@db3.ai/app 0.1.0-beta.1

package/src/queue/queueMonitor.ts

@@ -0,0 +1,368 @@
+import { randomUUID } from 'node:crypto';
+
+import {
+	createHttpEventDispatcher,
+	devtoolsEventsUrl,
+	isInteractiveDevelopmentEnvironment,
+	type DevtoolsEnvironment,
+	type DevtoolsEventDispatcher,
+	type HttpEventDispatcherOptions,
+} from '../devtools';
+import { isDevelopmentEnvironment } from '../db/sqlErrors';
+import type * as queue from './contracts';
+
+export type QueueMonitorMode = 'dev' | 'full';
+export type QueueMonitorAction = queue.QueueLifecycleAction;
+
+export interface QueueMonitorOptions {
+	/**
+	 * Human-readable source name copied to every emitted queue event.
+	 */
+	source?: string;
+
+	/**
+	 * Number of recent queue events retained in memory.
+	 */
+	maxEntries?: number;
+
+	/**
+	 * Whether queued payload data should be copied into emitted events.
+	 */
+	includePayload?: boolean;
+
+	/**
+	 * Optional event sink that receives every retained queue event.
+	 */
+	dispatch?: QueueMonitorDispatcher;
+}
+
+/**
+ * HTTP delivery configuration for queue monitor events.
+ */
+export type HttpQueueMonitorDispatcherOptions = HttpEventDispatcherOptions;
+
+export interface QueueMonitorEvent {
+	type: 'queue.job';
+	id: string;
+	source: string;
+	timestamp: string;
+	action: QueueMonitorAction;
+	queue: string;
+	jobName: string;
+	jobId?: queue.QueueJobId;
+	jobUuid?: string;
+	attempts?: number;
+	maxTries?: number;
+	delaySeconds?: number;
+	durationMs?: number;
+	errorMessage?: string;
+	origin?: queue.QueueJobOrigin;
+	payloadData?: Record<string, unknown>;
+}
+
+export interface QueueMonitorJobEventInput {
+	timestamp?: string;
+	action: QueueMonitorAction;
+	queue: string;
+	jobName: string;
+	jobId?: queue.QueueJobId;
+	jobUuid?: string;
+	attempts?: number;
+	maxTries?: number;
+	delaySeconds?: number;
+	durationMs?: number;
+	error?: unknown;
+	origin?: queue.QueueJobOrigin;
+	payloadData?: Record<string, unknown>;
+}
+
+export type QueueMonitorListener = (event: QueueMonitorEvent) => void;
+export type QueueMonitorDispatcher = (event: QueueMonitorEvent) => void;
+
+interface QueueMonitorEnvironment extends DevtoolsEnvironment {
+	PLATFORM_QUEUE_MONITOR?: string;
+	PLATFORM_QUEUE_MONITOR_MAX?: string;
+	PLATFORM_QUEUE_MONITOR_PAYLOAD?: string;
+	PLATFORM_QUEUE_MONITOR_SOURCE?: string;
+	npm_package_name?: string;
+}
+
+const defaultMaxEntries = 500;
+
+/**
+ * Captures queue lifecycle events for local development panels and tests.
+ */
+export class QueueMonitor {
+	private readonly entries: QueueMonitorEvent[] = [];
+	private readonly listeners = new Set<QueueMonitorListener>();
+	private readonly maxEntries: number;
+	private readonly source: string;
+	private readonly includePayload: boolean;
+	private readonly dispatch?: QueueMonitorDispatcher;
+
+	/**
+	 * Creates a queue lifecycle monitor.
+	 */
+	constructor(options: QueueMonitorOptions = {}) {
+		this.maxEntries = normalizePositiveInteger(options.maxEntries, defaultMaxEntries);
+		this.source = options.source || 'app';
+		this.includePayload = options.includePayload ?? false;
+		this.dispatch = options.dispatch;
+	}
+
+	/**
+	 * Returns queue events retained in memory.
+	 */
+	recent(): QueueMonitorEvent[] {
+		return [...this.entries];
+	}
+
+	/**
+	 * Removes queue events from the in-memory buffer.
+	 */
+	clear(): void {
+		this.entries.length = 0;
+	}
+
+	/**
+	 * Subscribes to queue events and returns an unsubscribe callback.
+	 */
+	subscribe(listener: QueueMonitorListener): () => void {
+		this.listeners.add(listener);
+
+		return () => {
+			this.listeners.delete(listener);
+		};
+	}
+
+	/**
+	 * Records one queue job lifecycle event.
+	 *
+	 * @param input - Queue job lifecycle fields.
+	 */
+	recordJob(input: QueueMonitorJobEventInput): void {
+		this.record({
+			type: 'queue.job',
+			id: this.eventId(input),
+			source: this.source,
+			timestamp: input.timestamp ?? new Date().toISOString(),
+			action: input.action,
+			queue: input.queue,
+			jobName: input.jobName,
+			jobId: input.jobId,
+			jobUuid: input.jobUuid,
+			attempts: input.attempts,
+			maxTries: input.maxTries,
+			delaySeconds: input.delaySeconds,
+			durationMs: input.durationMs === undefined ? undefined : roundDuration(input.durationMs),
+			errorMessage: input.error === undefined ? undefined : errorMessage(input.error),
+			origin: input.origin,
+			payloadData: this.includePayload ? input.payloadData : undefined,
+		});
+	}
+
+	/**
+	 * Records one event produced by the always-on queue lifecycle dispatcher.
+	 *
+	 * @param event - Queue lifecycle event to retain and forward.
+	 */
+	recordLifecycleEvent(event: queue.QueueLifecycleEvent): void {
+		this.recordJob({
+			timestamp: event.timestamp,
+			action: event.action,
+			queue: event.queue,
+			jobName: event.jobName,
+			jobId: event.jobId,
+			jobUuid: event.jobUuid,
+			attempts: event.attempts,
+			maxTries: event.maxTries,
+			delaySeconds: event.delaySeconds,
+			durationMs: event.durationMs,
+			error: event.error,
+			origin: event.origin,
+			payloadData: event.payloadData,
+		});
+	}
+
+	/**
+	 * Creates a stable id for terminal failures and a unique id for lifecycle events.
+	 *
+	 * A job UUID keeps repeated terminal instrumentation deliveries as one panel row.
+	 *
+	 * @param input - Queue event fields used to identify a terminal failure.
+	 * @returns Stable failed-event id or a fresh lifecycle event id.
+	 */
+	private eventId(input: QueueMonitorJobEventInput): string {
+		if (input.action === 'failed' && input.jobUuid) {
+			return `queue-failed:${input.jobUuid}`;
+		}
+
+		return randomUUID();
+	}
+
+	/**
+	 * Retains an event and notifies local plus external subscribers.
+	 */
+	private record(event: QueueMonitorEvent): void {
+		this.entries.push(event);
+
+		while (this.entries.length > this.maxEntries) {
+			this.entries.shift();
+		}
+
+		for (const listener of this.listeners) {
+			listener(event);
+		}
+
+		this.dispatch?.(event);
+	}
+}
+
+/**
+ * Enables the environment-default queue monitor when the runtime opts in.
+ */
+export function enableDefaultQueueMonitor(
+	options: QueueMonitorOptions | false | undefined = undefined,
+	env: QueueMonitorEnvironment = process.env,
+): QueueMonitor | null {
+	if (options === false) return null;
+
+	const defaults = defaultQueueMonitorOptions(env);
+	const resolved = defaults || options
+		? {
+			...(defaults ?? {}),
+			...(options ?? {}),
+		}
+		: null;
+
+	if (!resolved) return null;
+
+	return new QueueMonitor(resolved);
+}
+
+/**
+ * Resolves queue monitor defaults from development and observability env vars.
+ */
+export function defaultQueueMonitorOptions(
+	env: QueueMonitorEnvironment = process.env,
+): QueueMonitorOptions | null {
+	const mode = queueMonitorMode(env.PLATFORM_QUEUE_MONITOR, env.NODE_ENV);
+
+	if (!mode) return null;
+
+	return {
+		source: env.PLATFORM_QUEUE_MONITOR_SOURCE || env.npm_package_name || 'app',
+		maxEntries: parseIntegerEnv(env.PLATFORM_QUEUE_MONITOR_MAX, defaultMaxEntries),
+		includePayload: booleanEnv(
+			env.PLATFORM_QUEUE_MONITOR_PAYLOAD,
+			mode === 'full' || (mode === 'dev' && isDevelopmentEnvironment(env.NODE_ENV ?? '')),
+		),
+		dispatch: createHttpQueueMonitorDispatcher({
+			url: devtoolsEventsUrl(env),
+		}),
+	};
+}
+
+/**
+ * Creates a fire-and-forget batched HTTP dispatcher for queue events.
+ */
+export function createHttpQueueMonitorDispatcher(
+	options: HttpQueueMonitorDispatcherOptions = {},
+): DevtoolsEventDispatcher<QueueMonitorEvent> {
+	return createHttpEventDispatcher<QueueMonitorEvent>(options);
+}
+
+/**
+ * Resolves whether monitoring should be active for the current environment.
+ */
+function queueMonitorMode(
+	input: string | undefined,
+	environment: string | undefined,
+): QueueMonitorMode | null {
+	const normalized = (input || '').toLowerCase().trim();
+
+	switch (normalized) {
+		case '':
+			return isInteractiveDevelopmentEnvironment(environment) ? 'dev' : null;
+
+		case '1':
+		case 'true':
+		case 'on':
+		case 'dev':
+			return 'dev';
+
+		case 'full':
+			return 'full';
+
+		case '0':
+		case 'false':
+		case 'off':
+			return null;
+
+		default:
+			return isInteractiveDevelopmentEnvironment(environment) ? 'dev' : null;
+	}
+}
+
+/**
+ * Converts an unknown thrown value into a useful event message.
+ */
+function errorMessage(error: unknown): string {
+	if (error instanceof Error) return error.message;
+	if (error === undefined || error === null) return '';
+	return String(error);
+}
+
+/**
+ * Rounds a duration to one decimal place while keeping tiny timings visible.
+ */
+function roundDuration(durationMs: number): number {
+	if (!Number.isFinite(durationMs) || durationMs < 0) return 0;
+	return Math.round(durationMs * 10) / 10;
+}
+
+/**
+ * Parses a boolean env var while preserving a default value.
+ */
+function booleanEnv(input: string | undefined, fallback: boolean): boolean {
+	if (input === undefined || input === '') return fallback;
+
+	switch (input.toLowerCase().trim()) {
+		case '1':
+		case 'true':
+		case 'yes':
+		case 'on':
+			return true;
+
+		case '0':
+		case 'false':
+		case 'no':
+		case 'off':
+			return false;
+
+		default:
+			return fallback;
+	}
+}
+
+/**
+ * Parses an integer env var while enforcing a positive fallback.
+ */
+function parseIntegerEnv(input: string | undefined, fallback: number): number {
+	const parsed = Number(input);
+
+	if (!Number.isFinite(parsed)) return fallback;
+
+	return normalizePositiveInteger(parsed, fallback);
+}
+
+/**
+ * Returns a positive integer or the supplied fallback.
+ */
+function normalizePositiveInteger(input: unknown, fallback: number): number {
+	const value = Number(input);
+
+	if (!Number.isFinite(value) || value <= 0) return fallback;
+
+	return Math.trunc(value);
+}

package/src/scheduler/README.md

@@ -0,0 +1,139 @@
+# Scheduler
+
+The framework scheduler keeps recurring definitions in application code and
+stores one durable occurrence for every due named minute. It supports daily
+jobs and short inline calls:
+
+```ts
+schedule
+	.job(DatabaseBackupJob)
+	.dailyAt('02:00')
+	.timezone('Europe/London');
+
+schedule
+	.job(() => new ReportJob(organizationId))
+	.name('weekly-organization-report')
+	.dailyAt('06:00');
+
+schedule
+	.call(async () => {
+		await RecentUser.query().delete();
+	})
+	.name('clear-recent-users')
+	.daily();
+```
+
+A `QueueableJob` class uses its durable job name as the schedule name, so
+`.name(...)` is optional. Factories and inline calls must provide an explicit
+stable name because function identity is not durable. Names must be unique
+within one application schedule.
+
+The first implementation intentionally supports only `daily()` and
+`dailyAt('HH:mm')`. Times use UTC unless `.timezone(...)` selects an IANA
+timezone. The scheduler evaluates the current minute only; it does not run
+missed historical minutes after downtime.
+
+## Runtime
+
+`app().scheduler` exposes the scheduler service. Applications register their
+definitions once during scheduler and queue-worker bootstrap:
+
+```ts
+export function registerSchedule(app: App): void {
+	app.scheduler
+		.job(DatabaseBackupJob)
+		.dailyAt('02:00')
+		.timezone('Europe/London');
+}
+```
+
+There are two execution boundaries:
+
+- `scheduler.runDue()` evaluates one minute and is suitable for cron or a
+  one-off framework command.
+- `SchedulerWorker` evaluates immediately, waits to the next minute boundary,
+  and repeats without overlapping ticks in the same process.
+
+Run one dedicated scheduler process in production. Multiple processes are
+still safe: `scheduled_occurrences` has a unique `(name, scheduled_for)` claim,
+so only one process can dispatch a named task for a minute.
+
+Scheduled jobs are the preferred path. The scheduler only creates and
+dispatches the job; queue workers perform the expensive work. `call()` runs
+inside the scheduler process, so reserve it for short operations.
+
+## Occurrence History
+
+`ScheduledOccurrence` owns the `scheduled_occurrences` table. Its lifecycle
+states are:
+
+```text
+claimed -> queued -> running -> succeeded
+                     |       \
+                     |        -> retrying -> running
+                     \----------> deferred -> running
+                     \----------> failed
+```
+
+The occurrence records its schedule name, due minute, job name, queue ids,
+attempts, error, next-attempt time, and lifecycle timestamps. Queue origin
+metadata correlates the job without adding scheduler fields to application job
+payloads.
+
+The queue publishes awaited lifecycle events after each driver transition.
+Every queue-worker process must initialise `app().scheduler` so its recorder can
+update scheduled occurrences when that worker claims or finishes a job.
+Listener failures are isolated and logged; they cannot change an already
+committed queue outcome.
+
+Terminal occurrences are retained audit records. Replaying a failed scheduled
+job with `queue.retryFailed()` creates a new queue identity linked by `retryOf`;
+it does not turn the original failed occurrence into a success. Observe the
+replacement through queue lifecycle events/results and the application output.
+Queue does not persist a general success-history table. If an operations screen
+needs a durable recovered outcome, store the replacement identity and outcome
+on an application-owned report record. Do not erase the original failure.
+
+The shipped [replay runner](./examples/runScheduledReplay.ts) and
+[repairable job](./examples/RecoverableSummaryJob.ts) demonstrate this boundary:
+`npx tsx examples/runScheduledReplay.ts` fails on a missing local source, adds
+the source and successfully replays, while preserving the failed occurrence
+and rejecting a duplicate claim for the same minute. The copied Scheduler test
+asserts both the recovered output and the retained audit state.
+
+## Console
+
+Applications can expose the framework console with `runSchedulerConsole(...)`:
+
+```text
+scheduler:work
+scheduler:run
+scheduler:list
+scheduler:history --limit=25 --status=failed
+```
+
+`scheduler:work` is the long-running process. `scheduler:run` provides the same
+single-minute boundary that cron can invoke. `scheduler:list` validates and
+prints code-owned definitions. `scheduler:history` inspects durable occurrence
+state.
+
+## Runnable example and testing
+
+The shipped [daily summary job](./examples/WriteDailySummaryJob.ts),
+[registration function](./examples/registerDailySummary.ts) and
+[isolated runner](./examples/runDailySummary.ts) demonstrate a real SQL queue.
+Copy the example directory into an independent app's `examples` directory and
+run `npx tsx examples/runDailySummary.ts` with dedicated test SQL credentials.
+It creates/removes a `db3_app_test_*` database and temporary local storage.
+
+The runner evaluates a fixed UTC minute, rejects a second claim for that minute,
+processes the job and verifies successful occurrence history. It never sleeps
+or runs an application's historical schedules. The website includes the exact
+consumer-copyable test. Framework contributors run
+`npm run test:service --workspace packages/app -- scheduler`.
+
+A daily local time may not occur during a spring DST transition and may map to
+two UTC minutes during an autumn transition. Claims are keyed by UTC minute,
+not by a local business date. There is no catch-up or exactly-once side-effect
+guarantee. Claims and queue dispatch are separate operations; monitor and
+reconcile occurrences left claimed by an interrupted dispatch.

package/src/scheduler/ScheduledCall.ts

@@ -0,0 +1,36 @@
+import type * as scheduler from './contracts';
+import { ScheduledEvent } from './ScheduledEvent';
+
+/**
+ * Fluent daily schedule for one short inline callback.
+ */
+export class ScheduledCall extends ScheduledEvent {
+	readonly kind = 'call' as const;
+
+	/**
+	 * Creates a scheduled inline callback.
+	 *
+	 * @param callback - Short synchronous or asynchronous task.
+	 */
+	constructor(
+		private readonly callback: scheduler.ScheduledCallHandler,
+	) {
+		super();
+	}
+
+	/**
+	 * Runs the scheduled callback and awaits asynchronous completion.
+	 */
+	async run(): Promise<void> {
+		await this.callback();
+	}
+
+	/**
+	 * Requires every callback schedule to declare a stable explicit name.
+	 *
+	 * @returns Null because anonymous callback identity is not durable.
+	 */
+	protected override defaultName(): string | null {
+		return null;
+	}
+}

package/src/scheduler/ScheduledEvent.ts

@@ -0,0 +1,149 @@
+import type * as scheduler from './contracts';
+import { dailyScheduleIsDue, normalizeDailyTime, normalizeScheduleTimezone } from './scheduleTime';
+
+/**
+ * Shared fluent timing configuration for scheduled jobs and calls.
+ */
+export abstract class ScheduledEvent {
+	abstract readonly kind: scheduler.ScheduledTaskKind;
+	protected configuredName: string | null = null;
+	protected configuredFrequency: scheduler.DailyScheduleFrequency | null = null;
+	protected configuredTimezone = 'UTC';
+
+	/**
+	 * Sets the stable identity used for deduplication and history.
+	 *
+	 * @param name - Stable non-empty schedule name.
+	 * @returns This scheduled event.
+	 */
+	name(name: string): this {
+		const normalized = name.trim();
+
+		if (!normalized) {
+			throw new Error('Scheduled event names cannot be empty.');
+		}
+
+		if (normalized.length > 255) {
+			throw new Error('Scheduled event names cannot exceed 255 characters.');
+		}
+
+		this.configuredName = normalized;
+
+		return this;
+	}
+
+	/**
+	 * Schedules this task every day at local midnight.
+	 *
+	 * @returns This scheduled event.
+	 */
+	daily(): this {
+		return this.dailyAt('00:00');
+	}
+
+	/**
+	 * Schedules this task every day at one local wall-clock time.
+	 *
+	 * @param time - Time in 24-hour HH:mm form.
+	 * @returns This scheduled event.
+	 */
+	dailyAt(time: string): this {
+		this.configuredFrequency = {
+			type: 'daily',
+			time: normalizeDailyTime(time),
+		};
+
+		return this;
+	}
+
+	/**
+	 * Sets the IANA timezone used to interpret this event's local time.
+	 *
+	 * @param timezone - IANA timezone such as Europe/London.
+	 * @returns This scheduled event.
+	 */
+	timezone(timezone: string): this {
+		this.configuredTimezone = normalizeScheduleTimezone(timezone);
+
+		return this;
+	}
+
+	/**
+	 * Returns whether this event is due at one canonical UTC minute.
+	 *
+	 * @param instant - UTC minute being evaluated.
+	 * @returns True when the configured daily time matches.
+	 */
+	isDue(instant: Date): boolean {
+		const frequency = this.frequency();
+
+		return dailyScheduleIsDue(
+			instant,
+			frequency.time,
+			this.configuredTimezone,
+		);
+	}
+
+	/**
+	 * Returns the normalized public schedule definition.
+	 *
+	 * @returns Validated schedule definition.
+	 */
+	definition(): scheduler.ScheduledTaskDefinition {
+		const jobName = this.resolvedJobName();
+
+		return {
+			name: this.resolvedName(),
+			kind: this.kind,
+			frequency: this.frequency(),
+			timezone: this.configuredTimezone,
+			...(jobName ? {
+				jobName,
+			} : {}),
+		};
+	}
+
+	/**
+	 * Resolves the stable event name or throws when one is unavailable.
+	 *
+	 * @returns Stable event name.
+	 */
+	protected resolvedName(): string {
+		const name = this.configuredName ?? this.defaultName();
+
+		if (!name) {
+			throw new Error(`${this.constructor.name} requires an explicit stable name.`);
+		}
+
+		return name;
+	}
+
+	/**
+	 * Returns the daily frequency or throws when none was configured.
+	 *
+	 * @returns Configured daily frequency.
+	 */
+	private frequency(): scheduler.DailyScheduleFrequency {
+		if (!this.configuredFrequency) {
+			throw new Error(`Scheduled event "${this.resolvedName()}" requires a frequency.`);
+		}
+
+		return this.configuredFrequency;
+	}
+
+	/**
+	 * Returns the default stable name supplied by the concrete event type.
+	 *
+	 * @returns Default name, or null when explicit naming is required.
+	 */
+	protected abstract defaultName(): string | null;
+
+	/**
+	 * Returns the known queue job name for public schedule inspection.
+	 *
+	 * @returns Queue job name, or null for calls and deferred factories.
+	 */
+	protected resolvedJobName(): string | null {
+		return null;
+	}
+}