@signal24/dk-server-foundation 26.213.615

package/src/services/worker/CLAUDE.md

@@ -0,0 +1,77 @@
+# Worker System
+
+The worker system integrates BullMQ to queue background jobs and Deepkit services to run them. Enabling workers wires together the queue registry, runner, observer, CLI commands, and the `_jobs` audit table.
+
+## Enabling Workers
+
+```ts
+import { createApp } from '../app';
+
+const app = createApp({
+    config: MyConfig,
+    enableWorker: true
+    // …
+});
+```
+
+When `enableWorker` is set:
+
+- `WorkerService` becomes injectable for queueing jobs.
+- `WorkerRunnerService` and `WorkerObserverService` are registered (but only started when appropriate).
+- CLI controllers (`worker:start`, `worker:runner`, `worker:observer`, `worker:queue`) become available.
+- `JobEntity` is added to the database schema to persist job lifecycle data.
+
+At runtime, the runner/observer auto-start in non-production environments. In production, toggle them with configuration flags (`ENABLE_JOB_RUNNER`, `ENABLE_JOB_OBSERVER`). They can also be launched explicitly through the CLI commands listed above.
+
+## Defining Jobs
+
+Jobs extend `BaseJob` and are annotated with the `@WorkerJob()` decorator so the runner can discover them:
+
+```ts
+import { BaseJob, WorkerJob } from '../services/worker';
+
+@WorkerJob()
+class SendEmailJob extends BaseJob<{ to: string; subject: string }, void> {
+    constructor(private mailer: MailService) {
+        super();
+    }
+
+    async handle(data: { to: string; subject: string }) {
+        await this.mailer.send(data.to, data.subject);
+    }
+}
+```
+
+Optional static configuration:
+
+- `QUEUE_NAME` – assign the job to a named queue (defaults to the app's `BULL_QUEUE`).
+- `CRON_SCHEDULE` – schedule recurring execution using a cron pattern (registered automatically when the runner starts).
+
+The job instance receives dependencies via Deepkit's injector. There is no BullMQ `Job` object injected—log additional context yourself if needed.
+
+## Queueing Jobs
+
+Use `WorkerService.queueJob()` from request handlers or other services:
+
+```ts
+class NotificationController {
+    constructor(private worker: WorkerService) {}
+
+    async sendWelcome(to: string) {
+        await this.worker.queueJob(SendEmailJob, { to, subject: 'Welcome!' });
+    }
+}
+```
+
+`queueJob` accepts an optional `{ delay?: number }` option. When running in test environment (`APP_ENV=test`), the method simply logs a warning and skips queueing to keep tests deterministic.
+
+## Runner & Observer
+
+- **WorkerRunnerService** pulls jobs from the configured BullMQ queue and invokes the `handle()` method. It resolves job classes registered with `@WorkerJob()` and honours `CRON_SCHEDULE`.
+- **WorkerObserverService** listens for BullMQ lifecycle events, writes entries to the `_jobs` table via `JobEntity`, and clears completed/failed jobs from Redis after logging. It also wires into the health-check system to ensure Redis remains reachable.
+
+Both services close BullMQ resources during shutdown and support graceful termination through the Deepkit event system.
+
+## Redis & Configuration
+
+All worker components share Redis connection settings derived from `BaseAppConfig` (e.g. `BULL_QUEUE`, `REDIS_HOST` or sentinel configuration). Ensure these values are populated before starting the app or dedicated worker processes.

package/src/services/worker/bootstrap.ts

@@ -0,0 +1,58 @@
+import { App } from '@deepkit/app';
+import { eventDispatcher } from '@deepkit/event';
+import { onServerMainBootstrapDone, onServerShutdown } from '@deepkit/framework';
+import { Redis } from 'ioredis';
+
+import { getAppConfig } from '../../app/resolver';
+import { isDevelopment } from '../../app/const';
+import { globalState } from '../../app/state';
+import { WorkerQueueJobCommand, WorkerStartCommand, WorkerStartObserverCommand, WorkerStartRunnerCommand } from './cli';
+import { JobEntity } from './entity';
+import { WorkerObserverService } from './observer';
+import { WorkerQueueRegistry } from './queue';
+import { WorkerRunnerService } from './runner';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function installWorkerComponents(app: App<any>) {
+    globalState.enableWorker = true;
+    globalState.additionalEntities.push(JobEntity);
+
+    app.appModule.addProvider(WorkerObserverService);
+    app.appModule.addProvider(WorkerRunnerService);
+
+    app.appModule.addController(WorkerStartCommand);
+    app.appModule.addController(WorkerStartRunnerCommand);
+    app.appModule.addController(WorkerStartObserverCommand);
+    app.appModule.addController(WorkerQueueJobCommand);
+
+    class WorkerListener {
+        constructor(private runner: WorkerRunnerService) {}
+
+        @eventDispatcher.listen(onServerShutdown)
+        async shutdownRunner() {
+            await this.runner.shutdown();
+
+            // prevent new Redis connections from being created after shutdown
+            // some issue in dev keeping processes alive forever
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (Redis.prototype as any).connect = () => {};
+        }
+
+        // we want this to run very late in the process, after the worker and observer have shut down
+        @eventDispatcher.listen(onServerShutdown, 1000)
+        async closeQueues() {
+            await WorkerQueueRegistry.closeQueues();
+        }
+    }
+    app.appModule.addListener(WorkerListener);
+
+    app.listen(onServerMainBootstrapDone, () => {
+        if (!globalState.isCliService) {
+            setTimeout(() => {
+                const config = getAppConfig();
+                if (config.ENABLE_JOB_OBSERVER ?? isDevelopment) app.get(WorkerObserverService).start();
+                if (config.ENABLE_JOB_RUNNER ?? isDevelopment) app.get(WorkerRunnerService).start();
+            }, 1000);
+        }
+    });
+}

package/src/services/worker/cli.ts

@@ -0,0 +1,63 @@
+import { cli } from '@deepkit/app';
+
+import { CliServiceCommand } from '../cli';
+import { WorkerObserverService } from './observer';
+import { WorkerQueueRegistry } from './queue';
+import { WorkerRunnerService } from './runner';
+import { BaseJob } from './types';
+
+@cli.controller('worker:start', {
+    description: 'Start the worker runner and observer'
+})
+export class WorkerStartCommand extends CliServiceCommand {
+    constructor(
+        private runner: WorkerRunnerService,
+        private observer: WorkerObserverService
+    ) {
+        super();
+    }
+
+    async startService() {
+        await this.runner.start();
+        await this.observer.start();
+    }
+}
+
+@cli.controller('worker:runner', {
+    description: 'Start the worker runner'
+})
+export class WorkerStartRunnerCommand extends CliServiceCommand {
+    constructor(private runner: WorkerRunnerService) {
+        super();
+    }
+
+    async startService() {
+        await this.runner.start();
+    }
+}
+
+@cli.controller('worker:observer', {
+    description: 'Start the worker observer'
+})
+export class WorkerStartObserverCommand extends CliServiceCommand {
+    constructor(private observer: WorkerObserverService) {
+        super();
+    }
+
+    async startService() {
+        await this.observer.start();
+    }
+}
+
+@cli.controller('worker:queue', {
+    // todo: optional queue name?
+    description: 'Queue a job by name'
+})
+export class WorkerQueueJobCommand {
+    async execute(jobName: string, data?: string) {
+        data = data ? JSON.parse(data) : {};
+        const queue = WorkerQueueRegistry.getQueue(BaseJob.QUEUE_NAME);
+        await queue.add(jobName, data);
+        await queue.close();
+    }
+}

package/src/services/worker/entity.ts

@@ -0,0 +1,22 @@
+import { entity, Index, PrimaryKey } from '@deepkit/type';
+
+import { BaseEntity } from '../../database';
+
+@entity.name('_jobs')
+export class JobEntity extends BaseEntity {
+    id!: string & PrimaryKey;
+    queue!: string & Index;
+    queueId!: string;
+    attempt!: number;
+    name!: string;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    data!: any;
+    traceId!: string | null;
+    status!: 'completed' | 'failed';
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    result!: any;
+    createdAt!: Date;
+    shouldExecuteAt!: Date;
+    executedAt!: Date;
+    completedAt!: Date;
+}

package/src/services/worker/index.ts

@@ -0,0 +1,30 @@
+import { ClassType } from '@deepkit/core';
+
+import { isTest } from '../../app/const';
+import { createLogger } from '../logger';
+import { WorkerQueueRegistry } from './queue';
+import { BaseJob, IJobOptions, InputDataSymbol, JobClass } from './types';
+
+export * from './entity';
+export { BaseJob, WorkerJob } from './types';
+
+export class WorkerService {
+    private logger = createLogger(this);
+
+    async queueJob<I extends object, O, T extends BaseJob<I, O>>(jobClass: ClassType<T>, data: T[typeof InputDataSymbol], options?: IJobOptions) {
+        if (isTest) {
+            this.logger.warn('Not queueing job in test environment', { jobName: jobClass.name, data, options });
+            return;
+        }
+
+        try {
+            const typedJob = jobClass as unknown as JobClass;
+            const queue = WorkerQueueRegistry.getQueue(typedJob.QUEUE_NAME);
+            const job = await queue.add(jobClass.name, data, options);
+            this.logger.info('Queued job', { job: { name: jobClass.name, id: job.id } });
+        } catch (err) {
+            this.logger.error('Failed to queue job', err, { job: { name: jobClass.name } });
+            throw err;
+        }
+    }
+}

package/src/services/worker/observer.ts

@@ -0,0 +1,180 @@
+import { UniqueConstraintFailure } from '@deepkit/orm';
+import { SQLDatabaseAdapter } from '@deepkit/sql';
+import { Job, QueueEvents } from 'bullmq';
+
+import { getAppConfig } from '../../app/resolver';
+import { DBProvider } from '../../app/state';
+import { createPersistedEntity, getDialect, tableExistsSql } from '../../database';
+import { HealthcheckService } from '../../health/healthcheck.service';
+import { createRedisOptions } from '../../helpers/redis/redis';
+import { createLogger, ExtendedLogger } from '../logger';
+import { JobEntity } from './entity';
+import { WorkerQueueRegistry } from './queue';
+import { BaseAppConfig } from '../../app';
+
+export class WorkerObserverService {
+    private appConfig: BaseAppConfig;
+    private queueName: string;
+    private logger: ExtendedLogger;
+    private observer?: QueueEvents;
+    private db = this.dbProvider.db;
+
+    constructor(
+        private hcSvc: HealthcheckService,
+        private dbProvider: DBProvider
+    ) {
+        this.appConfig = getAppConfig();
+        this.queueName = this.appConfig.BULL_QUEUE;
+        this.logger = createLogger(this, { queueName: this.queueName });
+    }
+
+    async start() {
+        await this.ensureTableExists();
+
+        const { options, prefix } = createRedisOptions('BULL');
+        this.observer = new QueueEvents(this.queueName, {
+            connection: options,
+            prefix: `${prefix}:bmq`
+        });
+
+        const queue = WorkerQueueRegistry.getQueue(this.appConfig.BULL_QUEUE);
+
+        this.observer.on('added', args => {
+            this.logger.info('Job added', {
+                jobId: args.jobId,
+                jobName: args.name
+            });
+        });
+        this.observer.on('active', args => {
+            this.logger.info('Job activated', { jobId: args.jobId });
+        });
+        this.observer.on('stalled', args => {
+            this.logger.warn('Job stalled', { jobId: args.jobId });
+        });
+        this.observer.on('delayed', args => {
+            this.logger.info('Job delayed', { jobId: args.jobId, delay: args.delay });
+        });
+        this.observer.on('completed', async args => {
+            this.logger.info('Job completed', { jobId: args.jobId });
+            const job = await queue.getJob(args.jobId);
+            if (job) {
+                await this.logJob(job, 'completed', args.returnvalue);
+                await queue.remove(job.id!);
+            }
+        });
+        this.observer.on('failed', async args => {
+            this.logger.info('Job failed', { jobId: args.jobId });
+            const job = await queue.getJob(args.jobId);
+            if (job) {
+                await this.logJob(job, 'failed', { reason: job.failedReason, stack: job.failedReason });
+                await queue.remove(job.id!);
+            }
+        });
+        this.observer.on('error', err => {
+            this.logger.error('Observer error:', err);
+        });
+
+        this.logger.info('Observer started');
+
+        const completedJobs = await queue.getCompleted();
+        for (const job of completedJobs) {
+            this.logger.info('Logging previously completed job', { jobId: job.id });
+            await this.logJob(job, 'completed', job.returnvalue);
+            await queue.remove(job.id!);
+        }
+
+        const failedJobs = await queue.getFailed();
+        for (const job of failedJobs) {
+            this.logger.error('Logging previously failed job', { jobId: job.id });
+            await this.logJob(job, 'failed', { reason: job.failedReason, stack: job.stacktrace });
+            await queue.remove(job.id!);
+        }
+
+        this.hcSvc.register(async () => {
+            if (!this.isRedisReady()) {
+                throw new Error('Observer Redis connection is not ready');
+            }
+        });
+    }
+
+    private isRedisReady() {
+        return this.observer?.['connection']['_client'].status === 'ready' || this.observer?.['connection']['_client'].status === 'wait';
+    }
+
+    private async ensureTableExists() {
+        const dialect = getDialect(this.db.adapter as SQLDatabaseAdapter);
+        const tableInfoRows = await this.db.rawQuery(tableExistsSql(dialect, '_jobs'));
+        if (tableInfoRows.length) return;
+
+        if (dialect === 'postgres') {
+            await this.db.rawExecute(`
+                CREATE TABLE "_jobs" (
+                    "id" varchar(255) NOT NULL,
+                    "queue" varchar(255) NOT NULL,
+                    "queueId" varchar(255) NOT NULL,
+                    "attempt" smallint NOT NULL,
+                    "name" varchar(255) NOT NULL,
+                    "data" jsonb DEFAULT NULL,
+                    "traceId" char(32) DEFAULT NULL,
+                    "status" varchar(20) NOT NULL CHECK ("status" IN ('completed','failed')),
+                    "result" jsonb DEFAULT NULL,
+                    "createdAt" timestamp NOT NULL,
+                    "shouldExecuteAt" timestamp NOT NULL,
+                    "executedAt" timestamp NOT NULL,
+                    "completedAt" timestamp NOT NULL,
+                    PRIMARY KEY ("id","attempt")
+                );
+            `);
+        } else {
+            await this.db.rawExecute(`
+                CREATE TABLE \`_jobs\` (
+                    \`id\` varchar(255) NOT NULL,
+                    \`queue\` varchar(255) NOT NULL,
+                    \`queueId\` varchar(255) NOT NULL,
+                    \`attempt\` tinyint unsigned NOT NULL,
+                    \`name\` varchar(255) NOT NULL,
+                    \`data\` json DEFAULT NULL,
+                    \`traceId\` char(32) DEFAULT NULL,
+                    \`status\` enum('completed','failed') NOT NULL,
+                    \`result\` json DEFAULT NULL,
+                    \`createdAt\` datetime NOT NULL,
+                    \`shouldExecuteAt\` datetime NOT NULL,
+                    \`executedAt\` datetime NOT NULL,
+                    \`completedAt\` datetime NOT NULL,
+                    PRIMARY KEY (\`id\`,\`attempt\`)
+                ) ENGINE=InnoDB;
+            `);
+        }
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    async logJob(job: Job, status: 'completed' | 'failed', result: any) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const traceparent = (job.opts as any).traceparent;
+            const traceId = traceparent ? traceparent.split('-')[1] : null;
+
+            await createPersistedEntity(JobEntity, {
+                id: `${this.queueName}:${job.id}`,
+                queue: this.queueName,
+                queueId: job.id!,
+                attempt: job.attemptsMade,
+                name: job.name,
+                data: job.data,
+                traceId,
+                status,
+                result,
+                createdAt: new Date(job.timestamp),
+                shouldExecuteAt: new Date(job.timestamp + (job.opts.delay ?? 0)),
+                executedAt: new Date(job.processedOn!),
+                completedAt: new Date(job.finishedOn!)
+            });
+        } catch (err) {
+            if (err instanceof UniqueConstraintFailure) {
+                this.logger.warn('Job already logged', { jobId: job.id });
+            } else {
+                throw err;
+            }
+        }
+    }
+}

package/src/services/worker/queue.ts

@@ -0,0 +1,34 @@
+import { Queue } from 'bullmq';
+
+import { getAppConfig } from '../../app/resolver';
+import { createRedisOptions } from '../../helpers/redis/redis';
+
+export class WorkerQueueRegistry {
+    static registry = new Map<string, Queue>();
+
+    static getDefaultQueue() {
+        const appConfig = getAppConfig();
+        return this.getQueue(appConfig.BULL_QUEUE);
+    }
+
+    static getQueue(name: string): Queue {
+        if (!this.registry.has(name)) {
+            const { options, prefix } = createRedisOptions('BULL');
+            this.registry.set(
+                name,
+                new Queue(name, {
+                    connection: options,
+                    prefix: `${prefix}:bmq`
+                })
+            );
+        }
+
+        return this.registry.get(name)!;
+    }
+
+    static async closeQueues() {
+        for (const queue of this.registry.values()) {
+            await queue.close();
+        }
+    }
+}

package/src/services/worker/runner.ts

@@ -0,0 +1,146 @@
+import { SpanContext } from '@opentelemetry/api';
+import { Job, Queue, Worker } from 'bullmq';
+
+import { BaseAppConfig } from '../../app';
+import { getAppConfig, resolveDeep } from '../../app/resolver';
+import { HealthcheckService } from '../../health/healthcheck.service';
+import { sleepSecs, withContextData } from '../../helpers';
+import { getRegisteredClasses } from '../../helpers/framework/decorators';
+import { createRedisOptions } from '../../helpers/redis/redis';
+import { getTraceContext, setSpanAttributes, withRootSpan } from '../../telemetry';
+import { createLogger, ExtendedLogger } from '../logger';
+import { WorkerQueueRegistry } from './queue';
+import { JobClass, WorkerSymbol } from './types';
+
+export class WorkerRunnerService {
+    private appConfig: BaseAppConfig;
+    private queueName: string;
+    private logger: ExtendedLogger;
+    private worker?: Worker;
+    private queue?: Queue;
+    private jobHandlers = new Map<string, InstanceType<JobClass>>();
+    private runningJob?: Job;
+
+    constructor(private hcSvc: HealthcheckService) {
+        this.appConfig = getAppConfig();
+        this.queueName = this.appConfig.BULL_QUEUE;
+        this.logger = createLogger(this, { queue: this.queueName });
+    }
+
+    async start() {
+        this.queue = WorkerQueueRegistry.getQueue(this.queueName);
+
+        const allJobClasses = getRegisteredClasses<JobClass>(WorkerSymbol);
+
+        for (const jobClass of allJobClasses) {
+            if (jobClass.QUEUE_NAME === this.queueName) {
+                this.logger.info('Registering job', { id: { name: jobClass.name, schedule: jobClass.CRON_SCHEDULE } });
+
+                const handlerInstance = resolveDeep(jobClass);
+                if (!handlerInstance) throw new Error(`Cannot resolve job handler: ${jobClass.name}`);
+                this.jobHandlers.set(jobClass.name, handlerInstance);
+
+                if (jobClass.CRON_SCHEDULE) {
+                    this.queue.add(jobClass.name, {}, { repeat: { pattern: jobClass.CRON_SCHEDULE } });
+                }
+            }
+        }
+
+        const { options, prefix } = createRedisOptions('BULL');
+
+        this.worker = new Worker(
+            this.queueName,
+            async job =>
+                this.withJobSpan(job, () =>
+                    withContextData(
+                        {
+                            job: {
+                                queue: this.queueName,
+                                id: job.id,
+                                name: job.name
+                            },
+                            traceId: getTraceContext()?.traceId
+                        },
+                        async () => {
+                            const handler = this.jobHandlers.get(job.name);
+                            if (!handler) {
+                                throw new Error(`Job ${job.name} is not registered`);
+                            }
+                            try {
+                                this.runningJob = job;
+                                return await handler.handle(job.data);
+                            } finally {
+                                this.runningJob = undefined;
+                            }
+                        }
+                    )
+                ),
+            {
+                concurrency: 1,
+                connection: options,
+                prefix: `${prefix}:bmq`
+            }
+        );
+
+        this.worker.on('active', job => {
+            this.logger.info('Job activated', { job: { name: job.name, id: job.id } });
+        });
+        this.worker.on('completed', job => {
+            this.logger.info('Job completed', { job: { name: job.name, id: job.id } });
+        });
+        this.worker.on('failed', (job, err) => {
+            this.logger.error(`Job failed: ${err.message}`, err, { job: job ? { name: job.name, id: job.id } : undefined });
+        });
+        this.worker.on('stalled', jobId => {
+            this.logger.warn('Job stalled', { jobId });
+        });
+        this.worker.on('error', err => {
+            this.logger.error('Worker error', err);
+        });
+        this.worker.on('ready', () => {
+            this.logger.info('Worker ready');
+        });
+        this.logger.info('Worker started');
+
+        this.hcSvc.register(async () => {
+            if (!this.isRedisReady()) {
+                throw new Error('Worker Redis connection is not ready');
+            }
+        });
+    }
+
+    private async withJobSpan<T>(job: Job, fn: () => Promise<T>) {
+        if (job.repeatJobKey) {
+            let innerSpan: SpanContext | undefined;
+            const result = await withRootSpan(`Job ${job.name}`, { jobId: job.id, schedulerTrace: getTraceContext()?.traceId }, () => {
+                innerSpan = getTraceContext();
+                return fn();
+            });
+            if (innerSpan) setSpanAttributes({ jobTraceId: innerSpan.traceId });
+            return result;
+        } else {
+            return fn();
+        }
+    }
+
+    private isRedisReady() {
+        return this.worker?.['blockingConnection']['_client'].status === 'ready' || this.worker?.['blockingConnection']['_client'].status === 'wait';
+    }
+
+    async shutdown() {
+        // there's something crazy going on with worker shutdown when it hasn't successfully connected to Redis
+        if (this.isRedisReady()) {
+            await this.worker?.pause(true);
+
+            while (this.runningJob) {
+                this.logger.warn('Waiting for job to finish', { job: { name: this.runningJob.name, id: this.runningJob.id } });
+                await sleepSecs(1);
+            }
+
+            await this.worker?.close();
+        } else {
+            this.worker?.close(true);
+            this.worker?.disconnect();
+        }
+    }
+}

package/src/services/worker/types.ts

@@ -0,0 +1,32 @@
+import { ClassType } from '@deepkit/core';
+
+import { createRegistryClassDecorator } from '../../helpers/framework/decorators';
+
+export const JobSymbol = Symbol('Job');
+export const InputDataSymbol = Symbol('InputData');
+export const OutputDataSymbol = Symbol('OutputData');
+
+export const WorkerSymbol = Symbol('Worker');
+export const WorkerJob = createRegistryClassDecorator(WorkerSymbol);
+
+export abstract class BaseJob<I = void, O = void> {
+    [JobSymbol] = JobSymbol;
+    [InputDataSymbol]!: I;
+    [OutputDataSymbol]!: O;
+
+    static QUEUE_NAME = 'default';
+    static CRON_SCHEDULE: string | null = null;
+
+    abstract handle(data: I): Promise<O>;
+}
+
+export interface BaseJobClass {
+    QUEUE_NAME: string;
+    CRON_SCHEDULE: string | null;
+}
+
+export type JobClass = ClassType<BaseJob> & BaseJobClass;
+
+export interface IJobOptions {
+    delay?: number;
+}