@zyno-io/dk-server-foundation 26.216.430

package/src/services/worker/recorder.ts

@@ -0,0 +1,172 @@
+import { UniqueConstraintFailure } from '@deepkit/orm';
+import { SQLDatabaseAdapter } from '@deepkit/sql';
+import { Job, QueueEvents } from 'bullmq';
+
+import { getAppConfig } from '../../app/resolver';
+import { DBProvider } from '../../app/state';
+import { createPersistedEntity, getDialect, tableExistsSql } from '../../database';
+import { createRedisOptions } from '../../helpers/redis/redis';
+import { createLogger, ExtendedLogger } from '../logger';
+import { JobEntity } from './entity';
+import { WorkerQueueRegistry } from './queue';
+import { BaseAppConfig } from '../../app';
+
+export class WorkerRecorderService {
+    private appConfig: BaseAppConfig;
+    private queueName: string;
+    private logger: ExtendedLogger;
+    private observer?: QueueEvents;
+    private db = this.dbProvider.db;
+
+    constructor(private dbProvider: DBProvider) {
+        this.appConfig = getAppConfig();
+        this.queueName = this.appConfig.BULL_QUEUE;
+        this.logger = createLogger(this, { queueName: this.queueName });
+    }
+
+    async ensureTableExists() {
+        const dialect = getDialect(this.db.adapter as SQLDatabaseAdapter);
+        const tableInfoRows = await this.db.rawQuery(tableExistsSql(dialect, '_jobs'));
+        if (tableInfoRows.length) return;
+
+        if (dialect === 'postgres') {
+            await this.db.rawExecute(`
+                CREATE TABLE "_jobs" (
+                    "id" varchar(255) NOT NULL,
+                    "queue" varchar(255) NOT NULL,
+                    "queueId" varchar(255) NOT NULL,
+                    "attempt" smallint NOT NULL,
+                    "name" varchar(255) NOT NULL,
+                    "data" jsonb DEFAULT NULL,
+                    "traceId" char(32) DEFAULT NULL,
+                    "status" varchar(20) NOT NULL CHECK ("status" IN ('completed','failed')),
+                    "result" jsonb DEFAULT NULL,
+                    "createdAt" timestamp NOT NULL,
+                    "shouldExecuteAt" timestamp NOT NULL,
+                    "executedAt" timestamp NOT NULL,
+                    "completedAt" timestamp NOT NULL,
+                    PRIMARY KEY ("id","attempt")
+                );
+            `);
+        } else {
+            await this.db.rawExecute(`
+                CREATE TABLE \`_jobs\` (
+                    \`id\` varchar(255) NOT NULL,
+                    \`queue\` varchar(255) NOT NULL,
+                    \`queueId\` varchar(255) NOT NULL,
+                    \`attempt\` tinyint unsigned NOT NULL,
+                    \`name\` varchar(255) NOT NULL,
+                    \`data\` json DEFAULT NULL,
+                    \`traceId\` char(32) DEFAULT NULL,
+                    \`status\` enum('completed','failed') NOT NULL,
+                    \`result\` json DEFAULT NULL,
+                    \`createdAt\` datetime NOT NULL,
+                    \`shouldExecuteAt\` datetime NOT NULL,
+                    \`executedAt\` datetime NOT NULL,
+                    \`completedAt\` datetime NOT NULL,
+                    PRIMARY KEY (\`id\`,\`attempt\`)
+                ) ENGINE=InnoDB;
+            `);
+        }
+    }
+
+    async start() {
+        const { options, prefix } = createRedisOptions('BULL');
+        this.observer = new QueueEvents(this.queueName, {
+            connection: options,
+            prefix: `${prefix}:bmq`
+        });
+
+        const queue = WorkerQueueRegistry.getQueue(this.appConfig.BULL_QUEUE);
+
+        this.observer.on('added', args => {
+            this.logger.info('Job added', {
+                jobId: args.jobId,
+                jobName: args.name
+            });
+        });
+        this.observer.on('active', args => {
+            this.logger.info('Job activated', { jobId: args.jobId });
+        });
+        this.observer.on('stalled', args => {
+            this.logger.warn('Job stalled', { jobId: args.jobId });
+        });
+        this.observer.on('delayed', args => {
+            this.logger.info('Job delayed', { jobId: args.jobId, delay: args.delay });
+        });
+        this.observer.on('completed', async args => {
+            this.logger.info('Job completed', { jobId: args.jobId });
+            const job = await queue.getJob(args.jobId);
+            if (job) {
+                await this.logJob(job, 'completed', args.returnvalue);
+                await queue.remove(job.id!);
+            }
+        });
+        this.observer.on('failed', async args => {
+            this.logger.info('Job failed', { jobId: args.jobId });
+            const job = await queue.getJob(args.jobId);
+            if (job) {
+                await this.logJob(job, 'failed', { reason: job.failedReason, stack: job.failedReason });
+                await queue.remove(job.id!);
+            }
+        });
+        this.observer.on('error', err => {
+            this.logger.error('Recorder error:', err);
+        });
+
+        this.logger.info('Recorder started');
+
+        const completedJobs = await queue.getCompleted();
+        for (const job of completedJobs) {
+            this.logger.info('Logging previously completed job', { jobId: job.id });
+            await this.logJob(job, 'completed', job.returnvalue);
+            await queue.remove(job.id!);
+        }
+
+        const failedJobs = await queue.getFailed();
+        for (const job of failedJobs) {
+            this.logger.error('Logging previously failed job', { jobId: job.id });
+            await this.logJob(job, 'failed', { reason: job.failedReason, stack: job.stacktrace });
+            await queue.remove(job.id!);
+        }
+    }
+
+    async stop() {
+        if (this.observer) {
+            await this.observer.close();
+            this.observer = undefined;
+            this.logger.info('Recorder stopped');
+        }
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    async logJob(job: Job, status: 'completed' | 'failed', result: any) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const traceparent = (job.opts as any).traceparent;
+            const traceId = traceparent ? traceparent.split('-')[1] : null;
+
+            await createPersistedEntity(JobEntity, {
+                id: `${this.queueName}:${job.id}`,
+                queue: this.queueName,
+                queueId: job.id!,
+                attempt: job.attemptsMade,
+                name: job.name,
+                data: job.data,
+                traceId,
+                status,
+                result,
+                createdAt: new Date(job.timestamp),
+                shouldExecuteAt: new Date(job.timestamp + (job.opts.delay ?? 0)),
+                executedAt: new Date(job.processedOn!),
+                completedAt: new Date(job.finishedOn!)
+            });
+        } catch (err) {
+            if (err instanceof UniqueConstraintFailure) {
+                this.logger.warn('Job already logged', { jobId: job.id });
+            } else {
+                throw err;
+            }
+        }
+    }
+}

package/src/services/worker/runner.ts

@@ -0,0 +1,179 @@
+import { SpanContext } from '@opentelemetry/api';
+import { Job, Queue, Worker } from 'bullmq';
+
+import { BaseAppConfig } from '../../app';
+import { getAppConfig, resolveDeep } from '../../app/resolver';
+import { DBProvider } from '../../app/state';
+import { HealthcheckService } from '../../health/healthcheck.service';
+import { sleepSecs, withContextData } from '../../helpers';
+import { getRegisteredClasses } from '../../helpers/framework/decorators';
+import { createRedisOptions } from '../../helpers/redis/redis';
+import { getTraceContext, setSpanAttributes, withRootSpan } from '../../telemetry';
+import { createLogger, ExtendedLogger } from '../logger';
+import { LeaderService } from '../leader';
+import { WorkerQueueRegistry } from './queue';
+import { WorkerRecorderService } from './recorder';
+import { JobClass, WorkerSymbol } from './types';
+
+export class WorkerRunnerService {
+    private appConfig: BaseAppConfig;
+    private queueName: string;
+    private logger: ExtendedLogger;
+    private worker?: Worker;
+    private queue?: Queue;
+    private jobHandlers = new Map<string, InstanceType<JobClass>>();
+    private runningJob?: Job;
+    private recorder?: WorkerRecorderService;
+    private leader?: LeaderService;
+
+    constructor(
+        private hcSvc: HealthcheckService,
+        private dbProvider: DBProvider
+    ) {
+        this.appConfig = getAppConfig();
+        this.queueName = this.appConfig.BULL_QUEUE;
+        this.logger = createLogger(this, { queue: this.queueName });
+    }
+
+    async start() {
+        this.queue = WorkerQueueRegistry.getQueue(this.queueName);
+
+        const allJobClasses = getRegisteredClasses<JobClass>(WorkerSymbol);
+
+        for (const jobClass of allJobClasses) {
+            if (jobClass.QUEUE_NAME === this.queueName) {
+                this.logger.info('Registering job', { id: { name: jobClass.name, schedule: jobClass.CRON_SCHEDULE } });
+
+                const handlerInstance = resolveDeep(jobClass);
+                if (!handlerInstance) throw new Error(`Cannot resolve job handler: ${jobClass.name}`);
+                this.jobHandlers.set(jobClass.name, handlerInstance);
+
+                if (jobClass.CRON_SCHEDULE) {
+                    this.queue.add(jobClass.name, {}, { repeat: { pattern: jobClass.CRON_SCHEDULE } });
+                }
+            }
+        }
+
+        const { options, prefix } = createRedisOptions('BULL');
+
+        this.worker = new Worker(
+            this.queueName,
+            async job =>
+                this.withJobSpan(job, () =>
+                    withContextData(
+                        {
+                            job: {
+                                queue: this.queueName,
+                                id: job.id,
+                                name: job.name
+                            },
+                            traceId: getTraceContext()?.traceId
+                        },
+                        async () => {
+                            const handler = this.jobHandlers.get(job.name);
+                            if (!handler) {
+                                throw new Error(`Job ${job.name} is not registered`);
+                            }
+                            try {
+                                this.runningJob = job;
+                                return await handler.handle(job.data);
+                            } finally {
+                                this.runningJob = undefined;
+                            }
+                        }
+                    )
+                ),
+            {
+                concurrency: 1,
+                connection: options,
+                prefix: `${prefix}:bmq`
+            }
+        );
+
+        this.worker.on('active', job => {
+            this.logger.info('Job activated', { job: { name: job.name, id: job.id } });
+        });
+        this.worker.on('completed', job => {
+            this.logger.info('Job completed', { job: { name: job.name, id: job.id } });
+        });
+        this.worker.on('failed', (job, err) => {
+            this.logger.error(`Job failed: ${err.message}`, err, { job: job ? { name: job.name, id: job.id } : undefined });
+        });
+        this.worker.on('stalled', jobId => {
+            this.logger.warn('Job stalled', { jobId });
+        });
+        this.worker.on('error', err => {
+            this.logger.error('Worker error', err);
+        });
+        this.worker.on('ready', () => {
+            this.logger.info('Worker ready');
+        });
+        this.logger.info('Worker started');
+
+        this.hcSvc.register('Worker Runner', async () => {
+            if (!this.isRedisReady()) {
+                throw new Error('Worker Redis connection is not ready');
+            }
+        });
+
+        // Set up recorder with leader election
+        this.recorder = new WorkerRecorderService(this.dbProvider);
+        await this.recorder.ensureTableExists();
+
+        this.leader = new LeaderService('worker-recorder');
+        this.leader.setBecameLeaderCallback(async () => {
+            this.logger.info('This runner is now the recorder leader');
+            await this.recorder!.start();
+        });
+        this.leader.setLostLeaderCallback(async () => {
+            this.logger.info('This runner lost recorder leadership');
+            await this.recorder!.stop();
+        });
+        this.leader.start();
+    }
+
+    private async withJobSpan<T>(job: Job, fn: () => Promise<T>) {
+        if (job.repeatJobKey) {
+            let innerSpan: SpanContext | undefined;
+            const result = await withRootSpan(`Job ${job.name}`, { jobId: job.id, schedulerTrace: getTraceContext()?.traceId }, () => {
+                innerSpan = getTraceContext();
+                return fn();
+            });
+            if (innerSpan) setSpanAttributes({ jobTraceId: innerSpan.traceId });
+            return result;
+        } else {
+            return fn();
+        }
+    }
+
+    private isRedisReady() {
+        return this.worker?.['blockingConnection']['_client'].status === 'ready' || this.worker?.['blockingConnection']['_client'].status === 'wait';
+    }
+
+    async shutdown() {
+        // Stop leader election first (releases Redis lock)
+        if (this.leader) {
+            await this.leader.stop();
+        }
+
+        // Explicitly stop recorder (LeaderService.stop() does NOT call lostLeaderCallback)
+        if (this.recorder) {
+            await this.recorder.stop();
+        }
+
+        // there's something crazy going on with worker shutdown when it hasn't successfully connected to Redis
+        if (this.isRedisReady()) {
+            await this.worker?.pause(true);
+
+            while (this.runningJob) {
+                this.logger.warn('Waiting for job to finish', { job: { name: this.runningJob.name, id: this.runningJob.id } });
+                await sleepSecs(1);
+            }
+
+            await this.worker?.close();
+        } else {
+            this.worker?.close(true);
+            this.worker?.disconnect();
+        }
+    }
+}

package/src/services/worker/types.ts

@@ -0,0 +1,32 @@
+import { ClassType } from '@deepkit/core';
+
+import { createRegistryClassDecorator } from '../../helpers/framework/decorators';
+
+export const JobSymbol = Symbol('Job');
+export const InputDataSymbol = Symbol('InputData');
+export const OutputDataSymbol = Symbol('OutputData');
+
+export const WorkerSymbol = Symbol('Worker');
+export const WorkerJob = createRegistryClassDecorator(WorkerSymbol);
+
+export abstract class BaseJob<I = void, O = void> {
+    [JobSymbol] = JobSymbol;
+    [InputDataSymbol]!: I;
+    [OutputDataSymbol]!: O;
+
+    static QUEUE_NAME = 'default';
+    static CRON_SCHEDULE: string | null = null;
+
+    abstract handle(data: I): Promise<O>;
+}
+
+export interface BaseJobClass {
+    QUEUE_NAME: string;
+    CRON_SCHEDULE: string | null;
+}
+
+export type JobClass = ClassType<BaseJob> & BaseJobClass;
+
+export interface IJobOptions {
+    delay?: number;
+}