@zyno-io/dk-server-foundation 26.216.430

package/src/services/mesh.ts

@@ -0,0 +1,508 @@
+import { hostname } from 'os';
+
+import { uuid } from '@deepkit/type';
+import { memoize } from 'lodash';
+
+import { createRedis } from '../helpers/redis/redis';
+import { LeaderService, LeaderServiceOptions } from './leader';
+import { createLogger } from './logger';
+
+// --- Types ---
+
+export type MeshMessageMap = Record<string, { request: unknown; response: unknown }>;
+
+export interface MeshNode {
+    instanceId: number;
+    hostname: string;
+    self: boolean;
+}
+
+export interface MeshServiceOptions {
+    heartbeatIntervalMs?: number;
+    nodeTtlMs?: number;
+    requestTimeoutMs?: number;
+    leaderOptions?: LeaderServiceOptions;
+}
+
+export class MeshRequestTimeoutError extends Error {
+    constructor(instanceId: number, type: string) {
+        super(`Mesh request to instance ${instanceId} timed out (type: ${type})`);
+        this.name = 'MeshRequestTimeoutError';
+    }
+}
+
+export class MeshHandlerError extends Error {
+    constructor(message: string) {
+        super(message);
+        this.name = 'MeshHandlerError';
+    }
+}
+
+export class MeshNoHandlerError extends Error {
+    constructor(type: string) {
+        super(`No handler registered for mesh message type: ${type}`);
+        this.name = 'MeshNoHandlerError';
+    }
+}
+
+// --- Constants ---
+
+const NO_HANDLER_ERROR_PREFIX = 'MESH_NO_HANDLER:';
+
+// --- Lua Scripts ---
+
+const HEARTBEAT_SCRIPT = `
+local time = redis.call("time")
+local now_ms = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
+redis.call("zadd", KEYS[1], now_ms, ARGV[1])
+return 1
+`;
+
+const CLEANUP_SCRIPT = `
+local time = redis.call("time")
+local now_ms = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
+local cutoff = now_ms - tonumber(ARGV[1])
+local expired = redis.call("zrangebyscore", KEYS[1], "-inf", cutoff)
+for i, id in ipairs(expired) do
+    redis.call("zrem", KEYS[1], id)
+    redis.call("hdel", KEYS[2], id)
+end
+return expired
+`;
+
+// --- Redis Client ---
+
+type MeshRedisClient = ReturnType<typeof createRedis>['client'] & {
+    HEARTBEAT: (key: string, instanceId: string) => Promise<number>;
+    CLEANUP: (heartbeatsKey: string, nodesKey: string, ttlMs: string) => Promise<string[]>;
+};
+
+const getMeshRedis = memoize(() => {
+    const { client, prefix } = createRedis('MESH');
+    client.defineCommand('HEARTBEAT', { lua: HEARTBEAT_SCRIPT, numberOfKeys: 1 });
+    client.defineCommand('CLEANUP', { lua: CLEANUP_SCRIPT, numberOfKeys: 2 });
+    return { client: client as MeshRedisClient, prefix };
+});
+
+// --- Channel Message Types ---
+
+interface MeshRequest {
+    requestId: string;
+    senderInstanceId: number;
+    type: string;
+    data: unknown;
+    timeoutMs?: number;
+}
+
+interface MeshResponse {
+    requestId: string;
+    reply: true;
+    data?: unknown;
+    error?: string;
+}
+
+interface MeshHeartbeat {
+    requestId: string;
+    heartbeat: true;
+}
+
+// --- Pending Request ---
+
+interface PendingRequest {
+    resolve: (value: unknown) => void;
+    reject: (reason: Error) => void;
+    timer: ReturnType<typeof setTimeout>;
+    type: string;
+    targetInstanceId: number;
+}
+
+// --- MeshService ---
+
+export class MeshService<T extends MeshMessageMap> {
+    private _instanceId: number = 0;
+    private key: string;
+    private prefix: string = '';
+    private running = false;
+
+    private heartbeatIntervalMs: number;
+    private nodeTtlMs: number;
+    private requestTimeoutMs: number;
+    private leaderOptions?: LeaderServiceOptions;
+
+    private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
+    private leaderService: LeaderService | null = null;
+    private subscriberClient: ReturnType<typeof createRedis>['client'] | null = null;
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    private handlers = new Map<string, (data: any) => any>();
+    private pendingRequests = new Map<string, PendingRequest>();
+    private activeHandlerIntervals = new Set<ReturnType<typeof setInterval>>();
+    private nodeCleanedUpCallback: ((instanceId: number) => void | Promise<void>) | null = null;
+
+    private logger = createLogger(this);
+
+    constructor(key: string, options?: MeshServiceOptions) {
+        this.key = key;
+        this.heartbeatIntervalMs = options?.heartbeatIntervalMs ?? 5000;
+        this.nodeTtlMs = options?.nodeTtlMs ?? 15000;
+        this.requestTimeoutMs = options?.requestTimeoutMs ?? 10000;
+        this.leaderOptions = options?.leaderOptions;
+    }
+
+    get instanceId(): number {
+        return this._instanceId;
+    }
+
+    registerHandler<K extends keyof T & string>(type: K, handler: (data: T[K]['request']) => T[K]['response'] | Promise<T[K]['response']>): void {
+        this.handlers.set(type, handler);
+    }
+
+    setNodeCleanedUpCallback(cb: (instanceId: number) => void | Promise<void>): void {
+        this.nodeCleanedUpCallback = cb;
+    }
+
+    async getNodes(): Promise<MeshNode[]> {
+        if (!this.running) {
+            throw new Error('MeshService is not running');
+        }
+
+        const { client } = getMeshRedis();
+        const members = await client.zrange(this.heartbeatsKey(), 0, -1);
+        if (members.length === 0) return [];
+
+        const hostnames = await client.hmget(this.nodesKey(), ...members);
+        return members.map((id, i) => ({
+            instanceId: parseInt(id, 10),
+            hostname: hostnames[i] ?? 'unknown',
+            self: parseInt(id, 10) === this._instanceId
+        }));
+    }
+
+    async invoke<K extends keyof T & string>(instanceId: number, type: K, data: T[K]['request']): Promise<T[K]['response']> {
+        if (!this.running) {
+            throw new Error('MeshService is not running');
+        }
+
+        // Local invocation — call handler directly
+        if (instanceId === this._instanceId) {
+            const handler = this.handlers.get(type);
+            if (!handler) throw new MeshNoHandlerError(type);
+            return handler(data);
+        }
+
+        // Remote invocation via pub/sub
+        const requestId = uuid();
+        const channel = this.channelForInstance(instanceId);
+
+        // Serialize before registering pending request to avoid leaked timers on stringify failure
+        const message: MeshRequest = {
+            requestId,
+            senderInstanceId: this._instanceId,
+            type,
+            data,
+            timeoutMs: this.requestTimeoutMs
+        };
+        const payload = JSON.stringify(message);
+
+        return new Promise<T[K]['response']>((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pendingRequests.delete(requestId);
+                reject(new MeshRequestTimeoutError(instanceId, type));
+            }, this.requestTimeoutMs);
+
+            this.pendingRequests.set(requestId, {
+                resolve: resolve as (value: unknown) => void,
+                reject,
+                timer,
+                type,
+                targetInstanceId: instanceId
+            });
+
+            const { client } = getMeshRedis();
+            client.publish(channel, payload).catch(err => {
+                clearTimeout(timer);
+                this.pendingRequests.delete(requestId);
+                reject(new Error(`Failed to publish mesh request: ${err instanceof Error ? err.message : String(err)}`));
+            });
+        });
+    }
+
+    async start(): Promise<void> {
+        if (this.running) {
+            throw new Error('MeshService is already running');
+        }
+
+        const { client, prefix } = getMeshRedis();
+        this.prefix = prefix;
+
+        // Acquire unique instance ID
+        this._instanceId = await client.incr(this.nextIdKey());
+
+        // Create subscriber and subscribe to own channel
+        const { client: subClient } = createRedis('MESH');
+        this.subscriberClient = subClient;
+
+        try {
+            const channel = this.channelForInstance(this._instanceId);
+            await subClient.subscribe(channel);
+            subClient.on('message', (_channel: string, message: string) => {
+                this.handleMessage(message);
+            });
+
+            // Register heartbeat and node metadata
+            await client.HEARTBEAT(this.heartbeatsKey(), String(this._instanceId));
+            await client.hset(this.nodesKey(), String(this._instanceId), hostname());
+        } catch (err) {
+            // Clean up subscriber on partial init failure
+            try {
+                await subClient.unsubscribe();
+                await subClient.quit();
+            } catch {
+                // ignore cleanup errors
+            }
+            this.subscriberClient = null;
+            throw err;
+        }
+
+        this.running = true;
+
+        // Start heartbeat interval
+        this.heartbeatTimer = setInterval(() => this.doHeartbeat(), this.heartbeatIntervalMs);
+
+        // Start leader service for cleanup duties
+        this.leaderService = new LeaderService(`mesh:${this.key}`, this.leaderOptions);
+        this.leaderService.start();
+
+        this.logger.info('mesh node started', { instanceId: this._instanceId, key: this.key });
+    }
+
+    async stop(): Promise<void> {
+        this.running = false;
+
+        // Stop leader service
+        if (this.leaderService) {
+            await this.leaderService.stop();
+            this.leaderService = null;
+        }
+
+        // Stop heartbeat
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = null;
+        }
+
+        // Clear all active handler heartbeat intervals
+        for (const interval of this.activeHandlerIntervals) {
+            clearInterval(interval);
+        }
+        this.activeHandlerIntervals.clear();
+
+        // Reject all pending requests
+        for (const [requestId, pending] of this.pendingRequests) {
+            clearTimeout(pending.timer);
+            pending.reject(new Error('MeshService stopped'));
+            this.pendingRequests.delete(requestId);
+        }
+
+        // Unsubscribe and quit subscriber
+        if (this.subscriberClient) {
+            try {
+                await this.subscriberClient.unsubscribe();
+                await this.subscriberClient.quit();
+            } catch {
+                // ignore errors during cleanup
+            }
+            this.subscriberClient = null;
+        }
+
+        // Remove self from heartbeats and nodes
+        try {
+            const { client } = getMeshRedis();
+            await client.zrem(this.heartbeatsKey(), String(this._instanceId));
+            await client.hdel(this.nodesKey(), String(this._instanceId));
+        } catch {
+            // ignore errors during cleanup
+        }
+
+        this.logger.info('mesh node stopped', { instanceId: this._instanceId, key: this.key });
+    }
+
+    // --- Private ---
+
+    private nextIdKey(): string {
+        return `${this.prefix}:mesh:${this.key}:next_id`;
+    }
+
+    private heartbeatsKey(): string {
+        return `${this.prefix}:mesh:${this.key}:heartbeats`;
+    }
+
+    private nodesKey(): string {
+        return `${this.prefix}:mesh:${this.key}:nodes`;
+    }
+
+    private channelForInstance(instanceId: number): string {
+        return `${this.prefix}:mesh:${this.key}:node:${instanceId}`;
+    }
+
+    private async doHeartbeat(): Promise<void> {
+        if (!this.running) return;
+
+        try {
+            const { client } = getMeshRedis();
+            await client.HEARTBEAT(this.heartbeatsKey(), String(this._instanceId));
+        } catch (err) {
+            this.logger.warn('mesh heartbeat failed', { err });
+        }
+
+        // If leader, run cleanup
+        if (this.leaderService?.isLeader) {
+            await this.doCleanup();
+        }
+    }
+
+    private async doCleanup(): Promise<void> {
+        try {
+            const { client } = getMeshRedis();
+            const expired = await client.CLEANUP(this.heartbeatsKey(), this.nodesKey(), String(this.nodeTtlMs));
+
+            if (expired.length > 0) {
+                this.logger.info('cleaned up expired mesh nodes', { expired });
+
+                for (const idStr of expired) {
+                    const id = parseInt(idStr, 10);
+                    try {
+                        await this.nodeCleanedUpCallback?.(id);
+                    } catch (err) {
+                        this.logger.warn('node cleanup callback error', { err, instanceId: id });
+                    }
+                }
+            }
+        } catch (err) {
+            this.logger.warn('mesh cleanup failed', { err });
+        }
+    }
+
+    private handleMessage(raw: string): void {
+        if (!this.running) return;
+
+        let msg: unknown;
+        try {
+            msg = JSON.parse(raw);
+        } catch (err) {
+            this.logger.warn('failed to parse mesh message', { err, raw });
+            return;
+        }
+
+        if (typeof msg !== 'object' || msg === null) {
+            this.logger.warn('invalid mesh message: not an object', { raw });
+            return;
+        }
+
+        const obj = msg as Record<string, unknown>;
+
+        if ('heartbeat' in obj && obj.heartbeat) {
+            this.handleHeartbeatMessage(obj as unknown as MeshHeartbeat);
+        } else if ('reply' in obj && obj.reply) {
+            this.handleResponseMessage(obj as unknown as MeshResponse);
+        } else {
+            this.handleRequestMessage(obj as unknown as MeshRequest);
+        }
+    }
+
+    private handleHeartbeatMessage(msg: MeshHeartbeat): void {
+        const pending = this.pendingRequests.get(msg.requestId);
+        if (!pending) return;
+
+        // Reset timeout
+        clearTimeout(pending.timer);
+        pending.timer = setTimeout(() => {
+            this.pendingRequests.delete(msg.requestId);
+            pending.reject(new MeshRequestTimeoutError(pending.targetInstanceId, pending.type));
+        }, this.requestTimeoutMs);
+    }
+
+    private handleResponseMessage(msg: MeshResponse): void {
+        const pending = this.pendingRequests.get(msg.requestId);
+        if (!pending) return;
+
+        clearTimeout(pending.timer);
+        this.pendingRequests.delete(msg.requestId);
+
+        if (msg.error !== undefined) {
+            if (typeof msg.error === 'string' && msg.error.startsWith(NO_HANDLER_ERROR_PREFIX)) {
+                pending.reject(new MeshNoHandlerError(msg.error.slice(NO_HANDLER_ERROR_PREFIX.length)));
+            } else {
+                pending.reject(new MeshHandlerError(msg.error));
+            }
+        } else {
+            pending.resolve(msg.data);
+        }
+    }
+
+    private handleRequestMessage(msg: MeshRequest): void {
+        if (!this.running) return;
+
+        if (typeof msg.requestId !== 'string' || typeof msg.senderInstanceId !== 'number' || typeof msg.type !== 'string') {
+            this.logger.warn('invalid mesh request: missing or malformed fields', { msg });
+            return;
+        }
+
+        const handler = this.handlers.get(msg.type);
+        const { client } = getMeshRedis();
+        const senderChannel = this.channelForInstance(msg.senderInstanceId);
+
+        if (!handler) {
+            const response: MeshResponse = {
+                requestId: msg.requestId,
+                reply: true,
+                error: `${NO_HANDLER_ERROR_PREFIX}${msg.type}`
+            };
+            client.publish(senderChannel, JSON.stringify(response)).catch(err => {
+                this.logger.warn('failed to publish no-handler response', { err });
+            });
+            return;
+        }
+
+        // Use the caller's timeout for heartbeat interval (fall back to our own)
+        const effectiveTimeoutMs = msg.timeoutMs ?? this.requestTimeoutMs;
+        const heartbeatInterval = setInterval(() => {
+            const heartbeat: MeshHeartbeat = {
+                requestId: msg.requestId,
+                heartbeat: true
+            };
+            client.publish(senderChannel, JSON.stringify(heartbeat)).catch(err => {
+                this.logger.warn('failed to publish handler heartbeat', { err });
+            });
+        }, effectiveTimeoutMs * 0.75);
+        this.activeHandlerIntervals.add(heartbeatInterval);
+
+        // Execute handler
+        Promise.resolve()
+            .then(() => handler(msg.data))
+            .then(result => {
+                clearInterval(heartbeatInterval);
+                this.activeHandlerIntervals.delete(heartbeatInterval);
+                const response: MeshResponse = {
+                    requestId: msg.requestId,
+                    reply: true,
+                    data: result
+                };
+                client.publish(senderChannel, JSON.stringify(response)).catch(err => {
+                    this.logger.warn('failed to publish handler response', { err });
+                });
+            })
+            .catch(err => {
+                clearInterval(heartbeatInterval);
+                this.activeHandlerIntervals.delete(heartbeatInterval);
+                const response: MeshResponse = {
+                    requestId: msg.requestId,
+                    reply: true,
+                    error: err instanceof Error ? err.message : String(err)
+                };
+                client.publish(senderChannel, JSON.stringify(response)).catch(publishErr => {
+                    this.logger.warn('failed to publish handler error response', { err: publishErr });
+                });
+            });
+    }
+}

package/src/services/worker/bootstrap.ts

@@ -0,0 +1,53 @@
+import { App } from '@deepkit/app';
+import { eventDispatcher } from '@deepkit/event';
+import { onServerMainBootstrapDone, onServerShutdown } from '@deepkit/framework';
+import { Redis } from 'ioredis';
+
+import { getAppConfig } from '../../app/resolver';
+import { isDevelopment } from '../../app/const';
+import { globalState } from '../../app/state';
+import { WorkerQueueJobCommand, WorkerStartCommand } from './cli';
+import { JobEntity } from './entity';
+import { WorkerQueueRegistry } from './queue';
+import { WorkerRunnerService } from './runner';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function installWorkerComponents(app: App<any>) {
+    globalState.enableWorker = true;
+    globalState.additionalEntities.push(JobEntity);
+
+    app.appModule.addProvider(WorkerRunnerService);
+
+    app.appModule.addController(WorkerStartCommand);
+    app.appModule.addController(WorkerQueueJobCommand);
+
+    class WorkerListener {
+        constructor(private runner: WorkerRunnerService) {}
+
+        @eventDispatcher.listen(onServerShutdown)
+        async shutdownRunner() {
+            await this.runner.shutdown();
+
+            // prevent new Redis connections from being created after shutdown
+            // some issue in dev keeping processes alive forever
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (Redis.prototype as any).connect = () => {};
+        }
+
+        // we want this to run very late in the process, after the worker has shut down
+        @eventDispatcher.listen(onServerShutdown, 1000)
+        async closeQueues() {
+            await WorkerQueueRegistry.closeQueues();
+        }
+    }
+    app.appModule.addListener(WorkerListener);
+
+    app.listen(onServerMainBootstrapDone, () => {
+        if (!globalState.isCliService) {
+            setTimeout(() => {
+                const config = getAppConfig();
+                if (config.ENABLE_JOB_RUNNER ?? isDevelopment) app.get(WorkerRunnerService).start();
+            }, 1000);
+        }
+    });
+}

package/src/services/worker/cli.ts

@@ -0,0 +1,32 @@
+import { cli } from '@deepkit/app';
+
+import { CliServiceCommand } from '../cli';
+import { WorkerQueueRegistry } from './queue';
+import { WorkerRunnerService } from './runner';
+import { BaseJob } from './types';
+
+@cli.controller('worker:start', {
+    description: 'Start the worker runner'
+})
+export class WorkerStartCommand extends CliServiceCommand {
+    constructor(private runner: WorkerRunnerService) {
+        super();
+    }
+
+    async startService() {
+        await this.runner.start();
+    }
+}
+
+@cli.controller('worker:queue', {
+    // todo: optional queue name?
+    description: 'Queue a job by name'
+})
+export class WorkerQueueJobCommand {
+    async execute(jobName: string, data?: string) {
+        data = data ? JSON.parse(data) : {};
+        const queue = WorkerQueueRegistry.getQueue(BaseJob.QUEUE_NAME);
+        await queue.add(jobName, data);
+        await WorkerQueueRegistry.closeQueues();
+    }
+}

package/src/services/worker/entity.ts

@@ -0,0 +1,22 @@
+import { entity, Index, PrimaryKey } from '@deepkit/type';
+
+import { BaseEntity } from '../../database';
+
+@entity.name('_jobs')
+export class JobEntity extends BaseEntity {
+    id!: string & PrimaryKey;
+    queue!: string & Index;
+    queueId!: string;
+    attempt!: number;
+    name!: string;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    data!: any;
+    traceId!: string | null;
+    status!: 'completed' | 'failed';
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    result!: any;
+    createdAt!: Date;
+    shouldExecuteAt!: Date;
+    executedAt!: Date;
+    completedAt!: Date;
+}

package/src/services/worker/index.ts

@@ -0,0 +1,30 @@
+import { ClassType } from '@deepkit/core';
+
+import { isTest } from '../../app/const';
+import { createLogger } from '../logger';
+import { WorkerQueueRegistry } from './queue';
+import { BaseJob, IJobOptions, InputDataSymbol, JobClass } from './types';
+
+export * from './entity';
+export { BaseJob, WorkerJob } from './types';
+
+export class WorkerService {
+    private logger = createLogger(this);
+
+    async queueJob<I extends object, O, T extends BaseJob<I, O>>(jobClass: ClassType<T>, data: T[typeof InputDataSymbol], options?: IJobOptions) {
+        if (isTest) {
+            this.logger.warn('Not queueing job in test environment', { jobName: jobClass.name, data, options });
+            return;
+        }
+
+        try {
+            const typedJob = jobClass as unknown as JobClass;
+            const queue = WorkerQueueRegistry.getQueue(typedJob.QUEUE_NAME);
+            const job = await queue.add(jobClass.name, data, options);
+            this.logger.info('Queued job', { job: { name: jobClass.name, id: job.id } });
+        } catch (err) {
+            this.logger.error('Failed to queue job', err, { job: { name: jobClass.name } });
+            throw err;
+        }
+    }
+}

package/src/services/worker/queue.ts

@@ -0,0 +1,35 @@
+import { Queue } from 'bullmq';
+
+import { getAppConfig } from '../../app/resolver';
+import { createRedisOptions } from '../../helpers/redis/redis';
+
+export class WorkerQueueRegistry {
+    static registry = new Map<string, Queue>();
+
+    static getDefaultQueue() {
+        const appConfig = getAppConfig();
+        return this.getQueue(appConfig.BULL_QUEUE);
+    }
+
+    static getQueue(name: string): Queue {
+        if (!this.registry.has(name)) {
+            const { options, prefix } = createRedisOptions('BULL');
+            this.registry.set(
+                name,
+                new Queue(name, {
+                    connection: options,
+                    prefix: `${prefix}:bmq`
+                })
+            );
+        }
+
+        return this.registry.get(name)!;
+    }
+
+    static async closeQueues() {
+        for (const queue of this.registry.values()) {
+            await queue.close();
+        }
+        this.registry.clear();
+    }
+}