camera.ui3 0.0.30-alpha.0

package/dist/server/src/polyglot/node/decoder/child.js

@@ -0,0 +1,644 @@
+process.title = `camera.ui - ${process.argv[2]}`;
+import { ErrorCode, NatsError } from 'nats';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { ReplaySubject, firstValueFrom, fromEvent, timeout } from 'rxjs';
+import { first, takeUntil } from 'rxjs/operators';
+import { ProxyConnection } from '../../../nats/connection.js';
+import { pack, unpack } from '../../../utils/packer.js';
+import { readLength } from '../../../utils/reader.js';
+import { Subscribed } from '../../../utils/subscribed.js';
+import { sleep } from '../../../utils/utils.js';
+import { PluginLogger } from '../plugins/pluginLogger.js';
+import { ImageUtils } from './imageUtils.js';
+const PIX_FMT = 'yuv420p';
+class FrameWorkerChild extends Subscribed {
+    clients = new Map();
+    logger;
+    config;
+    subject;
+    ipcSubject;
+    imageSubject;
+    signalingSubject;
+    frameSubject;
+    publisher;
+    signalPublisher;
+    imagePublisher;
+    framePublisher;
+    subscriber;
+    imageSubscriber;
+    signalSubscriber;
+    maxDiffFrame = 3;
+    stopped = false;
+    closed = false;
+    restarting = false;
+    restartDelay = 5000;
+    restartTimeout;
+    closeDelay = 5000;
+    closeTimeout;
+    restartWatchdogDelay = 5000;
+    imageUtils;
+    ffmpegProcess;
+    prebufferFull = false;
+    prebufferDuration = 7000;
+    prebufferContainer = [];
+    metadataSubject = new ReplaySubject(1);
+    get metadata() {
+        return firstValueFrom(this.metadataSubject);
+    }
+    get logPrefix() {
+        return `${this.config.cameraName}:`;
+    }
+    constructor() {
+        super();
+        this.config = {
+            frameWorkerId: process.env.FRAME_WORKER_ID,
+            cameraId: process.env.CAMERA_ID,
+            cameraName: process.env.CAMERA_NAME,
+            ffmpegPath: process.env.FFMPEG_PATH,
+            logLevel: process.env.LOGGER_LEVEL ?? 'debug',
+            disableTimestamps: process.env.LOGGER_DISABLE_TIMESTAMPS === 'true',
+            auth: { user: process.env.PROXY_USER, password: process.env.PROXY_PASSWORD },
+            proxyEndpoints: process.env.PROXY_ENDPOINTS.split(','),
+            fps: parseInt(process.env.FPS),
+            resolution: parseInt(process.env.RESOLUTION),
+        };
+        delete process.env.FRAME_WORKER_ID;
+        delete process.env.CAMERA_ID;
+        delete process.env.CAMERA_NAME;
+        delete process.env.FFMPEG_PATH;
+        delete process.env.PROXY_USER;
+        delete process.env.PROXY_PASSWORD;
+        delete process.env.PROXY_ENDPOINTS;
+        delete process.env.FPS;
+        delete process.env.RESOLUTION;
+        this.logger = new PluginLogger('Frame Worker', this.config.disableTimestamps ?? false);
+        this.logger.debugEnabled = this.config.logLevel === 'debug' || this.config.logLevel === 'trace';
+        this.logger.traceEnabled = this.config.logLevel === 'trace';
+        this.subject = `${this.config.frameWorkerId}.frameWorkerChild`;
+        this.ipcSubject = `${this.config.frameWorkerId}.frameWorker`;
+        this.signalingSubject = `${this.config.cameraId}.frameGenerator.signaling`;
+        this.imageSubject = `${this.config.cameraId}.frameGenerator`;
+        this.frameSubject = `${this.config.cameraId}.frameReceiver`;
+        this.imageUtils = new ImageUtils(this.logger);
+        this.signalHander();
+        this.listen();
+    }
+    async listen() {
+        const servers = this.config.proxyEndpoints.filter((endpoint) => endpoint.startsWith('nats://'));
+        this.publisher = new ProxyConnection(this.subject, servers, this.config.auth);
+        await this.publisher.connect();
+        this.imagePublisher = new ProxyConnection(this.imageSubject, servers, this.config.auth);
+        await this.imagePublisher.connect();
+        this.framePublisher = new ProxyConnection(this.frameSubject, servers, this.config.auth);
+        await this.framePublisher.connect();
+        this.signalPublisher = new ProxyConnection(this.signalingSubject, servers, this.config.auth);
+        await this.signalPublisher.connect();
+        this.subscriber = this.publisher.subscribe(this.subject);
+        this.subscriber.on('message', this.onWorkerMessage.bind(this));
+        this.imageSubscriber = this.imagePublisher.subscribe(this.imageSubject);
+        this.imageSubscriber.on('message', this.onImageRequest.bind(this));
+        this.signalSubscriber = this.signalPublisher.subscribe(this.signalingSubject);
+        this.signalSubscriber.on('message', this.onSignalingMessage.bind(this));
+        await this.imageUtils.initiate();
+        this.send('started');
+    }
+    async close() {
+        this.closed = true;
+        // await this.stop();
+        await Promise.allSettled([this.publisher?.close(), this.imagePublisher?.close(), this.signalPublisher?.close(), this.framePublisher?.close()]);
+    }
+    async start() {
+        try {
+            this.closed = false;
+            this.restarting = false;
+            this.stopped = false;
+            clearTimeout(this.closeTimeout);
+            await this.startVideoSession();
+        }
+        catch (error) {
+            this.logger.error(this.logPrefix, 'An error occurred while starting frame session:', error);
+            await this.restartFFmpeg();
+        }
+    }
+    async stop() {
+        this.stopped = true;
+        await this.disconnectAllClients();
+        this.reset();
+    }
+    reset() {
+        this.resetPrebuffer();
+        this.resetMetadata();
+        this.resetVideoSession();
+        this.unsubscribe();
+    }
+    resetPrebuffer() {
+        this.prebufferFull = false;
+        this.prebufferContainer = [];
+    }
+    resetMetadata() {
+        this.metadataSubject = new ReplaySubject(1);
+    }
+    resetVideoSession() {
+        clearTimeout(this.restartTimeout);
+        clearTimeout(this.closeTimeout);
+        if (this.ffmpegProcess) {
+            this.logger.debug(this.logPrefix, 'Stopping FFmpeg process...');
+            this.ffmpegProcess.kill('SIGKILL');
+            this.ffmpegProcess = undefined;
+        }
+    }
+    async startVideoSession() {
+        this.resetPrebuffer();
+        this.resetVideoSession();
+        this.unsubscribe();
+        try {
+            await this.spawnFFmpeg();
+            await firstValueFrom(this.metadataSubject.pipe(timeout(15000)));
+            this.logger.debug(this.logPrefix, 'Frame session started');
+        }
+        catch {
+            throw new Error('FFmpeg process timed out');
+        }
+    }
+    buildFfmpegArguments(cameraSource, ffmpegArgs) {
+        return [
+            '-hide_banner',
+            // '-vsync',
+            // '0',
+            '-hwaccel',
+            ffmpegArgs.hwaccel,
+            ...ffmpegArgs.hwaccelArgs,
+            '-fflags',
+            'nobuffer',
+            '-flags',
+            'low_delay',
+            '-analyzeduration',
+            '0',
+            '-probesize',
+            '500000',
+            '-reorder_queue_size',
+            '0',
+            '-user_agent',
+            'camera.ui/decoder',
+            '-rtsp_transport',
+            'tcp',
+            '-i',
+            cameraSource,
+            '-threads',
+            ffmpegArgs.threads,
+            '-vf',
+            `showinfo=checksum=0,${ffmpegArgs.hwaccelFilter.replace('640', this.config.resolution.toString())},fps=fps=${this.config.fps}`,
+            '-an',
+            '-dn',
+            '-sn',
+            '-vcodec',
+            'rawvideo',
+            '-f',
+            'rawvideo',
+            '-pix_fmt',
+            PIX_FMT,
+            '-',
+        ];
+    }
+    async spawnFFmpeg() {
+        const response = (await this.send('getFFmpegArgs'));
+        if (!response) {
+            throw new Error('Error getting FFmpeg args from main process');
+        }
+        const args = this.buildFfmpegArguments(response.cameraSource, response.ffmpegArgs);
+        this.logger.debug(this.logPrefix, 'Starting FFmpeg process:', this.config.ffmpegPath, args.join(' '));
+        this.ffmpegProcess = spawn(this.config.ffmpegPath, args);
+        this.addSubscriptions(this.handleFFmpegStderr(), this.handleFFmpegClose());
+        this.handleFFmpegStdout();
+    }
+    async restartFFmpeg() {
+        if (!this.stopped && !this.closed && !this.restarting && this.clients.size > 0) {
+            this.restarting = true;
+            this.logger.debug(this.logPrefix, `Restarting FFmpeg process in ${this.restartDelay / 1000} seconds...`);
+            await sleep(this.restartDelay);
+            if (!this.stopped && !this.closed && this.clients.size > 0) {
+                await this.start();
+            }
+            else {
+                this.restarting = false;
+            }
+        }
+    }
+    handleFFmpegStderr() {
+        const stderrStream = fromEvent(this.ffmpegProcess.stderr, 'data');
+        return stderrStream.pipe(takeUntil(this.metadataSubject.pipe(first()))).subscribe((data) => {
+            const messageString = data.toString();
+            const metadata = this.parseFrameInfo(messageString);
+            if (metadata) {
+                this.logger.debug(this.logPrefix, 'Metadata:', metadata);
+                this.metadataSubject.next(metadata);
+            }
+        });
+    }
+    async handleFFmpegStdout() {
+        const metadata = await this.metadata;
+        let shifts = 0;
+        let warned = false;
+        const isClosed = () => this.stopped || this.closed || !this.framePublisher || !this.ffmpegProcess || !this.ffmpegProcess.stdout || this.ffmpegProcess.exitCode !== null;
+        const restartWatchdog = () => {
+            clearTimeout(this.restartTimeout);
+            this.restartTimeout = setTimeout(() => {
+                if (!this.stopped && !this.closed && !this.restarting) {
+                    const message = 'FFmpeg process is not producing frames, terminating...';
+                    if (!warned) {
+                        this.logger.debug(this.logPrefix, message);
+                        warned = true;
+                    }
+                    else {
+                        this.logger.error(this.logPrefix, message);
+                    }
+                    this.reset();
+                    this.restartFFmpeg();
+                }
+            }, this.restartWatchdogDelay);
+        };
+        while (!isClosed()) {
+            try {
+                restartWatchdog();
+                const frame = await readLength(this.ffmpegProcess.stdout, metadata.frameSize);
+                warned = false;
+                if (isClosed()) {
+                    clearTimeout(this.restartTimeout);
+                    break;
+                }
+                const frameId = this.generateId();
+                const now = Date.now();
+                this.prebufferContainer.push({
+                    frameId: frameId,
+                    time: now,
+                    frame,
+                });
+                if (this.prebufferContainer.length > 0 && now - this.prebufferContainer[0].time >= this.prebufferDuration && !this.prebufferFull) {
+                    this.logger.debug(this.logPrefix, 'Prebuffer is full');
+                    this.prebufferFull = true;
+                }
+                while (this.prebufferContainer.length > 0 && this.prebufferContainer[0].time < now - this.prebufferDuration) {
+                    this.prebufferContainer.shift();
+                    shifts++;
+                }
+                if (shifts > 1000) {
+                    this.prebufferContainer = [...this.prebufferContainer];
+                    shifts = 0;
+                }
+                if (this.prebufferFull) {
+                    const frameData = {
+                        frameId,
+                        timestamp: now,
+                    };
+                    this.framePublisher.publish(this.frameSubject, frameData);
+                }
+            }
+            catch (error) {
+                clearTimeout(this.restartTimeout);
+                if (!isClosed() && !this.restarting) {
+                    this.logger.error(this.logPrefix, 'An error occurred while reading ffmpeg stdout:', error);
+                }
+                break;
+            }
+        }
+    }
+    handleFFmpegClose() {
+        const onClose = fromEvent(this.ffmpegProcess, 'exit');
+        return onClose.subscribe(() => {
+            if (this.ffmpegProcess) {
+                if (this.ffmpegProcess.exitCode !== 0) {
+                    this.logger.error(this.logPrefix, 'FFmpeg process exited with non-zero code', this.ffmpegProcess.exitCode);
+                }
+                else {
+                    this.logger.debug(this.logPrefix, 'FFmpeg process exited with code', this.ffmpegProcess.exitCode);
+                }
+            }
+            this.ffmpegProcess = undefined;
+            this.reset();
+            this.restartFFmpeg();
+        });
+    }
+    parseFrameInfo(stderr) {
+        const patterns = {
+            s: /s:\s*(\d+x\d+)/,
+        };
+        const matches = Object.entries(patterns).reduce((acc, [key, pattern]) => {
+            const match = stderr.match(pattern);
+            if (match) {
+                acc[key] = match[1];
+            }
+            return acc;
+        }, {});
+        if (!matches.s) {
+            return;
+        }
+        const [width, height] = matches.s.split('x').map(Number);
+        const newWidth = this.config.resolution;
+        const newHeight = Math.round((newWidth * height) / width);
+        const format = PIX_FMT === 'yuv420p' ? 'yuv' : 'rgb';
+        const bytesPerPixel = format === 'yuv' ? 1.5 : 3; // YUV420P or RGB24
+        const frameSize = newWidth * newHeight * bytesPerPixel;
+        const metadata = {
+            format,
+            frameSize,
+            width: newWidth,
+            height: newHeight,
+            origHeight: height,
+            origWidth: width,
+        };
+        return metadata;
+    }
+    async send(message, data = {}) {
+        if (!process.send) {
+            throw new Error('Main Process not initialized');
+        }
+        if (!this.publisher) {
+            throw new Error('Publisher not initialized');
+        }
+        const workerMessage = { message, data };
+        const response = await this.publisher.request(this.ipcSubject, workerMessage, { timeout: 30000 });
+        return response.response;
+    }
+    async addClient(pluginId, requestId) {
+        if (!this.clients.has(pluginId)) {
+            this.clients.set(pluginId, { pluginId, requestIds: [requestId] });
+        }
+        const client = this.clients.get(pluginId);
+        if (!client.requestIds.includes(requestId)) {
+            client.requestIds.push(requestId);
+        }
+        this.logger.trace(this.logPrefix, `New client ${this.clientIdentifier(pluginId, requestId)}`);
+        clearTimeout(this.closeTimeout);
+        await this.send('updateClient', client);
+        if (this.clients.size > 0 && !this.ffmpegProcess && !this.restarting) {
+            await this.start();
+        }
+    }
+    async removeClient(pluginId, requestId) {
+        const client = this.clients.get(pluginId);
+        if (!client || !client.requestIds.includes(requestId)) {
+            return;
+        }
+        this.logger.trace(this.logPrefix, `Removing client ${this.clientIdentifier(pluginId, requestId)}`);
+        client.requestIds = client.requestIds.filter((id) => id !== requestId);
+        if (client.requestIds.length === 0) {
+            this.clients.delete(pluginId);
+            await this.send('removeClient', client);
+        }
+        else {
+            await this.send('updateClient', client);
+        }
+        if (this.clients.size === 0) {
+            clearTimeout(this.closeTimeout);
+            this.closeTimeout = setTimeout(() => this.stop(), this.closeDelay);
+        }
+    }
+    async recoverClients(clients) {
+        clients.forEach((client) => {
+            client.requestIds.forEach((requestId) => this.logger.trace(this.logPrefix, `Recovering client: ${this.clientIdentifier(client.pluginId, requestId)}`));
+            this.clients.set(client.pluginId, client);
+        });
+        if (this.clients.size > 0) {
+            await this.start();
+        }
+    }
+    disconnectClient(cameraId, pluginId, requestId) {
+        const client = this.clients.get(pluginId);
+        if (!this.signalPublisher || !client || !client.requestIds.includes(requestId)) {
+            return;
+        }
+        const pingTargetSubject = `${pluginId}.camera.${cameraId}.signaling`;
+        const response = { type: 'disconnect', requestId };
+        this.signalPublisher.publish(pingTargetSubject, response);
+        this.removeClient(pluginId, requestId);
+    }
+    async disconnectAllClients() {
+        this.clients.forEach((client) => {
+            client.requestIds.forEach(async (requestId) => {
+                this.disconnectClient(this.config.cameraId, client.pluginId, requestId);
+            });
+        });
+    }
+    async handleConnectMessage(message) {
+        await this.addClient(message.pluginId, message.requestId);
+        this.startPingPongForClient(message.cameraId, message.pluginId, message.requestId);
+    }
+    async handleDisconnectMessage(message) {
+        this.removeClient(message.pluginId, message.requestId);
+    }
+    async handleMetadataMessage(msg, message) {
+        const { requestId } = message;
+        const metadata = await this.metadata;
+        const response = { type: 'metadata', requestId, metadata };
+        msg.respond(pack(response));
+    }
+    async startPingPongForClient(cameraId, pluginId, requestId) {
+        const pingTargetSubject = `${pluginId}.camera.${cameraId}.signaling`;
+        const pingInterval = 3000;
+        const pongTimeout = 2000;
+        const requestIdExist = () => this.clients.has(pluginId) && this.clients.get(pluginId)?.requestIds.includes(requestId);
+        while (this.signalPublisher && requestIdExist()) {
+            try {
+                const message = { type: 'ping', requestId };
+                // this.logger.trace(`Sending ping to ${this.clientIdentifier(pluginId, requestId)}`);
+                const response = await this.signalPublisher.request(pingTargetSubject, message, { timeout: pongTimeout });
+                if (!requestIdExist()) {
+                    break;
+                }
+                if (response.type !== 'pong') {
+                    throw new Error('Invalid response to ping');
+                }
+                // this.logger.trace(`Received pong from ${this.clientIdentifier(pluginId, requestId)}`);
+                await sleep(pingInterval);
+            }
+            catch (error) {
+                if (requestIdExist()) {
+                    if (error instanceof NatsError && (error.code === ErrorCode.Timeout || error.code === ErrorCode.NoResponders)) {
+                        // this.logger.trace(`Ping/Pong timeout for ${this.clientIdentifier(pluginId, requestId)}`);
+                        this.removeClient(pluginId, requestId);
+                    }
+                    else {
+                        this.logger.error('Error during ping/pong:', error);
+                    }
+                }
+                break;
+            }
+        }
+    }
+    async onWorkerMessage(msg) {
+        if (this.closed) {
+            return;
+        }
+        this.respondOnIpcMessage(msg);
+    }
+    async onSignalingMessage(msg) {
+        if (this.closed) {
+            return;
+        }
+        this.respondOnSignalingMessage(msg);
+    }
+    async onImageRequest(msg) {
+        if (this.closed) {
+            return;
+        }
+        this.respondOnPrebufferedImageMessage(msg);
+    }
+    async respondOnIpcMessage(msg) {
+        try {
+            const mainMessage = unpack(msg.data);
+            const response = {};
+            const message = mainMessage.message;
+            switch (message) {
+                case 'setCameraName':
+                    this.config.cameraName = mainMessage.data.cameraName;
+                    break;
+                case 'recoverClients':
+                    this.recoverClients(mainMessage.data.clients);
+                    break;
+            }
+            msg.respond(pack(response));
+        }
+        catch (error) {
+            this.logger.error(this.logPrefix, 'Error while processing IPC respond:', error);
+        }
+    }
+    async respondOnSignalingMessage(msg) {
+        try {
+            const message = unpack(msg.data);
+            switch (message.type) {
+                case 'connect':
+                    this.handleConnectMessage(message);
+                    break;
+                case 'disconnect':
+                    this.handleDisconnectMessage(message);
+                    break;
+                case 'metadata':
+                    this.handleMetadataMessage(msg, message);
+                    break;
+                default:
+                    break;
+            }
+        }
+        catch (error) {
+            this.logger.error(this.logPrefix, 'Error while processing signaling respond:', error);
+        }
+    }
+    async respondOnPrebufferedImageMessage(msg) {
+        try {
+            const message = unpack(msg.data);
+            const { metadata, frameData, prebufferDuration, options } = message;
+            const { width, height, format } = metadata;
+            const targetFrame = this.findTargetFrame(frameData, prebufferDuration);
+            if (!targetFrame) {
+                this.logFrameNotFound();
+            }
+            else {
+                const processedImage = await this.processFrame(targetFrame.frame, width, height, format, options);
+                msg.respond(pack(processedImage));
+                return;
+            }
+            msg.respond(pack(new Uint8Array(0)));
+        }
+        catch (error) {
+            this.logger.error(this.logPrefix, 'Error while processing image respond:', error);
+        }
+    }
+    findTargetFrame(frameData, prebufferDuration) {
+        const targetIndex = this.prebufferContainer.findIndex((frame) => frame.frameId === frameData.frameId);
+        if (targetIndex === -1) {
+            return this.findNearestFrame(frameData);
+        }
+        if (prebufferDuration > 0) {
+            const targetTime = this.prebufferContainer[targetIndex].time - prebufferDuration * 1000;
+            const result = this.findNearestFrameByTime(targetTime);
+            if (result) {
+                return result;
+            }
+        }
+        return this.prebufferContainer[targetIndex];
+    }
+    findNearestFrame(frameData) {
+        const maxTimeDiff = this.maxDiffFrame * (1000 / this.config.fps);
+        return this.prebufferContainer.reduce((nearest, current) => {
+            const currentTimeDiff = Math.abs(current.time - frameData.timestamp);
+            if (currentTimeDiff <= maxTimeDiff) {
+                if (!nearest)
+                    return current;
+                const nearestTimeDiff = Math.abs(nearest.time - frameData.timestamp);
+                return currentTimeDiff < nearestTimeDiff ? current : nearest;
+            }
+            return nearest;
+        }, undefined);
+    }
+    findNearestFrameByTime(targetTime) {
+        const maxTimeDiff = (this.maxDiffFrame / this.config.fps) * 1000;
+        return this.prebufferContainer.reduce((nearest, current) => {
+            if (current.time <= targetTime && Math.abs(current.time - targetTime) <= maxTimeDiff) {
+                if (!nearest || Math.abs(current.time - targetTime) < Math.abs(nearest.time - targetTime)) {
+                    return current;
+                }
+            }
+            return nearest;
+        }, undefined);
+    }
+    async processFrame(frame, width, height, format, options) {
+        const newFrame = new Uint8Array(frame);
+        return await this.imageUtils.processImage(newFrame, width, height, format, options);
+    }
+    logFrameNotFound() {
+        if (this.prebufferContainer.length > 0) {
+            this.logger.warn(this.logPrefix, 'Frame not found in prebuffer');
+        }
+        else {
+            this.logger.warn(this.logPrefix, 'Prebuffer is empty! FFmpeg is not producing frames?');
+        }
+    }
+    clientIdentifier(pluginId, requestId) {
+        return `${pluginId}:${requestId}`;
+    }
+    generateId() {
+        return Math.random().toString(36).slice(2, 9);
+    }
+    signalHander() {
+        let closePromise;
+        const gracefullyClose = async (signal, reason, error, promise) => {
+            if (closePromise) {
+                return closePromise;
+            }
+            if (error || (promise && reason)) {
+                this.logger.debug(this.logPrefix, `is ungracefully closing from signal="${signal}" reason="${reason || 'unknown'}"...`);
+                if (error) {
+                    if (typeof error === 'string') {
+                        this.logger.error(this.logPrefix, error);
+                    }
+                    else {
+                        this.logger.error(this.logPrefix, `${signal}`, error);
+                    }
+                }
+                else {
+                    this.logger.error(this.logPrefix, `${signal}: Unhandled promise rejection:`, reason || promise);
+                }
+            }
+            else {
+                this.logger.log(this.logPrefix, `is gracefully closing from signal="${signal}" reason="${reason || 'unknown'}"...`);
+            }
+            const shutdown = setTimeout(() => {
+                this.logger.warn(this.logPrefix, 'Failed to gracefully close before timeout, force quit!');
+                process.kill(process.pid, 'SIGKILL');
+            }, 5000);
+            closePromise = this.close();
+            await closePromise;
+            clearTimeout(shutdown);
+            process.exit(0);
+        };
+        process.on('uncaughtException', (error) => gracefullyClose('uncaughtException', undefined, error));
+        process.on('unhandledRejection', (reason, promise) => gracefullyClose('unhandledRejection', reason, undefined, promise));
+        process.on('SIGTERM', gracefullyClose);
+        process.on('SIGINT', gracefullyClose);
+    }
+}
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+    new FrameWorkerChild();
+}
+//# sourceMappingURL=child.js.map