edge-impulse-linux 1.19.0 → 1.21.0

package/build/library/sensors/gstreamer.js

@@ -20,13 +20,14 @@ const DEFAULT_GST_VIDEO_SOURCE = 'v4l2src';
 class GStreamer extends tsee_1.EventEmitter {
     constructor(verbose, options) {
         super();
-        this._originalFilesSeen = new Set();
+        this._originalAndRgbFilesSeen = new Set();
         this._handledFiles = {};
         this._processing = false;
         this._mode = 'default';
         this._isStarted = false;
         this._isRestarting = false;
         this._emitsOriginalSizeImages = false;
+        this._emitsRgbBuffers = false;
         this._lastGstLaunchCommand = '';
         this._imagesReceived = 0;
         this._offset = 0;
@@ -35,6 +36,10 @@ class GStreamer extends tsee_1.EventEmitter {
         this._spawnHelper = options?.spawnHelperOverride || spawn_helper_1.spawnHelper;
         this._modeOverride = options?.modeOverride;
         this._profiling = options?.profiling || false;
+        this._overrideColorFormat = options?.colorFormat;
+        this._outputRgbBuffers = options?.dontOutputRgbBuffers === true ?
+            false :
+            true;
         if (options?.dontRunCleanupLoop === true) {
             // skip cleanup loop
         }
@@ -60,31 +65,43 @@ class GStreamer extends tsee_1.EventEmitter {
             console.log(PREFIX, 'checking for /etc/os-release');
             osRelease = await fs_1.default.promises.readFile('/etc/os-release', 'utf-8');
         }
+        else {
+            // so we don't need to check for undefined below
+            osRelease = '';
+        }
         let firmwareModel;
         // using /proc/device-tree as recommended in user space.
         if (await this.exists('/proc/device-tree/model')) {
             firmwareModel = await fs_1.default.promises.readFile('/proc/device-tree/model', 'utf-8');
         }
-        if (osRelease) {
-            if ((osRelease.indexOf('bullseye') > -1)
-                || (osRelease.indexOf('bookworm') > -1)) {
-                if (osRelease.indexOf('ID=raspbian') > -1) {
-                    this._mode = 'rpi';
-                }
-                if (firmwareModel && firmwareModel.indexOf('Raspberry Pi') > -1) {
-                    this._mode = 'rpi';
-                }
-            }
-        }
-        if (firmwareModel && firmwareModel.indexOf('Microchip SAMA7G5') > -1) {
-            this._mode = 'microchip';
+        else {
+            // so we don't need to check for undefined below
+            firmwareModel = '';
         }
-        else if (firmwareModel && firmwareModel.indexOf('RB3gen2') > -1 && firmwareModel.indexOf('vision') > -1) {
+        if (firmwareModel.indexOf('RB3gen2') > -1 && firmwareModel.indexOf('vision') > -1) {
             this._mode = 'qualcomm-rb3gen2';
         }
-        else if (firmwareModel && firmwareModel.indexOf('Qualcomm') > -1 && firmwareModel.indexOf('Yupik') > -1) {
+        else if (firmwareModel.indexOf('Qualcomm') > -1 && firmwareModel.indexOf('Yupik') > -1) {
             this._mode = 'qualcomm-yupik';
         }
+        else if (firmwareModel.indexOf('Raspberry Pi') > -1) {
+            if (((osRelease.indexOf('bullseye') > -1) || (osRelease.indexOf('bookworm') > -1))
+                && (osRelease.indexOf('ID=raspbian') === -1)) {
+                this._mode = 'rpi';
+            }
+            else if (osRelease.indexOf('trixie') > -1) {
+                this._mode = 'rpi';
+            }
+            // override to rpi5 if needed
+            if (firmwareModel.indexOf('Raspberry Pi 5') > -1) {
+                this._mode = 'rpi5';
+            }
+        }
+        else if ((firmwareModel.indexOf('Arduino') > -1 && firmwareModel.indexOf('Imola') > -1) ||
+            // this may be incorrect on the another platforms in the future
+            (process.env.EI_CLI_ENV === 'arduino')) {
+            this._mode = 'unoq';
+        }
         this._mode = (this._modeOverride) ? this._modeOverride : this._mode;
     }
     async listDevices() {
@@ -240,16 +257,20 @@ class GStreamer extends tsee_1.EventEmitter {
         }
         let lastPhoto = 0;
         let nextFrame = Date.now();
-        this._originalFilesSeen = new Set();
+        this._originalAndRgbFilesSeen = new Set();
         this._watcher = fs_1.default.watch(this._tempDir, async (eventType, fileName) => {
             if (eventType !== 'rename')
                 return;
             if (fileName === null)
                 return;
-            if (!(fileName.endsWith('.jpeg') || fileName.endsWith('.jpg')))
+            if (!(fileName.endsWith('.jpeg') || fileName.endsWith('.jpg') || fileName.endsWith('.rgb')))
                 return;
             if (fileName.startsWith('original')) {
-                this._originalFilesSeen.add(fileName);
+                this._originalAndRgbFilesSeen.add(fileName);
+                return;
+            }
+            if (fileName.endsWith('.rgb')) {
+                this._originalAndRgbFilesSeen.add(fileName);
                 return;
             }
             if (!this._tempDir)
@@ -257,10 +278,11 @@ class GStreamer extends tsee_1.EventEmitter {
             if (this._handledFiles[fileName])
                 return;
             this._imagesReceived++;
+            const tempDir = this._tempDir; // this can get unset later on -> so cache here
             // not next frame yet?
             if (this._processing || Date.now() < nextFrame) {
                 this._handledFiles[fileName] = true;
-                await this.safeUnlinkFile(path_1.default.join(this._tempDir, fileName));
+                await this.safeUnlinkFile(path_1.default.join(tempDir, fileName));
                 return;
             }
             nextFrame = Date.now() + options.intervalMs;
@@ -269,7 +291,7 @@ class GStreamer extends tsee_1.EventEmitter {
                 this._handledFiles[fileName] = true;
                 const originalName = fileName.replace('resized', 'original');
                 if (this._emitsOriginalSizeImages) {
-                    if (!this._originalFilesSeen.has(originalName)) {
+                    if (!this._originalAndRgbFilesSeen.has(originalName)) {
                         let waitForOriginalStart = Date.now();
                         if (this._verbose) {
                             console.log(PREFIX, `Waiting for original file "${originalName}"...`);
@@ -277,13 +299,13 @@ class GStreamer extends tsee_1.EventEmitter {
                         await new Promise(async (resolve, reject) => {
                             let start = Date.now();
                             while (1) {
-                                if (this._originalFilesSeen.has(originalName)) {
+                                if (this._originalAndRgbFilesSeen.has(originalName)) {
                                     return resolve();
                                 }
                                 if (Date.now() - start > 1000) {
                                     return reject(`Did not find original image ("${originalName}") within 1sec`);
                                 }
-                                await this.wait(10);
+                                await this.wait(1);
                             }
                         });
                         if (this._verbose) {
@@ -292,19 +314,46 @@ class GStreamer extends tsee_1.EventEmitter {
                         }
                     }
                 }
+                const rgbName = fileName.replace('.jpg', '.rgb');
+                if (this._emitsRgbBuffers) {
+                    if (!this._originalAndRgbFilesSeen.has(rgbName)) {
+                        let waitForRgbStart = Date.now();
+                        if (this._verbose) {
+                            console.log(PREFIX, `Waiting for RGB file "${rgbName}"...`);
+                        }
+                        await new Promise(async (resolve, reject) => {
+                            let start = Date.now();
+                            while (1) {
+                                if (this._originalAndRgbFilesSeen.has(rgbName)) {
+                                    return resolve();
+                                }
+                                if (Date.now() - start > 1000) {
+                                    return reject(`Did not find RGB image ("${rgbName}") within 1sec`);
+                                }
+                                await this.wait(1);
+                            }
+                        });
+                        if (this._verbose) {
+                            console.log(PREFIX, `Waiting for RGB file "${rgbName}" OK ` +
+                                `(took ${Date.now() - waitForRgbStart}ms.)`);
+                        }
+                    }
+                }
                 if (lastPhoto !== 0 && this._verbose) {
                     console.log(PREFIX, 'Got snapshot', fileName, 'time since last:', (Date.now() - lastPhoto) + 'ms.', 'size');
                 }
                 if (this._keepAliveTimeout) {
                     clearTimeout(this._keepAliveTimeout);
                 }
-                const tempDir = this._tempDir;
                 try {
-                    let [imgBuffer, originalImgBuffer] = await Promise.all([
+                    let [imgBuffer, originalImgBuffer, rgbBuffer,] = await Promise.all([
                         fs_1.default.promises.readFile(path_1.default.join(tempDir, fileName)),
                         this._emitsOriginalSizeImages ?
                             fs_1.default.promises.readFile(path_1.default.join(tempDir, originalName)) :
                             null,
+                        this._emitsRgbBuffers ?
+                            fs_1.default.promises.readFile(path_1.default.join(tempDir, rgbName)) :
+                            null,
                     ]);
                     // hash not changed? don't emit another event (streamer does this on Rpi)
                     let hash = crypto_1.default.createHash('sha256').update(imgBuffer).digest('hex');
@@ -312,7 +361,12 @@ class GStreamer extends tsee_1.EventEmitter {
                         // snapshot() sends out the original size image
                         this.emit('snapshot', originalImgBuffer || imgBuffer, path_1.default.basename(fileName));
                         // snapshotForInference() sends out the resized image
-                        this.emit('snapshotForInference', imgBuffer, path_1.default.basename(fileName));
+                        this.emit('snapshotForInference', {
+                            imageForInferenceJpg: imgBuffer,
+                            filename: path_1.default.basename(fileName),
+                            imageFromCameraJpg: originalImgBuffer || imgBuffer,
+                            imageForInferenceRgb: rgbBuffer || undefined,
+                        });
                         lastPhoto = Date.now();
                         // 2 seconds no new data? trigger timeout
                         if (this._keepAliveTimeout) {
@@ -341,7 +395,13 @@ class GStreamer extends tsee_1.EventEmitter {
                     (async () => {
                         if (originalName) {
                             await this.safeUnlinkFile(path_1.default.join(tempDir, originalName));
-                            this._originalFilesSeen.delete(originalName);
+                            this._originalAndRgbFilesSeen.delete(originalName);
+                        }
+                    })(),
+                    (async () => {
+                        if (rgbName) {
+                            await this.safeUnlinkFile(path_1.default.join(tempDir, rgbName));
+                            this._originalAndRgbFilesSeen.delete(rgbName);
                         }
                     })(),
                 ]);
@@ -376,9 +436,25 @@ class GStreamer extends tsee_1.EventEmitter {
                                 return;
                             }
                             else {
+                                const errMsg = `GStreamer (gst-launch-1.0) stopped before emitting any images. This most likely ` +
+                                    `means that the launch command is incorrect or that your camera is unresponsive. Here is the launch command:\n\n` +
+                                    `${this._lastGstLaunchCommand}\n\n` +
+                                    `You can try one of the following:\n\n` +
+                                    `* If your camera used to work:\n` +
+                                    `    * Disconnect and reconnect the camera (if you use an external camera)\n` +
+                                    `    * Kill all other GStreamer commands, via: 'sudo killall gst-launch-1.0'\n` +
+                                    `* Run with '--verbose' to see the raw GStreamer output. It might contain a hint why the process fails.\n` +
+                                    `* Run with '--dont-output-rgb-buffers' - this will disable RGB output buffer creation which can help with ` +
+                                    `targets that advertise RGB capabilities on the video source, but don't actually support this.\n\n` +
+                                    `If this does not resolve your issue, then please open a forum post at https://forum.edgeimpulse.com and include:\n\n` +
+                                    `* Your device, operating system, what camera you're using, and how the camera is connected (e.g. USB, CSI)\n` +
+                                    `* The launch command (above)\n` +
+                                    `* The verbose output (run this application with --verbose)\n` +
+                                    `* The output of 'gst-device-monitor-1.0'\n` +
+                                    `* The output of 'gst-inspect-1.0'`;
                                 if (this._imagesReceived === 0 && this._lastGstLaunchCommand) {
                                     reject(`Capture process failed with code ${code}\n\n` +
-                                        `command: ${this._lastGstLaunchCommand}`);
+                                        `${errMsg}`);
                                 }
                                 else {
                                     reject('Capture process failed with code ' + code);
@@ -429,6 +505,7 @@ class GStreamer extends tsee_1.EventEmitter {
     }
     async getGstreamerLaunchCommand(device, dimensions, inferenceDims) {
         this._emitsOriginalSizeImages = false;
+        this._emitsRgbBuffers = false;
         if (device.id === CUSTOM_GST_LAUNCH_COMMAND) {
             if (!this._customLaunchCommand) {
                 throw new Error('_customLaunchCommand is null');
@@ -443,7 +520,7 @@ class GStreamer extends tsee_1.EventEmitter {
                 pipeline: customArgs.join(' '),
             };
         }
-        // now we need to determine the resolution... we want something as close as possible to dimensions.widthx480
+        // now we need to determine the resolution... we want something as close as possible to dimensions.
         let caps = device.caps.filter(c => {
             return c.width >= dimensions.width && c.height >= dimensions.height;
         }).sort((a, b) => {
@@ -474,20 +551,28 @@ class GStreamer extends tsee_1.EventEmitter {
         if (device.id) {
             videoSource.push(`device=${device.id}`);
         }
-        if ((this._mode === 'rpi') || (this._mode === 'microchip')) {
-            // Rpi camera
+        if ((this._mode === 'rpi') || (this._mode === 'rpi5') || device.videoSource === 'libcamerasrc') {
+            // libcamera devices don't have id set (or have some unique on Raspberry Pi or Microchip)
             if ((!device.id)
                 || (device.name.indexOf('unicam') > -1)
                 || (device.name.indexOf('bcm2835-isp') > -1)) {
                 videoSource = [
                     ...(this._profiling ? ['-m'] : []),
                     'libcamerasrc',
+                    ...device.name ? ['camera-name="' + device.name + '"'] : [],
                 ];
                 const hasPlugin = await this.hasGstPlugin('libcamerasrc');
                 if (!hasPlugin) {
                     throw new Error('Missing "libcamerasrc" gstreamer element. Install via `sudo apt install -y gstreamer1.0-libcamera`');
                 }
             }
+            // FIXME: dirty hack for IPASoft on QRB2210
+            // Some resolutions reported by camera causes the pipeline to hang. To fix that, it seems
+            // we need to increase width and/or height by 8 pixels. This hack should be removed once
+            // the hardware driver for ISP on QRB2210 is released.
+            if (this._mode === 'unoq') {
+                cap.width += 8;
+            }
         }
         else if ((this._mode === 'qualcomm-rb3gen2') || (this._mode === 'qualcomm-yupik')) {
             videoSource = [
@@ -503,6 +588,24 @@ class GStreamer extends tsee_1.EventEmitter {
                 'pylonsrc',
             ];
         }
+        const frameReadyArgs = this._profiling ? [`!`, `identity name=frame_ready silent=false`] : [];
+        const jpgencDoneArgs = this._profiling ? [`!`, `identity name=jpegenc_done silent=false`] : [];
+        const resizeDoneArgs = this._profiling ? [`!`, `identity name=resize_done silent=false`] : [];
+        let teeToBothResizedJpgAndRgb;
+        let videoFormat = '';
+        if (this._outputRgbBuffers) {
+            teeToBothResizedJpgAndRgb = [
+                `tee name=u`,
+                `u. ! queue ! jpegenc ${jpgencDoneArgs.join(' ')} ! multifilesink location=resized%05d.jpg post-messages=true sync=false `,
+                `u. ! queue ! multifilesink location=resized%05d.rgb post-messages=true sync=false `,
+            ];
+            videoFormat = ',format=RGB'; // Don't do this if outputRgbBuffers is RGB, because we don't want to force RGB video format
+        }
+        else {
+            teeToBothResizedJpgAndRgb = [
+                `jpegenc ${jpgencDoneArgs.join(' ')} ! multifilesink location=resized%05d.jpg post-messages=true sync=false`,
+            ];
+        }
         let cropArgs = [];
         if (inferenceDims) {
             // fast path for fit-shortest and squash
@@ -514,14 +617,14 @@ class GStreamer extends tsee_1.EventEmitter {
                     `!`,
                     `videoscale`, `method=lanczos`,
                     `!`,
-                    `video/x-raw,width=${inferenceDims.width},height=${inferenceDims.height}`,
+                    `video/x-raw${videoFormat},width=${inferenceDims.width},height=${inferenceDims.height}`,
                 ]);
             }
-            else if (inferenceDims.resizeMode === 'squash' || inferenceDims.resizeMode === 'none' /* old model */) {
+            else if (inferenceDims.resizeMode === 'squash' || inferenceDims.resizeMode === 'none' /* old model -> squash */) {
                 cropArgs.push(`!`);
                 cropArgs.push(`videoscale`, `method=lanczos`);
                 cropArgs.push(`!`);
-                cropArgs.push(`video/x-raw,width=${inferenceDims.width},height=${inferenceDims.height}`);
+                cropArgs.push(`video/x-raw${videoFormat},width=${inferenceDims.width},height=${inferenceDims.height}`);
             }
         }
         let args;
@@ -532,14 +635,35 @@ class GStreamer extends tsee_1.EventEmitter {
                     `video/x-raw,width=${cap.width},height=${cap.height},format=YUY2`
                 ]);
             }
+            else if (device.videoSource === 'qtiqmmfsrc') {
+                videoSource = videoSource.concat([
+                    `!`,
+                    `video/x-raw,width=${cap.width},height=${cap.height},format=NV12`
+                ]);
+            }
+            else if (this._mode === 'rpi5') {
+                // on RPi 5 we need to set color format
+                let colorFormat = '';
+                if (this._overrideColorFormat) {
+                    colorFormat = this._overrideColorFormat;
+                }
+                else if (cap.formats && cap.formats.indexOf('YUY2') > -1) {
+                    colorFormat = 'YUY2';
+                }
+                else if (cap.formats) {
+                    throw new Error('Detected RPi 5 camera. Please provide the color format with `--camera-color-format`, supported formats: ' + cap.formats.join(', '));
+                }
+                videoSource = videoSource.concat([
+                    `!`,
+                    `video/x-raw,width=${cap.width},height=${cap.height},format=${colorFormat}`
+                ]);
+            }
             else {
                 videoSource = videoSource.concat([
                     `!`,
                     `video/x-raw,width=${cap.width},height=${cap.height}`
                 ]);
             }
-            let frameReadyArgs = this._profiling ? [`!`, `identity name=frame_ready silent=false`] : [];
-            let jpgencDoneArgs = this._profiling ? [`!`, `identity name=jpegenc_done silent=false`] : [];
             if (cropArgs.length === 0) {
                 args = videoSource.concat([
                     ...frameReadyArgs,
@@ -557,11 +681,9 @@ class GStreamer extends tsee_1.EventEmitter {
                 let resized = [
                     `t. ! queue`,
                     ...cropArgs,
+                    ...resizeDoneArgs,
                     `!`,
-                    `jpegenc`,
-                    ...jpgencDoneArgs,
-                    `!`,
-                    `multifilesink location=resized%05d.jpg post-messages=true sync=false`,
+                    ...teeToBothResizedJpgAndRgb,
                 ];
                 args = videoSource.concat([
                     ...frameReadyArgs,
@@ -573,15 +695,51 @@ class GStreamer extends tsee_1.EventEmitter {
                     ...resized,
                 ]);
                 this._emitsOriginalSizeImages = true;
+                if (this._outputRgbBuffers) {
+                    this._emitsRgbBuffers = true;
+                }
             }
         }
         else if (cap.type === 'image/jpeg') {
-            args = videoSource.concat([
+            videoSource = videoSource.concat([
                 `!`,
                 `image/jpeg,width=${cap.width},height=${cap.height}`,
-                `!`,
-                `multifilesink location=resized%05d.jpg post-messages=true sync=false`
             ]);
+            if (cropArgs.length === 0) {
+                args = videoSource.concat([
+                    ...frameReadyArgs,
+                    `!`,
+                    `multifilesink`,
+                    `location=resized%05d.jpg`,
+                    `post-messages=true`,
+                    `sync=false`,
+                ]);
+            }
+            else {
+                let original = `t. ! queue ! multifilesink location=original%05d.jpg  post-messages=true sync=false`;
+                let resized = [
+                    `t. ! queue`,
+                    `!`,
+                    `jpegdec`,
+                    `!`,
+                    `videoconvert`,
+                    ...cropArgs,
+                    ...resizeDoneArgs,
+                    `!`,
+                    ...teeToBothResizedJpgAndRgb,
+                ];
+                args = videoSource.concat([
+                    ...frameReadyArgs,
+                    `!`,
+                    `tee name=t`,
+                    original,
+                    ...resized,
+                ]);
+                this._emitsOriginalSizeImages = true;
+                if (this._outputRgbBuffers) {
+                    this._emitsRgbBuffers = true;
+                }
+            }
         }
         else if (cap.type === 'nvarguscamerasrc') {
             args = [
@@ -595,7 +753,8 @@ class GStreamer extends tsee_1.EventEmitter {
         }
         return {
             command: 'gst-launch-1.0',
-            pipeline: args.join(' '),
+            // replace multiple spaces with one space
+            pipeline: args.join(' ').trim().replace(/(\s+)/g, ' '),
         };
     }
     async stop() {
@@ -607,7 +766,10 @@ class GStreamer extends tsee_1.EventEmitter {
                 this._captureProcess.on('close', code => {
                     if (this._watcher) {
                         this._watcher.on('close', async () => {
-                            await this.cleanupTempDirAsync();
+                            try {
+                                await this.cleanupTempDirAsync();
+                            }
+                            catch (ex) { /* noop */ }
                         });
                         this._watcher.close();
                     }
@@ -696,7 +858,8 @@ class GStreamer extends tsee_1.EventEmitter {
             if (!currDevice)
                 continue;
             if (l.startsWith('name  :')) {
-                currDevice.name = l.split(':')[1].trim();
+                // extract name from the l between 'name  : ' and '\n', because name may contain colons as well
+                currDevice.name = l.split(':').slice(1).join(':').trim();
                 continue;
             }
             if (l.startsWith('class :')) {
@@ -731,6 +894,12 @@ class GStreamer extends tsee_1.EventEmitter {
                     if (currDevice.videoSource === 'pipewiresrc') {
                         currDevice.videoSource = DEFAULT_GST_VIDEO_SOURCE;
                     }
+                    if (currDevice.videoSource === 'avfvideosrc') {
+                        const m = l.match(/gst-launch-1.0 avfvideosrc device-index=(\d+) \!/);
+                        if (m && m.length >= 2) {
+                            currDevice.videoSource = `avfvideosrc device-index=${m[1]}`;
+                        }
+                    }
                 }
             }
         }
@@ -742,6 +911,7 @@ class GStreamer extends tsee_1.EventEmitter {
                 let width = (l.match(/width=[^\d]+(\d+)/) || [])[1];
                 let height = (l.match(/height=[^\d]+(\d+)/) || [])[1];
                 let framerate = (l.match(/framerate=[^\d]+(\d+)/) || [])[1];
+                let format = (l.match(/format=([a-zA-Z0-9]+)/) || [])[1];
                 // Rpi on bullseye has lines like this..
                 // eslint-disable-next-line @stylistic/max-len
                 // image/jpeg, width=160, height=120, pixel-aspect-ratio=1/1, framerate={ (fraction)30/1, (fraction)24/1, (fraction)20/1, (fraction)15/1, (fraction)10/1, (fraction)15/2, (fraction)5/1 }
@@ -759,10 +929,11 @@ class GStreamer extends tsee_1.EventEmitter {
                     width: Number(width || '0'),
                     height: Number(height || '0'),
                     framerate: Number(framerate || '0'),
+                    formats: format ? [format] : []
                 };
                 return r;
             });
-            if (this._mode === 'rpi' || this._mode === 'microchip') { // no framerate here...
+            if (this._mode === 'rpi' || this._mode === 'rpi5' || d.videoSource === 'libcamerasrc') { // no framerate here...
                 c = c.filter(x => x.width && x.height);
             }
             else if (d.name === 'RZG2L_CRU') {
@@ -780,14 +951,23 @@ class GStreamer extends tsee_1.EventEmitter {
             else {
                 c = c.filter(x => x.width && x.height && x.framerate);
             }
+            // get list of all formats form the c list
+            const uniqueFormats = Array.from(new Set(c.map(cap => cap.formats).flat()));
             c = c.reduce((curr, o) => {
-                // deduplicate caps
-                if (!curr.some(obj => obj.framerate === o.framerate &&
+                // deduplicate caps, prioritize 'video/x-raw' type
+                const existingIndex = curr.findIndex(obj => obj.framerate === o.framerate &&
                     obj.width === o.width &&
-                    obj.height === o.height &&
-                    obj.framerate === o.framerate)) {
+                    obj.height === o.height);
+                // store all supported formats (YUY2 etc.)
+                if (uniqueFormats) {
+                    o.formats = uniqueFormats.filter((format) => format !== undefined);
+                }
+                if (existingIndex === -1) {
                     curr.push(o);
                 }
+                else if (o.type === 'video/x-raw' && curr[existingIndex].type !== 'video/x-raw') {
+                    curr[existingIndex] = o;
+                }
                 return curr;
             }, []);
             d.caps = c;
@@ -804,7 +984,7 @@ class GStreamer extends tsee_1.EventEmitter {
         // Qualcomm has their own plugin, query its too
         // because listQtiqmmsrc* returns hardcoded devices ONLY if a specific gst plugin
         // is available we need to detect device type before, as it may be RubikPi, RB3Gen2 or
-        // any other with IMSDK loaded and unkniwn cameras
+        // any other with IMSDK loaded and unknown cameras
         if (this._mode === 'qualcomm-rb3gen2') {
             devices = devices.concat(await this.listQtiqmmsrcDevices());
         }
@@ -822,10 +1002,18 @@ class GStreamer extends tsee_1.EventEmitter {
                 videoSource: d.videoSource,
             };
         });
-        // deduplicate (by id)
+        // deduplicate (by id if present, otherwise by name)
         mapped = mapped.reduce((curr, m) => {
-            if (curr.find(x => x.id === m.id))
-                return curr;
+            if (m.id) {
+                if (curr.find(x => x.id === m.id)) {
+                    return curr;
+                }
+            }
+            else if (m.name) {
+                if (curr.find(x => x.name === m.name && !x.id)) {
+                    return curr;
+                }
+            }
             curr.push(m);
             return curr;
         }, []);
@@ -1253,6 +1441,8 @@ class GStreamer extends tsee_1.EventEmitter {
         const cropTop = Math.max(0, Math.floor((srcH - cropH) / 2));
         const cropBottom = Math.max(0, srcH - cropH - cropTop);
         return {
+            width: cropW,
+            height: cropH,
             left: cropLeft,
             right: cropRight,
             top: cropTop,
@@ -1277,7 +1467,7 @@ class GStreamer extends tsee_1.EventEmitter {
                 }
                 let filesToUnlink = [];
                 // copy as we're manipulating this set
-                for (let originalFileName of Array.from(this._originalFilesSeen)) {
+                for (let originalFileName of Array.from(this._originalAndRgbFilesSeen)) {
                     let originalNumberMatch = originalFileName.match(/(\d+)/);
                     let originalNumber = originalNumberMatch ? Number(originalNumberMatch[1]) : NaN;
                     if (isNaN(originalNumber)) {
@@ -1292,13 +1482,13 @@ class GStreamer extends tsee_1.EventEmitter {
                     filesToUnlink.push(originalFileName);
                 }
                 if (this._verbose) {
-                    console.log(PREFIX, `Unlinking ${filesToUnlink.length} original files...`);
+                    console.log(PREFIX, `Unlinking ${filesToUnlink.length} original / RGB files...`);
                 }
                 if (filesToUnlink.length > 0) {
                     const tempDir = this._tempDir;
                     await (0, async_pool_1.asyncPool)(10, filesToUnlink, async (originalFileName) => {
                         await this.safeUnlinkFile(path_1.default.join(tempDir, originalFileName));
-                        this._originalFilesSeen.delete(originalFileName);
+                        this._originalAndRgbFilesSeen.delete(originalFileName);
                     });
                 }
             }