edge-impulse-linux 1.18.3 → 1.20.1

package/build/library/sensors/gstreamer.js

@@ -27,11 +27,15 @@ class GStreamer extends tsee_1.EventEmitter {
         this._isStarted = false;
         this._isRestarting = false;
         this._emitsOriginalSizeImages = false;
+        this._lastGstLaunchCommand = '';
+        this._imagesReceived = 0;
+        this._offset = 0;
         this._verbose = verbose;
         this._customLaunchCommand = options?.customLaunchCommand;
         this._spawnHelper = options?.spawnHelperOverride || spawn_helper_1.spawnHelper;
         this._modeOverride = options?.modeOverride;
         this._profiling = options?.profiling || false;
+        this._overrideColorFormat = options?.colorFormat;
         if (options?.dontRunCleanupLoop === true) {
             // skip cleanup loop
         }
@@ -57,31 +61,43 @@ class GStreamer extends tsee_1.EventEmitter {
             console.log(PREFIX, 'checking for /etc/os-release');
             osRelease = await fs_1.default.promises.readFile('/etc/os-release', 'utf-8');
         }
+        else {
+            // so we don't need to check for undefined below
+            osRelease = '';
+        }
         let firmwareModel;
         // using /proc/device-tree as recommended in user space.
         if (await this.exists('/proc/device-tree/model')) {
             firmwareModel = await fs_1.default.promises.readFile('/proc/device-tree/model', 'utf-8');
         }
-        if (osRelease) {
-            if ((osRelease.indexOf('bullseye') > -1)
-                || (osRelease.indexOf('bookworm') > -1)) {
-                if (osRelease.indexOf('ID=raspbian') > -1) {
-                    this._mode = 'rpi';
-                }
-                if (firmwareModel && firmwareModel.indexOf('Raspberry Pi') > -1) {
-                    this._mode = 'rpi';
-                }
-            }
-        }
-        if (firmwareModel && firmwareModel.indexOf('Microchip SAMA7G5') > -1) {
-            this._mode = 'microchip';
+        else {
+            // so we don't need to check for undefined below
+            firmwareModel = '';
         }
-        else if (firmwareModel && firmwareModel.indexOf('RB3gen2') > -1 && firmwareModel.indexOf('vision') > -1) {
+        if (firmwareModel.indexOf('RB3gen2') > -1 && firmwareModel.indexOf('vision') > -1) {
             this._mode = 'qualcomm-rb3gen2';
         }
-        else if (firmwareModel && firmwareModel.indexOf('Qualcomm') > -1 && firmwareModel.indexOf('Yupik') > -1) {
+        else if (firmwareModel.indexOf('Qualcomm') > -1 && firmwareModel.indexOf('Yupik') > -1) {
             this._mode = 'qualcomm-yupik';
         }
+        else if (firmwareModel.indexOf('Raspberry Pi') > -1) {
+            if (((osRelease.indexOf('bullseye') > -1) || (osRelease.indexOf('bookworm') > -1))
+                && (osRelease.indexOf('ID=raspbian') === -1)) {
+                this._mode = 'rpi';
+            }
+            else if (osRelease.indexOf('trixie') > -1) {
+                this._mode = 'rpi';
+            }
+            // override to rpi5 if needed
+            if (firmwareModel.indexOf('Raspberry Pi 5') > -1) {
+                this._mode = 'rpi5';
+            }
+        }
+        else if ((firmwareModel.indexOf('Arduino') > -1 && firmwareModel.indexOf('Imola') > -1) ||
+            // this may be incorrect on the another platforms in the future
+            (process.env.EI_CLI_ENV === 'arduino')) {
+            this._mode = 'unoq';
+        }
         this._mode = (this._modeOverride) ? this._modeOverride : this._mode;
     }
     async listDevices() {
@@ -141,6 +157,8 @@ class GStreamer extends tsee_1.EventEmitter {
             args.push(`-v`);
         }
         args.push(pipeline);
+        // for logging purposes
+        this._lastGstLaunchCommand = `${command} ${args.join(' ')}`;
         if (this._verbose) {
             console.log(PREFIX, `Starting ${command} ${args.join(' ')}`);
         }
@@ -154,11 +172,77 @@ class GStreamer extends tsee_1.EventEmitter {
                 if (this._verbose) {
                     console.log(PREFIX, d.toString('utf-8'));
                 }
-                if (d.toString('utf-8').indexOf('frame_ready:sink') > -1) {
-                    this.emit('profilingInfo', new Date(), 'frame_ready');
-                }
-                if (d.toString('utf-8').indexOf('jpegenc_done:sink') > -1) {
-                    this.emit('profilingInfo', new Date(), 'jpegenc_done');
+                const ts = new Date();
+                const parsePtsFromLine = (line) => {
+                    const m = line.match(/pts: ([\d\.\:]+)/);
+                    if (m && m.length >= 2) {
+                        return m[1];
+                    }
+                    return null;
+                };
+                const parseOffsetFromLine = (line) => {
+                    const m = line.match(/offset: (-?[\d\.\:]+)/);
+                    if (m && m.length >= 2) {
+                        return Number(m[1]);
+                    }
+                    return null;
+                };
+                const str = d.toString('utf-8');
+                for (const line of str.split('\n')) {
+                    if (line.trim().startsWith('/GstPipeline:pipeline0/GstIdentity:')) {
+                        // identity elements
+                        const pts = parsePtsFromLine(line);
+                        let offset = parseOffsetFromLine(line);
+                        const identity = line.replace('/GstPipeline:pipeline0/GstIdentity:', '').split(' ')[0]
+                            .replace(':', '');
+                        // no pts -> skip
+                        if (!pts) {
+                            continue;
+                        }
+                        // frame_ready w/o offset -> also skip
+                        if (identity === 'frame_ready') {
+                            if (offset === null) {
+                                continue;
+                            }
+                            // e.g. qtiqmmfsrc always has offset -1 for frame_ready
+                            if (offset === -1) {
+                                offset = ++this._offset;
+                            }
+                            this.emit('profilingInfo', {
+                                type: 'frame_ready',
+                                ts: ts,
+                                pts: pts,
+                                offset: offset,
+                            });
+                        }
+                        else {
+                            this.emit('profilingInfo', {
+                                type: 'event-without-filename',
+                                ts: ts,
+                                name: identity,
+                                pts: pts,
+                            });
+                        }
+                    }
+                    else if (line.includes('GstMultiFileSink, filename=')) {
+                        let filenameM = line.match(/filename=\(string\)([\w\.]+)/);
+                        if (!filenameM || filenameM.length < 2) {
+                            console.log(PREFIX, `Failed to parse filename from GstMultiFileSink: ` + line.trim());
+                            continue;
+                        }
+                        let timestampM = line.match(/timestamp=\(guint64\)(\d+)/);
+                        if (!timestampM || timestampM.length < 2) {
+                            console.log(PREFIX, `Failed to parse timestamp from GstMultiFileSink: ` + line.trim());
+                            continue;
+                        }
+                        let filename = filenameM[1];
+                        let pts = this.timestampNsToPts(timestampM[1]);
+                        this.emit('profilingInfo', {
+                            type: 'pts-to-filename',
+                            pts: pts,
+                            filename: filename,
+                        });
+                    }
                 }
             });
             this._captureProcess.stderr.on('data', (d) => {
@@ -185,6 +269,7 @@ class GStreamer extends tsee_1.EventEmitter {
                 return;
             if (this._handledFiles[fileName])
                 return;
+            this._imagesReceived++;
             // not next frame yet?
             if (this._processing || Date.now() < nextFrame) {
                 this._handledFiles[fileName] = true;
@@ -226,11 +311,12 @@ class GStreamer extends tsee_1.EventEmitter {
                 if (this._keepAliveTimeout) {
                     clearTimeout(this._keepAliveTimeout);
                 }
+                const tempDir = this._tempDir;
                 try {
                     let [imgBuffer, originalImgBuffer] = await Promise.all([
-                        fs_1.default.promises.readFile(path_1.default.join(this._tempDir, fileName)),
+                        fs_1.default.promises.readFile(path_1.default.join(tempDir, fileName)),
                         this._emitsOriginalSizeImages ?
-                            fs_1.default.promises.readFile(path_1.default.join(this._tempDir, originalName)) :
+                            fs_1.default.promises.readFile(path_1.default.join(tempDir, originalName)) :
                             null,
                     ]);
                     // hash not changed? don't emit another event (streamer does this on Rpi)
@@ -259,9 +345,8 @@ class GStreamer extends tsee_1.EventEmitter {
                     };
                 }
                 catch (ex) {
-                    console.error(PREFIX, 'Failed to load file', path_1.default.join(this._tempDir, fileName), ex);
+                    console.error(PREFIX, 'Failed to load file', path_1.default.join(tempDir, fileName), ex);
                 }
-                const tempDir = this._tempDir;
                 await Promise.all([
                     (async () => {
                         await this.safeUnlinkFile(path_1.default.join(tempDir, fileName));
@@ -304,7 +389,13 @@ class GStreamer extends tsee_1.EventEmitter {
                                 return;
                             }
                             else {
-                                reject('Capture process failed with code ' + code);
+                                if (this._imagesReceived === 0 && this._lastGstLaunchCommand) {
+                                    reject(`Capture process failed with code ${code}\n\n` +
+                                        `command: ${this._lastGstLaunchCommand}`);
+                                }
+                                else {
+                                    reject('Capture process failed with code ' + code);
+                                }
                             }
                         }
                         else {
@@ -358,15 +449,14 @@ class GStreamer extends tsee_1.EventEmitter {
             let customArgs = (0, argv_split_1.split)(this._customLaunchCommand);
             customArgs = customArgs.concat([
                 `!`,
-                `multifilesink`,
-                `location=resized%05d.jpg`
+                `multifilesink location=resized%05d.jpg post-messages=true sync=false`,
             ]);
             return {
                 command: 'gst-launch-1.0',
                 pipeline: customArgs.join(' '),
             };
         }
-        // now we need to determine the resolution... we want something as close as possible to dimensions.widthx480
+        // now we need to determine the resolution... we want something as close as possible to dimensions.
         let caps = device.caps.filter(c => {
             return c.width >= dimensions.width && c.height >= dimensions.height;
         }).sort((a, b) => {
@@ -389,24 +479,50 @@ class GStreamer extends tsee_1.EventEmitter {
                 framerate: 30,
             };
         }
-        let videoSource = [device.videoSource, 'device=' + device.id];
-        if ((this._mode === 'rpi') || (this._mode === 'microchip')) {
-            // Rpi camera
+        let videoSource = [];
+        if (this._profiling) {
+            videoSource.push('-m');
+        }
+        videoSource.push(device.videoSource);
+        if (device.id) {
+            videoSource.push(`device=${device.id}`);
+        }
+        if ((this._mode === 'rpi') || (this._mode === 'rpi5') || device.videoSource === 'libcamerasrc') {
+            // libcamera devices don't have id set (or have some unique on Raspberry Pi or Microchip)
             if ((!device.id)
                 || (device.name.indexOf('unicam') > -1)
                 || (device.name.indexOf('bcm2835-isp') > -1)) {
-                videoSource = ['libcamerasrc'];
+                videoSource = [
+                    ...(this._profiling ? ['-m'] : []),
+                    'libcamerasrc',
+                    ...device.name ? ['camera-name="' + device.name + '"'] : [],
+                ];
                 const hasPlugin = await this.hasGstPlugin('libcamerasrc');
                 if (!hasPlugin) {
                     throw new Error('Missing "libcamerasrc" gstreamer element. Install via `sudo apt install -y gstreamer1.0-libcamera`');
                 }
             }
+            // FIXME: dirty hack for IPASoft on QRB2210
+            // Some resolutions reported by camera causes the pipeline to hang. To fix that, it seems
+            // we need to increase width and/or height by 8 pixels. This hack should be removed once
+            // the hardware driver for ISP on QRB2210 is released.
+            if (this._mode === 'unoq') {
+                cap.width += 8;
+            }
         }
         else if ((this._mode === 'qualcomm-rb3gen2') || (this._mode === 'qualcomm-yupik')) {
-            videoSource = ['qtiqmmfsrc', 'name=camsrc', 'camera=' + device.id];
+            videoSource = [
+                ...(this._profiling ? ['-m'] : []),
+                'qtiqmmfsrc',
+                'name=camsrc',
+                'camera=' + device.id,
+            ];
         }
         if (device.id === 'pylonsrc') {
-            videoSource = ['pylonsrc'];
+            videoSource = [
+                ...(this._profiling ? ['-m'] : []),
+                'pylonsrc',
+            ];
         }
         let cropArgs = [];
         if (inferenceDims) {
@@ -437,6 +553,23 @@ class GStreamer extends tsee_1.EventEmitter {
                     `video/x-raw,width=${cap.width},height=${cap.height},format=YUY2`
                 ]);
             }
+            else if (this._mode === 'rpi5') {
+                // on RPi 5 we need to set color format
+                let colorFormat = '';
+                if (this._overrideColorFormat) {
+                    colorFormat = this._overrideColorFormat;
+                }
+                else if (cap.formats && cap.formats.indexOf('YUY2') > -1) {
+                    colorFormat = 'YUY2';
+                }
+                else if (cap.formats) {
+                    throw new Error('Detected RPi 5 camera. Please provide the color format with `--camera-color-format`, supported formats: ' + cap.formats.join(', '));
+                }
+                videoSource = videoSource.concat([
+                    `!`,
+                    `video/x-raw,width=${cap.width},height=${cap.height},format=${colorFormat}`
+                ]);
+            }
             else {
                 videoSource = videoSource.concat([
                     `!`,
@@ -454,12 +587,11 @@ class GStreamer extends tsee_1.EventEmitter {
                     `jpegenc`,
                     ...jpgencDoneArgs,
                     `!`,
-                    `multifilesink`,
-                    `location=resized%05d.jpg`,
+                    `multifilesink location=resized%05d.jpg post-messages=true sync=false`,
                 ]);
             }
             else {
-                let original = `t. ! queue ! jpegenc ! multifilesink location=original%05d.jpg`;
+                let original = `t. ! queue ! jpegenc ! multifilesink location=original%05d.jpg post-messages=true sync=false`;
                 let resized = [
                     `t. ! queue`,
                     ...cropArgs,
@@ -467,8 +599,7 @@ class GStreamer extends tsee_1.EventEmitter {
                     `jpegenc`,
                     ...jpgencDoneArgs,
                     `!`,
-                    `multifilesink`,
-                    `location=resized%05d.jpg`,
+                    `multifilesink location=resized%05d.jpg post-messages=true sync=false`,
                 ];
                 args = videoSource.concat([
                     ...frameReadyArgs,
@@ -487,15 +618,14 @@ class GStreamer extends tsee_1.EventEmitter {
                 `!`,
                 `image/jpeg,width=${cap.width},height=${cap.height}`,
                 `!`,
-                `multifilesink`,
-                `location=resized%05d.jpg`
+                `multifilesink location=resized%05d.jpg post-messages=true sync=false`
             ]);
         }
         else if (cap.type === 'nvarguscamerasrc') {
             args = [
                 `nvarguscamerasrc ! "video/x-raw(memory:NVMM),width=${cap.width},height=${cap.height}" ! ` +
                     `nvvidconv flip-method=0 ! video/x-raw,width=${cap.width},height=${cap.height} ! nvvidconv ! ` +
-                    `jpegenc ! multifilesink location=resized%05d.jpg`
+                    `jpegenc ! multifilesink location=resized%05d.jpg post-messages=true sync=false`
             ];
         }
         else {
@@ -604,7 +734,8 @@ class GStreamer extends tsee_1.EventEmitter {
             if (!currDevice)
                 continue;
             if (l.startsWith('name  :')) {
-                currDevice.name = l.split(':')[1].trim();
+                // extract name from the l between 'name  : ' and '\n', because name may contain colons as well
+                currDevice.name = l.split(':').slice(1).join(':').trim();
                 continue;
             }
             if (l.startsWith('class :')) {
@@ -639,6 +770,12 @@ class GStreamer extends tsee_1.EventEmitter {
                     if (currDevice.videoSource === 'pipewiresrc') {
                         currDevice.videoSource = DEFAULT_GST_VIDEO_SOURCE;
                     }
+                    if (currDevice.videoSource === 'avfvideosrc') {
+                        const m = l.match(/gst-launch-1.0 avfvideosrc device-index=(\d+) \!/);
+                        if (m && m.length >= 2) {
+                            currDevice.videoSource = `avfvideosrc device-index=${m[1]}`;
+                        }
+                    }
                 }
             }
         }
@@ -650,6 +787,7 @@ class GStreamer extends tsee_1.EventEmitter {
                 let width = (l.match(/width=[^\d]+(\d+)/) || [])[1];
                 let height = (l.match(/height=[^\d]+(\d+)/) || [])[1];
                 let framerate = (l.match(/framerate=[^\d]+(\d+)/) || [])[1];
+                let format = (l.match(/format=([a-zA-Z0-9]+)/) || [])[1];
                 // Rpi on bullseye has lines like this..
                 // eslint-disable-next-line @stylistic/max-len
                 // image/jpeg, width=160, height=120, pixel-aspect-ratio=1/1, framerate={ (fraction)30/1, (fraction)24/1, (fraction)20/1, (fraction)15/1, (fraction)10/1, (fraction)15/2, (fraction)5/1 }
@@ -667,10 +805,11 @@ class GStreamer extends tsee_1.EventEmitter {
                     width: Number(width || '0'),
                     height: Number(height || '0'),
                     framerate: Number(framerate || '0'),
+                    formats: format ? [format] : []
                 };
                 return r;
             });
-            if (this._mode === 'rpi' || this._mode === 'microchip') { // no framerate here...
+            if (this._mode === 'rpi' || this._mode === 'rpi5' || d.videoSource === 'libcamerasrc') { // no framerate here...
                 c = c.filter(x => x.width && x.height);
             }
             else if (d.name === 'RZG2L_CRU') {
@@ -688,14 +827,23 @@ class GStreamer extends tsee_1.EventEmitter {
             else {
                 c = c.filter(x => x.width && x.height && x.framerate);
             }
+            // get list of all formats form the c list
+            const uniqueFormats = Array.from(new Set(c.map(cap => cap.formats).flat()));
             c = c.reduce((curr, o) => {
-                // deduplicate caps
-                if (!curr.some(obj => obj.framerate === o.framerate &&
+                // deduplicate caps, prioritize 'video/x-raw' type
+                const existingIndex = curr.findIndex(obj => obj.framerate === o.framerate &&
                     obj.width === o.width &&
-                    obj.height === o.height &&
-                    obj.framerate === o.framerate)) {
+                    obj.height === o.height);
+                // store all supported formats (YUY2 etc.)
+                if (uniqueFormats) {
+                    o.formats = uniqueFormats.filter((format) => format !== undefined);
+                }
+                if (existingIndex === -1) {
                     curr.push(o);
                 }
+                else if (o.type === 'video/x-raw' && curr[existingIndex].type !== 'video/x-raw') {
+                    curr[existingIndex] = o;
+                }
                 return curr;
             }, []);
             d.caps = c;
@@ -712,7 +860,7 @@ class GStreamer extends tsee_1.EventEmitter {
         // Qualcomm has their own plugin, query its too
         // because listQtiqmmsrc* returns hardcoded devices ONLY if a specific gst plugin
         // is available we need to detect device type before, as it may be RubikPi, RB3Gen2 or
-        // any other with IMSDK loaded and unkniwn cameras
+        // any other with IMSDK loaded and unknown cameras
         if (this._mode === 'qualcomm-rb3gen2') {
             devices = devices.concat(await this.listQtiqmmsrcDevices());
         }
@@ -730,10 +878,18 @@ class GStreamer extends tsee_1.EventEmitter {
                 videoSource: d.videoSource,
             };
         });
-        // deduplicate (by id)
+        // deduplicate (by id if present, otherwise by name)
         mapped = mapped.reduce((curr, m) => {
-            if (curr.find(x => x.id === m.id))
-                return curr;
+            if (m.id) {
+                if (curr.find(x => x.id === m.id)) {
+                    return curr;
+                }
+            }
+            else if (m.name) {
+                if (curr.find(x => x.name === m.name && !x.id)) {
+                    return curr;
+                }
+            }
             curr.push(m);
             return curr;
         }, []);
@@ -1235,6 +1391,17 @@ class GStreamer extends tsee_1.EventEmitter {
             }
         }
     }
+    timestampNsToPts(nsStr) {
+        const ns = BigInt(nsStr); // important: avoid precision loss
+        const HOUR = 3600000000000n;
+        const MIN = 60000000000n;
+        const SEC = 1000000000n;
+        const h = ns / HOUR;
+        const m = (ns / MIN) % 60n;
+        const s = (ns / SEC) % 60n;
+        const n = ns % SEC;
+        return `${h}:${m.toString().padStart(2, '0')}:${s.toString().padStart(2, '0')}.${n.toString().padStart(9, '0')}`;
+    }
 }
 exports.GStreamer = GStreamer;
 //# sourceMappingURL=gstreamer.js.map