edge-impulse-linux 1.16.2 → 1.17.2

package/cli/linux/webserver/public/webserver.js

@@ -2,6 +2,7 @@ window.WebServer = async () => {
 
     const els = {
         title: document.querySelector('#header-row h1'),
+        cameraOuterContainer: document.querySelector('#capture-camera .capture-camera-outer'),
         cameraContainer: document.querySelector('#capture-camera .capture-camera-inner'),
         cameraImg: document.querySelector('#capture-camera img'),
         timePerInference: document.querySelector('#time-per-inference'),
@@ -29,6 +30,9 @@ window.WebServer = async () => {
     ];
     let colorIx = 0;
     const labelToColor = { };
+    let isFirstClassification = true;
+    let inferenceIx = 0;
+    let lastClassification;
 
     function switchView(el) {
         for (let k of Object.keys(els.views)) {
@@ -150,6 +154,7 @@ window.WebServer = async () => {
                 inputEl.oninput = () => {
                     if (typeof threshold[k] === 'number') {
                         if (!inputEl.value || isNaN(Number(inputEl.value))) return;
+                        if (k === 'min_score' && Number(inputEl.value) === 0) return;
 
                         threshold[k] = Number(inputEl.value);
                     }
@@ -194,14 +199,65 @@ window.WebServer = async () => {
         bindThresholdSettings(opts.thresholds);
     });
 
+    const onWindowResize = () => {
+        if (els.cameraContainer.naturalWidth === 0) {
+            return;
+        }
+
+        let oldStyleWidth = els.cameraImg.style.width;
+        const containerWidth = els.cameraOuterContainer.getBoundingClientRect().width;
+
+        // height >480
+        if (els.cameraImg.naturalHeight > 480) {
+            // and fits within the container? just display as is
+            if (els.cameraImg.naturalWidth < containerWidth) {
+                els.cameraImg.style.width = els.cameraImg.naturalWidth + 'px';
+            }
+            else {
+                // does not fit within container? just use 100%
+                els.cameraImg.style.width = containerWidth + 'px';
+            }
+        }
+        else {
+            // what if we resize to 480 high?
+            const factor = els.cameraImg.naturalWidth / els.cameraImg.naturalHeight;
+            let newHeight = 480;
+            let newWidth = newHeight * factor;
+
+            // fits within the container? just display as is
+            if (newWidth < containerWidth) {
+                els.cameraImg.style.width = newWidth + 'px';
+            }
+            else {
+                // does not fit within container? just use 100%
+                els.cameraImg.style.width = containerWidth + 'px';
+            }
+        }
+
+        if (oldStyleWidth !== els.cameraImg.style.width && lastClassification) {
+            onClassification({
+                dontUpdateTable: true,
+                ...lastClassification,
+            });
+        }
+    };
+
+    let isFirstImage = true;
     socket.on('image', (opts) => {
+        if (isFirstImage) {
+            els.cameraImg.onload = () => {
+                onWindowResize();
+                els.cameraImg.onload = null;
+            };
+            isFirstImage = false;
+        }
         els.cameraImg.src = opts.img;
     });
+    window.addEventListener('resize', onWindowResize);
 
-    let isFirstClassification = true;
-    let inferenceIx = 0;
+    function onClassification(opts) {
+        lastClassification = opts;
 
-    socket.on('classification', (opts) => {
         let result = opts.result;
         let modelType = opts.modelType;
 
@@ -218,16 +274,39 @@ window.WebServer = async () => {
 
         els.imageClassify.row.style.display = 'none';
 
-        if (result.classification) {
+        if (result.classification && !opts.dontUpdateTable) {
+            const showOnlyTopResults = Object.keys(result.classification).length > 10 && !result.visual_anomaly_grid;
+
             if (isFirstClassification) {
-                for (let ix = 0; ix < Object.keys(result.classification).length; ix++) {
-                    const key = Object.keys(result.classification)[ix];
+                if (showOnlyTopResults) {
+                    // only 1 results th
+                    {
+                        let th = document.createElement('th');
+                        th.scope = 'col';
+                        th.textContent = th.title = 'Top 5 results';
+                        th.classList.add('px-0', 'text-center');
+                        els.resultsThead.appendChild(th);
+                    }
 
-                    let th = document.createElement('th');
-                    th.scope = 'col';
-                    th.classList.add('px-0', 'text-center');
-                    th.textContent = th.title = key;
-                    els.resultsThead.appendChild(th);
+                    // unless also have anomaly...
+                    if (Object.keys(result.classification).indexOf('anomaly') > -1) {
+                        let th = document.createElement('th');
+                        th.scope = 'col';
+                        th.textContent = th.title = 'anomaly';
+                        th.classList.add('px-0', 'text-center');
+                        els.resultsThead.appendChild(th);
+                    }
+                }
+                else {
+                    for (let ix = 0; ix < Object.keys(result.classification).length; ix++) {
+                        const key = Object.keys(result.classification)[ix];
+
+                        let th = document.createElement('th');
+                        th.scope = 'col';
+                        th.classList.add('px-0', 'text-center');
+                        th.textContent = th.title = key;
+                        els.resultsThead.appendChild(th);
+                    }
                 }
 
                 if (result.visual_anomaly_grid) {
@@ -264,15 +343,53 @@ window.WebServer = async () => {
             let td1 = document.createElement('td');
             td1.textContent = (++inferenceIx).toString();
             tr.appendChild(td1);
-            for (let k of Object.keys(result.classification)) {
+            if (showOnlyTopResults) {
+                // only print top 5
                 let td = document.createElement('td');
-                td.classList.add('text-center');
-                td.textContent = result.classification[k].toFixed(2);
-                if (result.classification[k] === highest && !isVisualAnomaly) {
-                    td.style.fontWeight = 600;
+
+                let results = [];
+                for (const key of Object.keys(result.classification)) {
+                    if (key === 'anomaly') continue;
+
+                    results.push({
+                        label: key,
+                        value: result.classification[key],
+                    });
+                }
+
+                const top = results.sort((a, b) => b.value - a.value).slice(0, 5);
+                for (let ix = 0; ix < top.length; ix++) {
+                    let span = ix === 0 ? document.createElement('strong') : document.createElement('span');
+                    span.textContent = `${top[ix].label}: ${top[ix].value.toFixed(2)}`;
+                    td.appendChild(span);
+
+                    if (ix !== top.length - 1) {
+                        let commaSpan = document.createElement('span');
+                        commaSpan.textContent = ', ';
+                        td.appendChild(commaSpan);
+                    }
                 }
                 tr.appendChild(td);
+
+                if (Object.keys(result.classification).indexOf('anomaly') > -1) {
+                    let anomalyTd = document.createElement('td');
+                    anomalyTd.classList.add('text-center');
+                    anomalyTd.textContent = result.classification.anomaly.toFixed(2);
+                    tr.appendChild(anomalyTd);
+                }
+            }
+            else {
+                for (let k of Object.keys(result.classification)) {
+                    let td = document.createElement('td');
+                    td.classList.add('text-center');
+                    td.textContent = result.classification[k].toFixed(2);
+                    if (result.classification[k] === highest && !isVisualAnomaly) {
+                        td.style.fontWeight = 600;
+                    }
+                    tr.appendChild(td);
+                }
             }
+
             if (result.visual_anomaly_grid) {
                 let td = document.createElement('td');
                 td.classList.add('text-center');
@@ -406,7 +523,9 @@ window.WebServer = async () => {
                 els.cameraContainer.appendChild(el);
             }
         }
-    });
+    }
+
+    socket.on('classification', onClassification);
 
     if (els.websocketAddress) {
         els.websocketAddress.textContent = `ws://${location.host}`;

package/examples/js/classify-camera.js

@@ -48,9 +48,7 @@ const { ImageClassifier, LinuxImpulseRunner, Ffmpeg, Imagesnap, RunnerHelloHasAn
             camera = new Imagesnap();
         }
         else if (process.platform === 'linux') {
-            camera = new Ffmpeg(false /* verbose */, {
-                scaleAndCropInPipeline: true,
-            });
+            camera = new Ffmpeg(false /* verbose */);
         }
         else {
             throw new Error('Unsupported platform "' + process.platform + '"');

package/examples/ts/classify-camera.ts

@@ -49,9 +49,7 @@ import { RunnerHelloHasAnomaly } from "../../library/classifier/linux-impulse-ru
             camera = new Imagesnap();
         }
         else if (process.platform === 'linux') {
-            camera = new GStreamer(false /* verbose */, {
-                scaleAndCropInPipeline: true,
-            });
+            camera = new GStreamer(false /* verbose */);
         }
         else {
             throw new Error('Unsupported platform "' + process.platform + '"');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "edge-impulse-linux",
-  "version": "1.16.2",
+  "version": "1.17.2",
   "description": "Node.js SDK and tools for Edge Impulse for Linux",
   "directories": {
     "example": "examples"
@@ -42,6 +42,7 @@
     "form-data": "4.0.4",
     "inquirer": "7.0.0",
     "inquirer-search-list": "1.2.6",
+    "koffi": "2.12.1",
     "multer": "1.4.5-lts.1",
     "node-fetch": "2.7.0",
     "npm-registry-fetch": "17.1.0",

package/test/gstreamer.test.ts

@@ -1288,8 +1288,6 @@ Device found:
 
         });
 
-
-
         // https://forum.edgeimpulse.com/t/edge-impulse-on-coral-edgetpu/2311
         it("coral edge tpu with iMX6S", async () => {
             const gstOutput = `Probing devices...
@@ -5234,7 +5232,7 @@ Freeing pipeline ...
 
             const gstreamer = new GStreamer(false, {
                 spawnHelperOverride: spawnHelper,
-                scaleAndCropInPipeline: true,
+                dontRunCleanupLoop: true,
             });
             await gstreamer.init();
             const launchResp = await gstreamer.getGstreamerLaunchCommand({
@@ -5251,10 +5249,9 @@ Freeing pipeline ...
 
             // console.log('launchResp', launchResp);
 
-            assert.equal(launchResp.invokeProcess, 'spawn');
             assert.equal(launchResp.command, 'gst-launch-1.0');
-            assert.equal(launchResp.args.join(' '),
-                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! jpegenc ! multifilesink location=test%05d.jpg');
+            assert.equal(launchResp.pipeline,
+                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! jpegenc ! multifilesink location=resized%05d.jpg');
         });
 
         it("w/ inference dims #1", async () => {
@@ -5277,7 +5274,7 @@ Freeing pipeline ...
 
             const gstreamer = new GStreamer(false, {
                 spawnHelperOverride: spawnHelper,
-                scaleAndCropInPipeline: true,
+                dontRunCleanupLoop: true,
             });
             await gstreamer.init();
             const launchResp = await gstreamer.getGstreamerLaunchCommand({
@@ -5298,10 +5295,13 @@ Freeing pipeline ...
 
             // console.log('launchResp', launchResp);
 
-            assert.equal(launchResp.invokeProcess, 'spawn');
             assert.equal(launchResp.command, 'gst-launch-1.0');
-            assert.equal(launchResp.args.join(' '),
-                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! videocrop left=180 right=180 ! videoscale method=lanczos ! video/x-raw,width=320,height=320 ! jpegenc ! multifilesink location=test%05d.jpg');
+            assert.equal(launchResp.pipeline,
+                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! ' +
+                'tee name=t ' +
+                    't. ! queue ! jpegenc ! multifilesink location=original%05d.jpg ' +
+                    't. ! queue ! videocrop left=180 right=180 ! videoscale method=lanczos ! video/x-raw,width=320,height=320 ! jpegenc ! multifilesink location=resized%05d.jpg'
+            );
         });
 
         it("w/ inference dims #2", async () => {
@@ -5324,7 +5324,7 @@ Freeing pipeline ...
 
             const gstreamer = new GStreamer(false, {
                 spawnHelperOverride: spawnHelper,
-                scaleAndCropInPipeline: true,
+                dontRunCleanupLoop: true,
             });
             await gstreamer.init();
             const launchResp = await gstreamer.getGstreamerLaunchCommand({
@@ -5345,10 +5345,13 @@ Freeing pipeline ...
 
             // console.log('launchResp', launchResp);
 
-            assert.equal(launchResp.invokeProcess, 'spawn');
             assert.equal(launchResp.command, 'gst-launch-1.0');
-            assert.equal(launchResp.args.join(' '),
-                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! videoscale method=lanczos ! video/x-raw,width=320,height=320 ! jpegenc ! multifilesink location=test%05d.jpg');
+            assert.equal(launchResp.pipeline,
+                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! ' +
+                'tee name=t ' +
+                    't. ! queue ! jpegenc ! multifilesink location=original%05d.jpg ' +
+                    't. ! queue ! videoscale method=lanczos ! video/x-raw,width=320,height=320 ! jpegenc ! multifilesink location=resized%05d.jpg'
+            );
         });
 
         it("w/ inference dims #3", async () => {
@@ -5371,7 +5374,7 @@ Freeing pipeline ...
 
             const gstreamer = new GStreamer(false, {
                 spawnHelperOverride: spawnHelper,
-                scaleAndCropInPipeline: true,
+                dontRunCleanupLoop: true,
             });
             await gstreamer.init();
             const launchResp = await gstreamer.getGstreamerLaunchCommand({
@@ -5392,10 +5395,9 @@ Freeing pipeline ...
 
             // console.log('launchResp', launchResp);
 
-            assert.equal(launchResp.invokeProcess, 'spawn');
             assert.equal(launchResp.command, 'gst-launch-1.0');
-            assert.equal(launchResp.args.join(' '),
-                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! jpegenc ! multifilesink location=test%05d.jpg');
+            assert.equal(launchResp.pipeline,
+                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! jpegenc ! multifilesink location=resized%05d.jpg');
         });
 
         it("w/ inference dims #4", async () => {
@@ -5418,7 +5420,7 @@ Freeing pipeline ...
 
             const gstreamer = new GStreamer(false, {
                 spawnHelperOverride: spawnHelper,
-                scaleAndCropInPipeline: undefined,
+                dontRunCleanupLoop: true,
             });
             await gstreamer.init();
             const launchResp = await gstreamer.getGstreamerLaunchCommand({
@@ -5439,10 +5441,13 @@ Freeing pipeline ...
 
             // console.log('launchResp', launchResp);
 
-            assert.equal(launchResp.invokeProcess, 'spawn');
             assert.equal(launchResp.command, 'gst-launch-1.0');
-            assert.equal(launchResp.args.join(' '),
-                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! jpegenc ! multifilesink location=test%05d.jpg');
+            assert.equal(launchResp.pipeline,
+                'pylonsrc ! video/x-raw,width=1440,height=1080 ! videoconvert ! ' +
+                'tee name=t ' +
+                    't. ! queue ! jpegenc ! multifilesink location=original%05d.jpg ' +
+                    't. ! queue ! videoscale method=lanczos ! video/x-raw,width=320,height=320 ! jpegenc ! multifilesink location=resized%05d.jpg'
+            );
         });
     });
 });
@@ -5491,7 +5496,7 @@ async function testGetDevices(output: {
     const gstreamer = new GStreamer(false, {
         spawnHelperOverride: spawnHelper,
         modeOverride: output?.modeOverride,
-        scaleAndCropInPipeline: true,
+        dontRunCleanupLoop: true,
     });
     await gstreamer.init();
     const devices = await gstreamer.getAllDevices();

package/examples/ts/classify-custom-from-test.ts

@@ -1,96 +0,0 @@
-import { LinuxImpulseRunner } from "../../library";
-import fs from 'fs';
-
-// This script expects two arguments:
-// 1. The model file
-// 2. A features file with raw data. You can get this by going to **Live classification > Load sample**
-//    then copy the features from the 'Raw features' block.
-
-(async () => {
-    try  {
-        if (!process.argv[2]) {
-            console.log('Missing one argument (model file)');
-            process.exit(1);
-        }
-
-        if (!process.argv[3]) {
-            console.log('Missing second argument (features file)');
-            process.exit(1);
-        }
-
-        // Load the model
-        let runner = new LinuxImpulseRunner(process.argv[2]);
-        let model = await runner.init();
-
-        const windowLengthMs = ((model.modelParameters.input_features_count /
-            model.modelParameters.frequency /
-            model.modelParameters.axis_count) * 1000);
-
-        console.log('Starting the custom classifier for',
-            model.project.owner + ' / ' + model.project.name, '(v' + model.project.deploy_version + ')');
-        console.log('Parameters', 'freq', model.modelParameters.frequency + 'Hz',
-            'window length', windowLengthMs + 'ms.',
-            'classes', model.modelParameters.labels);
-
-        // read the features file (comma separated numbers)
-        // Newlines deliminate samples
-        // Create an array of array of Numbers.  Each inner array is a sample.
-        let samples = (<string>await fs.promises.readFile(process.argv[3], 'utf-8'))
-            .trim().split('\n').map(line => line.split(',').map(n => Number(n)));
-
-        for (let features of samples) {
-            // and classify the data, this should match the classification in the Studio
-            let res = await runner.classify(features);
-
-            console.log('Begin output');
-            if (res.result.classification) {
-                let ret: number[] = [];
-                for (let k of Object.keys(res.result.classification)) {
-                    ret.push(res.result.classification[k]);
-                }
-                if (typeof res.result.visual_anomaly_max === 'number') {
-                    ret.push(res.result.visual_anomaly_max);
-                }
-                else if (typeof res.result.anomaly === 'number') {
-                    ret.push(res.result.anomaly);
-                }
-                console.log('[' + ret.map(x => x.toFixed(5)).join(', ') + ']');
-            }
-            if (res.result.bounding_boxes) {
-                for (let bb of res.result.bounding_boxes) {
-                    if (bb.value === 0) {
-                        continue;
-                    }
-                    console.log(`${bb.label} (${bb.value.toFixed(5)}) ` +
-                        `[ x: ${bb.x}, y: ${bb.y}, width: ${bb.width}, height: ${bb.height} ]`);
-                }
-            }
-            if (res.result.visual_anomaly_grid) {
-                for (let bb of res.result.visual_anomaly_grid) {
-                    console.log(`${bb.label} (${bb.value.toFixed(5)}) ` +
-                        `[ x: ${bb.x}, y: ${bb.y}, width: ${bb.width}, height: ${bb.height} ]`);
-                }
-            }
-            console.log('End output');
-
-            // if you want to fill in data on the fly you can also do this; e.g. if you have 3-axis accelerometer
-            // with 2 second window and 100Hz, then input_features_count=600, axis_count=3
-            // let features = [];
-            // for (let ix = 0; ix <
-            //   model.modelParameters.input_features_count; ix += model.modelParameters.axis_count) {
-            //     features.push(x);
-            //     features.push(y);
-            //     features.push(z);
-            // }
-            // and classify features
-        }
-        await runner.stop();
-    }
-    catch (ex) {
-        console.error(ex);
-        process.exit(1);
-    }
-    finally {
-        process.exit(0);
-    }
-})();