edge-impulse-linux 1.17.2 → 1.17.4

package/build/sdk/studio/sdk/model/listOrganizationPortalsResponseAllOfPortals.js

@@ -60,6 +60,11 @@ ListOrganizationPortalsResponseAllOfPortals.attributeTypeMap = [
         "baseName": "bucketUrl",
         "type": "string"
     },
+    {
+        "name": "storageProvider",
+        "baseName": "storageProvider",
+        "type": "string"
+    },
     {
         "name": "created",
         "baseName": "created",

package/build/sdk/studio/sdk/model/listOrganizationPortalsResponseAllOfPortals.js.map

@@ -1 +1 @@
-{"version":3,"file":"listOrganizationPortalsResponseAllOfPortals.js","sourceRoot":"","sources":["../../../../../sdk/studio/sdk/model/listOrganizationPortalsResponseAllOfPortals.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAGH,MAAa,2CAA2C;IA4DpD,MAAM,CAAC,mBAAmB;QACtB,OAAO,2CAA2C,CAAC,gBAAgB,CAAC;IACxE,CAAC;;AA9DL,kGA+DC;AApDU,yDAAa,GAAuB,SAAS,CAAC;AAE9C,4DAAgB,GAA0D;IAC7E;QACI,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,IAAI;QAChB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,MAAM;QACd,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,aAAa;QACrB,UAAU,EAAE,aAAa;QACzB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,KAAK;QACb,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,UAAU;QAClB,UAAU,EAAE,UAAU;QACtB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,YAAY;QACpB,UAAU,EAAE,YAAY;QACxB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,YAAY;QACpB,UAAU,EAAE,YAAY;QACxB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,WAAW;QACvB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,MAAM;KACjB;CAAK,CAAC"}
+{"version":3,"file":"listOrganizationPortalsResponseAllOfPortals.js","sourceRoot":"","sources":["../../../../../sdk/studio/sdk/model/listOrganizationPortalsResponseAllOfPortals.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAGH,MAAa,2CAA2C;IAkEpD,MAAM,CAAC,mBAAmB;QACtB,OAAO,2CAA2C,CAAC,gBAAgB,CAAC;IACxE,CAAC;;AApEL,kGAqEC;AAzDU,yDAAa,GAAuB,SAAS,CAAC;AAE9C,4DAAgB,GAA0D;IAC7E;QACI,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,IAAI;QAChB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,MAAM;QACd,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,aAAa;QACrB,UAAU,EAAE,aAAa;QACzB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,KAAK;QACb,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,UAAU;QAClB,UAAU,EAAE,UAAU;QACtB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,YAAY;QACpB,UAAU,EAAE,YAAY;QACxB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,YAAY;QACpB,UAAU,EAAE,YAAY;QACxB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,WAAW;QACvB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,iBAAiB;QACzB,UAAU,EAAE,iBAAiB;QAC7B,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,MAAM;KACjB;CAAK,CAAC"}

package/build/sdk/studio/sdk/model/modelPrediction.d.ts

@@ -9,6 +9,7 @@
  * https://openapi-generator.tech
  * Do not edit the class manually.
  */
+import { BoundingBoxWithScore } from './boundingBoxWithScore';
 export declare class ModelPrediction {
     'sampleId': number;
     'startMs': number;
@@ -24,6 +25,10 @@ export declare class ModelPrediction {
     * Only set for visual anomaly projects. 2D array of shape (n, n) with raw anomaly scores, where n varies based on the image input size and the specific visual anomaly algorithm used. The scores corresponds to each grid cell in the image\'s spatial matrix.
     */
     'anomalyScores'?: Array<Array<number>>;
+    /**
+    * Only set for object detection projects. Coordinates are scaled 0..1, not absolute values.
+    */
+    'boundingBoxes'?: Array<BoundingBoxWithScore>;
     static discriminator: string | undefined;
     static attributeTypeMap: Array<{
         name: string;

package/build/sdk/studio/sdk/model/modelPrediction.js

@@ -59,6 +59,11 @@ ModelPrediction.attributeTypeMap = [
         "name": "anomalyScores",
         "baseName": "anomalyScores",
         "type": "Array<Array<number>>"
+    },
+    {
+        "name": "boundingBoxes",
+        "baseName": "boundingBoxes",
+        "type": "Array<BoundingBoxWithScore>"
     }
 ];
 //# sourceMappingURL=modelPrediction.js.map

package/build/sdk/studio/sdk/model/modelPrediction.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelPrediction.js","sourceRoot":"","sources":["../../../../../sdk/studio/sdk/model/modelPrediction.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAGH,MAAa,eAAe;IA4DxB,MAAM,CAAC,mBAAmB;QACtB,OAAO,eAAe,CAAC,gBAAgB,CAAC;IAC5C,CAAC;;AA9DL,0CA+DC;AA/CU,6BAAa,GAAuB,SAAS,CAAC;AAE9C,gCAAgB,GAA0D;IAC7E;QACI,MAAM,EAAE,UAAU;QAClB,UAAU,EAAE,UAAU;QACtB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,OAAO;QACf,UAAU,EAAE,OAAO;QACnB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,OAAO;QACf,UAAU,EAAE,OAAO;QACnB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,YAAY;QACpB,UAAU,EAAE,YAAY;QACxB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,mBAAmB;QAC3B,UAAU,EAAE,mBAAmB;QAC/B,MAAM,EAAE,SAAS;KACpB;IACD;QACI,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,eAAe;QACvB,UAAU,EAAE,eAAe;QAC3B,MAAM,EAAE,sBAAsB;KACjC;CAAK,CAAC"}
+{"version":3,"file":"modelPrediction.js","sourceRoot":"","sources":["../../../../../sdk/studio/sdk/model/modelPrediction.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;GAUG;;;AAIH,MAAa,eAAe;IAqExB,MAAM,CAAC,mBAAmB;QACtB,OAAO,eAAe,CAAC,gBAAgB,CAAC;IAC5C,CAAC;;AAvEL,0CAwEC;AApDU,6BAAa,GAAuB,SAAS,CAAC;AAE9C,gCAAgB,GAA0D;IAC7E;QACI,MAAM,EAAE,UAAU;QAClB,UAAU,EAAE,UAAU;QACtB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,OAAO;QACf,UAAU,EAAE,OAAO;QACnB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,OAAO;QACf,UAAU,EAAE,OAAO;QACnB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,YAAY;QACpB,UAAU,EAAE,YAAY;QACxB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,mBAAmB;QAC3B,UAAU,EAAE,mBAAmB;QAC/B,MAAM,EAAE,SAAS;KACpB;IACD;QACI,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,QAAQ;KACnB;IACD;QACI,MAAM,EAAE,eAAe;QACvB,UAAU,EAAE,eAAe;QAC3B,MAAM,EAAE,sBAAsB;KACjC;IACD;QACI,MAAM,EAAE,eAAe;QACvB,UAAU,EAAE,eAAe;QAC3B,MAAM,EAAE,6BAA6B;KACxC;CAAK,CAAC"}

package/cli/linux/webserver/public/assets/mobileclient.css

@@ -4239,3 +4239,13 @@ video {
 .embed-view #capture-camera {
     margin-top: -68px !important;
 }
+
+.classification-top-overlay {
+    position: absolute;
+    left: 0.5rem;
+    top: 0.5rem;
+    color: white;
+    text-align: left;;
+    font-family: monospace;
+    font-size: 0.75rem;
+}

package/cli/linux/webserver/public/webserver.js

@@ -1,4 +1,7 @@
-window.WebServer = async () => {
+window.WebServer = async (vmStr) => {
+
+    const vm = JSON.parse(decodeURIComponent(vmStr));
+    console.log('vm', vm);
 
     const els = {
         title: document.querySelector('#header-row h1'),
@@ -255,6 +258,8 @@ window.WebServer = async () => {
     });
     window.addEventListener('resize', onWindowResize);
 
+    let lastFivePerfCalPredictions = [];
+
     function onClassification(opts) {
         lastClassification = opts;
 
@@ -263,7 +268,9 @@ window.WebServer = async () => {
 
         els.timePerInference.textContent = opts.timeMs;
         els.additionalInfo.textContent = opts.additionalInfo;
-        els.timePerInferenceContainer.style.display = '';
+        if (!vm.isEmbedView) {
+            els.timePerInferenceContainer.style.display = '';
+        }
         els.additionalInfoContainer.style.display = '';
 
         console.log('classification', opts.result, opts.timeMs);
@@ -323,12 +330,31 @@ window.WebServer = async () => {
 
             els.imageClassify.row.style.display = '';
 
-            let conclusion = 'uncertain';
+            let conclusion = vm.hasPerformanceCalibration ? '...' : 'uncertain';
             let highest = Math.max(...Object.values(result.classification));
 
             for (let k of Object.keys(result.classification)) {
                 if (result.classification[k] >= 0.55) {
-                    conclusion = k + ' (' + result.classification[k].toFixed(2) + ')';
+                    if (vm.hasPerformanceCalibration) {
+                        conclusion = k;
+                    }
+                    else {
+                        conclusion = k + ' (' + result.classification[k].toFixed(2) + ')';
+                    }
+                }
+            }
+
+            // for perfcal models, if one of the last 5 predictions was the keyword => select that
+            if (vm.hasPerformanceCalibration) {
+                lastFivePerfCalPredictions.push(conclusion);
+                if (lastFivePerfCalPredictions.length > 5) {
+                    lastFivePerfCalPredictions = lastFivePerfCalPredictions.slice(
+                        lastFivePerfCalPredictions.length - 5);
+                }
+
+                const dedup = Array.from(new Set(lastFivePerfCalPredictions));
+                if (dedup.length === 2 && dedup.find(x => x === '...')) {
+                    conclusion = dedup.find(x => x !== '...');
                 }
             }
 
@@ -417,18 +443,61 @@ window.WebServer = async () => {
             }
 
             els.imageClassify.text.textContent = conclusion;
+
+            // for image classification models, draw overlay on top of the image with top 3 conclusions in embed mode
+            if (vm.sensorType === 'camera' && vm.isEmbedView) {
+                let results = [];
+                for (const key of Object.keys(result.classification)) {
+                    if (key === 'anomaly') continue;
+
+                    results.push({
+                        label: key,
+                        value: result.classification[key],
+                    });
+                }
+
+                // don't change order if we only have 3
+                const top = results.length > 3 ?
+                    results.sort((a, b) => b.value - a.value).slice(0, 3) :
+                    results;
+                if (result.visual_anomaly_grid) {
+                    // also visual AD? add to top results
+                    top.push({
+                        label: 'anomaly',
+                        value: result.visual_anomaly_max,
+                    });
+                }
+
+                let classificationOverlayEl = els.cameraContainer.querySelector('.classification-top-overlay');
+                if (!classificationOverlayEl) {
+                    classificationOverlayEl = document.createElement('div');
+                    classificationOverlayEl.classList.add('classification-top-overlay');
+                    els.cameraContainer.appendChild(classificationOverlayEl);
+                }
+                classificationOverlayEl.textContent = '';
+
+                for (const topRes of top) {
+                    const topDiv = document.createElement('div');
+                    topDiv.textContent = `${topRes.label} (${topRes.value.toFixed(2)})`;
+                    classificationOverlayEl.appendChild(topDiv);
+                }
+
+                els.imageClassify.row.style.display = 'none';
+            }
         }
         if (result.bounding_boxes) {
             let factor = els.cameraImg.naturalHeight / els.cameraImg.clientHeight;
 
-            for (let b of result.bounding_boxes) {
+            for (let b of result.object_tracking || result.bounding_boxes) {
                 let bb = {
                     x: b.x / factor,
                     y: b.y / factor,
                     width: b.width / factor,
                     height: b.height / factor,
-                    label: b.label,
-                    value: b.value
+                    label: 'object_id' in b ?
+                        `${b.label} (ID ${b.object_id})` :
+                        b.label,
+                    value: 'value' in b ? b.value : undefined,
                 };
 
                 if (!labelToColor[bb.label]) {
@@ -462,7 +531,10 @@ window.WebServer = async () => {
                 let label = document.createElement('div');
                 label.classList.add('bounding-box-label');
                 label.style.background = color;
-                label.textContent = bb.label + ' (' + bb.value.toFixed(2) + ')';
+                label.textContent = bb.label;
+                if (typeof bb.value === 'number') {
+                    label.textContent += ' (' + bb.value.toFixed(2) + ')';
+                }
                 if (modelType === 'constrained_object_detection') {
                     el.style.whiteSpace = 'nowrap';
                 }
@@ -530,4 +602,68 @@ window.WebServer = async () => {
     if (els.websocketAddress) {
         els.websocketAddress.textContent = `ws://${location.host}`;
     }
+
+    // Here's a helper function that'll loop every second, checks if "classification-top-overlay" is present
+    // and then switches between white/black text automatically. If this element is not created, it just sits idle
+    (async () => {
+        function getAvgBrightness(img, x, y, w, h) {
+            const canvas = document.createElement('canvas');
+            const ctx = canvas.getContext('2d');
+            canvas.width = img.naturalWidth;
+            canvas.height = img.naturalHeight;
+            ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+
+            // Scale from CSS coords (relative to image on page) to natural pixel coords
+            const scaleX = img.naturalWidth / img.clientWidth;
+            const scaleY = img.naturalHeight / img.clientHeight;
+
+            const sx = (x - img.getBoundingClientRect().x) * scaleX;
+            const sy = (y - img.getBoundingClientRect().y) * scaleY;
+            const sw = w * scaleX;
+            const sh = h * scaleY;
+
+            const data = ctx.getImageData(sx, sy, sw, sh).data;
+
+            let total = 0;
+            for (let i = 0; i < data.length; i += 4) {
+                const r = data[i];
+                const g = data[i + 1];
+                const b = data[i + 2];
+                total += (r * 299 + g * 587 + b * 114) / 1000;
+            }
+
+            // debug info
+            // const cropCanvas = document.createElement('canvas');
+            // cropCanvas.width  = sw;
+            // cropCanvas.height = sh;
+            // const cctx = cropCanvas.getContext('2d');
+            // cctx.drawImage(img, sx, sy, sw, sh, 0, 0, sw, sh);
+            // const cropEl = document.querySelector('#crop') || document.createElement('img');
+            // cropEl.id = 'crop';
+            // cropEl.src = cropCanvas.toDataURL('image/png');
+            // document.body.appendChild(cropEl);
+
+            return total / (data.length / 4);
+        }
+
+        if (!vm.isEmbedView || vm.sensorType !== 'camera') return;
+
+        while (1) {
+            const labelEl = document.querySelector('.classification-top-overlay');
+            if (!labelEl) {
+                await new Promise(resolve => setTimeout(resolve, 1000));
+                continue;
+            }
+
+            const rect = labelEl.getBoundingClientRect();
+            const brightness = getAvgBrightness(els.cameraImg, rect.x, rect.y, rect.width, rect.height);
+            if (brightness > 180) {
+                labelEl.style.color = 'black';
+            }
+            else {
+                labelEl.style.color = 'white';
+            }
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+    })();
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "edge-impulse-linux",
-  "version": "1.17.2",
+  "version": "1.17.4",
   "description": "Node.js SDK and tools for Edge Impulse for Linux",
   "directories": {
     "example": "examples"