edge-impulse-linux 1.5.0 → 1.6.0

package/build/shared/viewmodels/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","sourceRoot":"","sources":["../../../shared/viewmodels/init.ts"],"names":[],"mappings":";;;AAAA,IAAY,QAuEX;AAvED,WAAY,QAAQ;IAChB,2CAAU,CAAA;IACV,qDAAe,CAAA;IACf,qCAAO,CAAA;IACP,yDAAiB,CAAA;IACjB,iEAAqB,CAAA;IACrB,2DAAkB,CAAA;IAClB,mDAAc,CAAA;IACd,yDAAiB,CAAA;IACjB,+DAAoB,CAAA;IACpB,yDAAiB,CAAA;IACjB,8DAAoB,CAAA;IACpB,8CAAY,CAAA;IACZ,8DAAoB,CAAA;IACpB,gEAAqB,CAAA;IACrB,kEAAsB,CAAA;IACtB,8EAA4B,CAAA;IAC5B,8CAAY,CAAA;IACZ,wCAAS,CAAA;IACT,0CAAU,CAAA;IACV,4CAAW,CAAA;IACX,4DAAmB,CAAA;IACnB,gFAA6B,CAAA;IAC7B,0DAAkB,CAAA;IAClB,4CAAW,CAAA;IACX,4CAAW,CAAA;IACX,gDAAa,CAAA;IACb,8CAAY,CAAA;IACZ,gDAAa,CAAA;IACb,0CAAU,CAAA;IACV,wEAAyB,CAAA;IACzB,8EAA4B,CAAA;IAC5B,4EAA2B,CAAA;IAC3B,wCAAS,CAAA;IACT,oEAAuB,CAAA;IACvB,gDAAa,CAAA;IACb,wDAAiB,CAAA;IACjB,oEAAuB,CAAA;IACvB,sDAAgB,CAAA;IAChB,4EAA2B,CAAA;IAC3B,gEAAqB,CAAA;IACrB,8DAAoB,CAAA;IACpB,kDAAc,CAAA;IACd,8DAAoB,CAAA;IACpB,0DAAkB,CAAA;IAClB,oDAAe,CAAA;IACf,sDAAgB,CAAA;IAChB,gDAAa,CAAA;IACb,0EAA0B,CAAA;IAC1B,kEAAsB,CAAA;IACtB,gEAAqB,CAAA;IACrB,sEAAwB,CAAA;IACxB,gEAAqB,CAAA;IACrB,kFAA8B,CAAA;IAC9B,oFAA+B,CAAA;IAC/B,gGAAqC,CAAA;IACrC,0FAAkC,CAAA;IAClC,wEAAyB,CAAA;IACzB,qEAAwB,CAAA;IACxB,uEAAyB,CAAA;IACzB,mFAA+B,CAAA;IAC/B,+DAAqB,CAAA;IACrB,yFAAkC,CAAA;IAClC,yEAA0B,CAAA;IAC1B,yEAA0B,CAAA;IAC1B,iEAAsB,CAAA;IACtB,uEAAyB,CAAA;IACzB,qGAAwC,CAAA;IACxC,2EAA2B,CAAA;IAC3B,yDAAkB,CAAA;IAClB,yDAAkB,CAAA;AACtB,CAAC,EAvEW,QAAQ,GAAR,gBAAQ,KAAR,gBAAQ,QAuEnB"}
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../../../shared/viewmodels/init.ts"],"names":[],"mappings":";;;AAAA,IAAY,QA4EX;AA5ED,WAAY,QAAQ;IAChB,2CAAU,CAAA;IACV,qDAAe,CAAA;IACf,qCAAO,CAAA;IACP,yDAAiB,CAAA;IACjB,iEAAqB,CAAA;IACrB,2DAAkB,CAAA;IAClB,mDAAc,CAAA;IACd,yDAAiB,CAAA;IACjB,+DAAoB,CAAA;IACpB,yDAAiB,CAAA;IACjB,8DAAoB,CAAA;IACpB,8CAAY,CAAA;IACZ,8DAAoB,CAAA;IACpB,gEAAqB,CAAA;IACrB,kEAAsB,CAAA;IACtB,8EAA4B,CAAA;IAC5B,8CAAY,CAAA;IACZ,wCAAS,CAAA;IACT,0CAAU,CAAA;IACV,4CAAW,CAAA;IACX,4DAAmB,CAAA;IACnB,gFAA6B,CAAA;IAC7B,0DAAkB,CAAA;IAClB,4CAAW,CAAA;IACX,4CAAW,CAAA;IACX,gDAAa,CAAA;IACb,8CAAY,CAAA;IACZ,gDAAa,CAAA;IACb,0CAAU,CAAA;IACV,wEAAyB,CAAA;IACzB,8EAA4B,CAAA;IAC5B,4EAA2B,CAAA;IAC3B,wCAAS,CAAA;IACT,oEAAuB,CAAA;IACvB,gDAAa,CAAA;IACb,wDAAiB,CAAA;IACjB,oEAAuB,CAAA;IACvB,sDAAgB,CAAA;IAChB,4EAA2B,CAAA;IAC3B,gEAAqB,CAAA;IACrB,kDAAc,CAAA;IACd,8DAAoB,CAAA;IACpB,0DAAkB,CAAA;IAClB,oDAAe,CAAA;IACf,sDAAgB,CAAA;IAChB,gDAAa,CAAA;IACb,oFAA+B,CAAA;IAC/B,0FAAkC,CAAA;IAClC,wFAAiC,CAAA;IACjC,oFAA+B,CAAA;IAC/B,oEAAuB,CAAA;IACvB,0EAA0B,CAAA;IAC1B,kEAAsB,CAAA;IACtB,gEAAqB,CAAA;IACrB,sEAAwB,CAAA;IACxB,gEAAqB,CAAA;IACrB,kFAA8B,CAAA;IAC9B,oFAA+B,CAAA;IAC/B,gGAAqC,CAAA;IACrC,0FAAkC,CAAA;IAClC,wEAAyB,CAAA;IACzB,qEAAwB,CAAA;IACxB,uEAAyB,CAAA;IACzB,mFAA+B,CAAA;IAC/B,+DAAqB,CAAA;IACrB,yFAAkC,CAAA;IAClC,yEAA0B,CAAA;IAC1B,yEAA0B,CAAA;IAC1B,iEAAsB,CAAA;IACtB,uEAAyB,CAAA;IACzB,qGAAwC,CAAA;IACxC,2EAA2B,CAAA;IAC3B,uEAAyB,CAAA;IACzB,yDAAkB,CAAA;IAClB,yDAAkB,CAAA;AACtB,CAAC,EA5EW,QAAQ,GAAR,gBAAQ,KAAR,gBAAQ,QA4EnB"}

package/cli/linux/runner.ts

@@ -24,7 +24,7 @@ import { Prophesee } from '../../library/sensors/prophesee';
 import { asyncMiddleware } from './webserver/middleware/asyncMiddleware';
 import multer from 'multer';
 import util from 'util';
-import { renderInferenceServerView } from './webserver/views/inference-server-view';
+import { InferenceServerModelViewModel, InferenceServerViewModel, renderInferenceServerView } from './webserver/views/inference-server-view';
 
 const RUNNER_PREFIX = '\x1b[33m[RUN]\x1b[0m';
 const BUILD_PREFIX = '\x1b[32m[BLD]\x1b[0m';
@@ -581,18 +581,35 @@ function startApiServer(model: ModelInformation, runner: LinuxImpulseRunner, pro
             text = text.concat([
                 `curl -v -X POST -H "Content-Type: application/json" -d '{"features": [5, 10, 15, 20]}' http://localhost:${port}/api/features`,
                 '',
-                '(Expecting ' + runner.getModel().modelParameters.input_features_count + ' features for this model)',
+                '(Expecting ' + model.modelParameters.input_features_count + ' features for this model)',
             ]);
         }
 
+        let modelVm: InferenceServerModelViewModel;
+        if (model.modelParameters.sensorType === 'camera') {
+            modelVm = {
+                mode: 'image',
+                width: model.modelParameters.image_input_width,
+                height: model.modelParameters.image_input_height,
+                depth: model.modelParameters.image_channel_count === 3 ? 'RGB' : 'Grayscale',
+                isObjectDetection: model.modelParameters.model_type === 'constrained_object_detection' ||
+                    model.modelParameters.model_type === 'object_detection',
+            };
+        }
+        else {
+            modelVm = {
+                mode: 'features',
+                featuresCount: model.modelParameters.input_features_count,
+            };
+        }
+
         const view = renderInferenceServerView({
             owner: model.project.owner,
             projectName: model.project.name,
             projectVersion: model.project.deploy_version,
-            featuresCount: runner.getModel().modelParameters.input_features_count,
-            mode: model.modelParameters.sensorType === 'camera' ? 'image' : 'features',
             serverPort: port,
             studioLink: projectStudioUrl,
+            model: modelVm,
         });
 
         res.status(200);
@@ -600,6 +617,11 @@ function startApiServer(model: ModelInformation, runner: LinuxImpulseRunner, pro
         res.end(view.toString());
     }));
 
+    app.get('/api/info', asyncMiddleware(async (req, res) => {
+        res.header('Content-Type', 'application/json');
+        res.end(JSON.stringify(model, null, 4) + '\n');
+    }));
+
     app.post('/api/features', asyncMiddleware(async (req, res) => {
         const body = <{ features: number[] }>req.body;
 
@@ -637,7 +659,13 @@ function startApiServer(model: ModelInformation, runner: LinuxImpulseRunner, pro
     }));
 
     app.post('/api/image', asyncMiddleware(async (req, res) => {
-        let features: number[];
+        let resized: {
+            features: number[],
+            originalWidth: number,
+            originalHeight: number,
+            newWidth: number,
+            newHeight: number,
+        };
 
         try {
             const multipartUpload = multer({ limits: { files: 1, fileSize: 100 * 1024 * 1024 } });
@@ -660,7 +688,7 @@ function startApiServer(model: ModelInformation, runner: LinuxImpulseRunner, pro
             console.log(RUNNER_PREFIX, 'Incoming inference request (/api/image) for ' +
                 (file.originalname || 'a file with size ' + file.size + ' bytes'));
 
-            features = (await ImageClassifier.resizeImage(model, file.buffer)).features;
+            resized = (await ImageClassifier.resizeImage(model, file.buffer, 'contain'));
         }
         catch (ex2) {
             const ex = <Error>ex2;
@@ -669,7 +697,40 @@ function startApiServer(model: ModelInformation, runner: LinuxImpulseRunner, pro
             return res.end(ex.message || ex.toString() + '\n');
         }
 
-        let response = await runner.classify(features);
+        let response = await runner.classify(resized.features);
+
+        let origFactor = resized.originalWidth / resized.originalHeight;
+        let newFactor = resized.newWidth / resized.newHeight;
+
+        let factor: number;
+        let offsetX: number;
+        let offsetY: number;
+
+        if (origFactor > newFactor) {
+            // boxed in with bands top/bottom
+            factor = resized.newWidth / resized.originalWidth;
+            offsetX = 0;
+            offsetY = (resized.newHeight - (resized.originalHeight * factor)) / 2;
+        }
+        else if (origFactor < newFactor) {
+            // boxed in with bands left/right
+            factor = resized.newHeight / resized.originalHeight;
+            offsetX = (resized.newWidth - (resized.originalWidth * factor)) / 2;
+            offsetY = 0;
+        }
+        else {
+            // image was already at the right aspect ratio
+            offsetX = 0;
+            offsetY = 0;
+            factor = resized.newWidth / resized.originalWidth;
+        }
+
+        for (const bb of response.result.bounding_boxes || []) {
+            bb.x = Math.round((bb.x / factor) - (offsetX / factor));
+            bb.width = Math.round((bb.width / factor));
+            bb.y = Math.round((bb.y / factor) - (offsetY / factor));
+            bb.height = Math.round((bb.height / factor));
+        }
 
         res.header('Content-Type', 'application/json');
         res.end(JSON.stringify(response, null, 4) + '\n');

package/cli/linux/webserver/public/assets/mobileclient.css

@@ -4169,7 +4169,11 @@ video {
     margin-left: -1.7rem;
     margin-right: -1.7rem;
     margin-bottom: 0.75rem;
-    margin-top: 2rem;
+    margin-top: 1.5rem;
     padding-left: 0.75rem;
     padding-right: 0.75rem;
 }
+
+#preview-image-container {
+    position: relative;
+}

package/cli/linux/webserver/public/inference-server.js

@@ -5,9 +5,18 @@ window.InferenceServer = async () => {
         runInferenceBtn: document.querySelector('#run-inference'),
         resultSection: document.querySelector('#result-section'),
         resultSectionPre: document.querySelector('#result-section pre'),
+        previewSection: document.querySelector('#preview-section'),
+        previewSectionImgContainer: document.querySelector('#preview-image-container'),
+        previewSectionImg:  document.querySelector('#preview-section img'),
         features: document.querySelector('#features'),
     };
 
+    const colors = [
+        '#e6194B', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#42d4f4', '#f032e6', '#fabed4',
+        '#469990', '#dcbeff', '#9A6324', '#fffac8', '#800000', '#aaffc3',
+    ];
+    const labelToColor = { };
+
     if (els.uploadFile) {
         els.uploadFile.oninput = () => {
             if (els.uploadFile.files.length > 0) {
@@ -83,15 +92,87 @@ window.InferenceServer = async () => {
             }
 
             const text = await resp.text();
+            let body;
             try {
-                const body = JSON.parse(text);
-
-                els.resultSection.style.display = '';
-                els.resultSectionPre.textContent = JSON.stringify(body, null, 4);
+                body = JSON.parse(text);
             }
             catch (ex) {
                 throw new Error(text);
             }
+
+            if (els.uploadFile && body.result.bounding_boxes) {
+                els.previewSection.style.display = '';
+
+                const dataUrl = await new Promise((resolve, reject) => {
+                    const reader = new FileReader();
+
+                    reader.onload = e => {
+                        resolve(e.target.result);
+                    };
+                    reader.onerror = err => {
+                        reject(err);
+                    };
+
+                    reader.readAsDataURL(els.uploadFile.files[0]);
+                });
+
+                els.previewSectionImg.src = dataUrl;
+
+                const result = body.result;
+                let factor = 1;
+
+                for (let oldEl of els.previewSectionImgContainer.querySelectorAll('.bounding-box-container')) {
+                    oldEl.parentNode.removeChild(oldEl);
+                }
+
+                for (let b of result.bounding_boxes) {
+                    let bb = {
+                        x: b.x / factor,
+                        y: b.y / factor,
+                        width: b.width / factor,
+                        height: b.height / factor,
+                        label: b.label,
+                        value: b.value
+                    };
+                    if (!labelToColor[bb.label]) {
+                        labelToColor[bb.label] = colors[0];
+                        colors.splice(0, 1);
+                    }
+                    let color = labelToColor[bb.label];
+                    let el = document.createElement('div');
+                    el.classList.add('bounding-box-container');
+                    el.style.position = 'absolute';
+                    el.style.border = 'solid 3px ' + color;
+                    // if (modelType === 'object_detection') {
+                        el.style.width = (bb.width) + 'px';
+                        el.style.height = (bb.height) + 'px';
+                        el.style.left = (bb.x) + 'px';
+                        el.style.top = (bb.y) + 'px';
+                    // }
+                    // else if (modelType === 'constrained_object_detection') {
+                    //     let centerX = bb.x + (bb.width / 2);
+                    //     let centerY = bb.y + (bb.height / 2);
+                    //     el.style.borderRadius = '10px';
+                    //     el.style.width = 20 + 'px';
+                    //     el.style.height = 20 + 'px';
+                    //     el.style.left = (centerX - 10) + 'px';
+                    //     el.style.top = (centerY - 10) + 'px';
+                    // }
+                    let label = document.createElement('div');
+                    label.classList.add('bounding-box-label');
+                    label.style.background = color;
+                    label.textContent = bb.label + ' (' + bb.value.toFixed(2) + ')';
+                    el.style.whiteSpace = 'nowrap';
+                    el.appendChild(label);
+                    els.previewSectionImgContainer.appendChild(el);
+                }
+            }
+            else {
+                els.previewSection.style.display = 'none';
+            }
+
+            els.resultSection.style.display = '';
+            els.resultSectionPre.textContent = JSON.stringify(body, null, 4);
         }
         catch (ex) {
             alert('Failed to run inference: ' + (ex.message || ex.toString()));

package/cli/linux/webserver/views/inference-server-view.ts

@@ -1,16 +1,31 @@
 import html from './escape-html-template-tag';
 
+export type InferenceServerModelViewModel = {
+    mode: 'features',
+    featuresCount: number,
+} | {
+    mode: 'image',
+    width: number,
+    height: number,
+    depth: 'Grayscale' | 'RGB',
+    isObjectDetection: boolean,
+};
+
 export type InferenceServerViewModel = {
     studioLink: string,
     owner: string,
     projectName: string,
     projectVersion: number,
     serverPort: number,
-    featuresCount: number,
-    mode: 'features' | 'image',
+    model: InferenceServerModelViewModel,
 };
 
 export const renderInferenceServerView = (vm: InferenceServerViewModel) => {
+    let resultsColClass = `col-12`;
+    if (vm.model.mode === 'image' && vm.model.isObjectDetection) {
+        resultsColClass = `col-12 col-lg-6`;
+    }
+
     return html`<!DOCTYPE html>
     <html>
 
@@ -63,24 +78,35 @@ export const renderInferenceServerView = (vm: InferenceServerViewModel) => {
                                     <h4>Model information</h4>
                                 </div>
                             </div>
+                            <div class="mb-2 text-sm">
+                                Inference server for "<a href="${vm.studioLink}" target="_blank">${vm.owner} / ${vm.projectName}</a>" (v${vm.projectVersion}).
+                            </div>
+                            <div class="row card-sub-header border-bottom">
+                                <div class="col">
+                                    <h4>Get model info</h4>
+                                </div>
+                            </div>
                             <div class="mb-3 text-sm">
-                                Inference server for "<a href="${vm.studioLink}" target="_blank">${vm.owner} / ${vm.projectName}</a>" (v${vm.projectVersion})
+                                <code>curl -v -X GET http://localhost:${vm.serverPort}/api/info</code>
                             </div>
                             <div class="row card-sub-header border-bottom">
                                 <div class="col">
                                     <h4>How to run inference</h4>
                                 </div>
                             </div>
-                            ${vm.mode === 'image' ? html`
+                            ${vm.model.mode === 'image' ? html`
                             <div class="mb-3 text-sm">
                                 <code>curl -v -X POST -F 'file=@path-to-an-image.jpg' http://localhost:${vm.serverPort}/api/image</code>
                             </div>
+                            <div class="mb-4 text-xs">
+                                (Expecting a ${vm.model.width} x ${vm.model.height} (${vm.model.depth}) image, images in other resolutions will be resized)
+                            </div>
                             ` : html`
                             <div class="mb-3 text-sm">
                                 <code>curl -v -X POST -H "Content-Type: application/json" -d '{"features": [5, 10, 15, 20]}' http://localhost:${vm.serverPort}/api/features</code>
                             </div>
                             <div class="mb-4 text-xs">
-                                (Expecting ${vm.featuresCount} features for this model)
+                                (Expecting ${vm.model.featuresCount} features for this model)
                             </div>
                             `}
                         </div>
@@ -89,16 +115,16 @@ export const renderInferenceServerView = (vm: InferenceServerViewModel) => {
                         <div class="card-body text-left mb--3">
                             <div class="row card-sub-header border-bottom mt--2">
                                 <div class="col">
-                                    <h4>Try it out</h4>
+                                    <h4>Try out inferencing</h4>
                                 </div>
                             </div>
-                            ${vm.mode === 'image' ? html`
+                            ${vm.model.mode === 'image' ? html`
                             <div class="mb-3">
                                 <input type="file" class="form-control form-control-sm border-0 px-0" id="upload-file">
                             </div>
                             ` : html`
                             <div class="mb-2 text-sm">
-                                Enter ${vm.featuresCount} features, split by ',' - for example from "Live classification".
+                                Enter ${vm.model.featuresCount} features, split by ',' - for example from "Live classification".
                             </div>
                             <div class="mb-3">
                                 <input class="form-control form-control-sm" id="features" placeholder="0, 0, 0, 0, ...">
@@ -107,14 +133,25 @@ export const renderInferenceServerView = (vm: InferenceServerViewModel) => {
                             <div class="mb-3">
                                 <button class="btn-sm btn btn-primary" id="run-inference" disabled>Run inference</button>
                             </div>
-                            <section id="result-section" style="display: none">
-                                <div class="row card-sub-header border-bottom">
-                                    <div class="col">
-                                        <h4>Result</h4>
+
+                            <div class="row mb-3" id="result-section" style="display: none">
+                                <div class="${resultsColClass}">
+                                    <div class="row card-sub-header border-bottom">
+                                        <div class="col">
+                                            <h4>Result</h4>
+                                        </div>
+                                    </div>
+                                    <pre class="text-xs mb-0">Bleep bloop</pre>
+                                </div>
+                                <div id="preview-section" class="${resultsColClass}" style="display: none">
+                                    <div class="row card-sub-header border-bottom">
+                                        <div class="col">
+                                            <h4>Preview</h4>
+                                        </div>
                                     </div>
+                                    <div id="preview-image-container" class="mt-3"><img id="preview-image"></img></div>
                                 </div>
-                                <pre class="mb-3 text-xs">Bleep bloop</pre>
-                            </section>
+                            </div>
                         </div>
                     </div>
                 </div>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "edge-impulse-linux",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Node.js SDK and tools for Edge Impulse for Linux",
   "directories": {
     "example": "examples"
@@ -8,7 +8,8 @@
   "scripts": {
     "build": "./node_modules/.bin/tsc -p .",
     "lint": "./node_modules/tslint/bin/tslint -p tsconfig.json",
-    "test": "tsc -b . && TS_NODE_PROJECT=./tsconfig.json mocha --config ./test/.mocharc.js"
+    "test": "tsc -b . && TS_NODE_PROJECT=./tsconfig.json mocha --config ./test/.mocharc.js",
+    "watch-runner": "func() { tsc-watch -b . --onSuccess \"node build/cli/linux/runner.js $1\" --noClear; }; func"
   },
   "bin": {
     "edge-impulse-linux": "build/cli/linux/linux.js",
@@ -65,6 +66,7 @@
     "@types/ws": "6.0.3",
     "mocha": "9.1.3",
     "ts-node": "10.4.0",
+    "tsc-watch": "4.6.2",
     "tslint": "6.1.2",
     "typescript": "3.9.3"
   }