edge-impulse-linux 1.2.8 → 1.3.0

package/cli/linux/linux.ts

@@ -5,7 +5,7 @@ import inquirer from 'inquirer';
 import { initCliApp, setupCliApp } from "../init-cli-app";
 import { RemoteMgmt, RemoteMgmtDevice, RemoteMgmtDeviceSampleEmitter } from "../../shared/daemon/remote-mgmt-service";
 import { MgmtInterfaceSampleRequestSample } from "../../shared/MgmtInterfaceTypes";
-import { makeImage, makeWav, upload } from '../make-image';
+import { makeImage, makeVideo, makeWav, upload } from '../make-image';
 import { Config, EdgeImpulseConfig } from "../config";
 import { EventEmitter } from "tsee";
 import { Mutex } from 'async-mutex';
@@ -19,6 +19,8 @@ import Path from 'path';
 import fs from 'fs';
 import Websocket from 'ws';
 import TypedEmitter from 'typed-emitter';
+import { Prophesee } from "../../library/sensors/prophesee";
+import { VideoRecorder } from "../../library/sensors/video-recorder";
 
 const packageVersion = (<{ version: string }>JSON.parse(fs.readFileSync(
     Path.join(__dirname, '..', '..', '..', 'package.json'), 'utf-8'))).version;
@@ -30,6 +32,8 @@ program
     .option('--hmac-key <key>', 'HMAC key to sign new data with (overrides current credentials)')
     .option('--disable-camera', `Don't prompt for camera`)
     .option('--disable-microphone', `Don't prompt for microphone`)
+    .option('--width <px>', 'Desired width of the camera stream')
+    .option('--height <px>', 'Desired height of the camera stream')
     .option('--clean', 'Clear credentials')
     .option('--silent', `Run in silent mode, don't prompt for credentials`)
     .option('--dev', 'List development servers, alternatively you can use the EI_HOST environmental variable ' +
@@ -46,6 +50,16 @@ const apiKeyArgv = <string | undefined>program.apiKey;
 const hmacKeyArgv = <string | undefined>program.hmacKey;
 const noCamera: boolean = !!program.disableCamera;
 const noMicrophone: boolean = !!program.disableMicrophone;
+const isProphesee = process.env.PROPHESEE_CAM === '1';
+const dimensions = program.width && program.height ? {
+    width: Number(program.width),
+    height: Number(program.height)
+} : undefined;
+
+if ((program.width && !program.height) || (!program.width && program.height)) {
+    console.error('--width and --height need to either be both specified or both omitted');
+    process.exit(1);
+}
 
 const SERIAL_PREFIX = '\x1b[33m[SER]\x1b[0m';
 
@@ -67,7 +81,7 @@ const cliOptions = {
 };
 
 class LinuxDevice extends (EventEmitter as new () => TypedEmitter<{
-    snapshot: (buffer: Buffer) => void
+    snapshot: (buffer: Buffer, filename: string) => void
 }>) implements RemoteMgmtDevice  {
     private _camera: ICamera | undefined;
     private _config: EdgeImpulseConfig;
@@ -86,24 +100,21 @@ class LinuxDevice extends (EventEmitter as new () => TypedEmitter<{
         this._devKeys = devKeys;
 
         if (this._camera) {
-            this._camera.on('snapshot', async (buffer) => {
+            this._camera.on('snapshot', async (buffer, filename) => {
                 const id = ++this._snapshotId;
                 const release = await this._snapshotMutex.acquire();
 
-                // limit to 5 frames a second & no new frames should have come in...
+                // limit to 10 frames a second & no new frames should have come in...
                 try {
                     if (this._snapshotStreaming &&
-                        Date.now() - +this._lastSnapshot > 200 &&
+                        Date.now() - +this._lastSnapshot >= 100 &&
                         id === this._snapshotId) {
 
                         const jpg = sharp(buffer);
 
                         const resized = await jpg.resize(undefined, 96).jpeg().toBuffer();
 
-                        if (verboseArgv) {
-                            console.log(Date.now(), 'sending snapshot');
-                        }
-                        this.emit('snapshot', resized);
+                        this.emit('snapshot', resized, filename);
                         this._lastSnapshot = new Date();
                     }
                 }
@@ -149,11 +160,19 @@ class LinuxDevice extends (EventEmitter as new () => TypedEmitter<{
             });
         }
         if (camera) {
+            let str = dimensions ? `(${dimensions.width}x${dimensions.height})` : `640x480`;
             sensors.push({
-                name: 'Camera (640x480)',
+                name: `Camera (${str})`,
                 frequencies: [],
                 maxSampleLengthS: 60000
             });
+            if (isProphesee) {
+                sensors.push({
+                    name: 'Video (1280x720)',
+                    frequencies: [],
+                    maxSampleLengthS: 60000
+                });
+            }
         }
         return sensors;
     }
@@ -162,6 +181,10 @@ class LinuxDevice extends (EventEmitter as new () => TypedEmitter<{
         return true;
     }
 
+    supportsSnapshotStreamingWhileCapturing() {
+        return true;
+    }
+
     beforeConnect() {
         return Promise.resolve();
     }
@@ -216,6 +239,60 @@ class LinuxDevice extends (EventEmitter as new () => TypedEmitter<{
 
             console.log(SERIAL_PREFIX, 'Sampling finished');
         }
+        else if (data.sensor?.startsWith('Video')) {
+            if (!this._camera) {
+                throw new Error('Linux daemon was started with --no-camera');
+            }
+
+            console.log(SERIAL_PREFIX, 'Waiting 2 seconds');
+
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+
+            ee.emit('started');
+
+            // give some time to emit...
+            await await new Promise((resolve) => setTimeout(resolve, 10));
+
+            let video = new VideoRecorder(this._camera, verboseArgv);
+            let videoEe = await video.record(data.length);
+
+            videoEe.on('processing', () => ee.emit('processing'));
+
+            let mp4 = await new Promise<Buffer>((resolve, reject) => {
+                if (!this._camera) {
+                    return reject('No camera');
+                }
+
+                videoEe.on('error', err => {
+                    reject(err);
+                });
+
+                videoEe.on('done', buffer => {
+                    resolve(buffer);
+                });
+            });
+
+            let img = makeVideo(mp4, this._devKeys.hmacKey, data.label + '.mp4');
+
+            console.log(SERIAL_PREFIX, 'Uploading sample to',
+                this._config.endpoints.internal.ingestion + data.path + '...');
+
+            ee.emit('uploading');
+
+            await upload({
+                apiKey: this._devKeys.apiKey,
+                filename: data.label + '.mp4',
+                processed: img,
+                allowDuplicates: false,
+                category: data.path.indexOf('/training') > -1 ? 'training' : 'testing',
+                config: this._config,
+                dataBuffer: mp4,
+                label: data.label,
+                boundingBoxes: undefined
+            });
+
+            console.log(SERIAL_PREFIX, 'Sampling finished');
+        }
         else if (data.sensor === 'Microphone') {
             if (noMicrophone) {
                 throw new Error('Linux daemon was started with --no-microphone');
@@ -348,7 +425,10 @@ let configFactory: Config;
         await configFactory.setLinuxProjectId(projectId);
 
         if (!noCamera) {
-            if (process.platform === 'darwin') {
+            if (isProphesee) {
+                camera = new Prophesee(verboseArgv);
+            }
+            else if (process.platform === 'darwin') {
                 camera = new Imagesnap();
             }
             else if (process.platform === 'linux') {
@@ -458,13 +538,23 @@ let configFactory: Config;
 
             console.log(SERIAL_PREFIX, 'Using camera', cameraDevice, 'starting...');
 
-            await camera.start({
-                device: cameraDevice,
-                intervalMs: 200,
-            });
+            if (isProphesee) {
+                await camera.start({
+                    device: cameraDevice,
+                    intervalMs: 40,
+                    dimensions: dimensions
+                });
+            }
+            else {
+                await camera.start({
+                    device: cameraDevice,
+                    intervalMs: 200,
+                    dimensions: dimensions
+                });
+            }
 
             camera.on('error', error => {
-                console.log('imagesnap error', error);
+                console.log('camera error', error);
             });
 
             console.log(SERIAL_PREFIX, 'Connected to camera');

package/cli/linux/runner-downloader.ts

@@ -30,9 +30,9 @@ export class RunnerDownloader extends EventEmitter<{
 
         let downloadType: string;
         if (process.platform === 'darwin') {
-            if (process.arch !== 'x64') {
+            if (process.arch !== 'x64' && process.arch !== 'arm64') {
                 throw new Error('Unsupported architecture "' + process.arch + '", only ' +
-                    'x64 supported for now');
+                    'x64 or arm64 supported for now');
             }
             downloadType = 'runner-mac-x86_64';
         }

package/cli/linux/runner.ts

@@ -20,6 +20,7 @@ import socketIO from 'socket.io';
 import sharp from 'sharp';
 import { AudioRecorder } from '../../library';
 import { ips } from '../get-ips';
+import { Prophesee } from '../../library/sensors/prophesee';
 
 const RUNNER_PREFIX = '\x1b[33m[RUN]\x1b[0m';
 const BUILD_PREFIX = '\x1b[32m[BLD]\x1b[0m';
@@ -309,7 +310,10 @@ function getModelPath(projectId: number, version: number) {
 
 async function connectCamera(cf: Config) {
     let camera: ICamera;
-    if (process.platform === 'darwin') {
+    if (process.env.PROPHESEE_CAM === '1') {
+        camera = new Prophesee(verboseArgv);
+    }
+    else if (process.platform === 'darwin') {
         camera = new Imagesnap();
     }
     else if (process.platform === 'linux') {
@@ -412,7 +416,14 @@ function startWebServer(model: ModelInformation, camera: ICamera, imgClassifier:
     // you can also get the actual image being classified from 'imageClassifier.on("result")',
     // but then you're limited by the inference speed.
     // here we get a direct feed from the camera so we guarantee the fps that we set earlier.
+
+    let nextFrame = Date.now();
+    let processingFrame = false;
     camera.on('snapshot', async (data) => {
+        if (nextFrame > Date.now() || processingFrame) return;
+
+        processingFrame = true;
+
         let img;
         if (model.modelParameters.image_channel_count === 3) {
             img = sharp(data).resize({
@@ -430,6 +441,9 @@ function startWebServer(model: ModelInformation, camera: ICamera, imgClassifier:
         io.emit('image', {
             img: 'data:image/jpeg;base64,' + (await img.jpeg().toBuffer()).toString('base64')
         });
+
+        nextFrame = Date.now() + 50;
+        processingFrame = false;
     });
 
     imgClassifier.on('result', async (result, timeMs, imgAsJpg) => {

package/cli/linux/webserver/public/index.html

@@ -63,14 +63,14 @@
                         <div class="row mb-4">
                             <div class="col">
                                 <div class="capture-camera-inner">
-                                    <img style="height: 480px">
+                                    <img style="min-height: 480px">
                                 </div>
                             </div>
                         </div>
                         <div class="row mt-2" id="image-classification-conclusion">
                             <h2 class="col"></h2>
                         </div>
-                        <div class="row mt-2">
+                        <div class="row mt-2" id="time-per-inference-container" style="display: none">
                             <div class="col text-xs">Time per inference: <span id="time-per-inference"></span> ms.</div>
                         </div>
                     </div>

package/cli/linux/webserver/public/webserver.js

@@ -5,6 +5,7 @@ window.WebServer = async () => {
         cameraContainer: document.querySelector('#capture-camera .capture-camera-inner'),
         cameraImg: document.querySelector('#capture-camera img'),
         timePerInference: document.querySelector('#time-per-inference'),
+        timePerInferenceContainer: document.querySelector('#time-per-inference-container'),
         imageClassify: {
             row: document.querySelector('#image-classification-conclusion'),
             text: document.querySelector('#image-classification-conclusion .col'),
@@ -48,6 +49,7 @@ window.WebServer = async () => {
         let result = opts.result;
 
         els.timePerInference.textContent = opts.timeMs;
+        els.timePerInferenceContainer.style.display = '';
 
         console.log('classification', opts.result, opts.timeMs);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "edge-impulse-linux",
-  "version": "1.2.8",
+  "version": "1.3.0",
   "description": "Node.js SDK and tools for Edge Impulse for Linux",
   "directories": {
     "example": "examples"
@@ -11,7 +11,8 @@
   },
   "bin": {
     "edge-impulse-linux": "build/cli/linux/linux.js",
-    "edge-impulse-linux-runner": "build/cli/linux/runner.js"
+    "edge-impulse-linux-runner": "build/cli/linux/runner.js",
+    "edge-impulse-camera-debug": "build/cli/linux/camera-debug.js"
   },
   "main": "build/library/index.js",
   "types": "build/library/index.d.ts",
@@ -26,7 +27,7 @@
   },
   "homepage": "https://github.com/edgeimpulse/linux-sdk-node#readme",
   "dependencies": {
-    "@types/sharp": "0.27.1",
+    "@types/sharp": "0.29.1",
     "async-mutex": "0.2.6",
     "borc": "2.1.2",
     "cbor": "5.0.1",
@@ -37,7 +38,7 @@
     "npm-registry-fetch": "5.0.0",
     "request": "2.88.0",
     "request-promise": "4.2.4",
-    "sharp": "0.27.2",
+    "sharp": "0.29.1",
     "socket.io": "2.3.0",
     "tiny-async-pool": "1.1.0",
     "tsee": "1.3.0",