@zappar/zappar-cv 0.4.0-beta.4 → 2.0.0-beta.0

package/lib/sequencerecorder.js

@@ -1,20 +1,18 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SequenceDecoder = exports.SequenceRecorder = exports.SequenceRecorderPacketType = void 0;
-const loglevel_1 = require("./loglevel");
-var SequenceRecorderPacketType;
+import { zcerr } from "./loglevel";
+export var SequenceRecorderPacketType;
 (function (SequenceRecorderPacketType) {
     SequenceRecorderPacketType[SequenceRecorderPacketType["CAMERA_FRAME"] = 0] = "CAMERA_FRAME";
     SequenceRecorderPacketType[SequenceRecorderPacketType["ACCELEROMETER"] = 1] = "ACCELEROMETER";
     SequenceRecorderPacketType[SequenceRecorderPacketType["ROTATION_RATE"] = 2] = "ROTATION_RATE";
     SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE"] = 3] = "ATTITUDE";
     SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE_MATRIX"] = 4] = "ATTITUDE_MATRIX";
-})(SequenceRecorderPacketType = exports.SequenceRecorderPacketType || (exports.SequenceRecorderPacketType = {}));
-class SequenceRecorder {
+})(SequenceRecorderPacketType || (SequenceRecorderPacketType = {}));
+export class SequenceRecorder {
     constructor(estimatedFrames) {
         this._insertionByte = 0;
         this._numberPackets = 0;
         this._started = false;
+        this._hasLoggedUnsuppored = false;
         this._data = new ArrayBuffer(this._estimatedSize(estimatedFrames, 320, 240));
         this._dataView = new DataView(this._data);
         this._dataUint8 = new Uint8Array(this._data);
@@ -38,6 +36,9 @@ class SequenceRecorder {
         this._started = false;
         let trimmed = this._data.slice(0, this._insertionByte);
         this._data = trimmed;
+        this._dataUint8 = new Uint8Array(this._data);
+        this._dataFloat32 = new Float32Array(this._data);
+        this._dataView = new DataView(this._data);
     }
     data() {
         return this._dataUint8;
@@ -45,8 +46,15 @@ class SequenceRecorder {
     appendCameraFrame(c) {
         if (!this._started)
             return;
+        if (!c.data) {
+            if (this._hasLoggedUnsuppored)
+                return;
+            this._hasLoggedUnsuppored = true;
+            zcerr("Unable to record frames from this camera source");
+            return;
+        }
         if (c.data.byteLength !== (c.dataWidth * c.dataHeight)) {
-            loglevel_1.zcerr("Unable to record frame to sequence: greyscale array doesn't match dimensions");
+            zcerr("Unable to record frame to sequence: greyscale array doesn't match dimensions");
             return;
         }
         let increment = SequenceRecorder.psbPacketType + SequenceRecorder.cameraPacketSizeBytes + (c.dataWidth * c.dataHeight);
@@ -120,7 +128,6 @@ class SequenceRecorder {
         }
     }
 }
-exports.SequenceRecorder = SequenceRecorder;
 // Header (8 bytes): "UAR1" + number of frames (uint32)
 SequenceRecorder.headerSizeBytes = 8;
 // Per-frame packet byte sizes
@@ -144,7 +151,7 @@ SequenceRecorder.rotationRatePacketSizeBytes = 4 * 4; // t, x, y, z
 SequenceRecorder.attitudePacketSizeBytes = 4 * 4; // t, x, y, z
 // Attitude matrix
 SequenceRecorder.attitudeMatrixPacketSizeBytes = 4 * 16;
-class SequenceDecoder {
+export class SequenceDecoder {
     constructor(_data) {
         this._data = _data;
         this.dataByPacket = new Map();
@@ -269,4 +276,3 @@ class SequenceDecoder {
         }
     }
 }
-exports.SequenceDecoder = SequenceDecoder;

package/lib/sequencesource.d.ts

@@ -4,14 +4,20 @@ export declare class SequenceSource extends Source {
     private _impl;
     private _pipeline;
     private _packetNumber;
+    private _motionUpdatesQueued;
+    private _motionUpdatesSent;
     private _decoder;
+    private _minFrameGapMs;
+    private _lastUpdateTime;
     static create(p: zappar_pipeline_t): zappar_sequence_source_t;
     static get(m: zappar_sequence_source_t): SequenceSource | undefined;
     constructor(_impl: zappar_sequence_source_t, _pipeline: zappar_pipeline_t);
     loadFromMemory(data: ArrayBuffer): void;
-    getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
+    maxPlaybackFpsSet(fps: number): void;
+    getFrame(currentlyProcessing: boolean): void;
     destroy(): void;
     start(): void;
     pause(): void;
     uploadGL(info: CameraFrameInfo): void;
+    private _submitMotionUpdates;
 }

package/lib/sequencesource.js

@@ -1,24 +1,25 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SequenceSource = void 0;
-const loglevel_1 = require("./loglevel");
-const sequencerecorder_1 = require("./sequencerecorder");
-const source_1 = require("./source");
-const pipeline_1 = require("./pipeline");
-const gl_matrix_1 = require("gl-matrix");
+import { zcerr, zcout } from "./loglevel";
+import { SequenceDecoder, SequenceRecorderPacketType } from "./sequencerecorder";
+import { Source } from "./source";
+import { Pipeline } from "./pipeline";
+import { mat4 } from "gl-matrix";
 let latest = 1;
 let byId = new Map();
-class SequenceSource extends source_1.Source {
+export class SequenceSource extends Source {
     constructor(_impl, _pipeline) {
         super();
         this._impl = _impl;
         this._pipeline = _pipeline;
         this._packetNumber = 0;
+        this._motionUpdatesQueued = 0;
+        this._motionUpdatesSent = 0;
+        this._minFrameGapMs = 0;
+        this._lastUpdateTime = 0;
     }
     static create(p) {
         let ret = (latest++);
         byId.set(ret, new SequenceSource(ret, p));
-        loglevel_1.zcout("sequence_source_source_t initialized");
+        zcout("sequence_source_source_t initialized");
         return ret;
     }
     static get(m) {
@@ -26,68 +27,70 @@ class SequenceSource extends source_1.Source {
     }
     loadFromMemory(data) {
         try {
-            this._decoder = new sequencerecorder_1.SequenceDecoder(data);
+            this._decoder = new SequenceDecoder(data);
             this._packetNumber = 0;
+            this._lastUpdateTime = 0;
         }
         catch (err) {
-            loglevel_1.zcerr("Unable to decode sequence:", err);
+            zcerr("Unable to decode sequence:", err);
         }
     }
+    maxPlaybackFpsSet(fps) {
+        if (fps <= 0) {
+            this._minFrameGapMs = 0;
+            return;
+        }
+        this._minFrameGapMs = 1000.0 / fps;
+    }
     getFrame(currentlyProcessing) {
-        let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+        let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
+        if (!this._decoder)
+            return;
+        this._submitMotionUpdates(pipeline);
+        if (this._motionUpdatesQueued > this._motionUpdatesSent)
+            return;
         if (currentlyProcessing)
             return;
-        if (!this._decoder)
+        let curTime = performance.now();
+        if (curTime < this._lastUpdateTime + this._minFrameGapMs)
             return;
-        while (this._packetNumber < this._decoder.numberPackets) {
-            let packet = this._decoder.dataByPacket.get(this._packetNumber);
-            this._packetNumber++;
-            if (!packet)
-                return;
-            switch (packet.type) {
-                case sequencerecorder_1.SequenceRecorderPacketType.ACCELEROMETER:
-                    pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ROTATION_RATE:
-                    pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE:
-                    pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE_MATRIX:
-                    pipeline.motionAttitudeMatrix(packet.attitude);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME:
-                    {
-                        let pixels = pipeline.cameraPixelArrays.pop();
-                        while (pixels) {
-                            if (pixels.byteLength === packet.greyscale.byteLength)
-                                break;
-                            pixels = pipeline.cameraPixelArrays.pop();
-                        }
-                        if (!pixels) {
-                            pixels = new ArrayBuffer(packet.greyscale.byteLength);
-                        }
-                        (new Uint8Array(pixels)).set(packet.greyscale);
-                        return {
-                            data: pixels,
-                            cameraSourceData: packet.greyscale,
-                            cameraModel: packet.cameraModel,
-                            cameraToDevice: packet.cameraToDevice,
-                            dataWidth: packet.width,
-                            dataHeight: packet.height,
-                            texture: undefined,
-                            userFacing: packet.userFacing,
-                            uvTransform: gl_matrix_1.mat4.create(),
-                            cameraSource: this
-                        };
-                        break;
-                    }
-            }
+        if (this._packetNumber >= this._decoder.numberPackets)
+            return;
+        let packet = this._decoder.dataByPacket.get(this._packetNumber);
+        if (packet.type != SequenceRecorderPacketType.CAMERA_FRAME)
+            return;
+        this._packetNumber++;
+        let pixels = pipeline.cameraPixelArrays.pop();
+        while (pixels) {
+            if (pixels.byteLength === packet.greyscale.byteLength)
+                break;
+            pixels = pipeline.cameraPixelArrays.pop();
+        }
+        if (!pixels) {
+            pixels = new ArrayBuffer(packet.greyscale.byteLength);
         }
-        return undefined;
+        (new Uint8Array(pixels)).set(packet.greyscale);
+        this._lastUpdateTime = curTime;
+        let info = {
+            data: pixels,
+            cameraSourceData: packet.greyscale,
+            cameraModel: packet.cameraModel,
+            cameraToDevice: packet.cameraToDevice,
+            dataWidth: packet.width,
+            dataHeight: packet.height,
+            texture: undefined,
+            userFacing: packet.userFacing,
+            uvTransform: mat4.create(),
+            cameraSource: this
+        };
+        // We can queue the motion updates for the follwing frame immediately
+        // as this camera frame will be sent immediately, but the next motion
+        // updates will be one frame later
+        this._submitMotionUpdates(pipeline);
+        const token = pipeline.registerToken(info);
+        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
     }
     destroy() {
         byId.delete(this._impl);
@@ -95,19 +98,19 @@ class SequenceSource extends source_1.Source {
     }
     start() {
         var _a;
-        let p = pipeline_1.Pipeline.get(this._pipeline);
+        let p = Pipeline.get(this._pipeline);
         if (p && p.currentCameraSource !== this) {
             (_a = p.currentCameraSource) === null || _a === void 0 ? void 0 : _a.pause();
             p.currentCameraSource = this;
         }
     }
     pause() {
-        let p = pipeline_1.Pipeline.get(this._pipeline);
+        let p = Pipeline.get(this._pipeline);
         if (p && p.currentCameraSource === this)
             p.currentCameraSource = undefined;
     }
     uploadGL(info) {
-        let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+        let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
         let gl = pipeline === null || pipeline === void 0 ? void 0 : pipeline.glContext;
@@ -124,5 +127,40 @@ class SequenceSource extends source_1.Source {
         gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, info.dataWidth, info.dataHeight, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, new Uint8Array(info.cameraSourceData));
         gl.bindTexture(gl.TEXTURE_2D, null);
     }
+    _submitMotionUpdates(pipeline) {
+        let pendingMotionUpdate = false;
+        while (this._packetNumber < this._decoder.numberPackets) {
+            let packet = this._decoder.dataByPacket.get(this._packetNumber);
+            if (packet.type == SequenceRecorderPacketType.CAMERA_FRAME)
+                break;
+            switch (packet.type) {
+                case SequenceRecorderPacketType.ACCELEROMETER:
+                    pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case SequenceRecorderPacketType.ROTATION_RATE:
+                    pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case SequenceRecorderPacketType.ATTITUDE:
+                    pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case SequenceRecorderPacketType.ATTITUDE_MATRIX:
+                    pipeline.motionAttitudeMatrix(packet.attitude);
+                    pendingMotionUpdate = true;
+                    break;
+            }
+            this._packetNumber++;
+        }
+        // If we've submitted motion updates, add a setTimeout to reflect when
+        // the message serializer will actually have sent them over to the worker
+        if (pendingMotionUpdate) {
+            this._motionUpdatesQueued++;
+            setTimeout(() => {
+                this._motionUpdatesSent++;
+            }, 0);
+        }
+        return;
+    }
 }
-exports.SequenceSource = SequenceSource;

package/lib/serializer.js

@@ -1,7 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.MessageSerializer = void 0;
-class MessageSerializer {
+export class MessageSerializer {
     constructor(_messageSender) {
         this._messageSender = _messageSender;
         this._freeBufferPool = [];
@@ -229,4 +226,3 @@ class MessageSerializer {
             this._offset++;
     }
 }
-exports.MessageSerializer = MessageSerializer;

package/lib/shader.js

@@ -1,7 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.linkProgram = exports.compileShader = void 0;
-function compileShader(gl, type, src) {
+export function compileShader(gl, type, src) {
     let ret = gl.createShader(type);
     if (!ret)
         throw new Error("Unable to create shader");
@@ -12,11 +9,9 @@ function compileShader(gl, type, src) {
         throw new Error("Shader compile error: " + msg);
     return ret;
 }
-exports.compileShader = compileShader;
-function linkProgram(gl, prog) {
+export function linkProgram(gl, prog) {
     gl.linkProgram(prog);
     let msg = gl.getProgramInfoLog(prog);
     if (msg && msg.trim().length > 0)
         throw new Error("Unable to link: " + msg);
 }
-exports.linkProgram = linkProgram;

package/lib/source.d.ts

@@ -1,17 +1,19 @@
+/// <reference types="dom-webcodecs" />
 export interface CameraFrameInfo {
-    data: ArrayBuffer;
+    data?: ArrayBuffer;
     texture: WebGLTexture | undefined;
     dataWidth: number;
     dataHeight: number;
     cameraModel: Float32Array;
     cameraToDevice: Float32Array;
-    uvTransform: Float32Array;
-    userFacing: boolean;
     cameraSource: Source;
     cameraSourceData?: any;
+    uvTransform?: Float32Array;
+    userFacing: boolean;
+    frame?: VideoFrame | ImageBitmap;
 }
 export declare abstract class Source {
-    abstract getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
+    abstract getFrame(currentlyProcessing: boolean): void;
     abstract pause(): void;
     abstract uploadGL(info: CameraFrameInfo): void;
 }

package/lib/source.js

@@ -1,6 +1,2 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Source = void 0;
-class Source {
+export class Source {
 }
-exports.Source = Source;

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.4.0-beta.4";
+export declare const VERSION = "2.0.0-beta.0";

package/lib/version.js

@@ -1,4 +1 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.VERSION = void 0;
-exports.VERSION = "0.4.0-beta.4";
+export const VERSION = "2.0.0-beta.0";

package/lib/worker-client.d.ts

@@ -1,3 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-export declare function launchWorker(): Promise<void>;
+export declare function launchWorker(worker?: Worker): Promise<void>;

package/lib/worker-client.js

@@ -1,4 +1,3 @@
-"use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,46 +7,30 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.launchWorker = exports.messageManager = void 0;
-const bundleworker_mainthread_1 = require("./bundleworker-mainthread");
-const messages_1 = require("./messages");
-exports.messageManager = new messages_1.MsgManager();
-let Worker;
-if (typeof Z_STANDALONE !== 'undefined' && Z_STANDALONE) {
-    Worker = require('worker-loader?inline=fallback!./worker');
-}
-else {
-    Worker = require('worker-loader!./worker');
-}
-if (Worker.default)
-    Worker = Worker.default;
-function launchWorker() {
+import { MsgManager } from "./messages";
+export let messageManager = new MsgManager();
+export function launchWorker(worker) {
     return __awaiter(this, void 0, void 0, function* () {
-        let worker = new Worker();
+        if (!worker)
+            worker = new Worker(new URL("./worker", import.meta.url), { type: 'module' });
         worker.postMessage({
             t: "wasm",
-            url: require("./zcv.wasm").default
+            url: new URL("./zcv.wasm", import.meta.url).toString()
         });
-        worker.postMessage({
-            t: "port",
-            port: bundleworker_mainthread_1.bundleWorkerMessageChannel.port1
-        }, [bundleworker_mainthread_1.bundleWorkerMessageChannel.port1]);
         yield waitForLoad(worker);
         function sendOutgoing() {
-            let msgs = exports.messageManager.getOutgoingMessages();
+            let msgs = messageManager.getOutgoingMessages();
             for (let msg of msgs) {
                 worker.postMessage(msg.msg, msg.transferables);
             }
         }
-        exports.messageManager.onOutgoingMessage.bind(sendOutgoing);
+        messageManager.onOutgoingMessage.bind(sendOutgoing);
         sendOutgoing();
         worker.addEventListener("message", evt => {
-            exports.messageManager.postIncomingMessage(evt.data);
+            messageManager.postIncomingMessage(evt.data);
         });
     });
 }
-exports.launchWorker = launchWorker;
 function waitForLoad(w) {
     return new Promise(resolve => {
         let listener = (msg) => {

package/lib/worker-imagebitmap.d.ts

@@ -0,0 +1,5 @@
+import { zappar_cwrap } from "./gen/zappar-native";
+import { zappar_server } from "./gen/zappar-server";
+import { MsgManager } from "./messages";
+import { ImageBitmapC2S } from "./workerinterface";
+export declare function handleImageBitmap(m: ImageBitmapC2S, r: zappar_cwrap, server: zappar_server, mgr: MsgManager): void;

package/lib/worker-imagebitmap.js

@@ -0,0 +1,59 @@
+import { ImageProcessGL } from "./image-process-gl";
+import { profile } from "./profile";
+import { mat4 } from "gl-matrix";
+let pixels;
+let texture;
+let _imageProcessor;
+let _gl;
+function getImageProcessor() {
+    if (!_imageProcessor || !_gl) {
+        const canvas = new OffscreenCanvas(1, 1);
+        _gl = canvas.getContext("webgl");
+        if (!_gl)
+            throw new Error("Unable to get offscreen GL context");
+        _imageProcessor = new ImageProcessGL(_gl);
+    }
+    return [_imageProcessor, _gl];
+}
+export function handleImageBitmap(m, r, server, mgr) {
+    // img: ImageBitmap,
+    //                                 rot: number,
+    //                                 tokenId: number,
+    //                                 r: zappar_cwrap,
+    //                                 p: zappar_pipeline_t,
+    //                                 userFacing: boolean,
+    //                                 server: zappar_server,
+    //                                 mgr: MsgManager) {
+    const [imageProcessor, gl] = getImageProcessor();
+    if (!texture) {
+        texture = gl.createTexture();
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    }
+    if (!texture)
+        return;
+    if (!pixels || pixels.byteLength < profile.dataWidth * profile.dataHeight) {
+        console.log("Generating pixel buffer", profile.dataWidth * profile.dataHeight);
+        pixels = new ArrayBuffer(profile.dataWidth * profile.dataHeight);
+    }
+    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing);
+    let info = imageProcessor.readFrame(texture, pixels);
+    let msg = {
+        t: "imageBitmapS2C",
+        dataWidth: info.dataWidth,
+        dataHeight: info.dataHeight,
+        frame: m.i,
+        userFacing: info.userFacing,
+        uvTransform: info.uvTransform || mat4.create(),
+        tokenId: m.tokenId, p: m.p
+    };
+    mgr.postOutgoingMessage(msg, [m.i]);
+    let pipeline = server._pipeline_by_instance.get(m.p);
+    if (pipeline) {
+        r.pipeline_camera_frame_submit(pipeline, pixels, info.dataWidth, info.dataHeight, m.tokenId, m.cameraToDevice, m.cameraModel, info.userFacing);
+        r.pipeline_frame_update(pipeline);
+        server.exploreState();
+    }
+}

package/lib/worker-server.d.ts

@@ -1,8 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-declare global {
-    interface Window {
-        bundle_adjust_message: any;
-    }
-}
 export declare function launchWorkerServer(wasmUrl: string): Promise<void>;