@zappar/zappar-cv 0.4.0-beta.1 → 0.4.0-beta.10

package/lib/pipeline.js

@@ -10,7 +10,9 @@ const drawplane_1 = require("./drawplane");
 const event_1 = require("./event");
 const loglevel_1 = require("./loglevel");
 const sequencerecorder_1 = require("./sequencerecorder");
+const camera_source_map_1 = require("./camera-source-map");
 let byId = new Map();
+let identity = gl_matrix_1.mat4.create();
 class Pipeline {
     constructor(_client, _impl, _mgr) {
         this._client = _client;
@@ -23,6 +25,7 @@ class Pipeline {
         this.videoTextures = [];
         this.cameraPixelArrays = [];
         this._sequenceRecordDeviceAttitudeMatrices = true;
+        this._sequenceRecorderFirstCameraToken = 0;
         this.onGLContextReset = new event_1.Event();
     }
     static create(client, mgr) {
@@ -43,6 +46,10 @@ class Pipeline {
             }, [msg]);
         }
         this.pendingMessages = [];
+        this.cleanOldFrames();
+    }
+    cleanOldFrames() {
+        var _a, _b;
         let currentToken = this._client.pipeline_camera_frame_user_data(this._impl);
         if (!currentToken)
             return;
@@ -50,12 +57,14 @@ class Pipeline {
             if (t[0] < currentToken) {
                 if (t[1].texture)
                     this.videoTextures.push(t[1].texture);
+                (_b = (_a = t[1].frame) === null || _a === void 0 ? void 0 : _a.close) === null || _b === void 0 ? void 0 : _b.call(_a);
                 this.cameraTokens.delete(t[0]);
             }
         }
     }
     cameraTokenReturn(msg) {
-        if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices) {
+        if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices
+            && msg.token >= this._sequenceRecorderFirstCameraToken) {
             let info = this.cameraTokens.get(msg.token);
             if (info) {
                 if (msg.att)
@@ -71,6 +80,7 @@ class Pipeline {
         if (!this._sequenceRecorder)
             this._sequenceRecorder = new sequencerecorder_1.SequenceRecorder(expectedFrames);
         this._sequenceRecorder.start();
+        this._sequenceRecorderFirstCameraToken = this.nextCameraToken;
     }
     sequenceRecordStop() {
         var _a;
@@ -174,7 +184,7 @@ class Pipeline {
         let info = this.getCurrentCameraInfo();
         if (!info)
             return gl_matrix_1.mat4.create();
-        return drawcamera_1.cameraFrameTextureMatrix(info.dataWidth, info.dataHeight, sw, sh, info.uvTransform, mirror);
+        return drawcamera_1.cameraFrameTextureMatrix(info.dataWidth, info.dataHeight, sw, sh, info.uvTransform || identity, mirror);
     }
     cameraFrameUserFacing() {
         var _a;
@@ -191,8 +201,48 @@ class Pipeline {
         gl_matrix_1.mat4.invert(res, res);
         return res;
     }
+    videoFrameFromWorker(msg) {
+        let tokenId = this.nextCameraToken++;
+        const cameraSource = camera_source_map_1.getCameraSource(msg.source);
+        if (!cameraSource)
+            return;
+        this.cameraTokens.set(tokenId, {
+            dataWidth: msg.w,
+            dataHeight: msg.h,
+            texture: undefined,
+            frame: msg.d,
+            userFacing: msg.userFacing,
+            uvTransform: msg.uvTransform,
+            cameraModel: msg.cameraModel,
+            cameraToDevice: msg.cameraToDevice,
+            cameraSource
+        });
+        this.cleanOldFrames();
+    }
+    imageBitmapFromWorker(msg) {
+        let data = this.cameraTokens.get(msg.tokenId);
+        if (!data)
+            return;
+        data.dataWidth = msg.dataWidth;
+        data.dataHeight = msg.dataHeight;
+        data.frame = msg.frame;
+        data.userFacing = msg.userFacing;
+        data.uvTransform = msg.uvTransform;
+        this.tokensInFlight--;
+        this.cleanOldFrames();
+    }
+    uploadGL() {
+        var _a, _b;
+        let info = this.getCurrentCameraInfo();
+        (_b = (_a = info === null || info === void 0 ? void 0 : info.cameraSource) === null || _a === void 0 ? void 0 : _a.uploadGL) === null || _b === void 0 ? void 0 : _b.call(_a, info);
+    }
+    registerToken(info) {
+        let tokenId = this.nextCameraToken++;
+        this.cameraTokens.set(tokenId, info);
+        this.tokensInFlight++;
+        return tokenId;
+    }
     processGL() {
-        var _a;
         if (!this.glContext) {
             loglevel_1.zcerr("no GL context for camera frames - please call pipeline_gl_context_set");
             return;
@@ -203,26 +253,7 @@ class Pipeline {
             this.currentCameraSource.getFrame(true);
             return;
         }
-        let info = this.currentCameraSource.getFrame(false);
-        if (!info)
-            return;
-        let tokenId = this.nextCameraToken++;
-        this.cameraTokens.set(tokenId, info);
-        let msg = {
-            d: info.data,
-            p: this._impl,
-            width: info.dataWidth,
-            height: info.dataHeight,
-            token: tokenId,
-            userFacing: info.userFacing,
-            c2d: info.cameraToDevice,
-            cm: info.cameraModel,
-            t: "cameraFrameC2S"
-        };
-        if (!this._sequenceRecordDeviceAttitudeMatrices)
-            (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendCameraFrame(info);
-        this.tokensInFlight++;
-        this._mgr.postOutgoingMessage(msg, [info.data]);
+        this.currentCameraSource.getFrame(false);
     }
     motionAccelerometerSubmit(timestamp, x, y, z) {
         var _a;
@@ -246,9 +277,47 @@ class Pipeline {
         // This doesn't need to be added to the sequence since that's done on frame update instead
         this._client.pipeline_motion_attitude_matrix_submit(this._impl, m);
     }
-    uploadGL() {
-        let info = this.getCurrentCameraInfo();
-        info === null || info === void 0 ? void 0 : info.cameraSource.uploadGL(info);
+    sendCameraStreamToWorker(source, stream, userFacing) {
+        let msg = {
+            t: "streamC2S",
+            p: this._impl,
+            s: stream,
+            userFacing,
+            source
+        };
+        this._mgr.postOutgoingMessage(msg, [msg.s]);
+    }
+    sendCameraToScreenRotationToWorker(rot) {
+        let msg = {
+            p: this._impl,
+            t: "cameraToScreenC2S",
+            r: rot
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
+    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice) {
+        let msg = {
+            p: this._impl,
+            t: "imageBitmapC2S",
+            i: img,
+            r: rot,
+            tokenId,
+            userFacing,
+            cameraModel,
+            cameraToDevice
+        };
+        this._mgr.postOutgoingMessage(msg, [img]);
+    }
+    sendDataToWorker(data, token, width, height, userFacing, cameraToDevice, cameraModel) {
+        let msg = {
+            d: data,
+            p: this._impl,
+            width, height, token, userFacing,
+            c2d: cameraToDevice,
+            cm: cameraModel,
+            t: "cameraFrameC2S"
+        };
+        this._mgr.postOutgoingMessage(msg, [data]);
     }
 }
 exports.Pipeline = Pipeline;

package/lib/profile.d.ts

@@ -13,4 +13,6 @@ export declare let profile: {
     dataWidth: number;
     dataHeight: number;
     videoElementInDOM: boolean;
+    preferMediaStreamTrackProcessorCamera: boolean;
+    preferImageBitmapCamera: boolean;
 };

package/lib/profile.js

@@ -17,56 +17,67 @@ exports.profile = {
     videoHeight: 480,
     dataWidth: 320,
     dataHeight: 240,
-    videoElementInDOM: false
+    videoElementInDOM: false,
+    preferMediaStreamTrackProcessorCamera: false,
+    preferImageBitmapCamera: false
 };
-window["zeeProfile"] = exports.profile;
+if (typeof window !== "undefined") {
+    window["zeeProfile"] = exports.profile;
+    if (window.location.href.indexOf("_mstppipeline") >= 0) {
+        console.log("Configuring for MSTP camera pipeline (if supported)");
+        exports.profile.preferMediaStreamTrackProcessorCamera = true;
+    }
+    if (window.location.href.indexOf("_imagebitmappipeline") >= 0) {
+        console.log("Configuring for ImageBitmap camera pipeline (if supported)");
+        exports.profile.preferImageBitmapCamera = true;
+    }
+}
 let agent = new UAParser.UAParser();
-switch (agent.getBrowser().name) {
-    case "Safari":
-        exports.profile.deviceMotionMutliplier = 1.0;
-        if (window.orientation === undefined) {
-            // Desktop Safari
-            break;
-        }
-        exports.profile.videoElementInDOM = true;
-        exports.profile.showSafariPermissionsResetIfNecessary = true;
-        break;
-    case "Chrome":
-        let engine = agent.getEngine();
-        if (!engine || engine.name !== "WebKit") {
-            break;
-        }
-    case "Mobile Safari":
-        exports.profile.deviceMotionMutliplier = 1.0;
-        let version = agent.getOS().version;
-        if (version) {
-            let versionParts = version.split(".");
-            if (versionParts.length >= 2) {
-                const majorVersion = parseInt(versionParts[0]);
-                const minorVersion = parseInt(versionParts[1]);
-                if (majorVersion < 11 ||
-                    (majorVersion === 11 && minorVersion < 3)) {
-                    exports.profile.blacklisted = true;
-                }
-                if (majorVersion < 12 ||
-                    (majorVersion === 12 && minorVersion < 2)) {
-                    exports.profile.videoElementInDOM = true;
-                }
-                if ((majorVersion === 12 && minorVersion >= 2) || (majorVersion >= 13))
-                    exports.profile.showGyroPermissionsWarningIfNecessary = true;
-                if (majorVersion >= 13) {
-                    exports.profile.showSafariPermissionsResetIfNecessary = true;
-                }
-                if (((majorVersion >= 12 && minorVersion > 1) || (majorVersion >= 13)) &&
-                    navigator.mediaDevices &&
-                    navigator.mediaDevices.getSupportedConstraints &&
-                    navigator.mediaDevices.getSupportedConstraints().frameRate) {
-                    exports.profile.requestHighFrameRate = true;
+let os = (agent.getOS().name || "unknown").toLowerCase();
+let engine = (agent.getEngine().name || "unknown").toLowerCase();
+if (engine === "webkit" && os !== "ios") {
+    exports.profile.deviceMotionMutliplier = 1.0;
+    if (typeof window !== "undefined" && window.orientation !== undefined) {
+        // iPad
+        iDevice();
+    }
+}
+if (engine === "webkit" && os === "ios") {
+    exports.profile.deviceMotionMutliplier = 1.0;
+    iDevice();
+}
+function iDevice() {
+    let version = agent.getOS().version;
+    if (version) {
+        let versionParts = version.split(".");
+        if (versionParts.length >= 2) {
+            const majorVersion = parseInt(versionParts[0]);
+            const minorVersion = parseInt(versionParts[1]);
+            if (majorVersion < 11 ||
+                (majorVersion === 11 && minorVersion < 3)) {
+                exports.profile.blacklisted = true;
+            }
+            if (majorVersion < 12 ||
+                (majorVersion === 12 && minorVersion < 2)) {
+                exports.profile.videoElementInDOM = true;
+            }
+            if ((majorVersion === 12 && minorVersion >= 2) || (majorVersion >= 13))
+                exports.profile.showGyroPermissionsWarningIfNecessary = true;
+            if (majorVersion >= 13) {
+                exports.profile.showSafariPermissionsResetIfNecessary = true;
+            }
+            if (((majorVersion >= 12 && minorVersion > 1) || (majorVersion >= 13)) &&
+                navigator.mediaDevices &&
+                navigator.mediaDevices.getSupportedConstraints &&
+                navigator.mediaDevices.getSupportedConstraints().frameRate) {
+                exports.profile.requestHighFrameRate = true;
+                if (majorVersion < 14) {
                     // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
+                    // Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
                     exports.profile.videoHeight = 360;
                     exports.profile.dataHeight = 180;
                 }
             }
         }
-        break;
+    }
 }

package/lib/riff-reader.d.ts

@@ -0,0 +1,18 @@
+export interface Chunk {
+    ident: string;
+    data: Uint8Array;
+    subident?: string;
+    subchunks?: Chunk[];
+}
+export declare class RiffReader {
+    private _data;
+    private _paddingBytes;
+    truncatedChunks: boolean;
+    private _view;
+    root: Chunk;
+    constructor(_data: ArrayBuffer, _paddingBytes?: boolean);
+    private _parse;
+    private _parseChunk;
+    find(str: string, inChunk?: Chunk): Chunk | undefined;
+    has(str: string): boolean;
+}

package/lib/riff-reader.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RiffReader = void 0;
+const decoder = new TextDecoder();
+class RiffReader {
+    constructor(_data, _paddingBytes = false) {
+        this._data = _data;
+        this._paddingBytes = _paddingBytes;
+        this.truncatedChunks = false;
+        this._view = new DataView(this._data);
+        this.root = this._parse();
+    }
+    _parse() {
+        const header = decoder.decode(this._data.slice(0, 4));
+        if (header !== "RIFF")
+            throw new Error("Not a valid ZPT file");
+        return this._parseChunk(0);
+    }
+    _parseChunk(offset) {
+        const ident = decoder.decode(this._data.slice(offset, offset + 4));
+        const length = this._view.getUint32(offset + 4, true);
+        const data = new Uint8Array(this._data, offset + 8, length);
+        if (data.byteLength !== length)
+            this.truncatedChunks = true;
+        let subchunks;
+        let subident;
+        if (ident === "RIFF" || ident === "LIST") {
+            subident = decoder.decode(this._data.slice(offset + 8, offset + 12));
+            subchunks = [];
+            let currentOffset = 4;
+            while (currentOffset < data.byteLength - 1) {
+                const subchunk = this._parseChunk(offset + 8 + currentOffset);
+                subchunks.push(subchunk);
+                currentOffset += subchunk.data.byteLength + 8;
+                if (this._paddingBytes && subchunk.data.byteLength & 0x1)
+                    currentOffset++;
+            }
+        }
+        return { ident, data, subident, subchunks };
+    }
+    find(str, inChunk = this.root) {
+        if (inChunk.ident === str)
+            return inChunk;
+        if (!inChunk.subchunks)
+            return;
+        for (let c of inChunk.subchunks) {
+            const found = this.find(str, c);
+            if (found)
+                return found;
+        }
+    }
+    has(str) {
+        return this.find(str) !== undefined;
+    }
+}
+exports.RiffReader = RiffReader;

package/lib/sequencerecorder.d.ts

@@ -26,6 +26,7 @@ export declare class SequenceRecorder {
     private _insertionByte;
     private _numberPackets;
     private _started;
+    private _hasLoggedUnsuppored;
     constructor(estimatedFrames: number);
     private _estimatedSize;
     start(): void;

package/lib/sequencerecorder.js

@@ -15,6 +15,7 @@ class SequenceRecorder {
         this._insertionByte = 0;
         this._numberPackets = 0;
         this._started = false;
+        this._hasLoggedUnsuppored = false;
         this._data = new ArrayBuffer(this._estimatedSize(estimatedFrames, 320, 240));
         this._dataView = new DataView(this._data);
         this._dataUint8 = new Uint8Array(this._data);
@@ -38,6 +39,9 @@ class SequenceRecorder {
         this._started = false;
         let trimmed = this._data.slice(0, this._insertionByte);
         this._data = trimmed;
+        this._dataUint8 = new Uint8Array(this._data);
+        this._dataFloat32 = new Float32Array(this._data);
+        this._dataView = new DataView(this._data);
     }
     data() {
         return this._dataUint8;
@@ -45,6 +49,13 @@ class SequenceRecorder {
     appendCameraFrame(c) {
         if (!this._started)
             return;
+        if (!c.data) {
+            if (this._hasLoggedUnsuppored)
+                return;
+            this._hasLoggedUnsuppored = true;
+            loglevel_1.zcerr("Unable to record frames from this camera source");
+            return;
+        }
         if (c.data.byteLength !== (c.dataWidth * c.dataHeight)) {
             loglevel_1.zcerr("Unable to record frame to sequence: greyscale array doesn't match dimensions");
             return;

package/lib/sequencesource.d.ts

@@ -4,14 +4,20 @@ export declare class SequenceSource extends Source {
     private _impl;
     private _pipeline;
     private _packetNumber;
+    private _motionUpdatesQueued;
+    private _motionUpdatesSent;
     private _decoder;
+    private _minFrameGapMs;
+    private _lastUpdateTime;
     static create(p: zappar_pipeline_t): zappar_sequence_source_t;
     static get(m: zappar_sequence_source_t): SequenceSource | undefined;
     constructor(_impl: zappar_sequence_source_t, _pipeline: zappar_pipeline_t);
     loadFromMemory(data: ArrayBuffer): void;
-    getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
+    maxPlaybackFpsSet(fps: number): void;
+    getFrame(currentlyProcessing: boolean): void;
     destroy(): void;
     start(): void;
     pause(): void;
     uploadGL(info: CameraFrameInfo): void;
+    private _submitMotionUpdates;
 }

package/lib/sequencesource.js

@@ -14,6 +14,10 @@ class SequenceSource extends source_1.Source {
         this._impl = _impl;
         this._pipeline = _pipeline;
         this._packetNumber = 0;
+        this._motionUpdatesQueued = 0;
+        this._motionUpdatesSent = 0;
+        this._minFrameGapMs = 0;
+        this._lastUpdateTime = 0;
     }
     static create(p) {
         let ret = (latest++);
@@ -28,66 +32,68 @@ class SequenceSource extends source_1.Source {
         try {
             this._decoder = new sequencerecorder_1.SequenceDecoder(data);
             this._packetNumber = 0;
+            this._lastUpdateTime = 0;
         }
         catch (err) {
             loglevel_1.zcerr("Unable to decode sequence:", err);
         }
     }
+    maxPlaybackFpsSet(fps) {
+        if (fps <= 0) {
+            this._minFrameGapMs = 0;
+            return;
+        }
+        this._minFrameGapMs = 1000.0 / fps;
+    }
     getFrame(currentlyProcessing) {
         let pipeline = pipeline_1.Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
+        if (!this._decoder)
+            return;
+        this._submitMotionUpdates(pipeline);
+        if (this._motionUpdatesQueued > this._motionUpdatesSent)
+            return;
         if (currentlyProcessing)
             return;
-        if (!this._decoder)
+        let curTime = performance.now();
+        if (curTime < this._lastUpdateTime + this._minFrameGapMs)
             return;
-        while (this._packetNumber < this._decoder.numberPackets) {
-            let packet = this._decoder.dataByPacket.get(this._packetNumber);
-            this._packetNumber++;
-            if (!packet)
-                return;
-            switch (packet.type) {
-                case sequencerecorder_1.SequenceRecorderPacketType.ACCELEROMETER:
-                    pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ROTATION_RATE:
-                    pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE:
-                    pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE_MATRIX:
-                    pipeline.motionAttitudeMatrix(packet.attitude);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME:
-                    {
-                        let pixels = pipeline.cameraPixelArrays.pop();
-                        while (pixels) {
-                            if (pixels.byteLength === packet.greyscale.byteLength)
-                                break;
-                            pixels = pipeline.cameraPixelArrays.pop();
-                        }
-                        if (!pixels) {
-                            pixels = new ArrayBuffer(packet.greyscale.byteLength);
-                        }
-                        (new Uint8Array(pixels)).set(packet.greyscale);
-                        return {
-                            data: pixels,
-                            cameraSourceData: packet.greyscale,
-                            cameraModel: packet.cameraModel,
-                            cameraToDevice: packet.cameraToDevice,
-                            dataWidth: packet.width,
-                            dataHeight: packet.height,
-                            texture: undefined,
-                            userFacing: packet.userFacing,
-                            uvTransform: gl_matrix_1.mat4.create(),
-                            cameraSource: this
-                        };
-                        break;
-                    }
-            }
+        if (this._packetNumber >= this._decoder.numberPackets)
+            return;
+        let packet = this._decoder.dataByPacket.get(this._packetNumber);
+        if (packet.type != sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME)
+            return;
+        this._packetNumber++;
+        let pixels = pipeline.cameraPixelArrays.pop();
+        while (pixels) {
+            if (pixels.byteLength === packet.greyscale.byteLength)
+                break;
+            pixels = pipeline.cameraPixelArrays.pop();
+        }
+        if (!pixels) {
+            pixels = new ArrayBuffer(packet.greyscale.byteLength);
         }
-        return undefined;
+        (new Uint8Array(pixels)).set(packet.greyscale);
+        this._lastUpdateTime = curTime;
+        let info = {
+            data: pixels,
+            cameraSourceData: packet.greyscale,
+            cameraModel: packet.cameraModel,
+            cameraToDevice: packet.cameraToDevice,
+            dataWidth: packet.width,
+            dataHeight: packet.height,
+            texture: undefined,
+            userFacing: packet.userFacing,
+            uvTransform: gl_matrix_1.mat4.create(),
+            cameraSource: this
+        };
+        // We can queue the motion updates for the follwing frame immediately
+        // as this camera frame will be sent immediately, but the next motion
+        // updates will be one frame later
+        this._submitMotionUpdates(pipeline);
+        const token = pipeline.registerToken(info);
+        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
     }
     destroy() {
         byId.delete(this._impl);
@@ -124,5 +130,41 @@ class SequenceSource extends source_1.Source {
         gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, info.dataWidth, info.dataHeight, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, new Uint8Array(info.cameraSourceData));
         gl.bindTexture(gl.TEXTURE_2D, null);
     }
+    _submitMotionUpdates(pipeline) {
+        let pendingMotionUpdate = false;
+        while (this._packetNumber < this._decoder.numberPackets) {
+            let packet = this._decoder.dataByPacket.get(this._packetNumber);
+            if (packet.type == sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME)
+                break;
+            switch (packet.type) {
+                case sequencerecorder_1.SequenceRecorderPacketType.ACCELEROMETER:
+                    pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ROTATION_RATE:
+                    pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE:
+                    pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE_MATRIX:
+                    pipeline.motionAttitudeMatrix(packet.attitude);
+                    pendingMotionUpdate = true;
+                    break;
+            }
+            this._packetNumber++;
+        }
+        // If we've submitted motion updates, add a setTimeout to reflect when
+        // the message serializer will actually have sent them over to the worker
+        if (pendingMotionUpdate) {
+            this._motionUpdatesQueued++;
+            setTimeout(() => {
+                this._motionUpdatesSent++;
+            }, 0);
+        }
+        return;
+    }
 }
 exports.SequenceSource = SequenceSource;

package/lib/source.d.ts

@@ -1,17 +1,19 @@
+/// <reference types="dom-webcodecs" />
 export interface CameraFrameInfo {
-    data: ArrayBuffer;
+    data?: ArrayBuffer;
     texture: WebGLTexture | undefined;
     dataWidth: number;
     dataHeight: number;
     cameraModel: Float32Array;
     cameraToDevice: Float32Array;
-    uvTransform: Float32Array;
-    userFacing: boolean;
     cameraSource: Source;
     cameraSourceData?: any;
+    uvTransform?: Float32Array;
+    userFacing: boolean;
+    frame?: VideoFrame | ImageBitmap;
 }
 export declare abstract class Source {
-    abstract getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
+    abstract getFrame(currentlyProcessing: boolean): void;
     abstract pause(): void;
     abstract uploadGL(info: CameraFrameInfo): void;
 }

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.4.0-beta.1";
+export declare const VERSION = "0.4.0-beta.10";

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = "0.4.0-beta.1";
+exports.VERSION = "0.4.0-beta.10";

package/lib/worker-client.js

@@ -10,7 +10,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.launchWorker = exports.messageManager = void 0;
-const bundleworker_mainthread_1 = require("./bundleworker-mainthread");
 const messages_1 = require("./messages");
 exports.messageManager = new messages_1.MsgManager();
 let Worker;
@@ -29,10 +28,6 @@ function launchWorker() {
             t: "wasm",
             url: require("./zcv.wasm").default
         });
-        worker.postMessage({
-            t: "port",
-            port: bundleworker_mainthread_1.bundleWorkerMessageChannel.port1
-        }, [bundleworker_mainthread_1.bundleWorkerMessageChannel.port1]);
         yield waitForLoad(worker);
         function sendOutgoing() {
             let msgs = exports.messageManager.getOutgoingMessages();