@zappar/zappar-cv 0.4.0-beta.4 → 0.4.0-beta.7

package/lib/profile.js

@@ -17,56 +17,67 @@ exports.profile = {
     videoHeight: 480,
     dataWidth: 320,
     dataHeight: 240,
-    videoElementInDOM: false
+    videoElementInDOM: false,
+    preferMediaStreamTrackProcessorCamera: false,
+    preferImageBitmapCamera: false
 };
-window["zeeProfile"] = exports.profile;
+if (typeof window !== "undefined") {
+    window["zeeProfile"] = exports.profile;
+    if (window.location.href.indexOf("_mstppipeline") >= 0) {
+        console.log("Configuring for MSTP camera pipeline (if supported)");
+        exports.profile.preferMediaStreamTrackProcessorCamera = true;
+    }
+    if (window.location.href.indexOf("_imagebitmappipeline") >= 0) {
+        console.log("Configuring for ImageBitmap camera pipeline (if supported)");
+        exports.profile.preferImageBitmapCamera = true;
+    }
+}
 let agent = new UAParser.UAParser();
-switch (agent.getBrowser().name) {
-    case "Safari":
-        exports.profile.deviceMotionMutliplier = 1.0;
-        if (window.orientation === undefined) {
-            // Desktop Safari
-            break;
-        }
-        exports.profile.videoElementInDOM = true;
-        exports.profile.showSafariPermissionsResetIfNecessary = true;
-        break;
-    case "Chrome":
-        let engine = agent.getEngine();
-        if (!engine || engine.name !== "WebKit") {
-            break;
-        }
-    case "Mobile Safari":
-        exports.profile.deviceMotionMutliplier = 1.0;
-        let version = agent.getOS().version;
-        if (version) {
-            let versionParts = version.split(".");
-            if (versionParts.length >= 2) {
-                const majorVersion = parseInt(versionParts[0]);
-                const minorVersion = parseInt(versionParts[1]);
-                if (majorVersion < 11 ||
-                    (majorVersion === 11 && minorVersion < 3)) {
-                    exports.profile.blacklisted = true;
-                }
-                if (majorVersion < 12 ||
-                    (majorVersion === 12 && minorVersion < 2)) {
-                    exports.profile.videoElementInDOM = true;
-                }
-                if ((majorVersion === 12 && minorVersion >= 2) || (majorVersion >= 13))
-                    exports.profile.showGyroPermissionsWarningIfNecessary = true;
-                if (majorVersion >= 13) {
-                    exports.profile.showSafariPermissionsResetIfNecessary = true;
-                }
-                if (((majorVersion >= 12 && minorVersion > 1) || (majorVersion >= 13)) &&
-                    navigator.mediaDevices &&
-                    navigator.mediaDevices.getSupportedConstraints &&
-                    navigator.mediaDevices.getSupportedConstraints().frameRate) {
-                    exports.profile.requestHighFrameRate = true;
+let os = (agent.getOS().name || "unknown").toLowerCase();
+let engine = (agent.getEngine().name || "unknown").toLowerCase();
+if (engine === "webkit" && os !== "ios") {
+    exports.profile.deviceMotionMutliplier = 1.0;
+    if (typeof window !== "undefined" && window.orientation !== undefined) {
+        // iPad
+        iDevice();
+    }
+}
+if (engine === "webkit" && os === "ios") {
+    exports.profile.deviceMotionMutliplier = 1.0;
+    iDevice();
+}
+function iDevice() {
+    let version = agent.getOS().version;
+    if (version) {
+        let versionParts = version.split(".");
+        if (versionParts.length >= 2) {
+            const majorVersion = parseInt(versionParts[0]);
+            const minorVersion = parseInt(versionParts[1]);
+            if (majorVersion < 11 ||
+                (majorVersion === 11 && minorVersion < 3)) {
+                exports.profile.blacklisted = true;
+            }
+            if (majorVersion < 12 ||
+                (majorVersion === 12 && minorVersion < 2)) {
+                exports.profile.videoElementInDOM = true;
+            }
+            if ((majorVersion === 12 && minorVersion >= 2) || (majorVersion >= 13))
+                exports.profile.showGyroPermissionsWarningIfNecessary = true;
+            if (majorVersion >= 13) {
+                exports.profile.showSafariPermissionsResetIfNecessary = true;
+            }
+            if (((majorVersion >= 12 && minorVersion > 1) || (majorVersion >= 13)) &&
+                navigator.mediaDevices &&
+                navigator.mediaDevices.getSupportedConstraints &&
+                navigator.mediaDevices.getSupportedConstraints().frameRate) {
+                exports.profile.requestHighFrameRate = true;
+                if (majorVersion < 14) {
                     // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
+                    // Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
                     exports.profile.videoHeight = 360;
                     exports.profile.dataHeight = 180;
                 }
             }
         }
-        break;
+    }
 }

package/lib/sequencerecorder.d.ts

@@ -26,6 +26,7 @@ export declare class SequenceRecorder {
     private _insertionByte;
     private _numberPackets;
     private _started;
+    private _hasLoggedUnsuppored;
     constructor(estimatedFrames: number);
     private _estimatedSize;
     start(): void;

package/lib/sequencerecorder.js

@@ -15,6 +15,7 @@ class SequenceRecorder {
         this._insertionByte = 0;
         this._numberPackets = 0;
         this._started = false;
+        this._hasLoggedUnsuppored = false;
         this._data = new ArrayBuffer(this._estimatedSize(estimatedFrames, 320, 240));
         this._dataView = new DataView(this._data);
         this._dataUint8 = new Uint8Array(this._data);
@@ -38,6 +39,9 @@ class SequenceRecorder {
         this._started = false;
         let trimmed = this._data.slice(0, this._insertionByte);
         this._data = trimmed;
+        this._dataUint8 = new Uint8Array(this._data);
+        this._dataFloat32 = new Float32Array(this._data);
+        this._dataView = new DataView(this._data);
     }
     data() {
         return this._dataUint8;
@@ -45,6 +49,13 @@ class SequenceRecorder {
     appendCameraFrame(c) {
         if (!this._started)
             return;
+        if (!c.data) {
+            if (this._hasLoggedUnsuppored)
+                return;
+            this._hasLoggedUnsuppored = true;
+            loglevel_1.zcerr("Unable to record frames from this camera source");
+            return;
+        }
         if (c.data.byteLength !== (c.dataWidth * c.dataHeight)) {
             loglevel_1.zcerr("Unable to record frame to sequence: greyscale array doesn't match dimensions");
             return;

package/lib/sequencesource.d.ts

@@ -4,14 +4,20 @@ export declare class SequenceSource extends Source {
     private _impl;
     private _pipeline;
     private _packetNumber;
+    private _motionUpdatesQueued;
+    private _motionUpdatesSent;
     private _decoder;
+    private _minFrameGapMs;
+    private _lastUpdateTime;
     static create(p: zappar_pipeline_t): zappar_sequence_source_t;
     static get(m: zappar_sequence_source_t): SequenceSource | undefined;
     constructor(_impl: zappar_sequence_source_t, _pipeline: zappar_pipeline_t);
     loadFromMemory(data: ArrayBuffer): void;
-    getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
+    maxPlaybackFpsSet(fps: number): void;
+    getFrame(currentlyProcessing: boolean): void;
     destroy(): void;
     start(): void;
     pause(): void;
     uploadGL(info: CameraFrameInfo): void;
+    private _submitMotionUpdates;
 }

package/lib/sequencesource.js

@@ -14,6 +14,10 @@ class SequenceSource extends source_1.Source {
         this._impl = _impl;
         this._pipeline = _pipeline;
         this._packetNumber = 0;
+        this._motionUpdatesQueued = 0;
+        this._motionUpdatesSent = 0;
+        this._minFrameGapMs = 0;
+        this._lastUpdateTime = 0;
     }
     static create(p) {
         let ret = (latest++);
@@ -28,66 +32,68 @@ class SequenceSource extends source_1.Source {
         try {
             this._decoder = new sequencerecorder_1.SequenceDecoder(data);
             this._packetNumber = 0;
+            this._lastUpdateTime = 0;
         }
         catch (err) {
             loglevel_1.zcerr("Unable to decode sequence:", err);
         }
     }
+    maxPlaybackFpsSet(fps) {
+        if (fps <= 0) {
+            this._minFrameGapMs = 0;
+            return;
+        }
+        this._minFrameGapMs = 1000.0 / fps;
+    }
     getFrame(currentlyProcessing) {
         let pipeline = pipeline_1.Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
+        if (!this._decoder)
+            return;
+        this._submitMotionUpdates(pipeline);
+        if (this._motionUpdatesQueued > this._motionUpdatesSent)
+            return;
         if (currentlyProcessing)
             return;
-        if (!this._decoder)
+        let curTime = performance.now();
+        if (curTime < this._lastUpdateTime + this._minFrameGapMs)
             return;
-        while (this._packetNumber < this._decoder.numberPackets) {
-            let packet = this._decoder.dataByPacket.get(this._packetNumber);
-            this._packetNumber++;
-            if (!packet)
-                return;
-            switch (packet.type) {
-                case sequencerecorder_1.SequenceRecorderPacketType.ACCELEROMETER:
-                    pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ROTATION_RATE:
-                    pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE:
-                    pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE_MATRIX:
-                    pipeline.motionAttitudeMatrix(packet.attitude);
-                    break;
-                case sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME:
-                    {
-                        let pixels = pipeline.cameraPixelArrays.pop();
-                        while (pixels) {
-                            if (pixels.byteLength === packet.greyscale.byteLength)
-                                break;
-                            pixels = pipeline.cameraPixelArrays.pop();
-                        }
-                        if (!pixels) {
-                            pixels = new ArrayBuffer(packet.greyscale.byteLength);
-                        }
-                        (new Uint8Array(pixels)).set(packet.greyscale);
-                        return {
-                            data: pixels,
-                            cameraSourceData: packet.greyscale,
-                            cameraModel: packet.cameraModel,
-                            cameraToDevice: packet.cameraToDevice,
-                            dataWidth: packet.width,
-                            dataHeight: packet.height,
-                            texture: undefined,
-                            userFacing: packet.userFacing,
-                            uvTransform: gl_matrix_1.mat4.create(),
-                            cameraSource: this
-                        };
-                        break;
-                    }
-            }
+        if (this._packetNumber >= this._decoder.numberPackets)
+            return;
+        let packet = this._decoder.dataByPacket.get(this._packetNumber);
+        if (packet.type != sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME)
+            return;
+        this._packetNumber++;
+        let pixels = pipeline.cameraPixelArrays.pop();
+        while (pixels) {
+            if (pixels.byteLength === packet.greyscale.byteLength)
+                break;
+            pixels = pipeline.cameraPixelArrays.pop();
+        }
+        if (!pixels) {
+            pixels = new ArrayBuffer(packet.greyscale.byteLength);
         }
-        return undefined;
+        (new Uint8Array(pixels)).set(packet.greyscale);
+        this._lastUpdateTime = curTime;
+        let info = {
+            data: pixels,
+            cameraSourceData: packet.greyscale,
+            cameraModel: packet.cameraModel,
+            cameraToDevice: packet.cameraToDevice,
+            dataWidth: packet.width,
+            dataHeight: packet.height,
+            texture: undefined,
+            userFacing: packet.userFacing,
+            uvTransform: gl_matrix_1.mat4.create(),
+            cameraSource: this
+        };
+        // We can queue the motion updates for the follwing frame immediately
+        // as this camera frame will be sent immediately, but the next motion
+        // updates will be one frame later
+        this._submitMotionUpdates(pipeline);
+        const token = pipeline.registerToken(info);
+        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
     }
     destroy() {
         byId.delete(this._impl);
@@ -124,5 +130,41 @@ class SequenceSource extends source_1.Source {
         gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, info.dataWidth, info.dataHeight, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, new Uint8Array(info.cameraSourceData));
         gl.bindTexture(gl.TEXTURE_2D, null);
     }
+    _submitMotionUpdates(pipeline) {
+        let pendingMotionUpdate = false;
+        while (this._packetNumber < this._decoder.numberPackets) {
+            let packet = this._decoder.dataByPacket.get(this._packetNumber);
+            if (packet.type == sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME)
+                break;
+            switch (packet.type) {
+                case sequencerecorder_1.SequenceRecorderPacketType.ACCELEROMETER:
+                    pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ROTATION_RATE:
+                    pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE:
+                    pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE_MATRIX:
+                    pipeline.motionAttitudeMatrix(packet.attitude);
+                    pendingMotionUpdate = true;
+                    break;
+            }
+            this._packetNumber++;
+        }
+        // If we've submitted motion updates, add a setTimeout to reflect when
+        // the message serializer will actually have sent them over to the worker
+        if (pendingMotionUpdate) {
+            this._motionUpdatesQueued++;
+            setTimeout(() => {
+                this._motionUpdatesSent++;
+            }, 0);
+        }
+        return;
+    }
 }
 exports.SequenceSource = SequenceSource;

package/lib/source.d.ts

@@ -1,17 +1,19 @@
+/// <reference types="dom-webcodecs" />
 export interface CameraFrameInfo {
-    data: ArrayBuffer;
+    data?: ArrayBuffer;
     texture: WebGLTexture | undefined;
     dataWidth: number;
     dataHeight: number;
     cameraModel: Float32Array;
     cameraToDevice: Float32Array;
-    uvTransform: Float32Array;
-    userFacing: boolean;
     cameraSource: Source;
     cameraSourceData?: any;
+    uvTransform?: Float32Array;
+    userFacing: boolean;
+    frame?: VideoFrame | ImageBitmap;
 }
 export declare abstract class Source {
-    abstract getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
+    abstract getFrame(currentlyProcessing: boolean): void;
     abstract pause(): void;
     abstract uploadGL(info: CameraFrameInfo): void;
 }

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.4.0-beta.4";
+export declare const VERSION = "0.4.0-beta.7";

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = "0.4.0-beta.4";
+exports.VERSION = "0.4.0-beta.7";

package/lib/worker-client.js

@@ -10,7 +10,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.launchWorker = exports.messageManager = void 0;
-const bundleworker_mainthread_1 = require("./bundleworker-mainthread");
 const messages_1 = require("./messages");
 exports.messageManager = new messages_1.MsgManager();
 let Worker;
@@ -29,10 +28,6 @@ function launchWorker() {
             t: "wasm",
             url: require("./zcv.wasm").default
         });
-        worker.postMessage({
-            t: "port",
-            port: bundleworker_mainthread_1.bundleWorkerMessageChannel.port1
-        }, [bundleworker_mainthread_1.bundleWorkerMessageChannel.port1]);
         yield waitForLoad(worker);
         function sendOutgoing() {
             let msgs = exports.messageManager.getOutgoingMessages();

package/lib/worker-imagebitmap.d.ts

@@ -0,0 +1,5 @@
+import { zappar_cwrap } from "./gen/zappar-native";
+import { zappar_server } from "./gen/zappar-server";
+import { MsgManager } from "./messages";
+import { ImageBitmapC2S } from "./workerinterface";
+export declare function handleImageBitmap(m: ImageBitmapC2S, r: zappar_cwrap, server: zappar_server, mgr: MsgManager): void;

package/lib/worker-imagebitmap.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleImageBitmap = void 0;
+const image_process_gl_1 = require("./image-process-gl");
+const profile_1 = require("./profile");
+const gl_matrix_1 = require("gl-matrix");
+let pixels;
+let texture;
+let _imageProcessor;
+let _gl;
+function getImageProcessor() {
+    if (!_imageProcessor || !_gl) {
+        const canvas = new OffscreenCanvas(1, 1);
+        _gl = canvas.getContext("webgl");
+        if (!_gl)
+            throw new Error("Unable to get offscreen GL context");
+        _imageProcessor = new image_process_gl_1.ImageProcessGL(_gl);
+    }
+    return [_imageProcessor, _gl];
+}
+function handleImageBitmap(m, r, server, mgr) {
+    // img: ImageBitmap,
+    //                                 rot: number,
+    //                                 tokenId: number,
+    //                                 r: zappar_cwrap,
+    //                                 p: zappar_pipeline_t,
+    //                                 userFacing: boolean,
+    //                                 server: zappar_server,
+    //                                 mgr: MsgManager) {
+    const [imageProcessor, gl] = getImageProcessor();
+    if (!texture) {
+        texture = gl.createTexture();
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    }
+    if (!texture)
+        return;
+    if (!pixels || pixels.byteLength < profile_1.profile.dataWidth * profile_1.profile.dataHeight) {
+        console.log("Generating pixel buffer", profile_1.profile.dataWidth * profile_1.profile.dataHeight);
+        pixels = new ArrayBuffer(profile_1.profile.dataWidth * profile_1.profile.dataHeight);
+    }
+    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing);
+    let info = imageProcessor.readFrame(texture, pixels);
+    let msg = {
+        t: "imageBitmapS2C",
+        dataWidth: info.dataWidth,
+        dataHeight: info.dataHeight,
+        frame: m.i,
+        userFacing: info.userFacing,
+        uvTransform: info.uvTransform || gl_matrix_1.mat4.create(),
+        tokenId: m.tokenId, p: m.p
+    };
+    mgr.postOutgoingMessage(msg, [m.i]);
+    let pipeline = server._pipeline_by_instance.get(m.p);
+    if (pipeline) {
+        r.pipeline_camera_frame_submit(pipeline, pixels, info.dataWidth, info.dataHeight, m.tokenId, m.cameraToDevice, m.cameraModel, info.userFacing);
+        r.pipeline_frame_update(pipeline);
+        server.exploreState();
+    }
+}
+exports.handleImageBitmap = handleImageBitmap;

package/lib/worker-server.d.ts

@@ -1,8 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-declare global {
-    interface Window {
-        bundle_adjust_message: any;
-    }
-}
 export declare function launchWorkerServer(wasmUrl: string): Promise<void>;

package/lib/worker-server.js

@@ -14,21 +14,12 @@ const ZNM = require("./zcv");
 const zappar_cwrap_1 = require("./gen/zappar-cwrap");
 const zappar_server_1 = require("./gen/zappar-server");
 const messages_1 = require("./messages");
+const gl_matrix_1 = require("gl-matrix");
+const zappar_native_1 = require("./gen/zappar-native");
+const worker_imagebitmap_1 = require("./worker-imagebitmap");
+const profile_1 = require("./profile");
 exports.messageManager = new messages_1.MsgManager();
-let bundleWorkerPort;
-self.addEventListener("message", msg => {
-    if (msg && msg.data && msg.data.t === "port") {
-        bundleWorkerPort = msg.data.port;
-    }
-});
-self.bundle_adjust_message = function (buf) {
-    var buf_copy = new ArrayBuffer(buf.byteLength);
-    new Uint8Array(buf_copy).set(new Uint8Array(buf));
-    bundleWorkerPort === null || bundleWorkerPort === void 0 ? void 0 : bundleWorkerPort.postMessage({
-        t: "bundle_adjust_message",
-        buf: buf_copy
-    }, [buf_copy]);
-};
+let latestCameraToScreenRotation = 0;
 function launchWorkerServer(wasmUrl) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
@@ -48,15 +39,6 @@ function launchWorkerServer(wasmUrl) {
                     }, [ab]);
                 });
                 exports.messageManager.postOutgoingMessage("loaded", []);
-                bundleWorkerPort.onmessage = function (msg) {
-                    // Received message from other worker
-                    var a_buf = msg.data.buf;
-                    var buf = mod._malloc(a_buf.byteLength);
-                    mod.HEAPU8.set(new Uint8Array(a_buf), buf);
-                    /// call the exported native function to update the world map
-                    mod._bundle_adjust_message_callback(buf, msg.data.buf.byteLength, 0);
-                    mod._free(buf);
-                };
                 exports.messageManager.onIncomingMessage.bind((msg) => {
                     var _a;
                     switch (msg.t) {
@@ -67,7 +49,7 @@ function launchWorkerServer(wasmUrl) {
                         case "buf":
                             (_a = server.serializersByPipelineId.get(msg.p)) === null || _a === void 0 ? void 0 : _a.bufferReturn(msg.d);
                             break;
-                        case "cameraFrameC2S":
+                        case "cameraFrameC2S": {
                             let msgt = msg;
                             let pipeline = server._pipeline_by_instance.get(msgt.p);
                             let att;
@@ -86,6 +68,22 @@ function launchWorkerServer(wasmUrl) {
                             };
                             exports.messageManager.postOutgoingMessage(ret, [msgt.d]);
                             break;
+                        }
+                        case "streamC2S": {
+                            let msgt = msg;
+                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source);
+                            break;
+                        }
+                        case "cameraToScreenC2S": {
+                            let msgt = msg;
+                            latestCameraToScreenRotation = msgt.r;
+                            break;
+                        }
+                        case "imageBitmapC2S": {
+                            let msgt = msg;
+                            worker_imagebitmap_1.handleImageBitmap(msgt, r, server, exports.messageManager);
+                            break;
+                        }
                     }
                 });
             }
@@ -94,3 +92,101 @@ function launchWorkerServer(wasmUrl) {
 }
 exports.launchWorkerServer = launchWorkerServer;
 ;
+let streamDataBufferPointer = 0;
+let streamDataBufferLength = 0;
+let tokenId = 1;
+const cameraToDeviceTransform = gl_matrix_1.mat4.create();
+const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+function consumeStream(mod, r, stream, p, userFacing, server, source) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const reader = yield stream.getReader();
+        while (true) {
+            let result = yield reader.read();
+            if (result.done) {
+                console.log("Stream done");
+                return;
+            }
+            let frame = result.value;
+            let size = frame.allocationSize();
+            if (size > streamDataBufferLength) {
+                if (streamDataBufferPointer > 0)
+                    mod._free(streamDataBufferPointer);
+                streamDataBufferPointer = mod._malloc(size);
+                streamDataBufferLength = size;
+            }
+            yield frame.copyTo(mod.HEAPU8.subarray(streamDataBufferPointer, streamDataBufferPointer + streamDataBufferLength));
+            let token = tokenId;
+            tokenId++;
+            const width = frame.visibleRect.width;
+            const height = frame.visibleRect.height;
+            let uvTransform;
+            let dataWidth = width;
+            let dataHeight = height;
+            switch (latestCameraToScreenRotation) {
+                case 270:
+                    uvTransform = new Float32Array([0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1]);
+                    dataWidth = height;
+                    dataHeight = width;
+                    break;
+                case 180:
+                    uvTransform = new Float32Array([-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1]);
+                    break;
+                case 90:
+                    uvTransform = new Float32Array([0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1]);
+                    dataWidth = height;
+                    dataHeight = width;
+                    break;
+                default:
+                    uvTransform = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
+                    break;
+            }
+            let clone = frame.clone();
+            if (userFacing)
+                gl_matrix_1.mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
+            else
+                gl_matrix_1.mat4.identity(cameraToDeviceTransform);
+            let focalLength = 300.0 * profile_1.profile.dataWidth / 320.0;
+            cameraModel[0] = focalLength;
+            cameraModel[1] = focalLength;
+            cameraModel[2] = profile_1.profile.dataWidth * 0.5;
+            cameraModel[3] = profile_1.profile.dataHeight * 0.5;
+            const ret = {
+                token: token,
+                d: clone,
+                p: p,
+                t: "videoFrameS2C",
+                userFacing,
+                uvTransform,
+                w: dataWidth,
+                h: dataHeight,
+                cameraToDevice: cameraToDeviceTransform,
+                cameraModel,
+                source
+            };
+            exports.messageManager.postOutgoingMessage(ret, [ret.d, ret.uvTransform.buffer]);
+            const pipeline = server._pipeline_by_instance.get(p);
+            if (pipeline) {
+                r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                r.pipeline_frame_update(pipeline);
+                server.exploreState();
+            }
+            frame.close();
+        }
+    });
+}
+function framePixelFormatFromFormat(f) {
+    switch (f) {
+        case "I420": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420;
+        case "I420A": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420A;
+        case "I422": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I422;
+        case "I444": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I444;
+        case "NV12": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_NV12;
+        case "RGBA":
+        case "RGBX":
+            return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_RGBA;
+        case "BGRA":
+        case "BGRX":
+            return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_BGRA;
+    }
+    return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_Y;
+}