@zappar/zappar-cv 3.0.0-beta.7 → 3.0.1-alpha.2

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/3.0.0-beta.7/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/3.0.1-alpha.2/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/3.0.0-beta.7/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.0-beta.7/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/3.0.1-alpha.2/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.1-alpha.2/zappar-cv.zip)

package/lib/camera-source.js

@@ -35,11 +35,15 @@ export class CameraSource extends HTMLElementSource {
             if (!pipeline)
                 return;
             let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
-            let interval = ev.interval * profile.intervalMultiplier;
-            if (this._lastTimestamp > -1 && !profile.trustSensorIntervals) {
-                interval = Math.max(timeStamp - this._lastTimestamp, interval);
+            const interval = ev.interval * profile.intervalMultiplier;
+            if (profile.trustSensorIntervals) {
+                if (this._lastTimestamp === -1)
+                    this._lastTimestamp = timeStamp;
+                else {
+                    this._lastTimestamp += interval;
+                    timeStamp = this._lastTimestamp;
+                }
             }
-            this._lastTimestamp = timeStamp;
             if (ev.acceleration !== null &&
                 ev.acceleration.x !== null &&
                 ev.acceleration.y !== null &&

package/lib/ceres-worker.d.ts

@@ -0,0 +1 @@
+export declare function launchCeresWorker(wasmUrl: string, module: WebAssembly.Module, port: MessagePort): Promise<void>;

package/lib/ceres-worker.js

@@ -0,0 +1,67 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { getWorkerMessageAPI } from "./worker-messages";
+import * as ZNM from "./zappar-cv";
+const ctx = self;
+let launchHandler = (evt) => {
+    if (evt && evt.data && evt.data.t === "wasm") {
+        let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zappar-cv.wasm", import.meta.url).toString();
+        launchCeresWorker(url, evt.data.module, evt.data.port);
+        ctx.removeEventListener("message", launchHandler);
+    }
+};
+ctx.addEventListener("message", launchHandler);
+export function launchCeresWorker(wasmUrl, module, port) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let mod = ZNM.default({
+            locateFile: (path, prefix) => {
+                if (path.endsWith("zappar-cv.wasm")) {
+                    return wasmUrl;
+                }
+                return prefix + path;
+            },
+            instantiateWasm: (imports, successCallback) => {
+                const instance = new WebAssembly.Instance(module, imports);
+                successCallback(instance);
+                return instance.exports;
+            },
+            onRuntimeInitialized: () => {
+                const ceres_worker = mod.cwrap('ceres_worker', null, []);
+                const messageAPI = getWorkerMessageAPI(mod);
+                port.addEventListener('message', evt => {
+                    if (evt.data.t !== 'msgsend')
+                        return;
+                    const data = evt.data.data;
+                    const reference = evt.data.reference;
+                    const ptr = mod._malloc(data.byteLength);
+                    mod.HEAPU8.set(data, ptr);
+                    messageAPI.worker_message_receive(reference, data.byteLength, ptr);
+                    mod._free(ptr);
+                    ceres_worker();
+                    const toSend = messageAPI.worker_message_send_count();
+                    for (let i = 0; i < toSend; i++) {
+                        const reference = messageAPI.worker_message_send_reference(i);
+                        const dataSize = messageAPI.worker_message_send_data_size(i);
+                        const dataPtr = messageAPI.worker_message_send_data(i);
+                        const data = mod.HEAPU8.slice(dataPtr, dataPtr + dataSize);
+                        port.postMessage({
+                            t: 'msgrec',
+                            data,
+                            reference
+                        }, [data.buffer]);
+                    }
+                    messageAPI.worker_message_send_clear();
+                });
+                port.start();
+            }
+        });
+    });
+}
+;

package/lib/direct.js

@@ -13,6 +13,8 @@ import { messageManager as workerMessageManager } from "./worker-server";
 export let messageManager = new MsgManager();
 export function launchWorker(wasm) {
     return __awaiter(this, void 0, void 0, function* () {
+        const response = yield fetch(new URL("./zappar-cv.wasm", import.meta.url).toString());
+        const module = yield WebAssembly.compileStreaming(response);
         messageManager.onOutgoingMessage.bind(() => {
             let msgs = messageManager.getOutgoingMessages();
             for (let msg of msgs)
@@ -23,6 +25,6 @@ export function launchWorker(wasm) {
             for (let msg of msgs)
                 messageManager.postIncomingMessage(msg.msg);
         });
-        launchWorkerServer(wasm);
+        launchWorkerServer(wasm, module);
     });
 }

package/lib/gen/zappar-cwrap.js

@@ -38,7 +38,7 @@ export function getRuntimeObject(mod) {
     ]);
     let pipeline_camera_frame_submit_raw_pointer_wrapped = mod.cwrap("zappar_pipeline_camera_frame_submit_raw_pointer", null, [
         "number",
-        "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"
+        "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"
     ]);
     let pipeline_camera_frame_camera_attitude_wrapped = mod.cwrap("zappar_pipeline_camera_frame_camera_attitude", "number", [
         "number"
@@ -324,7 +324,7 @@ export function getRuntimeObject(mod) {
             let ret = pipeline_camera_frame_submit_wrapped(o, arg_data, arg_len_data, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_camera_model, arg_user_facing);
             return ret;
         },
-        pipeline_camera_frame_submit_raw_pointer: (o, data, dataLength, format, width, height, user_data, camera_to_device_transform, rotation, camera_model, user_facing) => {
+        pipeline_camera_frame_submit_raw_pointer: (o, data, dataLength, format, width, height, user_data, camera_to_device_transform, rotation, camera_model, user_facing, timestamp) => {
             let arg_data = data;
             let arg_dataLength = dataLength;
             let arg_format = format;
@@ -337,7 +337,8 @@ export function getRuntimeObject(mod) {
             let arg_camera_model = getFloatDataArrayForArgIndex(8, camera_model.byteLength);
             mod.HEAPF32.set(camera_model, arg_camera_model / 4);
             let arg_user_facing = user_facing ? 1 : 0;
-            let ret = pipeline_camera_frame_submit_raw_pointer_wrapped(o, arg_data, arg_dataLength, arg_format, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_rotation, arg_camera_model, arg_user_facing);
+            let arg_timestamp = timestamp;
+            let ret = pipeline_camera_frame_submit_raw_pointer_wrapped(o, arg_data, arg_dataLength, arg_format, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_rotation, arg_camera_model, arg_user_facing, arg_timestamp);
             return ret;
         },
         pipeline_camera_frame_camera_attitude: (o) => {

package/lib/gen/zappar-native.d.ts

@@ -111,7 +111,7 @@ export interface zappar_cwrap {
     pipeline_camera_model(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_user_data(o: zappar_pipeline_t): number;
     pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean): void;
-    pipeline_camera_frame_submit_raw_pointer(o: zappar_pipeline_t, data: number, dataLength: number, format: frame_pixel_format_t, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, rotation: number, camera_model: Float32Array, user_facing: boolean): void;
+    pipeline_camera_frame_submit_raw_pointer(o: zappar_pipeline_t, data: number, dataLength: number, format: frame_pixel_format_t, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, rotation: number, camera_model: Float32Array, user_facing: boolean, timestamp: number): void;
     pipeline_camera_frame_camera_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_device_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_motion_accelerometer_submit(o: zappar_pipeline_t, time: number, x: number, y: number, z: number): void;

package/lib/mstp-camera-source.d.ts

@@ -11,6 +11,9 @@ export declare class MSTPCameraSource extends Source {
     private _isPaused;
     private _isUserFacing;
     private _cameraToScreenRotation;
+    private _accelerometer?;
+    private _linearAcceleration?;
+    private _gyroscope?;
     constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
     destroy(): void;
     private _stop;
@@ -26,6 +29,12 @@ export declare class MSTPCameraSource extends Source {
     private _deviceMotionListener;
     private _startDeviceOrientation;
     private _startDeviceMotion;
+    private _lastAccelerometerTimestamp;
+    private _accelerometerReading;
+    private _lastLinearAccelerationTimestamp;
+    private _linearAccelerationReading;
+    private _lastGyroscopeReading;
+    private _gyroscopeReading;
     private _stopDeviceMotion;
     uploadGL(info: CameraFrameInfo): void;
 }

package/lib/mstp-camera-source.js

@@ -31,12 +31,15 @@ export class MSTPCameraSource extends Source {
             if (!pipeline)
                 return;
             let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
-            let interval = ev.interval;
-            if (this._lastTimestamp > -1 && !profile.trustSensorIntervals) {
-                interval = Math.max(timeStamp - this._lastTimestamp, interval);
+            const interval = ev.interval * profile.intervalMultiplier;
+            if (profile.trustSensorIntervals) {
+                if (this._lastTimestamp === -1)
+                    this._lastTimestamp = timeStamp;
+                else {
+                    this._lastTimestamp += interval;
+                    timeStamp = this._lastTimestamp;
+                }
             }
-            this._lastTimestamp = timeStamp;
-            interval *= profile.intervalMultiplier;
             if (ev.acceleration !== null &&
                 ev.acceleration.x !== null &&
                 ev.acceleration.y !== null &&
@@ -61,6 +64,47 @@ export class MSTPCameraSource extends Source {
                 this._startDeviceOrientation();
             }
         };
+        this._lastAccelerometerTimestamp = -1;
+        this._accelerometerReading = () => {
+            var _a;
+            if (!this._accelerometer || this._accelerometer.x === undefined || this._accelerometer.y === undefined || this._accelerometer.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._accelerometer.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastAccelerometerTimestamp > -1 ? timestamp - this._lastAccelerometerTimestamp : 30;
+            this._lastAccelerometerTimestamp = timestamp;
+            pipeline.motionAccelerometerSubmit(timestamp, this._accelerometer.x * profile.deviceMotionMutliplier, this._accelerometer.y * profile.deviceMotionMutliplier, this._accelerometer.z * profile.deviceMotionMutliplier);
+            pipeline.motionAccelerometerWithGravitySubmitInt(timestamp, interval, this._accelerometer.x * profile.deviceMotionMutliplier, this._accelerometer.y * profile.deviceMotionMutliplier, this._accelerometer.z * profile.deviceMotionMutliplier);
+        };
+        this._lastLinearAccelerationTimestamp = -1;
+        this._linearAccelerationReading = () => {
+            var _a;
+            if (!this._linearAcceleration || this._linearAcceleration.x === undefined || this._linearAcceleration.y === undefined || this._linearAcceleration.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._linearAcceleration.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastLinearAccelerationTimestamp > -1 ? timestamp - this._lastLinearAccelerationTimestamp : 30;
+            this._lastLinearAccelerationTimestamp = timestamp;
+            pipeline.motionAccelerometerWithoutGravitySubmitInt(timestamp, interval, this._linearAcceleration.x * profile.deviceMotionMutliplier, this._linearAcceleration.y * profile.deviceMotionMutliplier, this._linearAcceleration.z * profile.deviceMotionMutliplier);
+        };
+        this._lastGyroscopeReading = -1;
+        this._gyroscopeReading = () => {
+            var _a;
+            if (!this._gyroscope || this._gyroscope.x === undefined || this._gyroscope.y === undefined || this._gyroscope.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._gyroscope.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastGyroscopeReading > -1 ? timestamp - this._lastGyroscopeReading : 30;
+            this._lastGyroscopeReading = timestamp;
+            pipeline.motionRotationRateSubmit(timestamp, -this._gyroscope.x, -this._gyroscope.y, -this._gyroscope.z);
+            pipeline.motionRotationRateSubmitInt(timestamp, interval, this._gyroscope.x * 180 / Math.PI, this._gyroscope.y * 180 / Math.PI, this._gyroscope.z * 180 / Math.PI);
+        };
         zcout("Using MSTP camera source");
     }
     destroy() {
@@ -221,9 +265,38 @@ export class MSTPCameraSource extends Source {
         });
     }
     _startDeviceMotion() {
-        window.addEventListener("devicemotion", this._deviceMotionListener, false);
+        try {
+            this._accelerometer = new Accelerometer({ frequency: 60 });
+            this._linearAcceleration = new LinearAccelerationSensor({ frequency: 60 });
+            this._gyroscope = new Gyroscope({ frequency: 60 });
+            // throw new Error();
+            this._accelerometer.addEventListener('reading', this._accelerometerReading);
+            this._accelerometer.start();
+            this._linearAcceleration.addEventListener('reading', this._linearAccelerationReading);
+            this._linearAcceleration.start();
+            this._gyroscope.addEventListener('reading', this._gyroscopeReading);
+            this._gyroscope.start();
+        }
+        catch (err) {
+            window.addEventListener("devicemotion", this._deviceMotionListener, false);
+        }
     }
     _stopDeviceMotion() {
+        if (this._accelerometer) {
+            this._accelerometer.stop();
+            this._accelerometer.removeEventListener('reading', this._accelerometerReading);
+            delete this._accelerometer;
+        }
+        if (this._linearAcceleration) {
+            this._linearAcceleration.stop();
+            this._linearAcceleration.removeEventListener('reading', this._linearAccelerationReading);
+            delete this._linearAcceleration;
+        }
+        if (this._gyroscope) {
+            this._gyroscope.stop();
+            this._gyroscope.removeEventListener('reading', this._gyroscopeReading);
+            delete this._gyroscope;
+        }
         window.removeEventListener("devicemotion", this._deviceMotionListener);
     }
     uploadGL(info) {

package/lib/native.js

@@ -12,7 +12,7 @@ import { zappar_client } from "./gen/zappar-client";
 import { drawPlane } from "./drawplane";
 import { cameraRotationForScreenOrientation, projectionMatrix } from "./cameramodel";
 import { mat4, vec3 } from "gl-matrix";
-import { launchWorker, messageManager } from "./worker-client";
+import { launchCeresWorker, launchWorker, messageManager } from "./worker-client";
 import { permissionRequestUI, permissionGrantedAll, permissionGrantedCamera, permissionGrantedMotion, permissionDeniedAny, permissionDeniedCamera, permissionDeniedMotion, permissionRequestMotion, permissionRequestCamera, permissionRequestAll, permissionDeniedUI } from "./permission";
 import { createFaceMesh, destroyFaceMesh, getFaceMesh } from "./facemesh";
 import { Pipeline, applyScreenCounterRotation } from "./pipeline";
@@ -84,6 +84,10 @@ export function initialize(opts) {
                     src.streamEnded();
                 break;
             }
+            case "setupCeresWorker": {
+                launchCeresWorker(msg.port);
+                break;
+            }
             case "licerr": {
                 let div = document.createElement("div");
                 div.innerHTML = "Visit <a href='https://docs.zap.works/universal-ar/licensing/' style='color: white;'>our licensing page</a> to find out about hosting on your own domain.";

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.0-beta.7";
+export declare const VERSION = "3.0.1-alpha.2";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.0-beta.7";
+export const VERSION = "3.0.1-alpha.2";

package/lib/worker-client.d.ts

@@ -1,3 +1,4 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
 export declare function launchWorker(worker?: Worker): Promise<void>;
+export declare function launchCeresWorker(port: MessagePort, worker?: Worker): Promise<void>;

package/lib/worker-client.js

@@ -9,13 +9,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 import { MsgManager } from "./messages";
 export let messageManager = new MsgManager();
+const wasmModule = (() => __awaiter(void 0, void 0, void 0, function* () {
+    const response = yield fetch(new URL("./zappar-cv.wasm", import.meta.url).toString());
+    const module = yield WebAssembly.compileStreaming(response);
+    return module;
+}))();
 export function launchWorker(worker) {
     return __awaiter(this, void 0, void 0, function* () {
         if (!worker)
             worker = new Worker(new URL("./worker", import.meta.url), { type: 'module' });
         worker.postMessage({
             t: "wasm",
-            url: new URL("./zappar-cv.wasm", import.meta.url).toString()
+            url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
+            module: yield wasmModule
         });
         yield waitForLoad(worker);
         function sendOutgoing() {
@@ -31,6 +37,18 @@ export function launchWorker(worker) {
         });
     });
 }
+export function launchCeresWorker(port, worker) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!worker)
+            worker = new Worker(new URL("./ceres-worker", import.meta.url), { type: 'module' });
+        worker.postMessage({
+            t: "wasm",
+            url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
+            module: yield wasmModule,
+            port
+        }, [port]);
+    });
+}
 function waitForLoad(w) {
     return new Promise(resolve => {
         let listener = (msg) => {

package/lib/worker-messages.d.ts

@@ -0,0 +1,10 @@
+import type * as ZNM from "./zappar-cv";
+export interface WorkerMessageAPI {
+    worker_message_send_count: () => number;
+    worker_message_send_clear: () => void;
+    worker_message_send_data_size: (index: number) => number;
+    worker_message_send_data: (index: number) => number;
+    worker_message_send_reference: (index: number) => number;
+    worker_message_receive: (ref: number, size: number, data: number) => void;
+}
+export declare function getWorkerMessageAPI(mod: ZNM.Module): WorkerMessageAPI;

package/lib/worker-messages.js

@@ -0,0 +1,10 @@
+export function getWorkerMessageAPI(mod) {
+    return {
+        worker_message_send_count: mod.cwrap("worker_message_send_count", "number", []),
+        worker_message_send_clear: mod.cwrap("worker_message_send_clear", null, []),
+        worker_message_send_data_size: mod.cwrap("worker_message_send_data_size", "number", ["number"]),
+        worker_message_send_data: mod.cwrap("worker_message_send_data", "number", ["number"]),
+        worker_message_send_reference: mod.cwrap("worker_message_send_reference", "number", ["number"]),
+        worker_message_receive: mod.cwrap("worker_message_receive", null, ["number", "number", "number"]),
+    };
+}

package/lib/worker-server.d.ts

@@ -1,3 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-export declare function launchWorkerServer(wasmUrl: string): Promise<void>;
+export declare function launchWorkerServer(wasmUrl: string, module: WebAssembly.Module): Promise<void>;

package/lib/worker-server.js

@@ -15,11 +15,13 @@ import { mat4 } from "gl-matrix";
 import { frame_pixel_format_t } from "./gen/zappar-native";
 import { handleImageBitmap } from "./worker-imagebitmap";
 import { profile } from "./profile";
+import { getWorkerMessageAPI } from "./worker-messages";
 export let messageManager = new MsgManager();
+let workerMessageChannel;
 let latestCameraToScreenRotation = 0;
 let rawDataAccessEnabled = false;
 const rawCameraFrameBuffersByPipeline = new Map();
-export function launchWorkerServer(wasmUrl) {
+export function launchWorkerServer(wasmUrl, module) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
             locateFile: (path, prefix) => {
@@ -28,8 +30,14 @@ export function launchWorkerServer(wasmUrl) {
                 }
                 return prefix + path;
             },
+            instantiateWasm: (imports, successCallback) => {
+                const instance = new WebAssembly.Instance(module, imports);
+                successCallback(instance);
+                return instance.exports;
+            },
             onRuntimeInitialized: () => {
                 let r = getRuntimeObject(mod);
+                const workerMessageAPI = getWorkerMessageAPI(mod);
                 let server = new zappar_server(r, (pipelineId, ab) => {
                     messageManager.postOutgoingMessage({
                         p: pipelineId,
@@ -57,6 +65,7 @@ export function launchWorkerServer(wasmUrl) {
                                 r.pipeline_frame_update(pipeline);
                                 att = r.pipeline_camera_frame_device_attitude(pipeline);
                                 server.exploreState();
+                                handleMessages(mod, workerMessageAPI);
                             }
                             let ret = {
                                 token: msgt.token,
@@ -87,7 +96,7 @@ export function launchWorkerServer(wasmUrl) {
                         }
                         case "streamC2S": {
                             let msgt = msg;
-                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source).then(() => {
+                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source, workerMessageAPI).then(() => {
                                 let m = { t: "streamEndedS2C", p: msgt.p, source: msgt.source };
                                 // console.log('Stream ended, requesting new stream');
                                 messageManager.postOutgoingMessage(m, []);
@@ -151,7 +160,7 @@ export function launchWorkerServer(wasmUrl) {
     });
 }
 ;
-function consumeStream(mod, r, stream, p, userFacing, server, source) {
+function consumeStream(mod, r, stream, p, userFacing, server, source, workerMessageAPI) {
     return __awaiter(this, void 0, void 0, function* () {
         while (true) {
             let reader;
@@ -164,7 +173,7 @@ function consumeStream(mod, r, stream, p, userFacing, server, source) {
                 continue;
             }
             try {
-                yield consumeReader(mod, r, reader, p, userFacing, server, source);
+                yield consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI);
                 return;
             }
             catch (err) {
@@ -192,8 +201,8 @@ function getFrameOrTimeout(reader) {
 }
 const cameraToDeviceTransform = mat4.create();
 const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
-function consumeReader(mod, r, reader, p, userFacing, server, source) {
-    var _a;
+function consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI) {
+    var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         while (true) {
             let result = yield getFrameOrTimeout(reader);
@@ -263,10 +272,11 @@ function consumeReader(mod, r, reader, p, userFacing, server, source) {
             const pipeline = server._pipeline_by_instance.get(p);
             if (pipeline) {
                 try {
-                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, (_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1);
+                    handleMessages(mod, workerMessageAPI);
                 }
                 catch (err) {
-                    // console.log('Exception during camera processing', err);
+                    console.log('Exception during camera processing', err);
                     // const func = mod.cwrap('what_to_stderr', 'number', ['number']);
                     // func(err);
                     // (mod as any).ccall('what_to_stderr', 'number', ['number'], [err]);
@@ -326,3 +336,34 @@ function framePixelFormatFromFormat(f) {
     }
     return frame_pixel_format_t.FRAME_PIXEL_FORMAT_Y;
 }
+function handleMessages(mod, m) {
+    const toSend = m.worker_message_send_count();
+    if (toSend === 0)
+        return;
+    if (!workerMessageChannel) {
+        workerMessageChannel = new MessageChannel();
+        workerMessageChannel.port1.start();
+        workerMessageChannel.port1.addEventListener('message', evt => {
+            if (evt.data.t !== 'msgrec')
+                return;
+            const data = evt.data.data;
+            const ptr = mod._malloc(data.byteLength);
+            mod.HEAPU8.set(data, ptr);
+            m.worker_message_receive(evt.data.reference, data.byteLength, ptr);
+            mod._free(ptr);
+        });
+        messageManager.postOutgoingMessage({ t: 'setupCeresWorker', port: workerMessageChannel.port2 }, [workerMessageChannel.port2]);
+    }
+    for (let i = 0; i < toSend; i++) {
+        const reference = m.worker_message_send_reference(i);
+        const dataSize = m.worker_message_send_data_size(i);
+        const dataPtr = m.worker_message_send_data(i);
+        const data = mod.HEAPU8.slice(dataPtr, dataPtr + dataSize);
+        workerMessageChannel.port1.postMessage({
+            t: 'msgsend',
+            data,
+            reference
+        }, [data.buffer]);
+    }
+    m.worker_message_send_clear();
+}

package/lib/worker.js

@@ -9,7 +9,7 @@ messageManager.onOutgoingMessage.bind(() => {
 let launchHandler = (evt) => {
     if (evt && evt.data && evt.data.t === "wasm") {
         let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zappar-cv.wasm", import.meta.url).toString();
-        launchWorkerServer(url);
+        launchWorkerServer(url, evt.data.module);
         ctx.removeEventListener("message", launchHandler);
     }
 };