@zappar/zappar-cv 3.0.0-beta.7 → 3.0.1-alpha.10

package/lib/native.js

@@ -12,7 +12,7 @@ import { zappar_client } from "./gen/zappar-client";
 import { drawPlane } from "./drawplane";
 import { cameraRotationForScreenOrientation, projectionMatrix } from "./cameramodel";
 import { mat4, vec3 } from "gl-matrix";
-import { launchWorker, messageManager } from "./worker-client";
+import { launchCeresWorker, launchWorker, messageManager } from "./worker-client";
 import { permissionRequestUI, permissionGrantedAll, permissionGrantedCamera, permissionGrantedMotion, permissionDeniedAny, permissionDeniedCamera, permissionDeniedMotion, permissionRequestMotion, permissionRequestCamera, permissionRequestAll, permissionDeniedUI } from "./permission";
 import { createFaceMesh, destroyFaceMesh, getFaceMesh } from "./facemesh";
 import { Pipeline, applyScreenCounterRotation } from "./pipeline";
@@ -26,7 +26,10 @@ import { createCameraSource, getCameraSource } from "./camera-source-map";
 import { gfx } from "./gfx";
 import { ImageTracker } from "./imagetracker";
 import { MSTPCameraSource } from "./mstp-camera-source";
+import { drawGrid } from "./drawgrid";
+import { getPointsDataMatrix } from "./drawpoints";
 let client;
+const pipelineByWorldTracker = new Map();
 export function initialize(opts) {
     if (client)
         return client;
@@ -84,6 +87,19 @@ export function initialize(opts) {
                     src.streamEnded();
                 break;
             }
+            case "setupCeresWorker": {
+                launchCeresWorker(msg.port);
+                break;
+            }
+            case "_z_datadownload": {
+                let blob = new Blob([msg.data], { type: "application/octet-stream" });
+                let url = URL.createObjectURL(blob);
+                let a = document.createElement("a");
+                a.download = "sequencedata.bin";
+                a.href = url;
+                a.click();
+                break;
+            }
             case "licerr": {
                 let div = document.createElement("div");
                 div.innerHTML = "Visit <a href='https://docs.zap.works/universal-ar/licensing/' style='color: white;'>our licensing page</a> to find out about hosting on your own domain.";
@@ -133,7 +149,7 @@ export function initialize(opts) {
             }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_source_profile_set: (cam, p) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setProfile(p); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
             let res = applyScreenCounterRotation(undefined, raw_pose);
             if (mirror) {
                 let scale = mat4.create();
@@ -145,8 +161,18 @@ export function initialize(opts) {
         }, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
+        }, pipeline_camera_frame_data_width: (pipeline) => {
+            var _a, _b;
+            const info = (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo();
+            return (_b = info === null || info === void 0 ? void 0 : info.dataWidth) !== null && _b !== void 0 ? _b : 0;
+        }, pipeline_camera_frame_data_height: (pipeline) => {
+            var _a, _b;
+            const info = (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo();
+            return (_b = info === null || info === void 0 ? void 0 : info.dataHeight) !== null && _b !== void 0 ? _b : 0;
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
             drawPlane(gl, projectionMatrix, cameraMatrix, targetMatrix, texture);
+        }, draw_grid: (gl, projectionMatrix, cameraMatrix, targetMatrix) => {
+            drawGrid(gl, projectionMatrix, cameraMatrix, targetMatrix);
         }, pipeline_draw_face: (p, projectionMatrix, cameraMatrix, targetMatrix, o) => {
             var _a;
             let obj = getFaceMesh(o);
@@ -353,6 +379,25 @@ export function initialize(opts) {
                 return 0;
             }
             return obj.getPreviewMesh(indx).normals.length;
+        }, zapcode_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.zapcode_tracker_anchor_pose_raw(o, indx));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, zapcode_tracker_anchor_pose: (o, indx, cameraPose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.zapcode_tracker_anchor_pose_raw(o, indx));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            mat4.multiply(res, cameraPose, res);
+            return res;
         }, face_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.face_tracker_anchor_pose_raw(o, indx));
             if (mirror) {
@@ -500,6 +545,15 @@ export function initialize(opts) {
                 return mat4.create();
             }
             return obj.anchor_pose;
+        }, world_tracker_create: (pipeline) => {
+            const ret = c.impl.world_tracker_create(pipeline);
+            pipelineByWorldTracker.set(ret, pipeline);
+            return ret;
+        }, world_tracker_points_data_matrix: (wt, screenWidth, screenHeight, mirror) => {
+            const pipeline = pipelineByWorldTracker.get(wt);
+            if (pipeline === undefined || !client)
+                return mat4.create();
+            return getPointsDataMatrix(client.pipeline_camera_frame_data_width(pipeline), client.pipeline_camera_frame_data_height(pipeline), screenWidth, screenHeight, mirror);
         }, world_tracker_plane_pose_camera_relative: (o, indx, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_plane_pose_raw(o, indx));
             if (mirror) {

package/lib/pipeline.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="dom-webcodecs" />
 import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
+import { camera_profile_t } from "./gen/zappar-native";
 import { zappar_client } from "./gen/zappar-client";
 import { MsgManager } from "./messages";
 import { Source, CameraFrameInfo } from "./source";
@@ -74,8 +75,9 @@ export declare class Pipeline {
     motionAttitudeSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionAttitudeMatrix(m: Float32Array): void;
     sendCameraStreamToWorker(source: zappar_camera_source_t, stream: ReadableStream<VideoFrame>, userFacing: boolean): void;
+    sendCameraProfileToWorker(source: zappar_camera_source_t, profile: camera_profile_t): void;
     sendCameraToScreenRotationToWorker(rot: number): void;
-    sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array): void;
+    sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array, cp: camera_profile_t): void;
     sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
 }
 export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): Float32Array;

package/lib/pipeline.js

@@ -448,6 +448,15 @@ export class Pipeline {
         };
         this._mgr.postOutgoingMessage(msg, [msg.s]);
     }
+    sendCameraProfileToWorker(source, profile) {
+        let msg = {
+            t: "cameraProfileC2S",
+            p: this._impl,
+            source,
+            profile
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
     sendCameraToScreenRotationToWorker(rot) {
         let msg = {
             p: this._impl,
@@ -456,7 +465,7 @@ export class Pipeline {
         };
         this._mgr.postOutgoingMessage(msg, []);
     }
-    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice) {
+    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice, cp) {
         let msg = {
             p: this._impl,
             t: "imageBitmapC2S",
@@ -465,7 +474,8 @@ export class Pipeline {
             tokenId,
             userFacing,
             cameraModel,
-            cameraToDevice
+            cameraToDevice,
+            cp
         };
         this._mgr.postOutgoingMessage(msg, [img]);
     }

package/lib/profile.d.ts

@@ -1,3 +1,4 @@
+import { camera_profile_t } from "./gen/zappar-native";
 export declare enum EmbeddedVideoImplementation {
     OBJECT_URL = 0,
     SRC_OBJECT = 1
@@ -10,8 +11,7 @@ export declare let profile: {
     requestHighFrameRate: boolean;
     videoWidth: number;
     videoHeight: number;
-    dataWidth: number;
-    dataHeight: number;
+    getDataSize: (p: camera_profile_t) => number[];
     videoElementInDOM: boolean;
     preferMediaStreamTrackProcessorCamera: boolean;
     preferImageBitmapCamera: boolean;

package/lib/profile.js

@@ -1,4 +1,5 @@
 import * as UAParser from "ua-parser-js";
+import { camera_profile_t } from "./gen/zappar-native";
 export var EmbeddedVideoImplementation;
 (function (EmbeddedVideoImplementation) {
     EmbeddedVideoImplementation[EmbeddedVideoImplementation["OBJECT_URL"] = 0] = "OBJECT_URL";
@@ -12,8 +13,7 @@ export let profile = {
     requestHighFrameRate: false,
     videoWidth: 640,
     videoHeight: 480,
-    dataWidth: 320,
-    dataHeight: 240,
+    getDataSize: (p) => p === camera_profile_t.HIGH ? [640, 480] : [320, 240],
     videoElementInDOM: false,
     preferMediaStreamTrackProcessorCamera: true,
     preferImageBitmapCamera: false,
@@ -80,7 +80,7 @@ function iDevice(version) {
                 // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
                 // Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
                 profile.videoHeight = 360;
-                profile.dataHeight = 180;
+                profile.getDataSize = (p) => p === camera_profile_t.HIGH ? [640, 360] : [320, 180];
             }
         }
         if (majorVersion === 16 && minorVersion >= 4) {

package/lib/serializer.d.ts

@@ -8,6 +8,7 @@ export interface MessageAppender {
     type: (t: number) => void;
     matrix4x4: (a: Float32Array) => void;
     matrix3x3: (a: Float32Array) => void;
+    floatArray: (a: Float32Array) => void;
     cameraModel: (a: Float32Array) => void;
     identityCoefficients: (a: Float32Array) => void;
     expressionCoefficients: (a: Float32Array) => void;

package/lib/serializer.js

@@ -22,6 +22,7 @@ export class MessageSerializer {
             type: i => this.int(i),
             matrix4x4: i => this.float32ArrayBuffer(i),
             matrix3x3: i => this.float32ArrayBuffer(i),
+            floatArray: i => this.float32ArrayBuffer(i),
             identityCoefficients: i => this.float32ArrayBuffer(i),
             expressionCoefficients: i => this.float32ArrayBuffer(i),
             cameraModel: i => this.float32ArrayBuffer(i),

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.0-beta.7";
+export declare const VERSION = "3.0.1-alpha.10";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.0-beta.7";
+export const VERSION = "3.0.1-alpha.10";

package/lib/worker-client.d.ts

@@ -1,3 +1,4 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
 export declare function launchWorker(worker?: Worker): Promise<void>;
+export declare function launchCeresWorker(port: MessagePort, worker?: Worker): Promise<void>;

package/lib/worker-client.js

@@ -7,15 +7,25 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var _a;
 import { MsgManager } from "./messages";
 export let messageManager = new MsgManager();
+const params = new URLSearchParams(window.location.search);
+const shouldRecordData = parseInt((_a = params.get('_zrecorddata')) !== null && _a !== void 0 ? _a : '0');
+const wasmModule = (() => __awaiter(void 0, void 0, void 0, function* () {
+    const response = yield fetch(new URL("./zappar-cv.wasm", import.meta.url).toString());
+    const module = yield WebAssembly.compileStreaming(response);
+    return module;
+}))();
 export function launchWorker(worker) {
     return __awaiter(this, void 0, void 0, function* () {
         if (!worker)
             worker = new Worker(new URL("./worker", import.meta.url), { type: 'module' });
         worker.postMessage({
             t: "wasm",
-            url: new URL("./zappar-cv.wasm", import.meta.url).toString()
+            url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
+            module: yield wasmModule,
+            shouldRecordData
         });
         yield waitForLoad(worker);
         function sendOutgoing() {
@@ -31,6 +41,18 @@ export function launchWorker(worker) {
         });
     });
 }
+export function launchCeresWorker(port, worker) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!worker)
+            worker = new Worker(new URL("./ceres-worker", import.meta.url), { type: 'module' });
+        worker.postMessage({
+            t: "wasm",
+            url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
+            module: yield wasmModule,
+            port
+        }, [port]);
+    });
+}
 function waitForLoad(w) {
     return new Promise(resolve => {
         let listener = (msg) => {

package/lib/worker-imagebitmap.js

@@ -34,12 +34,13 @@ export function handleImageBitmap(m, r, server, mgr) {
     }
     if (!texture)
         return;
-    if (!pixels || pixels.byteLength < profile.dataWidth * profile.dataHeight) {
+    const [dataWidth, dataHeight] = profile.getDataSize(m.cp);
+    if (!pixels || pixels.byteLength < dataWidth * dataHeight) {
         // console.log("Generating pixel buffer", profile.dataWidth * profile.dataHeight);
-        pixels = new ArrayBuffer(profile.dataWidth * profile.dataHeight);
+        pixels = new ArrayBuffer(dataWidth * dataHeight);
     }
-    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing);
-    let info = imageProcessor.readFrame(texture, pixels);
+    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing, m.cp);
+    let info = imageProcessor.readFrame(texture, pixels, m.cp);
     let msg = {
         t: "imageBitmapS2C",
         dataWidth: info.dataWidth,

package/lib/worker-messages.d.ts

@@ -0,0 +1,10 @@
+import type * as ZNM from "./zappar-cv";
+export interface WorkerMessageAPI {
+    worker_message_send_count: () => number;
+    worker_message_send_clear: () => void;
+    worker_message_send_data_size: (index: number) => number;
+    worker_message_send_data: (index: number) => number;
+    worker_message_send_reference: (index: number) => number;
+    worker_message_receive: (ref: number, size: number, data: number) => void;
+}
+export declare function getWorkerMessageAPI(mod: ZNM.Module): WorkerMessageAPI;

package/lib/worker-messages.js

@@ -0,0 +1,10 @@
+export function getWorkerMessageAPI(mod) {
+    return {
+        worker_message_send_count: mod.cwrap("worker_message_send_count", "number", []),
+        worker_message_send_clear: mod.cwrap("worker_message_send_clear", null, []),
+        worker_message_send_data_size: mod.cwrap("worker_message_send_data_size", "number", ["number"]),
+        worker_message_send_data: mod.cwrap("worker_message_send_data", "number", ["number"]),
+        worker_message_send_reference: mod.cwrap("worker_message_send_reference", "number", ["number"]),
+        worker_message_receive: mod.cwrap("worker_message_receive", null, ["number", "number", "number"]),
+    };
+}

package/lib/worker-server.d.ts

@@ -1,3 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-export declare function launchWorkerServer(wasmUrl: string): Promise<void>;
+export declare function launchWorkerServer(wasmUrl: string, module: WebAssembly.Module, shouldRecordData: number): Promise<void>;

package/lib/worker-server.js

@@ -12,14 +12,17 @@ import { getRuntimeObject } from "./gen/zappar-cwrap";
 import { zappar_server } from "./gen/zappar-server";
 import { MsgManager } from "./messages";
 import { mat4 } from "gl-matrix";
-import { frame_pixel_format_t } from "./gen/zappar-native";
+import { camera_profile_t, frame_pixel_format_t } from "./gen/zappar-native";
 import { handleImageBitmap } from "./worker-imagebitmap";
-import { profile } from "./profile";
+import { getWorkerMessageAPI } from "./worker-messages";
+import { getDataDownloadAPI } from "./data-download";
 export let messageManager = new MsgManager();
+let workerMessageChannel;
 let latestCameraToScreenRotation = 0;
 let rawDataAccessEnabled = false;
 const rawCameraFrameBuffersByPipeline = new Map();
-export function launchWorkerServer(wasmUrl) {
+const cameraProfileBySource = new Map();
+export function launchWorkerServer(wasmUrl, module, shouldRecordData) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
             locateFile: (path, prefix) => {
@@ -28,8 +31,16 @@ export function launchWorkerServer(wasmUrl) {
                 }
                 return prefix + path;
             },
+            instantiateWasm: (imports, successCallback) => {
+                const instance = new WebAssembly.Instance(module, imports);
+                successCallback(instance);
+                return instance.exports;
+            },
             onRuntimeInitialized: () => {
                 let r = getRuntimeObject(mod);
+                const workerMessageAPI = getWorkerMessageAPI(mod);
+                const dataDownloadAPI = (shouldRecordData > 0) ? getDataDownloadAPI(mod) : undefined;
+                dataDownloadAPI === null || dataDownloadAPI === void 0 ? void 0 : dataDownloadAPI.data_should_record_set(shouldRecordData); // Only happens if dataDownloadAPI is defined
                 let server = new zappar_server(r, (pipelineId, ab) => {
                     messageManager.postOutgoingMessage({
                         p: pipelineId,
@@ -57,6 +68,9 @@ export function launchWorkerServer(wasmUrl) {
                                 r.pipeline_frame_update(pipeline);
                                 att = r.pipeline_camera_frame_device_attitude(pipeline);
                                 server.exploreState();
+                                handleMessages(mod, workerMessageAPI);
+                                if (dataDownloadAPI)
+                                    handleDataDownload(mod, dataDownloadAPI);
                             }
                             let ret = {
                                 token: msgt.token,
@@ -85,9 +99,14 @@ export function launchWorkerServer(wasmUrl) {
                             messageManager.postOutgoingMessage(ret, []);
                             break;
                         }
+                        case "cameraProfileC2S": {
+                            let msgt = msg;
+                            cameraProfileBySource.set(msgt.source, msgt.p);
+                            break;
+                        }
                         case "streamC2S": {
                             let msgt = msg;
-                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source).then(() => {
+                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source, workerMessageAPI, dataDownloadAPI).then(() => {
                                 let m = { t: "streamEndedS2C", p: msgt.p, source: msgt.source };
                                 // console.log('Stream ended, requesting new stream');
                                 messageManager.postOutgoingMessage(m, []);
@@ -151,7 +170,7 @@ export function launchWorkerServer(wasmUrl) {
     });
 }
 ;
-function consumeStream(mod, r, stream, p, userFacing, server, source) {
+function consumeStream(mod, r, stream, p, userFacing, server, source, workerMessageAPI, dataDownloadAPI) {
     return __awaiter(this, void 0, void 0, function* () {
         while (true) {
             let reader;
@@ -164,7 +183,7 @@ function consumeStream(mod, r, stream, p, userFacing, server, source) {
                 continue;
             }
             try {
-                yield consumeReader(mod, r, reader, p, userFacing, server, source);
+                yield consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI, dataDownloadAPI);
                 return;
             }
             catch (err) {
@@ -192,8 +211,8 @@ function getFrameOrTimeout(reader) {
 }
 const cameraToDeviceTransform = mat4.create();
 const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
-function consumeReader(mod, r, reader, p, userFacing, server, source) {
-    var _a;
+function consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI, dataDownloadAPI) {
+    var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         while (true) {
             let result = yield getFrameOrTimeout(reader);
@@ -236,16 +255,22 @@ function consumeReader(mod, r, reader, p, userFacing, server, source) {
                     uvTransform = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
                     break;
             }
+            let downSample = false;
+            if (cameraProfileBySource.get(source) !== camera_profile_t.HIGH) {
+                dataWidth /= 2;
+                dataHeight /= 2;
+                downSample = true;
+            }
             let clone = frame.clone();
             if (userFacing)
                 mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
             else
                 mat4.identity(cameraToDeviceTransform);
-            let focalLength = 300.0 * profile.dataWidth / 320.0;
+            let focalLength = 300.0 * dataWidth / 320.0;
             cameraModel[0] = focalLength;
             cameraModel[1] = focalLength;
-            cameraModel[2] = profile.dataWidth * 0.5;
-            cameraModel[3] = profile.dataHeight * 0.5;
+            cameraModel[2] = dataWidth * 0.5;
+            cameraModel[3] = dataHeight * 0.5;
             const ret = {
                 token: token,
                 d: clone,
@@ -253,8 +278,8 @@ function consumeReader(mod, r, reader, p, userFacing, server, source) {
                 t: "videoFrameS2C",
                 userFacing,
                 uvTransform,
-                w: profile.dataWidth,
-                h: profile.dataHeight,
+                w: dataWidth,
+                h: dataHeight,
                 cameraToDevice: cameraToDeviceTransform,
                 cameraModel,
                 source
@@ -263,10 +288,13 @@ function consumeReader(mod, r, reader, p, userFacing, server, source) {
             const pipeline = server._pipeline_by_instance.get(p);
             if (pipeline) {
                 try {
-                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, (_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1, downSample);
+                    handleMessages(mod, workerMessageAPI);
+                    if (dataDownloadAPI)
+                        handleDataDownload(mod, dataDownloadAPI);
                 }
                 catch (err) {
-                    // console.log('Exception during camera processing', err);
+                    console.log('Exception during camera processing', err);
                     // const func = mod.cwrap('what_to_stderr', 'number', ['number']);
                     // func(err);
                     // (mod as any).ccall('what_to_stderr', 'number', ['number'], [err]);
@@ -326,3 +354,43 @@ function framePixelFormatFromFormat(f) {
     }
     return frame_pixel_format_t.FRAME_PIXEL_FORMAT_Y;
 }
+function handleMessages(mod, m) {
+    const toSend = m.worker_message_send_count();
+    if (toSend === 0)
+        return;
+    if (!workerMessageChannel) {
+        workerMessageChannel = new MessageChannel();
+        workerMessageChannel.port1.start();
+        workerMessageChannel.port1.addEventListener('message', evt => {
+            if (evt.data.t !== 'msgrec')
+                return;
+            const data = evt.data.data;
+            const ptr = mod._malloc(data.byteLength);
+            mod.HEAPU8.set(data, ptr);
+            m.worker_message_receive(evt.data.reference, data.byteLength, ptr);
+            mod._free(ptr);
+        });
+        messageManager.postOutgoingMessage({ t: 'setupCeresWorker', port: workerMessageChannel.port2 }, [workerMessageChannel.port2]);
+    }
+    for (let i = 0; i < toSend; i++) {
+        const reference = m.worker_message_send_reference(i);
+        const dataSize = m.worker_message_send_data_size(i);
+        const dataPtr = m.worker_message_send_data(i);
+        const data = mod.HEAPU8.slice(dataPtr, dataPtr + dataSize);
+        workerMessageChannel.port1.postMessage({
+            t: 'msgsend',
+            data,
+            reference
+        }, [data.buffer]);
+    }
+    m.worker_message_send_clear();
+}
+function handleDataDownload(mod, d) {
+    const size = d.data_download_size();
+    if (size === 0)
+        return;
+    const dataPtr = d.data_download();
+    const data = mod.HEAPU8.slice(dataPtr, dataPtr + size);
+    messageManager.postOutgoingMessage({ t: '_z_datadownload', data }, [data.buffer]);
+    d.data_download_clear();
+}

package/lib/worker.js

@@ -7,9 +7,10 @@ messageManager.onOutgoingMessage.bind(() => {
     }
 });
 let launchHandler = (evt) => {
+    var _a;
     if (evt && evt.data && evt.data.t === "wasm") {
         let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zappar-cv.wasm", import.meta.url).toString();
-        launchWorkerServer(url);
+        launchWorkerServer(url, evt.data.module, (_a = evt.data.shouldRecordData) !== null && _a !== void 0 ? _a : 0);
         ctx.removeEventListener("message", launchHandler);
     }
 };

package/lib/workerinterface.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="dom-webcodecs" />
 import { CameraFrameData } from "./camera-frame-data";
-import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
+import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
 export interface CameraFrameC2S {
     t: "cameraFrameC2S";
     d: ArrayBuffer;
@@ -39,6 +39,12 @@ export interface StreamC2S {
     userFacing: boolean;
     source: zappar_camera_source_t;
 }
+export interface CameraProfileC2S {
+    t: "cameraProfileC2S";
+    p: zappar_pipeline_t;
+    source: zappar_camera_source_t;
+    profile: camera_profile_t;
+}
 export interface CameraToScreenRotationC2S {
     t: "cameraToScreenC2S";
     p: zappar_pipeline_t;
@@ -53,6 +59,7 @@ export interface ImageBitmapC2S {
     tokenId: number;
     cameraModel: Float32Array;
     cameraToDevice: Float32Array;
+    cp: camera_profile_t;
 }
 export interface ImageBitmapS2C {
     t: "imageBitmapS2C";