@zappar/zappar-cv 3.0.1-alpha.1 → 3.0.1-alpha.10

package/lib/native.js

@@ -26,7 +26,10 @@ import { createCameraSource, getCameraSource } from "./camera-source-map";
 import { gfx } from "./gfx";
 import { ImageTracker } from "./imagetracker";
 import { MSTPCameraSource } from "./mstp-camera-source";
+import { drawGrid } from "./drawgrid";
+import { getPointsDataMatrix } from "./drawpoints";
 let client;
+const pipelineByWorldTracker = new Map();
 export function initialize(opts) {
     if (client)
         return client;
@@ -49,7 +52,7 @@ export function initialize(opts) {
             c.impl.analytics_project_id_set(".wiz" + pathParts[1], uid);
     }
     messageManager.onIncomingMessage.bind(msg => {
-        var _a, _b, _c, _d, _e, _f, _g;
+        var _a, _b, _c, _d, _e, _f, _g, _h;
         switch (msg.t) {
             case "zappar":
                 (_a = Pipeline.get(msg.p)) === null || _a === void 0 ? void 0 : _a.pendingMessages.push(msg.d);
@@ -72,6 +75,11 @@ export function initialize(opts) {
                 (_g = (_f = Pipeline.get(msgt.p)) === null || _f === void 0 ? void 0 : _f.imageBitmapFromWorker) === null || _g === void 0 ? void 0 : _g.call(_f, msgt);
                 break;
             }
+            case "raw": {
+                let msgt = msg;
+                (_h = Pipeline.get(msgt.p)) === null || _h === void 0 ? void 0 : _h.cameraFrameDataRawResult(msgt);
+                break;
+            }
             case "streamEndedS2C": {
                 let msgt = msg;
                 const src = getCameraSource(msgt.source);
@@ -83,6 +91,15 @@ export function initialize(opts) {
                 launchCeresWorker(msg.port);
                 break;
             }
+            case "_z_datadownload": {
+                let blob = new Blob([msg.data], { type: "application/octet-stream" });
+                let url = URL.createObjectURL(blob);
+                let a = document.createElement("a");
+                a.download = "sequencedata.bin";
+                a.href = url;
+                a.click();
+                break;
+            }
             case "licerr": {
                 let div = document.createElement("div");
                 div.innerHTML = "Visit <a href='https://docs.zap.works/universal-ar/licensing/' style='color: white;'>our licensing page</a> to find out about hosting on your own domain.";
@@ -132,11 +149,30 @@ export function initialize(opts) {
             }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_source_profile_set: (cam, p) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setProfile(p); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, raw_pose);
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
+        }, pipeline_camera_frame_data_width: (pipeline) => {
+            var _a, _b;
+            const info = (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo();
+            return (_b = info === null || info === void 0 ? void 0 : info.dataWidth) !== null && _b !== void 0 ? _b : 0;
+        }, pipeline_camera_frame_data_height: (pipeline) => {
+            var _a, _b;
+            const info = (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo();
+            return (_b = info === null || info === void 0 ? void 0 : info.dataHeight) !== null && _b !== void 0 ? _b : 0;
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
             drawPlane(gl, projectionMatrix, cameraMatrix, targetMatrix, texture);
+        }, draw_grid: (gl, projectionMatrix, cameraMatrix, targetMatrix) => {
+            drawGrid(gl, projectionMatrix, cameraMatrix, targetMatrix);
         }, pipeline_draw_face: (p, projectionMatrix, cameraMatrix, targetMatrix, o) => {
             var _a;
             let obj = getFaceMesh(o);
@@ -156,7 +192,7 @@ export function initialize(opts) {
                 return;
             }
             (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, obj);
-        }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
+        }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, pipeline_camera_frame_data_raw: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.getCameraFrameDataRaw()) || Promise.reject(); }, pipeline_camera_frame_data_raw_enabled_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.setCameraFrameDataRawEnabled(v); }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.instant_world_tracker_anchor_pose_raw(o));
             if (mirror) {
                 let scale = mat4.create();
@@ -343,6 +379,25 @@ export function initialize(opts) {
                 return 0;
             }
             return obj.getPreviewMesh(indx).normals.length;
+        }, zapcode_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.zapcode_tracker_anchor_pose_raw(o, indx));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, zapcode_tracker_anchor_pose: (o, indx, cameraPose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.zapcode_tracker_anchor_pose_raw(o, indx));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            mat4.multiply(res, cameraPose, res);
+            return res;
         }, face_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.face_tracker_anchor_pose_raw(o, indx));
             if (mirror) {
@@ -490,6 +545,15 @@ export function initialize(opts) {
                 return mat4.create();
             }
             return obj.anchor_pose;
+        }, world_tracker_create: (pipeline) => {
+            const ret = c.impl.world_tracker_create(pipeline);
+            pipelineByWorldTracker.set(ret, pipeline);
+            return ret;
+        }, world_tracker_points_data_matrix: (wt, screenWidth, screenHeight, mirror) => {
+            const pipeline = pipelineByWorldTracker.get(wt);
+            if (pipeline === undefined || !client)
+                return mat4.create();
+            return getPointsDataMatrix(client.pipeline_camera_frame_data_width(pipeline), client.pipeline_camera_frame_data_height(pipeline), screenWidth, screenHeight, mirror);
         }, world_tracker_plane_pose_camera_relative: (o, indx, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_plane_pose_raw(o, indx));
             if (mirror) {

package/lib/pipeline.d.ts

@@ -1,13 +1,15 @@
 /// <reference types="dom-webcodecs" />
 import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
-import { zappar_cwrap } from "./gen/zappar-native";
+import { camera_profile_t } from "./gen/zappar-native";
 import { zappar_client } from "./gen/zappar-client";
 import { MsgManager } from "./messages";
 import { Source, CameraFrameInfo } from "./source";
-import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
+import { CameraDataRawS2C, CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
 import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
 import { ImageTracker } from "./imagetracker";
+import { zappar } from "./gen/zappar-bridge";
+import { CameraFrameData } from "./camera-frame-data";
 export declare class Pipeline {
     private _client;
     private _impl;
@@ -27,12 +29,17 @@ export declare class Pipeline {
     private _sequenceRecorder;
     private _sequenceRecordDeviceAttitudeMatrices;
     private _sequenceRecorderFirstCameraToken;
+    private _cameraFrameDataPromises;
+    private _cameraFrameDataResolves;
+    private _cameraFrameDataRejects;
+    private _cameraFrameDataEnabled;
     onGLContextReset: Event;
-    static create(client: zappar_cwrap, mgr: MsgManager): zappar_pipeline_t;
+    static create(client: zappar, mgr: MsgManager): zappar_pipeline_t;
     static get(p: zappar_pipeline_t): Pipeline | undefined;
     private constructor();
     frameUpdate(client: zappar_client): void;
     cleanOldFrames(): void;
+    cameraFrameDataRawResult(msg: CameraDataRawS2C): void;
     cameraTokenReturn(msg: CameraFrameReturnS2C): void;
     sequenceRecordStart(expectedFrames: number): void;
     sequenceRecordStop(): void;
@@ -40,6 +47,8 @@ export declare class Pipeline {
     sequenceRecordClear(): void;
     sequenceRecordDeviceAttitudeMatrices(v: boolean): void;
     getVideoTexture(): WebGLTexture | undefined;
+    getCameraFrameDataRaw(): Promise<CameraFrameData | undefined>;
+    setCameraFrameDataRawEnabled(e: boolean): void;
     destroy(): void;
     getCurrentCameraInfo(): CameraFrameInfo | undefined;
     cameraFrameDrawGL(screenWidth: number, screenHeight: number, mirror?: boolean): void;
@@ -66,8 +75,9 @@ export declare class Pipeline {
     motionAttitudeSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionAttitudeMatrix(m: Float32Array): void;
     sendCameraStreamToWorker(source: zappar_camera_source_t, stream: ReadableStream<VideoFrame>, userFacing: boolean): void;
+    sendCameraProfileToWorker(source: zappar_camera_source_t, profile: camera_profile_t): void;
     sendCameraToScreenRotationToWorker(rot: number): void;
-    sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array): void;
+    sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array, cp: camera_profile_t): void;
     sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
 }
 export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): Float32Array;

package/lib/pipeline.js

@@ -24,6 +24,10 @@ export class Pipeline {
         this.cameraPixelArrays = [];
         this._sequenceRecordDeviceAttitudeMatrices = true;
         this._sequenceRecorderFirstCameraToken = 0;
+        this._cameraFrameDataPromises = new Map();
+        this._cameraFrameDataResolves = new Map();
+        this._cameraFrameDataRejects = new Map();
+        this._cameraFrameDataEnabled = false;
         this.onGLContextReset = new Event();
     }
     static create(client, mgr) {
@@ -56,14 +60,34 @@ export class Pipeline {
                 if (t[1].texture)
                     this.videoTextures.push(t[1].texture);
                 (_b = (_a = t[1].frame) === null || _a === void 0 ? void 0 : _a.close) === null || _b === void 0 ? void 0 : _b.call(_a);
+                if (t[1].data) {
+                    this.cameraPixelArrays.push(t[1].data);
+                    t[1].data = undefined;
+                }
                 this.cameraTokens.delete(t[0]);
             }
         }
     }
+    cameraFrameDataRawResult(msg) {
+        var _a, _b;
+        let info = this.cameraTokens.get(msg.token);
+        if (!info)
+            return;
+        if (msg.data === null) {
+            (_a = this._cameraFrameDataRejects.get(info)) === null || _a === void 0 ? void 0 : _a(new Error('Raw camera data not available'));
+        }
+        else {
+            info.data = msg.data.data;
+            (_b = this._cameraFrameDataResolves.get(info)) === null || _b === void 0 ? void 0 : _b(msg.data);
+        }
+        this._cameraFrameDataRejects.delete(info);
+        this._cameraFrameDataResolves.delete(info);
+        this._cameraFrameDataPromises.delete(info);
+    }
     cameraTokenReturn(msg) {
+        let info = this.cameraTokens.get(msg.token);
         if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices
             && msg.token >= this._sequenceRecorderFirstCameraToken) {
-            let info = this.cameraTokens.get(msg.token);
             if (info) {
                 if (msg.att)
                     this._sequenceRecorder.appendAttitudeMatrix(msg.att);
@@ -71,7 +95,20 @@ export class Pipeline {
                 this._sequenceRecorder.appendCameraFrame(info);
             }
         }
-        this.cameraPixelArrays.push(msg.d);
+        if (info && !info.data && msg.d) {
+            info.data = msg.d;
+            const rawRequestResolve = this._cameraFrameDataResolves.get(info);
+            if (rawRequestResolve) {
+                rawRequestResolve({
+                    data: info.data,
+                    width: info.dataWidth,
+                    height: info.dataHeight,
+                });
+            }
+            this._cameraFrameDataRejects.delete(info);
+            this._cameraFrameDataResolves.delete(info);
+            this._cameraFrameDataPromises.delete(info);
+        }
         this.tokensInFlight--;
     }
     sequenceRecordStart(expectedFrames) {
@@ -97,6 +134,37 @@ export class Pipeline {
     getVideoTexture() {
         return this.videoTextures.pop();
     }
+    getCameraFrameDataRaw() {
+        if (!this._cameraFrameDataEnabled)
+            throw new Error('Raw camera frame data is not enabled.');
+        const info = this.getCurrentCameraInfo();
+        if (!info)
+            return Promise.resolve(undefined);
+        if (info.data)
+            return Promise.resolve({ data: info.data.slice(0), width: info.dataWidth, height: info.dataHeight });
+        let existing = this._cameraFrameDataPromises.get(info);
+        if (!existing) {
+            existing = new Promise((resolve, reject) => {
+                if (typeof info.remoteToken !== 'undefined') {
+                    const msg = {
+                        t: "rawrequest",
+                        token: info.remoteToken,
+                        p: this._impl,
+                    };
+                    this._mgr.postOutgoingMessage(msg, []);
+                }
+                this._cameraFrameDataResolves.set(info, resolve);
+                this._cameraFrameDataRejects.set(info, reject);
+            });
+            this._cameraFrameDataPromises.set(info, existing);
+        }
+        return existing;
+    }
+    setCameraFrameDataRawEnabled(e) {
+        this._cameraFrameDataEnabled = e;
+        const msg = { t: 'rawenabled', v: e };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
     destroy() {
         this._client.pipeline_destroy(this._impl);
         byId.delete(this._impl);
@@ -226,7 +294,8 @@ export class Pipeline {
             uvTransform: msg.uvTransform,
             cameraModel: msg.cameraModel,
             cameraToDevice: msg.cameraToDevice,
-            cameraSource
+            cameraSource,
+            remoteToken: msg.token,
         });
         this.cleanOldFrames();
     }
@@ -379,6 +448,15 @@ export class Pipeline {
         };
         this._mgr.postOutgoingMessage(msg, [msg.s]);
     }
+    sendCameraProfileToWorker(source, profile) {
+        let msg = {
+            t: "cameraProfileC2S",
+            p: this._impl,
+            source,
+            profile
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
     sendCameraToScreenRotationToWorker(rot) {
         let msg = {
             p: this._impl,
@@ -387,7 +465,7 @@ export class Pipeline {
         };
         this._mgr.postOutgoingMessage(msg, []);
     }
-    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice) {
+    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice, cp) {
         let msg = {
             p: this._impl,
             t: "imageBitmapC2S",
@@ -396,7 +474,8 @@ export class Pipeline {
             tokenId,
             userFacing,
             cameraModel,
-            cameraToDevice
+            cameraToDevice,
+            cp
         };
         this._mgr.postOutgoingMessage(msg, [img]);
     }

package/lib/profile.d.ts

@@ -1,3 +1,4 @@
+import { camera_profile_t } from "./gen/zappar-native";
 export declare enum EmbeddedVideoImplementation {
     OBJECT_URL = 0,
     SRC_OBJECT = 1
@@ -10,8 +11,7 @@ export declare let profile: {
     requestHighFrameRate: boolean;
     videoWidth: number;
     videoHeight: number;
-    dataWidth: number;
-    dataHeight: number;
+    getDataSize: (p: camera_profile_t) => number[];
     videoElementInDOM: boolean;
     preferMediaStreamTrackProcessorCamera: boolean;
     preferImageBitmapCamera: boolean;

package/lib/profile.js

@@ -1,4 +1,5 @@
 import * as UAParser from "ua-parser-js";
+import { camera_profile_t } from "./gen/zappar-native";
 export var EmbeddedVideoImplementation;
 (function (EmbeddedVideoImplementation) {
     EmbeddedVideoImplementation[EmbeddedVideoImplementation["OBJECT_URL"] = 0] = "OBJECT_URL";
@@ -12,8 +13,7 @@ export let profile = {
     requestHighFrameRate: false,
     videoWidth: 640,
     videoHeight: 480,
-    dataWidth: 320,
-    dataHeight: 240,
+    getDataSize: (p) => p === camera_profile_t.HIGH ? [640, 480] : [320, 240],
     videoElementInDOM: false,
     preferMediaStreamTrackProcessorCamera: true,
     preferImageBitmapCamera: false,
@@ -80,7 +80,7 @@ function iDevice(version) {
                 // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
                 // Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
                 profile.videoHeight = 360;
-                profile.dataHeight = 180;
+                profile.getDataSize = (p) => p === camera_profile_t.HIGH ? [640, 360] : [320, 180];
             }
         }
         if (majorVersion === 16 && minorVersion >= 4) {

package/lib/serializer.d.ts

@@ -8,6 +8,7 @@ export interface MessageAppender {
     type: (t: number) => void;
     matrix4x4: (a: Float32Array) => void;
     matrix3x3: (a: Float32Array) => void;
+    floatArray: (a: Float32Array) => void;
     cameraModel: (a: Float32Array) => void;
     identityCoefficients: (a: Float32Array) => void;
     expressionCoefficients: (a: Float32Array) => void;

package/lib/serializer.js

@@ -22,6 +22,7 @@ export class MessageSerializer {
             type: i => this.int(i),
             matrix4x4: i => this.float32ArrayBuffer(i),
             matrix3x3: i => this.float32ArrayBuffer(i),
+            floatArray: i => this.float32ArrayBuffer(i),
             identityCoefficients: i => this.float32ArrayBuffer(i),
             expressionCoefficients: i => this.float32ArrayBuffer(i),
             cameraModel: i => this.float32ArrayBuffer(i),

package/lib/source.d.ts

@@ -10,6 +10,7 @@ export interface CameraFrameInfo {
     cameraSourceData?: any;
     uvTransform?: Float32Array;
     userFacing: boolean;
+    remoteToken?: number;
     frame?: VideoFrame | ImageBitmap;
 }
 export declare abstract class Source {

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.1-alpha.1";
+export declare const VERSION = "3.0.1-alpha.10";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.1-alpha.1";
+export const VERSION = "3.0.1-alpha.10";

package/lib/worker-client.js

@@ -7,8 +7,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var _a;
 import { MsgManager } from "./messages";
 export let messageManager = new MsgManager();
+const params = new URLSearchParams(window.location.search);
+const shouldRecordData = parseInt((_a = params.get('_zrecorddata')) !== null && _a !== void 0 ? _a : '0');
 const wasmModule = (() => __awaiter(void 0, void 0, void 0, function* () {
     const response = yield fetch(new URL("./zappar-cv.wasm", import.meta.url).toString());
     const module = yield WebAssembly.compileStreaming(response);
@@ -21,7 +24,8 @@ export function launchWorker(worker) {
         worker.postMessage({
             t: "wasm",
             url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
-            module: yield wasmModule
+            module: yield wasmModule,
+            shouldRecordData
         });
         yield waitForLoad(worker);
         function sendOutgoing() {

package/lib/worker-imagebitmap.js

@@ -34,12 +34,13 @@ export function handleImageBitmap(m, r, server, mgr) {
     }
     if (!texture)
         return;
-    if (!pixels || pixels.byteLength < profile.dataWidth * profile.dataHeight) {
+    const [dataWidth, dataHeight] = profile.getDataSize(m.cp);
+    if (!pixels || pixels.byteLength < dataWidth * dataHeight) {
         // console.log("Generating pixel buffer", profile.dataWidth * profile.dataHeight);
-        pixels = new ArrayBuffer(profile.dataWidth * profile.dataHeight);
+        pixels = new ArrayBuffer(dataWidth * dataHeight);
     }
-    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing);
-    let info = imageProcessor.readFrame(texture, pixels);
+    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing, m.cp);
+    let info = imageProcessor.readFrame(texture, pixels, m.cp);
     let msg = {
         t: "imageBitmapS2C",
         dataWidth: info.dataWidth,

package/lib/worker-server.d.ts

@@ -1,3 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-export declare function launchWorkerServer(wasmUrl: string, module: WebAssembly.Module): Promise<void>;
+export declare function launchWorkerServer(wasmUrl: string, module: WebAssembly.Module, shouldRecordData: number): Promise<void>;