@zappar/zappar-cv 3.0.1-alpha.1 → 3.0.1-alpha.3

package/lib/gen/zappar-cwrap.js

@@ -8,6 +8,18 @@ export function getRuntimeObject(mod) {
     ]);
     let pipeline_create_wrapped = mod.cwrap("zappar_pipeline_create", "number", []);
     let pipeline_destroy_wrapped = mod.cwrap("zappar_pipeline_destroy", null, ["number"]);
+    let pipeline_camera_frame_data_raw_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw", "number", [
+        "number"
+    ]);
+    let pipeline_camera_frame_data_raw_size_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_size", "number", [
+        "number"
+    ]);
+    let pipeline_camera_frame_data_raw_width_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_width", "number", [
+        "number"
+    ]);
+    let pipeline_camera_frame_data_raw_height_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_height", "number", [
+        "number"
+    ]);
     let pipeline_frame_update_wrapped = mod.cwrap("zappar_pipeline_frame_update", null, [
         "number"
     ]);
@@ -220,6 +232,25 @@ export function getRuntimeObject(mod) {
     let world_tracker_reset_wrapped = mod.cwrap("zappar_world_tracker_reset", null, [
         "number"
     ]);
+    let world_tracker_debug_enabled_wrapped = mod.cwrap("zappar_world_tracker_debug_enabled", "number", [
+        "number"
+    ]);
+    let world_tracker_debug_enabled_set_wrapped = mod.cwrap("zappar_world_tracker_debug_enabled_set", null, [
+        "number",
+        "number"
+    ]);
+    let world_tracker_debug_tracks_data_size_wrapped = mod.cwrap("zappar_world_tracker_debug_tracks_data_size", "number", [
+        "number"
+    ]);
+    let world_tracker_debug_tracks_data_wrapped = mod.cwrap("zappar_world_tracker_debug_tracks_data", "number", [
+        "number"
+    ]);
+    let world_tracker_debug_projections_data_size_wrapped = mod.cwrap("zappar_world_tracker_debug_projections_data_size", "number", [
+        "number"
+    ]);
+    let world_tracker_debug_projections_data_wrapped = mod.cwrap("zappar_world_tracker_debug_projections_data", "number", [
+        "number"
+    ]);
     let dataArrayArgLength = 32;
     let dataArrayArg = mod._malloc(dataArrayArgLength);
     let floatDataArrayArgLength = 16 * 4;
@@ -257,6 +288,22 @@ export function getRuntimeObject(mod) {
         pipeline_destroy: () => {
             pipeline_destroy_wrapped();
         },
+        pipeline_camera_frame_data_raw: (o) => {
+            let ret = pipeline_camera_frame_data_raw_wrapped(o);
+            return ret;
+        },
+        pipeline_camera_frame_data_raw_size: (o) => {
+            let ret = pipeline_camera_frame_data_raw_size_wrapped(o);
+            return ret;
+        },
+        pipeline_camera_frame_data_raw_width: (o) => {
+            let ret = pipeline_camera_frame_data_raw_width_wrapped(o);
+            return ret;
+        },
+        pipeline_camera_frame_data_raw_height: (o) => {
+            let ret = pipeline_camera_frame_data_raw_height_wrapped(o);
+            return ret;
+        },
         pipeline_frame_update: (o) => {
             let ret = pipeline_frame_update_wrapped(o);
             return ret;
@@ -679,5 +726,39 @@ export function getRuntimeObject(mod) {
             let ret = world_tracker_reset_wrapped(o);
             return ret;
         },
+        world_tracker_debug_enabled: (o) => {
+            let ret = world_tracker_debug_enabled_wrapped(o);
+            ret = ret === 1;
+            return ret;
+        },
+        world_tracker_debug_enabled_set: (o, debug_enabled) => {
+            let arg_debug_enabled = debug_enabled ? 1 : 0;
+            let ret = world_tracker_debug_enabled_set_wrapped(o, arg_debug_enabled);
+            return ret;
+        },
+        world_tracker_debug_tracks_data_size: (o) => {
+            let ret = world_tracker_debug_tracks_data_size_wrapped(o);
+            return ret;
+        },
+        world_tracker_debug_tracks_data: (o) => {
+            let ret = world_tracker_debug_tracks_data_wrapped(o);
+            let retsize = world_tracker_debug_tracks_data_size_wrapped(o);
+            let ab = new Float32Array(retsize);
+            ab.set(mod.HEAPF32.subarray(ret / 4, retsize + ret / 4));
+            ret = ab;
+            return ret;
+        },
+        world_tracker_debug_projections_data_size: (o) => {
+            let ret = world_tracker_debug_projections_data_size_wrapped(o);
+            return ret;
+        },
+        world_tracker_debug_projections_data: (o) => {
+            let ret = world_tracker_debug_projections_data_wrapped(o);
+            let retsize = world_tracker_debug_projections_data_size_wrapped(o);
+            let ab = new Float32Array(retsize);
+            ab.set(mod.HEAPF32.subarray(ret / 4, retsize + ret / 4));
+            ret = ab;
+            return ret;
+        },
     };
 }

package/lib/gen/zappar-native.d.ts

@@ -102,6 +102,10 @@ export interface zappar_cwrap {
     analytics_project_id_set(id: string, uid: string): void;
     pipeline_create(): zappar_pipeline_t;
     pipeline_destroy(o: zappar_pipeline_t): void;
+    pipeline_camera_frame_data_raw(o: zappar_pipeline_t): number;
+    pipeline_camera_frame_data_raw_size(o: zappar_pipeline_t): number;
+    pipeline_camera_frame_data_raw_width(o: zappar_pipeline_t): number;
+    pipeline_camera_frame_data_raw_height(o: zappar_pipeline_t): number;
     pipeline_frame_update(o: zappar_pipeline_t): void;
     pipeline_frame_number(o: zappar_pipeline_t): number;
     pipeline_camera_model(o: zappar_pipeline_t): Float32Array;
@@ -175,4 +179,10 @@ export interface zappar_cwrap {
     world_tracker_ground_anchor_valid(o: zappar_world_tracker_t): boolean;
     world_tracker_ground_anchor_pose_raw(o: zappar_world_tracker_t): Float32Array;
     world_tracker_reset(o: zappar_world_tracker_t): void;
+    world_tracker_debug_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_debug_enabled_set(o: zappar_world_tracker_t, debug_enabled: boolean): void;
+    world_tracker_debug_tracks_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_debug_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_debug_projections_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_debug_projections_data(o: zappar_world_tracker_t): Float32Array;
 }

package/lib/gen/zappar-server.js

@@ -384,6 +384,14 @@ export class zappar_server {
                     this._impl.world_tracker_reset(obj);
                     break;
                 }
+                case 45: {
+                    let clientId = msg.type();
+                    let obj = this._world_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.world_tracker_debug_enabled_set(obj, msg.bool());
+                    break;
+                }
             }
         });
     }
@@ -598,6 +606,22 @@ export class zappar_server {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_ground_anchor_pose_raw(v));
             });
+            serializer.sendMessage(29, msg => {
+                msg.type(k);
+                msg.int(this._impl.world_tracker_debug_tracks_data_size(v));
+            });
+            serializer.sendMessage(28, msg => {
+                msg.type(k);
+                msg.floatArray(this._impl.world_tracker_debug_tracks_data(v));
+            });
+            serializer.sendMessage(31, msg => {
+                msg.type(k);
+                msg.int(this._impl.world_tracker_debug_projections_data_size(v));
+            });
+            serializer.sendMessage(30, msg => {
+                msg.type(k);
+                msg.floatArray(this._impl.world_tracker_debug_projections_data(v));
+            });
         }
     }
 }

package/lib/gen/zappar.d.ts

@@ -49,6 +49,7 @@ export interface zappar {
     camera_user_facing(indx: number): boolean;
     projection_matrix_from_camera_model(model: Float32Array, renderWidth: number, renderHeight: number): Float32Array;
     projection_matrix_from_camera_model_ext(model: Float32Array, renderWidth: number, renderHeight: number, zNear: number, zFar: number): Float32Array;
+    pose_from_raw(raw_pose: Float32Array, mirror: boolean): Float32Array;
     log_level(): log_level_t;
     log_level_set(level: log_level_t): void;
     permission_request_ui(): void;
@@ -200,4 +201,10 @@ export interface zappar {
     world_tracker_ground_anchor_pose_camera_relative(o: zappar_world_tracker_t, mirror: boolean): Float32Array;
     world_tracker_ground_anchor_pose(o: zappar_world_tracker_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
     world_tracker_reset(o: zappar_world_tracker_t): void;
+    world_tracker_debug_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_debug_enabled_set(o: zappar_world_tracker_t, debug_enabled: boolean): void;
+    world_tracker_debug_tracks_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_debug_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_debug_projections_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_debug_projections_data(o: zappar_world_tracker_t): Float32Array;
 }

package/lib/html-element-source.js

@@ -89,6 +89,7 @@ export class HTMLElementSource extends Source {
             if (info) {
                 let token = pipeline.registerToken(info);
                 pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
+                info.data = undefined;
             }
         }
         catch (ex) {

package/lib/imagetracker.d.ts

@@ -1,5 +1,6 @@
 import { zappar_image_tracker_t, zappar_pipeline_t } from "./gen/zappar";
-import { image_target_type_t, zappar_cwrap } from "./gen/zappar-native";
+import { zappar } from "./gen/zappar-bridge";
+import { image_target_type_t } from "./gen/zappar-native";
 interface PreviewInfo {
     compressed: Uint8Array;
     mimeType: string;
@@ -26,7 +27,7 @@ export declare class ImageTracker {
     private _client;
     private _impl;
     private _targets;
-    static create(pipeline: zappar_pipeline_t, client: zappar_cwrap): zappar_image_tracker_t;
+    static create(pipeline: zappar_pipeline_t, client: zappar): zappar_image_tracker_t;
     static get(p: zappar_image_tracker_t): ImageTracker | undefined;
     private constructor();
     destroy(): void;

package/lib/index.d.ts

@@ -3,4 +3,5 @@ import { Additional } from "./additional";
 import { Options } from "./options";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(opts?: Options): Zappar;
+export { CameraFrameData } from './camera-frame-data';
 export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";

package/lib/native.js

@@ -26,6 +26,7 @@ import { createCameraSource, getCameraSource } from "./camera-source-map";
 import { gfx } from "./gfx";
 import { ImageTracker } from "./imagetracker";
 import { MSTPCameraSource } from "./mstp-camera-source";
+import { drawGrid } from "./drawgrid";
 let client;
 export function initialize(opts) {
     if (client)
@@ -49,7 +50,7 @@ export function initialize(opts) {
             c.impl.analytics_project_id_set(".wiz" + pathParts[1], uid);
     }
     messageManager.onIncomingMessage.bind(msg => {
-        var _a, _b, _c, _d, _e, _f, _g;
+        var _a, _b, _c, _d, _e, _f, _g, _h;
         switch (msg.t) {
             case "zappar":
                 (_a = Pipeline.get(msg.p)) === null || _a === void 0 ? void 0 : _a.pendingMessages.push(msg.d);
@@ -72,6 +73,11 @@ export function initialize(opts) {
                 (_g = (_f = Pipeline.get(msgt.p)) === null || _f === void 0 ? void 0 : _f.imageBitmapFromWorker) === null || _g === void 0 ? void 0 : _g.call(_f, msgt);
                 break;
             }
+            case "raw": {
+                let msgt = msg;
+                (_h = Pipeline.get(msgt.p)) === null || _h === void 0 ? void 0 : _h.cameraFrameDataRawResult(msgt);
+                break;
+            }
             case "streamEndedS2C": {
                 let msgt = msg;
                 const src = getCameraSource(msgt.source);
@@ -132,11 +138,30 @@ export function initialize(opts) {
             }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, raw_pose);
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
+        }, pipeline_camera_frame_data_width: (pipeline) => {
+            var _a, _b;
+            const info = (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo();
+            return (_b = info === null || info === void 0 ? void 0 : info.dataWidth) !== null && _b !== void 0 ? _b : 0;
+        }, pipeline_camera_frame_data_height: (pipeline) => {
+            var _a, _b;
+            const info = (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo();
+            return (_b = info === null || info === void 0 ? void 0 : info.dataHeight) !== null && _b !== void 0 ? _b : 0;
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
             drawPlane(gl, projectionMatrix, cameraMatrix, targetMatrix, texture);
+        }, draw_grid: (gl, projectionMatrix, cameraMatrix, targetMatrix) => {
+            drawGrid(gl, projectionMatrix, cameraMatrix, targetMatrix);
         }, pipeline_draw_face: (p, projectionMatrix, cameraMatrix, targetMatrix, o) => {
             var _a;
             let obj = getFaceMesh(o);
@@ -156,7 +181,7 @@ export function initialize(opts) {
                 return;
             }
             (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, obj);
-        }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
+        }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, pipeline_camera_frame_data_raw: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.getCameraFrameDataRaw()) || Promise.reject(); }, pipeline_camera_frame_data_raw_enabled_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.setCameraFrameDataRawEnabled(v); }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.instant_world_tracker_anchor_pose_raw(o));
             if (mirror) {
                 let scale = mat4.create();

package/lib/pipeline.d.ts

@@ -1,13 +1,14 @@
 /// <reference types="dom-webcodecs" />
 import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
-import { zappar_cwrap } from "./gen/zappar-native";
 import { zappar_client } from "./gen/zappar-client";
 import { MsgManager } from "./messages";
 import { Source, CameraFrameInfo } from "./source";
-import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
+import { CameraDataRawS2C, CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
 import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
 import { ImageTracker } from "./imagetracker";
+import { zappar } from "./gen/zappar-bridge";
+import { CameraFrameData } from "./camera-frame-data";
 export declare class Pipeline {
     private _client;
     private _impl;
@@ -27,12 +28,17 @@ export declare class Pipeline {
     private _sequenceRecorder;
     private _sequenceRecordDeviceAttitudeMatrices;
     private _sequenceRecorderFirstCameraToken;
+    private _cameraFrameDataPromises;
+    private _cameraFrameDataResolves;
+    private _cameraFrameDataRejects;
+    private _cameraFrameDataEnabled;
     onGLContextReset: Event;
-    static create(client: zappar_cwrap, mgr: MsgManager): zappar_pipeline_t;
+    static create(client: zappar, mgr: MsgManager): zappar_pipeline_t;
     static get(p: zappar_pipeline_t): Pipeline | undefined;
     private constructor();
     frameUpdate(client: zappar_client): void;
     cleanOldFrames(): void;
+    cameraFrameDataRawResult(msg: CameraDataRawS2C): void;
     cameraTokenReturn(msg: CameraFrameReturnS2C): void;
     sequenceRecordStart(expectedFrames: number): void;
     sequenceRecordStop(): void;
@@ -40,6 +46,8 @@ export declare class Pipeline {
     sequenceRecordClear(): void;
     sequenceRecordDeviceAttitudeMatrices(v: boolean): void;
     getVideoTexture(): WebGLTexture | undefined;
+    getCameraFrameDataRaw(): Promise<CameraFrameData | undefined>;
+    setCameraFrameDataRawEnabled(e: boolean): void;
     destroy(): void;
     getCurrentCameraInfo(): CameraFrameInfo | undefined;
     cameraFrameDrawGL(screenWidth: number, screenHeight: number, mirror?: boolean): void;

package/lib/pipeline.js

@@ -24,6 +24,10 @@ export class Pipeline {
         this.cameraPixelArrays = [];
         this._sequenceRecordDeviceAttitudeMatrices = true;
         this._sequenceRecorderFirstCameraToken = 0;
+        this._cameraFrameDataPromises = new Map();
+        this._cameraFrameDataResolves = new Map();
+        this._cameraFrameDataRejects = new Map();
+        this._cameraFrameDataEnabled = false;
         this.onGLContextReset = new Event();
     }
     static create(client, mgr) {
@@ -56,14 +60,34 @@ export class Pipeline {
                 if (t[1].texture)
                     this.videoTextures.push(t[1].texture);
                 (_b = (_a = t[1].frame) === null || _a === void 0 ? void 0 : _a.close) === null || _b === void 0 ? void 0 : _b.call(_a);
+                if (t[1].data) {
+                    this.cameraPixelArrays.push(t[1].data);
+                    t[1].data = undefined;
+                }
                 this.cameraTokens.delete(t[0]);
             }
         }
     }
+    cameraFrameDataRawResult(msg) {
+        var _a, _b;
+        let info = this.cameraTokens.get(msg.token);
+        if (!info)
+            return;
+        if (msg.data === null) {
+            (_a = this._cameraFrameDataRejects.get(info)) === null || _a === void 0 ? void 0 : _a(new Error('Raw camera data not available'));
+        }
+        else {
+            info.data = msg.data.data;
+            (_b = this._cameraFrameDataResolves.get(info)) === null || _b === void 0 ? void 0 : _b(msg.data);
+        }
+        this._cameraFrameDataRejects.delete(info);
+        this._cameraFrameDataResolves.delete(info);
+        this._cameraFrameDataPromises.delete(info);
+    }
     cameraTokenReturn(msg) {
+        let info = this.cameraTokens.get(msg.token);
         if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices
             && msg.token >= this._sequenceRecorderFirstCameraToken) {
-            let info = this.cameraTokens.get(msg.token);
             if (info) {
                 if (msg.att)
                     this._sequenceRecorder.appendAttitudeMatrix(msg.att);
@@ -71,7 +95,20 @@ export class Pipeline {
                 this._sequenceRecorder.appendCameraFrame(info);
             }
         }
-        this.cameraPixelArrays.push(msg.d);
+        if (info && !info.data && msg.d) {
+            info.data = msg.d;
+            const rawRequestResolve = this._cameraFrameDataResolves.get(info);
+            if (rawRequestResolve) {
+                rawRequestResolve({
+                    data: info.data,
+                    width: info.dataWidth,
+                    height: info.dataHeight,
+                });
+            }
+            this._cameraFrameDataRejects.delete(info);
+            this._cameraFrameDataResolves.delete(info);
+            this._cameraFrameDataPromises.delete(info);
+        }
         this.tokensInFlight--;
     }
     sequenceRecordStart(expectedFrames) {
@@ -97,6 +134,37 @@ export class Pipeline {
     getVideoTexture() {
         return this.videoTextures.pop();
     }
+    getCameraFrameDataRaw() {
+        if (!this._cameraFrameDataEnabled)
+            throw new Error('Raw camera frame data is not enabled.');
+        const info = this.getCurrentCameraInfo();
+        if (!info)
+            return Promise.resolve(undefined);
+        if (info.data)
+            return Promise.resolve({ data: info.data.slice(0), width: info.dataWidth, height: info.dataHeight });
+        let existing = this._cameraFrameDataPromises.get(info);
+        if (!existing) {
+            existing = new Promise((resolve, reject) => {
+                if (typeof info.remoteToken !== 'undefined') {
+                    const msg = {
+                        t: "rawrequest",
+                        token: info.remoteToken,
+                        p: this._impl,
+                    };
+                    this._mgr.postOutgoingMessage(msg, []);
+                }
+                this._cameraFrameDataResolves.set(info, resolve);
+                this._cameraFrameDataRejects.set(info, reject);
+            });
+            this._cameraFrameDataPromises.set(info, existing);
+        }
+        return existing;
+    }
+    setCameraFrameDataRawEnabled(e) {
+        this._cameraFrameDataEnabled = e;
+        const msg = { t: 'rawenabled', v: e };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
     destroy() {
         this._client.pipeline_destroy(this._impl);
         byId.delete(this._impl);
@@ -226,7 +294,8 @@ export class Pipeline {
             uvTransform: msg.uvTransform,
             cameraModel: msg.cameraModel,
             cameraToDevice: msg.cameraToDevice,
-            cameraSource
+            cameraSource,
+            remoteToken: msg.token,
         });
         this.cleanOldFrames();
     }

package/lib/serializer.d.ts

@@ -8,6 +8,7 @@ export interface MessageAppender {
     type: (t: number) => void;
     matrix4x4: (a: Float32Array) => void;
     matrix3x3: (a: Float32Array) => void;
+    floatArray: (a: Float32Array) => void;
     cameraModel: (a: Float32Array) => void;
     identityCoefficients: (a: Float32Array) => void;
     expressionCoefficients: (a: Float32Array) => void;

package/lib/serializer.js

@@ -22,6 +22,7 @@ export class MessageSerializer {
             type: i => this.int(i),
             matrix4x4: i => this.float32ArrayBuffer(i),
             matrix3x3: i => this.float32ArrayBuffer(i),
+            floatArray: i => this.float32ArrayBuffer(i),
             identityCoefficients: i => this.float32ArrayBuffer(i),
             expressionCoefficients: i => this.float32ArrayBuffer(i),
             cameraModel: i => this.float32ArrayBuffer(i),

package/lib/source.d.ts

@@ -10,6 +10,7 @@ export interface CameraFrameInfo {
     cameraSourceData?: any;
     uvTransform?: Float32Array;
     userFacing: boolean;
+    remoteToken?: number;
     frame?: VideoFrame | ImageBitmap;
 }
 export declare abstract class Source {

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.1-alpha.1";
+export declare const VERSION = "3.0.1-alpha.3";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.1-alpha.1";
+export const VERSION = "3.0.1-alpha.3";

package/lib/worker-server.js

@@ -19,6 +19,8 @@ import { getWorkerMessageAPI } from "./worker-messages";
 export let messageManager = new MsgManager();
 let workerMessageChannel;
 let latestCameraToScreenRotation = 0;
+let rawDataAccessEnabled = false;
+const rawCameraFrameBuffersByPipeline = new Map();
 export function launchWorkerServer(wasmUrl, module) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
@@ -45,7 +47,7 @@ export function launchWorkerServer(wasmUrl, module) {
                 });
                 messageManager.postOutgoingMessage("loaded", []);
                 messageManager.onIncomingMessage.bind((msg) => {
-                    var _a;
+                    var _a, _b, _c;
                     switch (msg.t) {
                         case "zappar":
                             server.processBuffer(msg.d);
@@ -75,6 +77,23 @@ export function launchWorkerServer(wasmUrl, module) {
                             messageManager.postOutgoingMessage(ret, [msgt.d]);
                             break;
                         }
+                        case "rawenabled": {
+                            const msgt = msg;
+                            rawDataAccessEnabled = msgt.v;
+                            break;
+                        }
+                        case "rawrequest": {
+                            const msgt = msg;
+                            const buffers = rawCameraFrameBuffersByPipeline.get(msgt.p);
+                            const ret = {
+                                t: "raw",
+                                token: msgt.token,
+                                p: msgt.p,
+                                data: buffers ? (_c = (_b = buffers.ready.find(entry => entry.token === msgt.token)) === null || _b === void 0 ? void 0 : _b.data) !== null && _c !== void 0 ? _c : null : null,
+                            };
+                            messageManager.postOutgoingMessage(ret, []);
+                            break;
+                        }
                         case "streamC2S": {
                             let msgt = msg;
                             consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source, workerMessageAPI).then(() => {
@@ -243,8 +262,8 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                 t: "videoFrameS2C",
                 userFacing,
                 uvTransform,
-                w: dataWidth,
-                h: dataHeight,
+                w: profile.dataWidth,
+                h: profile.dataHeight,
                 cameraToDevice: cameraToDeviceTransform,
                 cameraModel,
                 source
@@ -263,6 +282,32 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                     // (mod as any).ccall('what_to_stderr', 'number', ['number'], [err]);
                 }
                 r.pipeline_frame_update(pipeline);
+                if (rawDataAccessEnabled) {
+                    let buffers = rawCameraFrameBuffersByPipeline.get(p);
+                    if (!buffers) {
+                        buffers = { available: [], ready: [] };
+                        rawCameraFrameBuffersByPipeline.set(p, buffers);
+                    }
+                    if (buffers.ready.length > 4) {
+                        const removed = buffers.ready.splice(0, 1);
+                        for (const entry of removed)
+                            buffers.available.push(new Uint8Array(entry.data.data));
+                    }
+                    const size = r.pipeline_camera_frame_data_raw_size(pipeline);
+                    let buffer;
+                    while (!buffer || buffer.byteLength < size) {
+                        if (buffers.available.length < 1)
+                            buffers.available.push(new Uint8Array(size));
+                        buffer = buffers.available.pop();
+                    }
+                    const dataPointer = r.pipeline_camera_frame_data_raw(pipeline);
+                    buffer.set(mod.HEAPU8.subarray(dataPointer, dataPointer + size));
+                    buffers.ready.push({ token, data: {
+                            data: buffer,
+                            width: r.pipeline_camera_frame_data_raw_width(p),
+                            height: r.pipeline_camera_frame_data_raw_height(p),
+                        } });
+                }
                 server.exploreState();
             }
             frame.close();

package/lib/workerinterface.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="dom-webcodecs" />
+import { CameraFrameData } from "./camera-frame-data";
 import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
 export interface CameraFrameC2S {
     t: "cameraFrameC2S";
@@ -83,3 +84,18 @@ export interface AttitudeMatrixC2S {
     p: zappar_pipeline_t;
     m: Float32Array;
 }
+export interface CameraDataRawEnabledC2S {
+    t: "rawenabled";
+    v: boolean;
+}
+export interface CameraDataRawRequestC2S {
+    t: "rawrequest";
+    p: zappar_pipeline_t;
+    token: number;
+}
+export interface CameraDataRawS2C {
+    t: "raw";
+    token: number;
+    p: zappar_pipeline_t;
+    data: CameraFrameData | null;
+}