@zappar/zappar-cv 3.0.1-alpha.1 → 3.0.1-alpha.2

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/3.0.1-alpha.1/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/3.0.1-alpha.2/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/3.0.1-alpha.1/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.1-alpha.1/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/3.0.1-alpha.2/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.1-alpha.2/zappar-cv.zip)

package/lib/additional.d.ts

@@ -1,3 +1,4 @@
+import { CameraFrameData } from "./camera-frame-data";
 import { zappar_face_mesh_t, zappar_face_tracker_t, zappar_pipeline_t, zappar_image_tracker_t, zappar_sequence_source_t } from "./gen/zappar";
 export declare type zappar_html_element_source_t = number & {
     _: 'zappar_html_element_source_t';
@@ -8,6 +9,8 @@ export interface Additional {
     pipeline_draw_face(pipeline: zappar_pipeline_t, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: zappar_face_mesh_t): void;
     pipeline_draw_image_target_preview(pipeline: zappar_pipeline_t, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: zappar_image_tracker_t, indx: number): void;
     pipeline_draw_face_project(pipeline: zappar_pipeline_t, drawwMatrix: Float32Array, vertices: Float32Array, uvMatrix: Float32Array, uvs: Float32Array, indices: Uint16Array, texture: WebGLTexture): void;
+    pipeline_camera_frame_data_raw_enabled_set(pipeline: zappar_pipeline_t, enabled: boolean): void;
+    pipeline_camera_frame_data_raw(pipeline: zappar_pipeline_t): Promise<CameraFrameData | undefined>;
     draw_plane(gl: WebGLRenderingContext, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, texture: string): void;
     image_tracker_target_image(o: zappar_image_tracker_t, indx: number): HTMLImageElement | undefined;
     face_mesh_load_default(o: zappar_face_mesh_t): Promise<void>;

package/lib/camera-frame-data.d.ts

@@ -0,0 +1,5 @@
+export interface CameraFrameData {
+    width: number;
+    height: number;
+    data: ArrayBuffer;
+}

package/lib/camera-frame-data.js

@@ -0,0 +1 @@
+export {};

package/lib/gen/zappar-bridge.d.ts

@@ -0,0 +1,120 @@
+import { barcode_format_t } from "./zappar-native";
+export { barcode_format_t } from "./zappar-native";
+import { face_landmark_name_t } from "./zappar-native";
+export { face_landmark_name_t } from "./zappar-native";
+export { frame_pixel_format_t } from "./zappar-native";
+import { instant_world_tracker_transform_orientation_t } from "./zappar-native";
+export { instant_world_tracker_transform_orientation_t } from "./zappar-native";
+import { log_level_t } from "./zappar-native";
+export { log_level_t } from "./zappar-native";
+export { image_target_type_t } from "./zappar-native";
+export { world_tracker_quality_t } from "./zappar-native";
+export declare type zappar_pipeline_t = number & {
+    _: 'zappar_pipeline_t';
+};
+export declare type zappar_camera_source_t = number & {
+    _: 'zappar_camera_source_t';
+};
+export declare type zappar_sequence_source_t = number & {
+    _: 'zappar_sequence_source_t';
+};
+export declare type zappar_image_tracker_t = number & {
+    _: 'zappar_image_tracker_t';
+};
+export declare type zappar_face_tracker_t = number & {
+    _: 'zappar_face_tracker_t';
+};
+export declare type zappar_face_mesh_t = number & {
+    _: 'zappar_face_mesh_t';
+};
+export declare type zappar_face_landmark_t = number & {
+    _: 'zappar_face_landmark_t';
+};
+export declare type zappar_barcode_finder_t = number & {
+    _: 'zappar_barcode_finder_t';
+};
+export declare type zappar_instant_world_tracker_t = number & {
+    _: 'zappar_instant_world_tracker_t';
+};
+export declare type zappar_world_tracker_t = number & {
+    _: 'zappar_world_tracker_t';
+};
+export interface zappar {
+    log_level(): log_level_t;
+    log_level_set(level: log_level_t): void;
+    analytics_project_id_set(id: string, uid: string): void;
+    pipeline_create(): zappar_pipeline_t;
+    pipeline_destroy(o: zappar_pipeline_t): void;
+    pipeline_frame_update(o: zappar_pipeline_t): void;
+    pipeline_frame_number(o: zappar_pipeline_t): number;
+    pipeline_camera_model(o: zappar_pipeline_t): Float32Array;
+    pipeline_camera_frame_user_data(o: zappar_pipeline_t): number;
+    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean): void;
+    pipeline_camera_frame_camera_attitude(o: zappar_pipeline_t): Float32Array;
+    pipeline_camera_frame_device_attitude(o: zappar_pipeline_t): Float32Array;
+    pipeline_motion_accelerometer_submit(o: zappar_pipeline_t, time: number, x: number, y: number, z: number): void;
+    pipeline_motion_accelerometer_with_gravity_submit_int(o: zappar_pipeline_t, time: number, interval: number, x: number, y: number, z: number): void;
+    pipeline_motion_accelerometer_without_gravity_submit_int(o: zappar_pipeline_t, time: number, interval: number, x: number, y: number, z: number): void;
+    pipeline_motion_rotation_rate_submit(o: zappar_pipeline_t, time: number, x: number, y: number, z: number): void;
+    pipeline_motion_rotation_rate_submit_int(o: zappar_pipeline_t, time: number, interval: number, x: number, y: number, z: number): void;
+    pipeline_motion_attitude_submit(o: zappar_pipeline_t, time: number, x: number, y: number, z: number): void;
+    pipeline_motion_attitude_submit_int(o: zappar_pipeline_t, time: number, interval: number, x: number, y: number, z: number): void;
+    pipeline_motion_attitude_matrix_submit(o: zappar_pipeline_t, mat: Float32Array): void;
+    camera_source_create(pipeline: zappar_pipeline_t, device_id: string): zappar_camera_source_t;
+    camera_source_destroy(o: zappar_camera_source_t): void;
+    sequence_source_create(pipeline: zappar_pipeline_t): zappar_sequence_source_t;
+    sequence_source_destroy(o: zappar_sequence_source_t): void;
+    image_tracker_create(pipeline: zappar_pipeline_t): zappar_image_tracker_t;
+    image_tracker_destroy(o: zappar_image_tracker_t): void;
+    image_tracker_target_load_from_memory(o: zappar_image_tracker_t, data: ArrayBuffer): void;
+    image_tracker_target_loaded_version(o: zappar_image_tracker_t): number;
+    image_tracker_enabled(o: zappar_image_tracker_t): boolean;
+    image_tracker_enabled_set(o: zappar_image_tracker_t, enabled: boolean): void;
+    image_tracker_anchor_count(o: zappar_image_tracker_t): number;
+    image_tracker_anchor_id(o: zappar_image_tracker_t, indx: number): string;
+    image_tracker_anchor_pose_raw(o: zappar_image_tracker_t, indx: number): Float32Array;
+    face_tracker_create(pipeline: zappar_pipeline_t): zappar_face_tracker_t;
+    face_tracker_destroy(o: zappar_face_tracker_t): void;
+    face_tracker_model_load_from_memory(o: zappar_face_tracker_t, data: ArrayBuffer): void;
+    face_tracker_model_loaded_version(o: zappar_face_tracker_t): number;
+    face_tracker_enabled_set(o: zappar_face_tracker_t, enabled: boolean): void;
+    face_tracker_enabled(o: zappar_face_tracker_t): boolean;
+    face_tracker_max_faces_set(o: zappar_face_tracker_t, num: number): void;
+    face_tracker_max_faces(o: zappar_face_tracker_t): number;
+    face_tracker_anchor_count(o: zappar_face_tracker_t): number;
+    face_tracker_anchor_id(o: zappar_face_tracker_t, indx: number): string;
+    face_tracker_anchor_pose_raw(o: zappar_face_tracker_t, indx: number): Float32Array;
+    face_tracker_anchor_identity_coefficients(o: zappar_face_tracker_t, indx: number): Float32Array;
+    face_tracker_anchor_expression_coefficients(o: zappar_face_tracker_t, indx: number): Float32Array;
+    face_mesh_create(): zappar_face_mesh_t;
+    face_mesh_destroy(o: zappar_face_mesh_t): void;
+    face_landmark_create(landmark: face_landmark_name_t): zappar_face_landmark_t;
+    face_landmark_destroy(o: zappar_face_landmark_t): void;
+    barcode_finder_create(pipeline: zappar_pipeline_t): zappar_barcode_finder_t;
+    barcode_finder_destroy(o: zappar_barcode_finder_t): void;
+    barcode_finder_enabled_set(o: zappar_barcode_finder_t, enabled: boolean): void;
+    barcode_finder_enabled(o: zappar_barcode_finder_t): boolean;
+    barcode_finder_found_number(o: zappar_barcode_finder_t): number;
+    barcode_finder_found_text(o: zappar_barcode_finder_t, indx: number): string;
+    barcode_finder_found_format(o: zappar_barcode_finder_t, indx: number): barcode_format_t;
+    barcode_finder_formats(o: zappar_barcode_finder_t): barcode_format_t;
+    barcode_finder_formats_set(o: zappar_barcode_finder_t, f: barcode_format_t): void;
+    instant_world_tracker_create(pipeline: zappar_pipeline_t): zappar_instant_world_tracker_t;
+    instant_world_tracker_destroy(o: zappar_instant_world_tracker_t): void;
+    instant_world_tracker_enabled_set(o: zappar_instant_world_tracker_t, enabled: boolean): void;
+    instant_world_tracker_enabled(o: zappar_instant_world_tracker_t): boolean;
+    instant_world_tracker_anchor_pose_raw(o: zappar_instant_world_tracker_t): Float32Array;
+    instant_world_tracker_anchor_pose_set_from_camera_offset_raw(o: zappar_instant_world_tracker_t, x: number, y: number, z: number, orientation: instant_world_tracker_transform_orientation_t): void;
+    world_tracker_create(pipeline: zappar_pipeline_t): zappar_world_tracker_t;
+    world_tracker_destroy(o: zappar_world_tracker_t): void;
+    world_tracker_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_enabled_set(o: zappar_world_tracker_t, enabled: boolean): void;
+    world_tracker_quality(o: zappar_world_tracker_t): number;
+    world_tracker_plane_count(o: zappar_world_tracker_t): number;
+    world_tracker_plane_pose_raw(o: zappar_world_tracker_t, indx: number): Float32Array;
+    world_tracker_world_anchor_valid(o: zappar_world_tracker_t): boolean;
+    world_tracker_world_anchor_pose_raw(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_ground_anchor_valid(o: zappar_world_tracker_t): boolean;
+    world_tracker_ground_anchor_pose_raw(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_reset(o: zappar_world_tracker_t): void;
+}

package/lib/gen/zappar-bridge.js

@@ -0,0 +1,7 @@
+export { barcode_format_t } from "./zappar-native";
+export { face_landmark_name_t } from "./zappar-native";
+export { frame_pixel_format_t } from "./zappar-native";
+export { instant_world_tracker_transform_orientation_t } from "./zappar-native";
+export { log_level_t } from "./zappar-native";
+export { image_target_type_t } from "./zappar-native";
+export { world_tracker_quality_t } from "./zappar-native";

package/lib/gen/zappar-client.d.ts

@@ -1,6 +1,6 @@
 import { MessageSerializer } from "../serializer";
 import { MessageDeserializer } from "../deserializer";
-import { zappar_cwrap } from "./zappar-native";
+import { zappar } from "./zappar-bridge";
 export declare class zappar_client {
     private _messageSender;
     constructor(_messageSender: (ab: ArrayBuffer) => void);
@@ -18,6 +18,6 @@ export declare class zappar_client {
     private _barcode_finder_state_by_instance;
     private _instant_world_tracker_state_by_instance;
     private _world_tracker_state_by_instance;
-    impl: zappar_cwrap;
+    impl: zappar;
     processMessages(a: ArrayBuffer): void;
 }

package/lib/gen/zappar-client.js

@@ -103,11 +103,6 @@ export class zappar_client {
                     m.bool(user_facing);
                 });
             },
-            pipeline_camera_frame_submit_raw_pointer: (o, data, dataLength, format, width, height, user_data, camera_to_device_transform, rotation, camera_model, user_facing, timestamp) => {
-                let s = this._pipeline_state_by_instance.get(o);
-                if (!s)
-                    throw new Error("This object has been destroyed");
-            },
             pipeline_camera_frame_camera_attitude: (o) => {
                 let s = this._pipeline_state_by_instance.get(o);
                 if (!s)

package/lib/gen/zappar-cwrap.js

@@ -8,6 +8,18 @@ export function getRuntimeObject(mod) {
     ]);
     let pipeline_create_wrapped = mod.cwrap("zappar_pipeline_create", "number", []);
     let pipeline_destroy_wrapped = mod.cwrap("zappar_pipeline_destroy", null, ["number"]);
+    let pipeline_camera_frame_data_raw_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw", "number", [
+        "number"
+    ]);
+    let pipeline_camera_frame_data_raw_size_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_size", "number", [
+        "number"
+    ]);
+    let pipeline_camera_frame_data_raw_width_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_width", "number", [
+        "number"
+    ]);
+    let pipeline_camera_frame_data_raw_height_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_height", "number", [
+        "number"
+    ]);
     let pipeline_frame_update_wrapped = mod.cwrap("zappar_pipeline_frame_update", null, [
         "number"
     ]);
@@ -257,6 +269,22 @@ export function getRuntimeObject(mod) {
         pipeline_destroy: () => {
             pipeline_destroy_wrapped();
         },
+        pipeline_camera_frame_data_raw: (o) => {
+            let ret = pipeline_camera_frame_data_raw_wrapped(o);
+            return ret;
+        },
+        pipeline_camera_frame_data_raw_size: (o) => {
+            let ret = pipeline_camera_frame_data_raw_size_wrapped(o);
+            return ret;
+        },
+        pipeline_camera_frame_data_raw_width: (o) => {
+            let ret = pipeline_camera_frame_data_raw_width_wrapped(o);
+            return ret;
+        },
+        pipeline_camera_frame_data_raw_height: (o) => {
+            let ret = pipeline_camera_frame_data_raw_height_wrapped(o);
+            return ret;
+        },
         pipeline_frame_update: (o) => {
             let ret = pipeline_frame_update_wrapped(o);
             return ret;

package/lib/gen/zappar-native.d.ts

@@ -102,6 +102,10 @@ export interface zappar_cwrap {
     analytics_project_id_set(id: string, uid: string): void;
     pipeline_create(): zappar_pipeline_t;
     pipeline_destroy(o: zappar_pipeline_t): void;
+    pipeline_camera_frame_data_raw(o: zappar_pipeline_t): number;
+    pipeline_camera_frame_data_raw_size(o: zappar_pipeline_t): number;
+    pipeline_camera_frame_data_raw_width(o: zappar_pipeline_t): number;
+    pipeline_camera_frame_data_raw_height(o: zappar_pipeline_t): number;
     pipeline_frame_update(o: zappar_pipeline_t): void;
     pipeline_frame_number(o: zappar_pipeline_t): number;
     pipeline_camera_model(o: zappar_pipeline_t): Float32Array;

package/lib/gen/zappar.d.ts

@@ -49,6 +49,7 @@ export interface zappar {
     camera_user_facing(indx: number): boolean;
     projection_matrix_from_camera_model(model: Float32Array, renderWidth: number, renderHeight: number): Float32Array;
     projection_matrix_from_camera_model_ext(model: Float32Array, renderWidth: number, renderHeight: number, zNear: number, zFar: number): Float32Array;
+    pose_from_raw(raw_pose: Float32Array, mirror: boolean): Float32Array;
     log_level(): log_level_t;
     log_level_set(level: log_level_t): void;
     permission_request_ui(): void;

package/lib/html-element-source.js

@@ -89,6 +89,7 @@ export class HTMLElementSource extends Source {
             if (info) {
                 let token = pipeline.registerToken(info);
                 pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
+                info.data = undefined;
             }
         }
         catch (ex) {

package/lib/imagetracker.d.ts

@@ -1,5 +1,6 @@
 import { zappar_image_tracker_t, zappar_pipeline_t } from "./gen/zappar";
-import { image_target_type_t, zappar_cwrap } from "./gen/zappar-native";
+import { zappar } from "./gen/zappar-bridge";
+import { image_target_type_t } from "./gen/zappar-native";
 interface PreviewInfo {
     compressed: Uint8Array;
     mimeType: string;
@@ -26,7 +27,7 @@ export declare class ImageTracker {
     private _client;
     private _impl;
     private _targets;
-    static create(pipeline: zappar_pipeline_t, client: zappar_cwrap): zappar_image_tracker_t;
+    static create(pipeline: zappar_pipeline_t, client: zappar): zappar_image_tracker_t;
     static get(p: zappar_image_tracker_t): ImageTracker | undefined;
     private constructor();
     destroy(): void;

package/lib/index.d.ts

@@ -3,4 +3,5 @@ import { Additional } from "./additional";
 import { Options } from "./options";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(opts?: Options): Zappar;
+export { CameraFrameData } from './camera-frame-data';
 export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";

package/lib/native.js

@@ -49,7 +49,7 @@ export function initialize(opts) {
             c.impl.analytics_project_id_set(".wiz" + pathParts[1], uid);
     }
     messageManager.onIncomingMessage.bind(msg => {
-        var _a, _b, _c, _d, _e, _f, _g;
+        var _a, _b, _c, _d, _e, _f, _g, _h;
         switch (msg.t) {
             case "zappar":
                 (_a = Pipeline.get(msg.p)) === null || _a === void 0 ? void 0 : _a.pendingMessages.push(msg.d);
@@ -72,6 +72,11 @@ export function initialize(opts) {
                 (_g = (_f = Pipeline.get(msgt.p)) === null || _f === void 0 ? void 0 : _f.imageBitmapFromWorker) === null || _g === void 0 ? void 0 : _g.call(_f, msgt);
                 break;
             }
+            case "raw": {
+                let msgt = msg;
+                (_h = Pipeline.get(msgt.p)) === null || _h === void 0 ? void 0 : _h.cameraFrameDataRawResult(msgt);
+                break;
+            }
             case "streamEndedS2C": {
                 let msgt = msg;
                 const src = getCameraSource(msgt.source);
@@ -132,7 +137,16 @@ export function initialize(opts) {
             }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, raw_pose);
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
@@ -156,7 +170,7 @@ export function initialize(opts) {
                 return;
             }
             (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, obj);
-        }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
+        }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, pipeline_camera_frame_data_raw: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.getCameraFrameDataRaw()) || Promise.reject(); }, pipeline_camera_frame_data_raw_enabled_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.setCameraFrameDataRawEnabled(v); }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.instant_world_tracker_anchor_pose_raw(o));
             if (mirror) {
                 let scale = mat4.create();

package/lib/pipeline.d.ts

@@ -1,13 +1,14 @@
 /// <reference types="dom-webcodecs" />
 import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
-import { zappar_cwrap } from "./gen/zappar-native";
 import { zappar_client } from "./gen/zappar-client";
 import { MsgManager } from "./messages";
 import { Source, CameraFrameInfo } from "./source";
-import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
+import { CameraDataRawS2C, CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
 import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
 import { ImageTracker } from "./imagetracker";
+import { zappar } from "./gen/zappar-bridge";
+import { CameraFrameData } from "./camera-frame-data";
 export declare class Pipeline {
     private _client;
     private _impl;
@@ -27,12 +28,17 @@ export declare class Pipeline {
     private _sequenceRecorder;
     private _sequenceRecordDeviceAttitudeMatrices;
     private _sequenceRecorderFirstCameraToken;
+    private _cameraFrameDataPromises;
+    private _cameraFrameDataResolves;
+    private _cameraFrameDataRejects;
+    private _cameraFrameDataEnabled;
     onGLContextReset: Event;
-    static create(client: zappar_cwrap, mgr: MsgManager): zappar_pipeline_t;
+    static create(client: zappar, mgr: MsgManager): zappar_pipeline_t;
     static get(p: zappar_pipeline_t): Pipeline | undefined;
     private constructor();
     frameUpdate(client: zappar_client): void;
     cleanOldFrames(): void;
+    cameraFrameDataRawResult(msg: CameraDataRawS2C): void;
     cameraTokenReturn(msg: CameraFrameReturnS2C): void;
     sequenceRecordStart(expectedFrames: number): void;
     sequenceRecordStop(): void;
@@ -40,6 +46,8 @@ export declare class Pipeline {
     sequenceRecordClear(): void;
     sequenceRecordDeviceAttitudeMatrices(v: boolean): void;
     getVideoTexture(): WebGLTexture | undefined;
+    getCameraFrameDataRaw(): Promise<CameraFrameData | undefined>;
+    setCameraFrameDataRawEnabled(e: boolean): void;
     destroy(): void;
     getCurrentCameraInfo(): CameraFrameInfo | undefined;
     cameraFrameDrawGL(screenWidth: number, screenHeight: number, mirror?: boolean): void;

package/lib/pipeline.js

@@ -24,6 +24,10 @@ export class Pipeline {
         this.cameraPixelArrays = [];
         this._sequenceRecordDeviceAttitudeMatrices = true;
         this._sequenceRecorderFirstCameraToken = 0;
+        this._cameraFrameDataPromises = new Map();
+        this._cameraFrameDataResolves = new Map();
+        this._cameraFrameDataRejects = new Map();
+        this._cameraFrameDataEnabled = false;
         this.onGLContextReset = new Event();
     }
     static create(client, mgr) {
@@ -56,14 +60,34 @@ export class Pipeline {
                 if (t[1].texture)
                     this.videoTextures.push(t[1].texture);
                 (_b = (_a = t[1].frame) === null || _a === void 0 ? void 0 : _a.close) === null || _b === void 0 ? void 0 : _b.call(_a);
+                if (t[1].data) {
+                    this.cameraPixelArrays.push(t[1].data);
+                    t[1].data = undefined;
+                }
                 this.cameraTokens.delete(t[0]);
             }
         }
     }
+    cameraFrameDataRawResult(msg) {
+        var _a, _b;
+        let info = this.cameraTokens.get(msg.token);
+        if (!info)
+            return;
+        if (msg.data === null) {
+            (_a = this._cameraFrameDataRejects.get(info)) === null || _a === void 0 ? void 0 : _a(new Error('Raw camera data not available'));
+        }
+        else {
+            info.data = msg.data.data;
+            (_b = this._cameraFrameDataResolves.get(info)) === null || _b === void 0 ? void 0 : _b(msg.data);
+        }
+        this._cameraFrameDataRejects.delete(info);
+        this._cameraFrameDataResolves.delete(info);
+        this._cameraFrameDataPromises.delete(info);
+    }
     cameraTokenReturn(msg) {
+        let info = this.cameraTokens.get(msg.token);
         if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices
             && msg.token >= this._sequenceRecorderFirstCameraToken) {
-            let info = this.cameraTokens.get(msg.token);
             if (info) {
                 if (msg.att)
                     this._sequenceRecorder.appendAttitudeMatrix(msg.att);
@@ -71,7 +95,20 @@ export class Pipeline {
                 this._sequenceRecorder.appendCameraFrame(info);
             }
         }
-        this.cameraPixelArrays.push(msg.d);
+        if (info && !info.data && msg.d) {
+            info.data = msg.d;
+            const rawRequestResolve = this._cameraFrameDataResolves.get(info);
+            if (rawRequestResolve) {
+                rawRequestResolve({
+                    data: info.data,
+                    width: info.dataWidth,
+                    height: info.dataHeight,
+                });
+            }
+            this._cameraFrameDataRejects.delete(info);
+            this._cameraFrameDataResolves.delete(info);
+            this._cameraFrameDataPromises.delete(info);
+        }
         this.tokensInFlight--;
     }
     sequenceRecordStart(expectedFrames) {
@@ -97,6 +134,37 @@ export class Pipeline {
     getVideoTexture() {
         return this.videoTextures.pop();
     }
+    getCameraFrameDataRaw() {
+        if (!this._cameraFrameDataEnabled)
+            throw new Error('Raw camera frame data is not enabled.');
+        const info = this.getCurrentCameraInfo();
+        if (!info)
+            return Promise.resolve(undefined);
+        if (info.data)
+            return Promise.resolve({ data: info.data.slice(0), width: info.dataWidth, height: info.dataHeight });
+        let existing = this._cameraFrameDataPromises.get(info);
+        if (!existing) {
+            existing = new Promise((resolve, reject) => {
+                if (typeof info.remoteToken !== 'undefined') {
+                    const msg = {
+                        t: "rawrequest",
+                        token: info.remoteToken,
+                        p: this._impl,
+                    };
+                    this._mgr.postOutgoingMessage(msg, []);
+                }
+                this._cameraFrameDataResolves.set(info, resolve);
+                this._cameraFrameDataRejects.set(info, reject);
+            });
+            this._cameraFrameDataPromises.set(info, existing);
+        }
+        return existing;
+    }
+    setCameraFrameDataRawEnabled(e) {
+        this._cameraFrameDataEnabled = e;
+        const msg = { t: 'rawenabled', v: e };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
     destroy() {
         this._client.pipeline_destroy(this._impl);
         byId.delete(this._impl);
@@ -226,7 +294,8 @@ export class Pipeline {
             uvTransform: msg.uvTransform,
             cameraModel: msg.cameraModel,
             cameraToDevice: msg.cameraToDevice,
-            cameraSource
+            cameraSource,
+            remoteToken: msg.token,
         });
         this.cleanOldFrames();
     }

package/lib/source.d.ts

@@ -10,6 +10,7 @@ export interface CameraFrameInfo {
     cameraSourceData?: any;
     uvTransform?: Float32Array;
     userFacing: boolean;
+    remoteToken?: number;
     frame?: VideoFrame | ImageBitmap;
 }
 export declare abstract class Source {

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.1-alpha.1";
+export declare const VERSION = "3.0.1-alpha.2";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.1-alpha.1";
+export const VERSION = "3.0.1-alpha.2";

package/lib/worker-server.js

@@ -19,6 +19,8 @@ import { getWorkerMessageAPI } from "./worker-messages";
 export let messageManager = new MsgManager();
 let workerMessageChannel;
 let latestCameraToScreenRotation = 0;
+let rawDataAccessEnabled = false;
+const rawCameraFrameBuffersByPipeline = new Map();
 export function launchWorkerServer(wasmUrl, module) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
@@ -45,7 +47,7 @@ export function launchWorkerServer(wasmUrl, module) {
                 });
                 messageManager.postOutgoingMessage("loaded", []);
                 messageManager.onIncomingMessage.bind((msg) => {
-                    var _a;
+                    var _a, _b, _c;
                     switch (msg.t) {
                         case "zappar":
                             server.processBuffer(msg.d);
@@ -75,6 +77,23 @@ export function launchWorkerServer(wasmUrl, module) {
                             messageManager.postOutgoingMessage(ret, [msgt.d]);
                             break;
                         }
+                        case "rawenabled": {
+                            const msgt = msg;
+                            rawDataAccessEnabled = msgt.v;
+                            break;
+                        }
+                        case "rawrequest": {
+                            const msgt = msg;
+                            const buffers = rawCameraFrameBuffersByPipeline.get(msgt.p);
+                            const ret = {
+                                t: "raw",
+                                token: msgt.token,
+                                p: msgt.p,
+                                data: buffers ? (_c = (_b = buffers.ready.find(entry => entry.token === msgt.token)) === null || _b === void 0 ? void 0 : _b.data) !== null && _c !== void 0 ? _c : null : null,
+                            };
+                            messageManager.postOutgoingMessage(ret, []);
+                            break;
+                        }
                         case "streamC2S": {
                             let msgt = msg;
                             consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source, workerMessageAPI).then(() => {
@@ -243,8 +262,8 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                 t: "videoFrameS2C",
                 userFacing,
                 uvTransform,
-                w: dataWidth,
-                h: dataHeight,
+                w: profile.dataWidth,
+                h: profile.dataHeight,
                 cameraToDevice: cameraToDeviceTransform,
                 cameraModel,
                 source
@@ -263,6 +282,32 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                     // (mod as any).ccall('what_to_stderr', 'number', ['number'], [err]);
                 }
                 r.pipeline_frame_update(pipeline);
+                if (rawDataAccessEnabled) {
+                    let buffers = rawCameraFrameBuffersByPipeline.get(p);
+                    if (!buffers) {
+                        buffers = { available: [], ready: [] };
+                        rawCameraFrameBuffersByPipeline.set(p, buffers);
+                    }
+                    if (buffers.ready.length > 4) {
+                        const removed = buffers.ready.splice(0, 1);
+                        for (const entry of removed)
+                            buffers.available.push(new Uint8Array(entry.data.data));
+                    }
+                    const size = r.pipeline_camera_frame_data_raw_size(pipeline);
+                    let buffer;
+                    while (!buffer || buffer.byteLength < size) {
+                        if (buffers.available.length < 1)
+                            buffers.available.push(new Uint8Array(size));
+                        buffer = buffers.available.pop();
+                    }
+                    const dataPointer = r.pipeline_camera_frame_data_raw(pipeline);
+                    buffer.set(mod.HEAPU8.subarray(dataPointer, dataPointer + size));
+                    buffers.ready.push({ token, data: {
+                            data: buffer,
+                            width: r.pipeline_camera_frame_data_raw_width(p),
+                            height: r.pipeline_camera_frame_data_raw_height(p),
+                        } });
+                }
                 server.exploreState();
             }
             frame.close();

package/lib/workerinterface.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="dom-webcodecs" />
+import { CameraFrameData } from "./camera-frame-data";
 import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
 export interface CameraFrameC2S {
     t: "cameraFrameC2S";
@@ -83,3 +84,18 @@ export interface AttitudeMatrixC2S {
     p: zappar_pipeline_t;
     m: Float32Array;
 }
+export interface CameraDataRawEnabledC2S {
+    t: "rawenabled";
+    v: boolean;
+}
+export interface CameraDataRawRequestC2S {
+    t: "rawrequest";
+    p: zappar_pipeline_t;
+    token: number;
+}
+export interface CameraDataRawS2C {
+    t: "raw";
+    token: number;
+    p: zappar_pipeline_t;
+    data: CameraFrameData | null;
+}