@zappar/zappar-cv 3.0.1-alpha.7 → 3.0.1-beta.0

package/lib/gen/zappar-server.js

@@ -28,6 +28,8 @@ export class zappar_server {
         this._zapcode_tracker_by_instance = new Map();
         this._pipeline_id_by_world_tracker_id = new Map();
         this._world_tracker_by_instance = new Map();
+        this._pipeline_id_by_custom_anchor_id = new Map();
+        this._custom_anchor_by_instance = new Map();
     }
     processBuffer(b) {
         this._deserializer.setData(b);
@@ -73,7 +75,7 @@ export class zappar_server {
                     let obj = this._pipeline_by_instance.get(clientId);
                     if (obj === undefined)
                         return;
-                    this._impl.pipeline_camera_frame_submit(obj, msg.dataWithLength(), msg.int(), msg.int(), msg.int(), msg.matrix4x4(), msg.cameraModel(), msg.bool());
+                    this._impl.pipeline_camera_frame_submit(obj, msg.dataWithLength(), msg.int(), msg.int(), msg.int(), msg.matrix4x4(), msg.cameraModel(), msg.bool(), msg.float());
                     break;
                 }
                 case 10: {
@@ -417,7 +419,7 @@ export class zappar_server {
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
                         return;
-                    this._impl.world_tracker_reset(obj);
+                    this._impl.world_tracker_horizontal_plane_detection_enabled_set(obj, msg.bool());
                     break;
                 }
                 case 49: {
@@ -425,7 +427,76 @@ export class zappar_server {
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
                         return;
-                    this._impl.world_tracker_debug_enabled_set(obj, msg.bool());
+                    this._impl.world_tracker_vertical_plane_detection_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 50: {
+                    let clientId = msg.type();
+                    let obj = this._world_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.world_tracker_reset(obj);
+                    break;
+                }
+                case 51: {
+                    let clientId = msg.type();
+                    let obj = this._world_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.world_tracker_tracks_data_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 52: {
+                    let clientId = msg.type();
+                    let obj = this._world_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.world_tracker_projections_data_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 53: {
+                    let clientId = msg.type();
+                    let arg_pipeline_id = msg.type();
+                    let arg_pipeline = this._pipeline_by_instance.get(arg_pipeline_id);
+                    let arg_worldtracker_id = msg.type();
+                    let arg_worldtracker = this._world_tracker_by_instance.get(arg_worldtracker_id);
+                    let arg_id = msg.string();
+                    let handle = this._impl.custom_anchor_create(arg_pipeline, arg_worldtracker, arg_id);
+                    this._custom_anchor_by_instance.set(clientId, handle);
+                    this._pipeline_id_by_custom_anchor_id.set(clientId, arg_pipeline_id);
+                    break;
+                }
+                case 54: {
+                    let clientId = msg.type();
+                    let obj = this._custom_anchor_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.custom_anchor_destroy(obj);
+                    this._custom_anchor_by_instance.delete(clientId);
+                    break;
+                }
+                case 55: {
+                    let clientId = msg.type();
+                    let obj = this._custom_anchor_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.custom_anchor_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 56: {
+                    let clientId = msg.type();
+                    let obj = this._custom_anchor_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.custom_anchor_pose_set_from_camera_offset_raw(obj, msg.float(), msg.float(), msg.float(), msg.transformOrientation());
+                    break;
+                }
+                case 57: {
+                    let clientId = msg.type();
+                    let obj = this._custom_anchor_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.custom_anchor_pose_set_from_anchor_offset(obj, msg.string(), msg.float(), msg.float(), msg.float(), msg.transformOrientation());
                     break;
                 }
             }
@@ -641,52 +712,126 @@ export class zappar_server {
             let serializer = this.serializersByPipelineId.get(pipeline);
             if (!serializer)
                 continue;
-            serializer.sendMessage(31, msg => {
+            serializer.sendMessage(40, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_quality(v));
             });
             serializer.sendMessage(25, msg => {
                 msg.type(k);
-                msg.int(this._impl.world_tracker_plane_count(v));
+                msg.int(this._impl.world_tracker_plane_anchor_count(v));
             });
-            for (let i = 0; i < this._impl.world_tracker_plane_count(v); i++) {
+            for (let i = 0; i < this._impl.world_tracker_plane_anchor_count(v); i++) {
+                serializer.sendMessage(33, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.string(this._impl.world_tracker_plane_anchor_id(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.world_tracker_plane_anchor_count(v); i++) {
                 serializer.sendMessage(26, msg => {
                     msg.type(k);
                     msg.int(i);
-                    msg.matrix4x4(this._impl.world_tracker_plane_pose_raw(v, i));
+                    msg.matrix4x4(this._impl.world_tracker_plane_anchor_pose_raw(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.world_tracker_plane_anchor_count(v); i++) {
+                serializer.sendMessage(28, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.anchorStatus(this._impl.world_tracker_plane_anchor_status(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.world_tracker_plane_anchor_count(v); i++) {
+                serializer.sendMessage(29, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.int(this._impl.world_tracker_plane_anchor_polygon_data_size(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.world_tracker_plane_anchor_count(v); i++) {
+                serializer.sendMessage(30, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.floatArray(this._impl.world_tracker_plane_anchor_polygon_data(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.world_tracker_plane_anchor_count(v); i++) {
+                serializer.sendMessage(31, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.int(this._impl.world_tracker_plane_anchor_polygon_version(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.world_tracker_plane_anchor_count(v); i++) {
+                serializer.sendMessage(32, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.planeOrientation(this._impl.world_tracker_plane_anchor_orientation(v, i));
                 });
             }
-            serializer.sendMessage(27, msg => {
+            serializer.sendMessage(36, msg => {
                 msg.type(k);
-                msg.bool(this._impl.world_tracker_world_anchor_valid(v));
+                msg.anchorStatus(this._impl.world_tracker_world_anchor_status(v));
             });
-            serializer.sendMessage(28, msg => {
+            serializer.sendMessage(35, msg => {
+                msg.type(k);
+                msg.string(this._impl.world_tracker_world_anchor_id(v));
+            });
+            serializer.sendMessage(34, msg => {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_world_anchor_pose_raw(v));
             });
-            serializer.sendMessage(27, msg => {
+            serializer.sendMessage(38, msg => {
+                msg.type(k);
+                msg.string(this._impl.world_tracker_ground_anchor_id(v));
+            });
+            serializer.sendMessage(39, msg => {
                 msg.type(k);
-                msg.bool(this._impl.world_tracker_ground_anchor_valid(v));
+                msg.anchorStatus(this._impl.world_tracker_ground_anchor_status(v));
             });
-            serializer.sendMessage(30, msg => {
+            serializer.sendMessage(37, msg => {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_ground_anchor_pose_raw(v));
             });
-            serializer.sendMessage(33, msg => {
+            serializer.sendMessage(43, msg => {
                 msg.type(k);
-                msg.int(this._impl.world_tracker_debug_tracks_data_size(v));
+                msg.int(this._impl.world_tracker_tracks_data_size(v));
             });
-            serializer.sendMessage(32, msg => {
+            serializer.sendMessage(42, msg => {
                 msg.type(k);
-                msg.floatArray(this._impl.world_tracker_debug_tracks_data(v));
+                msg.floatArray(this._impl.world_tracker_tracks_data(v));
             });
-            serializer.sendMessage(35, msg => {
+            serializer.sendMessage(45, msg => {
                 msg.type(k);
-                msg.int(this._impl.world_tracker_debug_projections_data_size(v));
+                msg.int(this._impl.world_tracker_tracks_type_data_size(v));
             });
-            serializer.sendMessage(34, msg => {
+            serializer.sendMessage(44, msg => {
+                msg.type(k);
+                msg.ucharArray(this._impl.world_tracker_tracks_type_data(v));
+            });
+            serializer.sendMessage(48, msg => {
+                msg.type(k);
+                msg.int(this._impl.world_tracker_projections_data_size(v));
+            });
+            serializer.sendMessage(47, msg => {
+                msg.type(k);
+                msg.floatArray(this._impl.world_tracker_projections_data(v));
+            });
+        }
+        for (let [k, v] of this._custom_anchor_by_instance) {
+            let pipeline = this._pipeline_id_by_custom_anchor_id.get(k);
+            if (!pipeline)
+                continue;
+            let serializer = this.serializersByPipelineId.get(pipeline);
+            if (!serializer)
+                continue;
+            serializer.sendMessage(50, msg => {
+                msg.type(k);
+                msg.anchorStatus(this._impl.custom_anchor_status(v));
+            });
+            serializer.sendMessage(49, msg => {
                 msg.type(k);
-                msg.floatArray(this._impl.world_tracker_debug_projections_data(v));
+                msg.matrix4x4(this._impl.custom_anchor_pose_raw(v));
             });
         }
     }

package/lib/gen/zappar.d.ts

@@ -5,11 +5,17 @@ export { face_landmark_name_t } from "./zappar-native";
 export { frame_pixel_format_t } from "./zappar-native";
 import { instant_world_tracker_transform_orientation_t } from "./zappar-native";
 export { instant_world_tracker_transform_orientation_t } from "./zappar-native";
+import { transform_orientation_t } from "./zappar-native";
+export { transform_orientation_t } from "./zappar-native";
+import { plane_orientation_t } from "./zappar-native";
+export { plane_orientation_t } from "./zappar-native";
 import { log_level_t } from "./zappar-native";
 export { log_level_t } from "./zappar-native";
 import { image_target_type_t } from "./zappar-native";
 export { image_target_type_t } from "./zappar-native";
 export { world_tracker_quality_t } from "./zappar-native";
+import { anchor_status_t } from "./zappar-native";
+export { anchor_status_t } from "./zappar-native";
 import { camera_profile_t } from "./zappar-native";
 export { camera_profile_t } from "./zappar-native";
 export declare type zappar_pipeline_t = number & {
@@ -45,6 +51,9 @@ export declare type zappar_zapcode_tracker_t = number & {
 export declare type zappar_world_tracker_t = number & {
     _: 'zappar_world_tracker_t';
 };
+export declare type zappar_custom_anchor_t = number & {
+    _: 'zappar_custom_anchor_t';
+};
 export interface zappar {
     loaded(): boolean;
     camera_default_device_id(userFacing: boolean): string;
@@ -82,7 +91,7 @@ export interface zappar {
     pipeline_camera_pose_with_attitude(o: zappar_pipeline_t, mirror: boolean): Float32Array;
     pipeline_camera_pose_with_origin(o: zappar_pipeline_t, pose: Float32Array): Float32Array;
     pipeline_camera_frame_user_data(o: zappar_pipeline_t): number;
-    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean): void;
+    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean, timestampSeconds: number): void;
     pipeline_camera_frame_camera_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_device_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_user_facing(o: zappar_pipeline_t): boolean;
@@ -205,23 +214,52 @@ export interface zappar {
     world_tracker_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_enabled_set(o: zappar_world_tracker_t, enabled: boolean): void;
     world_tracker_quality(o: zappar_world_tracker_t): number;
-    world_tracker_plane_count(o: zappar_world_tracker_t): number;
-    world_tracker_plane_pose_raw(o: zappar_world_tracker_t, indx: number): Float32Array;
-    world_tracker_plane_pose_camera_relative(o: zappar_world_tracker_t, indx: number, mirror: boolean): Float32Array;
-    world_tracker_plane_pose(o: zappar_world_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
-    world_tracker_world_anchor_valid(o: zappar_world_tracker_t): boolean;
+    world_tracker_horizontal_plane_detection_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_horizontal_plane_detection_enabled_set(o: zappar_world_tracker_t, horizontal_plane_detection_enabled: boolean): void;
+    world_tracker_vertical_plane_detection_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_vertical_plane_detection_enabled_set(o: zappar_world_tracker_t, vertical_plane_detection_enabled: boolean): void;
+    world_tracker_vertical_plane_detection_supported(o: zappar_world_tracker_t): boolean;
+    world_tracker_plane_anchor_count(o: zappar_world_tracker_t): number;
+    world_tracker_plane_anchor_id(o: zappar_world_tracker_t, indx: number): string;
+    world_tracker_plane_anchor_pose_raw(o: zappar_world_tracker_t, indx: number): Float32Array;
+    world_tracker_plane_anchor_status(o: zappar_world_tracker_t, indx: number): anchor_status_t;
+    world_tracker_plane_anchor_pose_camera_relative(o: zappar_world_tracker_t, indx: number, mirror: boolean): Float32Array;
+    world_tracker_plane_anchor_pose(o: zappar_world_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
+    world_tracker_plane_anchor_polygon_data_size(o: zappar_world_tracker_t, indx: number): number;
+    world_tracker_plane_anchor_polygon_data(o: zappar_world_tracker_t, indx: number): Float32Array;
+    world_tracker_plane_anchor_polygon_version(o: zappar_world_tracker_t, indx: number): number;
+    world_tracker_plane_anchor_orientation(o: zappar_world_tracker_t, indx: number): plane_orientation_t;
+    world_tracker_world_anchor_status(o: zappar_world_tracker_t): anchor_status_t;
+    world_tracker_world_anchor_id(o: zappar_world_tracker_t): string;
     world_tracker_world_anchor_pose_raw(o: zappar_world_tracker_t): Float32Array;
     world_tracker_world_anchor_pose_camera_relative(o: zappar_world_tracker_t, mirror: boolean): Float32Array;
     world_tracker_world_anchor_pose(o: zappar_world_tracker_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
-    world_tracker_ground_anchor_valid(o: zappar_world_tracker_t): boolean;
+    world_tracker_ground_anchor_id(o: zappar_world_tracker_t): string;
+    world_tracker_ground_anchor_status(o: zappar_world_tracker_t): anchor_status_t;
     world_tracker_ground_anchor_pose_raw(o: zappar_world_tracker_t): Float32Array;
     world_tracker_ground_anchor_pose_camera_relative(o: zappar_world_tracker_t, mirror: boolean): Float32Array;
     world_tracker_ground_anchor_pose(o: zappar_world_tracker_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
     world_tracker_reset(o: zappar_world_tracker_t): void;
-    world_tracker_debug_enabled(o: zappar_world_tracker_t): boolean;
-    world_tracker_debug_enabled_set(o: zappar_world_tracker_t, debug_enabled: boolean): void;
-    world_tracker_debug_tracks_data_size(o: zappar_world_tracker_t): number;
-    world_tracker_debug_tracks_data(o: zappar_world_tracker_t): Float32Array;
-    world_tracker_debug_projections_data_size(o: zappar_world_tracker_t): number;
-    world_tracker_debug_projections_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_tracks_data_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_tracks_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
+    world_tracker_tracks_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_tracks_type_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_tracks_type_data(o: zappar_world_tracker_t): Uint8Array;
+    world_tracker_projections_data_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_projections_data_enabled_set(o: zappar_world_tracker_t, projections_data_enabled: boolean): void;
+    world_tracker_projections_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_projections_data(o: zappar_world_tracker_t): Float32Array;
+    custom_anchor_create(pipeline: zappar_pipeline_t, worldtracker: zappar_world_tracker_t, id: string): zappar_custom_anchor_t;
+    custom_anchor_destroy(o: zappar_custom_anchor_t): void;
+    custom_anchor_id(o: zappar_custom_anchor_t): string;
+    custom_anchor_status(o: zappar_custom_anchor_t): anchor_status_t;
+    custom_anchor_enabled_set(o: zappar_custom_anchor_t, enabled: boolean): void;
+    custom_anchor_enabled(o: zappar_custom_anchor_t): boolean;
+    custom_anchor_pose_raw(o: zappar_custom_anchor_t): Float32Array;
+    custom_anchor_pose_camera_relative(o: zappar_custom_anchor_t, mirror: boolean): Float32Array;
+    custom_anchor_pose(o: zappar_custom_anchor_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
+    custom_anchor_pose_set_from_camera_offset_raw(o: zappar_custom_anchor_t, x: number, y: number, z: number, orientation: transform_orientation_t): void;
+    custom_anchor_pose_set_from_camera_offset(o: zappar_custom_anchor_t, x: number, y: number, z: number, orientation: transform_orientation_t): void;
+    custom_anchor_pose_set_from_anchor_offset(o: zappar_custom_anchor_t, anchor_id: string, x: number, y: number, z: number, orientation: transform_orientation_t): void;
 }

package/lib/gen/zappar.js

@@ -2,7 +2,10 @@ export { barcode_format_t } from "./zappar-native";
 export { face_landmark_name_t } from "./zappar-native";
 export { frame_pixel_format_t } from "./zappar-native";
 export { instant_world_tracker_transform_orientation_t } from "./zappar-native";
+export { transform_orientation_t } from "./zappar-native";
+export { plane_orientation_t } from "./zappar-native";
 export { log_level_t } from "./zappar-native";
 export { image_target_type_t } from "./zappar-native";
 export { world_tracker_quality_t } from "./zappar-native";
+export { anchor_status_t } from "./zappar-native";
 export { camera_profile_t } from "./zappar-native";

package/lib/html-element-source.d.ts

@@ -15,6 +15,7 @@ export declare class HTMLElementSource extends Source {
     private _cameraToDeviceTransformUserFacing;
     private _cameraModel;
     private _profile;
+    private _waitingForFrame;
     static createVideoElementSource(p: zappar_pipeline_t, element: HTMLVideoElement | HTMLImageElement): zappar_html_element_source_t;
     static getVideoElementSource(m: zappar_html_element_source_t): HTMLElementSource | undefined;
     constructor(_video: HTMLVideoElement | HTMLImageElement, _pipeline: zappar_pipeline_t);
@@ -23,6 +24,9 @@ export declare class HTMLElementSource extends Source {
     pause(): void;
     start(): void;
     getFrame(currentlyProcessing: boolean): void;
+    private _lastPresentedFrames;
+    private _frameTime;
+    private _requestVideoFrameCallback;
     private _processFrame;
     uploadGL(): void;
     setProfile(p: camera_profile_t): void;

package/lib/html-element-source.js

@@ -21,6 +21,8 @@ export class HTMLElementSource extends Source {
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
         this._profile = camera_profile_t.DEFAULT;
+        this._waitingForFrame = false;
+        this._lastPresentedFrames = -1;
         mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
         let video = this._video;
         if (this._video instanceof HTMLVideoElement) {
@@ -76,6 +78,7 @@ export class HTMLElementSource extends Source {
         }
     }
     getFrame(currentlyProcessing) {
+        var _a;
         let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
@@ -90,7 +93,7 @@ export class HTMLElementSource extends Source {
             let info = this._processFrame(gl, this._cameraToScreenRotation, currentlyProcessing);
             if (info) {
                 let token = pipeline.registerToken(info);
-                pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
+                pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel, (_a = this._frameTime) !== null && _a !== void 0 ? _a : performance.now());
                 info.data = undefined;
             }
         }
@@ -99,6 +102,28 @@ export class HTMLElementSource extends Source {
         }
         return;
     }
+    _requestVideoFrameCallback() {
+        const video = this._video;
+        if (!(video === null || video === void 0 ? void 0 : video.requestVideoFrameCallback))
+            return;
+        video.requestVideoFrameCallback((time, metadata) => {
+            var _a;
+            if (typeof metadata === 'object')
+                this._frameTime = ((_a = metadata.captureTime) !== null && _a !== void 0 ? _a : -1) / 1000.0;
+            if (typeof metadata === 'object' && typeof metadata.presentedFrames === 'number') {
+                if (this._lastPresentedFrames === metadata.presentedFrames) {
+                    console.log('Avoided identical frame');
+                    this._requestVideoFrameCallback();
+                    return;
+                }
+                else {
+                    this._lastPresentedFrames = metadata.presentedFrames;
+                }
+            }
+            this._waitingForFrame = false;
+        });
+        this._waitingForFrame = true;
+    }
     _processFrame(gl, rotation, currentlyProcessing) {
         let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
@@ -106,12 +131,15 @@ export class HTMLElementSource extends Source {
         if (!this._imageProcessor)
             this._imageProcessor = new ImageProcessGL(gl);
         if (this._isUploadFrame) {
+            if (this._waitingForFrame)
+                return;
             if (!this._currentVideoTexture) {
                 this._currentVideoTexture = pipeline.getVideoTexture();
             }
             if (!this._currentVideoTexture)
                 return undefined;
             this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
+            this._requestVideoFrameCallback();
             this._isUploadFrame = !this._isUploadFrame;
             return undefined;
         }
@@ -131,7 +159,7 @@ export class HTMLElementSource extends Source {
         }
         let tex = this._currentVideoTexture;
         this._currentVideoTexture = undefined;
-        let focalLength = 300.0 * dataWidth / 320.0;
+        let focalLength = 240.0 * dataWidth / 320.0;
         this._cameraModel[0] = focalLength;
         this._cameraModel[1] = focalLength;
         this._cameraModel[2] = dataWidth * 0.5;

package/lib/index.d.ts

@@ -4,4 +4,4 @@ import { Options } from "./options";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(opts?: Options): Zappar;
 export { CameraFrameData } from './camera-frame-data';
-export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, zappar_zapcode_tracker_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";
+export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, zappar_zapcode_tracker_t, zappar_custom_anchor_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t, camera_profile_t, transform_orientation_t, plane_orientation_t, anchor_status_t, } from "./gen/zappar";

package/lib/index.js

@@ -4,4 +4,4 @@ export function initialize(opts) {
     console.log(`Zappar CV v${VERSION}`);
     return nativeInitialize(opts);
 }
-export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";
+export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t, camera_profile_t, transform_orientation_t, plane_orientation_t, anchor_status_t, } from "./gen/zappar";

package/lib/mstp-camera-source.js

@@ -234,7 +234,8 @@ export class MSTPCameraSource extends Source {
                     if (videoTracks.length > 0) {
                         this._isUserFacing = videoTracks[0].getSettings().facingMode === "user";
                         let processor = new MediaStreamTrackProcessor({
-                            track: videoTracks[0]
+                            track: videoTracks[0],
+                            maxBufferSize: 1
                         });
                         const str = this._currentStream;
                         yield delay(1000);

package/lib/native.js

@@ -27,7 +27,10 @@ import { gfx } from "./gfx";
 import { ImageTracker } from "./imagetracker";
 import { MSTPCameraSource } from "./mstp-camera-source";
 import { drawGrid } from "./drawgrid";
+import { getPointsDataMatrix } from "./drawpoints";
+import { drawAxis } from "./drawaxis";
 let client;
+const pipelineByWorldTracker = new Map();
 export function initialize(opts) {
     if (client)
         return client;
@@ -147,6 +150,7 @@ export function initialize(opts) {
             }
         }
     });
+    const customAnchorIDs = new Map();
     client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_source_profile_set: (cam, p) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setProfile(p); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
             let res = applyScreenCounterRotation(undefined, raw_pose);
             if (mirror) {
@@ -171,6 +175,8 @@ export function initialize(opts) {
             drawPlane(gl, projectionMatrix, cameraMatrix, targetMatrix, texture);
         }, draw_grid: (gl, projectionMatrix, cameraMatrix, targetMatrix) => {
             drawGrid(gl, projectionMatrix, cameraMatrix, targetMatrix);
+        }, draw_axis: (gl, projectionMatrix, cameraMatrix, targetMatrix) => {
+            drawAxis(gl, projectionMatrix, cameraMatrix, targetMatrix);
         }, pipeline_draw_face: (p, projectionMatrix, cameraMatrix, targetMatrix, o) => {
             var _a;
             let obj = getFaceMesh(o);
@@ -222,6 +228,44 @@ export function initialize(opts) {
             let rawVec = vec3.create();
             vec3.transformMat4(rawVec, [x, y, z], rotationMat);
             c.impl.instant_world_tracker_anchor_pose_set_from_camera_offset_raw(o, rawVec[0], rawVec[1], rawVec[2], orientation);
+        }, custom_anchor_pose_camera_relative: (o, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.custom_anchor_pose_raw(o));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, custom_anchor_pose: (o, cameraPose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.custom_anchor_pose_raw(o));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            mat4.multiply(res, cameraPose, res);
+            return res;
+        }, custom_anchor_create: (p, wt, id) => {
+            const ret = c.impl.custom_anchor_create(p, wt, id);
+            customAnchorIDs.set(ret, id);
+            return ret;
+        }, custom_anchor_id: (o) => {
+            let id = customAnchorIDs.get(o);
+            if (!id) {
+                zcwarn("attempting to call custom_anchor_id on a destroyed zappar_custom_anchor_t");
+                return "";
+            }
+            return id;
+        }, custom_anchor_pose_set_from_camera_offset: (o, x, y, z, orientation) => {
+            let userFacing = false;
+            let rotation = cameraRotationForScreenOrientation(userFacing) * Math.PI / 180.0;
+            let rotationMat = mat4.create();
+            mat4.fromRotation(rotationMat, -rotation, [0, 0, 1]);
+            let rawVec = vec3.create();
+            vec3.transformMat4(rawVec, [x, y, z], rotationMat);
+            c.impl.custom_anchor_pose_set_from_camera_offset_raw(o, rawVec[0], rawVec[1], rawVec[2], orientation);
         }, image_tracker_create: pipeline => ImageTracker.create(pipeline, c.impl), image_tracker_destroy: t => { var _a; return (_a = ImageTracker.get(t)) === null || _a === void 0 ? void 0 : _a.destroy(); }, image_tracker_target_type: (t, i) => {
             let obj = ImageTracker.get(t);
             if (!obj) {
@@ -543,8 +587,17 @@ export function initialize(opts) {
                 return mat4.create();
             }
             return obj.anchor_pose;
-        }, world_tracker_plane_pose_camera_relative: (o, indx, mirror) => {
-            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_plane_pose_raw(o, indx));
+        }, world_tracker_create: (pipeline) => {
+            const ret = c.impl.world_tracker_create(pipeline);
+            pipelineByWorldTracker.set(ret, pipeline);
+            return ret;
+        }, world_tracker_points_data_matrix: (wt, screenWidth, screenHeight, mirror) => {
+            const pipeline = pipelineByWorldTracker.get(wt);
+            if (pipeline === undefined || !client)
+                return mat4.create();
+            return getPointsDataMatrix(client.pipeline_camera_frame_data_width(pipeline), client.pipeline_camera_frame_data_height(pipeline), screenWidth, screenHeight, mirror);
+        }, world_tracker_plane_anchor_pose_camera_relative: (o, indx, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_plane_anchor_pose_raw(o, indx));
             if (mirror) {
                 let scale = mat4.create();
                 mat4.fromScaling(scale, [-1, 1, 1]);
@@ -552,8 +605,8 @@ export function initialize(opts) {
                 mat4.multiply(res, res, scale);
             }
             return res;
-        }, world_tracker_plane_pose: (o, indx, cameraPose, mirror) => {
-            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_plane_pose_raw(o, indx));
+        }, world_tracker_plane_anchor_pose: (o, indx, cameraPose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_plane_anchor_pose_raw(o, indx));
             if (mirror) {
                 let scale = mat4.create();
                 mat4.fromScaling(scale, [-1, 1, 1]);
@@ -600,6 +653,8 @@ export function initialize(opts) {
                 mat4.multiply(res, res, scale);
             }
             return res;
+        }, world_tracker_vertical_plane_detection_supported: (o) => {
+            return false;
         }, html_element_source_create: (pipeline, elm) => HTMLElementSource.createVideoElementSource(pipeline, elm), html_element_source_start: o => { var _a; return (_a = HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.start(); }, html_element_source_pause: o => { var _a; return (_a = HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, html_element_source_destroy: o => { var _a; return (_a = HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, sequence_source_create: p => SequenceSource.create(p), sequence_source_load_from_memory: (o, data) => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.loadFromMemory(data); }, sequence_source_pause: o => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, sequence_source_start: o => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.start(); }, sequence_source_max_playback_fps_set: (o, fps) => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.maxPlaybackFpsSet(fps); }, sequence_source_time_set: (o, t) => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.setTime(t); }, sequence_source_destroy: o => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, permission_granted_all: permissionGrantedAll, permission_granted_camera: permissionGrantedCamera, permission_granted_motion: permissionGrantedMotion, permission_denied_any: permissionDeniedAny, permission_denied_camera: permissionDeniedCamera, permission_denied_motion: permissionDeniedMotion, permission_request_motion: permissionRequestMotion, permission_request_camera: permissionRequestCamera, permission_request_all: permissionRequestAll, permission_request_ui: permissionRequestUI, permission_request_ui_promise: permissionRequestUI, permission_denied_ui: permissionDeniedUI, browser_incompatible: compatibility.incompatible, browser_incompatible_ui: compatibility.incompatible_ui, log_level_set: l => {
             setLogLevel(l);
             c.impl.log_level_set(l);