@zappar/zappar-cv 3.0.1-alpha.14 → 3.0.1-alpha.16

package/lib/gen/zappar-cwrap.js

@@ -237,6 +237,13 @@ export function getRuntimeObject(mod) {
     let world_tracker_quality_wrapped = mod.cwrap("zappar_world_tracker_quality", "number", [
         "number"
     ]);
+    let world_tracker_plane_detection_enabled_wrapped = mod.cwrap("zappar_world_tracker_plane_detection_enabled", "number", [
+        "number"
+    ]);
+    let world_tracker_plane_detection_enabled_set_wrapped = mod.cwrap("zappar_world_tracker_plane_detection_enabled_set", null, [
+        "number",
+        "number"
+    ]);
     let world_tracker_plane_count_wrapped = mod.cwrap("zappar_world_tracker_plane_count", "number", [
         "number"
     ]);
@@ -244,6 +251,18 @@ export function getRuntimeObject(mod) {
         "number",
         "number"
     ]);
+    let world_tracker_plane_visible_wrapped = mod.cwrap("zappar_world_tracker_plane_visible", "number", [
+        "number",
+        "number"
+    ]);
+    let world_tracker_plane_polygon_data_size_wrapped = mod.cwrap("zappar_world_tracker_plane_polygon_data_size", "number", [
+        "number",
+        "number"
+    ]);
+    let world_tracker_plane_polygon_data_wrapped = mod.cwrap("zappar_world_tracker_plane_polygon_data", "number", [
+        "number",
+        "number"
+    ]);
     let world_tracker_world_anchor_valid_wrapped = mod.cwrap("zappar_world_tracker_world_anchor_valid", "number", [
         "number"
     ]);
@@ -272,6 +291,12 @@ export function getRuntimeObject(mod) {
     let world_tracker_tracks_data_wrapped = mod.cwrap("zappar_world_tracker_tracks_data", "number", [
         "number"
     ]);
+    let world_tracker_tracks_type_data_size_wrapped = mod.cwrap("zappar_world_tracker_tracks_type_data_size", "number", [
+        "number"
+    ]);
+    let world_tracker_tracks_type_data_wrapped = mod.cwrap("zappar_world_tracker_tracks_type_data", "number", [
+        "number"
+    ]);
     let world_tracker_projections_data_enabled_wrapped = mod.cwrap("zappar_world_tracker_projections_data_enabled", "number", [
         "number"
     ]);
@@ -771,6 +796,16 @@ export function getRuntimeObject(mod) {
             let ret = world_tracker_quality_wrapped(o);
             return ret;
         },
+        world_tracker_plane_detection_enabled: (o) => {
+            let ret = world_tracker_plane_detection_enabled_wrapped(o);
+            ret = ret === 1;
+            return ret;
+        },
+        world_tracker_plane_detection_enabled_set: (o, plane_detection_enabled) => {
+            let arg_plane_detection_enabled = plane_detection_enabled ? 1 : 0;
+            let ret = world_tracker_plane_detection_enabled_set_wrapped(o, arg_plane_detection_enabled);
+            return ret;
+        },
         world_tracker_plane_count: (o) => {
             let ret = world_tracker_plane_count_wrapped(o);
             return ret;
@@ -783,6 +818,26 @@ export function getRuntimeObject(mod) {
             ret = ab;
             return ret;
         },
+        world_tracker_plane_visible: (o, indx) => {
+            let arg_indx = indx;
+            let ret = world_tracker_plane_visible_wrapped(o, arg_indx);
+            ret = ret === 1;
+            return ret;
+        },
+        world_tracker_plane_polygon_data_size: (o, indx) => {
+            let arg_indx = indx;
+            let ret = world_tracker_plane_polygon_data_size_wrapped(o, arg_indx);
+            return ret;
+        },
+        world_tracker_plane_polygon_data: (o, indx) => {
+            let arg_indx = indx;
+            let ret = world_tracker_plane_polygon_data_wrapped(o, arg_indx);
+            let retsize = world_tracker_plane_polygon_data_size_wrapped(o, indx);
+            let ab = new Float32Array(retsize);
+            ab.set(mod.HEAPF32.subarray(ret / 4, retsize + ret / 4));
+            ret = ab;
+            return ret;
+        },
         world_tracker_world_anchor_valid: (o) => {
             let ret = world_tracker_world_anchor_valid_wrapped(o);
             ret = ret === 1;
@@ -833,6 +888,18 @@ export function getRuntimeObject(mod) {
             ret = ab;
             return ret;
         },
+        world_tracker_tracks_type_data_size: (o) => {
+            let ret = world_tracker_tracks_type_data_size_wrapped(o);
+            return ret;
+        },
+        world_tracker_tracks_type_data: (o) => {
+            let ret = world_tracker_tracks_type_data_wrapped(o);
+            let retsize = world_tracker_tracks_type_data_size_wrapped(o);
+            let ab = new Uint8Array(retsize);
+            ab.set(mod.HEAPU8.subarray(ret, retsize + ret));
+            ret = ab;
+            return ret;
+        },
         world_tracker_projections_data_enabled: (o) => {
             let ret = world_tracker_projections_data_enabled_wrapped(o);
             ret = ret === 1;

package/lib/gen/zappar-native.d.ts

@@ -188,8 +188,13 @@ export interface zappar_cwrap {
     world_tracker_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_enabled_set(o: zappar_world_tracker_t, enabled: boolean): void;
     world_tracker_quality(o: zappar_world_tracker_t): number;
+    world_tracker_plane_detection_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_plane_detection_enabled_set(o: zappar_world_tracker_t, plane_detection_enabled: boolean): void;
     world_tracker_plane_count(o: zappar_world_tracker_t): number;
     world_tracker_plane_pose_raw(o: zappar_world_tracker_t, indx: number): Float32Array;
+    world_tracker_plane_visible(o: zappar_world_tracker_t, indx: number): boolean;
+    world_tracker_plane_polygon_data_size(o: zappar_world_tracker_t, indx: number): number;
+    world_tracker_plane_polygon_data(o: zappar_world_tracker_t, indx: number): Float32Array;
     world_tracker_world_anchor_valid(o: zappar_world_tracker_t): boolean;
     world_tracker_world_anchor_pose_raw(o: zappar_world_tracker_t): Float32Array;
     world_tracker_ground_anchor_valid(o: zappar_world_tracker_t): boolean;
@@ -199,6 +204,8 @@ export interface zappar_cwrap {
     world_tracker_tracks_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
     world_tracker_tracks_data_size(o: zappar_world_tracker_t): number;
     world_tracker_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_tracks_type_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_tracks_type_data(o: zappar_world_tracker_t): Uint8Array;
     world_tracker_projections_data_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_projections_data_enabled_set(o: zappar_world_tracker_t, projections_data_enabled: boolean): void;
     world_tracker_projections_data_size(o: zappar_world_tracker_t): number;

package/lib/gen/zappar-server.js

@@ -417,7 +417,7 @@ export class zappar_server {
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
                         return;
-                    this._impl.world_tracker_reset(obj);
+                    this._impl.world_tracker_plane_detection_enabled_set(obj, msg.bool());
                     break;
                 }
                 case 49: {
@@ -425,10 +425,18 @@ export class zappar_server {
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
                         return;
-                    this._impl.world_tracker_tracks_data_enabled_set(obj, msg.bool());
+                    this._impl.world_tracker_reset(obj);
                     break;
                 }
                 case 50: {
+                    let clientId = msg.type();
+                    let obj = this._world_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.world_tracker_tracks_data_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 51: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -649,7 +657,7 @@ export class zappar_server {
             let serializer = this.serializersByPipelineId.get(pipeline);
             if (!serializer)
                 continue;
-            serializer.sendMessage(31, msg => {
+            serializer.sendMessage(34, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_quality(v));
             });
@@ -664,35 +672,64 @@ export class zappar_server {
                     msg.matrix4x4(this._impl.world_tracker_plane_pose_raw(v, i));
                 });
             }
-            serializer.sendMessage(27, msg => {
+            for (let i = 0; i < this._impl.world_tracker_plane_count(v); i++) {
+                serializer.sendMessage(27, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.bool(this._impl.world_tracker_plane_visible(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.world_tracker_plane_count(v); i++) {
+                serializer.sendMessage(28, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.int(this._impl.world_tracker_plane_polygon_data_size(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.world_tracker_plane_count(v); i++) {
+                serializer.sendMessage(29, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.floatArray(this._impl.world_tracker_plane_polygon_data(v, i));
+                });
+            }
+            serializer.sendMessage(30, msg => {
                 msg.type(k);
                 msg.bool(this._impl.world_tracker_world_anchor_valid(v));
             });
-            serializer.sendMessage(28, msg => {
+            serializer.sendMessage(31, msg => {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_world_anchor_pose_raw(v));
             });
-            serializer.sendMessage(27, msg => {
+            serializer.sendMessage(30, msg => {
                 msg.type(k);
                 msg.bool(this._impl.world_tracker_ground_anchor_valid(v));
             });
-            serializer.sendMessage(30, msg => {
+            serializer.sendMessage(33, msg => {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_ground_anchor_pose_raw(v));
             });
-            serializer.sendMessage(34, msg => {
+            serializer.sendMessage(37, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_tracks_data_size(v));
             });
-            serializer.sendMessage(33, msg => {
+            serializer.sendMessage(36, msg => {
                 msg.type(k);
                 msg.floatArray(this._impl.world_tracker_tracks_data(v));
             });
-            serializer.sendMessage(37, msg => {
+            serializer.sendMessage(39, msg => {
+                msg.type(k);
+                msg.int(this._impl.world_tracker_tracks_type_data_size(v));
+            });
+            serializer.sendMessage(38, msg => {
+                msg.type(k);
+                msg.ucharArray(this._impl.world_tracker_tracks_type_data(v));
+            });
+            serializer.sendMessage(42, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_projections_data_size(v));
             });
-            serializer.sendMessage(36, msg => {
+            serializer.sendMessage(41, msg => {
                 msg.type(k);
                 msg.floatArray(this._impl.world_tracker_projections_data(v));
             });

package/lib/gen/zappar.d.ts

@@ -205,10 +205,15 @@ export interface zappar {
     world_tracker_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_enabled_set(o: zappar_world_tracker_t, enabled: boolean): void;
     world_tracker_quality(o: zappar_world_tracker_t): number;
+    world_tracker_plane_detection_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_plane_detection_enabled_set(o: zappar_world_tracker_t, plane_detection_enabled: boolean): void;
     world_tracker_plane_count(o: zappar_world_tracker_t): number;
     world_tracker_plane_pose_raw(o: zappar_world_tracker_t, indx: number): Float32Array;
+    world_tracker_plane_visible(o: zappar_world_tracker_t, indx: number): boolean;
     world_tracker_plane_pose_camera_relative(o: zappar_world_tracker_t, indx: number, mirror: boolean): Float32Array;
     world_tracker_plane_pose(o: zappar_world_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
+    world_tracker_plane_polygon_data_size(o: zappar_world_tracker_t, indx: number): number;
+    world_tracker_plane_polygon_data(o: zappar_world_tracker_t, indx: number): Float32Array;
     world_tracker_world_anchor_valid(o: zappar_world_tracker_t): boolean;
     world_tracker_world_anchor_pose_raw(o: zappar_world_tracker_t): Float32Array;
     world_tracker_world_anchor_pose_camera_relative(o: zappar_world_tracker_t, mirror: boolean): Float32Array;
@@ -222,6 +227,8 @@ export interface zappar {
     world_tracker_tracks_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
     world_tracker_tracks_data_size(o: zappar_world_tracker_t): number;
     world_tracker_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_tracks_type_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_tracks_type_data(o: zappar_world_tracker_t): Uint8Array;
     world_tracker_projections_data_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_projections_data_enabled_set(o: zappar_world_tracker_t, projections_data_enabled: boolean): void;
     world_tracker_projections_data_size(o: zappar_world_tracker_t): number;

package/lib/html-element-source.d.ts

@@ -15,6 +15,7 @@ export declare class HTMLElementSource extends Source {
     private _cameraToDeviceTransformUserFacing;
     private _cameraModel;
     private _profile;
+    private _waitingForFrame;
     static createVideoElementSource(p: zappar_pipeline_t, element: HTMLVideoElement | HTMLImageElement): zappar_html_element_source_t;
     static getVideoElementSource(m: zappar_html_element_source_t): HTMLElementSource | undefined;
     constructor(_video: HTMLVideoElement | HTMLImageElement, _pipeline: zappar_pipeline_t);
@@ -23,6 +24,8 @@ export declare class HTMLElementSource extends Source {
     pause(): void;
     start(): void;
     getFrame(currentlyProcessing: boolean): void;
+    private _lastPresentedFrames;
+    private _requestVideoFrameCallback;
     private _processFrame;
     uploadGL(): void;
     setProfile(p: camera_profile_t): void;

package/lib/html-element-source.js

@@ -21,6 +21,8 @@ export class HTMLElementSource extends Source {
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
         this._profile = camera_profile_t.DEFAULT;
+        this._waitingForFrame = false;
+        this._lastPresentedFrames = -1;
         mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
         let video = this._video;
         if (this._video instanceof HTMLVideoElement) {
@@ -99,6 +101,25 @@ export class HTMLElementSource extends Source {
         }
         return;
     }
+    _requestVideoFrameCallback() {
+        const video = this._video;
+        if (!(video === null || video === void 0 ? void 0 : video.requestVideoFrameCallback))
+            return;
+        video.requestVideoFrameCallback((time, metadata) => {
+            if (typeof metadata === 'object' && typeof metadata.presentedFrames === 'number') {
+                if (this._lastPresentedFrames === metadata.presentedFrames) {
+                    console.log('Avoided identical frame');
+                    this._requestVideoFrameCallback();
+                    return;
+                }
+                else {
+                    this._lastPresentedFrames = metadata.presentedFrames;
+                }
+            }
+            this._waitingForFrame = false;
+        });
+        this._waitingForFrame = true;
+    }
     _processFrame(gl, rotation, currentlyProcessing) {
         let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
@@ -106,12 +127,15 @@ export class HTMLElementSource extends Source {
         if (!this._imageProcessor)
             this._imageProcessor = new ImageProcessGL(gl);
         if (this._isUploadFrame) {
+            if (this._waitingForFrame)
+                return;
             if (!this._currentVideoTexture) {
                 this._currentVideoTexture = pipeline.getVideoTexture();
             }
             if (!this._currentVideoTexture)
                 return undefined;
             this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
+            this._requestVideoFrameCallback();
             this._isUploadFrame = !this._isUploadFrame;
             return undefined;
         }
@@ -131,7 +155,7 @@ export class HTMLElementSource extends Source {
         }
         let tex = this._currentVideoTexture;
         this._currentVideoTexture = undefined;
-        let focalLength = 300.0 * dataWidth / 320.0;
+        let focalLength = 240.0 * dataWidth / 320.0;
         this._cameraModel[0] = focalLength;
         this._cameraModel[1] = focalLength;
         this._cameraModel[2] = dataWidth * 0.5;

package/lib/serializer.d.ts

@@ -8,6 +8,7 @@ export interface MessageAppender {
     type: (t: number) => void;
     matrix4x4: (a: Float32Array) => void;
     matrix3x3: (a: Float32Array) => void;
+    ucharArray: (a: Uint8Array) => void;
     floatArray: (a: Float32Array) => void;
     cameraModel: (a: Float32Array) => void;
     identityCoefficients: (a: Float32Array) => void;

package/lib/serializer.js

@@ -23,6 +23,7 @@ export class MessageSerializer {
             matrix4x4: i => this.float32ArrayBuffer(i),
             matrix3x3: i => this.float32ArrayBuffer(i),
             floatArray: i => this.float32ArrayBuffer(i),
+            ucharArray: i => this.uint8ArrayBuffer(i),
             identityCoefficients: i => this.float32ArrayBuffer(i),
             expressionCoefficients: i => this.float32ArrayBuffer(i),
             cameraModel: i => this.float32ArrayBuffer(i),

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.1-alpha.14";
+export declare const VERSION = "3.0.1-alpha.16";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.1-alpha.14";
+export const VERSION = "3.0.1-alpha.16";

package/lib/worker-server.js

@@ -266,7 +266,7 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                 mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
             else
                 mat4.identity(cameraToDeviceTransform);
-            let focalLength = 300.0 * dataWidth / 320.0;
+            let focalLength = 240.0 * dataWidth / 320.0;
             cameraModel[0] = focalLength;
             cameraModel[1] = focalLength;
             cameraModel[2] = dataWidth * 0.5;