@zappar/zappar-cv 3.0.1-alpha.14 → 3.0.1-alpha.15

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/3.0.1-alpha.14/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/3.0.1-alpha.15/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/3.0.1-alpha.14/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.1-alpha.14/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/3.0.1-alpha.15/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.1-alpha.15/zappar-cv.zip)

package/lib/deserializer.d.ts

@@ -8,6 +8,7 @@ interface Message {
     type: () => number;
     matrix4x4: () => Float32Array;
     matrix3x3: () => Float32Array;
+    ucharArray: () => Uint8Array;
     floatArray: () => Float32Array;
     identityCoefficients: () => Float32Array;
     expressionCoefficients: () => Float32Array;

package/lib/deserializer.js

@@ -40,6 +40,15 @@ export class MessageDeserializer {
                     this._startOffset++;
                 return ret.buffer;
             },
+            ucharArray: () => {
+                let len = this._i32View[this._startOffset++];
+                let ret = new Uint8Array(len);
+                ret.set(this._u8View.subarray(this._startOffset * 4, this._startOffset * 4 + len));
+                this._startOffset += (ret.byteLength >> 2);
+                if ((ret.byteLength & 3) !== 0)
+                    this._startOffset++;
+                return ret;
+            },
             floatArray: () => {
                 let len = this._i32View[this._startOffset++];
                 let ret = new Float32Array(len);

package/lib/drawgrid.js

@@ -2,6 +2,7 @@ import { compileShader, linkProgram } from "./shader";
 let shader;
 let vbo;
 let uvbo;
+const gridSize = 10;
 export function disposeDrawGrid() {
     shader = undefined;
     vbo = undefined;
@@ -19,6 +20,7 @@ function generate(gl) {
         0.5, 0, 0.5,
         0.5, 0, -0.5
     ];
+    coords = coords.map(entry => entry * gridSize);
     gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
     gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(coords), gl.STATIC_DRAW);
     gl.bindBuffer(gl.ARRAY_BUFFER, null);
@@ -36,6 +38,7 @@ function generateUVBO(gl) {
         1, 1,
         1, 0
     ];
+    coords = coords.map(entry => entry * gridSize);
     gl.bindBuffer(gl.ARRAY_BUFFER, uvbo);
     gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(coords), gl.STATIC_DRAW);
     gl.bindBuffer(gl.ARRAY_BUFFER, null);

package/lib/drawpointswithtype.d.ts

@@ -0,0 +1,13 @@
+export declare class PointsWithTypeDraw {
+    private _gl;
+    private _vbo;
+    private _typebo;
+    private _shader;
+    constructor(_gl: WebGL2RenderingContext);
+    dispose(): void;
+    private _generate;
+    private _generateTypes;
+    drawPoints(screenWidth: number, screenHeight: number, dataWidth: number, dataHeight: number, points: Float32Array, mirror: boolean, type: Uint8Array, pointSize: number): void;
+    private _getCameraShader;
+}
+export declare function getPointsDataMatrix(frameWidth: number, frameHeight: number, screenWidth: number, screenHeight: number, mirror: boolean): Float32Array;

package/lib/drawpointswithtype.js

@@ -0,0 +1,190 @@
+import { profile } from "./profile";
+import { compileShader, linkProgram } from "./shader";
+import { mat4 } from "gl-matrix";
+let identity = mat4.create();
+export class PointsWithTypeDraw {
+    constructor(_gl) {
+        this._gl = _gl;
+    }
+    dispose() {
+        if (this._vbo)
+            this._gl.deleteBuffer(this._vbo);
+        this._vbo = undefined;
+        if (this._shader)
+            this._gl.deleteProgram(this._shader.prog);
+        this._shader = undefined;
+    }
+    _generate(gl, points) {
+        if (!this._vbo)
+            this._vbo = gl.createBuffer();
+        if (!this._vbo)
+            throw new Error("Unable to create buffer object");
+        gl.bindBuffer(gl.ARRAY_BUFFER, this._vbo);
+        gl.bufferData(gl.ARRAY_BUFFER, points, gl.DYNAMIC_DRAW);
+        gl.bindBuffer(gl.ARRAY_BUFFER, null);
+        return this._vbo;
+    }
+    _generateTypes(gl, points) {
+        if (!this._typebo)
+            this._typebo = gl.createBuffer();
+        if (!this._typebo)
+            throw new Error("Unable to create buffer object");
+        gl.bindBuffer(gl.ARRAY_BUFFER, this._typebo);
+        gl.bufferData(gl.ARRAY_BUFFER, points, gl.DYNAMIC_DRAW);
+        gl.bindBuffer(gl.ARRAY_BUFFER, null);
+        return this._typebo;
+    }
+    drawPoints(screenWidth, screenHeight, dataWidth, dataHeight, points, mirror, type, pointSize) {
+        if (points.length === 0)
+            return; // points = new Float32Array([0, 1, 0, 1]);
+        let gl = this._gl;
+        const reenableDepthTest = gl.isEnabled(gl.DEPTH_TEST);
+        const reenableScissorTest = gl.isEnabled(gl.SCISSOR_TEST);
+        const reenableCullFace = gl.isEnabled(gl.CULL_FACE);
+        const reenableStencilTest = gl.isEnabled(gl.STENCIL_TEST);
+        const reenableBlend = gl.isEnabled(gl.BLEND);
+        const previousBoundTexture = gl.getParameter(gl.TEXTURE_BINDING_2D);
+        const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
+        const previousProgram = gl.getParameter(gl.CURRENT_PROGRAM);
+        const previousActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE);
+        gl.disable(gl.DEPTH_TEST);
+        gl.disable(gl.SCISSOR_TEST);
+        gl.disable(gl.CULL_FACE);
+        gl.disable(gl.STENCIL_TEST);
+        gl.disable(gl.BLEND);
+        let shader = this._getCameraShader(gl);
+        let vbo = this._generate(gl, points);
+        let typebo = this._generateTypes(gl, type);
+        gl.activeTexture(gl.TEXTURE0);
+        gl.useProgram(shader.prog);
+        gl.uniform1f(shader.unif_pointSize, pointSize);
+        gl.uniformMatrix4fv(shader.unif_skinTexTransform, false, getPointsDataMatrix(dataWidth, dataHeight, screenWidth, screenHeight, mirror));
+        gl.bindTexture(gl.TEXTURE_2D, null);
+        gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
+        gl.vertexAttribPointer(shader.attr_position, 2, gl.FLOAT, false, 0, 0);
+        gl.enableVertexAttribArray(shader.attr_position);
+        gl.bindBuffer(gl.ARRAY_BUFFER, typebo);
+        gl.vertexAttribIPointer(shader.attr_type, 1, gl.UNSIGNED_BYTE, 0, 0);
+        gl.enableVertexAttribArray(shader.attr_type);
+        gl.drawArrays(gl.POINTS, 0, points.length / 2);
+        gl.disableVertexAttribArray(shader.attr_type);
+        gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
+        gl.disableVertexAttribArray(shader.attr_position);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
+        gl.useProgram(previousProgram);
+        gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
+        gl.activeTexture(previousActiveTexture);
+        if (reenableBlend)
+            gl.enable(gl.BLEND);
+        if (reenableCullFace)
+            gl.enable(gl.CULL_FACE);
+        if (reenableDepthTest)
+            gl.enable(gl.DEPTH_TEST);
+        if (reenableScissorTest)
+            gl.enable(gl.SCISSOR_TEST);
+        if (reenableStencilTest)
+            gl.enable(gl.STENCIL_TEST);
+    }
+    _getCameraShader(gl) {
+        if (this._shader)
+            return this._shader;
+        let prog = gl.createProgram();
+        if (!prog)
+            throw new Error("Unable to create program");
+        let vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSrc);
+        let fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSrc);
+        gl.attachShader(prog, vertexShader);
+        gl.attachShader(prog, fragmentShader);
+        linkProgram(gl, prog);
+        let unif_skinTexTransform = gl.getUniformLocation(prog, "skinTexTransform");
+        if (!unif_skinTexTransform)
+            throw new Error("Unable to get uniform location skinTexTransform");
+        let unif_pointSize = gl.getUniformLocation(prog, "pointSize");
+        if (!unif_pointSize)
+            throw new Error("Unable to get uniform location pointSize");
+        this._shader = {
+            prog,
+            unif_skinTexTransform,
+            unif_pointSize,
+            attr_position: gl.getAttribLocation(prog, "pos"),
+            attr_type: gl.getAttribLocation(prog, "aType"),
+        };
+        return this._shader;
+    }
+}
+let vertexShaderSrc = `#version 300 es
+
+in vec2 pos;
+in uint aType;
+uniform mat4 skinTexTransform;
+uniform float pointSize;
+
+out vec4 vColor;
+
+void main()
+{
+    gl_Position = skinTexTransform * vec4(pos, 0.0, 1.0);
+    vColor = vec4(1.0, 0.0, 0.0, 1.0);
+    if (aType == 1u) {
+        vColor = vec4(1.0, 0.7, 0.0, 1.0);
+    } else if (aType >= 4u) {
+        vColor = vec4(0.0, 0.0, 1.0, 1.0);
+    } else if (aType > 1u) {
+        vColor = vec4(0.0, 1.0, 0.0, 1.0);
+    }
+    gl_PointSize = pointSize;
+}`;
+let fragmentShaderSrc = `#version 300 es
+precision highp float;
+
+in vec4 vColor;
+out vec4 outColor;
+
+void main()
+{
+    outColor = vColor;
+}`;
+function cameraRotationForScreenOrientation() {
+    if (window.screen.orientation && !profile.forceWindowOrientation) {
+        switch (window.screen.orientation.type) {
+            case "portrait-primary":
+                return 270;
+            case "landscape-secondary":
+                return 180;
+            case "portrait-secondary":
+                return 90;
+            default:
+                return 0;
+        }
+    }
+    else if (window.orientation !== undefined) {
+        switch (window.orientation) {
+            case 0: return 270;
+            case 90: return 0;
+            case 180: return 90;
+            case -90: return 180;
+        }
+    }
+    return 0;
+}
+export function getPointsDataMatrix(frameWidth, frameHeight, screenWidth, screenHeight, mirror) {
+    let ret = mat4.create();
+    let trans = mat4.create();
+    // Translate to centre UV coords
+    mat4.fromTranslation(trans, [-0.5 * frameWidth, -0.5 * frameHeight, 0]);
+    mat4.multiply(ret, trans, ret);
+    mat4.fromScaling(trans, [2 / frameWidth, 2 / -frameHeight, 1]);
+    mat4.multiply(ret, trans, ret);
+    if (mirror) {
+        mat4.fromScaling(trans, [-1, 1, 1]);
+        mat4.multiply(ret, trans, ret);
+    }
+    let screenAspect = screenWidth > screenHeight ? (screenWidth / screenHeight) : (screenHeight / screenWidth);
+    let frameAspect = frameWidth / frameHeight;
+    mat4.fromScaling(trans, [1, screenAspect / frameAspect, 1]);
+    mat4.multiply(ret, trans, ret);
+    // Apply rotation back into ZCV's landscape space
+    mat4.fromRotation(trans, cameraRotationForScreenOrientation() * Math.PI / 180.0, [0, 0, 1]);
+    mat4.multiply(ret, trans, ret);
+    return ret;
+}

package/lib/gen/zappar-bridge.d.ts

@@ -123,6 +123,8 @@ export interface zappar {
     world_tracker_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_enabled_set(o: zappar_world_tracker_t, enabled: boolean): void;
     world_tracker_quality(o: zappar_world_tracker_t): number;
+    world_tracker_plane_detection_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_plane_detection_enabled_set(o: zappar_world_tracker_t, plane_detection_enabled: boolean): void;
     world_tracker_plane_count(o: zappar_world_tracker_t): number;
     world_tracker_plane_pose_raw(o: zappar_world_tracker_t, indx: number): Float32Array;
     world_tracker_world_anchor_valid(o: zappar_world_tracker_t): boolean;
@@ -134,6 +136,8 @@ export interface zappar {
     world_tracker_tracks_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
     world_tracker_tracks_data_size(o: zappar_world_tracker_t): number;
     world_tracker_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_tracks_type_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_tracks_type_data(o: zappar_world_tracker_t): Uint8Array;
     world_tracker_projections_data_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_projections_data_enabled_set(o: zappar_world_tracker_t, projections_data_enabled: boolean): void;
     world_tracker_projections_data_size(o: zappar_world_tracker_t): number;

package/lib/gen/zappar-client.js

@@ -680,6 +680,7 @@ export class zappar_client {
                 let newId = (this._latestId++);
                 let s = {
                     enabled: true,
+                    plane_detection_enabled: true,
                     plane_count: 0,
                     plane_pose: [],
                     world_anchor_valid: false,
@@ -689,6 +690,8 @@ export class zappar_client {
                     tracks_data_enabled: false,
                     tracks_data: new Float32Array([]),
                     tracks_data_size: 0,
+                    tracks_type_data: new Uint8Array([]),
+                    tracks_type_data_size: 0,
                     projections_data_enabled: false,
                     projections_data: new Float32Array([]),
                     projections_data_size: 0,
@@ -731,6 +734,21 @@ export class zappar_client {
                     throw new Error("This object has been destroyed");
                 return s.quality;
             },
+            world_tracker_plane_detection_enabled: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.plane_detection_enabled;
+            },
+            world_tracker_plane_detection_enabled_set: (o, plane_detection_enabled) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                this.serializer.sendMessage(48, m => {
+                    m.type(o);
+                    m.bool(plane_detection_enabled);
+                });
+            },
             world_tracker_plane_count: (o) => {
                 let s = this._world_tracker_state_by_instance.get(o);
                 if (!s)
@@ -771,7 +789,7 @@ export class zappar_client {
                 let s = this._world_tracker_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(48, m => {
+                this.serializer.sendMessage(49, m => {
                     m.type(o);
                 });
             },
@@ -785,7 +803,7 @@ export class zappar_client {
                 let s = this._world_tracker_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(49, m => {
+                this.serializer.sendMessage(50, m => {
                     m.type(o);
                     m.bool(tracks_data_enabled);
                 });
@@ -802,6 +820,18 @@ export class zappar_client {
                     throw new Error("This object has been destroyed");
                 return s.tracks_data;
             },
+            world_tracker_tracks_type_data_size: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.tracks_type_data_size;
+            },
+            world_tracker_tracks_type_data: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.tracks_type_data;
+            },
             world_tracker_projections_data_enabled: (o) => {
                 let s = this._world_tracker_state_by_instance.get(o);
                 if (!s)
@@ -812,7 +842,7 @@ export class zappar_client {
                 let s = this._world_tracker_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(50, m => {
+                this.serializer.sendMessage(51, m => {
                     m.type(o);
                     m.bool(projections_data_enabled);
                 });
@@ -1102,7 +1132,23 @@ export class zappar_client {
                     inst.tracks_data = msg.floatArray();
                     break;
                 }
-                case 37: {
+                case 36: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    inst.tracks_type_data_size = msg.int();
+                    break;
+                }
+                case 35: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    inst.tracks_type_data = msg.ucharArray();
+                    break;
+                }
+                case 39: {
                     let handle = msg.type();
                     let inst = this._world_tracker_state_by_instance.get(handle);
                     if (!inst)
@@ -1110,7 +1156,7 @@ export class zappar_client {
                     inst.projections_data_size = msg.int();
                     break;
                 }
-                case 36: {
+                case 38: {
                     let handle = msg.type();
                     let inst = this._world_tracker_state_by_instance.get(handle);
                     if (!inst)

package/lib/gen/zappar-cwrap.js

@@ -237,6 +237,13 @@ export function getRuntimeObject(mod) {
     let world_tracker_quality_wrapped = mod.cwrap("zappar_world_tracker_quality", "number", [
         "number"
     ]);
+    let world_tracker_plane_detection_enabled_wrapped = mod.cwrap("zappar_world_tracker_plane_detection_enabled", "number", [
+        "number"
+    ]);
+    let world_tracker_plane_detection_enabled_set_wrapped = mod.cwrap("zappar_world_tracker_plane_detection_enabled_set", null, [
+        "number",
+        "number"
+    ]);
     let world_tracker_plane_count_wrapped = mod.cwrap("zappar_world_tracker_plane_count", "number", [
         "number"
     ]);
@@ -272,6 +279,12 @@ export function getRuntimeObject(mod) {
     let world_tracker_tracks_data_wrapped = mod.cwrap("zappar_world_tracker_tracks_data", "number", [
         "number"
     ]);
+    let world_tracker_tracks_type_data_size_wrapped = mod.cwrap("zappar_world_tracker_tracks_type_data_size", "number", [
+        "number"
+    ]);
+    let world_tracker_tracks_type_data_wrapped = mod.cwrap("zappar_world_tracker_tracks_type_data", "number", [
+        "number"
+    ]);
     let world_tracker_projections_data_enabled_wrapped = mod.cwrap("zappar_world_tracker_projections_data_enabled", "number", [
         "number"
     ]);
@@ -771,6 +784,16 @@ export function getRuntimeObject(mod) {
             let ret = world_tracker_quality_wrapped(o);
             return ret;
         },
+        world_tracker_plane_detection_enabled: (o) => {
+            let ret = world_tracker_plane_detection_enabled_wrapped(o);
+            ret = ret === 1;
+            return ret;
+        },
+        world_tracker_plane_detection_enabled_set: (o, plane_detection_enabled) => {
+            let arg_plane_detection_enabled = plane_detection_enabled ? 1 : 0;
+            let ret = world_tracker_plane_detection_enabled_set_wrapped(o, arg_plane_detection_enabled);
+            return ret;
+        },
         world_tracker_plane_count: (o) => {
             let ret = world_tracker_plane_count_wrapped(o);
             return ret;
@@ -833,6 +856,18 @@ export function getRuntimeObject(mod) {
             ret = ab;
             return ret;
         },
+        world_tracker_tracks_type_data_size: (o) => {
+            let ret = world_tracker_tracks_type_data_size_wrapped(o);
+            return ret;
+        },
+        world_tracker_tracks_type_data: (o) => {
+            let ret = world_tracker_tracks_type_data_wrapped(o);
+            let retsize = world_tracker_tracks_type_data_size_wrapped(o);
+            let ab = new Uint8Array(retsize);
+            ab.set(mod.HEAPU8.subarray(ret, retsize + ret));
+            ret = ab;
+            return ret;
+        },
         world_tracker_projections_data_enabled: (o) => {
             let ret = world_tracker_projections_data_enabled_wrapped(o);
             ret = ret === 1;

package/lib/gen/zappar-native.d.ts

@@ -188,6 +188,8 @@ export interface zappar_cwrap {
     world_tracker_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_enabled_set(o: zappar_world_tracker_t, enabled: boolean): void;
     world_tracker_quality(o: zappar_world_tracker_t): number;
+    world_tracker_plane_detection_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_plane_detection_enabled_set(o: zappar_world_tracker_t, plane_detection_enabled: boolean): void;
     world_tracker_plane_count(o: zappar_world_tracker_t): number;
     world_tracker_plane_pose_raw(o: zappar_world_tracker_t, indx: number): Float32Array;
     world_tracker_world_anchor_valid(o: zappar_world_tracker_t): boolean;
@@ -199,6 +201,8 @@ export interface zappar_cwrap {
     world_tracker_tracks_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
     world_tracker_tracks_data_size(o: zappar_world_tracker_t): number;
     world_tracker_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_tracks_type_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_tracks_type_data(o: zappar_world_tracker_t): Uint8Array;
     world_tracker_projections_data_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_projections_data_enabled_set(o: zappar_world_tracker_t, projections_data_enabled: boolean): void;
     world_tracker_projections_data_size(o: zappar_world_tracker_t): number;

package/lib/gen/zappar-server.js

@@ -417,7 +417,7 @@ export class zappar_server {
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
                         return;
-                    this._impl.world_tracker_reset(obj);
+                    this._impl.world_tracker_plane_detection_enabled_set(obj, msg.bool());
                     break;
                 }
                 case 49: {
@@ -425,10 +425,18 @@ export class zappar_server {
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
                         return;
-                    this._impl.world_tracker_tracks_data_enabled_set(obj, msg.bool());
+                    this._impl.world_tracker_reset(obj);
                     break;
                 }
                 case 50: {
+                    let clientId = msg.type();
+                    let obj = this._world_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.world_tracker_tracks_data_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 51: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -688,11 +696,19 @@ export class zappar_server {
                 msg.type(k);
                 msg.floatArray(this._impl.world_tracker_tracks_data(v));
             });
-            serializer.sendMessage(37, msg => {
+            serializer.sendMessage(36, msg => {
+                msg.type(k);
+                msg.int(this._impl.world_tracker_tracks_type_data_size(v));
+            });
+            serializer.sendMessage(35, msg => {
+                msg.type(k);
+                msg.ucharArray(this._impl.world_tracker_tracks_type_data(v));
+            });
+            serializer.sendMessage(39, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_projections_data_size(v));
             });
-            serializer.sendMessage(36, msg => {
+            serializer.sendMessage(38, msg => {
                 msg.type(k);
                 msg.floatArray(this._impl.world_tracker_projections_data(v));
             });

package/lib/gen/zappar.d.ts

@@ -205,6 +205,8 @@ export interface zappar {
     world_tracker_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_enabled_set(o: zappar_world_tracker_t, enabled: boolean): void;
     world_tracker_quality(o: zappar_world_tracker_t): number;
+    world_tracker_plane_detection_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_plane_detection_enabled_set(o: zappar_world_tracker_t, plane_detection_enabled: boolean): void;
     world_tracker_plane_count(o: zappar_world_tracker_t): number;
     world_tracker_plane_pose_raw(o: zappar_world_tracker_t, indx: number): Float32Array;
     world_tracker_plane_pose_camera_relative(o: zappar_world_tracker_t, indx: number, mirror: boolean): Float32Array;
@@ -222,6 +224,8 @@ export interface zappar {
     world_tracker_tracks_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
     world_tracker_tracks_data_size(o: zappar_world_tracker_t): number;
     world_tracker_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_tracks_type_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_tracks_type_data(o: zappar_world_tracker_t): Uint8Array;
     world_tracker_projections_data_enabled(o: zappar_world_tracker_t): boolean;
     world_tracker_projections_data_enabled_set(o: zappar_world_tracker_t, projections_data_enabled: boolean): void;
     world_tracker_projections_data_size(o: zappar_world_tracker_t): number;

package/lib/html-element-source.d.ts

@@ -15,6 +15,7 @@ export declare class HTMLElementSource extends Source {
     private _cameraToDeviceTransformUserFacing;
     private _cameraModel;
     private _profile;
+    private _waitingForFrame;
     static createVideoElementSource(p: zappar_pipeline_t, element: HTMLVideoElement | HTMLImageElement): zappar_html_element_source_t;
     static getVideoElementSource(m: zappar_html_element_source_t): HTMLElementSource | undefined;
     constructor(_video: HTMLVideoElement | HTMLImageElement, _pipeline: zappar_pipeline_t);
@@ -23,6 +24,8 @@ export declare class HTMLElementSource extends Source {
     pause(): void;
     start(): void;
     getFrame(currentlyProcessing: boolean): void;
+    private _lastPresentedFrames;
+    private _requestVideoFrameCallback;
     private _processFrame;
     uploadGL(): void;
     setProfile(p: camera_profile_t): void;

package/lib/html-element-source.js

@@ -21,6 +21,8 @@ export class HTMLElementSource extends Source {
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
         this._profile = camera_profile_t.DEFAULT;
+        this._waitingForFrame = false;
+        this._lastPresentedFrames = -1;
         mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
         let video = this._video;
         if (this._video instanceof HTMLVideoElement) {
@@ -99,6 +101,25 @@ export class HTMLElementSource extends Source {
         }
         return;
     }
+    _requestVideoFrameCallback() {
+        const video = this._video;
+        if (!(video === null || video === void 0 ? void 0 : video.requestVideoFrameCallback))
+            return;
+        video.requestVideoFrameCallback((time, metadata) => {
+            if (typeof metadata === 'object' && typeof metadata.presentedFrames === 'number') {
+                if (this._lastPresentedFrames === metadata.presentedFrames) {
+                    console.log('Avoided identical frame');
+                    this._requestVideoFrameCallback();
+                    return;
+                }
+                else {
+                    this._lastPresentedFrames = metadata.presentedFrames;
+                }
+            }
+            this._waitingForFrame = false;
+        });
+        this._waitingForFrame = true;
+    }
     _processFrame(gl, rotation, currentlyProcessing) {
         let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
@@ -106,12 +127,15 @@ export class HTMLElementSource extends Source {
         if (!this._imageProcessor)
             this._imageProcessor = new ImageProcessGL(gl);
         if (this._isUploadFrame) {
+            if (this._waitingForFrame)
+                return;
             if (!this._currentVideoTexture) {
                 this._currentVideoTexture = pipeline.getVideoTexture();
             }
             if (!this._currentVideoTexture)
                 return undefined;
             this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
+            this._requestVideoFrameCallback();
             this._isUploadFrame = !this._isUploadFrame;
             return undefined;
         }
@@ -131,7 +155,7 @@ export class HTMLElementSource extends Source {
         }
         let tex = this._currentVideoTexture;
         this._currentVideoTexture = undefined;
-        let focalLength = 300.0 * dataWidth / 320.0;
+        let focalLength = 240.0 * dataWidth / 320.0;
         this._cameraModel[0] = focalLength;
         this._cameraModel[1] = focalLength;
         this._cameraModel[2] = dataWidth * 0.5;

package/lib/serializer.d.ts

@@ -8,6 +8,7 @@ export interface MessageAppender {
     type: (t: number) => void;
     matrix4x4: (a: Float32Array) => void;
     matrix3x3: (a: Float32Array) => void;
+    ucharArray: (a: Uint8Array) => void;
     floatArray: (a: Float32Array) => void;
     cameraModel: (a: Float32Array) => void;
     identityCoefficients: (a: Float32Array) => void;

package/lib/serializer.js

@@ -23,6 +23,7 @@ export class MessageSerializer {
             matrix4x4: i => this.float32ArrayBuffer(i),
             matrix3x3: i => this.float32ArrayBuffer(i),
             floatArray: i => this.float32ArrayBuffer(i),
+            ucharArray: i => this.uint8ArrayBuffer(i),
             identityCoefficients: i => this.float32ArrayBuffer(i),
             expressionCoefficients: i => this.float32ArrayBuffer(i),
             cameraModel: i => this.float32ArrayBuffer(i),

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.1-alpha.14";
+export declare const VERSION = "3.0.1-alpha.15";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.1-alpha.14";
+export const VERSION = "3.0.1-alpha.15";

package/lib/worker-server.js

@@ -266,7 +266,7 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                 mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
             else
                 mat4.identity(cameraToDeviceTransform);
-            let focalLength = 300.0 * dataWidth / 320.0;
+            let focalLength = 240.0 * dataWidth / 320.0;
             cameraModel[0] = focalLength;
             cameraModel[1] = focalLength;
             cameraModel[2] = dataWidth * 0.5;