@zappar/zappar-cv 2.1.3 → 3.0.0-alpha.0

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [2.1.4] - 2023-07-06
+
+- Whitelisted `*.ngrok-free.app` domain.
+
+## [2.1.3] - 2023-06-23
+
+### Fixed
+
+- Performance issues when using curved tracking.
+
 ## [2.0.2] - 2023-05-04
 
 ### Fixed
@@ -76,6 +86,7 @@
 ## [0.3.10] - 2021-09-17
 
 ### Added
+
 - loaded() function which returns true once the library is fully loaded and available for use
 
 ### Changed

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/2.1.3/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/3.0.0-alpha.0/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/2.1.3/zappar-cv.zip](https://libs.zappar.com/zappar-cv/2.1.3/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/3.0.0-alpha.0/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.0-alpha.0/zappar-cv.zip)

package/lib/camera-source.d.ts

@@ -19,6 +19,7 @@ export declare class CameraSource extends HTMLElementSource {
     private _getUserMedia;
     private _syncCamera;
     private _hasStartedOrientation;
+    private _lastTimestamp;
     private _deviceMotionListener;
     private _deviceOrientationListener;
     private _startDeviceOrientation;

package/lib/camera-source.js

@@ -29,23 +29,36 @@ export class CameraSource extends HTMLElementSource {
         this._currentStream = null;
         this._activeDeviceId = null;
         this._hasStartedOrientation = false;
+        this._lastTimestamp = -1;
         this._deviceMotionListener = (ev) => {
             let pipeline = Pipeline.get(this._pipeline);
             if (!pipeline)
                 return;
             let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            let interval = ev.interval * profile.intervalMultiplier;
+            if (this._lastTimestamp > -1 && !profile.trustSensorIntervals) {
+                interval = Math.max(timeStamp - this._lastTimestamp, interval);
+            }
+            this._lastTimestamp = timeStamp;
+            if (ev.acceleration !== null &&
+                ev.acceleration.x !== null &&
+                ev.acceleration.y !== null &&
+                ev.acceleration.z !== null) {
+                pipeline.motionAccelerometerWithoutGravitySubmitInt(timeStamp, interval, ev.acceleration.x * profile.deviceMotionMutliplier, ev.acceleration.y * profile.deviceMotionMutliplier, ev.acceleration.z * profile.deviceMotionMutliplier);
+            }
             if (ev.accelerationIncludingGravity !== null &&
                 ev.accelerationIncludingGravity.x !== null &&
                 ev.accelerationIncludingGravity.y !== null &&
                 ev.accelerationIncludingGravity.z !== null) {
                 pipeline.motionAccelerometerSubmit(timeStamp, ev.accelerationIncludingGravity.x * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile.deviceMotionMutliplier);
+                pipeline.motionAccelerometerWithGravitySubmitInt(timeStamp, interval, ev.accelerationIncludingGravity.x * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile.deviceMotionMutliplier);
             }
             if (ev.rotationRate !== null &&
                 ev.rotationRate.alpha !== null &&
                 ev.rotationRate.beta !== null &&
-                ev.rotationRate.gamma !== null && !this._hasStartedOrientation) {
-                ev.timeStamp;
-                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha * Math.PI / -180.0, ev.rotationRate.beta * Math.PI / -180.0, ev.rotationRate.gamma * Math.PI / -180.0);
+                ev.rotationRate.gamma !== null && this._hasStartedOrientation) {
+                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha, ev.rotationRate.beta, ev.rotationRate.gamma);
+                pipeline.motionRotationRateSubmitInt(timeStamp, interval, ev.rotationRate.alpha, ev.rotationRate.beta, ev.rotationRate.gamma);
             }
             else if (!this._hasStartedOrientation) {
                 this._startDeviceOrientation();
@@ -197,11 +210,13 @@ export class CameraSource extends HTMLElementSource {
             if (ev.alpha === null || ev.beta === null || ev.gamma === null)
                 return;
             pipeline.motionAttitudeSubmit(timeStamp, ev.alpha, ev.beta, ev.gamma);
+            pipeline.motionAttitudeSubmitInt(timeStamp, 0, ev.alpha, ev.beta, ev.gamma);
         };
         window.addEventListener("deviceorientation", this._deviceOrientationListener);
     }
     _startDeviceMotion() {
         window.addEventListener("devicemotion", this._deviceMotionListener, false);
+        this._startDeviceOrientation();
     }
     _stopDeviceMotion() {
         window.removeEventListener("devicemotion", this._deviceMotionListener);

package/lib/drawcamera.js

@@ -38,9 +38,19 @@ export class CameraDraw {
         if (!i.texture)
             return;
         let gl = this._gl;
+        const reenableDepthTest = gl.isEnabled(gl.DEPTH_TEST);
+        const reenableScissorTest = gl.isEnabled(gl.SCISSOR_TEST);
+        const reenableCullFace = gl.isEnabled(gl.CULL_FACE);
+        const reenableStencilTest = gl.isEnabled(gl.STENCIL_TEST);
+        const reenableBlend = gl.isEnabled(gl.BLEND);
+        const previousBoundTexture = gl.getParameter(gl.TEXTURE_BINDING_2D);
+        const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
+        const previousProgram = gl.getParameter(gl.CURRENT_PROGRAM);
+        const previousActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE);
         gl.disable(gl.DEPTH_TEST);
         gl.disable(gl.SCISSOR_TEST);
         gl.disable(gl.CULL_FACE);
+        gl.disable(gl.STENCIL_TEST);
         gl.disable(gl.BLEND);
         let shader = this._getCameraShader(gl);
         let vbo = this._generate(gl, i);
@@ -57,9 +67,20 @@ export class CameraDraw {
         gl.drawArrays(gl.TRIANGLES, 0, 6);
         gl.disableVertexAttribArray(shader.attr_position);
         gl.disableVertexAttribArray(shader.attr_texCoord);
-        gl.bindTexture(gl.TEXTURE_2D, null);
-        gl.bindBuffer(gl.ARRAY_BUFFER, null);
-        gl.useProgram(null);
+        gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
+        gl.useProgram(previousProgram);
+        gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
+        gl.activeTexture(previousActiveTexture);
+        if (reenableBlend)
+            gl.enable(gl.BLEND);
+        if (reenableCullFace)
+            gl.enable(gl.CULL_FACE);
+        if (reenableDepthTest)
+            gl.enable(gl.DEPTH_TEST);
+        if (reenableScissorTest)
+            gl.enable(gl.SCISSOR_TEST);
+        if (reenableStencilTest)
+            gl.enable(gl.STENCIL_TEST);
     }
     _getCameraShader(gl) {
         if (this._shader)

package/lib/gen/zappar-client.d.ts

@@ -17,6 +17,7 @@ export declare class zappar_client {
     private _face_landmark_state_by_instance;
     private _barcode_finder_state_by_instance;
     private _instant_world_tracker_state_by_instance;
+    private _world_tracker_state_by_instance;
     impl: zappar_cwrap;
     processMessages(a: ArrayBuffer): void;
 }

package/lib/gen/zappar-client.js

@@ -20,17 +20,18 @@ export class zappar_client {
         this._face_landmark_state_by_instance = new Map();
         this._barcode_finder_state_by_instance = new Map();
         this._instant_world_tracker_state_by_instance = new Map();
+        this._world_tracker_state_by_instance = new Map();
         this.impl = {
             log_level: () => {
                 return this._globalState.log_level;
             },
             log_level_set: (level) => {
-                this.serializer.sendMessage(34, m => {
+                this.serializer.sendMessage(38, m => {
                     m.logLevel(level);
                 });
             },
             analytics_project_id_set: (id, uid) => {
-                this.serializer.sendMessage(31, m => {
+                this.serializer.sendMessage(35, m => {
                     m.string(id);
                     m.string(uid);
                 });
@@ -47,7 +48,7 @@ export class zappar_client {
                     frame_number: 0,
                 };
                 this._pipeline_state_by_instance.set(newId, s);
-                this.serializer.sendMessage(27, m => {
+                this.serializer.sendMessage(31, m => {
                     m.type(newId);
                 });
                 return newId;
@@ -57,7 +58,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._pipeline_state_by_instance.delete(o);
-                this.serializer.sendMessage(28, m => {
+                this.serializer.sendMessage(32, m => {
                     m.type(o);
                 });
             },
@@ -131,13 +132,52 @@ export class zappar_client {
                     m.float(z);
                 });
             },
-            pipeline_motion_rotation_rate_submit: (o, time, x, y, z) => {
+            pipeline_motion_accelerometer_with_gravity_submit_int: (o, time, interval, x, y, z) => {
+                let s = this._pipeline_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                this.serializer.sendMessage(12, m => {
+                    m.type(o);
+                    m.timestamp(time);
+                    m.timestamp(interval);
+                    m.float(x);
+                    m.float(y);
+                    m.float(z);
+                });
+            },
+            pipeline_motion_accelerometer_without_gravity_submit_int: (o, time, interval, x, y, z) => {
                 let s = this._pipeline_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this.serializer.sendMessage(11, m => {
                     m.type(o);
                     m.timestamp(time);
+                    m.timestamp(interval);
+                    m.float(x);
+                    m.float(y);
+                    m.float(z);
+                });
+            },
+            pipeline_motion_rotation_rate_submit: (o, time, x, y, z) => {
+                let s = this._pipeline_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                this.serializer.sendMessage(15, m => {
+                    m.type(o);
+                    m.timestamp(time);
+                    m.float(x);
+                    m.float(y);
+                    m.float(z);
+                });
+            },
+            pipeline_motion_rotation_rate_submit_int: (o, time, interval, x, y, z) => {
+                let s = this._pipeline_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                this.serializer.sendMessage(13, m => {
+                    m.type(o);
+                    m.timestamp(time);
+                    m.timestamp(interval);
                     m.float(x);
                     m.float(y);
                     m.float(z);
@@ -147,7 +187,7 @@ export class zappar_client {
                 let s = this._pipeline_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(12, m => {
+                this.serializer.sendMessage(16, m => {
                     m.type(o);
                     m.timestamp(time);
                     m.float(x);
@@ -155,11 +195,24 @@ export class zappar_client {
                     m.float(z);
                 });
             },
+            pipeline_motion_attitude_submit_int: (o, time, interval, x, y, z) => {
+                let s = this._pipeline_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                this.serializer.sendMessage(14, m => {
+                    m.type(o);
+                    m.timestamp(time);
+                    m.timestamp(interval);
+                    m.float(x);
+                    m.float(y);
+                    m.float(z);
+                });
+            },
             pipeline_motion_attitude_matrix_submit: (o, mat) => {
                 let s = this._pipeline_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(13, m => {
+                this.serializer.sendMessage(17, m => {
                     m.type(o);
                     m.matrix4x4(mat);
                 });
@@ -169,7 +222,7 @@ export class zappar_client {
                 let newId = (this._latestId++);
                 let s = {};
                 this._camera_source_state_by_instance.set(newId, s);
-                this.serializer.sendMessage(29, m => {
+                this.serializer.sendMessage(33, m => {
                     m.type(newId);
                     m.type(pipeline);
                     m.string(device_id);
@@ -181,7 +234,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._camera_source_state_by_instance.delete(o);
-                this.serializer.sendMessage(30, m => {
+                this.serializer.sendMessage(34, m => {
                     m.type(o);
                 });
             },
@@ -190,7 +243,7 @@ export class zappar_client {
                 let newId = (this._latestId++);
                 let s = {};
                 this._sequence_source_state_by_instance.set(newId, s);
-                this.serializer.sendMessage(35, m => {
+                this.serializer.sendMessage(39, m => {
                     m.type(newId);
                     m.type(pipeline);
                 });
@@ -201,7 +254,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._sequence_source_state_by_instance.delete(o);
-                this.serializer.sendMessage(36, m => {
+                this.serializer.sendMessage(40, m => {
                     m.type(o);
                 });
             },
@@ -227,7 +280,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._image_tracker_state_by_instance.delete(o);
-                this.serializer.sendMessage(14, m => {
+                this.serializer.sendMessage(18, m => {
                     m.type(o);
                 });
             },
@@ -293,7 +346,7 @@ export class zappar_client {
                     anchor_expression_coefficients: [],
                 };
                 this._face_tracker_state_by_instance.set(newId, s);
-                this.serializer.sendMessage(20, m => {
+                this.serializer.sendMessage(24, m => {
                     m.type(newId);
                     m.type(pipeline);
                 });
@@ -304,7 +357,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._face_tracker_state_by_instance.delete(o);
-                this.serializer.sendMessage(21, m => {
+                this.serializer.sendMessage(25, m => {
                     m.type(o);
                 });
             },
@@ -312,7 +365,7 @@ export class zappar_client {
                 let s = this._face_tracker_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(22, m => {
+                this.serializer.sendMessage(26, m => {
                     m.type(o);
                     m.dataWithLength(data);
                 });
@@ -327,7 +380,7 @@ export class zappar_client {
                 let s = this._face_tracker_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(23, m => {
+                this.serializer.sendMessage(27, m => {
                     m.type(o);
                     m.bool(enabled);
                 });
@@ -342,7 +395,7 @@ export class zappar_client {
                 let s = this._face_tracker_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(24, m => {
+                this.serializer.sendMessage(28, m => {
                     m.type(o);
                     m.int(num);
                 });
@@ -388,7 +441,7 @@ export class zappar_client {
                 let newId = (this._latestId++);
                 let s = {};
                 this._face_mesh_state_by_instance.set(newId, s);
-                this.serializer.sendMessage(25, m => {
+                this.serializer.sendMessage(29, m => {
                     m.type(newId);
                 });
                 return newId;
@@ -398,7 +451,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._face_mesh_state_by_instance.delete(o);
-                this.serializer.sendMessage(26, m => {
+                this.serializer.sendMessage(30, m => {
                     m.type(o);
                 });
             },
@@ -407,7 +460,7 @@ export class zappar_client {
                 let newId = (this._latestId++);
                 let s = {};
                 this._face_landmark_state_by_instance.set(newId, s);
-                this.serializer.sendMessage(32, m => {
+                this.serializer.sendMessage(36, m => {
                     m.type(newId);
                     m.faceLandmarkName(landmark);
                 });
@@ -418,7 +471,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._face_landmark_state_by_instance.delete(o);
-                this.serializer.sendMessage(33, m => {
+                this.serializer.sendMessage(37, m => {
                     m.type(o);
                 });
             },
@@ -433,7 +486,7 @@ export class zappar_client {
                     formats: (1 << 17) - 1,
                 };
                 this._barcode_finder_state_by_instance.set(newId, s);
-                this.serializer.sendMessage(16, m => {
+                this.serializer.sendMessage(20, m => {
                     m.type(newId);
                     m.type(pipeline);
                 });
@@ -444,7 +497,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._barcode_finder_state_by_instance.delete(o);
-                this.serializer.sendMessage(17, m => {
+                this.serializer.sendMessage(21, m => {
                     m.type(o);
                 });
             },
@@ -452,7 +505,7 @@ export class zappar_client {
                 let s = this._barcode_finder_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(18, m => {
+                this.serializer.sendMessage(22, m => {
                     m.type(o);
                     m.bool(enabled);
                 });
@@ -491,7 +544,7 @@ export class zappar_client {
                 let s = this._barcode_finder_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
-                this.serializer.sendMessage(19, m => {
+                this.serializer.sendMessage(23, m => {
                     m.type(o);
                     m.barcodeFormat(f);
                 });
@@ -515,7 +568,7 @@ export class zappar_client {
                 if (!s)
                     throw new Error("This object has been destroyed");
                 this._instant_world_tracker_state_by_instance.delete(o);
-                this.serializer.sendMessage(15, m => {
+                this.serializer.sendMessage(19, m => {
                     m.type(o);
                 });
             },
@@ -552,6 +605,100 @@ export class zappar_client {
                     m.instantTrackerTransformOrientation(orientation);
                 });
             },
+            // #### world_tracker ####
+            world_tracker_create: (pipeline) => {
+                let newId = (this._latestId++);
+                let s = {
+                    enabled: true,
+                    plane_count: 0,
+                    plane_pose: [],
+                    world_anchor_valid: true,
+                    world_anchor_pose: new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]),
+                    ground_anchor_valid: true,
+                    ground_anchor_pose: new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]),
+                    quality: 0,
+                };
+                this._world_tracker_state_by_instance.set(newId, s);
+                this.serializer.sendMessage(41, m => {
+                    m.type(newId);
+                    m.type(pipeline);
+                });
+                return newId;
+            },
+            world_tracker_destroy: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                this._world_tracker_state_by_instance.delete(o);
+                this.serializer.sendMessage(42, m => {
+                    m.type(o);
+                });
+            },
+            world_tracker_enabled: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.enabled;
+            },
+            world_tracker_enabled_set: (o, enabled) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                this.serializer.sendMessage(43, m => {
+                    m.type(o);
+                    m.bool(enabled);
+                });
+            },
+            world_tracker_quality: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.quality;
+            },
+            world_tracker_plane_count: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.plane_count;
+            },
+            world_tracker_plane_pose_raw: (o, indx) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.plane_pose[indx];
+            },
+            world_tracker_world_anchor_valid: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.world_anchor_valid;
+            },
+            world_tracker_world_anchor_pose_raw: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.world_anchor_pose;
+            },
+            world_tracker_ground_anchor_valid: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.world_anchor_valid;
+            },
+            world_tracker_ground_anchor_pose_raw: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                return s.ground_anchor_pose;
+            },
+            world_tracker_reset: (o) => {
+                let s = this._world_tracker_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+                this.serializer.sendMessage(44, m => {
+                    m.type(o);
+                });
+            },
         };
     }
     processMessages(a) {
@@ -718,6 +865,63 @@ export class zappar_client {
                     inst.pose = msg.matrix4x4();
                     break;
                 }
+                case 27: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    inst.quality = msg.int();
+                    break;
+                }
+                case 21: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    inst.plane_count = msg.int();
+                    break;
+                }
+                case 22: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    let indx = msg.int();
+                    inst.plane_pose[indx] = msg.matrix4x4();
+                    break;
+                }
+                case 23: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    inst.world_anchor_valid = msg.bool();
+                    break;
+                }
+                case 24: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    inst.world_anchor_pose = msg.matrix4x4();
+                    break;
+                }
+                case 23: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    inst.world_anchor_valid = msg.bool();
+                    break;
+                }
+                case 26: {
+                    let handle = msg.type();
+                    let inst = this._world_tracker_state_by_instance.get(handle);
+                    if (!inst)
+                        return;
+                    inst.ground_anchor_pose = msg.matrix4x4();
+                    break;
+                }
             }
         });
     }