@zappar/zappar-cv 3.0.0-beta.7 → 3.0.1-alpha.10

package/lib/gen/zappar-server.js

@@ -24,6 +24,8 @@ export class zappar_server {
         this._barcode_finder_by_instance = new Map();
         this._pipeline_id_by_instant_world_tracker_id = new Map();
         this._instant_world_tracker_by_instance = new Map();
+        this._pipeline_id_by_zapcode_tracker_id = new Map();
+        this._zapcode_tracker_by_instance = new Map();
         this._pipeline_id_by_world_tracker_id = new Map();
         this._world_tracker_by_instance = new Map();
     }
@@ -351,6 +353,40 @@ export class zappar_server {
                     break;
                 }
                 case 41: {
+                    let clientId = msg.type();
+                    let arg_pipeline_id = msg.type();
+                    let arg_pipeline = this._pipeline_by_instance.get(arg_pipeline_id);
+                    let handle = this._impl.zapcode_tracker_create(arg_pipeline);
+                    this._zapcode_tracker_by_instance.set(clientId, handle);
+                    this._pipeline_id_by_zapcode_tracker_id.set(clientId, arg_pipeline_id);
+                    break;
+                }
+                case 44: {
+                    let clientId = msg.type();
+                    let obj = this._zapcode_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.zapcode_tracker_destroy(obj);
+                    this._zapcode_tracker_by_instance.delete(clientId);
+                    break;
+                }
+                case 43: {
+                    let clientId = msg.type();
+                    let obj = this._zapcode_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.zapcode_tracker_target_load_from_memory(obj, msg.dataWithLength());
+                    break;
+                }
+                case 42: {
+                    let clientId = msg.type();
+                    let obj = this._zapcode_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.zapcode_tracker_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 45: {
                     let clientId = msg.type();
                     let arg_pipeline_id = msg.type();
                     let arg_pipeline = this._pipeline_by_instance.get(arg_pipeline_id);
@@ -359,7 +395,7 @@ export class zappar_server {
                     this._pipeline_id_by_world_tracker_id.set(clientId, arg_pipeline_id);
                     break;
                 }
-                case 42: {
+                case 46: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -368,7 +404,7 @@ export class zappar_server {
                     this._world_tracker_by_instance.delete(clientId);
                     break;
                 }
-                case 43: {
+                case 47: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -376,7 +412,7 @@ export class zappar_server {
                     this._impl.world_tracker_enabled_set(obj, msg.bool());
                     break;
                 }
-                case 44: {
+                case 48: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -384,6 +420,22 @@ export class zappar_server {
                     this._impl.world_tracker_reset(obj);
                     break;
                 }
+                case 49: {
+                    let clientId = msg.type();
+                    let obj = this._world_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.world_tracker_tracks_data_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 50: {
+                    let clientId = msg.type();
+                    let obj = this._world_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.world_tracker_projections_data_enabled_set(obj, msg.bool());
+                    break;
+                }
             }
         });
     }
@@ -560,6 +612,36 @@ export class zappar_server {
                 msg.matrix4x4(this._impl.instant_world_tracker_anchor_pose_raw(v));
             });
         }
+        for (let [k, v] of this._zapcode_tracker_by_instance) {
+            let pipeline = this._pipeline_id_by_zapcode_tracker_id.get(k);
+            if (!pipeline)
+                continue;
+            let serializer = this.serializersByPipelineId.get(pipeline);
+            if (!serializer)
+                continue;
+            serializer.sendMessage(24, msg => {
+                msg.type(k);
+                msg.int(this._impl.zapcode_tracker_target_loaded_version(v));
+            });
+            serializer.sendMessage(21, msg => {
+                msg.type(k);
+                msg.int(this._impl.zapcode_tracker_anchor_count(v));
+            });
+            for (let i = 0; i < this._impl.zapcode_tracker_anchor_count(v); i++) {
+                serializer.sendMessage(22, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.string(this._impl.zapcode_tracker_anchor_id(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.zapcode_tracker_anchor_count(v); i++) {
+                serializer.sendMessage(23, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.matrix4x4(this._impl.zapcode_tracker_anchor_pose_raw(v, i));
+                });
+            }
+        }
         for (let [k, v] of this._world_tracker_by_instance) {
             let pipeline = this._pipeline_id_by_world_tracker_id.get(k);
             if (!pipeline)
@@ -567,37 +649,53 @@ export class zappar_server {
             let serializer = this.serializersByPipelineId.get(pipeline);
             if (!serializer)
                 continue;
-            serializer.sendMessage(27, msg => {
+            serializer.sendMessage(31, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_quality(v));
             });
-            serializer.sendMessage(21, msg => {
+            serializer.sendMessage(25, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_plane_count(v));
             });
             for (let i = 0; i < this._impl.world_tracker_plane_count(v); i++) {
-                serializer.sendMessage(22, msg => {
+                serializer.sendMessage(26, msg => {
                     msg.type(k);
                     msg.int(i);
                     msg.matrix4x4(this._impl.world_tracker_plane_pose_raw(v, i));
                 });
             }
-            serializer.sendMessage(23, msg => {
+            serializer.sendMessage(27, msg => {
                 msg.type(k);
                 msg.bool(this._impl.world_tracker_world_anchor_valid(v));
             });
-            serializer.sendMessage(24, msg => {
+            serializer.sendMessage(28, msg => {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_world_anchor_pose_raw(v));
             });
-            serializer.sendMessage(23, msg => {
+            serializer.sendMessage(27, msg => {
                 msg.type(k);
                 msg.bool(this._impl.world_tracker_ground_anchor_valid(v));
             });
-            serializer.sendMessage(26, msg => {
+            serializer.sendMessage(30, msg => {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_ground_anchor_pose_raw(v));
             });
+            serializer.sendMessage(34, msg => {
+                msg.type(k);
+                msg.int(this._impl.world_tracker_tracks_data_size(v));
+            });
+            serializer.sendMessage(33, msg => {
+                msg.type(k);
+                msg.floatArray(this._impl.world_tracker_tracks_data(v));
+            });
+            serializer.sendMessage(37, msg => {
+                msg.type(k);
+                msg.int(this._impl.world_tracker_projections_data_size(v));
+            });
+            serializer.sendMessage(36, msg => {
+                msg.type(k);
+                msg.floatArray(this._impl.world_tracker_projections_data(v));
+            });
         }
     }
 }

package/lib/gen/zappar.d.ts

@@ -10,6 +10,8 @@ export { log_level_t } from "./zappar-native";
 import { image_target_type_t } from "./zappar-native";
 export { image_target_type_t } from "./zappar-native";
 export { world_tracker_quality_t } from "./zappar-native";
+import { camera_profile_t } from "./zappar-native";
+export { camera_profile_t } from "./zappar-native";
 export declare type zappar_pipeline_t = number & {
     _: 'zappar_pipeline_t';
 };
@@ -37,6 +39,9 @@ export declare type zappar_barcode_finder_t = number & {
 export declare type zappar_instant_world_tracker_t = number & {
     _: 'zappar_instant_world_tracker_t';
 };
+export declare type zappar_zapcode_tracker_t = number & {
+    _: 'zappar_zapcode_tracker_t';
+};
 export declare type zappar_world_tracker_t = number & {
     _: 'zappar_world_tracker_t';
 };
@@ -99,6 +104,7 @@ export interface zappar {
     camera_source_destroy(o: zappar_camera_source_t): void;
     camera_source_start(o: zappar_camera_source_t): void;
     camera_source_pause(o: zappar_camera_source_t): void;
+    camera_source_profile_set(o: zappar_camera_source_t, profile: camera_profile_t): void;
     sequence_source_create(pipeline: zappar_pipeline_t): zappar_sequence_source_t;
     sequence_source_destroy(o: zappar_sequence_source_t): void;
     sequence_source_start(o: zappar_sequence_source_t): void;
@@ -183,6 +189,17 @@ export interface zappar {
     instant_world_tracker_anchor_pose(o: zappar_instant_world_tracker_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
     instant_world_tracker_anchor_pose_set_from_camera_offset_raw(o: zappar_instant_world_tracker_t, x: number, y: number, z: number, orientation: instant_world_tracker_transform_orientation_t): void;
     instant_world_tracker_anchor_pose_set_from_camera_offset(o: zappar_instant_world_tracker_t, x: number, y: number, z: number, orientation: instant_world_tracker_transform_orientation_t): void;
+    zapcode_tracker_create(pipeline: zappar_pipeline_t): zappar_zapcode_tracker_t;
+    zapcode_tracker_destroy(o: zappar_zapcode_tracker_t): void;
+    zapcode_tracker_target_load_from_memory(o: zappar_zapcode_tracker_t, data: ArrayBuffer): void;
+    zapcode_tracker_target_loaded_version(o: zappar_zapcode_tracker_t): number;
+    zapcode_tracker_enabled(o: zappar_zapcode_tracker_t): boolean;
+    zapcode_tracker_enabled_set(o: zappar_zapcode_tracker_t, enabled: boolean): void;
+    zapcode_tracker_anchor_count(o: zappar_zapcode_tracker_t): number;
+    zapcode_tracker_anchor_id(o: zappar_zapcode_tracker_t, indx: number): string;
+    zapcode_tracker_anchor_pose_raw(o: zappar_zapcode_tracker_t, indx: number): Float32Array;
+    zapcode_tracker_anchor_pose_camera_relative(o: zappar_zapcode_tracker_t, indx: number, mirror: boolean): Float32Array;
+    zapcode_tracker_anchor_pose(o: zappar_zapcode_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
     world_tracker_create(pipeline: zappar_pipeline_t): zappar_world_tracker_t;
     world_tracker_destroy(o: zappar_world_tracker_t): void;
     world_tracker_enabled(o: zappar_world_tracker_t): boolean;
@@ -201,4 +218,11 @@ export interface zappar {
     world_tracker_ground_anchor_pose_camera_relative(o: zappar_world_tracker_t, mirror: boolean): Float32Array;
     world_tracker_ground_anchor_pose(o: zappar_world_tracker_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
     world_tracker_reset(o: zappar_world_tracker_t): void;
+    world_tracker_tracks_data_enabled(o: zappar_world_tracker_t): boolean;
+    world_tracker_tracks_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
+    world_tracker_tracks_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_tracks_data(o: zappar_world_tracker_t): Float32Array;
+    world_tracker_projections_data_enabled_set(o: zappar_world_tracker_t, tracks_data_enabled: boolean): void;
+    world_tracker_projections_data_size(o: zappar_world_tracker_t): number;
+    world_tracker_projections_data(o: zappar_world_tracker_t): Float32Array;
 }

package/lib/gen/zappar.js

@@ -5,3 +5,4 @@ export { instant_world_tracker_transform_orientation_t } from "./zappar-native";
 export { log_level_t } from "./zappar-native";
 export { image_target_type_t } from "./zappar-native";
 export { world_tracker_quality_t } from "./zappar-native";
+export { camera_profile_t } from "./zappar-native";

package/lib/html-element-source.d.ts

@@ -1,4 +1,4 @@
-import { zappar_pipeline_t } from "./gen/zappar";
+import { camera_profile_t, zappar_pipeline_t } from "./gen/zappar";
 import { Source } from "./source";
 import { zappar_html_element_source_t } from "./additional";
 export declare class HTMLElementSource extends Source {
@@ -14,6 +14,7 @@ export declare class HTMLElementSource extends Source {
     private _cameraToDeviceTransform;
     private _cameraToDeviceTransformUserFacing;
     private _cameraModel;
+    private _profile;
     static createVideoElementSource(p: zappar_pipeline_t, element: HTMLVideoElement | HTMLImageElement): zappar_html_element_source_t;
     static getVideoElementSource(m: zappar_html_element_source_t): HTMLElementSource | undefined;
     constructor(_video: HTMLVideoElement | HTMLImageElement, _pipeline: zappar_pipeline_t);
@@ -24,4 +25,5 @@ export declare class HTMLElementSource extends Source {
     getFrame(currentlyProcessing: boolean): void;
     private _processFrame;
     uploadGL(): void;
+    setProfile(p: camera_profile_t): void;
 }

package/lib/html-element-source.js

@@ -1,3 +1,4 @@
+import { camera_profile_t } from "./gen/zappar";
 import { Pipeline } from "./pipeline";
 import { Source } from "./source";
 import { profile } from "./profile";
@@ -19,6 +20,7 @@ export class HTMLElementSource extends Source {
         this._cameraToDeviceTransform = mat4.create();
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+        this._profile = camera_profile_t.DEFAULT;
         mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
         let video = this._video;
         if (this._video instanceof HTMLVideoElement) {
@@ -109,14 +111,15 @@ export class HTMLElementSource extends Source {
             }
             if (!this._currentVideoTexture)
                 return undefined;
-            this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing);
+            this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
             this._isUploadFrame = !this._isUploadFrame;
             return undefined;
         }
         if (currentlyProcessing || !this._currentVideoTexture)
             return undefined;
         this._isUploadFrame = !this._isUploadFrame;
-        let greySize = profile.dataWidth * profile.dataHeight;
+        const [dataWidth, dataHeight] = profile.getDataSize(this._profile);
+        let greySize = dataWidth * dataHeight;
         let pixels = pipeline.cameraPixelArrays.pop();
         while (pixels) {
             if (pixels.byteLength === greySize)
@@ -128,14 +131,17 @@ export class HTMLElementSource extends Source {
         }
         let tex = this._currentVideoTexture;
         this._currentVideoTexture = undefined;
-        let focalLength = 300.0 * profile.dataWidth / 320.0;
+        let focalLength = 300.0 * dataWidth / 320.0;
         this._cameraModel[0] = focalLength;
         this._cameraModel[1] = focalLength;
-        this._cameraModel[2] = profile.dataWidth * 0.5;
-        this._cameraModel[3] = profile.dataHeight * 0.5;
-        return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
+        this._cameraModel[2] = dataWidth * 0.5;
+        this._cameraModel[3] = dataHeight * 0.5;
+        return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels, this._profile)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
     }
     uploadGL() {
         // No-op as already uploaded
     }
+    setProfile(p) {
+        this._profile = p;
+    }
 }

package/lib/image-process-gl.d.ts

@@ -1,3 +1,4 @@
+import { camera_profile_t } from "./gen/zappar-native";
 export interface ProcessedFrame {
     data?: ArrayBuffer;
     texture: WebGLTexture | undefined;
@@ -28,8 +29,8 @@ export declare class ImageProcessGL {
     constructor(_gl: WebGL2RenderingContext | WebGLRenderingContext);
     resetGLContext(): void;
     destroy(): void;
-    uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean): void;
-    readFrame(texture: WebGLTexture, pixels: ArrayBuffer): ProcessedFrame;
+    uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t): void;
+    readFrame(texture: WebGLTexture, pixels: ArrayBuffer, cp: camera_profile_t): ProcessedFrame;
     private _updateTransforms;
     private _getCameraUvTransform;
     private _getFramebuffer;

package/lib/image-process-gl.js

@@ -33,7 +33,7 @@ export class ImageProcessGL {
     destroy() {
         this.resetGLContext();
     }
-    uploadFrame(texture, img, rotation, fc) {
+    uploadFrame(texture, img, rotation, fc, cp) {
         let gl = this._gl;
         const glStateManager = GLStateManager.get(gl);
         glStateManager.push();
@@ -75,7 +75,8 @@ export class ImageProcessGL {
         if (videoHeight > videoWidth)
             videoHeight = [videoWidth, videoWidth = videoHeight][0];
         this._updateTransforms(rotation, fc);
-        let framebuffer = this._getFramebuffer(gl, profile.dataWidth / 4, profile.dataHeight);
+        const [dataWidth, dataHeight] = profile.getDataSize(cp);
+        let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
         let vbo = this._getVertexBuffer(gl);
         let ibo = this._getIndexBuffer(gl);
         let shader = this._getGreyscaleShader(gl);
@@ -123,7 +124,7 @@ export class ImageProcessGL {
         // Tell WebGL to use our program when drawing
         gl.useProgram(shader.program);
         // Specify greyscale width for the correct offsets, and the uv transform
-        gl.uniform1f(shader.uTexWidthLoc, profile.dataWidth);
+        gl.uniform1f(shader.uTexWidthLoc, dataWidth);
         gl.uniformMatrix4fv(shader.uUvTransformLoc, false, this._cameraUvTransform);
         gl.activeTexture(gl.TEXTURE0);
         // Bind the texture to texture unit 0
@@ -167,11 +168,12 @@ export class ImageProcessGL {
         if (reenableStencilTest)
             gl.enable(gl.STENCIL_TEST);
     }
-    readFrame(texture, pixels) {
+    readFrame(texture, pixels, cp) {
         let gl = this._gl;
         let pixelsView = new Uint8Array(pixels);
         const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
-        let framebuffer = this._getFramebuffer(gl, profile.dataWidth / 4, profile.dataHeight);
+        const [dataWidth, dataHeight] = profile.getDataSize(cp);
+        let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
         gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
         gl.readPixels(0, 0, this._framebufferWidth, this._framebufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixelsView);
         gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
@@ -179,8 +181,8 @@ export class ImageProcessGL {
             uvTransform: this._cameraUvTransform,
             data: pixels,
             texture,
-            dataWidth: profile.dataWidth,
-            dataHeight: profile.dataHeight,
+            dataWidth,
+            dataHeight,
             userFacing: this._computedFrontCameraRotation
         };
     }

package/lib/imagebitmap-camera-source.d.ts

@@ -1,4 +1,4 @@
-import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
+import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
 import { CameraFrameInfo, Source } from "./source";
 export declare class ImageBitmapCameraSource extends Source {
     private _impl;
@@ -18,8 +18,10 @@ export declare class ImageBitmapCameraSource extends Source {
     private _cameraToDeviceTransform;
     private _cameraToDeviceTransformUserFacing;
     private _cameraModel;
+    private _profile;
     constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
     destroy(): void;
+    setProfile(p: camera_profile_t): void;
     private _stop;
     pause(): void;
     start(): void;

package/lib/imagebitmap-camera-source.js

@@ -7,6 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+import { camera_profile_t } from "./gen/zappar";
 import { profile } from "./profile";
 import { Pipeline } from "./pipeline";
 import { Source } from "./source";
@@ -32,6 +33,7 @@ export class ImageBitmapCameraSource extends Source {
         this._cameraToDeviceTransform = mat4.create();
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+        this._profile = camera_profile_t.DEFAULT;
         this._hasStartedOrientation = false;
         this._lastTimestamp = -1;
         this._deviceMotionListener = (ev) => {
@@ -85,6 +87,9 @@ export class ImageBitmapCameraSource extends Source {
         this.pause();
         deleteCameraSource(this._impl);
     }
+    setProfile(p) {
+        this._profile = p;
+    }
     _stop() {
         if (!this._currentStream)
             return;
@@ -185,11 +190,12 @@ export class ImageBitmapCameraSource extends Source {
         let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
-        let focalLength = 300.0 * profile.dataWidth / 320.0;
+        const [dataWidth, dataHeight] = profile.getDataSize(this._profile);
+        let focalLength = 300.0 * dataWidth / 320.0;
         this._cameraModel[0] = focalLength;
         this._cameraModel[1] = focalLength;
-        this._cameraModel[2] = profile.dataWidth * 0.5;
-        this._cameraModel[3] = profile.dataHeight * 0.5;
+        this._cameraModel[2] = dataWidth * 0.5;
+        this._cameraModel[3] = dataHeight * 0.5;
         let token = pipeline.registerToken({
             dataWidth: this._canvas.width,
             dataHeight: this._canvas.height,
@@ -199,7 +205,7 @@ export class ImageBitmapCameraSource extends Source {
             cameraModel: this._cameraModel.slice(),
             cameraToDevice: this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform
         });
-        (_b = (_a = Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendImageBitmapToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, imageBitmap, rotation, this._isUserFacing, token, this._cameraModel, this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform);
+        (_b = (_a = Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendImageBitmapToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, imageBitmap, rotation, this._isUserFacing, token, this._cameraModel, this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform, this._profile);
         return;
     }
     _getUserMedia() {

package/lib/index.d.ts

@@ -4,4 +4,4 @@ import { Options } from "./options";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(opts?: Options): Zappar;
 export { CameraFrameData } from './camera-frame-data';
-export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";
+export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, zappar_zapcode_tracker_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";

package/lib/index.js

@@ -4,4 +4,4 @@ export function initialize(opts) {
     console.log(`Zappar CV v${VERSION}`);
     return nativeInitialize(opts);
 }
-export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";
+export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";

package/lib/mstp-camera-source.d.ts

@@ -1,4 +1,4 @@
-import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
+import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
 import { CameraFrameInfo, Source } from "./source";
 export declare class MSTPCameraSource extends Source {
     private _impl;
@@ -11,6 +11,9 @@ export declare class MSTPCameraSource extends Source {
     private _isPaused;
     private _isUserFacing;
     private _cameraToScreenRotation;
+    private _accelerometer?;
+    private _linearAcceleration?;
+    private _gyroscope?;
     constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
     destroy(): void;
     private _stop;
@@ -21,11 +24,18 @@ export declare class MSTPCameraSource extends Source {
     private _getUserMedia;
     streamEnded(): void;
     private _syncCamera;
+    setProfile(profile: camera_profile_t): void;
     private _hasStartedOrientation;
     private _lastTimestamp;
     private _deviceMotionListener;
     private _startDeviceOrientation;
     private _startDeviceMotion;
+    private _lastAccelerometerTimestamp;
+    private _accelerometerReading;
+    private _lastLinearAccelerationTimestamp;
+    private _linearAccelerationReading;
+    private _lastGyroscopeReading;
+    private _gyroscopeReading;
     private _stopDeviceMotion;
     uploadGL(info: CameraFrameInfo): void;
 }

package/lib/mstp-camera-source.js

@@ -31,12 +31,15 @@ export class MSTPCameraSource extends Source {
             if (!pipeline)
                 return;
             let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
-            let interval = ev.interval;
-            if (this._lastTimestamp > -1 && !profile.trustSensorIntervals) {
-                interval = Math.max(timeStamp - this._lastTimestamp, interval);
+            const interval = ev.interval * profile.intervalMultiplier;
+            if (profile.trustSensorIntervals) {
+                if (this._lastTimestamp === -1)
+                    this._lastTimestamp = timeStamp;
+                else {
+                    this._lastTimestamp += interval;
+                    timeStamp = this._lastTimestamp;
+                }
             }
-            this._lastTimestamp = timeStamp;
-            interval *= profile.intervalMultiplier;
             if (ev.acceleration !== null &&
                 ev.acceleration.x !== null &&
                 ev.acceleration.y !== null &&
@@ -61,6 +64,47 @@ export class MSTPCameraSource extends Source {
                 this._startDeviceOrientation();
             }
         };
+        this._lastAccelerometerTimestamp = -1;
+        this._accelerometerReading = () => {
+            var _a;
+            if (!this._accelerometer || this._accelerometer.x === undefined || this._accelerometer.y === undefined || this._accelerometer.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._accelerometer.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastAccelerometerTimestamp > -1 ? timestamp - this._lastAccelerometerTimestamp : 30;
+            this._lastAccelerometerTimestamp = timestamp;
+            pipeline.motionAccelerometerSubmit(timestamp, this._accelerometer.x * profile.deviceMotionMutliplier, this._accelerometer.y * profile.deviceMotionMutliplier, this._accelerometer.z * profile.deviceMotionMutliplier);
+            pipeline.motionAccelerometerWithGravitySubmitInt(timestamp, interval, this._accelerometer.x * profile.deviceMotionMutliplier, this._accelerometer.y * profile.deviceMotionMutliplier, this._accelerometer.z * profile.deviceMotionMutliplier);
+        };
+        this._lastLinearAccelerationTimestamp = -1;
+        this._linearAccelerationReading = () => {
+            var _a;
+            if (!this._linearAcceleration || this._linearAcceleration.x === undefined || this._linearAcceleration.y === undefined || this._linearAcceleration.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._linearAcceleration.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastLinearAccelerationTimestamp > -1 ? timestamp - this._lastLinearAccelerationTimestamp : 30;
+            this._lastLinearAccelerationTimestamp = timestamp;
+            pipeline.motionAccelerometerWithoutGravitySubmitInt(timestamp, interval, this._linearAcceleration.x * profile.deviceMotionMutliplier, this._linearAcceleration.y * profile.deviceMotionMutliplier, this._linearAcceleration.z * profile.deviceMotionMutliplier);
+        };
+        this._lastGyroscopeReading = -1;
+        this._gyroscopeReading = () => {
+            var _a;
+            if (!this._gyroscope || this._gyroscope.x === undefined || this._gyroscope.y === undefined || this._gyroscope.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._gyroscope.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastGyroscopeReading > -1 ? timestamp - this._lastGyroscopeReading : 30;
+            this._lastGyroscopeReading = timestamp;
+            pipeline.motionRotationRateSubmit(timestamp, -this._gyroscope.x, -this._gyroscope.y, -this._gyroscope.z);
+            pipeline.motionRotationRateSubmitInt(timestamp, interval, this._gyroscope.x * 180 / Math.PI, this._gyroscope.y * 180 / Math.PI, this._gyroscope.z * 180 / Math.PI);
+        };
         zcout("Using MSTP camera source");
     }
     destroy() {
@@ -205,6 +249,11 @@ export class MSTPCameraSource extends Source {
             }
         });
     }
+    setProfile(profile) {
+        let p = Pipeline.get(this._pipeline);
+        if (p)
+            p.sendCameraProfileToWorker(this._impl, profile);
+    }
     _startDeviceOrientation() {
         if (this._hasStartedOrientation)
             return;
@@ -221,9 +270,38 @@ export class MSTPCameraSource extends Source {
         });
     }
     _startDeviceMotion() {
-        window.addEventListener("devicemotion", this._deviceMotionListener, false);
+        try {
+            this._accelerometer = new Accelerometer({ frequency: 60 });
+            this._linearAcceleration = new LinearAccelerationSensor({ frequency: 60 });
+            this._gyroscope = new Gyroscope({ frequency: 60 });
+            // throw new Error();
+            this._accelerometer.addEventListener('reading', this._accelerometerReading);
+            this._accelerometer.start();
+            this._linearAcceleration.addEventListener('reading', this._linearAccelerationReading);
+            this._linearAcceleration.start();
+            this._gyroscope.addEventListener('reading', this._gyroscopeReading);
+            this._gyroscope.start();
+        }
+        catch (err) {
+            window.addEventListener("devicemotion", this._deviceMotionListener, false);
+        }
     }
     _stopDeviceMotion() {
+        if (this._accelerometer) {
+            this._accelerometer.stop();
+            this._accelerometer.removeEventListener('reading', this._accelerometerReading);
+            delete this._accelerometer;
+        }
+        if (this._linearAcceleration) {
+            this._linearAcceleration.stop();
+            this._linearAcceleration.removeEventListener('reading', this._linearAccelerationReading);
+            delete this._linearAcceleration;
+        }
+        if (this._gyroscope) {
+            this._gyroscope.stop();
+            this._gyroscope.removeEventListener('reading', this._gyroscopeReading);
+            delete this._gyroscope;
+        }
         window.removeEventListener("devicemotion", this._deviceMotionListener);
     }
     uploadGL(info) {