@zappar/zappar-cv 3.9.2 → 3.11.0

package/lib/gen/zappar-server.js

@@ -551,6 +551,14 @@ export class zappar_server {
                     this._d3_tracker_by_instance.delete(clientId);
                     break;
                 }
+                case 64: {
+                    let clientId = msg.type();
+                    let obj = this._d3_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.d3_tracker_enabled_set(obj, msg.bool());
+                    break;
+                }
             }
         });
     }
@@ -913,6 +921,73 @@ export class zappar_server {
             let serializer = this.serializersByPipelineId.get(pipeline);
             if (!serializer)
                 continue;
+            serializer.sendMessage(56, msg => {
+                msg.type(k);
+                msg.int(this._impl.d3_tracker_count(v));
+            });
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(57, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.string(this._impl.d3_tracker_id(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(58, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.matrix4x4(this._impl.d3_tracker_pose0_raw(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(59, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.matrix4x4(this._impl.d3_tracker_pose1_raw(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(60, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.int(this._impl.d3_tracker_type(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(61, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.int(this._impl.d3_tracker_category(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(62, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.string(this._impl.d3_tracker_qr(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(63, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.int(this._impl.d3_tracker_dense(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(64, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.floatArray(this._impl.d3_tracker_landmarks_data(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.d3_tracker_count(v); i++) {
+                serializer.sendMessage(65, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.int(this._impl.d3_tracker_landmarks_data_size(v, i));
+                });
+            }
         }
     }
 }

package/lib/gen/zappar.d.ts

@@ -125,6 +125,7 @@ export interface zappar {
     camera_source_start(o: zappar_camera_source_t): void;
     camera_source_pause(o: zappar_camera_source_t): void;
     camera_source_profile_set(o: zappar_camera_source_t, profile: camera_profile_t): void;
+    camera_source_preferred_resolution_set(o: zappar_camera_source_t, width: number, height: number): void;
     sequence_source_create(pipeline: zappar_pipeline_t): zappar_sequence_source_t;
     sequence_source_destroy(o: zappar_sequence_source_t): void;
     sequence_source_start(o: zappar_sequence_source_t): void;
@@ -295,25 +296,25 @@ export interface zappar {
     custom_anchor_pose_set_with_parent(o: zappar_custom_anchor_t, pose: Float32Array, anchor_id: string): void;
     d3_tracker_create(pipeline: zappar_pipeline_t): zappar_d3_tracker_t;
     d3_tracker_destroy(o: zappar_d3_tracker_t): void;
-    d3_tracker_id(o: zappar_d3_tracker_t, indx: number): string;
     d3_tracker_enabled(o: zappar_d3_tracker_t): boolean;
     d3_tracker_enabled_set(o: zappar_d3_tracker_t, enabled: boolean): void;
     d3_tracker_process_max_resolution_set(o: zappar_d3_tracker_t, v: boolean): void;
     d3_tracker_count(o: zappar_d3_tracker_t): number;
-    d3_tracker_qr(o: zappar_d3_tracker_t, indx: number): string;
-    d3_tracker_pose_raw(o: zappar_d3_tracker_t, indx: number): Float32Array;
-    d3_tracker_pose_camera_relative(o: zappar_d3_tracker_t, indx: number, mirror: boolean): Float32Array;
-    d3_tracker_pose(o: zappar_d3_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
+    d3_tracker_id(o: zappar_d3_tracker_t, indx: number): string;
     d3_tracker_pose0_raw(o: zappar_d3_tracker_t, indx: number): Float32Array;
     d3_tracker_pose0_camera_relative(o: zappar_d3_tracker_t, indx: number, mirror: boolean): Float32Array;
     d3_tracker_pose0(o: zappar_d3_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
     d3_tracker_pose1_raw(o: zappar_d3_tracker_t, indx: number): Float32Array;
     d3_tracker_pose1_camera_relative(o: zappar_d3_tracker_t, indx: number, mirror: boolean): Float32Array;
     d3_tracker_pose1(o: zappar_d3_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
-    d3_tracker_landmarks_data(o: zappar_d3_tracker_t, indx: number): Float32Array;
-    d3_tracker_landmarks_data_size(o: zappar_d3_tracker_t, indx: number): number;
     d3_tracker_type(o: zappar_d3_tracker_t, indx: number): number;
-    d3_tracker_dense(o: zappar_d3_tracker_t, indx: number): number;
     d3_tracker_category(o: zappar_d3_tracker_t, indx: number): number;
+    d3_tracker_qr(o: zappar_d3_tracker_t, indx: number): string;
+    d3_tracker_dense(o: zappar_d3_tracker_t, indx: number): number;
+    d3_tracker_landmarks_data(o: zappar_d3_tracker_t, indx: number): Float32Array;
+    d3_tracker_landmarks_data_size(o: zappar_d3_tracker_t, indx: number): number;
+    d3_tracker_pose_raw(o: zappar_d3_tracker_t, indx: number): Float32Array;
+    d3_tracker_pose_camera_relative(o: zappar_d3_tracker_t, indx: number, mirror: boolean): Float32Array;
+    d3_tracker_pose(o: zappar_d3_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
     d3_tracker_size_for_id_set(o: zappar_d3_tracker_t, id: string, size: number): void;
 }

package/lib/html-element-source.d.ts

@@ -10,6 +10,7 @@ export declare class HTMLElementSource extends Source {
     protected _cameraToScreenRotation: number;
     private _isUploadFrame;
     private _currentVideoTexture;
+    private _currentDataSize;
     private _imageProcessor;
     private _cameraToDeviceTransform;
     private _cameraToDeviceTransformUserFacing;

package/lib/html-element-source.js

@@ -17,6 +17,7 @@ export class HTMLElementSource extends Source {
         this._isUserFacing = false;
         this._cameraToScreenRotation = 0;
         this._isUploadFrame = true;
+        this._currentDataSize = [320, 240];
         this._cameraToDeviceTransform = mat4.create();
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
@@ -148,7 +149,7 @@ export class HTMLElementSource extends Source {
                 return undefined;
             if (pipeline.getCameraFrameDataRGBEnabled())
                 this._currentRgbFrame = this._generateRGBFrame();
-            this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
+            this._currentDataSize = this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
             this._requestVideoFrameCallback();
             this._isUploadFrame = !this._isUploadFrame;
             return undefined;
@@ -156,7 +157,7 @@ export class HTMLElementSource extends Source {
         if (currentlyProcessing || !this._currentVideoTexture)
             return undefined;
         this._isUploadFrame = !this._isUploadFrame;
-        const [dataWidth, dataHeight] = profile.getDataSize(this._profile);
+        const [dataWidth, dataHeight] = this._currentDataSize;
         let greySize = dataWidth * dataHeight;
         let pixels = pipeline.cameraPixelArrays.pop();
         while (pixels) {
@@ -174,7 +175,7 @@ export class HTMLElementSource extends Source {
         this._cameraModel[1] = focalLength;
         this._cameraModel[2] = dataWidth * 0.5;
         this._cameraModel[3] = dataHeight * 0.5;
-        const info = Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels, this._profile)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
+        const info = Object.assign(Object.assign({}, this._imageProcessor.readFrame(pixels)), { texture: tex, cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
         if (this._currentRgbFrame) {
             info.rgbFrame = this._currentRgbFrame;
             this._currentRgbFrame = undefined;

package/lib/image-process-gl.d.ts

@@ -1,7 +1,6 @@
 import { camera_profile_t } from "./gen/zappar-native";
 export interface ProcessedFrame {
     data?: ArrayBuffer;
-    texture: WebGLTexture | undefined;
     dataWidth: number;
     dataHeight: number;
     uvTransform?: Float32Array;
@@ -17,6 +16,7 @@ export declare class ImageProcessGL {
     private _computedTransformRotation;
     private _computedFrontCameraRotation;
     private _cameraUvTransform;
+    private _dataSize;
     private _framebufferWidth;
     private _framebufferHeight;
     private _framebufferId;
@@ -29,10 +29,9 @@ export declare class ImageProcessGL {
     constructor(_gl: WebGL2RenderingContext | WebGLRenderingContext);
     resetGLContext(): void;
     destroy(): void;
-    uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t): void;
-    uploadFrameForSize(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, dataWidth: number, dataHeight: number): void;
-    readFrame(texture: WebGLTexture, pixels: ArrayBuffer, cp: camera_profile_t): ProcessedFrame;
-    readFrameForSize(texture: WebGLTexture, pixels: ArrayBuffer, dataWidth: number, dataHeight: number): ProcessedFrame;
+    uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t): [number, number];
+    uploadFrameForSize(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap | null, rotation: number, fc: boolean, dataWidth: number, dataHeight: number): void;
+    readFrame(pixels: ArrayBuffer): ProcessedFrame;
     private _updateTransforms;
     private _getCameraUvTransform;
     private _getFramebuffer;

package/lib/image-process-gl.js

@@ -1,6 +1,6 @@
 import { mat4 } from "gl-matrix";
 import { GLStateManager } from "./gl-state-manager";
-import { profile } from "./profile";
+import { getDataSizeForSource } from "./profile";
 import { compileShader, linkProgram } from "./shader";
 export class ImageProcessGL {
     constructor(_gl) {
@@ -13,6 +13,7 @@ export class ImageProcessGL {
         this._computedTransformRotation = -1;
         this._computedFrontCameraRotation = false;
         this._cameraUvTransform = mat4.create();
+        this._dataSize = [0, 0];
         this._framebufferWidth = 0;
         this._framebufferHeight = 0;
         this._framebufferId = null;
@@ -34,8 +35,9 @@ export class ImageProcessGL {
         this.resetGLContext();
     }
     uploadFrame(texture, img, rotation, fc, cp) {
-        const [dataWidth, dataHeight] = profile.getDataSize(cp);
-        this.uploadFrameForSize(texture, img, rotation, fc, dataWidth, dataHeight);
+        const dataSize = getDataSizeForSource(cp, img, rotation);
+        this.uploadFrameForSize(texture, img, rotation, fc, ...dataSize);
+        return dataSize;
     }
     uploadFrameForSize(texture, img, rotation, fc, dataWidth, dataHeight) {
         let gl = this._gl;
@@ -61,24 +63,15 @@ export class ImageProcessGL {
         gl.disable(gl.STENCIL_TEST);
         gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
         gl.bindTexture(gl.TEXTURE_2D, texture);
-        const level = 0;
-        const internalFormat = gl.RGBA;
-        const srcFormat = gl.RGBA;
-        const srcType = gl.UNSIGNED_BYTE;
-        gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, img);
-        let videoWidth = 0;
-        let videoHeight = 0;
-        if (typeof HTMLVideoElement !== "undefined" && img instanceof HTMLVideoElement) {
-            videoWidth = img.videoWidth;
-            videoHeight = img.videoHeight;
+        if (img) {
+            const level = 0;
+            const internalFormat = gl.RGBA;
+            const srcFormat = gl.RGBA;
+            const srcType = gl.UNSIGNED_BYTE;
+            gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, img);
         }
-        else {
-            videoWidth = img.width;
-            videoHeight = img.height;
-        }
-        if (videoHeight > videoWidth)
-            videoHeight = [videoWidth, videoWidth = videoHeight][0];
         this._updateTransforms(rotation, fc);
+        this._dataSize = [dataWidth, dataHeight];
         let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
         let vbo = this._getVertexBuffer(gl);
         let ibo = this._getIndexBuffer(gl);
@@ -171,13 +164,10 @@ export class ImageProcessGL {
         if (reenableStencilTest)
             gl.enable(gl.STENCIL_TEST);
     }
-    readFrame(texture, pixels, cp) {
-        const [dataWidth, dataHeight] = profile.getDataSize(cp);
-        return this.readFrameForSize(texture, pixels, dataWidth, dataHeight);
-    }
-    readFrameForSize(texture, pixels, dataWidth, dataHeight) {
+    readFrame(pixels) {
         let gl = this._gl;
         let pixelsView = new Uint8Array(pixels);
+        const [dataWidth, dataHeight] = this._dataSize;
         const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
         let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
         gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
@@ -186,7 +176,6 @@ export class ImageProcessGL {
         return {
             uvTransform: this._cameraUvTransform,
             data: pixels,
-            texture,
             dataWidth,
             dataHeight,
             userFacing: this._computedFrontCameraRotation

package/lib/imagebitmap-camera-source.d.ts

@@ -8,6 +8,8 @@ export declare class ImageBitmapCameraSource extends Source {
     static DEFAULT_DEVICE_ID: string;
     private _currentStream;
     private _activeDeviceId;
+    private _preferredWidth;
+    private _preferredHeight;
     private _isPaused;
     private _isUserFacing;
     private _hadFrames;
@@ -22,6 +24,7 @@ export declare class ImageBitmapCameraSource extends Source {
     constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
     destroy(): void;
     setProfile(p: camera_profile_t): void;
+    setPreferredResolution(w: number, h: number): void;
     private _stop;
     pause(): void;
     start(): void;

package/lib/imagebitmap-camera-source.js

@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { camera_profile_t } from "./gen/zappar";
-import { profile } from "./profile";
+import { profile, getDataSizeForSource } from "./profile";
 import { Pipeline } from "./pipeline";
 import { Source } from "./source";
 import { cameraRotationForScreenOrientation } from "./cameramodel";
@@ -23,6 +23,8 @@ export class ImageBitmapCameraSource extends Source {
         this._deviceId = _deviceId;
         this._currentStream = null;
         this._activeDeviceId = null;
+        this._preferredWidth = profile.videoWidth;
+        this._preferredHeight = profile.videoHeight;
         this._isPaused = true;
         this._isUserFacing = false;
         this._hadFrames = false;
@@ -90,6 +92,10 @@ export class ImageBitmapCameraSource extends Source {
     setProfile(p) {
         this._profile = p;
     }
+    setPreferredResolution(w, h) {
+        this._preferredWidth = w;
+        this._preferredHeight = h;
+    }
     _stop() {
         if (!this._currentStream)
             return;
@@ -132,8 +138,8 @@ export class ImageBitmapCameraSource extends Source {
                 audio: false,
                 video: {
                     facingMode: facingMode,
-                    width: profile.videoWidth,
-                    height: profile.videoHeight,
+                    width: this._preferredWidth,
+                    height: this._preferredHeight,
                     frameRate: profile.requestHighFrameRate ? 60 : undefined,
                     deviceId: deviceId
                 }
@@ -190,7 +196,7 @@ export class ImageBitmapCameraSource extends Source {
         let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
-        const [dataWidth, dataHeight] = profile.getDataSize(this._profile);
+        const [dataWidth, dataHeight] = getDataSizeForSource(this._profile, imageBitmap, rotation);
         let focalLength = 300.0 * dataWidth / 320.0;
         this._cameraModel[0] = focalLength;
         this._cameraModel[1] = focalLength;
@@ -281,10 +287,11 @@ export class ImageBitmapCameraSource extends Source {
         let texture = pipeline.getVideoTexture();
         if (!texture)
             return;
+        const previousBoundTexture = gl.getParameter(gl.TEXTURE_BINDING_2D);
         gl.bindTexture(gl.TEXTURE_2D, texture);
         gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
         gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, info.frame);
-        gl.bindTexture(gl.TEXTURE_2D, null);
+        gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
         info.texture = texture;
     }
 }

package/lib/mstp-camera-source.d.ts

@@ -8,6 +8,8 @@ export declare class MSTPCameraSource extends Source {
     static DEFAULT_DEVICE_ID: string;
     private _currentStream;
     private _activeDeviceId;
+    private _preferredWidth;
+    private _preferredHeight;
     private _isPaused;
     private _isUserFacing;
     private _cameraToScreenRotation;
@@ -20,6 +22,7 @@ export declare class MSTPCameraSource extends Source {
     private _stop;
     pause(): void;
     start(): void;
+    setPreferredResolution(w: number, h: number): void;
     private _getConstraints;
     getFrame(allowRead: boolean): void;
     private _getUserMedia;

package/lib/mstp-camera-source.js

@@ -21,6 +21,8 @@ export class MSTPCameraSource extends Source {
         this._deviceId = _deviceId;
         this._currentStream = null;
         this._activeDeviceId = null;
+        this._preferredWidth = profile.videoWidth;
+        this._preferredHeight = profile.videoHeight;
         this._isPaused = true;
         this._isUserFacing = false;
         this._cameraToScreenRotation = 0;
@@ -150,6 +152,10 @@ export class MSTPCameraSource extends Source {
         this._startDeviceMotion();
         this._syncCamera();
     }
+    setPreferredResolution(w, h) {
+        this._preferredWidth = w;
+        this._preferredHeight = h;
+    }
     _getConstraints() {
         return __awaiter(this, void 0, void 0, function* () {
             let deviceId;
@@ -166,8 +172,8 @@ export class MSTPCameraSource extends Source {
                 audio: false,
                 video: {
                     facingMode: facingMode,
-                    width: profile.videoWidth,
-                    height: profile.videoHeight,
+                    width: this._preferredWidth,
+                    height: this._preferredHeight,
                     frameRate: profile.requestHighFrameRate ? 60 : 30,
                     deviceId: deviceId
                 }

package/lib/native.js

@@ -36,6 +36,20 @@ import { WebXRCameraSource } from "./webxr-camera-source";
 import { WebXRWorldTracker } from "./webxr-world-tracker";
 let client;
 const pipelineByWorldTracker = new Map();
+const d3TrackerSizeById = new Map();
+function applyD3SizeToPose(o, indx, pose) {
+    var _a;
+    const anchor_id = client.d3_tracker_id(o, indx);
+    if (anchor_id.length <= 0)
+        return;
+    const size = (_a = d3TrackerSizeById.get(o)) === null || _a === void 0 ? void 0 : _a.get(anchor_id);
+    if (size === undefined)
+        return pose;
+    const scale = 0.5 * size;
+    pose[12] *= scale;
+    pose[13] *= scale;
+    pose[14] *= scale;
+}
 const _temporaryMatrix = mat4.create();
 const _temporaryVec3 = vec3.create();
 export function initialize(opts) {
@@ -194,25 +208,31 @@ export function initialize(opts) {
     let bridgedWtImpl = (bridgedWtApi === null || bridgedWtApi === void 0 ? void 0 : bridgedWtApi.impl) || {};
     const wtImpl = (bridgedWtApi === null || bridgedWtApi === void 0 ? void 0 : bridgedWtApi.impl) || c.impl;
     let bridgedD3Impl = (bridgedD3Api === null || bridgedD3Api === void 0 ? void 0 : bridgedD3Api.impl) || {};
-    client = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, c.impl), { world_tracker_mesh_anchor_count: () => 0, world_tracker_mesh_anchor_id: () => "", world_tracker_mesh_anchor_indices: () => new Uint32Array(0), world_tracker_mesh_anchor_indices_count: () => 0, world_tracker_mesh_anchor_indices_size: () => 0, world_tracker_mesh_anchor_vertices: () => new Float32Array(0), world_tracker_mesh_anchor_vertices_count: () => 0, world_tracker_mesh_anchor_vertices_size: () => 0, world_tracker_mesh_anchor_vertices_offset: () => 0, world_tracker_mesh_anchor_vertices_stride: () => 0, world_tracker_mesh_anchor_pose_raw: () => new Float32Array(), world_tracker_mesh_anchor_status: () => anchor_status_t.ANCHOR_STATUS_STOPPED, world_tracker_mesh_anchors_enabled: () => false, world_tracker_mesh_anchors_enabled_set: () => { }, d3_tracker_create: (p) => { return -1; }, d3_tracker_destroy: (o) => { }, d3_tracker_count: (o) => { return 0; }, d3_tracker_enabled: (o) => { return false; }, d3_tracker_enabled_set: (o, v) => { }, d3_tracker_process_max_resolution_set: (o, v) => { }, d3_tracker_pose_raw: (o, indx) => { return mat4.create(); }, d3_tracker_pose0_raw: (o, indx) => { return mat4.create(); }, d3_tracker_pose1_raw: (o, indx) => { return mat4.create(); }, d3_tracker_id: (o, indx) => { return ''; }, d3_tracker_qr: (o, indx) => { return ''; }, d3_tracker_landmarks_data: (o, indx) => { return new Float32Array(0); }, d3_tracker_landmarks_data_size: (o, indx) => { return 0; }, d3_tracker_type: (o, indx) => { return 0; }, d3_tracker_dense: (o, indx) => { return 0; }, d3_tracker_category: (o, indx) => { return 0; }, d3_tracker_size_for_id_set: (o, id, size) => { return 0; }, d3_tracker_pose: (o, indx, cameraPose, mirror) => {
-            let res = applyScreenCounterRotation(undefined, client.d3_tracker_pose_raw(o, indx));
-            if (mirror) {
-                let scale = mat4.create();
-                mat4.fromScaling(scale, [-1, 1, 1]);
-                mat4.multiply(res, scale, res);
-                mat4.multiply(res, res, scale);
-            }
-            mat4.multiply(res, cameraPose, res);
-            return res;
+    let d3Impl = (bridgedD3Api === null || bridgedD3Api === void 0 ? void 0 : bridgedD3Api.impl) || c.impl;
+    client = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, c.impl), { world_tracker_mesh_anchor_count: () => 0, world_tracker_mesh_anchor_id: () => "", world_tracker_mesh_anchor_indices: () => new Uint32Array(0), world_tracker_mesh_anchor_indices_count: () => 0, world_tracker_mesh_anchor_indices_size: () => 0, world_tracker_mesh_anchor_vertices: () => new Float32Array(0), world_tracker_mesh_anchor_vertices_count: () => 0, world_tracker_mesh_anchor_vertices_size: () => 0, world_tracker_mesh_anchor_vertices_offset: () => 0, world_tracker_mesh_anchor_vertices_stride: () => 0, world_tracker_mesh_anchor_pose_raw: () => new Float32Array(), world_tracker_mesh_anchor_status: () => anchor_status_t.ANCHOR_STATUS_STOPPED, world_tracker_mesh_anchors_enabled: () => false, world_tracker_mesh_anchors_enabled_set: () => { }, d3_tracker_process_max_resolution_set: () => { } }), bridgedWtImpl), bridgedD3Impl), { d3_tracker_create: (pipeline) => {
+            let ret = d3Impl.d3_tracker_create(pipeline);
+            d3TrackerSizeById.set(ret, new Map());
+            return ret;
+        }, d3_tracker_destroy: (o) => {
+            d3TrackerSizeById.delete(o);
+            d3Impl.d3_tracker_destroy(o);
+        }, d3_tracker_pose0_raw: (o, indx) => {
+            let pose = d3Impl.d3_tracker_pose0_raw(o, indx).slice();
+            applyD3SizeToPose(o, indx, pose);
+            return pose;
+        }, d3_tracker_pose1_raw: (o, indx) => {
+            let pose = d3Impl.d3_tracker_pose1_raw(o, indx).slice();
+            applyD3SizeToPose(o, indx, pose);
+            return pose;
+        }, d3_tracker_size_for_id_set: (o, id, size) => {
+            var _a;
+            (_a = d3TrackerSizeById.get(o)) === null || _a === void 0 ? void 0 : _a.set(id, size);
+        }, d3_tracker_pose_raw: (o, indx) => {
+            return client.d3_tracker_pose0_raw(o, indx);
+        }, d3_tracker_pose: (o, indx, cameraPose, mirror) => {
+            return client.d3_tracker_pose0(o, indx, cameraPose, mirror);
         }, d3_tracker_pose_camera_relative: (o, indx, mirror) => {
-            let res = applyScreenCounterRotation(undefined, client.d3_tracker_pose_raw(o, indx));
-            if (mirror) {
-                let scale = mat4.create();
-                mat4.fromScaling(scale, [-1, 1, 1]);
-                mat4.multiply(res, scale, res);
-                mat4.multiply(res, res, scale);
-            }
-            return res;
+            return client.d3_tracker_pose0_camera_relative(o, indx, mirror);
         }, d3_tracker_pose0: (o, indx, cameraPose, mirror) => {
             let res = applyScreenCounterRotation(undefined, client.d3_tracker_pose0_raw(o, indx));
             if (mirror) {
@@ -251,7 +271,7 @@ export function initialize(opts) {
                 mat4.multiply(res, res, scale);
             }
             return res;
-        } }), bridgedWtImpl), bridgedD3Impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId, preferWebXRCamera), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_source_profile_set: (cam, p) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setProfile(p); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
+        }, loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId, preferWebXRCamera), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_source_profile_set: (cam, p) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setProfile(p); }, camera_source_preferred_resolution_set: (cam, w, h) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setPreferredResolution(w, h); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
             let res = applyScreenCounterRotation(undefined, raw_pose);
             if (mirror) {
                 let scale = mat4.create();

package/lib/profile.d.ts

@@ -11,7 +11,6 @@ export declare let profile: {
     requestHighFrameRate: boolean;
     videoWidth: number;
     videoHeight: number;
-    getDataSize: (p: camera_profile_t) => number[];
     videoElementInDOM: boolean;
     preferMediaStreamTrackProcessorCamera: boolean;
     preferImageBitmapCamera: boolean;
@@ -22,3 +21,5 @@ export declare let profile: {
     trustSensorIntervals: boolean;
     offscreenCanvasSupported: boolean;
 };
+export declare function getDataSize(p: camera_profile_t, fullWidth: number, fullHeight: number): [number, number];
+export declare function getDataSizeForSource(p: camera_profile_t, source: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation?: number): [number, number];

package/lib/profile.js

@@ -13,7 +13,6 @@ export let profile = {
     requestHighFrameRate: false,
     videoWidth: 1280,
     videoHeight: 720,
-    getDataSize: (p) => p === camera_profile_t.HIGH ? [640, 360] : [320, 180],
     videoElementInDOM: false,
     preferMediaStreamTrackProcessorCamera: false,
     preferImageBitmapCamera: false,
@@ -24,6 +23,39 @@ export let profile = {
     trustSensorIntervals: false,
     offscreenCanvasSupported: typeof globalThis.OffscreenCanvas !== "undefined",
 };
+export function getDataSize(p, fullWidth, fullHeight) {
+    if (p === camera_profile_t.FULL)
+        return [fullWidth, fullHeight];
+    let fullMax = fullWidth > fullHeight ? fullWidth : fullHeight;
+    let dataMax = p === camera_profile_t.HIGH ? 640 : 320;
+    let scale = dataMax / fullMax;
+    if (scale >= 1)
+        return [fullWidth, fullHeight];
+    // Ensure width is a multiple of 4 (required by ImageProcessGL greyscale conversion)
+    let width = Math.round(scale * fullWidth / 4) * 4;
+    let height = Math.round(width * fullHeight / fullWidth);
+    return [width, height];
+}
+export function getDataSizeForSource(p, source, rotation = 0) {
+    let fullWidth = source.width;
+    let fullHeight = source.height;
+    // NB: undefined checks needed as these types aren't available in workers
+    if (typeof HTMLVideoElement !== "undefined" && source instanceof HTMLVideoElement) {
+        fullWidth = source.videoWidth;
+        fullHeight = source.videoHeight;
+    }
+    else if (typeof HTMLImageElement !== "undefined" && source instanceof HTMLImageElement) {
+        fullWidth = source.naturalWidth;
+        fullHeight = source.naturalHeight;
+    }
+    if ((rotation == 90) || (rotation == 270)) {
+        // Will be rotated as part of greyscale conversion
+        return getDataSize(p, fullHeight, fullWidth);
+    }
+    else {
+        return getDataSize(p, fullWidth, fullHeight);
+    }
+}
 if (typeof window !== "undefined") {
     window["zeeProfile"] = profile;
     if (window.location.href.indexOf("_mstppipeline") >= 0) {
@@ -81,7 +113,6 @@ function iDevice(version) {
                 // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
                 // Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
                 profile.videoHeight = 360;
-                profile.getDataSize = (p) => p === camera_profile_t.HIGH ? [640, 360] : [320, 180];
             }
         }
         if (majorVersion === 16 && minorVersion >= 4) {

package/lib/sequencesource.js

@@ -97,7 +97,7 @@ export class SequenceSource extends Source {
         // updates will be one frame later
         this._submitMotionUpdates(pipeline);
         const token = pipeline.registerToken(info);
-        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel, -1);
+        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice.slice(), info.cameraModel.slice(), -1);
     }
     destroy() {
         byId.delete(this._impl);
@@ -154,7 +154,7 @@ export class SequenceSource extends Source {
                     pendingMotionUpdate = true;
                     break;
                 case SequenceRecorderPacketType.ATTITUDE_MATRIX:
-                    pipeline.motionAttitudeMatrix(packet.attitude);
+                    pipeline.motionAttitudeMatrix(packet.attitude.slice());
                     pendingMotionUpdate = true;
                     break;
                 case SequenceRecorderPacketType.ACCELEROMETER_WO_GRAVITY_INT:

package/lib/src/bridged-camera-source.d.ts

@@ -39,6 +39,7 @@ export declare class BridgedCameraSource extends Source {
     uploadGL(info: CameraFrameInfo): void;
     recycle(info: CameraFrameInfo): void;
     setProfile(p: camera_profile_t): void;
+    setPreferredResolution(w: number, h: number): void;
     private _anchorPoseFloat32;
     private _anchorPoseUint16;
     setCustomAnchorPose(anchor: zappar_custom_anchor_t, pose_version: number, pose: Float32Array): void;

package/lib/src/camera-source.d.ts

@@ -7,6 +7,8 @@ export declare class CameraSource extends HTMLElementSource {
     static DEFAULT_DEVICE_ID: string;
     private _currentStream;
     private _activeDeviceId;
+    private _preferredWidth;
+    private _preferredHeight;
     private _explicitUserCameraId;
     private _explicitEnvironmentCameraId;
     constructor(_impl: zappar_camera_source_t, pipeline: zappar_pipeline_t, _deviceId: string);
@@ -14,6 +16,7 @@ export declare class CameraSource extends HTMLElementSource {
     private _stop;
     pause(): void;
     start(): void;
+    setPreferredResolution(w: number, h: number): void;
     private _getConstraints;
     getFrame(allowRead: boolean): void;
     private _getUserMedia;

package/lib/src/gen/zappar-bridge.d.ts

@@ -186,4 +186,16 @@ export interface zappar {
     custom_anchor_pose_set_with_parent(o: zappar_custom_anchor_t, pose: Float32Array, anchor_id: string): void;
     d3_tracker_create(pipeline: zappar_pipeline_t): zappar_d3_tracker_t;
     d3_tracker_destroy(o: zappar_d3_tracker_t): void;
+    d3_tracker_enabled(o: zappar_d3_tracker_t): boolean;
+    d3_tracker_enabled_set(o: zappar_d3_tracker_t, enabled: boolean): void;
+    d3_tracker_count(o: zappar_d3_tracker_t): number;
+    d3_tracker_id(o: zappar_d3_tracker_t, indx: number): string;
+    d3_tracker_pose0_raw(o: zappar_d3_tracker_t, indx: number): Float32Array;
+    d3_tracker_pose1_raw(o: zappar_d3_tracker_t, indx: number): Float32Array;
+    d3_tracker_type(o: zappar_d3_tracker_t, indx: number): number;
+    d3_tracker_category(o: zappar_d3_tracker_t, indx: number): number;
+    d3_tracker_qr(o: zappar_d3_tracker_t, indx: number): string;
+    d3_tracker_dense(o: zappar_d3_tracker_t, indx: number): number;
+    d3_tracker_landmarks_data(o: zappar_d3_tracker_t, indx: number): Float32Array;
+    d3_tracker_landmarks_data_size(o: zappar_d3_tracker_t, indx: number): number;
 }