@zappar/zappar-cv 3.6.0 → 3.7.0-beta.0

package/lib/webxr-world-tracker.js

@@ -0,0 +1,331 @@
+import { anchor_status_t, plane_orientation_t, world_scale_mode_t } from "./gen/zappar";
+import { Pipeline } from "./pipeline";
+import { WebXRCameraSource } from "./webxr-camera-source";
+const _emptyUint8Array = new Uint8Array(0);
+export class WebXRWorldTracker {
+    constructor() {
+        this._latestId = 1;
+        this._wtStateById = new Map();
+        this._anchorStateById = new Map();
+        this._identityPose = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
+        this._emptyArray = new Float32Array([]);
+        this.impl = {
+            world_tracker_create: (pipeline) => {
+                let p = Pipeline.get(pipeline);
+                if (!p)
+                    throw new Error("Invalid pipeline in world_tracker_create");
+                let newId = (this._latestId++);
+                let s = {
+                    pipeline: p,
+                    enabled: true,
+                    horizontal_plane_detection_enabled: true,
+                    vertical_plane_detection_enabled: false,
+                    world_anchor_id: 'world',
+                    ground_anchor_id: 'ground',
+                    tracks_data_enabled: false,
+                    projections_data_enabled: false,
+                    mesh_anchors_enabled: false
+                };
+                this._wtStateById.set(newId, s);
+                this._notifyOptionsUpdated(s.pipeline);
+                return newId;
+            },
+            world_tracker_destroy: (o) => {
+                let s = this._getWtState(o);
+                this._wtStateById.delete(o);
+            },
+            world_tracker_enabled: (o) => {
+                let s = this._getWtState(o);
+                return s.enabled;
+            },
+            world_tracker_enabled_set: (o, enabled) => {
+                let s = this._getWtState(o);
+                s.enabled = enabled;
+                this._notifyOptionsUpdated(s.pipeline);
+            },
+            world_tracker_quality: (o) => {
+                let wtData = this._getWtData(o);
+                if (!wtData)
+                    return 0;
+                return wtData.quality;
+            },
+            world_tracker_horizontal_plane_detection_enabled: (o) => {
+                let s = this._getWtState(o);
+                return s.horizontal_plane_detection_enabled;
+            },
+            world_tracker_horizontal_plane_detection_enabled_set: (o, enabled) => {
+                let s = this._getWtState(o);
+                s.horizontal_plane_detection_enabled = enabled;
+                this._notifyOptionsUpdated(s.pipeline);
+            },
+            world_tracker_vertical_plane_detection_enabled: (o) => {
+                let s = this._getWtState(o);
+                return s.vertical_plane_detection_enabled;
+            },
+            world_tracker_vertical_plane_detection_enabled_set: (o, enabled) => {
+                let s = this._getWtState(o);
+                s.vertical_plane_detection_enabled = enabled;
+                this._notifyOptionsUpdated(s.pipeline);
+            },
+            world_tracker_plane_anchor_count: (o) => {
+                let wtData = this._getWtData(o);
+                let plane_anchors = wtData === null || wtData === void 0 ? void 0 : wtData.plane_anchors;
+                if (!plane_anchors)
+                    return 0;
+                return plane_anchors.length;
+            },
+            world_tracker_plane_anchor_id: (o, indx) => {
+                var _a, _b, _c;
+                let wtData = this._getWtData(o);
+                return (_c = (_b = (_a = wtData === null || wtData === void 0 ? void 0 : wtData.plane_anchors) === null || _a === void 0 ? void 0 : _a[indx]) === null || _b === void 0 ? void 0 : _b.id) !== null && _c !== void 0 ? _c : "";
+            },
+            world_tracker_plane_anchor_pose_raw: (o, indx) => {
+                var _a, _b;
+                let wtData = this._getWtData(o);
+                let pose = (_b = (_a = wtData === null || wtData === void 0 ? void 0 : wtData.plane_anchors) === null || _a === void 0 ? void 0 : _a[indx]) === null || _b === void 0 ? void 0 : _b.pose;
+                if (pose === undefined)
+                    return this._identityPose;
+                return pose;
+            },
+            world_tracker_plane_anchor_status: (o, indx) => {
+                var _a, _b, _c, _d;
+                return (_d = (_c = (_b = (_a = this._getWtData(o)) === null || _a === void 0 ? void 0 : _a.plane_anchors) === null || _b === void 0 ? void 0 : _b[indx]) === null || _c === void 0 ? void 0 : _c.status) !== null && _d !== void 0 ? _d : anchor_status_t.ANCHOR_STATUS_STOPPED;
+            },
+            world_tracker_plane_anchor_polygon_data_size: (o, indx) => {
+                var _a, _b, _c, _d, _e;
+                return (_e = (_d = (_c = (_b = (_a = this._getWtData(o)) === null || _a === void 0 ? void 0 : _a.plane_anchors) === null || _b === void 0 ? void 0 : _b[indx]) === null || _c === void 0 ? void 0 : _c.boundary) === null || _d === void 0 ? void 0 : _d.length) !== null && _e !== void 0 ? _e : 0;
+            },
+            world_tracker_plane_anchor_polygon_data: (o, indx) => {
+                var _a, _b, _c;
+                let boundary = (_c = (_b = (_a = this._getWtData(o)) === null || _a === void 0 ? void 0 : _a.plane_anchors) === null || _b === void 0 ? void 0 : _b[indx]) === null || _c === void 0 ? void 0 : _c.boundary;
+                if (!boundary)
+                    return this._emptyArray;
+                return boundary;
+            },
+            world_tracker_plane_anchor_polygon_version: (o, indx) => {
+                var _a, _b, _c, _d;
+                return (_d = (_c = (_b = (_a = this._getWtData(o)) === null || _a === void 0 ? void 0 : _a.plane_anchors) === null || _b === void 0 ? void 0 : _b[indx]) === null || _c === void 0 ? void 0 : _c.boundaryVersion) !== null && _d !== void 0 ? _d : 0;
+            },
+            world_tracker_plane_anchor_orientation: (o, indx) => {
+                var _a, _b, _c, _d;
+                return ((_d = (_c = (_b = (_a = this._getWtData(o)) === null || _a === void 0 ? void 0 : _a.plane_anchors) === null || _b === void 0 ? void 0 : _b[indx]) === null || _c === void 0 ? void 0 : _c.horizontal) !== null && _d !== void 0 ? _d : true) ? plane_orientation_t.PLANE_ORIENTATION_HORIZONTAL : plane_orientation_t.PLANE_ORIENTATION_VERTICAL;
+            },
+            world_tracker_world_anchor_status: (o) => {
+                var _a, _b;
+                return (_b = (_a = this._getWtData(o)) === null || _a === void 0 ? void 0 : _a.world_anchor_status) !== null && _b !== void 0 ? _b : anchor_status_t.ANCHOR_STATUS_INITIALIZING;
+            },
+            world_tracker_world_anchor_id: (o) => {
+                let s = this._getWtState(o);
+                return s.world_anchor_id;
+            },
+            world_tracker_world_anchor_pose_raw: (o) => {
+                let wtData = this._getWtData(o);
+                let pose = wtData === null || wtData === void 0 ? void 0 : wtData.world_anchor_pose;
+                if (pose === undefined)
+                    return this._identityPose;
+                return pose;
+            },
+            world_tracker_ground_anchor_status: (o) => {
+                var _a, _b;
+                return (_b = (_a = this._getWtData(o)) === null || _a === void 0 ? void 0 : _a.ground_anchor_status) !== null && _b !== void 0 ? _b : anchor_status_t.ANCHOR_STATUS_INITIALIZING;
+            },
+            world_tracker_ground_anchor_id: (o) => {
+                let s = this._getWtState(o);
+                return s.ground_anchor_id;
+            },
+            world_tracker_ground_anchor_pose_raw: (o) => {
+                let wtData = this._getWtData(o);
+                let pose = wtData === null || wtData === void 0 ? void 0 : wtData.ground_anchor_pose;
+                if (pose === undefined)
+                    return this._identityPose;
+                return pose;
+            },
+            world_tracker_reset: (o) => {
+                // Reset local pose_version state
+                for (const entry of this._anchorStateById.values()) {
+                    if (entry.wt === o)
+                        entry.pose_version = 0;
+                }
+                let s = this._getWtState(o);
+                let source = s.pipeline.currentCameraSource;
+                if (source instanceof WebXRCameraSource) {
+                    source.resetTracking();
+                }
+            },
+            world_tracker_tracks_data_enabled: (o) => {
+                let s = this._getWtState(o);
+                return s.tracks_data_enabled;
+            },
+            world_tracker_tracks_data_enabled_set: (o, tracks_data_enabled) => {
+                let s = this._getWtState(o);
+                s.tracks_data_enabled = tracks_data_enabled;
+                this._notifyOptionsUpdated(s.pipeline);
+            },
+            world_tracker_tracks_data_size: (o) => {
+                // Stub
+                let s = this._getWtState(o);
+                return 0;
+            },
+            world_tracker_tracks_data: (o) => {
+                // Stub
+                let s = this._getWtState(o);
+                return this._emptyArray;
+            },
+            world_tracker_tracks_type_data_size: (o) => {
+                // Stub
+                let s = this._getWtState(o);
+                return 0;
+            },
+            world_tracker_tracks_type_data: (o) => {
+                // Stub
+                return _emptyUint8Array;
+            },
+            world_tracker_projections_data_enabled: (o) => {
+                let s = this._getWtState(o);
+                return s.projections_data_enabled;
+            },
+            world_tracker_projections_data_enabled_set: (o, projections_data_enabled) => {
+                let s = this._getWtState(o);
+                s.projections_data_enabled = projections_data_enabled;
+                this._notifyOptionsUpdated(s.pipeline);
+            },
+            world_tracker_projections_data_size: (o) => {
+                // Stub
+                let s = this._getWtState(o);
+                return 0;
+            },
+            world_tracker_projections_data: (o) => {
+                // Stub
+                let s = this._getWtState(o);
+                return this._emptyArray;
+            },
+            world_tracker_session_number: (o) => {
+                var _a, _b;
+                let s = this._getWtState(o);
+                let source = (_b = (_a = s.pipeline) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo()) === null || _b === void 0 ? void 0 : _b.cameraSource;
+                if (source instanceof WebXRCameraSource) {
+                    return source.sessionNumber;
+                }
+                return 0;
+            },
+            world_tracker_scale_mode_set: (o, mode) => {
+                // Stub - no-op
+            },
+            world_tracker_scale_mode: (o) => {
+                // Stub - no-op
+                return world_scale_mode_t.DEFAULT;
+            },
+            // #### custom_anchor ####
+            custom_anchor_create: (pipeline, worldtracker, id) => {
+                let newId = (this._latestId++);
+                let wtState = this._getWtState(worldtracker);
+                let s = {
+                    pipeline: wtState.pipeline,
+                    wt: worldtracker,
+                    pose_version: 0,
+                    anchor_id: id,
+                };
+                this._anchorStateById.set(newId, s);
+                return newId;
+            },
+            custom_anchor_destroy: (o) => {
+                var _a, _b;
+                let s = this._getAnchorState(o);
+                this._anchorStateById.delete(o);
+                let source = (_b = (_a = s.pipeline) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo()) === null || _b === void 0 ? void 0 : _b.cameraSource;
+                if (source instanceof WebXRCameraSource) {
+                    source.deleteCustomAnchor(o);
+                }
+            },
+            custom_anchor_status: (o) => {
+                var _a, _b;
+                let s = this._getAnchorState(o);
+                let wtData = this._getWtDataFromPipeline(s.pipeline);
+                const anchorData = ((_a = wtData === null || wtData === void 0 ? void 0 : wtData.custom_anchors) !== null && _a !== void 0 ? _a : []).find(entry => entry.id === o);
+                return (_b = anchorData === null || anchorData === void 0 ? void 0 : anchorData.status) !== null && _b !== void 0 ? _b : anchor_status_t.ANCHOR_STATUS_INITIALIZING;
+            },
+            custom_anchor_pose_version: (o) => {
+                var _a, _b;
+                let s = this._getAnchorState(o);
+                let wtData = this._getWtDataFromPipeline(s.pipeline);
+                const anchorData = ((_a = wtData === null || wtData === void 0 ? void 0 : wtData.custom_anchors) !== null && _a !== void 0 ? _a : []).find(entry => entry.id === o);
+                return (_b = anchorData === null || anchorData === void 0 ? void 0 : anchorData.poseVersion) !== null && _b !== void 0 ? _b : -1;
+            },
+            custom_anchor_pose_raw: (o) => {
+                var _a;
+                let s = this._getAnchorState(o);
+                let wtData = this._getWtDataFromPipeline(s.pipeline);
+                const anchorData = ((_a = wtData === null || wtData === void 0 ? void 0 : wtData.custom_anchors) !== null && _a !== void 0 ? _a : []).find(entry => entry.id === o);
+                if (!anchorData)
+                    return this._identityPose;
+                return anchorData.pose;
+            },
+            custom_anchor_pose_set: (o, pose) => {
+                var _a, _b;
+                let s = this._getAnchorState(o);
+                s.pose_version++;
+                let source = (_b = (_a = s.pipeline) === null || _a === void 0 ? void 0 : _a.getCurrentCameraInfo()) === null || _b === void 0 ? void 0 : _b.cameraSource;
+                if (source instanceof WebXRCameraSource) {
+                    source.setCustomAnchorPose(o, s.pose_version, pose);
+                }
+            },
+            custom_anchor_pose_set_with_parent: (o, pose, anchor_id) => {
+                var _a, _b;
+                (_b = (_a = this.impl).custom_anchor_pose_set) === null || _b === void 0 ? void 0 : _b.call(_a, o, pose);
+            },
+        };
+    }
+    static SharedInstance() {
+        if (!this._sharedInstance) {
+            this._sharedInstance = new WebXRWorldTracker();
+        }
+        return this._sharedInstance;
+    }
+    _getWtState(o) {
+        let s = this._wtStateById.get(o);
+        if (!s)
+            throw new Error("This object has been destroyed");
+        return s;
+    }
+    _getAnchorState(o) {
+        let s = this._anchorStateById.get(o);
+        if (!s)
+            throw new Error("This object has been destroyed");
+        return s;
+    }
+    _getWtDataFromPipeline(p) {
+        var _a, _b;
+        return (_b = (_a = p === null || p === void 0 ? void 0 : p.getCurrentCameraInfo()) === null || _a === void 0 ? void 0 : _a.cameraSourceData) === null || _b === void 0 ? void 0 : _b.wtData;
+    }
+    _getWtData(o) {
+        let s = this._getWtState(o);
+        return this._getWtDataFromPipeline(s.pipeline);
+    }
+    // private _getFrameData(o : zappar_world_tracker_t): ParsedBridgeArrayBuffer | undefined {
+    //     let s = this._getWtState(o);
+    //     return s?.pipeline?.getCurrentCameraInfo()?.cameraSourceData?.frameData;
+    // }
+    _notifyOptionsUpdated(p) {
+        if (p.currentCameraSource instanceof WebXRCameraSource) {
+            p.currentCameraSource.optionsUpdated(this.getCombinedOptionsForPipeline(p));
+        }
+    }
+    getCombinedOptionsForPipeline(p) {
+        const options = {
+            horizontalPlaneDetection: false,
+            verticalPlaneDetection: false
+        };
+        for (let state of this._wtStateById.values()) {
+            if (state.pipeline != p)
+                continue;
+            if (!state.enabled)
+                continue;
+            if (state.horizontal_plane_detection_enabled)
+                options.horizontalPlaneDetection = true;
+            if (state.vertical_plane_detection_enabled)
+                options.verticalPlaneDetection = true;
+        }
+        return options;
+    }
+}
+WebXRWorldTracker._sharedInstance = null;

package/lib/worker-imagebitmap.js

@@ -34,13 +34,23 @@ export function handleImageBitmap(m, r, server, mgr) {
     }
     if (!texture)
         return;
-    const [dataWidth, dataHeight] = profile.getDataSize(m.cp);
+    let [dataWidth, dataHeight] = profile.getDataSize(m.cp);
+    let isPortrait = (m.r == 90) || (m.r == 270);
+    let ibWidth = isPortrait ? m.i.height : m.i.width;
+    let ibHeight = isPortrait ? m.i.width : m.i.height;
+    let cropHeight = Math.round(dataWidth * ibHeight / ibWidth);
+    if (cropHeight < dataHeight) {
+        dataHeight = cropHeight;
+    }
+    else {
+        dataWidth = Math.round(dataHeight * ibWidth / ibHeight);
+    }
     if (!pixels || pixels.byteLength < dataWidth * dataHeight) {
         // console.log("Generating pixel buffer", profile.dataWidth * profile.dataHeight);
         pixels = new ArrayBuffer(dataWidth * dataHeight);
     }
-    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing, m.cp);
-    let info = imageProcessor.readFrame(texture, pixels, m.cp);
+    imageProcessor.uploadFrameForSize(texture, m.i, m.r, m.userFacing, dataWidth, dataHeight);
+    let info = imageProcessor.readFrameForSize(texture, pixels, dataWidth, dataHeight);
     let msg = {
         t: "imageBitmapS2C",
         dataWidth: info.dataWidth,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zappar/zappar-cv",
-  "version": "3.6.0",
+  "version": "3.7.0-beta.0",
   "description": "Zappar's core computer vision library, supporting image, face and world tracking.",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -33,6 +33,7 @@
     "@types/offscreencanvas": "2019.6.4",
     "@types/pngjs": "6.0.0",
     "@types/ua-parser-js": "0.7.33",
+    "@types/webxr": "0.5.22",
     "@zappar/zapworks-cli": "2.0.6",
     "better-npm-audit": "^3.7.3",
     "dat.gui": "0.7.7",