@zappar/zappar-cv 2.0.0-beta.3 → 2.0.0-beta.7

package/lib/imagetracker.js

@@ -1,3 +1,5 @@
+import { image_target_type_t } from "./gen/zappar-native";
+import { getPreviewMesh } from "./imagetracker-previewmesh";
 import { RiffReader } from "./riff-reader";
 let byId = new Map();
 const decoder = new TextDecoder();
@@ -34,7 +36,10 @@ export class ImageTracker {
             topRadius: -1,
             bottomRadius: -1,
             sideLength: -1,
-            physicalScaleFactor: -1
+            physicalScaleFactor: -1,
+            trainedWidth: 0,
+            trainedHeight: 0,
+            type: image_target_type_t.IMAGE_TRACKER_TYPE_PLANAR
         };
         try {
             const reader = new RiffReader(current.data, false);
@@ -59,54 +64,90 @@ export class ImageTracker {
         let currentOffset = 0;
         let version = "0";
         [version, currentOffset] = readLine(currentOffset, data);
-        if (version === "3") {
-            let treeOrFlat = "0";
-            [treeOrFlat, currentOffset] = readLine(currentOffset, data);
-            if (treeOrFlat !== "0" && treeOrFlat !== "1")
-                return;
-            let numberTargets = "0";
-            [numberTargets, currentOffset] = readLine(currentOffset, data);
-            const parsedTargets = parseInt(numberTargets);
-            if (isNaN(parsedTargets) || parsedTargets < 1)
-                return;
-            let emptyLine = "";
-            [emptyLine, currentOffset] = readLine(currentOffset, data);
-            if (emptyLine.length !== 0)
-                return;
-            let infoLine = "";
-            [infoLine, currentOffset] = readLine(currentOffset, data);
-            const infoLineParts = infoLine.split(" ");
-            if (infoLineParts.length < 7)
-                return;
+        if (version === "1")
+            return this._parseOdleV1(data, currentOffset, info);
+        else if (version === "3")
+            return this._parseOdleV3(data, currentOffset, info);
+    }
+    _parseOdleV1(data, currentOffset, info) {
+        let treeOrFlat = "0";
+        [treeOrFlat, currentOffset] = readLine(currentOffset, data);
+        if (treeOrFlat !== "0" && treeOrFlat !== "1")
+            return;
+        let emptyLine = "";
+        [emptyLine, currentOffset] = readLine(currentOffset, data);
+        if (emptyLine.length !== 0)
+            return;
+        let infoLine = "";
+        [infoLine, currentOffset] = readLine(currentOffset, data);
+        const infoLineParts = infoLine.split(" ");
+        if (infoLineParts.length < 5)
+            return;
+        info.trainedWidth = parseInt(infoLineParts[3].replace("[", ""));
+        info.trainedHeight = parseInt(infoLineParts[4].replace("]", ""));
+    }
+    _parseOdleV3(data, currentOffset, info) {
+        let treeOrFlat = "0";
+        [treeOrFlat, currentOffset] = readLine(currentOffset, data);
+        if (treeOrFlat !== "0" && treeOrFlat !== "1")
+            return;
+        let numberTargets = "0";
+        [numberTargets, currentOffset] = readLine(currentOffset, data);
+        const parsedTargets = parseInt(numberTargets);
+        if (isNaN(parsedTargets) || parsedTargets < 1)
+            return;
+        let emptyLine = "";
+        [emptyLine, currentOffset] = readLine(currentOffset, data);
+        if (emptyLine.length !== 0)
+            return;
+        let infoLine = "";
+        [infoLine, currentOffset] = readLine(currentOffset, data);
+        const infoLineParts = infoLine.split(" ");
+        if (infoLineParts.length < 6)
+            return;
+        const targetType = parseInt(infoLineParts[0]);
+        if (targetType === 0 || targetType === 1 || targetType === 2)
+            info.type = targetType;
+        info.trainedWidth = parseInt(infoLineParts[4].replace("[", ""));
+        info.trainedHeight = parseInt(infoLineParts[5].replace("]", ""));
+        if (infoLineParts.length >= 7) {
             info.physicalScaleFactor = parseFloat(infoLineParts[6]);
             if (isNaN(info.physicalScaleFactor))
                 info.physicalScaleFactor = -1;
-            if (infoLineParts.length >= 8) {
-                info.topRadius = parseFloat(infoLineParts[7]);
-                if (isNaN(info.topRadius))
-                    info.topRadius = -1;
-                info.bottomRadius = info.topRadius;
-            }
-            if (infoLineParts.length >= 9) {
-                info.bottomRadius = parseFloat(infoLineParts[8]);
-                if (isNaN(info.bottomRadius))
-                    info.bottomRadius = -1;
-            }
-            if (infoLineParts.length >= 10) {
-                info.sideLength = parseFloat(infoLineParts[9]);
-                if (isNaN(info.sideLength))
-                    info.sideLength = -1;
-            }
+        }
+        if (infoLineParts.length >= 8) {
+            info.topRadius = parseFloat(infoLineParts[7]);
+            if (isNaN(info.topRadius))
+                info.topRadius = -1;
+            info.bottomRadius = info.topRadius;
+        }
+        if (infoLineParts.length >= 9) {
+            info.bottomRadius = parseFloat(infoLineParts[8]);
+            if (isNaN(info.bottomRadius))
+                info.bottomRadius = -1;
+        }
+        if (infoLineParts.length >= 10) {
+            info.sideLength = parseFloat(infoLineParts[9]);
+            if (isNaN(info.sideLength))
+                info.sideLength = -1;
         }
     }
     getDecodedPreview(i) {
         const info = this.getTargetInfo(i);
         if (!info.preview)
             return undefined;
-        const blob = new Blob([info.preview.compressed], { type: info.preview.mimeType });
-        const image = new Image();
-        image.src = URL.createObjectURL(blob);
-        return image;
+        if (!info.preview.image) {
+            const blob = new Blob([info.preview.compressed], { type: info.preview.mimeType });
+            info.preview.image = new Image();
+            info.preview.image.src = URL.createObjectURL(blob);
+        }
+        return info.preview.image;
+    }
+    getPreviewMesh(i) {
+        const info = this.getTargetInfo(i);
+        if (!info.previewMesh)
+            info.previewMesh = getPreviewMesh(info);
+        return info.previewMesh;
     }
 }
 function readLine(offset, str) {

package/lib/native.js

@@ -7,6 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+import { image_target_type_t } from "./gen/zappar";
 import { zappar_client } from "./gen/zappar-client";
 import { drawPlane } from "./drawplane";
 import { cameraRotationForScreenOrientation, projectionMatrix } from "./cameramodel";
@@ -39,7 +40,7 @@ export function initialize(opts) {
             d: ab
         }, [ab]);
     });
-    if (window.location.hostname.toLowerCase().indexOf(".zappar.io") > 0) {
+    if (window.location.hostname.toLowerCase().indexOf(".zappar.io") > 0 || window.location.hostname.toLowerCase().indexOf(".webar.run") > 0) {
         let pathParts = window.location.pathname.split("/");
         if (pathParts.length > 1 && pathParts[1].length > 0)
             c.impl.analytics_project_id_set(".wiz" + pathParts[1]);
@@ -117,7 +118,7 @@ export function initialize(opts) {
             }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
@@ -133,6 +134,14 @@ export function initialize(opts) {
         }, pipeline_draw_face_project: (p, matrix, vertices, uvMatrix, uvs, indices, texture) => {
             var _a;
             (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawFaceProject(matrix, vertices, uvMatrix, uvs, indices, texture);
+        }, pipeline_draw_image_target_preview: (p, projectionMatrix, cameraMatrix, targetMatrix, o, indx) => {
+            var _a;
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("image tracker not found");
+                return;
+            }
+            (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, obj);
         }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.instant_world_tracker_anchor_pose_raw(o));
             if (mirror) {
@@ -165,7 +174,14 @@ export function initialize(opts) {
             let rawVec = vec3.create();
             vec3.transformMat4(rawVec, [x, y, z], rotationMat);
             c.impl.instant_world_tracker_anchor_pose_set_from_camera_offset_raw(o, rawVec[0], rawVec[1], rawVec[2], orientation);
-        }, image_tracker_create: pipeline => ImageTracker.create(pipeline, c.impl), image_tracker_destroy: t => { var _a; return (_a = ImageTracker.get(t)) === null || _a === void 0 ? void 0 : _a.destroy(); }, image_tracker_target_count: t => {
+        }, image_tracker_create: pipeline => ImageTracker.create(pipeline, c.impl), image_tracker_destroy: t => { var _a; return (_a = ImageTracker.get(t)) === null || _a === void 0 ? void 0 : _a.destroy(); }, image_tracker_target_type: (t, i) => {
+            let obj = ImageTracker.get(t);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_type on a destroyed zappar_image_tracker_t");
+                return image_target_type_t.IMAGE_TRACKER_TYPE_PLANAR;
+            }
+            return obj.getTargetInfo(i).type;
+        }, image_tracker_target_count: t => {
             let obj = ImageTracker.get(t);
             if (!obj) {
                 zcwarn("attempting to call image_tracker_target_count on a destroyed zappar_image_tracker_t");
@@ -257,6 +273,62 @@ export function initialize(opts) {
             }
             mat4.multiply(res, cameraPose, res);
             return res;
+        }, image_tracker_target_preview_mesh_indices: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_indices on a destroyed zappar_image_tracker_t");
+                return new Uint16Array();
+            }
+            return obj.getPreviewMesh(indx).indices;
+        }, image_tracker_target_preview_mesh_vertices: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_vertices on a destroyed zappar_image_tracker_t");
+                return new Float32Array();
+            }
+            return obj.getPreviewMesh(indx).vertices;
+        }, image_tracker_target_preview_mesh_uvs: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_uvs on a destroyed zappar_image_tracker_t");
+                return new Float32Array();
+            }
+            return obj.getPreviewMesh(indx).uvs;
+        }, image_tracker_target_preview_mesh_normals: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_normals on a destroyed zappar_image_tracker_t");
+                return new Float32Array();
+            }
+            return obj.getPreviewMesh(indx).normals;
+        }, image_tracker_target_preview_mesh_indices_size: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_indices_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getPreviewMesh(indx).indices.length;
+        }, image_tracker_target_preview_mesh_vertices_size: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_vertices_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getPreviewMesh(indx).vertices.length;
+        }, image_tracker_target_preview_mesh_uvs_size: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_uvs_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getPreviewMesh(indx).uvs.length;
+        }, image_tracker_target_preview_mesh_normals_size: (o, indx) => {
+            let obj = ImageTracker.get(o);
+            if (!obj) {
+                zcwarn("attempting to call image_tracker_target_preview_mesh_normals_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getPreviewMesh(indx).normals.length;
         }, face_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.face_tracker_anchor_pose_raw(o, indx));
             if (mirror) {

package/lib/pipeline.d.ts

@@ -7,6 +7,7 @@ import { Source, CameraFrameInfo } from "./source";
 import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
 import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
+import { ImageTracker } from "./imagetracker";
 export declare class Pipeline {
     private _client;
     private _impl;
@@ -21,6 +22,7 @@ export declare class Pipeline {
     cameraPixelArrays: ArrayBuffer[];
     private _cameraDraw;
     private _faceDraw;
+    private _imageTargetPreviewDraw;
     private _faceProjectDraw;
     private _sequenceRecorder;
     private _sequenceRecordDeviceAttitudeMatrices;
@@ -44,6 +46,7 @@ export declare class Pipeline {
     glContextLost(): void;
     glContextSet(gl: WebGLRenderingContext, texturePool?: WebGLTexture[]): void;
     drawFace(projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: FaceMesh): void;
+    drawImageTargetPreview(projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, indx: number, o: ImageTracker): void;
     drawFaceProject(matrix: Float32Array, vertices: Float32Array, uvMatrix: Float32Array, uvs: Float32Array, indices: Uint16Array, texture: WebGLTexture): void;
     cameraFrameTexture(): WebGLTexture | undefined;
     cameraFrameTextureMatrix(sw: number, sh: number, mirror: boolean): Float32Array;

package/lib/pipeline.js

@@ -8,6 +8,7 @@ import { Event } from "./event";
 import { zcerr } from "./loglevel";
 import { SequenceRecorder } from "./sequencerecorder";
 import { getCameraSource } from "./camera-source-map";
+import { PreviewMeshDraw } from "./drawpreviewmesh";
 let byId = new Map();
 let identity = mat4.create();
 export class Pipeline {
@@ -121,10 +122,13 @@ export class Pipeline {
             this._cameraDraw.dispose();
         if (this._faceDraw)
             this._faceDraw.dispose();
+        if (this._imageTargetPreviewDraw)
+            this._imageTargetPreviewDraw.dispose();
         if (this._faceProjectDraw)
             this._faceProjectDraw.dispose();
         delete this._cameraDraw;
         delete this._faceDraw;
+        delete this._imageTargetPreviewDraw;
         delete this._faceProjectDraw;
         disposeDrawPlane();
         this.onGLContextReset.emit();
@@ -166,6 +170,16 @@ export class Pipeline {
         mat4.multiply(mat, mat, targetMatrix);
         this._faceDraw.drawFace(mat, o);
     }
+    drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, o) {
+        if (!this.glContext)
+            return;
+        if (!this._imageTargetPreviewDraw)
+            this._imageTargetPreviewDraw = new PreviewMeshDraw(this.glContext);
+        let mat = mat4.create();
+        mat4.multiply(mat, projectionMatrix, cameraMatrix);
+        mat4.multiply(mat, mat, targetMatrix);
+        this._imageTargetPreviewDraw.draw(mat, o, indx);
+    }
     drawFaceProject(matrix, vertices, uvMatrix, uvs, indices, texture) {
         if (!this.glContext)
             return;

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "2.0.0-beta.3";
+export declare const VERSION = "2.0.0-beta.7";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "2.0.0-beta.3";
+export const VERSION = "2.0.0-beta.7";