@zappar/zappar-cv 3.0.1-alpha.5 → 3.0.1-alpha.8

package/lib/gen/zappar-server.js

@@ -24,6 +24,8 @@ export class zappar_server {
         this._barcode_finder_by_instance = new Map();
         this._pipeline_id_by_instant_world_tracker_id = new Map();
         this._instant_world_tracker_by_instance = new Map();
+        this._pipeline_id_by_zapcode_tracker_id = new Map();
+        this._zapcode_tracker_by_instance = new Map();
         this._pipeline_id_by_world_tracker_id = new Map();
         this._world_tracker_by_instance = new Map();
     }
@@ -351,6 +353,40 @@ export class zappar_server {
                     break;
                 }
                 case 41: {
+                    let clientId = msg.type();
+                    let arg_pipeline_id = msg.type();
+                    let arg_pipeline = this._pipeline_by_instance.get(arg_pipeline_id);
+                    let handle = this._impl.zapcode_tracker_create(arg_pipeline);
+                    this._zapcode_tracker_by_instance.set(clientId, handle);
+                    this._pipeline_id_by_zapcode_tracker_id.set(clientId, arg_pipeline_id);
+                    break;
+                }
+                case 44: {
+                    let clientId = msg.type();
+                    let obj = this._zapcode_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.zapcode_tracker_destroy(obj);
+                    this._zapcode_tracker_by_instance.delete(clientId);
+                    break;
+                }
+                case 43: {
+                    let clientId = msg.type();
+                    let obj = this._zapcode_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.zapcode_tracker_target_load_from_memory(obj, msg.dataWithLength());
+                    break;
+                }
+                case 42: {
+                    let clientId = msg.type();
+                    let obj = this._zapcode_tracker_by_instance.get(clientId);
+                    if (obj === undefined)
+                        return;
+                    this._impl.zapcode_tracker_enabled_set(obj, msg.bool());
+                    break;
+                }
+                case 45: {
                     let clientId = msg.type();
                     let arg_pipeline_id = msg.type();
                     let arg_pipeline = this._pipeline_by_instance.get(arg_pipeline_id);
@@ -359,7 +395,7 @@ export class zappar_server {
                     this._pipeline_id_by_world_tracker_id.set(clientId, arg_pipeline_id);
                     break;
                 }
-                case 42: {
+                case 46: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -368,7 +404,7 @@ export class zappar_server {
                     this._world_tracker_by_instance.delete(clientId);
                     break;
                 }
-                case 43: {
+                case 47: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -376,7 +412,7 @@ export class zappar_server {
                     this._impl.world_tracker_enabled_set(obj, msg.bool());
                     break;
                 }
-                case 44: {
+                case 48: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -384,7 +420,7 @@ export class zappar_server {
                     this._impl.world_tracker_reset(obj);
                     break;
                 }
-                case 45: {
+                case 49: {
                     let clientId = msg.type();
                     let obj = this._world_tracker_by_instance.get(clientId);
                     if (obj === undefined)
@@ -568,6 +604,36 @@ export class zappar_server {
                 msg.matrix4x4(this._impl.instant_world_tracker_anchor_pose_raw(v));
             });
         }
+        for (let [k, v] of this._zapcode_tracker_by_instance) {
+            let pipeline = this._pipeline_id_by_zapcode_tracker_id.get(k);
+            if (!pipeline)
+                continue;
+            let serializer = this.serializersByPipelineId.get(pipeline);
+            if (!serializer)
+                continue;
+            serializer.sendMessage(24, msg => {
+                msg.type(k);
+                msg.int(this._impl.zapcode_tracker_target_loaded_version(v));
+            });
+            serializer.sendMessage(21, msg => {
+                msg.type(k);
+                msg.int(this._impl.zapcode_tracker_anchor_count(v));
+            });
+            for (let i = 0; i < this._impl.zapcode_tracker_anchor_count(v); i++) {
+                serializer.sendMessage(22, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.string(this._impl.zapcode_tracker_anchor_id(v, i));
+                });
+            }
+            for (let i = 0; i < this._impl.zapcode_tracker_anchor_count(v); i++) {
+                serializer.sendMessage(23, msg => {
+                    msg.type(k);
+                    msg.int(i);
+                    msg.matrix4x4(this._impl.zapcode_tracker_anchor_pose_raw(v, i));
+                });
+            }
+        }
         for (let [k, v] of this._world_tracker_by_instance) {
             let pipeline = this._pipeline_id_by_world_tracker_id.get(k);
             if (!pipeline)
@@ -575,50 +641,50 @@ export class zappar_server {
             let serializer = this.serializersByPipelineId.get(pipeline);
             if (!serializer)
                 continue;
-            serializer.sendMessage(27, msg => {
+            serializer.sendMessage(31, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_quality(v));
             });
-            serializer.sendMessage(21, msg => {
+            serializer.sendMessage(25, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_plane_count(v));
             });
             for (let i = 0; i < this._impl.world_tracker_plane_count(v); i++) {
-                serializer.sendMessage(22, msg => {
+                serializer.sendMessage(26, msg => {
                     msg.type(k);
                     msg.int(i);
                     msg.matrix4x4(this._impl.world_tracker_plane_pose_raw(v, i));
                 });
             }
-            serializer.sendMessage(23, msg => {
+            serializer.sendMessage(27, msg => {
                 msg.type(k);
                 msg.bool(this._impl.world_tracker_world_anchor_valid(v));
             });
-            serializer.sendMessage(24, msg => {
+            serializer.sendMessage(28, msg => {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_world_anchor_pose_raw(v));
             });
-            serializer.sendMessage(23, msg => {
+            serializer.sendMessage(27, msg => {
                 msg.type(k);
                 msg.bool(this._impl.world_tracker_ground_anchor_valid(v));
             });
-            serializer.sendMessage(26, msg => {
+            serializer.sendMessage(30, msg => {
                 msg.type(k);
                 msg.matrix4x4(this._impl.world_tracker_ground_anchor_pose_raw(v));
             });
-            serializer.sendMessage(29, msg => {
+            serializer.sendMessage(33, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_debug_tracks_data_size(v));
             });
-            serializer.sendMessage(28, msg => {
+            serializer.sendMessage(32, msg => {
                 msg.type(k);
                 msg.floatArray(this._impl.world_tracker_debug_tracks_data(v));
             });
-            serializer.sendMessage(31, msg => {
+            serializer.sendMessage(35, msg => {
                 msg.type(k);
                 msg.int(this._impl.world_tracker_debug_projections_data_size(v));
             });
-            serializer.sendMessage(30, msg => {
+            serializer.sendMessage(34, msg => {
                 msg.type(k);
                 msg.floatArray(this._impl.world_tracker_debug_projections_data(v));
             });

package/lib/gen/zappar.d.ts

@@ -10,6 +10,8 @@ export { log_level_t } from "./zappar-native";
 import { image_target_type_t } from "./zappar-native";
 export { image_target_type_t } from "./zappar-native";
 export { world_tracker_quality_t } from "./zappar-native";
+import { camera_profile_t } from "./zappar-native";
+export { camera_profile_t } from "./zappar-native";
 export declare type zappar_pipeline_t = number & {
     _: 'zappar_pipeline_t';
 };
@@ -37,6 +39,9 @@ export declare type zappar_barcode_finder_t = number & {
 export declare type zappar_instant_world_tracker_t = number & {
     _: 'zappar_instant_world_tracker_t';
 };
+export declare type zappar_zapcode_tracker_t = number & {
+    _: 'zappar_zapcode_tracker_t';
+};
 export declare type zappar_world_tracker_t = number & {
     _: 'zappar_world_tracker_t';
 };
@@ -99,6 +104,7 @@ export interface zappar {
     camera_source_destroy(o: zappar_camera_source_t): void;
     camera_source_start(o: zappar_camera_source_t): void;
     camera_source_pause(o: zappar_camera_source_t): void;
+    camera_source_profile_set(o: zappar_camera_source_t, profile: camera_profile_t): void;
     sequence_source_create(pipeline: zappar_pipeline_t): zappar_sequence_source_t;
     sequence_source_destroy(o: zappar_sequence_source_t): void;
     sequence_source_start(o: zappar_sequence_source_t): void;
@@ -183,6 +189,17 @@ export interface zappar {
     instant_world_tracker_anchor_pose(o: zappar_instant_world_tracker_t, camera_pose: Float32Array, mirror: boolean): Float32Array;
     instant_world_tracker_anchor_pose_set_from_camera_offset_raw(o: zappar_instant_world_tracker_t, x: number, y: number, z: number, orientation: instant_world_tracker_transform_orientation_t): void;
     instant_world_tracker_anchor_pose_set_from_camera_offset(o: zappar_instant_world_tracker_t, x: number, y: number, z: number, orientation: instant_world_tracker_transform_orientation_t): void;
+    zapcode_tracker_create(pipeline: zappar_pipeline_t): zappar_zapcode_tracker_t;
+    zapcode_tracker_destroy(o: zappar_zapcode_tracker_t): void;
+    zapcode_tracker_target_load_from_memory(o: zappar_zapcode_tracker_t, data: ArrayBuffer): void;
+    zapcode_tracker_target_loaded_version(o: zappar_zapcode_tracker_t): number;
+    zapcode_tracker_enabled(o: zappar_zapcode_tracker_t): boolean;
+    zapcode_tracker_enabled_set(o: zappar_zapcode_tracker_t, enabled: boolean): void;
+    zapcode_tracker_anchor_count(o: zappar_zapcode_tracker_t): number;
+    zapcode_tracker_anchor_id(o: zappar_zapcode_tracker_t, indx: number): string;
+    zapcode_tracker_anchor_pose_raw(o: zappar_zapcode_tracker_t, indx: number): Float32Array;
+    zapcode_tracker_anchor_pose_camera_relative(o: zappar_zapcode_tracker_t, indx: number, mirror: boolean): Float32Array;
+    zapcode_tracker_anchor_pose(o: zappar_zapcode_tracker_t, indx: number, camera_pose: Float32Array, mirror: boolean): Float32Array;
     world_tracker_create(pipeline: zappar_pipeline_t): zappar_world_tracker_t;
     world_tracker_destroy(o: zappar_world_tracker_t): void;
     world_tracker_enabled(o: zappar_world_tracker_t): boolean;

package/lib/gen/zappar.js

@@ -5,3 +5,4 @@ export { instant_world_tracker_transform_orientation_t } from "./zappar-native";
 export { log_level_t } from "./zappar-native";
 export { image_target_type_t } from "./zappar-native";
 export { world_tracker_quality_t } from "./zappar-native";
+export { camera_profile_t } from "./zappar-native";

package/lib/html-element-source.d.ts

@@ -1,4 +1,4 @@
-import { zappar_pipeline_t } from "./gen/zappar";
+import { camera_profile_t, zappar_pipeline_t } from "./gen/zappar";
 import { Source } from "./source";
 import { zappar_html_element_source_t } from "./additional";
 export declare class HTMLElementSource extends Source {
@@ -14,6 +14,7 @@ export declare class HTMLElementSource extends Source {
     private _cameraToDeviceTransform;
     private _cameraToDeviceTransformUserFacing;
     private _cameraModel;
+    private _profile;
     static createVideoElementSource(p: zappar_pipeline_t, element: HTMLVideoElement | HTMLImageElement): zappar_html_element_source_t;
     static getVideoElementSource(m: zappar_html_element_source_t): HTMLElementSource | undefined;
     constructor(_video: HTMLVideoElement | HTMLImageElement, _pipeline: zappar_pipeline_t);
@@ -24,4 +25,5 @@ export declare class HTMLElementSource extends Source {
     getFrame(currentlyProcessing: boolean): void;
     private _processFrame;
     uploadGL(): void;
+    setProfile(p: camera_profile_t): void;
 }

package/lib/html-element-source.js

@@ -1,3 +1,4 @@
+import { camera_profile_t } from "./gen/zappar";
 import { Pipeline } from "./pipeline";
 import { Source } from "./source";
 import { profile } from "./profile";
@@ -19,6 +20,7 @@ export class HTMLElementSource extends Source {
         this._cameraToDeviceTransform = mat4.create();
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+        this._profile = camera_profile_t.DEFAULT;
         mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
         let video = this._video;
         if (this._video instanceof HTMLVideoElement) {
@@ -109,14 +111,15 @@ export class HTMLElementSource extends Source {
             }
             if (!this._currentVideoTexture)
                 return undefined;
-            this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing);
+            this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
             this._isUploadFrame = !this._isUploadFrame;
             return undefined;
         }
         if (currentlyProcessing || !this._currentVideoTexture)
             return undefined;
         this._isUploadFrame = !this._isUploadFrame;
-        let greySize = profile.dataWidth * profile.dataHeight;
+        const [dataWidth, dataHeight] = profile.getDataSize(this._profile);
+        let greySize = dataWidth * dataHeight;
         let pixels = pipeline.cameraPixelArrays.pop();
         while (pixels) {
             if (pixels.byteLength === greySize)
@@ -128,14 +131,17 @@ export class HTMLElementSource extends Source {
         }
         let tex = this._currentVideoTexture;
         this._currentVideoTexture = undefined;
-        let focalLength = 300.0 * profile.dataWidth / 320.0;
+        let focalLength = 300.0 * dataWidth / 320.0;
         this._cameraModel[0] = focalLength;
         this._cameraModel[1] = focalLength;
-        this._cameraModel[2] = profile.dataWidth * 0.5;
-        this._cameraModel[3] = profile.dataHeight * 0.5;
-        return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
+        this._cameraModel[2] = dataWidth * 0.5;
+        this._cameraModel[3] = dataHeight * 0.5;
+        return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels, this._profile)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
     }
     uploadGL() {
         // No-op as already uploaded
     }
+    setProfile(p) {
+        this._profile = p;
+    }
 }

package/lib/image-process-gl.d.ts

@@ -1,3 +1,4 @@
+import { camera_profile_t } from "./gen/zappar-native";
 export interface ProcessedFrame {
     data?: ArrayBuffer;
     texture: WebGLTexture | undefined;
@@ -28,8 +29,8 @@ export declare class ImageProcessGL {
     constructor(_gl: WebGL2RenderingContext | WebGLRenderingContext);
     resetGLContext(): void;
     destroy(): void;
-    uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean): void;
-    readFrame(texture: WebGLTexture, pixels: ArrayBuffer): ProcessedFrame;
+    uploadFrame(texture: WebGLTexture, img: HTMLVideoElement | HTMLImageElement | ImageBitmap, rotation: number, fc: boolean, cp: camera_profile_t): void;
+    readFrame(texture: WebGLTexture, pixels: ArrayBuffer, cp: camera_profile_t): ProcessedFrame;
     private _updateTransforms;
     private _getCameraUvTransform;
     private _getFramebuffer;

package/lib/image-process-gl.js

@@ -33,7 +33,7 @@ export class ImageProcessGL {
     destroy() {
         this.resetGLContext();
     }
-    uploadFrame(texture, img, rotation, fc) {
+    uploadFrame(texture, img, rotation, fc, cp) {
         let gl = this._gl;
         const glStateManager = GLStateManager.get(gl);
         glStateManager.push();
@@ -75,7 +75,8 @@ export class ImageProcessGL {
         if (videoHeight > videoWidth)
             videoHeight = [videoWidth, videoWidth = videoHeight][0];
         this._updateTransforms(rotation, fc);
-        let framebuffer = this._getFramebuffer(gl, profile.dataWidth / 4, profile.dataHeight);
+        const [dataWidth, dataHeight] = profile.getDataSize(cp);
+        let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
         let vbo = this._getVertexBuffer(gl);
         let ibo = this._getIndexBuffer(gl);
         let shader = this._getGreyscaleShader(gl);
@@ -123,7 +124,7 @@ export class ImageProcessGL {
         // Tell WebGL to use our program when drawing
         gl.useProgram(shader.program);
         // Specify greyscale width for the correct offsets, and the uv transform
-        gl.uniform1f(shader.uTexWidthLoc, profile.dataWidth);
+        gl.uniform1f(shader.uTexWidthLoc, dataWidth);
         gl.uniformMatrix4fv(shader.uUvTransformLoc, false, this._cameraUvTransform);
         gl.activeTexture(gl.TEXTURE0);
         // Bind the texture to texture unit 0
@@ -167,11 +168,12 @@ export class ImageProcessGL {
         if (reenableStencilTest)
             gl.enable(gl.STENCIL_TEST);
     }
-    readFrame(texture, pixels) {
+    readFrame(texture, pixels, cp) {
         let gl = this._gl;
         let pixelsView = new Uint8Array(pixels);
         const previousBoundFramebuffer = gl.getParameter(gl.FRAMEBUFFER_BINDING);
-        let framebuffer = this._getFramebuffer(gl, profile.dataWidth / 4, profile.dataHeight);
+        const [dataWidth, dataHeight] = profile.getDataSize(cp);
+        let framebuffer = this._getFramebuffer(gl, dataWidth / 4, dataHeight);
         gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
         gl.readPixels(0, 0, this._framebufferWidth, this._framebufferHeight, gl.RGBA, gl.UNSIGNED_BYTE, pixelsView);
         gl.bindFramebuffer(gl.FRAMEBUFFER, previousBoundFramebuffer);
@@ -179,8 +181,8 @@ export class ImageProcessGL {
             uvTransform: this._cameraUvTransform,
             data: pixels,
             texture,
-            dataWidth: profile.dataWidth,
-            dataHeight: profile.dataHeight,
+            dataWidth,
+            dataHeight,
             userFacing: this._computedFrontCameraRotation
         };
     }

package/lib/imagebitmap-camera-source.d.ts

@@ -1,4 +1,4 @@
-import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
+import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
 import { CameraFrameInfo, Source } from "./source";
 export declare class ImageBitmapCameraSource extends Source {
     private _impl;
@@ -18,8 +18,10 @@ export declare class ImageBitmapCameraSource extends Source {
     private _cameraToDeviceTransform;
     private _cameraToDeviceTransformUserFacing;
     private _cameraModel;
+    private _profile;
     constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
     destroy(): void;
+    setProfile(p: camera_profile_t): void;
     private _stop;
     pause(): void;
     start(): void;

package/lib/imagebitmap-camera-source.js

@@ -7,6 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+import { camera_profile_t } from "./gen/zappar";
 import { profile } from "./profile";
 import { Pipeline } from "./pipeline";
 import { Source } from "./source";
@@ -32,6 +33,7 @@ export class ImageBitmapCameraSource extends Source {
         this._cameraToDeviceTransform = mat4.create();
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+        this._profile = camera_profile_t.DEFAULT;
         this._hasStartedOrientation = false;
         this._lastTimestamp = -1;
         this._deviceMotionListener = (ev) => {
@@ -85,6 +87,9 @@ export class ImageBitmapCameraSource extends Source {
         this.pause();
         deleteCameraSource(this._impl);
     }
+    setProfile(p) {
+        this._profile = p;
+    }
     _stop() {
         if (!this._currentStream)
             return;
@@ -185,11 +190,12 @@ export class ImageBitmapCameraSource extends Source {
         let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
-        let focalLength = 300.0 * profile.dataWidth / 320.0;
+        const [dataWidth, dataHeight] = profile.getDataSize(this._profile);
+        let focalLength = 300.0 * dataWidth / 320.0;
         this._cameraModel[0] = focalLength;
         this._cameraModel[1] = focalLength;
-        this._cameraModel[2] = profile.dataWidth * 0.5;
-        this._cameraModel[3] = profile.dataHeight * 0.5;
+        this._cameraModel[2] = dataWidth * 0.5;
+        this._cameraModel[3] = dataHeight * 0.5;
         let token = pipeline.registerToken({
             dataWidth: this._canvas.width,
             dataHeight: this._canvas.height,
@@ -199,7 +205,7 @@ export class ImageBitmapCameraSource extends Source {
             cameraModel: this._cameraModel.slice(),
             cameraToDevice: this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform
         });
-        (_b = (_a = Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendImageBitmapToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, imageBitmap, rotation, this._isUserFacing, token, this._cameraModel, this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform);
+        (_b = (_a = Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendImageBitmapToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, imageBitmap, rotation, this._isUserFacing, token, this._cameraModel, this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform, this._profile);
         return;
     }
     _getUserMedia() {

package/lib/index.d.ts

@@ -4,4 +4,4 @@ import { Options } from "./options";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(opts?: Options): Zappar;
 export { CameraFrameData } from './camera-frame-data';
-export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";
+export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, zappar_zapcode_tracker_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";

package/lib/index.js

@@ -4,4 +4,4 @@ export function initialize(opts) {
     console.log(`Zappar CV v${VERSION}`);
     return nativeInitialize(opts);
 }
-export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";
+export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t, camera_profile_t } from "./gen/zappar";

package/lib/mstp-camera-source.d.ts

@@ -1,4 +1,4 @@
-import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
+import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
 import { CameraFrameInfo, Source } from "./source";
 export declare class MSTPCameraSource extends Source {
     private _impl;
@@ -24,6 +24,7 @@ export declare class MSTPCameraSource extends Source {
     private _getUserMedia;
     streamEnded(): void;
     private _syncCamera;
+    setProfile(profile: camera_profile_t): void;
     private _hasStartedOrientation;
     private _lastTimestamp;
     private _deviceMotionListener;

package/lib/mstp-camera-source.js

@@ -249,6 +249,11 @@ export class MSTPCameraSource extends Source {
             }
         });
     }
+    setProfile(profile) {
+        let p = Pipeline.get(this._pipeline);
+        if (p)
+            p.sendCameraProfileToWorker(this._impl, profile);
+    }
     _startDeviceOrientation() {
         if (this._hasStartedOrientation)
             return;

package/lib/native.js

@@ -147,7 +147,7 @@ export function initialize(opts) {
             }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_source_profile_set: (cam, p) => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.setProfile(p); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
             let res = applyScreenCounterRotation(undefined, raw_pose);
             if (mirror) {
                 let scale = mat4.create();
@@ -377,6 +377,25 @@ export function initialize(opts) {
                 return 0;
             }
             return obj.getPreviewMesh(indx).normals.length;
+        }, zapcode_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.zapcode_tracker_anchor_pose_raw(o, indx));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, zapcode_tracker_anchor_pose: (o, indx, cameraPose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.zapcode_tracker_anchor_pose_raw(o, indx));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            mat4.multiply(res, cameraPose, res);
+            return res;
         }, face_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.face_tracker_anchor_pose_raw(o, indx));
             if (mirror) {

package/lib/pipeline.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="dom-webcodecs" />
 import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
+import { camera_profile_t } from "./gen/zappar-native";
 import { zappar_client } from "./gen/zappar-client";
 import { MsgManager } from "./messages";
 import { Source, CameraFrameInfo } from "./source";
@@ -74,8 +75,9 @@ export declare class Pipeline {
     motionAttitudeSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionAttitudeMatrix(m: Float32Array): void;
     sendCameraStreamToWorker(source: zappar_camera_source_t, stream: ReadableStream<VideoFrame>, userFacing: boolean): void;
+    sendCameraProfileToWorker(source: zappar_camera_source_t, profile: camera_profile_t): void;
     sendCameraToScreenRotationToWorker(rot: number): void;
-    sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array): void;
+    sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array, cp: camera_profile_t): void;
     sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
 }
 export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): Float32Array;

package/lib/pipeline.js

@@ -448,6 +448,15 @@ export class Pipeline {
         };
         this._mgr.postOutgoingMessage(msg, [msg.s]);
     }
+    sendCameraProfileToWorker(source, profile) {
+        let msg = {
+            t: "cameraProfileC2S",
+            p: this._impl,
+            source,
+            profile
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
     sendCameraToScreenRotationToWorker(rot) {
         let msg = {
             p: this._impl,
@@ -456,7 +465,7 @@ export class Pipeline {
         };
         this._mgr.postOutgoingMessage(msg, []);
     }
-    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice) {
+    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice, cp) {
         let msg = {
             p: this._impl,
             t: "imageBitmapC2S",
@@ -465,7 +474,8 @@ export class Pipeline {
             tokenId,
             userFacing,
             cameraModel,
-            cameraToDevice
+            cameraToDevice,
+            cp
         };
         this._mgr.postOutgoingMessage(msg, [img]);
     }

package/lib/profile.d.ts

@@ -1,3 +1,4 @@
+import { camera_profile_t } from "./gen/zappar-native";
 export declare enum EmbeddedVideoImplementation {
     OBJECT_URL = 0,
     SRC_OBJECT = 1
@@ -10,8 +11,7 @@ export declare let profile: {
     requestHighFrameRate: boolean;
     videoWidth: number;
     videoHeight: number;
-    dataWidth: number;
-    dataHeight: number;
+    getDataSize: (p: camera_profile_t) => number[];
     videoElementInDOM: boolean;
     preferMediaStreamTrackProcessorCamera: boolean;
     preferImageBitmapCamera: boolean;

package/lib/profile.js

@@ -1,4 +1,5 @@
 import * as UAParser from "ua-parser-js";
+import { camera_profile_t } from "./gen/zappar-native";
 export var EmbeddedVideoImplementation;
 (function (EmbeddedVideoImplementation) {
     EmbeddedVideoImplementation[EmbeddedVideoImplementation["OBJECT_URL"] = 0] = "OBJECT_URL";
@@ -12,8 +13,7 @@ export let profile = {
     requestHighFrameRate: false,
     videoWidth: 640,
     videoHeight: 480,
-    dataWidth: 320,
-    dataHeight: 240,
+    getDataSize: (p) => p === camera_profile_t.HIGH ? [640, 480] : [320, 240],
     videoElementInDOM: false,
     preferMediaStreamTrackProcessorCamera: true,
     preferImageBitmapCamera: false,
@@ -80,7 +80,7 @@ function iDevice(version) {
                 // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
                 // Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
                 profile.videoHeight = 360;
-                profile.dataHeight = 180;
+                profile.getDataSize = (p) => p === camera_profile_t.HIGH ? [640, 360] : [320, 180];
             }
         }
         if (majorVersion === 16 && minorVersion >= 4) {

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.1-alpha.5";
+export declare const VERSION = "3.0.1-alpha.8";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.1-alpha.5";
+export const VERSION = "3.0.1-alpha.8";

package/lib/worker-imagebitmap.js

@@ -34,12 +34,13 @@ export function handleImageBitmap(m, r, server, mgr) {
     }
     if (!texture)
         return;
-    if (!pixels || pixels.byteLength < profile.dataWidth * profile.dataHeight) {
+    const [dataWidth, dataHeight] = profile.getDataSize(m.cp);
+    if (!pixels || pixels.byteLength < dataWidth * dataHeight) {
         // console.log("Generating pixel buffer", profile.dataWidth * profile.dataHeight);
-        pixels = new ArrayBuffer(profile.dataWidth * profile.dataHeight);
+        pixels = new ArrayBuffer(dataWidth * dataHeight);
     }
-    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing);
-    let info = imageProcessor.readFrame(texture, pixels);
+    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing, m.cp);
+    let info = imageProcessor.readFrame(texture, pixels, m.cp);
     let msg = {
         t: "imageBitmapS2C",
         dataWidth: info.dataWidth,