@zappar/zappar-cv 3.0.1-alpha.7 → 3.0.1-beta.0

package/lib/permission.js

@@ -82,6 +82,7 @@ export function permissionRequestUI() {
         }
         let div = document.createElement("div");
         div.classList.add("zappar-permission-request");
+        div.classList.add("zappar-permission-request-without-button");
         div.innerHTML = `
     <style>
         .zappar-permission-request {
@@ -99,7 +100,21 @@ export function permissionRequestUI() {
             align-items: center;
             justify-content: center;
         }
+        .zappar-permission-request-without-button {
+            justify-content: flex-start;
+        }
+
+        .zappar-permission-request-without-button #zappar-permission-request-button {
+            display: none;
+        }
+
+        .zappar-permission-request-without-button .zappar-inner {
+            max-width: 300px;
+        }
+
         .zappar-inner {
+            margin-top: 30%;
+            margin-bottom: 30%;
             max-width: 400px;
             text-align: center;
         }
@@ -127,24 +142,35 @@ export function permissionRequestUI() {
     </div>
 `;
         document.body.append(div);
-        let button = div.querySelector("#zappar-permission-request-button");
-        return yield new Promise(resolve => {
-            button === null || button === void 0 ? void 0 : button.addEventListener("click", () => __awaiter(this, void 0, void 0, function* () {
+        function attempt() {
+            return __awaiter(this, void 0, void 0, function* () {
                 yield permissionRequestAll();
                 div.remove();
                 while (true) {
                     yield nextFrame();
                     if (permissionDeniedAny()) {
-                        resolve(false);
-                        return;
+                        return false;
                     }
                     if (permissionGrantedAll()) {
-                        resolve(true);
-                        return;
+                        return true;
                     }
                 }
+                return false;
+            });
+        }
+        try {
+            return yield attempt();
+        }
+        catch (err) {
+            // We may need a user interaction
+            div.classList.remove("zappar-permission-request-without-button");
+            let button = div.querySelector("#zappar-permission-request-button");
+            return yield new Promise((resolve) => __awaiter(this, void 0, void 0, function* () {
+                button === null || button === void 0 ? void 0 : button.addEventListener('click', () => __awaiter(this, void 0, void 0, function* () {
+                    resolve(yield attempt());
+                }));
             }));
-        });
+        }
     });
 }
 function nextFrame() {
@@ -254,7 +280,7 @@ function permissionDeniedUIChrome() {
     </style>
     <div class="zappar-inner">
         <div class="zappar-title">Permission is Needed</div>
-        <div class="zappar-text">Permission to access your device's camera and motion sensors is necessary for this experience.<br/><br/>To grant access, please tap the ! button in the address bar of your browser, then "Site settings", and finally "Clear and reset". You can then reload the page to try again.</div>
+        <div class="zappar-text">Permission to access your device's camera and motion sensors is necessary for this experience.<br/><br/>To grant access, please tap the <svg xmlns="http://www.w3.org/2000/svg" height="16px" viewBox="0 -960 960 960" width="16px" fill="white"><path d="M696-144q-60.48 0-102.24-41.76T552-288q0-60.48 41.76-102.24T696-432q60.48 0 102.24 41.76T840-288q0 60.48-41.76 102.24T696-144Zm.12-72Q726-216 747-237.12q21-21.12 21-51T746.88-339q-21.12-21-51-21T645-338.88q-21 21.12-21 51T645.12-237q21.12 21 51 21ZM168-252v-72h312v72H168Zm96-276q-60.48 0-102.24-41.76T120-672q0-60.48 41.76-102.24T264-816q60.48 0 102.24 41.76T408-672q0 60.48-41.76 102.24T264-528Zm.12-72Q294-600 315-621.12q21-21.12 21-51T314.88-723q-21.12-21-51-21T213-722.88q-21 21.12-21 51T213.12-621q21.12 21 51 21ZM480-636v-72h312v72H480Zm216 348ZM264-672Z"/></svg> button in the address bar of your browser, then "Permissions", and finally "Reset permissions". You can then reload the page to try again.</div>
         <button id="zappar-permission-reload-button">Reload</button>
     </div>
 `;

package/lib/pipeline.d.ts

@@ -78,6 +78,6 @@ export declare class Pipeline {
     sendCameraProfileToWorker(source: zappar_camera_source_t, profile: camera_profile_t): void;
     sendCameraToScreenRotationToWorker(rot: number): void;
     sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array, cp: camera_profile_t): void;
-    sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
+    sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array, captureTime: number): void;
 }
 export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): Float32Array;

package/lib/pipeline.js

@@ -479,14 +479,15 @@ export class Pipeline {
         };
         this._mgr.postOutgoingMessage(msg, [img]);
     }
-    sendDataToWorker(data, token, width, height, userFacing, cameraToDevice, cameraModel) {
+    sendDataToWorker(data, token, width, height, userFacing, cameraToDevice, cameraModel, captureTime) {
         let msg = {
             d: data,
             p: this._impl,
             width, height, token, userFacing,
             c2d: cameraToDevice,
             cm: cameraModel,
-            t: "cameraFrameC2S"
+            t: "cameraFrameC2S",
+            captureTime
         };
         this._mgr.postOutgoingMessage(msg, [data]);
     }

package/lib/sequencesource.js

@@ -97,7 +97,7 @@ export class SequenceSource extends Source {
         // updates will be one frame later
         this._submitMotionUpdates(pipeline);
         const token = pipeline.registerToken(info);
-        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
+        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel, -1);
     }
     destroy() {
         byId.delete(this._impl);

package/lib/serializer.d.ts

@@ -1,4 +1,4 @@
-import { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t } from "./gen/zappar-native";
+import { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, transform_orientation_t, log_level_t, plane_orientation_t, anchor_status_t } from "./gen/zappar-native";
 export interface MessageAppender {
     int: (i: number) => void;
     bool: (i: boolean) => void;
@@ -8,6 +8,7 @@ export interface MessageAppender {
     type: (t: number) => void;
     matrix4x4: (a: Float32Array) => void;
     matrix3x3: (a: Float32Array) => void;
+    ucharArray: (a: Uint8Array) => void;
     floatArray: (a: Float32Array) => void;
     cameraModel: (a: Float32Array) => void;
     identityCoefficients: (a: Float32Array) => void;
@@ -16,6 +17,9 @@ export interface MessageAppender {
     barcodeFormat: (f: barcode_format_t) => void;
     faceLandmarkName: (f: face_landmark_name_t) => void;
     instantTrackerTransformOrientation: (f: instant_world_tracker_transform_orientation_t) => void;
+    transformOrientation: (f: transform_orientation_t) => void;
+    planeOrientation: (f: plane_orientation_t) => void;
+    anchorStatus: (f: anchor_status_t) => void;
     logLevel: (f: log_level_t) => void;
 }
 export declare class MessageSerializer {

package/lib/serializer.js

@@ -23,6 +23,7 @@ export class MessageSerializer {
             matrix4x4: i => this.float32ArrayBuffer(i),
             matrix3x3: i => this.float32ArrayBuffer(i),
             floatArray: i => this.float32ArrayBuffer(i),
+            ucharArray: i => this.uint8ArrayBuffer(i),
             identityCoefficients: i => this.float32ArrayBuffer(i),
             expressionCoefficients: i => this.float32ArrayBuffer(i),
             cameraModel: i => this.float32ArrayBuffer(i),
@@ -30,6 +31,9 @@ export class MessageSerializer {
             barcodeFormat: i => this.int(i),
             faceLandmarkName: i => this.int(i),
             instantTrackerTransformOrientation: i => this.int(i),
+            transformOrientation: i => this.int(i),
+            planeOrientation: i => this.int(i),
+            anchorStatus: i => this.int(i),
             logLevel: i => this.int(i),
         };
         this._freeBufferPool.push(new ArrayBuffer(16));

package/lib/tr.js

@@ -35,7 +35,7 @@ export function tr(str) {
     }
     switch (str) {
         case STRINGS.PermissionTitle: return "Almost there...";
-        case STRINGS.PermissionDescription: return "In order to provide this augmented reality experience, we need access to your device's camera and motion sensors.";
+        case STRINGS.PermissionDescription: return "In order to provide this immersive experience, we need access to your device's camera and motion sensors.";
         case STRINGS.PermissionButton: return "Grant Access";
     }
     return "";

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.1-alpha.7";
+export declare const VERSION = "3.0.1-beta.0";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.1-alpha.7";
+export const VERSION = "3.0.1-beta.0";

package/lib/worker-client.js

@@ -7,8 +7,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var _a;
 import { MsgManager } from "./messages";
 export let messageManager = new MsgManager();
+const params = new URLSearchParams(window.location.search);
+const shouldRecordData = parseInt((_a = params.get('_zrecorddata')) !== null && _a !== void 0 ? _a : '0');
 const wasmModule = (() => __awaiter(void 0, void 0, void 0, function* () {
     const response = yield fetch(new URL("./zappar-cv.wasm", import.meta.url).toString());
     const module = yield WebAssembly.compileStreaming(response);
@@ -22,7 +25,7 @@ export function launchWorker(worker) {
             t: "wasm",
             url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
             module: yield wasmModule,
-            shouldRecordData: window.location.href.indexOf("_zrecorddata") >= 0
+            shouldRecordData
         });
         yield waitForLoad(worker);
         function sendOutgoing() {

package/lib/worker-imagebitmap.js

@@ -53,7 +53,7 @@ export function handleImageBitmap(m, r, server, mgr) {
     mgr.postOutgoingMessage(msg, [m.i]);
     let pipeline = server._pipeline_by_instance.get(m.p);
     if (pipeline) {
-        r.pipeline_camera_frame_submit(pipeline, pixels, info.dataWidth, info.dataHeight, m.tokenId, m.cameraToDevice, m.cameraModel, info.userFacing);
+        r.pipeline_camera_frame_submit(pipeline, pixels, info.dataWidth, info.dataHeight, m.tokenId, m.cameraToDevice, m.cameraModel, info.userFacing, performance.now());
         r.pipeline_frame_update(pipeline);
         server.exploreState();
     }

package/lib/worker-server.d.ts

@@ -1,3 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-export declare function launchWorkerServer(wasmUrl: string, module: WebAssembly.Module, shouldRecordData: boolean): Promise<void>;
+export declare function launchWorkerServer(wasmUrl: string, module: WebAssembly.Module, shouldRecordData: number): Promise<void>;

package/lib/worker-server.js

@@ -39,8 +39,8 @@ export function launchWorkerServer(wasmUrl, module, shouldRecordData) {
             onRuntimeInitialized: () => {
                 let r = getRuntimeObject(mod);
                 const workerMessageAPI = getWorkerMessageAPI(mod);
-                const dataDownloadAPI = shouldRecordData ? getDataDownloadAPI(mod) : undefined;
-                dataDownloadAPI === null || dataDownloadAPI === void 0 ? void 0 : dataDownloadAPI.data_should_record_set(1); // Only happens if dataDownloadAPI is defined
+                const dataDownloadAPI = (shouldRecordData > 0) ? getDataDownloadAPI(mod) : undefined;
+                dataDownloadAPI === null || dataDownloadAPI === void 0 ? void 0 : dataDownloadAPI.data_should_record_set(shouldRecordData); // Only happens if dataDownloadAPI is defined
                 let server = new zappar_server(r, (pipelineId, ab) => {
                     messageManager.postOutgoingMessage({
                         p: pipelineId,
@@ -64,7 +64,7 @@ export function launchWorkerServer(wasmUrl, module, shouldRecordData) {
                             let pipeline = server._pipeline_by_instance.get(msgt.p);
                             let att;
                             if (pipeline) {
-                                r.pipeline_camera_frame_submit(pipeline, msgt.d, msgt.width, msgt.height, msgt.token, msgt.c2d, msgt.cm, msgt.userFacing);
+                                r.pipeline_camera_frame_submit(pipeline, msgt.d, msgt.width, msgt.height, msgt.token, msgt.c2d, msgt.cm, msgt.userFacing, msgt.captureTime);
                                 r.pipeline_frame_update(pipeline);
                                 att = r.pipeline_camera_frame_device_attitude(pipeline);
                                 server.exploreState();
@@ -266,7 +266,7 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                 mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
             else
                 mat4.identity(cameraToDeviceTransform);
-            let focalLength = 300.0 * dataWidth / 320.0;
+            let focalLength = 240.0 * dataWidth / 320.0;
             cameraModel[0] = focalLength;
             cameraModel[1] = focalLength;
             cameraModel[2] = dataWidth * 0.5;
@@ -288,7 +288,7 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
             const pipeline = server._pipeline_by_instance.get(p);
             if (pipeline) {
                 try {
-                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, (_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1, downSample);
+                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, ((_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1) * 1e-6, downSample);
                     handleMessages(mod, workerMessageAPI);
                     if (dataDownloadAPI)
                         handleDataDownload(mod, dataDownloadAPI);

package/lib/worker.js

@@ -10,7 +10,7 @@ let launchHandler = (evt) => {
     var _a;
     if (evt && evt.data && evt.data.t === "wasm") {
         let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zappar-cv.wasm", import.meta.url).toString();
-        launchWorkerServer(url, evt.data.module, (_a = evt.data.shouldRecordData) !== null && _a !== void 0 ? _a : false);
+        launchWorkerServer(url, evt.data.module, (_a = evt.data.shouldRecordData) !== null && _a !== void 0 ? _a : 0);
         ctx.removeEventListener("message", launchHandler);
     }
 };

package/lib/workerinterface.d.ts

@@ -11,6 +11,7 @@ export interface CameraFrameC2S {
     token: number;
     width: number;
     height: number;
+    captureTime: number;
 }
 export interface CameraFrameReturnS2C {
     t: "cameraFrameRecycleS2C";