@zappar/zappar-cv 3.0.0-beta.7 → 3.0.1-alpha.1

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/3.0.0-beta.7/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/3.0.1-alpha.1/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/3.0.0-beta.7/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.0-beta.7/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/3.0.1-alpha.1/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.1-alpha.1/zappar-cv.zip)

package/lib/additional.d.ts

@@ -1,4 +1,3 @@
-import { CameraFrameData } from "./camera-frame-data";
 import { zappar_face_mesh_t, zappar_face_tracker_t, zappar_pipeline_t, zappar_image_tracker_t, zappar_sequence_source_t } from "./gen/zappar";
 export declare type zappar_html_element_source_t = number & {
     _: 'zappar_html_element_source_t';
@@ -9,8 +8,6 @@ export interface Additional {
     pipeline_draw_face(pipeline: zappar_pipeline_t, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: zappar_face_mesh_t): void;
     pipeline_draw_image_target_preview(pipeline: zappar_pipeline_t, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: zappar_image_tracker_t, indx: number): void;
     pipeline_draw_face_project(pipeline: zappar_pipeline_t, drawwMatrix: Float32Array, vertices: Float32Array, uvMatrix: Float32Array, uvs: Float32Array, indices: Uint16Array, texture: WebGLTexture): void;
-    pipeline_camera_frame_data_raw_enabled_set(pipeline: zappar_pipeline_t, enabled: boolean): void;
-    pipeline_camera_frame_data_raw(pipeline: zappar_pipeline_t): Promise<CameraFrameData | undefined>;
     draw_plane(gl: WebGLRenderingContext, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, texture: string): void;
     image_tracker_target_image(o: zappar_image_tracker_t, indx: number): HTMLImageElement | undefined;
     face_mesh_load_default(o: zappar_face_mesh_t): Promise<void>;

package/lib/camera-source.js

@@ -35,11 +35,15 @@ export class CameraSource extends HTMLElementSource {
             if (!pipeline)
                 return;
             let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
-            let interval = ev.interval * profile.intervalMultiplier;
-            if (this._lastTimestamp > -1 && !profile.trustSensorIntervals) {
-                interval = Math.max(timeStamp - this._lastTimestamp, interval);
+            const interval = ev.interval * profile.intervalMultiplier;
+            if (profile.trustSensorIntervals) {
+                if (this._lastTimestamp === -1)
+                    this._lastTimestamp = timeStamp;
+                else {
+                    this._lastTimestamp += interval;
+                    timeStamp = this._lastTimestamp;
+                }
             }
-            this._lastTimestamp = timeStamp;
             if (ev.acceleration !== null &&
                 ev.acceleration.x !== null &&
                 ev.acceleration.y !== null &&

package/lib/ceres-worker.d.ts

@@ -0,0 +1 @@
+export declare function launchCeresWorker(wasmUrl: string, module: WebAssembly.Module, port: MessagePort): Promise<void>;

package/lib/ceres-worker.js

@@ -0,0 +1,67 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { getWorkerMessageAPI } from "./worker-messages";
+import * as ZNM from "./zappar-cv";
+const ctx = self;
+let launchHandler = (evt) => {
+    if (evt && evt.data && evt.data.t === "wasm") {
+        let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zappar-cv.wasm", import.meta.url).toString();
+        launchCeresWorker(url, evt.data.module, evt.data.port);
+        ctx.removeEventListener("message", launchHandler);
+    }
+};
+ctx.addEventListener("message", launchHandler);
+export function launchCeresWorker(wasmUrl, module, port) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let mod = ZNM.default({
+            locateFile: (path, prefix) => {
+                if (path.endsWith("zappar-cv.wasm")) {
+                    return wasmUrl;
+                }
+                return prefix + path;
+            },
+            instantiateWasm: (imports, successCallback) => {
+                const instance = new WebAssembly.Instance(module, imports);
+                successCallback(instance);
+                return instance.exports;
+            },
+            onRuntimeInitialized: () => {
+                const ceres_worker = mod.cwrap('ceres_worker', null, []);
+                const messageAPI = getWorkerMessageAPI(mod);
+                port.addEventListener('message', evt => {
+                    if (evt.data.t !== 'msgsend')
+                        return;
+                    const data = evt.data.data;
+                    const reference = evt.data.reference;
+                    const ptr = mod._malloc(data.byteLength);
+                    mod.HEAPU8.set(data, ptr);
+                    messageAPI.worker_message_receive(reference, data.byteLength, ptr);
+                    mod._free(ptr);
+                    ceres_worker();
+                    const toSend = messageAPI.worker_message_send_count();
+                    for (let i = 0; i < toSend; i++) {
+                        const reference = messageAPI.worker_message_send_reference(i);
+                        const dataSize = messageAPI.worker_message_send_data_size(i);
+                        const dataPtr = messageAPI.worker_message_send_data(i);
+                        const data = mod.HEAPU8.slice(dataPtr, dataPtr + dataSize);
+                        port.postMessage({
+                            t: 'msgrec',
+                            data,
+                            reference
+                        }, [data.buffer]);
+                    }
+                    messageAPI.worker_message_send_clear();
+                });
+                port.start();
+            }
+        });
+    });
+}
+;

package/lib/direct.js

@@ -13,6 +13,8 @@ import { messageManager as workerMessageManager } from "./worker-server";
 export let messageManager = new MsgManager();
 export function launchWorker(wasm) {
     return __awaiter(this, void 0, void 0, function* () {
+        const response = yield fetch(new URL("./zappar-cv.wasm", import.meta.url).toString());
+        const module = yield WebAssembly.compileStreaming(response);
         messageManager.onOutgoingMessage.bind(() => {
             let msgs = messageManager.getOutgoingMessages();
             for (let msg of msgs)
@@ -23,6 +25,6 @@ export function launchWorker(wasm) {
             for (let msg of msgs)
                 messageManager.postIncomingMessage(msg.msg);
         });
-        launchWorkerServer(wasm);
+        launchWorkerServer(wasm, module);
     });
 }

package/lib/gen/zappar-client.d.ts

@@ -1,6 +1,6 @@
 import { MessageSerializer } from "../serializer";
 import { MessageDeserializer } from "../deserializer";
-import { zappar } from "./zappar-bridge";
+import { zappar_cwrap } from "./zappar-native";
 export declare class zappar_client {
     private _messageSender;
     constructor(_messageSender: (ab: ArrayBuffer) => void);
@@ -18,6 +18,6 @@ export declare class zappar_client {
     private _barcode_finder_state_by_instance;
     private _instant_world_tracker_state_by_instance;
     private _world_tracker_state_by_instance;
-    impl: zappar;
+    impl: zappar_cwrap;
     processMessages(a: ArrayBuffer): void;
 }

package/lib/gen/zappar-client.js

@@ -103,6 +103,11 @@ export class zappar_client {
                     m.bool(user_facing);
                 });
             },
+            pipeline_camera_frame_submit_raw_pointer: (o, data, dataLength, format, width, height, user_data, camera_to_device_transform, rotation, camera_model, user_facing, timestamp) => {
+                let s = this._pipeline_state_by_instance.get(o);
+                if (!s)
+                    throw new Error("This object has been destroyed");
+            },
             pipeline_camera_frame_camera_attitude: (o) => {
                 let s = this._pipeline_state_by_instance.get(o);
                 if (!s)

package/lib/gen/zappar-cwrap.js

@@ -8,18 +8,6 @@ export function getRuntimeObject(mod) {
     ]);
     let pipeline_create_wrapped = mod.cwrap("zappar_pipeline_create", "number", []);
     let pipeline_destroy_wrapped = mod.cwrap("zappar_pipeline_destroy", null, ["number"]);
-    let pipeline_camera_frame_data_raw_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw", "number", [
-        "number"
-    ]);
-    let pipeline_camera_frame_data_raw_size_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_size", "number", [
-        "number"
-    ]);
-    let pipeline_camera_frame_data_raw_width_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_width", "number", [
-        "number"
-    ]);
-    let pipeline_camera_frame_data_raw_height_wrapped = mod.cwrap("zappar_pipeline_camera_frame_data_raw_height", "number", [
-        "number"
-    ]);
     let pipeline_frame_update_wrapped = mod.cwrap("zappar_pipeline_frame_update", null, [
         "number"
     ]);
@@ -38,7 +26,7 @@ export function getRuntimeObject(mod) {
     ]);
     let pipeline_camera_frame_submit_raw_pointer_wrapped = mod.cwrap("zappar_pipeline_camera_frame_submit_raw_pointer", null, [
         "number",
-        "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"
+        "number", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"
     ]);
     let pipeline_camera_frame_camera_attitude_wrapped = mod.cwrap("zappar_pipeline_camera_frame_camera_attitude", "number", [
         "number"
@@ -269,22 +257,6 @@ export function getRuntimeObject(mod) {
         pipeline_destroy: () => {
             pipeline_destroy_wrapped();
         },
-        pipeline_camera_frame_data_raw: (o) => {
-            let ret = pipeline_camera_frame_data_raw_wrapped(o);
-            return ret;
-        },
-        pipeline_camera_frame_data_raw_size: (o) => {
-            let ret = pipeline_camera_frame_data_raw_size_wrapped(o);
-            return ret;
-        },
-        pipeline_camera_frame_data_raw_width: (o) => {
-            let ret = pipeline_camera_frame_data_raw_width_wrapped(o);
-            return ret;
-        },
-        pipeline_camera_frame_data_raw_height: (o) => {
-            let ret = pipeline_camera_frame_data_raw_height_wrapped(o);
-            return ret;
-        },
         pipeline_frame_update: (o) => {
             let ret = pipeline_frame_update_wrapped(o);
             return ret;
@@ -324,7 +296,7 @@ export function getRuntimeObject(mod) {
             let ret = pipeline_camera_frame_submit_wrapped(o, arg_data, arg_len_data, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_camera_model, arg_user_facing);
             return ret;
         },
-        pipeline_camera_frame_submit_raw_pointer: (o, data, dataLength, format, width, height, user_data, camera_to_device_transform, rotation, camera_model, user_facing) => {
+        pipeline_camera_frame_submit_raw_pointer: (o, data, dataLength, format, width, height, user_data, camera_to_device_transform, rotation, camera_model, user_facing, timestamp) => {
             let arg_data = data;
             let arg_dataLength = dataLength;
             let arg_format = format;
@@ -337,7 +309,8 @@ export function getRuntimeObject(mod) {
             let arg_camera_model = getFloatDataArrayForArgIndex(8, camera_model.byteLength);
             mod.HEAPF32.set(camera_model, arg_camera_model / 4);
             let arg_user_facing = user_facing ? 1 : 0;
-            let ret = pipeline_camera_frame_submit_raw_pointer_wrapped(o, arg_data, arg_dataLength, arg_format, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_rotation, arg_camera_model, arg_user_facing);
+            let arg_timestamp = timestamp;
+            let ret = pipeline_camera_frame_submit_raw_pointer_wrapped(o, arg_data, arg_dataLength, arg_format, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_rotation, arg_camera_model, arg_user_facing, arg_timestamp);
             return ret;
         },
         pipeline_camera_frame_camera_attitude: (o) => {

package/lib/gen/zappar-native.d.ts

@@ -102,16 +102,12 @@ export interface zappar_cwrap {
     analytics_project_id_set(id: string, uid: string): void;
     pipeline_create(): zappar_pipeline_t;
     pipeline_destroy(o: zappar_pipeline_t): void;
-    pipeline_camera_frame_data_raw(o: zappar_pipeline_t): number;
-    pipeline_camera_frame_data_raw_size(o: zappar_pipeline_t): number;
-    pipeline_camera_frame_data_raw_width(o: zappar_pipeline_t): number;
-    pipeline_camera_frame_data_raw_height(o: zappar_pipeline_t): number;
     pipeline_frame_update(o: zappar_pipeline_t): void;
     pipeline_frame_number(o: zappar_pipeline_t): number;
     pipeline_camera_model(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_user_data(o: zappar_pipeline_t): number;
     pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean): void;
-    pipeline_camera_frame_submit_raw_pointer(o: zappar_pipeline_t, data: number, dataLength: number, format: frame_pixel_format_t, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, rotation: number, camera_model: Float32Array, user_facing: boolean): void;
+    pipeline_camera_frame_submit_raw_pointer(o: zappar_pipeline_t, data: number, dataLength: number, format: frame_pixel_format_t, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, rotation: number, camera_model: Float32Array, user_facing: boolean, timestamp: number): void;
     pipeline_camera_frame_camera_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_device_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_motion_accelerometer_submit(o: zappar_pipeline_t, time: number, x: number, y: number, z: number): void;

package/lib/gen/zappar.d.ts

@@ -49,7 +49,6 @@ export interface zappar {
     camera_user_facing(indx: number): boolean;
     projection_matrix_from_camera_model(model: Float32Array, renderWidth: number, renderHeight: number): Float32Array;
     projection_matrix_from_camera_model_ext(model: Float32Array, renderWidth: number, renderHeight: number, zNear: number, zFar: number): Float32Array;
-    pose_from_raw(raw_pose: Float32Array, mirror: boolean): Float32Array;
     log_level(): log_level_t;
     log_level_set(level: log_level_t): void;
     permission_request_ui(): void;

package/lib/html-element-source.js

@@ -89,7 +89,6 @@ export class HTMLElementSource extends Source {
             if (info) {
                 let token = pipeline.registerToken(info);
                 pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
-                info.data = undefined;
             }
         }
         catch (ex) {

package/lib/imagetracker.d.ts

@@ -1,6 +1,5 @@
 import { zappar_image_tracker_t, zappar_pipeline_t } from "./gen/zappar";
-import { zappar } from "./gen/zappar-bridge";
-import { image_target_type_t } from "./gen/zappar-native";
+import { image_target_type_t, zappar_cwrap } from "./gen/zappar-native";
 interface PreviewInfo {
     compressed: Uint8Array;
     mimeType: string;
@@ -27,7 +26,7 @@ export declare class ImageTracker {
     private _client;
     private _impl;
     private _targets;
-    static create(pipeline: zappar_pipeline_t, client: zappar): zappar_image_tracker_t;
+    static create(pipeline: zappar_pipeline_t, client: zappar_cwrap): zappar_image_tracker_t;
     static get(p: zappar_image_tracker_t): ImageTracker | undefined;
     private constructor();
     destroy(): void;

package/lib/index.d.ts

@@ -3,5 +3,4 @@ import { Additional } from "./additional";
 import { Options } from "./options";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(opts?: Options): Zappar;
-export { CameraFrameData } from './camera-frame-data';
 export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";

package/lib/mstp-camera-source.d.ts

@@ -11,6 +11,9 @@ export declare class MSTPCameraSource extends Source {
     private _isPaused;
     private _isUserFacing;
     private _cameraToScreenRotation;
+    private _accelerometer?;
+    private _linearAcceleration?;
+    private _gyroscope?;
     constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
     destroy(): void;
     private _stop;
@@ -26,6 +29,12 @@ export declare class MSTPCameraSource extends Source {
     private _deviceMotionListener;
     private _startDeviceOrientation;
     private _startDeviceMotion;
+    private _lastAccelerometerTimestamp;
+    private _accelerometerReading;
+    private _lastLinearAccelerationTimestamp;
+    private _linearAccelerationReading;
+    private _lastGyroscopeReading;
+    private _gyroscopeReading;
     private _stopDeviceMotion;
     uploadGL(info: CameraFrameInfo): void;
 }

package/lib/mstp-camera-source.js

@@ -31,12 +31,15 @@ export class MSTPCameraSource extends Source {
             if (!pipeline)
                 return;
             let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
-            let interval = ev.interval;
-            if (this._lastTimestamp > -1 && !profile.trustSensorIntervals) {
-                interval = Math.max(timeStamp - this._lastTimestamp, interval);
+            const interval = ev.interval * profile.intervalMultiplier;
+            if (profile.trustSensorIntervals) {
+                if (this._lastTimestamp === -1)
+                    this._lastTimestamp = timeStamp;
+                else {
+                    this._lastTimestamp += interval;
+                    timeStamp = this._lastTimestamp;
+                }
             }
-            this._lastTimestamp = timeStamp;
-            interval *= profile.intervalMultiplier;
             if (ev.acceleration !== null &&
                 ev.acceleration.x !== null &&
                 ev.acceleration.y !== null &&
@@ -61,6 +64,47 @@ export class MSTPCameraSource extends Source {
                 this._startDeviceOrientation();
             }
         };
+        this._lastAccelerometerTimestamp = -1;
+        this._accelerometerReading = () => {
+            var _a;
+            if (!this._accelerometer || this._accelerometer.x === undefined || this._accelerometer.y === undefined || this._accelerometer.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._accelerometer.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastAccelerometerTimestamp > -1 ? timestamp - this._lastAccelerometerTimestamp : 30;
+            this._lastAccelerometerTimestamp = timestamp;
+            pipeline.motionAccelerometerSubmit(timestamp, this._accelerometer.x * profile.deviceMotionMutliplier, this._accelerometer.y * profile.deviceMotionMutliplier, this._accelerometer.z * profile.deviceMotionMutliplier);
+            pipeline.motionAccelerometerWithGravitySubmitInt(timestamp, interval, this._accelerometer.x * profile.deviceMotionMutliplier, this._accelerometer.y * profile.deviceMotionMutliplier, this._accelerometer.z * profile.deviceMotionMutliplier);
+        };
+        this._lastLinearAccelerationTimestamp = -1;
+        this._linearAccelerationReading = () => {
+            var _a;
+            if (!this._linearAcceleration || this._linearAcceleration.x === undefined || this._linearAcceleration.y === undefined || this._linearAcceleration.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._linearAcceleration.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastLinearAccelerationTimestamp > -1 ? timestamp - this._lastLinearAccelerationTimestamp : 30;
+            this._lastLinearAccelerationTimestamp = timestamp;
+            pipeline.motionAccelerometerWithoutGravitySubmitInt(timestamp, interval, this._linearAcceleration.x * profile.deviceMotionMutliplier, this._linearAcceleration.y * profile.deviceMotionMutliplier, this._linearAcceleration.z * profile.deviceMotionMutliplier);
+        };
+        this._lastGyroscopeReading = -1;
+        this._gyroscopeReading = () => {
+            var _a;
+            if (!this._gyroscope || this._gyroscope.x === undefined || this._gyroscope.y === undefined || this._gyroscope.z === undefined)
+                return;
+            let pipeline = Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            const timestamp = (_a = this._gyroscope.timestamp) !== null && _a !== void 0 ? _a : performance.now();
+            const interval = this._lastGyroscopeReading > -1 ? timestamp - this._lastGyroscopeReading : 30;
+            this._lastGyroscopeReading = timestamp;
+            pipeline.motionRotationRateSubmit(timestamp, -this._gyroscope.x, -this._gyroscope.y, -this._gyroscope.z);
+            pipeline.motionRotationRateSubmitInt(timestamp, interval, this._gyroscope.x * 180 / Math.PI, this._gyroscope.y * 180 / Math.PI, this._gyroscope.z * 180 / Math.PI);
+        };
         zcout("Using MSTP camera source");
     }
     destroy() {
@@ -221,9 +265,38 @@ export class MSTPCameraSource extends Source {
         });
     }
     _startDeviceMotion() {
-        window.addEventListener("devicemotion", this._deviceMotionListener, false);
+        try {
+            this._accelerometer = new Accelerometer({ frequency: 60 });
+            this._linearAcceleration = new LinearAccelerationSensor({ frequency: 60 });
+            this._gyroscope = new Gyroscope({ frequency: 60 });
+            // throw new Error();
+            this._accelerometer.addEventListener('reading', this._accelerometerReading);
+            this._accelerometer.start();
+            this._linearAcceleration.addEventListener('reading', this._linearAccelerationReading);
+            this._linearAcceleration.start();
+            this._gyroscope.addEventListener('reading', this._gyroscopeReading);
+            this._gyroscope.start();
+        }
+        catch (err) {
+            window.addEventListener("devicemotion", this._deviceMotionListener, false);
+        }
     }
     _stopDeviceMotion() {
+        if (this._accelerometer) {
+            this._accelerometer.stop();
+            this._accelerometer.removeEventListener('reading', this._accelerometerReading);
+            delete this._accelerometer;
+        }
+        if (this._linearAcceleration) {
+            this._linearAcceleration.stop();
+            this._linearAcceleration.removeEventListener('reading', this._linearAccelerationReading);
+            delete this._linearAcceleration;
+        }
+        if (this._gyroscope) {
+            this._gyroscope.stop();
+            this._gyroscope.removeEventListener('reading', this._gyroscopeReading);
+            delete this._gyroscope;
+        }
         window.removeEventListener("devicemotion", this._deviceMotionListener);
     }
     uploadGL(info) {

package/lib/native.js

@@ -12,7 +12,7 @@ import { zappar_client } from "./gen/zappar-client";
 import { drawPlane } from "./drawplane";
 import { cameraRotationForScreenOrientation, projectionMatrix } from "./cameramodel";
 import { mat4, vec3 } from "gl-matrix";
-import { launchWorker, messageManager } from "./worker-client";
+import { launchCeresWorker, launchWorker, messageManager } from "./worker-client";
 import { permissionRequestUI, permissionGrantedAll, permissionGrantedCamera, permissionGrantedMotion, permissionDeniedAny, permissionDeniedCamera, permissionDeniedMotion, permissionRequestMotion, permissionRequestCamera, permissionRequestAll, permissionDeniedUI } from "./permission";
 import { createFaceMesh, destroyFaceMesh, getFaceMesh } from "./facemesh";
 import { Pipeline, applyScreenCounterRotation } from "./pipeline";
@@ -49,7 +49,7 @@ export function initialize(opts) {
             c.impl.analytics_project_id_set(".wiz" + pathParts[1], uid);
     }
     messageManager.onIncomingMessage.bind(msg => {
-        var _a, _b, _c, _d, _e, _f, _g, _h;
+        var _a, _b, _c, _d, _e, _f, _g;
         switch (msg.t) {
             case "zappar":
                 (_a = Pipeline.get(msg.p)) === null || _a === void 0 ? void 0 : _a.pendingMessages.push(msg.d);
@@ -72,11 +72,6 @@ export function initialize(opts) {
                 (_g = (_f = Pipeline.get(msgt.p)) === null || _f === void 0 ? void 0 : _f.imageBitmapFromWorker) === null || _g === void 0 ? void 0 : _g.call(_f, msgt);
                 break;
             }
-            case "raw": {
-                let msgt = msg;
-                (_h = Pipeline.get(msgt.p)) === null || _h === void 0 ? void 0 : _h.cameraFrameDataRawResult(msgt);
-                break;
-            }
             case "streamEndedS2C": {
                 let msgt = msg;
                 const src = getCameraSource(msgt.source);
@@ -84,6 +79,10 @@ export function initialize(opts) {
                     src.streamEnded();
                 break;
             }
+            case "setupCeresWorker": {
+                launchCeresWorker(msg.port);
+                break;
+            }
             case "licerr": {
                 let div = document.createElement("div");
                 div.innerHTML = "Visit <a href='https://docs.zap.works/universal-ar/licensing/' style='color: white;'>our licensing page</a> to find out about hosting on your own domain.";
@@ -133,16 +132,7 @@ export function initialize(opts) {
             }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pose_from_raw: (raw_pose, mirror) => {
-            let res = applyScreenCounterRotation(undefined, raw_pose);
-            if (mirror) {
-                let scale = mat4.create();
-                mat4.fromScaling(scale, [-1, 1, 1]);
-                mat4.multiply(res, scale, res);
-                mat4.multiply(res, res, scale);
-            }
-            return res;
-        }, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? CameraSource.USER_DEFAULT_DEVICE_ID : CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, camera_count: () => 2, camera_id: indx => indx === 0 ? CameraSource.DEFAULT_DEVICE_ID : CameraSource.USER_DEFAULT_DEVICE_ID, camera_name: indx => indx === 0 ? "Rear-facing Camera" : "User-facing Camera", camera_user_facing: indx => indx !== 0, pipeline_create: () => Pipeline.create(c.impl, messageManager), pipeline_frame_update: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
@@ -166,7 +156,7 @@ export function initialize(opts) {
                 return;
             }
             (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, obj);
-        }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, pipeline_camera_frame_data_raw: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.getCameraFrameDataRaw()) || Promise.reject(); }, pipeline_camera_frame_data_raw_enabled_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.setCameraFrameDataRawEnabled(v); }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
+        }, projection_matrix_from_camera_model: projectionMatrix, projection_matrix_from_camera_model_ext: projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.instant_world_tracker_anchor_pose_raw(o));
             if (mirror) {
                 let scale = mat4.create();

package/lib/pipeline.d.ts

@@ -1,14 +1,13 @@
 /// <reference types="dom-webcodecs" />
 import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
+import { zappar_cwrap } from "./gen/zappar-native";
 import { zappar_client } from "./gen/zappar-client";
 import { MsgManager } from "./messages";
 import { Source, CameraFrameInfo } from "./source";
-import { CameraDataRawS2C, CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
+import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
 import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
 import { ImageTracker } from "./imagetracker";
-import { zappar } from "./gen/zappar-bridge";
-import { CameraFrameData } from "./camera-frame-data";
 export declare class Pipeline {
     private _client;
     private _impl;
@@ -28,17 +27,12 @@ export declare class Pipeline {
     private _sequenceRecorder;
     private _sequenceRecordDeviceAttitudeMatrices;
     private _sequenceRecorderFirstCameraToken;
-    private _cameraFrameDataPromises;
-    private _cameraFrameDataResolves;
-    private _cameraFrameDataRejects;
-    private _cameraFrameDataEnabled;
     onGLContextReset: Event;
-    static create(client: zappar, mgr: MsgManager): zappar_pipeline_t;
+    static create(client: zappar_cwrap, mgr: MsgManager): zappar_pipeline_t;
     static get(p: zappar_pipeline_t): Pipeline | undefined;
     private constructor();
     frameUpdate(client: zappar_client): void;
     cleanOldFrames(): void;
-    cameraFrameDataRawResult(msg: CameraDataRawS2C): void;
     cameraTokenReturn(msg: CameraFrameReturnS2C): void;
     sequenceRecordStart(expectedFrames: number): void;
     sequenceRecordStop(): void;
@@ -46,8 +40,6 @@ export declare class Pipeline {
     sequenceRecordClear(): void;
     sequenceRecordDeviceAttitudeMatrices(v: boolean): void;
     getVideoTexture(): WebGLTexture | undefined;
-    getCameraFrameDataRaw(): Promise<CameraFrameData | undefined>;
-    setCameraFrameDataRawEnabled(e: boolean): void;
     destroy(): void;
     getCurrentCameraInfo(): CameraFrameInfo | undefined;
     cameraFrameDrawGL(screenWidth: number, screenHeight: number, mirror?: boolean): void;