@zappar/zappar-cv 2.1.4 → 3.0.0-alpha.0

package/lib/imagebitmap-camera-source.d.ts

@@ -28,6 +28,7 @@ export declare class ImageBitmapCameraSource extends Source {
     private _getUserMedia;
     private _syncCamera;
     private _hasStartedOrientation;
+    private _lastTimestamp;
     private _deviceMotionListener;
     private _startDeviceOrientation;
     private _startDeviceMotion;

package/lib/imagebitmap-camera-source.js

@@ -33,23 +33,37 @@ export class ImageBitmapCameraSource extends Source {
         this._cameraToDeviceTransformUserFacing = mat4.create();
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
         this._hasStartedOrientation = false;
+        this._lastTimestamp = -1;
         this._deviceMotionListener = (ev) => {
             let pipeline = Pipeline.get(this._pipeline);
             if (!pipeline)
                 return;
             let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            let interval = ev.interval;
+            if (this._lastTimestamp > -1 && !profile.trustSensorIntervals) {
+                interval = Math.max(timeStamp - this._lastTimestamp, interval);
+            }
+            this._lastTimestamp = timeStamp;
+            interval *= profile.intervalMultiplier;
+            if (ev.acceleration !== null &&
+                ev.acceleration.x !== null &&
+                ev.acceleration.y !== null &&
+                ev.acceleration.z !== null) {
+                pipeline.motionAccelerometerWithoutGravitySubmitInt(timeStamp, interval, ev.acceleration.x * profile.deviceMotionMutliplier, ev.acceleration.y * profile.deviceMotionMutliplier, ev.acceleration.z * profile.deviceMotionMutliplier);
+            }
             if (ev.accelerationIncludingGravity !== null &&
                 ev.accelerationIncludingGravity.x !== null &&
                 ev.accelerationIncludingGravity.y !== null &&
                 ev.accelerationIncludingGravity.z !== null) {
                 pipeline.motionAccelerometerSubmit(timeStamp, ev.accelerationIncludingGravity.x * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile.deviceMotionMutliplier);
+                pipeline.motionAccelerometerWithGravitySubmitInt(timeStamp, interval, ev.accelerationIncludingGravity.x * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile.deviceMotionMutliplier);
             }
             if (ev.rotationRate !== null &&
                 ev.rotationRate.alpha !== null &&
                 ev.rotationRate.beta !== null &&
-                ev.rotationRate.gamma !== null && !this._hasStartedOrientation) {
-                ev.timeStamp;
-                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha * Math.PI / -180.0, ev.rotationRate.beta * Math.PI / -180.0, ev.rotationRate.gamma * Math.PI / -180.0);
+                ev.rotationRate.gamma !== null && this._hasStartedOrientation) {
+                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha, ev.rotationRate.beta, ev.rotationRate.gamma);
+                pipeline.motionRotationRateSubmitInt(timeStamp, interval, ev.rotationRate.alpha, ev.rotationRate.beta, ev.rotationRate.gamma);
             }
             else if (!this._hasStartedOrientation) {
                 this._startDeviceOrientation();
@@ -240,6 +254,7 @@ export class ImageBitmapCameraSource extends Source {
             if (ev.alpha === null || ev.beta === null || ev.gamma === null)
                 return;
             pipeline.motionAttitudeSubmit(timeStamp, ev.alpha, ev.beta, ev.gamma);
+            pipeline.motionAttitudeSubmitInt(timeStamp, 0, ev.alpha, ev.beta, ev.gamma);
         });
     }
     _startDeviceMotion() {

package/lib/index.d.ts

@@ -3,4 +3,4 @@ import { Additional } from "./additional";
 import { Options } from "./options";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(opts?: Options): Zappar;
-export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t } from "./gen/zappar";
+export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";

package/lib/index.js

@@ -4,4 +4,4 @@ export function initialize(opts) {
     console.log(`Zappar CV v${VERSION}`);
     return nativeInitialize(opts);
 }
-export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t } from "./gen/zappar";
+export { barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, image_target_type_t, world_tracker_quality_t } from "./gen/zappar";

package/lib/mstp-camera-source.d.ts

@@ -21,6 +21,7 @@ export declare class MSTPCameraSource extends Source {
     private _getUserMedia;
     private _syncCamera;
     private _hasStartedOrientation;
+    private _lastTimestamp;
     private _deviceMotionListener;
     private _startDeviceOrientation;
     private _startDeviceMotion;

package/lib/mstp-camera-source.js

@@ -25,23 +25,37 @@ export class MSTPCameraSource extends Source {
         this._isUserFacing = false;
         this._cameraToScreenRotation = 0;
         this._hasStartedOrientation = false;
+        this._lastTimestamp = -1;
         this._deviceMotionListener = (ev) => {
             let pipeline = Pipeline.get(this._pipeline);
             if (!pipeline)
                 return;
             let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            let interval = ev.interval;
+            if (this._lastTimestamp > -1 && !profile.trustSensorIntervals) {
+                interval = Math.max(timeStamp - this._lastTimestamp, interval);
+            }
+            this._lastTimestamp = timeStamp;
+            interval *= profile.intervalMultiplier;
+            if (ev.acceleration !== null &&
+                ev.acceleration.x !== null &&
+                ev.acceleration.y !== null &&
+                ev.acceleration.z !== null) {
+                pipeline.motionAccelerometerWithoutGravitySubmitInt(timeStamp, interval, ev.acceleration.x * profile.deviceMotionMutliplier, ev.acceleration.y * profile.deviceMotionMutliplier, ev.acceleration.z * profile.deviceMotionMutliplier);
+            }
             if (ev.accelerationIncludingGravity !== null &&
                 ev.accelerationIncludingGravity.x !== null &&
                 ev.accelerationIncludingGravity.y !== null &&
                 ev.accelerationIncludingGravity.z !== null) {
                 pipeline.motionAccelerometerSubmit(timeStamp, ev.accelerationIncludingGravity.x * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile.deviceMotionMutliplier);
+                pipeline.motionAccelerometerWithGravitySubmitInt(timeStamp, interval, ev.accelerationIncludingGravity.x * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile.deviceMotionMutliplier);
             }
             if (ev.rotationRate !== null &&
                 ev.rotationRate.alpha !== null &&
                 ev.rotationRate.beta !== null &&
-                ev.rotationRate.gamma !== null && !this._hasStartedOrientation) {
-                ev.timeStamp;
-                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha * Math.PI / -180.0, ev.rotationRate.beta * Math.PI / -180.0, ev.rotationRate.gamma * Math.PI / -180.0);
+                ev.rotationRate.gamma !== null && this._hasStartedOrientation) {
+                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha, ev.rotationRate.beta, ev.rotationRate.gamma);
+                pipeline.motionRotationRateSubmitInt(timeStamp, interval, ev.rotationRate.alpha, ev.rotationRate.beta, ev.rotationRate.gamma);
             }
             else if (!this._hasStartedOrientation) {
                 this._startDeviceOrientation();
@@ -193,6 +207,7 @@ export class MSTPCameraSource extends Source {
             if (ev.alpha === null || ev.beta === null || ev.gamma === null)
                 return;
             pipeline.motionAttitudeSubmit(timeStamp, ev.alpha, ev.beta, ev.gamma);
+            pipeline.motionAttitudeSubmitInt(timeStamp, 0, ev.alpha, ev.beta, ev.gamma);
         });
     }
     _startDeviceMotion() {

package/lib/native.js

@@ -478,6 +478,63 @@ export function initialize(opts) {
                 return mat4.create();
             }
             return obj.anchor_pose;
+        }, world_tracker_plane_pose_camera_relative: (o, indx, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_plane_pose_raw(o, indx));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, world_tracker_plane_pose: (o, indx, cameraPose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_plane_pose_raw(o, indx));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            mat4.multiply(res, cameraPose, res);
+            return res;
+        }, world_tracker_world_anchor_pose: (o, cameraPose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_world_anchor_pose_raw(o));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            mat4.multiply(res, cameraPose, res);
+            return res;
+        }, world_tracker_world_anchor_pose_camera_relative: (o, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_world_anchor_pose_raw(o));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
+        }, world_tracker_ground_anchor_pose: (o, cameraPose, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_ground_anchor_pose_raw(o));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            mat4.multiply(res, cameraPose, res);
+            return res;
+        }, world_tracker_ground_anchor_pose_camera_relative: (o, mirror) => {
+            let res = applyScreenCounterRotation(undefined, c.impl.world_tracker_ground_anchor_pose_raw(o));
+            if (mirror) {
+                let scale = mat4.create();
+                mat4.fromScaling(scale, [-1, 1, 1]);
+                mat4.multiply(res, scale, res);
+                mat4.multiply(res, res, scale);
+            }
+            return res;
         }, html_element_source_create: (pipeline, elm) => HTMLElementSource.createVideoElementSource(pipeline, elm), html_element_source_start: o => { var _a; return (_a = HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.start(); }, html_element_source_pause: o => { var _a; return (_a = HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, html_element_source_destroy: o => { var _a; return (_a = HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, sequence_source_create: p => SequenceSource.create(p), sequence_source_load_from_memory: (o, data) => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.loadFromMemory(data); }, sequence_source_pause: o => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, sequence_source_start: o => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.start(); }, sequence_source_max_playback_fps_set: (o, fps) => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.maxPlaybackFpsSet(fps); }, sequence_source_time_set: (o, t) => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.setTime(t); }, sequence_source_destroy: o => { var _a; return (_a = SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, permission_granted_all: permissionGrantedAll, permission_granted_camera: permissionGrantedCamera, permission_granted_motion: permissionGrantedMotion, permission_denied_any: permissionDeniedAny, permission_denied_camera: permissionDeniedCamera, permission_denied_motion: permissionDeniedMotion, permission_request_motion: permissionRequestMotion, permission_request_camera: permissionRequestCamera, permission_request_all: permissionRequestAll, permission_request_ui: permissionRequestUI, permission_request_ui_promise: permissionRequestUI, permission_denied_ui: permissionDeniedUI, browser_incompatible: compatibility.incompatible, browser_incompatible_ui: compatibility.incompatible_ui, log_level_set: l => {
             setLogLevel(l);
             c.impl.log_level_set(l);

package/lib/pipeline.d.ts

@@ -58,6 +58,10 @@ export declare class Pipeline {
     registerToken(info: CameraFrameInfo): number;
     processGL(): void;
     motionAccelerometerSubmit(timestamp: number, x: number, y: number, z: number): void;
+    motionAccelerometerWithoutGravitySubmitInt(timestamp: number, interval: number, x: number, y: number, z: number): void;
+    motionAccelerometerWithGravitySubmitInt(timestamp: number, interval: number, x: number, y: number, z: number): void;
+    motionRotationRateSubmitInt(timestamp: number, interval: number, x: number, y: number, z: number): void;
+    motionAttitudeSubmitInt(timestamp: number, interval: number, x: number, y: number, z: number): void;
     motionRotationRateSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionAttitudeSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionAttitudeMatrix(m: Float32Array): void;

package/lib/pipeline.js

@@ -270,23 +270,104 @@ export class Pipeline {
         var _a;
         if (!this._sequenceRecordDeviceAttitudeMatrices)
             (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendAccelerometer(timestamp, x, y, z);
-        this._client.pipeline_motion_accelerometer_submit(this._impl, timestamp, x, y, z);
+        const msg = {
+            t: 'sensorDataC2S',
+            sensor: 'accel',
+            interval: 0,
+            timestamp,
+            x, y, z,
+            p: this._impl
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
+    motionAccelerometerWithoutGravitySubmitInt(timestamp, interval, x, y, z) {
+        var _a;
+        (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendAccelerometerWithoutGravityInt(timestamp, interval, x, y, z);
+        const msg = {
+            t: 'sensorDataC2S',
+            sensor: 'accel_wo_gravity_int',
+            interval,
+            timestamp,
+            x, y, z,
+            p: this._impl
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
+    motionAccelerometerWithGravitySubmitInt(timestamp, interval, x, y, z) {
+        var _a;
+        (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendAccelerometerWithGravityInt(timestamp, interval, x, y, z);
+        const msg = {
+            t: 'sensorDataC2S',
+            sensor: 'accel_w_gravity_int',
+            interval,
+            timestamp,
+            x, y, z,
+            p: this._impl
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
+    motionRotationRateSubmitInt(timestamp, interval, x, y, z) {
+        var _a;
+        (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendRotationRateInt(timestamp, interval, x, y, z);
+        const msg = {
+            t: 'sensorDataC2S',
+            sensor: 'rotation_rate_int',
+            interval,
+            timestamp,
+            x, y, z,
+            p: this._impl
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
+    motionAttitudeSubmitInt(timestamp, interval, x, y, z) {
+        var _a;
+        (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendAttitudeInt(timestamp, interval, x, y, z);
+        const msg = {
+            t: 'sensorDataC2S',
+            sensor: 'attitude_int',
+            interval,
+            timestamp,
+            x, y, z,
+            p: this._impl
+        };
+        this._mgr.postOutgoingMessage(msg, []);
     }
     motionRotationRateSubmit(timestamp, x, y, z) {
         var _a;
         if (!this._sequenceRecordDeviceAttitudeMatrices)
             (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendRotationRate(timestamp, x, y, z);
-        this._client.pipeline_motion_rotation_rate_submit(this._impl, timestamp, x, y, z);
+        const msg = {
+            t: 'sensorDataC2S',
+            sensor: 'rotation_rate',
+            interval: 0,
+            timestamp,
+            x, y, z,
+            p: this._impl
+        };
+        this._mgr.postOutgoingMessage(msg, []);
     }
     motionAttitudeSubmit(timestamp, x, y, z) {
         var _a;
         if (!this._sequenceRecordDeviceAttitudeMatrices)
             (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendAttitude(timestamp, x, y, z);
-        this._client.pipeline_motion_attitude_submit(this._impl, timestamp, x, y, z);
+        const msg = {
+            t: 'sensorDataC2S',
+            sensor: 'attitude',
+            interval: 0,
+            timestamp,
+            x, y, z,
+            p: this._impl
+        };
+        this._mgr.postOutgoingMessage(msg, []);
     }
     motionAttitudeMatrix(m) {
         // This doesn't need to be added to the sequence since that's done on frame update instead
-        this._client.pipeline_motion_attitude_matrix_submit(this._impl, m);
+        const msg = {
+            t: 'attitudeMatrixC2S',
+            p: this._impl,
+            m,
+        };
+        this._mgr.postOutgoingMessage(msg, []);
     }
     sendCameraStreamToWorker(source, stream, userFacing) {
         let msg = {

package/lib/profile.d.ts

@@ -17,4 +17,6 @@ export declare let profile: {
     preferImageBitmapCamera: boolean;
     ios164CameraSelection: boolean;
     forceWindowOrientation: boolean;
+    intervalMultiplier: number;
+    trustSensorIntervals: boolean;
 };

package/lib/profile.js

@@ -15,10 +15,12 @@ export let profile = {
     dataWidth: 320,
     dataHeight: 240,
     videoElementInDOM: false,
-    preferMediaStreamTrackProcessorCamera: false,
+    preferMediaStreamTrackProcessorCamera: true,
     preferImageBitmapCamera: false,
     ios164CameraSelection: false,
     forceWindowOrientation: false,
+    intervalMultiplier: 1,
+    trustSensorIntervals: false,
 };
 if (typeof window !== "undefined") {
     window["zeeProfile"] = profile;
@@ -48,6 +50,9 @@ if (engine === "webkit" && os === "ios") {
 }
 function iDevice(version) {
     profile.forceWindowOrientation = true;
+    profile.preferMediaStreamTrackProcessorCamera = false;
+    profile.intervalMultiplier = 1000;
+    profile.trustSensorIntervals = true;
     let versionParts = version.split(".");
     if (versionParts.length >= 2) {
         const majorVersion = parseInt(versionParts[0]);

package/lib/sequencerecorder.d.ts

@@ -4,7 +4,11 @@ export declare enum SequenceRecorderPacketType {
     ACCELEROMETER = 1,
     ROTATION_RATE = 2,
     ATTITUDE = 3,
-    ATTITUDE_MATRIX = 4
+    ATTITUDE_MATRIX = 4,
+    ACCELEROMETER_WO_GRAVITY_INT = 5,
+    ACCELEROMETER_W_GRAVITY_INT = 6,
+    ROTATION_RATE_INT = 7,
+    ATTITUDE_INT = 8
 }
 export declare class SequenceRecorder {
     static headerSizeBytes: number;
@@ -16,6 +20,7 @@ export declare class SequenceRecorder {
     static psbCameraModel: number;
     static cameraPacketSizeBytes: number;
     static accelerometerPacketSizeBytes: number;
+    static sensorWithIntervalPacketSizeBytes: number;
     static rotationRatePacketSizeBytes: number;
     static attitudePacketSizeBytes: number;
     static attitudeMatrixPacketSizeBytes: number;
@@ -36,6 +41,11 @@ export declare class SequenceRecorder {
     appendAccelerometer(t: number, x: number, y: number, z: number): void;
     appendRotationRate(t: number, x: number, y: number, z: number): void;
     appendAttitude(t: number, x: number, y: number, z: number): void;
+    appendAccelerometerWithoutGravityInt(t: number, int: number, x: number, y: number, z: number): void;
+    appendAccelerometerWithGravityInt(t: number, int: number, x: number, y: number, z: number): void;
+    appendRotationRateInt(t: number, int: number, x: number, y: number, z: number): void;
+    appendAttitudeInt(t: number, int: number, x: number, y: number, z: number): void;
+    private _appendSensorWithInterval;
     private _appendTimestampedVec3;
     appendAttitudeMatrix(m: Float32Array): void;
     private _growArrayBuffer;
@@ -64,11 +74,16 @@ export interface SequenceRotationRateData extends SequenceVec3Data {
 export interface SequenceAttitudeData extends SequenceVec3Data {
     type: SequenceRecorderPacketType.ATTITUDE;
 }
+declare type SensorSourceType = SequenceRecorderPacketType.ACCELEROMETER_WO_GRAVITY_INT | SequenceRecorderPacketType.ACCELEROMETER_W_GRAVITY_INT | SequenceRecorderPacketType.ATTITUDE_INT | SequenceRecorderPacketType.ROTATION_RATE_INT;
+export interface SequenceSensorData extends SequenceVec3Data {
+    type: SensorSourceType;
+    int: number;
+}
 export interface SequenceAttitudeMatrixData {
     type: SequenceRecorderPacketType.ATTITUDE_MATRIX;
     attitude: Float32Array;
 }
-export declare type SequencePacketData = SequenceCameraData | SequenceAccelerometerData | SequenceRotationRateData | SequenceAttitudeData | SequenceAttitudeMatrixData;
+export declare type SequencePacketData = SequenceCameraData | SequenceAccelerometerData | SequenceRotationRateData | SequenceAttitudeData | SequenceAttitudeMatrixData | SequenceSensorData;
 export declare class SequenceDecoder {
     private _data;
     dataByPacket: Map<number, SequencePacketData>;
@@ -81,3 +96,4 @@ export declare class SequenceDecoder {
     numberAttitudeMatrixPackets: number;
     constructor(_data: ArrayBuffer);
 }
+export {};

package/lib/sequencerecorder.js

@@ -6,6 +6,10 @@ export var SequenceRecorderPacketType;
     SequenceRecorderPacketType[SequenceRecorderPacketType["ROTATION_RATE"] = 2] = "ROTATION_RATE";
     SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE"] = 3] = "ATTITUDE";
     SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE_MATRIX"] = 4] = "ATTITUDE_MATRIX";
+    SequenceRecorderPacketType[SequenceRecorderPacketType["ACCELEROMETER_WO_GRAVITY_INT"] = 5] = "ACCELEROMETER_WO_GRAVITY_INT";
+    SequenceRecorderPacketType[SequenceRecorderPacketType["ACCELEROMETER_W_GRAVITY_INT"] = 6] = "ACCELEROMETER_W_GRAVITY_INT";
+    SequenceRecorderPacketType[SequenceRecorderPacketType["ROTATION_RATE_INT"] = 7] = "ROTATION_RATE_INT";
+    SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE_INT"] = 8] = "ATTITUDE_INT";
 })(SequenceRecorderPacketType || (SequenceRecorderPacketType = {}));
 export class SequenceRecorder {
     constructor(estimatedFrames) {
@@ -85,6 +89,38 @@ export class SequenceRecorder {
     appendAttitude(t, x, y, z) {
         this._appendTimestampedVec3(SequenceRecorderPacketType.ATTITUDE, t, x, y, z);
     }
+    appendAccelerometerWithoutGravityInt(t, int, x, y, z) {
+        this._appendSensorWithInterval(SequenceRecorderPacketType.ACCELEROMETER_WO_GRAVITY_INT, t, int, x, y, z);
+    }
+    appendAccelerometerWithGravityInt(t, int, x, y, z) {
+        this._appendSensorWithInterval(SequenceRecorderPacketType.ACCELEROMETER_W_GRAVITY_INT, t, int, x, y, z);
+    }
+    appendRotationRateInt(t, int, x, y, z) {
+        this._appendSensorWithInterval(SequenceRecorderPacketType.ROTATION_RATE_INT, t, int, x, y, z);
+    }
+    appendAttitudeInt(t, int, x, y, z) {
+        this._appendSensorWithInterval(SequenceRecorderPacketType.ATTITUDE_INT, t, int, x, y, z);
+    }
+    _appendSensorWithInterval(type, t, int, x, y, z) {
+        if (!this._started)
+            return;
+        let increment = SequenceRecorder.psbPacketType + SequenceRecorder.sensorWithIntervalPacketSizeBytes;
+        this._growArrayBuffer(increment);
+        this._dataView.setUint32(this._insertionByte, type, true);
+        this._insertionByte += SequenceRecorder.psbPacketType;
+        this._dataView.setUint32(this._insertionByte, t, true);
+        this._insertionByte += 4;
+        this._dataView.setUint32(this._insertionByte, int, true);
+        this._insertionByte += 4;
+        this._dataView.setFloat32(this._insertionByte, x, true);
+        this._insertionByte += 4;
+        this._dataView.setFloat32(this._insertionByte, y, true);
+        this._insertionByte += 4;
+        this._dataView.setFloat32(this._insertionByte, z, true);
+        this._insertionByte += 4;
+        this._numberPackets++;
+        this._dataView.setUint32(4, this._numberPackets, true);
+    }
     _appendTimestampedVec3(type, t, x, y, z) {
         if (!this._started)
             return;
@@ -145,6 +181,8 @@ SequenceRecorder.cameraPacketSizeBytes = SequenceRecorder.psbCameraFrameWidth +
     SequenceRecorder.psbFlags;
 // Accelerometer
 SequenceRecorder.accelerometerPacketSizeBytes = 4 * 4; // t, x, y, z
+// Sensor with interval
+SequenceRecorder.sensorWithIntervalPacketSizeBytes = 4 * 5; // t, interval, x, y, z
 // Rotation rate
 SequenceRecorder.rotationRatePacketSizeBytes = 4 * 4; // t, x, y, z
 // Attitude
@@ -272,6 +310,28 @@ export class SequenceDecoder {
                         this.numberAttitudeMatrixPackets++;
                         break;
                     }
+                case SequenceRecorderPacketType.ACCELEROMETER_WO_GRAVITY_INT:
+                case SequenceRecorderPacketType.ACCELEROMETER_W_GRAVITY_INT:
+                case SequenceRecorderPacketType.ATTITUDE_INT:
+                case SequenceRecorderPacketType.ROTATION_RATE_INT:
+                    {
+                        let t = dataView.getUint32(readPoint, true);
+                        readPoint += 4;
+                        let int = dataView.getUint32(readPoint, true);
+                        readPoint += 4;
+                        let x = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        let y = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        let z = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        this.dataByPacket.set(i, {
+                            type: type,
+                            t, int, x, y, z
+                        });
+                        this.numberAccelerometerPackets++;
+                        break;
+                    }
             }
         }
     }

package/lib/sequencesource.js

@@ -157,6 +157,22 @@ export class SequenceSource extends Source {
                     pipeline.motionAttitudeMatrix(packet.attitude);
                     pendingMotionUpdate = true;
                     break;
+                case SequenceRecorderPacketType.ACCELEROMETER_WO_GRAVITY_INT:
+                    pipeline.motionAccelerometerWithoutGravitySubmitInt(packet.t, packet.int, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case SequenceRecorderPacketType.ACCELEROMETER_W_GRAVITY_INT:
+                    pipeline.motionAccelerometerWithGravitySubmitInt(packet.t, packet.int, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case SequenceRecorderPacketType.ATTITUDE_INT:
+                    pipeline.motionAttitudeSubmitInt(packet.t, packet.int, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case SequenceRecorderPacketType.ROTATION_RATE_INT:
+                    pipeline.motionRotationRateSubmitInt(packet.t, packet.int, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
             }
             this._packetNumber++;
         }

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "2.1.4";
+export declare const VERSION = "3.0.0-alpha.0";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "2.1.4";
+export const VERSION = "3.0.0-alpha.0";

package/lib/worker-imagebitmap.js

@@ -35,7 +35,7 @@ export function handleImageBitmap(m, r, server, mgr) {
     if (!texture)
         return;
     if (!pixels || pixels.byteLength < profile.dataWidth * profile.dataHeight) {
-        console.log("Generating pixel buffer", profile.dataWidth * profile.dataHeight);
+        // console.log("Generating pixel buffer", profile.dataWidth * profile.dataHeight);
         pixels = new ArrayBuffer(profile.dataWidth * profile.dataHeight);
     }
     imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing);

package/lib/worker-server.js

@@ -81,6 +81,44 @@ export function launchWorkerServer(wasmUrl) {
                             handleImageBitmap(msgt, r, server, messageManager);
                             break;
                         }
+                        case "sensorDataC2S": {
+                            const msgt = msg;
+                            const pipeline = server._pipeline_by_instance.get(msgt.p);
+                            if (!pipeline)
+                                break;
+                            switch (msgt.sensor) {
+                                case "accel":
+                                    r.pipeline_motion_accelerometer_submit(pipeline, msgt.timestamp, msgt.x, msgt.y, msgt.z);
+                                    break;
+                                case "accel_w_gravity_int":
+                                    r.pipeline_motion_accelerometer_with_gravity_submit_int(pipeline, msgt.timestamp, msgt.interval, msgt.x, msgt.y, msgt.z);
+                                    break;
+                                case "accel_wo_gravity_int":
+                                    r.pipeline_motion_accelerometer_without_gravity_submit_int(pipeline, msgt.timestamp, msgt.interval, msgt.x, msgt.y, msgt.z);
+                                    break;
+                                case "attitude_int":
+                                    r.pipeline_motion_attitude_submit_int(pipeline, msgt.timestamp, msgt.interval, msgt.x, msgt.y, msgt.z);
+                                    break;
+                                case "attitude":
+                                    r.pipeline_motion_attitude_submit(pipeline, msgt.timestamp, msgt.x, msgt.y, msgt.z);
+                                    break;
+                                case "rotation_rate_int":
+                                    r.pipeline_motion_rotation_rate_submit_int(pipeline, msgt.timestamp, msgt.interval, msgt.x, msgt.y, msgt.z);
+                                    break;
+                                case "rotation_rate":
+                                    r.pipeline_motion_rotation_rate_submit(pipeline, msgt.timestamp, msgt.x, msgt.y, msgt.z);
+                                    break;
+                            }
+                            break;
+                        }
+                        case "attitudeMatrixC2S": {
+                            const msgt = msg;
+                            const pipeline = server._pipeline_by_instance.get(msgt.p);
+                            if (!pipeline)
+                                break;
+                            r.pipeline_motion_attitude_matrix_submit(pipeline, msgt.m);
+                            break;
+                        }
                     }
                 });
             }
@@ -88,14 +126,27 @@ export function launchWorkerServer(wasmUrl) {
     });
 }
 ;
+function consumeStream(mod, r, stream, p, userFacing, server, source) {
+    return __awaiter(this, void 0, void 0, function* () {
+        while (true) {
+            try {
+                const reader = yield stream.getReader();
+                yield consumeReader(mod, r, reader, p, userFacing, server, source);
+            }
+            catch (err) {
+                console.log("Consuming reader error", err);
+            }
+            yield delay(1000);
+        }
+    });
+}
 let streamDataBufferPointer = 0;
 let streamDataBufferLength = 0;
 let tokenId = 1;
 const cameraToDeviceTransform = mat4.create();
 const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
-function consumeStream(mod, r, stream, p, userFacing, server, source) {
+function consumeReader(mod, r, reader, p, userFacing, server, source) {
     return __awaiter(this, void 0, void 0, function* () {
-        const reader = yield stream.getReader();
         while (true) {
             let result = yield reader.read();
             if (result.done) {
@@ -162,7 +213,12 @@ function consumeStream(mod, r, stream, p, userFacing, server, source) {
             messageManager.postOutgoingMessage(ret, [ret.d, ret.uvTransform.buffer]);
             const pipeline = server._pipeline_by_instance.get(p);
             if (pipeline) {
-                r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                try {
+                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                }
+                catch (err) {
+                    console.log('Exception during camera processing', err);
+                }
                 r.pipeline_frame_update(pipeline);
                 server.exploreState();
             }
@@ -170,6 +226,12 @@ function consumeStream(mod, r, stream, p, userFacing, server, source) {
         }
     });
 }
+function delay(ms) {
+    return new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+}
+;
 function framePixelFormatFromFormat(f) {
     switch (f) {
         case "I420": return frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420;