@zappar/zappar-cv 0.3.13 → 0.4.0-beta.10

package/lib/pipeline.js

@@ -9,7 +9,10 @@ const drawfaceproject_1 = require("./drawfaceproject");
 const drawplane_1 = require("./drawplane");
 const event_1 = require("./event");
 const loglevel_1 = require("./loglevel");
+const sequencerecorder_1 = require("./sequencerecorder");
+const camera_source_map_1 = require("./camera-source-map");
 let byId = new Map();
+let identity = gl_matrix_1.mat4.create();
 class Pipeline {
     constructor(_client, _impl, _mgr) {
         this._client = _client;
@@ -21,6 +24,8 @@ class Pipeline {
         this.tokensInFlight = 0;
         this.videoTextures = [];
         this.cameraPixelArrays = [];
+        this._sequenceRecordDeviceAttitudeMatrices = true;
+        this._sequenceRecorderFirstCameraToken = 0;
         this.onGLContextReset = new event_1.Event();
     }
     static create(client, mgr) {
@@ -41,6 +46,10 @@ class Pipeline {
             }, [msg]);
         }
         this.pendingMessages = [];
+        this.cleanOldFrames();
+    }
+    cleanOldFrames() {
+        var _a, _b;
         let currentToken = this._client.pipeline_camera_frame_user_data(this._impl);
         if (!currentToken)
             return;
@@ -48,14 +57,45 @@ class Pipeline {
             if (t[0] < currentToken) {
                 if (t[1].texture)
                     this.videoTextures.push(t[1].texture);
+                (_b = (_a = t[1].frame) === null || _a === void 0 ? void 0 : _a.close) === null || _b === void 0 ? void 0 : _b.call(_a);
                 this.cameraTokens.delete(t[0]);
             }
         }
     }
-    cameraTokenReturn(tokenId, pixelArray) {
-        this.cameraPixelArrays.push(pixelArray);
+    cameraTokenReturn(msg) {
+        if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices
+            && msg.token >= this._sequenceRecorderFirstCameraToken) {
+            let info = this.cameraTokens.get(msg.token);
+            if (info) {
+                if (msg.att)
+                    this._sequenceRecorder.appendAttitudeMatrix(msg.att);
+                info.data = msg.d;
+                this._sequenceRecorder.appendCameraFrame(info);
+            }
+        }
+        this.cameraPixelArrays.push(msg.d);
         this.tokensInFlight--;
     }
+    sequenceRecordStart(expectedFrames) {
+        if (!this._sequenceRecorder)
+            this._sequenceRecorder = new sequencerecorder_1.SequenceRecorder(expectedFrames);
+        this._sequenceRecorder.start();
+        this._sequenceRecorderFirstCameraToken = this.nextCameraToken;
+    }
+    sequenceRecordStop() {
+        var _a;
+        (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.stop();
+    }
+    sequenceRecordData() {
+        var _a;
+        return ((_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.data()) || new Uint8Array(0);
+    }
+    sequenceRecordClear() {
+        delete this._sequenceRecorder;
+    }
+    sequenceRecordDeviceAttitudeMatrices(v) {
+        this._sequenceRecordDeviceAttitudeMatrices = v;
+    }
     getVideoTexture() {
         return this.videoTextures.pop();
     }
@@ -144,7 +184,7 @@ class Pipeline {
         let info = this.getCurrentCameraInfo();
         if (!info)
             return gl_matrix_1.mat4.create();
-        return drawcamera_1.cameraFrameTextureMatrix(info.dataWidth, info.dataHeight, sw, sh, info.uvTransform, mirror);
+        return drawcamera_1.cameraFrameTextureMatrix(info.dataWidth, info.dataHeight, sw, sh, info.uvTransform || identity, mirror);
     }
     cameraFrameUserFacing() {
         var _a;
@@ -161,6 +201,47 @@ class Pipeline {
         gl_matrix_1.mat4.invert(res, res);
         return res;
     }
+    videoFrameFromWorker(msg) {
+        let tokenId = this.nextCameraToken++;
+        const cameraSource = camera_source_map_1.getCameraSource(msg.source);
+        if (!cameraSource)
+            return;
+        this.cameraTokens.set(tokenId, {
+            dataWidth: msg.w,
+            dataHeight: msg.h,
+            texture: undefined,
+            frame: msg.d,
+            userFacing: msg.userFacing,
+            uvTransform: msg.uvTransform,
+            cameraModel: msg.cameraModel,
+            cameraToDevice: msg.cameraToDevice,
+            cameraSource
+        });
+        this.cleanOldFrames();
+    }
+    imageBitmapFromWorker(msg) {
+        let data = this.cameraTokens.get(msg.tokenId);
+        if (!data)
+            return;
+        data.dataWidth = msg.dataWidth;
+        data.dataHeight = msg.dataHeight;
+        data.frame = msg.frame;
+        data.userFacing = msg.userFacing;
+        data.uvTransform = msg.uvTransform;
+        this.tokensInFlight--;
+        this.cleanOldFrames();
+    }
+    uploadGL() {
+        var _a, _b;
+        let info = this.getCurrentCameraInfo();
+        (_b = (_a = info === null || info === void 0 ? void 0 : info.cameraSource) === null || _a === void 0 ? void 0 : _a.uploadGL) === null || _b === void 0 ? void 0 : _b.call(_a, info);
+    }
+    registerToken(info) {
+        let tokenId = this.nextCameraToken++;
+        this.cameraTokens.set(tokenId, info);
+        this.tokensInFlight++;
+        return tokenId;
+    }
     processGL() {
         if (!this.glContext) {
             loglevel_1.zcerr("no GL context for camera frames - please call pipeline_gl_context_set");
@@ -172,32 +253,72 @@ class Pipeline {
             this.currentCameraSource.getFrame(true);
             return;
         }
-        let info = this.currentCameraSource.getFrame(false);
-        if (!info)
-            return;
-        let tokenId = this.nextCameraToken++;
-        this.cameraTokens.set(tokenId, info);
-        let msg = {
-            d: info.data,
-            p: this._impl,
-            width: info.dataWidth,
-            height: info.dataHeight,
-            token: tokenId,
-            userFacing: info.userFacing,
-            t: "cameraFrameC2S"
-        };
-        this.tokensInFlight++;
-        this._mgr.postOutgoingMessage(msg, [info.data]);
+        this.currentCameraSource.getFrame(false);
     }
     motionAccelerometerSubmit(timestamp, x, y, z) {
+        var _a;
+        if (!this._sequenceRecordDeviceAttitudeMatrices)
+            (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendAccelerometer(timestamp, x, y, z);
         this._client.pipeline_motion_accelerometer_submit(this._impl, timestamp, x, y, z);
     }
     motionRotationRateSubmit(timestamp, x, y, z) {
+        var _a;
+        if (!this._sequenceRecordDeviceAttitudeMatrices)
+            (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendRotationRate(timestamp, x, y, z);
         this._client.pipeline_motion_rotation_rate_submit(this._impl, timestamp, x, y, z);
     }
     motionAttitudeSubmit(timestamp, x, y, z) {
+        var _a;
+        if (!this._sequenceRecordDeviceAttitudeMatrices)
+            (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendAttitude(timestamp, x, y, z);
         this._client.pipeline_motion_attitude_submit(this._impl, timestamp, x, y, z);
     }
+    motionAttitudeMatrix(m) {
+        // This doesn't need to be added to the sequence since that's done on frame update instead
+        this._client.pipeline_motion_attitude_matrix_submit(this._impl, m);
+    }
+    sendCameraStreamToWorker(source, stream, userFacing) {
+        let msg = {
+            t: "streamC2S",
+            p: this._impl,
+            s: stream,
+            userFacing,
+            source
+        };
+        this._mgr.postOutgoingMessage(msg, [msg.s]);
+    }
+    sendCameraToScreenRotationToWorker(rot) {
+        let msg = {
+            p: this._impl,
+            t: "cameraToScreenC2S",
+            r: rot
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
+    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice) {
+        let msg = {
+            p: this._impl,
+            t: "imageBitmapC2S",
+            i: img,
+            r: rot,
+            tokenId,
+            userFacing,
+            cameraModel,
+            cameraToDevice
+        };
+        this._mgr.postOutgoingMessage(msg, [img]);
+    }
+    sendDataToWorker(data, token, width, height, userFacing, cameraToDevice, cameraModel) {
+        let msg = {
+            d: data,
+            p: this._impl,
+            width, height, token, userFacing,
+            c2d: cameraToDevice,
+            cm: cameraModel,
+            t: "cameraFrameC2S"
+        };
+        this._mgr.postOutgoingMessage(msg, [data]);
+    }
 }
 exports.Pipeline = Pipeline;
 function applyScreenCounterRotation(info, inp) {

package/lib/profile.d.ts

@@ -13,4 +13,6 @@ export declare let profile: {
     dataWidth: number;
     dataHeight: number;
     videoElementInDOM: boolean;
+    preferMediaStreamTrackProcessorCamera: boolean;
+    preferImageBitmapCamera: boolean;
 };

package/lib/profile.js

@@ -17,15 +17,27 @@ exports.profile = {
     videoHeight: 480,
     dataWidth: 320,
     dataHeight: 240,
-    videoElementInDOM: false
+    videoElementInDOM: false,
+    preferMediaStreamTrackProcessorCamera: false,
+    preferImageBitmapCamera: false
 };
-window["zeeProfile"] = exports.profile;
+if (typeof window !== "undefined") {
+    window["zeeProfile"] = exports.profile;
+    if (window.location.href.indexOf("_mstppipeline") >= 0) {
+        console.log("Configuring for MSTP camera pipeline (if supported)");
+        exports.profile.preferMediaStreamTrackProcessorCamera = true;
+    }
+    if (window.location.href.indexOf("_imagebitmappipeline") >= 0) {
+        console.log("Configuring for ImageBitmap camera pipeline (if supported)");
+        exports.profile.preferImageBitmapCamera = true;
+    }
+}
 let agent = new UAParser.UAParser();
 let os = (agent.getOS().name || "unknown").toLowerCase();
 let engine = (agent.getEngine().name || "unknown").toLowerCase();
 if (engine === "webkit" && os !== "ios") {
     exports.profile.deviceMotionMutliplier = 1.0;
-    if (window.orientation !== undefined) {
+    if (typeof window !== "undefined" && window.orientation !== undefined) {
         // iPad
         iDevice();
     }
@@ -59,9 +71,12 @@ function iDevice() {
                 navigator.mediaDevices.getSupportedConstraints &&
                 navigator.mediaDevices.getSupportedConstraints().frameRate) {
                 exports.profile.requestHighFrameRate = true;
-                // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
-                exports.profile.videoHeight = 360;
-                exports.profile.dataHeight = 180;
+                if (majorVersion < 14) {
+                    // Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
+                    // Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
+                    exports.profile.videoHeight = 360;
+                    exports.profile.dataHeight = 180;
+                }
             }
         }
     }

package/lib/riff-reader.d.ts

@@ -0,0 +1,18 @@
+export interface Chunk {
+    ident: string;
+    data: Uint8Array;
+    subident?: string;
+    subchunks?: Chunk[];
+}
+export declare class RiffReader {
+    private _data;
+    private _paddingBytes;
+    truncatedChunks: boolean;
+    private _view;
+    root: Chunk;
+    constructor(_data: ArrayBuffer, _paddingBytes?: boolean);
+    private _parse;
+    private _parseChunk;
+    find(str: string, inChunk?: Chunk): Chunk | undefined;
+    has(str: string): boolean;
+}

package/lib/riff-reader.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RiffReader = void 0;
+const decoder = new TextDecoder();
+class RiffReader {
+    constructor(_data, _paddingBytes = false) {
+        this._data = _data;
+        this._paddingBytes = _paddingBytes;
+        this.truncatedChunks = false;
+        this._view = new DataView(this._data);
+        this.root = this._parse();
+    }
+    _parse() {
+        const header = decoder.decode(this._data.slice(0, 4));
+        if (header !== "RIFF")
+            throw new Error("Not a valid ZPT file");
+        return this._parseChunk(0);
+    }
+    _parseChunk(offset) {
+        const ident = decoder.decode(this._data.slice(offset, offset + 4));
+        const length = this._view.getUint32(offset + 4, true);
+        const data = new Uint8Array(this._data, offset + 8, length);
+        if (data.byteLength !== length)
+            this.truncatedChunks = true;
+        let subchunks;
+        let subident;
+        if (ident === "RIFF" || ident === "LIST") {
+            subident = decoder.decode(this._data.slice(offset + 8, offset + 12));
+            subchunks = [];
+            let currentOffset = 4;
+            while (currentOffset < data.byteLength - 1) {
+                const subchunk = this._parseChunk(offset + 8 + currentOffset);
+                subchunks.push(subchunk);
+                currentOffset += subchunk.data.byteLength + 8;
+                if (this._paddingBytes && subchunk.data.byteLength & 0x1)
+                    currentOffset++;
+            }
+        }
+        return { ident, data, subident, subchunks };
+    }
+    find(str, inChunk = this.root) {
+        if (inChunk.ident === str)
+            return inChunk;
+        if (!inChunk.subchunks)
+            return;
+        for (let c of inChunk.subchunks) {
+            const found = this.find(str, c);
+            if (found)
+                return found;
+        }
+    }
+    has(str) {
+        return this.find(str) !== undefined;
+    }
+}
+exports.RiffReader = RiffReader;

package/lib/sequencerecorder.d.ts

@@ -0,0 +1,83 @@
+import { CameraFrameInfo } from "./source";
+export declare enum SequenceRecorderPacketType {
+    CAMERA_FRAME = 0,
+    ACCELEROMETER = 1,
+    ROTATION_RATE = 2,
+    ATTITUDE = 3,
+    ATTITUDE_MATRIX = 4
+}
+export declare class SequenceRecorder {
+    static headerSizeBytes: number;
+    static psbPacketType: number;
+    static psbCameraFrameWidth: number;
+    static psbCameraFrameHeight: number;
+    static psbFlags: number;
+    static psbCameraToDeviceTransform: number;
+    static psbCameraModel: number;
+    static cameraPacketSizeBytes: number;
+    static accelerometerPacketSizeBytes: number;
+    static rotationRatePacketSizeBytes: number;
+    static attitudePacketSizeBytes: number;
+    static attitudeMatrixPacketSizeBytes: number;
+    private _data;
+    private _dataView;
+    private _dataUint8;
+    private _dataFloat32;
+    private _insertionByte;
+    private _numberPackets;
+    private _started;
+    private _hasLoggedUnsuppored;
+    constructor(estimatedFrames: number);
+    private _estimatedSize;
+    start(): void;
+    stop(): void;
+    data(): Uint8Array;
+    appendCameraFrame(c: CameraFrameInfo): void;
+    appendAccelerometer(t: number, x: number, y: number, z: number): void;
+    appendRotationRate(t: number, x: number, y: number, z: number): void;
+    appendAttitude(t: number, x: number, y: number, z: number): void;
+    private _appendTimestampedVec3;
+    appendAttitudeMatrix(m: Float32Array): void;
+    private _growArrayBuffer;
+}
+export interface SequenceCameraData {
+    type: SequenceRecorderPacketType.CAMERA_FRAME;
+    width: number;
+    height: number;
+    greyscale: Uint8Array;
+    userFacing: boolean;
+    cameraToDevice: Float32Array;
+    cameraModel: Float32Array;
+}
+export interface SequenceVec3Data {
+    t: number;
+    x: number;
+    y: number;
+    z: number;
+}
+export interface SequenceAccelerometerData extends SequenceVec3Data {
+    type: SequenceRecorderPacketType.ACCELEROMETER;
+}
+export interface SequenceRotationRateData extends SequenceVec3Data {
+    type: SequenceRecorderPacketType.ROTATION_RATE;
+}
+export interface SequenceAttitudeData extends SequenceVec3Data {
+    type: SequenceRecorderPacketType.ATTITUDE;
+}
+export interface SequenceAttitudeMatrixData {
+    type: SequenceRecorderPacketType.ATTITUDE_MATRIX;
+    attitude: Float32Array;
+}
+export declare type SequencePacketData = SequenceCameraData | SequenceAccelerometerData | SequenceRotationRateData | SequenceAttitudeData | SequenceAttitudeMatrixData;
+export declare class SequenceDecoder {
+    private _data;
+    dataByPacket: Map<number, SequencePacketData>;
+    dataByCameraFrame: Map<number, SequenceCameraData>;
+    numberPackets: number;
+    numberCameraFrames: number;
+    numberAccelerometerPackets: number;
+    numberRotationRatePackets: number;
+    numberAttitudePackets: number;
+    numberAttitudeMatrixPackets: number;
+    constructor(_data: ArrayBuffer);
+}