@zappar/zappar-cv 0.4.0-beta.4 → 0.4.0-beta.7

package/lib/mstp-camera-source.js

@@ -0,0 +1,230 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MSTPCameraSource = void 0;
+const profile_1 = require("./profile");
+const pipeline_1 = require("./pipeline");
+const source_1 = require("./source");
+const cameramodel_1 = require("./cameramodel");
+const loglevel_1 = require("./loglevel");
+const camera_source_map_1 = require("./camera-source-map");
+class MSTPCameraSource extends source_1.Source {
+    constructor(_impl, _pipeline, _deviceId) {
+        super();
+        this._impl = _impl;
+        this._pipeline = _pipeline;
+        this._deviceId = _deviceId;
+        this._currentStream = null;
+        this._activeDeviceId = null;
+        this._isPaused = true;
+        this._isUserFacing = false;
+        this._cameraToScreenRotation = 0;
+        this._hasStartedOrientation = false;
+        this._deviceMotionListener = (ev) => {
+            let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            if (ev.accelerationIncludingGravity !== null &&
+                ev.accelerationIncludingGravity.x !== null &&
+                ev.accelerationIncludingGravity.y !== null &&
+                ev.accelerationIncludingGravity.z !== null) {
+                pipeline.motionAccelerometerSubmit(timeStamp, ev.accelerationIncludingGravity.x * profile_1.profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile_1.profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile_1.profile.deviceMotionMutliplier);
+            }
+            if (ev.rotationRate !== null &&
+                ev.rotationRate.alpha !== null &&
+                ev.rotationRate.beta !== null &&
+                ev.rotationRate.gamma !== null && !this._hasStartedOrientation) {
+                ev.timeStamp;
+                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha * Math.PI / -180.0, ev.rotationRate.beta * Math.PI / -180.0, ev.rotationRate.gamma * Math.PI / -180.0);
+            }
+            else if (!this._hasStartedOrientation) {
+                this._startDeviceOrientation();
+            }
+        };
+        loglevel_1.zcout("Using MSTP camera source");
+    }
+    destroy() {
+        this.pause();
+        camera_source_map_1.deleteCameraSource(this._impl);
+    }
+    _stop() {
+        if (!this._currentStream)
+            return;
+        let tracks = this._currentStream.getTracks();
+        tracks.forEach(t => t.stop());
+        this._currentStream = null;
+    }
+    pause() {
+        this._isPaused = true;
+        let p = pipeline_1.Pipeline.get(this._pipeline);
+        if (p && p.currentCameraSource === this)
+            p.currentCameraSource = undefined;
+        this._stopDeviceMotion();
+        this._syncCamera();
+    }
+    start() {
+        var _a;
+        let p = pipeline_1.Pipeline.get(this._pipeline);
+        if (p && p.currentCameraSource !== this) {
+            (_a = p.currentCameraSource) === null || _a === void 0 ? void 0 : _a.pause();
+            p.currentCameraSource = this;
+        }
+        this._isPaused = false;
+        this._startDeviceMotion();
+        this._syncCamera();
+    }
+    _getConstraints() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let deviceId;
+            let facingMode;
+            if (this._deviceId !== MSTPCameraSource.DEFAULT_DEVICE_ID &&
+                this._deviceId !== MSTPCameraSource.USER_DEFAULT_DEVICE_ID) {
+                // Custom device
+                deviceId = this._deviceId;
+            }
+            else {
+                facingMode = (this._deviceId === MSTPCameraSource.DEFAULT_DEVICE_ID) ? "environment" : "user";
+            }
+            let constraints = {
+                audio: false,
+                video: {
+                    facingMode: facingMode,
+                    width: profile_1.profile.videoWidth,
+                    height: profile_1.profile.videoHeight,
+                    frameRate: profile_1.profile.requestHighFrameRate ? 60 : undefined,
+                    deviceId: deviceId
+                }
+            };
+            if (deviceId)
+                return constraints;
+            if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices)
+                return constraints;
+            let devices = yield navigator.mediaDevices.enumerateDevices();
+            let hasHadCapabilities = false;
+            devices = devices.filter(val => {
+                // Remove non-video devices
+                if (val.kind !== "videoinput")
+                    return false;
+                // If the media info object contains capabilities, use it to filter to the correct facing cameras
+                if (val.getCapabilities) {
+                    hasHadCapabilities = true;
+                    let capabilities = val.getCapabilities();
+                    if (capabilities && capabilities.facingMode && capabilities.facingMode.indexOf(facingMode === "user" ? "user" : "environment") < 0)
+                        return false;
+                }
+                return true;
+            });
+            // If none of the devices had capability info, or we have no devices left, fall back to the standard constraints
+            if (!hasHadCapabilities || devices.length === 0) {
+                return constraints;
+            }
+            if (typeof constraints.video === "object") {
+                loglevel_1.zcout("choosing device ID", devices[devices.length - 1].deviceId);
+                constraints.video.deviceId = devices[devices.length - 1].deviceId;
+            }
+            return constraints;
+        });
+    }
+    getFrame(allowRead) {
+        var _a, _b;
+        let rotation = cameramodel_1.cameraRotationForScreenOrientation(false);
+        if (rotation != this._cameraToScreenRotation) {
+            (_b = (_a = pipeline_1.Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendCameraToScreenRotationToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, rotation);
+            this._cameraToScreenRotation = rotation;
+        }
+        return;
+    }
+    _getUserMedia() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let constraints = yield this._getConstraints();
+            if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia)
+                return yield navigator.mediaDevices.getUserMedia(constraints);
+            return yield new Promise((resolve, reject) => {
+                navigator.getUserMedia(constraints, resolve, reject);
+            });
+        });
+    }
+    _syncCamera() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._currentStream && this._isPaused) {
+                this._stop();
+                return;
+            }
+            if (this._currentStream && this._activeDeviceId !== this._deviceId)
+                this._stop();
+            if (!this._isPaused) {
+                this._activeDeviceId = this._deviceId;
+                this._currentStream = yield this._getUserMedia();
+                if (this._isPaused) {
+                    yield this._syncCamera();
+                    return;
+                }
+                this._isUserFacing = false;
+                if (this._currentStream) {
+                    let videoTracks = this._currentStream.getVideoTracks();
+                    if (videoTracks.length > 0) {
+                        this._isUserFacing = videoTracks[0].getSettings().facingMode === "user";
+                        let processor = new MediaStreamTrackProcessor({
+                            track: videoTracks[0]
+                        });
+                        let p = pipeline_1.Pipeline.get(this._pipeline);
+                        if (p)
+                            p.sendCameraStreamToWorker(this._impl, processor.readable, this._isUserFacing);
+                    }
+                }
+            }
+        });
+    }
+    _startDeviceOrientation() {
+        if (this._hasStartedOrientation)
+            return;
+        this._hasStartedOrientation = true;
+        window.addEventListener("deviceorientation", (ev) => {
+            let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            if (ev.alpha === null || ev.beta === null || ev.gamma === null)
+                return;
+            pipeline.motionAttitudeSubmit(timeStamp, ev.alpha, ev.beta, ev.gamma);
+        });
+    }
+    _startDeviceMotion() {
+        window.addEventListener("devicemotion", this._deviceMotionListener, false);
+    }
+    _stopDeviceMotion() {
+        window.removeEventListener("devicemotion", this._deviceMotionListener);
+    }
+    uploadGL(info) {
+        const pipeline = pipeline_1.Pipeline.get(this._pipeline);
+        const gl = pipeline === null || pipeline === void 0 ? void 0 : pipeline.glContext;
+        if (!info ||
+            info.texture ||
+            !info.frame ||
+            !pipeline ||
+            !gl)
+            return;
+        let texture = pipeline.getVideoTexture();
+        if (!texture)
+            return;
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, info.frame);
+        gl.bindTexture(gl.TEXTURE_2D, null);
+        info.texture = texture;
+        info.frame.close();
+        delete info.frame;
+    }
+}
+exports.MSTPCameraSource = MSTPCameraSource;
+MSTPCameraSource.USER_DEFAULT_DEVICE_ID = "Simulated User Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";
+MSTPCameraSource.DEFAULT_DEVICE_ID = "Simulated Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";

package/lib/native.js

@@ -23,14 +23,18 @@ const html_element_source_1 = require("./html-element-source");
 const facelandmark_1 = require("./facelandmark");
 const compatibility_1 = require("./compatibility");
 const loglevel_1 = require("./loglevel");
-const bundleworker_mainthread_1 = require("./bundleworker-mainthread");
 const sequencesource_1 = require("./sequencesource");
+const camera_source_map_1 = require("./camera-source-map");
+const gfx_1 = require("./gfx");
 let client;
 function initialize() {
     if (client)
         return client;
-    worker_client_1.launchWorker();
-    bundleworker_mainthread_1.launchBundleWorker();
+    let loaded = false;
+    worker_client_1.launchWorker().then(() => {
+        loglevel_1.zcout("Fully loaded");
+        loaded = true;
+    });
     let c = new zappar_client_1.zappar_client(ab => {
         worker_client_1.messageManager.postOutgoingMessage({
             t: "zappar",
@@ -43,7 +47,7 @@ function initialize() {
             c.impl.analytics_project_id_set(".wiz" + pathParts[1]);
     }
     worker_client_1.messageManager.onIncomingMessage.bind(msg => {
-        var _a, _b;
+        var _a, _b, _c, _d, _e, _f, _g;
         switch (msg.t) {
             case "zappar":
                 (_a = pipeline_1.Pipeline.get(msg.p)) === null || _a === void 0 ? void 0 : _a.pendingMessages.push(msg.d);
@@ -51,10 +55,21 @@ function initialize() {
             case "buf":
                 c.serializer.bufferReturn(msg.d);
                 break;
-            case "cameraFrameRecycleS2C":
+            case "cameraFrameRecycleS2C": {
+                let msgt = msg;
+                (_c = (_b = pipeline_1.Pipeline.get(msgt.p)) === null || _b === void 0 ? void 0 : _b.cameraTokenReturn) === null || _c === void 0 ? void 0 : _c.call(_b, msg);
+                break;
+            }
+            case "videoFrameS2C": {
+                let msgt = msg;
+                (_e = (_d = pipeline_1.Pipeline.get(msgt.p)) === null || _d === void 0 ? void 0 : _d.videoFrameFromWorker) === null || _e === void 0 ? void 0 : _e.call(_d, msgt);
+                break;
+            }
+            case "imageBitmapS2C": {
                 let msgt = msg;
-                (_b = pipeline_1.Pipeline.get(msgt.p)) === null || _b === void 0 ? void 0 : _b.cameraTokenReturn(msgt);
+                (_g = (_f = pipeline_1.Pipeline.get(msgt.p)) === null || _f === void 0 ? void 0 : _f.imageBitmapFromWorker) === null || _g === void 0 ? void 0 : _g.call(_f, msgt);
                 break;
+            }
             case "licerr": {
                 let div = document.createElement("div");
                 div.innerHTML = "Visit <a href='https://docs.zap.works/universal-ar/licensing/' style='color: white;'>our licensing page</a> to find out about hosting on your own domain.";
@@ -80,9 +95,31 @@ function initialize() {
                 }, 1000);
                 document.body.append(div);
             }
+            case "gfx": {
+                let div = document.createElement("div");
+                div.innerHTML = gfx_1.gfx;
+                div.style.position = "absolute";
+                div.style.bottom = "20px";
+                div.style.width = "250px";
+                div.style.left = "50%";
+                div.style.marginLeft = "-125px";
+                div.style.zIndex = Number.MAX_SAFE_INTEGER.toString();
+                div.style.opacity = "0";
+                div.style.transition = "opacity 0.5s";
+                document.body.append(div);
+                setTimeout(function () {
+                    div.style.opacity = "1";
+                }, 500);
+                setTimeout(function () {
+                    div.style.opacity = "0";
+                }, 3000);
+                setTimeout(function () {
+                    div.remove();
+                }, 4000);
+            }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { camera_default_device_id: userFacing => userFacing ? camera_source_1.CameraSource.USER_DEFAULT_DEVICE_ID : camera_source_1.CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => camera_source_1.CameraSource.create(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = camera_source_1.CameraSource.get(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = camera_source_1.CameraSource.get(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = camera_source_1.CameraSource.get(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, pipeline_create: () => pipeline_1.Pipeline.create(c.impl, worker_client_1.messageManager), pipeline_frame_update: (p) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? camera_source_1.CameraSource.USER_DEFAULT_DEVICE_ID : camera_source_1.CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => camera_source_map_1.createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = camera_source_map_1.getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = camera_source_map_1.getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = camera_source_map_1.getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, pipeline_create: () => pipeline_1.Pipeline.create(c.impl, worker_client_1.messageManager), pipeline_frame_update: (p) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = pipeline_1.Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
@@ -302,7 +339,7 @@ function initialize() {
                 return gl_matrix_1.mat4.create();
             }
             return obj.anchor_pose;
-        }, html_element_source_create: (pipeline, elm) => html_element_source_1.HTMLElementSource.createVideoElementSource(pipeline, elm), html_element_source_start: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.start(); }, html_element_source_pause: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, html_element_source_destroy: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, sequence_source_create: p => sequencesource_1.SequenceSource.create(p), sequence_source_load_from_memory: (o, data) => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.loadFromMemory(data); }, sequence_source_pause: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, sequence_source_start: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.start(); }, sequence_source_destroy: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, permission_granted_all: permission_1.permissionGrantedAll, permission_granted_camera: permission_1.permissionGrantedCamera, permission_granted_motion: permission_1.permissionGrantedMotion, permission_denied_any: permission_1.permissionDeniedAny, permission_denied_camera: permission_1.permissionDeniedCamera, permission_denied_motion: permission_1.permissionDeniedMotion, permission_request_motion: permission_1.permissionRequestMotion, permission_request_camera: permission_1.permissionRequestCamera, permission_request_all: permission_1.permissionRequestAll, permission_request_ui: permission_1.permissionRequestUI, permission_request_ui_promise: permission_1.permissionRequestUI, permission_denied_ui: permission_1.permissionDeniedUI, browser_incompatible: compatibility_1.default.incompatible, browser_incompatible_ui: compatibility_1.default.incompatible_ui, log_level_set: l => {
+        }, html_element_source_create: (pipeline, elm) => html_element_source_1.HTMLElementSource.createVideoElementSource(pipeline, elm), html_element_source_start: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.start(); }, html_element_source_pause: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, html_element_source_destroy: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, sequence_source_create: p => sequencesource_1.SequenceSource.create(p), sequence_source_load_from_memory: (o, data) => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.loadFromMemory(data); }, sequence_source_pause: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, sequence_source_start: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.start(); }, sequence_source_max_playback_fps_set: (o, fps) => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.maxPlaybackFpsSet(fps); }, sequence_source_destroy: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, permission_granted_all: permission_1.permissionGrantedAll, permission_granted_camera: permission_1.permissionGrantedCamera, permission_granted_motion: permission_1.permissionGrantedMotion, permission_denied_any: permission_1.permissionDeniedAny, permission_denied_camera: permission_1.permissionDeniedCamera, permission_denied_motion: permission_1.permissionDeniedMotion, permission_request_motion: permission_1.permissionRequestMotion, permission_request_camera: permission_1.permissionRequestCamera, permission_request_all: permission_1.permissionRequestAll, permission_request_ui: permission_1.permissionRequestUI, permission_request_ui_promise: permission_1.permissionRequestUI, permission_denied_ui: permission_1.permissionDeniedUI, browser_incompatible: compatibility_1.default.incompatible, browser_incompatible_ui: compatibility_1.default.incompatible_ui, log_level_set: l => {
             loglevel_1.setLogLevel(l);
             c.impl.log_level_set(l);
         } });

package/lib/pipeline.d.ts

@@ -1,10 +1,11 @@
-import { zappar_pipeline_t } from "./gen/zappar";
+/// <reference types="dom-webcodecs" />
+import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
 import { zappar_cwrap } from "./gen/zappar-native";
 import { zappar_client } from "./gen/zappar-client";
 import { MsgManager } from "./messages";
 import { mat4 } from "gl-matrix";
 import { Source, CameraFrameInfo } from "./source";
-import { CameraFrameReturnS2C } from "./workerinterface";
+import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
 import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
 export declare class Pipeline {
@@ -24,11 +25,13 @@ export declare class Pipeline {
     private _faceProjectDraw;
     private _sequenceRecorder;
     private _sequenceRecordDeviceAttitudeMatrices;
+    private _sequenceRecorderFirstCameraToken;
     onGLContextReset: Event;
     static create(client: zappar_cwrap, mgr: MsgManager): zappar_pipeline_t;
     static get(p: zappar_pipeline_t): Pipeline | undefined;
     private constructor();
     frameUpdate(client: zappar_client): void;
+    cleanOldFrames(): void;
     cameraTokenReturn(msg: CameraFrameReturnS2C): void;
     sequenceRecordStart(expectedFrames: number): void;
     sequenceRecordStop(): void;
@@ -47,11 +50,18 @@ export declare class Pipeline {
     cameraFrameTextureMatrix(sw: number, sh: number, mirror: boolean): Float32Array;
     cameraFrameUserFacing(): boolean;
     cameraPoseWithAttitude(mirror: boolean): Float32Array;
+    videoFrameFromWorker(msg: VideoFrameS2C): void;
+    imageBitmapFromWorker(msg: ImageBitmapS2C): void;
+    uploadGL(): void;
+    registerToken(info: CameraFrameInfo): number;
     processGL(): void;
     motionAccelerometerSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionRotationRateSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionAttitudeSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionAttitudeMatrix(m: Float32Array): void;
-    uploadGL(): void;
+    sendCameraStreamToWorker(source: zappar_camera_source_t, stream: ReadableStream<VideoFrame>, userFacing: boolean): void;
+    sendCameraToScreenRotationToWorker(rot: number): void;
+    sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array): void;
+    sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
 }
 export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): mat4;

package/lib/pipeline.js

@@ -10,7 +10,9 @@ const drawplane_1 = require("./drawplane");
 const event_1 = require("./event");
 const loglevel_1 = require("./loglevel");
 const sequencerecorder_1 = require("./sequencerecorder");
+const camera_source_map_1 = require("./camera-source-map");
 let byId = new Map();
+let identity = gl_matrix_1.mat4.create();
 class Pipeline {
     constructor(_client, _impl, _mgr) {
         this._client = _client;
@@ -23,6 +25,7 @@ class Pipeline {
         this.videoTextures = [];
         this.cameraPixelArrays = [];
         this._sequenceRecordDeviceAttitudeMatrices = true;
+        this._sequenceRecorderFirstCameraToken = 0;
         this.onGLContextReset = new event_1.Event();
     }
     static create(client, mgr) {
@@ -43,6 +46,10 @@ class Pipeline {
             }, [msg]);
         }
         this.pendingMessages = [];
+        this.cleanOldFrames();
+    }
+    cleanOldFrames() {
+        var _a, _b;
         let currentToken = this._client.pipeline_camera_frame_user_data(this._impl);
         if (!currentToken)
             return;
@@ -50,12 +57,14 @@ class Pipeline {
             if (t[0] < currentToken) {
                 if (t[1].texture)
                     this.videoTextures.push(t[1].texture);
+                (_b = (_a = t[1].frame) === null || _a === void 0 ? void 0 : _a.close) === null || _b === void 0 ? void 0 : _b.call(_a);
                 this.cameraTokens.delete(t[0]);
             }
         }
     }
     cameraTokenReturn(msg) {
-        if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices) {
+        if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices
+            && msg.token >= this._sequenceRecorderFirstCameraToken) {
             let info = this.cameraTokens.get(msg.token);
             if (info) {
                 if (msg.att)
@@ -71,6 +80,7 @@ class Pipeline {
         if (!this._sequenceRecorder)
             this._sequenceRecorder = new sequencerecorder_1.SequenceRecorder(expectedFrames);
         this._sequenceRecorder.start();
+        this._sequenceRecorderFirstCameraToken = this.nextCameraToken;
     }
     sequenceRecordStop() {
         var _a;
@@ -174,7 +184,7 @@ class Pipeline {
         let info = this.getCurrentCameraInfo();
         if (!info)
             return gl_matrix_1.mat4.create();
-        return drawcamera_1.cameraFrameTextureMatrix(info.dataWidth, info.dataHeight, sw, sh, info.uvTransform, mirror);
+        return drawcamera_1.cameraFrameTextureMatrix(info.dataWidth, info.dataHeight, sw, sh, info.uvTransform || identity, mirror);
     }
     cameraFrameUserFacing() {
         var _a;
@@ -191,8 +201,48 @@ class Pipeline {
         gl_matrix_1.mat4.invert(res, res);
         return res;
     }
+    videoFrameFromWorker(msg) {
+        let tokenId = this.nextCameraToken++;
+        const cameraSource = camera_source_map_1.getCameraSource(msg.source);
+        if (!cameraSource)
+            return;
+        this.cameraTokens.set(tokenId, {
+            dataWidth: msg.w,
+            dataHeight: msg.h,
+            texture: undefined,
+            frame: msg.d,
+            userFacing: msg.userFacing,
+            uvTransform: msg.uvTransform,
+            cameraModel: msg.cameraModel,
+            cameraToDevice: msg.cameraToDevice,
+            cameraSource
+        });
+        this.cleanOldFrames();
+    }
+    imageBitmapFromWorker(msg) {
+        let data = this.cameraTokens.get(msg.tokenId);
+        if (!data)
+            return;
+        data.dataWidth = msg.dataWidth;
+        data.dataHeight = msg.dataHeight;
+        data.frame = msg.frame;
+        data.userFacing = msg.userFacing;
+        data.uvTransform = msg.uvTransform;
+        this.tokensInFlight--;
+        this.cleanOldFrames();
+    }
+    uploadGL() {
+        var _a, _b;
+        let info = this.getCurrentCameraInfo();
+        (_b = (_a = info === null || info === void 0 ? void 0 : info.cameraSource) === null || _a === void 0 ? void 0 : _a.uploadGL) === null || _b === void 0 ? void 0 : _b.call(_a, info);
+    }
+    registerToken(info) {
+        let tokenId = this.nextCameraToken++;
+        this.cameraTokens.set(tokenId, info);
+        this.tokensInFlight++;
+        return tokenId;
+    }
     processGL() {
-        var _a;
         if (!this.glContext) {
             loglevel_1.zcerr("no GL context for camera frames - please call pipeline_gl_context_set");
             return;
@@ -203,26 +253,7 @@ class Pipeline {
             this.currentCameraSource.getFrame(true);
             return;
         }
-        let info = this.currentCameraSource.getFrame(false);
-        if (!info)
-            return;
-        let tokenId = this.nextCameraToken++;
-        this.cameraTokens.set(tokenId, info);
-        let msg = {
-            d: info.data,
-            p: this._impl,
-            width: info.dataWidth,
-            height: info.dataHeight,
-            token: tokenId,
-            userFacing: info.userFacing,
-            c2d: info.cameraToDevice,
-            cm: info.cameraModel,
-            t: "cameraFrameC2S"
-        };
-        if (!this._sequenceRecordDeviceAttitudeMatrices)
-            (_a = this._sequenceRecorder) === null || _a === void 0 ? void 0 : _a.appendCameraFrame(info);
-        this.tokensInFlight++;
-        this._mgr.postOutgoingMessage(msg, [info.data]);
+        this.currentCameraSource.getFrame(false);
     }
     motionAccelerometerSubmit(timestamp, x, y, z) {
         var _a;
@@ -246,9 +277,47 @@ class Pipeline {
         // This doesn't need to be added to the sequence since that's done on frame update instead
         this._client.pipeline_motion_attitude_matrix_submit(this._impl, m);
     }
-    uploadGL() {
-        let info = this.getCurrentCameraInfo();
-        info === null || info === void 0 ? void 0 : info.cameraSource.uploadGL(info);
+    sendCameraStreamToWorker(source, stream, userFacing) {
+        let msg = {
+            t: "streamC2S",
+            p: this._impl,
+            s: stream,
+            userFacing,
+            source
+        };
+        this._mgr.postOutgoingMessage(msg, [msg.s]);
+    }
+    sendCameraToScreenRotationToWorker(rot) {
+        let msg = {
+            p: this._impl,
+            t: "cameraToScreenC2S",
+            r: rot
+        };
+        this._mgr.postOutgoingMessage(msg, []);
+    }
+    sendImageBitmapToWorker(img, rot, userFacing, tokenId, cameraModel, cameraToDevice) {
+        let msg = {
+            p: this._impl,
+            t: "imageBitmapC2S",
+            i: img,
+            r: rot,
+            tokenId,
+            userFacing,
+            cameraModel,
+            cameraToDevice
+        };
+        this._mgr.postOutgoingMessage(msg, [img]);
+    }
+    sendDataToWorker(data, token, width, height, userFacing, cameraToDevice, cameraModel) {
+        let msg = {
+            d: data,
+            p: this._impl,
+            width, height, token, userFacing,
+            c2d: cameraToDevice,
+            cm: cameraModel,
+            t: "cameraFrameC2S"
+        };
+        this._mgr.postOutgoingMessage(msg, [data]);
     }
 }
 exports.Pipeline = Pipeline;

package/lib/profile.d.ts

@@ -13,4 +13,6 @@ export declare let profile: {
     dataWidth: number;
     dataHeight: number;
     videoElementInDOM: boolean;
+    preferMediaStreamTrackProcessorCamera: boolean;
+    preferImageBitmapCamera: boolean;
 };