@zappar/zappar-cv 0.3.13 → 0.4.0-beta.10

package/lib/imagetracker.js

@@ -0,0 +1,119 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ImageTracker = void 0;
+const riff_reader_1 = require("./riff-reader");
+let byId = new Map();
+const decoder = new TextDecoder();
+class ImageTracker {
+    constructor(_client, _impl) {
+        this._client = _client;
+        this._impl = _impl;
+        this._targets = [];
+    }
+    static create(pipeline, client) {
+        let ret = client.image_tracker_create(pipeline);
+        byId.set(ret, new ImageTracker(client, ret));
+        return ret;
+    }
+    static get(p) {
+        return byId.get(p);
+    }
+    destroy() {
+        this._client.image_tracker_destroy(this._impl);
+        byId.delete(this._impl);
+    }
+    loadFromMemory(data) {
+        this._targets.push({ data });
+        this._client.image_tracker_target_load_from_memory(this._impl, data);
+    }
+    targetCount() {
+        return this._targets.length;
+    }
+    getTargetInfo(i) {
+        let current = this._targets[i];
+        if (current && current.info)
+            return current.info;
+        current.info = {
+            topRadius: -1,
+            bottomRadius: -1,
+            sideLength: -1,
+            physicalScaleFactor: -1
+        };
+        try {
+            const reader = new riff_reader_1.RiffReader(current.data, false);
+            const imgChunk = reader.find("IMG ");
+            if (imgChunk) {
+                let mimeType = "image/png";
+                const mimeTypeChunk = reader.find("IMGM");
+                if (mimeTypeChunk)
+                    mimeType = decoder.decode(mimeTypeChunk.data);
+                current.info.preview = { mimeType, compressed: imgChunk.data };
+            }
+            const odleChunk = reader.find("ODLE");
+            if (odleChunk) {
+                const odle = decoder.decode(odleChunk.data);
+                this._parseOdle(odle, current.info);
+            }
+        }
+        catch (err) { }
+        return current.info;
+    }
+    _parseOdle(data, info) {
+        let currentOffset = 0;
+        let version = "0";
+        [version, currentOffset] = readLine(currentOffset, data);
+        if (version === "3") {
+            let treeOrFlat = "0";
+            [treeOrFlat, currentOffset] = readLine(currentOffset, data);
+            if (treeOrFlat !== "0" && treeOrFlat !== "1")
+                return;
+            let numberTargets = "0";
+            [numberTargets, currentOffset] = readLine(currentOffset, data);
+            const parsedTargets = parseInt(numberTargets);
+            if (isNaN(parsedTargets) || parsedTargets < 1)
+                return;
+            let emptyLine = "";
+            [emptyLine, currentOffset] = readLine(currentOffset, data);
+            if (emptyLine.length !== 0)
+                return;
+            let infoLine = "";
+            [infoLine, currentOffset] = readLine(currentOffset, data);
+            const infoLineParts = infoLine.split(" ");
+            if (infoLineParts.length < 7)
+                return;
+            info.physicalScaleFactor = parseFloat(infoLineParts[6]);
+            if (isNaN(info.physicalScaleFactor))
+                info.physicalScaleFactor = -1;
+            if (infoLineParts.length >= 8) {
+                info.topRadius = parseFloat(infoLineParts[7]);
+                if (isNaN(info.topRadius))
+                    info.topRadius = -1;
+                info.bottomRadius = info.topRadius;
+            }
+            if (infoLineParts.length >= 9) {
+                info.bottomRadius = parseFloat(infoLineParts[8]);
+                if (isNaN(info.bottomRadius))
+                    info.bottomRadius = -1;
+            }
+            if (infoLineParts.length >= 10) {
+                info.sideLength = parseFloat(infoLineParts[9]);
+                if (isNaN(info.sideLength))
+                    info.sideLength = -1;
+            }
+        }
+    }
+    getDecodedPreview(i) {
+        const info = this.getTargetInfo(i);
+        if (!info.preview)
+            return new Image();
+        const blob = new Blob([info.preview.compressed], { type: info.preview.mimeType });
+        const image = new Image();
+        image.src = URL.createObjectURL(blob);
+        return image;
+    }
+}
+exports.ImageTracker = ImageTracker;
+function readLine(offset, str) {
+    let indx = str.indexOf("\n", offset);
+    return [str.substring(offset, indx >= 0 ? indx : undefined).replace("\r", ""), indx + 1];
+}

package/lib/index.d.ts

@@ -2,4 +2,4 @@ import { zappar } from "./gen/zappar";
 import { Additional } from "./additional";
 export declare type Zappar = zappar & Additional;
 export declare function initialize(): Zappar;
-export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
+export { zappar_image_tracker_t, zappar_instant_world_tracker_t, zappar_barcode_finder_t, zappar_face_tracker_t, zappar_face_landmark_t, barcode_format_t, face_landmark_name_t, instant_world_tracker_transform_orientation_t, log_level_t, zappar_face_mesh_t, zappar_pipeline_t, zappar_camera_source_t, zappar_sequence_source_t } from "./gen/zappar";

package/lib/mstp-camera-source.d.ts

@@ -0,0 +1,29 @@
+import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar";
+import { CameraFrameInfo, Source } from "./source";
+export declare class MSTPCameraSource extends Source {
+    private _impl;
+    private _pipeline;
+    private _deviceId;
+    static USER_DEFAULT_DEVICE_ID: string;
+    static DEFAULT_DEVICE_ID: string;
+    private _currentStream;
+    private _activeDeviceId;
+    private _isPaused;
+    private _isUserFacing;
+    private _cameraToScreenRotation;
+    constructor(_impl: zappar_camera_source_t, _pipeline: zappar_pipeline_t, _deviceId: string);
+    destroy(): void;
+    private _stop;
+    pause(): void;
+    start(): void;
+    private _getConstraints;
+    getFrame(allowRead: boolean): void;
+    private _getUserMedia;
+    private _syncCamera;
+    private _hasStartedOrientation;
+    private _deviceMotionListener;
+    private _startDeviceOrientation;
+    private _startDeviceMotion;
+    private _stopDeviceMotion;
+    uploadGL(info: CameraFrameInfo): void;
+}

package/lib/mstp-camera-source.js

@@ -0,0 +1,230 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MSTPCameraSource = void 0;
+const profile_1 = require("./profile");
+const pipeline_1 = require("./pipeline");
+const source_1 = require("./source");
+const cameramodel_1 = require("./cameramodel");
+const loglevel_1 = require("./loglevel");
+const camera_source_map_1 = require("./camera-source-map");
+class MSTPCameraSource extends source_1.Source {
+    constructor(_impl, _pipeline, _deviceId) {
+        super();
+        this._impl = _impl;
+        this._pipeline = _pipeline;
+        this._deviceId = _deviceId;
+        this._currentStream = null;
+        this._activeDeviceId = null;
+        this._isPaused = true;
+        this._isUserFacing = false;
+        this._cameraToScreenRotation = 0;
+        this._hasStartedOrientation = false;
+        this._deviceMotionListener = (ev) => {
+            let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            if (ev.accelerationIncludingGravity !== null &&
+                ev.accelerationIncludingGravity.x !== null &&
+                ev.accelerationIncludingGravity.y !== null &&
+                ev.accelerationIncludingGravity.z !== null) {
+                pipeline.motionAccelerometerSubmit(timeStamp, ev.accelerationIncludingGravity.x * profile_1.profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile_1.profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile_1.profile.deviceMotionMutliplier);
+            }
+            if (ev.rotationRate !== null &&
+                ev.rotationRate.alpha !== null &&
+                ev.rotationRate.beta !== null &&
+                ev.rotationRate.gamma !== null && !this._hasStartedOrientation) {
+                ev.timeStamp;
+                pipeline.motionRotationRateSubmit(timeStamp, ev.rotationRate.alpha * Math.PI / -180.0, ev.rotationRate.beta * Math.PI / -180.0, ev.rotationRate.gamma * Math.PI / -180.0);
+            }
+            else if (!this._hasStartedOrientation) {
+                this._startDeviceOrientation();
+            }
+        };
+        loglevel_1.zcout("Using MSTP camera source");
+    }
+    destroy() {
+        this.pause();
+        camera_source_map_1.deleteCameraSource(this._impl);
+    }
+    _stop() {
+        if (!this._currentStream)
+            return;
+        let tracks = this._currentStream.getTracks();
+        tracks.forEach(t => t.stop());
+        this._currentStream = null;
+    }
+    pause() {
+        this._isPaused = true;
+        let p = pipeline_1.Pipeline.get(this._pipeline);
+        if (p && p.currentCameraSource === this)
+            p.currentCameraSource = undefined;
+        this._stopDeviceMotion();
+        this._syncCamera();
+    }
+    start() {
+        var _a;
+        let p = pipeline_1.Pipeline.get(this._pipeline);
+        if (p && p.currentCameraSource !== this) {
+            (_a = p.currentCameraSource) === null || _a === void 0 ? void 0 : _a.pause();
+            p.currentCameraSource = this;
+        }
+        this._isPaused = false;
+        this._startDeviceMotion();
+        this._syncCamera();
+    }
+    _getConstraints() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let deviceId;
+            let facingMode;
+            if (this._deviceId !== MSTPCameraSource.DEFAULT_DEVICE_ID &&
+                this._deviceId !== MSTPCameraSource.USER_DEFAULT_DEVICE_ID) {
+                // Custom device
+                deviceId = this._deviceId;
+            }
+            else {
+                facingMode = (this._deviceId === MSTPCameraSource.DEFAULT_DEVICE_ID) ? "environment" : "user";
+            }
+            let constraints = {
+                audio: false,
+                video: {
+                    facingMode: facingMode,
+                    width: profile_1.profile.videoWidth,
+                    height: profile_1.profile.videoHeight,
+                    frameRate: profile_1.profile.requestHighFrameRate ? 60 : 30,
+                    deviceId: deviceId
+                }
+            };
+            if (deviceId)
+                return constraints;
+            if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices)
+                return constraints;
+            let devices = yield navigator.mediaDevices.enumerateDevices();
+            let hasHadCapabilities = false;
+            devices = devices.filter(val => {
+                // Remove non-video devices
+                if (val.kind !== "videoinput")
+                    return false;
+                // If the media info object contains capabilities, use it to filter to the correct facing cameras
+                if (val.getCapabilities) {
+                    hasHadCapabilities = true;
+                    let capabilities = val.getCapabilities();
+                    if (capabilities && capabilities.facingMode && capabilities.facingMode.indexOf(facingMode === "user" ? "user" : "environment") < 0)
+                        return false;
+                }
+                return true;
+            });
+            // If none of the devices had capability info, or we have no devices left, fall back to the standard constraints
+            if (!hasHadCapabilities || devices.length === 0) {
+                return constraints;
+            }
+            if (typeof constraints.video === "object") {
+                loglevel_1.zcout("choosing device ID", devices[devices.length - 1].deviceId);
+                constraints.video.deviceId = devices[devices.length - 1].deviceId;
+            }
+            return constraints;
+        });
+    }
+    getFrame(allowRead) {
+        var _a, _b;
+        let rotation = cameramodel_1.cameraRotationForScreenOrientation(false);
+        if (rotation != this._cameraToScreenRotation) {
+            (_b = (_a = pipeline_1.Pipeline.get(this._pipeline)) === null || _a === void 0 ? void 0 : _a.sendCameraToScreenRotationToWorker) === null || _b === void 0 ? void 0 : _b.call(_a, rotation);
+            this._cameraToScreenRotation = rotation;
+        }
+        return;
+    }
+    _getUserMedia() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let constraints = yield this._getConstraints();
+            if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia)
+                return yield navigator.mediaDevices.getUserMedia(constraints);
+            return yield new Promise((resolve, reject) => {
+                navigator.getUserMedia(constraints, resolve, reject);
+            });
+        });
+    }
+    _syncCamera() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._currentStream && this._isPaused) {
+                this._stop();
+                return;
+            }
+            if (this._currentStream && this._activeDeviceId !== this._deviceId)
+                this._stop();
+            if (!this._isPaused) {
+                this._activeDeviceId = this._deviceId;
+                this._currentStream = yield this._getUserMedia();
+                if (this._isPaused) {
+                    yield this._syncCamera();
+                    return;
+                }
+                this._isUserFacing = false;
+                if (this._currentStream) {
+                    let videoTracks = this._currentStream.getVideoTracks();
+                    if (videoTracks.length > 0) {
+                        this._isUserFacing = videoTracks[0].getSettings().facingMode === "user";
+                        let processor = new MediaStreamTrackProcessor({
+                            track: videoTracks[0]
+                        });
+                        let p = pipeline_1.Pipeline.get(this._pipeline);
+                        if (p)
+                            p.sendCameraStreamToWorker(this._impl, processor.readable, this._isUserFacing);
+                    }
+                }
+            }
+        });
+    }
+    _startDeviceOrientation() {
+        if (this._hasStartedOrientation)
+            return;
+        this._hasStartedOrientation = true;
+        window.addEventListener("deviceorientation", (ev) => {
+            let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+            if (!pipeline)
+                return;
+            let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
+            if (ev.alpha === null || ev.beta === null || ev.gamma === null)
+                return;
+            pipeline.motionAttitudeSubmit(timeStamp, ev.alpha, ev.beta, ev.gamma);
+        });
+    }
+    _startDeviceMotion() {
+        window.addEventListener("devicemotion", this._deviceMotionListener, false);
+    }
+    _stopDeviceMotion() {
+        window.removeEventListener("devicemotion", this._deviceMotionListener);
+    }
+    uploadGL(info) {
+        const pipeline = pipeline_1.Pipeline.get(this._pipeline);
+        const gl = pipeline === null || pipeline === void 0 ? void 0 : pipeline.glContext;
+        if (!info ||
+            info.texture ||
+            !info.frame ||
+            !pipeline ||
+            !gl)
+            return;
+        let texture = pipeline.getVideoTexture();
+        if (!texture)
+            return;
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, info.frame);
+        gl.bindTexture(gl.TEXTURE_2D, null);
+        info.texture = texture;
+        info.frame.close();
+        delete info.frame;
+    }
+}
+exports.MSTPCameraSource = MSTPCameraSource;
+MSTPCameraSource.USER_DEFAULT_DEVICE_ID = "Simulated User Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";
+MSTPCameraSource.DEFAULT_DEVICE_ID = "Simulated Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";

package/lib/native.js

@@ -23,6 +23,10 @@ const html_element_source_1 = require("./html-element-source");
 const facelandmark_1 = require("./facelandmark");
 const compatibility_1 = require("./compatibility");
 const loglevel_1 = require("./loglevel");
+const sequencesource_1 = require("./sequencesource");
+const camera_source_map_1 = require("./camera-source-map");
+const gfx_1 = require("./gfx");
+const imagetracker_1 = require("./imagetracker");
 let client;
 function initialize() {
     if (client)
@@ -44,7 +48,7 @@ function initialize() {
             c.impl.analytics_project_id_set(".wiz" + pathParts[1]);
     }
     worker_client_1.messageManager.onIncomingMessage.bind(msg => {
-        var _a, _b;
+        var _a, _b, _c, _d, _e, _f, _g;
         switch (msg.t) {
             case "zappar":
                 (_a = pipeline_1.Pipeline.get(msg.p)) === null || _a === void 0 ? void 0 : _a.pendingMessages.push(msg.d);
@@ -52,10 +56,21 @@ function initialize() {
             case "buf":
                 c.serializer.bufferReturn(msg.d);
                 break;
-            case "cameraFrameRecycleS2C":
+            case "cameraFrameRecycleS2C": {
                 let msgt = msg;
-                (_b = pipeline_1.Pipeline.get(msgt.p)) === null || _b === void 0 ? void 0 : _b.cameraTokenReturn(msgt.token, msgt.d);
+                (_c = (_b = pipeline_1.Pipeline.get(msgt.p)) === null || _b === void 0 ? void 0 : _b.cameraTokenReturn) === null || _c === void 0 ? void 0 : _c.call(_b, msg);
                 break;
+            }
+            case "videoFrameS2C": {
+                let msgt = msg;
+                (_e = (_d = pipeline_1.Pipeline.get(msgt.p)) === null || _d === void 0 ? void 0 : _d.videoFrameFromWorker) === null || _e === void 0 ? void 0 : _e.call(_d, msgt);
+                break;
+            }
+            case "imageBitmapS2C": {
+                let msgt = msg;
+                (_g = (_f = pipeline_1.Pipeline.get(msgt.p)) === null || _f === void 0 ? void 0 : _f.imageBitmapFromWorker) === null || _g === void 0 ? void 0 : _g.call(_f, msgt);
+                break;
+            }
             case "licerr": {
                 let div = document.createElement("div");
                 div.innerHTML = "Visit <a href='https://docs.zap.works/universal-ar/licensing/' style='color: white;'>our licensing page</a> to find out about hosting on your own domain.";
@@ -81,9 +96,31 @@ function initialize() {
                 }, 1000);
                 document.body.append(div);
             }
+            case "gfx": {
+                let div = document.createElement("div");
+                div.innerHTML = gfx_1.gfx;
+                div.style.position = "absolute";
+                div.style.bottom = "20px";
+                div.style.width = "250px";
+                div.style.left = "50%";
+                div.style.marginLeft = "-125px";
+                div.style.zIndex = Number.MAX_SAFE_INTEGER.toString();
+                div.style.opacity = "0";
+                div.style.transition = "opacity 0.5s";
+                document.body.append(div);
+                setTimeout(function () {
+                    div.style.opacity = "1";
+                }, 500);
+                setTimeout(function () {
+                    div.style.opacity = "0";
+                }, 3000);
+                setTimeout(function () {
+                    div.remove();
+                }, 4000);
+            }
         }
     });
-    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? camera_source_1.CameraSource.USER_DEFAULT_DEVICE_ID : camera_source_1.CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => camera_source_1.CameraSource.create(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = camera_source_1.CameraSource.get(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = camera_source_1.CameraSource.get(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = camera_source_1.CameraSource.get(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, pipeline_create: () => pipeline_1.Pipeline.create(c.impl, worker_client_1.messageManager), pipeline_frame_update: (p) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
+    client = Object.assign(Object.assign({}, c.impl), { loaded: () => loaded, camera_default_device_id: userFacing => userFacing ? camera_source_1.CameraSource.USER_DEFAULT_DEVICE_ID : camera_source_1.CameraSource.DEFAULT_DEVICE_ID, camera_source_create: (p, deviceId) => camera_source_map_1.createCameraSource(p, deviceId), camera_source_destroy: cam => { var _a; return (_a = camera_source_map_1.getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.destroy(); }, camera_source_pause: cam => { var _a; return (_a = camera_source_map_1.getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.pause(); }, camera_source_start: cam => { var _a; return (_a = camera_source_map_1.getCameraSource(cam)) === null || _a === void 0 ? void 0 : _a.start(); }, pipeline_create: () => pipeline_1.Pipeline.create(c.impl, worker_client_1.messageManager), pipeline_frame_update: (p) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.frameUpdate(c); }, pipeline_camera_frame_draw_gl: (pipeline, screenWidth, screenHeight, mirror) => {
             var _a;
             (_a = pipeline_1.Pipeline.get(pipeline)) === null || _a === void 0 ? void 0 : _a.cameraFrameDrawGL(screenWidth, screenHeight, mirror);
         }, draw_plane: (gl, projectionMatrix, cameraMatrix, targetMatrix, texture) => {
@@ -99,7 +136,7 @@ function initialize() {
         }, pipeline_draw_face_project: (p, matrix, vertices, uvMatrix, uvs, indices, texture) => {
             var _a;
             (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawFaceProject(matrix, vertices, uvMatrix, uvs, indices, texture);
-        }, projection_matrix_from_camera_model: cameramodel_1.projectionMatrix, projection_matrix_from_camera_model_ext: cameramodel_1.projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: () => { }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || gl_matrix_1.mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => gl_matrix_1.mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || gl_matrix_1.mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = gl_matrix_1.mat4.create(); gl_matrix_1.mat4.invert(res, o); return res; }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
+        }, projection_matrix_from_camera_model: cameramodel_1.projectionMatrix, projection_matrix_from_camera_model_ext: cameramodel_1.projectionMatrix, pipeline_process_gl: p => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || gl_matrix_1.mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => gl_matrix_1.mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || gl_matrix_1.mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = gl_matrix_1.mat4.create(); gl_matrix_1.mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = pipeline_1.Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
             let res = pipeline_1.applyScreenCounterRotation(undefined, c.impl.instant_world_tracker_anchor_pose_raw(o));
             if (mirror) {
                 let scale = gl_matrix_1.mat4.create();
@@ -118,6 +155,92 @@ function initialize() {
             }
             gl_matrix_1.mat4.multiply(res, cameraPose, res);
             return res;
+        }, instant_world_tracker_anchor_pose_set_from_camera_offset: (o, x, y, z, orientation) => {
+            // TODO - add an _ext function if we need to support mirrored cameras with an extra param
+            // if(mirror) {
+            //     x *= -1;
+            // }
+            // TODO - can we access the appropriate pipeline here to call pipeline_camera_frame_user_facing(p)?
+            let userFacing = false;
+            let rotation = cameramodel_1.cameraRotationForScreenOrientation(userFacing) * Math.PI / 180.0;
+            let rotationMat = gl_matrix_1.mat4.create();
+            gl_matrix_1.mat4.fromRotation(rotationMat, -rotation, [0, 0, 1]);
+            let rawVec = gl_matrix_1.vec3.create();
+            gl_matrix_1.vec3.transformMat4(rawVec, [x, y, z], rotationMat);
+            c.impl.instant_world_tracker_anchor_pose_set_from_camera_offset_raw(o, rawVec[0], rawVec[1], rawVec[2], orientation);
+        }, image_tracker_create: pipeline => imagetracker_1.ImageTracker.create(pipeline, c.impl), image_tracker_destroy: t => { var _a; return (_a = imagetracker_1.ImageTracker.get(t)) === null || _a === void 0 ? void 0 : _a.destroy(); }, image_tracker_target_count: t => {
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_count on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.targetCount();
+        }, image_tracker_target_load_from_memory: (t, data) => {
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_load_from_memory on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            obj.loadFromMemory(data);
+        }, image_tracker_target_preview_compressed: (t, i) => {
+            var _a;
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_preview_compressed on a destroyed zappar_image_tracker_t");
+                return new Uint8Array(0);
+            }
+            return ((_a = obj.getTargetInfo(i).preview) === null || _a === void 0 ? void 0 : _a.compressed) || new Uint8Array(0);
+        }, image_tracker_target_preview_compressed_mimetype: (t, i) => {
+            var _a;
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_preview_compressed_mimetype on a destroyed zappar_image_tracker_t");
+                return "";
+            }
+            return ((_a = obj.getTargetInfo(i).preview) === null || _a === void 0 ? void 0 : _a.mimeType) || "";
+        }, image_tracker_target_preview_compressed_size: (t, i) => {
+            var _a, _b;
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_preview_compressed_size on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return ((_b = (_a = obj.getTargetInfo(i).preview) === null || _a === void 0 ? void 0 : _a.compressed) === null || _b === void 0 ? void 0 : _b.byteLength) || 0;
+        }, image_tracker_target_physical_scale_factor: (t, i) => {
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_physical_scale_factor on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getTargetInfo(i).physicalScaleFactor;
+        }, image_tracker_target_radius_top: (t, i) => {
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_radius_top on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getTargetInfo(i).topRadius;
+        }, image_tracker_target_radius_bottom: (t, i) => {
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_radius_bottom on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getTargetInfo(i).bottomRadius;
+        }, image_tracker_target_side_length: (t, i) => {
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_side_length on a destroyed zappar_image_tracker_t");
+                return 0;
+            }
+            return obj.getTargetInfo(i).sideLength;
+        }, image_tracker_target_image: (t, i) => {
+            let obj = imagetracker_1.ImageTracker.get(t);
+            if (!obj) {
+                loglevel_1.zcwarn("attempting to call image_tracker_target_image on a destroyed zappar_image_tracker_t");
+                return new Image();
+            }
+            return obj.getDecodedPreview(i);
         }, image_tracker_anchor_pose_camera_relative: (o, indx, mirror) => {
             let res = pipeline_1.applyScreenCounterRotation(undefined, c.impl.image_tracker_anchor_pose_raw(o, indx));
             if (mirror) {
@@ -290,7 +413,7 @@ function initialize() {
                 return gl_matrix_1.mat4.create();
             }
             return obj.anchor_pose;
-        }, html_element_source_create: (pipeline, elm) => html_element_source_1.HTMLElementSource.createVideoElementSource(pipeline, elm), html_element_source_start: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.start(); }, html_element_source_pause: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, html_element_source_destroy: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, permission_granted_all: permission_1.permissionGrantedAll, permission_granted_camera: permission_1.permissionGrantedCamera, permission_granted_motion: permission_1.permissionGrantedMotion, permission_denied_any: permission_1.permissionDeniedAny, permission_denied_camera: permission_1.permissionDeniedCamera, permission_denied_motion: permission_1.permissionDeniedMotion, permission_request_motion: permission_1.permissionRequestMotion, permission_request_camera: permission_1.permissionRequestCamera, permission_request_all: permission_1.permissionRequestAll, permission_request_ui: permission_1.permissionRequestUI, permission_request_ui_promise: permission_1.permissionRequestUI, permission_denied_ui: permission_1.permissionDeniedUI, browser_incompatible: compatibility_1.default.incompatible, browser_incompatible_ui: compatibility_1.default.incompatible_ui, log_level_set: l => {
+        }, html_element_source_create: (pipeline, elm) => html_element_source_1.HTMLElementSource.createVideoElementSource(pipeline, elm), html_element_source_start: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.start(); }, html_element_source_pause: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, html_element_source_destroy: o => { var _a; return (_a = html_element_source_1.HTMLElementSource.getVideoElementSource(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, sequence_source_create: p => sequencesource_1.SequenceSource.create(p), sequence_source_load_from_memory: (o, data) => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.loadFromMemory(data); }, sequence_source_pause: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.pause(); }, sequence_source_start: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.start(); }, sequence_source_max_playback_fps_set: (o, fps) => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.maxPlaybackFpsSet(fps); }, sequence_source_destroy: o => { var _a; return (_a = sequencesource_1.SequenceSource.get(o)) === null || _a === void 0 ? void 0 : _a.destroy(); }, permission_granted_all: permission_1.permissionGrantedAll, permission_granted_camera: permission_1.permissionGrantedCamera, permission_granted_motion: permission_1.permissionGrantedMotion, permission_denied_any: permission_1.permissionDeniedAny, permission_denied_camera: permission_1.permissionDeniedCamera, permission_denied_motion: permission_1.permissionDeniedMotion, permission_request_motion: permission_1.permissionRequestMotion, permission_request_camera: permission_1.permissionRequestCamera, permission_request_all: permission_1.permissionRequestAll, permission_request_ui: permission_1.permissionRequestUI, permission_request_ui_promise: permission_1.permissionRequestUI, permission_denied_ui: permission_1.permissionDeniedUI, browser_incompatible: compatibility_1.default.incompatible, browser_incompatible_ui: compatibility_1.default.incompatible_ui, log_level_set: l => {
             loglevel_1.setLogLevel(l);
             c.impl.log_level_set(l);
         } });

package/lib/pipeline.d.ts

@@ -1,9 +1,11 @@
-import { zappar_pipeline_t } from "./gen/zappar";
+/// <reference types="dom-webcodecs" />
+import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
 import { zappar_cwrap } from "./gen/zappar-native";
 import { zappar_client } from "./gen/zappar-client";
 import { MsgManager } from "./messages";
 import { mat4 } from "gl-matrix";
 import { Source, CameraFrameInfo } from "./source";
+import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
 import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
 export declare class Pipeline {
@@ -21,12 +23,21 @@ export declare class Pipeline {
     private _cameraDraw;
     private _faceDraw;
     private _faceProjectDraw;
+    private _sequenceRecorder;
+    private _sequenceRecordDeviceAttitudeMatrices;
+    private _sequenceRecorderFirstCameraToken;
     onGLContextReset: Event;
     static create(client: zappar_cwrap, mgr: MsgManager): zappar_pipeline_t;
     static get(p: zappar_pipeline_t): Pipeline | undefined;
     private constructor();
     frameUpdate(client: zappar_client): void;
-    cameraTokenReturn(tokenId: number, pixelArray: ArrayBuffer): void;
+    cleanOldFrames(): void;
+    cameraTokenReturn(msg: CameraFrameReturnS2C): void;
+    sequenceRecordStart(expectedFrames: number): void;
+    sequenceRecordStop(): void;
+    sequenceRecordData(): Uint8Array;
+    sequenceRecordClear(): void;
+    sequenceRecordDeviceAttitudeMatrices(v: boolean): void;
     getVideoTexture(): WebGLTexture | undefined;
     destroy(): void;
     getCurrentCameraInfo(): CameraFrameInfo | undefined;
@@ -39,9 +50,18 @@ export declare class Pipeline {
     cameraFrameTextureMatrix(sw: number, sh: number, mirror: boolean): Float32Array;
     cameraFrameUserFacing(): boolean;
     cameraPoseWithAttitude(mirror: boolean): Float32Array;
+    videoFrameFromWorker(msg: VideoFrameS2C): void;
+    imageBitmapFromWorker(msg: ImageBitmapS2C): void;
+    uploadGL(): void;
+    registerToken(info: CameraFrameInfo): number;
     processGL(): void;
     motionAccelerometerSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionRotationRateSubmit(timestamp: number, x: number, y: number, z: number): void;
     motionAttitudeSubmit(timestamp: number, x: number, y: number, z: number): void;
+    motionAttitudeMatrix(m: Float32Array): void;
+    sendCameraStreamToWorker(source: zappar_camera_source_t, stream: ReadableStream<VideoFrame>, userFacing: boolean): void;
+    sendCameraToScreenRotationToWorker(rot: number): void;
+    sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array): void;
+    sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
 }
 export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): mat4;