@zappar/zappar-cv 0.3.13 → 0.4.0-beta.10

package/lib/sequencerecorder.js

@@ -0,0 +1,283 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SequenceDecoder = exports.SequenceRecorder = exports.SequenceRecorderPacketType = void 0;
+const loglevel_1 = require("./loglevel");
+var SequenceRecorderPacketType;
+(function (SequenceRecorderPacketType) {
+    SequenceRecorderPacketType[SequenceRecorderPacketType["CAMERA_FRAME"] = 0] = "CAMERA_FRAME";
+    SequenceRecorderPacketType[SequenceRecorderPacketType["ACCELEROMETER"] = 1] = "ACCELEROMETER";
+    SequenceRecorderPacketType[SequenceRecorderPacketType["ROTATION_RATE"] = 2] = "ROTATION_RATE";
+    SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE"] = 3] = "ATTITUDE";
+    SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE_MATRIX"] = 4] = "ATTITUDE_MATRIX";
+})(SequenceRecorderPacketType = exports.SequenceRecorderPacketType || (exports.SequenceRecorderPacketType = {}));
+class SequenceRecorder {
+    constructor(estimatedFrames) {
+        this._insertionByte = 0;
+        this._numberPackets = 0;
+        this._started = false;
+        this._hasLoggedUnsuppored = false;
+        this._data = new ArrayBuffer(this._estimatedSize(estimatedFrames, 320, 240));
+        this._dataView = new DataView(this._data);
+        this._dataUint8 = new Uint8Array(this._data);
+        this._dataFloat32 = new Float32Array(this._data);
+    }
+    _estimatedSize(frames, cameraFrameWidth, cameraFrameHeight) {
+        return SequenceRecorder.headerSizeBytes + (SequenceRecorder.psbPacketType + SequenceRecorder.cameraPacketSizeBytes + cameraFrameWidth * cameraFrameHeight) * frames;
+    }
+    start() {
+        this._started = true;
+        this._insertionByte = 0;
+        this._numberPackets = 0;
+        this._growArrayBuffer(SequenceRecorder.headerSizeBytes);
+        let enc = new TextEncoder();
+        let headerData = enc.encode("UAR1");
+        this._dataUint8.set(headerData);
+        this._dataView.setUint32(4, 0, true);
+        this._insertionByte += 8;
+    }
+    stop() {
+        this._started = false;
+        let trimmed = this._data.slice(0, this._insertionByte);
+        this._data = trimmed;
+        this._dataUint8 = new Uint8Array(this._data);
+        this._dataFloat32 = new Float32Array(this._data);
+        this._dataView = new DataView(this._data);
+    }
+    data() {
+        return this._dataUint8;
+    }
+    appendCameraFrame(c) {
+        if (!this._started)
+            return;
+        if (!c.data) {
+            if (this._hasLoggedUnsuppored)
+                return;
+            this._hasLoggedUnsuppored = true;
+            loglevel_1.zcerr("Unable to record frames from this camera source");
+            return;
+        }
+        if (c.data.byteLength !== (c.dataWidth * c.dataHeight)) {
+            loglevel_1.zcerr("Unable to record frame to sequence: greyscale array doesn't match dimensions");
+            return;
+        }
+        let increment = SequenceRecorder.psbPacketType + SequenceRecorder.cameraPacketSizeBytes + (c.dataWidth * c.dataHeight);
+        this._growArrayBuffer(increment);
+        this._dataView.setUint32(this._insertionByte, SequenceRecorderPacketType.CAMERA_FRAME, true);
+        this._insertionByte += SequenceRecorder.psbPacketType;
+        this._dataView.setUint32(this._insertionByte, c.dataWidth, true);
+        this._insertionByte += SequenceRecorder.psbCameraFrameWidth;
+        this._dataView.setUint32(this._insertionByte, c.dataHeight, true);
+        this._insertionByte += SequenceRecorder.psbCameraFrameHeight;
+        this._dataFloat32.set(c.cameraToDevice, this._insertionByte / 4);
+        this._insertionByte += SequenceRecorder.psbCameraToDeviceTransform;
+        this._dataFloat32.set(c.cameraModel, this._insertionByte / 4);
+        this._insertionByte += SequenceRecorder.psbCameraModel;
+        this._dataView.setUint32(this._insertionByte, c.userFacing ? 1 : 0, true);
+        this._insertionByte += SequenceRecorder.psbFlags;
+        this._dataUint8.set(new Uint8Array(c.data), this._insertionByte);
+        this._insertionByte += c.data.byteLength;
+        this._numberPackets++;
+        this._dataView.setUint32(4, this._numberPackets, true);
+    }
+    appendAccelerometer(t, x, y, z) {
+        this._appendTimestampedVec3(SequenceRecorderPacketType.ACCELEROMETER, t, x, y, z);
+    }
+    appendRotationRate(t, x, y, z) {
+        this._appendTimestampedVec3(SequenceRecorderPacketType.ROTATION_RATE, t, x, y, z);
+    }
+    appendAttitude(t, x, y, z) {
+        this._appendTimestampedVec3(SequenceRecorderPacketType.ATTITUDE, t, x, y, z);
+    }
+    _appendTimestampedVec3(type, t, x, y, z) {
+        if (!this._started)
+            return;
+        let increment = SequenceRecorder.psbPacketType + SequenceRecorder.accelerometerPacketSizeBytes;
+        this._growArrayBuffer(increment);
+        this._dataView.setUint32(this._insertionByte, type, true);
+        this._insertionByte += SequenceRecorder.psbPacketType;
+        this._dataView.setUint32(this._insertionByte, t, true);
+        this._insertionByte += 4;
+        this._dataView.setFloat32(this._insertionByte, x, true);
+        this._insertionByte += 4;
+        this._dataView.setFloat32(this._insertionByte, y, true);
+        this._insertionByte += 4;
+        this._dataView.setFloat32(this._insertionByte, z, true);
+        this._insertionByte += 4;
+        this._numberPackets++;
+        this._dataView.setUint32(4, this._numberPackets, true);
+    }
+    appendAttitudeMatrix(m) {
+        if (!this._started)
+            return;
+        let increment = SequenceRecorder.psbPacketType + SequenceRecorder.attitudeMatrixPacketSizeBytes;
+        this._growArrayBuffer(increment);
+        this._dataView.setUint32(this._insertionByte, SequenceRecorderPacketType.ATTITUDE_MATRIX, true);
+        this._insertionByte += SequenceRecorder.psbPacketType;
+        this._dataFloat32.set(m, this._insertionByte / 4);
+        this._insertionByte += SequenceRecorder.attitudeMatrixPacketSizeBytes;
+        this._numberPackets++;
+        this._dataView.setUint32(4, this._numberPackets, true);
+    }
+    _growArrayBuffer(requiredIncrement) {
+        while (this._insertionByte + requiredIncrement > this._data.byteLength) {
+            let newSize = Math.max(this._data.byteLength * 2, this._estimatedSize(50, 320, 240));
+            let newData = new ArrayBuffer(newSize);
+            let newDataUint8 = new Uint8Array(newData);
+            newDataUint8.set(this._dataUint8);
+            this._data = newData;
+            this._dataUint8 = newDataUint8;
+            this._dataFloat32 = new Float32Array(this._data);
+            this._dataView = new DataView(this._data);
+        }
+    }
+}
+exports.SequenceRecorder = SequenceRecorder;
+// Header (8 bytes): "UAR1" + number of frames (uint32)
+SequenceRecorder.headerSizeBytes = 8;
+// Per-frame packet byte sizes
+SequenceRecorder.psbPacketType = 4;
+// Camera frames
+SequenceRecorder.psbCameraFrameWidth = 4;
+SequenceRecorder.psbCameraFrameHeight = 4;
+SequenceRecorder.psbFlags = 4;
+SequenceRecorder.psbCameraToDeviceTransform = 4 * 16;
+SequenceRecorder.psbCameraModel = 4 * 6;
+SequenceRecorder.cameraPacketSizeBytes = SequenceRecorder.psbCameraFrameWidth +
+    SequenceRecorder.psbCameraFrameHeight +
+    SequenceRecorder.psbCameraToDeviceTransform +
+    SequenceRecorder.psbCameraModel +
+    SequenceRecorder.psbFlags;
+// Accelerometer
+SequenceRecorder.accelerometerPacketSizeBytes = 4 * 4; // t, x, y, z
+// Rotation rate
+SequenceRecorder.rotationRatePacketSizeBytes = 4 * 4; // t, x, y, z
+// Attitude
+SequenceRecorder.attitudePacketSizeBytes = 4 * 4; // t, x, y, z
+// Attitude matrix
+SequenceRecorder.attitudeMatrixPacketSizeBytes = 4 * 16;
+class SequenceDecoder {
+    constructor(_data) {
+        this._data = _data;
+        this.dataByPacket = new Map();
+        this.dataByCameraFrame = new Map();
+        this.numberPackets = 0;
+        this.numberCameraFrames = 0;
+        this.numberAccelerometerPackets = 0;
+        this.numberRotationRatePackets = 0;
+        this.numberAttitudePackets = 0;
+        this.numberAttitudeMatrixPackets = 0;
+        let dec = new TextDecoder();
+        let txt = "";
+        try {
+            txt = dec.decode(this._data.slice(0, 4));
+        }
+        catch (err) {
+            throw new Error("Unable to decode header - perhaps this isn't a sequence file?");
+        }
+        if (txt !== "UAR1") {
+            throw new Error(`Invalid - perhaps this isn't a sequence file: ${txt}`);
+        }
+        let readPoint = 0;
+        let dataView = new DataView(this._data);
+        this.numberPackets = dataView.getUint32(4, true);
+        readPoint += SequenceRecorder.headerSizeBytes;
+        for (let i = 0; i < this.numberPackets; i++) {
+            let type = dataView.getUint32(readPoint, true);
+            readPoint += SequenceRecorder.psbPacketType;
+            switch (type) {
+                case SequenceRecorderPacketType.CAMERA_FRAME:
+                    {
+                        let width = dataView.getUint32(readPoint, true);
+                        readPoint += SequenceRecorder.psbCameraFrameWidth;
+                        let height = dataView.getUint32(readPoint, true);
+                        readPoint += SequenceRecorder.psbCameraFrameHeight;
+                        let cameraToDevice = new Float32Array(this._data, readPoint, SequenceRecorder.psbCameraToDeviceTransform / 4);
+                        readPoint += SequenceRecorder.psbCameraToDeviceTransform;
+                        let cameraModel = new Float32Array(this._data, readPoint, SequenceRecorder.psbCameraModel / 4);
+                        readPoint += SequenceRecorder.psbCameraModel;
+                        let flags = dataView.getUint32(readPoint, true);
+                        readPoint += SequenceRecorder.psbFlags;
+                        let userFacing = flags === 1;
+                        let greyscale = new Uint8Array(this._data, readPoint, width * height);
+                        readPoint += width * height;
+                        let d = {
+                            type: SequenceRecorderPacketType.CAMERA_FRAME,
+                            width,
+                            height,
+                            cameraToDevice,
+                            cameraModel,
+                            userFacing,
+                            greyscale
+                        };
+                        this.dataByPacket.set(i, d);
+                        this.dataByCameraFrame.set(this.numberCameraFrames, d);
+                        this.numberCameraFrames++;
+                        break;
+                    }
+                case SequenceRecorderPacketType.ACCELEROMETER:
+                    {
+                        let t = dataView.getUint32(readPoint, true);
+                        readPoint += 4;
+                        let x = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        let y = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        let z = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        this.dataByPacket.set(i, {
+                            type: SequenceRecorderPacketType.ACCELEROMETER,
+                            t, x, y, z
+                        });
+                        this.numberAccelerometerPackets++;
+                        break;
+                    }
+                case SequenceRecorderPacketType.ROTATION_RATE:
+                    {
+                        let t = dataView.getUint32(readPoint, true);
+                        readPoint += 4;
+                        let x = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        let y = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        let z = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        this.dataByPacket.set(i, {
+                            type: SequenceRecorderPacketType.ROTATION_RATE,
+                            t, x, y, z
+                        });
+                        this.numberRotationRatePackets++;
+                        break;
+                    }
+                case SequenceRecorderPacketType.ATTITUDE:
+                    {
+                        let t = dataView.getUint32(readPoint, true);
+                        readPoint += 4;
+                        let x = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        let y = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        let z = dataView.getFloat32(readPoint, true);
+                        readPoint += 4;
+                        this.dataByPacket.set(i, {
+                            type: SequenceRecorderPacketType.ATTITUDE,
+                            t, x, y, z
+                        });
+                        this.numberAttitudePackets++;
+                        break;
+                    }
+                case SequenceRecorderPacketType.ATTITUDE_MATRIX:
+                    {
+                        let attitude = new Float32Array(this._data, readPoint, SequenceRecorder.attitudeMatrixPacketSizeBytes / 4);
+                        readPoint += SequenceRecorder.attitudeMatrixPacketSizeBytes;
+                        this.dataByPacket.set(i, {
+                            type: SequenceRecorderPacketType.ATTITUDE_MATRIX,
+                            attitude
+                        });
+                        this.numberAttitudeMatrixPackets++;
+                        break;
+                    }
+            }
+        }
+    }
+}
+exports.SequenceDecoder = SequenceDecoder;

package/lib/sequencesource.d.ts

@@ -0,0 +1,23 @@
+import { CameraFrameInfo, Source } from "./source";
+import { zappar_sequence_source_t, zappar_pipeline_t } from "./gen/zappar";
+export declare class SequenceSource extends Source {
+    private _impl;
+    private _pipeline;
+    private _packetNumber;
+    private _motionUpdatesQueued;
+    private _motionUpdatesSent;
+    private _decoder;
+    private _minFrameGapMs;
+    private _lastUpdateTime;
+    static create(p: zappar_pipeline_t): zappar_sequence_source_t;
+    static get(m: zappar_sequence_source_t): SequenceSource | undefined;
+    constructor(_impl: zappar_sequence_source_t, _pipeline: zappar_pipeline_t);
+    loadFromMemory(data: ArrayBuffer): void;
+    maxPlaybackFpsSet(fps: number): void;
+    getFrame(currentlyProcessing: boolean): void;
+    destroy(): void;
+    start(): void;
+    pause(): void;
+    uploadGL(info: CameraFrameInfo): void;
+    private _submitMotionUpdates;
+}

package/lib/sequencesource.js

@@ -0,0 +1,170 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SequenceSource = void 0;
+const loglevel_1 = require("./loglevel");
+const sequencerecorder_1 = require("./sequencerecorder");
+const source_1 = require("./source");
+const pipeline_1 = require("./pipeline");
+const gl_matrix_1 = require("gl-matrix");
+let latest = 1;
+let byId = new Map();
+class SequenceSource extends source_1.Source {
+    constructor(_impl, _pipeline) {
+        super();
+        this._impl = _impl;
+        this._pipeline = _pipeline;
+        this._packetNumber = 0;
+        this._motionUpdatesQueued = 0;
+        this._motionUpdatesSent = 0;
+        this._minFrameGapMs = 0;
+        this._lastUpdateTime = 0;
+    }
+    static create(p) {
+        let ret = (latest++);
+        byId.set(ret, new SequenceSource(ret, p));
+        loglevel_1.zcout("sequence_source_source_t initialized");
+        return ret;
+    }
+    static get(m) {
+        return byId.get(m);
+    }
+    loadFromMemory(data) {
+        try {
+            this._decoder = new sequencerecorder_1.SequenceDecoder(data);
+            this._packetNumber = 0;
+            this._lastUpdateTime = 0;
+        }
+        catch (err) {
+            loglevel_1.zcerr("Unable to decode sequence:", err);
+        }
+    }
+    maxPlaybackFpsSet(fps) {
+        if (fps <= 0) {
+            this._minFrameGapMs = 0;
+            return;
+        }
+        this._minFrameGapMs = 1000.0 / fps;
+    }
+    getFrame(currentlyProcessing) {
+        let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+        if (!pipeline)
+            return;
+        if (!this._decoder)
+            return;
+        this._submitMotionUpdates(pipeline);
+        if (this._motionUpdatesQueued > this._motionUpdatesSent)
+            return;
+        if (currentlyProcessing)
+            return;
+        let curTime = performance.now();
+        if (curTime < this._lastUpdateTime + this._minFrameGapMs)
+            return;
+        if (this._packetNumber >= this._decoder.numberPackets)
+            return;
+        let packet = this._decoder.dataByPacket.get(this._packetNumber);
+        if (packet.type != sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME)
+            return;
+        this._packetNumber++;
+        let pixels = pipeline.cameraPixelArrays.pop();
+        while (pixels) {
+            if (pixels.byteLength === packet.greyscale.byteLength)
+                break;
+            pixels = pipeline.cameraPixelArrays.pop();
+        }
+        if (!pixels) {
+            pixels = new ArrayBuffer(packet.greyscale.byteLength);
+        }
+        (new Uint8Array(pixels)).set(packet.greyscale);
+        this._lastUpdateTime = curTime;
+        let info = {
+            data: pixels,
+            cameraSourceData: packet.greyscale,
+            cameraModel: packet.cameraModel,
+            cameraToDevice: packet.cameraToDevice,
+            dataWidth: packet.width,
+            dataHeight: packet.height,
+            texture: undefined,
+            userFacing: packet.userFacing,
+            uvTransform: gl_matrix_1.mat4.create(),
+            cameraSource: this
+        };
+        // We can queue the motion updates for the follwing frame immediately
+        // as this camera frame will be sent immediately, but the next motion
+        // updates will be one frame later
+        this._submitMotionUpdates(pipeline);
+        const token = pipeline.registerToken(info);
+        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
+    }
+    destroy() {
+        byId.delete(this._impl);
+        this.pause();
+    }
+    start() {
+        var _a;
+        let p = pipeline_1.Pipeline.get(this._pipeline);
+        if (p && p.currentCameraSource !== this) {
+            (_a = p.currentCameraSource) === null || _a === void 0 ? void 0 : _a.pause();
+            p.currentCameraSource = this;
+        }
+    }
+    pause() {
+        let p = pipeline_1.Pipeline.get(this._pipeline);
+        if (p && p.currentCameraSource === this)
+            p.currentCameraSource = undefined;
+    }
+    uploadGL(info) {
+        let pipeline = pipeline_1.Pipeline.get(this._pipeline);
+        if (!pipeline)
+            return;
+        let gl = pipeline === null || pipeline === void 0 ? void 0 : pipeline.glContext;
+        if (!gl)
+            return;
+        if (!info.texture)
+            info.texture = pipeline.getVideoTexture();
+        if (!info.texture)
+            return;
+        if (!info.cameraSourceData)
+            return;
+        gl.bindTexture(gl.TEXTURE_2D, info.texture);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, info.dataWidth, info.dataHeight, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, new Uint8Array(info.cameraSourceData));
+        gl.bindTexture(gl.TEXTURE_2D, null);
+    }
+    _submitMotionUpdates(pipeline) {
+        let pendingMotionUpdate = false;
+        while (this._packetNumber < this._decoder.numberPackets) {
+            let packet = this._decoder.dataByPacket.get(this._packetNumber);
+            if (packet.type == sequencerecorder_1.SequenceRecorderPacketType.CAMERA_FRAME)
+                break;
+            switch (packet.type) {
+                case sequencerecorder_1.SequenceRecorderPacketType.ACCELEROMETER:
+                    pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ROTATION_RATE:
+                    pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE:
+                    pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
+                    pendingMotionUpdate = true;
+                    break;
+                case sequencerecorder_1.SequenceRecorderPacketType.ATTITUDE_MATRIX:
+                    pipeline.motionAttitudeMatrix(packet.attitude);
+                    pendingMotionUpdate = true;
+                    break;
+            }
+            this._packetNumber++;
+        }
+        // If we've submitted motion updates, add a setTimeout to reflect when
+        // the message serializer will actually have sent them over to the worker
+        if (pendingMotionUpdate) {
+            this._motionUpdatesQueued++;
+            setTimeout(() => {
+                this._motionUpdatesSent++;
+            }, 0);
+        }
+        return;
+    }
+}
+exports.SequenceSource = SequenceSource;

package/lib/serializer.d.ts

@@ -7,6 +7,7 @@ export interface MessageAppender {
     dataWithLength: (i: ArrayBuffer) => void;
     type: (t: number) => void;
     matrix4x4: (a: Float32Array) => void;
+    matrix3x3: (a: Float32Array) => void;
     cameraModel: (a: Float32Array) => void;
     identityCoefficients: (a: Float32Array) => void;
     expressionCoefficients: (a: Float32Array) => void;

package/lib/serializer.js

@@ -24,6 +24,7 @@ class MessageSerializer {
             dataWithLength: i => this.arrayBuffer(i),
             type: i => this.int(i),
             matrix4x4: i => this.float32ArrayBuffer(i),
+            matrix3x3: i => this.float32ArrayBuffer(i),
             identityCoefficients: i => this.float32ArrayBuffer(i),
             expressionCoefficients: i => this.float32ArrayBuffer(i),
             cameraModel: i => this.float32ArrayBuffer(i),

package/lib/source.d.ts

@@ -1,12 +1,19 @@
+/// <reference types="dom-webcodecs" />
 export interface CameraFrameInfo {
-    data: ArrayBuffer;
+    data?: ArrayBuffer;
     texture: WebGLTexture | undefined;
     dataWidth: number;
     dataHeight: number;
-    uvTransform: Float32Array;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+    cameraSource: Source;
+    cameraSourceData?: any;
+    uvTransform?: Float32Array;
     userFacing: boolean;
+    frame?: VideoFrame | ImageBitmap;
 }
 export declare abstract class Source {
-    abstract getFrame(currentlyProcessing: boolean): CameraFrameInfo | undefined;
+    abstract getFrame(currentlyProcessing: boolean): void;
     abstract pause(): void;
+    abstract uploadGL(info: CameraFrameInfo): void;
 }

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.3.13";
+export declare const VERSION = "0.4.0-beta.10";

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = "0.3.13";
+exports.VERSION = "0.4.0-beta.10";

package/lib/worker-imagebitmap.d.ts

@@ -0,0 +1,5 @@
+import { zappar_cwrap } from "./gen/zappar-native";
+import { zappar_server } from "./gen/zappar-server";
+import { MsgManager } from "./messages";
+import { ImageBitmapC2S } from "./workerinterface";
+export declare function handleImageBitmap(m: ImageBitmapC2S, r: zappar_cwrap, server: zappar_server, mgr: MsgManager): void;

package/lib/worker-imagebitmap.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleImageBitmap = void 0;
+const image_process_gl_1 = require("./image-process-gl");
+const profile_1 = require("./profile");
+const gl_matrix_1 = require("gl-matrix");
+let pixels;
+let texture;
+let _imageProcessor;
+let _gl;
+function getImageProcessor() {
+    if (!_imageProcessor || !_gl) {
+        const canvas = new OffscreenCanvas(1, 1);
+        _gl = canvas.getContext("webgl");
+        if (!_gl)
+            throw new Error("Unable to get offscreen GL context");
+        _imageProcessor = new image_process_gl_1.ImageProcessGL(_gl);
+    }
+    return [_imageProcessor, _gl];
+}
+function handleImageBitmap(m, r, server, mgr) {
+    // img: ImageBitmap,
+    //                                 rot: number,
+    //                                 tokenId: number,
+    //                                 r: zappar_cwrap,
+    //                                 p: zappar_pipeline_t,
+    //                                 userFacing: boolean,
+    //                                 server: zappar_server,
+    //                                 mgr: MsgManager) {
+    const [imageProcessor, gl] = getImageProcessor();
+    if (!texture) {
+        texture = gl.createTexture();
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    }
+    if (!texture)
+        return;
+    if (!pixels || pixels.byteLength < profile_1.profile.dataWidth * profile_1.profile.dataHeight) {
+        console.log("Generating pixel buffer", profile_1.profile.dataWidth * profile_1.profile.dataHeight);
+        pixels = new ArrayBuffer(profile_1.profile.dataWidth * profile_1.profile.dataHeight);
+    }
+    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing);
+    let info = imageProcessor.readFrame(texture, pixels);
+    let msg = {
+        t: "imageBitmapS2C",
+        dataWidth: info.dataWidth,
+        dataHeight: info.dataHeight,
+        frame: m.i,
+        userFacing: info.userFacing,
+        uvTransform: info.uvTransform || gl_matrix_1.mat4.create(),
+        tokenId: m.tokenId, p: m.p
+    };
+    mgr.postOutgoingMessage(msg, [m.i]);
+    let pipeline = server._pipeline_by_instance.get(m.p);
+    if (pipeline) {
+        r.pipeline_camera_frame_submit(pipeline, pixels, info.dataWidth, info.dataHeight, m.tokenId, m.cameraToDevice, m.cameraModel, info.userFacing);
+        r.pipeline_frame_update(pipeline);
+        server.exploreState();
+    }
+}
+exports.handleImageBitmap = handleImageBitmap;