@zappar/zappar-cv 3.2.0-beta.2 → 3.3.0-beta.0

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/3.2.0-beta.2/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/3.3.0-beta.0/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/3.2.0-beta.2/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.2.0-beta.2/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/3.3.0-beta.0/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.3.0-beta.0/zappar-cv.zip)

package/lib/additional.d.ts

@@ -1,4 +1,4 @@
-import { CameraFrameData } from "./camera-frame-data";
+import { CameraFrameData, CameraFrameDataRGB } from "./camera-frame-data";
 import { zappar_face_mesh_t, zappar_face_tracker_t, zappar_world_tracker_t, zappar_pipeline_t, zappar_image_tracker_t, zappar_sequence_source_t } from "./gen/zappar";
 export declare type zappar_html_element_source_t = number & {
     _: 'zappar_html_element_source_t';
@@ -23,6 +23,8 @@ export interface Additional {
     pipeline_draw_face_project(pipeline: zappar_pipeline_t, drawwMatrix: Float32Array, vertices: Float32Array, uvMatrix: Float32Array, uvs: Float32Array, indices: Uint16Array, texture: WebGLTexture): void;
     pipeline_camera_frame_data_raw_enabled_set(pipeline: zappar_pipeline_t, enabled: boolean): void;
     pipeline_camera_frame_data_raw(pipeline: zappar_pipeline_t): Promise<CameraFrameData | undefined>;
+    pipeline_camera_frame_data_rgb_enabled_set(pipeline: zappar_pipeline_t, enabled: boolean): void;
+    pipeline_camera_frame_data_rgb(pipeline: zappar_pipeline_t): CameraFrameDataRGB | undefined;
     draw_plane(gl: WebGLRenderingContext, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, texture: string): void;
     draw_grid(gl: WebGLRenderingContext, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array): void;
     draw_axis(gl: WebGLRenderingContext, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array): void;

package/lib/android-bridge-message-handler.d.ts

@@ -2,10 +2,14 @@ export declare class AndroidBridgeMessageHandler {
     private _nativePort;
     private _messageId;
     private _replyResolvers;
+    private _queuedMessages;
     static IsSupported(): boolean;
+    private static _instance;
+    static SharedInstance(): AndroidBridgeMessageHandler;
     postMessage(message: string): Promise<any>;
     private _processMessage;
     private _processStringMessage;
     private _processArrayBufferMessage;
+    private _initRequested;
     private _initPort;
 }

package/lib/android-bridge-message-handler.js

@@ -38,6 +38,7 @@ export class AndroidBridgeMessageHandler {
     constructor() {
         this._messageId = 0;
         this._replyResolvers = new Map();
+        this._queuedMessages = [];
         this._processMessage = (msg) => {
             if (typeof msg.data === "string") {
                 traceStart("ABMH: processStringMessage");
@@ -51,25 +52,32 @@ export class AndroidBridgeMessageHandler {
                 traceEnd();
             }
         };
+        this._initRequested = false;
     }
     static IsSupported() {
         let hasAndroidInterface = !!(globalThis.zprCameraBridge);
         return hasAndroidInterface;
     }
+    static SharedInstance() {
+        if (AndroidBridgeMessageHandler._instance == null) {
+            AndroidBridgeMessageHandler._instance = new AndroidBridgeMessageHandler();
+        }
+        return AndroidBridgeMessageHandler._instance;
+    }
     postMessage(message) {
         return __awaiter(this, void 0, void 0, function* () {
             traceStart("ABMH: postMessage");
-            if (!this._nativePort) {
-                let portReady = this._initPort();
-                traceEnd();
-                yield portReady;
-                traceStart("ABMH: postMessage continuation");
-            }
             let fullMsg = `zpr:${this._messageId}:${message}`;
             let reply = new Promise((resolve, reject) => {
                 this._replyResolvers.set(this._messageId, { resolve, reject });
             });
-            this._nativePort.postMessage(fullMsg);
+            if (!this._nativePort) {
+                this._queuedMessages.push(fullMsg);
+                this._initPort();
+            }
+            else {
+                this._nativePort.postMessage(fullMsg);
+            }
             this._messageId++;
             traceEnd();
             return reply;
@@ -150,18 +158,28 @@ export class AndroidBridgeMessageHandler {
         replyResolvers.resolve(msgData);
     }
     _initPort() {
-        return new Promise(resolve => {
-            let listener = (msg) => {
-                if (msg.data === "zprCameraBridgePort" && msg.ports.length > 0) {
-                    this._nativePort = msg.ports[0];
-                    window.removeEventListener("message", listener);
-                    window.addEventListener("message", this._processMessage);
-                    resolve();
+        // Ensure we only call the request once
+        if (this._initRequested)
+            return;
+        this._initRequested = true;
+        // The one-time listener to receive the port from native ZapparCameraBridge
+        const listener = (msg) => {
+            if (msg.data === "zprCameraBridgePort" && msg.ports.length > 0) {
+                this._nativePort = msg.ports[0];
+                // Post initial messages in the order they were queued
+                for (const msg of this._queuedMessages) {
+                    this._nativePort.postMessage(msg);
                 }
-            };
-            window.addEventListener("message", listener);
-            let origin = window.origin || document.location.origin;
-            window.zprCameraBridge.requestMessageChannel(origin);
-        });
+                this._queuedMessages = [];
+                // Remove this listener, add the main processMessage one
+                window.removeEventListener("message", listener);
+                window.addEventListener("message", this._processMessage);
+            }
+        };
+        // Use the JSInterface from ZapparCameraBridge to request a message channel
+        window.addEventListener("message", listener);
+        let origin = window.origin || document.location.origin;
+        window.zprCameraBridge.requestMessageChannel(origin);
     }
 }
+AndroidBridgeMessageHandler._instance = null;

package/lib/bridged-camera-source.js

@@ -97,7 +97,7 @@ export class BridgedCameraSource extends Source {
             pipeline.motionAttitudeSubmitInt(timeStamp, 0, ev.alpha, ev.beta, ev.gamma);
         };
         if (AndroidBridgeMessageHandler.IsSupported()) {
-            this._messageHandler = new AndroidBridgeMessageHandler();
+            this._messageHandler = AndroidBridgeMessageHandler.SharedInstance();
         }
         else {
             this._messageHandler = (_b = (_a = window.webkit) === null || _a === void 0 ? void 0 : _a.messageHandlers) === null || _b === void 0 ? void 0 : _b.bridgedCamera;

package/lib/camera-frame-data.d.ts

@@ -1,5 +1,11 @@
-export interface CameraFrameData {
+/// <reference types="dom-webcodecs" />
+export interface BaseFrameData {
     width: number;
     height: number;
+}
+export interface CameraFrameData extends BaseFrameData {
     data: ArrayBuffer;
 }
+export interface CameraFrameDataRGB extends BaseFrameData {
+    frame: ImageBitmap | VideoFrame | ImageData;
+}

package/lib/html-element-source.d.ts

@@ -16,6 +16,12 @@ export declare class HTMLElementSource extends Source {
     private _cameraModel;
     private _profile;
     private _waitingForFrame;
+    private _rgbOffscreenCanvas;
+    private _rgbOffscreenContext;
+    private _rgbDomCanvas;
+    private _rgbDomContext;
+    private _currentRgbFrame;
+    private _isVideoElement;
     static createVideoElementSource(p: zappar_pipeline_t, element: HTMLVideoElement | HTMLImageElement): zappar_html_element_source_t;
     static getVideoElementSource(m: zappar_html_element_source_t): HTMLElementSource | undefined;
     constructor(_video: HTMLVideoElement | HTMLImageElement, _pipeline: zappar_pipeline_t);
@@ -30,4 +36,7 @@ export declare class HTMLElementSource extends Source {
     private _processFrame;
     uploadGL(): void;
     setProfile(p: camera_profile_t): void;
+    private _generateRGBFrame;
+    private _generateOffscreenRgbFrame;
+    private _generateDomRgbFrame;
 }

package/lib/html-element-source.js

@@ -3,7 +3,7 @@ import { Pipeline } from "./pipeline";
 import { Source } from "./source";
 import { profile } from "./profile";
 import { mat4 } from "gl-matrix";
-import { zcout } from "./loglevel";
+import { zcout, zcerr } from "./loglevel";
 import { ImageProcessGL } from "./image-process-gl";
 let latest = 1;
 let byId = new Map();
@@ -22,10 +22,14 @@ export class HTMLElementSource extends Source {
         this._cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
         this._profile = camera_profile_t.DEFAULT;
         this._waitingForFrame = false;
+        this._rgbOffscreenContext = null;
+        this._rgbDomContext = null;
+        this._isVideoElement = false;
         this._lastPresentedFrames = -1;
         mat4.fromScaling(this._cameraToDeviceTransformUserFacing, [-1, 1, -1]);
         let video = this._video;
-        if (this._video instanceof HTMLVideoElement) {
+        this._isVideoElement = video instanceof HTMLVideoElement;
+        if (this._isVideoElement) {
             video.addEventListener("loadedmetadata", () => { this._hadFrames = true; });
         }
         else {
@@ -56,6 +60,11 @@ export class HTMLElementSource extends Source {
             p.onGLContextReset.unbind(this._resetGLContext);
         this.pause();
         this._resetGLContext();
+        this._currentRgbFrame = undefined;
+        this._rgbDomCanvas = undefined;
+        this._rgbDomContext = null;
+        this._rgbOffscreenCanvas = undefined;
+        this._rgbOffscreenContext = null;
     }
     pause() {
         this._isPaused = true;
@@ -138,6 +147,8 @@ export class HTMLElementSource extends Source {
             }
             if (!this._currentVideoTexture)
                 return undefined;
+            if (pipeline.getCameraFrameDataRGBEnabled())
+                this._currentRgbFrame = this._generateRGBFrame();
             this._imageProcessor.uploadFrame(this._currentVideoTexture, this._video, rotation, this._isUserFacing, this._profile);
             this._requestVideoFrameCallback();
             this._isUploadFrame = !this._isUploadFrame;
@@ -164,7 +175,12 @@ export class HTMLElementSource extends Source {
         this._cameraModel[1] = focalLength;
         this._cameraModel[2] = dataWidth * 0.5;
         this._cameraModel[3] = dataHeight * 0.5;
-        return Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels, this._profile)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
+        const info = Object.assign(Object.assign({}, this._imageProcessor.readFrame(tex, pixels, this._profile)), { cameraModel: this._cameraModel, cameraSource: this, cameraToDevice: (this._isUserFacing ? this._cameraToDeviceTransformUserFacing : this._cameraToDeviceTransform) });
+        if (this._currentRgbFrame) {
+            info.rgbFrame = this._currentRgbFrame;
+            this._currentRgbFrame = undefined;
+        }
+        return info;
     }
     uploadGL() {
         // No-op as already uploaded
@@ -172,4 +188,57 @@ export class HTMLElementSource extends Source {
     setProfile(p) {
         this._profile = p;
     }
+    _generateRGBFrame() {
+        const width = this._isVideoElement ? this._video.videoWidth : this._video.naturalWidth;
+        const height = this._isVideoElement ? this._video.videoHeight : this._video.naturalHeight;
+        if (!(width > 0 && height > 0))
+            return undefined;
+        return profile.offscreenCanvasSupported
+            ? this._generateOffscreenRgbFrame(width, height)
+            : this._generateDomRgbFrame(width, height);
+    }
+    _generateOffscreenRgbFrame(width, height) {
+        var _a;
+        if (!this._rgbOffscreenCanvas) {
+            this._rgbOffscreenCanvas = new OffscreenCanvas(width, height);
+            this._rgbOffscreenContext = this._rgbOffscreenCanvas.getContext('2d', {
+                alpha: false,
+                willReadFrequently: true
+            });
+        }
+        else if (this._rgbOffscreenCanvas.width !== width || this._rgbOffscreenCanvas.height !== height) {
+            this._rgbOffscreenCanvas.width = width;
+            this._rgbOffscreenCanvas.height = height;
+        }
+        (_a = this._rgbOffscreenContext) === null || _a === void 0 ? void 0 : _a.drawImage(this._video, 0, 0, width, height);
+        try {
+            return this._rgbOffscreenCanvas.transferToImageBitmap();
+        }
+        catch (e) {
+            zcerr(`Failed to generate RGB frame: ${e}`);
+            return undefined;
+        }
+    }
+    _generateDomRgbFrame(width, height) {
+        var _a, _b;
+        if (!this._rgbDomCanvas) {
+            this._rgbDomCanvas = document.createElement('canvas');
+            this._rgbDomContext = this._rgbDomCanvas.getContext('2d', {
+                alpha: false,
+                willReadFrequently: true
+            });
+        }
+        else if (this._rgbDomCanvas.width !== width || this._rgbDomCanvas.height !== height) {
+            this._rgbDomCanvas.width = width;
+            this._rgbDomCanvas.height = height;
+        }
+        (_a = this._rgbDomContext) === null || _a === void 0 ? void 0 : _a.drawImage(this._video, 0, 0, width, height);
+        try {
+            return (_b = this._rgbDomContext) === null || _b === void 0 ? void 0 : _b.getImageData(0, 0, width, height);
+        }
+        catch (e) {
+            zcerr(`Failed to generate RGB frame: ${e}`);
+            return undefined;
+        }
+    }
 }

package/lib/imagebitmap-camera-source.js

@@ -286,8 +286,6 @@ export class ImageBitmapCameraSource extends Source {
         gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, info.frame);
         gl.bindTexture(gl.TEXTURE_2D, null);
         info.texture = texture;
-        info.frame.close();
-        delete info.frame;
     }
 }
 ImageBitmapCameraSource.USER_DEFAULT_DEVICE_ID = "Simulated User Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";

package/lib/mstp-camera-source.js

@@ -323,8 +323,6 @@ export class MSTPCameraSource extends Source {
         gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, info.frame);
         gl.bindTexture(gl.TEXTURE_2D, previousBoundTexture);
         info.texture = texture;
-        info.frame.close();
-        delete info.frame;
     }
 }
 MSTPCameraSource.USER_DEFAULT_DEVICE_ID = "Simulated User Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";

package/lib/native.js

@@ -277,7 +277,7 @@ export function initialize(opts) {
                 return;
             }
             (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.drawImageTargetPreview(projectionMatrix, cameraMatrix, targetMatrix, indx, obj);
-        }, projection_matrix_from_camera_model_and_size: projectionMatrixFromModelAndSize, projection_matrix_from_camera_model_and_size_ext: projectionMatrixFromModelAndSize, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, pipeline_camera_frame_data_raw: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.getCameraFrameDataRaw()) || Promise.reject(); }, pipeline_camera_frame_data_raw_enabled_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.setCameraFrameDataRawEnabled(v); }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
+        }, projection_matrix_from_camera_model_and_size: projectionMatrixFromModelAndSize, projection_matrix_from_camera_model_and_size_ext: projectionMatrixFromModelAndSize, pipeline_process_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.processGL(); }, pipeline_gl_context_set: (p, gl, texturePool) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextSet(gl, texturePool); }, pipeline_gl_context_lost: (p) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.glContextLost(); }, pipeline_camera_frame_upload_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.uploadGL(); }, pipeline_camera_frame_texture_gl: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTexture(); }, pipeline_camera_frame_texture_matrix: (p, sw, sh, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameTextureMatrix(sw, sh, mirror)) || mat4.create(); }, pipeline_camera_frame_user_facing: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraFrameUserFacing()) || false; }, pipeline_camera_pose_default: () => mat4.create(), pipeline_camera_pose_with_attitude: (p, mirror) => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.cameraPoseWithAttitude(mirror)) || mat4.create(); }, pipeline_camera_pose_with_origin: (p, o) => { let res = mat4.create(); mat4.invert(res, o); return res; }, pipeline_sequence_record_clear: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordClear(); }, pipeline_sequence_record_start: (p, expectedFrames) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStart(expectedFrames); }, pipeline_sequence_record_stop: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordStop(); }, pipeline_sequence_record_device_attitude_matrices_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordDeviceAttitudeMatrices(v); }, pipeline_sequence_record_data: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData()) || new Uint8Array(0); }, pipeline_sequence_record_data_size: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.sequenceRecordData().byteLength) || 0; }, pipeline_camera_frame_data_raw: p => { var _a; return ((_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.getCameraFrameDataRaw()) || Promise.reject(); }, pipeline_camera_frame_data_raw_enabled_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.setCameraFrameDataRawEnabled(v); }, pipeline_camera_frame_data_rgb: p => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.getCameraFrameDataRGB(); }, pipeline_camera_frame_data_rgb_enabled_set: (p, v) => { var _a; return (_a = Pipeline.get(p)) === null || _a === void 0 ? void 0 : _a.setCameraFrameDataRGBEnabled(v); }, instant_world_tracker_anchor_pose_camera_relative: (o, mirror) => {
             let res = applyScreenCounterRotation(undefined, c.impl.instant_world_tracker_anchor_pose_raw(o));
             if (mirror) {
                 let scale = mat4.create();
@@ -456,8 +456,8 @@ export function initialize(opts) {
                     }
                 }
             }
-            if (!anchor_pose) {
-                zcwarn("unable for find anchor with id", anchor_id);
+            if (!anchor_pose || anchor_pose.length !== 16) {
+                zcwarn("unable for find valid pose for anchor with id", anchor_id);
                 return;
             }
             const pose = mat4.create();

package/lib/pipeline.d.ts

@@ -9,7 +9,7 @@ import { FaceMesh } from "./facemesh";
 import { Event } from "./event";
 import { ImageTracker } from "./imagetracker";
 import { zappar } from "./gen/zappar-bridge";
-import { CameraFrameData } from "./camera-frame-data";
+import { CameraFrameData, CameraFrameDataRGB } from "./camera-frame-data";
 export declare class Pipeline {
     private _client;
     private _impl;
@@ -33,6 +33,7 @@ export declare class Pipeline {
     private _cameraFrameDataResolves;
     private _cameraFrameDataRejects;
     private _cameraFrameDataEnabled;
+    private _cameraFrameDataRGBEnabled;
     onGLContextReset: Event;
     static create(client: zappar, mgr: MsgManager): zappar_pipeline_t;
     static get(p: zappar_pipeline_t): Pipeline | undefined;
@@ -47,6 +48,9 @@ export declare class Pipeline {
     sequenceRecordClear(): void;
     sequenceRecordDeviceAttitudeMatrices(v: boolean): void;
     getVideoTexture(): WebGLTexture | undefined;
+    getCameraFrameDataRGB(): CameraFrameDataRGB | undefined;
+    setCameraFrameDataRGBEnabled(e: boolean): void;
+    getCameraFrameDataRGBEnabled(): boolean;
     getCameraFrameDataRaw(): Promise<CameraFrameData | undefined>;
     setCameraFrameDataRawEnabled(e: boolean): void;
     destroy(): void;

package/lib/pipeline.js

@@ -28,6 +28,7 @@ export class Pipeline {
         this._cameraFrameDataResolves = new Map();
         this._cameraFrameDataRejects = new Map();
         this._cameraFrameDataEnabled = false;
+        this._cameraFrameDataRGBEnabled = false;
         this.onGLContextReset = new Event();
     }
     static create(client, mgr) {
@@ -51,7 +52,7 @@ export class Pipeline {
         this.cleanOldFrames();
     }
     cleanOldFrames() {
-        var _a, _b, _c, _d;
+        var _a, _b, _c, _d, _e, _f;
         let currentToken = this._client.pipeline_camera_frame_user_data(this._impl);
         if (!currentToken)
             return;
@@ -60,7 +61,17 @@ export class Pipeline {
                 if (t[1].texture)
                     this.videoTextures.push(t[1].texture);
                 (_b = (_a = t[1].frame) === null || _a === void 0 ? void 0 : _a.close) === null || _b === void 0 ? void 0 : _b.call(_a);
-                (_d = (_c = t[1].cameraSource).recycle) === null || _d === void 0 ? void 0 : _d.call(_c, t[1]);
+                if (t[1].frame !== t[1].rgbFrame) {
+                    if (t[1].rgbFrame instanceof ImageData) {
+                        // auto gc'd
+                    }
+                    else {
+                        (_d = (_c = t[1].rgbFrame) === null || _c === void 0 ? void 0 : _c.close) === null || _d === void 0 ? void 0 : _d.call(_c);
+                    }
+                }
+                t[1].frame = undefined;
+                t[1].rgbFrame = undefined;
+                (_f = (_e = t[1].cameraSource).recycle) === null || _f === void 0 ? void 0 : _f.call(_e, t[1]);
                 if (t[1].data) {
                     this.cameraPixelArrays.push(t[1].data);
                     t[1].data = undefined;
@@ -135,6 +146,34 @@ export class Pipeline {
     getVideoTexture() {
         return this.videoTextures.pop();
     }
+    getCameraFrameDataRGB() {
+        if (!this._cameraFrameDataRGBEnabled)
+            return undefined;
+        const info = this.getCurrentCameraInfo();
+        if (!info || !info.rgbFrame)
+            return undefined;
+        let width = 0;
+        let height = 0;
+        if (info.rgbFrame instanceof VideoFrame) {
+            width = info.rgbFrame.displayWidth;
+            height = info.rgbFrame.displayHeight;
+        }
+        else {
+            width = info.rgbFrame.width;
+            height = info.rgbFrame.height;
+        }
+        return {
+            frame: info.rgbFrame,
+            width,
+            height,
+        };
+    }
+    setCameraFrameDataRGBEnabled(e) {
+        this._cameraFrameDataRGBEnabled = e;
+    }
+    getCameraFrameDataRGBEnabled() {
+        return this._cameraFrameDataRGBEnabled;
+    }
     getCameraFrameDataRaw() {
         if (!this._cameraFrameDataEnabled)
             throw new Error('Raw camera frame data is not enabled.');
@@ -297,6 +336,7 @@ export class Pipeline {
             cameraToDevice: msg.cameraToDevice,
             cameraSource,
             remoteToken: msg.token,
+            rgbFrame: this._cameraFrameDataRGBEnabled ? msg.d : undefined,
         });
         this.cleanOldFrames();
     }
@@ -309,6 +349,8 @@ export class Pipeline {
         data.frame = msg.frame;
         data.userFacing = msg.userFacing;
         data.uvTransform = msg.uvTransform;
+        if (this._cameraFrameDataRGBEnabled)
+            data.rgbFrame = msg.frame;
         this.tokensInFlight--;
         this.cleanOldFrames();
     }

package/lib/profile.d.ts

@@ -20,4 +20,5 @@ export declare let profile: {
     forceWindowOrientation: boolean;
     intervalMultiplier: number;
     trustSensorIntervals: boolean;
+    offscreenCanvasSupported: boolean;
 };

package/lib/profile.js

@@ -23,6 +23,7 @@ export let profile = {
     forceWindowOrientation: false,
     intervalMultiplier: 1,
     trustSensorIntervals: false,
+    offscreenCanvasSupported: typeof globalThis.OffscreenCanvas !== "undefined",
 };
 if (typeof window !== "undefined") {
     window["zeeProfile"] = profile;

package/lib/source.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="dom-webcodecs" />
 export interface CameraFrameInfo {
+    rgbFrame?: ImageBitmap | VideoFrame | ImageData;
     data?: ArrayBuffer;
     texture: WebGLTexture | undefined;
     dataWidth: number;

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.2.0-beta.2";
+export declare const VERSION = "3.3.0-beta.0";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.2.0-beta.2";
+export const VERSION = "3.3.0-beta.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zappar/zappar-cv",
-  "version": "3.2.0-beta.2",
+  "version": "3.3.0-beta.0",
   "description": "Zappar's core computer vision library, supporting image, face and world tracking.",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -31,7 +31,6 @@
   "devDependencies": {
     "@playwright/test": "^1.48.2",
     "@types/dom-mediacapture-transform": "0.1.1",
-    "@types/dom-webcodecs": "0.1.2",
     "@types/node": "14.14.7",
     "@types/offscreencanvas": "2019.6.4",
     "@types/pngjs": "6.0.0",
@@ -52,6 +51,7 @@
   "dependencies": {
     "@types/w3c-generic-sensor": "^1.0.9",
     "gl-matrix": "3.4.3",
-    "ua-parser-js": "1.0.33"
+    "ua-parser-js": "1.0.33",
+    "@types/dom-webcodecs": "0.1.2"
   }
 }