@zappar/zappar-cv 3.0.0-beta.7 → 3.0.1-alpha.1

package/lib/pipeline.js

@@ -24,10 +24,6 @@ export class Pipeline {
         this.cameraPixelArrays = [];
         this._sequenceRecordDeviceAttitudeMatrices = true;
         this._sequenceRecorderFirstCameraToken = 0;
-        this._cameraFrameDataPromises = new Map();
-        this._cameraFrameDataResolves = new Map();
-        this._cameraFrameDataRejects = new Map();
-        this._cameraFrameDataEnabled = false;
         this.onGLContextReset = new Event();
     }
     static create(client, mgr) {
@@ -60,34 +56,14 @@ export class Pipeline {
                 if (t[1].texture)
                     this.videoTextures.push(t[1].texture);
                 (_b = (_a = t[1].frame) === null || _a === void 0 ? void 0 : _a.close) === null || _b === void 0 ? void 0 : _b.call(_a);
-                if (t[1].data) {
-                    this.cameraPixelArrays.push(t[1].data);
-                    t[1].data = undefined;
-                }
                 this.cameraTokens.delete(t[0]);
             }
         }
     }
-    cameraFrameDataRawResult(msg) {
-        var _a, _b;
-        let info = this.cameraTokens.get(msg.token);
-        if (!info)
-            return;
-        if (msg.data === null) {
-            (_a = this._cameraFrameDataRejects.get(info)) === null || _a === void 0 ? void 0 : _a(new Error('Raw camera data not available'));
-        }
-        else {
-            info.data = msg.data.data;
-            (_b = this._cameraFrameDataResolves.get(info)) === null || _b === void 0 ? void 0 : _b(msg.data);
-        }
-        this._cameraFrameDataRejects.delete(info);
-        this._cameraFrameDataResolves.delete(info);
-        this._cameraFrameDataPromises.delete(info);
-    }
     cameraTokenReturn(msg) {
-        let info = this.cameraTokens.get(msg.token);
         if (this._sequenceRecorder && this._sequenceRecordDeviceAttitudeMatrices
             && msg.token >= this._sequenceRecorderFirstCameraToken) {
+            let info = this.cameraTokens.get(msg.token);
             if (info) {
                 if (msg.att)
                     this._sequenceRecorder.appendAttitudeMatrix(msg.att);
@@ -95,20 +71,7 @@ export class Pipeline {
                 this._sequenceRecorder.appendCameraFrame(info);
             }
         }
-        if (info && !info.data && msg.d) {
-            info.data = msg.d;
-            const rawRequestResolve = this._cameraFrameDataResolves.get(info);
-            if (rawRequestResolve) {
-                rawRequestResolve({
-                    data: info.data,
-                    width: info.dataWidth,
-                    height: info.dataHeight,
-                });
-            }
-            this._cameraFrameDataRejects.delete(info);
-            this._cameraFrameDataResolves.delete(info);
-            this._cameraFrameDataPromises.delete(info);
-        }
+        this.cameraPixelArrays.push(msg.d);
         this.tokensInFlight--;
     }
     sequenceRecordStart(expectedFrames) {
@@ -134,37 +97,6 @@ export class Pipeline {
     getVideoTexture() {
         return this.videoTextures.pop();
     }
-    getCameraFrameDataRaw() {
-        if (!this._cameraFrameDataEnabled)
-            throw new Error('Raw camera frame data is not enabled.');
-        const info = this.getCurrentCameraInfo();
-        if (!info)
-            return Promise.resolve(undefined);
-        if (info.data)
-            return Promise.resolve({ data: info.data.slice(0), width: info.dataWidth, height: info.dataHeight });
-        let existing = this._cameraFrameDataPromises.get(info);
-        if (!existing) {
-            existing = new Promise((resolve, reject) => {
-                if (typeof info.remoteToken !== 'undefined') {
-                    const msg = {
-                        t: "rawrequest",
-                        token: info.remoteToken,
-                        p: this._impl,
-                    };
-                    this._mgr.postOutgoingMessage(msg, []);
-                }
-                this._cameraFrameDataResolves.set(info, resolve);
-                this._cameraFrameDataRejects.set(info, reject);
-            });
-            this._cameraFrameDataPromises.set(info, existing);
-        }
-        return existing;
-    }
-    setCameraFrameDataRawEnabled(e) {
-        this._cameraFrameDataEnabled = e;
-        const msg = { t: 'rawenabled', v: e };
-        this._mgr.postOutgoingMessage(msg, []);
-    }
     destroy() {
         this._client.pipeline_destroy(this._impl);
         byId.delete(this._impl);
@@ -294,8 +226,7 @@ export class Pipeline {
             uvTransform: msg.uvTransform,
             cameraModel: msg.cameraModel,
             cameraToDevice: msg.cameraToDevice,
-            cameraSource,
-            remoteToken: msg.token,
+            cameraSource
         });
         this.cleanOldFrames();
     }

package/lib/source.d.ts

@@ -10,7 +10,6 @@ export interface CameraFrameInfo {
     cameraSourceData?: any;
     uvTransform?: Float32Array;
     userFacing: boolean;
-    remoteToken?: number;
     frame?: VideoFrame | ImageBitmap;
 }
 export declare abstract class Source {

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.0-beta.7";
+export declare const VERSION = "3.0.1-alpha.1";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.0-beta.7";
+export const VERSION = "3.0.1-alpha.1";

package/lib/worker-client.d.ts

@@ -1,3 +1,4 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
 export declare function launchWorker(worker?: Worker): Promise<void>;
+export declare function launchCeresWorker(port: MessagePort, worker?: Worker): Promise<void>;

package/lib/worker-client.js

@@ -9,13 +9,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 import { MsgManager } from "./messages";
 export let messageManager = new MsgManager();
+const wasmModule = (() => __awaiter(void 0, void 0, void 0, function* () {
+    const response = yield fetch(new URL("./zappar-cv.wasm", import.meta.url).toString());
+    const module = yield WebAssembly.compileStreaming(response);
+    return module;
+}))();
 export function launchWorker(worker) {
     return __awaiter(this, void 0, void 0, function* () {
         if (!worker)
             worker = new Worker(new URL("./worker", import.meta.url), { type: 'module' });
         worker.postMessage({
             t: "wasm",
-            url: new URL("./zappar-cv.wasm", import.meta.url).toString()
+            url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
+            module: yield wasmModule
         });
         yield waitForLoad(worker);
         function sendOutgoing() {
@@ -31,6 +37,18 @@ export function launchWorker(worker) {
         });
     });
 }
+export function launchCeresWorker(port, worker) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!worker)
+            worker = new Worker(new URL("./ceres-worker", import.meta.url), { type: 'module' });
+        worker.postMessage({
+            t: "wasm",
+            url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
+            module: yield wasmModule,
+            port
+        }, [port]);
+    });
+}
 function waitForLoad(w) {
     return new Promise(resolve => {
         let listener = (msg) => {

package/lib/worker-messages.d.ts

@@ -0,0 +1,10 @@
+import type * as ZNM from "./zappar-cv";
+export interface WorkerMessageAPI {
+    worker_message_send_count: () => number;
+    worker_message_send_clear: () => void;
+    worker_message_send_data_size: (index: number) => number;
+    worker_message_send_data: (index: number) => number;
+    worker_message_send_reference: (index: number) => number;
+    worker_message_receive: (ref: number, size: number, data: number) => void;
+}
+export declare function getWorkerMessageAPI(mod: ZNM.Module): WorkerMessageAPI;

package/lib/worker-messages.js

@@ -0,0 +1,10 @@
+export function getWorkerMessageAPI(mod) {
+    return {
+        worker_message_send_count: mod.cwrap("worker_message_send_count", "number", []),
+        worker_message_send_clear: mod.cwrap("worker_message_send_clear", null, []),
+        worker_message_send_data_size: mod.cwrap("worker_message_send_data_size", "number", ["number"]),
+        worker_message_send_data: mod.cwrap("worker_message_send_data", "number", ["number"]),
+        worker_message_send_reference: mod.cwrap("worker_message_send_reference", "number", ["number"]),
+        worker_message_receive: mod.cwrap("worker_message_receive", null, ["number", "number", "number"]),
+    };
+}

package/lib/worker-server.d.ts

@@ -1,3 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-export declare function launchWorkerServer(wasmUrl: string): Promise<void>;
+export declare function launchWorkerServer(wasmUrl: string, module: WebAssembly.Module): Promise<void>;

package/lib/worker-server.js

@@ -15,11 +15,11 @@ import { mat4 } from "gl-matrix";
 import { frame_pixel_format_t } from "./gen/zappar-native";
 import { handleImageBitmap } from "./worker-imagebitmap";
 import { profile } from "./profile";
+import { getWorkerMessageAPI } from "./worker-messages";
 export let messageManager = new MsgManager();
+let workerMessageChannel;
 let latestCameraToScreenRotation = 0;
-let rawDataAccessEnabled = false;
-const rawCameraFrameBuffersByPipeline = new Map();
-export function launchWorkerServer(wasmUrl) {
+export function launchWorkerServer(wasmUrl, module) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
             locateFile: (path, prefix) => {
@@ -28,8 +28,14 @@ export function launchWorkerServer(wasmUrl) {
                 }
                 return prefix + path;
             },
+            instantiateWasm: (imports, successCallback) => {
+                const instance = new WebAssembly.Instance(module, imports);
+                successCallback(instance);
+                return instance.exports;
+            },
             onRuntimeInitialized: () => {
                 let r = getRuntimeObject(mod);
+                const workerMessageAPI = getWorkerMessageAPI(mod);
                 let server = new zappar_server(r, (pipelineId, ab) => {
                     messageManager.postOutgoingMessage({
                         p: pipelineId,
@@ -39,7 +45,7 @@ export function launchWorkerServer(wasmUrl) {
                 });
                 messageManager.postOutgoingMessage("loaded", []);
                 messageManager.onIncomingMessage.bind((msg) => {
-                    var _a, _b, _c;
+                    var _a;
                     switch (msg.t) {
                         case "zappar":
                             server.processBuffer(msg.d);
@@ -57,6 +63,7 @@ export function launchWorkerServer(wasmUrl) {
                                 r.pipeline_frame_update(pipeline);
                                 att = r.pipeline_camera_frame_device_attitude(pipeline);
                                 server.exploreState();
+                                handleMessages(mod, workerMessageAPI);
                             }
                             let ret = {
                                 token: msgt.token,
@@ -68,26 +75,9 @@ export function launchWorkerServer(wasmUrl) {
                             messageManager.postOutgoingMessage(ret, [msgt.d]);
                             break;
                         }
-                        case "rawenabled": {
-                            const msgt = msg;
-                            rawDataAccessEnabled = msgt.v;
-                            break;
-                        }
-                        case "rawrequest": {
-                            const msgt = msg;
-                            const buffers = rawCameraFrameBuffersByPipeline.get(msgt.p);
-                            const ret = {
-                                t: "raw",
-                                token: msgt.token,
-                                p: msgt.p,
-                                data: buffers ? (_c = (_b = buffers.ready.find(entry => entry.token === msgt.token)) === null || _b === void 0 ? void 0 : _b.data) !== null && _c !== void 0 ? _c : null : null,
-                            };
-                            messageManager.postOutgoingMessage(ret, []);
-                            break;
-                        }
                         case "streamC2S": {
                             let msgt = msg;
-                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source).then(() => {
+                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source, workerMessageAPI).then(() => {
                                 let m = { t: "streamEndedS2C", p: msgt.p, source: msgt.source };
                                 // console.log('Stream ended, requesting new stream');
                                 messageManager.postOutgoingMessage(m, []);
@@ -151,7 +141,7 @@ export function launchWorkerServer(wasmUrl) {
     });
 }
 ;
-function consumeStream(mod, r, stream, p, userFacing, server, source) {
+function consumeStream(mod, r, stream, p, userFacing, server, source, workerMessageAPI) {
     return __awaiter(this, void 0, void 0, function* () {
         while (true) {
             let reader;
@@ -164,7 +154,7 @@ function consumeStream(mod, r, stream, p, userFacing, server, source) {
                 continue;
             }
             try {
-                yield consumeReader(mod, r, reader, p, userFacing, server, source);
+                yield consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI);
                 return;
             }
             catch (err) {
@@ -192,8 +182,8 @@ function getFrameOrTimeout(reader) {
 }
 const cameraToDeviceTransform = mat4.create();
 const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
-function consumeReader(mod, r, reader, p, userFacing, server, source) {
-    var _a;
+function consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI) {
+    var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         while (true) {
             let result = yield getFrameOrTimeout(reader);
@@ -253,8 +243,8 @@ function consumeReader(mod, r, reader, p, userFacing, server, source) {
                 t: "videoFrameS2C",
                 userFacing,
                 uvTransform,
-                w: profile.dataWidth,
-                h: profile.dataHeight,
+                w: dataWidth,
+                h: dataHeight,
                 cameraToDevice: cameraToDeviceTransform,
                 cameraModel,
                 source
@@ -263,41 +253,16 @@ function consumeReader(mod, r, reader, p, userFacing, server, source) {
             const pipeline = server._pipeline_by_instance.get(p);
             if (pipeline) {
                 try {
-                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, (_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1);
+                    handleMessages(mod, workerMessageAPI);
                 }
                 catch (err) {
-                    // console.log('Exception during camera processing', err);
+                    console.log('Exception during camera processing', err);
                     // const func = mod.cwrap('what_to_stderr', 'number', ['number']);
                     // func(err);
                     // (mod as any).ccall('what_to_stderr', 'number', ['number'], [err]);
                 }
                 r.pipeline_frame_update(pipeline);
-                if (rawDataAccessEnabled) {
-                    let buffers = rawCameraFrameBuffersByPipeline.get(p);
-                    if (!buffers) {
-                        buffers = { available: [], ready: [] };
-                        rawCameraFrameBuffersByPipeline.set(p, buffers);
-                    }
-                    if (buffers.ready.length > 4) {
-                        const removed = buffers.ready.splice(0, 1);
-                        for (const entry of removed)
-                            buffers.available.push(new Uint8Array(entry.data.data));
-                    }
-                    const size = r.pipeline_camera_frame_data_raw_size(pipeline);
-                    let buffer;
-                    while (!buffer || buffer.byteLength < size) {
-                        if (buffers.available.length < 1)
-                            buffers.available.push(new Uint8Array(size));
-                        buffer = buffers.available.pop();
-                    }
-                    const dataPointer = r.pipeline_camera_frame_data_raw(pipeline);
-                    buffer.set(mod.HEAPU8.subarray(dataPointer, dataPointer + size));
-                    buffers.ready.push({ token, data: {
-                            data: buffer,
-                            width: r.pipeline_camera_frame_data_raw_width(p),
-                            height: r.pipeline_camera_frame_data_raw_height(p),
-                        } });
-                }
                 server.exploreState();
             }
             frame.close();
@@ -326,3 +291,34 @@ function framePixelFormatFromFormat(f) {
     }
     return frame_pixel_format_t.FRAME_PIXEL_FORMAT_Y;
 }
+function handleMessages(mod, m) {
+    const toSend = m.worker_message_send_count();
+    if (toSend === 0)
+        return;
+    if (!workerMessageChannel) {
+        workerMessageChannel = new MessageChannel();
+        workerMessageChannel.port1.start();
+        workerMessageChannel.port1.addEventListener('message', evt => {
+            if (evt.data.t !== 'msgrec')
+                return;
+            const data = evt.data.data;
+            const ptr = mod._malloc(data.byteLength);
+            mod.HEAPU8.set(data, ptr);
+            m.worker_message_receive(evt.data.reference, data.byteLength, ptr);
+            mod._free(ptr);
+        });
+        messageManager.postOutgoingMessage({ t: 'setupCeresWorker', port: workerMessageChannel.port2 }, [workerMessageChannel.port2]);
+    }
+    for (let i = 0; i < toSend; i++) {
+        const reference = m.worker_message_send_reference(i);
+        const dataSize = m.worker_message_send_data_size(i);
+        const dataPtr = m.worker_message_send_data(i);
+        const data = mod.HEAPU8.slice(dataPtr, dataPtr + dataSize);
+        workerMessageChannel.port1.postMessage({
+            t: 'msgsend',
+            data,
+            reference
+        }, [data.buffer]);
+    }
+    m.worker_message_send_clear();
+}

package/lib/worker.js

@@ -9,7 +9,7 @@ messageManager.onOutgoingMessage.bind(() => {
 let launchHandler = (evt) => {
     if (evt && evt.data && evt.data.t === "wasm") {
         let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zappar-cv.wasm", import.meta.url).toString();
-        launchWorkerServer(url);
+        launchWorkerServer(url, evt.data.module);
         ctx.removeEventListener("message", launchHandler);
     }
 };

package/lib/workerinterface.d.ts

@@ -1,5 +1,4 @@
 /// <reference types="dom-webcodecs" />
-import { CameraFrameData } from "./camera-frame-data";
 import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
 export interface CameraFrameC2S {
     t: "cameraFrameC2S";
@@ -84,18 +83,3 @@ export interface AttitudeMatrixC2S {
     p: zappar_pipeline_t;
     m: Float32Array;
 }
-export interface CameraDataRawEnabledC2S {
-    t: "rawenabled";
-    v: boolean;
-}
-export interface CameraDataRawRequestC2S {
-    t: "rawrequest";
-    p: zappar_pipeline_t;
-    token: number;
-}
-export interface CameraDataRawS2C {
-    t: "raw";
-    token: number;
-    p: zappar_pipeline_t;
-    data: CameraFrameData | null;
-}