@zappar/zappar-cv 3.0.1-alpha.1 → 3.0.1-alpha.10

package/lib/worker-server.js

@@ -12,14 +12,17 @@ import { getRuntimeObject } from "./gen/zappar-cwrap";
 import { zappar_server } from "./gen/zappar-server";
 import { MsgManager } from "./messages";
 import { mat4 } from "gl-matrix";
-import { frame_pixel_format_t } from "./gen/zappar-native";
+import { camera_profile_t, frame_pixel_format_t } from "./gen/zappar-native";
 import { handleImageBitmap } from "./worker-imagebitmap";
-import { profile } from "./profile";
 import { getWorkerMessageAPI } from "./worker-messages";
+import { getDataDownloadAPI } from "./data-download";
 export let messageManager = new MsgManager();
 let workerMessageChannel;
 let latestCameraToScreenRotation = 0;
-export function launchWorkerServer(wasmUrl, module) {
+let rawDataAccessEnabled = false;
+const rawCameraFrameBuffersByPipeline = new Map();
+const cameraProfileBySource = new Map();
+export function launchWorkerServer(wasmUrl, module, shouldRecordData) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
             locateFile: (path, prefix) => {
@@ -36,6 +39,8 @@ export function launchWorkerServer(wasmUrl, module) {
             onRuntimeInitialized: () => {
                 let r = getRuntimeObject(mod);
                 const workerMessageAPI = getWorkerMessageAPI(mod);
+                const dataDownloadAPI = (shouldRecordData > 0) ? getDataDownloadAPI(mod) : undefined;
+                dataDownloadAPI === null || dataDownloadAPI === void 0 ? void 0 : dataDownloadAPI.data_should_record_set(shouldRecordData); // Only happens if dataDownloadAPI is defined
                 let server = new zappar_server(r, (pipelineId, ab) => {
                     messageManager.postOutgoingMessage({
                         p: pipelineId,
@@ -45,7 +50,7 @@ export function launchWorkerServer(wasmUrl, module) {
                 });
                 messageManager.postOutgoingMessage("loaded", []);
                 messageManager.onIncomingMessage.bind((msg) => {
-                    var _a;
+                    var _a, _b, _c;
                     switch (msg.t) {
                         case "zappar":
                             server.processBuffer(msg.d);
@@ -64,6 +69,8 @@ export function launchWorkerServer(wasmUrl, module) {
                                 att = r.pipeline_camera_frame_device_attitude(pipeline);
                                 server.exploreState();
                                 handleMessages(mod, workerMessageAPI);
+                                if (dataDownloadAPI)
+                                    handleDataDownload(mod, dataDownloadAPI);
                             }
                             let ret = {
                                 token: msgt.token,
@@ -75,9 +82,31 @@ export function launchWorkerServer(wasmUrl, module) {
                             messageManager.postOutgoingMessage(ret, [msgt.d]);
                             break;
                         }
+                        case "rawenabled": {
+                            const msgt = msg;
+                            rawDataAccessEnabled = msgt.v;
+                            break;
+                        }
+                        case "rawrequest": {
+                            const msgt = msg;
+                            const buffers = rawCameraFrameBuffersByPipeline.get(msgt.p);
+                            const ret = {
+                                t: "raw",
+                                token: msgt.token,
+                                p: msgt.p,
+                                data: buffers ? (_c = (_b = buffers.ready.find(entry => entry.token === msgt.token)) === null || _b === void 0 ? void 0 : _b.data) !== null && _c !== void 0 ? _c : null : null,
+                            };
+                            messageManager.postOutgoingMessage(ret, []);
+                            break;
+                        }
+                        case "cameraProfileC2S": {
+                            let msgt = msg;
+                            cameraProfileBySource.set(msgt.source, msgt.p);
+                            break;
+                        }
                         case "streamC2S": {
                             let msgt = msg;
-                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source, workerMessageAPI).then(() => {
+                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source, workerMessageAPI, dataDownloadAPI).then(() => {
                                 let m = { t: "streamEndedS2C", p: msgt.p, source: msgt.source };
                                 // console.log('Stream ended, requesting new stream');
                                 messageManager.postOutgoingMessage(m, []);
@@ -141,7 +170,7 @@ export function launchWorkerServer(wasmUrl, module) {
     });
 }
 ;
-function consumeStream(mod, r, stream, p, userFacing, server, source, workerMessageAPI) {
+function consumeStream(mod, r, stream, p, userFacing, server, source, workerMessageAPI, dataDownloadAPI) {
     return __awaiter(this, void 0, void 0, function* () {
         while (true) {
             let reader;
@@ -154,7 +183,7 @@ function consumeStream(mod, r, stream, p, userFacing, server, source, workerMess
                 continue;
             }
             try {
-                yield consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI);
+                yield consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI, dataDownloadAPI);
                 return;
             }
             catch (err) {
@@ -182,7 +211,7 @@ function getFrameOrTimeout(reader) {
 }
 const cameraToDeviceTransform = mat4.create();
 const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
-function consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI) {
+function consumeReader(mod, r, reader, p, userFacing, server, source, workerMessageAPI, dataDownloadAPI) {
     var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         while (true) {
@@ -226,16 +255,22 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                     uvTransform = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
                     break;
             }
+            let downSample = false;
+            if (cameraProfileBySource.get(source) !== camera_profile_t.HIGH) {
+                dataWidth /= 2;
+                dataHeight /= 2;
+                downSample = true;
+            }
             let clone = frame.clone();
             if (userFacing)
                 mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
             else
                 mat4.identity(cameraToDeviceTransform);
-            let focalLength = 300.0 * profile.dataWidth / 320.0;
+            let focalLength = 300.0 * dataWidth / 320.0;
             cameraModel[0] = focalLength;
             cameraModel[1] = focalLength;
-            cameraModel[2] = profile.dataWidth * 0.5;
-            cameraModel[3] = profile.dataHeight * 0.5;
+            cameraModel[2] = dataWidth * 0.5;
+            cameraModel[3] = dataHeight * 0.5;
             const ret = {
                 token: token,
                 d: clone,
@@ -253,8 +288,10 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
             const pipeline = server._pipeline_by_instance.get(p);
             if (pipeline) {
                 try {
-                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, (_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1);
+                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, (_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1, downSample);
                     handleMessages(mod, workerMessageAPI);
+                    if (dataDownloadAPI)
+                        handleDataDownload(mod, dataDownloadAPI);
                 }
                 catch (err) {
                     console.log('Exception during camera processing', err);
@@ -263,6 +300,32 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
                     // (mod as any).ccall('what_to_stderr', 'number', ['number'], [err]);
                 }
                 r.pipeline_frame_update(pipeline);
+                if (rawDataAccessEnabled) {
+                    let buffers = rawCameraFrameBuffersByPipeline.get(p);
+                    if (!buffers) {
+                        buffers = { available: [], ready: [] };
+                        rawCameraFrameBuffersByPipeline.set(p, buffers);
+                    }
+                    if (buffers.ready.length > 4) {
+                        const removed = buffers.ready.splice(0, 1);
+                        for (const entry of removed)
+                            buffers.available.push(new Uint8Array(entry.data.data));
+                    }
+                    const size = r.pipeline_camera_frame_data_raw_size(pipeline);
+                    let buffer;
+                    while (!buffer || buffer.byteLength < size) {
+                        if (buffers.available.length < 1)
+                            buffers.available.push(new Uint8Array(size));
+                        buffer = buffers.available.pop();
+                    }
+                    const dataPointer = r.pipeline_camera_frame_data_raw(pipeline);
+                    buffer.set(mod.HEAPU8.subarray(dataPointer, dataPointer + size));
+                    buffers.ready.push({ token, data: {
+                            data: buffer,
+                            width: r.pipeline_camera_frame_data_raw_width(p),
+                            height: r.pipeline_camera_frame_data_raw_height(p),
+                        } });
+                }
                 server.exploreState();
             }
             frame.close();
@@ -322,3 +385,12 @@ function handleMessages(mod, m) {
     }
     m.worker_message_send_clear();
 }
+function handleDataDownload(mod, d) {
+    const size = d.data_download_size();
+    if (size === 0)
+        return;
+    const dataPtr = d.data_download();
+    const data = mod.HEAPU8.slice(dataPtr, dataPtr + size);
+    messageManager.postOutgoingMessage({ t: '_z_datadownload', data }, [data.buffer]);
+    d.data_download_clear();
+}

package/lib/worker.js

@@ -7,9 +7,10 @@ messageManager.onOutgoingMessage.bind(() => {
     }
 });
 let launchHandler = (evt) => {
+    var _a;
     if (evt && evt.data && evt.data.t === "wasm") {
         let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zappar-cv.wasm", import.meta.url).toString();
-        launchWorkerServer(url, evt.data.module);
+        launchWorkerServer(url, evt.data.module, (_a = evt.data.shouldRecordData) !== null && _a !== void 0 ? _a : 0);
         ctx.removeEventListener("message", launchHandler);
     }
 };

package/lib/workerinterface.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="dom-webcodecs" />
-import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
+import { CameraFrameData } from "./camera-frame-data";
+import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
 export interface CameraFrameC2S {
     t: "cameraFrameC2S";
     d: ArrayBuffer;
@@ -38,6 +39,12 @@ export interface StreamC2S {
     userFacing: boolean;
     source: zappar_camera_source_t;
 }
+export interface CameraProfileC2S {
+    t: "cameraProfileC2S";
+    p: zappar_pipeline_t;
+    source: zappar_camera_source_t;
+    profile: camera_profile_t;
+}
 export interface CameraToScreenRotationC2S {
     t: "cameraToScreenC2S";
     p: zappar_pipeline_t;
@@ -52,6 +59,7 @@ export interface ImageBitmapC2S {
     tokenId: number;
     cameraModel: Float32Array;
     cameraToDevice: Float32Array;
+    cp: camera_profile_t;
 }
 export interface ImageBitmapS2C {
     t: "imageBitmapS2C";
@@ -83,3 +91,18 @@ export interface AttitudeMatrixC2S {
     p: zappar_pipeline_t;
     m: Float32Array;
 }
+export interface CameraDataRawEnabledC2S {
+    t: "rawenabled";
+    v: boolean;
+}
+export interface CameraDataRawRequestC2S {
+    t: "rawrequest";
+    p: zappar_pipeline_t;
+    token: number;
+}
+export interface CameraDataRawS2C {
+    t: "raw";
+    token: number;
+    p: zappar_pipeline_t;
+    data: CameraFrameData | null;
+}