@zappar/zappar-cv 0.4.0-beta.1 → 0.4.0-beta.10

package/lib/worker-imagebitmap.d.ts

@@ -0,0 +1,5 @@
+import { zappar_cwrap } from "./gen/zappar-native";
+import { zappar_server } from "./gen/zappar-server";
+import { MsgManager } from "./messages";
+import { ImageBitmapC2S } from "./workerinterface";
+export declare function handleImageBitmap(m: ImageBitmapC2S, r: zappar_cwrap, server: zappar_server, mgr: MsgManager): void;

package/lib/worker-imagebitmap.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleImageBitmap = void 0;
+const image_process_gl_1 = require("./image-process-gl");
+const profile_1 = require("./profile");
+const gl_matrix_1 = require("gl-matrix");
+let pixels;
+let texture;
+let _imageProcessor;
+let _gl;
+function getImageProcessor() {
+    if (!_imageProcessor || !_gl) {
+        const canvas = new OffscreenCanvas(1, 1);
+        _gl = canvas.getContext("webgl");
+        if (!_gl)
+            throw new Error("Unable to get offscreen GL context");
+        _imageProcessor = new image_process_gl_1.ImageProcessGL(_gl);
+    }
+    return [_imageProcessor, _gl];
+}
+function handleImageBitmap(m, r, server, mgr) {
+    // img: ImageBitmap,
+    //                                 rot: number,
+    //                                 tokenId: number,
+    //                                 r: zappar_cwrap,
+    //                                 p: zappar_pipeline_t,
+    //                                 userFacing: boolean,
+    //                                 server: zappar_server,
+    //                                 mgr: MsgManager) {
+    const [imageProcessor, gl] = getImageProcessor();
+    if (!texture) {
+        texture = gl.createTexture();
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    }
+    if (!texture)
+        return;
+    if (!pixels || pixels.byteLength < profile_1.profile.dataWidth * profile_1.profile.dataHeight) {
+        console.log("Generating pixel buffer", profile_1.profile.dataWidth * profile_1.profile.dataHeight);
+        pixels = new ArrayBuffer(profile_1.profile.dataWidth * profile_1.profile.dataHeight);
+    }
+    imageProcessor.uploadFrame(texture, m.i, m.r, m.userFacing);
+    let info = imageProcessor.readFrame(texture, pixels);
+    let msg = {
+        t: "imageBitmapS2C",
+        dataWidth: info.dataWidth,
+        dataHeight: info.dataHeight,
+        frame: m.i,
+        userFacing: info.userFacing,
+        uvTransform: info.uvTransform || gl_matrix_1.mat4.create(),
+        tokenId: m.tokenId, p: m.p
+    };
+    mgr.postOutgoingMessage(msg, [m.i]);
+    let pipeline = server._pipeline_by_instance.get(m.p);
+    if (pipeline) {
+        r.pipeline_camera_frame_submit(pipeline, pixels, info.dataWidth, info.dataHeight, m.tokenId, m.cameraToDevice, m.cameraModel, info.userFacing);
+        r.pipeline_frame_update(pipeline);
+        server.exploreState();
+    }
+}
+exports.handleImageBitmap = handleImageBitmap;

package/lib/worker-server.d.ts

@@ -1,8 +1,3 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
-declare global {
-    interface Window {
-        send_ba_request: any;
-    }
-}
 export declare function launchWorkerServer(wasmUrl: string): Promise<void>;

package/lib/worker-server.js

@@ -14,24 +14,12 @@ const ZNM = require("./zcv");
 const zappar_cwrap_1 = require("./gen/zappar-cwrap");
 const zappar_server_1 = require("./gen/zappar-server");
 const messages_1 = require("./messages");
+const gl_matrix_1 = require("gl-matrix");
+const zappar_native_1 = require("./gen/zappar-native");
+const worker_imagebitmap_1 = require("./worker-imagebitmap");
+const profile_1 = require("./profile");
 exports.messageManager = new messages_1.MsgManager();
-// let bundleWorkerPort : MessagePort;
-self.addEventListener("message", msg => {
-    if (msg && msg.data && msg.data.t === "port") {
-        // console.log("Got bundle worker port in zcv worker");
-        // bundleWorkerPort = msg.data.port;
-    }
-});
-/// export a function accisible from EM_ASM to post a bundle adjustment request data message
-self.send_ba_request = function (buf) {
-    var buf_copy = new ArrayBuffer(buf.byteLength);
-    new Uint8Array(buf_copy).set(new Uint8Array(buf));
-    // console.log("send_ba_request", bundleWorkerPort);
-    // bundleWorkerPort?.postMessage({
-    //     t: "send_ba_request",
-    //     buf: buf_copy
-    // }, [buf_copy]);
-};
+let latestCameraToScreenRotation = 0;
 function launchWorkerServer(wasmUrl) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
@@ -51,18 +39,6 @@ function launchWorkerServer(wasmUrl) {
                     }, [ab]);
                 });
                 exports.messageManager.postOutgoingMessage("loaded", []);
-                /// addEventListener doesn't seem to work on ports
-                // bundleWorkerPort.onmessage = function (msg) {
-                //     console.log("Message from bundle worker", msg);
-                //     // Received message from other worker
-                //     var a_buf = msg.data.buf;
-                //     var buf = mod._malloc(a_buf.byteLength);
-                //     mod.HEAPU8.set(new Uint8Array(a_buf), buf);
-                //     console.log(mod, r);
-                //     /// call the exported native function to update the world map
-                //     mod._ba_response_update(buf, msg.data.buf.byteLength);
-                //     mod._free(buf);
-                // }
                 exports.messageManager.onIncomingMessage.bind((msg) => {
                     var _a;
                     switch (msg.t) {
@@ -73,7 +49,7 @@ function launchWorkerServer(wasmUrl) {
                         case "buf":
                             (_a = server.serializersByPipelineId.get(msg.p)) === null || _a === void 0 ? void 0 : _a.bufferReturn(msg.d);
                             break;
-                        case "cameraFrameC2S":
+                        case "cameraFrameC2S": {
                             let msgt = msg;
                             let pipeline = server._pipeline_by_instance.get(msgt.p);
                             let att;
@@ -92,6 +68,22 @@ function launchWorkerServer(wasmUrl) {
                             };
                             exports.messageManager.postOutgoingMessage(ret, [msgt.d]);
                             break;
+                        }
+                        case "streamC2S": {
+                            let msgt = msg;
+                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source);
+                            break;
+                        }
+                        case "cameraToScreenC2S": {
+                            let msgt = msg;
+                            latestCameraToScreenRotation = msgt.r;
+                            break;
+                        }
+                        case "imageBitmapC2S": {
+                            let msgt = msg;
+                            worker_imagebitmap_1.handleImageBitmap(msgt, r, server, exports.messageManager);
+                            break;
+                        }
                     }
                 });
             }
@@ -100,3 +92,101 @@ function launchWorkerServer(wasmUrl) {
 }
 exports.launchWorkerServer = launchWorkerServer;
 ;
+let streamDataBufferPointer = 0;
+let streamDataBufferLength = 0;
+let tokenId = 1;
+const cameraToDeviceTransform = gl_matrix_1.mat4.create();
+const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+function consumeStream(mod, r, stream, p, userFacing, server, source) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const reader = yield stream.getReader();
+        while (true) {
+            let result = yield reader.read();
+            if (result.done) {
+                console.log("Stream done");
+                return;
+            }
+            let frame = result.value;
+            let size = frame.allocationSize();
+            if (size > streamDataBufferLength) {
+                if (streamDataBufferPointer > 0)
+                    mod._free(streamDataBufferPointer);
+                streamDataBufferPointer = mod._malloc(size);
+                streamDataBufferLength = size;
+            }
+            yield frame.copyTo(mod.HEAPU8.subarray(streamDataBufferPointer, streamDataBufferPointer + streamDataBufferLength));
+            let token = tokenId;
+            tokenId++;
+            const width = frame.visibleRect.width;
+            const height = frame.visibleRect.height;
+            let uvTransform;
+            let dataWidth = width;
+            let dataHeight = height;
+            switch (latestCameraToScreenRotation) {
+                case 270:
+                    uvTransform = new Float32Array([0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1]);
+                    dataWidth = height;
+                    dataHeight = width;
+                    break;
+                case 180:
+                    uvTransform = new Float32Array([-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1]);
+                    break;
+                case 90:
+                    uvTransform = new Float32Array([0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1]);
+                    dataWidth = height;
+                    dataHeight = width;
+                    break;
+                default:
+                    uvTransform = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
+                    break;
+            }
+            let clone = frame.clone();
+            if (userFacing)
+                gl_matrix_1.mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
+            else
+                gl_matrix_1.mat4.identity(cameraToDeviceTransform);
+            let focalLength = 300.0 * profile_1.profile.dataWidth / 320.0;
+            cameraModel[0] = focalLength;
+            cameraModel[1] = focalLength;
+            cameraModel[2] = profile_1.profile.dataWidth * 0.5;
+            cameraModel[3] = profile_1.profile.dataHeight * 0.5;
+            const ret = {
+                token: token,
+                d: clone,
+                p: p,
+                t: "videoFrameS2C",
+                userFacing,
+                uvTransform,
+                w: dataWidth,
+                h: dataHeight,
+                cameraToDevice: cameraToDeviceTransform,
+                cameraModel,
+                source
+            };
+            exports.messageManager.postOutgoingMessage(ret, [ret.d, ret.uvTransform.buffer]);
+            const pipeline = server._pipeline_by_instance.get(p);
+            if (pipeline) {
+                r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                r.pipeline_frame_update(pipeline);
+                server.exploreState();
+            }
+            frame.close();
+        }
+    });
+}
+function framePixelFormatFromFormat(f) {
+    switch (f) {
+        case "I420": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420;
+        case "I420A": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420A;
+        case "I422": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I422;
+        case "I444": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I444;
+        case "NV12": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_NV12;
+        case "RGBA":
+        case "RGBX":
+            return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_RGBA;
+        case "BGRA":
+        case "BGRX":
+            return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_BGRA;
+    }
+    return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_Y;
+}

package/lib/workerinterface.d.ts

@@ -1,4 +1,5 @@
-import { zappar_pipeline_t } from "./gen/zappar-native";
+/// <reference types="dom-webcodecs" />
+import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
 export interface CameraFrameC2S {
     t: "cameraFrameC2S";
     d: ArrayBuffer;
@@ -17,3 +18,48 @@ export interface CameraFrameReturnS2C {
     p: zappar_pipeline_t;
     att?: Float32Array;
 }
+export interface VideoFrameS2C {
+    t: "videoFrameS2C";
+    token: number;
+    d: VideoFrame;
+    p: zappar_pipeline_t;
+    source: zappar_camera_source_t;
+    userFacing: boolean;
+    uvTransform: Float32Array;
+    w: number;
+    h: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+}
+export interface StreamC2S {
+    t: "streamC2S";
+    p: zappar_pipeline_t;
+    s: ReadableStream<VideoFrame>;
+    userFacing: boolean;
+    source: zappar_camera_source_t;
+}
+export interface CameraToScreenRotationC2S {
+    t: "cameraToScreenC2S";
+    p: zappar_pipeline_t;
+    r: number;
+}
+export interface ImageBitmapC2S {
+    t: "imageBitmapC2S";
+    p: zappar_pipeline_t;
+    i: ImageBitmap;
+    r: number;
+    userFacing: boolean;
+    tokenId: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+}
+export interface ImageBitmapS2C {
+    t: "imageBitmapS2C";
+    p: zappar_pipeline_t;
+    dataWidth: number;
+    dataHeight: number;
+    uvTransform: Float32Array;
+    userFacing: boolean;
+    frame: ImageBitmap;
+    tokenId: number;
+}