@zappar/zappar-cv 0.4.0-beta.4 → 2.0.0-beta.0

package/lib/worker-server.js

@@ -1,4 +1,3 @@
-"use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,28 +7,17 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.launchWorkerServer = exports.messageManager = void 0;
-const ZNM = require("./zcv");
-const zappar_cwrap_1 = require("./gen/zappar-cwrap");
-const zappar_server_1 = require("./gen/zappar-server");
-const messages_1 = require("./messages");
-exports.messageManager = new messages_1.MsgManager();
-let bundleWorkerPort;
-self.addEventListener("message", msg => {
-    if (msg && msg.data && msg.data.t === "port") {
-        bundleWorkerPort = msg.data.port;
-    }
-});
-self.bundle_adjust_message = function (buf) {
-    var buf_copy = new ArrayBuffer(buf.byteLength);
-    new Uint8Array(buf_copy).set(new Uint8Array(buf));
-    bundleWorkerPort === null || bundleWorkerPort === void 0 ? void 0 : bundleWorkerPort.postMessage({
-        t: "bundle_adjust_message",
-        buf: buf_copy
-    }, [buf_copy]);
-};
-function launchWorkerServer(wasmUrl) {
+import * as ZNM from "./zcv";
+import { getRuntimeObject } from "./gen/zappar-cwrap";
+import { zappar_server } from "./gen/zappar-server";
+import { MsgManager } from "./messages";
+import { mat4 } from "gl-matrix";
+import { frame_pixel_format_t } from "./gen/zappar-native";
+import { handleImageBitmap } from "./worker-imagebitmap";
+import { profile } from "./profile";
+export let messageManager = new MsgManager();
+let latestCameraToScreenRotation = 0;
+export function launchWorkerServer(wasmUrl) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
             locateFile: (path, prefix) => {
@@ -39,35 +27,26 @@ function launchWorkerServer(wasmUrl) {
                 return prefix + path;
             },
             onRuntimeInitialized: () => {
-                let r = zappar_cwrap_1.getRuntimeObject(mod);
-                let server = new zappar_server_1.zappar_server(r, (pipelineId, ab) => {
-                    exports.messageManager.postOutgoingMessage({
+                let r = getRuntimeObject(mod);
+                let server = new zappar_server(r, (pipelineId, ab) => {
+                    messageManager.postOutgoingMessage({
                         p: pipelineId,
                         t: "zappar",
                         d: ab
                     }, [ab]);
                 });
-                exports.messageManager.postOutgoingMessage("loaded", []);
-                bundleWorkerPort.onmessage = function (msg) {
-                    // Received message from other worker
-                    var a_buf = msg.data.buf;
-                    var buf = mod._malloc(a_buf.byteLength);
-                    mod.HEAPU8.set(new Uint8Array(a_buf), buf);
-                    /// call the exported native function to update the world map
-                    mod._bundle_adjust_message_callback(buf, msg.data.buf.byteLength, 0);
-                    mod._free(buf);
-                };
-                exports.messageManager.onIncomingMessage.bind((msg) => {
+                messageManager.postOutgoingMessage("loaded", []);
+                messageManager.onIncomingMessage.bind((msg) => {
                     var _a;
                     switch (msg.t) {
                         case "zappar":
                             server.processBuffer(msg.d);
-                            exports.messageManager.postOutgoingMessage({ t: "buf", d: msg.d }, [msg.d]);
+                            messageManager.postOutgoingMessage({ t: "buf", d: msg.d }, [msg.d]);
                             break;
                         case "buf":
                             (_a = server.serializersByPipelineId.get(msg.p)) === null || _a === void 0 ? void 0 : _a.bufferReturn(msg.d);
                             break;
-                        case "cameraFrameC2S":
+                        case "cameraFrameC2S": {
                             let msgt = msg;
                             let pipeline = server._pipeline_by_instance.get(msgt.p);
                             let att;
@@ -84,13 +63,126 @@ function launchWorkerServer(wasmUrl) {
                                 t: "cameraFrameRecycleS2C",
                                 att
                             };
-                            exports.messageManager.postOutgoingMessage(ret, [msgt.d]);
+                            messageManager.postOutgoingMessage(ret, [msgt.d]);
                             break;
+                        }
+                        case "streamC2S": {
+                            let msgt = msg;
+                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source);
+                            break;
+                        }
+                        case "cameraToScreenC2S": {
+                            let msgt = msg;
+                            latestCameraToScreenRotation = msgt.r;
+                            break;
+                        }
+                        case "imageBitmapC2S": {
+                            let msgt = msg;
+                            handleImageBitmap(msgt, r, server, messageManager);
+                            break;
+                        }
                     }
                 });
             }
         });
     });
 }
-exports.launchWorkerServer = launchWorkerServer;
 ;
+let streamDataBufferPointer = 0;
+let streamDataBufferLength = 0;
+let tokenId = 1;
+const cameraToDeviceTransform = mat4.create();
+const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+function consumeStream(mod, r, stream, p, userFacing, server, source) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const reader = yield stream.getReader();
+        while (true) {
+            let result = yield reader.read();
+            if (result.done) {
+                console.log("Stream done");
+                return;
+            }
+            let frame = result.value;
+            let size = frame.allocationSize();
+            if (size > streamDataBufferLength) {
+                if (streamDataBufferPointer > 0)
+                    mod._free(streamDataBufferPointer);
+                streamDataBufferPointer = mod._malloc(size);
+                streamDataBufferLength = size;
+            }
+            yield frame.copyTo(mod.HEAPU8.subarray(streamDataBufferPointer, streamDataBufferPointer + streamDataBufferLength));
+            let token = tokenId;
+            tokenId++;
+            const width = frame.visibleRect.width;
+            const height = frame.visibleRect.height;
+            let uvTransform;
+            let dataWidth = width;
+            let dataHeight = height;
+            switch (latestCameraToScreenRotation) {
+                case 270:
+                    uvTransform = new Float32Array([0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1]);
+                    dataWidth = height;
+                    dataHeight = width;
+                    break;
+                case 180:
+                    uvTransform = new Float32Array([-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1]);
+                    break;
+                case 90:
+                    uvTransform = new Float32Array([0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1]);
+                    dataWidth = height;
+                    dataHeight = width;
+                    break;
+                default:
+                    uvTransform = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
+                    break;
+            }
+            let clone = frame.clone();
+            if (userFacing)
+                mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
+            else
+                mat4.identity(cameraToDeviceTransform);
+            let focalLength = 300.0 * profile.dataWidth / 320.0;
+            cameraModel[0] = focalLength;
+            cameraModel[1] = focalLength;
+            cameraModel[2] = profile.dataWidth * 0.5;
+            cameraModel[3] = profile.dataHeight * 0.5;
+            const ret = {
+                token: token,
+                d: clone,
+                p: p,
+                t: "videoFrameS2C",
+                userFacing,
+                uvTransform,
+                w: dataWidth,
+                h: dataHeight,
+                cameraToDevice: cameraToDeviceTransform,
+                cameraModel,
+                source
+            };
+            messageManager.postOutgoingMessage(ret, [ret.d, ret.uvTransform.buffer]);
+            const pipeline = server._pipeline_by_instance.get(p);
+            if (pipeline) {
+                r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                r.pipeline_frame_update(pipeline);
+                server.exploreState();
+            }
+            frame.close();
+        }
+    });
+}
+function framePixelFormatFromFormat(f) {
+    switch (f) {
+        case "I420": return frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420;
+        case "I420A": return frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420A;
+        case "I422": return frame_pixel_format_t.FRAME_PIXEL_FORMAT_I422;
+        case "I444": return frame_pixel_format_t.FRAME_PIXEL_FORMAT_I444;
+        case "NV12": return frame_pixel_format_t.FRAME_PIXEL_FORMAT_NV12;
+        case "RGBA":
+        case "RGBX":
+            return frame_pixel_format_t.FRAME_PIXEL_FORMAT_RGBA;
+        case "BGRA":
+        case "BGRX":
+            return frame_pixel_format_t.FRAME_PIXEL_FORMAT_BGRA;
+    }
+    return frame_pixel_format_t.FRAME_PIXEL_FORMAT_Y;
+}

package/lib/worker.js

@@ -1,23 +1,19 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const worker_server_1 = require("./worker-server");
+import { launchWorkerServer, messageManager } from "./worker-server";
 const ctx = self;
-worker_server_1.messageManager.onOutgoingMessage.bind(() => {
-    let msgs = worker_server_1.messageManager.getOutgoingMessages();
+messageManager.onOutgoingMessage.bind(() => {
+    let msgs = messageManager.getOutgoingMessages();
     for (let msg of msgs) {
         ctx.postMessage(msg.msg, msg.transferables);
     }
 });
 let launchHandler = (evt) => {
     if (evt && evt.data && evt.data.t === "wasm") {
-        let url = location.href.startsWith("blob") ? evt.data.url : require("./zcv.wasm");
-        if (url.default)
-            url = url.default;
-        worker_server_1.launchWorkerServer(url);
+        let url = location.href.startsWith("blob") ? evt.data.url : new URL("./zcv.wasm", import.meta.url).toString();
+        launchWorkerServer(url);
         ctx.removeEventListener("message", launchHandler);
     }
 };
 ctx.addEventListener("message", launchHandler);
 ctx.addEventListener("message", evt => {
-    worker_server_1.messageManager.postIncomingMessage(evt.data);
+    messageManager.postIncomingMessage(evt.data);
 });

package/lib/workerinterface.d.ts

@@ -1,4 +1,5 @@
-import { zappar_pipeline_t } from "./gen/zappar-native";
+/// <reference types="dom-webcodecs" />
+import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
 export interface CameraFrameC2S {
     t: "cameraFrameC2S";
     d: ArrayBuffer;
@@ -17,3 +18,48 @@ export interface CameraFrameReturnS2C {
     p: zappar_pipeline_t;
     att?: Float32Array;
 }
+export interface VideoFrameS2C {
+    t: "videoFrameS2C";
+    token: number;
+    d: VideoFrame;
+    p: zappar_pipeline_t;
+    source: zappar_camera_source_t;
+    userFacing: boolean;
+    uvTransform: Float32Array;
+    w: number;
+    h: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+}
+export interface StreamC2S {
+    t: "streamC2S";
+    p: zappar_pipeline_t;
+    s: ReadableStream<VideoFrame>;
+    userFacing: boolean;
+    source: zappar_camera_source_t;
+}
+export interface CameraToScreenRotationC2S {
+    t: "cameraToScreenC2S";
+    p: zappar_pipeline_t;
+    r: number;
+}
+export interface ImageBitmapC2S {
+    t: "imageBitmapC2S";
+    p: zappar_pipeline_t;
+    i: ImageBitmap;
+    r: number;
+    userFacing: boolean;
+    tokenId: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+}
+export interface ImageBitmapS2C {
+    t: "imageBitmapS2C";
+    p: zappar_pipeline_t;
+    dataWidth: number;
+    dataHeight: number;
+    uvTransform: Float32Array;
+    userFacing: boolean;
+    frame: ImageBitmap;
+    tokenId: number;
+}

package/lib/workerinterface.js

@@ -1,2 +1 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
+export {};