@zappar/zappar-cv 0.3.13 → 0.4.0-beta.10

package/lib/worker-server.js

@@ -15,7 +15,11 @@ const zappar_cwrap_1 = require("./gen/zappar-cwrap");
 const zappar_server_1 = require("./gen/zappar-server");
 const messages_1 = require("./messages");
 const gl_matrix_1 = require("gl-matrix");
+const zappar_native_1 = require("./gen/zappar-native");
+const worker_imagebitmap_1 = require("./worker-imagebitmap");
+const profile_1 = require("./profile");
 exports.messageManager = new messages_1.MsgManager();
+let latestCameraToScreenRotation = 0;
 function launchWorkerServer(wasmUrl) {
     return __awaiter(this, void 0, void 0, function* () {
         let mod = ZNM.default({
@@ -45,27 +49,41 @@ function launchWorkerServer(wasmUrl) {
                         case "buf":
                             (_a = server.serializersByPipelineId.get(msg.p)) === null || _a === void 0 ? void 0 : _a.bufferReturn(msg.d);
                             break;
-                        case "cameraFrameC2S":
+                        case "cameraFrameC2S": {
                             let msgt = msg;
-                            // PROCESS DATA
-                            let cameraToDeviceTransform = gl_matrix_1.mat4.create();
-                            if (msgt.userFacing) {
-                                gl_matrix_1.mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
-                            }
                             let pipeline = server._pipeline_by_instance.get(msgt.p);
+                            let att;
                             if (pipeline) {
-                                r.pipeline_camera_frame_submit(pipeline, msgt.d, msgt.width, msgt.height, msgt.token, cameraToDeviceTransform);
+                                r.pipeline_camera_frame_submit(pipeline, msgt.d, msgt.width, msgt.height, msgt.token, msgt.c2d, msgt.cm, msgt.userFacing);
                                 r.pipeline_frame_update(pipeline);
+                                att = r.pipeline_camera_frame_device_attitude(pipeline);
                                 server.exploreState();
                             }
                             let ret = {
                                 token: msgt.token,
                                 d: msgt.d,
                                 p: msgt.p,
-                                t: "cameraFrameRecycleS2C"
+                                t: "cameraFrameRecycleS2C",
+                                att
                             };
                             exports.messageManager.postOutgoingMessage(ret, [msgt.d]);
                             break;
+                        }
+                        case "streamC2S": {
+                            let msgt = msg;
+                            consumeStream(mod, r, msgt.s, msgt.p, msgt.userFacing, server, msgt.source);
+                            break;
+                        }
+                        case "cameraToScreenC2S": {
+                            let msgt = msg;
+                            latestCameraToScreenRotation = msgt.r;
+                            break;
+                        }
+                        case "imageBitmapC2S": {
+                            let msgt = msg;
+                            worker_imagebitmap_1.handleImageBitmap(msgt, r, server, exports.messageManager);
+                            break;
+                        }
                     }
                 });
             }
@@ -74,3 +92,101 @@ function launchWorkerServer(wasmUrl) {
 }
 exports.launchWorkerServer = launchWorkerServer;
 ;
+let streamDataBufferPointer = 0;
+let streamDataBufferLength = 0;
+let tokenId = 1;
+const cameraToDeviceTransform = gl_matrix_1.mat4.create();
+const cameraModel = new Float32Array([300, 300, 160, 120, 0, 0]);
+function consumeStream(mod, r, stream, p, userFacing, server, source) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const reader = yield stream.getReader();
+        while (true) {
+            let result = yield reader.read();
+            if (result.done) {
+                console.log("Stream done");
+                return;
+            }
+            let frame = result.value;
+            let size = frame.allocationSize();
+            if (size > streamDataBufferLength) {
+                if (streamDataBufferPointer > 0)
+                    mod._free(streamDataBufferPointer);
+                streamDataBufferPointer = mod._malloc(size);
+                streamDataBufferLength = size;
+            }
+            yield frame.copyTo(mod.HEAPU8.subarray(streamDataBufferPointer, streamDataBufferPointer + streamDataBufferLength));
+            let token = tokenId;
+            tokenId++;
+            const width = frame.visibleRect.width;
+            const height = frame.visibleRect.height;
+            let uvTransform;
+            let dataWidth = width;
+            let dataHeight = height;
+            switch (latestCameraToScreenRotation) {
+                case 270:
+                    uvTransform = new Float32Array([0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1]);
+                    dataWidth = height;
+                    dataHeight = width;
+                    break;
+                case 180:
+                    uvTransform = new Float32Array([-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1]);
+                    break;
+                case 90:
+                    uvTransform = new Float32Array([0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1]);
+                    dataWidth = height;
+                    dataHeight = width;
+                    break;
+                default:
+                    uvTransform = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);
+                    break;
+            }
+            let clone = frame.clone();
+            if (userFacing)
+                gl_matrix_1.mat4.fromScaling(cameraToDeviceTransform, [-1, 1, -1]);
+            else
+                gl_matrix_1.mat4.identity(cameraToDeviceTransform);
+            let focalLength = 300.0 * profile_1.profile.dataWidth / 320.0;
+            cameraModel[0] = focalLength;
+            cameraModel[1] = focalLength;
+            cameraModel[2] = profile_1.profile.dataWidth * 0.5;
+            cameraModel[3] = profile_1.profile.dataHeight * 0.5;
+            const ret = {
+                token: token,
+                d: clone,
+                p: p,
+                t: "videoFrameS2C",
+                userFacing,
+                uvTransform,
+                w: dataWidth,
+                h: dataHeight,
+                cameraToDevice: cameraToDeviceTransform,
+                cameraModel,
+                source
+            };
+            exports.messageManager.postOutgoingMessage(ret, [ret.d, ret.uvTransform.buffer]);
+            const pipeline = server._pipeline_by_instance.get(p);
+            if (pipeline) {
+                r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing);
+                r.pipeline_frame_update(pipeline);
+                server.exploreState();
+            }
+            frame.close();
+        }
+    });
+}
+function framePixelFormatFromFormat(f) {
+    switch (f) {
+        case "I420": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420;
+        case "I420A": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I420A;
+        case "I422": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I422;
+        case "I444": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_I444;
+        case "NV12": return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_NV12;
+        case "RGBA":
+        case "RGBX":
+            return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_RGBA;
+        case "BGRA":
+        case "BGRX":
+            return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_BGRA;
+    }
+    return zappar_native_1.frame_pixel_format_t.FRAME_PIXEL_FORMAT_Y;
+}

package/lib/workerinterface.d.ts

@@ -1,9 +1,12 @@
-import { zappar_pipeline_t } from "./gen/zappar-native";
+/// <reference types="dom-webcodecs" />
+import { zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
 export interface CameraFrameC2S {
     t: "cameraFrameC2S";
     d: ArrayBuffer;
     p: zappar_pipeline_t;
     userFacing: boolean;
+    c2d: Float32Array;
+    cm: Float32Array;
     token: number;
     width: number;
     height: number;
@@ -13,4 +16,50 @@ export interface CameraFrameReturnS2C {
     token: number;
     d: ArrayBuffer;
     p: zappar_pipeline_t;
+    att?: Float32Array;
+}
+export interface VideoFrameS2C {
+    t: "videoFrameS2C";
+    token: number;
+    d: VideoFrame;
+    p: zappar_pipeline_t;
+    source: zappar_camera_source_t;
+    userFacing: boolean;
+    uvTransform: Float32Array;
+    w: number;
+    h: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+}
+export interface StreamC2S {
+    t: "streamC2S";
+    p: zappar_pipeline_t;
+    s: ReadableStream<VideoFrame>;
+    userFacing: boolean;
+    source: zappar_camera_source_t;
+}
+export interface CameraToScreenRotationC2S {
+    t: "cameraToScreenC2S";
+    p: zappar_pipeline_t;
+    r: number;
+}
+export interface ImageBitmapC2S {
+    t: "imageBitmapC2S";
+    p: zappar_pipeline_t;
+    i: ImageBitmap;
+    r: number;
+    userFacing: boolean;
+    tokenId: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+}
+export interface ImageBitmapS2C {
+    t: "imageBitmapS2C";
+    p: zappar_pipeline_t;
+    dataWidth: number;
+    dataHeight: number;
+    uvTransform: Float32Array;
+    userFacing: boolean;
+    frame: ImageBitmap;
+    tokenId: number;
 }