@zappar/zappar-cv 3.3.0-beta.1 → 3.4.0-beta.0

package/lib/src/shader.d.ts

@@ -0,0 +1,2 @@
+export declare function compileShader(gl: WebGLRenderingContext, type: number, src: string): WebGLShader;
+export declare function linkProgram(gl: WebGLRenderingContext, prog: WebGLProgram): void;

package/lib/src/source.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="dom-webcodecs" />
+export interface CameraFrameInfo {
+    rgbFrame?: ImageBitmap | VideoFrame | ImageData;
+    data?: ArrayBuffer;
+    texture: WebGLTexture | undefined;
+    dataWidth: number;
+    dataHeight: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+    cameraSource: Source;
+    cameraSourceData?: any;
+    uvTransform?: Float32Array;
+    userFacing: boolean;
+    remoteToken?: number;
+    frame?: VideoFrame | ImageBitmap;
+}
+export declare abstract class Source {
+    abstract getFrame(currentlyProcessing: boolean): void;
+    abstract pause(): void;
+    abstract uploadGL(info: CameraFrameInfo): void;
+    recycle?(info: CameraFrameInfo): void;
+}

package/lib/src/tr.d.ts

@@ -0,0 +1,6 @@
+export declare enum STRINGS {
+    PermissionTitle = 0,
+    PermissionDescription = 1,
+    PermissionButton = 2
+}
+export declare function tr(str: STRINGS): string;

package/lib/src/tracing/jsontracer.d.ts

@@ -0,0 +1,14 @@
+import { PerformanceTracer } from "./performancetracer";
+export declare class JSONTracer implements PerformanceTracer {
+    private stackDepth;
+    private events;
+    private static beginTemplate;
+    private static endTemplate;
+    private static namingEvents;
+    constructor();
+    clear(): void;
+    toJSON(): string;
+    postJSON(url: string): void;
+    start(name: string): void;
+    end(): void;
+}

package/lib/src/tracing/performancetracer.d.ts

@@ -0,0 +1,11 @@
+export interface PerformanceTracer {
+    clear: () => void;
+    start: (name: string) => void;
+    end: () => void;
+}
+export declare class PerformanceTracerManager implements PerformanceTracer {
+    tracers: PerformanceTracer[];
+    clear(): void;
+    start(name: string): void;
+    end(): void;
+}

package/lib/src/tracing/sharedtracer.d.ts

@@ -0,0 +1,2 @@
+import { PerformanceTracerManager } from "./performancetracer";
+export declare function getSharedTracer(): PerformanceTracerManager;

package/lib/src/tracing/usertimingtracer.d.ts

@@ -0,0 +1,10 @@
+import { PerformanceTracer } from "./performancetracer";
+export declare class UserTimingTracer implements PerformanceTracer {
+    private useFixedName;
+    private markCount;
+    private traceStack;
+    constructor(useFixedName?: boolean);
+    clear(): void;
+    start(name: string): void;
+    end(): void;
+}

package/lib/src/version.d.ts

@@ -0,0 +1 @@
+export declare const VERSION = "3.4.0-beta.0";

package/lib/src/worker-client.d.ts

@@ -0,0 +1,4 @@
+import { MsgManager } from "./messages";
+export declare let messageManager: MsgManager;
+export declare function launchWorker(worker?: Worker): Promise<void>;
+export declare function launchCeresWorkers(port0: MessagePort, port1: MessagePort, worker0?: Worker, worker1?: Worker): Promise<void>;

package/lib/src/worker-imagebitmap.d.ts

@@ -0,0 +1,5 @@
+import { zappar_cwrap } from "./gen/zappar-native";
+import { zappar_server } from "./gen/zappar-server";
+import { MsgManager } from "./messages";
+import { ImageBitmapC2S } from "./workerinterface";
+export declare function handleImageBitmap(m: ImageBitmapC2S, r: zappar_cwrap, server: zappar_server, mgr: MsgManager): void;

package/lib/src/worker-messages.d.ts

@@ -0,0 +1,11 @@
+import type * as ZNM from "./zappar-cv";
+export interface WorkerMessageAPI {
+    worker_message_send_count: () => number;
+    worker_message_send_clear: () => void;
+    worker_message_send_data_size: (index: number) => number;
+    worker_message_send_data: (index: number) => number;
+    worker_message_send_reference: (index: number) => number;
+    worker_message_send_instance: (index: number) => number;
+    worker_message_receive: (ref: number, size: number, data: number, instance: number) => void;
+}
+export declare function getWorkerMessageAPI(mod: ZNM.Module): WorkerMessageAPI;

package/lib/src/worker-server.d.ts

@@ -0,0 +1,3 @@
+import { MsgManager } from "./messages";
+export declare let messageManager: MsgManager;
+export declare function launchWorkerServer(wasmUrl: string, module: WebAssembly.Module, shouldRecordData: number): Promise<void>;

package/lib/src/worker.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/src/workerinterface.d.ts

@@ -0,0 +1,110 @@
+/// <reference types="dom-webcodecs" />
+import { CameraFrameData } from "./camera-frame-data";
+import { camera_profile_t, zappar_camera_source_t, zappar_pipeline_t } from "./gen/zappar-native";
+export interface CameraFrameC2S {
+    t: "cameraFrameC2S";
+    d: ArrayBuffer;
+    p: zappar_pipeline_t;
+    userFacing: boolean;
+    c2d: Float32Array;
+    cm: Float32Array;
+    token: number;
+    width: number;
+    height: number;
+    captureTime: number;
+}
+export interface CameraFrameReturnS2C {
+    t: "cameraFrameRecycleS2C";
+    token: number;
+    d: ArrayBuffer;
+    p: zappar_pipeline_t;
+    att?: Float32Array;
+}
+export interface VideoFrameS2C {
+    t: "videoFrameS2C";
+    token: number;
+    d: VideoFrame;
+    p: zappar_pipeline_t;
+    source: zappar_camera_source_t;
+    userFacing: boolean;
+    uvTransform: Float32Array;
+    w: number;
+    h: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+}
+export interface StreamC2S {
+    t: "streamC2S";
+    p: zappar_pipeline_t;
+    s: ReadableStream<VideoFrame>;
+    userFacing: boolean;
+    source: zappar_camera_source_t;
+}
+export interface CameraProfileC2S {
+    t: "cameraProfileC2S";
+    p: zappar_pipeline_t;
+    source: zappar_camera_source_t;
+    profile: camera_profile_t;
+}
+export interface CameraToScreenRotationC2S {
+    t: "cameraToScreenC2S";
+    p: zappar_pipeline_t;
+    r: number;
+}
+export interface ImageBitmapC2S {
+    t: "imageBitmapC2S";
+    p: zappar_pipeline_t;
+    i: ImageBitmap;
+    r: number;
+    userFacing: boolean;
+    tokenId: number;
+    cameraModel: Float32Array;
+    cameraToDevice: Float32Array;
+    cp: camera_profile_t;
+}
+export interface ImageBitmapS2C {
+    t: "imageBitmapS2C";
+    p: zappar_pipeline_t;
+    dataWidth: number;
+    dataHeight: number;
+    uvTransform: Float32Array;
+    userFacing: boolean;
+    frame: ImageBitmap;
+    tokenId: number;
+}
+export interface StreamEndedS2C {
+    t: "streamEndedS2C";
+    p: zappar_pipeline_t;
+    source: zappar_camera_source_t;
+}
+export interface SensorDataC2S {
+    t: "sensorDataC2S";
+    sensor: "accel" | "accel_wo_gravity_int" | "accel_w_gravity_int" | "rotation_rate_int" | "attitude_int" | "rotation_rate" | "attitude" | "relative_orientation";
+    p: zappar_pipeline_t;
+    x: number;
+    y: number;
+    z: number;
+    w?: number;
+    timestamp: number;
+    interval: number;
+}
+export interface AttitudeMatrixC2S {
+    t: "attitudeMatrixC2S";
+    p: zappar_pipeline_t;
+    m: Float32Array;
+}
+export interface CameraDataRawEnabledC2S {
+    t: "rawenabled";
+    v: boolean;
+}
+export interface CameraDataRawRequestC2S {
+    t: "rawrequest";
+    p: zappar_pipeline_t;
+    token: number;
+}
+export interface CameraDataRawS2C {
+    t: "raw";
+    token: number;
+    p: zappar_pipeline_t;
+    data: CameraFrameData | null;
+}

package/lib/src/yuv-conversion-gl.d.ts

@@ -0,0 +1,44 @@
+export interface PlaneData {
+    width: number;
+    height: number;
+    data: Uint8Array;
+}
+export interface UvPlaneData {
+    width: number;
+    height: number;
+    data: Uint8Array;
+    dataVPlane?: Uint8Array;
+}
+export declare enum UvLayout {
+    INTERLEAVED_UV = 0,
+    INTERLEAVED_VU = 1,
+    PLANAR_UV = 2
+}
+export declare class YUVConversionGL {
+    private _gl;
+    private _yTexture;
+    private _uvTexture;
+    private _uvTextureSinglePlane;
+    private _vTexture;
+    private _framebufferId;
+    private _vao;
+    private _vertexBuffer;
+    private _conversionShaders;
+    private _isWebGL2;
+    private _useVao;
+    private _oneComponentInternalFormat;
+    private _oneComponentFormat;
+    private _twoComponentInternalFormat;
+    private _twoComponentFormat;
+    private _vaoExtension;
+    private _instancedArraysExtension;
+    constructor(_gl: WebGL2RenderingContext | WebGLRenderingContext);
+    resetGLContext(): void;
+    destroy(): void;
+    yuvToTexture(y: PlaneData, uv: UvPlaneData, uvLayout: UvLayout, destination: WebGLTexture): void;
+    private _getVertexAttribState;
+    private _prepareContext;
+    private _prepareConversionShader;
+    private _prepareTextures;
+    private _createSizedTexture;
+}

package/lib/tests/barcode-finder.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/camera-data-raw.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/camera-data-rgb.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/camera-far-near.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/camera-only.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/common.d.ts

@@ -0,0 +1,15 @@
+import { Zappar, zappar_pipeline_t } from "../src";
+import { zappar_sequence_source_t } from "../src/gen/zappar";
+import { Event1 } from "../src/event";
+export declare let canvas: HTMLCanvasElement;
+export declare let mirrorPoses: boolean;
+export declare let pipeline: zappar_pipeline_t;
+export declare let onSequenceLoaded: Event1<zappar_sequence_source_t>;
+export declare function setup(webgl2?: boolean): Zappar;
+export declare function currentPoseSetting(): PoseSetting;
+export declare enum PoseSetting {
+    default = 0,
+    attitude = 1,
+    target = 2
+}
+export declare function getGLContext(): WebGLRenderingContext | undefined;

package/lib/tests/d3-tracking.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/drawanchor.d.ts

@@ -0,0 +1,3 @@
+export declare function drawCylinderMVP(gl: WebGLRenderingContext, segments: number, mvpMatrix: Float32Array, color: number[]): void;
+export declare function drawCylinder(gl: WebGLRenderingContext, segments: number, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, color: number[]): void;
+export declare function drawAnchor(gl: WebGLRenderingContext, segments: number, length: number, diameter: number, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array): void;

package/lib/tests/face-landmarks.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/face-tracking.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/image-tracking-cylinder.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/image-tracking-preview-mesh.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/image-tracking.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/instant-tracking.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/no-rendering.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/sequence-preview.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/two-pipelines.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/video-face-tracking.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/world-tracking-camera-offset-anchor.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/world-tracking-meshes.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/world-tracking-plane-offset-anchor.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/world-tracking.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/zapcode-tracking.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.3.0-beta.1";
+export declare const VERSION = "3.4.0-beta.0";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.3.0-beta.1";
+export const VERSION = "3.4.0-beta.0";

package/lib/worker-client.d.ts

@@ -1,4 +1,4 @@
 import { MsgManager } from "./messages";
 export declare let messageManager: MsgManager;
 export declare function launchWorker(worker?: Worker): Promise<void>;
-export declare function launchCeresWorker(port: MessagePort, worker?: Worker): Promise<void>;
+export declare function launchCeresWorkers(port0: MessagePort, port1: MessagePort, worker0?: Worker, worker1?: Worker): Promise<void>;

package/lib/worker-client.js

@@ -41,16 +41,30 @@ export function launchWorker(worker) {
         });
     });
 }
-export function launchCeresWorker(port, worker) {
+export function launchCeresWorkers(port0, port1, worker0, worker1) {
     return __awaiter(this, void 0, void 0, function* () {
-        if (!worker)
-            worker = new Worker(new URL("./ceres-worker", import.meta.url), { type: 'module' });
-        worker.postMessage({
-            t: "wasm",
-            url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
-            module: yield wasmModule,
-            port
-        }, [port]);
+        {
+            if (!worker0)
+                worker0 = new Worker(new URL("./ceres-worker", import.meta.url), { type: 'module' });
+            worker0.postMessage({
+                t: "wasm",
+                url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
+                module: yield wasmModule,
+                port: port0,
+                instance: 0,
+            }, [port0]);
+        }
+        {
+            if (!worker1)
+                worker1 = new Worker(new URL("./ceres-worker", import.meta.url), { type: 'module' });
+            worker1.postMessage({
+                t: "wasm",
+                url: new URL("./zappar-cv.wasm", import.meta.url).toString(),
+                module: yield wasmModule,
+                port: port1,
+                instance: 1,
+            }, [port1]);
+        }
     });
 }
 function waitForLoad(w) {

package/lib/worker-messages.d.ts

@@ -5,6 +5,7 @@ export interface WorkerMessageAPI {
     worker_message_send_data_size: (index: number) => number;
     worker_message_send_data: (index: number) => number;
     worker_message_send_reference: (index: number) => number;
-    worker_message_receive: (ref: number, size: number, data: number) => void;
+    worker_message_send_instance: (index: number) => number;
+    worker_message_receive: (ref: number, size: number, data: number, instance: number) => void;
 }
 export declare function getWorkerMessageAPI(mod: ZNM.Module): WorkerMessageAPI;

package/lib/worker-messages.js

@@ -5,6 +5,7 @@ export function getWorkerMessageAPI(mod) {
         worker_message_send_data_size: mod.cwrap("worker_message_send_data_size", "number", ["number"]),
         worker_message_send_data: mod.cwrap("worker_message_send_data", "number", ["number"]),
         worker_message_send_reference: mod.cwrap("worker_message_send_reference", "number", ["number"]),
-        worker_message_receive: mod.cwrap("worker_message_receive", null, ["number", "number", "number"]),
+        worker_message_send_instance: mod.cwrap("worker_message_send_instance", "number", ["number"]),
+        worker_message_receive: mod.cwrap("worker_message_receive", null, ["number", "number", "number", "number"]),
     };
 }

package/lib/worker-server.js

@@ -17,7 +17,8 @@ import { handleImageBitmap } from "./worker-imagebitmap";
 import { getWorkerMessageAPI } from "./worker-messages";
 import { getDataDownloadAPI } from "./data-download";
 export let messageManager = new MsgManager();
-let workerMessageChannel;
+let workerMessageChannel0;
+let workerMessageChannel1;
 let latestCameraToScreenRotation = 0;
 let rawDataAccessEnabled = false;
 const rawCameraFrameBuffersByPipeline = new Map();
@@ -152,6 +153,9 @@ export function launchWorkerServer(wasmUrl, module, shouldRecordData) {
                                 case "rotation_rate":
                                     r.pipeline_motion_rotation_rate_submit(pipeline, msgt.timestamp, msgt.x, msgt.y, msgt.z);
                                     break;
+                                case "relative_orientation":
+                                    r.pipeline_motion_relative_orientation_submit_int(pipeline, msgt.timestamp, msgt.interval, msgt.x, msgt.y, msgt.z, msgt.w);
+                                    break;
                             }
                             break;
                         }
@@ -288,7 +292,7 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
             const pipeline = server._pipeline_by_instance.get(p);
             if (pipeline) {
                 try {
-                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, ((_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1) * 1e-6, downSample);
+                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, (_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1, downSample);
                     handleMessages(mod, workerMessageAPI);
                     if (dataDownloadAPI)
                         handleDataDownload(mod, dataDownloadAPI);
@@ -358,29 +362,42 @@ function handleMessages(mod, m) {
     const toSend = m.worker_message_send_count();
     if (toSend === 0)
         return;
-    if (!workerMessageChannel) {
-        workerMessageChannel = new MessageChannel();
-        workerMessageChannel.port1.start();
-        workerMessageChannel.port1.addEventListener('message', evt => {
+    if (!workerMessageChannel0 || !workerMessageChannel1) {
+        workerMessageChannel0 = new MessageChannel();
+        workerMessageChannel0.port1.start();
+        workerMessageChannel0.port1.addEventListener('message', evt => {
+            if (evt.data.t !== 'msgrec')
+                return;
+            const data = evt.data.data;
+            const ptr = mod._malloc(data.byteLength);
+            mod.HEAPU8.set(data, ptr);
+            m.worker_message_receive(evt.data.reference, data.byteLength, ptr, 0);
+            mod._free(ptr);
+        });
+        workerMessageChannel1 = new MessageChannel();
+        workerMessageChannel1.port1.start();
+        workerMessageChannel1.port1.addEventListener('message', evt => {
             if (evt.data.t !== 'msgrec')
                 return;
             const data = evt.data.data;
             const ptr = mod._malloc(data.byteLength);
             mod.HEAPU8.set(data, ptr);
-            m.worker_message_receive(evt.data.reference, data.byteLength, ptr);
+            m.worker_message_receive(evt.data.reference, data.byteLength, ptr, 1);
             mod._free(ptr);
         });
-        messageManager.postOutgoingMessage({ t: 'setupCeresWorker', port: workerMessageChannel.port2 }, [workerMessageChannel.port2]);
+        messageManager.postOutgoingMessage({ t: 'setupCeresWorker', port0: workerMessageChannel0.port2, port1: workerMessageChannel1.port2 }, [workerMessageChannel0.port2, workerMessageChannel1.port2]);
     }
     for (let i = 0; i < toSend; i++) {
         const reference = m.worker_message_send_reference(i);
         const dataSize = m.worker_message_send_data_size(i);
+        const instance = m.worker_message_send_instance(i);
         const dataPtr = m.worker_message_send_data(i);
         const data = mod.HEAPU8.slice(dataPtr, dataPtr + dataSize);
-        workerMessageChannel.port1.postMessage({
+        (instance === 0 ? workerMessageChannel0 : workerMessageChannel1).port1.postMessage({
             t: 'msgsend',
             data,
-            reference
+            reference,
+            instance
         }, [data.buffer]);
     }
     m.worker_message_send_clear();

package/lib/workerinterface.d.ts

@@ -79,11 +79,12 @@ export interface StreamEndedS2C {
 }
 export interface SensorDataC2S {
     t: "sensorDataC2S";
-    sensor: "accel" | "accel_wo_gravity_int" | "accel_w_gravity_int" | "rotation_rate_int" | "attitude_int" | "rotation_rate" | "attitude";
+    sensor: "accel" | "accel_wo_gravity_int" | "accel_w_gravity_int" | "rotation_rate_int" | "attitude_int" | "rotation_rate" | "attitude" | "relative_orientation";
     p: zappar_pipeline_t;
     x: number;
     y: number;
     z: number;
+    w?: number;
     timestamp: number;
     interval: number;
 }