@zappar/zappar-cv 3.0.1-alpha.17 → 3.0.1-alpha.19

package/README.md

@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
 
 You can use our CDN from within your HTML:
 ```
-<script src="https://libs.zappar.com/zappar-cv/3.0.1-alpha.17/zappar-cv.js"></script>
+<script src="https://libs.zappar.com/zappar-cv/3.0.1-alpha.19/zappar-cv.js"></script>
 ```
 
 Or you can download and host our standalone JavaScript bundle:
-[https://libs.zappar.com/zappar-cv/3.0.1-alpha.17/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.1-alpha.17/zappar-cv.zip)
+[https://libs.zappar.com/zappar-cv/3.0.1-alpha.19/zappar-cv.zip](https://libs.zappar.com/zappar-cv/3.0.1-alpha.19/zappar-cv.zip)

package/lib/drawaxis.js

@@ -16,9 +16,9 @@ function generate(gl) {
     if (!vbo)
         throw new Error("Unable to create buffer object");
     let coords = [
-        0, 0, 0, 1, 0, 0,
-        0, 0, 0, 0, 1, 0,
-        0, 0, 0, 0, 0, 1
+        0, 0, 0, 0.1, 0, 0,
+        0, 0, 0, 0, 0.1, 0,
+        0, 0, 0, 0, 0, 0.1
     ];
     gl.bindBuffer(gl.ARRAY_BUFFER, vbo);
     gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(coords), gl.STATIC_DRAW);
@@ -58,7 +58,7 @@ export function drawAxis(gl, projectionMatrix, cameraMatrix, targetMatrix) {
     gl.bindBuffer(gl.ARRAY_BUFFER, uvbo);
     gl.vertexAttribPointer(shader.attr_color, 4, gl.FLOAT, false, 4 * 4, 0);
     gl.enableVertexAttribArray(shader.attr_color);
-    gl.lineWidth(10);
+    gl.lineWidth(2);
     gl.drawArrays(gl.LINES, 0, 6);
     gl.disableVertexAttribArray(shader.attr_position);
     gl.disableVertexAttribArray(shader.attr_color);
@@ -135,38 +135,3 @@ function getShader(gl) {
     };
     return shader;
 }
-function loadTexture(gl, url) {
-    if (texturesByUrl[url])
-        return texturesByUrl[url];
-    let texture = gl.createTexture();
-    if (!texture)
-        throw new Error("Unable to create texture");
-    texturesByUrl[url] = texture;
-    gl.bindTexture(gl.TEXTURE_2D, texture);
-    // Because images have to be download over the internet
-    // they might take a moment until they are ready.
-    // Until then put a single pixel in the texture so we can
-    // use it immediately. When the image has finished downloading
-    // we'll update the texture with the contents of the image.
-    const level = 0;
-    const internalFormat = gl.RGBA;
-    const width = 1;
-    const height = 1;
-    const border = 0;
-    const srcFormat = gl.RGBA;
-    const srcType = gl.UNSIGNED_BYTE;
-    const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue
-    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
-    gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, width, height, border, srcFormat, srcType, pixel);
-    const image = new Image();
-    image.onload = function () {
-        gl.bindTexture(gl.TEXTURE_2D, texture);
-        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
-        gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, image);
-        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
-        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
-        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
-    };
-    image.src = url;
-    return texture;
-}

package/lib/gen/zappar-bridge.d.ts

@@ -62,7 +62,7 @@ export interface zappar {
     pipeline_frame_number(o: zappar_pipeline_t): number;
     pipeline_camera_model(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_user_data(o: zappar_pipeline_t): number;
-    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean): void;
+    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean, timestampSeconds: number): void;
     pipeline_camera_frame_camera_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_device_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_motion_accelerometer_submit(o: zappar_pipeline_t, time: number, x: number, y: number, z: number): void;

package/lib/gen/zappar-client.js

@@ -90,7 +90,7 @@ export class zappar_client {
                     throw new Error("This object has been destroyed");
                 return s.current_frame_user_data;
             },
-            pipeline_camera_frame_submit: (o, data, width, height, user_data, camera_to_device_transform, camera_model, user_facing) => {
+            pipeline_camera_frame_submit: (o, data, width, height, user_data, camera_to_device_transform, camera_model, user_facing, timestampSeconds) => {
                 let s = this._pipeline_state_by_instance.get(o);
                 if (!s)
                     throw new Error("This object has been destroyed");
@@ -103,6 +103,7 @@ export class zappar_client {
                     m.matrix4x4(camera_to_device_transform);
                     m.cameraModel(camera_model);
                     m.bool(user_facing);
+                    m.float(timestampSeconds);
                 });
             },
             pipeline_camera_frame_camera_attitude: (o) => {

package/lib/gen/zappar-cwrap.js

@@ -34,7 +34,7 @@ export function getRuntimeObject(mod) {
     ]);
     let pipeline_camera_frame_submit_wrapped = mod.cwrap("zappar_pipeline_camera_frame_submit", null, [
         "number",
-        "number", "number", "number", "number", "number", "number", "number", "number"
+        "number", "number", "number", "number", "number", "number", "number", "number", "number"
     ]);
     let pipeline_camera_frame_submit_raw_pointer_wrapped = mod.cwrap("zappar_pipeline_camera_frame_submit_raw_pointer", null, [
         "number",
@@ -430,7 +430,7 @@ export function getRuntimeObject(mod) {
             let ret = pipeline_camera_frame_user_data_wrapped(o);
             return ret;
         },
-        pipeline_camera_frame_submit: (o, data, width, height, user_data, camera_to_device_transform, camera_model, user_facing) => {
+        pipeline_camera_frame_submit: (o, data, width, height, user_data, camera_to_device_transform, camera_model, user_facing, timestampSeconds) => {
             if (dataArrayArgLength < data.byteLength) {
                 mod._free(dataArrayArg);
                 dataArrayArgLength = data.byteLength;
@@ -447,10 +447,11 @@ export function getRuntimeObject(mod) {
             let arg_camera_model = getFloatDataArrayForArgIndex(5, camera_model.byteLength);
             mod.HEAPF32.set(camera_model, arg_camera_model / 4);
             let arg_user_facing = user_facing ? 1 : 0;
-            let ret = pipeline_camera_frame_submit_wrapped(o, arg_data, arg_len_data, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_camera_model, arg_user_facing);
+            let arg_timestampSeconds = timestampSeconds;
+            let ret = pipeline_camera_frame_submit_wrapped(o, arg_data, arg_len_data, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_camera_model, arg_user_facing, arg_timestampSeconds);
             return ret;
         },
-        pipeline_camera_frame_submit_raw_pointer: (o, data, dataLength, format, width, height, user_data, camera_to_device_transform, rotation, camera_model, user_facing, timestamp, halfSample) => {
+        pipeline_camera_frame_submit_raw_pointer: (o, data, dataLength, format, width, height, user_data, camera_to_device_transform, rotation, camera_model, user_facing, timestampSeconds, halfSample) => {
             let arg_data = data;
             let arg_dataLength = dataLength;
             let arg_format = format;
@@ -463,9 +464,9 @@ export function getRuntimeObject(mod) {
             let arg_camera_model = getFloatDataArrayForArgIndex(8, camera_model.byteLength);
             mod.HEAPF32.set(camera_model, arg_camera_model / 4);
             let arg_user_facing = user_facing ? 1 : 0;
-            let arg_timestamp = timestamp;
+            let arg_timestampSeconds = timestampSeconds;
             let arg_halfSample = halfSample ? 1 : 0;
-            let ret = pipeline_camera_frame_submit_raw_pointer_wrapped(o, arg_data, arg_dataLength, arg_format, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_rotation, arg_camera_model, arg_user_facing, arg_timestamp, arg_halfSample);
+            let ret = pipeline_camera_frame_submit_raw_pointer_wrapped(o, arg_data, arg_dataLength, arg_format, arg_width, arg_height, arg_user_data, arg_camera_to_device_transform, arg_rotation, arg_camera_model, arg_user_facing, arg_timestampSeconds, arg_halfSample);
             return ret;
         },
         pipeline_camera_frame_camera_attitude: (o) => {

package/lib/gen/zappar-native.d.ts

@@ -136,8 +136,8 @@ export interface zappar_cwrap {
     pipeline_frame_number(o: zappar_pipeline_t): number;
     pipeline_camera_model(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_user_data(o: zappar_pipeline_t): number;
-    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean): void;
-    pipeline_camera_frame_submit_raw_pointer(o: zappar_pipeline_t, data: number, dataLength: number, format: frame_pixel_format_t, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, rotation: number, camera_model: Float32Array, user_facing: boolean, timestamp: number, halfSample: boolean): void;
+    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean, timestampSeconds: number): void;
+    pipeline_camera_frame_submit_raw_pointer(o: zappar_pipeline_t, data: number, dataLength: number, format: frame_pixel_format_t, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, rotation: number, camera_model: Float32Array, user_facing: boolean, timestampSeconds: number, halfSample: boolean): void;
     pipeline_camera_frame_camera_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_device_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_motion_accelerometer_submit(o: zappar_pipeline_t, time: number, x: number, y: number, z: number): void;

package/lib/gen/zappar-server.js

@@ -75,7 +75,7 @@ export class zappar_server {
                     let obj = this._pipeline_by_instance.get(clientId);
                     if (obj === undefined)
                         return;
-                    this._impl.pipeline_camera_frame_submit(obj, msg.dataWithLength(), msg.int(), msg.int(), msg.int(), msg.matrix4x4(), msg.cameraModel(), msg.bool());
+                    this._impl.pipeline_camera_frame_submit(obj, msg.dataWithLength(), msg.int(), msg.int(), msg.int(), msg.matrix4x4(), msg.cameraModel(), msg.bool(), msg.float());
                     break;
                 }
                 case 10: {

package/lib/gen/zappar.d.ts

@@ -91,7 +91,7 @@ export interface zappar {
     pipeline_camera_pose_with_attitude(o: zappar_pipeline_t, mirror: boolean): Float32Array;
     pipeline_camera_pose_with_origin(o: zappar_pipeline_t, pose: Float32Array): Float32Array;
     pipeline_camera_frame_user_data(o: zappar_pipeline_t): number;
-    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean): void;
+    pipeline_camera_frame_submit(o: zappar_pipeline_t, data: ArrayBuffer, width: number, height: number, user_data: number, camera_to_device_transform: Float32Array, camera_model: Float32Array, user_facing: boolean, timestampSeconds: number): void;
     pipeline_camera_frame_camera_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_device_attitude(o: zappar_pipeline_t): Float32Array;
     pipeline_camera_frame_user_facing(o: zappar_pipeline_t): boolean;

package/lib/html-element-source.d.ts

@@ -25,6 +25,7 @@ export declare class HTMLElementSource extends Source {
     start(): void;
     getFrame(currentlyProcessing: boolean): void;
     private _lastPresentedFrames;
+    private _frameTime;
     private _requestVideoFrameCallback;
     private _processFrame;
     uploadGL(): void;

package/lib/html-element-source.js

@@ -78,6 +78,7 @@ export class HTMLElementSource extends Source {
         }
     }
     getFrame(currentlyProcessing) {
+        var _a;
         let pipeline = Pipeline.get(this._pipeline);
         if (!pipeline)
             return;
@@ -92,7 +93,7 @@ export class HTMLElementSource extends Source {
             let info = this._processFrame(gl, this._cameraToScreenRotation, currentlyProcessing);
             if (info) {
                 let token = pipeline.registerToken(info);
-                pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
+                pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel, (_a = this._frameTime) !== null && _a !== void 0 ? _a : performance.now());
                 info.data = undefined;
             }
         }
@@ -106,6 +107,9 @@ export class HTMLElementSource extends Source {
         if (!(video === null || video === void 0 ? void 0 : video.requestVideoFrameCallback))
             return;
         video.requestVideoFrameCallback((time, metadata) => {
+            var _a;
+            if (typeof metadata === 'object')
+                this._frameTime = ((_a = metadata.captureTime) !== null && _a !== void 0 ? _a : -1) / 1000.0;
             if (typeof metadata === 'object' && typeof metadata.presentedFrames === 'number') {
                 if (this._lastPresentedFrames === metadata.presentedFrames) {
                     console.log('Avoided identical frame');

package/lib/mstp-camera-source.js

@@ -234,7 +234,8 @@ export class MSTPCameraSource extends Source {
                     if (videoTracks.length > 0) {
                         this._isUserFacing = videoTracks[0].getSettings().facingMode === "user";
                         let processor = new MediaStreamTrackProcessor({
-                            track: videoTracks[0]
+                            track: videoTracks[0],
+                            maxBufferSize: 1
                         });
                         const str = this._currentStream;
                         yield delay(1000);

package/lib/pipeline.d.ts

@@ -78,6 +78,6 @@ export declare class Pipeline {
     sendCameraProfileToWorker(source: zappar_camera_source_t, profile: camera_profile_t): void;
     sendCameraToScreenRotationToWorker(rot: number): void;
     sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array, cp: camera_profile_t): void;
-    sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
+    sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array, captureTime: number): void;
 }
 export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): Float32Array;

package/lib/pipeline.js

@@ -479,14 +479,15 @@ export class Pipeline {
         };
         this._mgr.postOutgoingMessage(msg, [img]);
     }
-    sendDataToWorker(data, token, width, height, userFacing, cameraToDevice, cameraModel) {
+    sendDataToWorker(data, token, width, height, userFacing, cameraToDevice, cameraModel, captureTime) {
         let msg = {
             d: data,
             p: this._impl,
             width, height, token, userFacing,
             c2d: cameraToDevice,
             cm: cameraModel,
-            t: "cameraFrameC2S"
+            t: "cameraFrameC2S",
+            captureTime
         };
         this._mgr.postOutgoingMessage(msg, [data]);
     }

package/lib/sequencesource.js

@@ -97,7 +97,7 @@ export class SequenceSource extends Source {
         // updates will be one frame later
         this._submitMotionUpdates(pipeline);
         const token = pipeline.registerToken(info);
-        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel);
+        pipeline.sendDataToWorker(info.data || new ArrayBuffer(0), token, info.dataWidth, info.dataHeight, info.userFacing, info.cameraToDevice, info.cameraModel, -1);
     }
     destroy() {
         byId.delete(this._impl);

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "3.0.1-alpha.17";
+export declare const VERSION = "3.0.1-alpha.19";

package/lib/version.js

@@ -1 +1 @@
-export const VERSION = "3.0.1-alpha.17";
+export const VERSION = "3.0.1-alpha.19";

package/lib/worker-imagebitmap.js

@@ -53,7 +53,7 @@ export function handleImageBitmap(m, r, server, mgr) {
     mgr.postOutgoingMessage(msg, [m.i]);
     let pipeline = server._pipeline_by_instance.get(m.p);
     if (pipeline) {
-        r.pipeline_camera_frame_submit(pipeline, pixels, info.dataWidth, info.dataHeight, m.tokenId, m.cameraToDevice, m.cameraModel, info.userFacing);
+        r.pipeline_camera_frame_submit(pipeline, pixels, info.dataWidth, info.dataHeight, m.tokenId, m.cameraToDevice, m.cameraModel, info.userFacing, performance.now());
         r.pipeline_frame_update(pipeline);
         server.exploreState();
     }

package/lib/worker-server.js

@@ -64,7 +64,7 @@ export function launchWorkerServer(wasmUrl, module, shouldRecordData) {
                             let pipeline = server._pipeline_by_instance.get(msgt.p);
                             let att;
                             if (pipeline) {
-                                r.pipeline_camera_frame_submit(pipeline, msgt.d, msgt.width, msgt.height, msgt.token, msgt.c2d, msgt.cm, msgt.userFacing);
+                                r.pipeline_camera_frame_submit(pipeline, msgt.d, msgt.width, msgt.height, msgt.token, msgt.c2d, msgt.cm, msgt.userFacing, msgt.captureTime);
                                 r.pipeline_frame_update(pipeline);
                                 att = r.pipeline_camera_frame_device_attitude(pipeline);
                                 server.exploreState();
@@ -288,7 +288,7 @@ function consumeReader(mod, r, reader, p, userFacing, server, source, workerMess
             const pipeline = server._pipeline_by_instance.get(p);
             if (pipeline) {
                 try {
-                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, (_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1, downSample);
+                    r.pipeline_camera_frame_submit_raw_pointer(pipeline, streamDataBufferPointer, size, framePixelFormatFromFormat(frame.format), width, height, token, cameraToDeviceTransform, latestCameraToScreenRotation, cameraModel, userFacing, ((_b = frame.timestamp) !== null && _b !== void 0 ? _b : -1) * 1e-6, downSample);
                     handleMessages(mod, workerMessageAPI);
                     if (dataDownloadAPI)
                         handleDataDownload(mod, dataDownloadAPI);

package/lib/workerinterface.d.ts

@@ -11,6 +11,7 @@ export interface CameraFrameC2S {
     token: number;
     width: number;
     height: number;
+    captureTime: number;
 }
 export interface CameraFrameReturnS2C {
     t: "cameraFrameRecycleS2C";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zappar/zappar-cv",
-  "version": "3.0.1-alpha.17",
+  "version": "3.0.1-alpha.19",
   "description": "Zappar's core computer vision library, supporting image, face and world tracking.",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",