@zappar/zappar-cv 0.4.0-beta.7 → 2.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +2 -2
- package/lib/additional.d.ts +2 -1
- package/lib/additional.js +1 -2
- package/lib/camera-source-map.js +16 -24
- package/lib/camera-source.js +17 -21
- package/lib/cameramodel.js +18 -23
- package/lib/compatibility.js +1 -3
- package/lib/deserializer.js +1 -5
- package/lib/direct.js +12 -16
- package/lib/drawcamera.js +26 -31
- package/lib/drawface.js +5 -9
- package/lib/drawfaceproject.js +5 -9
- package/lib/drawplane.js +6 -11
- package/lib/event.js +5 -13
- package/lib/facelandmark.d.ts +1 -2
- package/lib/facelandmark.js +11 -18
- package/lib/facelandmarkdata.js +1 -4
- package/lib/facelandmarkinterface.js +1 -2
- package/lib/facemesh.js +10 -17
- package/lib/gen/zappar-client.js +5 -24
- package/lib/gen/zappar-cwrap.js +1 -12
- package/lib/gen/zappar-native.d.ts +0 -1
- package/lib/gen/zappar-native.js +10 -13
- package/lib/gen/zappar-server.js +5 -13
- package/lib/gen/zappar.d.ts +7 -2
- package/lib/gen/zappar.js +5 -13
- package/lib/gfx.js +1 -4
- package/lib/gl-state-manager.d.ts +9 -0
- package/lib/gl-state-manager.js +31 -0
- package/lib/html-element-source.js +23 -27
- package/lib/image-process-gl.js +73 -32
- package/lib/imagebitmap-camera-source.js +30 -34
- package/lib/imagetracker.d.ts +28 -0
- package/lib/imagetracker.js +115 -0
- package/lib/index-standalone.d.ts +3 -0
- package/lib/index-standalone.js +5 -0
- package/lib/index.d.ts +2 -1
- package/lib/index.js +6 -27
- package/lib/landmarkdatagenerate.js +25 -30
- package/lib/loglevel.js +9 -16
- package/lib/messages.js +4 -8
- package/lib/mstp-camera-source.js +22 -26
- package/lib/native.d.ts +2 -1
- package/lib/native.js +194 -132
- package/lib/options.d.ts +3 -0
- package/lib/options.js +1 -0
- package/lib/permission.js +13 -27
- package/lib/pipeline.d.ts +1 -2
- package/lib/pipeline.js +34 -39
- package/lib/profile.js +40 -45
- package/lib/riff-reader.d.ts +18 -0
- package/lib/riff-reader.js +52 -0
- package/lib/sequencerecorder.js +7 -12
- package/lib/sequencesource.js +20 -24
- package/lib/serializer.js +1 -5
- package/lib/shader.js +2 -7
- package/lib/source.js +1 -5
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -4
- package/lib/worker-client.d.ts +1 -1
- package/lib/worker-client.js +9 -21
- package/lib/worker-imagebitmap.js +9 -13
- package/lib/worker-server.js +33 -37
- package/lib/worker.js +6 -10
- package/lib/workerinterface.js +1 -2
- package/lib/zcv.js +124 -132
- package/lib/zcv.wasm +0 -0
- package/package.json +14 -16
- package/umd/169.zappar-cv.js +1 -0
- package/umd/{482727c0e7dd40d73f5aae47f238ad61.zbin → 482727c0e7dd40d73f5a.zbin} +0 -0
- package/umd/752.zappar-cv.js +1 -0
- package/umd/{b4f46e148a3b3b5aef90717d8d60e3fc.zbin → b4f46e148a3b3b5aef90.zbin} +0 -0
- package/umd/e486cd197ef362198043.wasm +0 -0
- package/umd/{f7b62a3a53b5c9b3222ba8dc53cb8b11.zbin → f7b62a3a53b5c9b3222b.zbin} +0 -0
- package/umd/zappar-cv.js +1 -1
- package/umd/zappar-cv.worker.js +1 -1
- package/umd/9d9c4b542b40c41e3ef7587fb91e5d38.wasm +0 -0
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -18,8 +18,8 @@ npm i @zappar/zappar-cv
|
|
|
18
18
|
|
|
19
19
|
You can use our CDN from within your HTML:
|
|
20
20
|
```
|
|
21
|
-
<script src="https://libs.zappar.com/zappar-cv/0.
|
|
21
|
+
<script src="https://libs.zappar.com/zappar-cv/2.0.0-beta.1/zappar-cv.js"></script>
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
Or you can download and host our standalone JavaScript bundle:
|
|
25
|
-
[https://libs.zappar.com/zappar-cv/0.
|
|
25
|
+
[https://libs.zappar.com/zappar-cv/2.0.0-beta.1/zappar-cv.zip](https://libs.zappar.com/zappar-cv/2.0.0-beta.1/zappar-cv.zip)
|
package/lib/additional.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { zappar_face_mesh_t, zappar_face_tracker_t, zappar_pipeline_t } from "./gen/zappar";
|
|
1
|
+
import { zappar_face_mesh_t, zappar_face_tracker_t, zappar_pipeline_t, zappar_image_tracker_t } from "./gen/zappar";
|
|
2
2
|
export declare type zappar_html_element_source_t = number & {
|
|
3
3
|
_: 'zappar_html_element_source_t';
|
|
4
4
|
};
|
|
@@ -8,6 +8,7 @@ export interface Additional {
|
|
|
8
8
|
pipeline_draw_face(pipeline: zappar_pipeline_t, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, o: zappar_face_mesh_t): void;
|
|
9
9
|
pipeline_draw_face_project(pipeline: zappar_pipeline_t, drawwMatrix: Float32Array, vertices: Float32Array, uvMatrix: Float32Array, uvs: Float32Array, indices: Uint16Array, texture: WebGLTexture): void;
|
|
10
10
|
draw_plane(gl: WebGLRenderingContext, projectionMatrix: Float32Array, cameraMatrix: Float32Array, targetMatrix: Float32Array, texture: string): void;
|
|
11
|
+
image_tracker_target_image(o: zappar_image_tracker_t, indx: number): HTMLImageElement | undefined;
|
|
11
12
|
face_mesh_load_default(o: zappar_face_mesh_t): Promise<void>;
|
|
12
13
|
face_mesh_load_default_face(o: zappar_face_mesh_t, fillMouth: boolean, fillEyeL: boolean, fillEyeR: boolean): Promise<void>;
|
|
13
14
|
face_mesh_load_default_full_head_simplified(o: zappar_face_mesh_t, fillMouth: boolean, fillEyeL: boolean, fillEyeR: boolean, fillNeck: boolean): Promise<void>;
|
package/lib/additional.js
CHANGED
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
package/lib/camera-source-map.js
CHANGED
|
@@ -1,40 +1,32 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const loglevel_1 = require("./loglevel");
|
|
7
|
-
const mstp_camera_source_1 = require("./mstp-camera-source");
|
|
8
|
-
const profile_1 = require("./profile");
|
|
1
|
+
import { CameraSource } from "./camera-source";
|
|
2
|
+
import { ImageBitmapCameraSource } from "./imagebitmap-camera-source";
|
|
3
|
+
import { zcout } from "./loglevel";
|
|
4
|
+
import { MSTPCameraSource } from "./mstp-camera-source";
|
|
5
|
+
import { profile } from "./profile";
|
|
9
6
|
let latest = 1;
|
|
10
7
|
let byId = new Map();
|
|
11
|
-
function getNextCameraSourceId() {
|
|
8
|
+
export function getNextCameraSourceId() {
|
|
12
9
|
return (latest++);
|
|
13
10
|
}
|
|
14
|
-
|
|
15
|
-
function setCameraSourceId(id, c) {
|
|
11
|
+
export function setCameraSourceId(id, c) {
|
|
16
12
|
byId.set(id, c);
|
|
17
13
|
}
|
|
18
|
-
|
|
19
|
-
function getCameraSource(id) {
|
|
14
|
+
export function getCameraSource(id) {
|
|
20
15
|
return byId.get(id);
|
|
21
16
|
}
|
|
22
|
-
|
|
23
|
-
function deleteCameraSource(id) {
|
|
17
|
+
export function deleteCameraSource(id) {
|
|
24
18
|
byId.delete(id);
|
|
25
19
|
}
|
|
26
|
-
|
|
27
|
-
function createCameraSource(p, deviceId) {
|
|
20
|
+
export function createCameraSource(p, deviceId) {
|
|
28
21
|
let ret = getNextCameraSourceId();
|
|
29
|
-
if (
|
|
22
|
+
if (profile.preferMediaStreamTrackProcessorCamera &&
|
|
30
23
|
'MediaStreamTrackProcessor' in window &&
|
|
31
24
|
'MediaStreamTrackGenerator' in window)
|
|
32
|
-
setCameraSourceId(ret, new
|
|
33
|
-
else if (
|
|
34
|
-
setCameraSourceId(ret, new
|
|
25
|
+
setCameraSourceId(ret, new MSTPCameraSource(ret, p, deviceId));
|
|
26
|
+
else if (profile.preferImageBitmapCamera && typeof OffscreenCanvas !== "undefined")
|
|
27
|
+
setCameraSourceId(ret, new ImageBitmapCameraSource(ret, p, deviceId));
|
|
35
28
|
else
|
|
36
|
-
setCameraSourceId(ret, new
|
|
37
|
-
|
|
29
|
+
setCameraSourceId(ret, new CameraSource(ret, p, deviceId));
|
|
30
|
+
zcout("camera_source_t initialized");
|
|
38
31
|
return ret;
|
|
39
32
|
}
|
|
40
|
-
exports.createCameraSource = createCameraSource;
|
package/lib/camera-source.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,23 +7,21 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
8
|
});
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
const loglevel_1 = require("./loglevel");
|
|
18
|
-
const camera_source_map_1 = require("./camera-source-map");
|
|
10
|
+
import { HTMLElementSource } from "./html-element-source";
|
|
11
|
+
import { profile } from "./profile";
|
|
12
|
+
import { Pipeline } from "./pipeline";
|
|
13
|
+
import { cameraRotationForScreenOrientation } from "./cameramodel";
|
|
14
|
+
import { zcout } from "./loglevel";
|
|
15
|
+
import { deleteCameraSource } from "./camera-source-map";
|
|
19
16
|
let _videoSingleton = document.createElement("video");
|
|
20
17
|
_videoSingleton.setAttribute('playsinline', '');
|
|
21
18
|
_videoSingleton.setAttribute('webkit-playsinline', '');
|
|
22
|
-
if (
|
|
19
|
+
if (profile.videoElementInDOM) {
|
|
23
20
|
_videoSingleton.style.width = "0px";
|
|
24
21
|
_videoSingleton.style.height = "0px";
|
|
25
22
|
document.body.appendChild(_videoSingleton);
|
|
26
23
|
}
|
|
27
|
-
class CameraSource extends
|
|
24
|
+
export class CameraSource extends HTMLElementSource {
|
|
28
25
|
constructor(_impl, pipeline, _deviceId) {
|
|
29
26
|
super(_videoSingleton, pipeline);
|
|
30
27
|
this._impl = _impl;
|
|
@@ -33,7 +30,7 @@ class CameraSource extends html_element_source_1.HTMLElementSource {
|
|
|
33
30
|
this._activeDeviceId = null;
|
|
34
31
|
this._hasStartedOrientation = false;
|
|
35
32
|
this._deviceMotionListener = (ev) => {
|
|
36
|
-
let pipeline =
|
|
33
|
+
let pipeline = Pipeline.get(this._pipeline);
|
|
37
34
|
if (!pipeline)
|
|
38
35
|
return;
|
|
39
36
|
let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
|
|
@@ -41,7 +38,7 @@ class CameraSource extends html_element_source_1.HTMLElementSource {
|
|
|
41
38
|
ev.accelerationIncludingGravity.x !== null &&
|
|
42
39
|
ev.accelerationIncludingGravity.y !== null &&
|
|
43
40
|
ev.accelerationIncludingGravity.z !== null) {
|
|
44
|
-
pipeline.motionAccelerometerSubmit(timeStamp, ev.accelerationIncludingGravity.x *
|
|
41
|
+
pipeline.motionAccelerometerSubmit(timeStamp, ev.accelerationIncludingGravity.x * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.y * profile.deviceMotionMutliplier, ev.accelerationIncludingGravity.z * profile.deviceMotionMutliplier);
|
|
45
42
|
}
|
|
46
43
|
if (ev.rotationRate !== null &&
|
|
47
44
|
ev.rotationRate.alpha !== null &&
|
|
@@ -56,7 +53,7 @@ class CameraSource extends html_element_source_1.HTMLElementSource {
|
|
|
56
53
|
};
|
|
57
54
|
}
|
|
58
55
|
destroy() {
|
|
59
|
-
|
|
56
|
+
deleteCameraSource(this._impl);
|
|
60
57
|
super.destroy();
|
|
61
58
|
}
|
|
62
59
|
_stop() {
|
|
@@ -92,9 +89,9 @@ class CameraSource extends html_element_source_1.HTMLElementSource {
|
|
|
92
89
|
audio: false,
|
|
93
90
|
video: {
|
|
94
91
|
facingMode: facingMode,
|
|
95
|
-
width:
|
|
96
|
-
height:
|
|
97
|
-
frameRate:
|
|
92
|
+
width: profile.videoWidth,
|
|
93
|
+
height: profile.videoHeight,
|
|
94
|
+
frameRate: profile.requestHighFrameRate ? 60 : undefined,
|
|
98
95
|
deviceId: deviceId
|
|
99
96
|
}
|
|
100
97
|
};
|
|
@@ -122,14 +119,14 @@ class CameraSource extends html_element_source_1.HTMLElementSource {
|
|
|
122
119
|
return constraints;
|
|
123
120
|
}
|
|
124
121
|
if (typeof constraints.video === "object") {
|
|
125
|
-
|
|
122
|
+
zcout("choosing device ID", devices[devices.length - 1].deviceId);
|
|
126
123
|
constraints.video.deviceId = devices[devices.length - 1].deviceId;
|
|
127
124
|
}
|
|
128
125
|
return constraints;
|
|
129
126
|
});
|
|
130
127
|
}
|
|
131
128
|
getFrame(allowRead) {
|
|
132
|
-
this._cameraToScreenRotation =
|
|
129
|
+
this._cameraToScreenRotation = cameraRotationForScreenOrientation(false);
|
|
133
130
|
super.getFrame(allowRead);
|
|
134
131
|
}
|
|
135
132
|
_getUserMedia() {
|
|
@@ -179,7 +176,7 @@ class CameraSource extends html_element_source_1.HTMLElementSource {
|
|
|
179
176
|
return;
|
|
180
177
|
this._hasStartedOrientation = true;
|
|
181
178
|
this._deviceOrientationListener = (ev) => {
|
|
182
|
-
let pipeline =
|
|
179
|
+
let pipeline = Pipeline.get(this._pipeline);
|
|
183
180
|
if (!pipeline)
|
|
184
181
|
return;
|
|
185
182
|
let timeStamp = (ev.timeStamp !== undefined && ev.timeStamp !== null) ? ev.timeStamp : performance.now();
|
|
@@ -198,6 +195,5 @@ class CameraSource extends html_element_source_1.HTMLElementSource {
|
|
|
198
195
|
window.removeEventListener("deviceorientation", this._deviceOrientationListener);
|
|
199
196
|
}
|
|
200
197
|
}
|
|
201
|
-
exports.CameraSource = CameraSource;
|
|
202
198
|
CameraSource.USER_DEFAULT_DEVICE_ID = "Simulated User Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";
|
|
203
199
|
CameraSource.DEFAULT_DEVICE_ID = "Simulated Default Device ID: a908df7f-5661-4d20-b227-a1c15d2fdb4b";
|
package/lib/cameramodel.js
CHANGED
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.cameraRotationForScreenOrientation = exports.projectionMatrix = void 0;
|
|
4
|
-
const gl_matrix_1 = require("gl-matrix");
|
|
5
|
-
function projectionMatrix(params, screenWidth, screenHeight, near = 0.01, far = 100.0) {
|
|
1
|
+
import { mat4, vec3 } from "gl-matrix";
|
|
2
|
+
export function projectionMatrix(params, screenWidth, screenHeight, near = 0.01, far = 100.0) {
|
|
6
3
|
let cam_x = params[2] * 2;
|
|
7
4
|
let cam_y = params[3] * 2;
|
|
8
|
-
let projection =
|
|
9
|
-
|
|
5
|
+
let projection = mat4.create();
|
|
6
|
+
mat4.frustum(projection, near * (-0.5 - params[2]) / params[0], near * (cam_x - 0.5 - params[2]) / params[0], near * (cam_y - 0.5 - params[3]) / params[1], near * (-0.5 - params[3]) / params[1], near, far);
|
|
10
7
|
// Flip camera y axis (before any other transformations) - converts from
|
|
11
8
|
// the y axis downwards of the tracking camera models to the y axis upwards
|
|
12
9
|
// of rendering coordinate systems
|
|
@@ -14,18 +11,18 @@ function projectionMatrix(params, screenWidth, screenHeight, near = 0.01, far =
|
|
|
14
11
|
projection[5] *= -1.0;
|
|
15
12
|
projection[6] *= -1.0;
|
|
16
13
|
projection[7] *= -1.0;
|
|
17
|
-
let mult =
|
|
18
|
-
|
|
19
|
-
|
|
14
|
+
let mult = mat4.create();
|
|
15
|
+
mat4.fromScaling(mult, [0.5 * cam_x, 0.5 * cam_y, 1]);
|
|
16
|
+
mat4.multiply(projection, mult, projection);
|
|
20
17
|
// If it's user facing or not doesn't matter here as long as we apply the same-but-opposite
|
|
21
18
|
// rotation at the end of the function
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
let vec =
|
|
19
|
+
mat4.fromRotation(mult, cameraRotationForScreenOrientation(false) * Math.PI / 180.0, [0, 0, 1]);
|
|
20
|
+
mat4.multiply(projection, mult, projection);
|
|
21
|
+
let vec = vec3.create();
|
|
25
22
|
vec[0] = cam_x;
|
|
26
23
|
vec[1] = cam_y;
|
|
27
24
|
vec[2] = 0;
|
|
28
|
-
|
|
25
|
+
vec3.transformMat4(vec, vec, mult);
|
|
29
26
|
let absWidth = Math.abs(vec[0]);
|
|
30
27
|
let absHeight = Math.abs(vec[1]);
|
|
31
28
|
let scaleFactor = 1;
|
|
@@ -35,16 +32,15 @@ function projectionMatrix(params, screenWidth, screenHeight, near = 0.01, far =
|
|
|
35
32
|
else {
|
|
36
33
|
scaleFactor = screenWidth / absWidth;
|
|
37
34
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
35
|
+
mat4.fromScaling(mult, [scaleFactor, scaleFactor, 1]);
|
|
36
|
+
mat4.multiply(projection, mult, projection);
|
|
37
|
+
mat4.fromScaling(mult, [2 / screenWidth, 2 / screenHeight, 1]);
|
|
38
|
+
mat4.multiply(projection, mult, projection);
|
|
39
|
+
mat4.fromRotation(mult, cameraRotationForScreenOrientation(false) * Math.PI / -180.0, [0, 0, 1]);
|
|
40
|
+
mat4.multiply(projection, projection, mult);
|
|
44
41
|
return projection;
|
|
45
42
|
}
|
|
46
|
-
|
|
47
|
-
function cameraRotationForScreenOrientation(isUserFacing) {
|
|
43
|
+
export function cameraRotationForScreenOrientation(isUserFacing) {
|
|
48
44
|
if (window.screen.orientation) {
|
|
49
45
|
switch (window.screen.orientation.type) {
|
|
50
46
|
case "portrait-primary":
|
|
@@ -67,4 +63,3 @@ function cameraRotationForScreenOrientation(isUserFacing) {
|
|
|
67
63
|
}
|
|
68
64
|
return 0;
|
|
69
65
|
}
|
|
70
|
-
exports.cameraRotationForScreenOrientation = cameraRotationForScreenOrientation;
|
package/lib/compatibility.js
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
1
|
function getUnsupportedMessage() {
|
|
4
2
|
let ret = document.createElement("div");
|
|
5
3
|
let idealBrowser = "a recent web browser";
|
|
@@ -160,7 +158,7 @@ function isSupportedBrowser() {
|
|
|
160
158
|
return false;
|
|
161
159
|
return true;
|
|
162
160
|
}
|
|
163
|
-
|
|
161
|
+
export default {
|
|
164
162
|
incompatible: () => !isSupportedBrowser(),
|
|
165
163
|
incompatible_ui: incompatibleUI,
|
|
166
164
|
};
|
package/lib/deserializer.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.MessageDeserializer = void 0;
|
|
4
|
-
class MessageDeserializer {
|
|
1
|
+
export class MessageDeserializer {
|
|
5
2
|
constructor() {
|
|
6
3
|
this._buffer = new ArrayBuffer(0);
|
|
7
4
|
this._i32View = new Int32Array(this._buffer);
|
|
@@ -113,4 +110,3 @@ class MessageDeserializer {
|
|
|
113
110
|
}
|
|
114
111
|
}
|
|
115
112
|
}
|
|
116
|
-
exports.MessageDeserializer = MessageDeserializer;
|
package/lib/direct.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,25 +7,22 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
8
|
});
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
exports.messageManager = new messages_1.MsgManager();
|
|
17
|
-
function launchWorker(wasm) {
|
|
10
|
+
import { MsgManager } from "./messages";
|
|
11
|
+
import { launchWorkerServer } from "./worker-server";
|
|
12
|
+
import { messageManager as workerMessageManager } from "./worker-server";
|
|
13
|
+
export let messageManager = new MsgManager();
|
|
14
|
+
export function launchWorker(wasm) {
|
|
18
15
|
return __awaiter(this, void 0, void 0, function* () {
|
|
19
|
-
|
|
20
|
-
let msgs =
|
|
16
|
+
messageManager.onOutgoingMessage.bind(() => {
|
|
17
|
+
let msgs = messageManager.getOutgoingMessages();
|
|
21
18
|
for (let msg of msgs)
|
|
22
|
-
|
|
19
|
+
workerMessageManager.postIncomingMessage(msg.msg);
|
|
23
20
|
});
|
|
24
|
-
|
|
25
|
-
let msgs =
|
|
21
|
+
workerMessageManager.onOutgoingMessage.bind(() => {
|
|
22
|
+
let msgs = workerMessageManager.getOutgoingMessages();
|
|
26
23
|
for (let msg of msgs)
|
|
27
|
-
|
|
24
|
+
messageManager.postIncomingMessage(msg.msg);
|
|
28
25
|
});
|
|
29
|
-
|
|
26
|
+
launchWorkerServer(wasm);
|
|
30
27
|
});
|
|
31
28
|
}
|
|
32
|
-
exports.launchWorker = launchWorker;
|
package/lib/drawcamera.js
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const gl_matrix_1 = require("gl-matrix");
|
|
6
|
-
let identity = gl_matrix_1.mat4.create();
|
|
7
|
-
class CameraDraw {
|
|
1
|
+
import { compileShader, linkProgram } from "./shader";
|
|
2
|
+
import { mat4, vec3 } from "gl-matrix";
|
|
3
|
+
let identity = mat4.create();
|
|
4
|
+
export class CameraDraw {
|
|
8
5
|
constructor(_gl) {
|
|
9
6
|
this._gl = _gl;
|
|
10
7
|
}
|
|
@@ -69,11 +66,11 @@ class CameraDraw {
|
|
|
69
66
|
let prog = gl.createProgram();
|
|
70
67
|
if (!prog)
|
|
71
68
|
throw new Error("Unable to create program");
|
|
72
|
-
let vertexShader =
|
|
73
|
-
let fragmentShader =
|
|
69
|
+
let vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSrc);
|
|
70
|
+
let fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSrc);
|
|
74
71
|
gl.attachShader(prog, vertexShader);
|
|
75
72
|
gl.attachShader(prog, fragmentShader);
|
|
76
|
-
|
|
73
|
+
linkProgram(gl, prog);
|
|
77
74
|
let unif_skinTexTransform = gl.getUniformLocation(prog, "skinTexTransform");
|
|
78
75
|
if (!unif_skinTexTransform)
|
|
79
76
|
throw new Error("Unable to get uniform location skinTexTransform");
|
|
@@ -90,7 +87,6 @@ class CameraDraw {
|
|
|
90
87
|
return this._shader;
|
|
91
88
|
}
|
|
92
89
|
}
|
|
93
|
-
exports.CameraDraw = CameraDraw;
|
|
94
90
|
let vertexShaderSrc = `
|
|
95
91
|
#ifndef GL_ES
|
|
96
92
|
#define highp
|
|
@@ -149,45 +145,44 @@ function cameraRotationForScreenOrientation() {
|
|
|
149
145
|
}
|
|
150
146
|
return 0;
|
|
151
147
|
}
|
|
152
|
-
function cameraFrameTextureMatrix(frameWidth, frameHeight, screenWidth, screenHeight, uvMatrix, mirror) {
|
|
153
|
-
let ret =
|
|
154
|
-
let trans =
|
|
148
|
+
export function cameraFrameTextureMatrix(frameWidth, frameHeight, screenWidth, screenHeight, uvMatrix, mirror) {
|
|
149
|
+
let ret = mat4.create();
|
|
150
|
+
let trans = mat4.create();
|
|
155
151
|
// Translate to centre UV coords
|
|
156
|
-
|
|
157
|
-
|
|
152
|
+
mat4.fromTranslation(trans, [-0.5, -0.5, 0]);
|
|
153
|
+
mat4.multiply(ret, trans, ret);
|
|
158
154
|
if (mirror) {
|
|
159
|
-
|
|
160
|
-
|
|
155
|
+
mat4.fromScaling(trans, [-1, 1, 1]);
|
|
156
|
+
mat4.multiply(ret, trans, ret);
|
|
161
157
|
}
|
|
162
158
|
// Apply rotation back into ZCV's landscape space
|
|
163
|
-
|
|
164
|
-
|
|
159
|
+
mat4.fromRotation(trans, -1 * cameraRotationForScreenOrientation() * Math.PI / 180.0, [0, 0, 1]);
|
|
160
|
+
mat4.multiply(ret, trans, ret);
|
|
165
161
|
// Get our screenWidth and screenHeight into that same space
|
|
166
|
-
let vec =
|
|
162
|
+
let vec = vec3.create();
|
|
167
163
|
vec[0] = screenWidth;
|
|
168
164
|
vec[1] = screenHeight;
|
|
169
165
|
vec[2] = 0;
|
|
170
|
-
|
|
166
|
+
vec3.transformMat4(vec, vec, trans);
|
|
171
167
|
let absScreenX = Math.abs(vec[0]);
|
|
172
168
|
let absScreenY = Math.abs(vec[1]);
|
|
173
169
|
// Apply a flip since the texture is upside-down
|
|
174
|
-
|
|
175
|
-
|
|
170
|
+
mat4.fromScaling(trans, [1, -1, 1]);
|
|
171
|
+
mat4.multiply(ret, trans, ret);
|
|
176
172
|
// Apply cropping
|
|
177
173
|
let screenAspect = absScreenX / absScreenY;
|
|
178
174
|
let frameAspect = frameWidth / frameHeight;
|
|
179
175
|
if (screenAspect > frameAspect) {
|
|
180
|
-
|
|
176
|
+
mat4.fromScaling(trans, [1, frameAspect / screenAspect, 1]);
|
|
181
177
|
}
|
|
182
178
|
else {
|
|
183
|
-
|
|
179
|
+
mat4.fromScaling(trans, [screenAspect / frameAspect, 1, 1]);
|
|
184
180
|
}
|
|
185
|
-
|
|
181
|
+
mat4.multiply(ret, trans, ret);
|
|
186
182
|
// Translate back to UV coords
|
|
187
|
-
|
|
188
|
-
|
|
183
|
+
mat4.fromTranslation(trans, [0.5, 0.5, 0]);
|
|
184
|
+
mat4.multiply(ret, trans, ret);
|
|
189
185
|
// Apply the camera frame's UV matrix
|
|
190
|
-
|
|
186
|
+
mat4.multiply(ret, uvMatrix, ret);
|
|
191
187
|
return ret;
|
|
192
188
|
}
|
|
193
|
-
exports.cameraFrameTextureMatrix = cameraFrameTextureMatrix;
|
package/lib/drawface.js
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.FaceDraw = void 0;
|
|
4
|
-
const shader_1 = require("./shader");
|
|
5
|
-
class FaceDraw {
|
|
1
|
+
import { compileShader, linkProgram } from "./shader";
|
|
2
|
+
export class FaceDraw {
|
|
6
3
|
constructor(_gl) {
|
|
7
4
|
this._gl = _gl;
|
|
8
5
|
}
|
|
@@ -82,11 +79,11 @@ class FaceDraw {
|
|
|
82
79
|
let prog = gl.createProgram();
|
|
83
80
|
if (!prog)
|
|
84
81
|
throw new Error("Unable to create program");
|
|
85
|
-
let vertexShader =
|
|
86
|
-
let fragmentShader =
|
|
82
|
+
let vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSrc);
|
|
83
|
+
let fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSrc);
|
|
87
84
|
gl.attachShader(prog, vertexShader);
|
|
88
85
|
gl.attachShader(prog, fragmentShader);
|
|
89
|
-
|
|
86
|
+
linkProgram(gl, prog);
|
|
90
87
|
let unif_matrix = gl.getUniformLocation(prog, "matrix");
|
|
91
88
|
if (!unif_matrix)
|
|
92
89
|
throw new Error("Unable to get uniform location mattrix");
|
|
@@ -99,7 +96,6 @@ class FaceDraw {
|
|
|
99
96
|
return this._shader;
|
|
100
97
|
}
|
|
101
98
|
}
|
|
102
|
-
exports.FaceDraw = FaceDraw;
|
|
103
99
|
let vertexShaderSrc = `
|
|
104
100
|
#ifndef GL_ES
|
|
105
101
|
#define highp
|
package/lib/drawfaceproject.js
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.FaceDrawProject = void 0;
|
|
4
|
-
const shader_1 = require("./shader");
|
|
5
|
-
class FaceDrawProject {
|
|
1
|
+
import { compileShader, linkProgram } from "./shader";
|
|
2
|
+
export class FaceDrawProject {
|
|
6
3
|
constructor(_gl) {
|
|
7
4
|
this._gl = _gl;
|
|
8
5
|
}
|
|
@@ -89,11 +86,11 @@ class FaceDrawProject {
|
|
|
89
86
|
let prog = gl.createProgram();
|
|
90
87
|
if (!prog)
|
|
91
88
|
throw new Error("Unable to create program");
|
|
92
|
-
let vertexShader =
|
|
93
|
-
let fragmentShader =
|
|
89
|
+
let vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSrc);
|
|
90
|
+
let fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSrc);
|
|
94
91
|
gl.attachShader(prog, vertexShader);
|
|
95
92
|
gl.attachShader(prog, fragmentShader);
|
|
96
|
-
|
|
93
|
+
linkProgram(gl, prog);
|
|
97
94
|
let unif_matrix = gl.getUniformLocation(prog, "matrix");
|
|
98
95
|
if (!unif_matrix)
|
|
99
96
|
throw new Error("Unable to get uniform location matrix");
|
|
@@ -115,7 +112,6 @@ class FaceDrawProject {
|
|
|
115
112
|
return this._shader;
|
|
116
113
|
}
|
|
117
114
|
}
|
|
118
|
-
exports.FaceDrawProject = FaceDrawProject;
|
|
119
115
|
let vertexShaderSrc = `
|
|
120
116
|
#ifndef GL_ES
|
|
121
117
|
#define highp
|
package/lib/drawplane.js
CHANGED
|
@@ -1,18 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.drawPlane = exports.disposeDrawPlane = void 0;
|
|
4
|
-
const shader_1 = require("./shader");
|
|
1
|
+
import { compileShader, linkProgram } from "./shader";
|
|
5
2
|
let shader;
|
|
6
3
|
let vbo;
|
|
7
4
|
let uvbo;
|
|
8
5
|
let texturesByUrl = {};
|
|
9
|
-
function disposeDrawPlane() {
|
|
6
|
+
export function disposeDrawPlane() {
|
|
10
7
|
shader = undefined;
|
|
11
8
|
vbo = undefined;
|
|
12
9
|
uvbo = undefined;
|
|
13
10
|
texturesByUrl = {};
|
|
14
11
|
}
|
|
15
|
-
exports.disposeDrawPlane = disposeDrawPlane;
|
|
16
12
|
function generate(gl) {
|
|
17
13
|
if (vbo)
|
|
18
14
|
return vbo;
|
|
@@ -47,7 +43,7 @@ function generateUVBO(gl) {
|
|
|
47
43
|
gl.bindBuffer(gl.ARRAY_BUFFER, null);
|
|
48
44
|
return uvbo;
|
|
49
45
|
}
|
|
50
|
-
function drawPlane(gl, projectionMatrix, cameraMatrix, targetMatrix, textureUrl) {
|
|
46
|
+
export function drawPlane(gl, projectionMatrix, cameraMatrix, targetMatrix, textureUrl) {
|
|
51
47
|
let shader = getShader(gl);
|
|
52
48
|
let v = generate(gl);
|
|
53
49
|
let uvbo = generateUVBO(gl);
|
|
@@ -69,7 +65,6 @@ function drawPlane(gl, projectionMatrix, cameraMatrix, targetMatrix, textureUrl)
|
|
|
69
65
|
gl.disableVertexAttribArray(shader.attr_position);
|
|
70
66
|
gl.bindBuffer(gl.ARRAY_BUFFER, null);
|
|
71
67
|
}
|
|
72
|
-
exports.drawPlane = drawPlane;
|
|
73
68
|
function generateLocalMatrix() {
|
|
74
69
|
let position = [0, 0, -5];
|
|
75
70
|
return new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, position[0], position[1], position[2], 1]);
|
|
@@ -118,11 +113,11 @@ function getShader(gl) {
|
|
|
118
113
|
let prog = gl.createProgram();
|
|
119
114
|
if (!prog)
|
|
120
115
|
throw new Error("Unable to create program");
|
|
121
|
-
let vertexShader =
|
|
122
|
-
let fragmentShader =
|
|
116
|
+
let vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSrc);
|
|
117
|
+
let fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentShaderSrc);
|
|
123
118
|
gl.attachShader(prog, vertexShader);
|
|
124
119
|
gl.attachShader(prog, fragmentShader);
|
|
125
|
-
|
|
120
|
+
linkProgram(gl, prog);
|
|
126
121
|
let unif_proj = gl.getUniformLocation(prog, "projMatrix");
|
|
127
122
|
if (!unif_proj)
|
|
128
123
|
throw new Error("Unable to get uniform location projMatrix");
|
package/lib/event.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Event5 = exports.Event3 = exports.Event2 = exports.Event1 = exports.Event = void 0;
|
|
4
|
-
class Event {
|
|
1
|
+
export class Event {
|
|
5
2
|
constructor() {
|
|
6
3
|
this._funcs = [];
|
|
7
4
|
}
|
|
@@ -20,8 +17,7 @@ class Event {
|
|
|
20
17
|
}
|
|
21
18
|
}
|
|
22
19
|
}
|
|
23
|
-
|
|
24
|
-
class Event1 {
|
|
20
|
+
export class Event1 {
|
|
25
21
|
constructor() {
|
|
26
22
|
this._funcs = [];
|
|
27
23
|
}
|
|
@@ -40,8 +36,7 @@ class Event1 {
|
|
|
40
36
|
}
|
|
41
37
|
}
|
|
42
38
|
}
|
|
43
|
-
|
|
44
|
-
class Event2 {
|
|
39
|
+
export class Event2 {
|
|
45
40
|
constructor() {
|
|
46
41
|
this._funcs = [];
|
|
47
42
|
}
|
|
@@ -60,8 +55,7 @@ class Event2 {
|
|
|
60
55
|
}
|
|
61
56
|
}
|
|
62
57
|
}
|
|
63
|
-
|
|
64
|
-
class Event3 {
|
|
58
|
+
export class Event3 {
|
|
65
59
|
constructor() {
|
|
66
60
|
this._funcs = [];
|
|
67
61
|
}
|
|
@@ -80,8 +74,7 @@ class Event3 {
|
|
|
80
74
|
}
|
|
81
75
|
}
|
|
82
76
|
}
|
|
83
|
-
|
|
84
|
-
class Event5 {
|
|
77
|
+
export class Event5 {
|
|
85
78
|
constructor() {
|
|
86
79
|
this._funcs = [];
|
|
87
80
|
}
|
|
@@ -100,4 +93,3 @@ class Event5 {
|
|
|
100
93
|
}
|
|
101
94
|
}
|
|
102
95
|
}
|
|
103
|
-
exports.Event5 = Event5;
|
package/lib/facelandmark.d.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { zappar_face_landmark_t, face_landmark_name_t } from "./gen/zappar";
|
|
2
|
-
import { mat4 } from "gl-matrix";
|
|
3
2
|
export declare function createFaceLandmark(n: face_landmark_name_t): zappar_face_landmark_t;
|
|
4
3
|
export declare function destroyFaceLandmark(m: zappar_face_landmark_t): void;
|
|
5
4
|
export declare function getFaceLandmark(m: zappar_face_landmark_t): FaceLandmark | undefined;
|
|
6
5
|
export declare class FaceLandmark {
|
|
7
6
|
private _name;
|
|
8
|
-
anchor_pose:
|
|
7
|
+
anchor_pose: Float32Array;
|
|
9
8
|
constructor(_name: face_landmark_name_t);
|
|
10
9
|
private _getVertex;
|
|
11
10
|
update(identity: Float32Array, expression: Float32Array, mirrored: boolean): void;
|