@zappar/zappar-cv 0.4.0-beta.9 → 2.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +2 -2
- package/lib/additional.d.ts +2 -1
- package/lib/additional.js +1 -2
- package/lib/camera-source-map.js +16 -24
- package/lib/camera-source.js +17 -21
- package/lib/cameramodel.js +18 -23
- package/lib/compatibility.js +1 -3
- package/lib/deserializer.js +1 -5
- package/lib/direct.js +12 -16
- package/lib/drawcamera.js +26 -31
- package/lib/drawface.js +5 -9
- package/lib/drawfaceproject.js +5 -9
- package/lib/drawplane.js +6 -11
- package/lib/event.js +5 -13
- package/lib/facelandmark.d.ts +1 -2
- package/lib/facelandmark.js +11 -18
- package/lib/facelandmarkdata.js +1 -4
- package/lib/facelandmarkinterface.js +1 -2
- package/lib/facemesh.js +10 -17
- package/lib/gen/zappar-client.js +5 -24
- package/lib/gen/zappar-cwrap.js +1 -12
- package/lib/gen/zappar-native.d.ts +0 -1
- package/lib/gen/zappar-native.js +10 -13
- package/lib/gen/zappar-server.js +5 -13
- package/lib/gen/zappar.d.ts +7 -0
- package/lib/gen/zappar.js +5 -13
- package/lib/gfx.js +1 -4
- package/lib/gl-state-manager.js +1 -5
- package/lib/html-element-source.js +23 -27
- package/lib/image-process-gl.js +29 -21
- package/lib/imagebitmap-camera-source.js +30 -34
- package/lib/imagetracker.d.ts +28 -0
- package/lib/imagetracker.js +115 -0
- package/lib/index-standalone.d.ts +3 -0
- package/lib/index-standalone.js +5 -0
- package/lib/index.d.ts +2 -1
- package/lib/index.js +6 -27
- package/lib/landmarkdatagenerate.js +25 -30
- package/lib/loglevel.js +9 -16
- package/lib/messages.js +4 -8
- package/lib/mstp-camera-source.js +22 -26
- package/lib/native.d.ts +2 -1
- package/lib/native.js +194 -132
- package/lib/options.d.ts +3 -0
- package/lib/options.js +1 -0
- package/lib/permission.js +13 -27
- package/lib/pipeline.d.ts +1 -2
- package/lib/pipeline.js +34 -39
- package/lib/profile.js +40 -45
- package/lib/riff-reader.d.ts +18 -0
- package/lib/riff-reader.js +52 -0
- package/lib/sequencerecorder.js +7 -12
- package/lib/sequencesource.js +20 -24
- package/lib/serializer.js +1 -5
- package/lib/shader.js +2 -7
- package/lib/source.js +1 -5
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -4
- package/lib/worker-client.d.ts +1 -1
- package/lib/worker-client.js +9 -21
- package/lib/worker-imagebitmap.js +9 -13
- package/lib/worker-server.js +33 -37
- package/lib/worker.js +6 -10
- package/lib/workerinterface.js +1 -2
- package/lib/zcv.js +57 -57
- package/lib/zcv.wasm +0 -0
- package/package.json +14 -16
- package/umd/169.zappar-cv.js +1 -0
- package/umd/{482727c0e7dd40d73f5aae47f238ad61.zbin → 482727c0e7dd40d73f5a.zbin} +0 -0
- package/umd/{a78f5ad0a44def457ee50c8bcd5fd27c.wasm → 6a8309f88f836a752c68.wasm} +0 -0
- package/umd/752.zappar-cv.js +1 -0
- package/umd/{b4f46e148a3b3b5aef90717d8d60e3fc.zbin → b4f46e148a3b3b5aef90.zbin} +0 -0
- package/umd/{f7b62a3a53b5c9b3222ba8dc53cb8b11.zbin → f7b62a3a53b5c9b3222b.zbin} +0 -0
- package/umd/zappar-cv.js +1 -1
- package/umd/zappar-cv.worker.js +1 -1
package/lib/pipeline.d.ts
CHANGED
|
@@ -3,7 +3,6 @@ import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
|
|
|
3
3
|
import { zappar_cwrap } from "./gen/zappar-native";
|
|
4
4
|
import { zappar_client } from "./gen/zappar-client";
|
|
5
5
|
import { MsgManager } from "./messages";
|
|
6
|
-
import { mat4 } from "gl-matrix";
|
|
7
6
|
import { Source, CameraFrameInfo } from "./source";
|
|
8
7
|
import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
|
|
9
8
|
import { FaceMesh } from "./facemesh";
|
|
@@ -64,4 +63,4 @@ export declare class Pipeline {
|
|
|
64
63
|
sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array): void;
|
|
65
64
|
sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
|
|
66
65
|
}
|
|
67
|
-
export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array):
|
|
66
|
+
export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): Float32Array;
|
package/lib/pipeline.js
CHANGED
|
@@ -1,19 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
const loglevel_1 = require("./loglevel");
|
|
12
|
-
const sequencerecorder_1 = require("./sequencerecorder");
|
|
13
|
-
const camera_source_map_1 = require("./camera-source-map");
|
|
1
|
+
import { cameraFrameTextureMatrix, CameraDraw } from "./drawcamera";
|
|
2
|
+
import { mat4 } from "gl-matrix";
|
|
3
|
+
import { cameraRotationForScreenOrientation } from "./cameramodel";
|
|
4
|
+
import { FaceDraw } from "./drawface";
|
|
5
|
+
import { FaceDrawProject } from "./drawfaceproject";
|
|
6
|
+
import { disposeDrawPlane } from "./drawplane";
|
|
7
|
+
import { Event } from "./event";
|
|
8
|
+
import { zcerr } from "./loglevel";
|
|
9
|
+
import { SequenceRecorder } from "./sequencerecorder";
|
|
10
|
+
import { getCameraSource } from "./camera-source-map";
|
|
14
11
|
let byId = new Map();
|
|
15
|
-
let identity =
|
|
16
|
-
class Pipeline {
|
|
12
|
+
let identity = mat4.create();
|
|
13
|
+
export class Pipeline {
|
|
17
14
|
constructor(_client, _impl, _mgr) {
|
|
18
15
|
this._client = _client;
|
|
19
16
|
this._impl = _impl;
|
|
@@ -26,7 +23,7 @@ class Pipeline {
|
|
|
26
23
|
this.cameraPixelArrays = [];
|
|
27
24
|
this._sequenceRecordDeviceAttitudeMatrices = true;
|
|
28
25
|
this._sequenceRecorderFirstCameraToken = 0;
|
|
29
|
-
this.onGLContextReset = new
|
|
26
|
+
this.onGLContextReset = new Event();
|
|
30
27
|
}
|
|
31
28
|
static create(client, mgr) {
|
|
32
29
|
let ret = client.pipeline_create();
|
|
@@ -78,7 +75,7 @@ class Pipeline {
|
|
|
78
75
|
}
|
|
79
76
|
sequenceRecordStart(expectedFrames) {
|
|
80
77
|
if (!this._sequenceRecorder)
|
|
81
|
-
this._sequenceRecorder = new
|
|
78
|
+
this._sequenceRecorder = new SequenceRecorder(expectedFrames);
|
|
82
79
|
this._sequenceRecorder.start();
|
|
83
80
|
this._sequenceRecorderFirstCameraToken = this.nextCameraToken;
|
|
84
81
|
}
|
|
@@ -116,7 +113,7 @@ class Pipeline {
|
|
|
116
113
|
if (!token)
|
|
117
114
|
return;
|
|
118
115
|
if (!this._cameraDraw)
|
|
119
|
-
this._cameraDraw = new
|
|
116
|
+
this._cameraDraw = new CameraDraw(this.glContext);
|
|
120
117
|
this._cameraDraw.drawCameraFrame(screenWidth, screenHeight, token, mirror === true);
|
|
121
118
|
}
|
|
122
119
|
glContextLost() {
|
|
@@ -129,7 +126,7 @@ class Pipeline {
|
|
|
129
126
|
delete this._cameraDraw;
|
|
130
127
|
delete this._faceDraw;
|
|
131
128
|
delete this._faceProjectDraw;
|
|
132
|
-
|
|
129
|
+
disposeDrawPlane();
|
|
133
130
|
this.onGLContextReset.emit();
|
|
134
131
|
for (let tex of this.videoTextures) {
|
|
135
132
|
if (this.glContext)
|
|
@@ -163,17 +160,17 @@ class Pipeline {
|
|
|
163
160
|
if (!this.glContext)
|
|
164
161
|
return;
|
|
165
162
|
if (!this._faceDraw)
|
|
166
|
-
this._faceDraw = new
|
|
167
|
-
let mat =
|
|
168
|
-
|
|
169
|
-
|
|
163
|
+
this._faceDraw = new FaceDraw(this.glContext);
|
|
164
|
+
let mat = mat4.create();
|
|
165
|
+
mat4.multiply(mat, projectionMatrix, cameraMatrix);
|
|
166
|
+
mat4.multiply(mat, mat, targetMatrix);
|
|
170
167
|
this._faceDraw.drawFace(mat, o);
|
|
171
168
|
}
|
|
172
169
|
drawFaceProject(matrix, vertices, uvMatrix, uvs, indices, texture) {
|
|
173
170
|
if (!this.glContext)
|
|
174
171
|
return;
|
|
175
172
|
if (!this._faceProjectDraw)
|
|
176
|
-
this._faceProjectDraw = new
|
|
173
|
+
this._faceProjectDraw = new FaceDrawProject(this.glContext);
|
|
177
174
|
this._faceProjectDraw.drawFace(matrix, vertices, uvMatrix, uvs, indices, texture);
|
|
178
175
|
}
|
|
179
176
|
cameraFrameTexture() {
|
|
@@ -183,8 +180,8 @@ class Pipeline {
|
|
|
183
180
|
cameraFrameTextureMatrix(sw, sh, mirror) {
|
|
184
181
|
let info = this.getCurrentCameraInfo();
|
|
185
182
|
if (!info)
|
|
186
|
-
return
|
|
187
|
-
return
|
|
183
|
+
return mat4.create();
|
|
184
|
+
return cameraFrameTextureMatrix(info.dataWidth, info.dataHeight, sw, sh, info.uvTransform || identity, mirror);
|
|
188
185
|
}
|
|
189
186
|
cameraFrameUserFacing() {
|
|
190
187
|
var _a;
|
|
@@ -193,17 +190,17 @@ class Pipeline {
|
|
|
193
190
|
cameraPoseWithAttitude(mirror) {
|
|
194
191
|
let res = applyScreenCounterRotation(this.getCurrentCameraInfo(), this._client.pipeline_camera_frame_camera_attitude(this._impl));
|
|
195
192
|
if (mirror) {
|
|
196
|
-
let scale =
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
193
|
+
let scale = mat4.create();
|
|
194
|
+
mat4.fromScaling(scale, [-1, 1, 1]);
|
|
195
|
+
mat4.multiply(res, scale, res);
|
|
196
|
+
mat4.multiply(res, res, scale);
|
|
200
197
|
}
|
|
201
|
-
|
|
198
|
+
mat4.invert(res, res);
|
|
202
199
|
return res;
|
|
203
200
|
}
|
|
204
201
|
videoFrameFromWorker(msg) {
|
|
205
202
|
let tokenId = this.nextCameraToken++;
|
|
206
|
-
const cameraSource =
|
|
203
|
+
const cameraSource = getCameraSource(msg.source);
|
|
207
204
|
if (!cameraSource)
|
|
208
205
|
return;
|
|
209
206
|
this.cameraTokens.set(tokenId, {
|
|
@@ -244,7 +241,7 @@ class Pipeline {
|
|
|
244
241
|
}
|
|
245
242
|
processGL() {
|
|
246
243
|
if (!this.glContext) {
|
|
247
|
-
|
|
244
|
+
zcerr("no GL context for camera frames - please call pipeline_gl_context_set");
|
|
248
245
|
return;
|
|
249
246
|
}
|
|
250
247
|
if (!this.currentCameraSource)
|
|
@@ -320,13 +317,11 @@ class Pipeline {
|
|
|
320
317
|
this._mgr.postOutgoingMessage(msg, [data]);
|
|
321
318
|
}
|
|
322
319
|
}
|
|
323
|
-
|
|
324
|
-
function applyScreenCounterRotation(info, inp) {
|
|
320
|
+
export function applyScreenCounterRotation(info, inp) {
|
|
325
321
|
let userFacing = false;
|
|
326
322
|
userFacing = info ? info.userFacing : false;
|
|
327
|
-
let mult =
|
|
328
|
-
|
|
329
|
-
|
|
323
|
+
let mult = mat4.create();
|
|
324
|
+
mat4.fromRotation(mult, cameraRotationForScreenOrientation(userFacing) * Math.PI / 180.0, [0, 0, 1]);
|
|
325
|
+
mat4.multiply(mult, mult, inp);
|
|
330
326
|
return mult;
|
|
331
327
|
}
|
|
332
|
-
exports.applyScreenCounterRotation = applyScreenCounterRotation;
|
package/lib/profile.js
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.profile = exports.EmbeddedVideoImplementation = void 0;
|
|
4
|
-
const UAParser = require("ua-parser-js");
|
|
5
|
-
var EmbeddedVideoImplementation;
|
|
1
|
+
import * as UAParser from "ua-parser-js";
|
|
2
|
+
export var EmbeddedVideoImplementation;
|
|
6
3
|
(function (EmbeddedVideoImplementation) {
|
|
7
4
|
EmbeddedVideoImplementation[EmbeddedVideoImplementation["OBJECT_URL"] = 0] = "OBJECT_URL";
|
|
8
5
|
EmbeddedVideoImplementation[EmbeddedVideoImplementation["SRC_OBJECT"] = 1] = "SRC_OBJECT";
|
|
9
|
-
})(EmbeddedVideoImplementation
|
|
10
|
-
|
|
6
|
+
})(EmbeddedVideoImplementation || (EmbeddedVideoImplementation = {}));
|
|
7
|
+
export let profile = {
|
|
11
8
|
deviceMotionMutliplier: -1.0,
|
|
12
9
|
blacklisted: false,
|
|
13
10
|
showGyroPermissionsWarningIfNecessary: false,
|
|
@@ -22,61 +19,59 @@ exports.profile = {
|
|
|
22
19
|
preferImageBitmapCamera: false
|
|
23
20
|
};
|
|
24
21
|
if (typeof window !== "undefined") {
|
|
25
|
-
window["zeeProfile"] =
|
|
22
|
+
window["zeeProfile"] = profile;
|
|
26
23
|
if (window.location.href.indexOf("_mstppipeline") >= 0) {
|
|
27
24
|
console.log("Configuring for MSTP camera pipeline (if supported)");
|
|
28
|
-
|
|
25
|
+
profile.preferMediaStreamTrackProcessorCamera = true;
|
|
29
26
|
}
|
|
30
27
|
if (window.location.href.indexOf("_imagebitmappipeline") >= 0) {
|
|
31
28
|
console.log("Configuring for ImageBitmap camera pipeline (if supported)");
|
|
32
|
-
|
|
29
|
+
profile.preferImageBitmapCamera = true;
|
|
33
30
|
}
|
|
34
31
|
}
|
|
35
32
|
let agent = new UAParser.UAParser();
|
|
36
33
|
let os = (agent.getOS().name || "unknown").toLowerCase();
|
|
37
34
|
let engine = (agent.getEngine().name || "unknown").toLowerCase();
|
|
38
35
|
if (engine === "webkit" && os !== "ios") {
|
|
39
|
-
|
|
36
|
+
profile.deviceMotionMutliplier = 1.0;
|
|
40
37
|
if (typeof window !== "undefined" && window.orientation !== undefined) {
|
|
41
38
|
// iPad
|
|
42
|
-
iDevice();
|
|
39
|
+
iDevice("15.0");
|
|
43
40
|
}
|
|
44
41
|
}
|
|
45
42
|
if (engine === "webkit" && os === "ios") {
|
|
46
|
-
|
|
47
|
-
|
|
43
|
+
profile.deviceMotionMutliplier = 1.0;
|
|
44
|
+
const version = agent.getOS().version || "15.0";
|
|
45
|
+
iDevice(version);
|
|
48
46
|
}
|
|
49
|
-
function iDevice() {
|
|
50
|
-
let
|
|
51
|
-
if (
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
exports.profile.videoHeight = 360;
|
|
78
|
-
exports.profile.dataHeight = 180;
|
|
79
|
-
}
|
|
47
|
+
function iDevice(version) {
|
|
48
|
+
let versionParts = version.split(".");
|
|
49
|
+
if (versionParts.length >= 2) {
|
|
50
|
+
const majorVersion = parseInt(versionParts[0]);
|
|
51
|
+
const minorVersion = parseInt(versionParts[1]);
|
|
52
|
+
if (majorVersion < 11 ||
|
|
53
|
+
(majorVersion === 11 && minorVersion < 3)) {
|
|
54
|
+
profile.blacklisted = true;
|
|
55
|
+
}
|
|
56
|
+
if (majorVersion < 12 ||
|
|
57
|
+
(majorVersion === 12 && minorVersion < 2)) {
|
|
58
|
+
profile.videoElementInDOM = true;
|
|
59
|
+
}
|
|
60
|
+
if ((majorVersion === 12 && minorVersion >= 2) || (majorVersion >= 13))
|
|
61
|
+
profile.showGyroPermissionsWarningIfNecessary = true;
|
|
62
|
+
if (majorVersion >= 13) {
|
|
63
|
+
profile.showSafariPermissionsResetIfNecessary = true;
|
|
64
|
+
}
|
|
65
|
+
if (((majorVersion >= 12 && minorVersion > 1) || (majorVersion >= 13)) &&
|
|
66
|
+
navigator.mediaDevices &&
|
|
67
|
+
navigator.mediaDevices.getSupportedConstraints &&
|
|
68
|
+
navigator.mediaDevices.getSupportedConstraints().frameRate) {
|
|
69
|
+
profile.requestHighFrameRate = true;
|
|
70
|
+
if (majorVersion < 14) {
|
|
71
|
+
// Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
|
|
72
|
+
// Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
|
|
73
|
+
profile.videoHeight = 360;
|
|
74
|
+
profile.dataHeight = 180;
|
|
80
75
|
}
|
|
81
76
|
}
|
|
82
77
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface Chunk {
|
|
2
|
+
ident: string;
|
|
3
|
+
data: Uint8Array;
|
|
4
|
+
subident?: string;
|
|
5
|
+
subchunks?: Chunk[];
|
|
6
|
+
}
|
|
7
|
+
export declare class RiffReader {
|
|
8
|
+
private _data;
|
|
9
|
+
private _paddingBytes;
|
|
10
|
+
truncatedChunks: boolean;
|
|
11
|
+
private _view;
|
|
12
|
+
root: Chunk;
|
|
13
|
+
constructor(_data: ArrayBuffer, _paddingBytes?: boolean);
|
|
14
|
+
private _parse;
|
|
15
|
+
private _parseChunk;
|
|
16
|
+
find(str: string, inChunk?: Chunk): Chunk | undefined;
|
|
17
|
+
has(str: string): boolean;
|
|
18
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
const decoder = new TextDecoder();
|
|
2
|
+
export class RiffReader {
|
|
3
|
+
constructor(_data, _paddingBytes = false) {
|
|
4
|
+
this._data = _data;
|
|
5
|
+
this._paddingBytes = _paddingBytes;
|
|
6
|
+
this.truncatedChunks = false;
|
|
7
|
+
this._view = new DataView(this._data);
|
|
8
|
+
this.root = this._parse();
|
|
9
|
+
}
|
|
10
|
+
_parse() {
|
|
11
|
+
const header = decoder.decode(this._data.slice(0, 4));
|
|
12
|
+
if (header !== "RIFF")
|
|
13
|
+
throw new Error("Not a valid ZPT file");
|
|
14
|
+
return this._parseChunk(0);
|
|
15
|
+
}
|
|
16
|
+
_parseChunk(offset) {
|
|
17
|
+
const ident = decoder.decode(this._data.slice(offset, offset + 4));
|
|
18
|
+
const length = this._view.getUint32(offset + 4, true);
|
|
19
|
+
const data = new Uint8Array(this._data, offset + 8, length);
|
|
20
|
+
if (data.byteLength !== length)
|
|
21
|
+
this.truncatedChunks = true;
|
|
22
|
+
let subchunks;
|
|
23
|
+
let subident;
|
|
24
|
+
if (ident === "RIFF" || ident === "LIST") {
|
|
25
|
+
subident = decoder.decode(this._data.slice(offset + 8, offset + 12));
|
|
26
|
+
subchunks = [];
|
|
27
|
+
let currentOffset = 4;
|
|
28
|
+
while (currentOffset < data.byteLength - 1) {
|
|
29
|
+
const subchunk = this._parseChunk(offset + 8 + currentOffset);
|
|
30
|
+
subchunks.push(subchunk);
|
|
31
|
+
currentOffset += subchunk.data.byteLength + 8;
|
|
32
|
+
if (this._paddingBytes && subchunk.data.byteLength & 0x1)
|
|
33
|
+
currentOffset++;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return { ident, data, subident, subchunks };
|
|
37
|
+
}
|
|
38
|
+
find(str, inChunk = this.root) {
|
|
39
|
+
if (inChunk.ident === str)
|
|
40
|
+
return inChunk;
|
|
41
|
+
if (!inChunk.subchunks)
|
|
42
|
+
return;
|
|
43
|
+
for (let c of inChunk.subchunks) {
|
|
44
|
+
const found = this.find(str, c);
|
|
45
|
+
if (found)
|
|
46
|
+
return found;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
has(str) {
|
|
50
|
+
return this.find(str) !== undefined;
|
|
51
|
+
}
|
|
52
|
+
}
|
package/lib/sequencerecorder.js
CHANGED
|
@@ -1,16 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.SequenceDecoder = exports.SequenceRecorder = exports.SequenceRecorderPacketType = void 0;
|
|
4
|
-
const loglevel_1 = require("./loglevel");
|
|
5
|
-
var SequenceRecorderPacketType;
|
|
1
|
+
import { zcerr } from "./loglevel";
|
|
2
|
+
export var SequenceRecorderPacketType;
|
|
6
3
|
(function (SequenceRecorderPacketType) {
|
|
7
4
|
SequenceRecorderPacketType[SequenceRecorderPacketType["CAMERA_FRAME"] = 0] = "CAMERA_FRAME";
|
|
8
5
|
SequenceRecorderPacketType[SequenceRecorderPacketType["ACCELEROMETER"] = 1] = "ACCELEROMETER";
|
|
9
6
|
SequenceRecorderPacketType[SequenceRecorderPacketType["ROTATION_RATE"] = 2] = "ROTATION_RATE";
|
|
10
7
|
SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE"] = 3] = "ATTITUDE";
|
|
11
8
|
SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE_MATRIX"] = 4] = "ATTITUDE_MATRIX";
|
|
12
|
-
})(SequenceRecorderPacketType
|
|
13
|
-
class SequenceRecorder {
|
|
9
|
+
})(SequenceRecorderPacketType || (SequenceRecorderPacketType = {}));
|
|
10
|
+
export class SequenceRecorder {
|
|
14
11
|
constructor(estimatedFrames) {
|
|
15
12
|
this._insertionByte = 0;
|
|
16
13
|
this._numberPackets = 0;
|
|
@@ -53,11 +50,11 @@ class SequenceRecorder {
|
|
|
53
50
|
if (this._hasLoggedUnsuppored)
|
|
54
51
|
return;
|
|
55
52
|
this._hasLoggedUnsuppored = true;
|
|
56
|
-
|
|
53
|
+
zcerr("Unable to record frames from this camera source");
|
|
57
54
|
return;
|
|
58
55
|
}
|
|
59
56
|
if (c.data.byteLength !== (c.dataWidth * c.dataHeight)) {
|
|
60
|
-
|
|
57
|
+
zcerr("Unable to record frame to sequence: greyscale array doesn't match dimensions");
|
|
61
58
|
return;
|
|
62
59
|
}
|
|
63
60
|
let increment = SequenceRecorder.psbPacketType + SequenceRecorder.cameraPacketSizeBytes + (c.dataWidth * c.dataHeight);
|
|
@@ -131,7 +128,6 @@ class SequenceRecorder {
|
|
|
131
128
|
}
|
|
132
129
|
}
|
|
133
130
|
}
|
|
134
|
-
exports.SequenceRecorder = SequenceRecorder;
|
|
135
131
|
// Header (8 bytes): "UAR1" + number of frames (uint32)
|
|
136
132
|
SequenceRecorder.headerSizeBytes = 8;
|
|
137
133
|
// Per-frame packet byte sizes
|
|
@@ -155,7 +151,7 @@ SequenceRecorder.rotationRatePacketSizeBytes = 4 * 4; // t, x, y, z
|
|
|
155
151
|
SequenceRecorder.attitudePacketSizeBytes = 4 * 4; // t, x, y, z
|
|
156
152
|
// Attitude matrix
|
|
157
153
|
SequenceRecorder.attitudeMatrixPacketSizeBytes = 4 * 16;
|
|
158
|
-
class SequenceDecoder {
|
|
154
|
+
export class SequenceDecoder {
|
|
159
155
|
constructor(_data) {
|
|
160
156
|
this._data = _data;
|
|
161
157
|
this.dataByPacket = new Map();
|
|
@@ -280,4 +276,3 @@ class SequenceDecoder {
|
|
|
280
276
|
}
|
|
281
277
|
}
|
|
282
278
|
}
|
|
283
|
-
exports.SequenceDecoder = SequenceDecoder;
|
package/lib/sequencesource.js
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const source_1 = require("./source");
|
|
7
|
-
const pipeline_1 = require("./pipeline");
|
|
8
|
-
const gl_matrix_1 = require("gl-matrix");
|
|
1
|
+
import { zcerr, zcout } from "./loglevel";
|
|
2
|
+
import { SequenceDecoder, SequenceRecorderPacketType } from "./sequencerecorder";
|
|
3
|
+
import { Source } from "./source";
|
|
4
|
+
import { Pipeline } from "./pipeline";
|
|
5
|
+
import { mat4 } from "gl-matrix";
|
|
9
6
|
let latest = 1;
|
|
10
7
|
let byId = new Map();
|
|
11
|
-
class SequenceSource extends
|
|
8
|
+
export class SequenceSource extends Source {
|
|
12
9
|
constructor(_impl, _pipeline) {
|
|
13
10
|
super();
|
|
14
11
|
this._impl = _impl;
|
|
@@ -22,7 +19,7 @@ class SequenceSource extends source_1.Source {
|
|
|
22
19
|
static create(p) {
|
|
23
20
|
let ret = (latest++);
|
|
24
21
|
byId.set(ret, new SequenceSource(ret, p));
|
|
25
|
-
|
|
22
|
+
zcout("sequence_source_source_t initialized");
|
|
26
23
|
return ret;
|
|
27
24
|
}
|
|
28
25
|
static get(m) {
|
|
@@ -30,12 +27,12 @@ class SequenceSource extends source_1.Source {
|
|
|
30
27
|
}
|
|
31
28
|
loadFromMemory(data) {
|
|
32
29
|
try {
|
|
33
|
-
this._decoder = new
|
|
30
|
+
this._decoder = new SequenceDecoder(data);
|
|
34
31
|
this._packetNumber = 0;
|
|
35
32
|
this._lastUpdateTime = 0;
|
|
36
33
|
}
|
|
37
34
|
catch (err) {
|
|
38
|
-
|
|
35
|
+
zcerr("Unable to decode sequence:", err);
|
|
39
36
|
}
|
|
40
37
|
}
|
|
41
38
|
maxPlaybackFpsSet(fps) {
|
|
@@ -46,7 +43,7 @@ class SequenceSource extends source_1.Source {
|
|
|
46
43
|
this._minFrameGapMs = 1000.0 / fps;
|
|
47
44
|
}
|
|
48
45
|
getFrame(currentlyProcessing) {
|
|
49
|
-
let pipeline =
|
|
46
|
+
let pipeline = Pipeline.get(this._pipeline);
|
|
50
47
|
if (!pipeline)
|
|
51
48
|
return;
|
|
52
49
|
if (!this._decoder)
|
|
@@ -62,7 +59,7 @@ class SequenceSource extends source_1.Source {
|
|
|
62
59
|
if (this._packetNumber >= this._decoder.numberPackets)
|
|
63
60
|
return;
|
|
64
61
|
let packet = this._decoder.dataByPacket.get(this._packetNumber);
|
|
65
|
-
if (packet.type !=
|
|
62
|
+
if (packet.type != SequenceRecorderPacketType.CAMERA_FRAME)
|
|
66
63
|
return;
|
|
67
64
|
this._packetNumber++;
|
|
68
65
|
let pixels = pipeline.cameraPixelArrays.pop();
|
|
@@ -85,7 +82,7 @@ class SequenceSource extends source_1.Source {
|
|
|
85
82
|
dataHeight: packet.height,
|
|
86
83
|
texture: undefined,
|
|
87
84
|
userFacing: packet.userFacing,
|
|
88
|
-
uvTransform:
|
|
85
|
+
uvTransform: mat4.create(),
|
|
89
86
|
cameraSource: this
|
|
90
87
|
};
|
|
91
88
|
// We can queue the motion updates for the follwing frame immediately
|
|
@@ -101,19 +98,19 @@ class SequenceSource extends source_1.Source {
|
|
|
101
98
|
}
|
|
102
99
|
start() {
|
|
103
100
|
var _a;
|
|
104
|
-
let p =
|
|
101
|
+
let p = Pipeline.get(this._pipeline);
|
|
105
102
|
if (p && p.currentCameraSource !== this) {
|
|
106
103
|
(_a = p.currentCameraSource) === null || _a === void 0 ? void 0 : _a.pause();
|
|
107
104
|
p.currentCameraSource = this;
|
|
108
105
|
}
|
|
109
106
|
}
|
|
110
107
|
pause() {
|
|
111
|
-
let p =
|
|
108
|
+
let p = Pipeline.get(this._pipeline);
|
|
112
109
|
if (p && p.currentCameraSource === this)
|
|
113
110
|
p.currentCameraSource = undefined;
|
|
114
111
|
}
|
|
115
112
|
uploadGL(info) {
|
|
116
|
-
let pipeline =
|
|
113
|
+
let pipeline = Pipeline.get(this._pipeline);
|
|
117
114
|
if (!pipeline)
|
|
118
115
|
return;
|
|
119
116
|
let gl = pipeline === null || pipeline === void 0 ? void 0 : pipeline.glContext;
|
|
@@ -134,22 +131,22 @@ class SequenceSource extends source_1.Source {
|
|
|
134
131
|
let pendingMotionUpdate = false;
|
|
135
132
|
while (this._packetNumber < this._decoder.numberPackets) {
|
|
136
133
|
let packet = this._decoder.dataByPacket.get(this._packetNumber);
|
|
137
|
-
if (packet.type ==
|
|
134
|
+
if (packet.type == SequenceRecorderPacketType.CAMERA_FRAME)
|
|
138
135
|
break;
|
|
139
136
|
switch (packet.type) {
|
|
140
|
-
case
|
|
137
|
+
case SequenceRecorderPacketType.ACCELEROMETER:
|
|
141
138
|
pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
|
|
142
139
|
pendingMotionUpdate = true;
|
|
143
140
|
break;
|
|
144
|
-
case
|
|
141
|
+
case SequenceRecorderPacketType.ROTATION_RATE:
|
|
145
142
|
pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
|
|
146
143
|
pendingMotionUpdate = true;
|
|
147
144
|
break;
|
|
148
|
-
case
|
|
145
|
+
case SequenceRecorderPacketType.ATTITUDE:
|
|
149
146
|
pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
|
|
150
147
|
pendingMotionUpdate = true;
|
|
151
148
|
break;
|
|
152
|
-
case
|
|
149
|
+
case SequenceRecorderPacketType.ATTITUDE_MATRIX:
|
|
153
150
|
pipeline.motionAttitudeMatrix(packet.attitude);
|
|
154
151
|
pendingMotionUpdate = true;
|
|
155
152
|
break;
|
|
@@ -167,4 +164,3 @@ class SequenceSource extends source_1.Source {
|
|
|
167
164
|
return;
|
|
168
165
|
}
|
|
169
166
|
}
|
|
170
|
-
exports.SequenceSource = SequenceSource;
|
package/lib/serializer.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.MessageSerializer = void 0;
|
|
4
|
-
class MessageSerializer {
|
|
1
|
+
export class MessageSerializer {
|
|
5
2
|
constructor(_messageSender) {
|
|
6
3
|
this._messageSender = _messageSender;
|
|
7
4
|
this._freeBufferPool = [];
|
|
@@ -229,4 +226,3 @@ class MessageSerializer {
|
|
|
229
226
|
this._offset++;
|
|
230
227
|
}
|
|
231
228
|
}
|
|
232
|
-
exports.MessageSerializer = MessageSerializer;
|
package/lib/shader.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.linkProgram = exports.compileShader = void 0;
|
|
4
|
-
function compileShader(gl, type, src) {
|
|
1
|
+
export function compileShader(gl, type, src) {
|
|
5
2
|
let ret = gl.createShader(type);
|
|
6
3
|
if (!ret)
|
|
7
4
|
throw new Error("Unable to create shader");
|
|
@@ -12,11 +9,9 @@ function compileShader(gl, type, src) {
|
|
|
12
9
|
throw new Error("Shader compile error: " + msg);
|
|
13
10
|
return ret;
|
|
14
11
|
}
|
|
15
|
-
|
|
16
|
-
function linkProgram(gl, prog) {
|
|
12
|
+
export function linkProgram(gl, prog) {
|
|
17
13
|
gl.linkProgram(prog);
|
|
18
14
|
let msg = gl.getProgramInfoLog(prog);
|
|
19
15
|
if (msg && msg.trim().length > 0)
|
|
20
16
|
throw new Error("Unable to link: " + msg);
|
|
21
17
|
}
|
|
22
|
-
exports.linkProgram = linkProgram;
|
package/lib/source.js
CHANGED
package/lib/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "0.
|
|
1
|
+
export declare const VERSION = "2.0.0-beta.0";
|
package/lib/version.js
CHANGED
package/lib/worker-client.d.ts
CHANGED
package/lib/worker-client.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,41 +7,30 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
8
|
});
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
exports.messageManager = new messages_1.MsgManager();
|
|
15
|
-
let Worker;
|
|
16
|
-
if (typeof Z_STANDALONE !== 'undefined' && Z_STANDALONE) {
|
|
17
|
-
Worker = require('worker-loader?inline=fallback!./worker');
|
|
18
|
-
}
|
|
19
|
-
else {
|
|
20
|
-
Worker = require('worker-loader!./worker');
|
|
21
|
-
}
|
|
22
|
-
if (Worker.default)
|
|
23
|
-
Worker = Worker.default;
|
|
24
|
-
function launchWorker() {
|
|
10
|
+
import { MsgManager } from "./messages";
|
|
11
|
+
export let messageManager = new MsgManager();
|
|
12
|
+
export function launchWorker(worker) {
|
|
25
13
|
return __awaiter(this, void 0, void 0, function* () {
|
|
26
|
-
|
|
14
|
+
if (!worker)
|
|
15
|
+
worker = new Worker(new URL("./worker", import.meta.url), { type: 'module' });
|
|
27
16
|
worker.postMessage({
|
|
28
17
|
t: "wasm",
|
|
29
|
-
url:
|
|
18
|
+
url: new URL("./zcv.wasm", import.meta.url).toString()
|
|
30
19
|
});
|
|
31
20
|
yield waitForLoad(worker);
|
|
32
21
|
function sendOutgoing() {
|
|
33
|
-
let msgs =
|
|
22
|
+
let msgs = messageManager.getOutgoingMessages();
|
|
34
23
|
for (let msg of msgs) {
|
|
35
24
|
worker.postMessage(msg.msg, msg.transferables);
|
|
36
25
|
}
|
|
37
26
|
}
|
|
38
|
-
|
|
27
|
+
messageManager.onOutgoingMessage.bind(sendOutgoing);
|
|
39
28
|
sendOutgoing();
|
|
40
29
|
worker.addEventListener("message", evt => {
|
|
41
|
-
|
|
30
|
+
messageManager.postIncomingMessage(evt.data);
|
|
42
31
|
});
|
|
43
32
|
});
|
|
44
33
|
}
|
|
45
|
-
exports.launchWorker = launchWorker;
|
|
46
34
|
function waitForLoad(w) {
|
|
47
35
|
return new Promise(resolve => {
|
|
48
36
|
let listener = (msg) => {
|