@zappar/zappar-cv 0.4.0-beta.8 → 2.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +2 -2
- package/lib/additional.d.ts +3 -1
- package/lib/additional.js +1 -2
- package/lib/camera-source-map.js +16 -24
- package/lib/camera-source.js +17 -21
- package/lib/cameramodel.js +18 -23
- package/lib/compatibility.js +1 -3
- package/lib/deserializer.js +1 -5
- package/lib/direct.js +12 -16
- package/lib/drawcamera.js +26 -31
- package/lib/drawface.js +5 -9
- package/lib/drawfaceproject.js +5 -9
- package/lib/drawplane.js +6 -11
- package/lib/event.js +5 -13
- package/lib/facelandmark.d.ts +1 -2
- package/lib/facelandmark.js +11 -18
- package/lib/facelandmarkdata.js +1 -4
- package/lib/facelandmarkinterface.js +1 -2
- package/lib/facemesh.js +10 -17
- package/lib/gen/zappar-client.js +5 -24
- package/lib/gen/zappar-cwrap.js +1 -12
- package/lib/gen/zappar-native.d.ts +0 -1
- package/lib/gen/zappar-native.js +10 -13
- package/lib/gen/zappar-server.js +5 -13
- package/lib/gen/zappar.d.ts +7 -2
- package/lib/gen/zappar.js +5 -13
- package/lib/gfx.js +1 -4
- package/lib/gl-state-manager.d.ts +9 -0
- package/lib/gl-state-manager.js +31 -0
- package/lib/html-element-source.js +23 -27
- package/lib/image-process-gl.js +73 -32
- package/lib/imagebitmap-camera-source.js +30 -34
- package/lib/imagetracker.d.ts +28 -0
- package/lib/imagetracker.js +115 -0
- package/lib/index-standalone.d.ts +3 -0
- package/lib/index-standalone.js +5 -0
- package/lib/index.d.ts +2 -1
- package/lib/index.js +6 -27
- package/lib/landmarkdatagenerate.js +25 -30
- package/lib/loglevel.js +9 -16
- package/lib/messages.js +4 -8
- package/lib/mstp-camera-source.js +22 -26
- package/lib/native.d.ts +2 -1
- package/lib/native.js +194 -132
- package/lib/options.d.ts +3 -0
- package/lib/options.js +1 -0
- package/lib/permission.js +13 -27
- package/lib/pipeline.d.ts +1 -2
- package/lib/pipeline.js +34 -39
- package/lib/profile.js +40 -45
- package/lib/riff-reader.d.ts +18 -0
- package/lib/riff-reader.js +52 -0
- package/lib/sequencerecorder.js +7 -12
- package/lib/sequencesource.d.ts +2 -0
- package/lib/sequencesource.js +28 -25
- package/lib/serializer.js +1 -5
- package/lib/shader.js +2 -7
- package/lib/source.js +1 -5
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -4
- package/lib/worker-client.d.ts +1 -1
- package/lib/worker-client.js +9 -21
- package/lib/worker-imagebitmap.js +9 -13
- package/lib/worker-server.js +33 -37
- package/lib/worker.js +6 -10
- package/lib/workerinterface.js +1 -2
- package/lib/zcv.js +125 -115
- package/lib/zcv.wasm +0 -0
- package/package.json +14 -16
- package/umd/169.zappar-cv.js +1 -0
- package/umd/{482727c0e7dd40d73f5aae47f238ad61.zbin → 482727c0e7dd40d73f5a.zbin} +0 -0
- package/umd/557cf735d27efb96e262.wasm +0 -0
- package/umd/752.zappar-cv.js +1 -0
- package/umd/{b4f46e148a3b3b5aef90717d8d60e3fc.zbin → b4f46e148a3b3b5aef90.zbin} +0 -0
- package/umd/{f7b62a3a53b5c9b3222ba8dc53cb8b11.zbin → f7b62a3a53b5c9b3222b.zbin} +0 -0
- package/umd/zappar-cv.js +1 -1
- package/umd/zappar-cv.worker.js +1 -1
- package/umd/fbf9061bc2d2826ecf97c8ba2b51e7a5.wasm +0 -0
package/lib/pipeline.d.ts
CHANGED
|
@@ -3,7 +3,6 @@ import { zappar_pipeline_t, zappar_camera_source_t } from "./gen/zappar";
|
|
|
3
3
|
import { zappar_cwrap } from "./gen/zappar-native";
|
|
4
4
|
import { zappar_client } from "./gen/zappar-client";
|
|
5
5
|
import { MsgManager } from "./messages";
|
|
6
|
-
import { mat4 } from "gl-matrix";
|
|
7
6
|
import { Source, CameraFrameInfo } from "./source";
|
|
8
7
|
import { CameraFrameReturnS2C, ImageBitmapS2C, VideoFrameS2C } from "./workerinterface";
|
|
9
8
|
import { FaceMesh } from "./facemesh";
|
|
@@ -64,4 +63,4 @@ export declare class Pipeline {
|
|
|
64
63
|
sendImageBitmapToWorker(img: ImageBitmap, rot: number, userFacing: boolean, tokenId: number, cameraModel: Float32Array, cameraToDevice: Float32Array): void;
|
|
65
64
|
sendDataToWorker(data: ArrayBuffer, token: number, width: number, height: number, userFacing: boolean, cameraToDevice: Float32Array, cameraModel: Float32Array): void;
|
|
66
65
|
}
|
|
67
|
-
export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array):
|
|
66
|
+
export declare function applyScreenCounterRotation(info: CameraFrameInfo | undefined, inp: Float32Array): Float32Array;
|
package/lib/pipeline.js
CHANGED
|
@@ -1,19 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
const loglevel_1 = require("./loglevel");
|
|
12
|
-
const sequencerecorder_1 = require("./sequencerecorder");
|
|
13
|
-
const camera_source_map_1 = require("./camera-source-map");
|
|
1
|
+
import { cameraFrameTextureMatrix, CameraDraw } from "./drawcamera";
|
|
2
|
+
import { mat4 } from "gl-matrix";
|
|
3
|
+
import { cameraRotationForScreenOrientation } from "./cameramodel";
|
|
4
|
+
import { FaceDraw } from "./drawface";
|
|
5
|
+
import { FaceDrawProject } from "./drawfaceproject";
|
|
6
|
+
import { disposeDrawPlane } from "./drawplane";
|
|
7
|
+
import { Event } from "./event";
|
|
8
|
+
import { zcerr } from "./loglevel";
|
|
9
|
+
import { SequenceRecorder } from "./sequencerecorder";
|
|
10
|
+
import { getCameraSource } from "./camera-source-map";
|
|
14
11
|
let byId = new Map();
|
|
15
|
-
let identity =
|
|
16
|
-
class Pipeline {
|
|
12
|
+
let identity = mat4.create();
|
|
13
|
+
export class Pipeline {
|
|
17
14
|
constructor(_client, _impl, _mgr) {
|
|
18
15
|
this._client = _client;
|
|
19
16
|
this._impl = _impl;
|
|
@@ -26,7 +23,7 @@ class Pipeline {
|
|
|
26
23
|
this.cameraPixelArrays = [];
|
|
27
24
|
this._sequenceRecordDeviceAttitudeMatrices = true;
|
|
28
25
|
this._sequenceRecorderFirstCameraToken = 0;
|
|
29
|
-
this.onGLContextReset = new
|
|
26
|
+
this.onGLContextReset = new Event();
|
|
30
27
|
}
|
|
31
28
|
static create(client, mgr) {
|
|
32
29
|
let ret = client.pipeline_create();
|
|
@@ -78,7 +75,7 @@ class Pipeline {
|
|
|
78
75
|
}
|
|
79
76
|
sequenceRecordStart(expectedFrames) {
|
|
80
77
|
if (!this._sequenceRecorder)
|
|
81
|
-
this._sequenceRecorder = new
|
|
78
|
+
this._sequenceRecorder = new SequenceRecorder(expectedFrames);
|
|
82
79
|
this._sequenceRecorder.start();
|
|
83
80
|
this._sequenceRecorderFirstCameraToken = this.nextCameraToken;
|
|
84
81
|
}
|
|
@@ -116,7 +113,7 @@ class Pipeline {
|
|
|
116
113
|
if (!token)
|
|
117
114
|
return;
|
|
118
115
|
if (!this._cameraDraw)
|
|
119
|
-
this._cameraDraw = new
|
|
116
|
+
this._cameraDraw = new CameraDraw(this.glContext);
|
|
120
117
|
this._cameraDraw.drawCameraFrame(screenWidth, screenHeight, token, mirror === true);
|
|
121
118
|
}
|
|
122
119
|
glContextLost() {
|
|
@@ -129,7 +126,7 @@ class Pipeline {
|
|
|
129
126
|
delete this._cameraDraw;
|
|
130
127
|
delete this._faceDraw;
|
|
131
128
|
delete this._faceProjectDraw;
|
|
132
|
-
|
|
129
|
+
disposeDrawPlane();
|
|
133
130
|
this.onGLContextReset.emit();
|
|
134
131
|
for (let tex of this.videoTextures) {
|
|
135
132
|
if (this.glContext)
|
|
@@ -163,17 +160,17 @@ class Pipeline {
|
|
|
163
160
|
if (!this.glContext)
|
|
164
161
|
return;
|
|
165
162
|
if (!this._faceDraw)
|
|
166
|
-
this._faceDraw = new
|
|
167
|
-
let mat =
|
|
168
|
-
|
|
169
|
-
|
|
163
|
+
this._faceDraw = new FaceDraw(this.glContext);
|
|
164
|
+
let mat = mat4.create();
|
|
165
|
+
mat4.multiply(mat, projectionMatrix, cameraMatrix);
|
|
166
|
+
mat4.multiply(mat, mat, targetMatrix);
|
|
170
167
|
this._faceDraw.drawFace(mat, o);
|
|
171
168
|
}
|
|
172
169
|
drawFaceProject(matrix, vertices, uvMatrix, uvs, indices, texture) {
|
|
173
170
|
if (!this.glContext)
|
|
174
171
|
return;
|
|
175
172
|
if (!this._faceProjectDraw)
|
|
176
|
-
this._faceProjectDraw = new
|
|
173
|
+
this._faceProjectDraw = new FaceDrawProject(this.glContext);
|
|
177
174
|
this._faceProjectDraw.drawFace(matrix, vertices, uvMatrix, uvs, indices, texture);
|
|
178
175
|
}
|
|
179
176
|
cameraFrameTexture() {
|
|
@@ -183,8 +180,8 @@ class Pipeline {
|
|
|
183
180
|
cameraFrameTextureMatrix(sw, sh, mirror) {
|
|
184
181
|
let info = this.getCurrentCameraInfo();
|
|
185
182
|
if (!info)
|
|
186
|
-
return
|
|
187
|
-
return
|
|
183
|
+
return mat4.create();
|
|
184
|
+
return cameraFrameTextureMatrix(info.dataWidth, info.dataHeight, sw, sh, info.uvTransform || identity, mirror);
|
|
188
185
|
}
|
|
189
186
|
cameraFrameUserFacing() {
|
|
190
187
|
var _a;
|
|
@@ -193,17 +190,17 @@ class Pipeline {
|
|
|
193
190
|
cameraPoseWithAttitude(mirror) {
|
|
194
191
|
let res = applyScreenCounterRotation(this.getCurrentCameraInfo(), this._client.pipeline_camera_frame_camera_attitude(this._impl));
|
|
195
192
|
if (mirror) {
|
|
196
|
-
let scale =
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
193
|
+
let scale = mat4.create();
|
|
194
|
+
mat4.fromScaling(scale, [-1, 1, 1]);
|
|
195
|
+
mat4.multiply(res, scale, res);
|
|
196
|
+
mat4.multiply(res, res, scale);
|
|
200
197
|
}
|
|
201
|
-
|
|
198
|
+
mat4.invert(res, res);
|
|
202
199
|
return res;
|
|
203
200
|
}
|
|
204
201
|
videoFrameFromWorker(msg) {
|
|
205
202
|
let tokenId = this.nextCameraToken++;
|
|
206
|
-
const cameraSource =
|
|
203
|
+
const cameraSource = getCameraSource(msg.source);
|
|
207
204
|
if (!cameraSource)
|
|
208
205
|
return;
|
|
209
206
|
this.cameraTokens.set(tokenId, {
|
|
@@ -244,7 +241,7 @@ class Pipeline {
|
|
|
244
241
|
}
|
|
245
242
|
processGL() {
|
|
246
243
|
if (!this.glContext) {
|
|
247
|
-
|
|
244
|
+
zcerr("no GL context for camera frames - please call pipeline_gl_context_set");
|
|
248
245
|
return;
|
|
249
246
|
}
|
|
250
247
|
if (!this.currentCameraSource)
|
|
@@ -320,13 +317,11 @@ class Pipeline {
|
|
|
320
317
|
this._mgr.postOutgoingMessage(msg, [data]);
|
|
321
318
|
}
|
|
322
319
|
}
|
|
323
|
-
|
|
324
|
-
function applyScreenCounterRotation(info, inp) {
|
|
320
|
+
export function applyScreenCounterRotation(info, inp) {
|
|
325
321
|
let userFacing = false;
|
|
326
322
|
userFacing = info ? info.userFacing : false;
|
|
327
|
-
let mult =
|
|
328
|
-
|
|
329
|
-
|
|
323
|
+
let mult = mat4.create();
|
|
324
|
+
mat4.fromRotation(mult, cameraRotationForScreenOrientation(userFacing) * Math.PI / 180.0, [0, 0, 1]);
|
|
325
|
+
mat4.multiply(mult, mult, inp);
|
|
330
326
|
return mult;
|
|
331
327
|
}
|
|
332
|
-
exports.applyScreenCounterRotation = applyScreenCounterRotation;
|
package/lib/profile.js
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.profile = exports.EmbeddedVideoImplementation = void 0;
|
|
4
|
-
const UAParser = require("ua-parser-js");
|
|
5
|
-
var EmbeddedVideoImplementation;
|
|
1
|
+
import * as UAParser from "ua-parser-js";
|
|
2
|
+
export var EmbeddedVideoImplementation;
|
|
6
3
|
(function (EmbeddedVideoImplementation) {
|
|
7
4
|
EmbeddedVideoImplementation[EmbeddedVideoImplementation["OBJECT_URL"] = 0] = "OBJECT_URL";
|
|
8
5
|
EmbeddedVideoImplementation[EmbeddedVideoImplementation["SRC_OBJECT"] = 1] = "SRC_OBJECT";
|
|
9
|
-
})(EmbeddedVideoImplementation
|
|
10
|
-
|
|
6
|
+
})(EmbeddedVideoImplementation || (EmbeddedVideoImplementation = {}));
|
|
7
|
+
export let profile = {
|
|
11
8
|
deviceMotionMutliplier: -1.0,
|
|
12
9
|
blacklisted: false,
|
|
13
10
|
showGyroPermissionsWarningIfNecessary: false,
|
|
@@ -22,61 +19,59 @@ exports.profile = {
|
|
|
22
19
|
preferImageBitmapCamera: false
|
|
23
20
|
};
|
|
24
21
|
if (typeof window !== "undefined") {
|
|
25
|
-
window["zeeProfile"] =
|
|
22
|
+
window["zeeProfile"] = profile;
|
|
26
23
|
if (window.location.href.indexOf("_mstppipeline") >= 0) {
|
|
27
24
|
console.log("Configuring for MSTP camera pipeline (if supported)");
|
|
28
|
-
|
|
25
|
+
profile.preferMediaStreamTrackProcessorCamera = true;
|
|
29
26
|
}
|
|
30
27
|
if (window.location.href.indexOf("_imagebitmappipeline") >= 0) {
|
|
31
28
|
console.log("Configuring for ImageBitmap camera pipeline (if supported)");
|
|
32
|
-
|
|
29
|
+
profile.preferImageBitmapCamera = true;
|
|
33
30
|
}
|
|
34
31
|
}
|
|
35
32
|
let agent = new UAParser.UAParser();
|
|
36
33
|
let os = (agent.getOS().name || "unknown").toLowerCase();
|
|
37
34
|
let engine = (agent.getEngine().name || "unknown").toLowerCase();
|
|
38
35
|
if (engine === "webkit" && os !== "ios") {
|
|
39
|
-
|
|
36
|
+
profile.deviceMotionMutliplier = 1.0;
|
|
40
37
|
if (typeof window !== "undefined" && window.orientation !== undefined) {
|
|
41
38
|
// iPad
|
|
42
|
-
iDevice();
|
|
39
|
+
iDevice("15.0");
|
|
43
40
|
}
|
|
44
41
|
}
|
|
45
42
|
if (engine === "webkit" && os === "ios") {
|
|
46
|
-
|
|
47
|
-
|
|
43
|
+
profile.deviceMotionMutliplier = 1.0;
|
|
44
|
+
const version = agent.getOS().version || "15.0";
|
|
45
|
+
iDevice(version);
|
|
48
46
|
}
|
|
49
|
-
function iDevice() {
|
|
50
|
-
let
|
|
51
|
-
if (
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
exports.profile.videoHeight = 360;
|
|
78
|
-
exports.profile.dataHeight = 180;
|
|
79
|
-
}
|
|
47
|
+
function iDevice(version) {
|
|
48
|
+
let versionParts = version.split(".");
|
|
49
|
+
if (versionParts.length >= 2) {
|
|
50
|
+
const majorVersion = parseInt(versionParts[0]);
|
|
51
|
+
const minorVersion = parseInt(versionParts[1]);
|
|
52
|
+
if (majorVersion < 11 ||
|
|
53
|
+
(majorVersion === 11 && minorVersion < 3)) {
|
|
54
|
+
profile.blacklisted = true;
|
|
55
|
+
}
|
|
56
|
+
if (majorVersion < 12 ||
|
|
57
|
+
(majorVersion === 12 && minorVersion < 2)) {
|
|
58
|
+
profile.videoElementInDOM = true;
|
|
59
|
+
}
|
|
60
|
+
if ((majorVersion === 12 && minorVersion >= 2) || (majorVersion >= 13))
|
|
61
|
+
profile.showGyroPermissionsWarningIfNecessary = true;
|
|
62
|
+
if (majorVersion >= 13) {
|
|
63
|
+
profile.showSafariPermissionsResetIfNecessary = true;
|
|
64
|
+
}
|
|
65
|
+
if (((majorVersion >= 12 && minorVersion > 1) || (majorVersion >= 13)) &&
|
|
66
|
+
navigator.mediaDevices &&
|
|
67
|
+
navigator.mediaDevices.getSupportedConstraints &&
|
|
68
|
+
navigator.mediaDevices.getSupportedConstraints().frameRate) {
|
|
69
|
+
profile.requestHighFrameRate = true;
|
|
70
|
+
if (majorVersion < 14) {
|
|
71
|
+
// Avoid bug where iOS letterboxes 16:9 into 4:3 for high fps
|
|
72
|
+
// Doesn't seem needed with newer versions of iOS, tested on 14.8.1 and 15.2
|
|
73
|
+
profile.videoHeight = 360;
|
|
74
|
+
profile.dataHeight = 180;
|
|
80
75
|
}
|
|
81
76
|
}
|
|
82
77
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface Chunk {
|
|
2
|
+
ident: string;
|
|
3
|
+
data: Uint8Array;
|
|
4
|
+
subident?: string;
|
|
5
|
+
subchunks?: Chunk[];
|
|
6
|
+
}
|
|
7
|
+
export declare class RiffReader {
|
|
8
|
+
private _data;
|
|
9
|
+
private _paddingBytes;
|
|
10
|
+
truncatedChunks: boolean;
|
|
11
|
+
private _view;
|
|
12
|
+
root: Chunk;
|
|
13
|
+
constructor(_data: ArrayBuffer, _paddingBytes?: boolean);
|
|
14
|
+
private _parse;
|
|
15
|
+
private _parseChunk;
|
|
16
|
+
find(str: string, inChunk?: Chunk): Chunk | undefined;
|
|
17
|
+
has(str: string): boolean;
|
|
18
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
const decoder = new TextDecoder();
|
|
2
|
+
export class RiffReader {
|
|
3
|
+
constructor(_data, _paddingBytes = false) {
|
|
4
|
+
this._data = _data;
|
|
5
|
+
this._paddingBytes = _paddingBytes;
|
|
6
|
+
this.truncatedChunks = false;
|
|
7
|
+
this._view = new DataView(this._data);
|
|
8
|
+
this.root = this._parse();
|
|
9
|
+
}
|
|
10
|
+
_parse() {
|
|
11
|
+
const header = decoder.decode(this._data.slice(0, 4));
|
|
12
|
+
if (header !== "RIFF")
|
|
13
|
+
throw new Error("Not a valid ZPT file");
|
|
14
|
+
return this._parseChunk(0);
|
|
15
|
+
}
|
|
16
|
+
_parseChunk(offset) {
|
|
17
|
+
const ident = decoder.decode(this._data.slice(offset, offset + 4));
|
|
18
|
+
const length = this._view.getUint32(offset + 4, true);
|
|
19
|
+
const data = new Uint8Array(this._data, offset + 8, length);
|
|
20
|
+
if (data.byteLength !== length)
|
|
21
|
+
this.truncatedChunks = true;
|
|
22
|
+
let subchunks;
|
|
23
|
+
let subident;
|
|
24
|
+
if (ident === "RIFF" || ident === "LIST") {
|
|
25
|
+
subident = decoder.decode(this._data.slice(offset + 8, offset + 12));
|
|
26
|
+
subchunks = [];
|
|
27
|
+
let currentOffset = 4;
|
|
28
|
+
while (currentOffset < data.byteLength - 1) {
|
|
29
|
+
const subchunk = this._parseChunk(offset + 8 + currentOffset);
|
|
30
|
+
subchunks.push(subchunk);
|
|
31
|
+
currentOffset += subchunk.data.byteLength + 8;
|
|
32
|
+
if (this._paddingBytes && subchunk.data.byteLength & 0x1)
|
|
33
|
+
currentOffset++;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return { ident, data, subident, subchunks };
|
|
37
|
+
}
|
|
38
|
+
find(str, inChunk = this.root) {
|
|
39
|
+
if (inChunk.ident === str)
|
|
40
|
+
return inChunk;
|
|
41
|
+
if (!inChunk.subchunks)
|
|
42
|
+
return;
|
|
43
|
+
for (let c of inChunk.subchunks) {
|
|
44
|
+
const found = this.find(str, c);
|
|
45
|
+
if (found)
|
|
46
|
+
return found;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
has(str) {
|
|
50
|
+
return this.find(str) !== undefined;
|
|
51
|
+
}
|
|
52
|
+
}
|
package/lib/sequencerecorder.js
CHANGED
|
@@ -1,16 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.SequenceDecoder = exports.SequenceRecorder = exports.SequenceRecorderPacketType = void 0;
|
|
4
|
-
const loglevel_1 = require("./loglevel");
|
|
5
|
-
var SequenceRecorderPacketType;
|
|
1
|
+
import { zcerr } from "./loglevel";
|
|
2
|
+
export var SequenceRecorderPacketType;
|
|
6
3
|
(function (SequenceRecorderPacketType) {
|
|
7
4
|
SequenceRecorderPacketType[SequenceRecorderPacketType["CAMERA_FRAME"] = 0] = "CAMERA_FRAME";
|
|
8
5
|
SequenceRecorderPacketType[SequenceRecorderPacketType["ACCELEROMETER"] = 1] = "ACCELEROMETER";
|
|
9
6
|
SequenceRecorderPacketType[SequenceRecorderPacketType["ROTATION_RATE"] = 2] = "ROTATION_RATE";
|
|
10
7
|
SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE"] = 3] = "ATTITUDE";
|
|
11
8
|
SequenceRecorderPacketType[SequenceRecorderPacketType["ATTITUDE_MATRIX"] = 4] = "ATTITUDE_MATRIX";
|
|
12
|
-
})(SequenceRecorderPacketType
|
|
13
|
-
class SequenceRecorder {
|
|
9
|
+
})(SequenceRecorderPacketType || (SequenceRecorderPacketType = {}));
|
|
10
|
+
export class SequenceRecorder {
|
|
14
11
|
constructor(estimatedFrames) {
|
|
15
12
|
this._insertionByte = 0;
|
|
16
13
|
this._numberPackets = 0;
|
|
@@ -53,11 +50,11 @@ class SequenceRecorder {
|
|
|
53
50
|
if (this._hasLoggedUnsuppored)
|
|
54
51
|
return;
|
|
55
52
|
this._hasLoggedUnsuppored = true;
|
|
56
|
-
|
|
53
|
+
zcerr("Unable to record frames from this camera source");
|
|
57
54
|
return;
|
|
58
55
|
}
|
|
59
56
|
if (c.data.byteLength !== (c.dataWidth * c.dataHeight)) {
|
|
60
|
-
|
|
57
|
+
zcerr("Unable to record frame to sequence: greyscale array doesn't match dimensions");
|
|
61
58
|
return;
|
|
62
59
|
}
|
|
63
60
|
let increment = SequenceRecorder.psbPacketType + SequenceRecorder.cameraPacketSizeBytes + (c.dataWidth * c.dataHeight);
|
|
@@ -131,7 +128,6 @@ class SequenceRecorder {
|
|
|
131
128
|
}
|
|
132
129
|
}
|
|
133
130
|
}
|
|
134
|
-
exports.SequenceRecorder = SequenceRecorder;
|
|
135
131
|
// Header (8 bytes): "UAR1" + number of frames (uint32)
|
|
136
132
|
SequenceRecorder.headerSizeBytes = 8;
|
|
137
133
|
// Per-frame packet byte sizes
|
|
@@ -155,7 +151,7 @@ SequenceRecorder.rotationRatePacketSizeBytes = 4 * 4; // t, x, y, z
|
|
|
155
151
|
SequenceRecorder.attitudePacketSizeBytes = 4 * 4; // t, x, y, z
|
|
156
152
|
// Attitude matrix
|
|
157
153
|
SequenceRecorder.attitudeMatrixPacketSizeBytes = 4 * 16;
|
|
158
|
-
class SequenceDecoder {
|
|
154
|
+
export class SequenceDecoder {
|
|
159
155
|
constructor(_data) {
|
|
160
156
|
this._data = _data;
|
|
161
157
|
this.dataByPacket = new Map();
|
|
@@ -280,4 +276,3 @@ class SequenceDecoder {
|
|
|
280
276
|
}
|
|
281
277
|
}
|
|
282
278
|
}
|
|
283
|
-
exports.SequenceDecoder = SequenceDecoder;
|
package/lib/sequencesource.d.ts
CHANGED
|
@@ -9,11 +9,13 @@ export declare class SequenceSource extends Source {
|
|
|
9
9
|
private _decoder;
|
|
10
10
|
private _minFrameGapMs;
|
|
11
11
|
private _lastUpdateTime;
|
|
12
|
+
private _time;
|
|
12
13
|
static create(p: zappar_pipeline_t): zappar_sequence_source_t;
|
|
13
14
|
static get(m: zappar_sequence_source_t): SequenceSource | undefined;
|
|
14
15
|
constructor(_impl: zappar_sequence_source_t, _pipeline: zappar_pipeline_t);
|
|
15
16
|
loadFromMemory(data: ArrayBuffer): void;
|
|
16
17
|
maxPlaybackFpsSet(fps: number): void;
|
|
18
|
+
setTime(t: number): void;
|
|
17
19
|
getFrame(currentlyProcessing: boolean): void;
|
|
18
20
|
destroy(): void;
|
|
19
21
|
start(): void;
|
package/lib/sequencesource.js
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const source_1 = require("./source");
|
|
7
|
-
const pipeline_1 = require("./pipeline");
|
|
8
|
-
const gl_matrix_1 = require("gl-matrix");
|
|
1
|
+
import { zcerr, zcout } from "./loglevel";
|
|
2
|
+
import { SequenceDecoder, SequenceRecorderPacketType } from "./sequencerecorder";
|
|
3
|
+
import { Source } from "./source";
|
|
4
|
+
import { Pipeline } from "./pipeline";
|
|
5
|
+
import { mat4 } from "gl-matrix";
|
|
9
6
|
let latest = 1;
|
|
10
7
|
let byId = new Map();
|
|
11
|
-
class SequenceSource extends
|
|
8
|
+
export class SequenceSource extends Source {
|
|
12
9
|
constructor(_impl, _pipeline) {
|
|
13
10
|
super();
|
|
14
11
|
this._impl = _impl;
|
|
@@ -22,7 +19,7 @@ class SequenceSource extends source_1.Source {
|
|
|
22
19
|
static create(p) {
|
|
23
20
|
let ret = (latest++);
|
|
24
21
|
byId.set(ret, new SequenceSource(ret, p));
|
|
25
|
-
|
|
22
|
+
zcout("sequence_source_source_t initialized");
|
|
26
23
|
return ret;
|
|
27
24
|
}
|
|
28
25
|
static get(m) {
|
|
@@ -30,12 +27,12 @@ class SequenceSource extends source_1.Source {
|
|
|
30
27
|
}
|
|
31
28
|
loadFromMemory(data) {
|
|
32
29
|
try {
|
|
33
|
-
this._decoder = new
|
|
30
|
+
this._decoder = new SequenceDecoder(data);
|
|
34
31
|
this._packetNumber = 0;
|
|
35
32
|
this._lastUpdateTime = 0;
|
|
36
33
|
}
|
|
37
34
|
catch (err) {
|
|
38
|
-
|
|
35
|
+
zcerr("Unable to decode sequence:", err);
|
|
39
36
|
}
|
|
40
37
|
}
|
|
41
38
|
maxPlaybackFpsSet(fps) {
|
|
@@ -45,8 +42,15 @@ class SequenceSource extends source_1.Source {
|
|
|
45
42
|
}
|
|
46
43
|
this._minFrameGapMs = 1000.0 / fps;
|
|
47
44
|
}
|
|
45
|
+
setTime(t) {
|
|
46
|
+
if (t < 0)
|
|
47
|
+
delete this._time;
|
|
48
|
+
else
|
|
49
|
+
this._time = t;
|
|
50
|
+
}
|
|
48
51
|
getFrame(currentlyProcessing) {
|
|
49
|
-
|
|
52
|
+
var _a;
|
|
53
|
+
let pipeline = Pipeline.get(this._pipeline);
|
|
50
54
|
if (!pipeline)
|
|
51
55
|
return;
|
|
52
56
|
if (!this._decoder)
|
|
@@ -56,13 +60,13 @@ class SequenceSource extends source_1.Source {
|
|
|
56
60
|
return;
|
|
57
61
|
if (currentlyProcessing)
|
|
58
62
|
return;
|
|
59
|
-
let curTime = performance.now();
|
|
63
|
+
let curTime = (_a = this._time) !== null && _a !== void 0 ? _a : performance.now();
|
|
60
64
|
if (curTime < this._lastUpdateTime + this._minFrameGapMs)
|
|
61
65
|
return;
|
|
62
66
|
if (this._packetNumber >= this._decoder.numberPackets)
|
|
63
67
|
return;
|
|
64
68
|
let packet = this._decoder.dataByPacket.get(this._packetNumber);
|
|
65
|
-
if (packet.type !=
|
|
69
|
+
if (packet.type != SequenceRecorderPacketType.CAMERA_FRAME)
|
|
66
70
|
return;
|
|
67
71
|
this._packetNumber++;
|
|
68
72
|
let pixels = pipeline.cameraPixelArrays.pop();
|
|
@@ -85,7 +89,7 @@ class SequenceSource extends source_1.Source {
|
|
|
85
89
|
dataHeight: packet.height,
|
|
86
90
|
texture: undefined,
|
|
87
91
|
userFacing: packet.userFacing,
|
|
88
|
-
uvTransform:
|
|
92
|
+
uvTransform: mat4.create(),
|
|
89
93
|
cameraSource: this
|
|
90
94
|
};
|
|
91
95
|
// We can queue the motion updates for the follwing frame immediately
|
|
@@ -101,19 +105,19 @@ class SequenceSource extends source_1.Source {
|
|
|
101
105
|
}
|
|
102
106
|
start() {
|
|
103
107
|
var _a;
|
|
104
|
-
let p =
|
|
108
|
+
let p = Pipeline.get(this._pipeline);
|
|
105
109
|
if (p && p.currentCameraSource !== this) {
|
|
106
110
|
(_a = p.currentCameraSource) === null || _a === void 0 ? void 0 : _a.pause();
|
|
107
111
|
p.currentCameraSource = this;
|
|
108
112
|
}
|
|
109
113
|
}
|
|
110
114
|
pause() {
|
|
111
|
-
let p =
|
|
115
|
+
let p = Pipeline.get(this._pipeline);
|
|
112
116
|
if (p && p.currentCameraSource === this)
|
|
113
117
|
p.currentCameraSource = undefined;
|
|
114
118
|
}
|
|
115
119
|
uploadGL(info) {
|
|
116
|
-
let pipeline =
|
|
120
|
+
let pipeline = Pipeline.get(this._pipeline);
|
|
117
121
|
if (!pipeline)
|
|
118
122
|
return;
|
|
119
123
|
let gl = pipeline === null || pipeline === void 0 ? void 0 : pipeline.glContext;
|
|
@@ -134,22 +138,22 @@ class SequenceSource extends source_1.Source {
|
|
|
134
138
|
let pendingMotionUpdate = false;
|
|
135
139
|
while (this._packetNumber < this._decoder.numberPackets) {
|
|
136
140
|
let packet = this._decoder.dataByPacket.get(this._packetNumber);
|
|
137
|
-
if (packet.type ==
|
|
141
|
+
if (packet.type == SequenceRecorderPacketType.CAMERA_FRAME)
|
|
138
142
|
break;
|
|
139
143
|
switch (packet.type) {
|
|
140
|
-
case
|
|
144
|
+
case SequenceRecorderPacketType.ACCELEROMETER:
|
|
141
145
|
pipeline.motionAccelerometerSubmit(packet.t, packet.x, packet.y, packet.z);
|
|
142
146
|
pendingMotionUpdate = true;
|
|
143
147
|
break;
|
|
144
|
-
case
|
|
148
|
+
case SequenceRecorderPacketType.ROTATION_RATE:
|
|
145
149
|
pipeline.motionRotationRateSubmit(packet.t, packet.x, packet.y, packet.z);
|
|
146
150
|
pendingMotionUpdate = true;
|
|
147
151
|
break;
|
|
148
|
-
case
|
|
152
|
+
case SequenceRecorderPacketType.ATTITUDE:
|
|
149
153
|
pipeline.motionAttitudeSubmit(packet.t, packet.x, packet.y, packet.z);
|
|
150
154
|
pendingMotionUpdate = true;
|
|
151
155
|
break;
|
|
152
|
-
case
|
|
156
|
+
case SequenceRecorderPacketType.ATTITUDE_MATRIX:
|
|
153
157
|
pipeline.motionAttitudeMatrix(packet.attitude);
|
|
154
158
|
pendingMotionUpdate = true;
|
|
155
159
|
break;
|
|
@@ -167,4 +171,3 @@ class SequenceSource extends source_1.Source {
|
|
|
167
171
|
return;
|
|
168
172
|
}
|
|
169
173
|
}
|
|
170
|
-
exports.SequenceSource = SequenceSource;
|
package/lib/serializer.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.MessageSerializer = void 0;
|
|
4
|
-
class MessageSerializer {
|
|
1
|
+
export class MessageSerializer {
|
|
5
2
|
constructor(_messageSender) {
|
|
6
3
|
this._messageSender = _messageSender;
|
|
7
4
|
this._freeBufferPool = [];
|
|
@@ -229,4 +226,3 @@ class MessageSerializer {
|
|
|
229
226
|
this._offset++;
|
|
230
227
|
}
|
|
231
228
|
}
|
|
232
|
-
exports.MessageSerializer = MessageSerializer;
|
package/lib/shader.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.linkProgram = exports.compileShader = void 0;
|
|
4
|
-
function compileShader(gl, type, src) {
|
|
1
|
+
export function compileShader(gl, type, src) {
|
|
5
2
|
let ret = gl.createShader(type);
|
|
6
3
|
if (!ret)
|
|
7
4
|
throw new Error("Unable to create shader");
|
|
@@ -12,11 +9,9 @@ function compileShader(gl, type, src) {
|
|
|
12
9
|
throw new Error("Shader compile error: " + msg);
|
|
13
10
|
return ret;
|
|
14
11
|
}
|
|
15
|
-
|
|
16
|
-
function linkProgram(gl, prog) {
|
|
12
|
+
export function linkProgram(gl, prog) {
|
|
17
13
|
gl.linkProgram(prog);
|
|
18
14
|
let msg = gl.getProgramInfoLog(prog);
|
|
19
15
|
if (msg && msg.trim().length > 0)
|
|
20
16
|
throw new Error("Unable to link: " + msg);
|
|
21
17
|
}
|
|
22
|
-
exports.linkProgram = linkProgram;
|
package/lib/source.js
CHANGED
package/lib/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "0.
|
|
1
|
+
export declare const VERSION = "2.0.0-beta.2";
|
package/lib/version.js
CHANGED
package/lib/worker-client.d.ts
CHANGED