@wonderlandengine/ar-provider-zappar 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +6 -0
- package/dist/src/face-tracking-mode-zappar.d.ts +40 -0
- package/dist/src/face-tracking-mode-zappar.js +261 -0
- package/dist/src/image-tracking-mode-zappar.d.ts +35 -0
- package/dist/src/image-tracking-mode-zappar.js +173 -0
- package/dist/src/slam-anchor-zappar.d.ts +16 -0
- package/dist/src/slam-anchor-zappar.js +39 -0
- package/dist/src/world-tracking-mode-zappar.d.ts +17 -0
- package/dist/src/world-tracking-mode-zappar.js +46 -0
- package/dist/src/zappar-module.d.ts +5 -0
- package/dist/src/zappar-module.js +43 -0
- package/dist/src/zappar-provider.d.ts +115 -0
- package/dist/src/zappar-provider.js +827 -0
- package/package.json +47 -0
- package/scripts/postinstall.mjs +184 -0
package/README.md
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# Wonderland Engine AR Tracking - Zappar Provider
|
|
2
|
+
|
|
3
|
+
Implementation of a Zappar-based AR tracking provider for the [Wonderland Engine AR framework](https://www.npmjs.com/package/@wonderlandengine/ar-tracking).
|
|
4
|
+
|
|
5
|
+
Learn more about Wonderland Engine at [https://wonderlandengine.com](https://wonderlandengine.com).
|
|
6
|
+
|
|
7
|
+
## Usage
|
|
8
|
+
|
|
9
|
+
Install this provider into your Wonderland Engine project:
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
npm i --save @wonderlandengine/ar-provider-zappar
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
For instructions on configuring AR sessions, refer to the
|
|
16
|
+
[repository README](https://github.com/WonderlandEngine/wonderland-ar-tracking#readme).
|
|
17
|
+
|
|
18
|
+
This package depends on `@zappar/zappar`; make sure your bundler is configured
|
|
19
|
+
to serve the `zcv.wasm` asset as described in the
|
|
20
|
+
[Zappar installation guide](https://docs.zap.works/universal-ar/javascript/getting-started/installation/).
|
|
21
|
+
|
|
22
|
+
## Supported Tracking Modes
|
|
23
|
+
|
|
24
|
+
The Zappar provider implements SLAM, face tracking, and image tracking. These
|
|
25
|
+
map to the standard Wonderland Engine components supplied by
|
|
26
|
+
`@wonderlandengine/ar-tracking` (`ARSLAMCamera`, `ARFaceTrackingCamera`, and
|
|
27
|
+
`ARImageTrackingCamera`).
|
|
28
|
+
|
|
29
|
+
- Face tracking loads Zappar's default model and mesh. Attachment points that
|
|
30
|
+
have no exact landmark are approximated to the closest available Zappar
|
|
31
|
+
landmark.
|
|
32
|
+
|
|
33
|
+
## Recording a Zappar Sequence (Camera + Motion)
|
|
34
|
+
|
|
35
|
+
If you want accurate offline replay (especially for SLAM), recording **only** an
|
|
36
|
+
MP4 is usually not enough. Zappar can also record a **sequence** that contains
|
|
37
|
+
camera frames **and** device motion data.
|
|
38
|
+
|
|
39
|
+
Record on a phone/tablet (where the IMU matches the camera movement), then copy
|
|
40
|
+
the saved file to your desktop for replay.
|
|
41
|
+
|
|
42
|
+
### Quick Record Script (DevTools Console)
|
|
43
|
+
|
|
44
|
+
This repo already exposes the current pipeline as `window.ZapparPipeline`.
|
|
45
|
+
|
|
46
|
+
1. Start your AR session normally.
|
|
47
|
+
2. Open DevTools on the capture device (or use remote debugging).
|
|
48
|
+
3. Paste the following:
|
|
49
|
+
|
|
50
|
+
```js
|
|
51
|
+
// Start recording (pre-alloc ~20 seconds @ 30fps)
|
|
52
|
+
window.ZapparPipeline.sequenceRecordClear();
|
|
53
|
+
window.ZapparPipeline.sequenceRecordStart(30 * 20);
|
|
54
|
+
|
|
55
|
+
// When you're done recording, run this to stop + download the sequence:
|
|
56
|
+
window.ZapparPipeline.sequenceRecordStop();
|
|
57
|
+
const data = window.ZapparPipeline.sequenceRecordData(); // Uint8Array
|
|
58
|
+
const blob = new Blob([data], {type: 'application/octet-stream'});
|
|
59
|
+
const a = document.createElement('a');
|
|
60
|
+
a.href = URL.createObjectURL(blob);
|
|
61
|
+
a.download = 'zappar-sequence.bin';
|
|
62
|
+
a.click();
|
|
63
|
+
URL.revokeObjectURL(a.href);
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Notes:
|
|
67
|
+
|
|
68
|
+
- Sequence recording captures camera + IMU, so it’s best done on a mobile device.
|
|
69
|
+
- You can replay sequences using Zappar’s `SequenceSource` API (preferred over MP4 replay for correct tracking).
|
|
70
|
+
|
|
71
|
+
### Registering Image Targets
|
|
72
|
+
|
|
73
|
+
To use image tracking, register each Zappar `.zpt` target file with the
|
|
74
|
+
provider before the session starts:
|
|
75
|
+
|
|
76
|
+
```ts
|
|
77
|
+
import {
|
|
78
|
+
ZapparProvider,
|
|
79
|
+
ZapparImageTargetOptions,
|
|
80
|
+
} from '@wonderlandengine/ar-provider-zappar';
|
|
81
|
+
import {ARSession} from '@wonderlandengine/ar-tracking';
|
|
82
|
+
|
|
83
|
+
const session = ARSession.getSessionForEngine(engine);
|
|
84
|
+
const provider = ZapparProvider.registerTrackingProviderWithARSession(session);
|
|
85
|
+
|
|
86
|
+
const posterTarget: ZapparImageTargetOptions = {
|
|
87
|
+
name: 'poster',
|
|
88
|
+
physicalWidthInMeters: 0.25,
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
await provider.registerImageTarget('/static/targets/poster.zpt', posterTarget);
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Once registered, the provider emits image scanning and tracking events via
|
|
95
|
+
`ARImageTrackingCamera`.
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/// <reference path="../src/types/global.d.ts" />
|
|
2
|
+
export * from './src/zappar-provider.js';
|
|
3
|
+
export * from './src/world-tracking-mode-zappar.js';
|
|
4
|
+
export * from './src/face-tracking-mode-zappar.js';
|
|
5
|
+
export * from './src/image-tracking-mode-zappar.js';
|
|
6
|
+
export * from './src/slam-anchor-zappar.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/// <reference path="src/types/global.d.ts" />
|
|
2
|
+
export * from './src/zappar-provider.js';
|
|
3
|
+
export * from './src/world-tracking-mode-zappar.js';
|
|
4
|
+
export * from './src/face-tracking-mode-zappar.js';
|
|
5
|
+
export * from './src/image-tracking-mode-zappar.js';
|
|
6
|
+
export * from './src/slam-anchor-zappar.js';
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { Emitter } from '@wonderlandengine/api';
|
|
2
|
+
import { FaceFoundEvent, FaceLoadingEvent, FaceLostEvent, FaceTrackingMode, TrackingMode } from '@wonderlandengine/ar-tracking';
|
|
3
|
+
export declare class FaceTracking_Zappar extends TrackingMode implements FaceTrackingMode {
|
|
4
|
+
private _zappar;
|
|
5
|
+
private _view?;
|
|
6
|
+
private _faceTracker?;
|
|
7
|
+
private _faceMesh?;
|
|
8
|
+
private _resourcesReady;
|
|
9
|
+
private _resourcesPromise;
|
|
10
|
+
private readonly _landmarks;
|
|
11
|
+
private readonly _sharedLandmarks;
|
|
12
|
+
private readonly _anchorNumericIds;
|
|
13
|
+
private _nextAnchorId;
|
|
14
|
+
private readonly _cameraMatrix;
|
|
15
|
+
private readonly _cameraPosition;
|
|
16
|
+
private readonly _cameraRotation;
|
|
17
|
+
private readonly _cameraScale;
|
|
18
|
+
private readonly _scratchMatrix;
|
|
19
|
+
private readonly _scratchPosition;
|
|
20
|
+
private readonly _scratchRotation;
|
|
21
|
+
private readonly _scratchScale;
|
|
22
|
+
private _loadingEvent;
|
|
23
|
+
readonly onFaceScanning: Emitter<[event: FaceLoadingEvent]>;
|
|
24
|
+
readonly onFaceLoading: Emitter<[event: FaceLoadingEvent]>;
|
|
25
|
+
readonly onFaceFound: Emitter<[event: FaceFoundEvent]>;
|
|
26
|
+
readonly onFaceUpdate: Emitter<[event: FaceFoundEvent]>;
|
|
27
|
+
readonly onFaceLost: Emitter<[event: FaceLostEvent]>;
|
|
28
|
+
init(): void;
|
|
29
|
+
startSession(): void;
|
|
30
|
+
endSession(): void;
|
|
31
|
+
update(): void;
|
|
32
|
+
private _prepareResources;
|
|
33
|
+
private _buildLandmarks;
|
|
34
|
+
private _buildLoadingEvent;
|
|
35
|
+
private _handleAnchorVisible;
|
|
36
|
+
private _handleAnchorNotVisible;
|
|
37
|
+
private _buildFaceEvent;
|
|
38
|
+
private _applyCameraPose;
|
|
39
|
+
private _anchorNumericId;
|
|
40
|
+
}
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import { Emitter } from '@wonderlandengine/api';
|
|
2
|
+
import { FaceAttachmentPoint, TrackingMode, } from '@wonderlandengine/ar-tracking';
|
|
3
|
+
import { mat4, quat, vec3 } from 'gl-matrix';
|
|
4
|
+
const AttachmentLandmarkKeys = {
|
|
5
|
+
[FaceAttachmentPoint.Forehead]: 'NOSE_BRIDGE',
|
|
6
|
+
[FaceAttachmentPoint.EyeOuterCornerLeft]: 'EYE_LEFT',
|
|
7
|
+
[FaceAttachmentPoint.EyeOuterCornerRight]: 'EYE_RIGHT',
|
|
8
|
+
[FaceAttachmentPoint.EyeLeft]: 'EYE_LEFT',
|
|
9
|
+
[FaceAttachmentPoint.EyeRight]: 'EYE_RIGHT',
|
|
10
|
+
[FaceAttachmentPoint.EyeBrowCenterLeft]: 'EYEBROW_LEFT',
|
|
11
|
+
[FaceAttachmentPoint.EyeBrowCenterRight]: 'EYEBROW_RIGHT',
|
|
12
|
+
[FaceAttachmentPoint.EyeBrowInnerLeft]: 'EYEBROW_LEFT',
|
|
13
|
+
[FaceAttachmentPoint.EyeBrowInnerRight]: 'EYEBROW_RIGHT',
|
|
14
|
+
[FaceAttachmentPoint.EyeBrowOuterLeft]: 'EYEBROW_LEFT',
|
|
15
|
+
[FaceAttachmentPoint.EyeBrowOuterRight]: 'EYEBROW_RIGHT',
|
|
16
|
+
[FaceAttachmentPoint.EarLeft]: 'EAR_LEFT',
|
|
17
|
+
[FaceAttachmentPoint.EarRight]: 'EAR_RIGHT',
|
|
18
|
+
[FaceAttachmentPoint.NoseBridge]: 'NOSE_BRIDGE',
|
|
19
|
+
[FaceAttachmentPoint.NoseTip]: 'NOSE_TIP',
|
|
20
|
+
[FaceAttachmentPoint.CheekLeft]: 'EAR_LEFT',
|
|
21
|
+
[FaceAttachmentPoint.CheekRight]: 'EAR_RIGHT',
|
|
22
|
+
[FaceAttachmentPoint.Mouth]: 'MOUTH_CENTER',
|
|
23
|
+
[FaceAttachmentPoint.MouthCornerLeft]: 'MOUTH_CENTER',
|
|
24
|
+
[FaceAttachmentPoint.MouthCornerRight]: 'MOUTH_CENTER',
|
|
25
|
+
[FaceAttachmentPoint.UpperLip]: 'LIP_TOP',
|
|
26
|
+
[FaceAttachmentPoint.LowerLip]: 'LIP_BOTTOM',
|
|
27
|
+
[FaceAttachmentPoint.Chin]: 'CHIN',
|
|
28
|
+
};
|
|
29
|
+
const AttachmentList = Object.values(FaceAttachmentPoint);
|
|
30
|
+
export class FaceTracking_Zappar extends TrackingMode {
|
|
31
|
+
_zappar = null;
|
|
32
|
+
_view;
|
|
33
|
+
_faceTracker;
|
|
34
|
+
_faceMesh;
|
|
35
|
+
_resourcesReady = false;
|
|
36
|
+
_resourcesPromise = null;
|
|
37
|
+
_landmarks = new Map();
|
|
38
|
+
_sharedLandmarks = new Map();
|
|
39
|
+
_anchorNumericIds = new Map();
|
|
40
|
+
_nextAnchorId = 0;
|
|
41
|
+
_cameraMatrix = mat4.create();
|
|
42
|
+
_cameraPosition = vec3.create();
|
|
43
|
+
_cameraRotation = quat.create();
|
|
44
|
+
_cameraScale = vec3.create();
|
|
45
|
+
_scratchMatrix = mat4.create();
|
|
46
|
+
_scratchPosition = vec3.create();
|
|
47
|
+
_scratchRotation = quat.create();
|
|
48
|
+
_scratchScale = vec3.create();
|
|
49
|
+
_loadingEvent = null;
|
|
50
|
+
onFaceScanning = new Emitter();
|
|
51
|
+
onFaceLoading = new Emitter();
|
|
52
|
+
onFaceFound = new Emitter();
|
|
53
|
+
onFaceUpdate = new Emitter();
|
|
54
|
+
onFaceLost = new Emitter();
|
|
55
|
+
init() {
|
|
56
|
+
this._view = this.component.object.getComponent('view') ?? undefined;
|
|
57
|
+
const input = this.component.object.getComponent('input');
|
|
58
|
+
if (input) {
|
|
59
|
+
input.active = false;
|
|
60
|
+
}
|
|
61
|
+
if (!this._resourcesPromise) {
|
|
62
|
+
this._resourcesPromise = this._prepareResources();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
startSession() {
|
|
66
|
+
void this.provider.startSession();
|
|
67
|
+
}
|
|
68
|
+
endSession() {
|
|
69
|
+
this.provider.endSession();
|
|
70
|
+
}
|
|
71
|
+
update() {
|
|
72
|
+
if (!this._resourcesReady || !this._faceTracker || !this._faceMesh) {
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
const Zappar = this._zappar;
|
|
76
|
+
if (!Zappar)
|
|
77
|
+
return;
|
|
78
|
+
const provider = this.provider;
|
|
79
|
+
const pipeline = provider.getPipeline();
|
|
80
|
+
const projectionMatrix = Zappar.projectionMatrixFromCameraModel(pipeline.cameraModel(), this.component.engine.canvas.width, this.component.engine.canvas.height);
|
|
81
|
+
if (this._view) {
|
|
82
|
+
this._view.projectionMatrix.set(projectionMatrix);
|
|
83
|
+
}
|
|
84
|
+
const cameraPose = pipeline.cameraPoseDefault();
|
|
85
|
+
this._applyCameraPose(cameraPose);
|
|
86
|
+
for (const anchor of this._faceTracker.visible) {
|
|
87
|
+
const event = this._buildFaceEvent(anchor);
|
|
88
|
+
this.onFaceUpdate.notify(event);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
async _prepareResources() {
|
|
92
|
+
const provider = this.provider;
|
|
93
|
+
this._zappar = await provider.ensureZapparNamespace();
|
|
94
|
+
await provider.ensureFaceResources();
|
|
95
|
+
if (this._resourcesReady) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
this._faceTracker = provider.getFaceTracker();
|
|
99
|
+
this._faceMesh = provider.getFaceMesh();
|
|
100
|
+
this._buildLandmarks();
|
|
101
|
+
this._faceTracker.onVisible.bind(this._handleAnchorVisible);
|
|
102
|
+
this._faceTracker.onNotVisible.bind(this._handleAnchorNotVisible);
|
|
103
|
+
this._loadingEvent = this._buildLoadingEvent();
|
|
104
|
+
if (this._loadingEvent) {
|
|
105
|
+
this.onFaceScanning.notify(this._loadingEvent);
|
|
106
|
+
this.onFaceLoading.notify(this._loadingEvent);
|
|
107
|
+
}
|
|
108
|
+
this._resourcesReady = true;
|
|
109
|
+
}
|
|
110
|
+
_buildLandmarks() {
|
|
111
|
+
const Zappar = this._zappar;
|
|
112
|
+
if (!Zappar)
|
|
113
|
+
return;
|
|
114
|
+
this._landmarks.clear();
|
|
115
|
+
this._sharedLandmarks.clear();
|
|
116
|
+
for (const attachment of AttachmentList) {
|
|
117
|
+
const key = AttachmentLandmarkKeys[attachment];
|
|
118
|
+
if (!key) {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const landmarkName = Zappar.FaceLandmarkName[key];
|
|
122
|
+
if (landmarkName === undefined)
|
|
123
|
+
continue;
|
|
124
|
+
let landmark = this._sharedLandmarks.get(landmarkName);
|
|
125
|
+
if (!landmark) {
|
|
126
|
+
landmark = new Zappar.FaceLandmark(landmarkName);
|
|
127
|
+
this._sharedLandmarks.set(landmarkName, landmark);
|
|
128
|
+
}
|
|
129
|
+
this._landmarks.set(attachment, landmark);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
_buildLoadingEvent() {
|
|
133
|
+
if (!this._faceTracker || !this._faceMesh) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
const indicesArray = this._faceMesh.indices;
|
|
137
|
+
const uvsArray = this._faceMesh.uvs;
|
|
138
|
+
const verticesArray = this._faceMesh.vertices;
|
|
139
|
+
const indices = [];
|
|
140
|
+
for (let i = 0; i < indicesArray.length; i += 3) {
|
|
141
|
+
indices.push({
|
|
142
|
+
a: indicesArray[i],
|
|
143
|
+
b: indicesArray[i + 1],
|
|
144
|
+
c: indicesArray[i + 2],
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
const uvs = [];
|
|
148
|
+
for (let i = 0; i < uvsArray.length; i += 2) {
|
|
149
|
+
uvs.push({ u: uvsArray[i], v: uvsArray[i + 1] });
|
|
150
|
+
}
|
|
151
|
+
return {
|
|
152
|
+
maxDetections: this._faceTracker.maxFaces,
|
|
153
|
+
pointsPerDetection: verticesArray.length / 3,
|
|
154
|
+
indices: indices,
|
|
155
|
+
uvs: uvs,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
_handleAnchorVisible = (anchor) => {
|
|
159
|
+
if (!this._resourcesReady) {
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const event = this._buildFaceEvent(anchor);
|
|
163
|
+
this.onFaceFound.notify(event);
|
|
164
|
+
};
|
|
165
|
+
_handleAnchorNotVisible = (anchor) => {
|
|
166
|
+
const id = this._anchorNumericId(anchor.id);
|
|
167
|
+
this.onFaceLost.notify({ id });
|
|
168
|
+
};
|
|
169
|
+
_buildFaceEvent(anchor) {
|
|
170
|
+
const provider = this.provider;
|
|
171
|
+
const pipeline = provider.getPipeline();
|
|
172
|
+
const cameraPose = pipeline.cameraPoseDefault();
|
|
173
|
+
const anchorPose = anchor.pose(cameraPose, false);
|
|
174
|
+
mat4.copy(this._scratchMatrix, anchorPose);
|
|
175
|
+
mat4.getTranslation(this._scratchPosition, this._scratchMatrix);
|
|
176
|
+
mat4.getRotation(this._scratchRotation, this._scratchMatrix);
|
|
177
|
+
mat4.getScaling(this._scratchScale, this._scratchMatrix);
|
|
178
|
+
const scale = (this._scratchScale[0] + this._scratchScale[1] + this._scratchScale[2]) / 3;
|
|
179
|
+
const anchorPosition = {
|
|
180
|
+
x: this._scratchPosition[0],
|
|
181
|
+
y: this._scratchPosition[1],
|
|
182
|
+
z: this._scratchPosition[2],
|
|
183
|
+
};
|
|
184
|
+
this._faceMesh.updateFromFaceAnchor(anchor, false);
|
|
185
|
+
const verticesArray = this._faceMesh.vertices;
|
|
186
|
+
const normalsArray = this._faceMesh.normals;
|
|
187
|
+
const vertices = [];
|
|
188
|
+
for (let i = 0; i < verticesArray.length; i += 3) {
|
|
189
|
+
vertices.push({
|
|
190
|
+
x: verticesArray[i],
|
|
191
|
+
y: verticesArray[i + 1],
|
|
192
|
+
z: verticesArray[i + 2],
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
const normals = [];
|
|
196
|
+
for (let i = 0; i < normalsArray.length; i += 3) {
|
|
197
|
+
normals.push({
|
|
198
|
+
x: normalsArray[i],
|
|
199
|
+
y: normalsArray[i + 1],
|
|
200
|
+
z: normalsArray[i + 2],
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
const attachmentPoints = {};
|
|
204
|
+
for (const attachment of AttachmentList) {
|
|
205
|
+
const landmark = this._landmarks.get(attachment);
|
|
206
|
+
if (landmark) {
|
|
207
|
+
landmark.updateFromFaceAnchor(anchor, false);
|
|
208
|
+
mat4.copy(this._scratchMatrix, landmark.pose);
|
|
209
|
+
mat4.getTranslation(this._scratchPosition, this._scratchMatrix);
|
|
210
|
+
attachmentPoints[attachment] = {
|
|
211
|
+
position: {
|
|
212
|
+
x: this._scratchPosition[0],
|
|
213
|
+
y: this._scratchPosition[1],
|
|
214
|
+
z: this._scratchPosition[2],
|
|
215
|
+
},
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
else {
|
|
219
|
+
attachmentPoints[attachment] = {
|
|
220
|
+
position: { ...anchorPosition },
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
const id = this._anchorNumericId(anchor.id);
|
|
225
|
+
return {
|
|
226
|
+
id,
|
|
227
|
+
vertices: vertices,
|
|
228
|
+
normals: normals,
|
|
229
|
+
attachmentPoints: attachmentPoints,
|
|
230
|
+
transform: {
|
|
231
|
+
position: { ...anchorPosition },
|
|
232
|
+
rotation: {
|
|
233
|
+
x: this._scratchRotation[0],
|
|
234
|
+
y: this._scratchRotation[1],
|
|
235
|
+
z: this._scratchRotation[2],
|
|
236
|
+
w: this._scratchRotation[3],
|
|
237
|
+
},
|
|
238
|
+
scale,
|
|
239
|
+
scaledWidth: this._scratchScale[0],
|
|
240
|
+
scaledHeight: this._scratchScale[1],
|
|
241
|
+
scaledDepth: this._scratchScale[2],
|
|
242
|
+
},
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
_applyCameraPose(matrix) {
|
|
246
|
+
mat4.copy(this._cameraMatrix, matrix);
|
|
247
|
+
mat4.getTranslation(this._cameraPosition, this._cameraMatrix);
|
|
248
|
+
mat4.getRotation(this._cameraRotation, this._cameraMatrix);
|
|
249
|
+
mat4.getScaling(this._cameraScale, this._cameraMatrix);
|
|
250
|
+
this.component.object.setPositionWorld(this._cameraPosition);
|
|
251
|
+
this.component.object.setRotationWorld(this._cameraRotation);
|
|
252
|
+
}
|
|
253
|
+
_anchorNumericId(anchorId) {
|
|
254
|
+
let id = this._anchorNumericIds.get(anchorId);
|
|
255
|
+
if (id === undefined) {
|
|
256
|
+
id = this._nextAnchorId++;
|
|
257
|
+
this._anchorNumericIds.set(anchorId, id);
|
|
258
|
+
}
|
|
259
|
+
return id;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Emitter } from '@wonderlandengine/api';
|
|
2
|
+
import { ImageScanningEvent, ImageTrackedEvent, ImageTrackingMode, TrackingMode } from '@wonderlandengine/ar-tracking';
|
|
3
|
+
export declare class ImageTracking_Zappar extends TrackingMode implements ImageTrackingMode {
|
|
4
|
+
private _zappar;
|
|
5
|
+
private _view?;
|
|
6
|
+
private _imageTracker?;
|
|
7
|
+
private _resourcesReady;
|
|
8
|
+
private _resourcesPromise;
|
|
9
|
+
private _targetsSubscriptionAdded;
|
|
10
|
+
private readonly _cameraMatrix;
|
|
11
|
+
private readonly _cameraPosition;
|
|
12
|
+
private readonly _cameraRotation;
|
|
13
|
+
private readonly _cameraScale;
|
|
14
|
+
private readonly _scratchMatrix;
|
|
15
|
+
private readonly _scratchPosition;
|
|
16
|
+
private readonly _scratchRotation;
|
|
17
|
+
private readonly _scratchScale;
|
|
18
|
+
private readonly _lastAnchorEvents;
|
|
19
|
+
readonly onImageScanning: Emitter<[event: ImageScanningEvent]>;
|
|
20
|
+
readonly onImageFound: Emitter<[event: ImageTrackedEvent]>;
|
|
21
|
+
readonly onImageUpdate: Emitter<[event: ImageTrackedEvent]>;
|
|
22
|
+
readonly onImageLost: Emitter<[event: ImageTrackedEvent]>;
|
|
23
|
+
init(): void;
|
|
24
|
+
startSession(): void;
|
|
25
|
+
endSession(): void;
|
|
26
|
+
update(): void;
|
|
27
|
+
private _prepareResources;
|
|
28
|
+
private _handleTargetsChanged;
|
|
29
|
+
private _emitScanningEvent;
|
|
30
|
+
private _handleAnchorVisible;
|
|
31
|
+
private _handleAnchorNotVisible;
|
|
32
|
+
private _buildImageEvent;
|
|
33
|
+
private _guessDescriptor;
|
|
34
|
+
private _applyCameraPose;
|
|
35
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { Emitter } from '@wonderlandengine/api';
|
|
2
|
+
import { TrackingMode, } from '@wonderlandengine/ar-tracking';
|
|
3
|
+
import { mat4, quat, vec3 } from 'gl-matrix';
|
|
4
|
+
export class ImageTracking_Zappar extends TrackingMode {
|
|
5
|
+
_zappar = null;
|
|
6
|
+
_view;
|
|
7
|
+
_imageTracker;
|
|
8
|
+
_resourcesReady = false;
|
|
9
|
+
_resourcesPromise = null;
|
|
10
|
+
_targetsSubscriptionAdded = false;
|
|
11
|
+
_cameraMatrix = mat4.create();
|
|
12
|
+
_cameraPosition = vec3.create();
|
|
13
|
+
_cameraRotation = quat.create();
|
|
14
|
+
_cameraScale = vec3.create();
|
|
15
|
+
_scratchMatrix = mat4.create();
|
|
16
|
+
_scratchPosition = vec3.create();
|
|
17
|
+
_scratchRotation = quat.create();
|
|
18
|
+
_scratchScale = vec3.create();
|
|
19
|
+
_lastAnchorEvents = new Map();
|
|
20
|
+
onImageScanning = new Emitter();
|
|
21
|
+
onImageFound = new Emitter();
|
|
22
|
+
onImageUpdate = new Emitter();
|
|
23
|
+
onImageLost = new Emitter();
|
|
24
|
+
init() {
|
|
25
|
+
this._view = this.component.object.getComponent('view') ?? undefined;
|
|
26
|
+
const input = this.component.object.getComponent('input');
|
|
27
|
+
if (input) {
|
|
28
|
+
input.active = false;
|
|
29
|
+
}
|
|
30
|
+
if (!this._resourcesPromise) {
|
|
31
|
+
this._resourcesPromise = this._prepareResources();
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
startSession() {
|
|
35
|
+
void this.provider.startSession();
|
|
36
|
+
}
|
|
37
|
+
endSession() {
|
|
38
|
+
this.provider.endSession();
|
|
39
|
+
}
|
|
40
|
+
update() {
|
|
41
|
+
if (!this._resourcesReady || !this._imageTracker) {
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
const Zappar = this._zappar;
|
|
45
|
+
if (!Zappar)
|
|
46
|
+
return;
|
|
47
|
+
const provider = this.provider;
|
|
48
|
+
const pipeline = provider.getPipeline();
|
|
49
|
+
const projectionMatrix = Zappar.projectionMatrixFromCameraModel(pipeline.cameraModel(), this.component.engine.canvas.width, this.component.engine.canvas.height);
|
|
50
|
+
if (this._view) {
|
|
51
|
+
this._view.projectionMatrix.set(projectionMatrix);
|
|
52
|
+
}
|
|
53
|
+
const cameraPose = pipeline.cameraPoseDefault();
|
|
54
|
+
this._applyCameraPose(cameraPose);
|
|
55
|
+
for (const anchor of this._imageTracker.visible) {
|
|
56
|
+
const event = this._buildImageEvent(anchor);
|
|
57
|
+
this.onImageUpdate.notify(event);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async _prepareResources() {
|
|
61
|
+
const provider = this.provider;
|
|
62
|
+
this._zappar = await provider.ensureZapparNamespace();
|
|
63
|
+
this._imageTracker = provider.ensureImageTracker();
|
|
64
|
+
if (!this._targetsSubscriptionAdded) {
|
|
65
|
+
provider.onImageTargetsChanged.add(this._handleTargetsChanged);
|
|
66
|
+
this._targetsSubscriptionAdded = true;
|
|
67
|
+
}
|
|
68
|
+
this._imageTracker.onVisible.bind(this._handleAnchorVisible);
|
|
69
|
+
this._imageTracker.onNotVisible.bind(this._handleAnchorNotVisible);
|
|
70
|
+
this._emitScanningEvent();
|
|
71
|
+
this._resourcesReady = true;
|
|
72
|
+
}
|
|
73
|
+
_handleTargetsChanged = () => {
|
|
74
|
+
if (!this._resourcesReady) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
this._emitScanningEvent();
|
|
78
|
+
};
|
|
79
|
+
_emitScanningEvent() {
|
|
80
|
+
const provider = this.provider;
|
|
81
|
+
const event = provider.getImageScanningEvent();
|
|
82
|
+
if (event.imageTargets.length > 0) {
|
|
83
|
+
this.onImageScanning.notify(event);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
_handleAnchorVisible = (anchor) => {
|
|
87
|
+
if (!this._resourcesReady) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const event = this._buildImageEvent(anchor);
|
|
91
|
+
this.onImageFound.notify(event);
|
|
92
|
+
};
|
|
93
|
+
_handleAnchorNotVisible = (anchor) => {
|
|
94
|
+
const last = this._lastAnchorEvents.get(anchor.id);
|
|
95
|
+
if (!last) {
|
|
96
|
+
this.onImageLost.notify({
|
|
97
|
+
name: anchor.id,
|
|
98
|
+
position: { x: 0, y: 0, z: 0 },
|
|
99
|
+
rotation: { x: 0, y: 0, z: 0, w: 1 },
|
|
100
|
+
scale: 1,
|
|
101
|
+
scaleWidth: 1,
|
|
102
|
+
scaledHeight: 1,
|
|
103
|
+
type: 'flat',
|
|
104
|
+
});
|
|
105
|
+
this._lastAnchorEvents.delete(anchor.id);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
this.onImageLost.notify(last);
|
|
109
|
+
this._lastAnchorEvents.delete(anchor.id);
|
|
110
|
+
};
|
|
111
|
+
_buildImageEvent(anchor) {
|
|
112
|
+
const provider = this.provider;
|
|
113
|
+
const pipeline = provider.getPipeline();
|
|
114
|
+
const cameraPose = pipeline.cameraPoseDefault();
|
|
115
|
+
const anchorPose = anchor.pose(cameraPose, false);
|
|
116
|
+
mat4.copy(this._scratchMatrix, anchorPose);
|
|
117
|
+
mat4.getTranslation(this._scratchPosition, this._scratchMatrix);
|
|
118
|
+
mat4.getRotation(this._scratchRotation, this._scratchMatrix);
|
|
119
|
+
mat4.getScaling(this._scratchScale, this._scratchMatrix);
|
|
120
|
+
const scale = (this._scratchScale[0] + this._scratchScale[1] + this._scratchScale[2]) / 3;
|
|
121
|
+
const descriptor = this._guessDescriptor(anchor);
|
|
122
|
+
const geometry = descriptor?.geometry;
|
|
123
|
+
const event = {
|
|
124
|
+
name: descriptor?.name ?? anchor.id,
|
|
125
|
+
position: {
|
|
126
|
+
x: this._scratchPosition[0],
|
|
127
|
+
y: this._scratchPosition[1],
|
|
128
|
+
z: this._scratchPosition[2],
|
|
129
|
+
},
|
|
130
|
+
rotation: {
|
|
131
|
+
x: this._scratchRotation[0],
|
|
132
|
+
y: this._scratchRotation[1],
|
|
133
|
+
z: this._scratchRotation[2],
|
|
134
|
+
w: this._scratchRotation[3],
|
|
135
|
+
},
|
|
136
|
+
scale,
|
|
137
|
+
scaleWidth: geometry?.scaleWidth ?? scale,
|
|
138
|
+
scaledHeight: geometry?.scaledHeight ?? scale,
|
|
139
|
+
type: descriptor?.type ?? 'flat',
|
|
140
|
+
height: geometry?.height,
|
|
141
|
+
radiusTop: geometry?.radiusTop,
|
|
142
|
+
radiusBottom: geometry?.radiusBottom,
|
|
143
|
+
arcStartRadians: geometry?.arcStartRadians,
|
|
144
|
+
arcLengthRadians: geometry?.arcLengthRadians,
|
|
145
|
+
};
|
|
146
|
+
this._lastAnchorEvents.set(anchor.id, event);
|
|
147
|
+
return event;
|
|
148
|
+
}
|
|
149
|
+
_guessDescriptor(anchor) {
|
|
150
|
+
const provider = this.provider;
|
|
151
|
+
const descriptors = provider.getImageTargetDescriptors();
|
|
152
|
+
if (descriptors.length === 0) {
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
const direct = descriptors.find((descriptor) => descriptor.name === anchor.id);
|
|
156
|
+
if (direct) {
|
|
157
|
+
return direct;
|
|
158
|
+
}
|
|
159
|
+
const numeric = Number.parseInt(anchor.id, 10);
|
|
160
|
+
if (!Number.isNaN(numeric) && descriptors[numeric]) {
|
|
161
|
+
return descriptors[numeric];
|
|
162
|
+
}
|
|
163
|
+
return descriptors[0];
|
|
164
|
+
}
|
|
165
|
+
_applyCameraPose(matrix) {
|
|
166
|
+
mat4.copy(this._cameraMatrix, matrix);
|
|
167
|
+
mat4.getTranslation(this._cameraPosition, this._cameraMatrix);
|
|
168
|
+
mat4.getRotation(this._cameraRotation, this._cameraMatrix);
|
|
169
|
+
mat4.getScaling(this._cameraScale, this._cameraMatrix);
|
|
170
|
+
this.component.object.setPositionWorld(this._cameraPosition);
|
|
171
|
+
this.component.object.setRotationWorld(this._cameraRotation);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { Component } from '@wonderlandengine/api';
|
|
2
|
+
/**
|
|
3
|
+
* Applies Zappar Instant World Tracking anchor pose to this object.
|
|
4
|
+
*
|
|
5
|
+
* Attach this component to a parent object of your content to verify/visualize
|
|
6
|
+
* the anchor pose coming from Zappar.
|
|
7
|
+
*/
|
|
8
|
+
export declare class SlamAnchorZappar extends Component {
|
|
9
|
+
static TypeName: string;
|
|
10
|
+
private _provider;
|
|
11
|
+
private readonly _tmpTransform;
|
|
12
|
+
start(): void;
|
|
13
|
+
update(): void;
|
|
14
|
+
private onSessionStart;
|
|
15
|
+
private onSessionEnd;
|
|
16
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { Component } from '@wonderlandengine/api';
|
|
2
|
+
import { quat2 } from 'gl-matrix';
|
|
3
|
+
import { ARSession } from '@wonderlandengine/ar-tracking';
|
|
4
|
+
import { ZapparProvider } from './zappar-provider.js';
|
|
5
|
+
/**
|
|
6
|
+
* Applies Zappar Instant World Tracking anchor pose to this object.
|
|
7
|
+
*
|
|
8
|
+
* Attach this component to a parent object of your content to verify/visualize
|
|
9
|
+
* the anchor pose coming from Zappar.
|
|
10
|
+
*/
|
|
11
|
+
export class SlamAnchorZappar extends Component {
|
|
12
|
+
static TypeName = 'slam-anchor-zappar';
|
|
13
|
+
_provider = null;
|
|
14
|
+
_tmpTransform = quat2.create();
|
|
15
|
+
start() {
|
|
16
|
+
const arSession = ARSession.getSessionForEngine(this.engine);
|
|
17
|
+
arSession.onSessionStart.add(this.onSessionStart);
|
|
18
|
+
arSession.onSessionEnd.add(this.onSessionEnd);
|
|
19
|
+
// If a session is already running, RetainEmitter will call immediately.
|
|
20
|
+
}
|
|
21
|
+
update() {
|
|
22
|
+
const provider = this._provider;
|
|
23
|
+
if (!provider)
|
|
24
|
+
return;
|
|
25
|
+
const anchorPose = provider.slamAnchorPoseMatrix;
|
|
26
|
+
if (!anchorPose)
|
|
27
|
+
return;
|
|
28
|
+
// Zappar matrices are column-major; gl-matrix expects column-major.
|
|
29
|
+
quat2.fromMat4(this._tmpTransform, anchorPose);
|
|
30
|
+
this.object.setTransformWorld(this._tmpTransform);
|
|
31
|
+
}
|
|
32
|
+
onSessionStart = (provider) => {
|
|
33
|
+
this._provider = provider instanceof ZapparProvider ? provider : null;
|
|
34
|
+
};
|
|
35
|
+
onSessionEnd = (provider) => {
|
|
36
|
+
if (provider === this._provider)
|
|
37
|
+
this._provider = null;
|
|
38
|
+
};
|
|
39
|
+
}
|