@multisetai/vps 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +258 -356
- package/dist/core/index.d.ts +106 -46
- package/dist/core/index.js +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3 -3
- package/dist/three/index.d.ts +53 -0
- package/dist/three/index.js +93 -0
- package/package.json +5 -5
- package/dist/webxr/index.d.ts +0 -52
- package/dist/webxr/index.js +0 -93
package/README.md
CHANGED
|
@@ -1,473 +1,375 @@
|
|
|
1
1
|
# MultiSet VPS WebXR
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
TypeScript SDK for integrating MultiSet's Visual Positioning System (VPS) into WebXR applications. Provides 6-DOF localization by matching camera frames against cloud-hosted maps.
|
|
4
4
|
|
|
5
|
-
##
|
|
5
|
+
## Architecture
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
7
|
+
The SDK is split into two independent entry points:
|
|
8
|
+
|
|
9
|
+
| Entry point | Contents | Three.js required |
|
|
10
|
+
|---|---|---|
|
|
11
|
+
| `@multisetai/vps/core` | `MultisetClient` + `XRSessionManager` | No |
|
|
12
|
+
| `@multisetai/vps/three` | `ThreeAdapter` | Yes (peer dep) |
|
|
13
|
+
|
|
14
|
+
`XRSessionManager` owns the full vanilla WebXR session lifecycle — frame loop, camera capture, localization, tracking-loss recovery — with zero Three.js dependency. `ThreeAdapter` wires it to a Three.js renderer and scene.
|
|
14
15
|
|
|
15
16
|
## Installation
|
|
16
17
|
|
|
17
18
|
```bash
|
|
19
|
+
# Core only (no Three.js)
|
|
20
|
+
npm install @multisetai/vps
|
|
21
|
+
|
|
22
|
+
# With Three.js adapter
|
|
18
23
|
npm install @multisetai/vps three
|
|
19
24
|
```
|
|
20
25
|
|
|
21
|
-
> **Note**: `three` is a peer dependency and must be installed separately.
|
|
22
|
-
|
|
23
26
|
## Requirements
|
|
24
27
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
- **
|
|
28
|
-
- **WebXR-capable device**: Android device with ARCore
|
|
29
|
-
- **Modern browser**: Chrome or Edge on Android
|
|
30
|
-
- **Three.js**: Version 0.176.0 or higher (peer dependency)
|
|
31
|
-
|
|
32
|
-
> **iOS is not supported.** This SDK requires the `camera-access` WebXR feature to capture frames for localization. Safari on iOS does not implement `camera-access`, so the AR session will fail to start on any iOS browser.
|
|
33
|
-
|
|
34
|
-
### Development Requirements
|
|
28
|
+
- **HTTPS** — WebXR requires a secure context (`https://` or `http://localhost`).
|
|
29
|
+
- **Android + ARCore** — Chrome or Edge on an ARCore-capable Android device.
|
|
30
|
+
- **Three.js ≥ 0.176.0** — only required when using `@multisetai/vps/three`.
|
|
35
31
|
|
|
36
|
-
-
|
|
37
|
-
- TypeScript 5.8+ (for TypeScript projects)
|
|
32
|
+
> **iOS is not supported.** This SDK requires the `camera-access` WebXR feature. Safari on iOS does not implement it.
|
|
38
33
|
|
|
39
|
-
## Configuration
|
|
34
|
+
## CORS Configuration
|
|
40
35
|
|
|
41
|
-
|
|
36
|
+
The SDK makes direct browser-to-API requests, so your domain must be whitelisted in the MultiSet dashboard.
|
|
42
37
|
|
|
43
|
-
|
|
38
|
+
1. Open the [MultiSet Dashboard](https://docs.multiset.ai/basics/credentials/configuring-allowed-domains-cors)
|
|
39
|
+
2. Go to **Credentials → Settings → Domains**
|
|
40
|
+
3. Click **Add +** and enter your origin (e.g. `https://localhost:5173` for dev, `https://your-app.com` for prod)
|
|
44
41
|
|
|
45
|
-
|
|
42
|
+
Without this, the browser will block every API request with a CORS error.
|
|
46
43
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
1. Navigate to the [MultiSet Dashboard](https://docs.multiset.ai/basics/credentials/configuring-allowed-domains-cors)
|
|
50
|
-
2. Go to **Credentials → Settings** tab
|
|
51
|
-
3. Locate the **Domains** section
|
|
52
|
-
4. Click the purple **Add +** button
|
|
53
|
-
5. Enter your application's full origin (protocol + domain + port if applicable)
|
|
54
|
-
|
|
55
|
-
#### Recommended Setup
|
|
56
|
-
|
|
57
|
-
Add entries for both your local development and production environments:
|
|
58
|
-
|
|
59
|
-
**Local Development:**
|
|
60
|
-
- **Format:** `https://localhost:PORT` (e.g., `https://localhost:3000`, `https://localhost:5173`)
|
|
61
|
-
- **Note:** Use the exact port your development server runs on
|
|
62
|
-
|
|
63
|
-
**Production:**
|
|
64
|
-
- **Format:** `https://your-domain.com` (e.g., `https://www.myapp.com`)
|
|
65
|
-
- **Note:** Do not include trailing slashes
|
|
66
|
-
|
|
67
|
-
> **Important:** Without whitelisting your domain, API requests will be blocked by the browser's CORS policy, and you'll see errors in the browser console.
|
|
68
|
-
|
|
69
|
-
For more details, see the [MultiSet CORS Configuration Documentation](https://docs.multiset.ai/basics/credentials/configuring-allowed-domains-cors).
|
|
44
|
+
---
|
|
70
45
|
|
|
71
46
|
## Quick Start
|
|
72
47
|
|
|
73
|
-
###
|
|
48
|
+
### With Three.js
|
|
74
49
|
|
|
75
50
|
```typescript
|
|
76
|
-
import
|
|
77
|
-
import {
|
|
78
|
-
|
|
51
|
+
import * as THREE from 'three';
|
|
52
|
+
import { MultisetClient, XRSessionManager } from '@multisetai/vps/core';
|
|
53
|
+
import { ThreeAdapter } from '@multisetai/vps/three';
|
|
79
54
|
|
|
80
|
-
|
|
55
|
+
// Check support before showing any AR UI
|
|
56
|
+
const supported = await ThreeAdapter.isSupported();
|
|
57
|
+
if (!supported) {
|
|
58
|
+
console.warn('WebXR immersive-ar is not supported on this device.');
|
|
59
|
+
}
|
|
81
60
|
|
|
82
|
-
```typescript
|
|
83
61
|
const client = new MultisetClient({
|
|
84
62
|
clientId: 'CLIENT_ID',
|
|
85
63
|
clientSecret: 'CLIENT_SECRET',
|
|
86
64
|
code: 'MAP_OR_MAPSET_CODE',
|
|
87
|
-
mapType: 'map',
|
|
88
|
-
onAuthorize: (token) => console.log('Authorized:', token),
|
|
89
|
-
onError: (error) => console.error('Error:', error),
|
|
65
|
+
mapType: 'map',
|
|
90
66
|
});
|
|
91
67
|
|
|
92
68
|
await client.authorize();
|
|
93
|
-
```
|
|
94
69
|
|
|
95
|
-
|
|
70
|
+
const renderer = new THREE.WebGLRenderer({ antialias: true });
|
|
71
|
+
document.body.appendChild(renderer.domElement);
|
|
96
72
|
|
|
97
|
-
|
|
98
|
-
const
|
|
73
|
+
const scene = new THREE.Scene();
|
|
74
|
+
const camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 0.01, 1000);
|
|
75
|
+
|
|
76
|
+
const session = new XRSessionManager(renderer.getContext() as WebGL2RenderingContext, {
|
|
99
77
|
client,
|
|
100
|
-
canvas: document.querySelector('canvas'),
|
|
101
78
|
overlayRoot: document.body,
|
|
102
|
-
|
|
103
|
-
|
|
79
|
+
autoLocalize: true,
|
|
80
|
+
confidenceCheck: true,
|
|
81
|
+
confidenceThreshold: 0.5,
|
|
82
|
+
onSessionStart: () => {
|
|
83
|
+
// Hide the canvas during AR — the XR compositor owns the display.
|
|
84
|
+
// Leaving it visible shows the stale preview frame on top of the AR scene.
|
|
85
|
+
renderer.domElement.style.display = 'none';
|
|
86
|
+
},
|
|
87
|
+
onSessionEnd: () => {
|
|
88
|
+
renderer.domElement.style.display = 'block';
|
|
89
|
+
},
|
|
90
|
+
onLocalizationSuccess: (result) => console.log('Localized:', result.localizeData.position),
|
|
91
|
+
onLocalizationFailure: (reason) => console.warn('Failed:', reason),
|
|
92
|
+
onError: (err) => console.error(err),
|
|
104
93
|
});
|
|
105
94
|
|
|
106
|
-
|
|
95
|
+
const adapter = new ThreeAdapter({ session, renderer, scene, camera, showMesh: true });
|
|
96
|
+
adapter.initialize(); // mounts the built-in START AR / STOP AR button
|
|
97
|
+
|
|
98
|
+
// Add your 3D content
|
|
99
|
+
scene.add(new THREE.Mesh(
|
|
100
|
+
new THREE.BoxGeometry(0.1, 0.1, 0.1),
|
|
101
|
+
new THREE.MeshBasicMaterial({ color: 0xff0077 })
|
|
102
|
+
));
|
|
107
103
|
```
|
|
108
104
|
|
|
109
|
-
###
|
|
105
|
+
### Without Three.js (vanilla WebXR)
|
|
110
106
|
|
|
111
107
|
```typescript
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
108
|
+
import { MultisetClient, XRSessionManager } from '@multisetai/vps/core';
|
|
109
|
+
|
|
110
|
+
const supported = await XRSessionManager.isSupported();
|
|
111
|
+
if (!supported) {
|
|
112
|
+
console.warn('WebXR immersive-ar is not supported on this device.');
|
|
116
113
|
}
|
|
117
|
-
```
|
|
118
114
|
|
|
119
|
-
|
|
115
|
+
const client = new MultisetClient({ clientId: '...', clientSecret: '...', code: '...', mapType: 'map' });
|
|
116
|
+
await client.authorize();
|
|
120
117
|
|
|
121
|
-
|
|
118
|
+
const gl = document.querySelector('canvas')!.getContext('webgl2')!;
|
|
122
119
|
|
|
123
|
-
|
|
120
|
+
const session = new XRSessionManager(gl, {
|
|
121
|
+
client,
|
|
122
|
+
overlayRoot: document.body,
|
|
123
|
+
autoLocalize: true,
|
|
124
|
+
onLocalizationSuccess: (result) => console.log('Localized:', result.localizeData.position),
|
|
125
|
+
onError: (err) => console.error(err),
|
|
126
|
+
});
|
|
124
127
|
|
|
125
|
-
|
|
128
|
+
// Wire your own render loop before creating the button
|
|
129
|
+
session.setXRFrameHandler((event) => {
|
|
130
|
+
// event: { time, frame, pose, view, viewport, baseLayer, referenceSpace, deltaSeconds }
|
|
131
|
+
// render here using event.baseLayer.framebuffer
|
|
132
|
+
});
|
|
126
133
|
|
|
127
|
-
|
|
128
|
-
new MultisetClient(config: IMultisetSdkConfig)
|
|
134
|
+
document.body.appendChild(session.createButton());
|
|
129
135
|
```
|
|
130
136
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
**Configuration Options:**
|
|
137
|
+
---
|
|
134
138
|
|
|
135
|
-
|
|
136
|
-
|---|---|---|
|
|
137
|
-
| `clientId` | `string` | Your MultiSet client ID |
|
|
138
|
-
| `clientSecret` | `string` | Your MultiSet client secret |
|
|
139
|
-
| `code` | `string` | Map or map-set code (e.g. `'MAP_XXXXX'`) |
|
|
140
|
-
| `mapType` | `'map' \| 'map-set'` | Whether `code` refers to a single map or a map set |
|
|
141
|
-
| `endpoints` | `Partial<IMultisetSdkEndpoints>` | Override default API endpoints |
|
|
142
|
-
| `showMesh` | `boolean` | Show the VPS map mesh in the AR scene. Default: `false` |
|
|
143
|
-
| `showGizmo` | `boolean` | Show coordinate gizmo when a pose is found. Default: `true` |
|
|
144
|
-
| `autoLocalize` | `boolean` | Automatically start localization when the AR session starts |
|
|
145
|
-
| `relocalization` | `boolean` | Automatically re-localize after tracking loss is recovered |
|
|
146
|
-
| `confidenceCheck` | `boolean` | Only accept results with `confidence >= confidenceThreshold` |
|
|
147
|
-
| `confidenceThreshold` | `number` | Minimum confidence (0.2–0.8). Used when `confidenceCheck` is `true`. Default: `0.5` |
|
|
148
|
-
| `isRightHanded` | `boolean` | Handedness of the coordinate system sent to the API. Default: `true` |
|
|
149
|
-
| `passGeoPose` | `boolean` | Use the browser Geolocation API to send a `geoHint` with each localization request |
|
|
150
|
-
| `use2DFiltering` | `boolean` | Skip altitude in geoHint spatial filtering. **Only valid when `passGeoPose: true`** — TypeScript will error if `passGeoPose` is omitted or `false` |
|
|
151
|
-
| `convertToGeoCoordinates` | `boolean` | Request the backend to return results as geographic coordinates |
|
|
152
|
-
| `hintMapCodes` | `string[]` | Narrow candidate maps by code. **Only valid when `mapType: 'map-set'`** — TypeScript will error on `mapType: 'map'` |
|
|
153
|
-
| `hintPosition` | `string` | Local-space position hint as `"x,y,z"` (e.g. `"2.5,0.1,8.0"`) |
|
|
154
|
-
| `hintRadius` | `number \| string` | Search radius in meters for spatial filtering (range 1–100). Requires `hintPosition` or `passGeoPose` |
|
|
155
|
-
| `localizationTrackingTimeoutMs` | `number` | Max ms to wait for a valid viewer pose before failing. Default: `10000` |
|
|
156
|
-
| `onLocalizationInit` | `() => void` | Called at the start of a localization run |
|
|
157
|
-
| `onLocalizationSuccess` | `(result) => void` | Called when localization succeeds (and passes confidence check if enabled) |
|
|
158
|
-
| `onLocalizationFailure` | `(reason?) => void` | Called when all attempts fail or confidence is below threshold |
|
|
159
|
-
| `onAuthorize` | `(token) => void` | Called after successful authorization |
|
|
160
|
-
| `onFrameCaptured` | `(payload) => void` | Called when a camera frame is captured |
|
|
161
|
-
| `onCameraIntrinsics` | `(intrinsics) => void` | Called with camera intrinsic parameters |
|
|
162
|
-
| `onPoseResult` | `(payload) => void` | Called with the raw pose result from the backend |
|
|
163
|
-
| `onError` | `(error) => void` | Called when any error occurs |
|
|
139
|
+
## Canvas Visibility During AR
|
|
164
140
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
##### `authorize(): Promise<string>`
|
|
141
|
+
> **Important** — the SDK renders the Three.js scene into the **XR framebuffer**, not the canvas element. During an active AR session the canvas element is not updated; it retains whatever was last drawn by the preview loop. If the canvas is visible during AR (e.g. as part of the WebXR DOM overlay) it will appear as a frozen image on top of the AR scene.
|
|
168
142
|
|
|
169
|
-
|
|
143
|
+
Always hide the canvas when the session starts and restore it when it ends:
|
|
170
144
|
|
|
171
145
|
```typescript
|
|
172
|
-
|
|
146
|
+
onSessionStart: () => { renderer.domElement.style.display = 'none'; },
|
|
147
|
+
onSessionEnd: () => { renderer.domElement.style.display = 'block'; },
|
|
173
148
|
```
|
|
174
149
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
#### Events and callbacks
|
|
178
|
-
|
|
179
|
-
The client and controller emit events through callback functions:
|
|
150
|
+
---
|
|
180
151
|
|
|
181
|
-
|
|
182
|
-
- **`onFrameCaptured`**: Called when a camera frame is captured for localization
|
|
183
|
-
- **`onCameraIntrinsics`**: Called with camera intrinsic parameters
|
|
184
|
-
- **`onPoseResult`**: Called with localization results (pose found/not found)
|
|
185
|
-
- **`onLocalizationInit`**: Called at the start of a localization run
|
|
186
|
-
- **`onLocalizationSuccess`**: Called when a localization run succeeds (optionally after confidence checks)
|
|
187
|
-
- **`onLocalizationFailure`**: Called when a localization run fails or the best result is below the confidence threshold
|
|
188
|
-
- **`onError`**: Called when any error occurs during authorization or localization
|
|
152
|
+
## API Reference
|
|
189
153
|
|
|
190
|
-
###
|
|
191
|
-
|
|
192
|
-
`MultisetClient` together with `WebxrController` supports a higher-level localization flow similar to the Unity SingleFrameLocalizationManager:
|
|
193
|
-
|
|
194
|
-
- **Confidence-based acceptance**: When `confidenceCheck` is `true`, only results with `confidence >= confidenceThreshold` (0.2–0.8) are treated as successful.
|
|
195
|
-
- **Auto-localize**: When `autoLocalize` is `true`, a localization run is started automatically when the AR session starts.
|
|
196
|
-
- **Re-localize on tracking loss**: When `relocalization` is `true`, the controller automatically tries to re-localize after tracking has been lost for a short period.
|
|
197
|
-
- **Geo hint**: When `passGeoPose` is `true`, the SDK uses the browser Geolocation API to compute a `geoHint` string `"lat,lon,alt"` and sends it with each localization request to improve candidate matching. When `convertToGeoCoordinates` is `true`, the backend may include geo coordinates in the response.
|
|
198
|
-
- **2D filtering**: When `use2DFiltering` is `true` (requires `passGeoPose: true`), altitude is ignored in the `geoHint` spatial filter — useful when elevation data is unreliable.
|
|
199
|
-
- **Position hint**: `hintPosition` provides a local-space `"x,y,z"` starting point for candidate search. `hintRadius` (1–100 m) constrains the search radius around either `hintPosition` or the `geoHint`.
|
|
200
|
-
- **Map code hints**: When `mapType` is `'map-set'`, `hintMapCodes` narrows the set of candidate maps queried during localization.
|
|
201
|
-
|
|
202
|
-
## WebXR Controller API
|
|
203
|
-
|
|
204
|
-
### `WebxrController`
|
|
205
|
-
|
|
206
|
-
Manages WebXR sessions, Three.js scene, camera, and renderer. Handles AR button creation and frame capture for localization.
|
|
154
|
+
### `MultisetClient`
|
|
207
155
|
|
|
208
|
-
|
|
156
|
+
Pure HTTP client for auth and localization. No WebXR or rendering concerns.
|
|
209
157
|
|
|
210
158
|
```typescript
|
|
211
|
-
new
|
|
159
|
+
new MultisetClient(config: IMultisetClientConfig)
|
|
212
160
|
```
|
|
213
161
|
|
|
214
|
-
|
|
162
|
+
#### `IMultisetClientConfig`
|
|
215
163
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
164
|
+
| Parameter | Type | Description |
|
|
165
|
+
|---|---|---|
|
|
166
|
+
| `clientId` | `string` | Your MultiSet client ID |
|
|
167
|
+
| `clientSecret` | `string` | Your MultiSet client secret |
|
|
168
|
+
| `code` | `string` | Map or map-set code |
|
|
169
|
+
| `mapType` | `'map' \| 'map-set'` | Whether `code` is a single map or a map set |
|
|
170
|
+
| `endpoints?` | `Partial<IMultisetSdkEndpoints>` | Override default API endpoints |
|
|
171
|
+
| `isRightHanded?` | `boolean` | Handedness sent to the API. Default `true` |
|
|
172
|
+
| `convertToGeoCoordinates?` | `boolean` | Request geographic coordinates in the response |
|
|
173
|
+
| `hintPosition?` | `string` | Local-space position hint `"x,y,z"` |
|
|
174
|
+
| `hintRadius?` | `number \| string` | Search radius in metres (1–100). Requires `hintPosition` or `passGeoPose` |
|
|
175
|
+
| `hintMapCodes?` | `string[]` | Narrow candidates by map code. Only valid when `mapType: 'map-set'` |
|
|
176
|
+
| `passGeoPose?` | `boolean` | Send a `geoHint` from the Geolocation API with each request |
|
|
177
|
+
| `use2DFiltering?` | `boolean` | Skip altitude in geo filtering. Only valid when `passGeoPose: true` |
|
|
227
178
|
|
|
228
179
|
#### Methods
|
|
229
180
|
|
|
230
|
-
|
|
181
|
+
| Method | Returns | Description |
|
|
182
|
+
|---|---|---|
|
|
183
|
+
| `authorize()` | `Promise<string>` | Authenticate and cache an access token. Call before any other method. |
|
|
184
|
+
| `localizeWithFrame(frame, intrinsics)` | `Promise<ILocalizeAndMapDetails \| null>` | Submit a captured frame for localization. |
|
|
185
|
+
| `fetchMapDetails(mapCode)` | `Promise<IGetMapsDetailsResponse \| null>` | Fetch map metadata by code (result is cached). |
|
|
186
|
+
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
### `XRSessionManager`
|
|
231
190
|
|
|
232
|
-
|
|
191
|
+
Owns the WebXR session lifecycle — frame loop, camera capture, localization, and tracking-loss recovery. Zero Three.js dependency.
|
|
233
192
|
|
|
234
193
|
```typescript
|
|
235
|
-
|
|
236
|
-
```
|
|
194
|
+
import { XRSessionManager } from '@multisetai/vps/core';
|
|
237
195
|
|
|
238
|
-
|
|
196
|
+
new XRSessionManager(gl: WebGL2RenderingContext, options: IXRSessionOptions)
|
|
197
|
+
```
|
|
239
198
|
|
|
240
|
-
|
|
199
|
+
#### `IXRSessionOptions`
|
|
241
200
|
|
|
242
|
-
|
|
201
|
+
| Parameter | Type | Description |
|
|
202
|
+
|---|---|---|
|
|
203
|
+
| `client` | `MultisetClient` | Required. |
|
|
204
|
+
| `overlayRoot?` | `HTMLElement` | Root element for the WebXR DOM overlay. |
|
|
205
|
+
| `autoLocalize?` | `boolean` | Run one localization automatically when the session starts. |
|
|
206
|
+
| `relocalization?` | `boolean` | Re-localize whenever tracking is recovered after loss. |
|
|
207
|
+
| `confidenceCheck?` | `boolean` | Only accept results with `confidence >= confidenceThreshold`. |
|
|
208
|
+
| `confidenceThreshold?` | `number` | Minimum confidence (0.2–0.8). Default `0.5`. |
|
|
209
|
+
| `localizationTrackingTimeoutMs?` | `number` | Max ms to wait for a viewer pose before failing. Default `10000`. |
|
|
210
|
+
| `referenceSpaceType?` | `XRReferenceSpaceType` | XR reference space type. Default `'local'`. Use `'local-floor'` for floor-relative tracking if the device supports it. |
|
|
211
|
+
| `framebufferScaleFactor?` | `number` | XR framebuffer scale relative to native resolution. Values `< 1` reduce GPU load; values `> 1` supersample. |
|
|
212
|
+
| `onSessionStart?` | `() => void` | Called when the AR session starts. |
|
|
213
|
+
| `onSessionEnd?` | `() => void` | Called when the AR session ends. |
|
|
214
|
+
| `onLocalizationInit?` | `() => void` | Called at the start of a localization run. |
|
|
215
|
+
| `onLocalizationSuccess?` | `(result: ILocalizeAndMapDetails) => void` | Called when localization succeeds (and passes the confidence check, if enabled). |
|
|
216
|
+
| `onLocalizationFailure?` | `(reason?: string) => void` | Called when localization fails or falls below the confidence threshold. |
|
|
217
|
+
| `onFrameCaptured?` | `(frame: IFrameCaptureEvent) => void` | Called when a camera frame is captured for localization. |
|
|
218
|
+
| `onCameraIntrinsics?` | `(intrinsics: ICameraIntrinsicsEvent) => void` | Called with camera intrinsic parameters for the captured frame. |
|
|
219
|
+
| `onPoseResult?` | `(pose: IPoseResultEvent) => void` | Called with the raw pose result from the VPS backend. |
|
|
220
|
+
| `onError?` | `(error: unknown) => void` | Called when any error occurs. |
|
|
221
|
+
| `onContextLost?` | `() => void` | Called when the WebGL context is lost. The active session is ended automatically. |
|
|
222
|
+
| `onContextRestored?` | `() => void` | Called when the WebGL context is restored. The user may restart the session. |
|
|
223
|
+
|
|
224
|
+
#### Static methods
|
|
225
|
+
|
|
226
|
+
| Method | Returns | Description |
|
|
227
|
+
|---|---|---|
|
|
228
|
+
| `XRSessionManager.isSupported()` | `Promise<boolean>` | Returns `true` if the browser supports `immersive-ar` WebXR sessions. Use this to conditionally show AR UI before creating any objects. |
|
|
243
229
|
|
|
244
|
-
|
|
245
|
-
const result = await controller.localizeFrame();
|
|
246
|
-
if (result?.localizeData?.poseFound) {
|
|
247
|
-
console.log('Localized at:', result.localizeData.position);
|
|
248
|
-
}
|
|
249
|
-
```
|
|
230
|
+
#### Methods
|
|
250
231
|
|
|
251
|
-
|
|
232
|
+
| Method | Returns | Description |
|
|
233
|
+
|---|---|---|
|
|
234
|
+
| `createButton()` | `HTMLButtonElement` | Create the built-in styled AR button. Shows **START AR** / **STOP AR** and toggles the session on click. |
|
|
235
|
+
| `startSession()` | `Promise<void>` | Start an AR session programmatically. **Must be called from within a user gesture handler** (click/tap). |
|
|
236
|
+
| `stopSession()` | `void` | Stop the active AR session. No-op if no session is running. |
|
|
237
|
+
| `localizeFrame()` | `Promise<ILocalizeAndMapDetails \| null>` | Capture and localize one frame. Requires an active session. |
|
|
238
|
+
| `isActive()` | `boolean` | Whether an XR session is currently running. |
|
|
239
|
+
| `isLocalizing` | `boolean` | Whether a localization run is currently in progress. |
|
|
240
|
+
| `getClient()` | `MultisetClient` | Access the underlying `MultisetClient`. |
|
|
241
|
+
| `dispose()` | `void` | End the session, remove context loss listeners, and release all resources. |
|
|
252
242
|
|
|
253
|
-
|
|
254
|
-
- Waits for a valid viewer pose (up to `localizationTrackingTimeoutMs`, default 10 s)
|
|
255
|
-
- Captures the camera frame and calls the localization API
|
|
256
|
-
- Applies the confidence threshold when `confidenceCheck` is enabled
|
|
257
|
-
- Calls `onLocalizationSuccess` or `onLocalizationFailure` accordingly
|
|
243
|
+
#### Adapter hooks
|
|
258
244
|
|
|
259
|
-
|
|
245
|
+
Used internally by `ThreeAdapter`. Only call these when building a custom renderer adapter.
|
|
260
246
|
|
|
261
|
-
|
|
247
|
+
| Method | Description |
|
|
248
|
+
|---|---|
|
|
249
|
+
| `setXRFrameHandler(fn)` | Called every XR frame with pose, view, viewport, and framebuffer info. |
|
|
250
|
+
| `setAdapterResultHandler(fn)` | Called after a successful localization with the result and tracker-space matrix. |
|
|
251
|
+
| `setAdapterSessionHandlers(onStart, onEnd)` | Called on session start/end, before user callbacks. |
|
|
262
252
|
|
|
263
|
-
|
|
264
|
-
const scene = controller.getScene();
|
|
265
|
-
const cube = new THREE.Mesh(geometry, material);
|
|
266
|
-
scene.add(cube);
|
|
267
|
-
```
|
|
253
|
+
---
|
|
268
254
|
|
|
269
|
-
|
|
255
|
+
### `ThreeAdapter`
|
|
270
256
|
|
|
271
|
-
|
|
257
|
+
Wires `XRSessionManager` to a Three.js renderer. Handles XR framebuffer binding, camera matrix sync, preview loop, resize, and optional map mesh / gizmo display.
|
|
272
258
|
|
|
273
259
|
```typescript
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
260
|
+
import { ThreeAdapter } from '@multisetai/vps/three';
|
|
261
|
+
|
|
262
|
+
new ThreeAdapter(options: IThreeAdapterOptions)
|
|
277
263
|
```
|
|
278
264
|
|
|
279
|
-
|
|
265
|
+
#### `IThreeAdapterOptions`
|
|
280
266
|
|
|
281
|
-
|
|
267
|
+
| Parameter | Type | Description |
|
|
268
|
+
|---|---|---|
|
|
269
|
+
| `session` | `XRSessionManager` | Required. |
|
|
270
|
+
| `renderer` | `THREE.WebGLRenderer` | Required. |
|
|
271
|
+
| `scene` | `THREE.Scene` | Required. |
|
|
272
|
+
| `camera` | `THREE.PerspectiveCamera` | Required. |
|
|
273
|
+
| `showMesh?` | `boolean` | Show the VPS map mesh after localization. Default `false`. |
|
|
274
|
+
| `showGizmo?` | `boolean` | Show a transform gizmo after localization. Default `true`. |
|
|
275
|
+
| `useDefaultButton?` | `boolean` | Mount the built-in START AR / STOP AR button. Default `true`. Set to `false` to drive the session via `startSession()` / `stopSession()`. |
|
|
276
|
+
| `buttonContainer?` | `HTMLElement` | Where to append the built-in button. Defaults to `overlayRoot` or `document.body`. |
|
|
277
|
+
| `onButtonCreated?` | `(button: HTMLButtonElement) => void` | Called after the built-in button is created. |
|
|
278
|
+
| `onXRFrame?` | `(event: IXRFrameEvent) => void` | Called every XR frame after camera matrices are synced, before the scene is rendered. Use this to update scene objects each frame. |
|
|
279
|
+
|
|
280
|
+
#### Static methods
|
|
281
|
+
|
|
282
|
+
| Method | Returns | Description |
|
|
283
|
+
|---|---|---|
|
|
284
|
+
| `ThreeAdapter.isSupported()` | `Promise<boolean>` | Returns `true` if the browser supports `immersive-ar` WebXR sessions. |
|
|
282
285
|
|
|
283
|
-
|
|
284
|
-
const renderer = controller.getRenderer();
|
|
285
|
-
renderer.shadowMap.enabled = true;
|
|
286
|
-
```
|
|
286
|
+
#### Methods
|
|
287
287
|
|
|
288
|
-
|
|
288
|
+
| Method | Returns | Description |
|
|
289
|
+
|---|---|---|
|
|
290
|
+
| `initialize(buttonContainer?)` | `HTMLButtonElement \| null` | Start the preview render loop, attach resize handler, and mount the built-in button. Returns `null` when `useDefaultButton: false`. |
|
|
291
|
+
| `isActive()` | `boolean` | Whether an XR session is currently running. |
|
|
292
|
+
| `isLocalizing` | `boolean` | Whether a localization run is currently in progress. |
|
|
293
|
+
| `startSession()` | `Promise<void>` | Start an AR session. **Must be called from within a user gesture handler.** |
|
|
294
|
+
| `stopSession()` | `void` | Stop the active AR session. No-op if no session is running. |
|
|
295
|
+
| `localizeFrame()` | `Promise<ILocalizeAndMapDetails \| null>` | Capture and localize one frame. |
|
|
296
|
+
| `dispose()` | `void` | Stop loops, remove listeners, dispose Three.js resources, and end the session. |
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
## Styling the AR Button
|
|
301
|
+
|
|
302
|
+
The built-in button ships with minimal inline styles. Use these CSS classes to override appearance from your own stylesheet:
|
|
303
|
+
|
|
304
|
+
| Class | When present |
|
|
305
|
+
|---|---|
|
|
306
|
+
| `.multiset-ar-button` | Always — use for base styles |
|
|
307
|
+
| `.multiset-ar-inactive` | No active session (START AR state) |
|
|
308
|
+
| `.multiset-ar-active` | During an active AR session (STOP AR state) |
|
|
309
|
+
|
|
310
|
+
```css
|
|
311
|
+
.multiset-ar-button {
|
|
312
|
+
font-family: 'Your Font', sans-serif;
|
|
313
|
+
border-radius: 8px;
|
|
314
|
+
}
|
|
289
315
|
|
|
290
|
-
|
|
316
|
+
.multiset-ar-inactive {
|
|
317
|
+
background: rgba(0, 0, 0, 0.5);
|
|
318
|
+
}
|
|
291
319
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
320
|
+
.multiset-ar-active {
|
|
321
|
+
background: rgba(200, 0, 0, 0.6);
|
|
322
|
+
border-color: #ff4444;
|
|
295
323
|
}
|
|
296
324
|
```
|
|
297
325
|
|
|
298
|
-
|
|
326
|
+
---
|
|
327
|
+
|
|
328
|
+
## Custom Start / Stop Button
|
|
299
329
|
|
|
300
|
-
|
|
330
|
+
Set `useDefaultButton: false` to manage the session yourself:
|
|
301
331
|
|
|
302
332
|
```typescript
|
|
303
|
-
|
|
333
|
+
const adapter = new ThreeAdapter({
|
|
334
|
+
session,
|
|
335
|
+
renderer, scene, camera,
|
|
336
|
+
useDefaultButton: false,
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
adapter.initialize();
|
|
340
|
+
|
|
341
|
+
myButton.addEventListener('click', () => {
|
|
342
|
+
if (adapter.isActive()) {
|
|
343
|
+
adapter.stopSession();
|
|
344
|
+
} else {
|
|
345
|
+
// startSession() must be the first call inside the gesture handler —
|
|
346
|
+
// any await before it consumes the browser's user activation token.
|
|
347
|
+
void adapter.startSession();
|
|
348
|
+
}
|
|
349
|
+
});
|
|
304
350
|
```
|
|
305
351
|
|
|
306
|
-
|
|
352
|
+
---
|
|
307
353
|
|
|
308
|
-
|
|
354
|
+
## Type Definitions
|
|
309
355
|
|
|
310
356
|
```typescript
|
|
311
|
-
// Core types
|
|
312
357
|
import type {
|
|
313
|
-
|
|
314
|
-
IMultisetPublicConfig,
|
|
358
|
+
IMultisetClientConfig,
|
|
315
359
|
IMultisetSdkEndpoints,
|
|
360
|
+
IXRSessionOptions,
|
|
361
|
+
IXRFrameEvent,
|
|
316
362
|
IFrameCaptureEvent,
|
|
317
363
|
ICameraIntrinsicsEvent,
|
|
318
364
|
IPoseResultEvent,
|
|
319
365
|
ILocalizeAndMapDetails,
|
|
320
|
-
|
|
366
|
+
IGetMapsDetailsResponse,
|
|
321
367
|
MapType,
|
|
322
368
|
} from '@multisetai/vps/core';
|
|
323
369
|
|
|
324
|
-
|
|
325
|
-
import type {
|
|
326
|
-
IWebxrControllerOptions,
|
|
327
|
-
} from '@multisetai/vps/webxr';
|
|
328
|
-
```
|
|
329
|
-
|
|
330
|
-
## Examples
|
|
331
|
-
|
|
332
|
-
> **Full working examples**: This repository includes complete, runnable example applications in the [`examples/`](./examples) directory:
|
|
333
|
-
> - **[React Example](./examples/react)** - Full React implementation with TypeScript
|
|
334
|
-
> - **[Vanilla JavaScript Example](./examples/vanilla)** - Vanilla JavaScript implementation
|
|
335
|
-
>
|
|
336
|
-
> Each example includes setup instructions, configuration, and demonstrates the complete SDK integration workflow.
|
|
337
|
-
|
|
338
|
-
### Vanilla JavaScript
|
|
339
|
-
|
|
340
|
-
```javascript
|
|
341
|
-
import * as THREE from 'three';
|
|
342
|
-
import { MultisetClient } from '@multisetai/vps/core';
|
|
343
|
-
import { WebxrController } from '@multisetai/vps/webxr';
|
|
344
|
-
|
|
345
|
-
const client = new MultisetClient({
|
|
346
|
-
clientId: 'CLIENT_ID',
|
|
347
|
-
clientSecret: 'CLIENT_SECRET',
|
|
348
|
-
code: 'MAP_OR_MAPSET_CODE',
|
|
349
|
-
mapType: 'map',
|
|
350
|
-
showMesh: true,
|
|
351
|
-
confidenceCheck: true,
|
|
352
|
-
confidenceThreshold: 0.5,
|
|
353
|
-
onLocalizationSuccess: (result) => console.log('Localized:', result.localizeData.position),
|
|
354
|
-
onLocalizationFailure: (reason) => console.warn('Localization failed:', reason),
|
|
355
|
-
onAuthorize: (token) => console.log('Authorized:', token),
|
|
356
|
-
onError: (error) => console.error('Error:', error),
|
|
357
|
-
});
|
|
358
|
-
|
|
359
|
-
const controller = new WebxrController({
|
|
360
|
-
client,
|
|
361
|
-
canvas: document.querySelector('canvas'),
|
|
362
|
-
overlayRoot: document.body,
|
|
363
|
-
onSessionStart: () => console.log('AR session started'),
|
|
364
|
-
onSessionEnd: () => console.log('AR session ended'),
|
|
365
|
-
});
|
|
366
|
-
|
|
367
|
-
// Authorize and initialize
|
|
368
|
-
await client.authorize();
|
|
369
|
-
await controller.initialize();
|
|
370
|
-
|
|
371
|
-
// Add 3D objects to scene
|
|
372
|
-
const scene = controller.getScene();
|
|
373
|
-
const cube = new THREE.Mesh(
|
|
374
|
-
new THREE.BoxGeometry(0.1, 0.1, 0.1),
|
|
375
|
-
new THREE.MeshBasicMaterial({ color: 0xff0077 })
|
|
376
|
-
);
|
|
377
|
-
cube.position.set(0, 0, -0.4);
|
|
378
|
-
scene.add(cube);
|
|
379
|
-
|
|
380
|
-
// Manually trigger a single localization attempt
|
|
381
|
-
const result = await controller.localizeFrame();
|
|
382
|
-
if (result?.localizeData?.poseFound) {
|
|
383
|
-
console.log('Position:', result.localizeData.position);
|
|
384
|
-
console.log('Rotation:', result.localizeData.rotation);
|
|
385
|
-
}
|
|
386
|
-
```
|
|
387
|
-
|
|
388
|
-
### React
|
|
389
|
-
|
|
390
|
-
```tsx
|
|
391
|
-
import { useEffect, useRef, useState } from 'react';
|
|
392
|
-
import * as THREE from 'three';
|
|
393
|
-
import { MultisetClient } from '@multisetai/vps/core';
|
|
394
|
-
import { WebxrController } from '@multisetai/vps/webxr';
|
|
395
|
-
|
|
396
|
-
export default function App() {
|
|
397
|
-
const [authorized, setAuthorized] = useState(false);
|
|
398
|
-
const [arSessionActive, setArSessionActive] = useState(false);
|
|
399
|
-
const canvasRef = useRef<HTMLCanvasElement>(null);
|
|
400
|
-
const clientRef = useRef<MultisetClient | null>(null);
|
|
401
|
-
const controllerRef = useRef<WebxrController | null>(null);
|
|
402
|
-
|
|
403
|
-
useEffect(() => {
|
|
404
|
-
return () => {
|
|
405
|
-
controllerRef.current?.dispose();
|
|
406
|
-
};
|
|
407
|
-
}, []);
|
|
408
|
-
|
|
409
|
-
const handleAuthorize = async () => {
|
|
410
|
-
if (!canvasRef.current) return;
|
|
411
|
-
|
|
412
|
-
clientRef.current = new MultisetClient({
|
|
413
|
-
clientId: 'CLIENT_ID',
|
|
414
|
-
clientSecret: 'CLIENT_SECRET',
|
|
415
|
-
code: 'MAP_OR_MAPSET_CODE',
|
|
416
|
-
mapType: 'map',
|
|
417
|
-
showMesh: true,
|
|
418
|
-
confidenceCheck: true,
|
|
419
|
-
confidenceThreshold: 0.5,
|
|
420
|
-
onLocalizationSuccess: (result) => console.log('Localized:', result.localizeData.position),
|
|
421
|
-
onLocalizationFailure: (reason) => console.warn('Localization failed:', reason),
|
|
422
|
-
onError: (error) => console.error('Error:', error),
|
|
423
|
-
});
|
|
424
|
-
|
|
425
|
-
controllerRef.current = new WebxrController({
|
|
426
|
-
client: clientRef.current,
|
|
427
|
-
canvas: canvasRef.current,
|
|
428
|
-
overlayRoot: document.body,
|
|
429
|
-
onSessionStart: () => setArSessionActive(true),
|
|
430
|
-
onSessionEnd: () => setArSessionActive(false),
|
|
431
|
-
});
|
|
432
|
-
|
|
433
|
-
await clientRef.current.authorize();
|
|
434
|
-
await controllerRef.current.initialize();
|
|
435
|
-
|
|
436
|
-
// Add 3D objects to the scene
|
|
437
|
-
const scene = controllerRef.current.getScene();
|
|
438
|
-
const cube = new THREE.Mesh(
|
|
439
|
-
new THREE.BoxGeometry(0.1, 0.1, 0.1),
|
|
440
|
-
new THREE.MeshBasicMaterial({ color: 0xff0077 })
|
|
441
|
-
);
|
|
442
|
-
cube.position.set(0, 0, -0.4);
|
|
443
|
-
scene.add(cube);
|
|
444
|
-
|
|
445
|
-
setAuthorized(true);
|
|
446
|
-
};
|
|
447
|
-
|
|
448
|
-
const handleCapture = async () => {
|
|
449
|
-
const result = await controllerRef.current!.localizeFrame();
|
|
450
|
-
if (result?.localizeData?.poseFound) {
|
|
451
|
-
console.log('Position:', result.localizeData.position);
|
|
452
|
-
console.log('Rotation:', result.localizeData.rotation);
|
|
453
|
-
}
|
|
454
|
-
};
|
|
455
|
-
|
|
456
|
-
return (
|
|
457
|
-
<div>
|
|
458
|
-
{!authorized && (
|
|
459
|
-
<button onClick={handleAuthorize}>Authorize</button>
|
|
460
|
-
)}
|
|
461
|
-
{arSessionActive && (
|
|
462
|
-
<button onClick={handleCapture}>Capture</button>
|
|
463
|
-
)}
|
|
464
|
-
<canvas ref={canvasRef} />
|
|
465
|
-
</div>
|
|
466
|
-
);
|
|
467
|
-
}
|
|
370
|
+
import type { IThreeAdapterOptions } from '@multisetai/vps/three';
|
|
468
371
|
```
|
|
469
372
|
|
|
470
373
|
## License
|
|
471
374
|
|
|
472
375
|
[](LICENSE)
|
|
473
|
-
|