@displayxr/inline3d 1.29.0 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +187 -0
- package/README.md +9 -5
- package/call.d.ts +318 -238
- package/camera.d.ts +257 -0
- package/dist/call.js +17 -12
- package/index.d.ts +56 -0
- package/js/call/capture.js +4 -212
- package/js/call/disparity.js +4 -432
- package/js/call/element.js +14 -10
- package/js/call/options.js +132 -0
- package/js/call/signaling.js +66 -9
- package/js/call/ui.js +106 -9
- package/js/call/wire.js +8 -153
- package/js/camera/capture.js +220 -0
- package/js/camera/converge.js +82 -0
- package/js/camera/disparity.js +432 -0
- package/js/camera/geometry.js +80 -0
- package/js/camera/metadata.js +398 -0
- package/js/inline3d-call.js +435 -320
- package/js/inline3d-camera.js +678 -0
- package/js/inline3d-model-playcanvas.js +1 -0
- package/js/inline3d-model.js +1 -0
- package/js/inline3d-splat-deferred.js +153 -0
- package/js/inline3d-splat-pc-entry.js +45 -0
- package/js/inline3d-splat-playcanvas.js +14 -3
- package/js/inline3d-splat.js +3 -145
- package/js/inline3d-viewer-ease.js +360 -0
- package/js/inline3d-viewer.js +15 -4
- package/js/inline3d.js +13 -0
- package/model.d.ts +8 -1
- package/package.json +22 -2
- package/splat-playcanvas.d.ts +4 -0
- package/splat.d.ts +8 -1
- package/viewer.d.ts +9 -0
package/js/call/disparity.js
CHANGED
|
@@ -1,432 +1,4 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
// No focal length, baseline or distance: those only PREDICT the disparity (f·B/Z); here it is read
|
|
6
|
-
// off the pixels. Works for any SBS sender, calibrated or not, as long as the rows roughly align (a
|
|
7
|
-
// small vertical search absorbs the residual of a raw, unrectified pair).
|
|
8
|
-
//
|
|
9
|
-
// The focus is ONE point, the midpoint between the eyes. It stays meaningful when the head turns (one
|
|
10
|
-
// eye may be hidden), unlike "both eyes" or "the whole face". Without a face detector the module
|
|
11
|
-
// finds the subject as the NEAREST strong disparity mode (in a call the person is in front of the
|
|
12
|
-
// room) and takes the eye line from the top of that blob.
|
|
13
|
-
//
|
|
14
|
-
// Pure: a grayscale SBS image in, numbers out. No DOM, no WebRTC. Sizes are in the INPUT's pixels;
|
|
15
|
-
// the caller scales back to source pixels.
|
|
16
|
-
|
|
17
|
-
/** Blocks with a luma standard deviation below this are too flat to match. */
|
|
18
|
-
const MIN_TEXTURE = 6;
|
|
19
|
-
|
|
20
|
-
/** A block match must reach this normalised cross-correlation to count. */
|
|
21
|
-
export const MIN_NCC = 0.8;
|
|
22
|
-
|
|
23
|
-
/** ...and beat its best rival peak (>= 3 px away) by this much, or it is ambiguous (periodic). */
|
|
24
|
-
export const UNIQUENESS = 0.08;
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Normalised cross-correlation of a `bw`×`bh` block of the left eye at (lx, ly) against the right
|
|
28
|
-
* eye at (rx, ry). `img` is the whole SBS image, `W` its full width, `E` the per-eye width.
|
|
29
|
-
* Returns -1 when the right block would leave the right eye or is flat.
|
|
30
|
-
*/
|
|
31
|
-
function ncc(img, W, E, lx, ly, rx, ry, bw, bh, lMean, lStd) {
|
|
32
|
-
if (rx < 0 || rx + bw > E || ry < 0) return -1;
|
|
33
|
-
let sum = 0;
|
|
34
|
-
let sum2 = 0;
|
|
35
|
-
let cross = 0;
|
|
36
|
-
const n = bw * bh;
|
|
37
|
-
for (let y = 0; y < bh; y++) {
|
|
38
|
-
const lo = (ly + y) * W + lx;
|
|
39
|
-
const ro = (ry + y) * W + E + rx;
|
|
40
|
-
for (let x = 0; x < bw; x++) {
|
|
41
|
-
const r = img[ro + x];
|
|
42
|
-
sum += r;
|
|
43
|
-
sum2 += r * r;
|
|
44
|
-
cross += img[lo + x] * r;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
const rMean = sum / n;
|
|
48
|
-
const rVar = sum2 / n - rMean * rMean;
|
|
49
|
-
if (rVar < 1) return -1;
|
|
50
|
-
return (cross / n - lMean * rMean) / (lStd * Math.sqrt(rVar));
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
function blockStats(img, W, x0, y0, bw, bh) {
|
|
54
|
-
let sum = 0;
|
|
55
|
-
let sum2 = 0;
|
|
56
|
-
for (let y = 0; y < bh; y++) {
|
|
57
|
-
const o = (y0 + y) * W + x0;
|
|
58
|
-
for (let x = 0; x < bw; x++) {
|
|
59
|
-
const v = img[o + x];
|
|
60
|
-
sum += v;
|
|
61
|
-
sum2 += v * v;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
const n = bw * bh;
|
|
65
|
-
const mean = sum / n;
|
|
66
|
-
return { mean, std: Math.sqrt(Math.max(0, sum2 / n - mean * mean)) };
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* Best horizontal disparity of one left-eye block: `d = xLeft − xRight` (positive = crossed = in
|
|
71
|
-
* front of the display plane), searched over [dMin, dMax] and ±dy rows. Sub-pixel by a parabola.
|
|
72
|
-
* @returns {{d:number, dy:number, c:number}|null}
|
|
73
|
-
*/
|
|
74
|
-
export function matchBlock(img, W, H, x0, y0, bw, bh, { dMin, dMax, dyMax = 2, uniq } = {}) {
|
|
75
|
-
const E = W / 2;
|
|
76
|
-
const { mean, std } = blockStats(img, W, x0, y0, bw, bh);
|
|
77
|
-
if (std < MIN_TEXTURE) return null;
|
|
78
|
-
let best = -2;
|
|
79
|
-
let bd = 0;
|
|
80
|
-
let bdy = 0;
|
|
81
|
-
const perD = new Map(); // best NCC seen at each disparity (any row), for the uniqueness test
|
|
82
|
-
const scan = (d0, d1, dStep, y0s, y1s) => {
|
|
83
|
-
for (let dy = y0s; dy <= y1s; dy++) {
|
|
84
|
-
const ry = y0 + dy;
|
|
85
|
-
if (ry < 0 || ry + bh > H) continue;
|
|
86
|
-
for (let d = d0; d <= d1; d += dStep) {
|
|
87
|
-
const c = ncc(img, W, E, x0, y0, x0 - d, ry, bw, bh, mean, std);
|
|
88
|
-
if (c > (perD.get(d) ?? -2)) perD.set(d, c);
|
|
89
|
-
if (c > best) {
|
|
90
|
-
best = c;
|
|
91
|
-
bd = d;
|
|
92
|
-
bdy = dy;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
};
|
|
97
|
-
// Coarse (every 2nd disparity, rows every 2nd), then a full-resolution pass around the winner:
|
|
98
|
-
// ~1/4 of the exhaustive cost, and textures here are never finer than the downscaled 2-px step.
|
|
99
|
-
const coarse = dMax - dMin > 8;
|
|
100
|
-
scan(dMin, dMax, coarse ? 2 : 1, -dyMax, dyMax);
|
|
101
|
-
if (coarse && best > -1) {
|
|
102
|
-
const cd = bd;
|
|
103
|
-
const cy = bdy;
|
|
104
|
-
scan(Math.max(dMin, cd - 2), Math.min(dMax, cd + 2), 1, Math.max(-dyMax, cy - 1), Math.min(dyMax, cy + 1));
|
|
105
|
-
}
|
|
106
|
-
if (best < -1) return null;
|
|
107
|
-
// Uniqueness: a repeating texture (a checkerboard, a striped shirt, blinds) matches equally well
|
|
108
|
-
// one period away. Reject the block unless its best peak clearly beats every peak >= 3 px away.
|
|
109
|
-
// The coarse scan samples every 2nd disparity, so a rival peak between two samples is under-read
|
|
110
|
-
// while the winner has been refined to its top; refine each rival local maximum at +-1 as well, or
|
|
111
|
-
// an exactly periodic pattern slips through by a hair.
|
|
112
|
-
let second = -2;
|
|
113
|
-
const peakAt = (d) => {
|
|
114
|
-
let v = -2;
|
|
115
|
-
for (let dy = -dyMax; dy <= dyMax; dy++) {
|
|
116
|
-
const ry = y0 + dy;
|
|
117
|
-
if (ry < 0 || ry + bh > H) continue;
|
|
118
|
-
for (let k = d - 1; k <= d + 1; k++) v = Math.max(v, ncc(img, W, E, x0, y0, x0 - k, ry, bw, bh, mean, std));
|
|
119
|
-
}
|
|
120
|
-
return v;
|
|
121
|
-
};
|
|
122
|
-
for (const [d, c] of perD) {
|
|
123
|
-
if (Math.abs(d - bd) < 3 || c <= second) continue;
|
|
124
|
-
const isLocalMax = c >= (perD.get(d - 2) ?? -2) && c >= (perD.get(d + 2) ?? -2);
|
|
125
|
-
const v = isLocalMax && c > 0 ? peakAt(d) : c;
|
|
126
|
-
if (v > second) second = v;
|
|
127
|
-
}
|
|
128
|
-
// Only for WIDE searches: in a narrow (tracking/refine) window a smooth face legitimately scores
|
|
129
|
-
// high a few px off its peak, and a window anchored on a known disparity cannot alias anyway.
|
|
130
|
-
if (dMax - dMin > 12 && second > -2 && best - second < (uniq ?? UNIQUENESS)) return null;
|
|
131
|
-
// Sub-pixel along d at the best row.
|
|
132
|
-
const cm = ncc(img, W, E, x0, y0, x0 - (bd - 1), y0 + bdy, bw, bh, mean, std);
|
|
133
|
-
const cp = ncc(img, W, E, x0, y0, x0 - (bd + 1), y0 + bdy, bw, bh, mean, std);
|
|
134
|
-
let sub = 0;
|
|
135
|
-
const den = cm - 2 * best + cp;
|
|
136
|
-
if (cm > -1 && cp > -1 && den < 0) sub = Math.max(-0.5, Math.min(0.5, (0.5 * (cm - cp)) / den));
|
|
137
|
-
return { d: bd + sub, dy: bdy, c: best };
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Block disparities over the central region of the LEFT eye.
|
|
142
|
-
* @returns {Array<{x:number,y:number,d:number,dy:number,c:number}>} block top-left corners
|
|
143
|
-
*/
|
|
144
|
-
export function blockDisparities(img, W, H, o = {}) {
|
|
145
|
-
const E = W / 2;
|
|
146
|
-
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
147
|
-
const step = o.step || Math.round(b * 1.5); // sparse grid: half the blocks, same modes
|
|
148
|
-
const dMax = o.dMax ?? Math.round(E * 0.3);
|
|
149
|
-
const dMin = o.dMin ?? -Math.round(E * 0.05);
|
|
150
|
-
const x0 = Math.round(E * (o.roiX0 ?? 0.15));
|
|
151
|
-
const x1 = Math.round(E * (o.roiX1 ?? 0.85)) - b;
|
|
152
|
-
const y0 = Math.round(H * (o.roiY0 ?? 0.05));
|
|
153
|
-
const y1 = Math.round(H * (o.roiY1 ?? 0.85)) - b;
|
|
154
|
-
const out = [];
|
|
155
|
-
for (let y = y0; y <= y1; y += step) {
|
|
156
|
-
for (let x = x0; x <= x1; x += step) {
|
|
157
|
-
const m = matchBlock(img, W, H, x, y, b, b, { dMin, dMax, dyMax: o.dyMax ?? 2, uniq: o.uniq });
|
|
158
|
-
if (m && m.c >= (o.minNcc ?? MIN_NCC)) out.push({ x, y, ...m });
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
return out;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
/**
|
|
165
|
-
* The subject = the NEAREST strong disparity mode: the largest `d` whose ±`tol` bin holds at least
|
|
166
|
-
* `minFrac` of the matched blocks (and `minBlocks`). Returns its members, or null.
|
|
167
|
-
*/
|
|
168
|
-
export function nearestMode(blocks, { tol = 2, minFrac = 0.12, minBlocks = 3 } = {}) {
|
|
169
|
-
if (!blocks.length) return null;
|
|
170
|
-
const need = Math.max(minBlocks, Math.ceil(blocks.length * minFrac));
|
|
171
|
-
const ds = blocks.map((b) => b.d).sort((a, b) => b - a); // descending: nearest first
|
|
172
|
-
for (const d of ds) {
|
|
173
|
-
const members = blocks.filter((b) => Math.abs(b.d - d) <= tol);
|
|
174
|
-
if (members.length >= need) {
|
|
175
|
-
const md = members.map((b) => b.d).sort((a, b) => a - b);
|
|
176
|
-
return { d: md[md.length >> 1], members };
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
return null;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* Where the eyes are in a subject blob: horizontally the centre of its TOP rows (the head, not the
|
|
184
|
-
* shoulders), vertically ~40% down a head whose height ≈ 1.3 × its width.
|
|
185
|
-
*
|
|
186
|
-
* Measured on real SR-camera frames (grayscale, dim), only a handful of blocks match on a face, so
|
|
187
|
-
* this lands on the upper face, often the forehead, a few cm above the eyes. That is the same depth
|
|
188
|
-
* plane to within ~1 cm: at 0.6 m the disparity differs by ~1-2 source px, under 1 px of shift per
|
|
189
|
-
* eye. A darkest-band eye-line search was tried and rejected: the darkest band is the hair. Pass
|
|
190
|
-
* `focus` (a face detector's eye midpoint) for the exact point.
|
|
191
|
-
*/
|
|
192
|
-
export function eyeMidpoint(members, block) {
|
|
193
|
-
const top = Math.min(...members.map((m) => m.y));
|
|
194
|
-
const headRows = members.filter((m) => m.y <= top + 2 * block);
|
|
195
|
-
const xs = headRows.map((m) => m.x);
|
|
196
|
-
const xMin = Math.min(...xs);
|
|
197
|
-
const xMax = Math.max(...xs) + block;
|
|
198
|
-
const headW = xMax - xMin;
|
|
199
|
-
return { x: (xMin + xMax) / 2, y: top + 0.4 * 1.3 * headW };
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
/**
|
|
203
|
-
* The disparity of the point between the eyes, in input pixels.
|
|
204
|
-
*
|
|
205
|
-
* @param {Uint8Array|Uint8ClampedArray|number[]} img grayscale SBS, row-major, `W`×`H`
|
|
206
|
-
* @param {number} W full SBS width (both eyes)
|
|
207
|
-
* @param {number} H
|
|
208
|
-
* @param {object} [o]
|
|
209
|
-
* @param {{x:number,y:number}} [o.focus] the eye midpoint in LEFT-eye pixels (e.g. from a face
|
|
210
|
-
* detector); skips the subject search
|
|
211
|
-
* @returns {{d:number, x:number, y:number, c:number, method:'focus'|'mode', blocks:number}|null}
|
|
212
|
-
*/
|
|
213
|
-
export function measureFocusDisparity(img, W, H, o = {}) {
|
|
214
|
-
const E = W / 2;
|
|
215
|
-
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
216
|
-
const dMax = o.dMax ?? Math.round(E * 0.3);
|
|
217
|
-
const dMin = o.dMin ?? -Math.round(E * 0.05);
|
|
218
|
-
let focus = o.focus || null;
|
|
219
|
-
let method = 'focus';
|
|
220
|
-
let blocks = 0;
|
|
221
|
-
let d0 = null;
|
|
222
|
-
if (!focus) {
|
|
223
|
-
const all = blockDisparities(img, W, H, { ...o, block: b, dMax, dMin });
|
|
224
|
-
blocks = all.length;
|
|
225
|
-
const mode = nearestMode(all, o);
|
|
226
|
-
if (!mode) return null;
|
|
227
|
-
focus = eyeMidpoint(mode.members, b);
|
|
228
|
-
d0 = mode.d;
|
|
229
|
-
method = 'mode';
|
|
230
|
-
}
|
|
231
|
-
// Refine with one wider window centred on the focus point: eyes + brow + nose bridge carry
|
|
232
|
-
// plenty of texture even when one eye is turned away.
|
|
233
|
-
const ww = Math.round(b * (o.refineW ?? 3));
|
|
234
|
-
const wh = Math.round(b * (o.refineH ?? 1.5));
|
|
235
|
-
const rx = Math.round(Math.max(0, Math.min(E - ww, focus.x - ww / 2)));
|
|
236
|
-
const ry = Math.round(Math.max(0, Math.min(H - wh, focus.y - wh / 2)));
|
|
237
|
-
const lo = d0 === null ? dMin : Math.max(dMin, Math.floor(d0) - 4);
|
|
238
|
-
const hi = d0 === null ? dMax : Math.min(dMax, Math.ceil(d0) + 4);
|
|
239
|
-
const m = matchBlock(img, W, H, rx, ry, ww, wh, { dMin: lo, dMax: hi, dyMax: o.dyMax ?? 2 });
|
|
240
|
-
if (m && m.c >= (o.minNcc ?? MIN_NCC)) return { d: m.d, x: focus.x, y: focus.y, c: m.c, method, blocks };
|
|
241
|
-
// A flat or occluded focus window: fall back to the subject's mode.
|
|
242
|
-
if (d0 !== null) return { d: d0, x: focus.x, y: focus.y, c: 0, method, blocks };
|
|
243
|
-
return null;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
/** RGBA pixels (ImageData.data) of a SBS frame → luma, same size. */
|
|
247
|
-
export function lumaFromRgba(rgba, n) {
|
|
248
|
-
const out = new Uint8Array(n);
|
|
249
|
-
for (let i = 0, j = 0; i < n; i++, j += 4) out[i] = (rgba[j] * 77 + rgba[j + 1] * 150 + rgba[j + 2] * 29) >> 8;
|
|
250
|
-
return out;
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
/**
|
|
254
|
-
* Temporal steadiness for measurements: the median of the last `n` accepted values, and a hold of
|
|
255
|
-
* the last good value when a measurement fails (a blink, a hand over the face) rather than snapping
|
|
256
|
-
* back to zero.
|
|
257
|
-
*/
|
|
258
|
-
export function createDisparityTrack({ n = 3 } = {}) {
|
|
259
|
-
const hist = [];
|
|
260
|
-
const t = {
|
|
261
|
-
value: null,
|
|
262
|
-
push(d) {
|
|
263
|
-
if (d === null || !Number.isFinite(d)) return t.value;
|
|
264
|
-
hist.push(d);
|
|
265
|
-
if (hist.length > n) hist.shift();
|
|
266
|
-
const s = [...hist].sort((a, b) => a - b);
|
|
267
|
-
t.value = s[s.length >> 1];
|
|
268
|
-
return t.value;
|
|
269
|
-
},
|
|
270
|
-
reset() {
|
|
271
|
-
hist.length = 0;
|
|
272
|
-
t.value = null;
|
|
273
|
-
},
|
|
274
|
-
};
|
|
275
|
-
return t;
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
/** Copy a `w`×`h` patch of the LEFT eye at (x, y). */
|
|
279
|
-
function copyPatch(img, W, x, y, w, h) {
|
|
280
|
-
const p = new Float32Array(w * h);
|
|
281
|
-
for (let r = 0; r < h; r++) for (let c = 0; c < w; c++) p[r * w + c] = img[(y + r) * W + x + c];
|
|
282
|
-
return p;
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/** Best NCC position of `tpl` (w×h) in the LEFT eye within ±rx/±ry of (x0, y0). */
|
|
286
|
-
function findInLeft(img, W, H, tpl, w, h, x0, y0, rx, ry) {
|
|
287
|
-
const E = W / 2;
|
|
288
|
-
const n = w * h;
|
|
289
|
-
let tm = 0;
|
|
290
|
-
let t2 = 0;
|
|
291
|
-
for (let i = 0; i < n; i++) {
|
|
292
|
-
tm += tpl[i];
|
|
293
|
-
t2 += tpl[i] * tpl[i];
|
|
294
|
-
}
|
|
295
|
-
tm /= n;
|
|
296
|
-
const ts = Math.sqrt(Math.max(1e-6, t2 / n - tm * tm));
|
|
297
|
-
let best = -2;
|
|
298
|
-
let bx = x0;
|
|
299
|
-
let by = y0;
|
|
300
|
-
for (let y = Math.max(0, y0 - ry); y <= Math.min(H - h, y0 + ry); y++) {
|
|
301
|
-
for (let x = Math.max(0, x0 - rx); x <= Math.min(E - w, x0 + rx); x++) {
|
|
302
|
-
let s = 0;
|
|
303
|
-
let s2 = 0;
|
|
304
|
-
let cr = 0;
|
|
305
|
-
for (let r = 0; r < h; r++) {
|
|
306
|
-
const o = (y + r) * W + x;
|
|
307
|
-
for (let c = 0; c < w; c++) {
|
|
308
|
-
const v = img[o + c];
|
|
309
|
-
s += v;
|
|
310
|
-
s2 += v * v;
|
|
311
|
-
cr += v * tpl[r * w + c];
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
const m = s / n;
|
|
315
|
-
const vr = s2 / n - m * m;
|
|
316
|
-
if (vr < 1) continue;
|
|
317
|
-
const cc = (cr / n - m * tm) / (Math.sqrt(vr) * ts);
|
|
318
|
-
if (cc > best) {
|
|
319
|
-
best = cc;
|
|
320
|
-
bx = x;
|
|
321
|
-
by = y;
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
return { x: bx, y: by, c: best };
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
/**
|
|
329
|
-
* Cheap frame-to-frame tracking of the focus point. A full subject search (measureFocusDisparity)
|
|
330
|
-
* locks on; after that each call only re-finds a small left-eye template around the last position
|
|
331
|
-
* (the head moved) and matches it to the right eye in a ±`dWin` disparity window. The full search
|
|
332
|
-
* re-runs every `fullEveryMs`, or as soon as tracking loses the template.
|
|
333
|
-
*/
|
|
334
|
-
export function createFocusTracker({ fullEveryMs = 2000, dWin = 4, rx = 12, ry = 8, minTrackNcc = 0.75, jump = 8 } = {}) {
|
|
335
|
-
let st = null; // { x, y, d, tpl, w, h, at }
|
|
336
|
-
let pending = null; // a re-search result that disagreed with a healthy track, awaiting confirmation
|
|
337
|
-
const t = {
|
|
338
|
-
/** @returns {{d:number,x:number,y:number,c:number,method:'mode'|'focus'|'track',blocks:number}|null} */
|
|
339
|
-
measure(img, W, H, now, o = {}) {
|
|
340
|
-
const E = W / 2;
|
|
341
|
-
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
342
|
-
const w = Math.round(b * 3);
|
|
343
|
-
const h = Math.round(b * 1.5);
|
|
344
|
-
// Track first, whenever there is something to track.
|
|
345
|
-
let tracked = null;
|
|
346
|
-
if (st && st.w === w && st.h === h) {
|
|
347
|
-
const f = findInLeft(img, W, H, st.tpl, w, h, st.x, st.y, rx, ry);
|
|
348
|
-
if (f.c >= minTrackNcc) {
|
|
349
|
-
const m = matchBlock(img, W, H, f.x, f.y, w, h, { dMin: Math.floor(st.d) - dWin, dMax: Math.ceil(st.d) + dWin, dyMax: 2 });
|
|
350
|
-
if (m && m.c >= MIN_NCC) tracked = { m, f };
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
const accept = ({ m, f }) => {
|
|
354
|
-
st = { ...st, x: f.x, y: f.y, d: m.d, tpl: copyPatch(img, W, f.x, f.y, w, h) };
|
|
355
|
-
return { d: m.d, x: f.x + w / 2, y: f.y + h / 2, c: m.c, method: 'track', blocks: 0 };
|
|
356
|
-
};
|
|
357
|
-
if (tracked && now - st.at < fullEveryMs) return accept(tracked);
|
|
358
|
-
const m = measureFocusDisparity(img, W, H, o);
|
|
359
|
-
if (tracked && (!m || Math.abs(m.d - tracked.m.d) > jump)) {
|
|
360
|
-
// A periodic re-search that disagrees with a HEALTHY track is not trusted on its own (a
|
|
361
|
-
// repeating background can alias one period away): switch only when the next re-search
|
|
362
|
-
// confirms it (a new, nearer person really did step in).
|
|
363
|
-
const confirmed = m && pending && Math.abs(pending.d - m.d) <= 3;
|
|
364
|
-
pending = m && !confirmed ? { d: m.d } : null;
|
|
365
|
-
if (!confirmed) {
|
|
366
|
-
st.at = now;
|
|
367
|
-
return accept(tracked);
|
|
368
|
-
}
|
|
369
|
-
} else pending = null;
|
|
370
|
-
if (!m) {
|
|
371
|
-
st = null;
|
|
372
|
-
return null;
|
|
373
|
-
}
|
|
374
|
-
const x = Math.round(Math.max(0, Math.min(E - w, m.x - w / 2)));
|
|
375
|
-
const y = Math.round(Math.max(0, Math.min(H - h, m.y - h / 2)));
|
|
376
|
-
st = { x, y, d: m.d, tpl: copyPatch(img, W, x, y, w, h), w, h, at: now };
|
|
377
|
-
return m;
|
|
378
|
-
},
|
|
379
|
-
reset() {
|
|
380
|
-
st = null;
|
|
381
|
-
pending = null;
|
|
382
|
-
},
|
|
383
|
-
};
|
|
384
|
-
return t;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
/**
|
|
388
|
-
* Downsample a luma plane by an integer `factor` (the Y plane of a decoded frame, read straight from
|
|
389
|
-
* WebCodecs: no RGBA conversion, no canvas). Averaging, not single-point sampling, so sensor noise
|
|
390
|
-
* does not become false texture: the full box for factors <= 2, a 2x2 sample per cell above.
|
|
391
|
-
* @param {Uint8Array} src the plane
|
|
392
|
-
* @param {number} offset byte offset of the plane in `src`
|
|
393
|
-
* @param {number} stride bytes per source row
|
|
394
|
-
* @returns {{img: Uint8Array, w: number, h: number}}
|
|
395
|
-
*/
|
|
396
|
-
export function downsampleLuma(src, offset, stride, srcW, srcH, factor) {
|
|
397
|
-
const f = Math.max(1, factor | 0);
|
|
398
|
-
let w = Math.floor(srcW / f);
|
|
399
|
-
w -= w & 1; // even, so the SBS halves split cleanly
|
|
400
|
-
const h = Math.floor(srcH / f);
|
|
401
|
-
const out = new Uint8Array(w * h);
|
|
402
|
-
if (f <= 2) {
|
|
403
|
-
// Small factors: the full box.
|
|
404
|
-
const n = f * f;
|
|
405
|
-
for (let y = 0; y < h; y++) {
|
|
406
|
-
for (let x = 0; x < w; x++) {
|
|
407
|
-
let sum = 0;
|
|
408
|
-
for (let dy = 0; dy < f; dy++) {
|
|
409
|
-
const o = offset + (y * f + dy) * stride + x * f;
|
|
410
|
-
for (let dx = 0; dx < f; dx++) sum += src[o + dx];
|
|
411
|
-
}
|
|
412
|
-
out[y * w + x] = (sum / n) | 0;
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
return { img: out, w, h };
|
|
416
|
-
}
|
|
417
|
-
// Larger factors: a 2x2 sample at the quarter points of each cell. Enough averaging to keep sensor
|
|
418
|
-
// and codec noise from reading as texture, at ~1/9 of the full box's cost for f = 6 (the full box
|
|
419
|
-
// over a 2560x720 luma plane was ~8 ms of JS per measurement).
|
|
420
|
-
const a = f >> 2;
|
|
421
|
-
const c = (3 * f) >> 2;
|
|
422
|
-
for (let y = 0; y < h; y++) {
|
|
423
|
-
const r0 = offset + (y * f + a) * stride;
|
|
424
|
-
const r1 = offset + (y * f + c) * stride;
|
|
425
|
-
for (let x = 0; x < w; x++) {
|
|
426
|
-
const x0 = x * f + a;
|
|
427
|
-
const x1 = x * f + c;
|
|
428
|
-
out[y * w + x] = (src[r0 + x0] + src[r0 + x1] + src[r1 + x0] + src[r1 + x1]) >> 2;
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
return { img: out, w, h };
|
|
432
|
-
}
|
|
1
|
+
// call/disparity.js — moved to js/camera/disparity.js in C2 (RFC 0003 §4: auto-convergence is a
|
|
2
|
+
// camera-side measurement that the call and `addCameraView` share). This path re-exports it so
|
|
3
|
+
// test imports and any in-tree reference keep resolving; new code imports the camera module.
|
|
4
|
+
export * from '../camera/disparity.js';
|
package/js/call/element.js
CHANGED
|
@@ -37,7 +37,7 @@ import { mountCall, dxrSignaling } from '../inline3d-call.js';
|
|
|
37
37
|
|
|
38
38
|
/** Every call event is re-dispatched as `dxr-call:<type>`; `dxr-call:ready` carries the handle. */
|
|
39
39
|
export const CALL_EVENT_PREFIX = 'dxr-call:';
|
|
40
|
-
const CALL_EVENTS = ['joined', 'left', 'peer', 'peerleft', 'state', '
|
|
40
|
+
const CALL_EVENTS = ['joined', 'left', 'peer', 'peerleft', 'state', 'display', 'speaker', 'quality', 'error', 'warning'];
|
|
41
41
|
const TAG = '[inline3d/call] <dxr-call>';
|
|
42
42
|
|
|
43
43
|
/** A boolean attribute: present is true, unless its value spells "false"/"0"/"off"/"no". */
|
|
@@ -61,18 +61,19 @@ const str = (v) => (v === null || v === undefined || String(v).trim() === '' ? u
|
|
|
61
61
|
* | `signaling` | `signaling: dxrSignaling(url)` |
|
|
62
62
|
* | `key` | `key` |
|
|
63
63
|
* | `camera` | `camera` (`auto`/`stereo`/`mono`/deviceId)|
|
|
64
|
-
* | `layout` | `layout` (`grid`/`speaker`)
|
|
65
|
-
* | `accent` | `accent`
|
|
64
|
+
* | `layout` | `layout` (`grid`/`speaker`/`none`) |
|
|
65
|
+
* | `accent` | `theme.accent` |
|
|
66
66
|
* | `max-peers` | `maxPeers` |
|
|
67
67
|
* | `no-ui` | `ui: false` |
|
|
68
|
+
* | `ui="tiles"` | `ui: 'tiles'` |
|
|
68
69
|
* | `auto-join` | `autoJoin: true` |
|
|
69
70
|
* | `mono3d="off"` | `mono3D: 'off'` |
|
|
70
71
|
* | `no-audio` | `audio: false` |
|
|
71
72
|
* | `no-self-view` | `selfView: false` |
|
|
72
73
|
* | `no-auto-converge` | `autoConverge: false` |
|
|
73
|
-
* | `tile-aspect` | `tileAspect` (a number, or `w/h`)
|
|
74
|
-
* | `invite-base` | `
|
|
75
|
-
* | `browser-url` | `browserUrl`
|
|
74
|
+
* | `tile-aspect` | `theme.tileAspect` (a number, or `w/h`) |
|
|
75
|
+
* | `invite-base` | `invite.base` |
|
|
76
|
+
* | `browser-url` | `landing.browserUrl` |
|
|
76
77
|
* | `debug` | `debug: true` |
|
|
77
78
|
*
|
|
78
79
|
* @param {{ getAttribute(name: string): string | null } | ((name: string) => string | null)} source
|
|
@@ -90,18 +91,21 @@ export function attrsToOpts(source) {
|
|
|
90
91
|
put('key', str(get('key')));
|
|
91
92
|
put('camera', str(get('camera')));
|
|
92
93
|
put('layout', str(get('layout')));
|
|
93
|
-
put('accent', str(get('accent')));
|
|
94
94
|
put('maxPeers', num(get('max-peers')));
|
|
95
|
+
const ui = str(get('ui'));
|
|
95
96
|
if (isOn(get('no-ui'))) o.ui = false;
|
|
97
|
+
else if (ui === 'tiles') o.ui = 'tiles';
|
|
96
98
|
if (isOn(get('auto-join'))) o.autoJoin = true;
|
|
97
99
|
const m3 = str(get('mono3d'));
|
|
98
100
|
if (m3 && /^(off|false|0|no)$/i.test(m3)) o.mono3D = 'off';
|
|
99
101
|
if (isOn(get('no-audio'))) o.audio = false;
|
|
100
102
|
if (isOn(get('no-self-view'))) o.selfView = false;
|
|
101
103
|
if (isOn(get('no-auto-converge'))) o.autoConverge = false;
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
104
|
+
// The C2 groups (RFC 0003 §2): the attribute names are the C1 ones, the option shapes are new.
|
|
105
|
+
const theme = { accent: str(get('accent')), tileAspect: num(get('tile-aspect')) };
|
|
106
|
+
if (theme.accent !== undefined || theme.tileAspect !== undefined) o.theme = Object.fromEntries(Object.entries(theme).filter(([, v]) => v !== undefined));
|
|
107
|
+
if (str(get('invite-base'))) o.invite = { base: str(get('invite-base')) };
|
|
108
|
+
if (str(get('browser-url'))) o.landing = { browserUrl: str(get('browser-url')) };
|
|
105
109
|
if (isOn(get('debug'))) o.debug = true;
|
|
106
110
|
return o;
|
|
107
111
|
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// call/options.js — `CallOptions` → the module's internal shape (RFC 0003 §2): defaults,
|
|
2
|
+
// validation, the C2 groups (`theme`, `invite`, `landing`, `liftOptions.max`) folded flat, and the
|
|
3
|
+
// 1.29 spellings accepted for one release with a warning. Pure (no DOM, no network) — see
|
|
4
|
+
// test/call.test.mjs. Internal: importable by path, not part of the `./call` entry.
|
|
5
|
+
|
|
6
|
+
import { isValidRoomId, parseInviteLink, DEFAULT_MAX_PEERS } from './wire.js';
|
|
7
|
+
import { clampMaxPeers } from './transport.js';
|
|
8
|
+
import { normalizeMono3D } from './lift.js';
|
|
9
|
+
import { dxrSignaling } from './signaling.js';
|
|
10
|
+
import { resolveCallAccent, THEME_VARS } from './ui.js';
|
|
11
|
+
|
|
12
|
+
const TAG = '[inline3d/call]';
|
|
13
|
+
const DEFAULT_BROWSER_URL = 'https://github.com/DisplayXR/displayxr-browser';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The 1.29 option spellings and where they went (RFC 0003 §2). Accepted for one release with a
|
|
17
|
+
* single console.warn per key; `normalizeCallOptions` folds them into the C2 shape. 1.31 drops
|
|
18
|
+
* them. The `<dxr-call>` ATTRIBUTES keep their names — the element maps them to the new shapes.
|
|
19
|
+
*/
|
|
20
|
+
export const LEGACY_OPTIONS = Object.freeze({
|
|
21
|
+
accent: 'theme.accent',
|
|
22
|
+
tileAspect: 'theme.tileAspect',
|
|
23
|
+
inviteBase: 'invite.base',
|
|
24
|
+
updateUrl: 'invite.updateUrl',
|
|
25
|
+
browserUrl: 'landing.browserUrl',
|
|
26
|
+
maxLifted: 'liftOptions.max',
|
|
27
|
+
calibration: "openCamera({ calibration }) of '@displayxr/inline3d/camera' (pass the camera as `camera`)",
|
|
28
|
+
rectify: "openCamera({ rectify }) of '@displayxr/inline3d/camera'",
|
|
29
|
+
log: 'debug: true (and diagnostics())',
|
|
30
|
+
recoverSession: "nothing — a call always recovers the document's session (sharedInline3D())",
|
|
31
|
+
wallOptions: "sharedInline3D(opts) — the document's wall is configured by the page",
|
|
32
|
+
scrollIntoView: 'nothing — always on with ui: true',
|
|
33
|
+
});
|
|
34
|
+
const legacyWarned = new Set();
|
|
35
|
+
function warnLegacy(key) {
|
|
36
|
+
if (legacyWarned.has(key)) return;
|
|
37
|
+
legacyWarned.add(key);
|
|
38
|
+
console.warn(`${TAG} option \`${key}\` is deprecated (removed in 1.31) — use ${LEGACY_OPTIONS[key]}`);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Apply defaults and validate. Pure (no DOM, no network) — see test/call.test.mjs. The result is
|
|
43
|
+
* the module's INTERNAL shape (flat), with the public groups (`theme`, `invite`, `landing`,
|
|
44
|
+
* `liftOptions`) folded in and the 1.29 spellings accepted with a warning (LEGACY_OPTIONS).
|
|
45
|
+
* @param {object} [opts]
|
|
46
|
+
*/
|
|
47
|
+
export function normalizeCallOptions(opts = {}) {
|
|
48
|
+
for (const k of Object.keys(LEGACY_OPTIONS)) if (opts[k] !== undefined) warnLegacy(k);
|
|
49
|
+
const theme = opts.theme && typeof opts.theme === 'object' ? opts.theme : {};
|
|
50
|
+
const invite = opts.invite && typeof opts.invite === 'object' ? opts.invite : {};
|
|
51
|
+
const landing = opts.landing && typeof opts.landing === 'object' ? opts.landing : {};
|
|
52
|
+
const ui = opts.ui === undefined ? true : opts.ui === 'tiles' ? 'tiles' : !!opts.ui;
|
|
53
|
+
const chrome = ui === true;
|
|
54
|
+
const tileAspect = theme.tileAspect !== undefined ? theme.tileAspect : opts.tileAspect;
|
|
55
|
+
const aspect = typeof tileAspect === 'number' && tileAspect > 0.3 && tileAspect < 4 ? tileAspect : 16 / 9;
|
|
56
|
+
let room = opts.room === undefined || opts.room === null ? 'auto' : opts.room;
|
|
57
|
+
if (room !== 'auto' && !isValidRoomId(room)) {
|
|
58
|
+
const fromLink = parseInviteLink(room);
|
|
59
|
+
if (!fromLink) throw new Error(`@displayxr/inline3d/call: room "${room}" is not a valid room id (16-64 base64url chars) or invite link`);
|
|
60
|
+
room = fromLink;
|
|
61
|
+
}
|
|
62
|
+
const liftOptions = opts.liftOptions && typeof opts.liftOptions === 'object' ? { ...opts.liftOptions } : null;
|
|
63
|
+
const maxLiftedRaw = liftOptions && liftOptions.max !== undefined ? liftOptions.max : opts.maxLifted;
|
|
64
|
+
if (liftOptions) delete liftOptions.max;
|
|
65
|
+
const accent = theme.accent !== undefined ? theme.accent : opts.accent;
|
|
66
|
+
const cssVars = {};
|
|
67
|
+
for (const [k, prop] of Object.entries(THEME_VARS)) {
|
|
68
|
+
const v = k === 'accent' ? accent : theme[k];
|
|
69
|
+
if (typeof v === 'string' && v) cssVars[prop] = k === 'accent' ? resolveCallAccent(v) : v;
|
|
70
|
+
else if (typeof v === 'number' && k === 'radius') cssVars[prop] = `${v}px`;
|
|
71
|
+
}
|
|
72
|
+
const inviteBase = invite.base !== undefined ? invite.base : opts.inviteBase;
|
|
73
|
+
const updateUrl = invite.updateUrl !== undefined ? invite.updateUrl : opts.updateUrl;
|
|
74
|
+
const browserUrl = landing.browserUrl !== undefined ? landing.browserUrl : opts.browserUrl;
|
|
75
|
+
return {
|
|
76
|
+
room,
|
|
77
|
+
// Default: the hosted DisplayXR signalling server (which also mints TURN credentials). A
|
|
78
|
+
// publishable key (RFC 0003 §5a) rides along on connect; the server may ignore it today.
|
|
79
|
+
signaling: opts.signaling || dxrSignaling(undefined, typeof opts.key === 'string' && opts.key ? { key: opts.key } : {}),
|
|
80
|
+
iceServers: Array.isArray(opts.iceServers) ? opts.iceServers : undefined,
|
|
81
|
+
// 'auto' | 'stereo' | 'mono' | deviceId | MediaStream | a StereoCamera from /camera.
|
|
82
|
+
camera: opts.camera === undefined ? 'auto' : opts.camera,
|
|
83
|
+
format: opts.format === 'sbs' ? 'sbs' : opts.format === 'mono' ? 'mono' : undefined,
|
|
84
|
+
// Camera-side knobs (moved to /camera in C2; still forwarded to openCamera() for one release).
|
|
85
|
+
calibration: opts.calibration && typeof opts.calibration === 'object' ? { ...opts.calibration } : {},
|
|
86
|
+
rectify: typeof opts.rectify === 'function' ? opts.rectify : null,
|
|
87
|
+
audio: opts.audio === undefined ? true : !!opts.audio,
|
|
88
|
+
// Auto-convergence (camera/disparity.js): measure the disparity of the point between each SBS
|
|
89
|
+
// peer's eyes and shift the eyes so it sits at the display plane. The depth slider stays an
|
|
90
|
+
// offset on top. Off = the pair as sent (plus any `hint`).
|
|
91
|
+
autoConverge: opts.autoConverge === undefined ? true : !!opts.autoConverge,
|
|
92
|
+
mono3D: normalizeMono3D(opts.mono3D),
|
|
93
|
+
maxPeers: clampMaxPeers(opts.maxPeers === undefined ? DEFAULT_MAX_PEERS : opts.maxPeers),
|
|
94
|
+
// Extra lift() options for lifted tiles (models, ort, quality, providers). The call's own
|
|
95
|
+
// keys (mode, wall, ui, convergence, priority) always win; `max` is the call's cap (below).
|
|
96
|
+
liftOptions: liftOptions && Object.keys(liftOptions).length ? liftOptions : null,
|
|
97
|
+
// Concurrent lifted tiles (`liftOptions.max`). Default = maxPeers (4): every mono peer can be lifted.
|
|
98
|
+
maxLifted: Number.isFinite(+maxLiftedRaw) && maxLiftedRaw !== null && maxLiftedRaw !== undefined ? Math.max(0, Math.min(DEFAULT_MAX_PEERS, Math.floor(+maxLiftedRaw))) : DEFAULT_MAX_PEERS,
|
|
99
|
+
// 'grid' | 'speaker' | 'none' (the page lays the tiles out; handle.tile(id) finds them).
|
|
100
|
+
layout: opts.layout === 'speaker' ? 'speaker' : opts.layout === 'none' ? 'none' : 'grid',
|
|
101
|
+
// true: full chrome | 'tiles': badges + plates only | false: nothing. `chrome` = the full set.
|
|
102
|
+
ui,
|
|
103
|
+
chrome,
|
|
104
|
+
autoJoin: opts.autoJoin === undefined ? !chrome : !!opts.autoJoin,
|
|
105
|
+
selfView: opts.selfView === undefined ? true : !!opts.selfView,
|
|
106
|
+
tileAspect: aspect,
|
|
107
|
+
// theme → the CSS custom properties written on the host, and the string table.
|
|
108
|
+
cssVars,
|
|
109
|
+
strings: theme.strings && typeof theme.strings === 'object' ? { ...theme.strings } : null,
|
|
110
|
+
invite: {
|
|
111
|
+
base: typeof inviteBase === 'string' ? inviteBase : null,
|
|
112
|
+
updateUrl: updateUrl === undefined ? chrome : !!updateUrl,
|
|
113
|
+
},
|
|
114
|
+
landing: {
|
|
115
|
+
browserUrl: typeof browserUrl === 'string' ? browserUrl : DEFAULT_BROWSER_URL,
|
|
116
|
+
allow2D: landing.allow2D === undefined ? true : !!landing.allow2D,
|
|
117
|
+
},
|
|
118
|
+
debug: !!opts.debug,
|
|
119
|
+
log: typeof opts.log === 'function' ? opts.log : opts.debug ? (tag, obj) => console.log(`${TAG} ${tag} ${JSON.stringify(obj)}`) : null,
|
|
120
|
+
// Internal (tests): replaces the lift module importer behind `mono3D: 'auto'`.
|
|
121
|
+
_liftImporter: typeof opts._liftImporter === 'function' ? opts._liftImporter : undefined,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** The 1.29 `./call` exports kept as deprecated wrappers for one release (removed in 1.31). */
|
|
126
|
+
export const DEPRECATED_CALL_EXPORTS = Object.freeze([
|
|
127
|
+
'normalizeCallOptions', 'newRoomId', 'isValidRoomId', 'buildInviteLink', 'normalizeHello', 'makeHello', 'routeFor', 'badgeFor', 'createLiveGate',
|
|
128
|
+
'convergenceShiftPx', 'lowPass', 'clampShift', 'eyeCropRect', 'mirrorSwapOps', 'mirrorSwapPixels', 'maxBitrateKbps', 'measureFocusDisparity',
|
|
129
|
+
'preferVideoCodecs', 'sortCodecCapabilities', 'VIDEO_CODEC_ORDER', 'MeshTransport', 'clampMaxPeers', 'qrEncode', 'roomKey', 'SIGNAL_PROTOCOL',
|
|
130
|
+
'WIRE_VERSION', 'CALL_SDK', 'PLATE_TEXT', 'CALL_ACCENTS', 'normalizeMono3D', 'resolveLift', 'createLiftPool', 'createFrameWatch',
|
|
131
|
+
'liftConvergenceFor', 'liftPriorityFor', 'setLiftPriority', 'defaultLiftSpecifier', 'LIFT_PRIORITY',
|
|
132
|
+
]);
|