@displayxr/inline3d 1.29.0 → 1.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/README.md +4 -0
- package/call.d.ts +269 -238
- package/camera.d.ts +257 -0
- package/dist/call.js +17 -12
- package/js/call/capture.js +4 -212
- package/js/call/disparity.js +4 -432
- package/js/call/element.js +14 -10
- package/js/call/options.js +132 -0
- package/js/call/ui.js +106 -9
- package/js/call/wire.js +8 -153
- package/js/camera/capture.js +220 -0
- package/js/camera/converge.js +82 -0
- package/js/camera/disparity.js +432 -0
- package/js/camera/geometry.js +80 -0
- package/js/camera/metadata.js +398 -0
- package/js/inline3d-call.js +404 -319
- package/js/inline3d-camera.js +678 -0
- package/js/inline3d-splat-deferred.js +153 -0
- package/js/inline3d-splat-pc-entry.js +45 -0
- package/js/inline3d-splat.js +2 -145
- package/package.json +19 -1
- package/splat-playcanvas.d.ts +4 -0
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
// camera/disparity.js — auto-convergence (RFC 0002 §3): MEASURE how far apart the remote person's eyes are between the two
|
|
2
|
+
// halves of their side-by-side frame, so the receiver can shift each eye by d/2 in opposite
|
|
3
|
+
// directions and put the point between the eyes at zero parallax (the display plane).
|
|
4
|
+
//
|
|
5
|
+
// No focal length, baseline or distance: those only PREDICT the disparity (f·B/Z); here it is read
|
|
6
|
+
// off the pixels. Works for any SBS sender, calibrated or not, as long as the rows roughly align (a
|
|
7
|
+
// small vertical search absorbs the residual of a raw, unrectified pair).
|
|
8
|
+
//
|
|
9
|
+
// The focus is ONE point, the midpoint between the eyes. It stays meaningful when the head turns (one
|
|
10
|
+
// eye may be hidden), unlike "both eyes" or "the whole face". Without a face detector the module
|
|
11
|
+
// finds the subject as the NEAREST strong disparity mode (in a call the person is in front of the
|
|
12
|
+
// room) and takes the eye line from the top of that blob.
|
|
13
|
+
//
|
|
14
|
+
// Pure: a grayscale SBS image in, numbers out. No DOM, no WebRTC. Sizes are in the INPUT's pixels;
|
|
15
|
+
// the caller scales back to source pixels.
|
|
16
|
+
|
|
17
|
+
/** Blocks with a luma standard deviation below this are too flat to match. */
|
|
18
|
+
const MIN_TEXTURE = 6;
|
|
19
|
+
|
|
20
|
+
/** A block match must reach this normalised cross-correlation to count. */
|
|
21
|
+
export const MIN_NCC = 0.8;
|
|
22
|
+
|
|
23
|
+
/** ...and beat its best rival peak (>= 3 px away) by this much, or it is ambiguous (periodic). */
|
|
24
|
+
export const UNIQUENESS = 0.08;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Normalised cross-correlation of a `bw`×`bh` block of the left eye at (lx, ly) against the right
|
|
28
|
+
* eye at (rx, ry). `img` is the whole SBS image, `W` its full width, `E` the per-eye width.
|
|
29
|
+
* Returns -1 when the right block would leave the right eye or is flat.
|
|
30
|
+
*/
|
|
31
|
+
function ncc(img, W, E, lx, ly, rx, ry, bw, bh, lMean, lStd) {
|
|
32
|
+
if (rx < 0 || rx + bw > E || ry < 0) return -1;
|
|
33
|
+
let sum = 0;
|
|
34
|
+
let sum2 = 0;
|
|
35
|
+
let cross = 0;
|
|
36
|
+
const n = bw * bh;
|
|
37
|
+
for (let y = 0; y < bh; y++) {
|
|
38
|
+
const lo = (ly + y) * W + lx;
|
|
39
|
+
const ro = (ry + y) * W + E + rx;
|
|
40
|
+
for (let x = 0; x < bw; x++) {
|
|
41
|
+
const r = img[ro + x];
|
|
42
|
+
sum += r;
|
|
43
|
+
sum2 += r * r;
|
|
44
|
+
cross += img[lo + x] * r;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const rMean = sum / n;
|
|
48
|
+
const rVar = sum2 / n - rMean * rMean;
|
|
49
|
+
if (rVar < 1) return -1;
|
|
50
|
+
return (cross / n - lMean * rMean) / (lStd * Math.sqrt(rVar));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function blockStats(img, W, x0, y0, bw, bh) {
|
|
54
|
+
let sum = 0;
|
|
55
|
+
let sum2 = 0;
|
|
56
|
+
for (let y = 0; y < bh; y++) {
|
|
57
|
+
const o = (y0 + y) * W + x0;
|
|
58
|
+
for (let x = 0; x < bw; x++) {
|
|
59
|
+
const v = img[o + x];
|
|
60
|
+
sum += v;
|
|
61
|
+
sum2 += v * v;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const n = bw * bh;
|
|
65
|
+
const mean = sum / n;
|
|
66
|
+
return { mean, std: Math.sqrt(Math.max(0, sum2 / n - mean * mean)) };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Best horizontal disparity of one left-eye block: `d = xLeft − xRight` (positive = crossed = in
|
|
71
|
+
* front of the display plane), searched over [dMin, dMax] and ±dy rows. Sub-pixel by a parabola.
|
|
72
|
+
* @returns {{d:number, dy:number, c:number}|null}
|
|
73
|
+
*/
|
|
74
|
+
export function matchBlock(img, W, H, x0, y0, bw, bh, { dMin, dMax, dyMax = 2, uniq } = {}) {
|
|
75
|
+
const E = W / 2;
|
|
76
|
+
const { mean, std } = blockStats(img, W, x0, y0, bw, bh);
|
|
77
|
+
if (std < MIN_TEXTURE) return null;
|
|
78
|
+
let best = -2;
|
|
79
|
+
let bd = 0;
|
|
80
|
+
let bdy = 0;
|
|
81
|
+
const perD = new Map(); // best NCC seen at each disparity (any row), for the uniqueness test
|
|
82
|
+
const scan = (d0, d1, dStep, y0s, y1s) => {
|
|
83
|
+
for (let dy = y0s; dy <= y1s; dy++) {
|
|
84
|
+
const ry = y0 + dy;
|
|
85
|
+
if (ry < 0 || ry + bh > H) continue;
|
|
86
|
+
for (let d = d0; d <= d1; d += dStep) {
|
|
87
|
+
const c = ncc(img, W, E, x0, y0, x0 - d, ry, bw, bh, mean, std);
|
|
88
|
+
if (c > (perD.get(d) ?? -2)) perD.set(d, c);
|
|
89
|
+
if (c > best) {
|
|
90
|
+
best = c;
|
|
91
|
+
bd = d;
|
|
92
|
+
bdy = dy;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
// Coarse (every 2nd disparity, rows every 2nd), then a full-resolution pass around the winner:
|
|
98
|
+
// ~1/4 of the exhaustive cost, and textures here are never finer than the downscaled 2-px step.
|
|
99
|
+
const coarse = dMax - dMin > 8;
|
|
100
|
+
scan(dMin, dMax, coarse ? 2 : 1, -dyMax, dyMax);
|
|
101
|
+
if (coarse && best > -1) {
|
|
102
|
+
const cd = bd;
|
|
103
|
+
const cy = bdy;
|
|
104
|
+
scan(Math.max(dMin, cd - 2), Math.min(dMax, cd + 2), 1, Math.max(-dyMax, cy - 1), Math.min(dyMax, cy + 1));
|
|
105
|
+
}
|
|
106
|
+
if (best < -1) return null;
|
|
107
|
+
// Uniqueness: a repeating texture (a checkerboard, a striped shirt, blinds) matches equally well
|
|
108
|
+
// one period away. Reject the block unless its best peak clearly beats every peak >= 3 px away.
|
|
109
|
+
// The coarse scan samples every 2nd disparity, so a rival peak between two samples is under-read
|
|
110
|
+
// while the winner has been refined to its top; refine each rival local maximum at +-1 as well, or
|
|
111
|
+
// an exactly periodic pattern slips through by a hair.
|
|
112
|
+
let second = -2;
|
|
113
|
+
const peakAt = (d) => {
|
|
114
|
+
let v = -2;
|
|
115
|
+
for (let dy = -dyMax; dy <= dyMax; dy++) {
|
|
116
|
+
const ry = y0 + dy;
|
|
117
|
+
if (ry < 0 || ry + bh > H) continue;
|
|
118
|
+
for (let k = d - 1; k <= d + 1; k++) v = Math.max(v, ncc(img, W, E, x0, y0, x0 - k, ry, bw, bh, mean, std));
|
|
119
|
+
}
|
|
120
|
+
return v;
|
|
121
|
+
};
|
|
122
|
+
for (const [d, c] of perD) {
|
|
123
|
+
if (Math.abs(d - bd) < 3 || c <= second) continue;
|
|
124
|
+
const isLocalMax = c >= (perD.get(d - 2) ?? -2) && c >= (perD.get(d + 2) ?? -2);
|
|
125
|
+
const v = isLocalMax && c > 0 ? peakAt(d) : c;
|
|
126
|
+
if (v > second) second = v;
|
|
127
|
+
}
|
|
128
|
+
// Only for WIDE searches: in a narrow (tracking/refine) window a smooth face legitimately scores
|
|
129
|
+
// high a few px off its peak, and a window anchored on a known disparity cannot alias anyway.
|
|
130
|
+
if (dMax - dMin > 12 && second > -2 && best - second < (uniq ?? UNIQUENESS)) return null;
|
|
131
|
+
// Sub-pixel along d at the best row.
|
|
132
|
+
const cm = ncc(img, W, E, x0, y0, x0 - (bd - 1), y0 + bdy, bw, bh, mean, std);
|
|
133
|
+
const cp = ncc(img, W, E, x0, y0, x0 - (bd + 1), y0 + bdy, bw, bh, mean, std);
|
|
134
|
+
let sub = 0;
|
|
135
|
+
const den = cm - 2 * best + cp;
|
|
136
|
+
if (cm > -1 && cp > -1 && den < 0) sub = Math.max(-0.5, Math.min(0.5, (0.5 * (cm - cp)) / den));
|
|
137
|
+
return { d: bd + sub, dy: bdy, c: best };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Block disparities over the central region of the LEFT eye.
|
|
142
|
+
* @returns {Array<{x:number,y:number,d:number,dy:number,c:number}>} block top-left corners
|
|
143
|
+
*/
|
|
144
|
+
export function blockDisparities(img, W, H, o = {}) {
|
|
145
|
+
const E = W / 2;
|
|
146
|
+
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
147
|
+
const step = o.step || Math.round(b * 1.5); // sparse grid: half the blocks, same modes
|
|
148
|
+
const dMax = o.dMax ?? Math.round(E * 0.3);
|
|
149
|
+
const dMin = o.dMin ?? -Math.round(E * 0.05);
|
|
150
|
+
const x0 = Math.round(E * (o.roiX0 ?? 0.15));
|
|
151
|
+
const x1 = Math.round(E * (o.roiX1 ?? 0.85)) - b;
|
|
152
|
+
const y0 = Math.round(H * (o.roiY0 ?? 0.05));
|
|
153
|
+
const y1 = Math.round(H * (o.roiY1 ?? 0.85)) - b;
|
|
154
|
+
const out = [];
|
|
155
|
+
for (let y = y0; y <= y1; y += step) {
|
|
156
|
+
for (let x = x0; x <= x1; x += step) {
|
|
157
|
+
const m = matchBlock(img, W, H, x, y, b, b, { dMin, dMax, dyMax: o.dyMax ?? 2, uniq: o.uniq });
|
|
158
|
+
if (m && m.c >= (o.minNcc ?? MIN_NCC)) out.push({ x, y, ...m });
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return out;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* The subject = the NEAREST strong disparity mode: the largest `d` whose ±`tol` bin holds at least
|
|
166
|
+
* `minFrac` of the matched blocks (and `minBlocks`). Returns its members, or null.
|
|
167
|
+
*/
|
|
168
|
+
export function nearestMode(blocks, { tol = 2, minFrac = 0.12, minBlocks = 3 } = {}) {
|
|
169
|
+
if (!blocks.length) return null;
|
|
170
|
+
const need = Math.max(minBlocks, Math.ceil(blocks.length * minFrac));
|
|
171
|
+
const ds = blocks.map((b) => b.d).sort((a, b) => b - a); // descending: nearest first
|
|
172
|
+
for (const d of ds) {
|
|
173
|
+
const members = blocks.filter((b) => Math.abs(b.d - d) <= tol);
|
|
174
|
+
if (members.length >= need) {
|
|
175
|
+
const md = members.map((b) => b.d).sort((a, b) => a - b);
|
|
176
|
+
return { d: md[md.length >> 1], members };
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Where the eyes are in a subject blob: horizontally the centre of its TOP rows (the head, not the
|
|
184
|
+
* shoulders), vertically ~40% down a head whose height ≈ 1.3 × its width.
|
|
185
|
+
*
|
|
186
|
+
* Measured on real SR-camera frames (grayscale, dim), only a handful of blocks match on a face, so
|
|
187
|
+
* this lands on the upper face, often the forehead, a few cm above the eyes. That is the same depth
|
|
188
|
+
* plane to within ~1 cm: at 0.6 m the disparity differs by ~1-2 source px, under 1 px of shift per
|
|
189
|
+
* eye. A darkest-band eye-line search was tried and rejected: the darkest band is the hair. Pass
|
|
190
|
+
* `focus` (a face detector's eye midpoint) for the exact point.
|
|
191
|
+
*/
|
|
192
|
+
export function eyeMidpoint(members, block) {
|
|
193
|
+
const top = Math.min(...members.map((m) => m.y));
|
|
194
|
+
const headRows = members.filter((m) => m.y <= top + 2 * block);
|
|
195
|
+
const xs = headRows.map((m) => m.x);
|
|
196
|
+
const xMin = Math.min(...xs);
|
|
197
|
+
const xMax = Math.max(...xs) + block;
|
|
198
|
+
const headW = xMax - xMin;
|
|
199
|
+
return { x: (xMin + xMax) / 2, y: top + 0.4 * 1.3 * headW };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* The disparity of the point between the eyes, in input pixels.
|
|
204
|
+
*
|
|
205
|
+
* @param {Uint8Array|Uint8ClampedArray|number[]} img grayscale SBS, row-major, `W`×`H`
|
|
206
|
+
* @param {number} W full SBS width (both eyes)
|
|
207
|
+
* @param {number} H
|
|
208
|
+
* @param {object} [o]
|
|
209
|
+
* @param {{x:number,y:number}} [o.focus] the eye midpoint in LEFT-eye pixels (e.g. from a face
|
|
210
|
+
* detector); skips the subject search
|
|
211
|
+
* @returns {{d:number, x:number, y:number, c:number, method:'focus'|'mode', blocks:number}|null}
|
|
212
|
+
*/
|
|
213
|
+
export function measureFocusDisparity(img, W, H, o = {}) {
|
|
214
|
+
const E = W / 2;
|
|
215
|
+
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
216
|
+
const dMax = o.dMax ?? Math.round(E * 0.3);
|
|
217
|
+
const dMin = o.dMin ?? -Math.round(E * 0.05);
|
|
218
|
+
let focus = o.focus || null;
|
|
219
|
+
let method = 'focus';
|
|
220
|
+
let blocks = 0;
|
|
221
|
+
let d0 = null;
|
|
222
|
+
if (!focus) {
|
|
223
|
+
const all = blockDisparities(img, W, H, { ...o, block: b, dMax, dMin });
|
|
224
|
+
blocks = all.length;
|
|
225
|
+
const mode = nearestMode(all, o);
|
|
226
|
+
if (!mode) return null;
|
|
227
|
+
focus = eyeMidpoint(mode.members, b);
|
|
228
|
+
d0 = mode.d;
|
|
229
|
+
method = 'mode';
|
|
230
|
+
}
|
|
231
|
+
// Refine with one wider window centred on the focus point: eyes + brow + nose bridge carry
|
|
232
|
+
// plenty of texture even when one eye is turned away.
|
|
233
|
+
const ww = Math.round(b * (o.refineW ?? 3));
|
|
234
|
+
const wh = Math.round(b * (o.refineH ?? 1.5));
|
|
235
|
+
const rx = Math.round(Math.max(0, Math.min(E - ww, focus.x - ww / 2)));
|
|
236
|
+
const ry = Math.round(Math.max(0, Math.min(H - wh, focus.y - wh / 2)));
|
|
237
|
+
const lo = d0 === null ? dMin : Math.max(dMin, Math.floor(d0) - 4);
|
|
238
|
+
const hi = d0 === null ? dMax : Math.min(dMax, Math.ceil(d0) + 4);
|
|
239
|
+
const m = matchBlock(img, W, H, rx, ry, ww, wh, { dMin: lo, dMax: hi, dyMax: o.dyMax ?? 2 });
|
|
240
|
+
if (m && m.c >= (o.minNcc ?? MIN_NCC)) return { d: m.d, x: focus.x, y: focus.y, c: m.c, method, blocks };
|
|
241
|
+
// A flat or occluded focus window: fall back to the subject's mode.
|
|
242
|
+
if (d0 !== null) return { d: d0, x: focus.x, y: focus.y, c: 0, method, blocks };
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** RGBA pixels (ImageData.data) of a SBS frame → luma, same size. */
|
|
247
|
+
export function lumaFromRgba(rgba, n) {
|
|
248
|
+
const out = new Uint8Array(n);
|
|
249
|
+
for (let i = 0, j = 0; i < n; i++, j += 4) out[i] = (rgba[j] * 77 + rgba[j + 1] * 150 + rgba[j + 2] * 29) >> 8;
|
|
250
|
+
return out;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Temporal steadiness for measurements: the median of the last `n` accepted values, and a hold of
|
|
255
|
+
* the last good value when a measurement fails (a blink, a hand over the face) rather than snapping
|
|
256
|
+
* back to zero.
|
|
257
|
+
*/
|
|
258
|
+
export function createDisparityTrack({ n = 3 } = {}) {
|
|
259
|
+
const hist = [];
|
|
260
|
+
const t = {
|
|
261
|
+
value: null,
|
|
262
|
+
push(d) {
|
|
263
|
+
if (d === null || !Number.isFinite(d)) return t.value;
|
|
264
|
+
hist.push(d);
|
|
265
|
+
if (hist.length > n) hist.shift();
|
|
266
|
+
const s = [...hist].sort((a, b) => a - b);
|
|
267
|
+
t.value = s[s.length >> 1];
|
|
268
|
+
return t.value;
|
|
269
|
+
},
|
|
270
|
+
reset() {
|
|
271
|
+
hist.length = 0;
|
|
272
|
+
t.value = null;
|
|
273
|
+
},
|
|
274
|
+
};
|
|
275
|
+
return t;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/** Copy a `w`×`h` patch of the LEFT eye at (x, y). */
|
|
279
|
+
function copyPatch(img, W, x, y, w, h) {
|
|
280
|
+
const p = new Float32Array(w * h);
|
|
281
|
+
for (let r = 0; r < h; r++) for (let c = 0; c < w; c++) p[r * w + c] = img[(y + r) * W + x + c];
|
|
282
|
+
return p;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Best NCC position of `tpl` (w×h) in the LEFT eye within ±rx/±ry of (x0, y0). */
|
|
286
|
+
function findInLeft(img, W, H, tpl, w, h, x0, y0, rx, ry) {
|
|
287
|
+
const E = W / 2;
|
|
288
|
+
const n = w * h;
|
|
289
|
+
let tm = 0;
|
|
290
|
+
let t2 = 0;
|
|
291
|
+
for (let i = 0; i < n; i++) {
|
|
292
|
+
tm += tpl[i];
|
|
293
|
+
t2 += tpl[i] * tpl[i];
|
|
294
|
+
}
|
|
295
|
+
tm /= n;
|
|
296
|
+
const ts = Math.sqrt(Math.max(1e-6, t2 / n - tm * tm));
|
|
297
|
+
let best = -2;
|
|
298
|
+
let bx = x0;
|
|
299
|
+
let by = y0;
|
|
300
|
+
for (let y = Math.max(0, y0 - ry); y <= Math.min(H - h, y0 + ry); y++) {
|
|
301
|
+
for (let x = Math.max(0, x0 - rx); x <= Math.min(E - w, x0 + rx); x++) {
|
|
302
|
+
let s = 0;
|
|
303
|
+
let s2 = 0;
|
|
304
|
+
let cr = 0;
|
|
305
|
+
for (let r = 0; r < h; r++) {
|
|
306
|
+
const o = (y + r) * W + x;
|
|
307
|
+
for (let c = 0; c < w; c++) {
|
|
308
|
+
const v = img[o + c];
|
|
309
|
+
s += v;
|
|
310
|
+
s2 += v * v;
|
|
311
|
+
cr += v * tpl[r * w + c];
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
const m = s / n;
|
|
315
|
+
const vr = s2 / n - m * m;
|
|
316
|
+
if (vr < 1) continue;
|
|
317
|
+
const cc = (cr / n - m * tm) / (Math.sqrt(vr) * ts);
|
|
318
|
+
if (cc > best) {
|
|
319
|
+
best = cc;
|
|
320
|
+
bx = x;
|
|
321
|
+
by = y;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
return { x: bx, y: by, c: best };
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Cheap frame-to-frame tracking of the focus point. A full subject search (measureFocusDisparity)
|
|
330
|
+
* locks on; after that each call only re-finds a small left-eye template around the last position
|
|
331
|
+
* (the head moved) and matches it to the right eye in a ±`dWin` disparity window. The full search
|
|
332
|
+
* re-runs every `fullEveryMs`, or as soon as tracking loses the template.
|
|
333
|
+
*/
|
|
334
|
+
export function createFocusTracker({ fullEveryMs = 2000, dWin = 4, rx = 12, ry = 8, minTrackNcc = 0.75, jump = 8 } = {}) {
|
|
335
|
+
let st = null; // { x, y, d, tpl, w, h, at }
|
|
336
|
+
let pending = null; // a re-search result that disagreed with a healthy track, awaiting confirmation
|
|
337
|
+
const t = {
|
|
338
|
+
/** @returns {{d:number,x:number,y:number,c:number,method:'mode'|'focus'|'track',blocks:number}|null} */
|
|
339
|
+
measure(img, W, H, now, o = {}) {
|
|
340
|
+
const E = W / 2;
|
|
341
|
+
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
342
|
+
const w = Math.round(b * 3);
|
|
343
|
+
const h = Math.round(b * 1.5);
|
|
344
|
+
// Track first, whenever there is something to track.
|
|
345
|
+
let tracked = null;
|
|
346
|
+
if (st && st.w === w && st.h === h) {
|
|
347
|
+
const f = findInLeft(img, W, H, st.tpl, w, h, st.x, st.y, rx, ry);
|
|
348
|
+
if (f.c >= minTrackNcc) {
|
|
349
|
+
const m = matchBlock(img, W, H, f.x, f.y, w, h, { dMin: Math.floor(st.d) - dWin, dMax: Math.ceil(st.d) + dWin, dyMax: 2 });
|
|
350
|
+
if (m && m.c >= MIN_NCC) tracked = { m, f };
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
const accept = ({ m, f }) => {
|
|
354
|
+
st = { ...st, x: f.x, y: f.y, d: m.d, tpl: copyPatch(img, W, f.x, f.y, w, h) };
|
|
355
|
+
return { d: m.d, x: f.x + w / 2, y: f.y + h / 2, c: m.c, method: 'track', blocks: 0 };
|
|
356
|
+
};
|
|
357
|
+
if (tracked && now - st.at < fullEveryMs) return accept(tracked);
|
|
358
|
+
const m = measureFocusDisparity(img, W, H, o);
|
|
359
|
+
if (tracked && (!m || Math.abs(m.d - tracked.m.d) > jump)) {
|
|
360
|
+
// A periodic re-search that disagrees with a HEALTHY track is not trusted on its own (a
|
|
361
|
+
// repeating background can alias one period away): switch only when the next re-search
|
|
362
|
+
// confirms it (a new, nearer person really did step in).
|
|
363
|
+
const confirmed = m && pending && Math.abs(pending.d - m.d) <= 3;
|
|
364
|
+
pending = m && !confirmed ? { d: m.d } : null;
|
|
365
|
+
if (!confirmed) {
|
|
366
|
+
st.at = now;
|
|
367
|
+
return accept(tracked);
|
|
368
|
+
}
|
|
369
|
+
} else pending = null;
|
|
370
|
+
if (!m) {
|
|
371
|
+
st = null;
|
|
372
|
+
return null;
|
|
373
|
+
}
|
|
374
|
+
const x = Math.round(Math.max(0, Math.min(E - w, m.x - w / 2)));
|
|
375
|
+
const y = Math.round(Math.max(0, Math.min(H - h, m.y - h / 2)));
|
|
376
|
+
st = { x, y, d: m.d, tpl: copyPatch(img, W, x, y, w, h), w, h, at: now };
|
|
377
|
+
return m;
|
|
378
|
+
},
|
|
379
|
+
reset() {
|
|
380
|
+
st = null;
|
|
381
|
+
pending = null;
|
|
382
|
+
},
|
|
383
|
+
};
|
|
384
|
+
return t;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* Downsample a luma plane by an integer `factor` (the Y plane of a decoded frame, read straight from
|
|
389
|
+
* WebCodecs: no RGBA conversion, no canvas). Averaging, not single-point sampling, so sensor noise
|
|
390
|
+
* does not become false texture: the full box for factors <= 2, a 2x2 sample per cell above.
|
|
391
|
+
* @param {Uint8Array} src the plane
|
|
392
|
+
* @param {number} offset byte offset of the plane in `src`
|
|
393
|
+
* @param {number} stride bytes per source row
|
|
394
|
+
* @returns {{img: Uint8Array, w: number, h: number}}
|
|
395
|
+
*/
|
|
396
|
+
export function downsampleLuma(src, offset, stride, srcW, srcH, factor) {
|
|
397
|
+
const f = Math.max(1, factor | 0);
|
|
398
|
+
let w = Math.floor(srcW / f);
|
|
399
|
+
w -= w & 1; // even, so the SBS halves split cleanly
|
|
400
|
+
const h = Math.floor(srcH / f);
|
|
401
|
+
const out = new Uint8Array(w * h);
|
|
402
|
+
if (f <= 2) {
|
|
403
|
+
// Small factors: the full box.
|
|
404
|
+
const n = f * f;
|
|
405
|
+
for (let y = 0; y < h; y++) {
|
|
406
|
+
for (let x = 0; x < w; x++) {
|
|
407
|
+
let sum = 0;
|
|
408
|
+
for (let dy = 0; dy < f; dy++) {
|
|
409
|
+
const o = offset + (y * f + dy) * stride + x * f;
|
|
410
|
+
for (let dx = 0; dx < f; dx++) sum += src[o + dx];
|
|
411
|
+
}
|
|
412
|
+
out[y * w + x] = (sum / n) | 0;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
return { img: out, w, h };
|
|
416
|
+
}
|
|
417
|
+
// Larger factors: a 2x2 sample at the quarter points of each cell. Enough averaging to keep sensor
|
|
418
|
+
// and codec noise from reading as texture, at ~1/9 of the full box's cost for f = 6 (the full box
|
|
419
|
+
// over a 2560x720 luma plane was ~8 ms of JS per measurement).
|
|
420
|
+
const a = f >> 2;
|
|
421
|
+
const c = (3 * f) >> 2;
|
|
422
|
+
for (let y = 0; y < h; y++) {
|
|
423
|
+
const r0 = offset + (y * f + a) * stride;
|
|
424
|
+
const r1 = offset + (y * f + c) * stride;
|
|
425
|
+
for (let x = 0; x < w; x++) {
|
|
426
|
+
const x0 = x * f + a;
|
|
427
|
+
const x1 = x * f + c;
|
|
428
|
+
out[y * w + x] = (src[r0 + x0] + src[r0 + x1] + src[r1 + x0] + src[r1 + x1]) >> 2;
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return { img: out, w, h };
|
|
432
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// camera/geometry.js — the pure geometry of a side-by-side frame: what counts as a pair, how one
|
|
2
|
+
// eye is cropped to a tile's aspect (with a convergence shift), and the self-view mirroring trap.
|
|
3
|
+
// Shared by `@displayxr/inline3d/camera` (the self view, capture) and `/call` (remote tiles).
|
|
4
|
+
// No DOM — see test/camera.test.mjs and test/call.test.mjs.
|
|
5
|
+
|
|
6
|
+
/** A delivered frame wider than this (w / h) is a side-by-side pair (e.g. 1280x480, 2560x720). */
|
|
7
|
+
export const SBS_ASPECT_MIN = 2.5;
|
|
8
|
+
|
|
9
|
+
/** Does a delivered frame look like a side-by-side pair? (`prefer: 'auto'` only) */
|
|
10
|
+
export function looksSbs(width, height) {
|
|
11
|
+
return width > 0 && height > 0 && width / height > SBS_ASPECT_MIN;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The source rectangle of ONE eye, cropped to the tile's aspect and shifted by the convergence
|
|
16
|
+
* offset. Never upscales past the source: a 640-wide eye (a real raw stereo camera) stays 640.
|
|
17
|
+
*
|
|
18
|
+
* @param {number} eyeW source per-eye width (px)
|
|
19
|
+
* @param {number} eyeH source height
|
|
20
|
+
* @param {number} aspect the tile's (= the woven buffer's per-eye) aspect, w/h
|
|
21
|
+
* @param {number} shift per-eye shift in source px (see convergenceShiftPx); + = push back
|
|
22
|
+
* @param {0|1} eye 0 = left, 1 = right
|
|
23
|
+
* @returns {{sx:number, sy:number, sw:number, sh:number}} relative to that eye's half
|
|
24
|
+
*/
|
|
25
|
+
export function eyeCropRect(eyeW, eyeH, aspect, shift, eye) {
|
|
26
|
+
const s = Math.abs(shift || 0);
|
|
27
|
+
let sw = Math.min(eyeW - 2 * s, eyeH * aspect);
|
|
28
|
+
sw = Math.max(1, sw);
|
|
29
|
+
const sh = Math.min(eyeH, sw / aspect);
|
|
30
|
+
sw = sh * aspect;
|
|
31
|
+
const cx = (eyeW - sw) / 2 + (eye === 0 ? shift : -shift);
|
|
32
|
+
return { sx: Math.max(0, Math.min(eyeW - sw, cx)), sy: (eyeH - sh) / 2, sw, sh };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The output per-eye size for a source eye and a tile aspect: no upscaling, even numbers. */
|
|
36
|
+
export function eyeOutputSize(eyeW, eyeH, aspect) {
|
|
37
|
+
const w = Math.min(eyeW, eyeH * aspect);
|
|
38
|
+
const h = w / aspect;
|
|
39
|
+
return { w: Math.max(2, Math.round(w / 2) * 2), h: Math.max(2, Math.round(h / 2) * 2) };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// ── self view: the mirroring trap (RFC 0002 §3, RFC 0003 §4) ───────────────────────────────
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* The draw operations for a MIRRORED stereo self-view: each half mirrored AND the halves swapped.
|
|
46
|
+
*
|
|
47
|
+
* Why both: mirroring a scene horizontally means the reflected left eye sees what the original
|
|
48
|
+
* right eye saw (mirrored), and vice versa. Mirroring each half IN PLACE keeps the eyes where they
|
|
49
|
+
* were and inverts every disparity — the face goes pseudoscopic (inside-out). Mirror + swap keeps
|
|
50
|
+
* crossed disparity crossed. (It is the same pixels as flipping the whole SBS frame; spelled out
|
|
51
|
+
* per half so the intent survives refactoring.) The WIRE is never mirrored — only this preview.
|
|
52
|
+
*
|
|
53
|
+
* @param {number} W full SBS frame width
|
|
54
|
+
* @param {number} H height
|
|
55
|
+
* @returns {Array<{src:'L'|'R', sx:number, sw:number, sy:number, sh:number, dx:number, dw:number, mirror:true}>}
|
|
56
|
+
*/
|
|
57
|
+
export function mirrorSwapOps(W, H) {
|
|
58
|
+
const half = W / 2;
|
|
59
|
+
return [
|
|
60
|
+
{ src: 'R', sx: half, sw: half, sy: 0, sh: H, dx: 0, dw: half, mirror: true },
|
|
61
|
+
{ src: 'L', sx: 0, sw: half, sy: 0, sh: H, dx: half, dw: half, mirror: true },
|
|
62
|
+
];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Apply mirrorSwapOps to a row-major single-channel frame (tests, and a reference for the canvas
|
|
67
|
+
* path). Returns a new array.
|
|
68
|
+
*/
|
|
69
|
+
export function mirrorSwapPixels(px, W, H) {
|
|
70
|
+
const out = new px.constructor(px.length);
|
|
71
|
+
for (const op of mirrorSwapOps(W, H)) {
|
|
72
|
+
for (let y = 0; y < H; y++) {
|
|
73
|
+
for (let x = 0; x < op.dw; x++) {
|
|
74
|
+
const srcX = op.sx + (op.sw - 1 - x); // mirrored within the half
|
|
75
|
+
out[y * W + op.dx + x] = px[y * W + srcX];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|