@displayxr/inline3d 1.29.0 → 1.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/README.md +4 -0
- package/call.d.ts +269 -238
- package/camera.d.ts +257 -0
- package/dist/call.js +17 -12
- package/js/call/capture.js +4 -212
- package/js/call/disparity.js +4 -432
- package/js/call/element.js +14 -10
- package/js/call/options.js +132 -0
- package/js/call/ui.js +106 -9
- package/js/call/wire.js +8 -153
- package/js/camera/capture.js +220 -0
- package/js/camera/converge.js +82 -0
- package/js/camera/disparity.js +432 -0
- package/js/camera/geometry.js +80 -0
- package/js/camera/metadata.js +398 -0
- package/js/inline3d-call.js +404 -319
- package/js/inline3d-camera.js +678 -0
- package/js/inline3d-splat-deferred.js +153 -0
- package/js/inline3d-splat-pc-entry.js +45 -0
- package/js/inline3d-splat.js +2 -145
- package/package.json +19 -1
- package/splat-playcanvas.d.ts +4 -0
package/js/call/capture.js
CHANGED
|
@@ -1,212 +1,4 @@
|
|
|
1
|
-
// call/capture.js —
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
// 1280x480, i.e. 640x480 per eye, grayscale and UNRECTIFIED). Found → sent as `sbs`. Otherwise the
|
|
6
|
-
// default webcam, sent as `mono`. Per-eye width is whatever the camera gives; nothing is upscaled.
|
|
7
|
-
//
|
|
8
|
-
// Two field facts shape the probing:
|
|
9
|
-
// - On some laptops the stereo camera is held EXCLUSIVELY by the eye tracker, so opening it
|
|
10
|
-
// fails with NotReadableError ("Device in use"). That device is skipped silently — a call must
|
|
11
|
-
// never fail because an optional camera is busy.
|
|
12
|
-
// - Labels are empty until one getUserMedia has succeeded, so the default camera is opened first
|
|
13
|
-
// (which is also the fallback), then the others are probed one at a time.
|
|
14
|
-
//
|
|
15
|
-
// RECTIFICATION is not done here (P2): it needs the camera's calibration (intrinsics/extrinsics
|
|
16
|
-
// for the ACTIVE device), which comes from a plug-in or the runtime. P1 sends the raw pair with
|
|
17
|
-
// `rectified: false`, and the `rectify` hook (addCall option) is where a calibrated step plugs in:
|
|
18
|
-
// `(stream, info) => MediaStream | Promise<MediaStream>`, returning a rectified SBS stream.
|
|
19
|
-
|
|
20
|
-
import { looksSbs } from './wire.js';
|
|
21
|
-
|
|
22
|
-
const MONO_CONSTRAINTS = { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30 } };
|
|
23
|
-
// Ask a candidate stereo device for its widest mode; a 2560x720 or 3840x1080 pair comes back as
|
|
24
|
-
// itself, a 1280x480 one as 1280x480. `ideal`, never `min`: a smaller real pair is still a pair.
|
|
25
|
-
const PROBE_CONSTRAINTS = { width: { ideal: 3840 }, height: { ideal: 1080 }, frameRate: { ideal: 30 } };
|
|
26
|
-
export const AUDIO_CONSTRAINTS = Object.freeze({ echoCancellation: true, noiseSuppression: true, autoGainControl: true });
|
|
27
|
-
|
|
28
|
-
/** Errors that mean "the device exists but another process holds it" (an eye tracker, say). */
|
|
29
|
-
export function isBusyError(err) {
|
|
30
|
-
return !!err && (err.name === 'NotReadableError' || err.name === 'TrackStartError' || /in use|busy/i.test(err.message || ''));
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Why no camera could be opened, from the per-device failures. `'camera-busy'` when every device
|
|
35
|
-
* that exists is held by another process — on a 3D laptop/monitor whose only camera is the eye
|
|
36
|
-
* tracker's, that is the normal case while tracking, and the call should go audio-only and say so
|
|
37
|
-
* rather than fail or send an empty picture. Pure.
|
|
38
|
-
* @param {Array<{error: string, busy?: boolean}>} skipped
|
|
39
|
-
*/
|
|
40
|
-
export function noCameraCode(skipped) {
|
|
41
|
-
return skipped.length && skipped.every((s) => s.busy) ? 'camera-busy' : 'no-camera';
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
/** A label that names a stereo camera, tried first when probing. Pure. */
|
|
45
|
-
export function stereoLabelHint(label) {
|
|
46
|
-
return /stereo|\b3d\b|\bsbs\b|dual/i.test(label || '');
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
/**
|
|
50
|
-
* Open the camera. Never rejects for a busy optional device; rejects only when NO camera can be
|
|
51
|
-
* opened at all (the caller then runs audio-only or reports the error).
|
|
52
|
-
*
|
|
53
|
-
* @param {'auto'|'stereo'|'mono'|string|MediaStream} want 'auto' | 'stereo' | 'mono' | a
|
|
54
|
-
* deviceId | a page-supplied MediaStream
|
|
55
|
-
* @param {{ format?: 'sbs'|'mono', calibration?: {baselineMm?: number, hfovDeg?: number, rectified?: boolean},
|
|
56
|
-
* mediaDevices?: MediaDevices, log?: Function }} [o]
|
|
57
|
-
* @returns {Promise<{ stream: MediaStream, format: 'sbs'|'mono', width: number, height: number,
|
|
58
|
-
* deviceId: string|null, label: string, owned: boolean, skipped: Array<{label: string, error: string}>,
|
|
59
|
-
* calibration: object }>}
|
|
60
|
-
*/
|
|
61
|
-
export async function openCamera(want = 'auto', o = {}) {
|
|
62
|
-
const md = o.mediaDevices || (globalThis.navigator && navigator.mediaDevices);
|
|
63
|
-
const log = o.log || (() => {});
|
|
64
|
-
const skipped = [];
|
|
65
|
-
const cal = o.calibration || {};
|
|
66
|
-
|
|
67
|
-
if (want && typeof want === 'object' && typeof want.getVideoTracks === 'function') {
|
|
68
|
-
const t = want.getVideoTracks()[0];
|
|
69
|
-
const st = t ? t.getSettings() : {};
|
|
70
|
-
// A page-supplied stream carries a DECLARED format; aspect is never used to guess 3D-ness here.
|
|
71
|
-
const format = o.format === 'sbs' ? 'sbs' : 'mono';
|
|
72
|
-
return { stream: want, format, width: st.width || 0, height: st.height || 0, deviceId: st.deviceId || null, label: t ? t.label : '', owned: false, skipped, calibration: cal };
|
|
73
|
-
}
|
|
74
|
-
if (!md || !md.getUserMedia) throw Object.assign(new Error('getUserMedia is not available (insecure context?)'), { code: 'no-camera' });
|
|
75
|
-
|
|
76
|
-
const open = async (constraints) => {
|
|
77
|
-
const s = await md.getUserMedia({ video: constraints, audio: false });
|
|
78
|
-
const t = s.getVideoTracks()[0];
|
|
79
|
-
const st = t ? t.getSettings() : {};
|
|
80
|
-
// Never hand back a 0x0 track: some engines open a device that then delivers nothing.
|
|
81
|
-
if (!t || t.readyState === 'ended' || !(st.width > 0) || !(st.height > 0)) {
|
|
82
|
-
s.getTracks().forEach((x) => x.stop());
|
|
83
|
-
throw Object.assign(new Error('the camera opened but delivers no picture (0x0)'), { name: 'NotReadableError' });
|
|
84
|
-
}
|
|
85
|
-
// The DisplayXR Browser's "3D Camera (DisplayXR)" (the runtime's stereo camera, e.g. an SR
|
|
86
|
-
// eye-tracking camera shared without stealing it) describes itself on the opened track:
|
|
87
|
-
// `displayxrStereo: {layout, rectified, baselineMm, horizontalFovDeg}`.
|
|
88
|
-
const stereo = st.displayxrStereo && st.displayxrStereo.layout === 'side-by-side' ? st.displayxrStereo : null;
|
|
89
|
-
return { stream: s, track: t, width: st.width, height: st.height, deviceId: st.deviceId || null, label: t.label || '', stereo };
|
|
90
|
-
};
|
|
91
|
-
const skip = (label, err) => {
|
|
92
|
-
skipped.push({ label, error: `${err.name}: ${err.message}`, busy: isBusyError(err) });
|
|
93
|
-
log('camera-skip', { label, error: err.name });
|
|
94
|
-
};
|
|
95
|
-
const stop = (r) => r && r.stream.getTracks().forEach((t) => t.stop());
|
|
96
|
-
const result = (r, format) => ({
|
|
97
|
-
stream: r.stream, format, width: r.width, height: r.height, deviceId: r.deviceId, label: r.label, owned: true, skipped,
|
|
98
|
-
calibration: {
|
|
99
|
-
rectified: false,
|
|
100
|
-
// A runtime stereo camera's own description, where the page gave none.
|
|
101
|
-
...(format === 'sbs' && r.stereo
|
|
102
|
-
? {
|
|
103
|
-
rectified: !!r.stereo.rectified,
|
|
104
|
-
...(r.stereo.baselineMm > 0 ? { baselineMm: r.stereo.baselineMm } : {}),
|
|
105
|
-
...(r.stereo.horizontalFovDeg > 0 ? { hfovDeg: r.stereo.horizontalFovDeg } : {}),
|
|
106
|
-
}
|
|
107
|
-
: {}),
|
|
108
|
-
...cal,
|
|
109
|
-
},
|
|
110
|
-
});
|
|
111
|
-
|
|
112
|
-
// An explicit device id.
|
|
113
|
-
if (typeof want === 'string' && !['auto', 'stereo', 'mono'].includes(want)) {
|
|
114
|
-
let r;
|
|
115
|
-
try {
|
|
116
|
-
r = await open({ deviceId: { exact: want }, ...PROBE_CONSTRAINTS });
|
|
117
|
-
} catch (err) {
|
|
118
|
-
skip(want, err);
|
|
119
|
-
throw Object.assign(new Error(`camera unavailable: ${err.message}`), { code: noCameraCode(skipped), skipped });
|
|
120
|
-
}
|
|
121
|
-
if (looksSbs(r.width, r.height)) return result(r, 'sbs');
|
|
122
|
-
stop(r);
|
|
123
|
-
return result(await open({ deviceId: { exact: want }, ...MONO_CONSTRAINTS }), 'mono');
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// Fast path: when labels are already visible (camera permission persisted), open a device that
|
|
127
|
-
// NAMES itself stereo — e.g. the DisplayXR Browser's "3D Camera (DisplayXR)" — directly. Every
|
|
128
|
-
// camera start can take seconds (a busy capture stack took 7-20 s per start on a field box), and
|
|
129
|
-
// the default-first probe below costs two or three of them before it reaches the pair.
|
|
130
|
-
const tried = new Set();
|
|
131
|
-
if (want !== 'mono') {
|
|
132
|
-
let early = [];
|
|
133
|
-
try {
|
|
134
|
-
early = (await md.enumerateDevices()).filter((d) => d.kind === 'videoinput' && d.deviceId && stereoLabelHint(d.label));
|
|
135
|
-
} catch {
|
|
136
|
-
/* no enumeration yet: the normal path below */
|
|
137
|
-
}
|
|
138
|
-
for (const d of early) {
|
|
139
|
-
tried.add(d.deviceId);
|
|
140
|
-
let r = null;
|
|
141
|
-
try {
|
|
142
|
-
r = await open({ deviceId: { exact: d.deviceId }, ...PROBE_CONSTRAINTS });
|
|
143
|
-
} catch (err) {
|
|
144
|
-
skip(d.label, err);
|
|
145
|
-
continue;
|
|
146
|
-
}
|
|
147
|
-
if (looksSbs(r.width, r.height)) {
|
|
148
|
-
log('camera-sbs', { label: r.label, width: r.width, height: r.height, direct: true });
|
|
149
|
-
return result(r, 'sbs');
|
|
150
|
-
}
|
|
151
|
-
stop(r);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// The default camera first: permission, labels, and the fallback.
|
|
156
|
-
let mono = null;
|
|
157
|
-
try {
|
|
158
|
-
mono = await open(MONO_CONSTRAINTS);
|
|
159
|
-
} catch (err) {
|
|
160
|
-
skip('default', err);
|
|
161
|
-
}
|
|
162
|
-
if (want === 'mono' && mono) return result(mono, 'mono');
|
|
163
|
-
if (mono && looksSbs(mono.width, mono.height)) return result(mono, 'sbs'); // the default IS the pair
|
|
164
|
-
|
|
165
|
-
let devices = [];
|
|
166
|
-
try {
|
|
167
|
-
devices = (await md.enumerateDevices()).filter((d) => d.kind === 'videoinput');
|
|
168
|
-
} catch {
|
|
169
|
-
/* no enumeration: keep the default */
|
|
170
|
-
}
|
|
171
|
-
devices.sort((a, b) => Number(stereoLabelHint(b.label)) - Number(stereoLabelHint(a.label)));
|
|
172
|
-
for (const d of devices) {
|
|
173
|
-
if (!d.deviceId) continue; // no permission yet: an anonymous entry cannot be opened by id
|
|
174
|
-
if (mono && d.deviceId === mono.deviceId) continue;
|
|
175
|
-
if (tried.has(d.deviceId)) continue; // the fast path already opened it (not a pair, or busy)
|
|
176
|
-
let r = null;
|
|
177
|
-
try {
|
|
178
|
-
r = await open({ deviceId: { exact: d.deviceId }, ...PROBE_CONSTRAINTS });
|
|
179
|
-
} catch (err) {
|
|
180
|
-
// NotReadableError = held by another process (e.g. an eye tracker). Skip, never fail.
|
|
181
|
-
skip(d.label, err);
|
|
182
|
-
continue;
|
|
183
|
-
}
|
|
184
|
-
if (looksSbs(r.width, r.height)) {
|
|
185
|
-
stop(mono);
|
|
186
|
-
log('camera-sbs', { label: r.label, width: r.width, height: r.height });
|
|
187
|
-
return result(r, 'sbs');
|
|
188
|
-
}
|
|
189
|
-
stop(r);
|
|
190
|
-
}
|
|
191
|
-
if (mono) {
|
|
192
|
-
if (want === 'stereo') log('camera-no-stereo', { fallback: mono.label });
|
|
193
|
-
return result(mono, 'mono');
|
|
194
|
-
}
|
|
195
|
-
const code = noCameraCode(skipped);
|
|
196
|
-
throw Object.assign(
|
|
197
|
-
new Error(code === 'camera-busy' ? 'every camera is in use by another app (e.g. eye tracking)' : 'no camera could be opened'),
|
|
198
|
-
{ code, skipped }
|
|
199
|
-
);
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
/** The microphone, with echo cancellation and noise suppression. Null if unavailable/denied. */
|
|
203
|
-
export async function openMic(o = {}) {
|
|
204
|
-
const md = o.mediaDevices || (globalThis.navigator && navigator.mediaDevices);
|
|
205
|
-
try {
|
|
206
|
-
const s = await md.getUserMedia({ audio: { ...AUDIO_CONSTRAINTS }, video: false });
|
|
207
|
-
return s.getAudioTracks()[0] || null;
|
|
208
|
-
} catch (err) {
|
|
209
|
-
(o.log || (() => {}))('mic-skip', { error: err && err.name });
|
|
210
|
-
return null;
|
|
211
|
-
}
|
|
212
|
-
}
|
|
1
|
+
// call/capture.js — moved to js/camera/capture.js in C2 (RFC 0003 §4: device selection,
|
|
2
|
+
// calibration and the busy-device skip belong to `@displayxr/inline3d/camera`). This path
|
|
3
|
+
// re-exports it so test imports and any in-tree reference keep resolving.
|
|
4
|
+
export * from '../camera/capture.js';
|
package/js/call/disparity.js
CHANGED
|
@@ -1,432 +1,4 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
// No focal length, baseline or distance: those only PREDICT the disparity (f·B/Z); here it is read
|
|
6
|
-
// off the pixels. Works for any SBS sender, calibrated or not, as long as the rows roughly align (a
|
|
7
|
-
// small vertical search absorbs the residual of a raw, unrectified pair).
|
|
8
|
-
//
|
|
9
|
-
// The focus is ONE point, the midpoint between the eyes. It stays meaningful when the head turns (one
|
|
10
|
-
// eye may be hidden), unlike "both eyes" or "the whole face". Without a face detector the module
|
|
11
|
-
// finds the subject as the NEAREST strong disparity mode (in a call the person is in front of the
|
|
12
|
-
// room) and takes the eye line from the top of that blob.
|
|
13
|
-
//
|
|
14
|
-
// Pure: a grayscale SBS image in, numbers out. No DOM, no WebRTC. Sizes are in the INPUT's pixels;
|
|
15
|
-
// the caller scales back to source pixels.
|
|
16
|
-
|
|
17
|
-
/** Blocks with a luma standard deviation below this are too flat to match. */
|
|
18
|
-
const MIN_TEXTURE = 6;
|
|
19
|
-
|
|
20
|
-
/** A block match must reach this normalised cross-correlation to count. */
|
|
21
|
-
export const MIN_NCC = 0.8;
|
|
22
|
-
|
|
23
|
-
/** ...and beat its best rival peak (>= 3 px away) by this much, or it is ambiguous (periodic). */
|
|
24
|
-
export const UNIQUENESS = 0.08;
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Normalised cross-correlation of a `bw`×`bh` block of the left eye at (lx, ly) against the right
|
|
28
|
-
* eye at (rx, ry). `img` is the whole SBS image, `W` its full width, `E` the per-eye width.
|
|
29
|
-
* Returns -1 when the right block would leave the right eye or is flat.
|
|
30
|
-
*/
|
|
31
|
-
function ncc(img, W, E, lx, ly, rx, ry, bw, bh, lMean, lStd) {
|
|
32
|
-
if (rx < 0 || rx + bw > E || ry < 0) return -1;
|
|
33
|
-
let sum = 0;
|
|
34
|
-
let sum2 = 0;
|
|
35
|
-
let cross = 0;
|
|
36
|
-
const n = bw * bh;
|
|
37
|
-
for (let y = 0; y < bh; y++) {
|
|
38
|
-
const lo = (ly + y) * W + lx;
|
|
39
|
-
const ro = (ry + y) * W + E + rx;
|
|
40
|
-
for (let x = 0; x < bw; x++) {
|
|
41
|
-
const r = img[ro + x];
|
|
42
|
-
sum += r;
|
|
43
|
-
sum2 += r * r;
|
|
44
|
-
cross += img[lo + x] * r;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
const rMean = sum / n;
|
|
48
|
-
const rVar = sum2 / n - rMean * rMean;
|
|
49
|
-
if (rVar < 1) return -1;
|
|
50
|
-
return (cross / n - lMean * rMean) / (lStd * Math.sqrt(rVar));
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
function blockStats(img, W, x0, y0, bw, bh) {
|
|
54
|
-
let sum = 0;
|
|
55
|
-
let sum2 = 0;
|
|
56
|
-
for (let y = 0; y < bh; y++) {
|
|
57
|
-
const o = (y0 + y) * W + x0;
|
|
58
|
-
for (let x = 0; x < bw; x++) {
|
|
59
|
-
const v = img[o + x];
|
|
60
|
-
sum += v;
|
|
61
|
-
sum2 += v * v;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
const n = bw * bh;
|
|
65
|
-
const mean = sum / n;
|
|
66
|
-
return { mean, std: Math.sqrt(Math.max(0, sum2 / n - mean * mean)) };
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* Best horizontal disparity of one left-eye block: `d = xLeft − xRight` (positive = crossed = in
|
|
71
|
-
* front of the display plane), searched over [dMin, dMax] and ±dy rows. Sub-pixel by a parabola.
|
|
72
|
-
* @returns {{d:number, dy:number, c:number}|null}
|
|
73
|
-
*/
|
|
74
|
-
export function matchBlock(img, W, H, x0, y0, bw, bh, { dMin, dMax, dyMax = 2, uniq } = {}) {
|
|
75
|
-
const E = W / 2;
|
|
76
|
-
const { mean, std } = blockStats(img, W, x0, y0, bw, bh);
|
|
77
|
-
if (std < MIN_TEXTURE) return null;
|
|
78
|
-
let best = -2;
|
|
79
|
-
let bd = 0;
|
|
80
|
-
let bdy = 0;
|
|
81
|
-
const perD = new Map(); // best NCC seen at each disparity (any row), for the uniqueness test
|
|
82
|
-
const scan = (d0, d1, dStep, y0s, y1s) => {
|
|
83
|
-
for (let dy = y0s; dy <= y1s; dy++) {
|
|
84
|
-
const ry = y0 + dy;
|
|
85
|
-
if (ry < 0 || ry + bh > H) continue;
|
|
86
|
-
for (let d = d0; d <= d1; d += dStep) {
|
|
87
|
-
const c = ncc(img, W, E, x0, y0, x0 - d, ry, bw, bh, mean, std);
|
|
88
|
-
if (c > (perD.get(d) ?? -2)) perD.set(d, c);
|
|
89
|
-
if (c > best) {
|
|
90
|
-
best = c;
|
|
91
|
-
bd = d;
|
|
92
|
-
bdy = dy;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
};
|
|
97
|
-
// Coarse (every 2nd disparity, rows every 2nd), then a full-resolution pass around the winner:
|
|
98
|
-
// ~1/4 of the exhaustive cost, and textures here are never finer than the downscaled 2-px step.
|
|
99
|
-
const coarse = dMax - dMin > 8;
|
|
100
|
-
scan(dMin, dMax, coarse ? 2 : 1, -dyMax, dyMax);
|
|
101
|
-
if (coarse && best > -1) {
|
|
102
|
-
const cd = bd;
|
|
103
|
-
const cy = bdy;
|
|
104
|
-
scan(Math.max(dMin, cd - 2), Math.min(dMax, cd + 2), 1, Math.max(-dyMax, cy - 1), Math.min(dyMax, cy + 1));
|
|
105
|
-
}
|
|
106
|
-
if (best < -1) return null;
|
|
107
|
-
// Uniqueness: a repeating texture (a checkerboard, a striped shirt, blinds) matches equally well
|
|
108
|
-
// one period away. Reject the block unless its best peak clearly beats every peak >= 3 px away.
|
|
109
|
-
// The coarse scan samples every 2nd disparity, so a rival peak between two samples is under-read
|
|
110
|
-
// while the winner has been refined to its top; refine each rival local maximum at +-1 as well, or
|
|
111
|
-
// an exactly periodic pattern slips through by a hair.
|
|
112
|
-
let second = -2;
|
|
113
|
-
const peakAt = (d) => {
|
|
114
|
-
let v = -2;
|
|
115
|
-
for (let dy = -dyMax; dy <= dyMax; dy++) {
|
|
116
|
-
const ry = y0 + dy;
|
|
117
|
-
if (ry < 0 || ry + bh > H) continue;
|
|
118
|
-
for (let k = d - 1; k <= d + 1; k++) v = Math.max(v, ncc(img, W, E, x0, y0, x0 - k, ry, bw, bh, mean, std));
|
|
119
|
-
}
|
|
120
|
-
return v;
|
|
121
|
-
};
|
|
122
|
-
for (const [d, c] of perD) {
|
|
123
|
-
if (Math.abs(d - bd) < 3 || c <= second) continue;
|
|
124
|
-
const isLocalMax = c >= (perD.get(d - 2) ?? -2) && c >= (perD.get(d + 2) ?? -2);
|
|
125
|
-
const v = isLocalMax && c > 0 ? peakAt(d) : c;
|
|
126
|
-
if (v > second) second = v;
|
|
127
|
-
}
|
|
128
|
-
// Only for WIDE searches: in a narrow (tracking/refine) window a smooth face legitimately scores
|
|
129
|
-
// high a few px off its peak, and a window anchored on a known disparity cannot alias anyway.
|
|
130
|
-
if (dMax - dMin > 12 && second > -2 && best - second < (uniq ?? UNIQUENESS)) return null;
|
|
131
|
-
// Sub-pixel along d at the best row.
|
|
132
|
-
const cm = ncc(img, W, E, x0, y0, x0 - (bd - 1), y0 + bdy, bw, bh, mean, std);
|
|
133
|
-
const cp = ncc(img, W, E, x0, y0, x0 - (bd + 1), y0 + bdy, bw, bh, mean, std);
|
|
134
|
-
let sub = 0;
|
|
135
|
-
const den = cm - 2 * best + cp;
|
|
136
|
-
if (cm > -1 && cp > -1 && den < 0) sub = Math.max(-0.5, Math.min(0.5, (0.5 * (cm - cp)) / den));
|
|
137
|
-
return { d: bd + sub, dy: bdy, c: best };
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Block disparities over the central region of the LEFT eye.
|
|
142
|
-
* @returns {Array<{x:number,y:number,d:number,dy:number,c:number}>} block top-left corners
|
|
143
|
-
*/
|
|
144
|
-
export function blockDisparities(img, W, H, o = {}) {
|
|
145
|
-
const E = W / 2;
|
|
146
|
-
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
147
|
-
const step = o.step || Math.round(b * 1.5); // sparse grid: half the blocks, same modes
|
|
148
|
-
const dMax = o.dMax ?? Math.round(E * 0.3);
|
|
149
|
-
const dMin = o.dMin ?? -Math.round(E * 0.05);
|
|
150
|
-
const x0 = Math.round(E * (o.roiX0 ?? 0.15));
|
|
151
|
-
const x1 = Math.round(E * (o.roiX1 ?? 0.85)) - b;
|
|
152
|
-
const y0 = Math.round(H * (o.roiY0 ?? 0.05));
|
|
153
|
-
const y1 = Math.round(H * (o.roiY1 ?? 0.85)) - b;
|
|
154
|
-
const out = [];
|
|
155
|
-
for (let y = y0; y <= y1; y += step) {
|
|
156
|
-
for (let x = x0; x <= x1; x += step) {
|
|
157
|
-
const m = matchBlock(img, W, H, x, y, b, b, { dMin, dMax, dyMax: o.dyMax ?? 2, uniq: o.uniq });
|
|
158
|
-
if (m && m.c >= (o.minNcc ?? MIN_NCC)) out.push({ x, y, ...m });
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
return out;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
/**
|
|
165
|
-
* The subject = the NEAREST strong disparity mode: the largest `d` whose ±`tol` bin holds at least
|
|
166
|
-
* `minFrac` of the matched blocks (and `minBlocks`). Returns its members, or null.
|
|
167
|
-
*/
|
|
168
|
-
export function nearestMode(blocks, { tol = 2, minFrac = 0.12, minBlocks = 3 } = {}) {
|
|
169
|
-
if (!blocks.length) return null;
|
|
170
|
-
const need = Math.max(minBlocks, Math.ceil(blocks.length * minFrac));
|
|
171
|
-
const ds = blocks.map((b) => b.d).sort((a, b) => b - a); // descending: nearest first
|
|
172
|
-
for (const d of ds) {
|
|
173
|
-
const members = blocks.filter((b) => Math.abs(b.d - d) <= tol);
|
|
174
|
-
if (members.length >= need) {
|
|
175
|
-
const md = members.map((b) => b.d).sort((a, b) => a - b);
|
|
176
|
-
return { d: md[md.length >> 1], members };
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
return null;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* Where the eyes are in a subject blob: horizontally the centre of its TOP rows (the head, not the
|
|
184
|
-
* shoulders), vertically ~40% down a head whose height ≈ 1.3 × its width.
|
|
185
|
-
*
|
|
186
|
-
* Measured on real SR-camera frames (grayscale, dim), only a handful of blocks match on a face, so
|
|
187
|
-
* this lands on the upper face, often the forehead, a few cm above the eyes. That is the same depth
|
|
188
|
-
* plane to within ~1 cm: at 0.6 m the disparity differs by ~1-2 source px, under 1 px of shift per
|
|
189
|
-
* eye. A darkest-band eye-line search was tried and rejected: the darkest band is the hair. Pass
|
|
190
|
-
* `focus` (a face detector's eye midpoint) for the exact point.
|
|
191
|
-
*/
|
|
192
|
-
export function eyeMidpoint(members, block) {
|
|
193
|
-
const top = Math.min(...members.map((m) => m.y));
|
|
194
|
-
const headRows = members.filter((m) => m.y <= top + 2 * block);
|
|
195
|
-
const xs = headRows.map((m) => m.x);
|
|
196
|
-
const xMin = Math.min(...xs);
|
|
197
|
-
const xMax = Math.max(...xs) + block;
|
|
198
|
-
const headW = xMax - xMin;
|
|
199
|
-
return { x: (xMin + xMax) / 2, y: top + 0.4 * 1.3 * headW };
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
/**
|
|
203
|
-
* The disparity of the point between the eyes, in input pixels.
|
|
204
|
-
*
|
|
205
|
-
* @param {Uint8Array|Uint8ClampedArray|number[]} img grayscale SBS, row-major, `W`×`H`
|
|
206
|
-
* @param {number} W full SBS width (both eyes)
|
|
207
|
-
* @param {number} H
|
|
208
|
-
* @param {object} [o]
|
|
209
|
-
* @param {{x:number,y:number}} [o.focus] the eye midpoint in LEFT-eye pixels (e.g. from a face
|
|
210
|
-
* detector); skips the subject search
|
|
211
|
-
* @returns {{d:number, x:number, y:number, c:number, method:'focus'|'mode', blocks:number}|null}
|
|
212
|
-
*/
|
|
213
|
-
export function measureFocusDisparity(img, W, H, o = {}) {
|
|
214
|
-
const E = W / 2;
|
|
215
|
-
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
216
|
-
const dMax = o.dMax ?? Math.round(E * 0.3);
|
|
217
|
-
const dMin = o.dMin ?? -Math.round(E * 0.05);
|
|
218
|
-
let focus = o.focus || null;
|
|
219
|
-
let method = 'focus';
|
|
220
|
-
let blocks = 0;
|
|
221
|
-
let d0 = null;
|
|
222
|
-
if (!focus) {
|
|
223
|
-
const all = blockDisparities(img, W, H, { ...o, block: b, dMax, dMin });
|
|
224
|
-
blocks = all.length;
|
|
225
|
-
const mode = nearestMode(all, o);
|
|
226
|
-
if (!mode) return null;
|
|
227
|
-
focus = eyeMidpoint(mode.members, b);
|
|
228
|
-
d0 = mode.d;
|
|
229
|
-
method = 'mode';
|
|
230
|
-
}
|
|
231
|
-
// Refine with one wider window centred on the focus point: eyes + brow + nose bridge carry
|
|
232
|
-
// plenty of texture even when one eye is turned away.
|
|
233
|
-
const ww = Math.round(b * (o.refineW ?? 3));
|
|
234
|
-
const wh = Math.round(b * (o.refineH ?? 1.5));
|
|
235
|
-
const rx = Math.round(Math.max(0, Math.min(E - ww, focus.x - ww / 2)));
|
|
236
|
-
const ry = Math.round(Math.max(0, Math.min(H - wh, focus.y - wh / 2)));
|
|
237
|
-
const lo = d0 === null ? dMin : Math.max(dMin, Math.floor(d0) - 4);
|
|
238
|
-
const hi = d0 === null ? dMax : Math.min(dMax, Math.ceil(d0) + 4);
|
|
239
|
-
const m = matchBlock(img, W, H, rx, ry, ww, wh, { dMin: lo, dMax: hi, dyMax: o.dyMax ?? 2 });
|
|
240
|
-
if (m && m.c >= (o.minNcc ?? MIN_NCC)) return { d: m.d, x: focus.x, y: focus.y, c: m.c, method, blocks };
|
|
241
|
-
// A flat or occluded focus window: fall back to the subject's mode.
|
|
242
|
-
if (d0 !== null) return { d: d0, x: focus.x, y: focus.y, c: 0, method, blocks };
|
|
243
|
-
return null;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
/** RGBA pixels (ImageData.data) of a SBS frame → luma, same size. */
|
|
247
|
-
export function lumaFromRgba(rgba, n) {
|
|
248
|
-
const out = new Uint8Array(n);
|
|
249
|
-
for (let i = 0, j = 0; i < n; i++, j += 4) out[i] = (rgba[j] * 77 + rgba[j + 1] * 150 + rgba[j + 2] * 29) >> 8;
|
|
250
|
-
return out;
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
/**
|
|
254
|
-
* Temporal steadiness for measurements: the median of the last `n` accepted values, and a hold of
|
|
255
|
-
* the last good value when a measurement fails (a blink, a hand over the face) rather than snapping
|
|
256
|
-
* back to zero.
|
|
257
|
-
*/
|
|
258
|
-
export function createDisparityTrack({ n = 3 } = {}) {
|
|
259
|
-
const hist = [];
|
|
260
|
-
const t = {
|
|
261
|
-
value: null,
|
|
262
|
-
push(d) {
|
|
263
|
-
if (d === null || !Number.isFinite(d)) return t.value;
|
|
264
|
-
hist.push(d);
|
|
265
|
-
if (hist.length > n) hist.shift();
|
|
266
|
-
const s = [...hist].sort((a, b) => a - b);
|
|
267
|
-
t.value = s[s.length >> 1];
|
|
268
|
-
return t.value;
|
|
269
|
-
},
|
|
270
|
-
reset() {
|
|
271
|
-
hist.length = 0;
|
|
272
|
-
t.value = null;
|
|
273
|
-
},
|
|
274
|
-
};
|
|
275
|
-
return t;
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
/** Copy a `w`×`h` patch of the LEFT eye at (x, y). */
|
|
279
|
-
function copyPatch(img, W, x, y, w, h) {
|
|
280
|
-
const p = new Float32Array(w * h);
|
|
281
|
-
for (let r = 0; r < h; r++) for (let c = 0; c < w; c++) p[r * w + c] = img[(y + r) * W + x + c];
|
|
282
|
-
return p;
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/** Best NCC position of `tpl` (w×h) in the LEFT eye within ±rx/±ry of (x0, y0). */
|
|
286
|
-
function findInLeft(img, W, H, tpl, w, h, x0, y0, rx, ry) {
|
|
287
|
-
const E = W / 2;
|
|
288
|
-
const n = w * h;
|
|
289
|
-
let tm = 0;
|
|
290
|
-
let t2 = 0;
|
|
291
|
-
for (let i = 0; i < n; i++) {
|
|
292
|
-
tm += tpl[i];
|
|
293
|
-
t2 += tpl[i] * tpl[i];
|
|
294
|
-
}
|
|
295
|
-
tm /= n;
|
|
296
|
-
const ts = Math.sqrt(Math.max(1e-6, t2 / n - tm * tm));
|
|
297
|
-
let best = -2;
|
|
298
|
-
let bx = x0;
|
|
299
|
-
let by = y0;
|
|
300
|
-
for (let y = Math.max(0, y0 - ry); y <= Math.min(H - h, y0 + ry); y++) {
|
|
301
|
-
for (let x = Math.max(0, x0 - rx); x <= Math.min(E - w, x0 + rx); x++) {
|
|
302
|
-
let s = 0;
|
|
303
|
-
let s2 = 0;
|
|
304
|
-
let cr = 0;
|
|
305
|
-
for (let r = 0; r < h; r++) {
|
|
306
|
-
const o = (y + r) * W + x;
|
|
307
|
-
for (let c = 0; c < w; c++) {
|
|
308
|
-
const v = img[o + c];
|
|
309
|
-
s += v;
|
|
310
|
-
s2 += v * v;
|
|
311
|
-
cr += v * tpl[r * w + c];
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
const m = s / n;
|
|
315
|
-
const vr = s2 / n - m * m;
|
|
316
|
-
if (vr < 1) continue;
|
|
317
|
-
const cc = (cr / n - m * tm) / (Math.sqrt(vr) * ts);
|
|
318
|
-
if (cc > best) {
|
|
319
|
-
best = cc;
|
|
320
|
-
bx = x;
|
|
321
|
-
by = y;
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
return { x: bx, y: by, c: best };
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
/**
|
|
329
|
-
* Cheap frame-to-frame tracking of the focus point. A full subject search (measureFocusDisparity)
|
|
330
|
-
* locks on; after that each call only re-finds a small left-eye template around the last position
|
|
331
|
-
* (the head moved) and matches it to the right eye in a ±`dWin` disparity window. The full search
|
|
332
|
-
* re-runs every `fullEveryMs`, or as soon as tracking loses the template.
|
|
333
|
-
*/
|
|
334
|
-
export function createFocusTracker({ fullEveryMs = 2000, dWin = 4, rx = 12, ry = 8, minTrackNcc = 0.75, jump = 8 } = {}) {
|
|
335
|
-
let st = null; // { x, y, d, tpl, w, h, at }
|
|
336
|
-
let pending = null; // a re-search result that disagreed with a healthy track, awaiting confirmation
|
|
337
|
-
const t = {
|
|
338
|
-
/** @returns {{d:number,x:number,y:number,c:number,method:'mode'|'focus'|'track',blocks:number}|null} */
|
|
339
|
-
measure(img, W, H, now, o = {}) {
|
|
340
|
-
const E = W / 2;
|
|
341
|
-
const b = o.block || Math.max(8, Math.round(E / 20));
|
|
342
|
-
const w = Math.round(b * 3);
|
|
343
|
-
const h = Math.round(b * 1.5);
|
|
344
|
-
// Track first, whenever there is something to track.
|
|
345
|
-
let tracked = null;
|
|
346
|
-
if (st && st.w === w && st.h === h) {
|
|
347
|
-
const f = findInLeft(img, W, H, st.tpl, w, h, st.x, st.y, rx, ry);
|
|
348
|
-
if (f.c >= minTrackNcc) {
|
|
349
|
-
const m = matchBlock(img, W, H, f.x, f.y, w, h, { dMin: Math.floor(st.d) - dWin, dMax: Math.ceil(st.d) + dWin, dyMax: 2 });
|
|
350
|
-
if (m && m.c >= MIN_NCC) tracked = { m, f };
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
const accept = ({ m, f }) => {
|
|
354
|
-
st = { ...st, x: f.x, y: f.y, d: m.d, tpl: copyPatch(img, W, f.x, f.y, w, h) };
|
|
355
|
-
return { d: m.d, x: f.x + w / 2, y: f.y + h / 2, c: m.c, method: 'track', blocks: 0 };
|
|
356
|
-
};
|
|
357
|
-
if (tracked && now - st.at < fullEveryMs) return accept(tracked);
|
|
358
|
-
const m = measureFocusDisparity(img, W, H, o);
|
|
359
|
-
if (tracked && (!m || Math.abs(m.d - tracked.m.d) > jump)) {
|
|
360
|
-
// A periodic re-search that disagrees with a HEALTHY track is not trusted on its own (a
|
|
361
|
-
// repeating background can alias one period away): switch only when the next re-search
|
|
362
|
-
// confirms it (a new, nearer person really did step in).
|
|
363
|
-
const confirmed = m && pending && Math.abs(pending.d - m.d) <= 3;
|
|
364
|
-
pending = m && !confirmed ? { d: m.d } : null;
|
|
365
|
-
if (!confirmed) {
|
|
366
|
-
st.at = now;
|
|
367
|
-
return accept(tracked);
|
|
368
|
-
}
|
|
369
|
-
} else pending = null;
|
|
370
|
-
if (!m) {
|
|
371
|
-
st = null;
|
|
372
|
-
return null;
|
|
373
|
-
}
|
|
374
|
-
const x = Math.round(Math.max(0, Math.min(E - w, m.x - w / 2)));
|
|
375
|
-
const y = Math.round(Math.max(0, Math.min(H - h, m.y - h / 2)));
|
|
376
|
-
st = { x, y, d: m.d, tpl: copyPatch(img, W, x, y, w, h), w, h, at: now };
|
|
377
|
-
return m;
|
|
378
|
-
},
|
|
379
|
-
reset() {
|
|
380
|
-
st = null;
|
|
381
|
-
pending = null;
|
|
382
|
-
},
|
|
383
|
-
};
|
|
384
|
-
return t;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
/**
|
|
388
|
-
* Downsample a luma plane by an integer `factor` (the Y plane of a decoded frame, read straight from
|
|
389
|
-
* WebCodecs: no RGBA conversion, no canvas). Averaging, not single-point sampling, so sensor noise
|
|
390
|
-
* does not become false texture: the full box for factors <= 2, a 2x2 sample per cell above.
|
|
391
|
-
* @param {Uint8Array} src the plane
|
|
392
|
-
* @param {number} offset byte offset of the plane in `src`
|
|
393
|
-
* @param {number} stride bytes per source row
|
|
394
|
-
* @returns {{img: Uint8Array, w: number, h: number}}
|
|
395
|
-
*/
|
|
396
|
-
export function downsampleLuma(src, offset, stride, srcW, srcH, factor) {
|
|
397
|
-
const f = Math.max(1, factor | 0);
|
|
398
|
-
let w = Math.floor(srcW / f);
|
|
399
|
-
w -= w & 1; // even, so the SBS halves split cleanly
|
|
400
|
-
const h = Math.floor(srcH / f);
|
|
401
|
-
const out = new Uint8Array(w * h);
|
|
402
|
-
if (f <= 2) {
|
|
403
|
-
// Small factors: the full box.
|
|
404
|
-
const n = f * f;
|
|
405
|
-
for (let y = 0; y < h; y++) {
|
|
406
|
-
for (let x = 0; x < w; x++) {
|
|
407
|
-
let sum = 0;
|
|
408
|
-
for (let dy = 0; dy < f; dy++) {
|
|
409
|
-
const o = offset + (y * f + dy) * stride + x * f;
|
|
410
|
-
for (let dx = 0; dx < f; dx++) sum += src[o + dx];
|
|
411
|
-
}
|
|
412
|
-
out[y * w + x] = (sum / n) | 0;
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
return { img: out, w, h };
|
|
416
|
-
}
|
|
417
|
-
// Larger factors: a 2x2 sample at the quarter points of each cell. Enough averaging to keep sensor
|
|
418
|
-
// and codec noise from reading as texture, at ~1/9 of the full box's cost for f = 6 (the full box
|
|
419
|
-
// over a 2560x720 luma plane was ~8 ms of JS per measurement).
|
|
420
|
-
const a = f >> 2;
|
|
421
|
-
const c = (3 * f) >> 2;
|
|
422
|
-
for (let y = 0; y < h; y++) {
|
|
423
|
-
const r0 = offset + (y * f + a) * stride;
|
|
424
|
-
const r1 = offset + (y * f + c) * stride;
|
|
425
|
-
for (let x = 0; x < w; x++) {
|
|
426
|
-
const x0 = x * f + a;
|
|
427
|
-
const x1 = x * f + c;
|
|
428
|
-
out[y * w + x] = (src[r0 + x0] + src[r0 + x1] + src[r1 + x0] + src[r1 + x1]) >> 2;
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
return { img: out, w, h };
|
|
432
|
-
}
|
|
1
|
+
// call/disparity.js — moved to js/camera/disparity.js in C2 (RFC 0003 §4: auto-convergence is a
|
|
2
|
+
// camera-side measurement that the call and `addCameraView` share). This path re-exports it so
|
|
3
|
+
// test imports and any in-tree reference keep resolving; new code imports the camera module.
|
|
4
|
+
export * from '../camera/disparity.js';
|