@displayxr/inline3d 1.29.0 → 1.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,212 +1,4 @@
1
- // call/capture.js — getting a camera (+ microphone) and knowing what it is (RFC §4).
2
- //
3
- // `camera: 'auto'` looks for a device that delivers a side-by-side PAIR — one device whose frame
4
- // is wider than 2.5:1 (a USB stereo camera enumerates as one wide device: field data has one at
5
- // 1280x480, i.e. 640x480 per eye, grayscale and UNRECTIFIED). Found → sent as `sbs`. Otherwise the
6
- // default webcam, sent as `mono`. Per-eye width is whatever the camera gives; nothing is upscaled.
7
- //
8
- // Two field facts shape the probing:
9
- // - On some laptops the stereo camera is held EXCLUSIVELY by the eye tracker, so opening it
10
- // fails with NotReadableError ("Device in use"). That device is skipped silently — a call must
11
- // never fail because an optional camera is busy.
12
- // - Labels are empty until one getUserMedia has succeeded, so the default camera is opened first
13
- // (which is also the fallback), then the others are probed one at a time.
14
- //
15
- // RECTIFICATION is not done here (P2): it needs the camera's calibration (intrinsics/extrinsics
16
- // for the ACTIVE device), which comes from a plug-in or the runtime. P1 sends the raw pair with
17
- // `rectified: false`, and the `rectify` hook (addCall option) is where a calibrated step plugs in:
18
- // `(stream, info) => MediaStream | Promise<MediaStream>`, returning a rectified SBS stream.
19
-
20
- import { looksSbs } from './wire.js';
21
-
22
- const MONO_CONSTRAINTS = { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30 } };
23
- // Ask a candidate stereo device for its widest mode; a 2560x720 or 3840x1080 pair comes back as
24
- // itself, a 1280x480 one as 1280x480. `ideal`, never `min`: a smaller real pair is still a pair.
25
- const PROBE_CONSTRAINTS = { width: { ideal: 3840 }, height: { ideal: 1080 }, frameRate: { ideal: 30 } };
26
- export const AUDIO_CONSTRAINTS = Object.freeze({ echoCancellation: true, noiseSuppression: true, autoGainControl: true });
27
-
28
- /** Errors that mean "the device exists but another process holds it" (an eye tracker, say). */
29
- export function isBusyError(err) {
30
- return !!err && (err.name === 'NotReadableError' || err.name === 'TrackStartError' || /in use|busy/i.test(err.message || ''));
31
- }
32
-
33
- /**
34
- * Why no camera could be opened, from the per-device failures. `'camera-busy'` when every device
35
- * that exists is held by another process — on a 3D laptop/monitor whose only camera is the eye
36
- * tracker's, that is the normal case while tracking, and the call should go audio-only and say so
37
- * rather than fail or send an empty picture. Pure.
38
- * @param {Array<{error: string, busy?: boolean}>} skipped
39
- */
40
- export function noCameraCode(skipped) {
41
- return skipped.length && skipped.every((s) => s.busy) ? 'camera-busy' : 'no-camera';
42
- }
43
-
44
- /** A label that names a stereo camera, tried first when probing. Pure. */
45
- export function stereoLabelHint(label) {
46
- return /stereo|\b3d\b|\bsbs\b|dual/i.test(label || '');
47
- }
48
-
49
- /**
50
- * Open the camera. Never rejects for a busy optional device; rejects only when NO camera can be
51
- * opened at all (the caller then runs audio-only or reports the error).
52
- *
53
- * @param {'auto'|'stereo'|'mono'|string|MediaStream} want 'auto' | 'stereo' | 'mono' | a
54
- * deviceId | a page-supplied MediaStream
55
- * @param {{ format?: 'sbs'|'mono', calibration?: {baselineMm?: number, hfovDeg?: number, rectified?: boolean},
56
- * mediaDevices?: MediaDevices, log?: Function }} [o]
57
- * @returns {Promise<{ stream: MediaStream, format: 'sbs'|'mono', width: number, height: number,
58
- * deviceId: string|null, label: string, owned: boolean, skipped: Array<{label: string, error: string}>,
59
- * calibration: object }>}
60
- */
61
- export async function openCamera(want = 'auto', o = {}) {
62
- const md = o.mediaDevices || (globalThis.navigator && navigator.mediaDevices);
63
- const log = o.log || (() => {});
64
- const skipped = [];
65
- const cal = o.calibration || {};
66
-
67
- if (want && typeof want === 'object' && typeof want.getVideoTracks === 'function') {
68
- const t = want.getVideoTracks()[0];
69
- const st = t ? t.getSettings() : {};
70
- // A page-supplied stream carries a DECLARED format; aspect is never used to guess 3D-ness here.
71
- const format = o.format === 'sbs' ? 'sbs' : 'mono';
72
- return { stream: want, format, width: st.width || 0, height: st.height || 0, deviceId: st.deviceId || null, label: t ? t.label : '', owned: false, skipped, calibration: cal };
73
- }
74
- if (!md || !md.getUserMedia) throw Object.assign(new Error('getUserMedia is not available (insecure context?)'), { code: 'no-camera' });
75
-
76
- const open = async (constraints) => {
77
- const s = await md.getUserMedia({ video: constraints, audio: false });
78
- const t = s.getVideoTracks()[0];
79
- const st = t ? t.getSettings() : {};
80
- // Never hand back a 0x0 track: some engines open a device that then delivers nothing.
81
- if (!t || t.readyState === 'ended' || !(st.width > 0) || !(st.height > 0)) {
82
- s.getTracks().forEach((x) => x.stop());
83
- throw Object.assign(new Error('the camera opened but delivers no picture (0x0)'), { name: 'NotReadableError' });
84
- }
85
- // The DisplayXR Browser's "3D Camera (DisplayXR)" (the runtime's stereo camera, e.g. an SR
86
- // eye-tracking camera shared without stealing it) describes itself on the opened track:
87
- // `displayxrStereo: {layout, rectified, baselineMm, horizontalFovDeg}`.
88
- const stereo = st.displayxrStereo && st.displayxrStereo.layout === 'side-by-side' ? st.displayxrStereo : null;
89
- return { stream: s, track: t, width: st.width, height: st.height, deviceId: st.deviceId || null, label: t.label || '', stereo };
90
- };
91
- const skip = (label, err) => {
92
- skipped.push({ label, error: `${err.name}: ${err.message}`, busy: isBusyError(err) });
93
- log('camera-skip', { label, error: err.name });
94
- };
95
- const stop = (r) => r && r.stream.getTracks().forEach((t) => t.stop());
96
- const result = (r, format) => ({
97
- stream: r.stream, format, width: r.width, height: r.height, deviceId: r.deviceId, label: r.label, owned: true, skipped,
98
- calibration: {
99
- rectified: false,
100
- // A runtime stereo camera's own description, where the page gave none.
101
- ...(format === 'sbs' && r.stereo
102
- ? {
103
- rectified: !!r.stereo.rectified,
104
- ...(r.stereo.baselineMm > 0 ? { baselineMm: r.stereo.baselineMm } : {}),
105
- ...(r.stereo.horizontalFovDeg > 0 ? { hfovDeg: r.stereo.horizontalFovDeg } : {}),
106
- }
107
- : {}),
108
- ...cal,
109
- },
110
- });
111
-
112
- // An explicit device id.
113
- if (typeof want === 'string' && !['auto', 'stereo', 'mono'].includes(want)) {
114
- let r;
115
- try {
116
- r = await open({ deviceId: { exact: want }, ...PROBE_CONSTRAINTS });
117
- } catch (err) {
118
- skip(want, err);
119
- throw Object.assign(new Error(`camera unavailable: ${err.message}`), { code: noCameraCode(skipped), skipped });
120
- }
121
- if (looksSbs(r.width, r.height)) return result(r, 'sbs');
122
- stop(r);
123
- return result(await open({ deviceId: { exact: want }, ...MONO_CONSTRAINTS }), 'mono');
124
- }
125
-
126
- // Fast path: when labels are already visible (camera permission persisted), open a device that
127
- // NAMES itself stereo — e.g. the DisplayXR Browser's "3D Camera (DisplayXR)" — directly. Every
128
- // camera start can take seconds (a busy capture stack took 7-20 s per start on a field box), and
129
- // the default-first probe below costs two or three of them before it reaches the pair.
130
- const tried = new Set();
131
- if (want !== 'mono') {
132
- let early = [];
133
- try {
134
- early = (await md.enumerateDevices()).filter((d) => d.kind === 'videoinput' && d.deviceId && stereoLabelHint(d.label));
135
- } catch {
136
- /* no enumeration yet: the normal path below */
137
- }
138
- for (const d of early) {
139
- tried.add(d.deviceId);
140
- let r = null;
141
- try {
142
- r = await open({ deviceId: { exact: d.deviceId }, ...PROBE_CONSTRAINTS });
143
- } catch (err) {
144
- skip(d.label, err);
145
- continue;
146
- }
147
- if (looksSbs(r.width, r.height)) {
148
- log('camera-sbs', { label: r.label, width: r.width, height: r.height, direct: true });
149
- return result(r, 'sbs');
150
- }
151
- stop(r);
152
- }
153
- }
154
-
155
- // The default camera first: permission, labels, and the fallback.
156
- let mono = null;
157
- try {
158
- mono = await open(MONO_CONSTRAINTS);
159
- } catch (err) {
160
- skip('default', err);
161
- }
162
- if (want === 'mono' && mono) return result(mono, 'mono');
163
- if (mono && looksSbs(mono.width, mono.height)) return result(mono, 'sbs'); // the default IS the pair
164
-
165
- let devices = [];
166
- try {
167
- devices = (await md.enumerateDevices()).filter((d) => d.kind === 'videoinput');
168
- } catch {
169
- /* no enumeration: keep the default */
170
- }
171
- devices.sort((a, b) => Number(stereoLabelHint(b.label)) - Number(stereoLabelHint(a.label)));
172
- for (const d of devices) {
173
- if (!d.deviceId) continue; // no permission yet: an anonymous entry cannot be opened by id
174
- if (mono && d.deviceId === mono.deviceId) continue;
175
- if (tried.has(d.deviceId)) continue; // the fast path already opened it (not a pair, or busy)
176
- let r = null;
177
- try {
178
- r = await open({ deviceId: { exact: d.deviceId }, ...PROBE_CONSTRAINTS });
179
- } catch (err) {
180
- // NotReadableError = held by another process (e.g. an eye tracker). Skip, never fail.
181
- skip(d.label, err);
182
- continue;
183
- }
184
- if (looksSbs(r.width, r.height)) {
185
- stop(mono);
186
- log('camera-sbs', { label: r.label, width: r.width, height: r.height });
187
- return result(r, 'sbs');
188
- }
189
- stop(r);
190
- }
191
- if (mono) {
192
- if (want === 'stereo') log('camera-no-stereo', { fallback: mono.label });
193
- return result(mono, 'mono');
194
- }
195
- const code = noCameraCode(skipped);
196
- throw Object.assign(
197
- new Error(code === 'camera-busy' ? 'every camera is in use by another app (e.g. eye tracking)' : 'no camera could be opened'),
198
- { code, skipped }
199
- );
200
- }
201
-
202
- /** The microphone, with echo cancellation and noise suppression. Null if unavailable/denied. */
203
- export async function openMic(o = {}) {
204
- const md = o.mediaDevices || (globalThis.navigator && navigator.mediaDevices);
205
- try {
206
- const s = await md.getUserMedia({ audio: { ...AUDIO_CONSTRAINTS }, video: false });
207
- return s.getAudioTracks()[0] || null;
208
- } catch (err) {
209
- (o.log || (() => {}))('mic-skip', { error: err && err.name });
210
- return null;
211
- }
212
- }
1
+ // call/capture.js — moved to js/camera/capture.js in C2 (RFC 0003 §4: device selection,
2
+ // calibration and the busy-device skip belong to `@displayxr/inline3d/camera`). This path
3
+ // re-exports it so test imports and any in-tree reference keep resolving.
4
+ export * from '../camera/capture.js';
@@ -1,432 +1,4 @@
1
- // Auto-convergence (RFC 0002 §3): MEASURE how far apart the remote person's eyes are between the two
2
- // halves of their side-by-side frame, so the receiver can shift each eye by d/2 in opposite
3
- // directions and put the point between the eyes at zero parallax (the display plane).
4
- //
5
- // No focal length, baseline or distance: those only PREDICT the disparity (f·B/Z); here it is read
6
- // off the pixels. Works for any SBS sender, calibrated or not, as long as the rows roughly align (a
7
- // small vertical search absorbs the residual of a raw, unrectified pair).
8
- //
9
- // The focus is ONE point, the midpoint between the eyes. It stays meaningful when the head turns (one
10
- // eye may be hidden), unlike "both eyes" or "the whole face". Without a face detector the module
11
- // finds the subject as the NEAREST strong disparity mode (in a call the person is in front of the
12
- // room) and takes the eye line from the top of that blob.
13
- //
14
- // Pure: a grayscale SBS image in, numbers out. No DOM, no WebRTC. Sizes are in the INPUT's pixels;
15
- // the caller scales back to source pixels.
16
-
17
- /** Blocks with a luma standard deviation below this are too flat to match. */
18
- const MIN_TEXTURE = 6;
19
-
20
- /** A block match must reach this normalised cross-correlation to count. */
21
- export const MIN_NCC = 0.8;
22
-
23
- /** ...and beat its best rival peak (>= 3 px away) by this much, or it is ambiguous (periodic). */
24
- export const UNIQUENESS = 0.08;
25
-
26
- /**
27
- * Normalised cross-correlation of a `bw`×`bh` block of the left eye at (lx, ly) against the right
28
- * eye at (rx, ry). `img` is the whole SBS image, `W` its full width, `E` the per-eye width.
29
- * Returns -1 when the right block would leave the right eye or is flat.
30
- */
31
- function ncc(img, W, E, lx, ly, rx, ry, bw, bh, lMean, lStd) {
32
- if (rx < 0 || rx + bw > E || ry < 0) return -1;
33
- let sum = 0;
34
- let sum2 = 0;
35
- let cross = 0;
36
- const n = bw * bh;
37
- for (let y = 0; y < bh; y++) {
38
- const lo = (ly + y) * W + lx;
39
- const ro = (ry + y) * W + E + rx;
40
- for (let x = 0; x < bw; x++) {
41
- const r = img[ro + x];
42
- sum += r;
43
- sum2 += r * r;
44
- cross += img[lo + x] * r;
45
- }
46
- }
47
- const rMean = sum / n;
48
- const rVar = sum2 / n - rMean * rMean;
49
- if (rVar < 1) return -1;
50
- return (cross / n - lMean * rMean) / (lStd * Math.sqrt(rVar));
51
- }
52
-
53
- function blockStats(img, W, x0, y0, bw, bh) {
54
- let sum = 0;
55
- let sum2 = 0;
56
- for (let y = 0; y < bh; y++) {
57
- const o = (y0 + y) * W + x0;
58
- for (let x = 0; x < bw; x++) {
59
- const v = img[o + x];
60
- sum += v;
61
- sum2 += v * v;
62
- }
63
- }
64
- const n = bw * bh;
65
- const mean = sum / n;
66
- return { mean, std: Math.sqrt(Math.max(0, sum2 / n - mean * mean)) };
67
- }
68
-
69
- /**
70
- * Best horizontal disparity of one left-eye block: `d = xLeft − xRight` (positive = crossed = in
71
- * front of the display plane), searched over [dMin, dMax] and ±dy rows. Sub-pixel by a parabola.
72
- * @returns {{d:number, dy:number, c:number}|null}
73
- */
74
- export function matchBlock(img, W, H, x0, y0, bw, bh, { dMin, dMax, dyMax = 2, uniq } = {}) {
75
- const E = W / 2;
76
- const { mean, std } = blockStats(img, W, x0, y0, bw, bh);
77
- if (std < MIN_TEXTURE) return null;
78
- let best = -2;
79
- let bd = 0;
80
- let bdy = 0;
81
- const perD = new Map(); // best NCC seen at each disparity (any row), for the uniqueness test
82
- const scan = (d0, d1, dStep, y0s, y1s) => {
83
- for (let dy = y0s; dy <= y1s; dy++) {
84
- const ry = y0 + dy;
85
- if (ry < 0 || ry + bh > H) continue;
86
- for (let d = d0; d <= d1; d += dStep) {
87
- const c = ncc(img, W, E, x0, y0, x0 - d, ry, bw, bh, mean, std);
88
- if (c > (perD.get(d) ?? -2)) perD.set(d, c);
89
- if (c > best) {
90
- best = c;
91
- bd = d;
92
- bdy = dy;
93
- }
94
- }
95
- }
96
- };
97
- // Coarse (every 2nd disparity, rows every 2nd), then a full-resolution pass around the winner:
98
- // ~1/4 of the exhaustive cost, and textures here are never finer than the downscaled 2-px step.
99
- const coarse = dMax - dMin > 8;
100
- scan(dMin, dMax, coarse ? 2 : 1, -dyMax, dyMax);
101
- if (coarse && best > -1) {
102
- const cd = bd;
103
- const cy = bdy;
104
- scan(Math.max(dMin, cd - 2), Math.min(dMax, cd + 2), 1, Math.max(-dyMax, cy - 1), Math.min(dyMax, cy + 1));
105
- }
106
- if (best < -1) return null;
107
- // Uniqueness: a repeating texture (a checkerboard, a striped shirt, blinds) matches equally well
108
- // one period away. Reject the block unless its best peak clearly beats every peak >= 3 px away.
109
- // The coarse scan samples every 2nd disparity, so a rival peak between two samples is under-read
110
- // while the winner has been refined to its top; refine each rival local maximum at +-1 as well, or
111
- // an exactly periodic pattern slips through by a hair.
112
- let second = -2;
113
- const peakAt = (d) => {
114
- let v = -2;
115
- for (let dy = -dyMax; dy <= dyMax; dy++) {
116
- const ry = y0 + dy;
117
- if (ry < 0 || ry + bh > H) continue;
118
- for (let k = d - 1; k <= d + 1; k++) v = Math.max(v, ncc(img, W, E, x0, y0, x0 - k, ry, bw, bh, mean, std));
119
- }
120
- return v;
121
- };
122
- for (const [d, c] of perD) {
123
- if (Math.abs(d - bd) < 3 || c <= second) continue;
124
- const isLocalMax = c >= (perD.get(d - 2) ?? -2) && c >= (perD.get(d + 2) ?? -2);
125
- const v = isLocalMax && c > 0 ? peakAt(d) : c;
126
- if (v > second) second = v;
127
- }
128
- // Only for WIDE searches: in a narrow (tracking/refine) window a smooth face legitimately scores
129
- // high a few px off its peak, and a window anchored on a known disparity cannot alias anyway.
130
- if (dMax - dMin > 12 && second > -2 && best - second < (uniq ?? UNIQUENESS)) return null;
131
- // Sub-pixel along d at the best row.
132
- const cm = ncc(img, W, E, x0, y0, x0 - (bd - 1), y0 + bdy, bw, bh, mean, std);
133
- const cp = ncc(img, W, E, x0, y0, x0 - (bd + 1), y0 + bdy, bw, bh, mean, std);
134
- let sub = 0;
135
- const den = cm - 2 * best + cp;
136
- if (cm > -1 && cp > -1 && den < 0) sub = Math.max(-0.5, Math.min(0.5, (0.5 * (cm - cp)) / den));
137
- return { d: bd + sub, dy: bdy, c: best };
138
- }
139
-
140
- /**
141
- * Block disparities over the central region of the LEFT eye.
142
- * @returns {Array<{x:number,y:number,d:number,dy:number,c:number}>} block top-left corners
143
- */
144
- export function blockDisparities(img, W, H, o = {}) {
145
- const E = W / 2;
146
- const b = o.block || Math.max(8, Math.round(E / 20));
147
- const step = o.step || Math.round(b * 1.5); // sparse grid: half the blocks, same modes
148
- const dMax = o.dMax ?? Math.round(E * 0.3);
149
- const dMin = o.dMin ?? -Math.round(E * 0.05);
150
- const x0 = Math.round(E * (o.roiX0 ?? 0.15));
151
- const x1 = Math.round(E * (o.roiX1 ?? 0.85)) - b;
152
- const y0 = Math.round(H * (o.roiY0 ?? 0.05));
153
- const y1 = Math.round(H * (o.roiY1 ?? 0.85)) - b;
154
- const out = [];
155
- for (let y = y0; y <= y1; y += step) {
156
- for (let x = x0; x <= x1; x += step) {
157
- const m = matchBlock(img, W, H, x, y, b, b, { dMin, dMax, dyMax: o.dyMax ?? 2, uniq: o.uniq });
158
- if (m && m.c >= (o.minNcc ?? MIN_NCC)) out.push({ x, y, ...m });
159
- }
160
- }
161
- return out;
162
- }
163
-
164
- /**
165
- * The subject = the NEAREST strong disparity mode: the largest `d` whose ±`tol` bin holds at least
166
- * `minFrac` of the matched blocks (and `minBlocks`). Returns its members, or null.
167
- */
168
- export function nearestMode(blocks, { tol = 2, minFrac = 0.12, minBlocks = 3 } = {}) {
169
- if (!blocks.length) return null;
170
- const need = Math.max(minBlocks, Math.ceil(blocks.length * minFrac));
171
- const ds = blocks.map((b) => b.d).sort((a, b) => b - a); // descending: nearest first
172
- for (const d of ds) {
173
- const members = blocks.filter((b) => Math.abs(b.d - d) <= tol);
174
- if (members.length >= need) {
175
- const md = members.map((b) => b.d).sort((a, b) => a - b);
176
- return { d: md[md.length >> 1], members };
177
- }
178
- }
179
- return null;
180
- }
181
-
182
- /**
183
- * Where the eyes are in a subject blob: horizontally the centre of its TOP rows (the head, not the
184
- * shoulders), vertically ~40% down a head whose height ≈ 1.3 × its width.
185
- *
186
- * Measured on real SR-camera frames (grayscale, dim), only a handful of blocks match on a face, so
187
- * this lands on the upper face, often the forehead, a few cm above the eyes. That is the same depth
188
- * plane to within ~1 cm: at 0.6 m the disparity differs by ~1-2 source px, under 1 px of shift per
189
- * eye. A darkest-band eye-line search was tried and rejected: the darkest band is the hair. Pass
190
- * `focus` (a face detector's eye midpoint) for the exact point.
191
- */
192
- export function eyeMidpoint(members, block) {
193
- const top = Math.min(...members.map((m) => m.y));
194
- const headRows = members.filter((m) => m.y <= top + 2 * block);
195
- const xs = headRows.map((m) => m.x);
196
- const xMin = Math.min(...xs);
197
- const xMax = Math.max(...xs) + block;
198
- const headW = xMax - xMin;
199
- return { x: (xMin + xMax) / 2, y: top + 0.4 * 1.3 * headW };
200
- }
201
-
202
- /**
203
- * The disparity of the point between the eyes, in input pixels.
204
- *
205
- * @param {Uint8Array|Uint8ClampedArray|number[]} img grayscale SBS, row-major, `W`×`H`
206
- * @param {number} W full SBS width (both eyes)
207
- * @param {number} H
208
- * @param {object} [o]
209
- * @param {{x:number,y:number}} [o.focus] the eye midpoint in LEFT-eye pixels (e.g. from a face
210
- * detector); skips the subject search
211
- * @returns {{d:number, x:number, y:number, c:number, method:'focus'|'mode', blocks:number}|null}
212
- */
213
- export function measureFocusDisparity(img, W, H, o = {}) {
214
- const E = W / 2;
215
- const b = o.block || Math.max(8, Math.round(E / 20));
216
- const dMax = o.dMax ?? Math.round(E * 0.3);
217
- const dMin = o.dMin ?? -Math.round(E * 0.05);
218
- let focus = o.focus || null;
219
- let method = 'focus';
220
- let blocks = 0;
221
- let d0 = null;
222
- if (!focus) {
223
- const all = blockDisparities(img, W, H, { ...o, block: b, dMax, dMin });
224
- blocks = all.length;
225
- const mode = nearestMode(all, o);
226
- if (!mode) return null;
227
- focus = eyeMidpoint(mode.members, b);
228
- d0 = mode.d;
229
- method = 'mode';
230
- }
231
- // Refine with one wider window centred on the focus point: eyes + brow + nose bridge carry
232
- // plenty of texture even when one eye is turned away.
233
- const ww = Math.round(b * (o.refineW ?? 3));
234
- const wh = Math.round(b * (o.refineH ?? 1.5));
235
- const rx = Math.round(Math.max(0, Math.min(E - ww, focus.x - ww / 2)));
236
- const ry = Math.round(Math.max(0, Math.min(H - wh, focus.y - wh / 2)));
237
- const lo = d0 === null ? dMin : Math.max(dMin, Math.floor(d0) - 4);
238
- const hi = d0 === null ? dMax : Math.min(dMax, Math.ceil(d0) + 4);
239
- const m = matchBlock(img, W, H, rx, ry, ww, wh, { dMin: lo, dMax: hi, dyMax: o.dyMax ?? 2 });
240
- if (m && m.c >= (o.minNcc ?? MIN_NCC)) return { d: m.d, x: focus.x, y: focus.y, c: m.c, method, blocks };
241
- // A flat or occluded focus window: fall back to the subject's mode.
242
- if (d0 !== null) return { d: d0, x: focus.x, y: focus.y, c: 0, method, blocks };
243
- return null;
244
- }
245
-
246
- /** RGBA pixels (ImageData.data) of a SBS frame → luma, same size. */
247
- export function lumaFromRgba(rgba, n) {
248
- const out = new Uint8Array(n);
249
- for (let i = 0, j = 0; i < n; i++, j += 4) out[i] = (rgba[j] * 77 + rgba[j + 1] * 150 + rgba[j + 2] * 29) >> 8;
250
- return out;
251
- }
252
-
253
- /**
254
- * Temporal steadiness for measurements: the median of the last `n` accepted values, and a hold of
255
- * the last good value when a measurement fails (a blink, a hand over the face) rather than snapping
256
- * back to zero.
257
- */
258
- export function createDisparityTrack({ n = 3 } = {}) {
259
- const hist = [];
260
- const t = {
261
- value: null,
262
- push(d) {
263
- if (d === null || !Number.isFinite(d)) return t.value;
264
- hist.push(d);
265
- if (hist.length > n) hist.shift();
266
- const s = [...hist].sort((a, b) => a - b);
267
- t.value = s[s.length >> 1];
268
- return t.value;
269
- },
270
- reset() {
271
- hist.length = 0;
272
- t.value = null;
273
- },
274
- };
275
- return t;
276
- }
277
-
278
- /** Copy a `w`×`h` patch of the LEFT eye at (x, y). */
279
- function copyPatch(img, W, x, y, w, h) {
280
- const p = new Float32Array(w * h);
281
- for (let r = 0; r < h; r++) for (let c = 0; c < w; c++) p[r * w + c] = img[(y + r) * W + x + c];
282
- return p;
283
- }
284
-
285
- /** Best NCC position of `tpl` (w×h) in the LEFT eye within ±rx/±ry of (x0, y0). */
286
- function findInLeft(img, W, H, tpl, w, h, x0, y0, rx, ry) {
287
- const E = W / 2;
288
- const n = w * h;
289
- let tm = 0;
290
- let t2 = 0;
291
- for (let i = 0; i < n; i++) {
292
- tm += tpl[i];
293
- t2 += tpl[i] * tpl[i];
294
- }
295
- tm /= n;
296
- const ts = Math.sqrt(Math.max(1e-6, t2 / n - tm * tm));
297
- let best = -2;
298
- let bx = x0;
299
- let by = y0;
300
- for (let y = Math.max(0, y0 - ry); y <= Math.min(H - h, y0 + ry); y++) {
301
- for (let x = Math.max(0, x0 - rx); x <= Math.min(E - w, x0 + rx); x++) {
302
- let s = 0;
303
- let s2 = 0;
304
- let cr = 0;
305
- for (let r = 0; r < h; r++) {
306
- const o = (y + r) * W + x;
307
- for (let c = 0; c < w; c++) {
308
- const v = img[o + c];
309
- s += v;
310
- s2 += v * v;
311
- cr += v * tpl[r * w + c];
312
- }
313
- }
314
- const m = s / n;
315
- const vr = s2 / n - m * m;
316
- if (vr < 1) continue;
317
- const cc = (cr / n - m * tm) / (Math.sqrt(vr) * ts);
318
- if (cc > best) {
319
- best = cc;
320
- bx = x;
321
- by = y;
322
- }
323
- }
324
- }
325
- return { x: bx, y: by, c: best };
326
- }
327
-
328
- /**
329
- * Cheap frame-to-frame tracking of the focus point. A full subject search (measureFocusDisparity)
330
- * locks on; after that each call only re-finds a small left-eye template around the last position
331
- * (the head moved) and matches it to the right eye in a ±`dWin` disparity window. The full search
332
- * re-runs every `fullEveryMs`, or as soon as tracking loses the template.
333
- */
334
- export function createFocusTracker({ fullEveryMs = 2000, dWin = 4, rx = 12, ry = 8, minTrackNcc = 0.75, jump = 8 } = {}) {
335
- let st = null; // { x, y, d, tpl, w, h, at }
336
- let pending = null; // a re-search result that disagreed with a healthy track, awaiting confirmation
337
- const t = {
338
- /** @returns {{d:number,x:number,y:number,c:number,method:'mode'|'focus'|'track',blocks:number}|null} */
339
- measure(img, W, H, now, o = {}) {
340
- const E = W / 2;
341
- const b = o.block || Math.max(8, Math.round(E / 20));
342
- const w = Math.round(b * 3);
343
- const h = Math.round(b * 1.5);
344
- // Track first, whenever there is something to track.
345
- let tracked = null;
346
- if (st && st.w === w && st.h === h) {
347
- const f = findInLeft(img, W, H, st.tpl, w, h, st.x, st.y, rx, ry);
348
- if (f.c >= minTrackNcc) {
349
- const m = matchBlock(img, W, H, f.x, f.y, w, h, { dMin: Math.floor(st.d) - dWin, dMax: Math.ceil(st.d) + dWin, dyMax: 2 });
350
- if (m && m.c >= MIN_NCC) tracked = { m, f };
351
- }
352
- }
353
- const accept = ({ m, f }) => {
354
- st = { ...st, x: f.x, y: f.y, d: m.d, tpl: copyPatch(img, W, f.x, f.y, w, h) };
355
- return { d: m.d, x: f.x + w / 2, y: f.y + h / 2, c: m.c, method: 'track', blocks: 0 };
356
- };
357
- if (tracked && now - st.at < fullEveryMs) return accept(tracked);
358
- const m = measureFocusDisparity(img, W, H, o);
359
- if (tracked && (!m || Math.abs(m.d - tracked.m.d) > jump)) {
360
- // A periodic re-search that disagrees with a HEALTHY track is not trusted on its own (a
361
- // repeating background can alias one period away): switch only when the next re-search
362
- // confirms it (a new, nearer person really did step in).
363
- const confirmed = m && pending && Math.abs(pending.d - m.d) <= 3;
364
- pending = m && !confirmed ? { d: m.d } : null;
365
- if (!confirmed) {
366
- st.at = now;
367
- return accept(tracked);
368
- }
369
- } else pending = null;
370
- if (!m) {
371
- st = null;
372
- return null;
373
- }
374
- const x = Math.round(Math.max(0, Math.min(E - w, m.x - w / 2)));
375
- const y = Math.round(Math.max(0, Math.min(H - h, m.y - h / 2)));
376
- st = { x, y, d: m.d, tpl: copyPatch(img, W, x, y, w, h), w, h, at: now };
377
- return m;
378
- },
379
- reset() {
380
- st = null;
381
- pending = null;
382
- },
383
- };
384
- return t;
385
- }
386
-
387
- /**
388
- * Downsample a luma plane by an integer `factor` (the Y plane of a decoded frame, read straight from
389
- * WebCodecs: no RGBA conversion, no canvas). Averaging, not single-point sampling, so sensor noise
390
- * does not become false texture: the full box for factors <= 2, a 2x2 sample per cell above.
391
- * @param {Uint8Array} src the plane
392
- * @param {number} offset byte offset of the plane in `src`
393
- * @param {number} stride bytes per source row
394
- * @returns {{img: Uint8Array, w: number, h: number}}
395
- */
396
- export function downsampleLuma(src, offset, stride, srcW, srcH, factor) {
397
- const f = Math.max(1, factor | 0);
398
- let w = Math.floor(srcW / f);
399
- w -= w & 1; // even, so the SBS halves split cleanly
400
- const h = Math.floor(srcH / f);
401
- const out = new Uint8Array(w * h);
402
- if (f <= 2) {
403
- // Small factors: the full box.
404
- const n = f * f;
405
- for (let y = 0; y < h; y++) {
406
- for (let x = 0; x < w; x++) {
407
- let sum = 0;
408
- for (let dy = 0; dy < f; dy++) {
409
- const o = offset + (y * f + dy) * stride + x * f;
410
- for (let dx = 0; dx < f; dx++) sum += src[o + dx];
411
- }
412
- out[y * w + x] = (sum / n) | 0;
413
- }
414
- }
415
- return { img: out, w, h };
416
- }
417
- // Larger factors: a 2x2 sample at the quarter points of each cell. Enough averaging to keep sensor
418
- // and codec noise from reading as texture, at ~1/9 of the full box's cost for f = 6 (the full box
419
- // over a 2560x720 luma plane was ~8 ms of JS per measurement).
420
- const a = f >> 2;
421
- const c = (3 * f) >> 2;
422
- for (let y = 0; y < h; y++) {
423
- const r0 = offset + (y * f + a) * stride;
424
- const r1 = offset + (y * f + c) * stride;
425
- for (let x = 0; x < w; x++) {
426
- const x0 = x * f + a;
427
- const x1 = x * f + c;
428
- out[y * w + x] = (src[r0 + x0] + src[r0 + x1] + src[r1 + x0] + src[r1 + x1]) >> 2;
429
- }
430
- }
431
- return { img: out, w, h };
432
- }
1
+ // call/disparity.js — moved to js/camera/disparity.js in C2 (RFC 0003 §4: auto-convergence is a
2
+ // camera-side measurement that the call and `addCameraView` share). This path re-exports it so
3
+ // test imports and any in-tree reference keep resolving; new code imports the camera module.
4
+ export * from '../camera/disparity.js';