@smartspectra/node-sdk 3.2.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +13 -0
- package/README.md +409 -0
- package/js/constants.js +125 -0
- package/js/ffi.js +494 -0
- package/js/index.d.ts +282 -0
- package/js/index.js +60 -0
- package/js/main/index.d.ts +32 -0
- package/js/main/index.js +404 -0
- package/js/messages/generated.d.ts +2083 -0
- package/js/messages/generated.js +5810 -0
- package/js/messages/index.d.ts +27 -0
- package/js/messages/index.js +67 -0
- package/js/preload/index.js +116 -0
- package/js/renderer/index.d.ts +154 -0
- package/js/renderer/index.js +670 -0
- package/js/resolve-native.js +113 -0
- package/js/smartspectra.js +293 -0
- package/package.json +81 -0
|
@@ -0,0 +1,670 @@
|
|
|
1
|
+
// js/renderer/index.js
|
|
2
|
+
// Copyright (C) 2026 Presage Technologies, Inc.
|
|
3
|
+
//
|
|
4
|
+
// SPDX-License-Identifier: LicenseRef-Proprietary
|
|
5
|
+
//
|
|
6
|
+
// Renderer-side `SmartSpectraSDK` for Electron applications. Reads
|
|
7
|
+
// VideoFrames directly from the MediaStreamTrack via
|
|
8
|
+
// `MediaStreamTrackProcessor`, draws each frame into an OffscreenCanvas to
|
|
9
|
+
// extract RGBA pixels, and ships the buffer (structured-cloned — Electron's
|
|
10
|
+
// MessagePortMain drops ArrayBuffer transfer lists) to the main-process SDK
|
|
11
|
+
// via the preload bridge. SDK callbacks come back over the same bridge and
|
|
12
|
+
// surface as events on this class.
|
|
13
|
+
//
|
|
14
|
+
// Requires Electron 16+ (Chromium 94+) for MediaStreamTrackProcessor.
|
|
15
|
+
|
|
16
|
+
'use strict';
|
|
17
|
+
|
|
18
|
+
const SUPPORTED_EVENTS = new Set([
|
|
19
|
+
'processingStatus',
|
|
20
|
+
'validationStatus',
|
|
21
|
+
'metrics',
|
|
22
|
+
'accumulatedMetrics',
|
|
23
|
+
'insight',
|
|
24
|
+
'frameSentThrough',
|
|
25
|
+
'error',
|
|
26
|
+
// Fires once the SDK has acquired (or re-acquired, after a Stop) the
|
|
27
|
+
// default MediaStream. Host apps attach this stream to their preview
|
|
28
|
+
// <video> element. Not fired when the host supplies its own stream
|
|
29
|
+
// via useMediaStream() — in that case the host already owns the
|
|
30
|
+
// reference.
|
|
31
|
+
'streamAvailable',
|
|
32
|
+
]);
|
|
33
|
+
|
|
34
|
+
const PIXEL_FORMAT_RGBA = 2; // mirrors PixelFormat.kRGBA in constants.js
|
|
35
|
+
|
|
36
|
+
// kStarting and kRunning enum values from constants.js. Mirrored here so
|
|
37
|
+
// the renderer doesn't need to pull in the constants module for these
|
|
38
|
+
// comparisons.
|
|
39
|
+
const PROCESSING_STATUS_STARTING = 2;
|
|
40
|
+
const PROCESSING_STATUS_RUNNING = 3;
|
|
41
|
+
|
|
42
|
+
// kProcessingFailed error code from constants.js. Used by the startup
|
|
43
|
+
// watchdog when the graph stays stuck at Starting without producing a
|
|
44
|
+
// single validation packet for STARTUP_WATCHDOG_MS.
|
|
45
|
+
const ERROR_CODE_PROCESSING_FAILED = 8;
|
|
46
|
+
const STARTUP_WATCHDOG_MS = 30_000;
|
|
47
|
+
|
|
48
|
+
// Default getUserMedia constraints — 1280x720 at 30 fps with the
|
|
49
|
+
// front-facing camera, matching what the signal pipeline is tuned for.
|
|
50
|
+
// `ideal` qualifiers let the browser fall back gracefully when a webcam
|
|
51
|
+
// doesn't natively support these numbers; the signal processing
|
|
52
|
+
// tolerates moderate deviations.
|
|
53
|
+
const DEFAULT_VIDEO_CONSTRAINTS = Object.freeze({
|
|
54
|
+
width: { ideal: 1280 },
|
|
55
|
+
height: { ideal: 720 },
|
|
56
|
+
frameRate: { ideal: 30 },
|
|
57
|
+
facingMode: 'user',
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
class SmartSpectraSDK {
|
|
61
|
+
/**
|
|
62
|
+
* @param {Object} [options]
|
|
63
|
+
* @param {string} [options.apiKey]
|
|
64
|
+
* @param {number[]} [options.requestedMetrics]
|
|
65
|
+
* @param {boolean} [options.enableAccumulatedOutput=false]
|
|
66
|
+
* @param {number} [options.sendTimeoutMs=30000] Per-call timeout for
|
|
67
|
+
* renderer→main IPC roundtrips (start/stop/reset/requestInsight/configure).
|
|
68
|
+
* If the main process crashes or stalls, the pending call rejects with a
|
|
69
|
+
* `SMARTSPECTRA_IPC_TIMEOUT` error instead of hanging the UI forever.
|
|
70
|
+
* Set to 0 to disable the timeout.
|
|
71
|
+
*/
|
|
72
|
+
constructor(options = {}) {
|
|
73
|
+
if (typeof window === 'undefined' || !window.__smartspectraBridge) {
|
|
74
|
+
throw new Error(
|
|
75
|
+
'@smartspectra/node-sdk/renderer: window.__smartspectraBridge is unavailable. ' +
|
|
76
|
+
'Set webPreferences.preload to require.resolve(\'@smartspectra/node-sdk/preload\') ' +
|
|
77
|
+
'on the BrowserWindow that loads this renderer, and call ' +
|
|
78
|
+
'bindSmartSpectraIpc(window) from the main process.');
|
|
79
|
+
}
|
|
80
|
+
for (const method of ['attach', 'postMessage', 'onMessage', 'close']) {
|
|
81
|
+
if (typeof window.__smartspectraBridge[method] !== 'function') {
|
|
82
|
+
throw new Error(
|
|
83
|
+
`@smartspectra/node-sdk/renderer: bridge is missing ${method}(). ` +
|
|
84
|
+
'The preload script may be out of sync with the renderer module.');
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (typeof MediaStreamTrackProcessor === 'undefined') {
|
|
88
|
+
throw new Error(
|
|
89
|
+
'@smartspectra/node-sdk/renderer: MediaStreamTrackProcessor is unavailable. ' +
|
|
90
|
+
'Requires Electron 16+ (Chromium 94+).');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
this._options = {
|
|
94
|
+
apiKey: options.apiKey ?? '',
|
|
95
|
+
requestedMetrics: options.requestedMetrics,
|
|
96
|
+
enableAccumulatedOutput: !!options.enableAccumulatedOutput,
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
this._listeners = new Map();
|
|
100
|
+
this._processingStatus = 0; // kUninitialized
|
|
101
|
+
|
|
102
|
+
this._stream = null;
|
|
103
|
+
// true when the SDK acquired the stream via getUserMedia and is
|
|
104
|
+
// therefore responsible for stopping its tracks on stop()/destroy().
|
|
105
|
+
// false when the host supplied the stream via useMediaStream() —
|
|
106
|
+
// the host owns the lifecycle in that case.
|
|
107
|
+
this._streamOwned = false;
|
|
108
|
+
this._track = null;
|
|
109
|
+
this._processor = null;
|
|
110
|
+
this._reader = null;
|
|
111
|
+
this._frameLoopActive = false;
|
|
112
|
+
this._frameLoopPromise = null;
|
|
113
|
+
// Monotonic frame-timestamp ratchet. Lives on the instance (not the
|
|
114
|
+
// frame loop) so it persists across stop()/start() cycles: the main
|
|
115
|
+
// process reuses one native session whose per-session monotonic guard
|
|
116
|
+
// is NOT reset by stop/start, so a fresh camera clock starting lower
|
|
117
|
+
// than the previous session's last timestamp would otherwise trip
|
|
118
|
+
// kNonMonotonicTimestamp.
|
|
119
|
+
this._lastFrameTs = 0n;
|
|
120
|
+
// Set true once the AE/AWB/focus lock has been attempted post-Running
|
|
121
|
+
// so we don't try to relock every frame after kRunning.
|
|
122
|
+
this._cameraLocked = false;
|
|
123
|
+
// Startup watchdog: if processingStatus stays at kStarting for
|
|
124
|
+
// STARTUP_WATCHDOG_MS without a single validationStatus packet,
|
|
125
|
+
// we synthesize an 'error' event so host apps can show a real
|
|
126
|
+
// failure instead of an indefinite "Starting…" UI. Cleared when
|
|
127
|
+
// we first see kRunning, the first validationStatus packet, or
|
|
128
|
+
// any other status transition out of kStarting.
|
|
129
|
+
this._startupWatchdog = null;
|
|
130
|
+
|
|
131
|
+
this._bridge = window.__smartspectraBridge;
|
|
132
|
+
// Diagnostic logging is opt-in via SMARTSPECTRA_DIAGNOSTICS=1 in the
|
|
133
|
+
// host process, mirrored onto the bridge by the preload script (this
|
|
134
|
+
// context can't read process.env). Older preload builds lack the
|
|
135
|
+
// property — strict-compare so they resolve to "off".
|
|
136
|
+
this._diagnostics = this._bridge.diagnostics === true;
|
|
137
|
+
this._nextReplyId = 1;
|
|
138
|
+
this._pendingAcks = new Map(); // replyId → { resolve, reject }
|
|
139
|
+
// Per-call IPC ack timeout. A finite, non-negative number overrides the
|
|
140
|
+
// 30s default; 0 disables the timeout entirely.
|
|
141
|
+
const sendTimeoutMs = options.sendTimeoutMs;
|
|
142
|
+
this._sendTimeoutMs = (typeof sendTimeoutMs === 'number'
|
|
143
|
+
&& Number.isFinite(sendTimeoutMs) && sendTimeoutMs >= 0)
|
|
144
|
+
? sendTimeoutMs
|
|
145
|
+
: 30_000;
|
|
146
|
+
|
|
147
|
+
this._bridge.onMessage((data) => this._handleBridgeMessage(data));
|
|
148
|
+
this._bridge.attach();
|
|
149
|
+
|
|
150
|
+
// Configure the main-process SDK eagerly so a frame pushed shortly
|
|
151
|
+
// after `start()` doesn't race against a still-unconstructed
|
|
152
|
+
// SmartSpectraSDK on the main side.
|
|
153
|
+
this._bridge.postMessage({ kind: 'configure', options: this._options });
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** SDK package version. */
|
|
157
|
+
static get version() { return require('../../package.json').version; }
|
|
158
|
+
|
|
159
|
+
/** Current ProcessingStatus integer value. */
|
|
160
|
+
get processingStatus() { return this._processingStatus; }
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Override the default camera with a host-supplied `MediaStream` from
|
|
164
|
+
* `navigator.mediaDevices.getUserMedia`, `desktopCapturer`, a
|
|
165
|
+
* `<canvas>.captureStream()`, etc. Optional — if not called, `start()`
|
|
166
|
+
* acquires its own MediaStream with sensible defaults (front camera,
|
|
167
|
+
* 1280x720 at 30 fps, AE/AWB lock after settle) and emits it via the
|
|
168
|
+
* `'streamAvailable'` event.
|
|
169
|
+
*
|
|
170
|
+
* When the host supplies the stream the host also owns its lifecycle —
|
|
171
|
+
* the SDK never stops the original tracks, only its own internal clone.
|
|
172
|
+
*/
|
|
173
|
+
useMediaStream(stream) {
|
|
174
|
+
if (!(stream instanceof MediaStream)) {
|
|
175
|
+
throw new TypeError('useMediaStream: argument must be a MediaStream');
|
|
176
|
+
}
|
|
177
|
+
this._stream = stream;
|
|
178
|
+
this._streamOwned = false;
|
|
179
|
+
return this;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Begin processing. If no stream has been attached via useMediaStream(),
|
|
184
|
+
* the SDK acquires the default front-facing camera and emits it through
|
|
185
|
+
* `'streamAvailable'` so the host can show a live preview. Resolves once
|
|
186
|
+
* the main-process SDK has armed the graph and is ready to accept frames.
|
|
187
|
+
* On failure, rolls back the renderer-side frame pump and (if the SDK
|
|
188
|
+
* acquired the stream itself) releases the camera so a retry isn't
|
|
189
|
+
* blocked.
|
|
190
|
+
*/
|
|
191
|
+
async start() {
|
|
192
|
+
// Acquire the default camera if the host didn't supply one. The
|
|
193
|
+
// streamAvailable event is dispatched synchronously after acquire
|
|
194
|
+
// so the host can attach it to a <video> element before the frame
|
|
195
|
+
// loop kicks off — that way the preview shows the same warm-up
|
|
196
|
+
// window the signal processor sees.
|
|
197
|
+
if (!this._stream) {
|
|
198
|
+
this._stream = await this._acquireDefaultStream();
|
|
199
|
+
this._streamOwned = true;
|
|
200
|
+
this._emit('streamAvailable', this._stream);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const tracks = this._stream.getVideoTracks();
|
|
204
|
+
if (tracks.length === 0) {
|
|
205
|
+
// If we acquired this stream ourselves, release it before
|
|
206
|
+
// surfacing the error — otherwise the camera stays open and
|
|
207
|
+
// the next start() races against a half-built state.
|
|
208
|
+
if (this._streamOwned) {
|
|
209
|
+
for (const t of this._stream.getTracks()) {
|
|
210
|
+
try { t.stop(); } catch { /* already stopped */ }
|
|
211
|
+
}
|
|
212
|
+
this._stream = null;
|
|
213
|
+
this._streamOwned = false;
|
|
214
|
+
}
|
|
215
|
+
throw new Error('start(): MediaStream has no video tracks');
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Clone the source track so MediaStreamTrackProcessor doesn't starve
|
|
219
|
+
// other consumers of the original stream (e.g. a preview <video>
|
|
220
|
+
// element the host app renders). Both tracks share the underlying
|
|
221
|
+
// camera source and receive identical frames; each can be consumed
|
|
222
|
+
// independently. The clone is stopped in _stopFrameLoop().
|
|
223
|
+
this._track = tracks[0].clone();
|
|
224
|
+
this._processor = new MediaStreamTrackProcessor({ track: this._track });
|
|
225
|
+
this._reader = this._processor.readable.getReader();
|
|
226
|
+
this._cameraLocked = false;
|
|
227
|
+
// Send `start` before arming the frame loop. The main process calls
|
|
228
|
+
// startCustom() inside the `start` handler; any frames posted before
|
|
229
|
+
// that returns would pass the JS guard (_customInputActive=true) but
|
|
230
|
+
// fail the C guard (custom_input=null), producing a misleading error.
|
|
231
|
+
// Frames captured while the ack is in flight are buffered by the reader
|
|
232
|
+
// and drained once the loop starts below.
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
await this._send({ kind: 'start', frameTransform: 0 });
|
|
236
|
+
} catch (err) {
|
|
237
|
+
// _stopFrameLoop is idempotent: cleans up reader/track even if the
|
|
238
|
+
// loop was never started.
|
|
239
|
+
await this._stopFrameLoop();
|
|
240
|
+
this._releaseOwnedStream();
|
|
241
|
+
throw err;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
this._frameLoopActive = true;
|
|
245
|
+
this._frameLoopPromise = this._runFrameLoop();
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Stop processing. Resolves once the main-process SDK has stopped.
|
|
250
|
+
* If the SDK acquired the camera itself, the stream is released here
|
|
251
|
+
* — start() will re-acquire on the next call. Host-supplied streams
|
|
252
|
+
* are left untouched; the host owns their lifecycle.
|
|
253
|
+
*/
|
|
254
|
+
async stop() {
|
|
255
|
+
this._clearStartupWatchdog();
|
|
256
|
+
await this._stopFrameLoop();
|
|
257
|
+
await this._send({ kind: 'stop' });
|
|
258
|
+
this._releaseOwnedStream();
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Rebuild the graph after a kError state. Releases any SDK-acquired
|
|
263
|
+
* camera so the next start() re-runs the acquire + settle sequence
|
|
264
|
+
* from scratch.
|
|
265
|
+
*/
|
|
266
|
+
async reset() {
|
|
267
|
+
this._clearStartupWatchdog();
|
|
268
|
+
await this._stopFrameLoop();
|
|
269
|
+
await this._send({ kind: 'reset' });
|
|
270
|
+
this._releaseOwnedStream();
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Dispatch an on-demand insight prompt. Resolves with the request id
|
|
275
|
+
* assigned to it; the matching Insight response arrives via the
|
|
276
|
+
* `'insight'` event with the same id.
|
|
277
|
+
*/
|
|
278
|
+
async requestInsight(text) {
|
|
279
|
+
if (typeof text !== 'string') {
|
|
280
|
+
throw new TypeError('requestInsight: text must be a string');
|
|
281
|
+
}
|
|
282
|
+
const ack = await this._send({ kind: 'requestInsight', text });
|
|
283
|
+
return ack.requestId | 0;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Tear down the renderer-side resources and instruct the main process
|
|
288
|
+
* to destroy its SDK instance. Idempotent.
|
|
289
|
+
*/
|
|
290
|
+
destroy() {
|
|
291
|
+
if (this._destroyed) return;
|
|
292
|
+
this._destroyed = true;
|
|
293
|
+
this._clearStartupWatchdog();
|
|
294
|
+
// Reject any in-flight IPC roundtrips so awaiters of start()/stop()/
|
|
295
|
+
// reset()/requestInsight() don't hang on a session that's gone (the
|
|
296
|
+
// bridge is closed below and no ack will ever arrive). Typed so
|
|
297
|
+
// cleanup code can distinguish it from a session-level error or an
|
|
298
|
+
// IPC timeout; each pending entry's wrapped reject also clears its
|
|
299
|
+
// own timeout timer. A one-sided bridge disconnect (main-process
|
|
300
|
+
// death) can't be caught here — the DOM MessagePort has no close
|
|
301
|
+
// event — but the per-call _send timeout rejects those waiters.
|
|
302
|
+
const destroyedErr = new Error('SmartSpectraSDK: destroyed before main-process ack');
|
|
303
|
+
destroyedErr.code = 'SMARTSPECTRA_DESTROYED';
|
|
304
|
+
this._rejectPendingAcks(destroyedErr);
|
|
305
|
+
this._frameLoopActive = false;
|
|
306
|
+
if (this._reader) {
|
|
307
|
+
try { this._reader.cancel().catch(() => {}); } catch { /* ignore */ }
|
|
308
|
+
this._reader = null;
|
|
309
|
+
}
|
|
310
|
+
this._processor = null;
|
|
311
|
+
if (this._track) {
|
|
312
|
+
try { this._track.stop(); } catch { /* already stopped */ }
|
|
313
|
+
this._track = null;
|
|
314
|
+
}
|
|
315
|
+
this._releaseOwnedStream();
|
|
316
|
+
this._stream = null;
|
|
317
|
+
try { this._bridge.postMessage({ kind: 'destroy' }); } catch { /* bridge may be dead */ }
|
|
318
|
+
try { this._bridge.close(); } catch { /* already closed */ }
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
on(event, callback) {
|
|
322
|
+
if (typeof event !== 'string' || typeof callback !== 'function') {
|
|
323
|
+
throw new TypeError('on(event: string, callback: Function)');
|
|
324
|
+
}
|
|
325
|
+
if (!SUPPORTED_EVENTS.has(event)) {
|
|
326
|
+
throw new RangeError(`on(): unsupported event ${event}`);
|
|
327
|
+
}
|
|
328
|
+
this._listeners.set(event, callback);
|
|
329
|
+
return this;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// ---------- internal --------------------------------------------------
|
|
333
|
+
|
|
334
|
+
_armStartupWatchdog() {
|
|
335
|
+
// Idempotent: if we're already armed (e.g. duplicate kStarting
|
|
336
|
+
// status events) keep the original timer so the deadline is
|
|
337
|
+
// measured from the first Starting transition, not the latest.
|
|
338
|
+
if (this._startupWatchdog) return;
|
|
339
|
+
this._startupWatchdog = setTimeout(() => {
|
|
340
|
+
this._startupWatchdog = null;
|
|
341
|
+
// Defensive: don't fire if we somehow advanced past Starting
|
|
342
|
+
// in the meantime (status race vs. timer).
|
|
343
|
+
if (this._processingStatus !== PROCESSING_STATUS_STARTING) return;
|
|
344
|
+
const message =
|
|
345
|
+
`Processing stalled at Starting for ${Math.round(STARTUP_WATCHDOG_MS / 1000)} s ` +
|
|
346
|
+
'without producing a validation packet. Common causes: model ' +
|
|
347
|
+
'download blocked by network/firewall, server-side service unavailable, ' +
|
|
348
|
+
'or an SDK-level error that didn\'t propagate through the bridge. ' +
|
|
349
|
+
'Check the main-process terminal for SDK log lines, then retry or fall ' +
|
|
350
|
+
'back to the basic metrics bundle (breathing, pulse rate), which has no ' +
|
|
351
|
+
'additional model-load requirement.';
|
|
352
|
+
this._emit(
|
|
353
|
+
'error',
|
|
354
|
+
ERROR_CODE_PROCESSING_FAILED,
|
|
355
|
+
message,
|
|
356
|
+
/*retryable=*/true);
|
|
357
|
+
}, STARTUP_WATCHDOG_MS);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
_clearStartupWatchdog() {
|
|
361
|
+
if (this._startupWatchdog) {
|
|
362
|
+
clearTimeout(this._startupWatchdog);
|
|
363
|
+
this._startupWatchdog = null;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
async _acquireDefaultStream() {
|
|
368
|
+
if (!navigator || !navigator.mediaDevices ||
|
|
369
|
+
typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
370
|
+
throw new Error(
|
|
371
|
+
'@smartspectra/node-sdk/renderer: navigator.mediaDevices.getUserMedia ' +
|
|
372
|
+
'is unavailable. Set webPreferences.contextIsolation=true and ensure ' +
|
|
373
|
+
'the BrowserWindow has been granted camera permission.');
|
|
374
|
+
}
|
|
375
|
+
try {
|
|
376
|
+
return await navigator.mediaDevices.getUserMedia({
|
|
377
|
+
video: DEFAULT_VIDEO_CONSTRAINTS,
|
|
378
|
+
});
|
|
379
|
+
} catch (err) {
|
|
380
|
+
// Surface the failure with a hint about the likely cause —
|
|
381
|
+
// macOS / Windows prompt the user the first time, and a denial
|
|
382
|
+
// surfaces as NotAllowedError. Chrome also raises NotReadable
|
|
383
|
+
// when another app holds the camera.
|
|
384
|
+
const detail = err && err.name
|
|
385
|
+
? ` (${err.name})`
|
|
386
|
+
: '';
|
|
387
|
+
const e = new Error(
|
|
388
|
+
`Failed to acquire default camera${detail}: ` +
|
|
389
|
+
`${err && err.message ? err.message : err}. ` +
|
|
390
|
+
'Either grant camera permission and retry, or call ' +
|
|
391
|
+
'useMediaStream() with your own MediaStream before start().');
|
|
392
|
+
e.cause = err;
|
|
393
|
+
throw e;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
_releaseOwnedStream() {
|
|
398
|
+
if (!this._streamOwned || !this._stream) return;
|
|
399
|
+
// Stop every track so the OS-level camera indicator turns off and
|
|
400
|
+
// a subsequent getUserMedia call doesn't race against a held
|
|
401
|
+
// device handle. Audio tracks are stopped too even though we only
|
|
402
|
+
// requested video — defensive in case constraints widen later.
|
|
403
|
+
for (const track of this._stream.getTracks()) {
|
|
404
|
+
try { track.stop(); } catch { /* already stopped */ }
|
|
405
|
+
}
|
|
406
|
+
this._stream = null;
|
|
407
|
+
this._streamOwned = false;
|
|
408
|
+
this._cameraLocked = false;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
_tryLockCameraParameters() {
|
|
412
|
+
if (this._cameraLocked || !this._stream) return;
|
|
413
|
+
this._cameraLocked = true; // gate retries even if applyConstraints fails
|
|
414
|
+
|
|
415
|
+
// Lock the ORIGINAL track, not the cloned one we feed into
|
|
416
|
+
// MediaStreamTrackProcessor. The original is what a host-app
|
|
417
|
+
// preview <video> renders; the clone shares the same underlying
|
|
418
|
+
// camera source so the lock applied here propagates.
|
|
419
|
+
const tracks = this._stream.getVideoTracks();
|
|
420
|
+
const track = tracks[0];
|
|
421
|
+
if (!track || typeof track.applyConstraints !== 'function') return;
|
|
422
|
+
|
|
423
|
+
// Probe capabilities first so we only request modes the hardware
|
|
424
|
+
// advertises — many built-in webcams don't expose manual exposure
|
|
425
|
+
// at all, and unsupported constraints are a hard reject. Each
|
|
426
|
+
// advanced entry is an OR, so the browser picks the first the
|
|
427
|
+
// device supports.
|
|
428
|
+
const caps = (typeof track.getCapabilities === 'function')
|
|
429
|
+
? track.getCapabilities() || {}
|
|
430
|
+
: {};
|
|
431
|
+
const advanced = [];
|
|
432
|
+
if (Array.isArray(caps.exposureMode) && caps.exposureMode.includes('manual')) {
|
|
433
|
+
advanced.push({ exposureMode: 'manual' });
|
|
434
|
+
}
|
|
435
|
+
if (Array.isArray(caps.whiteBalanceMode) && caps.whiteBalanceMode.includes('manual')) {
|
|
436
|
+
advanced.push({ whiteBalanceMode: 'manual' });
|
|
437
|
+
}
|
|
438
|
+
if (Array.isArray(caps.focusMode) && caps.focusMode.includes('manual')) {
|
|
439
|
+
advanced.push({ focusMode: 'manual' });
|
|
440
|
+
}
|
|
441
|
+
if (advanced.length === 0) return;
|
|
442
|
+
|
|
443
|
+
// Fire-and-forget — applyConstraints returns a Promise that
|
|
444
|
+
// rejects on unsupported constraints. We swallow the rejection
|
|
445
|
+
// because the graph tolerates auto-modes; the lock is an
|
|
446
|
+
// accuracy improvement, not a correctness requirement.
|
|
447
|
+
track.applyConstraints({ advanced }).catch((err) => {
|
|
448
|
+
console.warn(
|
|
449
|
+
'[smartspectra/renderer] camera lock applyConstraints rejected, ' +
|
|
450
|
+
'continuing in auto-mode:', err && err.message ? err.message : err);
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
async _stopFrameLoop() {
|
|
455
|
+
this._frameLoopActive = false;
|
|
456
|
+
if (this._reader) {
|
|
457
|
+
try { await this._reader.cancel(); } catch { /* already closed */ }
|
|
458
|
+
this._reader = null;
|
|
459
|
+
}
|
|
460
|
+
this._processor = null;
|
|
461
|
+
if (this._track) {
|
|
462
|
+
// Stop the cloned track to release the per-track camera reference.
|
|
463
|
+
// The original stream's track (which the host app may still be
|
|
464
|
+
// rendering as a preview) is unaffected.
|
|
465
|
+
try { this._track.stop(); } catch { /* already stopped */ }
|
|
466
|
+
this._track = null;
|
|
467
|
+
}
|
|
468
|
+
if (this._frameLoopPromise) {
|
|
469
|
+
try { await this._frameLoopPromise; } catch { /* loop errors are logged in-loop */ }
|
|
470
|
+
this._frameLoopPromise = null;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
async _runFrameLoop() {
|
|
475
|
+
const canvas = new OffscreenCanvas(1, 1);
|
|
476
|
+
const ctx = canvas.getContext('2d', { willReadFrequently: true });
|
|
477
|
+
let frameCount = 0;
|
|
478
|
+
|
|
479
|
+
try {
|
|
480
|
+
while (this._frameLoopActive && !this._destroyed) {
|
|
481
|
+
let read;
|
|
482
|
+
try {
|
|
483
|
+
read = await this._reader.read();
|
|
484
|
+
} catch (e) {
|
|
485
|
+
console.warn('[smartspectra/renderer] reader.read() threw:', e);
|
|
486
|
+
break;
|
|
487
|
+
}
|
|
488
|
+
if (read.done) break;
|
|
489
|
+
const vf = read.value;
|
|
490
|
+
if (!vf) continue;
|
|
491
|
+
|
|
492
|
+
try {
|
|
493
|
+
if (this._destroyed) break;
|
|
494
|
+
if (canvas.width !== vf.codedWidth || canvas.height !== vf.codedHeight) {
|
|
495
|
+
canvas.width = vf.codedWidth;
|
|
496
|
+
canvas.height = vf.codedHeight;
|
|
497
|
+
}
|
|
498
|
+
ctx.drawImage(vf, 0, 0);
|
|
499
|
+
const imageData = ctx.getImageData(0, 0, vf.codedWidth, vf.codedHeight);
|
|
500
|
+
|
|
501
|
+
// VideoFrame.timestamp is already in microseconds.
|
|
502
|
+
// Ratchet to guarantee strict monotonicity across stream
|
|
503
|
+
// pauses, re-attaches, and stop()/start() restarts (see
|
|
504
|
+
// _lastFrameTs).
|
|
505
|
+
let ts = BigInt(vf.timestamp || 0);
|
|
506
|
+
if (ts <= this._lastFrameTs) ts = this._lastFrameTs + 1n;
|
|
507
|
+
this._lastFrameTs = ts;
|
|
508
|
+
|
|
509
|
+
const ab = imageData.data.buffer;
|
|
510
|
+
this._bridge.postMessage({
|
|
511
|
+
kind: 'frame',
|
|
512
|
+
buffer: ab,
|
|
513
|
+
width: vf.codedWidth,
|
|
514
|
+
height: vf.codedHeight,
|
|
515
|
+
stride: vf.codedWidth * 4,
|
|
516
|
+
pixelFormat: PIXEL_FORMAT_RGBA,
|
|
517
|
+
timestampUs: Number(ts),
|
|
518
|
+
});
|
|
519
|
+
frameCount++;
|
|
520
|
+
if (this._diagnostics) {
|
|
521
|
+
if (frameCount === 1) {
|
|
522
|
+
console.log(
|
|
523
|
+
`[smartspectra/renderer] first frame forwarded ` +
|
|
524
|
+
`(${vf.codedWidth}x${vf.codedHeight}, ts=${ts}µs)`);
|
|
525
|
+
} else if (frameCount % 60 === 0) {
|
|
526
|
+
console.log(`[smartspectra/renderer] forwarded ${frameCount} frames`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
} finally {
|
|
530
|
+
vf.close();
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
} catch (e) {
|
|
534
|
+
console.error('[smartspectra/renderer] frame loop failed:', e);
|
|
535
|
+
} finally {
|
|
536
|
+
if (this._diagnostics) {
|
|
537
|
+
console.log(
|
|
538
|
+
`[smartspectra/renderer] frame loop exited after ${frameCount} frames`);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
_send(message) {
|
|
544
|
+
const replyId = this._nextReplyId++;
|
|
545
|
+
const timeoutMs = this._sendTimeoutMs;
|
|
546
|
+
return new Promise((resolve, reject) => {
|
|
547
|
+
const onTimeout = () => {
|
|
548
|
+
// delete() returns false if the ack already arrived and cleared
|
|
549
|
+
// the entry — guards against rejecting an already-settled call.
|
|
550
|
+
if (this._pendingAcks.delete(replyId)) {
|
|
551
|
+
const err = new Error(
|
|
552
|
+
`smartspectra: main-process ack for kind=${message && message.kind} ` +
|
|
553
|
+
`timed out after ${timeoutMs}ms`);
|
|
554
|
+
err.code = 'SMARTSPECTRA_IPC_TIMEOUT';
|
|
555
|
+
reject(err);
|
|
556
|
+
}
|
|
557
|
+
};
|
|
558
|
+
const timer = timeoutMs > 0 ? setTimeout(onTimeout, timeoutMs) : null;
|
|
559
|
+
const clear = () => { if (timer !== null) clearTimeout(timer); };
|
|
560
|
+
this._pendingAcks.set(replyId, {
|
|
561
|
+
resolve: (v) => { clear(); resolve(v); },
|
|
562
|
+
reject: (e) => { clear(); reject(e); },
|
|
563
|
+
});
|
|
564
|
+
try {
|
|
565
|
+
this._bridge.postMessage({ ...message, replyId });
|
|
566
|
+
} catch (e) {
|
|
567
|
+
clear();
|
|
568
|
+
this._pendingAcks.delete(replyId);
|
|
569
|
+
reject(e);
|
|
570
|
+
}
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
_rejectPendingAcks(err) {
|
|
575
|
+
// Snapshot then clear: the wrapped reject() doesn't touch the map, so
|
|
576
|
+
// clearing after the loop is safe and avoids mutate-while-iterate.
|
|
577
|
+
const pending = [...this._pendingAcks.values()];
|
|
578
|
+
this._pendingAcks.clear();
|
|
579
|
+
for (const { reject } of pending) reject(err);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
_handleBridgeMessage(msg) {
|
|
583
|
+
if (!msg || typeof msg.kind !== 'string') return;
|
|
584
|
+
switch (msg.kind) {
|
|
585
|
+
case 'processingStatus':
|
|
586
|
+
this._processingStatus = msg.status | 0;
|
|
587
|
+
// Arm the startup watchdog when the SDK reports
|
|
588
|
+
// Starting, and disarm on any subsequent transition.
|
|
589
|
+
// The watchdog catches startup stalls (model load, network
|
|
590
|
+
// setup, similar) that otherwise present as a silent
|
|
591
|
+
// "Starting…" UI forever.
|
|
592
|
+
if (this._processingStatus === PROCESSING_STATUS_STARTING) {
|
|
593
|
+
this._armStartupWatchdog();
|
|
594
|
+
} else {
|
|
595
|
+
this._clearStartupWatchdog();
|
|
596
|
+
}
|
|
597
|
+
// Once the graph reports Running the camera ISP has had
|
|
598
|
+
// long enough to converge — match the mobile SDKs by
|
|
599
|
+
// locking AE/AWB/focus at the converged values so the
|
|
600
|
+
// signal processor sees a stable exposure. Best-effort:
|
|
601
|
+
// capabilities vary across Chromium versions and webcams,
|
|
602
|
+
// and we treat unsupported as graceful degradation
|
|
603
|
+
// (auto-modes keep running, the graph still works).
|
|
604
|
+
if (this._processingStatus === PROCESSING_STATUS_RUNNING) {
|
|
605
|
+
this._tryLockCameraParameters();
|
|
606
|
+
}
|
|
607
|
+
this._emit('processingStatus', this._processingStatus);
|
|
608
|
+
return;
|
|
609
|
+
case 'validationStatus':
|
|
610
|
+
// First validation packet means the graph reached output —
|
|
611
|
+
// disarm the watchdog regardless of the code value (an
|
|
612
|
+
// error code is still progress, the graph isn't wedged).
|
|
613
|
+
this._clearStartupWatchdog();
|
|
614
|
+
this._emit('validationStatus', msg.code | 0, Number(msg.timestampUs), msg.hint || '');
|
|
615
|
+
return;
|
|
616
|
+
case 'metrics':
|
|
617
|
+
this._emit('metrics', bufFrom(msg.buffer), Number(msg.timestampUs));
|
|
618
|
+
return;
|
|
619
|
+
case 'accumulatedMetrics':
|
|
620
|
+
this._emit('accumulatedMetrics', bufFrom(msg.buffer), Number(msg.timestampUs));
|
|
621
|
+
return;
|
|
622
|
+
case 'insight':
|
|
623
|
+
this._emit('insight', bufFrom(msg.buffer), msg.requestId | 0);
|
|
624
|
+
return;
|
|
625
|
+
case 'frameSentThrough':
|
|
626
|
+
this._emit('frameSentThrough', !!msg.sent, Number(msg.timestampUs));
|
|
627
|
+
return;
|
|
628
|
+
case 'error':
|
|
629
|
+
this._emit('error', msg.code | 0, msg.message || '', !!msg.retryable);
|
|
630
|
+
return;
|
|
631
|
+
case 'ack': {
|
|
632
|
+
const pending = this._pendingAcks.get(msg.replyId);
|
|
633
|
+
if (!pending) return;
|
|
634
|
+
this._pendingAcks.delete(msg.replyId);
|
|
635
|
+
if (msg.ok) pending.resolve(msg.payload || {});
|
|
636
|
+
else pending.reject(makeAckError(msg.error));
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
_emit(event, ...args) {
|
|
643
|
+
const cb = this._listeners.get(event);
|
|
644
|
+
if (!cb) return;
|
|
645
|
+
try { cb(...args); }
|
|
646
|
+
catch (e) { console.error(`SmartSpectraSDK: listener for '${event}' threw:`, e); }
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
function bufFrom(arrayBuffer) {
|
|
651
|
+
// Renderer-side: Buffer isn't always available, but Uint8Array is the
|
|
652
|
+
// closest analogue and what most JS protobuf libraries accept.
|
|
653
|
+
return new Uint8Array(arrayBuffer);
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
function makeAckError(err) {
|
|
657
|
+
const e = new Error(err?.message || 'SmartSpectra IPC failure');
|
|
658
|
+
e.code = typeof err?.code === 'number' ? err.code : 0;
|
|
659
|
+
e.retryable = !!err?.retryable;
|
|
660
|
+
return e;
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// Re-export the public value constants alongside the class so the
|
|
664
|
+
// renderer-side consumer can import everything from a single entry point
|
|
665
|
+
// without bundling the koffi/FFI machinery that lives under the main
|
|
666
|
+
// package entry.
|
|
667
|
+
module.exports = Object.assign(
|
|
668
|
+
{ SmartSpectraSDK },
|
|
669
|
+
require('../constants'),
|
|
670
|
+
);
|