mujoco-react 9.2.0 → 9.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +225 -15
- package/dist/{chunk-33CV6HSV.js → chunk-VDSEPZYQ.js} +303 -14
- package/dist/chunk-VDSEPZYQ.js.map +1 -0
- package/dist/index.d.ts +274 -7
- package/dist/index.js +1172 -131
- package/dist/index.js.map +1 -1
- package/dist/spark.d.ts +24 -2
- package/dist/spark.js +89 -3
- package/dist/spark.js.map +1 -1
- package/dist/{types-S8ggQY2n.d.ts → types-BuJ4boaq.d.ts} +160 -5
- package/dist/vite.d.ts +1 -1
- package/dist/vite.js +14 -7
- package/dist/vite.js.map +1 -1
- package/package.json +1 -1
- package/src/components/SplatCollisionProxyPreview.tsx +350 -0
- package/src/components/VisualScenario.tsx +287 -11
- package/src/core/MujocoSimProvider.tsx +374 -30
- package/src/core/SceneLoader.ts +13 -0
- package/src/hooks/useMountedCameraSequenceRecorder.ts +155 -0
- package/src/index.ts +80 -0
- package/src/rendering/cameraFrameCapture.ts +195 -26
- package/src/rendering/cameraFrameSource.ts +747 -0
- package/src/spark.tsx +144 -0
- package/src/types.ts +166 -4
- package/src/vite.ts +14 -6
- package/dist/chunk-33CV6HSV.js.map +0 -1
|
@@ -0,0 +1,747 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
*
|
|
5
|
+
* Helpers for resolving dataset camera streams to mounted MuJoCo resources.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type {
|
|
9
|
+
Bodies,
|
|
10
|
+
CameraFrameCaptureOptions,
|
|
11
|
+
CameraFrameCaptureSource,
|
|
12
|
+
CameraFrameSequenceCamera,
|
|
13
|
+
CameraFrameSequenceOptions,
|
|
14
|
+
CameraFrameSequenceResult,
|
|
15
|
+
Cameras,
|
|
16
|
+
MujocoSimAPI,
|
|
17
|
+
Sites,
|
|
18
|
+
} from '../types';
|
|
19
|
+
|
|
20
|
+
export type MountedCameraFrameCaptureSource = Extract<
|
|
21
|
+
CameraFrameCaptureSource,
|
|
22
|
+
| { kind: 'mujoco-camera' }
|
|
23
|
+
| { kind: 'mujoco-site' }
|
|
24
|
+
| { kind: 'mujoco-body' }
|
|
25
|
+
>;
|
|
26
|
+
|
|
27
|
+
export type CameraFrameMountSelector =
|
|
28
|
+
| { cameraName: Cameras; siteName?: never; bodyName?: never }
|
|
29
|
+
| { siteName: Sites; cameraName?: never; bodyName?: never }
|
|
30
|
+
| { bodyName: Bodies; cameraName?: never; siteName?: never };
|
|
31
|
+
|
|
32
|
+
export interface NamedCameraFrameResource {
|
|
33
|
+
name: string | null | undefined;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface ResolveMountedCameraFrameSourceOptions {
|
|
37
|
+
cameras?: readonly (Cameras | NamedCameraFrameResource | null | undefined)[];
|
|
38
|
+
sites?: readonly (Sites | NamedCameraFrameResource | null | undefined)[];
|
|
39
|
+
bodies?: readonly (Bodies | NamedCameraFrameResource | null | undefined)[];
|
|
40
|
+
aliases?: Record<
|
|
41
|
+
string,
|
|
42
|
+
CameraFrameMountSelector | readonly CameraFrameMountSelector[]
|
|
43
|
+
>;
|
|
44
|
+
/**
|
|
45
|
+
* Accept the first valid alias selector even when the current resource
|
|
46
|
+
* inventory cannot verify it. This is useful when aliases come from a
|
|
47
|
+
* previously validated model inventory and the actual provider will validate
|
|
48
|
+
* again during capture.
|
|
49
|
+
*/
|
|
50
|
+
allowAliasFallback?: boolean;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface ResolvedMountedCameraFrameSource {
|
|
54
|
+
key: string;
|
|
55
|
+
selector: CameraFrameMountSelector;
|
|
56
|
+
source: MountedCameraFrameCaptureSource;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export const MountedCameraFrameSourceSuggestionMatch = {
|
|
60
|
+
Direct: 'direct',
|
|
61
|
+
Alias: 'alias',
|
|
62
|
+
Normalized: 'normalized',
|
|
63
|
+
Prefix: 'prefix',
|
|
64
|
+
Suffix: 'suffix',
|
|
65
|
+
Contains: 'contains',
|
|
66
|
+
} as const;
|
|
67
|
+
|
|
68
|
+
export type MountedCameraFrameSourceSuggestionMatch =
|
|
69
|
+
(typeof MountedCameraFrameSourceSuggestionMatch)[keyof typeof MountedCameraFrameSourceSuggestionMatch];
|
|
70
|
+
|
|
71
|
+
export interface MountedCameraFrameSourceSuggestion {
|
|
72
|
+
key: string;
|
|
73
|
+
selector: CameraFrameMountSelector;
|
|
74
|
+
source: MountedCameraFrameCaptureSource;
|
|
75
|
+
resourceName: string;
|
|
76
|
+
resourceKind: MountedCameraFrameCaptureSource['kind'];
|
|
77
|
+
match: MountedCameraFrameSourceSuggestionMatch;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export type MountedCameraFrameSequenceDefaults = Omit<
|
|
81
|
+
CameraFrameSequenceCamera,
|
|
82
|
+
'key' | 'cameraName' | 'siteName' | 'bodyName' | 'source'
|
|
83
|
+
>;
|
|
84
|
+
|
|
85
|
+
export type MountedCameraFrameSequenceCameraOptions = Partial<
|
|
86
|
+
MountedCameraFrameSequenceDefaults
|
|
87
|
+
>;
|
|
88
|
+
|
|
89
|
+
export interface CreateMountedCameraFrameSequencePlanOptions
|
|
90
|
+
extends ResolveMountedCameraFrameSourceOptions {
|
|
91
|
+
defaults?: MountedCameraFrameSequenceDefaults;
|
|
92
|
+
cameraOptions?: Record<string, MountedCameraFrameSequenceCameraOptions>;
|
|
93
|
+
requireAll?: boolean;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export interface MountedCameraFrameSequencePlan {
|
|
97
|
+
cameraKeys: string[];
|
|
98
|
+
cameras: CameraFrameSequenceCamera[];
|
|
99
|
+
resolved: Record<string, ResolvedMountedCameraFrameSource>;
|
|
100
|
+
missingKeys: string[];
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export const MountedCameraFrameSequenceReadinessStatus = {
|
|
104
|
+
Ready: 'ready',
|
|
105
|
+
Partial: 'partial',
|
|
106
|
+
Missing: 'missing',
|
|
107
|
+
} as const;
|
|
108
|
+
|
|
109
|
+
export type MountedCameraFrameSequenceReadinessStatus =
|
|
110
|
+
(typeof MountedCameraFrameSequenceReadinessStatus)[keyof typeof MountedCameraFrameSequenceReadinessStatus];
|
|
111
|
+
|
|
112
|
+
export interface MountedCameraFrameSequenceSourceReadiness {
|
|
113
|
+
key: string;
|
|
114
|
+
ready: boolean;
|
|
115
|
+
selector?: CameraFrameMountSelector;
|
|
116
|
+
source?: MountedCameraFrameCaptureSource;
|
|
117
|
+
message: string;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export interface MountedCameraFrameSequenceReadiness {
|
|
121
|
+
ready: boolean;
|
|
122
|
+
status: MountedCameraFrameSequenceReadinessStatus;
|
|
123
|
+
cameraKeys: string[];
|
|
124
|
+
resolvedKeys: string[];
|
|
125
|
+
missingKeys: string[];
|
|
126
|
+
cameras: Record<string, MountedCameraFrameSequenceSourceReadiness>;
|
|
127
|
+
message: string;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export type MountedCameraFrameSequencePlanOptions = Omit<
|
|
131
|
+
CreateMountedCameraFrameSequencePlanOptions,
|
|
132
|
+
'cameras' | 'sites' | 'bodies'
|
|
133
|
+
>;
|
|
134
|
+
|
|
135
|
+
export interface MountedCameraFrameSequenceRecordOptions
|
|
136
|
+
extends Omit<CameraFrameSequenceOptions, 'cameras'>,
|
|
137
|
+
MountedCameraFrameSequencePlanOptions {
|
|
138
|
+
cameraKeys: readonly string[];
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export interface MountedCameraFrameSequenceRecordResult
|
|
142
|
+
extends CameraFrameSequenceResult {
|
|
143
|
+
plan: MountedCameraFrameSequencePlan;
|
|
144
|
+
readiness: MountedCameraFrameSequenceReadiness;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export const MountedCameraFrameSequenceManifestStatus = {
|
|
148
|
+
Complete: 'complete',
|
|
149
|
+
Partial: 'partial',
|
|
150
|
+
Missing: 'missing',
|
|
151
|
+
} as const;
|
|
152
|
+
|
|
153
|
+
export type MountedCameraFrameSequenceManifestStatus =
|
|
154
|
+
(typeof MountedCameraFrameSequenceManifestStatus)[keyof typeof MountedCameraFrameSequenceManifestStatus];
|
|
155
|
+
|
|
156
|
+
export interface MountedCameraFrameSequenceStreamSummary {
|
|
157
|
+
key: string;
|
|
158
|
+
ready: boolean;
|
|
159
|
+
complete: boolean;
|
|
160
|
+
status: MountedCameraFrameSequenceManifestStatus;
|
|
161
|
+
source?: CameraFrameCaptureSource;
|
|
162
|
+
selector?: CameraFrameMountSelector;
|
|
163
|
+
target?: string;
|
|
164
|
+
width?: number;
|
|
165
|
+
height?: number;
|
|
166
|
+
expectedFrameCount: number;
|
|
167
|
+
recordedFrameCount: number;
|
|
168
|
+
missingFrameCount: number;
|
|
169
|
+
firstFrameIndex: number | null;
|
|
170
|
+
lastFrameIndex: number | null;
|
|
171
|
+
firstTimestamp: number | null;
|
|
172
|
+
lastTimestamp: number | null;
|
|
173
|
+
message: string;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export interface CreateMountedCameraFrameSequenceManifestOptions {
|
|
177
|
+
expectedFrameCount?: number;
|
|
178
|
+
cameraKeys?: readonly string[];
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export interface MountedCameraFrameSequenceManifest {
|
|
182
|
+
schema: 'mujoco-react/mounted-camera-frame-sequence-manifest@1';
|
|
183
|
+
ready: boolean;
|
|
184
|
+
complete: boolean;
|
|
185
|
+
status: MountedCameraFrameSequenceManifestStatus;
|
|
186
|
+
cameraKeys: string[];
|
|
187
|
+
resolvedKeys: string[];
|
|
188
|
+
missingKeys: string[];
|
|
189
|
+
expectedFrameCount: number;
|
|
190
|
+
recordedFrameCount: number;
|
|
191
|
+
missingFrameCount: number;
|
|
192
|
+
streamSummaries: Record<string, MountedCameraFrameSequenceStreamSummary>;
|
|
193
|
+
streams: MountedCameraFrameSequenceStreamSummary[];
|
|
194
|
+
readiness: MountedCameraFrameSequenceReadiness;
|
|
195
|
+
message: string;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export type MountedCameraFrameSequenceRecorderTarget = Pick<
|
|
199
|
+
MujocoSimAPI,
|
|
200
|
+
'getCameras' | 'getSites' | 'getBodies' | 'recordCameraSequence'
|
|
201
|
+
>;
|
|
202
|
+
|
|
203
|
+
function getResourceName(
|
|
204
|
+
resource: string | NamedCameraFrameResource | null | undefined
|
|
205
|
+
) {
|
|
206
|
+
if (!resource) return null;
|
|
207
|
+
return typeof resource === 'string' ? resource : resource.name ?? null;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function createNameSet(
|
|
211
|
+
resources:
|
|
212
|
+
| readonly (string | NamedCameraFrameResource | null | undefined)[]
|
|
213
|
+
| undefined
|
|
214
|
+
) {
|
|
215
|
+
return new Set(
|
|
216
|
+
(resources ?? [])
|
|
217
|
+
.map((resource) => getResourceName(resource))
|
|
218
|
+
.filter((name): name is string => Boolean(name))
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function createResourceNames(
|
|
223
|
+
resources:
|
|
224
|
+
| readonly (string | NamedCameraFrameResource | null | undefined)[]
|
|
225
|
+
| undefined
|
|
226
|
+
) {
|
|
227
|
+
return (resources ?? [])
|
|
228
|
+
.map((resource) => getResourceName(resource))
|
|
229
|
+
.filter((name): name is string => Boolean(name));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function normalizeAliasCandidates(
|
|
233
|
+
value: CameraFrameMountSelector | readonly CameraFrameMountSelector[] | undefined
|
|
234
|
+
) {
|
|
235
|
+
if (!value) return [];
|
|
236
|
+
return Array.isArray(value) ? value : [value];
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function countMountedSelectors(selector: CameraFrameMountSelector) {
|
|
240
|
+
return Number(Boolean(selector.cameraName)) +
|
|
241
|
+
Number(Boolean(selector.siteName)) +
|
|
242
|
+
Number(Boolean(selector.bodyName));
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function normalizeCameraSourceName(value: string) {
|
|
246
|
+
return value
|
|
247
|
+
.trim()
|
|
248
|
+
.toLowerCase()
|
|
249
|
+
.replace(/[^a-z0-9]+/g, '_')
|
|
250
|
+
.replace(/^_+|_+$/g, '');
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function createCameraSourceKeyVariants(key: string) {
|
|
254
|
+
const candidates = [
|
|
255
|
+
key,
|
|
256
|
+
key.startsWith('observation.images.')
|
|
257
|
+
? key.slice('observation.images.'.length)
|
|
258
|
+
: '',
|
|
259
|
+
key.includes('.') ? key.split('.').at(-1) ?? '' : '',
|
|
260
|
+
key.includes('/') ? key.split('/').at(-1) ?? '' : '',
|
|
261
|
+
];
|
|
262
|
+
return candidates
|
|
263
|
+
.map((candidate) => candidate.trim())
|
|
264
|
+
.filter((candidate, index, items) => candidate && items.indexOf(candidate) === index);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function getSelectorKey(selector: CameraFrameMountSelector) {
|
|
268
|
+
if (selector.cameraName) return `camera:${selector.cameraName}`;
|
|
269
|
+
if (selector.siteName) return `site:${selector.siteName}`;
|
|
270
|
+
if (selector.bodyName) return `body:${selector.bodyName}`;
|
|
271
|
+
return null;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
export function getMountedCameraFrameCaptureSource(
|
|
275
|
+
selector: CameraFrameMountSelector
|
|
276
|
+
): MountedCameraFrameCaptureSource | null {
|
|
277
|
+
if (countMountedSelectors(selector) !== 1) return null;
|
|
278
|
+
if (selector.cameraName) {
|
|
279
|
+
return { kind: 'mujoco-camera', cameraName: selector.cameraName };
|
|
280
|
+
}
|
|
281
|
+
if (selector.siteName) {
|
|
282
|
+
return { kind: 'mujoco-site', siteName: selector.siteName };
|
|
283
|
+
}
|
|
284
|
+
if (selector.bodyName) {
|
|
285
|
+
return { kind: 'mujoco-body', bodyName: selector.bodyName };
|
|
286
|
+
}
|
|
287
|
+
return null;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export function isMountedCameraFrameCaptureSource(
|
|
291
|
+
source: CameraFrameCaptureSource
|
|
292
|
+
): source is MountedCameraFrameCaptureSource {
|
|
293
|
+
return (
|
|
294
|
+
source.kind === 'mujoco-camera' ||
|
|
295
|
+
source.kind === 'mujoco-site' ||
|
|
296
|
+
source.kind === 'mujoco-body'
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
export function getCameraFrameCaptureSourceTarget(
|
|
301
|
+
source: CameraFrameCaptureSource
|
|
302
|
+
) {
|
|
303
|
+
if (source.kind === 'mujoco-camera') return source.cameraName;
|
|
304
|
+
if (source.kind === 'mujoco-site') return source.siteName;
|
|
305
|
+
if (source.kind === 'mujoco-body') return source.bodyName;
|
|
306
|
+
if (source.kind === 'custom-camera') return 'custom camera';
|
|
307
|
+
if (source.kind === 'explicit-pose') return 'explicit pose';
|
|
308
|
+
return 'fallback camera';
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function createMountedCameraFrameSourceSuggestion(
|
|
312
|
+
key: string,
|
|
313
|
+
selector: CameraFrameMountSelector,
|
|
314
|
+
resourceName: string,
|
|
315
|
+
match: MountedCameraFrameSourceSuggestionMatch
|
|
316
|
+
): MountedCameraFrameSourceSuggestion | null {
|
|
317
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
318
|
+
if (!source) return null;
|
|
319
|
+
return {
|
|
320
|
+
key,
|
|
321
|
+
selector,
|
|
322
|
+
source,
|
|
323
|
+
resourceName,
|
|
324
|
+
resourceKind: source.kind,
|
|
325
|
+
match,
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function addMountedCameraFrameSourceSuggestion(
|
|
330
|
+
suggestions: MountedCameraFrameSourceSuggestion[],
|
|
331
|
+
seen: Set<string>,
|
|
332
|
+
suggestion: MountedCameraFrameSourceSuggestion | null
|
|
333
|
+
) {
|
|
334
|
+
if (!suggestion) return;
|
|
335
|
+
const selectorKey = getSelectorKey(suggestion.selector);
|
|
336
|
+
if (!selectorKey || seen.has(selectorKey)) return;
|
|
337
|
+
seen.add(selectorKey);
|
|
338
|
+
suggestions.push(suggestion);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function getCameraFrameResourceMatch(
|
|
342
|
+
key: string,
|
|
343
|
+
resourceName: string
|
|
344
|
+
): MountedCameraFrameSourceSuggestionMatch | null {
|
|
345
|
+
if (resourceName === key) return MountedCameraFrameSourceSuggestionMatch.Direct;
|
|
346
|
+
|
|
347
|
+
const normalizedResource = normalizeCameraSourceName(resourceName);
|
|
348
|
+
if (!normalizedResource) return null;
|
|
349
|
+
|
|
350
|
+
for (const variant of createCameraSourceKeyVariants(key)) {
|
|
351
|
+
if (resourceName === variant) return MountedCameraFrameSourceSuggestionMatch.Direct;
|
|
352
|
+
|
|
353
|
+
const normalizedKey = normalizeCameraSourceName(variant);
|
|
354
|
+
if (!normalizedKey) continue;
|
|
355
|
+
if (normalizedResource === normalizedKey) {
|
|
356
|
+
return MountedCameraFrameSourceSuggestionMatch.Normalized;
|
|
357
|
+
}
|
|
358
|
+
if (normalizedResource.startsWith(`${normalizedKey}_`)) {
|
|
359
|
+
return MountedCameraFrameSourceSuggestionMatch.Prefix;
|
|
360
|
+
}
|
|
361
|
+
if (normalizedResource.endsWith(`_${normalizedKey}`)) {
|
|
362
|
+
return MountedCameraFrameSourceSuggestionMatch.Suffix;
|
|
363
|
+
}
|
|
364
|
+
if (normalizedResource.includes(`_${normalizedKey}_`)) {
|
|
365
|
+
return MountedCameraFrameSourceSuggestionMatch.Contains;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
return null;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function isSelectorMounted(
|
|
372
|
+
selector: CameraFrameMountSelector,
|
|
373
|
+
cameraNames: Set<string>,
|
|
374
|
+
siteNames: Set<string>,
|
|
375
|
+
bodyNames: Set<string>
|
|
376
|
+
) {
|
|
377
|
+
if (countMountedSelectors(selector) !== 1) return false;
|
|
378
|
+
return (
|
|
379
|
+
(selector.cameraName ? cameraNames.has(selector.cameraName) : false) ||
|
|
380
|
+
(selector.siteName ? siteNames.has(selector.siteName) : false) ||
|
|
381
|
+
(selector.bodyName ? bodyNames.has(selector.bodyName) : false)
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
export function createMountedCameraFrameSourceSuggestions(
|
|
386
|
+
key: string,
|
|
387
|
+
options: ResolveMountedCameraFrameSourceOptions
|
|
388
|
+
): MountedCameraFrameSourceSuggestion[] {
|
|
389
|
+
const cameraNames = createNameSet(options.cameras);
|
|
390
|
+
const siteNames = createNameSet(options.sites);
|
|
391
|
+
const bodyNames = createNameSet(options.bodies);
|
|
392
|
+
const suggestions: MountedCameraFrameSourceSuggestion[] = [];
|
|
393
|
+
const seen = new Set<string>();
|
|
394
|
+
|
|
395
|
+
for (const selector of normalizeAliasCandidates(options.aliases?.[key])) {
|
|
396
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
400
|
+
if (!source) continue;
|
|
401
|
+
addMountedCameraFrameSourceSuggestion(
|
|
402
|
+
suggestions,
|
|
403
|
+
seen,
|
|
404
|
+
createMountedCameraFrameSourceSuggestion(
|
|
405
|
+
key,
|
|
406
|
+
selector,
|
|
407
|
+
getCameraFrameCaptureSourceTarget(source),
|
|
408
|
+
MountedCameraFrameSourceSuggestionMatch.Alias
|
|
409
|
+
)
|
|
410
|
+
);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
for (const cameraName of createResourceNames(options.cameras)) {
|
|
414
|
+
const match = getCameraFrameResourceMatch(key, cameraName);
|
|
415
|
+
if (!match) continue;
|
|
416
|
+
addMountedCameraFrameSourceSuggestion(
|
|
417
|
+
suggestions,
|
|
418
|
+
seen,
|
|
419
|
+
createMountedCameraFrameSourceSuggestion(
|
|
420
|
+
key,
|
|
421
|
+
{ cameraName },
|
|
422
|
+
cameraName,
|
|
423
|
+
match
|
|
424
|
+
)
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
for (const siteName of createResourceNames(options.sites)) {
|
|
429
|
+
const match = getCameraFrameResourceMatch(key, siteName);
|
|
430
|
+
if (!match) continue;
|
|
431
|
+
addMountedCameraFrameSourceSuggestion(
|
|
432
|
+
suggestions,
|
|
433
|
+
seen,
|
|
434
|
+
createMountedCameraFrameSourceSuggestion(
|
|
435
|
+
key,
|
|
436
|
+
{ siteName },
|
|
437
|
+
siteName,
|
|
438
|
+
match
|
|
439
|
+
)
|
|
440
|
+
);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
for (const bodyName of createResourceNames(options.bodies)) {
|
|
444
|
+
const match = getCameraFrameResourceMatch(key, bodyName);
|
|
445
|
+
if (!match) continue;
|
|
446
|
+
addMountedCameraFrameSourceSuggestion(
|
|
447
|
+
suggestions,
|
|
448
|
+
seen,
|
|
449
|
+
createMountedCameraFrameSourceSuggestion(
|
|
450
|
+
key,
|
|
451
|
+
{ bodyName },
|
|
452
|
+
bodyName,
|
|
453
|
+
match
|
|
454
|
+
)
|
|
455
|
+
);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
return suggestions;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
export function resolveMountedCameraFrameSource(
|
|
462
|
+
key: string,
|
|
463
|
+
options: ResolveMountedCameraFrameSourceOptions
|
|
464
|
+
): ResolvedMountedCameraFrameSource | null {
|
|
465
|
+
const cameraNames = createNameSet(options.cameras);
|
|
466
|
+
const siteNames = createNameSet(options.sites);
|
|
467
|
+
const bodyNames = createNameSet(options.bodies);
|
|
468
|
+
const directCandidates: CameraFrameMountSelector[] = [
|
|
469
|
+
{ cameraName: key },
|
|
470
|
+
{ siteName: key },
|
|
471
|
+
{ bodyName: key },
|
|
472
|
+
];
|
|
473
|
+
const aliasCandidates = normalizeAliasCandidates(options.aliases?.[key]);
|
|
474
|
+
const candidates = [...directCandidates, ...aliasCandidates];
|
|
475
|
+
|
|
476
|
+
for (const selector of candidates) {
|
|
477
|
+
if (!isSelectorMounted(selector, cameraNames, siteNames, bodyNames)) {
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
481
|
+
if (!source) continue;
|
|
482
|
+
return { key, selector, source };
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
const [suggestion] = createMountedCameraFrameSourceSuggestions(key, options);
|
|
486
|
+
if (suggestion) {
|
|
487
|
+
return {
|
|
488
|
+
key,
|
|
489
|
+
selector: suggestion.selector,
|
|
490
|
+
source: suggestion.source,
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
if (options.allowAliasFallback) {
|
|
495
|
+
for (const selector of aliasCandidates) {
|
|
496
|
+
const source = getMountedCameraFrameCaptureSource(selector);
|
|
497
|
+
if (!source) continue;
|
|
498
|
+
return { key, selector, source };
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
return null;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
export function createMountedCameraFrameSequencePlan(
|
|
506
|
+
cameraKeys: readonly string[],
|
|
507
|
+
options: CreateMountedCameraFrameSequencePlanOptions
|
|
508
|
+
): MountedCameraFrameSequencePlan {
|
|
509
|
+
const cameras: CameraFrameSequenceCamera[] = [];
|
|
510
|
+
const resolved: Record<string, ResolvedMountedCameraFrameSource> = {};
|
|
511
|
+
const missingKeys: string[] = [];
|
|
512
|
+
|
|
513
|
+
for (const key of cameraKeys) {
|
|
514
|
+
const mountedSource = resolveMountedCameraFrameSource(key, options);
|
|
515
|
+
if (!mountedSource) {
|
|
516
|
+
missingKeys.push(key);
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
resolved[key] = mountedSource;
|
|
521
|
+
cameras.push({
|
|
522
|
+
key,
|
|
523
|
+
...options.defaults,
|
|
524
|
+
...options.cameraOptions?.[key],
|
|
525
|
+
...mountedSource.selector,
|
|
526
|
+
source: mountedSource.source,
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
if (options.requireAll && missingKeys.length > 0) {
|
|
531
|
+
throw new Error(
|
|
532
|
+
`Unable to resolve mounted MuJoCo camera source${
|
|
533
|
+
missingKeys.length === 1 ? '' : 's'
|
|
534
|
+
} for ${missingKeys.join(', ')}.`
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
return {
|
|
539
|
+
cameraKeys: [...cameraKeys],
|
|
540
|
+
cameras,
|
|
541
|
+
resolved,
|
|
542
|
+
missingKeys,
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
export function createMountedCameraFrameSequenceReadiness(
|
|
547
|
+
plan: MountedCameraFrameSequencePlan
|
|
548
|
+
): MountedCameraFrameSequenceReadiness {
|
|
549
|
+
const cameras: Record<string, MountedCameraFrameSequenceSourceReadiness> = {};
|
|
550
|
+
const resolvedKeys = plan.cameraKeys.filter((key) => Boolean(plan.resolved[key]));
|
|
551
|
+
|
|
552
|
+
for (const key of plan.cameraKeys) {
|
|
553
|
+
const resolved = plan.resolved[key];
|
|
554
|
+
cameras[key] = resolved
|
|
555
|
+
? {
|
|
556
|
+
key,
|
|
557
|
+
ready: true,
|
|
558
|
+
selector: resolved.selector,
|
|
559
|
+
source: resolved.source,
|
|
560
|
+
message: `Camera stream "${key}" resolves to ${resolved.source.kind}:${getCameraFrameCaptureSourceTarget(resolved.source)}.`,
|
|
561
|
+
}
|
|
562
|
+
: {
|
|
563
|
+
key,
|
|
564
|
+
ready: false,
|
|
565
|
+
message: `Camera stream "${key}" does not resolve to a mounted MuJoCo camera, site, or body.`,
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
const missingKeys = [...plan.missingKeys];
|
|
570
|
+
const ready = missingKeys.length === 0;
|
|
571
|
+
const status: MountedCameraFrameSequenceReadinessStatus = ready
|
|
572
|
+
? MountedCameraFrameSequenceReadinessStatus.Ready
|
|
573
|
+
: resolvedKeys.length > 0
|
|
574
|
+
? MountedCameraFrameSequenceReadinessStatus.Partial
|
|
575
|
+
: MountedCameraFrameSequenceReadinessStatus.Missing;
|
|
576
|
+
|
|
577
|
+
return {
|
|
578
|
+
ready,
|
|
579
|
+
status,
|
|
580
|
+
cameraKeys: [...plan.cameraKeys],
|
|
581
|
+
resolvedKeys,
|
|
582
|
+
missingKeys,
|
|
583
|
+
cameras,
|
|
584
|
+
message: ready
|
|
585
|
+
? `All ${plan.cameraKeys.length} requested camera stream${
|
|
586
|
+
plan.cameraKeys.length === 1 ? '' : 's'
|
|
587
|
+
} resolve to mounted MuJoCo sources.`
|
|
588
|
+
: `Missing mounted MuJoCo source${
|
|
589
|
+
missingKeys.length === 1 ? '' : 's'
|
|
590
|
+
} for ${missingKeys.join(', ')}.`,
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
function normalizeFrameCount(frameCount: number | undefined) {
|
|
595
|
+
return Number.isFinite(frameCount) && frameCount !== undefined
|
|
596
|
+
? Math.max(0, Math.floor(frameCount))
|
|
597
|
+
: 0;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
export function createMountedCameraFrameSequenceManifest(
|
|
601
|
+
result: MountedCameraFrameSequenceRecordResult,
|
|
602
|
+
options: CreateMountedCameraFrameSequenceManifestOptions = {}
|
|
603
|
+
): MountedCameraFrameSequenceManifest {
|
|
604
|
+
const cameraKeys = [
|
|
605
|
+
...(options.cameraKeys ??
|
|
606
|
+
result.readiness.cameraKeys ??
|
|
607
|
+
result.plan.cameraKeys ??
|
|
608
|
+
result.cameraKeys),
|
|
609
|
+
];
|
|
610
|
+
const expectedFrameCount = normalizeFrameCount(
|
|
611
|
+
options.expectedFrameCount ?? result.frameCount
|
|
612
|
+
);
|
|
613
|
+
const recordedFrameCount = normalizeFrameCount(result.frameCount);
|
|
614
|
+
const streamSummaries: Record<string, MountedCameraFrameSequenceStreamSummary> =
|
|
615
|
+
{};
|
|
616
|
+
const streams: MountedCameraFrameSequenceStreamSummary[] = [];
|
|
617
|
+
let missingFrameCount = 0;
|
|
618
|
+
let completeStreamCount = 0;
|
|
619
|
+
let resolvedOrRecordedStreamCount = 0;
|
|
620
|
+
|
|
621
|
+
for (const key of cameraKeys) {
|
|
622
|
+
const summary = result.cameraSummaries[key];
|
|
623
|
+
const readiness = result.readiness.cameras[key];
|
|
624
|
+
const source = summary?.source ?? readiness?.source;
|
|
625
|
+
const ready = readiness?.ready ?? Boolean(summary);
|
|
626
|
+
const recorded = normalizeFrameCount(summary?.frameCount);
|
|
627
|
+
const missing = Math.max(expectedFrameCount - recorded, 0);
|
|
628
|
+
const complete = ready && missing === 0;
|
|
629
|
+
const status = complete
|
|
630
|
+
? MountedCameraFrameSequenceManifestStatus.Complete
|
|
631
|
+
: ready || recorded > 0
|
|
632
|
+
? MountedCameraFrameSequenceManifestStatus.Partial
|
|
633
|
+
: MountedCameraFrameSequenceManifestStatus.Missing;
|
|
634
|
+
const target = source
|
|
635
|
+
? getCameraFrameCaptureSourceTarget(source)
|
|
636
|
+
: readiness?.message
|
|
637
|
+
? undefined
|
|
638
|
+
: 'missing MuJoCo camera';
|
|
639
|
+
const message = complete
|
|
640
|
+
? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${
|
|
641
|
+
expectedFrameCount === 1 ? '' : 's'
|
|
642
|
+
}.`
|
|
643
|
+
: ready || recorded > 0
|
|
644
|
+
? `Camera stream "${key}" recorded ${recorded} of ${expectedFrameCount} frame${
|
|
645
|
+
expectedFrameCount === 1 ? '' : 's'
|
|
646
|
+
}.`
|
|
647
|
+
: readiness?.message ??
|
|
648
|
+
`Camera stream "${key}" did not record any frames.`;
|
|
649
|
+
const stream = {
|
|
650
|
+
key,
|
|
651
|
+
ready,
|
|
652
|
+
complete,
|
|
653
|
+
status,
|
|
654
|
+
source,
|
|
655
|
+
selector: readiness?.selector,
|
|
656
|
+
target,
|
|
657
|
+
width: summary?.width,
|
|
658
|
+
height: summary?.height,
|
|
659
|
+
expectedFrameCount,
|
|
660
|
+
recordedFrameCount: recorded,
|
|
661
|
+
missingFrameCount: missing,
|
|
662
|
+
firstFrameIndex: summary?.firstFrameIndex ?? null,
|
|
663
|
+
lastFrameIndex: summary?.lastFrameIndex ?? null,
|
|
664
|
+
firstTimestamp: summary?.firstTimestamp ?? null,
|
|
665
|
+
lastTimestamp: summary?.lastTimestamp ?? null,
|
|
666
|
+
message,
|
|
667
|
+
};
|
|
668
|
+
|
|
669
|
+
streamSummaries[key] = stream;
|
|
670
|
+
streams.push(stream);
|
|
671
|
+
missingFrameCount += missing;
|
|
672
|
+
if (complete) completeStreamCount += 1;
|
|
673
|
+
if (ready || recorded > 0) resolvedOrRecordedStreamCount += 1;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
const complete =
|
|
677
|
+
result.readiness.ready &&
|
|
678
|
+
streams.length === completeStreamCount &&
|
|
679
|
+
missingFrameCount === 0;
|
|
680
|
+
const status = complete
|
|
681
|
+
? MountedCameraFrameSequenceManifestStatus.Complete
|
|
682
|
+
: resolvedOrRecordedStreamCount > 0
|
|
683
|
+
? MountedCameraFrameSequenceManifestStatus.Partial
|
|
684
|
+
: MountedCameraFrameSequenceManifestStatus.Missing;
|
|
685
|
+
|
|
686
|
+
return {
|
|
687
|
+
schema: 'mujoco-react/mounted-camera-frame-sequence-manifest@1',
|
|
688
|
+
ready: result.readiness.ready,
|
|
689
|
+
complete,
|
|
690
|
+
status,
|
|
691
|
+
cameraKeys,
|
|
692
|
+
resolvedKeys: [...result.readiness.resolvedKeys],
|
|
693
|
+
missingKeys: [...result.readiness.missingKeys],
|
|
694
|
+
expectedFrameCount,
|
|
695
|
+
recordedFrameCount,
|
|
696
|
+
missingFrameCount,
|
|
697
|
+
streamSummaries,
|
|
698
|
+
streams,
|
|
699
|
+
readiness: result.readiness,
|
|
700
|
+
message: complete
|
|
701
|
+
? `All ${cameraKeys.length} camera stream${
|
|
702
|
+
cameraKeys.length === 1 ? '' : 's'
|
|
703
|
+
} recorded ${expectedFrameCount} frame${
|
|
704
|
+
expectedFrameCount === 1 ? '' : 's'
|
|
705
|
+
}.`
|
|
706
|
+
: `Mounted camera sequence coverage is ${status}.`,
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
export function createMountedCameraFrameSequencePlanFromApi(
|
|
711
|
+
api: MountedCameraFrameSequenceRecorderTarget,
|
|
712
|
+
cameraKeys: readonly string[],
|
|
713
|
+
options: MountedCameraFrameSequencePlanOptions = {}
|
|
714
|
+
): MountedCameraFrameSequencePlan {
|
|
715
|
+
return createMountedCameraFrameSequencePlan(cameraKeys, {
|
|
716
|
+
...options,
|
|
717
|
+
cameras: api.getCameras(),
|
|
718
|
+
sites: api.getSites(),
|
|
719
|
+
bodies: api.getBodies(),
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
export async function recordMountedCameraFrameSequence(
|
|
724
|
+
api: MountedCameraFrameSequenceRecorderTarget,
|
|
725
|
+
options: MountedCameraFrameSequenceRecordOptions
|
|
726
|
+
): Promise<MountedCameraFrameSequenceRecordResult> {
|
|
727
|
+
const { cameraKeys, ...restOptions } = options;
|
|
728
|
+
const requireAll =
|
|
729
|
+
restOptions.requireAll ?? restOptions.requireMountedSources ?? true;
|
|
730
|
+
const plan = createMountedCameraFrameSequencePlanFromApi(
|
|
731
|
+
api,
|
|
732
|
+
cameraKeys,
|
|
733
|
+
{ ...restOptions, requireAll }
|
|
734
|
+
);
|
|
735
|
+
const readiness = createMountedCameraFrameSequenceReadiness(plan);
|
|
736
|
+
const result = await api.recordCameraSequence({
|
|
737
|
+
...restOptions,
|
|
738
|
+
cameras: plan.cameras,
|
|
739
|
+
requireMountedSources: restOptions.requireMountedSources ?? true,
|
|
740
|
+
});
|
|
741
|
+
|
|
742
|
+
return {
|
|
743
|
+
...result,
|
|
744
|
+
plan,
|
|
745
|
+
readiness,
|
|
746
|
+
};
|
|
747
|
+
}
|