@viji-dev/sdk 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +155 -60
  2. package/bin/viji.js +9 -29
  3. package/dist/assets/artist-dts-BHUsvSI6.js +613 -0
  4. package/dist/assets/artist-dts-p5-Cyw8vmy_.js +736 -0
  5. package/dist/assets/core-CiQx3w0t.js +12 -0
  6. package/dist/assets/dark-plus-C3mMm8J8.js +1 -0
  7. package/dist/assets/docs-api-PBLtY4Ni.js +12381 -0
  8. package/dist/assets/engine-javascript-CXyY7cc8.js +141 -0
  9. package/dist/assets/essentia-wasm.web-0S-sW98u-CYV1l1zv.js +38 -0
  10. package/dist/assets/essentia.js-core.es-DnrJE0uR-DOSrF5_G.js +32 -0
  11. package/dist/assets/glsl-DMyvO4G4.js +1 -0
  12. package/dist/assets/index-BhFxsauQ.js +215 -0
  13. package/dist/assets/index-BqhVeA7U.css +1 -0
  14. package/dist/assets/index-T4TOjvD0.js +1 -0
  15. package/dist/assets/index-Wz9WqGqz.js +52 -0
  16. package/dist/assets/index-t24aGwla.js +1 -0
  17. package/dist/assets/javascript-wDzz0qaB.js +1 -0
  18. package/dist/assets/shader-uniforms-GdaUkQPK.js +1 -0
  19. package/dist/assets/typescript-BPQ3VLAy.js +1 -0
  20. package/dist/assets/viji.worker-CQSJ0SiO-ljtBlcNZ.js +27018 -0
  21. package/{index.html → dist/index.html} +2 -1
  22. package/package.json +31 -35
  23. package/src/cli/commands/build.js +50 -99
  24. package/src/cli/commands/create.js +32 -47
  25. package/src/cli/commands/dev.js +30 -97
  26. package/src/cli/server/dev-server.js +233 -0
  27. package/src/cli/server/scene-scanner.js +93 -0
  28. package/src/cli/server/vite-scene-plugin.d.ts +2 -0
  29. package/src/cli/server/vite-scene-plugin.js +134 -0
  30. package/src/cli/utils/cli-utils.js +29 -139
  31. package/src/cli/utils/scene-compiler.js +10 -17
  32. package/src/templates/scene-templates.js +85 -0
  33. package/.gitignore +0 -29
  34. package/eslint.config.js +0 -37
  35. package/postcss.config.js +0 -6
  36. package/scenes/audio-visualizer/main.js +0 -287
  37. package/scenes/core-demo/main.js +0 -532
  38. package/scenes/demo-scene/main.js +0 -619
  39. package/scenes/global.d.ts +0 -15
  40. package/scenes/particle-system/main.js +0 -349
  41. package/scenes/tsconfig.json +0 -12
  42. package/scenes/video-mirror/main.ts +0 -436
  43. package/src/App.css +0 -42
  44. package/src/App.tsx +0 -279
  45. package/src/cli/commands/init.js +0 -262
  46. package/src/components/SDKPage.tsx +0 -337
  47. package/src/components/core/CoreContainer.tsx +0 -126
  48. package/src/components/ui/DeviceSelectionList.tsx +0 -137
  49. package/src/components/ui/FPSCounter.tsx +0 -78
  50. package/src/components/ui/FileDropzonePanel.tsx +0 -120
  51. package/src/components/ui/FileListPanel.tsx +0 -285
  52. package/src/components/ui/InputExpansionPanel.tsx +0 -31
  53. package/src/components/ui/MediaPlayerControls.tsx +0 -191
  54. package/src/components/ui/MenuContainer.tsx +0 -71
  55. package/src/components/ui/ParametersMenu.tsx +0 -797
  56. package/src/components/ui/ProjectSwitcherMenu.tsx +0 -192
  57. package/src/components/ui/QuickInputControls.tsx +0 -542
  58. package/src/components/ui/SDKMenuSystem.tsx +0 -96
  59. package/src/components/ui/SettingsMenu.tsx +0 -346
  60. package/src/components/ui/SimpleInputControls.tsx +0 -137
  61. package/src/index.css +0 -68
  62. package/src/main.tsx +0 -10
  63. package/src/scenes-hmr.ts +0 -158
  64. package/src/services/project-filesystem.ts +0 -436
  65. package/src/stores/scene-player/index.ts +0 -3
  66. package/src/stores/scene-player/input-manager.store.ts +0 -1045
  67. package/src/stores/scene-player/scene-session.store.ts +0 -659
  68. package/src/styles/globals.css +0 -111
  69. package/src/templates/minimal-template.js +0 -11
  70. package/src/utils/debounce.js +0 -34
  71. package/src/vite-env.d.ts +0 -1
  72. package/tailwind.config.js +0 -18
  73. package/tsconfig.app.json +0 -27
  74. package/tsconfig.json +0 -27
  75. package/tsconfig.node.json +0 -27
  76. package/vite.config.ts +0 -54
  77. /package/{public → dist}/favicon.png +0 -0
@@ -1,1045 +0,0 @@
1
- import { create } from 'zustand';
2
- import { persist, createJSONStorage } from 'zustand/middleware';
3
-
4
- // Input Source Types
5
- export enum AudioInputType {
6
- NONE = 'none',
7
- MICROPHONE = 'microphone',
8
- SCREEN_AUDIO = 'screen_audio',
9
- FILES = 'files'
10
- }
11
-
12
- export enum VideoInputType {
13
- NONE = 'none',
14
- CAMERA = 'camera',
15
- SCREEN_VIDEO = 'screen_video',
16
- FILES = 'files'
17
- }
18
-
19
- // File Input Types
20
- export interface AudioFile {
21
- id: string;
22
- name: string;
23
- url: string;
24
- size: number;
25
- type: string;
26
- duration?: number;
27
- }
28
-
29
- export interface VideoFile {
30
- id: string;
31
- name: string;
32
- url: string;
33
- size: number;
34
- type: string;
35
- duration?: number;
36
- width?: number;
37
- height?: number;
38
- }
39
-
40
- // Player State Types
41
- export interface AudioPlayerState {
42
- isPlaying: boolean;
43
- isPaused: boolean;
44
- currentTime: number;
45
- duration: number;
46
- volume: number;
47
- isMuted: boolean;
48
- playbackRate: number;
49
- currentTrackId?: string;
50
- }
51
-
52
- export interface VideoPlayerState {
53
- isPlaying: boolean;
54
- isPaused: boolean;
55
- currentTime: number;
56
- duration: number;
57
- volume: number;
58
- isMuted: boolean;
59
- playbackRate: number;
60
- currentFileId?: string;
61
- }
62
-
63
- export interface AudioInputConfig {
64
- type: AudioInputType;
65
- enabled: boolean;
66
-
67
- // Device Input (Microphone)
68
- deviceId?: string;
69
- deviceLabel?: string;
70
- volume: number;
71
- isMuted: boolean;
72
-
73
- // File Input
74
- files: AudioFile[];
75
- playerState: AudioPlayerState;
76
-
77
- // Screen Audio
78
- screenAudioEnabled: boolean;
79
-
80
- // Generated Stream
81
- stream?: MediaStream;
82
- }
83
-
84
- export interface VideoInputConfig {
85
- type: VideoInputType;
86
- enabled: boolean;
87
-
88
- // Device Input (Camera)
89
- deviceId?: string;
90
- deviceLabel?: string;
91
- resolution?: { width: number; height: number };
92
-
93
- // File Input
94
- files: VideoFile[];
95
- playerState: VideoPlayerState;
96
-
97
- // Screen Video
98
- screenVideoEnabled: boolean;
99
-
100
- // Generated Stream
101
- stream?: MediaStream;
102
- }
103
-
104
- export interface InputPermissions {
105
- audio: boolean;
106
- video: boolean;
107
- screenCapture: boolean;
108
- }
109
-
110
- export interface InputConfiguration {
111
- audio: AudioInputConfig;
112
- video: VideoInputConfig;
113
- permissions: InputPermissions;
114
- interactionEnabled: boolean;
115
- }
116
-
117
- // Input Manager Store State
118
- interface InputManagerState {
119
- inputConfiguration: InputConfiguration;
120
-
121
- // Available Devices
122
- availableDevices: {
123
- audioDevices: MediaDeviceInfo[];
124
- videoDevices: MediaDeviceInfo[];
125
- };
126
-
127
- // Permission Management
128
- isRequestingPermissions: boolean;
129
- lastPermissionError: string | null;
130
-
131
- // File Management
132
- isLoadingFiles: boolean;
133
- fileLoadError: string | null;
134
-
135
- // Stream Management
136
- isCreatingStream: boolean;
137
- streamError: string | null;
138
-
139
- // Player Management
140
- audioElement?: HTMLAudioElement;
141
- videoElement?: HTMLVideoElement;
142
- canvasElement?: HTMLCanvasElement;
143
- imageAnimationFrameId?: number;
144
-
145
- // Device Fallback Management
146
- previousWorkingAudioDevice: {
147
- deviceId?: string;
148
- deviceLabel?: string;
149
- } | null;
150
- previousWorkingVideoDevice: {
151
- deviceId?: string;
152
- deviceLabel?: string;
153
- } | null;
154
- }
155
-
156
- // Input Manager Store Actions
157
- interface InputManagerActions {
158
- // Configuration Management
159
- setInputConfiguration: (config: InputConfiguration) => void;
160
- updateAudioConfig: (config: Partial<AudioInputConfig>) => void;
161
- updateVideoConfig: (config: Partial<VideoInputConfig>) => void;
162
- updatePermissions: (permissions: Partial<InputPermissions>) => void;
163
-
164
- // Audio Input Management
165
- setAudioInputType: (type: AudioInputType) => Promise<void>;
166
- selectAudioDevice: (deviceId: string, deviceLabel: string) => Promise<void>;
167
- setAudioVolume: (volume: number) => void;
168
- setAudioMuted: (muted: boolean) => void;
169
- enableScreenAudio: () => Promise<void>;
170
-
171
- // Video Input Management
172
- setVideoInputType: (type: VideoInputType) => Promise<void>;
173
- selectVideoDevice: (deviceId: string, deviceLabel: string) => Promise<void>;
174
- setVideoResolution: (width: number, height: number) => void;
175
- enableScreenVideo: () => Promise<void>;
176
-
177
- // File Management
178
- addAudioFiles: (files: File[]) => Promise<void>;
179
- removeAudioFile: (fileId: string) => void;
180
- addVideoFiles: (files: File[]) => Promise<void>;
181
- removeVideoFile: (fileId: string) => void;
182
- clearAllFiles: () => void;
183
-
184
- // Audio Player Controls
185
- playAudio: (trackId?: string) => Promise<void>;
186
- pauseAudio: () => void;
187
- stopAudio: () => void;
188
- nextTrack: () => void;
189
- previousTrack: () => void;
190
- seekAudio: (time: number) => void;
191
- setPlaybackRate: (rate: number) => void;
192
-
193
- // Video Player Controls
194
- playVideo: (fileId?: string) => Promise<void>;
195
- pauseVideo: () => void;
196
- stopVideo: () => void;
197
- seekVideo: (time: number) => void;
198
- setVideoPlaybackRate: (rate: number) => void;
199
-
200
- // Device Management
201
- setAvailableDevices: (devices: { audioDevices: MediaDeviceInfo[], videoDevices: MediaDeviceInfo[] }) => void;
202
- refreshAvailableDevices: () => Promise<void>;
203
- syncDevicesFromActiveStreams: () => void;
204
-
205
- // Permission Management
206
- requestPermissions: (audio?: boolean, video?: boolean) => Promise<boolean>;
207
- setPermissionRequesting: (requesting: boolean) => void;
208
-
209
- // Stream Management
210
- createAudioStream: () => Promise<void>;
211
- createVideoStream: () => Promise<void>;
212
- updateStreamsForCore: () => Promise<{ audioStream?: MediaStream; videoStream?: MediaStream }>;
213
- clearStreams: () => void;
214
-
215
- // Interaction Management
216
- setInteractionEnabled: (enabled: boolean) => void;
217
-
218
- // Error Handling
219
- setPermissionError: (error: string) => void;
220
- setFileLoadError: (error: string) => void;
221
- setStreamError: (error: string | null) => void;
222
- clearErrors: () => void;
223
-
224
- // Device Fallback Management
225
- storePreviousWorkingAudioDevice: (deviceId?: string, deviceLabel?: string) => void;
226
- storePreviousWorkingVideoDevice: (deviceId?: string, deviceLabel?: string) => void;
227
- revertToWorkingAudioDevice: () => Promise<boolean>;
228
- revertToWorkingVideoDevice: () => Promise<boolean>;
229
-
230
- // Utility Methods
231
- hasAudioPermission: () => boolean;
232
- hasVideoPermission: () => boolean;
233
- hasScreenCapturePermission: () => boolean;
234
- hasActiveAudioStream: () => boolean;
235
- hasActiveVideoStream: () => boolean;
236
- resetToDefaults: () => void;
237
- }
238
-
239
- // Combined Store Type
240
- type InputManagerStore = InputManagerState & InputManagerActions;
241
-
242
- // Default configurations
243
- const createDefaultAudioConfig = (): AudioInputConfig => ({
244
- type: AudioInputType.NONE,
245
- enabled: false,
246
- volume: 1.0,
247
- isMuted: false,
248
- files: [],
249
- playerState: {
250
- isPlaying: false,
251
- isPaused: false,
252
- currentTime: 0,
253
- duration: 0,
254
- volume: 1.0,
255
- isMuted: false,
256
- playbackRate: 1.0,
257
- },
258
- screenAudioEnabled: false,
259
- });
260
-
261
- const createDefaultVideoConfig = (): VideoInputConfig => ({
262
- type: VideoInputType.NONE,
263
- enabled: false,
264
- files: [],
265
- playerState: {
266
- isPlaying: false,
267
- isPaused: false,
268
- currentTime: 0,
269
- duration: 0,
270
- volume: 1.0,
271
- isMuted: false,
272
- playbackRate: 1.0,
273
- },
274
- screenVideoEnabled: false,
275
- });
276
-
277
- const createDefaultInputConfiguration = (): InputConfiguration => ({
278
- audio: createDefaultAudioConfig(),
279
- video: createDefaultVideoConfig(),
280
- permissions: {
281
- audio: false,
282
- video: false,
283
- screenCapture: false,
284
- },
285
- interactionEnabled: true,
286
- });
287
-
288
- // Initial State
289
- const initialState: InputManagerState = {
290
- inputConfiguration: createDefaultInputConfiguration(),
291
- availableDevices: {
292
- audioDevices: [],
293
- videoDevices: [],
294
- },
295
- isRequestingPermissions: false,
296
- lastPermissionError: null,
297
- isLoadingFiles: false,
298
- fileLoadError: null,
299
- isCreatingStream: false,
300
- streamError: null,
301
- previousWorkingAudioDevice: null,
302
- previousWorkingVideoDevice: null,
303
- };
304
-
305
- // Create Zustand Store with Persistence
306
- export const useInputManagerStore = create<InputManagerStore>()(
307
- persist(
308
- (set, get) => ({
309
- // Initial State
310
- ...initialState,
311
-
312
- // Configuration Management Actions
313
- setInputConfiguration: (inputConfiguration: InputConfiguration) => {
314
- set({ inputConfiguration });
315
- },
316
-
317
- updateAudioConfig: (config: Partial<AudioInputConfig>) => {
318
- const { inputConfiguration } = get();
319
- set({
320
- inputConfiguration: {
321
- ...inputConfiguration,
322
- audio: {
323
- ...inputConfiguration.audio,
324
- ...config,
325
- },
326
- },
327
- });
328
- },
329
-
330
- updateVideoConfig: (config: Partial<VideoInputConfig>) => {
331
- const { inputConfiguration } = get();
332
- set({
333
- inputConfiguration: {
334
- ...inputConfiguration,
335
- video: {
336
- ...inputConfiguration.video,
337
- ...config,
338
- },
339
- },
340
- });
341
- },
342
-
343
- updatePermissions: (permissions: Partial<InputPermissions>) => {
344
- const { inputConfiguration } = get();
345
- set({
346
- inputConfiguration: {
347
- ...inputConfiguration,
348
- permissions: {
349
- ...inputConfiguration.permissions,
350
- ...permissions,
351
- },
352
- },
353
- });
354
- },
355
-
356
- // Audio Input Management (simplified implementations)
357
- setAudioInputType: async (type: AudioInputType) => {
358
- get().stopAudio();
359
- get().updateAudioConfig({ stream: undefined });
360
- get().updateAudioConfig({
361
- type,
362
- enabled: type !== AudioInputType.NONE,
363
- screenAudioEnabled: type === AudioInputType.SCREEN_AUDIO
364
- });
365
-
366
- if (type === AudioInputType.SCREEN_AUDIO) {
367
- await get().enableScreenAudio();
368
- const { useSceneSessionStore } = await import('./scene-session.store');
369
- const { updateCoreConfig } = useSceneSessionStore.getState();
370
- await updateCoreConfig({ audioStream: get().inputConfiguration.audio.stream });
371
- return;
372
- }
373
-
374
- if (type !== AudioInputType.NONE) {
375
- await get().createAudioStream();
376
- }
377
- },
378
-
379
- selectAudioDevice: async (deviceId: string, deviceLabel: string) => {
380
- console.debug(`🎤 [SDK INPUT] selectAudioDevice`, { deviceId, deviceLabel });
381
- get().updateAudioConfig({ deviceId, deviceLabel });
382
-
383
- if (get().inputConfiguration.audio.type === AudioInputType.MICROPHONE) {
384
- await get().createAudioStream();
385
- }
386
- },
387
-
388
- setAudioVolume: (volume: number) => {
389
- // Clamp and persist
390
- const clamped = Math.max(0, Math.min(1, volume));
391
- get().updateAudioConfig({ volume: clamped });
392
- // Apply to active audio element
393
- const el = get().audioElement;
394
- if (el) {
395
- el.volume = clamped;
396
- }
397
- },
398
-
399
- setAudioMuted: (isMuted: boolean) => {
400
- get().updateAudioConfig({ isMuted });
401
- const el = get().audioElement;
402
- if (el) {
403
- el.muted = isMuted;
404
- }
405
- },
406
-
407
- enableScreenAudio: async () => {
408
- try {
409
- set({ isCreatingStream: true, streamError: null });
410
- // Must include video:true to get audio in some browsers
411
- const screen = await (navigator.mediaDevices as any).getDisplayMedia({ audio: true, video: true });
412
- const audioTracks = screen.getAudioTracks();
413
- if (audioTracks.length === 0) {
414
- // Stop video tracks and error out
415
- screen.getVideoTracks().forEach((t: MediaStreamTrack) => t.stop());
416
- throw new Error('No audio available from screen share. Ensure "Share audio" is enabled.');
417
- }
418
- const audioOnly = new MediaStream(audioTracks);
419
- // Stop video tracks (we only need audio here)
420
- screen.getVideoTracks().forEach((t: MediaStreamTrack) => t.stop());
421
- get().updateAudioConfig({ stream: audioOnly, screenAudioEnabled: true });
422
- // Apply to core
423
- const { useSceneSessionStore } = await import('./scene-session.store');
424
- await useSceneSessionStore.getState().updateCoreConfig({ audioStream: audioOnly });
425
- } catch (err) {
426
- const msg = err instanceof Error ? err.message : 'Failed to capture screen audio';
427
- set({ streamError: msg });
428
- } finally {
429
- set({ isCreatingStream: false });
430
- }
431
- },
432
-
433
- // Video Input Management
434
- setVideoInputType: async (type: VideoInputType) => {
435
- get().stopVideo();
436
- get().updateVideoConfig({ stream: undefined });
437
- get().updateVideoConfig({
438
- type,
439
- enabled: type !== VideoInputType.NONE
440
- });
441
-
442
- if (type !== VideoInputType.NONE) {
443
- await get().createVideoStream();
444
- }
445
- },
446
-
447
- selectVideoDevice: async (deviceId: string, deviceLabel: string) => {
448
- console.debug(`📹 [SDK INPUT] selectVideoDevice`, { deviceId, deviceLabel });
449
- get().updateVideoConfig({ deviceId, deviceLabel });
450
-
451
- if (get().inputConfiguration.video.type === VideoInputType.CAMERA) {
452
- await get().createVideoStream();
453
- }
454
- },
455
-
456
- setVideoResolution: (width: number, height: number) => {
457
- get().updateVideoConfig({ resolution: { width, height } });
458
- },
459
-
460
- enableScreenVideo: async () => {
461
- console.debug('📺 [SDK INPUT] enableScreenVideo');
462
- get().updateVideoConfig({ screenVideoEnabled: true });
463
- },
464
-
465
- // File Management (simplified implementations)
466
- addAudioFiles: async (files: File[]) => {
467
- console.debug('🎵 [SDK INPUT] addAudioFiles', { count: files.length });
468
- const audioFiles: AudioFile[] = files.map(file => ({
469
- id: crypto.randomUUID(),
470
- name: file.name,
471
- url: URL.createObjectURL(file),
472
- size: file.size,
473
- type: file.type,
474
- }));
475
- const { inputConfiguration } = get();
476
- get().updateAudioConfig({ files: [...inputConfiguration.audio.files, ...audioFiles] });
477
- },
478
-
479
- removeAudioFile: (fileId: string) => {
480
- const { inputConfiguration } = get();
481
- const updatedFiles = inputConfiguration.audio.files.filter(f => f.id !== fileId);
482
- get().updateAudioConfig({ files: updatedFiles });
483
- },
484
-
485
- addVideoFiles: async (files: File[]) => {
486
- console.debug('🎬 [SDK INPUT] addVideoFiles', { count: files.length });
487
- const videoFiles: VideoFile[] = files.map(file => ({
488
- id: crypto.randomUUID(),
489
- name: file.name,
490
- url: URL.createObjectURL(file),
491
- size: file.size,
492
- type: file.type,
493
- }));
494
- const { inputConfiguration } = get();
495
- get().updateVideoConfig({ files: [...inputConfiguration.video.files, ...videoFiles] });
496
- },
497
-
498
- removeVideoFile: (fileId: string) => {
499
- const { inputConfiguration } = get();
500
- const updatedFiles = inputConfiguration.video.files.filter(f => f.id !== fileId);
501
- get().updateVideoConfig({ files: updatedFiles });
502
- },
503
-
504
- clearAllFiles: () => {
505
- get().updateAudioConfig({ files: [] });
506
- get().updateVideoConfig({ files: [] });
507
- },
508
-
509
- // Player controls
510
- playAudio: async (trackId?: string) => {
511
- try {
512
- const { inputConfiguration } = get();
513
- const files = inputConfiguration.audio.files;
514
- if (files.length === 0) return;
515
- const id = trackId || inputConfiguration.audio.playerState.currentTrackId || files[0].id;
516
- const file = files.find(f => f.id === id) || files[0];
517
-
518
- // Create or reuse audio element
519
- let el = get().audioElement;
520
- if (!el) {
521
- el = new Audio();
522
- el.crossOrigin = 'anonymous';
523
- set({ audioElement: el });
524
- }
525
-
526
- // Wire up events to keep store in sync
527
- el.onloadedmetadata = () => {
528
- // Update duration on file item and player state
529
- const state = get();
530
- const list = state.inputConfiguration.audio.files.map(f => (
531
- f.id === file.id ? { ...f, duration: el!.duration } : f
532
- ));
533
- state.updateAudioConfig({ files: list, playerState: { ...state.inputConfiguration.audio.playerState, duration: el!.duration } });
534
- };
535
- el.ontimeupdate = () => {
536
- const state = get();
537
- state.updateAudioConfig({ playerState: { ...state.inputConfiguration.audio.playerState, currentTime: el!.currentTime } });
538
- };
539
- el.onended = () => {
540
- try { get().nextTrack(); } catch {}
541
- };
542
-
543
- // Apply settings
544
- el.src = file.url;
545
- el.volume = inputConfiguration.audio.volume;
546
- el.muted = inputConfiguration.audio.isMuted;
547
- el.playbackRate = inputConfiguration.audio.playerState.playbackRate;
548
-
549
- await el.play();
550
-
551
- // Update player state
552
- get().updateAudioConfig({ playerState: { ...inputConfiguration.audio.playerState, currentTrackId: file.id, isPlaying: true, isPaused: false } });
553
-
554
- // Create/update stream and push to core
555
- await get().createAudioStream();
556
- const { audioStream } = await get().updateStreamsForCore();
557
- const { useSceneSessionStore } = await import('./scene-session.store');
558
- await useSceneSessionStore.getState().updateCoreConfig({ audioStream });
559
- } catch (error) {
560
- console.error('Failed to play audio:', error);
561
- }
562
- },
563
-
564
- pauseAudio: () => {
565
- const el = get().audioElement;
566
- if (el) {
567
- el.pause();
568
- }
569
- const ps = get().inputConfiguration.audio.playerState;
570
- get().updateAudioConfig({ playerState: { ...ps, isPlaying: false, isPaused: true } });
571
- },
572
-
573
- stopAudio: () => {
574
- const el = get().audioElement;
575
- if (el) {
576
- el.pause();
577
- el.currentTime = 0;
578
- }
579
- const ps = get().inputConfiguration.audio.playerState;
580
- get().updateAudioConfig({ playerState: { ...ps, isPlaying: false, isPaused: false, currentTime: 0 } });
581
- },
582
-
583
- nextTrack: () => {
584
- const audio = get().inputConfiguration.audio;
585
- if (audio.files.length === 0) return;
586
- const idx = audio.files.findIndex(f => f.id === audio.playerState.currentTrackId);
587
- const nextIdx = idx >= 0 ? (idx + 1) % audio.files.length : 0;
588
- get().playAudio(audio.files[nextIdx].id);
589
- },
590
-
591
- previousTrack: () => {
592
- const audio = get().inputConfiguration.audio;
593
- if (audio.files.length === 0) return;
594
- const idx = audio.files.findIndex(f => f.id === audio.playerState.currentTrackId);
595
- const prevIdx = idx > 0 ? idx - 1 : audio.files.length - 1;
596
- get().playAudio(audio.files[prevIdx].id);
597
- },
598
-
599
- seekAudio: (time: number) => {
600
- const el = get().audioElement;
601
- if (el) {
602
- el.currentTime = Math.max(0, Math.min(Number.isFinite(el.duration) ? el.duration : time, time));
603
- }
604
- const ps = get().inputConfiguration.audio.playerState;
605
- get().updateAudioConfig({ playerState: { ...ps, currentTime: time } });
606
- },
607
-
608
- setPlaybackRate: (rate: number) => {
609
- const el = get().audioElement;
610
- if (el) {
611
- el.playbackRate = rate;
612
- }
613
- const ps = get().inputConfiguration.audio.playerState;
614
- get().updateAudioConfig({ playerState: { ...ps, playbackRate: rate } });
615
- },
616
-
617
- // Video player controls
618
- playVideo: async (fileId?: string) => {
619
- try {
620
- const { inputConfiguration } = get();
621
- const files = inputConfiguration.video.files;
622
- if (files.length === 0) return;
623
- const id = fileId || inputConfiguration.video.playerState.currentFileId || files[0].id;
624
- const file = files.find(f => f.id === id) || files[0];
625
-
626
- // Create or reuse video element
627
- let el = get().videoElement;
628
- if (!el) {
629
- el = document.createElement('video');
630
- el.playsInline = true;
631
- set({ videoElement: el });
632
- }
633
-
634
- // Wire up events
635
- el.onloadedmetadata = () => {
636
- const state = get();
637
- const list = state.inputConfiguration.video.files.map(f => (
638
- f.id === file.id ? { ...f, duration: el!.duration } : f
639
- ));
640
- state.updateVideoConfig({ files: list, playerState: { ...state.inputConfiguration.video.playerState, duration: el!.duration } });
641
- };
642
- el.ontimeupdate = () => {
643
- const state = get();
644
- state.updateVideoConfig({ playerState: { ...state.inputConfiguration.video.playerState, currentTime: el!.currentTime } });
645
- };
646
-
647
- // Apply settings
648
- el.src = file.url;
649
- el.muted = true; // avoid autoplay restrictions
650
- el.loop = true;
651
- el.playbackRate = inputConfiguration.video.playerState.playbackRate;
652
-
653
- await el.play();
654
-
655
- // Update state
656
- get().updateVideoConfig({ playerState: { ...inputConfiguration.video.playerState, currentFileId: file.id, isPlaying: true, isPaused: false } });
657
-
658
- // Create/update stream and push to core
659
- await get().createVideoStream();
660
- const { videoStream } = await get().updateStreamsForCore();
661
- const { useSceneSessionStore } = await import('./scene-session.store');
662
- await useSceneSessionStore.getState().updateCoreConfig({ videoStream });
663
- } catch (error) {
664
- console.error('Failed to play video:', error);
665
- }
666
- },
667
-
668
- pauseVideo: () => {
669
- const el = get().videoElement;
670
- if (el) {
671
- el.pause();
672
- }
673
- const ps = get().inputConfiguration.video.playerState;
674
- get().updateVideoConfig({ playerState: { ...ps, isPlaying: false, isPaused: true } });
675
- },
676
-
677
- stopVideo: () => {
678
- const el = get().videoElement;
679
- if (el) {
680
- el.pause();
681
- el.currentTime = 0;
682
- }
683
- const ps = get().inputConfiguration.video.playerState;
684
- get().updateVideoConfig({ playerState: { ...ps, isPlaying: false, isPaused: false, currentTime: 0 } });
685
- },
686
-
687
- seekVideo: (time: number) => {
688
- const el = get().videoElement;
689
- if (el) {
690
- el.currentTime = Math.max(0, Math.min(Number.isFinite(el.duration) ? el.duration : time, time));
691
- }
692
- const ps = get().inputConfiguration.video.playerState;
693
- get().updateVideoConfig({ playerState: { ...ps, currentTime: time } });
694
- },
695
-
696
- setVideoPlaybackRate: (rate: number) => {
697
- const el = get().videoElement;
698
- if (el) {
699
- el.playbackRate = rate;
700
- }
701
- const ps = get().inputConfiguration.video.playerState;
702
- get().updateVideoConfig({ playerState: { ...ps, playbackRate: rate } });
703
- },
704
-
705
- // Device Management
706
- setAvailableDevices: (devices) => {
707
- set({ availableDevices: devices });
708
- },
709
-
710
- refreshAvailableDevices: async () => {
711
- try {
712
- const devices = await navigator.mediaDevices.enumerateDevices();
713
- const audioDevices = devices.filter(device => device.kind === 'audioinput');
714
- const videoDevices = devices.filter(device => device.kind === 'videoinput');
715
-
716
- get().setAvailableDevices({ audioDevices, videoDevices });
717
- } catch (error) {
718
- console.error('Failed to enumerate devices:', error);
719
- }
720
- },
721
-
722
- syncDevicesFromActiveStreams: () => {
723
- console.debug('🔄 [SDK INPUT] syncDevicesFromActiveStreams');
724
- },
725
-
726
- // Permission Management
727
- requestPermissions: async (audio = true, video = true) => {
728
- set({ isRequestingPermissions: true, lastPermissionError: null });
729
-
730
- try {
731
- const constraints: MediaStreamConstraints = {};
732
- if (audio) constraints.audio = true;
733
- if (video) constraints.video = true;
734
-
735
- const stream = await navigator.mediaDevices.getUserMedia(constraints);
736
-
737
- // Stop the stream immediately - we just wanted permissions
738
- stream.getTracks().forEach(track => track.stop());
739
-
740
- get().updatePermissions({
741
- audio: audio,
742
- video: video,
743
- });
744
-
745
- // Refresh available devices after getting permissions
746
- await get().refreshAvailableDevices();
747
-
748
- return true;
749
- } catch (error) {
750
- const errorMessage = error instanceof Error ? error.message : 'Permission denied';
751
- get().setPermissionError(errorMessage);
752
- console.error('Permission request failed:', error);
753
- return false;
754
- } finally {
755
- set({ isRequestingPermissions: false });
756
- }
757
- },
758
-
759
- setPermissionRequesting: (isRequestingPermissions: boolean) => {
760
- set({ isRequestingPermissions });
761
- },
762
-
763
- // Stream Management (simplified)
764
- createAudioStream: async () => {
765
- const { inputConfiguration } = get();
766
- const audioConfig = inputConfiguration.audio;
767
-
768
- if (audioConfig.type === AudioInputType.MICROPHONE && audioConfig.deviceId) {
769
- try {
770
- set({ isCreatingStream: true, streamError: null });
771
-
772
- const stream = await navigator.mediaDevices.getUserMedia({
773
- audio: { deviceId: { exact: audioConfig.deviceId } }
774
- });
775
-
776
- get().updateAudioConfig({ stream });
777
- console.debug('🎤 [SDK INPUT] Audio stream created');
778
-
779
- } catch (error) {
780
- const errorMessage = error instanceof Error ? error.message : 'Failed to create audio stream';
781
- get().setStreamError(errorMessage);
782
- console.error('Failed to create audio stream:', error);
783
- } finally {
784
- set({ isCreatingStream: false });
785
- }
786
- } else if (audioConfig.type === AudioInputType.FILES && audioConfig.files.length > 0) {
787
- try {
788
- set({ isCreatingStream: true, streamError: null });
789
- // Capture from existing audio element if present; otherwise construct and start it
790
- let audioEl = get().audioElement;
791
- const trackId = audioConfig.playerState.currentTrackId || audioConfig.files[0].id;
792
- const file = audioConfig.files.find(f => f.id === trackId) || audioConfig.files[0];
793
- if (!audioEl) {
794
- audioEl = new Audio();
795
- audioEl.crossOrigin = 'anonymous';
796
- audioEl.loop = true;
797
- audioEl.src = file.url;
798
- audioEl.volume = audioConfig.volume;
799
- audioEl.muted = audioConfig.isMuted;
800
- await audioEl.play().catch(() => {});
801
- set({ audioElement: audioEl });
802
- }
803
- // @ts-ignore
804
- const stream: MediaStream = (audioEl as any).captureStream ? (audioEl as any).captureStream() : (audioEl as any).mozCaptureStream?.();
805
- if (stream) {
806
- get().updateAudioConfig({ stream });
807
- console.debug('🎵 [SDK INPUT] Audio file stream captured');
808
- } else {
809
- throw new Error('Audio captureStream not supported');
810
- }
811
- } catch (error) {
812
- const errorMessage = error instanceof Error ? error.message : 'Failed to create audio stream from file';
813
- get().setStreamError(errorMessage);
814
- console.error('Failed to create audio file stream:', error);
815
- } finally {
816
- set({ isCreatingStream: false });
817
- }
818
- }
819
- },
820
-
821
- createVideoStream: async () => {
822
- const { inputConfiguration } = get();
823
- const videoConfig = inputConfiguration.video;
824
-
825
- if (videoConfig.type === VideoInputType.CAMERA && videoConfig.deviceId) {
826
- try {
827
- set({ isCreatingStream: true, streamError: null });
828
-
829
- const constraints: MediaStreamConstraints = {
830
- video: {
831
- deviceId: { exact: videoConfig.deviceId },
832
- ...(videoConfig.resolution && {
833
- width: { exact: videoConfig.resolution.width },
834
- height: { exact: videoConfig.resolution.height }
835
- })
836
- }
837
- };
838
-
839
- const stream = await navigator.mediaDevices.getUserMedia(constraints);
840
-
841
- get().updateVideoConfig({ stream });
842
- console.debug('📹 [SDK INPUT] Video stream created');
843
-
844
- } catch (error) {
845
- const errorMessage = error instanceof Error ? error.message : 'Failed to create video stream';
846
- get().setStreamError(errorMessage);
847
- console.error('Failed to create video stream:', error);
848
- } finally {
849
- set({ isCreatingStream: false });
850
- }
851
- } else if (videoConfig.type === VideoInputType.SCREEN_VIDEO) {
852
- try {
853
- set({ isCreatingStream: true, streamError: null });
854
- const stream = await navigator.mediaDevices.getDisplayMedia({
855
- audio: false,
856
- video: true,
857
- });
858
-
859
- // keep stream alive by binding to hidden video element
860
- let videoEl = get().videoElement;
861
- if (!videoEl) {
862
- videoEl = document.createElement('video');
863
- videoEl.style.display = 'none';
864
- videoEl.playsInline = true;
865
- videoEl.muted = true;
866
- document.body.appendChild(videoEl);
867
- set({ videoElement: videoEl });
868
- }
869
- try {
870
- // @ts-ignore
871
- if ('srcObject' in videoEl) {
872
- // @ts-ignore
873
- videoEl.srcObject = stream;
874
- } else {
875
- // @ts-ignore
876
- videoEl.src = URL.createObjectURL(stream);
877
- }
878
- await videoEl.play().catch(() => {});
879
- } catch {}
880
-
881
- get().updateVideoConfig({ stream, screenVideoEnabled: true });
882
- console.debug('📺 [SDK INPUT] Screen video stream created');
883
-
884
- } catch (error) {
885
- const errorMessage = error instanceof Error ? error.message : 'Failed to capture screen video';
886
- set({ streamError: errorMessage });
887
- } finally {
888
- set({ isCreatingStream: false });
889
- }
890
- } else if (videoConfig.type === VideoInputType.FILES && videoConfig.files.length > 0) {
891
- try {
892
- set({ isCreatingStream: true, streamError: null });
893
- const file = videoConfig.files.find(f => f.id === videoConfig.playerState.currentFileId) || videoConfig.files[0];
894
- let videoEl = get().videoElement;
895
- if (!videoEl) {
896
- videoEl = document.createElement('video');
897
- videoEl.src = file.url;
898
- videoEl.muted = true;
899
- videoEl.loop = true;
900
- videoEl.playsInline = true;
901
- await videoEl.play().catch(() => {});
902
- set({ videoElement: videoEl });
903
- }
904
- // @ts-ignore
905
- const stream: MediaStream = (videoEl as any).captureStream ? (videoEl as any).captureStream() : (videoEl as any).mozCaptureStream?.();
906
- if (stream) {
907
- get().updateVideoConfig({ stream });
908
- console.debug('🎬 [SDK INPUT] Video file stream captured');
909
- } else {
910
- throw new Error('Video captureStream not supported');
911
- }
912
- } catch (error) {
913
- const errorMessage = error instanceof Error ? error.message : 'Failed to create video stream from file';
914
- get().setStreamError(errorMessage);
915
- console.error('Failed to create video file stream:', error);
916
- } finally {
917
- set({ isCreatingStream: false });
918
- }
919
- }
920
- },
921
-
922
- updateStreamsForCore: async () => {
923
- const config = get().inputConfiguration;
924
- if (config.audio.enabled && config.audio.type !== AudioInputType.NONE && !config.audio.stream) {
925
- await get().createAudioStream();
926
- }
927
- if (config.video.enabled && config.video.type !== VideoInputType.NONE && !config.video.stream) {
928
- await get().createVideoStream();
929
- }
930
- return {
931
- audioStream: (config.audio.enabled && config.audio.type !== AudioInputType.NONE) ? get().inputConfiguration.audio.stream : undefined,
932
- videoStream: (config.video.enabled && config.video.type !== VideoInputType.NONE) ? get().inputConfiguration.video.stream : undefined,
933
- };
934
- },
935
-
936
- clearStreams: () => {
937
- const { inputConfiguration } = get();
938
-
939
- // Stop audio stream
940
- if (inputConfiguration.audio.stream) {
941
- inputConfiguration.audio.stream.getTracks().forEach(track => track.stop());
942
- get().updateAudioConfig({ stream: undefined });
943
- }
944
-
945
- // Stop video stream
946
- if (inputConfiguration.video.stream) {
947
- inputConfiguration.video.stream.getTracks().forEach(track => track.stop());
948
- get().updateVideoConfig({ stream: undefined });
949
- }
950
- },
951
-
952
- // Interaction Management
953
- setInteractionEnabled: (interactionEnabled: boolean) => {
954
- const { inputConfiguration } = get();
955
- set({
956
- inputConfiguration: {
957
- ...inputConfiguration,
958
- interactionEnabled,
959
- },
960
- });
961
- },
962
-
963
- // Error Handling
964
- setPermissionError: (lastPermissionError: string) => {
965
- set({ lastPermissionError });
966
- },
967
-
968
- setFileLoadError: (fileLoadError: string) => {
969
- set({ fileLoadError });
970
- },
971
-
972
- setStreamError: (streamError: string | null) => {
973
- set({ streamError });
974
- },
975
-
976
- clearErrors: () => {
977
- set({
978
- lastPermissionError: null,
979
- fileLoadError: null,
980
- streamError: null,
981
- });
982
- },
983
-
984
- // Device Fallback Management
985
- storePreviousWorkingAudioDevice: (deviceId?: string, deviceLabel?: string) => {
986
- set({
987
- previousWorkingAudioDevice: deviceId ? { deviceId, deviceLabel } : null
988
- });
989
- },
990
-
991
- storePreviousWorkingVideoDevice: (deviceId?: string, deviceLabel?: string) => {
992
- set({
993
- previousWorkingVideoDevice: deviceId ? { deviceId, deviceLabel } : null
994
- });
995
- },
996
-
997
- revertToWorkingAudioDevice: async () => {
998
- const { previousWorkingAudioDevice } = get();
999
- if (previousWorkingAudioDevice?.deviceId && previousWorkingAudioDevice?.deviceLabel) {
1000
- await get().selectAudioDevice(previousWorkingAudioDevice.deviceId, previousWorkingAudioDevice.deviceLabel);
1001
- return true;
1002
- }
1003
- return false;
1004
- },
1005
-
1006
- revertToWorkingVideoDevice: async () => {
1007
- const { previousWorkingVideoDevice } = get();
1008
- if (previousWorkingVideoDevice?.deviceId && previousWorkingVideoDevice?.deviceLabel) {
1009
- await get().selectVideoDevice(previousWorkingVideoDevice.deviceId, previousWorkingVideoDevice.deviceLabel);
1010
- return true;
1011
- }
1012
- return false;
1013
- },
1014
-
1015
- // Utility Methods
1016
- hasAudioPermission: () => get().inputConfiguration.permissions.audio,
1017
- hasVideoPermission: () => get().inputConfiguration.permissions.video,
1018
- hasScreenCapturePermission: () => get().inputConfiguration.permissions.screenCapture,
1019
- hasActiveAudioStream: () => !!get().inputConfiguration.audio.stream,
1020
- hasActiveVideoStream: () => !!get().inputConfiguration.video.stream,
1021
-
1022
- resetToDefaults: () => {
1023
- try {
1024
- get().clearStreams();
1025
- } catch {}
1026
- set({
1027
- inputConfiguration: createDefaultInputConfiguration(),
1028
- lastPermissionError: null,
1029
- fileLoadError: null,
1030
- streamError: null,
1031
- isCreatingStream: false,
1032
- });
1033
- },
1034
- }),
1035
- {
1036
- name: 'input-manager-store',
1037
- storage: createJSONStorage(() => sessionStorage),
1038
- // Drop all input settings on page refresh: don't rehydrate or persist anything
1039
- onRehydrateStorage: () => {
1040
- try { sessionStorage.removeItem('input-manager-store'); } catch {}
1041
- },
1042
- partialize: () => ({}),
1043
- }
1044
- )
1045
- );