@npcify/web 0.1.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -20
- package/dist/index.d.ts +583 -119
- package/dist/index.js +6294 -4309
- package/examples/three-game.js +107 -0
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { VRMExpressionPresetName,
|
|
1
|
+
import { VRM, VRMExpressionPresetName, VRMHumanoid, VRMHumanBoneName } from '@pixiv/three-vrm';
|
|
2
2
|
import { VRMAnimation } from '@pixiv/three-vrm-animation';
|
|
3
|
+
import { AnimationMixer, AnimationAction, Object3D, Group, SkinnedMesh, Mesh, Material, Skeleton, Vector3, Euler, Scene, PerspectiveCamera, WebGLRenderer } from 'three';
|
|
4
|
+
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
|
|
3
5
|
|
|
4
6
|
declare const TTS_PROVIDERS: readonly ["11labs"];
|
|
5
7
|
type TtsProviderId = (typeof TTS_PROVIDERS)[number];
|
|
@@ -333,14 +335,53 @@ interface AssetLoadProgress {
|
|
|
333
335
|
total: number;
|
|
334
336
|
}
|
|
335
337
|
|
|
336
|
-
interface
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
338
|
+
interface ClientTelemetryOptions {
|
|
339
|
+
enabled: boolean;
|
|
340
|
+
baseUrl: string;
|
|
341
|
+
apiToken: string;
|
|
342
|
+
}
|
|
343
|
+
declare class ClientTelemetry {
|
|
344
|
+
private readonly enabled;
|
|
345
|
+
private readonly baseUrl;
|
|
346
|
+
private readonly apiToken;
|
|
347
|
+
private state;
|
|
348
|
+
constructor({ enabled, baseUrl, apiToken }: ClientTelemetryOptions);
|
|
349
|
+
get traceHeaders(): Record<string, string> | undefined;
|
|
350
|
+
begin(conversationId: string): void;
|
|
351
|
+
clear(error?: Error): void;
|
|
352
|
+
recordStreamStart(taskId: string, stagingId: string): void;
|
|
353
|
+
recordFirstTextDelta(): void;
|
|
354
|
+
recordFirstStaging(): void;
|
|
355
|
+
reportAudioPlaybackStart(params: {
|
|
356
|
+
taskId: string;
|
|
357
|
+
stagingId: string;
|
|
358
|
+
speech: StagingSpeechBlock;
|
|
359
|
+
}): void;
|
|
360
|
+
private readonly createTurnSpan;
|
|
361
|
+
private readonly sendClientLatency;
|
|
362
|
+
reportPhonemePlayback(params: {
|
|
363
|
+
taskId: string;
|
|
364
|
+
stagingId: string;
|
|
365
|
+
speechId: string;
|
|
366
|
+
phonemeCount: number;
|
|
367
|
+
latePhonemeCount: number;
|
|
368
|
+
phonemeLateMs: number;
|
|
369
|
+
}): void;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** Deduplicate animation demand and discard results after model/catalog replacement. */
|
|
373
|
+
declare class LazyAnimations {
|
|
374
|
+
private readonly load;
|
|
375
|
+
private readonly attach;
|
|
376
|
+
private pending;
|
|
377
|
+
private generation;
|
|
378
|
+
private allowed;
|
|
379
|
+
constructor(load: (id: string) => Promise<VRMAnimation>, attach: (id: string, animation: VRMAnimation) => void);
|
|
380
|
+
reset(ids?: string[]): void;
|
|
381
|
+
get availableIds(): string[];
|
|
382
|
+
isAllowed(id: string): boolean;
|
|
383
|
+
ensure(id: string): Promise<void>;
|
|
384
|
+
ensureMany(ids: string[]): Promise<void>;
|
|
344
385
|
}
|
|
345
386
|
|
|
346
387
|
declare const PerformanceRecordingSchemaVersion = 3;
|
|
@@ -542,6 +583,162 @@ declare class PerformanceRecorder {
|
|
|
542
583
|
private elapsedMs;
|
|
543
584
|
}
|
|
544
585
|
|
|
586
|
+
interface AnimationConfig {
|
|
587
|
+
loop?: boolean;
|
|
588
|
+
weight?: number;
|
|
589
|
+
}
|
|
590
|
+
interface CharacterAnimationPlayOptions {
|
|
591
|
+
fadeDuration?: number;
|
|
592
|
+
loop?: boolean;
|
|
593
|
+
}
|
|
594
|
+
interface AnimationControllerLike {
|
|
595
|
+
prepareAnimations?: (ids: string[]) => Promise<void>;
|
|
596
|
+
previewAnimation?: (animation: VRMAnimation) => () => void;
|
|
597
|
+
hasAnimation?: (id: string) => boolean;
|
|
598
|
+
addAnimation?: (animationId: string, animation: VRMAnimation, config?: AnimationConfig) => void;
|
|
599
|
+
clearAnimations?: () => void;
|
|
600
|
+
setIdleAnimation?: (animationId: string | null) => void;
|
|
601
|
+
getAnimationMixer?: () => AnimationMixer;
|
|
602
|
+
getActiveAction?: () => AnimationAction | null;
|
|
603
|
+
getActiveAnimationId?: () => string | null;
|
|
604
|
+
getIdleAnimationId?: () => string | null;
|
|
605
|
+
getClipIds?: () => string[];
|
|
606
|
+
playAnimation: (animationId: string, fadeDuration?: number, loopOverride?: boolean) => void;
|
|
607
|
+
playIdle: (fadeDuration?: number) => void;
|
|
608
|
+
stopAnimation: (fadeDuration?: number) => void;
|
|
609
|
+
update: (delta: number) => void;
|
|
610
|
+
}
|
|
611
|
+
declare class RemoteAnimationController implements AnimationControllerLike {
|
|
612
|
+
playAnimation(_animationId: string, _fadeDuration?: number, _loopOverride?: boolean): void;
|
|
613
|
+
playIdle(_fadeDuration?: number): void;
|
|
614
|
+
stopAnimation(_fadeDuration?: number): void;
|
|
615
|
+
update(_delta: number): void;
|
|
616
|
+
}
|
|
617
|
+
declare class AnimationController implements AnimationControllerLike {
|
|
618
|
+
private readonly vrm;
|
|
619
|
+
prepareAnimations?: (ids: string[]) => Promise<void>;
|
|
620
|
+
private readonly mixer;
|
|
621
|
+
private readonly clips;
|
|
622
|
+
private readonly configs;
|
|
623
|
+
private previewMixer;
|
|
624
|
+
private restorePreview;
|
|
625
|
+
private activeAction;
|
|
626
|
+
private activeAnimationId;
|
|
627
|
+
private idleAnimationId;
|
|
628
|
+
private pendingReturnToIdleAction;
|
|
629
|
+
constructor(vrm: VRM, idleAnimationId?: string | null);
|
|
630
|
+
getAnimationMixer(): AnimationMixer;
|
|
631
|
+
getActiveAction(): AnimationAction | null;
|
|
632
|
+
getActiveAnimationId(): string | null;
|
|
633
|
+
getIdleAnimationId(): string | null;
|
|
634
|
+
getClipIds(): string[];
|
|
635
|
+
hasAnimation(id: string): boolean;
|
|
636
|
+
addAnimation(animationId: string, animation: VRMAnimation, config?: AnimationConfig): void;
|
|
637
|
+
previewAnimation(animation: VRMAnimation): () => void;
|
|
638
|
+
setIdleAnimation(animationId: string | null): void;
|
|
639
|
+
clearAnimations(): void;
|
|
640
|
+
playAnimation(animationId: string, fadeDuration?: number, loopOverride?: boolean): void;
|
|
641
|
+
playIdle(fadeDuration?: number): void;
|
|
642
|
+
stopAnimation(fadeDuration?: number): void;
|
|
643
|
+
update(delta: number): void;
|
|
644
|
+
private readonly handleActionFinished;
|
|
645
|
+
}
|
|
646
|
+
interface CharacterAnimationDependencies {
|
|
647
|
+
getController: () => AnimationControllerLike | undefined;
|
|
648
|
+
getAnimations: () => Map<string, VRMAnimation>;
|
|
649
|
+
getAvailableIds: () => string[];
|
|
650
|
+
ensureAnimation: (animationId: string) => Promise<void>;
|
|
651
|
+
}
|
|
652
|
+
declare class CharacterAnimation {
|
|
653
|
+
private readonly dependencies;
|
|
654
|
+
constructor(dependencies: CharacterAnimationDependencies);
|
|
655
|
+
/** Loaded VRMAnimation clips keyed by animation ID. */
|
|
656
|
+
get clips(): ReadonlyMap<string, VRMAnimation>;
|
|
657
|
+
/** Animation IDs available for this character (both loaded and lazy-loadable). */
|
|
658
|
+
get availableIds(): string[];
|
|
659
|
+
/** Configured default idle animation ID, or `null` if none. */
|
|
660
|
+
get idleAnimationId(): string | null;
|
|
661
|
+
/** Currently playing animation ID, or `null` if stopped/T-pose. */
|
|
662
|
+
get activeAnimationId(): string | null;
|
|
663
|
+
/** Underlying Three.js `AnimationMixer` driving the character's skeleton. */
|
|
664
|
+
get mixer(): AnimationMixer | null;
|
|
665
|
+
/** Currently active Three.js `AnimationAction`, if any. */
|
|
666
|
+
get activeAction(): AnimationAction | null;
|
|
667
|
+
/** Underlying animation controller instance. */
|
|
668
|
+
get controller(): AnimationControllerLike | undefined;
|
|
669
|
+
/** Check whether an animation clip is already loaded into the mixer. */
|
|
670
|
+
has(animationId: string): boolean;
|
|
671
|
+
/** Preload a lazy animation by ID so it is ready to play immediately. */
|
|
672
|
+
ensure(animationId: string): Promise<void>;
|
|
673
|
+
/**
|
|
674
|
+
* Play a character animation by ID, automatically fetching the clip first if it is lazy-loaded.
|
|
675
|
+
*/
|
|
676
|
+
play(animationId: string, options?: CharacterAnimationPlayOptions): Promise<void>;
|
|
677
|
+
/** Transition back to the configured idle animation. */
|
|
678
|
+
playIdle(fadeDuration?: number): void;
|
|
679
|
+
/** Fade out and stop the active animation. */
|
|
680
|
+
stop(fadeDuration?: number): void;
|
|
681
|
+
/** Preview a `VRMAnimation` clip and return a function that restores the previous pose. */
|
|
682
|
+
preview(animation: VRMAnimation): (() => void) | undefined;
|
|
683
|
+
private get instanceController();
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
declare class CharacterAudio {
|
|
687
|
+
private readonly getPlayer;
|
|
688
|
+
private readonly unlockHandler?;
|
|
689
|
+
constructor(getPlayer: () => AudioPlayer, unlockHandler?: (() => Promise<boolean>) | undefined);
|
|
690
|
+
/** Underlying `<audio>` element used for speech playback and Web Audio routing. */
|
|
691
|
+
get element(): HTMLAudioElement | null;
|
|
692
|
+
/** True while speech audio is actively playing. */
|
|
693
|
+
get playing(): boolean;
|
|
694
|
+
/** True when the audio element is paused. */
|
|
695
|
+
get paused(): boolean;
|
|
696
|
+
/** True when the current audio clip has reached its end. */
|
|
697
|
+
get ended(): boolean;
|
|
698
|
+
/** Current playback clock in seconds. */
|
|
699
|
+
get currentTime(): number;
|
|
700
|
+
/** Duration of the active audio clip in seconds, or `null` if unknown. */
|
|
701
|
+
get duration(): number | null;
|
|
702
|
+
/** Playback volume in `[0, 1]`. */
|
|
703
|
+
get volume(): number;
|
|
704
|
+
set volume(value: number);
|
|
705
|
+
/** Whether speech audio is muted. */
|
|
706
|
+
get muted(): boolean;
|
|
707
|
+
set muted(value: boolean);
|
|
708
|
+
/** Playback speed multiplier (`1` is normal speed). */
|
|
709
|
+
get playbackRate(): number;
|
|
710
|
+
set playbackRate(value: number);
|
|
711
|
+
/** CORS mode on the `<audio>` element (defaults to `'anonymous'` once Web Audio is connected). */
|
|
712
|
+
get crossOrigin(): string | null;
|
|
713
|
+
set crossOrigin(value: string | null);
|
|
714
|
+
/** Whether browser audio playback has been unlocked by user interaction. */
|
|
715
|
+
get unlocked(): boolean;
|
|
716
|
+
/** Viseme timeline entries currently bound to the active speech clip. */
|
|
717
|
+
get activeVisemes(): readonly ConversationTimelineEntry[];
|
|
718
|
+
/** Real-time telemetry/performance snapshot of the audio element. */
|
|
719
|
+
get snapshot(): AudioPerformanceFrameSnapshot;
|
|
720
|
+
/** Unlock browser audio playback inside a user gesture handler. */
|
|
721
|
+
unlock(): Promise<boolean>;
|
|
722
|
+
/** Stop the currently playing speech audio clip. */
|
|
723
|
+
stop(): void;
|
|
724
|
+
/**
|
|
725
|
+
* Create (or reuse) a `MediaElementAudioSourceNode` for this character's `<audio>` element.
|
|
726
|
+
* Useful for `THREE.PositionalAudio.setNodeSource(...)`, `AnalyserNode`, or custom Web Audio graphs.
|
|
727
|
+
*/
|
|
728
|
+
createMediaElementSource(audioContext: ResumableAudioContext): MediaElementAudioSourceNode;
|
|
729
|
+
/**
|
|
730
|
+
* Connect the character's `<audio>` element directly to a Web Audio `AudioNode`
|
|
731
|
+
* (such as `GainNode`, `PannerNode`, `AnalyserNode`, or `audioContext.destination`).
|
|
732
|
+
*/
|
|
733
|
+
connectAudioNode(destination: AudioNode): MediaElementAudioSourceNode;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
/** One random stream for the blinks and one for the eye moves. */
|
|
737
|
+
type IdleFaceRandom = {
|
|
738
|
+
blink: () => number;
|
|
739
|
+
gaze: () => number;
|
|
740
|
+
};
|
|
741
|
+
|
|
545
742
|
type FacialChannelName = VRMExpressionPresetName | string;
|
|
546
743
|
declare const FacialPerformanceLayer: {
|
|
547
744
|
readonly Emotion: "emotion";
|
|
@@ -555,15 +752,26 @@ interface FacialCueOptions {
|
|
|
555
752
|
weight?: number;
|
|
556
753
|
durationMs?: number;
|
|
557
754
|
}
|
|
755
|
+
interface FacialPerformanceMixerOptions {
|
|
756
|
+
/** The random numbers of the idle blinks and eye moves (tests: a fixed stream). */
|
|
757
|
+
idleRandom?: IdleFaceRandom | (() => number);
|
|
758
|
+
}
|
|
558
759
|
/**
|
|
559
760
|
* Adapts layered performance weights to Three-VRM facial channels.
|
|
560
761
|
*
|
|
561
762
|
* The built-in layers deliberately keep speech, emotion, blinking, and gaze
|
|
562
763
|
* state independent. More sources and blend policies can be added to the
|
|
563
764
|
* renderer-agnostic PerformanceWeightMixer without changing VRM ownership.
|
|
765
|
+
*
|
|
766
|
+
* A template face (`detectTemplateFace` of @npcify/vrm-renderer) also gets an
|
|
767
|
+
* IdleFace: it fills the Blink layer with idle blinks while no cue drives a
|
|
768
|
+
* blink preset, and it turns the eye bones toward the gaze target (or by the
|
|
769
|
+
* look presets of the Gaze layer when a cue drives them). Speech plays the
|
|
770
|
+
* same for every VRM: the viseme layer eases to each cue of the VisemePlayer.
|
|
564
771
|
*/
|
|
565
772
|
declare class FacialPerformanceMixer {
|
|
566
773
|
private readonly recorder?;
|
|
774
|
+
private readonly options;
|
|
567
775
|
private vrm;
|
|
568
776
|
private availableChannels;
|
|
569
777
|
private readonly availableChannelSet;
|
|
@@ -574,12 +782,21 @@ declare class FacialPerformanceMixer {
|
|
|
574
782
|
private readonly originalMouthOverrides;
|
|
575
783
|
private activeMouthMode;
|
|
576
784
|
private restoreMouthAfterVisemeRelease;
|
|
577
|
-
|
|
785
|
+
private idleFace;
|
|
786
|
+
private gazeTarget;
|
|
787
|
+
constructor(recorder?: PerformanceRecorder | undefined, options?: FacialPerformanceMixerOptions);
|
|
578
788
|
setVrm(vrm: VRM | null): void;
|
|
579
789
|
getChannelNames(): string[];
|
|
580
790
|
getEmotionChannelNames(): string[];
|
|
581
791
|
getVisemeChannelNames(): string[];
|
|
582
|
-
|
|
792
|
+
/** Whether the model blinks and moves its eyes by itself (template faces only). */
|
|
793
|
+
get hasIdleFace(): boolean;
|
|
794
|
+
get mouthMode(): FacialMouthMode;
|
|
795
|
+
getWeight(channel: FacialChannelName): number;
|
|
796
|
+
getWeights(): Record<string, number>;
|
|
797
|
+
getMouthOpenness(): number;
|
|
798
|
+
/** The object the eyes of a template face follow (the viewer's camera); null looks ahead. */
|
|
799
|
+
setGazeTarget(target: Object3D | null): void;
|
|
583
800
|
playCue(channel: FacialChannelName, options?: FacialCueOptions): void;
|
|
584
801
|
setEmotion(channel: FacialChannelName, weight: number, mouthMode: FacialMouthMode): void;
|
|
585
802
|
clearEmotion(mouthMode?: FacialMouthMode): void;
|
|
@@ -604,6 +821,7 @@ declare class FacialPerformanceMixer {
|
|
|
604
821
|
capturePerformanceModel(): FacialPerformanceModelSnapshot | null;
|
|
605
822
|
getPerformanceFrameSnapshot(): FacialPerformanceFrameSnapshot;
|
|
606
823
|
private configureLayers;
|
|
824
|
+
private updateIdleFace;
|
|
607
825
|
private getLayerChannels;
|
|
608
826
|
private areVisemesReleased;
|
|
609
827
|
private scheduleRelease;
|
|
@@ -614,40 +832,6 @@ declare class FacialPerformanceMixer {
|
|
|
614
832
|
private restoreMouthOverrides;
|
|
615
833
|
}
|
|
616
834
|
|
|
617
|
-
interface ClientTelemetryOptions {
|
|
618
|
-
enabled: boolean;
|
|
619
|
-
baseUrl: string;
|
|
620
|
-
apiToken: string;
|
|
621
|
-
}
|
|
622
|
-
declare class ClientTelemetry {
|
|
623
|
-
private readonly enabled;
|
|
624
|
-
private readonly baseUrl;
|
|
625
|
-
private readonly apiToken;
|
|
626
|
-
private state;
|
|
627
|
-
constructor({ enabled, baseUrl, apiToken }: ClientTelemetryOptions);
|
|
628
|
-
get traceHeaders(): Record<string, string> | undefined;
|
|
629
|
-
begin(conversationId: string): void;
|
|
630
|
-
clear(error?: Error): void;
|
|
631
|
-
recordStreamStart(taskId: string, stagingId: string): void;
|
|
632
|
-
recordFirstTextDelta(): void;
|
|
633
|
-
recordFirstStaging(): void;
|
|
634
|
-
reportAudioPlaybackStart(params: {
|
|
635
|
-
taskId: string;
|
|
636
|
-
stagingId: string;
|
|
637
|
-
speech: StagingSpeechBlock;
|
|
638
|
-
}): void;
|
|
639
|
-
private readonly createTurnSpan;
|
|
640
|
-
private readonly sendClientLatency;
|
|
641
|
-
reportPhonemePlayback(params: {
|
|
642
|
-
taskId: string;
|
|
643
|
-
stagingId: string;
|
|
644
|
-
speechId: string;
|
|
645
|
-
phonemeCount: number;
|
|
646
|
-
latePhonemeCount: number;
|
|
647
|
-
phonemeLateMs: number;
|
|
648
|
-
}): void;
|
|
649
|
-
}
|
|
650
|
-
|
|
651
835
|
declare class VisemePlayer {
|
|
652
836
|
private readonly facialMixer;
|
|
653
837
|
private readonly recorder?;
|
|
@@ -661,19 +845,17 @@ declare class VisemePlayer {
|
|
|
661
845
|
private phase;
|
|
662
846
|
private running;
|
|
663
847
|
private timelineRevision;
|
|
664
|
-
private readonly heldMouths;
|
|
665
848
|
constructor(facialMixer: FacialPerformanceMixer, recorder?: PerformanceRecorder | undefined);
|
|
666
849
|
attachAudioElement(element: HTMLAudioElement | null): void;
|
|
667
850
|
setVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
668
851
|
cleanup(): void;
|
|
669
852
|
updateVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
670
853
|
capturePerformanceState(): void;
|
|
854
|
+
getVisemes(): readonly ConversationTimelineEntry[];
|
|
671
855
|
getAudioPerformanceSnapshot(): AudioPerformanceFrameSnapshot;
|
|
672
856
|
getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
|
|
673
857
|
update(_delta: number): void;
|
|
674
858
|
private readonly handlePlaying;
|
|
675
|
-
private updateHeldMouth;
|
|
676
|
-
private cueDuration;
|
|
677
859
|
private readonly handleEnded;
|
|
678
860
|
private collectDueCueIndices;
|
|
679
861
|
private shouldCloseActiveCue;
|
|
@@ -692,11 +874,41 @@ type SpeechPlaybackOptions = {
|
|
|
692
874
|
signal?: AbortSignal;
|
|
693
875
|
onStart?: () => void;
|
|
694
876
|
};
|
|
877
|
+
type ResumableAudioContext = BaseAudioContext & {
|
|
878
|
+
resume?: () => Promise<void>;
|
|
879
|
+
createMediaElementSource: (mediaElement: HTMLMediaElement) => MediaElementAudioSourceNode;
|
|
880
|
+
};
|
|
695
881
|
declare class AudioPlayer {
|
|
696
882
|
private readonly visemePlayer;
|
|
697
883
|
private audioElement;
|
|
698
884
|
private readonly streams;
|
|
885
|
+
private unlocked;
|
|
886
|
+
private volumeValue;
|
|
887
|
+
private mutedValue;
|
|
888
|
+
private playbackRateValue;
|
|
889
|
+
private crossOriginValue;
|
|
890
|
+
private attachedAudioContext;
|
|
891
|
+
private unlockAttempt;
|
|
892
|
+
private playbackAttempt;
|
|
699
893
|
constructor(visemePlayer: VisemePlayer);
|
|
894
|
+
get element(): HTMLAudioElement | null;
|
|
895
|
+
get isUnlocked(): boolean;
|
|
896
|
+
get playing(): boolean;
|
|
897
|
+
get paused(): boolean;
|
|
898
|
+
get ended(): boolean;
|
|
899
|
+
get duration(): number | null;
|
|
900
|
+
get volume(): number;
|
|
901
|
+
set volume(value: number);
|
|
902
|
+
get muted(): boolean;
|
|
903
|
+
set muted(value: boolean);
|
|
904
|
+
get playbackRate(): number;
|
|
905
|
+
set playbackRate(value: number);
|
|
906
|
+
get crossOrigin(): string | null;
|
|
907
|
+
set crossOrigin(value: string | null);
|
|
908
|
+
get activeVisemes(): readonly ConversationTimelineEntry[];
|
|
909
|
+
get snapshot(): AudioPerformanceFrameSnapshot;
|
|
910
|
+
createMediaElementSource(audioContext: ResumableAudioContext): MediaElementAudioSourceNode;
|
|
911
|
+
connectAudioNode(destination: AudioNode): MediaElementAudioSourceNode;
|
|
700
912
|
appendStreamChunk(streamId: string, mimeType: string, data: string | Uint8Array): void;
|
|
701
913
|
completeStream(streamId: string): void;
|
|
702
914
|
failStream(streamId: string, message: string): void;
|
|
@@ -764,6 +976,63 @@ declare class StagingPlayer {
|
|
|
764
976
|
private clearPerformance;
|
|
765
977
|
}
|
|
766
978
|
|
|
979
|
+
type Vector3Tuple = readonly [number, number, number];
|
|
980
|
+
interface CharacterShadowOptions {
|
|
981
|
+
cast?: boolean;
|
|
982
|
+
receive?: boolean;
|
|
983
|
+
}
|
|
984
|
+
interface CharacterTransformOptions {
|
|
985
|
+
position?: Vector3 | Vector3Tuple;
|
|
986
|
+
rotation?: Euler | Vector3Tuple;
|
|
987
|
+
scale?: number | Vector3 | Vector3Tuple;
|
|
988
|
+
}
|
|
989
|
+
declare class CharacterSceneRig {
|
|
990
|
+
private currentVrm;
|
|
991
|
+
private cachedMeshes;
|
|
992
|
+
private cachedSkinnedMeshes;
|
|
993
|
+
private cachedMaterials;
|
|
994
|
+
private cachedSkeletons;
|
|
995
|
+
private shadowOptions;
|
|
996
|
+
private transformOptions;
|
|
997
|
+
get vrm(): VRM | null;
|
|
998
|
+
setVrm(vrm: VRM | null): void;
|
|
999
|
+
/** Root Three.js group containing the VRM hierarchy (`vrm.scene`). */
|
|
1000
|
+
get scene(): Group | null;
|
|
1001
|
+
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
1002
|
+
get object3d(): Group | null;
|
|
1003
|
+
/** Alias for `scene`. */
|
|
1004
|
+
get root(): Group | null;
|
|
1005
|
+
/** Primary rendered mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
1006
|
+
get mesh(): SkinnedMesh | Mesh | null;
|
|
1007
|
+
/** All Three.js `Mesh` nodes inside the character hierarchy. */
|
|
1008
|
+
get meshes(): readonly Mesh[];
|
|
1009
|
+
/** All Three.js `SkinnedMesh` nodes inside the character hierarchy. */
|
|
1010
|
+
get skinnedMeshes(): readonly SkinnedMesh[];
|
|
1011
|
+
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
1012
|
+
get materials(): readonly Material[];
|
|
1013
|
+
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
1014
|
+
get skeletons(): readonly Skeleton[];
|
|
1015
|
+
/** Three-VRM humanoid rig when loaded. */
|
|
1016
|
+
get humanoid(): VRMHumanoid | null;
|
|
1017
|
+
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
1018
|
+
getBone(name: VRMHumanBoneName): Object3D | null;
|
|
1019
|
+
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
1020
|
+
getNormalizedBone(name: VRMHumanBoneName): Object3D | null;
|
|
1021
|
+
/** Add the character root group to a Three.js `Scene` or parent `Object3D`. */
|
|
1022
|
+
attachTo(parent: Object3D): Group;
|
|
1023
|
+
/** Remove the character root group from its current parent `Object3D`. */
|
|
1024
|
+
detach(): void;
|
|
1025
|
+
/** Configure `castShadow` and `receiveShadow` across all character meshes. */
|
|
1026
|
+
setShadows(options?: CharacterShadowOptions): void;
|
|
1027
|
+
/** Configure position, rotation, and scale on the character root group. */
|
|
1028
|
+
setTransform(options: CharacterTransformOptions): void;
|
|
1029
|
+
/** Re-index meshes, materials, and skeletons after custom scene graph edits. */
|
|
1030
|
+
refreshMeshes(): void;
|
|
1031
|
+
disposeVrm(): void;
|
|
1032
|
+
private applyShadows;
|
|
1033
|
+
private applyTransform;
|
|
1034
|
+
}
|
|
1035
|
+
|
|
767
1036
|
type StagingSpeechPlaybackEvent = StagingSpeechBlock & {
|
|
768
1037
|
taskId: string;
|
|
769
1038
|
stagingId: string;
|
|
@@ -777,6 +1046,62 @@ type StagingPlaybackStart = {
|
|
|
777
1046
|
stagingId: string;
|
|
778
1047
|
taskId: string;
|
|
779
1048
|
};
|
|
1049
|
+
type CharacterPlaybackCursor = StagingPlaybackCursor & StagingPlaybackStart;
|
|
1050
|
+
interface NpcifyOptions {
|
|
1051
|
+
baseUrl: string;
|
|
1052
|
+
characterId: string;
|
|
1053
|
+
/** API token or key (`apiKey` is also accepted as an alias). */
|
|
1054
|
+
apiToken?: string;
|
|
1055
|
+
/** Alias for `apiToken`. */
|
|
1056
|
+
apiKey?: string;
|
|
1057
|
+
initialAssetInfo?: NpcifyAssetInfo | null;
|
|
1058
|
+
assetProfile?: 'original' | 'web-optimized-v1';
|
|
1059
|
+
telemetry?: {
|
|
1060
|
+
enabled?: boolean;
|
|
1061
|
+
};
|
|
1062
|
+
/** Editing uses live generation even when the character is published as baked. */
|
|
1063
|
+
authoring?: boolean;
|
|
1064
|
+
bakedConversation?: BakedConversationAsset | string | ArrayBuffer | Uint8Array;
|
|
1065
|
+
/** Optional Three.js `Scene` or `Object3D` to attach `character.scene` to once loaded. */
|
|
1066
|
+
parent?: Object3D | null;
|
|
1067
|
+
/** Optional Three.js `Camera` or `Object3D` for template-face eye tracking. */
|
|
1068
|
+
gazeTarget?: Object3D | null;
|
|
1069
|
+
/** Optional `castShadow` / `receiveShadow` configuration applied to character meshes. */
|
|
1070
|
+
shadows?: boolean | CharacterShadowOptions;
|
|
1071
|
+
/** Optional initial root transform (`position`, `rotation`, `scale`). */
|
|
1072
|
+
transform?: CharacterTransformOptions;
|
|
1073
|
+
/** Convenience initial root position (`Vector3` or `[x, y, z]`). */
|
|
1074
|
+
position?: CharacterTransformOptions['position'];
|
|
1075
|
+
/** Convenience initial root rotation (`Euler` or `[x, y, z]`). */
|
|
1076
|
+
rotation?: CharacterTransformOptions['rotation'];
|
|
1077
|
+
/** Convenience initial root scale (`number`, `Vector3`, or `[x, y, z]`). */
|
|
1078
|
+
scale?: CharacterTransformOptions['scale'];
|
|
1079
|
+
/** Initial speech audio volume in `[0, 1]`. */
|
|
1080
|
+
volume?: number;
|
|
1081
|
+
/** Initial speech audio muted flag. */
|
|
1082
|
+
muted?: boolean;
|
|
1083
|
+
/** Optional error callback. */
|
|
1084
|
+
onError?: (error: Error) => void;
|
|
1085
|
+
}
|
|
1086
|
+
type CharacterOptions = NpcifyOptions;
|
|
1087
|
+
interface CharacterDisposeOptions {
|
|
1088
|
+
/** Dispose Three.js VRM geometries, materials, and textures. Defaults to `true`. */
|
|
1089
|
+
disposeVrm?: boolean;
|
|
1090
|
+
}
|
|
1091
|
+
interface CharacterSpeechState {
|
|
1092
|
+
speaking: boolean;
|
|
1093
|
+
thinking: boolean;
|
|
1094
|
+
currentSpeech: StagingSpeechPlaybackEvent | null;
|
|
1095
|
+
currentText: string;
|
|
1096
|
+
revealedText: string;
|
|
1097
|
+
options: StagingResultPayload['options'];
|
|
1098
|
+
cursor: CharacterPlaybackCursor | null;
|
|
1099
|
+
}
|
|
1100
|
+
interface PlayStagingOptions {
|
|
1101
|
+
force?: boolean;
|
|
1102
|
+
silent?: boolean;
|
|
1103
|
+
}
|
|
1104
|
+
|
|
780
1105
|
interface StreamPlaybackDependencies {
|
|
781
1106
|
getStagingPlayer: () => StagingPlayer;
|
|
782
1107
|
telemetry: ClientTelemetry;
|
|
@@ -794,16 +1119,28 @@ declare class StreamPlaybackController {
|
|
|
794
1119
|
private timelineStartCallback;
|
|
795
1120
|
private cursorCallback;
|
|
796
1121
|
private activePlayback;
|
|
1122
|
+
private speakingState;
|
|
1123
|
+
private currentSpeechEvent;
|
|
1124
|
+
private currentTextValue;
|
|
1125
|
+
private revealedTextValue;
|
|
1126
|
+
private latestOptionsValue;
|
|
1127
|
+
private currentCursorValue;
|
|
797
1128
|
private readonly playedStagingIds;
|
|
798
1129
|
private readonly states;
|
|
799
1130
|
constructor(dependencies: StreamPlaybackDependencies);
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
1131
|
+
get speaking(): boolean;
|
|
1132
|
+
get currentSpeech(): StagingSpeechPlaybackEvent | null;
|
|
1133
|
+
get currentText(): string;
|
|
1134
|
+
get revealedText(): string;
|
|
1135
|
+
get options(): StagingResultPayload['options'];
|
|
1136
|
+
get cursor(): CharacterPlaybackCursor | null;
|
|
1137
|
+
onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): () => void;
|
|
1138
|
+
onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): () => void;
|
|
1139
|
+
onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): () => void;
|
|
1140
|
+
onPlaybackBlocked(callback: (stagingId: string) => void): () => void;
|
|
1141
|
+
onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): () => void;
|
|
1142
|
+
onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): () => void;
|
|
1143
|
+
onPlaybackCursor(callback: (cursor: CharacterPlaybackCursor) => void): () => void;
|
|
807
1144
|
hasPlayed(stagingId: string): boolean;
|
|
808
1145
|
playPayload(payload: StagingResultPayload, options?: {
|
|
809
1146
|
force?: boolean;
|
|
@@ -811,69 +1148,195 @@ declare class StreamPlaybackController {
|
|
|
811
1148
|
}): Promise<boolean>;
|
|
812
1149
|
handleEvent(event: StagingStreamEvent): void;
|
|
813
1150
|
cleanup(): void;
|
|
1151
|
+
private resetRealtimeSpeech;
|
|
814
1152
|
private playTimeline;
|
|
815
1153
|
private flushOptions;
|
|
816
1154
|
private recordStreamEvent;
|
|
817
1155
|
private recordPlaybackEvent;
|
|
818
1156
|
private errorReason;
|
|
819
1157
|
private toSpeechEvent;
|
|
820
|
-
private
|
|
821
|
-
private invokeTimelineStartCallback;
|
|
822
|
-
private invokeSpeechPlaybackStartCallback;
|
|
823
|
-
private invokeCharacterRevealCallback;
|
|
824
|
-
private invokeCallback;
|
|
1158
|
+
private safeInvoke;
|
|
825
1159
|
}
|
|
826
1160
|
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
/**
|
|
837
|
-
|
|
838
|
-
|
|
1161
|
+
declare class CharacterExpressions {
|
|
1162
|
+
readonly mixer: FacialPerformanceMixer;
|
|
1163
|
+
constructor(mixer: FacialPerformanceMixer);
|
|
1164
|
+
/** All available VRM expression channel names on the loaded character. */
|
|
1165
|
+
get all(): string[];
|
|
1166
|
+
/** Available emotion expression channel names (`happy`, `angry`, `sad`, etc.). */
|
|
1167
|
+
get emotions(): string[];
|
|
1168
|
+
/** Available mouth/viseme expression channel names (`aa`, `ee`, `ih`, `oh`, `ou`). */
|
|
1169
|
+
get visemes(): string[];
|
|
1170
|
+
/** Current applied weights for all expression channels in `[0, 1]`. */
|
|
1171
|
+
get weights(): Record<string, number>;
|
|
1172
|
+
/** Real-time aggregate mouth openness in `[0, 1]`. */
|
|
1173
|
+
get mouthOpenness(): number;
|
|
1174
|
+
/** Active mouth ownership mode (`'expression'` or `'viseme'`). */
|
|
1175
|
+
get mouthMode(): FacialMouthMode;
|
|
1176
|
+
/** Whether automatic idle blinking and eye movement are active on this face. */
|
|
1177
|
+
get hasIdleFace(): boolean;
|
|
1178
|
+
/** Full performance frame snapshot including layered weights and applied values. */
|
|
1179
|
+
get snapshot(): FacialPerformanceFrameSnapshot;
|
|
1180
|
+
/** Read the current applied weight of a single expression channel in `[0, 1]`. */
|
|
1181
|
+
getWeight(name: FacialChannelName): number;
|
|
1182
|
+
/** Drive an expression channel target weight in `[0, 1]` (eased during `update(delta)`). */
|
|
1183
|
+
setWeight(name: FacialChannelName, weight: number): void;
|
|
1184
|
+
/** Play a timed expression cue and ease back to neutral after `durationMs`. */
|
|
1185
|
+
play(name: FacialChannelName, options?: FacialCueOptions): void;
|
|
1186
|
+
/** Set the active emotion expression channel and weight. */
|
|
1187
|
+
setEmotion(name: FacialChannelName, weight?: number, mouthMode?: FacialMouthMode): void;
|
|
1188
|
+
/** Clear the active emotion expression layer. */
|
|
1189
|
+
clearEmotion(mouthMode?: FacialMouthMode): void;
|
|
1190
|
+
/** Reset all expression layer targets to zero. */
|
|
1191
|
+
reset(): void;
|
|
839
1192
|
}
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1193
|
+
|
|
1194
|
+
interface CharacterRuntimeInitOptions extends Omit<NpcifyOptions, 'apiToken' | 'assetProfile' | 'apiKey'> {
|
|
1195
|
+
apiToken: string;
|
|
1196
|
+
assetProfile: 'original' | 'web-optimized-v1';
|
|
1197
|
+
loadAnimation?: (animationId: string) => Promise<VRMAnimation>;
|
|
843
1198
|
}
|
|
844
|
-
declare class
|
|
1199
|
+
declare class CharacterRuntime {
|
|
845
1200
|
readonly baseUrl: string;
|
|
846
1201
|
readonly characterId: string;
|
|
847
1202
|
readonly apiToken: string;
|
|
848
1203
|
readonly assetProfile: 'original' | 'web-optimized-v1';
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
1204
|
+
readonly audio: CharacterAudio;
|
|
1205
|
+
readonly expressions: CharacterExpressions;
|
|
1206
|
+
readonly animation: CharacterAnimation;
|
|
1207
|
+
animations: Map<string, VRMAnimation>;
|
|
1208
|
+
animationController?: AnimationControllerLike;
|
|
1209
|
+
protected readonly sceneRig: CharacterSceneRig;
|
|
1210
|
+
protected readonly lazyAnimations: LazyAnimations;
|
|
1211
|
+
protected audioPlayer: AudioPlayer | null;
|
|
1212
|
+
protected stagingPlayer: StagingPlayer | null;
|
|
1213
|
+
protected readonly facialMixer: FacialPerformanceMixer;
|
|
1214
|
+
protected readonly visemePlayer: VisemePlayer;
|
|
1215
|
+
protected readonly performanceRecorder: PerformanceRecorder;
|
|
1216
|
+
protected readonly telemetry: ClientTelemetry;
|
|
1217
|
+
protected readonly streamPlayback: StreamPlaybackController;
|
|
1218
|
+
protected defaultParent: Object3D | null;
|
|
1219
|
+
protected currentGazeTarget: Object3D | null;
|
|
1220
|
+
protected thinkingState: boolean;
|
|
1221
|
+
protected errorCallback: ((error: Error) => void) | null;
|
|
1222
|
+
constructor(options: CharacterRuntimeInitOptions);
|
|
1223
|
+
/** Self-reference so `npcify.character` and `character` share the same interface. */
|
|
1224
|
+
get character(): this;
|
|
1225
|
+
/** Loaded Three-VRM instance, or `null` before `loadAssets()`. */
|
|
1226
|
+
get vrm(): VRM | null;
|
|
1227
|
+
set vrm(value: VRM | null);
|
|
1228
|
+
/** True once the character VRM model is loaded. */
|
|
1229
|
+
get ready(): boolean;
|
|
1230
|
+
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
1231
|
+
get scene(): Group | null;
|
|
1232
|
+
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
1233
|
+
get object3d(): Group | null;
|
|
1234
|
+
/** Alias for `scene`. */
|
|
1235
|
+
get root(): Group | null;
|
|
1236
|
+
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
1237
|
+
get mesh(): SkinnedMesh | Mesh | null;
|
|
1238
|
+
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
1239
|
+
get meshes(): readonly Mesh[];
|
|
1240
|
+
/** All Three.js `SkinnedMesh` instances in the character hierarchy. */
|
|
1241
|
+
get skinnedMeshes(): readonly SkinnedMesh[];
|
|
1242
|
+
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
1243
|
+
get materials(): readonly Material[];
|
|
1244
|
+
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
1245
|
+
get skeletons(): readonly Skeleton[];
|
|
1246
|
+
/** Three-VRM humanoid rig when loaded. */
|
|
1247
|
+
get humanoid(): VRMHumanoid | null;
|
|
1248
|
+
/** Underlying Three.js `AnimationMixer` driving skeletal animations. */
|
|
1249
|
+
get mixer(): AnimationMixer | null;
|
|
1250
|
+
/** The `Object3D` (such as a `THREE.Camera`) the character's eyes track. */
|
|
1251
|
+
get gazeTarget(): Object3D | null;
|
|
1252
|
+
set gazeTarget(target: Object3D | null);
|
|
1253
|
+
/** True while the character is actively delivering a speech block or playing voice audio. */
|
|
1254
|
+
get speaking(): boolean;
|
|
1255
|
+
/** True while waiting for the next AI turn to begin streaming or playing. */
|
|
1256
|
+
get thinking(): boolean;
|
|
1257
|
+
/** Active speech block metadata during playback, or `null` when silent. */
|
|
1258
|
+
get currentSpeech(): StagingSpeechPlaybackEvent | null;
|
|
1259
|
+
/** Full text of the current or most recent speech block. */
|
|
1260
|
+
get currentText(): string;
|
|
1261
|
+
/** Subtitle text revealed so far for the current or most recent speech block. */
|
|
1262
|
+
get revealedText(): string;
|
|
1263
|
+
/** Latest reply options delivered for the current turn. */
|
|
1264
|
+
get options(): StagingResultPayload['options'];
|
|
1265
|
+
/** Real-time playback cursor within the active dialogue block, or `null` when idle. */
|
|
1266
|
+
get cursor(): CharacterPlaybackCursor | null;
|
|
1267
|
+
/** Combined snapshot of real-time speech and dialogue state. */
|
|
1268
|
+
get speech(): CharacterSpeechState;
|
|
1269
|
+
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
1270
|
+
getBone(name: VRMHumanBoneName): Object3D | null;
|
|
1271
|
+
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
1272
|
+
getNormalizedBone(name: VRMHumanBoneName): Object3D | null;
|
|
1273
|
+
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
1274
|
+
attachTo(parent: Object3D): Group;
|
|
1275
|
+
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
1276
|
+
detach(): void;
|
|
1277
|
+
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
1278
|
+
setShadows(options?: CharacterShadowOptions): void;
|
|
1279
|
+
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
1280
|
+
setTransform(options: CharacterTransformOptions): void;
|
|
1281
|
+
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
1282
|
+
refreshMeshes(): void;
|
|
1283
|
+
ensureAnimation(animationId: string): Promise<void>;
|
|
1284
|
+
/** Play a character animation by ID, automatically loading it if lazy. */
|
|
1285
|
+
playAnimation(animationId: string, options?: CharacterAnimationPlayOptions): Promise<void>;
|
|
1286
|
+
/** Transition back to the character's idle animation. */
|
|
1287
|
+
playIdle(fadeDuration?: number): void;
|
|
1288
|
+
/** Stop the currently playing skeletal animation. */
|
|
1289
|
+
stopAnimation(fadeDuration?: number): void;
|
|
1290
|
+
unlockAudio(): Promise<boolean>;
|
|
1291
|
+
getExpressions(): string[];
|
|
1292
|
+
getMouthExpressionNames(): string[];
|
|
1293
|
+
getExpressionNames(): string[];
|
|
1294
|
+
playExpression(name: FacialChannelName, options?: FacialCueOptions): void;
|
|
1295
|
+
setExpressionWeight(name: FacialChannelName, weight: number): void;
|
|
1296
|
+
getExpressionWeight(name: FacialChannelName): number;
|
|
1297
|
+
setEmotion(name: FacialChannelName, weight?: number, mouthMode?: FacialMouthMode): void;
|
|
1298
|
+
clearEmotion(mouthMode?: FacialMouthMode): void;
|
|
1299
|
+
/** The object the eyes of a template face follow (the camera); null looks ahead. */
|
|
1300
|
+
setGazeTarget(target: Object3D | null): void;
|
|
1301
|
+
startPerformanceRecording(options?: PerformanceRecordingStartOptions): void;
|
|
1302
|
+
stopPerformanceRecording(): PerformanceRecording | null;
|
|
1303
|
+
getPerformanceRecording(): PerformanceRecording | null;
|
|
1304
|
+
recordStagingRoute(stagingId: string | null, state: 'selected' | 'emitted' | 'received'): void;
|
|
1305
|
+
onError(callback: (error: Error) => void): () => void;
|
|
1306
|
+
onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): () => void;
|
|
1307
|
+
onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): () => void;
|
|
1308
|
+
onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): () => void;
|
|
1309
|
+
onPlaybackBlocked(callback: (stagingId: string) => void): () => void;
|
|
1310
|
+
onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): () => void;
|
|
1311
|
+
onPlaybackCursor(callback: Parameters<StreamPlaybackController['onPlaybackCursor']>[0]): () => void;
|
|
1312
|
+
onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): () => void;
|
|
1313
|
+
/** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
|
|
1314
|
+
update(delta: number): void;
|
|
1315
|
+
protected readonly getAudioPlayer: () => AudioPlayer;
|
|
1316
|
+
protected readonly getStagingPlayer: () => StagingPlayer;
|
|
1317
|
+
protected resetRuntimeAssets(): void;
|
|
1318
|
+
protected disposeRuntimeAssets(): void;
|
|
1319
|
+
protected readonly handleCallbackError: (context: string, error: unknown) => void;
|
|
1320
|
+
protected readonly reportError: (error: Error, context?: string) => void;
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
declare class Character extends CharacterRuntime {
|
|
857
1324
|
private readonly conversationSocket;
|
|
858
1325
|
private renderSessionId;
|
|
859
1326
|
private initialAssetInfo;
|
|
860
1327
|
private stagingResultCallback;
|
|
861
1328
|
private thinkingCallback;
|
|
862
|
-
private errorCallback;
|
|
863
1329
|
private readonly notifiedStagingIds;
|
|
864
1330
|
private readonly liveConversation;
|
|
865
1331
|
private baked;
|
|
866
1332
|
private readonly authoring;
|
|
867
1333
|
private readonly bakedAsset?;
|
|
868
|
-
|
|
869
|
-
animations: Map<string, VRMAnimation>;
|
|
1334
|
+
private animationRefreshId;
|
|
870
1335
|
conversationId: string | null;
|
|
871
|
-
|
|
872
|
-
|
|
1336
|
+
constructor(options: NpcifyOptions);
|
|
1337
|
+
load(callback?: (info: AssetLoadProgress) => void): Promise<this>;
|
|
873
1338
|
loadAssets(callback?: (info: AssetLoadProgress) => void): Promise<void>;
|
|
874
|
-
private animationRefreshId;
|
|
875
1339
|
reloadAnimations(info?: NpcifyAssetInfo): Promise<void>;
|
|
876
|
-
ensureAnimation(animationId: string): Promise<void>;
|
|
877
1340
|
reloadAssets(): Promise<void>;
|
|
878
1341
|
initializeRemote(): Promise<void>;
|
|
879
1342
|
createConversation(options?: {
|
|
@@ -884,48 +1347,42 @@ declare class Npcify {
|
|
|
884
1347
|
connect(conversationId?: string): Promise<string>;
|
|
885
1348
|
disconnect(): Promise<void>;
|
|
886
1349
|
endConversation(): Promise<void>;
|
|
1350
|
+
dispose(options?: CharacterDisposeOptions): Promise<void>;
|
|
887
1351
|
sendMessage(message: string, options?: ConversationTurnOptions): Promise<StagingResultPayload>;
|
|
888
1352
|
sendMessage(conversationId: string, message: string, options?: ConversationTurnOptions): Promise<StagingResultPayload>;
|
|
889
1353
|
replayStaging(stagingId: string): Promise<StagingResultPayload>;
|
|
890
1354
|
replayStaging(conversationId: string, stagingId: string, options?: PlayStagingOptions): Promise<StagingResultPayload>;
|
|
891
1355
|
get isBakedConversation(): boolean;
|
|
892
1356
|
getConversationWorldline(conversationId?: string | null): Promise<ConversationWorldline>;
|
|
893
|
-
/** Feed an owner-authoring result through the normal event and playback path. */
|
|
894
1357
|
acceptAuthoringResult(payload: StagingResultPayload): Promise<void>;
|
|
895
1358
|
playStaging(payload: StagingResultPayload, options?: PlayStagingOptions): Promise<boolean>;
|
|
896
|
-
unlockAudio(): Promise<boolean>;
|
|
897
1359
|
setRenderSessionId(renderSessionId: string | null): void;
|
|
898
|
-
onStagingResult(callback: (payload: StagingResultPayload) => Promise<void> | void): void;
|
|
899
|
-
onThinking(callback: (payload: LlmThinkingPayload) => Promise<void> | void): void;
|
|
900
|
-
onError(callback: (error: Error) => void): void;
|
|
901
|
-
onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): void;
|
|
902
|
-
onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): void;
|
|
903
|
-
onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): void;
|
|
904
|
-
onPlaybackBlocked(callback: (stagingId: string) => void): void;
|
|
905
|
-
onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): void;
|
|
906
|
-
onPlaybackCursor(callback: Parameters<StreamPlaybackController['onPlaybackCursor']>[0]): void;
|
|
907
|
-
onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): void;
|
|
908
|
-
getExpressions(): string[];
|
|
909
|
-
getMouthExpressionNames(): string[];
|
|
910
|
-
getExpressionNames(): string[];
|
|
911
|
-
playExpression(name: string, options?: FacialCueOptions): void;
|
|
912
|
-
startPerformanceRecording(options?: PerformanceRecordingStartOptions): void;
|
|
913
|
-
stopPerformanceRecording(): PerformanceRecording | null;
|
|
914
|
-
getPerformanceRecording(): PerformanceRecording | null;
|
|
915
|
-
recordStagingRoute(stagingId: string | null, state: 'selected' | 'emitted' | 'received'): void;
|
|
916
|
-
update(delta: number): void;
|
|
1360
|
+
onStagingResult(callback: (payload: StagingResultPayload) => Promise<void> | void): () => void;
|
|
1361
|
+
onThinking(callback: (payload: LlmThinkingPayload) => Promise<void> | void): () => void;
|
|
917
1362
|
private resetAssets;
|
|
918
1363
|
private readonly getAssetInfo;
|
|
919
1364
|
private handleStagingResult;
|
|
920
|
-
private shouldNotifyResult;
|
|
921
|
-
private invokeStagingResultCallback;
|
|
922
1365
|
private invokeThinkingCallback;
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
private readonly handleCallbackError;
|
|
926
|
-
private readonly reportError;
|
|
1366
|
+
}
|
|
1367
|
+
declare class Npcify extends Character {
|
|
927
1368
|
}
|
|
928
1369
|
|
|
1370
|
+
interface LoadCharacterOptions extends CharacterOptions {
|
|
1371
|
+
/** Optional asset download progress callback. */
|
|
1372
|
+
onProgress?: (progress: AssetLoadProgress) => void;
|
|
1373
|
+
/** Automatically create and start a conversation once the model finishes loading. */
|
|
1374
|
+
autoStartConversation?: boolean;
|
|
1375
|
+
/** Start the opening turn silently when `autoStartConversation` is enabled. */
|
|
1376
|
+
silent?: boolean;
|
|
1377
|
+
}
|
|
1378
|
+
/** Instantiate an unloaded `Character` instance. Call `await character.load()` to load 3D assets. */
|
|
1379
|
+
declare const createCharacter: (options: CharacterOptions) => Character;
|
|
1380
|
+
/**
|
|
1381
|
+
* Load an AI character's VRM model, humanoid skeleton, facial expressions, and animations
|
|
1382
|
+
* for direct integration into a custom Three.js `Scene` and render loop.
|
|
1383
|
+
*/
|
|
1384
|
+
declare const loadCharacter: (options: LoadCharacterOptions) => Promise<Character>;
|
|
1385
|
+
|
|
929
1386
|
type PerformanceIssueSeverity = 'error' | 'warning';
|
|
930
1387
|
interface PerformanceIssue {
|
|
931
1388
|
code: 'render.frame-gap' | 'timeline.invalid-entry' | 'timeline.non-monotonic' | 'viseme.audio-coverage-gap' | 'viseme.cue-late';
|
|
@@ -973,6 +1430,7 @@ declare class BakedConversation {
|
|
|
973
1430
|
private activeId;
|
|
974
1431
|
readonly asset: BakedConversationAsset;
|
|
975
1432
|
constructor(asset: BakedConversationAsset, characterId: string);
|
|
1433
|
+
assertAnimationsAvailable(animationIds: Iterable<string>): void;
|
|
976
1434
|
start(): StagingResultPayload;
|
|
977
1435
|
select(stagingId: string): StagingResultPayload;
|
|
978
1436
|
reply(message: string, parentStagingId?: string | null): {
|
|
@@ -994,15 +1452,21 @@ interface MountNpcifyOptions {
|
|
|
994
1452
|
baseUrl: string;
|
|
995
1453
|
characterId: string;
|
|
996
1454
|
apiKey?: string;
|
|
1455
|
+
apiToken?: string;
|
|
997
1456
|
serverRendering?: boolean;
|
|
998
1457
|
assetProfile?: 'original' | 'web-optimized-v1';
|
|
999
1458
|
onError?: (error: Error) => void;
|
|
1000
1459
|
}
|
|
1001
1460
|
interface NpcifyView {
|
|
1002
1461
|
npcify: Npcify;
|
|
1462
|
+
character?: Character;
|
|
1463
|
+
scene?: Scene | null;
|
|
1464
|
+
camera?: PerspectiveCamera | null;
|
|
1465
|
+
renderer?: WebGLRenderer | null;
|
|
1466
|
+
controls?: OrbitControls | null;
|
|
1003
1467
|
dispose(): Promise<void>;
|
|
1004
1468
|
}
|
|
1005
1469
|
/** Mount into an existing canvas. Its CSS must give it a nonzero width and height. */
|
|
1006
1470
|
declare const mountNpcify: (options: MountNpcifyOptions) => Promise<NpcifyView>;
|
|
1007
1471
|
|
|
1008
|
-
export { type AudioPerformanceFrameSnapshot, BakedConversation, type BakedConversationAsset, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, type MountNpcifyOptions, Npcify, type NpcifyAssetInfo, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, type RepeatedVisemeTransitionAnalysis, type StagingOptionsReadyPayload, type StagingSpeechPlaybackEvent, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, decodePerformance, encodePerformance, loadBakedConversation, mountNpcify };
|
|
1472
|
+
export { AnimationController, type AnimationControllerLike, type AssetLoadProgress, type AudioPerformanceFrameSnapshot, AudioPlayer, BakedConversation, type BakedConversationAsset, Character, CharacterAnimation, type CharacterAnimationPlayOptions, CharacterAudio, type CharacterDisposeOptions, CharacterExpressions, type CharacterOptions, type CharacterPlaybackCursor, CharacterRuntime, type CharacterRuntimeInitOptions, CharacterSceneRig, type CharacterShadowOptions, type CharacterSpeechState, type CharacterTransformOptions, type ConversationTurnOptions, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, type LlmThinkingPayload, type LoadCharacterOptions, type MountNpcifyOptions, Npcify, type NpcifyAssetInfo, Character as NpcifyCharacter, type NpcifyOptions, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, RemoteAnimationController, type RepeatedVisemeTransitionAnalysis, type StagingOptionsReadyPayload, type StagingPlaybackStart, type StagingSpeechPlaybackEvent, type Vector3Tuple, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, createCharacter, decodePerformance, encodePerformance, loadBakedConversation, loadCharacter, mountNpcify };
|