@npcify/web 0.1.7 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -20
- package/dist/index.d.ts +631 -121
- package/dist/index.js +5743 -4354
- package/examples/three-game.js +107 -0
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { VRMExpressionPresetName,
|
|
1
|
+
import { VRM, VRMExpressionPresetName, VRMHumanoid, VRMHumanBoneName } from '@pixiv/three-vrm';
|
|
2
2
|
import { VRMAnimation } from '@pixiv/three-vrm-animation';
|
|
3
|
+
import { AnimationMixer, AnimationAction, Object3D, Vector3, Group, SkinnedMesh, Mesh, Material, Skeleton, Euler, Scene, PerspectiveCamera, WebGLRenderer } from 'three';
|
|
4
|
+
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
|
|
3
5
|
|
|
4
|
-
|
|
5
|
-
type TtsProviderId = (typeof TTS_PROVIDERS)[number];
|
|
6
|
+
type TtsProviderId = string;
|
|
6
7
|
type NullableTtsProviderId = TtsProviderId | null;
|
|
7
8
|
|
|
8
9
|
declare const EXPRESSION_CALL_RANGES: readonly ["linear", "bool"];
|
|
@@ -333,14 +334,53 @@ interface AssetLoadProgress {
|
|
|
333
334
|
total: number;
|
|
334
335
|
}
|
|
335
336
|
|
|
336
|
-
interface
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
337
|
+
interface ClientTelemetryOptions {
|
|
338
|
+
enabled: boolean;
|
|
339
|
+
baseUrl: string;
|
|
340
|
+
apiToken: string;
|
|
341
|
+
}
|
|
342
|
+
declare class ClientTelemetry {
|
|
343
|
+
private readonly enabled;
|
|
344
|
+
private readonly baseUrl;
|
|
345
|
+
private readonly apiToken;
|
|
346
|
+
private state;
|
|
347
|
+
constructor({ enabled, baseUrl, apiToken }: ClientTelemetryOptions);
|
|
348
|
+
get traceHeaders(): Record<string, string> | undefined;
|
|
349
|
+
begin(conversationId: string): void;
|
|
350
|
+
clear(error?: Error): void;
|
|
351
|
+
recordStreamStart(taskId: string, stagingId: string): void;
|
|
352
|
+
recordFirstTextDelta(): void;
|
|
353
|
+
recordFirstStaging(): void;
|
|
354
|
+
reportAudioPlaybackStart(params: {
|
|
355
|
+
taskId: string;
|
|
356
|
+
stagingId: string;
|
|
357
|
+
speech: StagingSpeechBlock;
|
|
358
|
+
}): void;
|
|
359
|
+
private readonly createTurnSpan;
|
|
360
|
+
private readonly sendClientLatency;
|
|
361
|
+
reportPhonemePlayback(params: {
|
|
362
|
+
taskId: string;
|
|
363
|
+
stagingId: string;
|
|
364
|
+
speechId: string;
|
|
365
|
+
phonemeCount: number;
|
|
366
|
+
latePhonemeCount: number;
|
|
367
|
+
phonemeLateMs: number;
|
|
368
|
+
}): void;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/** Deduplicate animation demand and discard results after model/catalog replacement. */
|
|
372
|
+
declare class LazyAnimations {
|
|
373
|
+
private readonly load;
|
|
374
|
+
private readonly attach;
|
|
375
|
+
private pending;
|
|
376
|
+
private generation;
|
|
377
|
+
private allowed;
|
|
378
|
+
constructor(load: (id: string) => Promise<VRMAnimation>, attach: (id: string, animation: VRMAnimation) => void);
|
|
379
|
+
reset(ids?: string[]): void;
|
|
380
|
+
get availableIds(): string[];
|
|
381
|
+
isAllowed(id: string): boolean;
|
|
382
|
+
ensure(id: string): Promise<void>;
|
|
383
|
+
ensureMany(ids: string[]): Promise<void>;
|
|
344
384
|
}
|
|
345
385
|
|
|
346
386
|
declare const PerformanceRecordingSchemaVersion = 3;
|
|
@@ -542,6 +582,162 @@ declare class PerformanceRecorder {
|
|
|
542
582
|
private elapsedMs;
|
|
543
583
|
}
|
|
544
584
|
|
|
585
|
+
interface AnimationConfig {
|
|
586
|
+
loop?: boolean;
|
|
587
|
+
weight?: number;
|
|
588
|
+
}
|
|
589
|
+
interface CharacterAnimationPlayOptions {
|
|
590
|
+
fadeDuration?: number;
|
|
591
|
+
loop?: boolean;
|
|
592
|
+
}
|
|
593
|
+
interface AnimationControllerLike {
|
|
594
|
+
prepareAnimations?: (ids: string[]) => Promise<void>;
|
|
595
|
+
previewAnimation?: (animation: VRMAnimation) => () => void;
|
|
596
|
+
hasAnimation?: (id: string) => boolean;
|
|
597
|
+
addAnimation?: (animationId: string, animation: VRMAnimation, config?: AnimationConfig) => void;
|
|
598
|
+
clearAnimations?: () => void;
|
|
599
|
+
setIdleAnimation?: (animationId: string | null) => void;
|
|
600
|
+
getAnimationMixer?: () => AnimationMixer;
|
|
601
|
+
getActiveAction?: () => AnimationAction | null;
|
|
602
|
+
getActiveAnimationId?: () => string | null;
|
|
603
|
+
getIdleAnimationId?: () => string | null;
|
|
604
|
+
getClipIds?: () => string[];
|
|
605
|
+
playAnimation: (animationId: string, fadeDuration?: number, loopOverride?: boolean) => void;
|
|
606
|
+
playIdle: (fadeDuration?: number) => void;
|
|
607
|
+
stopAnimation: (fadeDuration?: number) => void;
|
|
608
|
+
update: (delta: number) => void;
|
|
609
|
+
}
|
|
610
|
+
declare class RemoteAnimationController implements AnimationControllerLike {
|
|
611
|
+
playAnimation(_animationId: string, _fadeDuration?: number, _loopOverride?: boolean): void;
|
|
612
|
+
playIdle(_fadeDuration?: number): void;
|
|
613
|
+
stopAnimation(_fadeDuration?: number): void;
|
|
614
|
+
update(_delta: number): void;
|
|
615
|
+
}
|
|
616
|
+
declare class AnimationController implements AnimationControllerLike {
|
|
617
|
+
private readonly vrm;
|
|
618
|
+
prepareAnimations?: (ids: string[]) => Promise<void>;
|
|
619
|
+
private readonly mixer;
|
|
620
|
+
private readonly clips;
|
|
621
|
+
private readonly configs;
|
|
622
|
+
private previewMixer;
|
|
623
|
+
private restorePreview;
|
|
624
|
+
private activeAction;
|
|
625
|
+
private activeAnimationId;
|
|
626
|
+
private idleAnimationId;
|
|
627
|
+
private pendingReturnToIdleAction;
|
|
628
|
+
constructor(vrm: VRM, idleAnimationId?: string | null);
|
|
629
|
+
getAnimationMixer(): AnimationMixer;
|
|
630
|
+
getActiveAction(): AnimationAction | null;
|
|
631
|
+
getActiveAnimationId(): string | null;
|
|
632
|
+
getIdleAnimationId(): string | null;
|
|
633
|
+
getClipIds(): string[];
|
|
634
|
+
hasAnimation(id: string): boolean;
|
|
635
|
+
addAnimation(animationId: string, animation: VRMAnimation, config?: AnimationConfig): void;
|
|
636
|
+
previewAnimation(animation: VRMAnimation): () => void;
|
|
637
|
+
setIdleAnimation(animationId: string | null): void;
|
|
638
|
+
clearAnimations(): void;
|
|
639
|
+
playAnimation(animationId: string, fadeDuration?: number, loopOverride?: boolean): void;
|
|
640
|
+
playIdle(fadeDuration?: number): void;
|
|
641
|
+
stopAnimation(fadeDuration?: number): void;
|
|
642
|
+
update(delta: number): void;
|
|
643
|
+
private readonly handleActionFinished;
|
|
644
|
+
}
|
|
645
|
+
interface CharacterAnimationDependencies {
|
|
646
|
+
getController: () => AnimationControllerLike | undefined;
|
|
647
|
+
getAnimations: () => Map<string, VRMAnimation>;
|
|
648
|
+
getAvailableIds: () => string[];
|
|
649
|
+
ensureAnimation: (animationId: string) => Promise<void>;
|
|
650
|
+
}
|
|
651
|
+
declare class CharacterAnimation {
|
|
652
|
+
private readonly dependencies;
|
|
653
|
+
constructor(dependencies: CharacterAnimationDependencies);
|
|
654
|
+
/** Loaded VRMAnimation clips keyed by animation ID. */
|
|
655
|
+
get clips(): ReadonlyMap<string, VRMAnimation>;
|
|
656
|
+
/** Animation IDs available for this character (both loaded and lazy-loadable). */
|
|
657
|
+
get availableIds(): string[];
|
|
658
|
+
/** Configured default idle animation ID, or `null` if none. */
|
|
659
|
+
get idleAnimationId(): string | null;
|
|
660
|
+
/** Currently playing animation ID, or `null` if stopped/T-pose. */
|
|
661
|
+
get activeAnimationId(): string | null;
|
|
662
|
+
/** Underlying Three.js `AnimationMixer` driving the character's skeleton. */
|
|
663
|
+
get mixer(): AnimationMixer | null;
|
|
664
|
+
/** Currently active Three.js `AnimationAction`, if any. */
|
|
665
|
+
get activeAction(): AnimationAction | null;
|
|
666
|
+
/** Underlying animation controller instance. */
|
|
667
|
+
get controller(): AnimationControllerLike | undefined;
|
|
668
|
+
/** Check whether an animation clip is already loaded into the mixer. */
|
|
669
|
+
has(animationId: string): boolean;
|
|
670
|
+
/** Preload a lazy animation by ID so it is ready to play immediately. */
|
|
671
|
+
ensure(animationId: string): Promise<void>;
|
|
672
|
+
/**
|
|
673
|
+
* Play a character animation by ID, automatically fetching the clip first if it is lazy-loaded.
|
|
674
|
+
*/
|
|
675
|
+
play(animationId: string, options?: CharacterAnimationPlayOptions): Promise<void>;
|
|
676
|
+
/** Transition back to the configured idle animation. */
|
|
677
|
+
playIdle(fadeDuration?: number): void;
|
|
678
|
+
/** Fade out and stop the active animation. */
|
|
679
|
+
stop(fadeDuration?: number): void;
|
|
680
|
+
/** Preview a `VRMAnimation` clip and return a function that restores the previous pose. */
|
|
681
|
+
preview(animation: VRMAnimation): (() => void) | undefined;
|
|
682
|
+
private get instanceController();
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
declare class CharacterAudio {
|
|
686
|
+
private readonly getPlayer;
|
|
687
|
+
private readonly unlockHandler?;
|
|
688
|
+
constructor(getPlayer: () => AudioPlayer, unlockHandler?: (() => Promise<boolean>) | undefined);
|
|
689
|
+
/** Underlying `<audio>` element used for speech playback and Web Audio routing. */
|
|
690
|
+
get element(): HTMLAudioElement | null;
|
|
691
|
+
/** True while speech audio is actively playing. */
|
|
692
|
+
get playing(): boolean;
|
|
693
|
+
/** True when the audio element is paused. */
|
|
694
|
+
get paused(): boolean;
|
|
695
|
+
/** True when the current audio clip has reached its end. */
|
|
696
|
+
get ended(): boolean;
|
|
697
|
+
/** Current playback clock in seconds. */
|
|
698
|
+
get currentTime(): number;
|
|
699
|
+
/** Duration of the active audio clip in seconds, or `null` if unknown. */
|
|
700
|
+
get duration(): number | null;
|
|
701
|
+
/** Playback volume in `[0, 1]`. */
|
|
702
|
+
get volume(): number;
|
|
703
|
+
set volume(value: number);
|
|
704
|
+
/** Whether speech audio is muted. */
|
|
705
|
+
get muted(): boolean;
|
|
706
|
+
set muted(value: boolean);
|
|
707
|
+
/** Playback speed multiplier (`1` is normal speed). */
|
|
708
|
+
get playbackRate(): number;
|
|
709
|
+
set playbackRate(value: number);
|
|
710
|
+
/** CORS mode on the `<audio>` element (defaults to `'anonymous'` once Web Audio is connected). */
|
|
711
|
+
get crossOrigin(): string | null;
|
|
712
|
+
set crossOrigin(value: string | null);
|
|
713
|
+
/** Whether browser audio playback has been unlocked by user interaction. */
|
|
714
|
+
get unlocked(): boolean;
|
|
715
|
+
/** Viseme timeline entries currently bound to the active speech clip. */
|
|
716
|
+
get activeVisemes(): readonly ConversationTimelineEntry[];
|
|
717
|
+
/** Real-time telemetry/performance snapshot of the audio element. */
|
|
718
|
+
get snapshot(): AudioPerformanceFrameSnapshot;
|
|
719
|
+
/** Unlock browser audio playback inside a user gesture handler. */
|
|
720
|
+
unlock(): Promise<boolean>;
|
|
721
|
+
/** Stop the currently playing speech audio clip. */
|
|
722
|
+
stop(): void;
|
|
723
|
+
/**
|
|
724
|
+
* Create (or reuse) a `MediaElementAudioSourceNode` for this character's `<audio>` element.
|
|
725
|
+
* Useful for `THREE.PositionalAudio.setNodeSource(...)`, `AnalyserNode`, or custom Web Audio graphs.
|
|
726
|
+
*/
|
|
727
|
+
createMediaElementSource(audioContext: ResumableAudioContext): MediaElementAudioSourceNode;
|
|
728
|
+
/**
|
|
729
|
+
* Connect the character's `<audio>` element directly to a Web Audio `AudioNode`
|
|
730
|
+
* (such as `GainNode`, `PannerNode`, `AnalyserNode`, or `audioContext.destination`).
|
|
731
|
+
*/
|
|
732
|
+
connectAudioNode(destination: AudioNode): MediaElementAudioSourceNode;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
/** One random stream for the blinks and one for the eye moves. */
|
|
736
|
+
type IdleFaceRandom = {
|
|
737
|
+
blink: () => number;
|
|
738
|
+
gaze: () => number;
|
|
739
|
+
};
|
|
740
|
+
|
|
545
741
|
type FacialChannelName = VRMExpressionPresetName | string;
|
|
546
742
|
declare const FacialPerformanceLayer: {
|
|
547
743
|
readonly Emotion: "emotion";
|
|
@@ -555,15 +751,26 @@ interface FacialCueOptions {
|
|
|
555
751
|
weight?: number;
|
|
556
752
|
durationMs?: number;
|
|
557
753
|
}
|
|
754
|
+
interface FacialPerformanceMixerOptions {
|
|
755
|
+
/** The random numbers of the idle blinks and eye moves (tests: a fixed stream). */
|
|
756
|
+
idleRandom?: IdleFaceRandom | (() => number);
|
|
757
|
+
}
|
|
558
758
|
/**
|
|
559
759
|
* Adapts layered performance weights to Three-VRM facial channels.
|
|
560
760
|
*
|
|
561
761
|
* The built-in layers deliberately keep speech, emotion, blinking, and gaze
|
|
562
762
|
* state independent. More sources and blend policies can be added to the
|
|
563
763
|
* renderer-agnostic PerformanceWeightMixer without changing VRM ownership.
|
|
764
|
+
*
|
|
765
|
+
* A template face (`detectTemplateFace` of @npcify/vrm-renderer) also gets an
|
|
766
|
+
* IdleFace: it fills the Blink layer with idle blinks while no cue drives a
|
|
767
|
+
* blink preset, and it turns the eye bones toward the gaze target (or by the
|
|
768
|
+
* look presets of the Gaze layer when a cue drives them). Speech plays the
|
|
769
|
+
* same for every VRM: the viseme layer eases to each cue of the VisemePlayer.
|
|
564
770
|
*/
|
|
565
771
|
declare class FacialPerformanceMixer {
|
|
566
772
|
private readonly recorder?;
|
|
773
|
+
private readonly options;
|
|
567
774
|
private vrm;
|
|
568
775
|
private availableChannels;
|
|
569
776
|
private readonly availableChannelSet;
|
|
@@ -574,12 +781,21 @@ declare class FacialPerformanceMixer {
|
|
|
574
781
|
private readonly originalMouthOverrides;
|
|
575
782
|
private activeMouthMode;
|
|
576
783
|
private restoreMouthAfterVisemeRelease;
|
|
577
|
-
|
|
784
|
+
private idleFace;
|
|
785
|
+
private gazeTarget;
|
|
786
|
+
constructor(recorder?: PerformanceRecorder | undefined, options?: FacialPerformanceMixerOptions);
|
|
578
787
|
setVrm(vrm: VRM | null): void;
|
|
579
788
|
getChannelNames(): string[];
|
|
580
789
|
getEmotionChannelNames(): string[];
|
|
581
790
|
getVisemeChannelNames(): string[];
|
|
582
|
-
|
|
791
|
+
/** Whether the model blinks and moves its eyes by itself (template faces only). */
|
|
792
|
+
get hasIdleFace(): boolean;
|
|
793
|
+
get mouthMode(): FacialMouthMode;
|
|
794
|
+
getWeight(channel: FacialChannelName): number;
|
|
795
|
+
getWeights(): Record<string, number>;
|
|
796
|
+
getMouthOpenness(): number;
|
|
797
|
+
/** The object the eyes of a template face follow (the viewer's camera); null looks ahead. */
|
|
798
|
+
setGazeTarget(target: Object3D | null): void;
|
|
583
799
|
playCue(channel: FacialChannelName, options?: FacialCueOptions): void;
|
|
584
800
|
setEmotion(channel: FacialChannelName, weight: number, mouthMode: FacialMouthMode): void;
|
|
585
801
|
clearEmotion(mouthMode?: FacialMouthMode): void;
|
|
@@ -604,6 +820,7 @@ declare class FacialPerformanceMixer {
|
|
|
604
820
|
capturePerformanceModel(): FacialPerformanceModelSnapshot | null;
|
|
605
821
|
getPerformanceFrameSnapshot(): FacialPerformanceFrameSnapshot;
|
|
606
822
|
private configureLayers;
|
|
823
|
+
private updateIdleFace;
|
|
607
824
|
private getLayerChannels;
|
|
608
825
|
private areVisemesReleased;
|
|
609
826
|
private scheduleRelease;
|
|
@@ -614,40 +831,6 @@ declare class FacialPerformanceMixer {
|
|
|
614
831
|
private restoreMouthOverrides;
|
|
615
832
|
}
|
|
616
833
|
|
|
617
|
-
interface ClientTelemetryOptions {
|
|
618
|
-
enabled: boolean;
|
|
619
|
-
baseUrl: string;
|
|
620
|
-
apiToken: string;
|
|
621
|
-
}
|
|
622
|
-
declare class ClientTelemetry {
|
|
623
|
-
private readonly enabled;
|
|
624
|
-
private readonly baseUrl;
|
|
625
|
-
private readonly apiToken;
|
|
626
|
-
private state;
|
|
627
|
-
constructor({ enabled, baseUrl, apiToken }: ClientTelemetryOptions);
|
|
628
|
-
get traceHeaders(): Record<string, string> | undefined;
|
|
629
|
-
begin(conversationId: string): void;
|
|
630
|
-
clear(error?: Error): void;
|
|
631
|
-
recordStreamStart(taskId: string, stagingId: string): void;
|
|
632
|
-
recordFirstTextDelta(): void;
|
|
633
|
-
recordFirstStaging(): void;
|
|
634
|
-
reportAudioPlaybackStart(params: {
|
|
635
|
-
taskId: string;
|
|
636
|
-
stagingId: string;
|
|
637
|
-
speech: StagingSpeechBlock;
|
|
638
|
-
}): void;
|
|
639
|
-
private readonly createTurnSpan;
|
|
640
|
-
private readonly sendClientLatency;
|
|
641
|
-
reportPhonemePlayback(params: {
|
|
642
|
-
taskId: string;
|
|
643
|
-
stagingId: string;
|
|
644
|
-
speechId: string;
|
|
645
|
-
phonemeCount: number;
|
|
646
|
-
latePhonemeCount: number;
|
|
647
|
-
phonemeLateMs: number;
|
|
648
|
-
}): void;
|
|
649
|
-
}
|
|
650
|
-
|
|
651
834
|
declare class VisemePlayer {
|
|
652
835
|
private readonly facialMixer;
|
|
653
836
|
private readonly recorder?;
|
|
@@ -661,19 +844,17 @@ declare class VisemePlayer {
|
|
|
661
844
|
private phase;
|
|
662
845
|
private running;
|
|
663
846
|
private timelineRevision;
|
|
664
|
-
private readonly heldMouths;
|
|
665
847
|
constructor(facialMixer: FacialPerformanceMixer, recorder?: PerformanceRecorder | undefined);
|
|
666
848
|
attachAudioElement(element: HTMLAudioElement | null): void;
|
|
667
849
|
setVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
668
850
|
cleanup(): void;
|
|
669
851
|
updateVisemes(visemes: ConversationTimelineEntry[]): void;
|
|
670
852
|
capturePerformanceState(): void;
|
|
853
|
+
getVisemes(): readonly ConversationTimelineEntry[];
|
|
671
854
|
getAudioPerformanceSnapshot(): AudioPerformanceFrameSnapshot;
|
|
672
855
|
getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
|
|
673
856
|
update(_delta: number): void;
|
|
674
857
|
private readonly handlePlaying;
|
|
675
|
-
private updateHeldMouth;
|
|
676
|
-
private cueDuration;
|
|
677
858
|
private readonly handleEnded;
|
|
678
859
|
private collectDueCueIndices;
|
|
679
860
|
private shouldCloseActiveCue;
|
|
@@ -692,11 +873,41 @@ type SpeechPlaybackOptions = {
|
|
|
692
873
|
signal?: AbortSignal;
|
|
693
874
|
onStart?: () => void;
|
|
694
875
|
};
|
|
876
|
+
type ResumableAudioContext = BaseAudioContext & {
|
|
877
|
+
resume?: () => Promise<void>;
|
|
878
|
+
createMediaElementSource: (mediaElement: HTMLMediaElement) => MediaElementAudioSourceNode;
|
|
879
|
+
};
|
|
695
880
|
declare class AudioPlayer {
|
|
696
881
|
private readonly visemePlayer;
|
|
697
882
|
private audioElement;
|
|
698
883
|
private readonly streams;
|
|
884
|
+
private unlocked;
|
|
885
|
+
private volumeValue;
|
|
886
|
+
private mutedValue;
|
|
887
|
+
private playbackRateValue;
|
|
888
|
+
private crossOriginValue;
|
|
889
|
+
private attachedAudioContext;
|
|
890
|
+
private unlockAttempt;
|
|
891
|
+
private playbackAttempt;
|
|
699
892
|
constructor(visemePlayer: VisemePlayer);
|
|
893
|
+
get element(): HTMLAudioElement | null;
|
|
894
|
+
get isUnlocked(): boolean;
|
|
895
|
+
get playing(): boolean;
|
|
896
|
+
get paused(): boolean;
|
|
897
|
+
get ended(): boolean;
|
|
898
|
+
get duration(): number | null;
|
|
899
|
+
get volume(): number;
|
|
900
|
+
set volume(value: number);
|
|
901
|
+
get muted(): boolean;
|
|
902
|
+
set muted(value: boolean);
|
|
903
|
+
get playbackRate(): number;
|
|
904
|
+
set playbackRate(value: number);
|
|
905
|
+
get crossOrigin(): string | null;
|
|
906
|
+
set crossOrigin(value: string | null);
|
|
907
|
+
get activeVisemes(): readonly ConversationTimelineEntry[];
|
|
908
|
+
get snapshot(): AudioPerformanceFrameSnapshot;
|
|
909
|
+
createMediaElementSource(audioContext: ResumableAudioContext): MediaElementAudioSourceNode;
|
|
910
|
+
connectAudioNode(destination: AudioNode): MediaElementAudioSourceNode;
|
|
700
911
|
appendStreamChunk(streamId: string, mimeType: string, data: string | Uint8Array): void;
|
|
701
912
|
completeStream(streamId: string): void;
|
|
702
913
|
failStream(streamId: string, message: string): void;
|
|
@@ -764,6 +975,87 @@ declare class StagingPlayer {
|
|
|
764
975
|
private clearPerformance;
|
|
765
976
|
}
|
|
766
977
|
|
|
978
|
+
type Vector3Tuple = readonly [number, number, number];
|
|
979
|
+
interface Vector3Like {
|
|
980
|
+
readonly x: number;
|
|
981
|
+
readonly y: number;
|
|
982
|
+
readonly z: number;
|
|
983
|
+
}
|
|
984
|
+
type CharacterLookAtTarget = Object3D | Vector3 | Vector3Tuple | Vector3Like | null;
|
|
985
|
+
interface CharacterShadowOptions {
|
|
986
|
+
cast?: boolean;
|
|
987
|
+
receive?: boolean;
|
|
988
|
+
}
|
|
989
|
+
interface CharacterTransformOptions {
|
|
990
|
+
position?: Vector3 | Vector3Tuple;
|
|
991
|
+
rotation?: Euler | Vector3Tuple;
|
|
992
|
+
scale?: number | Vector3 | Vector3Tuple;
|
|
993
|
+
}
|
|
994
|
+
declare class CharacterSceneRig {
|
|
995
|
+
private currentVrm;
|
|
996
|
+
private cachedMeshes;
|
|
997
|
+
private cachedSkinnedMeshes;
|
|
998
|
+
private cachedMaterials;
|
|
999
|
+
private cachedSkeletons;
|
|
1000
|
+
private shadowOptions;
|
|
1001
|
+
private transformOptions;
|
|
1002
|
+
private lookAtAnchor;
|
|
1003
|
+
/** The target that setVrmLookAtTarget set on the current model. */
|
|
1004
|
+
private vrmLookAtTarget;
|
|
1005
|
+
get vrm(): VRM | null;
|
|
1006
|
+
setVrm(vrm: VRM | null): void;
|
|
1007
|
+
/**
|
|
1008
|
+
* The `Object3D` of a `lookAt()` argument: an `Object3D` as it is, a point
|
|
1009
|
+
* or three coordinates as a world position that this rig keeps, or null.
|
|
1010
|
+
* Coordinates that are not finite numbers throw a `RangeError` and change
|
|
1011
|
+
* nothing.
|
|
1012
|
+
*/
|
|
1013
|
+
resolveLookAtTarget(targetOrX: CharacterLookAtTarget | number, y?: number, z?: number): Object3D | null;
|
|
1014
|
+
/**
|
|
1015
|
+
* Sets three-vrm's own look-at target (`vrm.lookAt.target`) of the current
|
|
1016
|
+
* model. Null removes the target that this method set; a target that other
|
|
1017
|
+
* code set stays. The IdleFace of a template face does not move the eyes
|
|
1018
|
+
* while `vrm.lookAt.target` is set.
|
|
1019
|
+
*/
|
|
1020
|
+
setVrmLookAtTarget(target: Object3D | null): void;
|
|
1021
|
+
private updateLookAtAnchor;
|
|
1022
|
+
/** Root Three.js group containing the VRM hierarchy (`vrm.scene`). */
|
|
1023
|
+
get scene(): Group | null;
|
|
1024
|
+
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
1025
|
+
get object3d(): Group | null;
|
|
1026
|
+
/** Alias for `scene`. */
|
|
1027
|
+
get root(): Group | null;
|
|
1028
|
+
/** Primary rendered mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
1029
|
+
get mesh(): SkinnedMesh | Mesh | null;
|
|
1030
|
+
/** All Three.js `Mesh` nodes inside the character hierarchy. */
|
|
1031
|
+
get meshes(): readonly Mesh[];
|
|
1032
|
+
/** All Three.js `SkinnedMesh` nodes inside the character hierarchy. */
|
|
1033
|
+
get skinnedMeshes(): readonly SkinnedMesh[];
|
|
1034
|
+
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
1035
|
+
get materials(): readonly Material[];
|
|
1036
|
+
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
1037
|
+
get skeletons(): readonly Skeleton[];
|
|
1038
|
+
/** Three-VRM humanoid rig when loaded. */
|
|
1039
|
+
get humanoid(): VRMHumanoid | null;
|
|
1040
|
+
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
1041
|
+
getBone(name: VRMHumanBoneName): Object3D | null;
|
|
1042
|
+
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
1043
|
+
getNormalizedBone(name: VRMHumanBoneName): Object3D | null;
|
|
1044
|
+
/** Add the character root group to a Three.js `Scene` or parent `Object3D`. */
|
|
1045
|
+
attachTo(parent: Object3D): Group;
|
|
1046
|
+
/** Remove the character root group from its current parent `Object3D`. */
|
|
1047
|
+
detach(): void;
|
|
1048
|
+
/** Configure `castShadow` and `receiveShadow` across all character meshes. */
|
|
1049
|
+
setShadows(options?: CharacterShadowOptions): void;
|
|
1050
|
+
/** Configure position, rotation, and scale on the character root group. */
|
|
1051
|
+
setTransform(options: CharacterTransformOptions): void;
|
|
1052
|
+
/** Re-index meshes, materials, and skeletons after custom scene graph edits. */
|
|
1053
|
+
refreshMeshes(): void;
|
|
1054
|
+
disposeVrm(): void;
|
|
1055
|
+
private applyShadows;
|
|
1056
|
+
private applyTransform;
|
|
1057
|
+
}
|
|
1058
|
+
|
|
767
1059
|
type StagingSpeechPlaybackEvent = StagingSpeechBlock & {
|
|
768
1060
|
taskId: string;
|
|
769
1061
|
stagingId: string;
|
|
@@ -777,6 +1069,62 @@ type StagingPlaybackStart = {
|
|
|
777
1069
|
stagingId: string;
|
|
778
1070
|
taskId: string;
|
|
779
1071
|
};
|
|
1072
|
+
type CharacterPlaybackCursor = StagingPlaybackCursor & StagingPlaybackStart;
|
|
1073
|
+
interface NpcifyOptions {
|
|
1074
|
+
baseUrl: string;
|
|
1075
|
+
characterId: string;
|
|
1076
|
+
/** API token or key (`apiKey` is also accepted as an alias). */
|
|
1077
|
+
apiToken?: string;
|
|
1078
|
+
/** Alias for `apiToken`. */
|
|
1079
|
+
apiKey?: string;
|
|
1080
|
+
initialAssetInfo?: NpcifyAssetInfo | null;
|
|
1081
|
+
assetProfile?: 'original' | 'web-optimized-v1';
|
|
1082
|
+
telemetry?: {
|
|
1083
|
+
enabled?: boolean;
|
|
1084
|
+
};
|
|
1085
|
+
/** Editing uses live generation even when the character is published as baked. */
|
|
1086
|
+
authoring?: boolean;
|
|
1087
|
+
bakedConversation?: BakedConversationAsset | string | ArrayBuffer | Uint8Array;
|
|
1088
|
+
/** Optional Three.js `Scene` or `Object3D` to attach `character.scene` to once loaded. */
|
|
1089
|
+
parent?: Object3D | null;
|
|
1090
|
+
/** Optional Three.js `Camera` or `Object3D` for template-face eye tracking. */
|
|
1091
|
+
gazeTarget?: Object3D | null;
|
|
1092
|
+
/** Optional `castShadow` / `receiveShadow` configuration applied to character meshes. */
|
|
1093
|
+
shadows?: boolean | CharacterShadowOptions;
|
|
1094
|
+
/** Optional initial root transform (`position`, `rotation`, `scale`). */
|
|
1095
|
+
transform?: CharacterTransformOptions;
|
|
1096
|
+
/** Convenience initial root position (`Vector3` or `[x, y, z]`). */
|
|
1097
|
+
position?: CharacterTransformOptions['position'];
|
|
1098
|
+
/** Convenience initial root rotation (`Euler` or `[x, y, z]`). */
|
|
1099
|
+
rotation?: CharacterTransformOptions['rotation'];
|
|
1100
|
+
/** Convenience initial root scale (`number`, `Vector3`, or `[x, y, z]`). */
|
|
1101
|
+
scale?: CharacterTransformOptions['scale'];
|
|
1102
|
+
/** Initial speech audio volume in `[0, 1]`. */
|
|
1103
|
+
volume?: number;
|
|
1104
|
+
/** Initial speech audio muted flag. */
|
|
1105
|
+
muted?: boolean;
|
|
1106
|
+
/** Optional error callback. */
|
|
1107
|
+
onError?: (error: Error) => void;
|
|
1108
|
+
}
|
|
1109
|
+
type CharacterOptions = NpcifyOptions;
|
|
1110
|
+
interface CharacterDisposeOptions {
|
|
1111
|
+
/** Dispose Three.js VRM geometries, materials, and textures. Defaults to `true`. */
|
|
1112
|
+
disposeVrm?: boolean;
|
|
1113
|
+
}
|
|
1114
|
+
interface CharacterSpeechState {
|
|
1115
|
+
speaking: boolean;
|
|
1116
|
+
thinking: boolean;
|
|
1117
|
+
currentSpeech: StagingSpeechPlaybackEvent | null;
|
|
1118
|
+
currentText: string;
|
|
1119
|
+
revealedText: string;
|
|
1120
|
+
options: StagingResultPayload['options'];
|
|
1121
|
+
cursor: CharacterPlaybackCursor | null;
|
|
1122
|
+
}
|
|
1123
|
+
interface PlayStagingOptions {
|
|
1124
|
+
force?: boolean;
|
|
1125
|
+
silent?: boolean;
|
|
1126
|
+
}
|
|
1127
|
+
|
|
780
1128
|
interface StreamPlaybackDependencies {
|
|
781
1129
|
getStagingPlayer: () => StagingPlayer;
|
|
782
1130
|
telemetry: ClientTelemetry;
|
|
@@ -794,16 +1142,28 @@ declare class StreamPlaybackController {
|
|
|
794
1142
|
private timelineStartCallback;
|
|
795
1143
|
private cursorCallback;
|
|
796
1144
|
private activePlayback;
|
|
1145
|
+
private speakingState;
|
|
1146
|
+
private currentSpeechEvent;
|
|
1147
|
+
private currentTextValue;
|
|
1148
|
+
private revealedTextValue;
|
|
1149
|
+
private latestOptionsValue;
|
|
1150
|
+
private currentCursorValue;
|
|
797
1151
|
private readonly playedStagingIds;
|
|
798
1152
|
private readonly states;
|
|
799
1153
|
constructor(dependencies: StreamPlaybackDependencies);
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
1154
|
+
get speaking(): boolean;
|
|
1155
|
+
get currentSpeech(): StagingSpeechPlaybackEvent | null;
|
|
1156
|
+
get currentText(): string;
|
|
1157
|
+
get revealedText(): string;
|
|
1158
|
+
get options(): StagingResultPayload['options'];
|
|
1159
|
+
get cursor(): CharacterPlaybackCursor | null;
|
|
1160
|
+
onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): () => void;
|
|
1161
|
+
onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): () => void;
|
|
1162
|
+
onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): () => void;
|
|
1163
|
+
onPlaybackBlocked(callback: (stagingId: string) => void): () => void;
|
|
1164
|
+
onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): () => void;
|
|
1165
|
+
onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): () => void;
|
|
1166
|
+
onPlaybackCursor(callback: (cursor: CharacterPlaybackCursor) => void): () => void;
|
|
807
1167
|
hasPlayed(stagingId: string): boolean;
|
|
808
1168
|
playPayload(payload: StagingResultPayload, options?: {
|
|
809
1169
|
force?: boolean;
|
|
@@ -811,69 +1171,218 @@ declare class StreamPlaybackController {
|
|
|
811
1171
|
}): Promise<boolean>;
|
|
812
1172
|
handleEvent(event: StagingStreamEvent): void;
|
|
813
1173
|
cleanup(): void;
|
|
1174
|
+
private resetRealtimeSpeech;
|
|
814
1175
|
private playTimeline;
|
|
815
1176
|
private flushOptions;
|
|
816
1177
|
private recordStreamEvent;
|
|
817
1178
|
private recordPlaybackEvent;
|
|
818
1179
|
private errorReason;
|
|
819
1180
|
private toSpeechEvent;
|
|
820
|
-
private
|
|
821
|
-
private invokeTimelineStartCallback;
|
|
822
|
-
private invokeSpeechPlaybackStartCallback;
|
|
823
|
-
private invokeCharacterRevealCallback;
|
|
824
|
-
private invokeCallback;
|
|
1181
|
+
private safeInvoke;
|
|
825
1182
|
}
|
|
826
1183
|
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
/**
|
|
837
|
-
|
|
838
|
-
|
|
1184
|
+
declare class CharacterExpressions {
|
|
1185
|
+
readonly mixer: FacialPerformanceMixer;
|
|
1186
|
+
constructor(mixer: FacialPerformanceMixer);
|
|
1187
|
+
/** All available VRM expression channel names on the loaded character. */
|
|
1188
|
+
get all(): string[];
|
|
1189
|
+
/** Available emotion expression channel names (`happy`, `angry`, `sad`, etc.). */
|
|
1190
|
+
get emotions(): string[];
|
|
1191
|
+
/** Available mouth/viseme expression channel names (`aa`, `ee`, `ih`, `oh`, `ou`). */
|
|
1192
|
+
get visemes(): string[];
|
|
1193
|
+
/** Current applied weights for all expression channels in `[0, 1]`. */
|
|
1194
|
+
get weights(): Record<string, number>;
|
|
1195
|
+
/** Real-time aggregate mouth openness in `[0, 1]`. */
|
|
1196
|
+
get mouthOpenness(): number;
|
|
1197
|
+
/** Active mouth ownership mode (`'expression'` or `'viseme'`). */
|
|
1198
|
+
get mouthMode(): FacialMouthMode;
|
|
1199
|
+
/** Whether automatic idle blinking and eye movement are active on this face. */
|
|
1200
|
+
get hasIdleFace(): boolean;
|
|
1201
|
+
/** Full performance frame snapshot including layered weights and applied values. */
|
|
1202
|
+
get snapshot(): FacialPerformanceFrameSnapshot;
|
|
1203
|
+
/** Read the current applied weight of a single expression channel in `[0, 1]`. */
|
|
1204
|
+
getWeight(name: FacialChannelName): number;
|
|
1205
|
+
/** Drive an expression channel target weight in `[0, 1]` (eased during `update(delta)`). */
|
|
1206
|
+
setWeight(name: FacialChannelName, weight: number): void;
|
|
1207
|
+
/** Play a timed expression cue and ease back to neutral after `durationMs`. */
|
|
1208
|
+
play(name: FacialChannelName, options?: FacialCueOptions): void;
|
|
1209
|
+
/** Set the active emotion expression channel and weight. */
|
|
1210
|
+
setEmotion(name: FacialChannelName, weight?: number, mouthMode?: FacialMouthMode): void;
|
|
1211
|
+
/** Clear the active emotion expression layer. */
|
|
1212
|
+
clearEmotion(mouthMode?: FacialMouthMode): void;
|
|
1213
|
+
/** Reset all expression layer targets to zero. */
|
|
1214
|
+
reset(): void;
|
|
839
1215
|
}
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1216
|
+
|
|
1217
|
+
interface CharacterRuntimeInitOptions extends Omit<NpcifyOptions, 'apiToken' | 'assetProfile' | 'apiKey'> {
|
|
1218
|
+
apiToken: string;
|
|
1219
|
+
assetProfile: 'original' | 'web-optimized-v1';
|
|
1220
|
+
loadAnimation?: (animationId: string) => Promise<VRMAnimation>;
|
|
843
1221
|
}
|
|
844
|
-
declare class
|
|
1222
|
+
declare class CharacterRuntime {
|
|
845
1223
|
readonly baseUrl: string;
|
|
846
1224
|
readonly characterId: string;
|
|
847
1225
|
readonly apiToken: string;
|
|
848
1226
|
readonly assetProfile: 'original' | 'web-optimized-v1';
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
1227
|
+
readonly audio: CharacterAudio;
|
|
1228
|
+
readonly expressions: CharacterExpressions;
|
|
1229
|
+
readonly animation: CharacterAnimation;
|
|
1230
|
+
animations: Map<string, VRMAnimation>;
|
|
1231
|
+
animationController?: AnimationControllerLike;
|
|
1232
|
+
protected readonly sceneRig: CharacterSceneRig;
|
|
1233
|
+
protected readonly lazyAnimations: LazyAnimations;
|
|
1234
|
+
protected audioPlayer: AudioPlayer | null;
|
|
1235
|
+
protected stagingPlayer: StagingPlayer | null;
|
|
1236
|
+
protected readonly facialMixer: FacialPerformanceMixer;
|
|
1237
|
+
protected readonly visemePlayer: VisemePlayer;
|
|
1238
|
+
protected readonly performanceRecorder: PerformanceRecorder;
|
|
1239
|
+
protected readonly telemetry: ClientTelemetry;
|
|
1240
|
+
protected readonly streamPlayback: StreamPlaybackController;
|
|
1241
|
+
protected defaultParent: Object3D | null;
|
|
1242
|
+
protected currentGazeTarget: Object3D | null;
|
|
1243
|
+
/** The current gaze target came from `lookAt()`. */
|
|
1244
|
+
protected lookAtActive: boolean;
|
|
1245
|
+
protected thinkingState: boolean;
|
|
1246
|
+
protected errorCallback: ((error: Error) => void) | null;
|
|
1247
|
+
constructor(options: CharacterRuntimeInitOptions);
|
|
1248
|
+
/** Self-reference so `npcify.character` and `character` share the same interface. */
|
|
1249
|
+
get character(): this;
|
|
1250
|
+
/** Loaded Three-VRM instance, or `null` before `loadAssets()`. */
|
|
1251
|
+
get vrm(): VRM | null;
|
|
1252
|
+
set vrm(value: VRM | null);
|
|
1253
|
+
/** True once the character VRM model is loaded. */
|
|
1254
|
+
get ready(): boolean;
|
|
1255
|
+
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
1256
|
+
get scene(): Group | null;
|
|
1257
|
+
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
1258
|
+
get object3d(): Group | null;
|
|
1259
|
+
/** Alias for `scene`. */
|
|
1260
|
+
get root(): Group | null;
|
|
1261
|
+
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
1262
|
+
get mesh(): SkinnedMesh | Mesh | null;
|
|
1263
|
+
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
1264
|
+
get meshes(): readonly Mesh[];
|
|
1265
|
+
/** All Three.js `SkinnedMesh` instances in the character hierarchy. */
|
|
1266
|
+
get skinnedMeshes(): readonly SkinnedMesh[];
|
|
1267
|
+
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
1268
|
+
get materials(): readonly Material[];
|
|
1269
|
+
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
1270
|
+
get skeletons(): readonly Skeleton[];
|
|
1271
|
+
/** Three-VRM humanoid rig when loaded. */
|
|
1272
|
+
get humanoid(): VRMHumanoid | null;
|
|
1273
|
+
/** Underlying Three.js `AnimationMixer` driving skeletal animations. */
|
|
1274
|
+
get mixer(): AnimationMixer | null;
|
|
1275
|
+
/** The `Object3D` (such as a `THREE.Camera`) the character's eyes track. */
|
|
1276
|
+
get gazeTarget(): Object3D | null;
|
|
1277
|
+
set gazeTarget(target: Object3D | null);
|
|
1278
|
+
/** True while the character is actively delivering a speech block or playing voice audio. */
|
|
1279
|
+
get speaking(): boolean;
|
|
1280
|
+
/** True while waiting for the next AI turn to begin streaming or playing. */
|
|
1281
|
+
get thinking(): boolean;
|
|
1282
|
+
/** Active speech block metadata during playback, or `null` when silent. */
|
|
1283
|
+
get currentSpeech(): StagingSpeechPlaybackEvent | null;
|
|
1284
|
+
/** Full text of the current or most recent speech block. */
|
|
1285
|
+
get currentText(): string;
|
|
1286
|
+
/** Subtitle text revealed so far for the current or most recent speech block. */
|
|
1287
|
+
get revealedText(): string;
|
|
1288
|
+
/** Latest reply options delivered for the current turn. */
|
|
1289
|
+
get options(): StagingResultPayload['options'];
|
|
1290
|
+
/** Real-time playback cursor within the active dialogue block, or `null` when idle. */
|
|
1291
|
+
get cursor(): CharacterPlaybackCursor | null;
|
|
1292
|
+
/** Combined snapshot of real-time speech and dialogue state. */
|
|
1293
|
+
get speech(): CharacterSpeechState;
|
|
1294
|
+
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
1295
|
+
getBone(name: VRMHumanBoneName): Object3D | null;
|
|
1296
|
+
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
1297
|
+
getNormalizedBone(name: VRMHumanBoneName): Object3D | null;
|
|
1298
|
+
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
1299
|
+
attachTo(parent: Object3D): Group;
|
|
1300
|
+
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
1301
|
+
detach(): void;
|
|
1302
|
+
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
1303
|
+
setShadows(options?: CharacterShadowOptions): void;
|
|
1304
|
+
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
1305
|
+
setTransform(options: CharacterTransformOptions): void;
|
|
1306
|
+
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
1307
|
+
refreshMeshes(): void;
|
|
1308
|
+
ensureAnimation(animationId: string): Promise<void>;
|
|
1309
|
+
/** Play a character animation by ID, automatically loading it if lazy. */
|
|
1310
|
+
playAnimation(animationId: string, options?: CharacterAnimationPlayOptions): Promise<void>;
|
|
1311
|
+
/** Transition back to the character's idle animation. */
|
|
1312
|
+
playIdle(fadeDuration?: number): void;
|
|
1313
|
+
/** Stop the currently playing skeletal animation. */
|
|
1314
|
+
stopAnimation(fadeDuration?: number): void;
|
|
1315
|
+
unlockAudio(): Promise<boolean>;
|
|
1316
|
+
getExpressions(): string[];
|
|
1317
|
+
getMouthExpressionNames(): string[];
|
|
1318
|
+
getExpressionNames(): string[];
|
|
1319
|
+
playExpression(name: FacialChannelName, options?: FacialCueOptions): void;
|
|
1320
|
+
setExpressionWeight(name: FacialChannelName, weight: number): void;
|
|
1321
|
+
getExpressionWeight(name: FacialChannelName): number;
|
|
1322
|
+
setEmotion(name: FacialChannelName, weight?: number, mouthMode?: FacialMouthMode): void;
|
|
1323
|
+
clearEmotion(mouthMode?: FacialMouthMode): void;
|
|
1324
|
+
/**
|
|
1325
|
+
* The object the eyes of a template face follow (the camera); null looks
|
|
1326
|
+
* ahead. It replaces a `lookAt()` target. Other faces do not follow it.
|
|
1327
|
+
*/
|
|
1328
|
+
setGazeTarget(target: Object3D | null): void;
|
|
1329
|
+
/**
|
|
1330
|
+
* Direct the character's gaze toward a Three.js `Object3D` or world
|
|
1331
|
+
* coordinates. A template face keeps its gaze cone, idle eye moves and the
|
|
1332
|
+
* look cues of staging; other faces turn their eyes with three-vrm's
|
|
1333
|
+
* `vrm.lookAt.target`. Coordinates must be finite numbers, else this throws
|
|
1334
|
+
* a `RangeError` and keeps the current target.
|
|
1335
|
+
*/
|
|
1336
|
+
lookAt(target: CharacterLookAtTarget): void;
|
|
1337
|
+
lookAt(x: number, y: number, z: number): void;
|
|
1338
|
+
/** Reset the character's gaze target so the eyes look straight ahead. */
|
|
1339
|
+
clearLookAt(): void;
|
|
1340
|
+
startPerformanceRecording(options?: PerformanceRecordingStartOptions): void;
|
|
1341
|
+
stopPerformanceRecording(): PerformanceRecording | null;
|
|
1342
|
+
getPerformanceRecording(): PerformanceRecording | null;
|
|
1343
|
+
recordStagingRoute(stagingId: string | null, state: 'selected' | 'emitted' | 'received'): void;
|
|
1344
|
+
onError(callback: (error: Error) => void): () => void;
|
|
1345
|
+
onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): () => void;
|
|
1346
|
+
onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): () => void;
|
|
1347
|
+
onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): () => void;
|
|
1348
|
+
onPlaybackBlocked(callback: (stagingId: string) => void): () => void;
|
|
1349
|
+
onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): () => void;
|
|
1350
|
+
onPlaybackCursor(callback: Parameters<StreamPlaybackController['onPlaybackCursor']>[0]): () => void;
|
|
1351
|
+
onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): () => void;
|
|
1352
|
+
/** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
|
|
1353
|
+
update(delta: number): void;
|
|
1354
|
+
protected readonly getAudioPlayer: () => AudioPlayer;
|
|
1355
|
+
protected readonly getStagingPlayer: () => StagingPlayer;
|
|
1356
|
+
/**
|
|
1357
|
+
* A template face follows the gaze target with its IdleFace. Only a
|
|
1358
|
+
* `lookAt()` target moves the eyes of another face, through three-vrm's
|
|
1359
|
+
* `vrm.lookAt.target`; on a template face that target would stop the
|
|
1360
|
+
* IdleFace (its gaze cone, idle eye moves and look cues).
|
|
1361
|
+
*/
|
|
1362
|
+
protected applyGaze(): void;
|
|
1363
|
+
protected resetRuntimeAssets(): void;
|
|
1364
|
+
protected disposeRuntimeAssets(): void;
|
|
1365
|
+
protected readonly handleCallbackError: (context: string, error: unknown) => void;
|
|
1366
|
+
protected readonly reportError: (error: Error, context?: string) => void;
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
declare class Character extends CharacterRuntime {
|
|
857
1370
|
private readonly conversationSocket;
|
|
858
1371
|
private renderSessionId;
|
|
859
1372
|
private initialAssetInfo;
|
|
860
1373
|
private stagingResultCallback;
|
|
861
1374
|
private thinkingCallback;
|
|
862
|
-
private errorCallback;
|
|
863
1375
|
private readonly notifiedStagingIds;
|
|
864
1376
|
private readonly liveConversation;
|
|
865
1377
|
private baked;
|
|
866
1378
|
private readonly authoring;
|
|
867
1379
|
private readonly bakedAsset?;
|
|
868
|
-
|
|
869
|
-
animations: Map<string, VRMAnimation>;
|
|
1380
|
+
private animationRefreshId;
|
|
870
1381
|
conversationId: string | null;
|
|
871
|
-
|
|
872
|
-
|
|
1382
|
+
constructor(options: NpcifyOptions);
|
|
1383
|
+
load(callback?: (info: AssetLoadProgress) => void): Promise<this>;
|
|
873
1384
|
loadAssets(callback?: (info: AssetLoadProgress) => void): Promise<void>;
|
|
874
|
-
private animationRefreshId;
|
|
875
1385
|
reloadAnimations(info?: NpcifyAssetInfo): Promise<void>;
|
|
876
|
-
ensureAnimation(animationId: string): Promise<void>;
|
|
877
1386
|
reloadAssets(): Promise<void>;
|
|
878
1387
|
initializeRemote(): Promise<void>;
|
|
879
1388
|
createConversation(options?: {
|
|
@@ -884,48 +1393,42 @@ declare class Npcify {
|
|
|
884
1393
|
connect(conversationId?: string): Promise<string>;
|
|
885
1394
|
disconnect(): Promise<void>;
|
|
886
1395
|
endConversation(): Promise<void>;
|
|
1396
|
+
dispose(options?: CharacterDisposeOptions): Promise<void>;
|
|
887
1397
|
sendMessage(message: string, options?: ConversationTurnOptions): Promise<StagingResultPayload>;
|
|
888
1398
|
sendMessage(conversationId: string, message: string, options?: ConversationTurnOptions): Promise<StagingResultPayload>;
|
|
889
1399
|
replayStaging(stagingId: string): Promise<StagingResultPayload>;
|
|
890
1400
|
replayStaging(conversationId: string, stagingId: string, options?: PlayStagingOptions): Promise<StagingResultPayload>;
|
|
891
1401
|
get isBakedConversation(): boolean;
|
|
892
1402
|
getConversationWorldline(conversationId?: string | null): Promise<ConversationWorldline>;
|
|
893
|
-
/** Feed an owner-authoring result through the normal event and playback path. */
|
|
894
1403
|
acceptAuthoringResult(payload: StagingResultPayload): Promise<void>;
|
|
895
1404
|
playStaging(payload: StagingResultPayload, options?: PlayStagingOptions): Promise<boolean>;
|
|
896
|
-
unlockAudio(): Promise<boolean>;
|
|
897
1405
|
setRenderSessionId(renderSessionId: string | null): void;
|
|
898
|
-
onStagingResult(callback: (payload: StagingResultPayload) => Promise<void> | void): void;
|
|
899
|
-
onThinking(callback: (payload: LlmThinkingPayload) => Promise<void> | void): void;
|
|
900
|
-
onError(callback: (error: Error) => void): void;
|
|
901
|
-
onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): void;
|
|
902
|
-
onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): void;
|
|
903
|
-
onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): void;
|
|
904
|
-
onPlaybackBlocked(callback: (stagingId: string) => void): void;
|
|
905
|
-
onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): void;
|
|
906
|
-
onPlaybackCursor(callback: Parameters<StreamPlaybackController['onPlaybackCursor']>[0]): void;
|
|
907
|
-
onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): void;
|
|
908
|
-
getExpressions(): string[];
|
|
909
|
-
getMouthExpressionNames(): string[];
|
|
910
|
-
getExpressionNames(): string[];
|
|
911
|
-
playExpression(name: string, options?: FacialCueOptions): void;
|
|
912
|
-
startPerformanceRecording(options?: PerformanceRecordingStartOptions): void;
|
|
913
|
-
stopPerformanceRecording(): PerformanceRecording | null;
|
|
914
|
-
getPerformanceRecording(): PerformanceRecording | null;
|
|
915
|
-
recordStagingRoute(stagingId: string | null, state: 'selected' | 'emitted' | 'received'): void;
|
|
916
|
-
update(delta: number): void;
|
|
1406
|
+
onStagingResult(callback: (payload: StagingResultPayload) => Promise<void> | void): () => void;
|
|
1407
|
+
onThinking(callback: (payload: LlmThinkingPayload) => Promise<void> | void): () => void;
|
|
917
1408
|
private resetAssets;
|
|
918
1409
|
private readonly getAssetInfo;
|
|
919
1410
|
private handleStagingResult;
|
|
920
|
-
private shouldNotifyResult;
|
|
921
|
-
private invokeStagingResultCallback;
|
|
922
1411
|
private invokeThinkingCallback;
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
private readonly handleCallbackError;
|
|
926
|
-
private readonly reportError;
|
|
1412
|
+
}
|
|
1413
|
+
declare class Npcify extends Character {
|
|
927
1414
|
}
|
|
928
1415
|
|
|
1416
|
+
interface LoadCharacterOptions extends CharacterOptions {
|
|
1417
|
+
/** Optional asset download progress callback. */
|
|
1418
|
+
onProgress?: (progress: AssetLoadProgress) => void;
|
|
1419
|
+
/** Automatically create and start a conversation once the model finishes loading. */
|
|
1420
|
+
autoStartConversation?: boolean;
|
|
1421
|
+
/** Start the opening turn silently when `autoStartConversation` is enabled. */
|
|
1422
|
+
silent?: boolean;
|
|
1423
|
+
}
|
|
1424
|
+
/** Instantiate an unloaded `Character` instance. Call `await character.load()` to load 3D assets. */
|
|
1425
|
+
declare const createCharacter: (options: CharacterOptions) => Character;
|
|
1426
|
+
/**
|
|
1427
|
+
* Load an AI character's VRM model, humanoid skeleton, facial expressions, and animations
|
|
1428
|
+
* for direct integration into a custom Three.js `Scene` and render loop.
|
|
1429
|
+
*/
|
|
1430
|
+
declare const loadCharacter: (options: LoadCharacterOptions) => Promise<Character>;
|
|
1431
|
+
|
|
929
1432
|
type PerformanceIssueSeverity = 'error' | 'warning';
|
|
930
1433
|
interface PerformanceIssue {
|
|
931
1434
|
code: 'render.frame-gap' | 'timeline.invalid-entry' | 'timeline.non-monotonic' | 'viseme.audio-coverage-gap' | 'viseme.cue-late';
|
|
@@ -973,6 +1476,7 @@ declare class BakedConversation {
|
|
|
973
1476
|
private activeId;
|
|
974
1477
|
readonly asset: BakedConversationAsset;
|
|
975
1478
|
constructor(asset: BakedConversationAsset, characterId: string);
|
|
1479
|
+
assertAnimationsAvailable(animationIds: Iterable<string>): void;
|
|
976
1480
|
start(): StagingResultPayload;
|
|
977
1481
|
select(stagingId: string): StagingResultPayload;
|
|
978
1482
|
reply(message: string, parentStagingId?: string | null): {
|
|
@@ -994,15 +1498,21 @@ interface MountNpcifyOptions {
|
|
|
994
1498
|
baseUrl: string;
|
|
995
1499
|
characterId: string;
|
|
996
1500
|
apiKey?: string;
|
|
1501
|
+
apiToken?: string;
|
|
997
1502
|
serverRendering?: boolean;
|
|
998
1503
|
assetProfile?: 'original' | 'web-optimized-v1';
|
|
999
1504
|
onError?: (error: Error) => void;
|
|
1000
1505
|
}
|
|
1001
1506
|
interface NpcifyView {
|
|
1002
1507
|
npcify: Npcify;
|
|
1508
|
+
character?: Character;
|
|
1509
|
+
scene?: Scene | null;
|
|
1510
|
+
camera?: PerspectiveCamera | null;
|
|
1511
|
+
renderer?: WebGLRenderer | null;
|
|
1512
|
+
controls?: OrbitControls | null;
|
|
1003
1513
|
dispose(): Promise<void>;
|
|
1004
1514
|
}
|
|
1005
1515
|
/** Mount into an existing canvas. Its CSS must give it a nonzero width and height. */
|
|
1006
1516
|
declare const mountNpcify: (options: MountNpcifyOptions) => Promise<NpcifyView>;
|
|
1007
1517
|
|
|
1008
|
-
export { type AudioPerformanceFrameSnapshot, BakedConversation, type BakedConversationAsset, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, type MountNpcifyOptions, Npcify, type NpcifyAssetInfo, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, type RepeatedVisemeTransitionAnalysis, type StagingOptionsReadyPayload, type StagingSpeechPlaybackEvent, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, decodePerformance, encodePerformance, loadBakedConversation, mountNpcify };
|
|
1518
|
+
export { AnimationController, type AnimationControllerLike, type AssetLoadProgress, type AudioPerformanceFrameSnapshot, AudioPlayer, BakedConversation, type BakedConversationAsset, Character, CharacterAnimation, type CharacterAnimationPlayOptions, CharacterAudio, type CharacterDisposeOptions, CharacterExpressions, type CharacterLookAtTarget, type CharacterOptions, type CharacterPlaybackCursor, CharacterRuntime, type CharacterRuntimeInitOptions, CharacterSceneRig, type CharacterShadowOptions, type CharacterSpeechState, type CharacterTransformOptions, type ConversationTurnOptions, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, type LlmThinkingPayload, type LoadCharacterOptions, type MountNpcifyOptions, Npcify, type NpcifyAssetInfo, Character as NpcifyCharacter, type NpcifyOptions, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, RemoteAnimationController, type RepeatedVisemeTransitionAnalysis, type StagingOptionsReadyPayload, type StagingPlaybackStart, type StagingSpeechPlaybackEvent, type Vector3Like, type Vector3Tuple, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, createCharacter, decodePerformance, encodePerformance, loadBakedConversation, loadCharacter, mountNpcify };
|