@npcify/web 0.4.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -0
- package/dist/index.d.ts +463 -57
- package/dist/index.js +2757 -398
- package/examples/index.html +1 -0
- package/examples/main.js +22 -0
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -3,9 +3,85 @@ import { VRMAnimation } from '@pixiv/three-vrm-animation';
|
|
|
3
3
|
import { AnimationMixer, AnimationAction, Object3D, Vector3, Group, SkinnedMesh, Mesh, Material, Skeleton, Euler, Scene, PerspectiveCamera, WebGLRenderer } from 'three';
|
|
4
4
|
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Sight (docs/sight.md): a client captures frames, the server sends each
|
|
8
|
+
* frame to the vision provider that the owner of the character chose, and the
|
|
9
|
+
* character reads the text observations in its next replies. Frames are never
|
|
10
|
+
* stored.
|
|
11
|
+
*/
|
|
12
|
+
/** The vision providers. No other provider takes over when one fails. */
|
|
13
|
+
declare const SIGHT_PROVIDERS: readonly ["gemini", "openai"];
|
|
14
|
+
type SightProvider = (typeof SIGHT_PROVIDERS)[number];
|
|
15
|
+
/** The codes of a refused frame. A newer server can send other codes. */
|
|
16
|
+
declare const SIGHT_ERRORS: readonly ["SIGHT_DISABLED", "SIGHT_RATE_LIMITED", "SIGHT_FRAME_TOO_LARGE", "SIGHT_FRAME_INVALID", "SIGHT_PROVIDER_FAILED", "INSUFFICIENT_POINTS", "CONVERSATION_ACCESS_DENIED"];
|
|
17
|
+
type SightErrorCode = (typeof SIGHT_ERRORS)[number];
|
|
18
|
+
type SightPersonPosition = 'left' | 'center' | 'right';
|
|
19
|
+
/** One person or avatar in a frame. */
|
|
20
|
+
type SightPerson = {
|
|
21
|
+
position: SightPersonPosition;
|
|
22
|
+
/** What the person does, in five words or fewer. */
|
|
23
|
+
action: string;
|
|
24
|
+
/** The person faces or addresses the camera. */
|
|
25
|
+
facingCamera: boolean;
|
|
26
|
+
};
|
|
27
|
+
/** What the vision model saw in one frame. */
|
|
28
|
+
type SightObservation = {
|
|
29
|
+
/** The number of people or avatars in the frame. */
|
|
30
|
+
people: number;
|
|
31
|
+
persons: SightPerson[];
|
|
32
|
+
/** Each readable piece of text, verbatim. */
|
|
33
|
+
text: string[];
|
|
34
|
+
};
|
|
35
|
+
/** `sight_observation`: an observation of the conversation, sent to its room. */
|
|
36
|
+
type SightObservationEvent = {
|
|
37
|
+
conversationId: string;
|
|
38
|
+
observationId: string;
|
|
39
|
+
/** The capture time that the client sent, as an ISO 8601 string. */
|
|
40
|
+
capturedAt: string;
|
|
41
|
+
observation: SightObservation;
|
|
42
|
+
/** One sentence about the frame, for the character. */
|
|
43
|
+
summary: string;
|
|
44
|
+
model: string;
|
|
45
|
+
/** The points charged to the owner; null while the charge is pending. */
|
|
46
|
+
costPoints: number | null;
|
|
47
|
+
};
|
|
48
|
+
|
|
6
49
|
type TtsProviderId = string;
|
|
7
50
|
type NullableTtsProviderId = TtsProviderId | null;
|
|
8
51
|
|
|
52
|
+
declare const DEFAULT_VARIANT_KEY: "default";
|
|
53
|
+
declare const NUDE_VARIANT_KEY: "nude";
|
|
54
|
+
interface AvatarVariantSummary {
|
|
55
|
+
key: string;
|
|
56
|
+
label: string;
|
|
57
|
+
description: string;
|
|
58
|
+
r18: boolean;
|
|
59
|
+
portraitUrl: string | null;
|
|
60
|
+
creditText?: string | null;
|
|
61
|
+
}
|
|
62
|
+
declare const STAGING_TOOL_NAMES: readonly ["switch_variant"];
|
|
63
|
+
type StagingToolName = (typeof STAGING_TOOL_NAMES)[number];
|
|
64
|
+
declare const STAGING_TOOL_RESULT_STATUSES: readonly ["applied", "noop", "rejected"];
|
|
65
|
+
type StagingToolResultStatus = (typeof STAGING_TOOL_RESULT_STATUSES)[number];
|
|
66
|
+
interface StagingSwitchVariantArguments {
|
|
67
|
+
variant: string;
|
|
68
|
+
}
|
|
69
|
+
interface StagingToolCallResult {
|
|
70
|
+
ok: boolean;
|
|
71
|
+
status: StagingToolResultStatus;
|
|
72
|
+
previousVariant: string;
|
|
73
|
+
activeVariant: string;
|
|
74
|
+
error?: string;
|
|
75
|
+
}
|
|
76
|
+
interface StagingToolCall {
|
|
77
|
+
id: string;
|
|
78
|
+
name: StagingToolName;
|
|
79
|
+
/** Seconds from the start of the staging. */
|
|
80
|
+
timestamp: number;
|
|
81
|
+
arguments: StagingSwitchVariantArguments;
|
|
82
|
+
result: StagingToolCallResult;
|
|
83
|
+
}
|
|
84
|
+
|
|
9
85
|
declare const EXPRESSION_CALL_RANGES: readonly ["linear", "bool"];
|
|
10
86
|
type ExpressionCallRange = (typeof EXPRESSION_CALL_RANGES)[number];
|
|
11
87
|
type ExpressionCallRangeMap = Record<string, ExpressionCallRange>;
|
|
@@ -18,6 +94,14 @@ type Sqt = {
|
|
|
18
94
|
quaternion?: [number, number, number, number] | null;
|
|
19
95
|
translation?: [number, number, number] | null;
|
|
20
96
|
};
|
|
97
|
+
/**
|
|
98
|
+
* The owner's sight switch and vision provider (docs/sight.md). `enabled` is
|
|
99
|
+
* true only with a provider; an SDK captures frames only then.
|
|
100
|
+
*/
|
|
101
|
+
type AvatarSightSettings = {
|
|
102
|
+
enabled: boolean;
|
|
103
|
+
provider: SightProvider | null;
|
|
104
|
+
};
|
|
21
105
|
interface AvatarPublicInfo {
|
|
22
106
|
id: string;
|
|
23
107
|
name: string | null;
|
|
@@ -39,6 +123,7 @@ interface AvatarPublicInfo {
|
|
|
39
123
|
freeFirstNMessages?: number | null;
|
|
40
124
|
hostedCharacterEnabled?: boolean;
|
|
41
125
|
listedPublicly?: boolean;
|
|
126
|
+
allowFork?: boolean;
|
|
42
127
|
r18?: boolean;
|
|
43
128
|
bakedConversationEnabled?: boolean;
|
|
44
129
|
bakedConversationRevision?: number;
|
|
@@ -48,11 +133,23 @@ interface AvatarPublicInfo {
|
|
|
48
133
|
* `allowCustomReply`. Missing on older servers: off.
|
|
49
134
|
*/
|
|
50
135
|
speechInputEnabled?: boolean;
|
|
136
|
+
/** Sight (Sight tab). Missing on older servers: off. */
|
|
137
|
+
sight?: AvatarSightSettings;
|
|
51
138
|
autoPlay: boolean;
|
|
52
139
|
serverRenderingEnabled: boolean;
|
|
53
140
|
/** The character's body motion is generated live in each turn (missing on older servers). */
|
|
54
141
|
realtimeMotionEnabled?: boolean;
|
|
142
|
+
/** After the free messages, the player pays live motion and speech recognition (missing on older servers: off). */
|
|
143
|
+
playerPaysAiCosts?: boolean;
|
|
144
|
+
/** On the hosted web page, the character reads and saves memories about signed-in players (missing on older servers: off). */
|
|
145
|
+
memoryEnabled?: boolean;
|
|
146
|
+
/** In SDK chats, the character reads and saves memories keyed by memoryIdentifier or the owner when empty (missing on older servers: off). */
|
|
147
|
+
sdkMemoryEnabled?: boolean;
|
|
148
|
+
/** On live turns, the character gets the player's local message time (missing on older servers: off). */
|
|
149
|
+
messageTimeEnabled?: boolean;
|
|
55
150
|
cameraSqt?: Sqt | null;
|
|
151
|
+
/** Allowed appearance variants (`default` first) when the character has variants. */
|
|
152
|
+
variants?: AvatarVariantSummary[];
|
|
56
153
|
}
|
|
57
154
|
|
|
58
155
|
interface UsageChargeSummary {
|
|
@@ -78,6 +175,9 @@ interface ConversationWorldlineStaging {
|
|
|
78
175
|
text: string;
|
|
79
176
|
options: ConversationOption[];
|
|
80
177
|
createdAt: string;
|
|
178
|
+
initialVariant?: string;
|
|
179
|
+
activeVariant?: string;
|
|
180
|
+
toolCalls?: StagingToolCall[];
|
|
81
181
|
}
|
|
82
182
|
interface ConversationWorldline {
|
|
83
183
|
conversationId: string;
|
|
@@ -233,6 +333,12 @@ interface StagingTimeline {
|
|
|
233
333
|
motions?: StagingMotionClip[];
|
|
234
334
|
/** Stored live motion frames, on reads of a completed staging (NPCP 1.1). */
|
|
235
335
|
motion?: StagingMotionTrack;
|
|
336
|
+
/** Variant active at the start of the staging (NPCP 1.2; default is `default`). */
|
|
337
|
+
initialVariant?: string;
|
|
338
|
+
/** Variant active at the end of the staging (NPCP 1.2; default is `default`). */
|
|
339
|
+
activeVariant?: string;
|
|
340
|
+
/** Recorded agent tool calls and results on the staging clock (NPCP 1.2). */
|
|
341
|
+
toolCalls?: StagingToolCall[];
|
|
236
342
|
}
|
|
237
343
|
declare const STAGING_PROTOCOL_VERSION: 1;
|
|
238
344
|
interface StagingResultPayload {
|
|
@@ -402,7 +508,31 @@ interface ConversationTurnOptions {
|
|
|
402
508
|
requestId?: string;
|
|
403
509
|
}
|
|
404
510
|
|
|
405
|
-
type NpcifyAssetInfo = Pick<AvatarPublicInfo, 'animations' | 'idleAnimationId' | 'bakedConversationEnabled' | 'bakedConversationRevision'> & Partial<Pick<AvatarPublicInfo, 'allowCustomReply' | 'speechInputEnabled'>>;
|
|
511
|
+
type NpcifyAssetInfo = Pick<AvatarPublicInfo, 'animations' | 'idleAnimationId' | 'bakedConversationEnabled' | 'bakedConversationRevision'> & Partial<Pick<AvatarPublicInfo, 'allowCustomReply' | 'speechInputEnabled' | 'sight' | 'sdkMemoryEnabled' | 'variants'>>;
|
|
512
|
+
interface FetchVrmDownloadUrlOptions {
|
|
513
|
+
apiToken?: string;
|
|
514
|
+
assetProfile?: 'original' | 'web-optimized-v1';
|
|
515
|
+
signal?: AbortSignal;
|
|
516
|
+
}
|
|
517
|
+
type ResourceProgressCallback = (resourceId: string, loaded: number, total: number) => void;
|
|
518
|
+
declare const fetchVrmDownloadUrl: (baseUrl: string, characterId: string, options?: FetchVrmDownloadUrlOptions) => Promise<string>;
|
|
519
|
+
declare const fetchVariantVrmDownloadUrl: (baseUrl: string, characterId: string, variantKey: string, options?: FetchVrmDownloadUrlOptions) => Promise<string>;
|
|
520
|
+
declare const loadVrm: ({ baseUrl, characterId, variantKey, apiToken, onProgress, assetProfile, }: {
|
|
521
|
+
baseUrl: string;
|
|
522
|
+
characterId: string;
|
|
523
|
+
variantKey?: string;
|
|
524
|
+
apiToken?: string;
|
|
525
|
+
assetProfile?: "original" | "web-optimized-v1";
|
|
526
|
+
onProgress?: ResourceProgressCallback;
|
|
527
|
+
}) => Promise<VRM>;
|
|
528
|
+
declare const loadVariantVrm: ({ baseUrl, characterId, variantKey, apiToken, onProgress, assetProfile, }: {
|
|
529
|
+
baseUrl: string;
|
|
530
|
+
characterId: string;
|
|
531
|
+
variantKey: string;
|
|
532
|
+
apiToken?: string;
|
|
533
|
+
assetProfile?: "original" | "web-optimized-v1";
|
|
534
|
+
onProgress?: ResourceProgressCallback;
|
|
535
|
+
}) => Promise<VRM>;
|
|
406
536
|
|
|
407
537
|
interface LlmThinkingPayload {
|
|
408
538
|
taskId?: string | null;
|
|
@@ -683,6 +813,7 @@ interface AnimationControllerLike {
|
|
|
683
813
|
previewAnimation?: (animation: VRMAnimation) => () => void;
|
|
684
814
|
hasAnimation?: (id: string) => boolean;
|
|
685
815
|
addAnimation?: (animationId: string, animation: VRMAnimation, config?: AnimationConfig) => void;
|
|
816
|
+
setVrm?: (nextVrm: VRM, rawAnimations: ReadonlyMap<string, VRMAnimation>) => void;
|
|
686
817
|
clearAnimations?: () => void;
|
|
687
818
|
setIdleAnimation?: (animationId: string | null) => void;
|
|
688
819
|
getAnimationMixer?: () => AnimationMixer;
|
|
@@ -702,9 +833,9 @@ declare class RemoteAnimationController implements AnimationControllerLike {
|
|
|
702
833
|
update(_delta: number): void;
|
|
703
834
|
}
|
|
704
835
|
declare class AnimationController implements AnimationControllerLike {
|
|
705
|
-
private
|
|
836
|
+
private vrm;
|
|
706
837
|
prepareAnimations?: (ids: string[]) => Promise<void>;
|
|
707
|
-
private
|
|
838
|
+
private mixer;
|
|
708
839
|
private readonly clips;
|
|
709
840
|
private readonly rawClips;
|
|
710
841
|
private readonly normalizedIdleClips;
|
|
@@ -717,6 +848,7 @@ declare class AnimationController implements AnimationControllerLike {
|
|
|
717
848
|
private idleAnimationId;
|
|
718
849
|
private pendingReturnToIdleAction;
|
|
719
850
|
constructor(vrm: VRM, idleAnimationId?: string | null, options?: AnimationControllerOptions);
|
|
851
|
+
setVrm(nextVrm: VRM, rawAnimations: ReadonlyMap<string, VRMAnimation>): void;
|
|
720
852
|
getAnimationMixer(): AnimationMixer;
|
|
721
853
|
getActiveAction(): AnimationAction | null;
|
|
722
854
|
getActiveAnimationId(): string | null;
|
|
@@ -876,8 +1008,11 @@ declare class FacialPerformanceMixer {
|
|
|
876
1008
|
private restoreMouthAfterVisemeRelease;
|
|
877
1009
|
private idleFace;
|
|
878
1010
|
private gazeTarget;
|
|
1011
|
+
private readonly listeningFace;
|
|
879
1012
|
constructor(recorder?: PerformanceRecorder | undefined, options?: FacialPerformanceMixerOptions);
|
|
880
|
-
setVrm(vrm: VRM | null
|
|
1013
|
+
setVrm(vrm: VRM | null, options?: {
|
|
1014
|
+
preserveState?: boolean;
|
|
1015
|
+
}): void;
|
|
881
1016
|
getChannelNames(): string[];
|
|
882
1017
|
getEmotionChannelNames(): string[];
|
|
883
1018
|
getVisemeChannelNames(): string[];
|
|
@@ -895,6 +1030,17 @@ declare class FacialPerformanceMixer {
|
|
|
895
1030
|
/** Blend temporary emotions without changing timeline targets or mouth ownership. */
|
|
896
1031
|
setOverlay(weights: Readonly<Record<string, number>>): void;
|
|
897
1032
|
clearOverlay(): void;
|
|
1033
|
+
/**
|
|
1034
|
+
* The attentive face of the listening reflexes (`relaxed`, else `happy`)
|
|
1035
|
+
* at `weight`. It eases in and out on its own layer, gives way to any
|
|
1036
|
+
* emotion and leaves the mouth layers alone. `immediate` with weight 0
|
|
1037
|
+
* removes it in this frame.
|
|
1038
|
+
*/
|
|
1039
|
+
setListeningFace(weight: number, options?: {
|
|
1040
|
+
immediate?: boolean;
|
|
1041
|
+
}): void;
|
|
1042
|
+
/** The preset of the attentive face, or null when the model has none. */
|
|
1043
|
+
get listeningFaceChannel(): string | null;
|
|
898
1044
|
releaseVisemeMouthMode(): void;
|
|
899
1045
|
/**
|
|
900
1046
|
* Ends the speech source without disturbing any other performance layer.
|
|
@@ -1034,6 +1180,8 @@ type StagingPlaybackOptions = {
|
|
|
1034
1180
|
onCharacter?: (speech: StagingSpeechBlock, characterIndex: number, character: string) => void;
|
|
1035
1181
|
onSpeechStart?: (speech: StagingSpeechBlock) => void;
|
|
1036
1182
|
onTimelineStart?: (timeline: StagingTimeline) => void;
|
|
1183
|
+
onToolCall?: (toolCall: StagingToolCall) => void;
|
|
1184
|
+
onVariantCue?: (variantKey: string, toolCall: StagingToolCall | null) => Promise<void> | void;
|
|
1037
1185
|
onPhonemePlayback?: (speech: StagingSpeechBlock, stats: {
|
|
1038
1186
|
phonemeCount: number;
|
|
1039
1187
|
latePhonemeCount: number;
|
|
@@ -1056,6 +1204,9 @@ declare class StagingPlayer {
|
|
|
1056
1204
|
private activeExpressionMouthMode;
|
|
1057
1205
|
private activeAnimation;
|
|
1058
1206
|
private currentTrackTime;
|
|
1207
|
+
private playbackElapsed;
|
|
1208
|
+
private readonly appliedToolCallIds;
|
|
1209
|
+
private activeOptions?;
|
|
1059
1210
|
constructor(audioPlayer: AudioPlayer, facialMixer: FacialPerformanceMixer, animationController: AnimationControllerLike);
|
|
1060
1211
|
get paused(): boolean;
|
|
1061
1212
|
/**
|
|
@@ -1079,6 +1230,7 @@ declare class StagingPlayer {
|
|
|
1079
1230
|
private playSpeechAttempt;
|
|
1080
1231
|
private playSilentSpeech;
|
|
1081
1232
|
private playTimedRange;
|
|
1233
|
+
private applyToolCalls;
|
|
1082
1234
|
private applyTracks;
|
|
1083
1235
|
private clearPerformance;
|
|
1084
1236
|
}
|
|
@@ -1160,7 +1312,7 @@ declare class CharacterSceneRig {
|
|
|
1160
1312
|
/** The target that setVrmLookAtTarget set on the current model. */
|
|
1161
1313
|
private vrmLookAtTarget;
|
|
1162
1314
|
get vrm(): VRM | null;
|
|
1163
|
-
setVrm(vrm: VRM | null): void;
|
|
1315
|
+
setVrm(vrm: VRM | null, fallbackParent?: Object3D | null): void;
|
|
1164
1316
|
/**
|
|
1165
1317
|
* The `Object3D` of a `lookAt()` argument: an `Object3D` as it is, a point
|
|
1166
1318
|
* or three coordinates as a world position that this rig keeps, or null.
|
|
@@ -1209,6 +1361,7 @@ declare class CharacterSceneRig {
|
|
|
1209
1361
|
/** Re-index meshes, materials, and skeletons after custom scene graph edits. */
|
|
1210
1362
|
refreshMeshes(): void;
|
|
1211
1363
|
disposeVrm(): void;
|
|
1364
|
+
disposeAllVrms(vrms: Iterable<VRM>): void;
|
|
1212
1365
|
private applyShadows;
|
|
1213
1366
|
private applyTransform;
|
|
1214
1367
|
}
|
|
@@ -1241,7 +1394,19 @@ interface NpcifyOptions {
|
|
|
1241
1394
|
};
|
|
1242
1395
|
/** Editing uses live generation even when the character is published as baked. */
|
|
1243
1396
|
authoring?: boolean;
|
|
1397
|
+
/** Marks conversations started on `/npcs/:id` on the website; ignored when an API key is used. */
|
|
1398
|
+
hostedWebChat?: boolean;
|
|
1399
|
+
/**
|
|
1400
|
+
* Developer-supplied player identifier for SDK memory. Leave empty or omit
|
|
1401
|
+
* to share and manage SDK memories in Dashboard -> Memory.
|
|
1402
|
+
*/
|
|
1403
|
+
memoryIdentifier?: string | null;
|
|
1244
1404
|
bakedConversation?: BakedConversationAsset | string | ArrayBuffer | Uint8Array;
|
|
1405
|
+
/**
|
|
1406
|
+
* Optional loader of a variant's VRM. Without it the client asks the API for
|
|
1407
|
+
* the variant's download URL with `apiToken`.
|
|
1408
|
+
*/
|
|
1409
|
+
loadVariant?: (variantKey: string) => Promise<VRM>;
|
|
1245
1410
|
/** Optional Three.js `Scene` or `Object3D` to attach `character.scene` to once loaded. */
|
|
1246
1411
|
parent?: Object3D | null;
|
|
1247
1412
|
/** Optional Three.js `Camera` or `Object3D` for template-face eye tracking. */
|
|
@@ -1313,6 +1478,8 @@ interface StreamPlaybackDependencies {
|
|
|
1313
1478
|
onMotionFrames?: (event: StagingStreamMotionFramesEvent) => void;
|
|
1314
1479
|
/** The stored live motion of a staging that starts to play (a replay). */
|
|
1315
1480
|
onMotionTrack?: (stagingId: string, track: StagingMotionTrack) => void;
|
|
1481
|
+
onToolCall?: (toolCall: StagingToolCall) => void;
|
|
1482
|
+
onVariantCue?: (variantKey: string, toolCall: StagingToolCall | null) => Promise<void> | void;
|
|
1316
1483
|
}
|
|
1317
1484
|
declare class StreamPlaybackController {
|
|
1318
1485
|
private readonly dependencies;
|
|
@@ -1341,6 +1508,12 @@ declare class StreamPlaybackController {
|
|
|
1341
1508
|
get cursor(): CharacterPlaybackCursor | null;
|
|
1342
1509
|
/** The staging that plays or waits to play, or null. */
|
|
1343
1510
|
get activeStagingId(): string | null;
|
|
1511
|
+
/**
|
|
1512
|
+
* A staging plays or waits to play (its audio or the rest of its stream).
|
|
1513
|
+
* False after it completed or when autoplay blocked it; the last staging
|
|
1514
|
+
* stays the active staging after that.
|
|
1515
|
+
*/
|
|
1516
|
+
get playing(): boolean;
|
|
1344
1517
|
onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): () => void;
|
|
1345
1518
|
onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): () => void;
|
|
1346
1519
|
onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): () => void;
|
|
@@ -1366,6 +1539,53 @@ declare class StreamPlaybackController {
|
|
|
1366
1539
|
private safeInvoke;
|
|
1367
1540
|
}
|
|
1368
1541
|
|
|
1542
|
+
interface CharacterVariantChangeEvent {
|
|
1543
|
+
previousVariant: string;
|
|
1544
|
+
activeVariant: string;
|
|
1545
|
+
toolCall: StagingToolCall | null;
|
|
1546
|
+
}
|
|
1547
|
+
type VariantChangeEvent = CharacterVariantChangeEvent;
|
|
1548
|
+
interface NpcifyEventMap {
|
|
1549
|
+
variantChange: CharacterVariantChangeEvent;
|
|
1550
|
+
toolCall: StagingToolCall;
|
|
1551
|
+
vrmChange: VRM;
|
|
1552
|
+
}
|
|
1553
|
+
type NpcifyEventName = keyof NpcifyEventMap;
|
|
1554
|
+
interface CharacterVariantControllerOptions {
|
|
1555
|
+
loadVariant: (variantKey: string) => Promise<VRM>;
|
|
1556
|
+
applyVrmSwap: (nextVrm: VRM) => void;
|
|
1557
|
+
getCurrentVrm: () => VRM | null;
|
|
1558
|
+
handleCallbackError: (context: string, error: unknown) => void;
|
|
1559
|
+
}
|
|
1560
|
+
declare class CharacterVariantController {
|
|
1561
|
+
private readonly options;
|
|
1562
|
+
readonly variantVrms: Map<string, VRM>;
|
|
1563
|
+
private variantsList;
|
|
1564
|
+
private activeVariantKey;
|
|
1565
|
+
private remoteMode;
|
|
1566
|
+
private loadEpoch;
|
|
1567
|
+
private switchRevision;
|
|
1568
|
+
private readonly pendingLoads;
|
|
1569
|
+
private readonly listeners;
|
|
1570
|
+
constructor(options: CharacterVariantControllerOptions);
|
|
1571
|
+
get variants(): AvatarVariantSummary[];
|
|
1572
|
+
get activeVariant(): string;
|
|
1573
|
+
get isRemote(): boolean;
|
|
1574
|
+
on<K extends NpcifyEventName>(event: K, listener: (payload: NpcifyEventMap[K]) => void): () => void;
|
|
1575
|
+
off<K extends NpcifyEventName>(event: K, listener: (payload: NpcifyEventMap[K]) => void): void;
|
|
1576
|
+
emit<K extends NpcifyEventName>(event: K, payload: NpcifyEventMap[K]): void;
|
|
1577
|
+
configureVariants(variants: readonly AvatarVariantSummary[] | undefined, options?: {
|
|
1578
|
+
remote?: boolean;
|
|
1579
|
+
}): void;
|
|
1580
|
+
registerDefaultVrm(vrm: VRM): void;
|
|
1581
|
+
syncDirectVrm(vrm: VRM | null, previousVrm: VRM | null): void;
|
|
1582
|
+
preloadVariants(eager: boolean): Promise<void>;
|
|
1583
|
+
ensureVariant(variantKey: string): Promise<VRM>;
|
|
1584
|
+
switchVariant(variantKey: string, toolCall?: StagingToolCall | null): Promise<void>;
|
|
1585
|
+
reset(): VRM[];
|
|
1586
|
+
private applyActiveVariant;
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1369
1589
|
declare class CharacterExpressions {
|
|
1370
1590
|
readonly mixer: FacialPerformanceMixer;
|
|
1371
1591
|
constructor(mixer: FacialPerformanceMixer);
|
|
@@ -1403,12 +1623,147 @@ declare class CharacterExpressions {
|
|
|
1403
1623
|
reset(): void;
|
|
1404
1624
|
}
|
|
1405
1625
|
|
|
1626
|
+
/**
|
|
1627
|
+
* The scene graph access of a character: its root group, meshes, materials,
|
|
1628
|
+
* skeletons, bones and transform. `CharacterRuntime` adds the performance.
|
|
1629
|
+
*/
|
|
1630
|
+
declare class CharacterSceneAccess {
|
|
1631
|
+
protected readonly sceneRig: CharacterSceneRig;
|
|
1632
|
+
protected defaultParent: Object3D | null;
|
|
1633
|
+
/** True once the character VRM model is loaded. */
|
|
1634
|
+
get ready(): boolean;
|
|
1635
|
+
/** Root Three.js `Group` (`vrm.scene`) to add to a `THREE.Scene`. */
|
|
1636
|
+
get scene(): Group | null;
|
|
1637
|
+
/** Alias for `scene` for Three.js `Object3D` workflows. */
|
|
1638
|
+
get object3d(): Group | null;
|
|
1639
|
+
/** Alias for `scene`. */
|
|
1640
|
+
get root(): Group | null;
|
|
1641
|
+
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
1642
|
+
get mesh(): SkinnedMesh | Mesh | null;
|
|
1643
|
+
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
1644
|
+
get meshes(): readonly Mesh[];
|
|
1645
|
+
/** All Three.js `SkinnedMesh` instances in the character hierarchy. */
|
|
1646
|
+
get skinnedMeshes(): readonly SkinnedMesh[];
|
|
1647
|
+
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
1648
|
+
get materials(): readonly Material[];
|
|
1649
|
+
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
1650
|
+
get skeletons(): readonly Skeleton[];
|
|
1651
|
+
/** Three-VRM humanoid rig when loaded. */
|
|
1652
|
+
get humanoid(): VRMHumanoid | null;
|
|
1653
|
+
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
1654
|
+
getBone(name: VRMHumanBoneName): Object3D | null;
|
|
1655
|
+
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
1656
|
+
getNormalizedBone(name: VRMHumanBoneName): Object3D | null;
|
|
1657
|
+
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
1658
|
+
attachTo(parent: Object3D): Group;
|
|
1659
|
+
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
1660
|
+
detach(): void;
|
|
1661
|
+
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
1662
|
+
setShadows(options?: CharacterShadowOptions): void;
|
|
1663
|
+
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
1664
|
+
setTransform(options: CharacterTransformOptions): void;
|
|
1665
|
+
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
1666
|
+
refreshMeshes(): void;
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
interface SpeechNod {
|
|
1670
|
+
/** The largest pitch of the head and neck together, in degrees. */
|
|
1671
|
+
amplitudeDegrees: number;
|
|
1672
|
+
durationMs: number;
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
/** What the player does now, as the listening reflexes see it. */
|
|
1676
|
+
type PlayerActivity = 'idle' | 'typing' | 'speaking';
|
|
1677
|
+
interface PlayerActivityTrackerOptions {
|
|
1678
|
+
/** The random numbers of the nods (tests: a fixed stream). */
|
|
1679
|
+
random?: () => number;
|
|
1680
|
+
}
|
|
1681
|
+
/**
|
|
1682
|
+
* The state machine of the player's activity (docs/voice-input.md,
|
|
1683
|
+
* "Listening reflexes"). Speaking is a listening session in state
|
|
1684
|
+
* `listening`; typing is input in a text field during the last 1.5 s.
|
|
1685
|
+
* Speaking wins over typing. Only typed input and microphone levels go in:
|
|
1686
|
+
* nothing is sent to a server here.
|
|
1687
|
+
*/
|
|
1688
|
+
declare class PlayerActivityTracker {
|
|
1689
|
+
private listening;
|
|
1690
|
+
private typing;
|
|
1691
|
+
private typingTimer;
|
|
1692
|
+
private current;
|
|
1693
|
+
private readonly trigger;
|
|
1694
|
+
private readonly changeListeners;
|
|
1695
|
+
private readonly nodListeners;
|
|
1696
|
+
constructor(options?: PlayerActivityTrackerOptions);
|
|
1697
|
+
get state(): PlayerActivity;
|
|
1698
|
+
/** The listening session is (or is no longer) in state `listening`. */
|
|
1699
|
+
setListening(active: boolean): void;
|
|
1700
|
+
/**
|
|
1701
|
+
* Input in a text field: `true` at each input event, `false` when the
|
|
1702
|
+
* player sends or leaves the field. Without input, typing ends after 1.5 s.
|
|
1703
|
+
*/
|
|
1704
|
+
notifyTyping(active: boolean): void;
|
|
1705
|
+
/** One microphone frame of the listening session; returns a nod when one is due. */
|
|
1706
|
+
pushVoiceLevel(level: number, frameMs?: number): SpeechNod | null;
|
|
1707
|
+
onChange(listener: (state: PlayerActivity) => void): () => void;
|
|
1708
|
+
onNod(listener: (nod: SpeechNod) => void): () => void;
|
|
1709
|
+
/** Ends all activity, for example when the conversation ends. */
|
|
1710
|
+
reset(): void;
|
|
1711
|
+
private resolve;
|
|
1712
|
+
private clearTypingTimer;
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
interface ListeningReflexHost {
|
|
1716
|
+
vrm(): VRM | null;
|
|
1717
|
+
/** The gaze target of `setGazeTarget` or `lookAt`, or null. */
|
|
1718
|
+
gazeTarget(): Object3D | null;
|
|
1719
|
+
/** The host application set the gaze target with `lookAt()`. */
|
|
1720
|
+
lookAtActive(): boolean;
|
|
1721
|
+
/** Looks at `target` until the returned function restores the prior gaze. */
|
|
1722
|
+
lookAtTemporarily(target: Object3D): () => void;
|
|
1723
|
+
facialMixer: FacialPerformanceMixer;
|
|
1724
|
+
}
|
|
1725
|
+
interface ListeningReflexFrame {
|
|
1726
|
+
/** A reply (staging) plays or waits to play. */
|
|
1727
|
+
stagingActive: boolean;
|
|
1728
|
+
/** The weight of live motion frames, 0 when idle. */
|
|
1729
|
+
liveMotionWeight: number;
|
|
1730
|
+
}
|
|
1731
|
+
/**
|
|
1732
|
+
* The listening reflexes of the Web client (docs/voice-input.md): while the
|
|
1733
|
+
* player speaks or types, the character looks at the player, shows a subtle
|
|
1734
|
+
* attentive face, and nods lightly at pauses in the speech. A staging takes
|
|
1735
|
+
* over at once: the face, the gaze and a nod stop in the same frame, and
|
|
1736
|
+
* nothing changes while it plays. A `lookAt()` target of the host stays; the
|
|
1737
|
+
* prior gaze comes back when the reflexes end. Live motion stops a nod.
|
|
1738
|
+
*/
|
|
1739
|
+
declare class ListeningReflex {
|
|
1740
|
+
private readonly host;
|
|
1741
|
+
private readonly activity;
|
|
1742
|
+
private readonly nod;
|
|
1743
|
+
private pendingNod;
|
|
1744
|
+
private releaseGaze;
|
|
1745
|
+
private readonly front;
|
|
1746
|
+
private readonly origin;
|
|
1747
|
+
private readonly forward;
|
|
1748
|
+
private yielded;
|
|
1749
|
+
constructor(host: ListeningReflexHost, activity: PlayerActivityTracker);
|
|
1750
|
+
/** Call after the animation mixer and live motion, before the facial mixer and `vrm.update()`. */
|
|
1751
|
+
update(delta: number, frame: ListeningReflexFrame): void;
|
|
1752
|
+
/** A new model or the end of the conversation: no gaze, face or nod remains. */
|
|
1753
|
+
reset(): void;
|
|
1754
|
+
private beginGaze;
|
|
1755
|
+
private endGaze;
|
|
1756
|
+
/** The point in front of the head, along the character's forward axis. */
|
|
1757
|
+
private placeFront;
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1406
1760
|
interface CharacterRuntimeInitOptions extends Omit<NpcifyOptions, 'apiToken' | 'assetProfile' | 'apiKey'> {
|
|
1407
1761
|
apiToken: string;
|
|
1408
1762
|
assetProfile: 'original' | 'web-optimized-v1';
|
|
1409
1763
|
loadAnimation?: (animationId: string) => Promise<VRMAnimation>;
|
|
1764
|
+
loadVariant?: (variantKey: string) => Promise<VRM>;
|
|
1410
1765
|
}
|
|
1411
|
-
declare class CharacterRuntime {
|
|
1766
|
+
declare class CharacterRuntime extends CharacterSceneAccess {
|
|
1412
1767
|
readonly baseUrl: string;
|
|
1413
1768
|
readonly characterId: string;
|
|
1414
1769
|
readonly apiToken: string;
|
|
@@ -1418,8 +1773,8 @@ declare class CharacterRuntime {
|
|
|
1418
1773
|
readonly animation: CharacterAnimation;
|
|
1419
1774
|
animations: Map<string, VRMAnimation>;
|
|
1420
1775
|
animationController?: AnimationControllerLike;
|
|
1421
|
-
protected readonly sceneRig: CharacterSceneRig;
|
|
1422
1776
|
protected readonly lazyAnimations: LazyAnimations;
|
|
1777
|
+
protected readonly variantController: CharacterVariantController;
|
|
1423
1778
|
protected audioPlayer: AudioPlayer | null;
|
|
1424
1779
|
protected stagingPlayer: StagingPlayer | null;
|
|
1425
1780
|
protected readonly facialMixer: FacialPerformanceMixer;
|
|
@@ -1428,7 +1783,8 @@ declare class CharacterRuntime {
|
|
|
1428
1783
|
protected readonly telemetry: ClientTelemetry;
|
|
1429
1784
|
protected readonly streamPlayback: StreamPlaybackController;
|
|
1430
1785
|
protected readonly liveMotion: LiveMotionPlayer;
|
|
1431
|
-
protected
|
|
1786
|
+
protected readonly playerActivityTracker: PlayerActivityTracker;
|
|
1787
|
+
protected readonly listeningReflex: ListeningReflex;
|
|
1432
1788
|
protected currentGazeTarget: Object3D | null;
|
|
1433
1789
|
/** The current gaze target came from `lookAt()`. */
|
|
1434
1790
|
protected lookAtActive: boolean;
|
|
@@ -1440,26 +1796,14 @@ declare class CharacterRuntime {
|
|
|
1440
1796
|
/** Loaded Three-VRM instance, or `null` before `loadAssets()`. */
|
|
1441
1797
|
get vrm(): VRM | null;
|
|
1442
1798
|
set vrm(value: VRM | null);
|
|
1443
|
-
/**
|
|
1444
|
-
get
|
|
1445
|
-
/**
|
|
1446
|
-
get
|
|
1447
|
-
/**
|
|
1448
|
-
get
|
|
1449
|
-
/**
|
|
1450
|
-
|
|
1451
|
-
/** Primary character mesh (`SkinnedMesh` preferred, otherwise first `Mesh`). */
|
|
1452
|
-
get mesh(): SkinnedMesh | Mesh | null;
|
|
1453
|
-
/** All Three.js `Mesh` instances in the character hierarchy. */
|
|
1454
|
-
get meshes(): readonly Mesh[];
|
|
1455
|
-
/** All Three.js `SkinnedMesh` instances in the character hierarchy. */
|
|
1456
|
-
get skinnedMeshes(): readonly SkinnedMesh[];
|
|
1457
|
-
/** Deduplicated Three.js `Material` instances across all character meshes. */
|
|
1458
|
-
get materials(): readonly Material[];
|
|
1459
|
-
/** Deduplicated Three.js `Skeleton` instances across all character skinned meshes. */
|
|
1460
|
-
get skeletons(): readonly Skeleton[];
|
|
1461
|
-
/** Three-VRM humanoid rig when loaded. */
|
|
1462
|
-
get humanoid(): VRMHumanoid | null;
|
|
1799
|
+
/** Allowed appearance variants configured for this character (`default` first). */
|
|
1800
|
+
get variants(): AvatarVariantSummary[];
|
|
1801
|
+
/** Currently active appearance variant key (defaults to `'default'`). */
|
|
1802
|
+
get activeVariant(): string;
|
|
1803
|
+
/** Cached Three-VRM instances keyed by variant key. */
|
|
1804
|
+
get variantVrms(): Map<string, VRM>;
|
|
1805
|
+
/** Switch the character's active appearance variant, loading on-demand if needed. */
|
|
1806
|
+
switchVariant(variantKey: string): Promise<void>;
|
|
1463
1807
|
/** Underlying Three.js `AnimationMixer` driving skeletal animations. */
|
|
1464
1808
|
get mixer(): AnimationMixer | null;
|
|
1465
1809
|
/** The `Object3D` (such as a `THREE.Camera`) the character's eyes track. */
|
|
@@ -1481,20 +1825,6 @@ declare class CharacterRuntime {
|
|
|
1481
1825
|
get cursor(): CharacterPlaybackCursor | null;
|
|
1482
1826
|
/** Combined snapshot of real-time speech and dialogue state. */
|
|
1483
1827
|
get speech(): CharacterSpeechState;
|
|
1484
|
-
/** Look up a raw humanoid bone `Object3D` by VRM bone name (for example `'head'` or `'rightHand'`). */
|
|
1485
|
-
getBone(name: VRMHumanBoneName): Object3D | null;
|
|
1486
|
-
/** Look up a normalized humanoid bone `Object3D` by VRM bone name. */
|
|
1487
|
-
getNormalizedBone(name: VRMHumanBoneName): Object3D | null;
|
|
1488
|
-
/** Attach the character root `Group` to a Three.js `Scene` or parent `Object3D`. */
|
|
1489
|
-
attachTo(parent: Object3D): Group;
|
|
1490
|
-
/** Detach the character root `Group` from its current parent `Object3D`. */
|
|
1491
|
-
detach(): void;
|
|
1492
|
-
/** Configure `castShadow` and `receiveShadow` on all character meshes. */
|
|
1493
|
-
setShadows(options?: CharacterShadowOptions): void;
|
|
1494
|
-
/** Configure `position`, `rotation`, and `scale` on the character root `Group`. */
|
|
1495
|
-
setTransform(options: CharacterTransformOptions): void;
|
|
1496
|
-
/** Re-index cached meshes, materials, and skeletons after custom scene graph edits. */
|
|
1497
|
-
refreshMeshes(): void;
|
|
1498
1828
|
ensureAnimation(animationId: string): Promise<void>;
|
|
1499
1829
|
/** Play a character animation by ID, automatically loading it if lazy. */
|
|
1500
1830
|
playAnimation(animationId: string, options?: CharacterAnimationPlayOptions): Promise<void>;
|
|
@@ -1535,6 +1865,16 @@ declare class CharacterRuntime {
|
|
|
1535
1865
|
clearLookAt(): void;
|
|
1536
1866
|
/** Briefly look at an object; release restores the prior target and gaze mode. */
|
|
1537
1867
|
lookAtTemporarily(target: Object3D): () => void;
|
|
1868
|
+
/**
|
|
1869
|
+
* Tells the character that the player types (`true` at each input event
|
|
1870
|
+
* of the text field) or stopped (`false`, for example after sending). The
|
|
1871
|
+
* character looks at the player and shows a subtle attentive face; typing
|
|
1872
|
+
* ends by itself after 1.5 s without input. Local only: no request.
|
|
1873
|
+
*/
|
|
1874
|
+
notifyUserTyping(active: boolean): void;
|
|
1875
|
+
/** `'speaking'` while listening, `'typing'` after typed input, else `'idle'`. */
|
|
1876
|
+
get playerActivity(): PlayerActivity;
|
|
1877
|
+
onPlayerActivity(callback: (state: PlayerActivity) => void): () => void;
|
|
1538
1878
|
startPerformanceRecording(options?: PerformanceRecordingStartOptions): void;
|
|
1539
1879
|
stopPerformanceRecording(): PerformanceRecording | null;
|
|
1540
1880
|
getPerformanceRecording(): PerformanceRecording | null;
|
|
@@ -1547,6 +1887,11 @@ declare class CharacterRuntime {
|
|
|
1547
1887
|
onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): () => void;
|
|
1548
1888
|
onPlaybackCursor(callback: Parameters<StreamPlaybackController['onPlaybackCursor']>[0]): () => void;
|
|
1549
1889
|
onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): () => void;
|
|
1890
|
+
on<K extends NpcifyEventName>(event: K, listener: (payload: NpcifyEventMap[K]) => void): () => void;
|
|
1891
|
+
off<K extends NpcifyEventName>(event: K, listener: (payload: NpcifyEventMap[K]) => void): void;
|
|
1892
|
+
onVrmChange(callback: (vrm: VRM) => void): () => void;
|
|
1893
|
+
onVariantChange(callback: (event: CharacterVariantChangeEvent) => void): () => void;
|
|
1894
|
+
onToolCall(callback: (toolCall: StagingToolCall) => void): () => void;
|
|
1550
1895
|
/** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
|
|
1551
1896
|
update(delta: number): void;
|
|
1552
1897
|
protected readonly getAudioPlayer: () => AudioPlayer;
|
|
@@ -1558,7 +1903,9 @@ declare class CharacterRuntime {
|
|
|
1558
1903
|
* IdleFace (its gaze cone, idle eye moves and look cues).
|
|
1559
1904
|
*/
|
|
1560
1905
|
protected applyGaze(): void;
|
|
1561
|
-
protected resetRuntimeAssets(
|
|
1906
|
+
protected resetRuntimeAssets(options?: {
|
|
1907
|
+
disposeVrms?: boolean;
|
|
1908
|
+
}): void;
|
|
1562
1909
|
protected disposeRuntimeAssets(): void;
|
|
1563
1910
|
protected readonly handleCallbackError: (context: string, error: unknown) => void;
|
|
1564
1911
|
protected readonly reportError: (error: Error, context?: string) => void;
|
|
@@ -1600,6 +1947,49 @@ interface ConversationConnectionClosedEvent {
|
|
|
1600
1947
|
conversationId: string | null;
|
|
1601
1948
|
}
|
|
1602
1949
|
|
|
1950
|
+
/**
|
|
1951
|
+
* The code of a `SightError`: a refusal of the server (`SIGHT_ERRORS` of
|
|
1952
|
+
* `@npcify/shared`), a code of the conversation connection
|
|
1953
|
+
* (`CONVERSATION_DISCONNECTED`, `CONVERSATION_TIMEOUT`, ...), or a code of
|
|
1954
|
+
* the SDK. Another string is a code of a newer server.
|
|
1955
|
+
*/
|
|
1956
|
+
type SightFailureCode = SightErrorCode | 'CAMERA_PERMISSION_DENIED' | 'CAMERA_UNAVAILABLE' | 'SIGHT_SOURCE_ENDED' | 'SIGHT_ALREADY_STARTED' | 'CONVERSATION_REQUIRED' | (string & {});
|
|
1957
|
+
declare class SightError extends Error {
|
|
1958
|
+
readonly code: SightFailureCode;
|
|
1959
|
+
constructor(code: SightFailureCode, message?: string);
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
/** `'camera'` opens the camera; a `MediaStream` is a video of the caller. */
|
|
1963
|
+
type SightSourceOption = MediaStream | 'camera';
|
|
1964
|
+
|
|
1965
|
+
/**
|
|
1966
|
+
* `'unavailable'`: the character info did not turn sight on, or the
|
|
1967
|
+
* conversation is baked. `'off'`: sight can start. `'starting'`: the camera
|
|
1968
|
+
* and the connection open. `'on'`: frames go to the server.
|
|
1969
|
+
*/
|
|
1970
|
+
type SightState = 'unavailable' | 'off' | 'starting' | 'on';
|
|
1971
|
+
interface SightOptions {
|
|
1972
|
+
/**
|
|
1973
|
+
* `'camera'` (default) opens the camera, and `stopSight()` closes it. A
|
|
1974
|
+
* `MediaStream` (a camera or a screen share of your page) stays yours:
|
|
1975
|
+
* the SDK never stops its tracks.
|
|
1976
|
+
*/
|
|
1977
|
+
source?: SightSourceOption;
|
|
1978
|
+
/** The longest time between two frames. Default 5000 ms; the minimum is 1000 ms. */
|
|
1979
|
+
intervalMs?: number;
|
|
1980
|
+
/**
|
|
1981
|
+
* A change of the picture sends a frame at once (one each second at most).
|
|
1982
|
+
* `threshold` is the mean change of the 32 × 18 luma grid between two
|
|
1983
|
+
* checks, on a 0 to 255 scale (default 8). `false` sends only at the
|
|
1984
|
+
* interval.
|
|
1985
|
+
*/
|
|
1986
|
+
motion?: {
|
|
1987
|
+
threshold?: number;
|
|
1988
|
+
} | false;
|
|
1989
|
+
/** The error that stopped sight after `startSight()` resolved. */
|
|
1990
|
+
onError?: (error: SightError) => void;
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1603
1993
|
declare class MicrophoneError extends Error {
|
|
1604
1994
|
readonly code: 'MICROPHONE_PERMISSION_DENIED' | 'MICROPHONE_UNAVAILABLE';
|
|
1605
1995
|
constructor(code: 'MICROPHONE_PERMISSION_DENIED' | 'MICROPHONE_UNAVAILABLE', message: string);
|
|
@@ -1703,15 +2093,14 @@ declare class Character extends CharacterRuntime {
|
|
|
1703
2093
|
/** The conversation whose live motion was warmed up. */
|
|
1704
2094
|
private liveMotionPrepared;
|
|
1705
2095
|
private readonly authoring;
|
|
2096
|
+
private hostedWebChat;
|
|
2097
|
+
private memoryIdentifier;
|
|
1706
2098
|
protected readonly normalizeIdleOrientation: boolean;
|
|
1707
2099
|
private readonly bakedAsset?;
|
|
1708
2100
|
private animationRefreshId;
|
|
1709
2101
|
private readonly voice;
|
|
1710
|
-
private readonly
|
|
1711
|
-
private
|
|
1712
|
-
/** The character's voice input switch, after the character loads. */
|
|
1713
|
-
private speechInput;
|
|
1714
|
-
private latestOptions;
|
|
2102
|
+
private readonly sight;
|
|
2103
|
+
private readonly inputs;
|
|
1715
2104
|
private listeningStateCallback;
|
|
1716
2105
|
private transcriptCallback;
|
|
1717
2106
|
private connectionClosedCallback;
|
|
@@ -1768,6 +2157,25 @@ declare class Character extends CharacterRuntime {
|
|
|
1768
2157
|
onListeningState(callback: (state: ListeningState) => void): () => void;
|
|
1769
2158
|
/** Partial text while the player speaks (`final: false`), then the final text. */
|
|
1770
2159
|
onTranscript(callback: (event: TranscriptEvent) => void): () => void;
|
|
2160
|
+
/**
|
|
2161
|
+
* `'unavailable'` until the character info turns sight on (its owner's
|
|
2162
|
+
* Sight tab), and in a baked conversation; then `'off'`, `'starting'` or
|
|
2163
|
+
* `'on'`.
|
|
2164
|
+
*/
|
|
2165
|
+
get sightState(): SightState;
|
|
2166
|
+
/**
|
|
2167
|
+
* Lets the character see the player's camera (docs/sight.md). Needs a
|
|
2168
|
+
* conversation; the first call asks for the camera permission. Rejects
|
|
2169
|
+
* with a `SightError`: `SIGHT_DISABLED` when sight is unavailable,
|
|
2170
|
+
* `CONVERSATION_REQUIRED`, `CAMERA_PERMISSION_DENIED`, ...
|
|
2171
|
+
*/
|
|
2172
|
+
startSight(options?: SightOptions): Promise<void>;
|
|
2173
|
+
/** Stops the frames. Closes the camera that `startSight` opened. */
|
|
2174
|
+
stopSight(): void;
|
|
2175
|
+
/** What the character saw: each observation of the current conversation. */
|
|
2176
|
+
onSight(callback: (event: SightObservationEvent) => void): () => void;
|
|
2177
|
+
/** The sight state. `error` is the `SightError` that stopped sight. */
|
|
2178
|
+
onSightState(callback: (state: SightState, error?: SightError) => void): () => void;
|
|
1771
2179
|
/**
|
|
1772
2180
|
* The server closed the conversation connection: a refused or failed join,
|
|
1773
2181
|
* or an API key that was rotated, edited or deleted. The next `connect`,
|
|
@@ -1784,19 +2192,14 @@ declare class Character extends CharacterRuntime {
|
|
|
1784
2192
|
*/
|
|
1785
2193
|
playStreamEvent(event: StagingStreamEvent): Promise<boolean> | undefined;
|
|
1786
2194
|
setRenderSessionId(renderSessionId: string | null): void;
|
|
2195
|
+
setHostedWebChat(enabled: boolean): void;
|
|
2196
|
+
setMemoryIdentifier(memoryIdentifier: string | null | undefined): void;
|
|
2197
|
+
getMemoryIdentifier(): string | null;
|
|
1787
2198
|
onStagingResult(callback: (payload: StagingResultPayload) => Promise<void> | void): () => void;
|
|
1788
2199
|
onThinking(callback: (payload: LlmThinkingPayload) => Promise<void> | void): () => void;
|
|
1789
2200
|
private resetAssets;
|
|
1790
2201
|
private readonly getAssetInfo;
|
|
1791
2202
|
private handleStagingResult;
|
|
1792
|
-
/** Throws when the input modes do not include `mode`. */
|
|
1793
|
-
private assertInputMode;
|
|
1794
|
-
/**
|
|
1795
|
-
* A typed reply needs the `text` mode; one of the current reply options
|
|
1796
|
-
* needs `options` or `text`. Without a configuration the server's rules
|
|
1797
|
-
* apply alone, as before.
|
|
1798
|
-
*/
|
|
1799
|
-
private assertReplyMode;
|
|
1800
2203
|
private invokeThinkingCallback;
|
|
1801
2204
|
}
|
|
1802
2205
|
declare class Npcify extends Character {
|
|
@@ -1894,6 +2297,8 @@ interface MountNpcifyOptions {
|
|
|
1894
2297
|
normalizeIdleOrientation?: boolean;
|
|
1895
2298
|
/** How the player can reply: `'options'`, `'text'`, `'voice'` (docs/voice-input.md). */
|
|
1896
2299
|
inputModes?: InputMode[];
|
|
2300
|
+
/** Player memory identifier for SDK conversations; empty/null uses the character owner's dashboard memory bucket. */
|
|
2301
|
+
memoryIdentifier?: string | null;
|
|
1897
2302
|
onError?: (error: Error) => void;
|
|
1898
2303
|
}
|
|
1899
2304
|
interface NpcifyView {
|
|
@@ -1903,9 +2308,10 @@ interface NpcifyView {
|
|
|
1903
2308
|
camera?: PerspectiveCamera | null;
|
|
1904
2309
|
renderer?: WebGLRenderer | null;
|
|
1905
2310
|
controls?: OrbitControls | null;
|
|
2311
|
+
setMemoryIdentifier(memoryIdentifier: string | null | undefined): void;
|
|
1906
2312
|
dispose(): Promise<void>;
|
|
1907
2313
|
}
|
|
1908
2314
|
/** Mount into an existing canvas. Its CSS must give it a nonzero width and height. */
|
|
1909
2315
|
declare const mountNpcify: (options: MountNpcifyOptions) => Promise<NpcifyView>;
|
|
1910
2316
|
|
|
1911
|
-
export { AnimationController, type AnimationControllerLike, type AnimationControllerOptions, type AssetLoadProgress, type AudioCapture, type AudioPerformanceFrameSnapshot, AudioPlayer, BakedConversation, type BakedConversationAsset, type CaptureFrame, Character, CharacterAnimation, type CharacterAnimationPlayOptions, CharacterAudio, type CharacterDisposeOptions, CharacterExpressions, type CharacterLookAtTarget, type CharacterOptions, type CharacterPlaybackCursor, CharacterRuntime, type CharacterRuntimeInitOptions, CharacterSceneRig, type CharacterShadowOptions, type CharacterSpeechState, type CharacterTransformOptions, type ConversationConnectionClosedEvent, ConversationConnectionError, type ConversationConnectionErrorCode, type ConversationJoinFailure, type ConversationTurnOptions, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, INPUT_MODES, type InputMode, InputModeError, type ListeningOptions, type ListeningState, type LiveMotionOptions, LiveMotionPlayer, type LiveMotionRootMode, type LlmThinkingPayload, type LoadCharacterOptions, MicrophoneError, type MountNpcifyOptions, Npcify, type NpcifyAssetInfo, Character as NpcifyCharacter, type NpcifyOptions, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, type PushToTalkOptions, type PushToTalkTarget, RemoteAnimationController, type RepeatedVisemeTransitionAnalysis, SpeechInputError, type SpeechInputResult, type StagingOptionsReadyPayload, type StagingPlaybackStart, type StagingSpeechPlaybackEvent, type TranscriptEvent, type Vector3Like, type Vector3Tuple, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, bindPushToTalk, createCharacter, decodePerformance, encodePerformance, loadBakedConversation, loadCharacter, mountNpcify };
|
|
2317
|
+
export { AnimationController, type AnimationControllerLike, type AnimationControllerOptions, type AssetLoadProgress, type AudioCapture, type AudioPerformanceFrameSnapshot, AudioPlayer, type AvatarVariantSummary, BakedConversation, type BakedConversationAsset, type CaptureFrame, Character, CharacterAnimation, type CharacterAnimationPlayOptions, CharacterAudio, type CharacterDisposeOptions, CharacterExpressions, type CharacterLookAtTarget, type CharacterOptions, type CharacterPlaybackCursor, CharacterRuntime, type CharacterRuntimeInitOptions, CharacterSceneRig, type CharacterShadowOptions, type CharacterSpeechState, type CharacterTransformOptions, type CharacterVariantChangeEvent, type ConversationConnectionClosedEvent, ConversationConnectionError, type ConversationConnectionErrorCode, type ConversationJoinFailure, type ConversationTurnOptions, DEFAULT_VARIANT_KEY, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, type FetchVrmDownloadUrlOptions, INPUT_MODES, type InputMode, InputModeError, type ListeningOptions, type ListeningState, type LiveMotionOptions, LiveMotionPlayer, type LiveMotionRootMode, type LlmThinkingPayload, type LoadCharacterOptions, MicrophoneError, type MountNpcifyOptions, NUDE_VARIANT_KEY, Npcify, type NpcifyAssetInfo, Character as NpcifyCharacter, type NpcifyEventMap, type NpcifyEventName, type NpcifyOptions, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, type PlayerActivity, type PushToTalkOptions, type PushToTalkTarget, RemoteAnimationController, type RepeatedVisemeTransitionAnalysis, SightError, type SightFailureCode, type SightObservation, type SightObservationEvent, type SightOptions, type SightPerson, type SightState, SpeechInputError, type SpeechInputResult, type StagingOptionsReadyPayload, type StagingPlaybackStart, type StagingSpeechPlaybackEvent, type StagingSwitchVariantArguments, type StagingToolCall, type StagingToolCallResult, type StagingToolName, type StagingToolResultStatus, type TranscriptEvent, type VariantChangeEvent, type Vector3Like, type Vector3Tuple, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, bindPushToTalk, createCharacter, decodePerformance, encodePerformance, fetchVariantVrmDownloadUrl, fetchVrmDownloadUrl, loadBakedConversation, loadCharacter, loadVariantVrm, loadVrm, mountNpcify };
|