@npcify/web 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,970 @@
1
+ import { VRMExpressionPresetName, VRM } from '@pixiv/three-vrm';
2
+ import { VRMAnimation } from '@pixiv/three-vrm-animation';
3
+
4
+ declare const TTS_PROVIDERS: readonly ["11labs"];
5
+ type TtsProviderId = (typeof TTS_PROVIDERS)[number];
6
+ type NullableTtsProviderId = TtsProviderId | null;
7
+
8
+ declare const EXPRESSION_CALL_RANGES: readonly ["linear", "bool"];
9
+ type ExpressionCallRange = (typeof EXPRESSION_CALL_RANGES)[number];
10
+ type ExpressionCallRangeMap = Record<string, ExpressionCallRange>;
11
+ type AvatarAnimationSummary = {
12
+ id: string;
13
+ name: string;
14
+ };
15
+ type Sqt = {
16
+ scale?: [number, number, number] | null;
17
+ quaternion?: [number, number, number, number] | null;
18
+ translation?: [number, number, number] | null;
19
+ };
20
+ interface AvatarPublicInfo {
21
+ id: string;
22
+ name: string | null;
23
+ description?: string | null;
24
+ knowledgeContext: string | null;
25
+ language: string | null;
26
+ vrmId?: string;
27
+ portraitUrl: string | null;
28
+ animations: AvatarAnimationSummary[];
29
+ tones: string[];
30
+ expressions: string[];
31
+ expressionsConfigured: boolean;
32
+ expressionCallRanges: ExpressionCallRangeMap;
33
+ idleAnimationId: string | null;
34
+ ttsEnabled: boolean;
35
+ ttsVoiceId: string | null;
36
+ ttsProvider: NullableTtsProviderId;
37
+ chatPricePoints?: number | null;
38
+ freeFirstNMessages?: number | null;
39
+ hostedCharacterEnabled?: boolean;
40
+ listedPublicly?: boolean;
41
+ bakedConversationEnabled?: boolean;
42
+ bakedConversationRevision?: number;
43
+ allowCustomReply?: boolean;
44
+ autoPlay: boolean;
45
+ serverRenderingEnabled: boolean;
46
+ cameraSqt?: Sqt | null;
47
+ }
48
+
49
+ interface UsageChargeSummary {
50
+ id: string;
51
+ grossPoints: number;
52
+ developerPoints: number;
53
+ platformFeePoints: number;
54
+ currency: string;
55
+ }
56
+ interface ConversationSummary {
57
+ id: string;
58
+ }
59
+ interface ConversationCreationResult<TConversation extends ConversationSummary = ConversationSummary> {
60
+ conversation: TConversation;
61
+ openingStaging: StagingResultPayload;
62
+ chatPricePoints?: number;
63
+ availablePoints?: number;
64
+ }
65
+ interface ConversationWorldlineStaging {
66
+ id: string;
67
+ parentId: string | null;
68
+ triggerText: string | null;
69
+ text: string;
70
+ options: ConversationOption[];
71
+ createdAt: string;
72
+ }
73
+ interface ConversationWorldline {
74
+ conversationId: string;
75
+ openingStagingId: string | null;
76
+ activeStagingId: string | null;
77
+ stagings: ConversationWorldlineStaging[];
78
+ }
79
+
80
+ interface ConversationTimelineEntry {
81
+ name: string;
82
+ timestamp: number;
83
+ weight?: number;
84
+ }
85
+ interface WordTiming {
86
+ word: string;
87
+ startTime: number;
88
+ endTime: number;
89
+ }
90
+ interface CharacterTiming {
91
+ character: string;
92
+ startTime: number;
93
+ endTime: number;
94
+ }
95
+ interface ConversationOption {
96
+ label: string;
97
+ end?: boolean;
98
+ }
99
+ type ConversationExpressionIntensity = 'subtle' | 'normal' | 'strong' | 'extreme';
100
+ interface ConversationExpressionCue {
101
+ name: string;
102
+ intensity: ConversationExpressionIntensity;
103
+ weight: number;
104
+ }
105
+ type StagingAudioSource = {
106
+ type: 's3';
107
+ url: string;
108
+ } | {
109
+ type: 'base64';
110
+ data: string;
111
+ mimeType: string;
112
+ } | {
113
+ type: 'bytes';
114
+ data: Uint8Array;
115
+ mimeType: string;
116
+ } | {
117
+ type: 'stream';
118
+ streamId: string;
119
+ mimeType: string;
120
+ } | {
121
+ type: 'silent';
122
+ duration: number;
123
+ };
124
+ interface StagingSpeechBeat {
125
+ id: string;
126
+ text: string;
127
+ tone: string | null;
128
+ /** Seconds from the start of the staging. */
129
+ startTime: number;
130
+ /** Seconds from the start of the staging. */
131
+ endTime: number;
132
+ }
133
+ interface StagingSpeechBlock {
134
+ id: string;
135
+ type: 'speech';
136
+ /** Seconds from the start of the staging. */
137
+ startTime: number;
138
+ /** Seconds from the start of the staging. */
139
+ endTime: number;
140
+ text: string;
141
+ audio: StagingAudioSource;
142
+ beats: StagingSpeechBeat[];
143
+ /** Timings remain local to this audio block. */
144
+ visemes: ConversationTimelineEntry[];
145
+ /** Timings remain local to this audio block. */
146
+ wordTimings: WordTiming[];
147
+ /** Exact provider or deterministic timings, local to this audio block. */
148
+ characterTimings?: CharacterTiming[];
149
+ }
150
+ interface StagingPauseBlock {
151
+ id: string;
152
+ type: 'pause';
153
+ /** Seconds from the start of the staging. */
154
+ startTime: number;
155
+ /** Seconds from the start of the staging. */
156
+ endTime: number;
157
+ }
158
+ type StagingDialogueBlock = StagingSpeechBlock | StagingPauseBlock;
159
+ interface StagingExpressionClip {
160
+ /** Seconds from the start of the staging. */
161
+ startTime: number;
162
+ /** Seconds from the start of the staging. */
163
+ endTime: number;
164
+ expression: ConversationExpressionCue;
165
+ }
166
+ interface StagingAnimationClip {
167
+ /** Seconds from the start of the staging. */
168
+ startTime: number;
169
+ /** Seconds from the start of the staging. */
170
+ endTime: number;
171
+ animation: string;
172
+ loop: boolean;
173
+ }
174
+ interface StagingTimeline {
175
+ /** Total duration in seconds. */
176
+ duration: number;
177
+ dialogue: StagingDialogueBlock[];
178
+ expressions: StagingExpressionClip[];
179
+ animations: StagingAnimationClip[];
180
+ }
181
+ declare const STAGING_PROTOCOL_VERSION: 1;
182
+ interface StagingResultPayload {
183
+ protocolVersion: typeof STAGING_PROTOCOL_VERSION;
184
+ taskId: string | null;
185
+ stagingId: string;
186
+ text: string;
187
+ timeline: StagingTimeline;
188
+ options: ConversationOption[];
189
+ usageCharge?: UsageChargeSummary | null;
190
+ availablePoints?: number;
191
+ }
192
+ interface StagingStreamStartEvent {
193
+ type: 'start';
194
+ taskId: string;
195
+ stagingId: string;
196
+ }
197
+ interface StagingStreamTextDeltaEvent {
198
+ type: 'text_delta';
199
+ taskId: string;
200
+ stagingId: string;
201
+ delta: string;
202
+ }
203
+ interface StagingStreamTimelineEvent {
204
+ type: 'staging';
205
+ taskId: string;
206
+ stagingId: string;
207
+ revision: number;
208
+ /** False means an immutable dialogue prefix; more blocks may follow. */
209
+ complete?: boolean;
210
+ timeline: StagingTimeline;
211
+ }
212
+ /** Older SDKs ignore this event and wait for the complete `staging` event. */
213
+ interface StagingStreamPrefixEvent extends Omit<StagingStreamTimelineEvent, 'type' | 'complete'> {
214
+ type: 'staging_prefix';
215
+ complete: false;
216
+ }
217
+ interface StagingStreamSpeechChunkEvent {
218
+ type: 'speech_chunk';
219
+ taskId: string;
220
+ stagingId: string;
221
+ speechId: string;
222
+ streamId: string;
223
+ mimeType: string;
224
+ chunkIndex: number;
225
+ data: string | Uint8Array;
226
+ }
227
+ interface StagingStreamSpeechCompleteEvent {
228
+ type: 'speech_complete';
229
+ taskId: string;
230
+ stagingId: string;
231
+ speechId: string;
232
+ streamId: string;
233
+ }
234
+ interface StagingStreamSpeechErrorEvent {
235
+ type: 'speech_error';
236
+ taskId: string;
237
+ stagingId: string;
238
+ speechId: string;
239
+ streamId: string;
240
+ message: string;
241
+ }
242
+ interface StagingStreamOptionsEvent {
243
+ type: 'options';
244
+ taskId: string;
245
+ stagingId: string;
246
+ options: ConversationOption[];
247
+ text: string;
248
+ }
249
+ interface StagingStreamErrorEvent {
250
+ type: 'error';
251
+ taskId?: string;
252
+ stagingId?: string;
253
+ message: string;
254
+ }
255
+ interface StagingStreamBillingEvent {
256
+ type: 'billing';
257
+ taskId: string;
258
+ stagingId: string;
259
+ usageCharge?: UsageChargeSummary | null;
260
+ availablePoints?: number;
261
+ }
262
+ type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent;
263
+
264
+ declare const BAKED_CONVERSATION_VERSION: 1;
265
+ /** Portable, language-neutral asset. Audio is embedded binary, never a signed URL. */
266
+ interface BakedConversationAsset {
267
+ format: 'npcify.baked-conversation';
268
+ version: typeof BAKED_CONVERSATION_VERSION;
269
+ avatarId: string;
270
+ revision: number;
271
+ rootStagingId: string | null;
272
+ stagings: Array<{
273
+ id: string;
274
+ parentId: string | null;
275
+ triggerText: string | null;
276
+ payload: StagingResultPayload;
277
+ }>;
278
+ }
279
+
280
+ declare const PERFORMANCE_MEDIA_TYPE = "application/vnd.npcify.performance";
281
+ declare const PERFORMANCE_FILE_EXTENSION = ".npcp";
282
+ declare const PERFORMANCE_KINDS: {
283
+ readonly staging: 1;
284
+ readonly stream: 2;
285
+ readonly turn: 3;
286
+ readonly conversation: 4;
287
+ readonly baked: 5;
288
+ readonly bakedResult: 6;
289
+ readonly storedTimeline: 7;
290
+ readonly cachedOpening: 8;
291
+ };
292
+ type PerformanceKind = keyof typeof PERFORMANCE_KINDS;
293
+ /** NPCP v1. Gzip is for storage/assets; realtime events use uncompressed frames. */
294
+ declare const encodePerformance: (kind: PerformanceKind, value: unknown, options?: {
295
+ compress?: boolean;
296
+ }) => Uint8Array<ArrayBuffer>;
297
+ /** Envelope validation precedes decompression/allocation. Consumers validate their schema. */
298
+ declare const decodePerformance: <T = unknown>(input: ArrayBuffer | Uint8Array, expectedKind: PerformanceKind) => T;
299
+
300
+ interface ConversationTurnOptions {
301
+ silent?: boolean;
302
+ parentStagingId?: string | null;
303
+ }
304
+
305
+ type NpcifyAssetInfo = Pick<AvatarPublicInfo, 'animations' | 'idleAnimationId' | 'bakedConversationEnabled' | 'bakedConversationRevision'>;
306
+
307
+ interface LlmThinkingPayload {
308
+ taskId?: string | null;
309
+ stagingId?: string | null;
310
+ [key: string]: unknown;
311
+ }
312
+
313
+ interface AssetLoadProgress {
314
+ progress: number;
315
+ total: number;
316
+ }
317
+
318
+ interface AnimationControllerLike {
319
+ prepareAnimations?: (ids: string[]) => Promise<void>;
320
+ hasAnimation?: (id: string) => boolean;
321
+ playAnimation: (animationId: string, fadeDuration?: number, loopOverride?: boolean) => void;
322
+ playIdle: (fadeDuration?: number) => void;
323
+ stopAnimation: (fadeDuration?: number) => void;
324
+ update: (delta: number) => void;
325
+ }
326
+
327
+ declare const PerformanceRecordingSchemaVersion = 3;
328
+ type PerformanceMouthMode = 'expression' | 'viseme';
329
+ type PerformanceVisemePhase = 'idle' | 'open' | 'neutral' | 'pending';
330
+ interface PerformanceTimelineEntry {
331
+ name: string;
332
+ sourceIndex: number;
333
+ timestampSeconds: number;
334
+ weight?: number;
335
+ }
336
+ interface PerformanceTimelineSnapshot {
337
+ atMs: number;
338
+ revision: number;
339
+ rawEntries: PerformanceTimelineEntry[];
340
+ normalizedEntries: PerformanceTimelineEntry[];
341
+ droppedSourceIndices: number[];
342
+ inversionCount: number;
343
+ }
344
+ interface PerformanceMorphBindingSnapshot {
345
+ key: string;
346
+ primitiveName: string;
347
+ primitiveUuid: string;
348
+ morphTargetIndex: number;
349
+ weight: number;
350
+ affectedVertexCount: number | null;
351
+ mouthVertexOverlapCount: number | null;
352
+ mouthVertexOverlapRatio: number | null;
353
+ }
354
+ interface PerformanceExpressionModelSnapshot {
355
+ name: string;
356
+ layer: string;
357
+ overrideMouth: 'none' | 'block' | 'blend';
358
+ bindings: PerformanceMorphBindingSnapshot[];
359
+ }
360
+ interface FacialPerformanceModelSnapshot {
361
+ expressions: PerformanceExpressionModelSnapshot[];
362
+ mouthExpressionNames: string[];
363
+ }
364
+ interface PerformanceChannelSnapshot {
365
+ target: number;
366
+ value: number;
367
+ }
368
+ interface PerformanceLayerSnapshot {
369
+ target: number;
370
+ value: number;
371
+ channels: Record<string, PerformanceChannelSnapshot>;
372
+ }
373
+ interface FacialPerformanceFrameSnapshot {
374
+ expressionWeights: Record<string, number>;
375
+ layers: Record<string, PerformanceLayerSnapshot>;
376
+ morphTargetInfluences: Record<string, number>;
377
+ mouthExpressionMultiplier: number;
378
+ mouthMode: PerformanceMouthMode;
379
+ renderedExpressionWeights: Record<string, number>;
380
+ }
381
+ interface AudioPerformanceFrameSnapshot {
382
+ currentTimeSeconds: number | null;
383
+ durationSeconds: number | null;
384
+ elementAttached: boolean;
385
+ ended: boolean;
386
+ paused: boolean;
387
+ playbackRate: number;
388
+ readyState: number;
389
+ seeking: boolean;
390
+ }
391
+ interface VisemePerformanceFrameSnapshot {
392
+ activeCueIndex: number | null;
393
+ neutralFrameRequired: boolean;
394
+ nextCueIndex: number;
395
+ pendingCueIndex: number | null;
396
+ phase: PerformanceVisemePhase;
397
+ repeatedCueDipActive: boolean;
398
+ running: boolean;
399
+ timelineRevision: number;
400
+ }
401
+ interface PerformanceFrame {
402
+ atMs: number;
403
+ audio: AudioPerformanceFrameSnapshot;
404
+ deltaMs: number;
405
+ facial: FacialPerformanceFrameSnapshot;
406
+ index: number;
407
+ viseme: VisemePerformanceFrameSnapshot;
408
+ }
409
+ type PerformanceEventPayload = {
410
+ type: 'audio.state';
411
+ audioTimeSeconds: number | null;
412
+ state: 'attached' | 'detached' | 'playing' | 'ended';
413
+ } | {
414
+ type: 'viseme.cue';
415
+ audioTimeSeconds: number;
416
+ cueIndex: number;
417
+ decision: 'played' | 'expired' | 'deferred-for-neutral-frame';
418
+ name: string;
419
+ openDurationMs: number;
420
+ remainingDurationMs: number;
421
+ scheduledAudioTimeSeconds: number;
422
+ timelineRevision: number;
423
+ } | {
424
+ type: 'viseme.neutral';
425
+ afterCueIndex: number | null;
426
+ audioTimeSeconds: number | null;
427
+ reason: 'scheduled' | 'forced-before-cue' | 'timeline-replaced' | 'speech-ended';
428
+ timelineRevision: number;
429
+ } | {
430
+ type: 'viseme.repeat-dip';
431
+ audioTimeSeconds: number;
432
+ fromCueIndex: number;
433
+ name: string;
434
+ ratio: number;
435
+ scheduledAudioTimeSeconds: number;
436
+ targetWeight: number;
437
+ timelineRevision: number;
438
+ toCueIndex: number;
439
+ } | {
440
+ type: 'facial.cue';
441
+ channel: string;
442
+ durationMs: number;
443
+ layer: string;
444
+ weight: number;
445
+ } | {
446
+ type: 'facial.release';
447
+ channel: string | null;
448
+ immediate: boolean;
449
+ layer: string;
450
+ reason: string;
451
+ } | {
452
+ type: 'facial.mouth-mode';
453
+ mode: PerformanceMouthMode;
454
+ } | {
455
+ type: 'audio.unlock';
456
+ state: 'requested' | 'succeeded' | 'blocked';
457
+ } | {
458
+ type: 'staging.stream';
459
+ event: 'start' | 'timeline' | 'options' | 'error';
460
+ stagingId: string;
461
+ taskId: string;
462
+ } | {
463
+ type: 'staging.result';
464
+ source: 'api' | 'opening' | 'replay' | 'turn';
465
+ stagingId: string;
466
+ taskId: string;
467
+ playback: 'requested' | 'already-streamed';
468
+ } | {
469
+ type: 'staging.route';
470
+ state: 'selected' | 'emitted' | 'received';
471
+ stagingId: string;
472
+ } | {
473
+ type: 'staging.playback';
474
+ state: 'requested' | 'started' | 'completed' | 'blocked' | 'cancelled' | 'failed' | 'skipped';
475
+ stagingId: string;
476
+ taskId: string;
477
+ reason?: string;
478
+ };
479
+ type PerformanceEvent = PerformanceEventPayload & {
480
+ atMs: number;
481
+ };
482
+ interface PerformanceRecording {
483
+ characterId: string;
484
+ durationMs: number;
485
+ events: PerformanceEvent[];
486
+ frames: PerformanceFrame[];
487
+ id: string;
488
+ model: FacialPerformanceModelSnapshot | null;
489
+ schemaVersion: typeof PerformanceRecordingSchemaVersion;
490
+ startedAtUnixMs: number;
491
+ stoppedAtUnixMs: number | null;
492
+ timelines: PerformanceTimelineSnapshot[];
493
+ truncated: boolean;
494
+ }
495
+ interface PerformanceRecordingStartOptions {
496
+ maxEvents?: number;
497
+ maxFrames?: number;
498
+ }
499
+ interface PerformanceRecorderOptions {
500
+ characterId: string;
501
+ createId?: () => string;
502
+ nowMs?: () => number;
503
+ unixNowMs?: () => number;
504
+ }
505
+ declare class PerformanceRecorder {
506
+ private readonly characterId;
507
+ private readonly createId;
508
+ private readonly nowMs;
509
+ private readonly unixNowMs;
510
+ private maxEvents;
511
+ private maxFrames;
512
+ private startedAtMonotonicMs;
513
+ private recording;
514
+ constructor(options: PerformanceRecorderOptions);
515
+ get isRecording(): boolean;
516
+ start(options?: PerformanceRecordingStartOptions): void;
517
+ stop(): PerformanceRecording | null;
518
+ getRecording(): PerformanceRecording | null;
519
+ setModel(model: FacialPerformanceModelSnapshot): void;
520
+ recordTimeline(timeline: Omit<PerformanceTimelineSnapshot, 'atMs'>): void;
521
+ recordEvent(event: PerformanceEventPayload): void;
522
+ recordFrame(frame: Omit<PerformanceFrame, 'atMs' | 'index'>): void;
523
+ private elapsedMs;
524
+ }
525
+
526
+ type FacialChannelName = VRMExpressionPresetName | string;
527
+ declare const FacialPerformanceLayer: {
528
+ readonly Emotion: "emotion";
529
+ readonly Viseme: "viseme";
530
+ readonly Blink: "blink";
531
+ readonly Gaze: "gaze";
532
+ };
533
+ type FacialPerformanceLayerId = (typeof FacialPerformanceLayer)[keyof typeof FacialPerformanceLayer];
534
+ type FacialMouthMode = 'expression' | 'viseme';
535
+ interface FacialCueOptions {
536
+ weight?: number;
537
+ durationMs?: number;
538
+ }
539
+ /**
540
+ * Adapts layered performance weights to Three-VRM facial channels.
541
+ *
542
+ * The built-in layers deliberately keep speech, emotion, blinking, and gaze
543
+ * state independent. More sources and blend policies can be added to the
544
+ * renderer-agnostic PerformanceWeightMixer without changing VRM ownership.
545
+ */
546
+ declare class FacialPerformanceMixer {
547
+ private readonly recorder?;
548
+ private vrm;
549
+ private availableChannels;
550
+ private readonly availableChannelSet;
551
+ private readonly channelsByLayer;
552
+ private readonly weightMixer;
553
+ private readonly inspector;
554
+ private readonly cueTimeouts;
555
+ private readonly originalMouthOverrides;
556
+ private activeMouthMode;
557
+ private restoreMouthAfterVisemeRelease;
558
+ constructor(recorder?: PerformanceRecorder | undefined);
559
+ setVrm(vrm: VRM | null): void;
560
+ getChannelNames(): string[];
561
+ getEmotionChannelNames(): string[];
562
+ getVisemeChannelNames(): string[];
563
+ playCue(channel: FacialChannelName, options?: FacialCueOptions): void;
564
+ setEmotion(channel: FacialChannelName, weight: number, mouthMode: FacialMouthMode): void;
565
+ clearEmotion(mouthMode?: FacialMouthMode): void;
566
+ releaseVisemeMouthMode(): void;
567
+ /**
568
+ * Ends the speech source without disturbing any other performance layer.
569
+ * Viseme targets ease to neutral before emotion mouth ownership is restored.
570
+ */
571
+ endSpeech(): void;
572
+ /** Releases the speech layer smoothly without changing emotion ownership. */
573
+ releaseVisemes(reason: string): void;
574
+ attenuateViseme(channel: FacialChannelName, ratio: number, durationMs: number): number | null;
575
+ /**
576
+ * Produces a guaranteed neutral rendered mouth for the current frame.
577
+ * This is intentionally immediate: easing cannot represent a distinct
578
+ * boundary between two adjacent cues with the same viseme name.
579
+ */
580
+ releaseVisemesImmediately(reason: string): void;
581
+ setChannelTarget(channel: FacialChannelName, weight: number): void;
582
+ reset(): void;
583
+ update(delta: number): void;
584
+ capturePerformanceModel(): FacialPerformanceModelSnapshot | null;
585
+ getPerformanceFrameSnapshot(): FacialPerformanceFrameSnapshot;
586
+ private configureLayers;
587
+ private getLayerChannels;
588
+ private areVisemesReleased;
589
+ private scheduleRelease;
590
+ private clearCueTimeouts;
591
+ private clearLayerCueTimeouts;
592
+ private applyMouthMode;
593
+ private restoreExpressionMouthOwnership;
594
+ private restoreMouthOverrides;
595
+ }
596
+
597
+ interface ClientTelemetryOptions {
598
+ enabled: boolean;
599
+ baseUrl: string;
600
+ apiToken: string;
601
+ }
602
+ declare class ClientTelemetry {
603
+ private readonly enabled;
604
+ private readonly baseUrl;
605
+ private readonly apiToken;
606
+ private state;
607
+ constructor({ enabled, baseUrl, apiToken }: ClientTelemetryOptions);
608
+ get traceHeaders(): Record<string, string> | undefined;
609
+ begin(conversationId: string): void;
610
+ clear(error?: Error): void;
611
+ recordStreamStart(taskId: string, stagingId: string): void;
612
+ recordFirstTextDelta(): void;
613
+ recordFirstStaging(): void;
614
+ reportAudioPlaybackStart(params: {
615
+ taskId: string;
616
+ stagingId: string;
617
+ speech: StagingSpeechBlock;
618
+ }): void;
619
+ private readonly createTurnSpan;
620
+ private readonly sendClientLatency;
621
+ }
622
+
623
+ declare class VisemePlayer {
624
+ private readonly facialMixer;
625
+ private readonly recorder?;
626
+ private audioElement;
627
+ private rawVisemes;
628
+ private visemes;
629
+ private normalizedTimeline;
630
+ private nextVisemeIndex;
631
+ private activeCueIndex;
632
+ private repeatedCueDipActive;
633
+ private phase;
634
+ private running;
635
+ private timelineRevision;
636
+ constructor(facialMixer: FacialPerformanceMixer, recorder?: PerformanceRecorder | undefined);
637
+ attachAudioElement(element: HTMLAudioElement | null): void;
638
+ setVisemes(visemes: ConversationTimelineEntry[]): void;
639
+ cleanup(): void;
640
+ capturePerformanceState(): void;
641
+ getAudioPerformanceSnapshot(): AudioPerformanceFrameSnapshot;
642
+ getPerformanceSnapshot(): VisemePerformanceFrameSnapshot;
643
+ update(_delta: number): void;
644
+ private readonly handlePlaying;
645
+ private readonly handleEnded;
646
+ private collectDueCueIndices;
647
+ private shouldCloseActiveCue;
648
+ private shouldDipActiveCue;
649
+ private dipActiveCue;
650
+ private playCue;
651
+ private releaseToNeutral;
652
+ private recordCueDecision;
653
+ private recordTimeline;
654
+ private getFiniteAudioTime;
655
+ private detachAudioListeners;
656
+ private resetPlaybackState;
657
+ }
658
+
659
+ type SpeechPlaybackOptions = {
660
+ signal?: AbortSignal;
661
+ onStart?: () => void;
662
+ };
663
+ declare class AudioPlayer {
664
+ private readonly visemePlayer;
665
+ private audioElement;
666
+ private readonly streams;
667
+ constructor(visemePlayer: VisemePlayer);
668
+ appendStreamChunk(streamId: string, mimeType: string, data: string | Uint8Array): void;
669
+ completeStream(streamId: string): void;
670
+ failStream(streamId: string, message: string): void;
671
+ unlock(): Promise<boolean>;
672
+ playUntilFinished(source: Exclude<StagingAudioSource, {
673
+ type: 'silent';
674
+ }>, visemes: ConversationTimelineEntry[], options?: SpeechPlaybackOptions): Promise<void>;
675
+ getCurrentTime(): number;
676
+ stop(): void;
677
+ reset(): void;
678
+ cleanup(): void;
679
+ private discardStream;
680
+ private readonly buildSilentWav;
681
+ private resolveSourceUrl;
682
+ private getOrCreateStream;
683
+ private ensureAudioElement;
684
+ private prepareElement;
685
+ }
686
+
687
+ type StagingPlaybackCursor = {
688
+ blockId: string | null;
689
+ offsetSeconds: number;
690
+ };
691
+ type StagingPlaybackOptions = {
692
+ complete?: boolean;
693
+ onCursor?: (cursor: StagingPlaybackCursor) => void;
694
+ onCharacter?: (speech: StagingSpeechBlock, characterIndex: number, character: string) => void;
695
+ onSpeechStart?: (speech: StagingSpeechBlock) => void;
696
+ onTimelineStart?: (timeline: StagingTimeline) => void;
697
+ };
698
+ declare class StagingPlayer {
699
+ private readonly audioPlayer;
700
+ private readonly facialMixer;
701
+ private readonly animationController;
702
+ private playbackController;
703
+ private generation;
704
+ private incremental;
705
+ private cursor;
706
+ private onCursor;
707
+ private pendingSpeech;
708
+ private activeExpression;
709
+ private activeExpressionMouthMode;
710
+ private activeAnimation;
711
+ constructor(audioPlayer: AudioPlayer, facialMixer: FacialPerformanceMixer, animationController: AnimationControllerLike);
712
+ play(timeline: StagingTimeline, options?: StagingPlaybackOptions): Promise<void>;
713
+ updateTimeline(timeline: StagingTimeline, complete?: boolean): void;
714
+ private startCursor;
715
+ private clearCursor;
716
+ appendSpeechChunk(streamId: string, mimeType: string, data: string | Uint8Array): void;
717
+ completeStreamingSpeech(streamId: string): void;
718
+ failStreamingSpeech(streamId: string, message: string): void;
719
+ cancel(discardStreams?: boolean): void;
720
+ cleanup(): void;
721
+ private playSpeech;
722
+ private playSpeechAttempt;
723
+ private playSilentSpeech;
724
+ private playTimedRange;
725
+ private applyTracks;
726
+ private clearPerformance;
727
+ }
728
+
729
+ type StagingSpeechPlaybackEvent = StagingSpeechBlock & {
730
+ taskId: string;
731
+ stagingId: string;
732
+ };
733
+ type StagingOptionsReadyPayload = {
734
+ stagingId: string;
735
+ options: StagingResultPayload['options'];
736
+ text?: string;
737
+ };
738
+ type StagingPlaybackStart = {
739
+ stagingId: string;
740
+ taskId: string;
741
+ };
742
+ interface StreamPlaybackDependencies {
743
+ getStagingPlayer: () => StagingPlayer;
744
+ telemetry: ClientTelemetry;
745
+ handleCallbackError: (context: string, error: unknown) => void;
746
+ reportError: (error: Error, context?: string) => void;
747
+ recordEvent?: (event: PerformanceEventPayload) => void;
748
+ }
749
+ declare class StreamPlaybackController {
750
+ private readonly dependencies;
751
+ private streamEventCallback;
752
+ private speechPlaybackStartCallback;
753
+ private characterRevealCallback;
754
+ private playbackBlockedCallback;
755
+ private optionsReadyCallback;
756
+ private timelineStartCallback;
757
+ private cursorCallback;
758
+ private activePlayback;
759
+ private readonly playedStagingIds;
760
+ private readonly states;
761
+ constructor(dependencies: StreamPlaybackDependencies);
762
+ onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): void;
763
+ onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): void;
764
+ onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): void;
765
+ onPlaybackBlocked(callback: (stagingId: string) => void): void;
766
+ onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): void;
767
+ onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): void;
768
+ onPlaybackCursor(callback: (cursor: StagingPlaybackCursor & StagingPlaybackStart) => void): void;
769
+ hasPlayed(stagingId: string): boolean;
770
+ playPayload(payload: StagingResultPayload, options?: {
771
+ force?: boolean;
772
+ timeline?: StagingTimeline;
773
+ }): Promise<boolean>;
774
+ handleEvent(event: StagingStreamEvent): void;
775
+ cleanup(): void;
776
+ private playTimeline;
777
+ private flushOptions;
778
+ private recordStreamEvent;
779
+ private recordPlaybackEvent;
780
+ private errorReason;
781
+ private toSpeechEvent;
782
+ private invokeStreamEventCallback;
783
+ private invokeTimelineStartCallback;
784
+ private invokeSpeechPlaybackStartCallback;
785
+ private invokeCharacterRevealCallback;
786
+ private invokeCallback;
787
+ }
788
+
789
+ interface NpcifyOptions {
790
+ baseUrl: string;
791
+ characterId: string;
792
+ apiToken?: string;
793
+ initialAssetInfo?: NpcifyAssetInfo | null;
794
+ assetProfile?: 'original' | 'web-optimized-v1';
795
+ telemetry?: {
796
+ enabled?: boolean;
797
+ };
798
+ /** Editing uses live generation even when the character is published as baked. */
799
+ authoring?: boolean;
800
+ bakedConversation?: BakedConversationAsset | string | ArrayBuffer | Uint8Array;
801
+ }
802
+ interface PlayStagingOptions {
803
+ force?: boolean;
804
+ silent?: boolean;
805
+ }
806
+ declare class Npcify {
807
+ readonly baseUrl: string;
808
+ readonly characterId: string;
809
+ readonly apiToken: string;
810
+ readonly assetProfile: 'original' | 'web-optimized-v1';
811
+ private readonly lazyAnimations;
812
+ private audioPlayer;
813
+ private stagingPlayer;
814
+ private readonly facialMixer;
815
+ private readonly visemePlayer;
816
+ private readonly performanceRecorder;
817
+ private readonly telemetry;
818
+ private readonly streamPlayback;
819
+ private readonly conversationSocket;
820
+ private renderSessionId;
821
+ private initialAssetInfo;
822
+ private stagingResultCallback;
823
+ private thinkingCallback;
824
+ private errorCallback;
825
+ private readonly notifiedStagingIds;
826
+ private readonly liveConversation;
827
+ private baked;
828
+ private readonly authoring;
829
+ private readonly bakedAsset?;
830
+ vrm: VRM | null;
831
+ animations: Map<string, VRMAnimation>;
832
+ conversationId: string | null;
833
+ animationController?: AnimationControllerLike;
834
+ constructor({ baseUrl, characterId, apiToken, initialAssetInfo, assetProfile, telemetry, authoring, bakedConversation, }: NpcifyOptions);
835
+ loadAssets(callback?: (info: AssetLoadProgress) => void): Promise<void>;
836
+ private animationRefreshId;
837
+ reloadAnimations(info?: NpcifyAssetInfo): Promise<void>;
838
+ ensureAnimation(animationId: string): Promise<void>;
839
+ reloadAssets(): Promise<void>;
840
+ initializeRemote(): Promise<void>;
841
+ createConversation(options?: {
842
+ autoConnect?: boolean;
843
+ autoPlay?: boolean;
844
+ silent?: boolean;
845
+ }): Promise<ConversationCreationResult>;
846
+ connect(conversationId?: string): Promise<string>;
847
+ disconnect(): Promise<void>;
848
+ endConversation(): Promise<void>;
849
+ sendMessage(message: string, options?: ConversationTurnOptions): Promise<StagingResultPayload>;
850
+ sendMessage(conversationId: string, message: string, options?: ConversationTurnOptions): Promise<StagingResultPayload>;
851
+ replayStaging(stagingId: string): Promise<StagingResultPayload>;
852
+ replayStaging(conversationId: string, stagingId: string, options?: PlayStagingOptions): Promise<StagingResultPayload>;
853
+ get isBakedConversation(): boolean;
854
+ getConversationWorldline(conversationId?: string | null): Promise<ConversationWorldline>;
855
+ /** Feed an owner-authoring result through the normal event and playback path. */
856
+ acceptAuthoringResult(payload: StagingResultPayload): Promise<void>;
857
+ playStaging(payload: StagingResultPayload, options?: PlayStagingOptions): Promise<boolean>;
858
+ unlockAudio(): Promise<boolean>;
859
+ setRenderSessionId(renderSessionId: string | null): void;
860
+ onStagingResult(callback: (payload: StagingResultPayload) => Promise<void> | void): void;
861
+ onThinking(callback: (payload: LlmThinkingPayload) => Promise<void> | void): void;
862
+ onError(callback: (error: Error) => void): void;
863
+ onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): void;
864
+ onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): void;
865
+ onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): void;
866
+ onPlaybackBlocked(callback: (stagingId: string) => void): void;
867
+ onOptionsReady(callback: (payload: StagingOptionsReadyPayload) => void): void;
868
+ onPlaybackCursor(callback: Parameters<StreamPlaybackController['onPlaybackCursor']>[0]): void;
869
+ onTimelineStart(callback: (timeline: StagingTimeline, playback: StagingPlaybackStart) => Promise<void> | void): void;
870
+ getExpressions(): string[];
871
+ getMouthExpressionNames(): string[];
872
+ getExpressionNames(): string[];
873
+ playExpression(name: string, options?: FacialCueOptions): void;
874
+ startPerformanceRecording(options?: PerformanceRecordingStartOptions): void;
875
+ stopPerformanceRecording(): PerformanceRecording | null;
876
+ getPerformanceRecording(): PerformanceRecording | null;
877
+ recordStagingRoute(stagingId: string | null, state: 'selected' | 'emitted' | 'received'): void;
878
+ update(delta: number): void;
879
+ private resetAssets;
880
+ private readonly getAssetInfo;
881
+ private handleStagingResult;
882
+ private shouldNotifyResult;
883
+ private invokeStagingResultCallback;
884
+ private invokeThinkingCallback;
885
+ private readonly getAudioPlayer;
886
+ private readonly getStagingPlayer;
887
+ private readonly handleCallbackError;
888
+ private readonly reportError;
889
+ }
890
+
891
+ type PerformanceIssueSeverity = 'error' | 'warning';
892
+ interface PerformanceIssue {
893
+ code: 'render.frame-gap' | 'timeline.invalid-entry' | 'timeline.non-monotonic' | 'viseme.audio-coverage-gap' | 'viseme.cue-late';
894
+ details: Record<string, unknown>;
895
+ message: string;
896
+ severity: PerformanceIssueSeverity;
897
+ }
898
+ interface RepeatedVisemeTransitionAnalysis {
899
+ activeEmotionMouthOverlap: string[];
900
+ fromCueIndex: number;
901
+ hasNeutralFrame: boolean;
902
+ dipEventCount: number;
903
+ minRenderedMouthWeight: number | null;
904
+ name: string;
905
+ neutralFrameCount: number;
906
+ timelineRevision: number;
907
+ toCueIndex: number;
908
+ }
909
+ interface PerformanceAnalysis {
910
+ healthy: boolean;
911
+ issues: PerformanceIssue[];
912
+ repeatedVisemeTransitions: RepeatedVisemeTransitionAnalysis[];
913
+ summary: {
914
+ durationMs: number;
915
+ eventCount: number;
916
+ frameCount: number;
917
+ maxCueDelayMs: number;
918
+ maxFrameGapMs: number;
919
+ maxVisemeCoverageGapMs: number;
920
+ repeatedVisemeTransitionCount: number;
921
+ timelineCount: number;
922
+ };
923
+ }
924
+ interface PerformanceAnalysisOptions {
925
+ cueDelayWarningMs?: number;
926
+ frameGapWarningMs?: number;
927
+ neutralWeightThreshold?: number;
928
+ visemeCoverageWarningMs?: number;
929
+ }
930
+ declare const analyzePerformanceRecording: (recording: PerformanceRecording, options?: PerformanceAnalysisOptions) => PerformanceAnalysis;
931
+
932
+ /** An in-memory conversation runtime. No sockets, HTTP, billing, or telemetry. */
933
+ declare class BakedConversation {
934
+ private readonly nodes;
935
+ private activeId;
936
+ readonly asset: BakedConversationAsset;
937
+ constructor(asset: BakedConversationAsset, characterId: string);
938
+ start(): StagingResultPayload;
939
+ select(stagingId: string): StagingResultPayload;
940
+ reply(message: string, parentStagingId?: string | null): {
941
+ payload: StagingResultPayload;
942
+ ended: boolean;
943
+ };
944
+ worldline(conversationId: string): ConversationWorldline;
945
+ private child;
946
+ }
947
+ declare const loadBakedConversation: (options: {
948
+ baseUrl: string;
949
+ characterId: string;
950
+ apiToken?: string;
951
+ asset?: BakedConversationAsset | string | ArrayBuffer | Uint8Array;
952
+ }) => Promise<BakedConversation>;
953
+
954
+ interface MountNpcifyOptions {
955
+ canvas: HTMLCanvasElement;
956
+ baseUrl: string;
957
+ characterId: string;
958
+ apiKey?: string;
959
+ serverRendering?: boolean;
960
+ assetProfile?: 'original' | 'web-optimized-v1';
961
+ onError?: (error: Error) => void;
962
+ }
963
+ interface NpcifyView {
964
+ npcify: Npcify;
965
+ dispose(): Promise<void>;
966
+ }
967
+ /** Mount into an existing canvas. Its CSS must give it a nonzero width and height. */
968
+ declare const mountNpcify: (options: MountNpcifyOptions) => Promise<NpcifyView>;
969
+
970
+ export { type AudioPerformanceFrameSnapshot, BakedConversation, type BakedConversationAsset, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, type MountNpcifyOptions, Npcify, type NpcifyAssetInfo, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, type RepeatedVisemeTransitionAnalysis, type StagingOptionsReadyPayload, type StagingSpeechPlaybackEvent, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, decodePerformance, encodePerformance, loadBakedConversation, mountNpcify };