@npcify/web 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -125,6 +125,7 @@ animate()
125
125
  - **Dialogue & Speech State**:
126
126
  - `character.speaking`, `character.thinking`, `character.currentSpeech`, `character.currentText`, `character.revealedText`, `character.options`, `character.cursor`, `character.speech`.
127
127
  - `await character.createConversation()`, `await character.sendMessage(text)`, `await character.endConversation()`.
128
+ - `character.onConnectionClosed(callback)`: The server closed the conversation connection ([lifecycle](#runtime-and-lifecycle)).
128
129
  - **Voice Input** ([details](#voice-input-and-input-modes)):
129
130
  - `character.bindPushToTalk(options)`, `character.startListening(options)`, `character.stopListening()`, `character.cancelListening()`.
130
131
  - `character.listeningState`, `character.onListeningState(callback)`, `character.onTranscript(callback)`, `character.inputModes`.
@@ -277,6 +278,16 @@ character.onListeningState((state) => {
277
278
  baked playback requires browser rendering.
278
279
  - Keys may cover multiple characters. Editing scopes, rotating, or deleting keys
279
280
  affects subsequent requests and disconnects associated conversation sockets.
281
+ - `character.onConnectionClosed(({ reason, code, conversationId }) => …)` reports
282
+ a close of the conversation connection by the server: a refused or failed
283
+ join (`code`), or a key that was edited, rotated or deleted (`code: null`).
284
+ The SDK does not use that connection again: the next `connect`,
285
+ `sendMessage` or `startListening` opens a new one. Joins and requests that
286
+ wait on a closed connection fail at once with a `ConversationConnectionError`
287
+ (`CONVERSATION_DISCONNECTED`); a wait without an answer in time fails with
288
+ `CONVERSATION_TIMEOUT`. A refused conversation fails at once with
289
+ `CONVERSATION_ACCESS_DENIED` until `endConversation()`; after another join
290
+ code (`CONVERSATION_JOIN_FAILED` or a newer one) the next call tries again.
280
291
  - ESM only; `require()` is not a supported package entry point. Browser APIs are
281
292
  used when creating a view; do not call `mountNpcify` during server rendering.
282
293
 
package/dist/index.d.ts CHANGED
@@ -50,6 +50,8 @@ interface AvatarPublicInfo {
50
50
  speechInputEnabled?: boolean;
51
51
  autoPlay: boolean;
52
52
  serverRenderingEnabled: boolean;
53
+ /** The character's body motion is generated live in each turn (missing on older servers). */
54
+ realtimeMotionEnabled?: boolean;
53
55
  cameraSqt?: Sqt | null;
54
56
  }
55
57
 
@@ -192,12 +194,45 @@ interface StagingAnimationClip {
192
194
  animation: string;
193
195
  loop: boolean;
194
196
  }
197
+ /** A part of the staging whose body motion is generated live (NPCP 1.1). */
198
+ interface StagingMotionClip {
199
+ /** Seconds from the start of the staging. */
200
+ startTime: number;
201
+ /** Seconds from the start of the staging. */
202
+ endTime: number;
203
+ /** A short English description of the body motion. */
204
+ prompt: string;
205
+ }
206
+ type StagingMotionFramesSource = {
207
+ type: 's3';
208
+ url: string;
209
+ } | {
210
+ type: 'bytes';
211
+ data: Uint8Array;
212
+ };
213
+ /**
214
+ * The stored live motion of a staging (NPCP 1.1): pose frames on the staging
215
+ * clock, in the layout of `motion_frames` (see `motion-frames.ts`).
216
+ */
217
+ interface StagingMotionTrack {
218
+ fps: number;
219
+ frameCount: number;
220
+ /** VRM humanoid bone names, the order of the rotations of a frame. */
221
+ bones: string[];
222
+ /** Hips height of the source rest pose in meters. */
223
+ restHipsHeight: number;
224
+ frames: StagingMotionFramesSource;
225
+ }
195
226
  interface StagingTimeline {
196
227
  /** Total duration in seconds. */
197
228
  duration: number;
198
229
  dialogue: StagingDialogueBlock[];
199
230
  expressions: StagingExpressionClip[];
200
231
  animations: StagingAnimationClip[];
232
+ /** Live motion prompts of a character with real-time motion (NPCP 1.1). */
233
+ motions?: StagingMotionClip[];
234
+ /** Stored live motion frames, on reads of a completed staging (NPCP 1.1). */
235
+ motion?: StagingMotionTrack;
201
236
  }
202
237
  declare const STAGING_PROTOCOL_VERSION: 1;
203
238
  interface StagingResultPayload {
@@ -262,6 +297,7 @@ interface StagingStreamSpeechErrorEvent {
262
297
  stagingId: string;
263
298
  speechId: string;
264
299
  streamId: string;
300
+ /** Always `SPEECH_FAILED`; the cause stays in the server logs. */
265
301
  message: string;
266
302
  }
267
303
  interface StagingStreamOptionsEvent {
@@ -275,6 +311,10 @@ interface StagingStreamErrorEvent {
275
311
  type: 'error';
276
312
  taskId?: string;
277
313
  stagingId?: string;
314
+ /**
315
+ * A client error of the server, for example `INSUFFICIENT_POINTS`, or
316
+ * `TURN_FAILED`. Vendor text stays in the server logs.
317
+ */
278
318
  message: string;
279
319
  }
280
320
  interface StagingStreamBillingEvent {
@@ -284,7 +324,28 @@ interface StagingStreamBillingEvent {
284
324
  usageCharge?: UsageChargeSummary | null;
285
325
  availablePoints?: number;
286
326
  }
287
- type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamLiveEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent;
327
+ /**
328
+ * Live pose frames of a character with real-time motion (NPCP 1.1). Older
329
+ * receivers ignore the event. Frame `startFrame + i` plays at
330
+ * `(startFrame + i) / fps` seconds of the staging clock. A frame of a later
331
+ * event replaces the frame with the same index unless its `revision` is lower.
332
+ */
333
+ interface StagingStreamMotionFramesEvent {
334
+ type: 'motion_frames';
335
+ taskId: string;
336
+ stagingId: string;
337
+ revision: number;
338
+ fps: number;
339
+ startFrame: number;
340
+ frameCount: number;
341
+ /** VRM humanoid bone names, the order of the rotations of a frame. */
342
+ bones: string[];
343
+ /** Hips height of the source rest pose in meters. */
344
+ restHipsHeight: number;
345
+ /** `frameCount x (3 + 4 x bones.length)` float32 values, little endian. */
346
+ frames: Uint8Array;
347
+ }
348
+ type StagingStreamEvent = StagingStreamStartEvent | StagingStreamTextDeltaEvent | StagingStreamTimelineEvent | StagingStreamPrefixEvent | StagingStreamLiveEvent | StagingStreamSpeechChunkEvent | StagingStreamSpeechCompleteEvent | StagingStreamSpeechErrorEvent | StagingStreamOptionsEvent | StagingStreamErrorEvent | StagingStreamBillingEvent | StagingStreamMotionFramesEvent;
288
349
 
289
350
  /**
290
351
  * Voice input of conversations (docs/voice-input.md): the input modes of the
@@ -994,8 +1055,14 @@ declare class StagingPlayer {
994
1055
  private activeExpression;
995
1056
  private activeExpressionMouthMode;
996
1057
  private activeAnimation;
1058
+ private currentTrackTime;
997
1059
  constructor(audioPlayer: AudioPlayer, facialMixer: FacialPerformanceMixer, animationController: AnimationControllerLike);
998
1060
  get paused(): boolean;
1061
+ /**
1062
+ * Seconds of the staging clock that the tracks show now, or null when no
1063
+ * staging plays (also while playback waits for the next dialogue block).
1064
+ */
1065
+ get trackTime(): number | null;
999
1066
  pause(): void;
1000
1067
  resume(): Promise<void>;
1001
1068
  private waitUntilResumed;
@@ -1016,6 +1083,55 @@ declare class StagingPlayer {
1016
1083
  private clearPerformance;
1017
1084
  }
1018
1085
 
1086
+ /**
1087
+ * How the host uses the root of live motion. `anchored` (default) removes the
1088
+ * horizontal hips movement of the turn, so that the character stays where the
1089
+ * host placed it; height and turns stay. `world` applies it as generated.
1090
+ */
1091
+ type LiveMotionRootMode = 'anchored' | 'world';
1092
+ interface LiveMotionOptions {
1093
+ rootMode?: LiveMotionRootMode;
1094
+ /** Seconds to blend from idle into live frames. */
1095
+ blendInSeconds?: number;
1096
+ /** Seconds to blend from live frames back to idle. */
1097
+ blendOutSeconds?: number;
1098
+ }
1099
+ /**
1100
+ * Plays live motion frames (ADR 0011) on a VRM's normalized humanoid. Call
1101
+ * `update()` after the animation mixer and before `vrm.update()`: the live
1102
+ * pose blends over the mixer's pose (idle) with its own weight.
1103
+ */
1104
+ declare class LiveMotionPlayer {
1105
+ private vrm;
1106
+ private readonly tracks;
1107
+ private readonly rootMode;
1108
+ private readonly blendIn;
1109
+ private readonly blendOut;
1110
+ private weight;
1111
+ /** The last applied pose, to blend out of it after the frames end. */
1112
+ private lastPose;
1113
+ private readonly scratch;
1114
+ constructor(options?: LiveMotionOptions);
1115
+ setVrm(vrm: VRM | null): void;
1116
+ /** Adds the frames of a `motion_frames` event; invalid events are dropped. */
1117
+ push(received: StagingStreamMotionFramesEvent): boolean;
1118
+ /** Loads the stored frames of a staging (a replay). */
1119
+ load(stagingId: string, track: StagingMotionTrack, fetchBytes?: (url: string) => Promise<Uint8Array>): Promise<void>;
1120
+ has(stagingId: string): boolean;
1121
+ forget(stagingId: string): void;
1122
+ /** The current live weight, 0 (idle) to 1 (live frames). */
1123
+ get liveWeight(): number;
1124
+ /**
1125
+ * Applies the live pose of `stagingId` at `time` seconds of its staging
1126
+ * clock; `time` null means that no staging plays now.
1127
+ */
1128
+ update(delta: number, stagingId: string | null, time: number | null): void;
1129
+ private track;
1130
+ /** The interpolated frame at `time`, or null when its frames are missing. */
1131
+ private sample;
1132
+ private apply;
1133
+ }
1134
+
1019
1135
  type Vector3Tuple = readonly [number, number, number];
1020
1136
  interface Vector3Like {
1021
1137
  readonly x: number;
@@ -1151,6 +1267,11 @@ interface NpcifyOptions {
1151
1267
  * `Character` (`loadCharacter` / `createCharacter` Three.js game mode).
1152
1268
  */
1153
1269
  normalizeIdleOrientation?: boolean;
1270
+ /**
1271
+ * Playback of live motion frames (characters with real-time motion). The
1272
+ * default `rootMode: 'anchored'` keeps the character where it stands.
1273
+ */
1274
+ liveMotion?: LiveMotionOptions;
1154
1275
  /**
1155
1276
  * How the player can reply: `'options'`, `'text'` and `'voice'`, in any
1156
1277
  * combination; one mode forces it. Without it, every mode that the
@@ -1188,6 +1309,10 @@ interface StreamPlaybackDependencies {
1188
1309
  handleCallbackError: (context: string, error: unknown) => void;
1189
1310
  reportError: (error: Error, context?: string) => void;
1190
1311
  recordEvent?: (event: PerformanceEventPayload) => void;
1312
+ /** Live motion frames of the playing staging (NPCP 1.1). */
1313
+ onMotionFrames?: (event: StagingStreamMotionFramesEvent) => void;
1314
+ /** The stored live motion of a staging that starts to play (a replay). */
1315
+ onMotionTrack?: (stagingId: string, track: StagingMotionTrack) => void;
1191
1316
  }
1192
1317
  declare class StreamPlaybackController {
1193
1318
  private readonly dependencies;
@@ -1214,6 +1339,8 @@ declare class StreamPlaybackController {
1214
1339
  get revealedText(): string;
1215
1340
  get options(): StagingResultPayload['options'];
1216
1341
  get cursor(): CharacterPlaybackCursor | null;
1342
+ /** The staging that plays or waits to play, or null. */
1343
+ get activeStagingId(): string | null;
1217
1344
  onStreamEvent(callback: (event: StagingStreamEvent) => Promise<void> | void): () => void;
1218
1345
  onSpeechPlaybackStart(callback: (speech: StagingSpeechPlaybackEvent) => Promise<void> | void): () => void;
1219
1346
  onCharacterReveal(callback: (speech: StagingSpeechPlaybackEvent, characterIndex: number, character: string) => void): () => void;
@@ -1300,6 +1427,7 @@ declare class CharacterRuntime {
1300
1427
  protected readonly performanceRecorder: PerformanceRecorder;
1301
1428
  protected readonly telemetry: ClientTelemetry;
1302
1429
  protected readonly streamPlayback: StreamPlaybackController;
1430
+ protected readonly liveMotion: LiveMotionPlayer;
1303
1431
  protected defaultParent: Object3D | null;
1304
1432
  protected currentGazeTarget: Object3D | null;
1305
1433
  /** The current gaze target came from `lookAt()`. */
@@ -1436,6 +1564,42 @@ declare class CharacterRuntime {
1436
1564
  protected readonly reportError: (error: Error, context?: string) => void;
1437
1565
  }
1438
1566
 
1567
+ /**
1568
+ * The code of a join that the server did not accept. Only
1569
+ * `CONVERSATION_ACCESS_DENIED` is final; the next join tries any other code
1570
+ * again. Another string is a code of a newer server.
1571
+ */
1572
+ type ConversationJoinFailure = 'CONVERSATION_ACCESS_DENIED' | 'CONVERSATION_JOIN_FAILED' | (string & {});
1573
+ /**
1574
+ * `CONVERSATION_DISCONNECTED`: the connection closed, was lost or is not
1575
+ * open. `CONVERSATION_TIMEOUT`: no acknowledgement came in time.
1576
+ */
1577
+ type ConversationConnectionErrorCode = ConversationJoinFailure | 'CONVERSATION_DISCONNECTED' | 'CONVERSATION_TIMEOUT';
1578
+ /**
1579
+ * A join or a request that the conversation connection could not complete
1580
+ * (docs/streaming_reply.md#web-sdk).
1581
+ */
1582
+ declare class ConversationConnectionError extends Error {
1583
+ readonly code: ConversationConnectionErrorCode;
1584
+ /** The close or loss that failed the wait (`io server disconnect`, `transport close`, `connect_error`, ...), or null. */
1585
+ readonly reason: string | null;
1586
+ constructor(code: ConversationConnectionErrorCode, message: string,
1587
+ /** The close or loss that failed the wait (`io server disconnect`, `transport close`, `connect_error`, ...), or null. */
1588
+ reason?: string | null);
1589
+ }
1590
+ /** The server closed the conversation connection, or refused to open it. */
1591
+ interface ConversationConnectionClosedEvent {
1592
+ /** `io server disconnect`, or `connect_error` when the server refused the connection. */
1593
+ reason: string;
1594
+ /**
1595
+ * The join that made the server close the connection; null for another
1596
+ * close, for example after the API key was rotated, edited or deleted.
1597
+ */
1598
+ code: ConversationJoinFailure | null;
1599
+ /** The conversation of that join, or null. */
1600
+ conversationId: string | null;
1601
+ }
1602
+
1439
1603
  declare class MicrophoneError extends Error {
1440
1604
  readonly code: 'MICROPHONE_PERMISSION_DENIED' | 'MICROPHONE_UNAVAILABLE';
1441
1605
  constructor(code: 'MICROPHONE_PERMISSION_DENIED' | 'MICROPHONE_UNAVAILABLE', message: string);
@@ -1536,6 +1700,8 @@ declare class Character extends CharacterRuntime {
1536
1700
  private readonly notifiedStagingIds;
1537
1701
  private readonly liveConversation;
1538
1702
  private baked;
1703
+ /** The conversation whose live motion was warmed up. */
1704
+ private liveMotionPrepared;
1539
1705
  private readonly authoring;
1540
1706
  protected readonly normalizeIdleOrientation: boolean;
1541
1707
  private readonly bakedAsset?;
@@ -1548,6 +1714,7 @@ declare class Character extends CharacterRuntime {
1548
1714
  private latestOptions;
1549
1715
  private listeningStateCallback;
1550
1716
  private transcriptCallback;
1717
+ private connectionClosedCallback;
1551
1718
  conversationId: string | null;
1552
1719
  constructor(options: NpcifyOptions);
1553
1720
  load(callback?: (info: AssetLoadProgress) => void): Promise<this>;
@@ -1555,6 +1722,13 @@ declare class Character extends CharacterRuntime {
1555
1722
  reloadAnimations(info?: NpcifyAssetInfo): Promise<void>;
1556
1723
  reloadAssets(): Promise<void>;
1557
1724
  initializeRemote(): Promise<void>;
1725
+ /**
1726
+ * Warms up live motion for the current conversation, once, on the
1727
+ * player's intent: call it at the first focus or key in a reply field or
1728
+ * when a talk key goes down. It does nothing for a character without live
1729
+ * motion; the server limits warm-ups per player (ADR 0011).
1730
+ */
1731
+ prepareLiveMotion(): Promise<boolean>;
1558
1732
  createConversation(options?: {
1559
1733
  autoConnect?: boolean;
1560
1734
  autoPlay?: boolean;
@@ -1594,6 +1768,13 @@ declare class Character extends CharacterRuntime {
1594
1768
  onListeningState(callback: (state: ListeningState) => void): () => void;
1595
1769
  /** Partial text while the player speaks (`final: false`), then the final text. */
1596
1770
  onTranscript(callback: (event: TranscriptEvent) => void): () => void;
1771
+ /**
1772
+ * The server closed the conversation connection: a refused or failed join,
1773
+ * or an API key that was rotated, edited or deleted. The next `connect`,
1774
+ * `sendMessage` or `startListening` opens a new connection; a refused
1775
+ * conversation fails at once with `CONVERSATION_ACCESS_DENIED`.
1776
+ */
1777
+ onConnectionClosed(callback: (event: ConversationConnectionClosedEvent) => void): () => void;
1597
1778
  getConversationWorldline(conversationId?: string | null): Promise<ConversationWorldline>;
1598
1779
  acceptAuthoringResult(payload: StagingResultPayload): Promise<void>;
1599
1780
  playStaging(payload: StagingResultPayload, options?: PlayStagingOptions): Promise<boolean>;
@@ -1727,4 +1908,4 @@ interface NpcifyView {
1727
1908
  /** Mount into an existing canvas. Its CSS must give it a nonzero width and height. */
1728
1909
  declare const mountNpcify: (options: MountNpcifyOptions) => Promise<NpcifyView>;
1729
1910
 
1730
- export { AnimationController, type AnimationControllerLike, type AnimationControllerOptions, type AssetLoadProgress, type AudioCapture, type AudioPerformanceFrameSnapshot, AudioPlayer, BakedConversation, type BakedConversationAsset, type CaptureFrame, Character, CharacterAnimation, type CharacterAnimationPlayOptions, CharacterAudio, type CharacterDisposeOptions, CharacterExpressions, type CharacterLookAtTarget, type CharacterOptions, type CharacterPlaybackCursor, CharacterRuntime, type CharacterRuntimeInitOptions, CharacterSceneRig, type CharacterShadowOptions, type CharacterSpeechState, type CharacterTransformOptions, type ConversationTurnOptions, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, INPUT_MODES, type InputMode, InputModeError, type ListeningOptions, type ListeningState, type LlmThinkingPayload, type LoadCharacterOptions, MicrophoneError, type MountNpcifyOptions, Npcify, type NpcifyAssetInfo, Character as NpcifyCharacter, type NpcifyOptions, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, type PushToTalkOptions, type PushToTalkTarget, RemoteAnimationController, type RepeatedVisemeTransitionAnalysis, SpeechInputError, type SpeechInputResult, type StagingOptionsReadyPayload, type StagingPlaybackStart, type StagingSpeechPlaybackEvent, type TranscriptEvent, type Vector3Like, type Vector3Tuple, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, bindPushToTalk, createCharacter, decodePerformance, encodePerformance, loadBakedConversation, loadCharacter, mountNpcify };
1911
+ export { AnimationController, type AnimationControllerLike, type AnimationControllerOptions, type AssetLoadProgress, type AudioCapture, type AudioPerformanceFrameSnapshot, AudioPlayer, BakedConversation, type BakedConversationAsset, type CaptureFrame, Character, CharacterAnimation, type CharacterAnimationPlayOptions, CharacterAudio, type CharacterDisposeOptions, CharacterExpressions, type CharacterLookAtTarget, type CharacterOptions, type CharacterPlaybackCursor, CharacterRuntime, type CharacterRuntimeInitOptions, CharacterSceneRig, type CharacterShadowOptions, type CharacterSpeechState, type CharacterTransformOptions, type ConversationConnectionClosedEvent, ConversationConnectionError, type ConversationConnectionErrorCode, type ConversationJoinFailure, type ConversationTurnOptions, type FacialChannelName, type FacialCueOptions, type FacialMouthMode, type FacialPerformanceFrameSnapshot, FacialPerformanceLayer, type FacialPerformanceLayerId, FacialPerformanceMixer, type FacialPerformanceModelSnapshot, INPUT_MODES, type InputMode, InputModeError, type ListeningOptions, type ListeningState, type LiveMotionOptions, LiveMotionPlayer, type LiveMotionRootMode, type LlmThinkingPayload, type LoadCharacterOptions, MicrophoneError, type MountNpcifyOptions, Npcify, type NpcifyAssetInfo, Character as NpcifyCharacter, type NpcifyOptions, type NpcifyView, PERFORMANCE_FILE_EXTENSION, PERFORMANCE_MEDIA_TYPE, type PerformanceAnalysis, type PerformanceAnalysisOptions, type PerformanceEvent, type PerformanceFrame, type PerformanceIssue, type PerformanceKind, type PerformanceRecording, PerformanceRecordingSchemaVersion, type PerformanceRecordingStartOptions, type PerformanceTimelineSnapshot, type PlayStagingOptions, type PushToTalkOptions, type PushToTalkTarget, RemoteAnimationController, type RepeatedVisemeTransitionAnalysis, SpeechInputError, type SpeechInputResult, type StagingOptionsReadyPayload, type StagingPlaybackStart, type StagingSpeechPlaybackEvent, type TranscriptEvent, type Vector3Like, type Vector3Tuple, type VisemePerformanceFrameSnapshot, analyzePerformanceRecording, bindPushToTalk, createCharacter, decodePerformance, encodePerformance, loadBakedConversation, loadCharacter, mountNpcify };