@spatialwalk/avatarkit 1.0.0-beta.35 → 1.0.0-beta.36

package/CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0-beta.36] - 2025-12-20
+
+### ✨ New Features
+- **Audio Sample Rate Configuration** - Added configurable audio sample rate support in SDK initialization
+  - Added `audioFormat` configuration option with `channelCount` (fixed to 1) and `sampleRate` (supports: 8000, 16000, 22050, 24000, 32000, 44100, 48000 Hz, default: 16000)
+  - Audio recording and playback now automatically use the configured sample rate
+  - Vanilla demo updated with sample rate dropdown selector for easy testing
+
 ## [1.0.0-beta.35] - 2025-12-16
 
 ### 🔄 API Changes

package/README.md

@@ -45,6 +45,10 @@ const configuration: Configuration = {
   // - LogLevel.error: Only error logs
   // - LogLevel.warning: Warning and error logs
   // - LogLevel.all: All logs (info, warning, error)
+  audioFormat: { // Optional, default is { channelCount: 1, sampleRate: 16000 }
+    channelCount: 1, // Fixed to 1 (mono)
+    sampleRate: 16000 // Supported: 8000, 16000, 22050, 24000, 32000, 44100, 48000 Hz
+  }
 }
 
 await AvatarSDK.initialize('your-app-id', configuration)
@@ -68,8 +72,8 @@ const avatarView = new AvatarView(avatar, container)
 // 4. Start real-time communication (SDK mode only)
 await avatarView.avatarController.start()
 
-// 5. Send audio data (SDK mode, must be 16kHz mono PCM16 format)
-const audioData = new ArrayBuffer(1024) // Example: 16kHz PCM16 audio data
+// 5. Send audio data (SDK mode, must be mono PCM16 format matching configured sample rate)
+const audioData = new ArrayBuffer(1024) // Example: PCM16 audio data at configured sample rate
 avatarView.avatarController.send(audioData, false) // Send audio data
 avatarView.avatarController.send(audioData, true) // end=true marks the end of current conversation round
 ```
@@ -190,23 +194,37 @@ RenderSystem → WebGPU/WebGL → Canvas rendering
 
 ### Audio Format Requirements
 
-**⚠️ Important:** The SDK requires audio data to be in **16kHz mono PCM16** format:
+**⚠️ Important:** The SDK requires audio data to be in **mono PCM16** format:
 
-- **Sample Rate**: 16kHz (16000 Hz) - This is a backend requirement
-- **Channels**: Mono (single channel)
+- **Sample Rate**: Configurable via `audioFormat.sampleRate` in SDK initialization (default: 16000 Hz)
+  - Supported sample rates: 8000, 16000, 22050, 24000, 32000, 44100, 48000 Hz
+  - The configured sample rate will be used for both audio recording and playback
+- **Channels**: Mono (single channel) - Fixed to 1 channel
 - **Format**: PCM16 (16-bit signed integer, little-endian)
 - **Byte Order**: Little-endian
 
 **Audio Data Format:**
 - Each sample is 2 bytes (16-bit)
 - Audio data should be provided as `ArrayBuffer` or `Uint8Array`
-- For example: 1 second of audio = 16000 samples × 2 bytes = 32000 bytes
+- For example, with 16kHz sample rate: 1 second of audio = 16000 samples × 2 bytes = 32000 bytes
+- For 48kHz sample rate: 1 second of audio = 48000 samples × 2 bytes = 96000 bytes
 
 **Resampling:**
-- If your audio source is at a different sample rate (e.g., 24kHz, 48kHz), you must resample it to 16kHz before sending to the SDK
+- If your audio source is at a different sample rate, you must resample it to match the configured sample rate before sending to the SDK
 - For high-quality resampling, we recommend using Web Audio API's `OfflineAudioContext` with anti-aliasing filtering
 - See example projects for resampling implementation
 
+**Configuration Example:**
+```typescript
+const configuration: Configuration = {
+  environment: Environment.cn,
+  audioFormat: {
+    channelCount: 1, // Fixed to 1 (mono)
+    sampleRate: 48000 // Choose from: 8000, 16000, 22050, 24000, 32000, 44100, 48000
+  }
+}
+```
+
 ## 📚 API Reference
 
 ### AvatarSDK
@@ -431,6 +449,12 @@ interface Configuration {
   environment: Environment
   drivingServiceMode?: DrivingServiceMode  // Optional, default is 'sdk' (SDK mode)
   logLevel?: LogLevel  // Optional, default is 'off' (no logs)
+  audioFormat?: AudioFormat  // Optional, default is { channelCount: 1, sampleRate: 16000 }
+}
+
+interface AudioFormat {
+  readonly channelCount: 1  // Fixed to 1 (mono)
+  readonly sampleRate: number  // Supported: 8000, 16000, 22050, 24000, 32000, 44100, 48000 Hz, default: 16000
 }
 ```
 
@@ -459,6 +483,11 @@ enum LogLevel {
   - `LogLevel.error`: Only error logs
   - `LogLevel.warning`: Warning and error logs
   - `LogLevel.all`: All logs (info, warning, error)
+- `audioFormat`: Configures audio sample rate and channel count
+  - `channelCount`: Fixed to 1 (mono channel)
+  - `sampleRate`: Audio sample rate in Hz (default: 16000)
+    - Supported values: 8000, 16000, 22050, 24000, 32000, 44100, 48000
+    - The configured sample rate will be used for both audio recording and playback
 - `sessionToken`: Set separately via `AvatarSDK.setSessionToken()`, not in Configuration
 
 ```typescript
@@ -500,13 +529,15 @@ enum ConnectionState {
 ```typescript
 enum ConversationState {
   idle = 'idle',      // Idle state (breathing animation)
-  playing = 'playing' // Playing state (active conversation)
+  playing = 'playing', // Playing state (active conversation)
+  pausing = 'pausing' // Pausing state (paused during playback)
 }
 ```
 
 **State Description:**
 - `idle`: Avatar is in idle state (breathing animation), waiting for conversation to start
 - `playing`: Avatar is playing conversation content (including during transition animations)
+- `pausing`: Avatar playback is paused (e.g., when `end=false` and waiting for more audio data)
 
 **Note:** During transition animations, the target state is notified immediately:
 - When transitioning from `idle` to `playing`, the `playing` state is notified immediately

package/dist/{StreamingAudioPlayer-eWy8xl8G.js → StreamingAudioPlayer-GTu9p5GZ.js}

@@ -1,7 +1,7 @@
 var __defProp = Object.defineProperty;
 var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
-import { A as APP_CONFIG, e as errorToMessage, l as logEvent, a as logger } from "./index-DgkjYDp2.js";
+import { A as APP_CONFIG, e as errorToMessage, l as logEvent, a as logger } from "./index-B7llDHtO.js";
 class StreamingAudioPlayer {
   constructor(options) {
     __publicField(this, "audioContext", null);
@@ -17,9 +17,13 @@ class StreamingAudioPlayer {
     __publicField(this, "isPlaying", false);
     __publicField(this, "isPaused", false);
     __publicField(this, "autoStartEnabled", true);
+    __publicField(this, "autoContinue", false);
     __publicField(this, "audioChunks", []);
     __publicField(this, "scheduledChunks", 0);
     __publicField(this, "activeSources", /* @__PURE__ */ new Set());
+    __publicField(this, "lastScheduledChunkEndTime", 0);
+    __publicField(this, "lastGetCurrentTimeLog", 0);
+    __publicField(this, "scheduledChunkInfo", []);
     __publicField(this, "gainNode", null);
     __publicField(this, "volume", 1);
     __publicField(this, "onEndedCallback");
@@ -70,6 +74,13 @@ class StreamingAudioPlayer {
       isPlaying: this.isPlaying,
       scheduledChunks: this.scheduledChunks
     });
+    if (this.autoContinue && this.isPaused) {
+      this.log("[StreamingAudioPlayer] autoContinue=true, auto-resuming playback");
+      this.autoContinue = false;
+      this.resume().catch((err) => {
+        logger.errorWithError("Failed to auto-resume playback:", err);
+      });
+    }
     if (!this.isPlaying && this.autoStartEnabled && this.audioChunks.length > 0) {
       this.log("[StreamingAudioPlayer] Auto-starting playback from addChunk");
       this.startPlayback();
@@ -88,6 +99,7 @@ class StreamingAudioPlayer {
     this.pausedTimeOffset = 0;
     this.pausedAt = 0;
     this.pausedAudioContextTime = 0;
+    this.autoContinue = false;
     this.log("Starting new session", {
       chunks: audioChunks.length
     });
@@ -107,6 +119,9 @@ class StreamingAudioPlayer {
     this.isPlaying = true;
     this.sessionStartTime = this.audioContext.currentTime;
     this.scheduledTime = this.sessionStartTime;
+    this.lastScheduledChunkEndTime = 0;
+    this.scheduledChunkInfo = [];
+    this.autoContinue = false;
     this.log("[StreamingAudioPlayer] Starting playback", {
       sessionStartTime: this.sessionStartTime,
       bufferedChunks: this.audioChunks.length,
@@ -155,16 +170,30 @@ class StreamingAudioPlayer {
       const source = this.audioContext.createBufferSource();
       source.buffer = audioBuffer;
       source.connect(this.gainNode);
-      source.start(this.scheduledTime);
+      const chunkStartTime = this.scheduledTime;
+      source.start(chunkStartTime);
+      const actualStartTime = Math.max(chunkStartTime, this.audioContext.currentTime);
+      this.scheduledChunkInfo.push({
+        startTime: actualStartTime,
+        duration: audioBuffer.duration
+      });
       this.activeSources.add(source);
       source.onended = () => {
         this.activeSources.delete(source);
-        if (isLast && this.activeSources.size === 0) {
-          this.log("Last audio chunk ended, marking playback as ended");
-          this.markEnded();
+        if (this.activeSources.size === 0) {
+          const lastChunk = this.audioChunks[this.scheduledChunks - 1];
+          if (lastChunk && !lastChunk.isLast) {
+            this.log("All audio chunks ended but end=false, pausing and setting autoContinue");
+            this.autoContinue = true;
+            this.pause();
+          } else if (isLast) {
+            this.log("Last audio chunk ended, marking playback as ended");
+            this.markEnded();
+          }
         }
       };
       this.scheduledTime += audioBuffer.duration;
+      this.lastScheduledChunkEndTime = this.scheduledTime - this.sessionStartTime - this.pausedTimeOffset;
       this.scheduledChunks++;
       this.log(`[StreamingAudioPlayer] Scheduled chunk ${chunkIndex + 1}/${this.audioChunks.length}`, {
         startTime: this.scheduledTime - audioBuffer.duration,
@@ -225,8 +254,24 @@ class StreamingAudioPlayer {
       return this.pausedAt;
     }
     const currentAudioTime = this.audioContext.currentTime;
-    const elapsed = currentAudioTime - this.sessionStartTime - this.pausedTimeOffset;
-    return Math.max(0, elapsed);
+    if (this.activeSources.size === 0 && this.scheduledChunks > 0) {
+      return Math.max(0, this.lastScheduledChunkEndTime);
+    }
+    let totalPlayedDuration = 0;
+    for (let i = 0; i < this.scheduledChunkInfo.length; i++) {
+      const chunkInfo = this.scheduledChunkInfo[i];
+      const chunkEndTime = chunkInfo.startTime + chunkInfo.duration;
+      if (currentAudioTime < chunkInfo.startTime) {
+        break;
+      } else if (chunkEndTime <= currentAudioTime) {
+        totalPlayedDuration += chunkInfo.duration;
+      } else {
+        const playedTime = currentAudioTime - chunkInfo.startTime;
+        totalPlayedDuration += playedTime;
+        break;
+      }
+    }
+    return Math.max(0, totalPlayedDuration);
   }
   pause() {
     if (!this.isPlaying || this.isPaused || !this.audioContext) {
@@ -251,6 +296,7 @@ class StreamingAudioPlayer {
     if (!this.isPaused || !this.audioContext || !this.isPlaying) {
       return;
     }
+    this.autoContinue = false;
     if (this.audioContext.state === "suspended") {
       try {
         await this.audioContext.resume();
@@ -300,6 +346,7 @@ class StreamingAudioPlayer {
     this.activeSources.clear();
     this.audioChunks = [];
     this.scheduledChunks = 0;
+    this.autoContinue = false;
     this.log("[StreamingAudioPlayer] Playback stopped, state reset");
   }
   setAutoStart(enabled) {

package/dist/animation/AnimationWebSocketClient.d.ts

@@ -18,6 +18,7 @@ export declare class AnimationWebSocketClient extends EventEmitter {
     private isConnecting;
     private isManuallyDisconnected;
     private reconnectTimer;
+    private sessionConfigured;
     constructor(options: AnimationWebSocketClientOptions);
     connect(characterId: string): Promise<void>;
     disconnect(): void;
@@ -27,6 +28,8 @@ export declare class AnimationWebSocketClient extends EventEmitter {
     getCurrentCharacterId(): string;
     private buildWebSocketUrl;
     private connectWebSocket;
+    private sanitizeUrlForLog;
+    private configureSession;
     private handleMessage;
     private scheduleReconnect;
 }

package/dist/audio/StreamingAudioPlayer.d.ts

@@ -17,9 +17,13 @@ export declare class StreamingAudioPlayer {
     private isPlaying;
     private isPaused;
     private autoStartEnabled;
+    private autoContinue;
     private audioChunks;
     private scheduledChunks;
     private activeSources;
+    private lastScheduledChunkEndTime;
+    private lastGetCurrentTimeLog;
+    private scheduledChunkInfo;
     private gainNode;
     private volume;
     private onEndedCallback?;

package/dist/core/AvatarController.d.ts

@@ -28,6 +28,8 @@ export declare class AvatarController {
     private keyframesOffset;
     private readonly MAX_KEYFRAMES;
     private readonly KEYFRAMES_CLEANUP_THRESHOLD;
+    private lastSyncLogTime;
+    private lastOutOfBoundsState;
     private isAudioOnlyMode;
     private hostModeMetrics;
     private readonly audioBytesPerSecond;

package/dist/generated/common/v1/models.d.ts

@@ -0,0 +1,29 @@
+import { BinaryReader, BinaryWriter } from '@bufbuild/protobuf/wire';
+export declare const protobufPackage = "common.v1";
+
+export interface CustomAnimation {
+    key: string;
+    pbUrl: string;
+    wavUrl: string;
+    remark: string;
+}
+export declare const CustomAnimation: MessageFns<CustomAnimation>;
+type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
+export type DeepPartial<T> = T extends Builtin ? T : T extends globalThis.Array<infer U> ? globalThis.Array<DeepPartial<U>> : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>> : T extends {} ? {
+    [K in keyof T]?: DeepPartial<T[K]>;
+} : Partial<T>;
+type KeysOfUnion<T> = T extends T ? keyof T : never;
+export type Exact<P, I extends P> = P extends Builtin ? P : P & {
+    [K in keyof P]: Exact<P[K], I[K]>;
+} & {
+    [K in Exclude<keyof I, KeysOfUnion<P>>]: never;
+};
+export interface MessageFns<T> {
+    encode(message: T, writer?: BinaryWriter): BinaryWriter;
+    decode(input: BinaryReader | Uint8Array, length?: number): T;
+    fromJSON(object: any): T;
+    toJSON(message: T): unknown;
+    create<I extends Exact<DeepPartial<T>, I>>(base?: I): T;
+    fromPartial<I extends Exact<DeepPartial<T>, I>>(object: I): T;
+}
+export {};

package/dist/generated/driveningress/v2/driveningress.d.ts

@@ -1,6 +1,32 @@
 import { BinaryReader, BinaryWriter } from '@bufbuild/protobuf/wire';
+import { CustomAnimation } from '../../common/v1/models';
 import { Timestamp } from '../../google/protobuf/timestamp';
 export declare const protobufPackage = "driveningress.v2";
+export declare enum MessageType {
+    MESSAGE_UNSPECIFIED = 0,
+
+    MESSAGE_CLIENT_CONFIGURE_SESSION = 1,
+
+    MESSAGE_SERVER_CONFIRM_SESSION = 2,
+    MESSAGE_CLIENT_AUDIO_INPUT = 3,
+    MESSAGE_SERVER_ERROR = 4,
+    MESSAGE_SERVER_RESPONSE_ANIMATION = 5,
+    UNRECOGNIZED = -1
+}
+export declare function messageTypeFromJSON(object: any): MessageType;
+export declare function messageTypeToJSON(object: MessageType): string;
+export declare enum AudioFormat {
+    AUDIO_FORMAT_PCM_S16LE = 0,
+    UNRECOGNIZED = -1
+}
+export declare function audioFormatFromJSON(object: any): AudioFormat;
+export declare function audioFormatToJSON(object: AudioFormat): string;
+export declare enum TransportCompression {
+    TRANSPORT_COMPRESSION_NONE = 0,
+    UNRECOGNIZED = -1
+}
+export declare function transportCompressionFromJSON(object: any): TransportCompression;
+export declare function transportCompressionToJSON(object: TransportCompression): string;
 export interface GetCharacterInfoRequest {
     characterId: string;
 }
@@ -16,6 +42,7 @@ export interface CharacterAsset {
     characterSettings?: {
         [key: string]: any;
     } | undefined;
+    customAnimations: CustomAnimation[];
 }
 export interface Resource {
     type: string;
@@ -36,12 +63,66 @@ export interface Animations {
     frameMono?: ResourceHolder | undefined;
     audioMono?: ResourceHolder | undefined;
 }
+export interface ClientConfigureSession {
+    sampleRate: number;
+    bitrate: number;
+    audioFormat: AudioFormat;
+    transportCompression: TransportCompression;
+}
+export interface ServerConfirmSession {
+    connectionId: string;
+}
+export interface ClientAudioInput {
+    reqId: string;
+    end: boolean;
+    audio: Uint8Array;
+}
+export interface ServerError {
+    connectionId: string;
+    reqId: string;
+    code: number;
+    message: string;
+}
+export interface Flame {
+    translation: number[];
+    rotation: number[];
+    neckPose: number[];
+    jawPose: number[];
+    eyePose: number[];
+    eyeLid: number[];
+    expression: number[];
+}
+export interface FlameAnimation {
+    keyframes: Flame[];
+}
+export interface ServerResponseAnimation {
+    connectionId: string;
+    reqId: string;
+    end: boolean;
+    animation?: FlameAnimation | undefined;
+}
+export interface Message {
+    type: MessageType;
+    clientConfigureSession?: ClientConfigureSession | undefined;
+    serverConfirmSession?: ServerConfirmSession | undefined;
+    clientAudioInput?: ClientAudioInput | undefined;
+    serverError?: ServerError | undefined;
+    serverResponseAnimation?: ServerResponseAnimation | undefined;
+}
 export declare const GetCharacterInfoRequest: MessageFns<GetCharacterInfoRequest>;
 export declare const CharacterAsset: MessageFns<CharacterAsset>;
 export declare const Resource: MessageFns<Resource>;
 export declare const ResourceHolder: MessageFns<ResourceHolder>;
 export declare const Models: MessageFns<Models>;
 export declare const Animations: MessageFns<Animations>;
+export declare const ClientConfigureSession: MessageFns<ClientConfigureSession>;
+export declare const ServerConfirmSession: MessageFns<ServerConfirmSession>;
+export declare const ClientAudioInput: MessageFns<ClientAudioInput>;
+export declare const ServerError: MessageFns<ServerError>;
+export declare const Flame: MessageFns<Flame>;
+export declare const FlameAnimation: MessageFns<FlameAnimation>;
+export declare const ServerResponseAnimation: MessageFns<ServerResponseAnimation>;
+export declare const Message: MessageFns<Message>;
 export interface DrivenIngressService {
     GetCharacterInfo(request: GetCharacterInfoRequest): Promise<CharacterAsset>;
 }