@aether-stack-dev/client-sdk 1.2.4 → 1.3.0

package/README.md

@@ -137,6 +137,158 @@ The hook returns:
 
 Pass `autoConnect: true` in options to connect automatically on mount.
 
+The hook also accepts a `character` config for custom avatars (see below).
+
+## Custom Characters
+
+AStack supports custom VRM characters. Any VRM model with ARKit blendshapes will animate with the server's Audio2Face output.
+
+### VRMAvatar Props
+
+| Prop | Type | Default | Description |
+|------|------|---------|-------------|
+| `blendshapes` | `number[]` | **required** | 52 ARKit blendshape values |
+| `modelUrl` | `string` | Supabase-hosted VRM | URL of the VRM model |
+| `blendshapeMap` | `Record<string, string>` | — | Custom blendshape name mapping (ARKit → model) |
+| `expressionOverrides` | `Record<string, number>` | — | Manual VRM expression values (0–1) |
+| `onModelLoad` | `(report: ModelCompatibilityReport) => void` | — | Callback with compatibility report after load |
+| `maxModelSize` | `number` | `31457280` (30MB) | Max model file size in bytes (0 to disable) |
+| `width` | `number` | `400` | Canvas width |
+| `height` | `number` | `400` | Canvas height |
+| `backgroundColor` | `number \| null` | `0xffffff` | Background color, or null for transparent |
+| `cameraPosition` | `[number, number, number]` | `[0, 1.4, 1.2]` | Camera position in world space |
+| `cameraTarget` | `[number, number, number]` | `[0, 1.3, 0]` | Camera look-at target |
+| `cameraFov` | `number` | `30` | Camera field of view (degrees) |
+| `lightIntensity` | `number` | `1` | Multiplier for all scene lights |
+| `idleAnimationUrl` | `string` | — | URL of a .vrma animation clip for idle loop |
+| `postProcessing` | `PostProcessingConfig` | — | Post-processing effects (bloom, ao, dof) |
+| `environmentUrl` | `string` | — | URL of an .hdr environment map |
+| `environmentIntensity` | `number` | `1` | HDRI lighting intensity |
+| `environmentBlur` | `number` | `0` | HDRI background blur |
+| `environmentZoom` | `number` | `1` | HDRI background zoom level |
+| `onEnvironmentLoad` | `() => void` | — | Callback when HDRI environment finishes loading |
+| `animationSpeed` | `number` | `1` | Playback speed multiplier for idle animation clip |
+| `animationWeight` | `number` | `1` | Blend weight for idle animation clip (0–1) |
+| `animationCrossfade` | `number` | `0.5` | Crossfade duration (seconds) when switching animations |
+| `orbitAngle` | `number` | — | Camera azimuth angle (radians) for orbit control |
+| `orbitElevation` | `number` | — | Camera elevation angle (radians) for orbit control |
+| `avatarRotation` | `number` | — | Avatar Y-axis rotation (radians) |
+| `onOrbitChange` | `(angle: number, elevation: number) => void` | — | Callback when camera orbit changes |
+| `features` | `VRMAvatarFeatures` | — | Feature toggles (see below) |
+
+### VRMAvatarFeatures
+
+Control which built-in behaviors are active:
+
+```typescript
+interface VRMAvatarFeatures {
+  idleAnimation?: boolean;     // Subtle breathing/swaying (default: true)
+  microExpressions?: boolean;  // Random blinks, smiles, brow raises (default: true)
+  springBones?: boolean;       // Hair/clothing physics simulation (default: true)
+  postProcessing?: PostProcessingConfig;
+  expressionPresets?: boolean; // VRM expression preset support (default: true)
+}
+```
+
+### Custom Model URL
+
+```typescript
+<VRMAvatar blendshapes={blendshapes} modelUrl="https://cdn.example.com/my-character.vrm" />
+```
+
+### Blendshape Mapping
+
+For models with non-standard blendshape names, provide a mapping from ARKit names to your model's names:
+
+```typescript
+import { VROID_BLENDSHAPE_MAP } from '@aether-stack-dev/client-sdk';
+
+// Use the built-in VRoid Studio preset
+<VRMAvatar blendshapes={blendshapes} blendshapeMap={VROID_BLENDSHAPE_MAP} />
+
+// Or provide a custom mapping
+<VRMAvatar blendshapes={blendshapes} blendshapeMap={{ jawOpen: 'mouth_open', eyeBlinkLeft: 'blink_L' }} />
+```
+
+### Advanced Rendering
+
+VRMAvatar supports expression presets, animation clips, HDRI environments, post-processing (bloom, ambient occlusion, depth of field), and camera control. See the [Rendering Features documentation](https://astack.dev/docs/characters/rendering) for full details and examples.
+
+```typescript
+<VRMAvatar
+  blendshapes={blendshapes}
+  expressionOverrides={{ happy: 0.5 }}
+  idleAnimationUrl="https://cdn.example.com/idle.vrma"
+  environmentUrl="https://cdn.example.com/studio.hdr"
+  postProcessing={{ bloom: 0.3, ao: 0.5, dof: true }}
+/>
+```
+
+### Compatibility Report
+
+```typescript
+<VRMAvatar
+  blendshapes={blendshapes}
+  modelUrl="/custom.vrm"
+  onModelLoad={(report) => {
+    console.log(`${report.supported}/52 blendshapes supported`);
+    console.log('Missing:', report.missing);
+    console.log('Warnings:', report.warnings);
+    console.log('Stats:', report.modelStats);
+  }}
+/>
+```
+
+### Hook Character Config
+
+```typescript
+const { blendshapes, characterConfig } = useAStackCSR({
+  workerUrl, sessionToken,
+  character: {
+    modelUrl: 'https://cdn.example.com/my-character.vrm',
+    blendshapeMap: VROID_BLENDSHAPE_MAP,
+    onModelLoad: (report) => console.log(report),
+  },
+});
+
+// Spread characterConfig directly on VRMAvatar
+<VRMAvatar blendshapes={blendshapes} {...characterConfig} />
+```
+
+See the [full character documentation](https://astack.dev/docs/characters) for VRM requirements, ecosystem links, and integration guides.
+
+## Exports
+
+### Types
+
+| Export | Description |
+|--------|-------------|
+| `AStackCSRConfig` | Constructor config interface |
+| `AStackCSREvents` | Event name/payload map |
+| `CallStatus` | `'idle' \| 'starting' \| 'active' \| 'stopping' \| 'error'` |
+| `VRMAvatarProps` | VRMAvatar component props (react subpath) |
+| `VRMAvatarFeatures` | Feature toggle interface (react subpath) |
+| `ModelCompatibilityReport` | Model load report (react subpath) |
+| `PostProcessingConfig` | Post-processing config (react subpath) |
+| `ARKitBlendshapeName` | Union of 52 ARKit blendshape name strings |
+| `AudioChunk` | Audio player chunk interface |
+| `AudioPlayerEvents` | Audio player event map |
+| `CharacterConfig` | Character config from useAStackCSR (react subpath) |
+| `UseAStackCSROptions` | Hook options (react subpath) |
+| `UseAStackCSRReturn` | Hook return type (react subpath) |
+| `AStackError` | Error class with error code |
+| `ErrorCodes` | Error code constants |
+
+### Constants
+
+| Export | Description |
+|--------|-------------|
+| `ARKIT_BLENDSHAPES` | Array of 52 ARKit blendshape name strings |
+| `BLENDSHAPE_COUNT` | `52` |
+| `CRITICAL_BLENDSHAPES` | `['jawOpen', 'eyeBlinkLeft', 'eyeBlinkRight']` |
+| `DEFAULT_BLENDSHAPE_MAP` | Identity map (ARKit name → same name) |
+| `VROID_BLENDSHAPE_MAP` | VRoid Studio naming preset (react subpath) |
+
 ## License
 
 MIT

package/dist/avatar/VRMAvatar.d.ts

@@ -1,9 +1,43 @@
+import { type ModelCompatibilityReport } from './compatibility';
+import { type PostProcessingConfig } from './postProcessing';
+export type { ModelCompatibilityReport } from './compatibility';
+export type { PostProcessingConfig } from './postProcessing';
+export interface VRMAvatarFeatures {
+    idleAnimation?: boolean;
+    microExpressions?: boolean;
+    springBones?: boolean;
+    postProcessing?: PostProcessingConfig;
+    expressionPresets?: boolean;
+}
 export interface VRMAvatarProps {
     blendshapes: number[];
     width?: number;
     height?: number;
     modelUrl?: string;
-    backgroundColor?: number;
+    backgroundColor?: number | null;
+    blendshapeMap?: Record<string, string>;
+    expressionOverrides?: Record<string, number>;
+    onModelLoad?: (report: ModelCompatibilityReport) => void;
+    maxModelSize?: number;
+    cameraPosition?: [number, number, number];
+    cameraTarget?: [number, number, number];
+    cameraFov?: number;
+    lightIntensity?: number;
+    idleAnimationUrl?: string;
+    animationSpeed?: number;
+    animationWeight?: number;
+    animationCrossfade?: number;
+    postProcessing?: PostProcessingConfig;
+    environmentUrl?: string;
+    environmentIntensity?: number;
+    environmentBlur?: number;
+    environmentZoom?: number;
+    onEnvironmentLoad?: () => void;
+    features?: VRMAvatarFeatures;
+    orbitAngle?: number;
+    orbitElevation?: number;
+    avatarRotation?: number;
+    onOrbitChange?: (angle: number, elevation: number) => void;
 }
-export declare function VRMAvatar({ blendshapes, width, height, modelUrl, backgroundColor }: VRMAvatarProps): import("react/jsx-runtime").JSX.Element;
+export declare function VRMAvatar({ blendshapes, width, height, modelUrl, backgroundColor, blendshapeMap, expressionOverrides, onModelLoad, maxModelSize, cameraPosition, cameraTarget, cameraFov, lightIntensity, idleAnimationUrl, animationSpeed, animationWeight, animationCrossfade, postProcessing, environmentUrl, environmentIntensity, environmentBlur, environmentZoom, onEnvironmentLoad, features, orbitAngle, orbitElevation, avatarRotation, onOrbitChange, }: VRMAvatarProps): import("react/jsx-runtime").JSX.Element;
 export default VRMAvatar;

package/dist/avatar/compatibility.d.ts

@@ -0,0 +1,12 @@
+import { VRM } from '@pixiv/three-vrm';
+export interface ModelCompatibilityReport {
+    supported: number;
+    missing: string[];
+    warnings: string[];
+    modelStats: {
+        vertexCount: number;
+        textureCount: number;
+        morphTargetCount: number;
+    };
+}
+export declare function buildCompatibilityReport(vrm: VRM): ModelCompatibilityReport;

package/dist/avatar/constants.d.ts

@@ -1,3 +1,6 @@
 export declare const ARKIT_BLENDSHAPES: readonly ["jawOpen", "jawForward", "jawLeft", "jawRight", "mouthClose", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthSmileLeft", "mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthStretchLeft", "mouthStretchRight", "mouthRollLower", "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper", "mouthPressLeft", "mouthPressRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthUpperUpLeft", "mouthUpperUpRight", "tongueOut", "cheekPuff", "cheekSquintLeft", "cheekSquintRight", "noseSneerLeft", "noseSneerRight", "eyeBlinkLeft", "eyeBlinkRight", "eyeLookDownLeft", "eyeLookDownRight", "eyeLookInLeft", "eyeLookInRight", "eyeLookOutLeft", "eyeLookOutRight", "eyeLookUpLeft", "eyeLookUpRight", "eyeSquintLeft", "eyeSquintRight", "eyeWideLeft", "eyeWideRight", "browDownLeft", "browDownRight", "browInnerUp", "browOuterUpLeft", "browOuterUpRight"];
 export type ARKitBlendshapeName = typeof ARKIT_BLENDSHAPES[number];
 export declare const BLENDSHAPE_COUNT = 52;
+export declare const CRITICAL_BLENDSHAPES: ARKitBlendshapeName[];
+export declare const DEFAULT_BLENDSHAPE_MAP: Record<string, string>;
+export declare const VROID_BLENDSHAPE_MAP: Record<string, string>;

package/dist/avatar/environment.d.ts

@@ -0,0 +1,6 @@
+import * as THREE from 'three';
+export interface EnvironmentResult {
+    envMap: THREE.Texture;
+    dispose: () => void;
+}
+export declare function loadEnvironment(renderer: THREE.WebGLRenderer, url: string): Promise<EnvironmentResult>;

package/dist/avatar/index.d.ts

@@ -1,4 +1,4 @@
 export { VRMAvatar } from './VRMAvatar';
-export type { VRMAvatarProps } from './VRMAvatar';
-export { ARKIT_BLENDSHAPES, BLENDSHAPE_COUNT } from './constants';
+export type { VRMAvatarProps, VRMAvatarFeatures, ModelCompatibilityReport, PostProcessingConfig } from './VRMAvatar';
+export { ARKIT_BLENDSHAPES, BLENDSHAPE_COUNT, CRITICAL_BLENDSHAPES, DEFAULT_BLENDSHAPE_MAP, VROID_BLENDSHAPE_MAP, } from './constants';
 export type { ARKitBlendshapeName } from './constants';

package/dist/avatar/postProcessing.d.ts

@@ -0,0 +1,20 @@
+import * as THREE from 'three';
+import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer.js';
+export interface PostProcessingConfig {
+    bloom?: number;
+    ao?: number;
+    dof?: boolean;
+}
+export interface PostProcessingPipeline {
+    composer: EffectComposer;
+    dispose: () => void;
+    setSize: (w: number, h: number) => void;
+}
+export declare function createPostProcessing(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.PerspectiveCamera, config: PostProcessingConfig, width: number, height: number): PostProcessingPipeline;
+export declare class FpsMonitor {
+    private samples;
+    private lastTime;
+    disabled: boolean;
+    tick(time: number): void;
+    reset(): void;
+}

package/dist/index.esm.js

@@ -18,6 +18,9 @@ const ARKIT_BLENDSHAPES = [
     'browDownLeft', 'browDownRight', 'browInnerUp', 'browOuterUpLeft', 'browOuterUpRight'
 ];
 const BLENDSHAPE_COUNT = 52;
+const DEFAULT_BLENDSHAPE_MAP = Object.fromEntries(ARKIT_BLENDSHAPES.map(name => [name, name]));
+({
+    ...DEFAULT_BLENDSHAPE_MAP});
 
 const IDX_JAW_OPEN = ARKIT_BLENDSHAPES.indexOf('jawOpen');
 const IDX_MOUTH_FUNNEL = ARKIT_BLENDSHAPES.indexOf('mouthFunnel');

package/dist/index.js

@@ -22,6 +22,9 @@ const ARKIT_BLENDSHAPES = [
     'browDownLeft', 'browDownRight', 'browInnerUp', 'browOuterUpLeft', 'browOuterUpRight'
 ];
 const BLENDSHAPE_COUNT = 52;
+const DEFAULT_BLENDSHAPE_MAP = Object.fromEntries(ARKIT_BLENDSHAPES.map(name => [name, name]));
+({
+    ...DEFAULT_BLENDSHAPE_MAP});
 
 const IDX_JAW_OPEN = ARKIT_BLENDSHAPES.indexOf('jawOpen');
 const IDX_MOUTH_FUNNEL = ARKIT_BLENDSHAPES.indexOf('mouthFunnel');

package/dist/react/index.d.ts

@@ -1,4 +1,5 @@
 export { useAStackCSR } from './useAStackCSR';
-export type { UseAStackCSROptions, UseAStackCSRReturn } from './useAStackCSR';
+export type { UseAStackCSROptions, UseAStackCSRReturn, CharacterConfig } from './useAStackCSR';
 export { VRMAvatar } from '../avatar/VRMAvatar';
-export type { VRMAvatarProps } from '../avatar/VRMAvatar';
+export type { VRMAvatarProps, VRMAvatarFeatures, ModelCompatibilityReport, PostProcessingConfig } from '../avatar/VRMAvatar';
+export { VROID_BLENDSHAPE_MAP } from '../avatar/constants';

package/dist/react/useAStackCSR.d.ts

@@ -1,6 +1,14 @@
 import { AStackCSRClient, AStackCSRConfig, CallStatus } from '../AStackCSRClient';
+import type { ModelCompatibilityReport } from '../avatar/VRMAvatar';
+export interface CharacterConfig {
+    modelUrl?: string;
+    blendshapeMap?: Record<string, string>;
+    onModelLoad?: (report: ModelCompatibilityReport) => void;
+    maxModelSize?: number;
+}
 export interface UseAStackCSROptions extends AStackCSRConfig {
     autoConnect?: boolean;
+    character?: CharacterConfig;
 }
 export interface UseAStackCSRReturn {
     client: AStackCSRClient | null;
@@ -10,6 +18,7 @@ export interface UseAStackCSRReturn {
     transcript: string;
     response: string;
     error: Error | null;
+    characterConfig: CharacterConfig;
     connect: () => Promise<void>;
     disconnect: () => void;
     startCall: () => Promise<void>;