@spatialwalk/avatarkit 1.0.0-beta.4 → 1.0.0-beta.40

package/dist/withJoi/src/config/realtime-config.d.ts

@@ -0,0 +1,46 @@
+export declare const REALTIME_CONFIG: {
+    readonly characterList: {
+        readonly pageSize: 20;
+    };
+    readonly audio: {
+        readonly recording: {
+            readonly sampleRate: 16000;
+            readonly channelCount: 1;
+            readonly echoCancellation: true;
+            readonly noiseSuppression: true;
+            readonly format: "pcm_s16le";
+        };
+        readonly playback: {
+            readonly sampleRate: 24000;
+            readonly channelCount: 1;
+            readonly bitsPerSample: 16;
+            readonly codec: "pcm_s16le";
+        };
+    };
+    readonly sync: {
+        readonly maxBufferTime: 5000;
+        readonly alignmentTolerance: 100;
+    };
+    readonly connection: {
+        readonly reconnectAttempts: 3;
+        readonly maxReconnectAttempts: 5;
+        readonly heartbeatInterval: 30000;
+    };
+    readonly vad: {
+        readonly sampleRate: 16000;
+        readonly speechThreshold: 0.3;
+        readonly exitThreshold: 0.1;
+        readonly minSilenceDurationMs: 400;
+        readonly modelBaseUrl: "https://cdn.spatialwalk.top/public";
+        readonly modelName: "onnx-community/silero-vad";
+        readonly noiseGate: {
+            readonly enabled: true;
+            readonly openThresholdDb: -50;
+            readonly closeThresholdDb: -55;
+            readonly attackMs: 5;
+            readonly releaseMs: 50;
+            readonly holdMs: 100;
+            readonly highPassHz: 100;
+        };
+    };
+};

package/dist/withJoi/src/config/region-config.d.ts

@@ -0,0 +1,15 @@
+export type Region = 'us' | 'cn' | 'test';
+export type I18nLocale = 'en' | 'zhCN';
+export interface RegionConfig {
+    name: string;
+    label: string;
+    sdkApiBaseUrl: string;
+    realtimeApiBaseUrl: string;
+    realtimeWsUrl: string;
+    platformApiBaseUrl: string;
+    realtimePreviewWsUrl: string;
+}
+export declare const REGION_LIST: Region[];
+export declare const REGION_CONFIGS: Partial<Record<Region, RegionConfig>>;
+export declare function getEnvironmentI18nKey(env: Region): string;
+export declare const REGION_KEYS: Region[];

package/dist/withJoi/src/constants.d.ts

@@ -0,0 +1,22 @@
+import { Region } from './config/region-config';
+export declare const POSTHOG_HOST = "https://i.spatialwalk.ai";
+export declare const POSTHOG_API_KEY: any;
+export declare const POSTHOG_TRACK_EVENTS: boolean;
+export declare const POSTHOG_PROJECT_NAME: string;
+export declare const ENV_TEST: string | boolean;
+
+export declare function getRegionFromHostname(): Region;
+type WebListName = 'playground' | undefined;
+
+export declare function getWebListName(): WebListName;
+
+export declare function isDebugMode(): boolean;
+
+export declare const PROD_VOICES: readonly [{
+    readonly value: "6ccbfb76-1fc6-48f7-b71d-91ac6298247b";
+    readonly label: "女性";
+}, {
+    readonly value: "c961b81c-a935-4c17-bfb3-ba2239de8c2f";
+    readonly label: "男性";
+}];
+export {};

package/dist/withJoi/src/lib/utils.d.ts

@@ -0,0 +1,2 @@
+import { ClassValue } from 'clsx';
+export declare function cn(...inputs: ClassValue[]): string;

package/dist/withJoi/src/libs/vad/manager.d.ts

@@ -0,0 +1,37 @@
+import { SimpleVAD } from './simpleVad';
+import { VAD, VADConfig } from './vad';
+export interface VADAudioOptions {
+    audioContextOptions?: AudioContextOptions;
+    minChunkSize?: number;
+    vadConfig?: Partial<VADConfig>;
+}
+
+export declare class VADAudioManager {
+    private audioContext;
+    private audioWorkletNode;
+    private mediaStream;
+    private sourceNode;
+    private highPassFilter;
+    private vad;
+    private workletInitialized;
+    private audioContextOptions;
+    private workletUrl;
+    constructor(vad: VAD | SimpleVAD, options?: VADAudioOptions);
+    private ensureAudioContext;
+    initialize(workletUrl: string): Promise<void>;
+    private initializeWorklet;
+    startMicrophone(requestVideo?: boolean): Promise<MediaStream | null>;
+    stopMicrophone(): Promise<void>;
+    pauseMicrophone(): Promise<void>;
+    resumeMicrophone(): Promise<void>;
+    stop(): void;
+    updateNoiseGateConfig(config: {
+        openThresholdDb?: number;
+        closeThresholdDb?: number;
+        attackMs?: number;
+        releaseMs?: number;
+        holdMs?: number;
+        highPassHz?: number;
+    }): void;
+    dispose(): void;
+}

package/dist/withJoi/src/libs/vad/process.worklet.d.ts

@@ -0,0 +1,22 @@
+declare const MIN_CHUNK_SIZE = 512;
+
+declare let globalPointer: number;
+declare const globalBuffer: Float32Array<ArrayBuffer>;
+
+interface NoiseGateConfig {
+    enabled: boolean;
+    openThresholdDb: number;
+    closeThresholdDb: number;
+    attackMs: number;
+    releaseMs: number;
+    holdMs: number;
+}
+declare let noiseGateConfig: NoiseGateConfig;
+declare let gateEnvelope: number;
+declare let gateOpen: boolean;
+declare let holdCounter: number;
+
+declare class VADProcessor extends AudioWorkletProcessor {
+    constructor();
+    process(inputs: Float32Array[][], _outputs: Float32Array[][], _parameters: Record<string, Float32Array>): boolean;
+}

package/dist/withJoi/src/libs/vad/simpleVad.d.ts

@@ -0,0 +1,35 @@
+import { VADEvents } from './vad';
+export interface SimpleVADConfig {
+    sampleRate: number;
+    speechThreshold: number;
+    exitThreshold: number;
+    minSilenceDurationMs: number;
+    speechPadMs: number;
+    minSpeechDurationMs: number;
+    maxBufferDuration: number;
+    newBufferSize: number;
+}
+
+export declare class SimpleVAD {
+    private config;
+    private buffer;
+    private bufferPointer;
+    private isRecording;
+    private prevBuffers;
+    private eventListeners;
+    private silenceStartTime;
+    private speechStartTime;
+    private isReady;
+    constructor(userConfig?: Partial<SimpleVADConfig>);
+    initialize(): Promise<void>;
+    private calculateRMS;
+    processAudio(buffer: Float32Array): Promise<void>;
+    private processSpeechSegment;
+    private reset;
+    on<K extends keyof VADEvents>(event: K, callback: (event: VADEvents[K]) => void): void;
+    off<K extends keyof VADEvents>(event: K, callback: (event: VADEvents[K]) => void): void;
+    emit<K extends keyof VADEvents>(event: K, data: VADEvents[K]): void;
+    updateConfig(newConfig: Partial<SimpleVADConfig>): void;
+}
+
+export declare function createSimpleVAD(config?: Partial<SimpleVADConfig>): Promise<SimpleVAD>;

package/dist/withJoi/src/libs/vad/vad.d.ts

@@ -0,0 +1,56 @@
+export interface VADConfig {
+    sampleRate: number;
+    speechThreshold: number;
+    exitThreshold: number;
+    minSilenceDurationMs: number;
+    speechPadMs: number;
+    minSpeechDurationMs: number;
+    maxBufferDuration: number;
+    newBufferSize: number;
+    modelBaseUrl?: string;
+    modelName?: string;
+}
+export interface VADEvents {
+    'speech-start': void;
+    'speech-end': void;
+    'speech-ready': {
+        buffer: Float32Array;
+        duration: number;
+    };
+    'status': {
+        type: string;
+        message: string;
+    };
+    'debug': {
+        message: string;
+        data?: unknown;
+    };
+}
+export type VADEventCallback<K extends keyof VADEvents> = (event: VADEvents[K]) => void;
+
+export declare class VAD {
+    private config;
+    private model;
+    private state;
+    private sampleRateTensor;
+    private buffer;
+    private bufferPointer;
+    private isRecording;
+    private postSpeechSamples;
+    private prevBuffers;
+    private inferenceChain;
+    private eventListeners;
+    private isReady;
+    constructor(userConfig?: Partial<VADConfig>);
+    initialize(): Promise<void>;
+    on<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void;
+    off<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void;
+    emit<K extends keyof VADEvents>(event: K, data: VADEvents[K]): void;
+    processAudio(inputBuffer: Float32Array): Promise<void>;
+    private detectSpeech;
+    private processSpeechSegment;
+    private reset;
+    updateConfig(newConfig: Partial<VADConfig>): void;
+}
+
+export declare function createVAD(config?: Partial<VADConfig>): Promise<VAD>;

package/dist/withJoi/src/libs/vad/wav.d.ts

@@ -0,0 +1 @@
+export declare function toWav(buffer: Float32Array, sampleRate: number): ArrayBuffer;

package/dist/withJoi/src/locales/index.d.ts

@@ -0,0 +1,2 @@
+export { default as en } from './en.json';
+export { default as zhCN } from './zh-CN.json';

package/dist/withJoi/src/modules/i18n.d.ts

@@ -0,0 +1,3 @@
+import { I18n } from 'vue-i18n';
+export declare const i18n: I18n;
+export declare function getGlobalT(): (key: string, params?: Record<string, unknown>) => string;

package/dist/withJoi/src/modules/pwa.d.ts

@@ -0,0 +1 @@
+export { registerSW } from 'virtual:pwa-register';

package/dist/withJoi/src/realtime/PreviewWebSocketClient.d.ts

@@ -0,0 +1,18 @@
+export interface PreviewWebSocketOptions {
+    characterId: string;
+    speaker: string;
+    drivenConfig?: {
+        characterId: string;
+        drivenServerUrl: string;
+    };
+    debug?: boolean;
+}
+export declare function usePreviewWebSocket(options: PreviewWebSocketOptions): {
+    isConnected: import('vue').ComputedRef<boolean>;
+    isConnecting: import('vue').ComputedRef<boolean>;
+    connect: () => Promise<void>;
+    disconnect: () => Promise<void>;
+    sendText: (text: string) => boolean;
+    on: (event: string, handler: (...args: any[]) => void) => void;
+    off: (event: string, handler: (...args: any[]) => void) => void;
+};

package/dist/withJoi/src/realtime/RealtimeSyncManager.d.ts

@@ -0,0 +1,34 @@
+import { Flame } from '../../generated/realtime/v1/messages';
+import { SessionBuffer } from './types';
+import { EventEmitter } from './utils/eventEmitter';
+export interface RealtimeSyncManagerOptions {
+    maxBufferTime?: number;
+    alignmentTolerance?: number;
+    minKeyframesForStart?: number;
+    minAudioChunksForStart?: number;
+    debug?: boolean;
+}
+export declare class RealtimeSyncManager extends EventEmitter {
+    private sessions;
+    private maxBufferTime;
+    private alignmentTolerance;
+    private minKeyframesForStart;
+    private minAudioChunksForStart;
+    private debug;
+    constructor(options?: RealtimeSyncManagerOptions);
+    addAudioChunk(reqId: string, audio: Uint8Array, isLast?: boolean): void;
+    addFlameKeyframes(reqId: string, keyframes: Flame[]): void;
+    markSessionEnd(reqId: string): void;
+    isSessionReady(reqId: string): boolean;
+    getSessionData(reqId: string): SessionBuffer | null;
+    clearSession(reqId: string): void;
+    clearAllSessions(): void;
+    private getOrCreateSession;
+    private checkStreamingReady;
+    private checkSessionReady;
+    private validateAlignment;
+    private log;
+}
+
+export declare function getAudioNow(audioCtx: AudioContext): number;
+export declare function computeDriftMs(renderTimeSec: number, audioNowSec: number): number;

package/dist/withJoi/src/realtime/RealtimeWebSocketClient.d.ts

@@ -0,0 +1,18 @@
+export interface RealtimeWebSocketOptions {
+    wsUrl: string;
+    heartbeatInterval?: number;
+    debug?: boolean;
+    jwtToken?: string;
+}
+export declare function useRealtimeWebsocket(options: RealtimeWebSocketOptions): {
+    isConnected: import('vue').ComputedRef<boolean>;
+    connect: (characterId: string) => Promise<void>;
+    disconnect: () => Promise<void>;
+    sendSessionStart: (reqId: string, characterId: string) => void;
+    sendAudio: (reqId: string, audio: Uint8Array) => void;
+    sendTextInput: (reqId: string, text: string) => void;
+    sendSpeechStart: (reqId: string) => void;
+    generateReqId: () => string;
+    on: (event: string, handler: (...args: any[]) => void) => void;
+    off: (event: string, handler: (...args: any[]) => void) => void;
+};

package/dist/withJoi/src/realtime/index.d.ts

@@ -0,0 +1,3 @@
+export { RealtimeSyncManager } from './RealtimeSyncManager';
+export { useRealtimeWebsocket } from './RealtimeWebSocketClient';
+export * from './types';

package/dist/withJoi/src/realtime/types.d.ts

@@ -0,0 +1,29 @@
+import { Flame } from '../../generated/realtime/v1/messages';
+export interface RealtimeConfig {
+    wsUrl: string;
+    characterId: string;
+    audioSampleRate?: number;
+    reconnectAttempts?: number;
+    heartbeatInterval?: number;
+}
+export interface SessionBuffer {
+    reqId: string;
+    audioChunks: Array<{
+        data: Uint8Array;
+        isLast: boolean;
+    }>;
+    flameKeyframes: Flame[];
+    receivedEnd: boolean;
+    startTime: number;
+    isPlaying?: boolean;
+    lastEmittedKeyframeCount?: number;
+}
+export type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'error';
+export type RecordingState = 'idle' | 'recording' | 'processing';
+export type PlaybackState = 'idle' | 'buffering' | 'playing';
+export interface AudioRecordingOptions {
+    sampleRate: number;
+    channelCount: number;
+    echoCancellation: boolean;
+    noiseSuppression: boolean;
+}

package/dist/withJoi/src/realtime/utils/audioDebug.d.ts

@@ -0,0 +1,3 @@
+export declare function analyzeAudioData(data: Uint8Array): void;
+
+export declare function testAudioPlayback(audioData: Uint8Array): Promise<boolean>;

package/dist/withJoi/src/realtime/utils/audioUtils.d.ts

@@ -0,0 +1,11 @@
+export declare function mergeAudioChunks(chunks: Uint8Array[]): Uint8Array;
+
+export declare function estimateAudioDuration(audioData: Uint8Array, sampleRate?: number): number;
+
+export declare function createAudioBlobURL(chunks: Uint8Array[], sampleRate?: number): string;
+
+export declare function revokeAudioBlobURL(url: string): void;
+
+export declare function getSupportedAudioMimeTypes(): string[];
+
+export declare function getBestSupportedAudioMimeType(): string;

package/dist/withJoi/src/realtime/utils/eventEmitter.d.ts

@@ -0,0 +1,9 @@
+type EventHandler = (...args: any[]) => void;
+export declare class EventEmitter {
+    private events;
+    on(event: string, handler: EventHandler): void;
+    off(event: string, handler: EventHandler): void;
+    emit(event: string, ...args: unknown[]): void;
+    removeAllListeners(event?: string): void;
+}
+export {};

package/dist/withJoi/src/renderer/RenderSystem.d.ts

@@ -0,0 +1,34 @@
+import { CameraConfig } from '../types/avatar';
+import { RenderBackend } from './renderer';
+export interface RenderSystemOptions {
+    canvas: HTMLCanvasElement;
+    preferBackend?: RenderBackend;
+    backgroundColor?: [number, number, number, number];
+    camera?: CameraConfig;
+}
+export declare class RenderSystem {
+    private renderer;
+    private backend;
+    private canvas;
+    private options;
+    private camera;
+    private viewMatrix;
+    private projectionMatrix;
+    private originalPackedData;
+    renderTime: number;
+    sortTime: number;
+    constructor(options: RenderSystemOptions);
+    initialize(): Promise<void>;
+    loadSplatsFromPackedData(packedData: Float32Array): void;
+    renderFrame(): void;
+    updateCamera(params: Partial<CameraConfig>): void;
+    handleResize(): void;
+    getBackend(): RenderBackend | null;
+    dispose(): void;
+    private checkWebGPUSupport;
+    private updateCameraAspect;
+    private updateCameraMatrices;
+    private getCameraForward;
+    private updatePerspectiveMatrix;
+    private updateViewMatrix;
+}

package/dist/withJoi/src/renderer/covariance.d.ts

@@ -0,0 +1 @@
+export declare function computeCovariance3D(scale: number[], rotation: number[]): number[][];

package/dist/withJoi/src/renderer/renderer.d.ts

@@ -0,0 +1,7 @@
+export interface I3DGSRenderer {
+    initialize: () => Promise<void>;
+    loadSplatsFromPackedData: (packedData: Float32Array, pointCount: number, sortOrder?: Uint32Array) => void;
+    render: (viewMatrix: Float32Array, projectionMatrix: Float32Array, screenSize: [number, number]) => void;
+    dispose: () => void;
+}
+export type RenderBackend = 'webgl' | 'webgpu';

package/dist/withJoi/src/renderer/sortSplats.d.ts

@@ -0,0 +1 @@
+export declare function sortSplats(packedData: Float32Array, cameraPosition: [number, number, number], cameraForward: [number, number, number]): Uint32Array;

package/dist/withJoi/src/renderer/webgl/reorderData.d.ts

@@ -0,0 +1 @@
+export declare function reorderPackedData(packedData: Float32Array, sortOrder: Uint32Array): Float32Array;

package/dist/withJoi/src/renderer/webgl/webglRenderer.d.ts

@@ -0,0 +1,27 @@
+import { I3DGSRenderer } from '../renderer';
+export declare class WebGLRenderer implements I3DGSRenderer {
+    private canvas;
+    private backgroundColor;
+    private gl;
+    private shaderProgram;
+    private uniformLocations;
+    private attributeLocations;
+    private splatBuffer;
+    private quadVertexBuffer;
+    private vertexArray;
+    private splatCount;
+    private isInitialized;
+    private splatBufferSize;
+    constructor(canvas: HTMLCanvasElement, backgroundColor?: [number, number, number, number]);
+    initialize(): Promise<void>;
+    private setupShaderLocations;
+    private setupWebGLState;
+    private createBuffers;
+    private createQuadVertexBuffer;
+    loadSplatsFromPackedData(packedData: Float32Array, pointCount: number, _sortOrder?: Uint32Array): void;
+    private uploadToGPU;
+    private setupVertexAttributes;
+    render(viewMatrix: Float32Array, projectionMatrix: Float32Array, screenSize: [number, number]): void;
+    private createShaderProgram;
+    dispose(): void;
+}

package/dist/withJoi/src/renderer/webgpu/webgpuRenderer.d.ts

@@ -0,0 +1,26 @@
+import { I3DGSRenderer } from '../renderer';
+export declare class WebGPURenderer implements I3DGSRenderer {
+    private canvas;
+    private backgroundColor;
+    private device;
+    private context;
+    private renderPipeline;
+    private quadVertexBuffer;
+    private uniformBuffer;
+    private uniformBindGroup;
+    private sortIndexBuffer;
+    private splatDataBuffer;
+    private storageBindGroup;
+    private bindGroupNeedsUpdate;
+    private splatCount;
+    private presentationFormat;
+    constructor(canvas: HTMLCanvasElement, backgroundColor?: [number, number, number, number]);
+    initialize(): Promise<void>;
+    private createUniformBuffer;
+    private createQuadVertexBuffer;
+    private createRenderPipeline;
+    loadSplatsFromPackedData(packedData: Float32Array, pointCount: number, sortOrder?: Uint32Array): void;
+    render(viewMatrix: Float32Array, projectionMatrix: Float32Array, screenSize: [number, number]): void;
+    private updateUniforms;
+    dispose(): void;
+}