@aether-stack-dev/client-sdk 1.1.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AStackCSRClient.d.ts +2 -0
- package/dist/SupabaseSignalingClient.d.ts +2 -0
- package/dist/avatar/index.d.ts +0 -2
- package/dist/index.esm.js +29 -7
- package/dist/index.js +29 -7
- package/dist/react/index.d.ts +0 -2
- package/dist/react.esm.js +44 -158
- package/dist/react.js +43 -158
- package/dist/types.d.ts +1 -1
- package/package.json +8 -6
- package/dist/avatar/TalkingHeadAvatar.d.ts +0 -8
|
@@ -35,6 +35,8 @@ export interface AStackCSREvents {
|
|
|
35
35
|
model_loaded?: boolean;
|
|
36
36
|
blendshape_count?: number;
|
|
37
37
|
}) => void;
|
|
38
|
+
creditsExhausted: () => void;
|
|
39
|
+
sessionExpired: (reason: string) => void;
|
|
38
40
|
}
|
|
39
41
|
export type CallStatus = 'idle' | 'starting' | 'active' | 'stopping' | 'error';
|
|
40
42
|
export declare class AStackCSRClient extends EventEmitter<AStackCSREvents> {
|
|
@@ -13,6 +13,8 @@ export declare class SupabaseSignalingClient extends EventEmitter {
|
|
|
13
13
|
private maxReconnectAttempts;
|
|
14
14
|
private reconnectDelay;
|
|
15
15
|
private heartbeatInterval;
|
|
16
|
+
private supabaseUrl;
|
|
17
|
+
private supabasePublishableKey;
|
|
16
18
|
constructor(config: AStackConfig);
|
|
17
19
|
connect(sessionId: string, channelName: string, wsToken: string): Promise<void>;
|
|
18
20
|
private handleSignalingMessage;
|
package/dist/avatar/index.d.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
export { VRMAvatar } from './VRMAvatar';
|
|
2
2
|
export type { VRMAvatarProps } from './VRMAvatar';
|
|
3
|
-
export { TalkingHeadAvatar } from './TalkingHeadAvatar';
|
|
4
|
-
export type { TalkingHeadAvatarProps } from './TalkingHeadAvatar';
|
|
5
3
|
export { ARKIT_BLENDSHAPES, BLENDSHAPE_COUNT } from './constants';
|
|
6
4
|
export type { ARKitBlendshapeName } from './constants';
|
package/dist/index.esm.js
CHANGED
|
@@ -105,22 +105,28 @@ class AudioPlayer extends EventEmitter {
|
|
|
105
105
|
const blendshapes = hasBlendshapes
|
|
106
106
|
? chunk.blendshapes
|
|
107
107
|
: this.generateAmplitudeBlendshapes(floatArray);
|
|
108
|
-
const duration = audioBuffer.duration
|
|
109
|
-
const
|
|
108
|
+
const duration = audioBuffer.duration;
|
|
109
|
+
const audioStartTime = ctx.currentTime;
|
|
110
110
|
let frameIndex = 0;
|
|
111
111
|
this.isPlaying = true;
|
|
112
112
|
this.emit('playbackStarted');
|
|
113
|
+
source.onended = () => {
|
|
114
|
+
if (this.animationFrameId !== null) {
|
|
115
|
+
cancelAnimationFrame(this.animationFrameId);
|
|
116
|
+
this.animationFrameId = null;
|
|
117
|
+
}
|
|
118
|
+
this.emit('blendshapeUpdate', new Array(BLENDSHAPE_COUNT).fill(0));
|
|
119
|
+
this.emit('playbackEnded');
|
|
120
|
+
this.playNext();
|
|
121
|
+
};
|
|
113
122
|
const animate = () => {
|
|
114
123
|
if (!this.isPlaying) {
|
|
115
124
|
this.animationFrameId = null;
|
|
116
125
|
return;
|
|
117
126
|
}
|
|
118
|
-
const elapsed =
|
|
127
|
+
const elapsed = ctx.currentTime - audioStartTime;
|
|
119
128
|
if (elapsed >= duration || frameIndex >= blendshapes.length) {
|
|
120
129
|
this.animationFrameId = null;
|
|
121
|
-
this.emit('blendshapeUpdate', new Array(BLENDSHAPE_COUNT).fill(0));
|
|
122
|
-
this.emit('playbackEnded');
|
|
123
|
-
this.playNext();
|
|
124
130
|
return;
|
|
125
131
|
}
|
|
126
132
|
const progress = elapsed / duration;
|
|
@@ -340,6 +346,13 @@ class AStackCSRClient extends EventEmitter {
|
|
|
340
346
|
case 'connected':
|
|
341
347
|
if (typeof data.clientId === 'string')
|
|
342
348
|
this.clientId = data.clientId;
|
|
349
|
+
if (data.protocol_version) {
|
|
350
|
+
const major = parseInt(data.protocol_version.split('.')[0], 10);
|
|
351
|
+
if (major > 1) {
|
|
352
|
+
console.error(`[CSR] Protocol version mismatch: server=${data.protocol_version}, expected=1.x`);
|
|
353
|
+
this.emit('error', new Error(`Protocol version mismatch: server=${data.protocol_version}`));
|
|
354
|
+
}
|
|
355
|
+
}
|
|
343
356
|
break;
|
|
344
357
|
case 'authenticated':
|
|
345
358
|
if (this.pendingAuth) {
|
|
@@ -432,6 +445,14 @@ class AStackCSRClient extends EventEmitter {
|
|
|
432
445
|
blendshape_count: typeof data.blendshape_count === 'number' ? data.blendshape_count : undefined
|
|
433
446
|
});
|
|
434
447
|
break;
|
|
448
|
+
case 'session_expired': {
|
|
449
|
+
const reason = typeof data.reason === 'string' ? data.reason : 'unknown';
|
|
450
|
+
this.emit('sessionExpired', reason);
|
|
451
|
+
if (reason === 'credits_exhausted') {
|
|
452
|
+
this.emit('creditsExhausted');
|
|
453
|
+
}
|
|
454
|
+
break;
|
|
455
|
+
}
|
|
435
456
|
}
|
|
436
457
|
}
|
|
437
458
|
async startCall(options) {
|
|
@@ -472,10 +493,11 @@ class AStackCSRClient extends EventEmitter {
|
|
|
472
493
|
this.startImageCapture();
|
|
473
494
|
}
|
|
474
495
|
}
|
|
496
|
+
const { systemPrompt, priorContext, configOverrides, disableA2F, providers } = options || {};
|
|
475
497
|
this.ws.send(JSON.stringify({
|
|
476
498
|
type: 'call_start',
|
|
477
499
|
fps: this.config.fps,
|
|
478
|
-
|
|
500
|
+
systemPrompt, priorContext, configOverrides, disableA2F, providers
|
|
479
501
|
}));
|
|
480
502
|
const source = this.audioContext.createMediaStreamSource(this.mediaStream);
|
|
481
503
|
this.audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-processor');
|
package/dist/index.js
CHANGED
|
@@ -109,22 +109,28 @@ class AudioPlayer extends eventemitter3.EventEmitter {
|
|
|
109
109
|
const blendshapes = hasBlendshapes
|
|
110
110
|
? chunk.blendshapes
|
|
111
111
|
: this.generateAmplitudeBlendshapes(floatArray);
|
|
112
|
-
const duration = audioBuffer.duration
|
|
113
|
-
const
|
|
112
|
+
const duration = audioBuffer.duration;
|
|
113
|
+
const audioStartTime = ctx.currentTime;
|
|
114
114
|
let frameIndex = 0;
|
|
115
115
|
this.isPlaying = true;
|
|
116
116
|
this.emit('playbackStarted');
|
|
117
|
+
source.onended = () => {
|
|
118
|
+
if (this.animationFrameId !== null) {
|
|
119
|
+
cancelAnimationFrame(this.animationFrameId);
|
|
120
|
+
this.animationFrameId = null;
|
|
121
|
+
}
|
|
122
|
+
this.emit('blendshapeUpdate', new Array(BLENDSHAPE_COUNT).fill(0));
|
|
123
|
+
this.emit('playbackEnded');
|
|
124
|
+
this.playNext();
|
|
125
|
+
};
|
|
117
126
|
const animate = () => {
|
|
118
127
|
if (!this.isPlaying) {
|
|
119
128
|
this.animationFrameId = null;
|
|
120
129
|
return;
|
|
121
130
|
}
|
|
122
|
-
const elapsed =
|
|
131
|
+
const elapsed = ctx.currentTime - audioStartTime;
|
|
123
132
|
if (elapsed >= duration || frameIndex >= blendshapes.length) {
|
|
124
133
|
this.animationFrameId = null;
|
|
125
|
-
this.emit('blendshapeUpdate', new Array(BLENDSHAPE_COUNT).fill(0));
|
|
126
|
-
this.emit('playbackEnded');
|
|
127
|
-
this.playNext();
|
|
128
134
|
return;
|
|
129
135
|
}
|
|
130
136
|
const progress = elapsed / duration;
|
|
@@ -344,6 +350,13 @@ class AStackCSRClient extends eventemitter3.EventEmitter {
|
|
|
344
350
|
case 'connected':
|
|
345
351
|
if (typeof data.clientId === 'string')
|
|
346
352
|
this.clientId = data.clientId;
|
|
353
|
+
if (data.protocol_version) {
|
|
354
|
+
const major = parseInt(data.protocol_version.split('.')[0], 10);
|
|
355
|
+
if (major > 1) {
|
|
356
|
+
console.error(`[CSR] Protocol version mismatch: server=${data.protocol_version}, expected=1.x`);
|
|
357
|
+
this.emit('error', new Error(`Protocol version mismatch: server=${data.protocol_version}`));
|
|
358
|
+
}
|
|
359
|
+
}
|
|
347
360
|
break;
|
|
348
361
|
case 'authenticated':
|
|
349
362
|
if (this.pendingAuth) {
|
|
@@ -436,6 +449,14 @@ class AStackCSRClient extends eventemitter3.EventEmitter {
|
|
|
436
449
|
blendshape_count: typeof data.blendshape_count === 'number' ? data.blendshape_count : undefined
|
|
437
450
|
});
|
|
438
451
|
break;
|
|
452
|
+
case 'session_expired': {
|
|
453
|
+
const reason = typeof data.reason === 'string' ? data.reason : 'unknown';
|
|
454
|
+
this.emit('sessionExpired', reason);
|
|
455
|
+
if (reason === 'credits_exhausted') {
|
|
456
|
+
this.emit('creditsExhausted');
|
|
457
|
+
}
|
|
458
|
+
break;
|
|
459
|
+
}
|
|
439
460
|
}
|
|
440
461
|
}
|
|
441
462
|
async startCall(options) {
|
|
@@ -476,10 +497,11 @@ class AStackCSRClient extends eventemitter3.EventEmitter {
|
|
|
476
497
|
this.startImageCapture();
|
|
477
498
|
}
|
|
478
499
|
}
|
|
500
|
+
const { systemPrompt, priorContext, configOverrides, disableA2F, providers } = options || {};
|
|
479
501
|
this.ws.send(JSON.stringify({
|
|
480
502
|
type: 'call_start',
|
|
481
503
|
fps: this.config.fps,
|
|
482
|
-
|
|
504
|
+
systemPrompt, priorContext, configOverrides, disableA2F, providers
|
|
483
505
|
}));
|
|
484
506
|
const source = this.audioContext.createMediaStreamSource(this.mediaStream);
|
|
485
507
|
this.audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-processor');
|
package/dist/react/index.d.ts
CHANGED
|
@@ -2,5 +2,3 @@ export { useAStackCSR } from './useAStackCSR';
|
|
|
2
2
|
export type { UseAStackCSROptions, UseAStackCSRReturn } from './useAStackCSR';
|
|
3
3
|
export { VRMAvatar } from '../avatar/VRMAvatar';
|
|
4
4
|
export type { VRMAvatarProps } from '../avatar/VRMAvatar';
|
|
5
|
-
export { TalkingHeadAvatar } from '../avatar/TalkingHeadAvatar';
|
|
6
|
-
export type { TalkingHeadAvatarProps } from '../avatar/TalkingHeadAvatar';
|
package/dist/react.esm.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
'use client';
|
|
1
2
|
import { useState, useRef, useEffect, useCallback } from 'react';
|
|
2
3
|
import { EventEmitter } from 'eventemitter3';
|
|
3
4
|
import { jsxs, jsx } from 'react/jsx-runtime';
|
|
@@ -110,22 +111,28 @@ class AudioPlayer extends EventEmitter {
|
|
|
110
111
|
const blendshapes = hasBlendshapes
|
|
111
112
|
? chunk.blendshapes
|
|
112
113
|
: this.generateAmplitudeBlendshapes(floatArray);
|
|
113
|
-
const duration = audioBuffer.duration
|
|
114
|
-
const
|
|
114
|
+
const duration = audioBuffer.duration;
|
|
115
|
+
const audioStartTime = ctx.currentTime;
|
|
115
116
|
let frameIndex = 0;
|
|
116
117
|
this.isPlaying = true;
|
|
117
118
|
this.emit('playbackStarted');
|
|
119
|
+
source.onended = () => {
|
|
120
|
+
if (this.animationFrameId !== null) {
|
|
121
|
+
cancelAnimationFrame(this.animationFrameId);
|
|
122
|
+
this.animationFrameId = null;
|
|
123
|
+
}
|
|
124
|
+
this.emit('blendshapeUpdate', new Array(BLENDSHAPE_COUNT).fill(0));
|
|
125
|
+
this.emit('playbackEnded');
|
|
126
|
+
this.playNext();
|
|
127
|
+
};
|
|
118
128
|
const animate = () => {
|
|
119
129
|
if (!this.isPlaying) {
|
|
120
130
|
this.animationFrameId = null;
|
|
121
131
|
return;
|
|
122
132
|
}
|
|
123
|
-
const elapsed =
|
|
133
|
+
const elapsed = ctx.currentTime - audioStartTime;
|
|
124
134
|
if (elapsed >= duration || frameIndex >= blendshapes.length) {
|
|
125
135
|
this.animationFrameId = null;
|
|
126
|
-
this.emit('blendshapeUpdate', new Array(BLENDSHAPE_COUNT).fill(0));
|
|
127
|
-
this.emit('playbackEnded');
|
|
128
|
-
this.playNext();
|
|
129
136
|
return;
|
|
130
137
|
}
|
|
131
138
|
const progress = elapsed / duration;
|
|
@@ -345,6 +352,13 @@ class AStackCSRClient extends EventEmitter {
|
|
|
345
352
|
case 'connected':
|
|
346
353
|
if (typeof data.clientId === 'string')
|
|
347
354
|
this.clientId = data.clientId;
|
|
355
|
+
if (data.protocol_version) {
|
|
356
|
+
const major = parseInt(data.protocol_version.split('.')[0], 10);
|
|
357
|
+
if (major > 1) {
|
|
358
|
+
console.error(`[CSR] Protocol version mismatch: server=${data.protocol_version}, expected=1.x`);
|
|
359
|
+
this.emit('error', new Error(`Protocol version mismatch: server=${data.protocol_version}`));
|
|
360
|
+
}
|
|
361
|
+
}
|
|
348
362
|
break;
|
|
349
363
|
case 'authenticated':
|
|
350
364
|
if (this.pendingAuth) {
|
|
@@ -437,6 +451,14 @@ class AStackCSRClient extends EventEmitter {
|
|
|
437
451
|
blendshape_count: typeof data.blendshape_count === 'number' ? data.blendshape_count : undefined
|
|
438
452
|
});
|
|
439
453
|
break;
|
|
454
|
+
case 'session_expired': {
|
|
455
|
+
const reason = typeof data.reason === 'string' ? data.reason : 'unknown';
|
|
456
|
+
this.emit('sessionExpired', reason);
|
|
457
|
+
if (reason === 'credits_exhausted') {
|
|
458
|
+
this.emit('creditsExhausted');
|
|
459
|
+
}
|
|
460
|
+
break;
|
|
461
|
+
}
|
|
440
462
|
}
|
|
441
463
|
}
|
|
442
464
|
async startCall(options) {
|
|
@@ -477,10 +499,11 @@ class AStackCSRClient extends EventEmitter {
|
|
|
477
499
|
this.startImageCapture();
|
|
478
500
|
}
|
|
479
501
|
}
|
|
502
|
+
const { systemPrompt, priorContext, configOverrides, disableA2F, providers } = options || {};
|
|
480
503
|
this.ws.send(JSON.stringify({
|
|
481
504
|
type: 'call_start',
|
|
482
505
|
fps: this.config.fps,
|
|
483
|
-
|
|
506
|
+
systemPrompt, priorContext, configOverrides, disableA2F, providers
|
|
484
507
|
}));
|
|
485
508
|
const source = this.audioContext.createMediaStreamSource(this.mediaStream);
|
|
486
509
|
this.audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-processor');
|
|
@@ -786,11 +809,19 @@ function VRMAvatar({ blendshapes, width = 400, height = 400, modelUrl = '/models
|
|
|
786
809
|
const container = containerRef.current;
|
|
787
810
|
if (!container)
|
|
788
811
|
return;
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
812
|
+
let renderer;
|
|
813
|
+
try {
|
|
814
|
+
renderer = new THREE.WebGLRenderer({
|
|
815
|
+
antialias: true,
|
|
816
|
+
alpha: true,
|
|
817
|
+
powerPreference: 'high-performance'
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
catch {
|
|
821
|
+
setError('WebGL is not supported in this browser');
|
|
822
|
+
setLoading(false);
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
794
825
|
renderer.setSize(width, height);
|
|
795
826
|
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
|
796
827
|
renderer.outputColorSpace = THREE.SRGBColorSpace;
|
|
@@ -898,149 +929,4 @@ function VRMAvatar({ blendshapes, width = 400, height = 400, modelUrl = '/models
|
|
|
898
929
|
return (jsxs("div", { className: "relative", style: { width, height }, children: [jsx("div", { ref: containerRef, className: "rounded-lg overflow-hidden" }), loading && (jsx("div", { className: "absolute inset-0 flex items-center justify-center bg-gray-800 rounded-lg", children: jsx("div", { className: "text-white text-sm", children: "Loading VRM avatar..." }) })), error && (jsx("div", { className: "absolute inset-0 flex items-center justify-center bg-red-900/50 rounded-lg", children: jsx("div", { className: "text-white text-sm", children: error }) }))] }));
|
|
899
930
|
}
|
|
900
931
|
|
|
901
|
-
|
|
902
|
-
const iframeRef = useRef(null);
|
|
903
|
-
const [loading, setLoading] = useState(true);
|
|
904
|
-
const [error, setError] = useState(null);
|
|
905
|
-
useEffect(() => {
|
|
906
|
-
const iframe = iframeRef.current;
|
|
907
|
-
if (!iframe)
|
|
908
|
-
return;
|
|
909
|
-
const handleMessage = (event) => {
|
|
910
|
-
if (event.data?.type === 'talkinghead-ready') {
|
|
911
|
-
setLoading(false);
|
|
912
|
-
}
|
|
913
|
-
else if (event.data?.type === 'talkinghead-error') {
|
|
914
|
-
setError(event.data.message || 'Failed to load TalkingHead avatar');
|
|
915
|
-
setLoading(false);
|
|
916
|
-
}
|
|
917
|
-
};
|
|
918
|
-
window.addEventListener('message', handleMessage);
|
|
919
|
-
return () => window.removeEventListener('message', handleMessage);
|
|
920
|
-
}, []);
|
|
921
|
-
useEffect(() => {
|
|
922
|
-
const iframe = iframeRef.current;
|
|
923
|
-
if (!iframe?.contentWindow || blendshapes.length === 0)
|
|
924
|
-
return;
|
|
925
|
-
const blendshapeData = {};
|
|
926
|
-
ARKIT_BLENDSHAPES.forEach((name, i) => {
|
|
927
|
-
blendshapeData[name] = blendshapes[i] || 0;
|
|
928
|
-
});
|
|
929
|
-
iframe.contentWindow.postMessage({
|
|
930
|
-
type: 'update-blendshapes',
|
|
931
|
-
blendshapes: blendshapeData
|
|
932
|
-
}, '*');
|
|
933
|
-
}, [blendshapes]);
|
|
934
|
-
const iframeSrc = `data:text/html,${encodeURIComponent(`
|
|
935
|
-
<!DOCTYPE html>
|
|
936
|
-
<html>
|
|
937
|
-
<head>
|
|
938
|
-
<style>
|
|
939
|
-
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
940
|
-
html, body { width: 100%; height: 100%; overflow: hidden; background: #1a1a2e; }
|
|
941
|
-
#avatar { width: 100%; height: 100%; }
|
|
942
|
-
</style>
|
|
943
|
-
<script type="importmap">
|
|
944
|
-
{
|
|
945
|
-
"imports": {
|
|
946
|
-
"three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js",
|
|
947
|
-
"three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/"
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
${"</"}script>
|
|
951
|
-
</head>
|
|
952
|
-
<body>
|
|
953
|
-
<div id="avatar"></div>
|
|
954
|
-
<script type="module">
|
|
955
|
-
import { TalkingHead } from 'https://cdn.jsdelivr.net/npm/@met4citizen/talkinghead@1.6.0/modules/talkinghead.mjs';
|
|
956
|
-
|
|
957
|
-
let head = null;
|
|
958
|
-
|
|
959
|
-
async function init() {
|
|
960
|
-
try {
|
|
961
|
-
const container = document.getElementById('avatar');
|
|
962
|
-
head = new TalkingHead(container, {
|
|
963
|
-
ttsEndpoint: null,
|
|
964
|
-
cameraView: 'head',
|
|
965
|
-
cameraRotateEnable: false,
|
|
966
|
-
cameraPanEnable: false,
|
|
967
|
-
cameraZoomEnable: false,
|
|
968
|
-
lightAmbientColor: 0xffffff,
|
|
969
|
-
lightAmbientIntensity: 0.6,
|
|
970
|
-
lightDirectColor: 0xffffff,
|
|
971
|
-
lightDirectIntensity: 0.8,
|
|
972
|
-
lightSpotIntensity: 0,
|
|
973
|
-
avatarMood: 'neutral',
|
|
974
|
-
modelPixelRatio: Math.min(window.devicePixelRatio, 2),
|
|
975
|
-
modelFPS: 30
|
|
976
|
-
});
|
|
977
|
-
|
|
978
|
-
await head.showAvatar({
|
|
979
|
-
url: '${avatarUrl}',
|
|
980
|
-
body: 'F',
|
|
981
|
-
avatarMood: 'neutral',
|
|
982
|
-
lipsyncLang: 'en'
|
|
983
|
-
});
|
|
984
|
-
|
|
985
|
-
window.parent.postMessage({ type: 'talkinghead-ready' }, '*');
|
|
986
|
-
} catch (err) {
|
|
987
|
-
console.error('[TalkingHead] Error:', err);
|
|
988
|
-
window.parent.postMessage({ type: 'talkinghead-error', message: err.message }, '*');
|
|
989
|
-
}
|
|
990
|
-
}
|
|
991
|
-
|
|
992
|
-
let started = false;
|
|
993
|
-
|
|
994
|
-
window.addEventListener('message', (event) => {
|
|
995
|
-
if (event.data?.type === 'update-blendshapes' && head) {
|
|
996
|
-
if (!started) {
|
|
997
|
-
head.start();
|
|
998
|
-
started = true;
|
|
999
|
-
}
|
|
1000
|
-
|
|
1001
|
-
const shapes = event.data.blendshapes;
|
|
1002
|
-
const mtAvatar = head.mtAvatar;
|
|
1003
|
-
|
|
1004
|
-
if (mtAvatar) {
|
|
1005
|
-
for (const [name, value] of Object.entries(shapes)) {
|
|
1006
|
-
const v = Math.min(Number(value) || 0, 1);
|
|
1007
|
-
if (mtAvatar[name]) {
|
|
1008
|
-
mtAvatar[name].value = v;
|
|
1009
|
-
mtAvatar[name].applied = v;
|
|
1010
|
-
if (mtAvatar[name].ms && mtAvatar[name].is) {
|
|
1011
|
-
for (let i = 0; i < mtAvatar[name].ms.length; i++) {
|
|
1012
|
-
const influences = mtAvatar[name].ms[i];
|
|
1013
|
-
const idx = mtAvatar[name].is[i];
|
|
1014
|
-
if (influences && idx !== undefined) {
|
|
1015
|
-
influences[idx] = v;
|
|
1016
|
-
}
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
}
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
if (head.morphs) {
|
|
1024
|
-
for (const mesh of head.morphs) {
|
|
1025
|
-
if (mesh.morphTargetDictionary && mesh.morphTargetInfluences) {
|
|
1026
|
-
for (const [name, value] of Object.entries(shapes)) {
|
|
1027
|
-
const idx = mesh.morphTargetDictionary[name];
|
|
1028
|
-
if (idx !== undefined) {
|
|
1029
|
-
mesh.morphTargetInfluences[idx] = Math.min(Number(value) || 0, 1);
|
|
1030
|
-
}
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
}
|
|
1034
|
-
}
|
|
1035
|
-
}
|
|
1036
|
-
});
|
|
1037
|
-
|
|
1038
|
-
init();
|
|
1039
|
-
${"</"}script>
|
|
1040
|
-
</body>
|
|
1041
|
-
</html>
|
|
1042
|
-
`)}`;
|
|
1043
|
-
return (jsxs("div", { className: "relative bg-gray-900 rounded-lg overflow-hidden", style: { width, height }, children: [jsx("iframe", { ref: iframeRef, src: iframeSrc, style: { width: '100%', height: '100%', border: 'none' }, sandbox: "allow-scripts allow-same-origin" }), loading && (jsx("div", { className: "absolute inset-0 flex items-center justify-center bg-gray-800 rounded-lg", children: jsx("div", { className: "text-white text-sm", children: "Loading TalkingHead avatar..." }) })), error && (jsx("div", { className: "absolute inset-0 flex items-center justify-center bg-red-900/50 rounded-lg", children: jsx("div", { className: "text-white text-sm text-center p-4", children: error }) }))] }));
|
|
1044
|
-
}
|
|
1045
|
-
|
|
1046
|
-
export { TalkingHeadAvatar, VRMAvatar, useAStackCSR };
|
|
932
|
+
export { VRMAvatar, useAStackCSR };
|
package/dist/react.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
'use client';
|
|
1
2
|
'use strict';
|
|
2
3
|
|
|
3
4
|
var react = require('react');
|
|
@@ -131,22 +132,28 @@ class AudioPlayer extends eventemitter3.EventEmitter {
|
|
|
131
132
|
const blendshapes = hasBlendshapes
|
|
132
133
|
? chunk.blendshapes
|
|
133
134
|
: this.generateAmplitudeBlendshapes(floatArray);
|
|
134
|
-
const duration = audioBuffer.duration
|
|
135
|
-
const
|
|
135
|
+
const duration = audioBuffer.duration;
|
|
136
|
+
const audioStartTime = ctx.currentTime;
|
|
136
137
|
let frameIndex = 0;
|
|
137
138
|
this.isPlaying = true;
|
|
138
139
|
this.emit('playbackStarted');
|
|
140
|
+
source.onended = () => {
|
|
141
|
+
if (this.animationFrameId !== null) {
|
|
142
|
+
cancelAnimationFrame(this.animationFrameId);
|
|
143
|
+
this.animationFrameId = null;
|
|
144
|
+
}
|
|
145
|
+
this.emit('blendshapeUpdate', new Array(BLENDSHAPE_COUNT).fill(0));
|
|
146
|
+
this.emit('playbackEnded');
|
|
147
|
+
this.playNext();
|
|
148
|
+
};
|
|
139
149
|
const animate = () => {
|
|
140
150
|
if (!this.isPlaying) {
|
|
141
151
|
this.animationFrameId = null;
|
|
142
152
|
return;
|
|
143
153
|
}
|
|
144
|
-
const elapsed =
|
|
154
|
+
const elapsed = ctx.currentTime - audioStartTime;
|
|
145
155
|
if (elapsed >= duration || frameIndex >= blendshapes.length) {
|
|
146
156
|
this.animationFrameId = null;
|
|
147
|
-
this.emit('blendshapeUpdate', new Array(BLENDSHAPE_COUNT).fill(0));
|
|
148
|
-
this.emit('playbackEnded');
|
|
149
|
-
this.playNext();
|
|
150
157
|
return;
|
|
151
158
|
}
|
|
152
159
|
const progress = elapsed / duration;
|
|
@@ -366,6 +373,13 @@ class AStackCSRClient extends eventemitter3.EventEmitter {
|
|
|
366
373
|
case 'connected':
|
|
367
374
|
if (typeof data.clientId === 'string')
|
|
368
375
|
this.clientId = data.clientId;
|
|
376
|
+
if (data.protocol_version) {
|
|
377
|
+
const major = parseInt(data.protocol_version.split('.')[0], 10);
|
|
378
|
+
if (major > 1) {
|
|
379
|
+
console.error(`[CSR] Protocol version mismatch: server=${data.protocol_version}, expected=1.x`);
|
|
380
|
+
this.emit('error', new Error(`Protocol version mismatch: server=${data.protocol_version}`));
|
|
381
|
+
}
|
|
382
|
+
}
|
|
369
383
|
break;
|
|
370
384
|
case 'authenticated':
|
|
371
385
|
if (this.pendingAuth) {
|
|
@@ -458,6 +472,14 @@ class AStackCSRClient extends eventemitter3.EventEmitter {
|
|
|
458
472
|
blendshape_count: typeof data.blendshape_count === 'number' ? data.blendshape_count : undefined
|
|
459
473
|
});
|
|
460
474
|
break;
|
|
475
|
+
case 'session_expired': {
|
|
476
|
+
const reason = typeof data.reason === 'string' ? data.reason : 'unknown';
|
|
477
|
+
this.emit('sessionExpired', reason);
|
|
478
|
+
if (reason === 'credits_exhausted') {
|
|
479
|
+
this.emit('creditsExhausted');
|
|
480
|
+
}
|
|
481
|
+
break;
|
|
482
|
+
}
|
|
461
483
|
}
|
|
462
484
|
}
|
|
463
485
|
async startCall(options) {
|
|
@@ -498,10 +520,11 @@ class AStackCSRClient extends eventemitter3.EventEmitter {
|
|
|
498
520
|
this.startImageCapture();
|
|
499
521
|
}
|
|
500
522
|
}
|
|
523
|
+
const { systemPrompt, priorContext, configOverrides, disableA2F, providers } = options || {};
|
|
501
524
|
this.ws.send(JSON.stringify({
|
|
502
525
|
type: 'call_start',
|
|
503
526
|
fps: this.config.fps,
|
|
504
|
-
|
|
527
|
+
systemPrompt, priorContext, configOverrides, disableA2F, providers
|
|
505
528
|
}));
|
|
506
529
|
const source = this.audioContext.createMediaStreamSource(this.mediaStream);
|
|
507
530
|
this.audioProcessor = new AudioWorkletNode(this.audioContext, 'audio-processor');
|
|
@@ -807,11 +830,19 @@ function VRMAvatar({ blendshapes, width = 400, height = 400, modelUrl = '/models
|
|
|
807
830
|
const container = containerRef.current;
|
|
808
831
|
if (!container)
|
|
809
832
|
return;
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
833
|
+
let renderer;
|
|
834
|
+
try {
|
|
835
|
+
renderer = new THREE__namespace.WebGLRenderer({
|
|
836
|
+
antialias: true,
|
|
837
|
+
alpha: true,
|
|
838
|
+
powerPreference: 'high-performance'
|
|
839
|
+
});
|
|
840
|
+
}
|
|
841
|
+
catch {
|
|
842
|
+
setError('WebGL is not supported in this browser');
|
|
843
|
+
setLoading(false);
|
|
844
|
+
return;
|
|
845
|
+
}
|
|
815
846
|
renderer.setSize(width, height);
|
|
816
847
|
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
|
817
848
|
renderer.outputColorSpace = THREE__namespace.SRGBColorSpace;
|
|
@@ -919,151 +950,5 @@ function VRMAvatar({ blendshapes, width = 400, height = 400, modelUrl = '/models
|
|
|
919
950
|
return (jsxRuntime.jsxs("div", { className: "relative", style: { width, height }, children: [jsxRuntime.jsx("div", { ref: containerRef, className: "rounded-lg overflow-hidden" }), loading && (jsxRuntime.jsx("div", { className: "absolute inset-0 flex items-center justify-center bg-gray-800 rounded-lg", children: jsxRuntime.jsx("div", { className: "text-white text-sm", children: "Loading VRM avatar..." }) })), error && (jsxRuntime.jsx("div", { className: "absolute inset-0 flex items-center justify-center bg-red-900/50 rounded-lg", children: jsxRuntime.jsx("div", { className: "text-white text-sm", children: error }) }))] }));
|
|
920
951
|
}
|
|
921
952
|
|
|
922
|
-
function TalkingHeadAvatar({ blendshapes, width = 400, height = 400, avatarUrl = 'https://models.readyplayer.me/6460d95f9ae10f45bffb2864.glb?morphTargets=ARKit,Oculus+Visemes,mouthOpen,mouthSmile,eyesClosed,eyesLookUp,eyesLookDown&textureSizeLimit=1024&textureFormat=png' }) {
|
|
923
|
-
const iframeRef = react.useRef(null);
|
|
924
|
-
const [loading, setLoading] = react.useState(true);
|
|
925
|
-
const [error, setError] = react.useState(null);
|
|
926
|
-
react.useEffect(() => {
|
|
927
|
-
const iframe = iframeRef.current;
|
|
928
|
-
if (!iframe)
|
|
929
|
-
return;
|
|
930
|
-
const handleMessage = (event) => {
|
|
931
|
-
if (event.data?.type === 'talkinghead-ready') {
|
|
932
|
-
setLoading(false);
|
|
933
|
-
}
|
|
934
|
-
else if (event.data?.type === 'talkinghead-error') {
|
|
935
|
-
setError(event.data.message || 'Failed to load TalkingHead avatar');
|
|
936
|
-
setLoading(false);
|
|
937
|
-
}
|
|
938
|
-
};
|
|
939
|
-
window.addEventListener('message', handleMessage);
|
|
940
|
-
return () => window.removeEventListener('message', handleMessage);
|
|
941
|
-
}, []);
|
|
942
|
-
react.useEffect(() => {
|
|
943
|
-
const iframe = iframeRef.current;
|
|
944
|
-
if (!iframe?.contentWindow || blendshapes.length === 0)
|
|
945
|
-
return;
|
|
946
|
-
const blendshapeData = {};
|
|
947
|
-
ARKIT_BLENDSHAPES.forEach((name, i) => {
|
|
948
|
-
blendshapeData[name] = blendshapes[i] || 0;
|
|
949
|
-
});
|
|
950
|
-
iframe.contentWindow.postMessage({
|
|
951
|
-
type: 'update-blendshapes',
|
|
952
|
-
blendshapes: blendshapeData
|
|
953
|
-
}, '*');
|
|
954
|
-
}, [blendshapes]);
|
|
955
|
-
const iframeSrc = `data:text/html,${encodeURIComponent(`
|
|
956
|
-
<!DOCTYPE html>
|
|
957
|
-
<html>
|
|
958
|
-
<head>
|
|
959
|
-
<style>
|
|
960
|
-
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
961
|
-
html, body { width: 100%; height: 100%; overflow: hidden; background: #1a1a2e; }
|
|
962
|
-
#avatar { width: 100%; height: 100%; }
|
|
963
|
-
</style>
|
|
964
|
-
<script type="importmap">
|
|
965
|
-
{
|
|
966
|
-
"imports": {
|
|
967
|
-
"three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js",
|
|
968
|
-
"three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/"
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
${"</"}script>
|
|
972
|
-
</head>
|
|
973
|
-
<body>
|
|
974
|
-
<div id="avatar"></div>
|
|
975
|
-
<script type="module">
|
|
976
|
-
import { TalkingHead } from 'https://cdn.jsdelivr.net/npm/@met4citizen/talkinghead@1.6.0/modules/talkinghead.mjs';
|
|
977
|
-
|
|
978
|
-
let head = null;
|
|
979
|
-
|
|
980
|
-
async function init() {
|
|
981
|
-
try {
|
|
982
|
-
const container = document.getElementById('avatar');
|
|
983
|
-
head = new TalkingHead(container, {
|
|
984
|
-
ttsEndpoint: null,
|
|
985
|
-
cameraView: 'head',
|
|
986
|
-
cameraRotateEnable: false,
|
|
987
|
-
cameraPanEnable: false,
|
|
988
|
-
cameraZoomEnable: false,
|
|
989
|
-
lightAmbientColor: 0xffffff,
|
|
990
|
-
lightAmbientIntensity: 0.6,
|
|
991
|
-
lightDirectColor: 0xffffff,
|
|
992
|
-
lightDirectIntensity: 0.8,
|
|
993
|
-
lightSpotIntensity: 0,
|
|
994
|
-
avatarMood: 'neutral',
|
|
995
|
-
modelPixelRatio: Math.min(window.devicePixelRatio, 2),
|
|
996
|
-
modelFPS: 30
|
|
997
|
-
});
|
|
998
|
-
|
|
999
|
-
await head.showAvatar({
|
|
1000
|
-
url: '${avatarUrl}',
|
|
1001
|
-
body: 'F',
|
|
1002
|
-
avatarMood: 'neutral',
|
|
1003
|
-
lipsyncLang: 'en'
|
|
1004
|
-
});
|
|
1005
|
-
|
|
1006
|
-
window.parent.postMessage({ type: 'talkinghead-ready' }, '*');
|
|
1007
|
-
} catch (err) {
|
|
1008
|
-
console.error('[TalkingHead] Error:', err);
|
|
1009
|
-
window.parent.postMessage({ type: 'talkinghead-error', message: err.message }, '*');
|
|
1010
|
-
}
|
|
1011
|
-
}
|
|
1012
|
-
|
|
1013
|
-
let started = false;
|
|
1014
|
-
|
|
1015
|
-
window.addEventListener('message', (event) => {
|
|
1016
|
-
if (event.data?.type === 'update-blendshapes' && head) {
|
|
1017
|
-
if (!started) {
|
|
1018
|
-
head.start();
|
|
1019
|
-
started = true;
|
|
1020
|
-
}
|
|
1021
|
-
|
|
1022
|
-
const shapes = event.data.blendshapes;
|
|
1023
|
-
const mtAvatar = head.mtAvatar;
|
|
1024
|
-
|
|
1025
|
-
if (mtAvatar) {
|
|
1026
|
-
for (const [name, value] of Object.entries(shapes)) {
|
|
1027
|
-
const v = Math.min(Number(value) || 0, 1);
|
|
1028
|
-
if (mtAvatar[name]) {
|
|
1029
|
-
mtAvatar[name].value = v;
|
|
1030
|
-
mtAvatar[name].applied = v;
|
|
1031
|
-
if (mtAvatar[name].ms && mtAvatar[name].is) {
|
|
1032
|
-
for (let i = 0; i < mtAvatar[name].ms.length; i++) {
|
|
1033
|
-
const influences = mtAvatar[name].ms[i];
|
|
1034
|
-
const idx = mtAvatar[name].is[i];
|
|
1035
|
-
if (influences && idx !== undefined) {
|
|
1036
|
-
influences[idx] = v;
|
|
1037
|
-
}
|
|
1038
|
-
}
|
|
1039
|
-
}
|
|
1040
|
-
}
|
|
1041
|
-
}
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
if (head.morphs) {
|
|
1045
|
-
for (const mesh of head.morphs) {
|
|
1046
|
-
if (mesh.morphTargetDictionary && mesh.morphTargetInfluences) {
|
|
1047
|
-
for (const [name, value] of Object.entries(shapes)) {
|
|
1048
|
-
const idx = mesh.morphTargetDictionary[name];
|
|
1049
|
-
if (idx !== undefined) {
|
|
1050
|
-
mesh.morphTargetInfluences[idx] = Math.min(Number(value) || 0, 1);
|
|
1051
|
-
}
|
|
1052
|
-
}
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
}
|
|
1057
|
-
});
|
|
1058
|
-
|
|
1059
|
-
init();
|
|
1060
|
-
${"</"}script>
|
|
1061
|
-
</body>
|
|
1062
|
-
</html>
|
|
1063
|
-
`)}`;
|
|
1064
|
-
return (jsxRuntime.jsxs("div", { className: "relative bg-gray-900 rounded-lg overflow-hidden", style: { width, height }, children: [jsxRuntime.jsx("iframe", { ref: iframeRef, src: iframeSrc, style: { width: '100%', height: '100%', border: 'none' }, sandbox: "allow-scripts allow-same-origin" }), loading && (jsxRuntime.jsx("div", { className: "absolute inset-0 flex items-center justify-center bg-gray-800 rounded-lg", children: jsxRuntime.jsx("div", { className: "text-white text-sm", children: "Loading TalkingHead avatar..." }) })), error && (jsxRuntime.jsx("div", { className: "absolute inset-0 flex items-center justify-center bg-red-900/50 rounded-lg", children: jsxRuntime.jsx("div", { className: "text-white text-sm text-center p-4", children: error }) }))] }));
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
exports.TalkingHeadAvatar = TalkingHeadAvatar;
|
|
1068
953
|
exports.VRMAvatar = VRMAvatar;
|
|
1069
954
|
exports.useAStackCSR = useAStackCSR;
|
package/dist/types.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aether-stack-dev/client-sdk",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "JavaScript/TypeScript SDK for AStack video-to-video AI conversations",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"module": "dist/index.esm.js",
|
|
8
|
-
"types": "dist/
|
|
8
|
+
"types": "dist/index.d.ts",
|
|
9
9
|
"exports": {
|
|
10
10
|
".": {
|
|
11
11
|
"import": "./dist/index.esm.js",
|
|
12
12
|
"require": "./dist/index.js",
|
|
13
|
-
"types": "./dist/
|
|
13
|
+
"types": "./dist/index.d.ts"
|
|
14
14
|
},
|
|
15
15
|
"./react": {
|
|
16
16
|
"import": "./dist/react.esm.js",
|
|
@@ -28,8 +28,10 @@
|
|
|
28
28
|
"build:types": "tsc --declaration --emitDeclarationOnly --outDir dist",
|
|
29
29
|
"test": "jest",
|
|
30
30
|
"test:watch": "jest --watch",
|
|
31
|
+
"test:coverage": "jest --coverage",
|
|
31
32
|
"lint": "eslint src/**/*.{ts,tsx}",
|
|
32
|
-
"typecheck": "tsc --noEmit"
|
|
33
|
+
"typecheck": "tsc --noEmit",
|
|
34
|
+
"prepublishOnly": "npm run build"
|
|
33
35
|
},
|
|
34
36
|
"keywords": [
|
|
35
37
|
"ai",
|
|
@@ -42,7 +44,7 @@
|
|
|
42
44
|
"license": "MIT",
|
|
43
45
|
"dependencies": {
|
|
44
46
|
"@pixiv/three-vrm": "^3.4.4",
|
|
45
|
-
"@supabase/supabase-js": "^2.
|
|
47
|
+
"@supabase/supabase-js": "^2.100.0",
|
|
46
48
|
"eventemitter3": "^5.0.1",
|
|
47
49
|
"three": "^0.181.2",
|
|
48
50
|
"uuid": "^9.0.1"
|
|
@@ -78,7 +80,7 @@
|
|
|
78
80
|
},
|
|
79
81
|
"repository": {
|
|
80
82
|
"type": "git",
|
|
81
|
-
"url": "https://github.com/aether-stack/astack-mono.git",
|
|
83
|
+
"url": "git+https://github.com/aether-stack/astack-mono.git",
|
|
82
84
|
"directory": "client-sdk"
|
|
83
85
|
},
|
|
84
86
|
"bugs": {
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
export interface TalkingHeadAvatarProps {
|
|
2
|
-
blendshapes: number[];
|
|
3
|
-
width?: number;
|
|
4
|
-
height?: number;
|
|
5
|
-
avatarUrl?: string;
|
|
6
|
-
}
|
|
7
|
-
export declare function TalkingHeadAvatar({ blendshapes, width, height, avatarUrl }: TalkingHeadAvatarProps): import("react/jsx-runtime").JSX.Element;
|
|
8
|
-
export default TalkingHeadAvatar;
|