@bytexbyte/nxtlinq-ai-agent-sdk 1.6.7 → 1.6.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/components/context/ChatBotContext.d.ts.map +1 -1
- package/dist/components/context/ChatBotContext.js +85 -19
- package/dist/components/ui/MessageInput.d.ts.map +1 -1
- package/dist/components/ui/MessageInput.js +3 -3
- package/dist/components/ui/MessageList.js +6 -6
- package/dist/core/lib/useSpeechToTextFromMic/helper.d.ts +6 -3
- package/dist/core/lib/useSpeechToTextFromMic/helper.d.ts.map +1 -1
- package/dist/core/lib/useSpeechToTextFromMic/helper.js +18 -7
- package/dist/core/lib/useSpeechToTextFromMic/index.d.ts +3 -1
- package/dist/core/lib/useSpeechToTextFromMic/index.d.ts.map +1 -1
- package/dist/core/lib/useSpeechToTextFromMic/index.js +14 -3
- package/dist/types/ait-api.d.ts +1 -0
- package/dist/types/ait-api.d.ts.map +1 -1
- package/package.json +1 -1
- package/umd/nxtlinq-ai-agent.umd.js +132 -132
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ChatBotContext.d.ts","sourceRoot":"","sources":["../../../src/components/context/ChatBotContext.tsx"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAS/B,OAAO,EACL,kBAAkB,EAClB,YAAY,EAGb,MAAM,uBAAuB,CAAC;AAI/B,eAAO,MAAM,UAAU,0BAMtB,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,KAAK,CAAC,EAAE,CAAC,YAAY,
|
|
1
|
+
{"version":3,"file":"ChatBotContext.d.ts","sourceRoot":"","sources":["../../../src/components/context/ChatBotContext.tsx"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAS/B,OAAO,EACL,kBAAkB,EAClB,YAAY,EAGb,MAAM,uBAAuB,CAAC;AAI/B,eAAO,MAAM,UAAU,0BAMtB,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,KAAK,CAAC,EAAE,CAAC,YAAY,CAs/DlD,CAAC"}
|
|
@@ -35,7 +35,7 @@ idvBannerDismissSeconds = 86400,
|
|
|
35
35
|
isStopRecordingOnSend = false, }) => {
|
|
36
36
|
const nxtlinqApi = React.useMemo(() => createNxtlinqApi(apiKey, apiSecret), [apiKey, apiSecret]);
|
|
37
37
|
// Custom hook
|
|
38
|
-
const { isRecording, transcript, start: startRecording, stop: stopRecording, clear: clearRecording } = useSpeechToTextFromMic({
|
|
38
|
+
const { isRecording, transcript, partialTranscript, start: startRecording, stop: stopRecording, clear: clearRecording } = useSpeechToTextFromMic({
|
|
39
39
|
apiKey,
|
|
40
40
|
apiSecret
|
|
41
41
|
});
|
|
@@ -77,6 +77,40 @@ isStopRecordingOnSend = false, }) => {
|
|
|
77
77
|
// Refs for input value and recording state
|
|
78
78
|
const isRecordingRef = React.useRef(false);
|
|
79
79
|
const textInputRef = React.useRef(null);
|
|
80
|
+
const lastPartialRangeRef = React.useRef(null);
|
|
81
|
+
function insertPartial(input, partial, caret) {
|
|
82
|
+
let start = caret;
|
|
83
|
+
let end = caret;
|
|
84
|
+
if (lastPartialRangeRef.current) {
|
|
85
|
+
start = lastPartialRangeRef.current.start;
|
|
86
|
+
end = lastPartialRangeRef.current.end;
|
|
87
|
+
}
|
|
88
|
+
const before = input.slice(0, start);
|
|
89
|
+
const after = input.slice(end);
|
|
90
|
+
const next = before + partial + after;
|
|
91
|
+
const newCaret = start + partial.length;
|
|
92
|
+
lastPartialRangeRef.current = { start, end: newCaret };
|
|
93
|
+
return { next, caret: newCaret };
|
|
94
|
+
}
|
|
95
|
+
function finalizePartial(input, finalText) {
|
|
96
|
+
if (!lastPartialRangeRef.current) {
|
|
97
|
+
return { next: input, caret: input.length };
|
|
98
|
+
}
|
|
99
|
+
const { start, end } = lastPartialRangeRef.current;
|
|
100
|
+
const before = input.slice(0, start);
|
|
101
|
+
const after = input.slice(end);
|
|
102
|
+
const next = before + finalText + after;
|
|
103
|
+
const caret = start + finalText.length;
|
|
104
|
+
lastPartialRangeRef.current = null;
|
|
105
|
+
return { next, caret };
|
|
106
|
+
}
|
|
107
|
+
function normalizeTranscript(text) {
|
|
108
|
+
return text
|
|
109
|
+
.toLowerCase()
|
|
110
|
+
.replace(/[.,!?;:]/g, '')
|
|
111
|
+
.replace(/\s+/g, ' ')
|
|
112
|
+
.trim();
|
|
113
|
+
}
|
|
80
114
|
// Simple token cleanup function
|
|
81
115
|
const clearExpiredToken = React.useCallback(() => {
|
|
82
116
|
try {
|
|
@@ -104,10 +138,34 @@ isStopRecordingOnSend = false, }) => {
|
|
|
104
138
|
signerRef.current = signer;
|
|
105
139
|
}, [signer]);
|
|
106
140
|
React.useEffect(() => {
|
|
107
|
-
if (!
|
|
141
|
+
if (!textInputRef.current)
|
|
108
142
|
return;
|
|
143
|
+
const el = textInputRef.current;
|
|
144
|
+
const selStart = el.selectionStart ?? inputValue.length;
|
|
145
|
+
if (partialTranscript) {
|
|
146
|
+
const { next, caret } = insertPartial(inputValue, partialTranscript, selStart);
|
|
147
|
+
setInputValue(next);
|
|
148
|
+
setTimeout(() => {
|
|
149
|
+
if (textInputRef.current) {
|
|
150
|
+
textInputRef.current.selectionStart = caret;
|
|
151
|
+
textInputRef.current.selectionEnd = caret;
|
|
152
|
+
}
|
|
153
|
+
}, 0);
|
|
109
154
|
}
|
|
110
|
-
|
|
155
|
+
}, [partialTranscript]);
|
|
156
|
+
React.useEffect(() => {
|
|
157
|
+
if (!transcript)
|
|
158
|
+
return;
|
|
159
|
+
if (!textInputRef.current)
|
|
160
|
+
return;
|
|
161
|
+
const { next, caret } = finalizePartial(inputValue, transcript);
|
|
162
|
+
setInputValue(normalizeTranscript(next));
|
|
163
|
+
setTimeout(() => {
|
|
164
|
+
if (textInputRef.current) {
|
|
165
|
+
textInputRef.current.selectionStart = caret;
|
|
166
|
+
textInputRef.current.selectionEnd = caret;
|
|
167
|
+
}
|
|
168
|
+
}, 0);
|
|
111
169
|
}, [transcript]);
|
|
112
170
|
React.useEffect(() => {
|
|
113
171
|
isRecordingRef.current = isRecording;
|
|
@@ -448,7 +506,7 @@ isStopRecordingOnSend = false, }) => {
|
|
|
448
506
|
}
|
|
449
507
|
};
|
|
450
508
|
// Check permissions
|
|
451
|
-
const hasPermission = async (
|
|
509
|
+
const hasPermission = async (requiredPermission, autoRetry = true, onAutoConnect, onAutoSignIn) => {
|
|
452
510
|
// Use refs to get latest state values
|
|
453
511
|
const currentHitAddress = hitAddressRef.current;
|
|
454
512
|
const currentAit = aitRef.current;
|
|
@@ -490,7 +548,7 @@ isStopRecordingOnSend = false, }) => {
|
|
|
490
548
|
}
|
|
491
549
|
}
|
|
492
550
|
// If connection (and sign-in if needed) successful, continue with permission check
|
|
493
|
-
const result = await hasPermission(
|
|
551
|
+
const result = await hasPermission(requiredPermission, false);
|
|
494
552
|
return result;
|
|
495
553
|
}
|
|
496
554
|
catch (error) {
|
|
@@ -533,7 +591,7 @@ isStopRecordingOnSend = false, }) => {
|
|
|
533
591
|
// Wait a bit more to ensure permissions are also loaded
|
|
534
592
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
535
593
|
// If sign-in successful, continue with permission check
|
|
536
|
-
const result = await hasPermission(
|
|
594
|
+
const result = await hasPermission(requiredPermission, false);
|
|
537
595
|
return result;
|
|
538
596
|
}
|
|
539
597
|
else {
|
|
@@ -585,7 +643,7 @@ isStopRecordingOnSend = false, }) => {
|
|
|
585
643
|
// Wait a bit more to ensure permissions are also loaded
|
|
586
644
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
587
645
|
// If sign-in successful, continue with permission check
|
|
588
|
-
const result = await hasPermission(
|
|
646
|
+
const result = await hasPermission(requiredPermission, false);
|
|
589
647
|
return result;
|
|
590
648
|
}
|
|
591
649
|
else {
|
|
@@ -634,7 +692,7 @@ isStopRecordingOnSend = false, }) => {
|
|
|
634
692
|
// Wait a bit more to ensure permissions are also loaded
|
|
635
693
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
636
694
|
// If sign-in successful, continue with permission check
|
|
637
|
-
const result = await hasPermission(
|
|
695
|
+
const result = await hasPermission(requiredPermission, false);
|
|
638
696
|
return result;
|
|
639
697
|
}
|
|
640
698
|
else {
|
|
@@ -684,7 +742,7 @@ isStopRecordingOnSend = false, }) => {
|
|
|
684
742
|
// Wait a bit more to ensure permissions are also loaded
|
|
685
743
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
686
744
|
// If sign-in successful, continue with permission check
|
|
687
|
-
const result = await hasPermission(
|
|
745
|
+
const result = await hasPermission(requiredPermission, false);
|
|
688
746
|
return result;
|
|
689
747
|
}
|
|
690
748
|
else {
|
|
@@ -769,17 +827,17 @@ isStopRecordingOnSend = false, }) => {
|
|
|
769
827
|
}]);
|
|
770
828
|
return false;
|
|
771
829
|
}
|
|
772
|
-
if (!availablePermissionLabels.includes(
|
|
830
|
+
if (!availablePermissionLabels.includes(requiredPermission)) {
|
|
773
831
|
setIsLoading(false); // Stop thinking before showing message
|
|
774
832
|
setMessages(prev => [...prev, {
|
|
775
833
|
id: Date.now().toString(),
|
|
776
|
-
content: `This
|
|
834
|
+
content: `This permission (${requiredPermission}) is not available for your current identity provider.`,
|
|
777
835
|
role: 'assistant',
|
|
778
836
|
timestamp: new Date().toISOString()
|
|
779
837
|
}]);
|
|
780
838
|
return false;
|
|
781
839
|
}
|
|
782
|
-
if (!currentPermissions.includes(
|
|
840
|
+
if (!currentPermissions.includes(requiredPermission)) {
|
|
783
841
|
// Check if AIT is still loading or if we just auto-connected
|
|
784
842
|
if (isAITLoading || isAutoConnecting) {
|
|
785
843
|
setIsLoading(false); // Stop thinking before showing message
|
|
@@ -804,7 +862,7 @@ isStopRecordingOnSend = false, }) => {
|
|
|
804
862
|
}
|
|
805
863
|
// Only show permission error if AIT is loaded and permissions are not empty
|
|
806
864
|
setIsLoading(false); // Stop thinking before showing message
|
|
807
|
-
const permissionMsg = `You don't have the required AIT permission: ${
|
|
865
|
+
const permissionMsg = `You don't have the required AIT permission: ${requiredPermission}. Would you like to enable AIT permission?`;
|
|
808
866
|
setMessages(prev => [
|
|
809
867
|
...prev,
|
|
810
868
|
{
|
|
@@ -813,7 +871,7 @@ isStopRecordingOnSend = false, }) => {
|
|
|
813
871
|
role: 'assistant',
|
|
814
872
|
timestamp: new Date().toISOString(),
|
|
815
873
|
button: 'enableAIT',
|
|
816
|
-
metadata: {
|
|
874
|
+
metadata: { requiredPermission }
|
|
817
875
|
}
|
|
818
876
|
]);
|
|
819
877
|
return false;
|
|
@@ -989,9 +1047,11 @@ isStopRecordingOnSend = false, }) => {
|
|
|
989
1047
|
let wasAutoSignedIn = false;
|
|
990
1048
|
// Added: Mark if permission denied due to missing AIT permission
|
|
991
1049
|
let permissionDenied = false;
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
1050
|
+
// Use requiredPermission from response if available, otherwise fall back to toolUse.name
|
|
1051
|
+
const permissionToCheck = response?.requiredPermission || toolUse.name;
|
|
1052
|
+
const isToolAllowed = await hasPermission(permissionToCheck, true, () => { wasAutoConnected = true; }, () => { wasAutoSignedIn = true; });
|
|
1053
|
+
// If currentPermissions does not include permissionToCheck and availablePermissionLabels includes permissionToCheck, it means AIT permission is missing
|
|
1054
|
+
if (!isToolAllowed && !permissions.includes(permissionToCheck) && availablePermissions.map(p => p.label).includes(permissionToCheck)) {
|
|
995
1055
|
permissionDenied = true;
|
|
996
1056
|
}
|
|
997
1057
|
if (!isToolAllowed) {
|
|
@@ -1417,10 +1477,12 @@ isStopRecordingOnSend = false, }) => {
|
|
|
1417
1477
|
if (!newPermissions.includes(toolName)) {
|
|
1418
1478
|
newPermissions.push(toolName);
|
|
1419
1479
|
}
|
|
1480
|
+
// Filter out deleted permissions before saving
|
|
1481
|
+
const validPermissions = newPermissions.filter(permission => availablePermissionLabels.includes(permission));
|
|
1420
1482
|
// Generate and register AIT with new permissions
|
|
1421
1483
|
// For auto-enable, we should create a regular AIT (not AI Agent AIT) if user doesn't have existing AIT
|
|
1422
1484
|
const shouldCreateAsAIAgent = !!aitRef.current; // Only create as AI Agent if user already has an AIT
|
|
1423
|
-
await generateAndRegisterAITWithSigner(
|
|
1485
|
+
await generateAndRegisterAITWithSigner(validPermissions, shouldCreateAsAIAgent, currentSigner, currentHitAddress);
|
|
1424
1486
|
showSuccess('AIT permission enabled successfully! You can now use the AI agent.');
|
|
1425
1487
|
await refreshAIT(true);
|
|
1426
1488
|
setIsAITEnabling(false);
|
|
@@ -1442,7 +1504,11 @@ isStopRecordingOnSend = false, }) => {
|
|
|
1442
1504
|
const savePermissions = async (newPermissions) => {
|
|
1443
1505
|
setIsDisabled(true);
|
|
1444
1506
|
try {
|
|
1445
|
-
|
|
1507
|
+
// Filter out deleted permissions before saving
|
|
1508
|
+
const permissionsToSave = newPermissions || permissions;
|
|
1509
|
+
const availablePermissionLabels = availablePermissions.map(p => p.label);
|
|
1510
|
+
const validPermissions = permissionsToSave.filter(permission => availablePermissionLabels.includes(permission));
|
|
1511
|
+
await generateAndRegisterAIT(validPermissions, false);
|
|
1446
1512
|
showSuccess('AIT permissions saved successfully! You can now use the AI agent with your configured permissions.');
|
|
1447
1513
|
setShowPermissionForm(false);
|
|
1448
1514
|
setIsPermissionFormOpen(false);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MessageInput.d.ts","sourceRoot":"","sources":["../../../src/components/ui/MessageInput.tsx"],"names":[],"mappings":"AAKA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAI/B,eAAO,MAAM,YAAY,EAAE,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"MessageInput.d.ts","sourceRoot":"","sources":["../../../src/components/ui/MessageInput.tsx"],"names":[],"mappings":"AAKA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAI/B,eAAO,MAAM,YAAY,EAAE,KAAK,CAAC,EAoGhC,CAAC"}
|
|
@@ -22,16 +22,16 @@ export const MessageInput = () => {
|
|
|
22
22
|
align-items: center !important;
|
|
23
23
|
gap: 10px !important;
|
|
24
24
|
border-top: 1px solid #eee !important;
|
|
25
|
-
`, children: [_jsx(InputBase, { value: inputValue, onChange: (e) => setInputValue(e.target.value), onKeyPress: handleKeyPress, placeholder: inputPlaceholder,
|
|
25
|
+
`, children: [_jsx(InputBase, { value: inputValue, onChange: (e) => setInputValue(e.target.value), onKeyPress: handleKeyPress, placeholder: inputPlaceholder, fullWidth: true, inputProps: {
|
|
26
26
|
ref: textInputRef
|
|
27
|
-
}, endAdornment: _jsx(IconButton, { onClick: () => isRecording ? stopRecording() : startRecording(),
|
|
27
|
+
}, endAdornment: _jsx(IconButton, { onClick: () => isRecording ? stopRecording() : startRecording(), children: isRecording ? _jsx(MicIcon, {}) : _jsx(MicOffIcon, {}) }), css: css `
|
|
28
28
|
flex: 1 !important;
|
|
29
29
|
padding: 10px !important;
|
|
30
30
|
border: 1px solid #ddd !important;
|
|
31
31
|
border-radius: 20px !important;
|
|
32
32
|
outline: none !important;
|
|
33
33
|
font-size: 14px !important;
|
|
34
|
-
background-color:
|
|
34
|
+
background-color: #fff !important;
|
|
35
35
|
height: 40px !important;
|
|
36
36
|
box-sizing: border-box !important;
|
|
37
37
|
` }), _jsxs("button", { onClick: (e) => handleSubmit(e), disabled: isDisabled || !inputValue.trim(), css: css `
|
|
@@ -23,9 +23,9 @@ export const MessageList = () => {
|
|
|
23
23
|
signInWallet(true);
|
|
24
24
|
}
|
|
25
25
|
else if (buttonType === 'enableAIT') {
|
|
26
|
-
const
|
|
27
|
-
if (
|
|
28
|
-
const success = await enableAIT(
|
|
26
|
+
const requiredPermission = message.metadata?.requiredPermission;
|
|
27
|
+
if (requiredPermission) {
|
|
28
|
+
const success = await enableAIT(requiredPermission);
|
|
29
29
|
if (success) {
|
|
30
30
|
// Find the last user message
|
|
31
31
|
const lastUserMsg = [...messages].reverse().find(m => m.role === 'user');
|
|
@@ -61,16 +61,16 @@ export const MessageList = () => {
|
|
|
61
61
|
(message.button === 'connectWallet' && Boolean(hitAddress)) ||
|
|
62
62
|
(message.button === 'signIn' && !isNeedSignInWithWallet) ||
|
|
63
63
|
(message.button === 'enableAIT' && (isAITLoading || isAITEnabling ||
|
|
64
|
-
(message.metadata?.
|
|
64
|
+
(message.metadata?.requiredPermission && permissions.includes(message.metadata.requiredPermission)))) || false, css: (message.button === 'connectWallet' && Boolean(hitAddress)) ? connectedButton :
|
|
65
65
|
(message.button === 'signIn' && !isNeedSignInWithWallet) ? connectedButton :
|
|
66
66
|
(message.button === 'continue') ? connectedButton :
|
|
67
|
-
(message.button === 'enableAIT' && (message.metadata?.
|
|
67
|
+
(message.button === 'enableAIT' && (message.metadata?.requiredPermission && permissions.includes(message.metadata.requiredPermission))) ? connectedButton :
|
|
68
68
|
chatbotButton, children: isAutoConnecting ? 'Connecting...' :
|
|
69
69
|
message.button === 'connectWallet' ? (Boolean(hitAddress) ? 'Connected' : 'Connect Wallet') :
|
|
70
70
|
message.button === 'signIn' ? (!isNeedSignInWithWallet ? 'Signed In' : 'Sign In') :
|
|
71
71
|
message.button === 'continue' ? 'Continue' :
|
|
72
72
|
message.button === 'enableAIT' ?
|
|
73
73
|
((isAITLoading || isAITEnabling) ? 'Enabling...' :
|
|
74
|
-
(message.metadata?.
|
|
74
|
+
(message.metadata?.requiredPermission && permissions.includes(message.metadata.requiredPermission)) ? 'AIT Enabled' : 'Enable AIT Permissions') :
|
|
75
75
|
message.button }) }))] }), message.role === 'assistant' && message.metadata?.model && (_jsx("div", { css: modelIndicator, children: _jsxs("div", { css: modelBadge, children: [_jsx("span", { css: modelDot }), getModelDisplayName(message.metadata.model)] }) }))] }, message.id))), isLoading && (_jsx("div", { css: loadingIndicator, children: "Thinking..." })), _jsx("div", { ref: messagesEndRef })] }));
|
|
76
76
|
};
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { SpeechRecognizer } from 'microsoft-cognitiveservices-speech-sdk';
|
|
2
2
|
import { Dispatch, SetStateAction } from 'react';
|
|
3
3
|
/**
|
|
4
|
-
* Start speech recognition
|
|
4
|
+
* Start speech recognition
|
|
5
|
+
* - partialTranscript: 暫存逐字 (recognizing)
|
|
6
|
+
* - setSpeechToTextArray: 完整句子陣列 (recognized)
|
|
5
7
|
*/
|
|
6
|
-
export declare const startSpeechToTextFromMic: (setSpeechToTextArray: Dispatch<SetStateAction<string[]>>,
|
|
8
|
+
export declare const startSpeechToTextFromMic: (setSpeechToTextArray: Dispatch<SetStateAction<string[]>>, // 完整句子
|
|
9
|
+
config: {
|
|
7
10
|
apiKey: string;
|
|
8
11
|
apiSecret: string;
|
|
9
|
-
}, historyRef: React.MutableRefObject<string[]>, indexRef: React.MutableRefObject<number
|
|
12
|
+
}, historyRef: React.MutableRefObject<string[]>, indexRef: React.MutableRefObject<number>, setPartialTranscript?: Dispatch<SetStateAction<string>>) => Promise<SpeechRecognizer | undefined>;
|
|
10
13
|
/**
|
|
11
14
|
* Stop speech recognition
|
|
12
15
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../../../../src/core/lib/useSpeechToTextFromMic/helper.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../../../../src/core/lib/useSpeechToTextFromMic/helper.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,gBAAgB,EACjB,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAIjD;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,GACnC,sBAAsB,QAAQ,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,EAAI,OAAO;AACnE,QAAQ;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,EAC7C,YAAY,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAC5C,UAAU,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,EACxC,uBAAuB,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,KACtD,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAsDtC,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,eAAe,GAAI,YAAY,gBAAgB,GAAG,SAAS,SAIvE,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;;;;;;;;GAwBxE"}
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { AudioConfig, CancellationReason, PropertyId, ResultReason, SpeechConfig, SpeechRecognizer, } from 'microsoft-cognitiveservices-speech-sdk';
|
|
2
2
|
import Cookie from 'universal-cookie';
|
|
3
3
|
import { createNxtlinqApi } from '../../../api/nxtlinq-api';
|
|
4
|
-
let recognizer;
|
|
5
4
|
/**
|
|
6
|
-
* Start speech recognition
|
|
5
|
+
* Start speech recognition
|
|
6
|
+
* - partialTranscript: 暫存逐字 (recognizing)
|
|
7
|
+
* - setSpeechToTextArray: 完整句子陣列 (recognized)
|
|
7
8
|
*/
|
|
8
|
-
export const startSpeechToTextFromMic = async (setSpeechToTextArray,
|
|
9
|
+
export const startSpeechToTextFromMic = async (setSpeechToTextArray, // 完整句子
|
|
10
|
+
config, historyRef, indexRef, setPartialTranscript // ✅ 新增：暫存逐字
|
|
11
|
+
) => {
|
|
9
12
|
const tokenRes = await getTokenOrRefresh(config.apiKey, config.apiSecret);
|
|
10
13
|
if (!tokenRes.authToken || !tokenRes.region) {
|
|
11
14
|
console.error('Speech token retrieval failed:', tokenRes.error);
|
|
@@ -13,20 +16,28 @@ export const startSpeechToTextFromMic = async (setSpeechToTextArray, config, his
|
|
|
13
16
|
}
|
|
14
17
|
const speechConfig = SpeechConfig.fromAuthorizationToken(tokenRes.authToken, tokenRes.region);
|
|
15
18
|
speechConfig.speechRecognitionLanguage = 'en-US';
|
|
19
|
+
// 靜音判斷與 segmentation 設定
|
|
16
20
|
speechConfig.setProperty(PropertyId.SpeechServiceConnection_InitialSilenceTimeoutMs, '10000');
|
|
17
21
|
speechConfig.setProperty(PropertyId.SpeechServiceConnection_EndSilenceTimeoutMs, '86400000');
|
|
18
22
|
speechConfig.setProperty(PropertyId.Speech_SegmentationSilenceTimeoutMs, '1000');
|
|
19
23
|
const audioConfig = AudioConfig.fromDefaultMicrophoneInput();
|
|
20
24
|
const recognizer = new SpeechRecognizer(speechConfig, audioConfig);
|
|
25
|
+
// 暫存逐字
|
|
21
26
|
recognizer.recognizing = (_s, e) => {
|
|
22
|
-
|
|
23
|
-
|
|
27
|
+
if (setPartialTranscript) {
|
|
28
|
+
setPartialTranscript(e.result.text);
|
|
29
|
+
}
|
|
24
30
|
};
|
|
31
|
+
// 完整句子
|
|
25
32
|
recognizer.recognized = (_s, e) => {
|
|
26
33
|
if (e.result.reason === ResultReason.RecognizedSpeech) {
|
|
27
|
-
|
|
28
|
-
|
|
34
|
+
const text = e.result.text;
|
|
35
|
+
historyRef.current[indexRef.current] = text;
|
|
29
36
|
indexRef.current += 1;
|
|
37
|
+
// ✅ 關鍵：只把「本次完整句子」回推給 UI
|
|
38
|
+
setSpeechToTextArray([text]);
|
|
39
|
+
if (setPartialTranscript)
|
|
40
|
+
setPartialTranscript('');
|
|
30
41
|
}
|
|
31
42
|
};
|
|
32
43
|
recognizer.canceled = (_s, e) => {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
interface Props {
|
|
2
2
|
apiKey: string;
|
|
3
3
|
apiSecret: string;
|
|
4
|
+
autoClearTranscript?: boolean;
|
|
4
5
|
}
|
|
5
6
|
type UseSpeechToTextFromMicResult = {
|
|
6
7
|
start: () => Promise<void>;
|
|
@@ -8,7 +9,8 @@ type UseSpeechToTextFromMicResult = {
|
|
|
8
9
|
clear: () => void;
|
|
9
10
|
isRecording: boolean;
|
|
10
11
|
transcript: string;
|
|
12
|
+
partialTranscript: string;
|
|
11
13
|
};
|
|
12
|
-
export declare function useSpeechToTextFromMic({ apiKey, apiSecret }: Props): UseSpeechToTextFromMicResult;
|
|
14
|
+
export declare function useSpeechToTextFromMic({ apiKey, apiSecret, autoClearTranscript, }: Props): UseSpeechToTextFromMicResult;
|
|
13
15
|
export {};
|
|
14
16
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/core/lib/useSpeechToTextFromMic/index.ts"],"names":[],"mappings":"AAIA,UAAU,KAAK;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/core/lib/useSpeechToTextFromMic/index.ts"],"names":[],"mappings":"AAIA,UAAU,KAAK;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,KAAK,4BAA4B,GAAG;IAClC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,EAAE,MAAM,IAAI,CAAC;IACjB,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,EACrC,MAAM,EACN,SAAS,EACT,mBAA0B,GAC3B,EAAE,KAAK,GAAG,4BAA4B,CAuEtC"}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { useRef, useState } from 'react';
|
|
1
|
+
import { useEffect, useRef, useState } from 'react';
|
|
2
2
|
import { startSpeechToTextFromMic, stopRecognition } from './helper';
|
|
3
|
-
export function useSpeechToTextFromMic({ apiKey, apiSecret }) {
|
|
3
|
+
export function useSpeechToTextFromMic({ apiKey, apiSecret, autoClearTranscript = true, }) {
|
|
4
4
|
const [isRecording, setIsRecording] = useState(false);
|
|
5
5
|
const [transcriptArray, setTranscriptArray] = useState([]);
|
|
6
|
+
const [partialTranscript, setPartialTranscript] = useState('');
|
|
6
7
|
const [recognizer, setRecognizer] = useState();
|
|
7
8
|
const wakelock = useRef();
|
|
8
9
|
const historyRef = useRef([]);
|
|
@@ -25,7 +26,7 @@ export function useSpeechToTextFromMic({ apiKey, apiSecret }) {
|
|
|
25
26
|
clear();
|
|
26
27
|
await lockWakeState();
|
|
27
28
|
setIsRecording(true);
|
|
28
|
-
const recognizerInstance = await startSpeechToTextFromMic(setTranscriptArray, { apiKey, apiSecret }, historyRef, indexRef);
|
|
29
|
+
const recognizerInstance = await startSpeechToTextFromMic(setTranscriptArray, { apiKey, apiSecret }, historyRef, indexRef, setPartialTranscript);
|
|
29
30
|
setRecognizer(recognizerInstance);
|
|
30
31
|
};
|
|
31
32
|
const stop = () => {
|
|
@@ -38,7 +39,16 @@ export function useSpeechToTextFromMic({ apiKey, apiSecret }) {
|
|
|
38
39
|
historyRef.current = [];
|
|
39
40
|
indexRef.current = 0;
|
|
40
41
|
setTranscriptArray([]);
|
|
42
|
+
setPartialTranscript('');
|
|
41
43
|
};
|
|
44
|
+
useEffect(() => {
|
|
45
|
+
if (autoClearTranscript && transcriptArray.length > 0) {
|
|
46
|
+
const timer = setTimeout(() => {
|
|
47
|
+
setTranscriptArray([]);
|
|
48
|
+
}, 100);
|
|
49
|
+
return () => clearTimeout(timer);
|
|
50
|
+
}
|
|
51
|
+
}, [transcriptArray, autoClearTranscript]);
|
|
42
52
|
const transcript = transcriptArray.join(' ');
|
|
43
53
|
return {
|
|
44
54
|
start,
|
|
@@ -46,5 +56,6 @@ export function useSpeechToTextFromMic({ apiKey, apiSecret }) {
|
|
|
46
56
|
clear,
|
|
47
57
|
isRecording,
|
|
48
58
|
transcript,
|
|
59
|
+
partialTranscript,
|
|
49
60
|
};
|
|
50
61
|
}
|
package/dist/types/ait-api.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ait-api.d.ts","sourceRoot":"","sources":["../../src/types/ait-api.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE;QACT,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE;YACR,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAC5B,CAAC;QACF,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"ait-api.d.ts","sourceRoot":"","sources":["../../src/types/ait-api.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE;QACT,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE;YACR,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAC5B,CAAC;QACF,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;CACH;AAED,MAAM,WAAW,GAAG;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,WAAW,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,WAAW,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,oBAAqB,SAAQ,WAAW;CACxD;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,cAAc,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE;QACH,8BAA8B,EAAE,CAAC,MAAM,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAA;SAAE,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC3I,SAAS,EAAE,CAAC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAC7F,CAAC;IACF,MAAM,EAAE;QACN,YAAY,EAAE,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACrG,SAAS,EAAE,CAAC,MAAM,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACpG,CAAC;IACF,QAAQ,EAAE;QACR,cAAc,EAAE,CAAC,QAAQ,EAAE,oBAAoB,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACzH,CAAC;IACF,IAAI,EAAE;QACJ,QAAQ,EAAE,CAAC,MAAM,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC5G,MAAM,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,OAAO,CAAC;YAAE,WAAW,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACxF,CAAC;IACF,KAAK,EAAE;QACL,WAAW,EAAE,CAAC,MAAM,EAAE;YACpB,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,MAAM,EAAE,MAAM,CAAC;YACf,SAAS,EAAE,MAAM,CAAC;YAClB,QAAQ,EAAE,MAAM,CAAC;YACjB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACrC,OAAO,EAAE,MAAM,CAAC;YAChB,OAAO,CAAC,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;SACjD,KAAK,OAAO,CAAC;YACZ,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;gBAAE,IAAI,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;YACxC,QAAQ,CAAC,EAAE;gBACT,OAAO,EAAE;oBACP,IAAI,EAAE,MAAM,CAAC;oBACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;iBAC5B,CAAC;aACH,CAAC;YACF,YAAY,CAAC,EAAE;gBACb,MAAM,CAAC,EAAE;oBACP,OAAO,CAAC,EAAE;wBACR,OAAO,CAAC,EAAE,KAAK,CAAC;4BAAE,IAAI,EAAE,MAAM,CAAA;yBAAE,CAAC,CAAC;qBACnC,CAAC;iBACH,CAAC;aACH,CAAC;YACF,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACvB,mBAAmB,EAAE,CAAC,MAAM,EAAE;YAC5B,MAAM,EAAE,MAAM,CAAC;YACf,SAAS,EAAE,MAAM,CAAC;YAClB,QAAQ,EAAE,MAAM,CAAC;YACjB,UAAU,CAAC,EAAE,MAAM,CAAC;SACrB,KAAK,OAAO,CAAC;YAAE,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAClE,kCAAkC,EAAE,CAAC,MAAM,EAAE;YAC3C,MAAM,EAAE,MAAM,CAAC;YACf,SAAS,EAAE,MAAM,CAAC;YAClB,QAAQ,EAAE,MAAM,CAAC;YACjB,UAAU,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACvD,0BAA0B,EAAE,CAAC,MAAM,EAAE;YACnC,MAAM,EAAE,MAAM,CAAC;YACf,SAAS,EAAE,MAAM,CAAC;YAClB,QAAQ,EAAE,MAAM,CAAC;YACjB,UAAU,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACvD,wBAAwB,EAAE,CAAC,MAAM,EAAE;YACjC,MAAM,EAAE,MAAM,CAAC;YACf,SAAS,EAAE,MAAM,CAAC;YAClB,QAAQ,EAAE,MAAM,CAAC;YACjB,UAAU,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACxD,CAAC;IACF,WAAW,EAAE;QACX,qBAAqB,EAAE,CAAC,MAAM,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,SAAS,CAAC,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,WAAW,EAAE,iBAAiB,EAAE,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACjJ,CAAC;IACF,SAAS,EAAE;QACT,iBAAiB,EAAE,MAAM,OAAO,CAAC;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACzF,CAAC;CACH"}
|
package/package.json
CHANGED