handmux 0.26.0 → 0.27.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -93
- package/README.zh-CN.md +20 -93
- package/dist/package.json +19 -19
- package/dist/public/assets/index-B4pKf3IW.js +356 -0
- package/dist/public/assets/index-Cdrh2CUp.css +32 -0
- package/dist/public/assets/pcm-worklet-NzlBQ6-t.js +1 -0
- package/dist/public/index.html +2 -2
- package/dist/src/agent-runtime/adapter.js +2 -0
- package/dist/src/agent-runtime/builtinRuntime.js +3 -1
- package/dist/src/agent-runtime/conversationActivation.js +136 -4
- package/dist/src/agent-runtime/run.js +23 -2
- package/dist/src/agent-runtime/runtime.js +174 -15
- package/dist/src/agent-runtime/tmuxRuntime.js +3 -7
- package/dist/src/agents/codex.js +78 -10
- package/dist/src/agents/codexActivationReceipt.js +149 -0
- package/dist/src/agents/codexConversation.js +59 -20
- package/dist/src/agents/codexConversationActivation.js +265 -65
- package/dist/src/agents/codexOpenSession.js +164 -0
- package/dist/src/agents/nativeInbox.js +24 -56
- package/dist/src/agents/scanUtils.js +7 -1
- package/dist/src/apiErrors.js +10 -1
- package/dist/src/asr/config.js +29 -0
- package/dist/src/asr/providerRegistry.js +19 -0
- package/dist/src/asr/providers/tencent.js +135 -0
- package/dist/src/asr/providers/xfyun.js +73 -0
- package/dist/src/asr/tencentSentence.js +111 -0
- package/dist/src/asr/tencentSign.js +20 -0
- package/dist/src/asr/verify.js +80 -0
- package/dist/src/cli/codexManaged.js +13 -2
- package/dist/src/cli/i18n/en.js +16 -1
- package/dist/src/cli/i18n/zh.js +16 -1
- package/dist/src/cli/options.js +29 -2
- package/dist/src/cli/setupModel.js +49 -5
- package/dist/src/cli/setupWizard.js +149 -29
- package/dist/src/cli/supervisor.js +24 -8
- package/dist/src/cli/supervisorLaunch.js +26 -0
- package/dist/src/codexAppServer.js +41 -4
- package/dist/src/routes/agents.js +51 -14
- package/dist/src/routes/system.js +109 -10
- package/dist/src/server.js +11 -2
- package/dist/src/terminalStream.js +8 -30
- package/dist/src/tmux/commands.js +14 -0
- package/dist/src/tmux/controlProtocol.js +26 -0
- package/dist/src/tmux/paneOutputCapture.js +180 -0
- package/package.json +5 -5
- package/dist/public/assets/index-BVFKs5dN.css +0 -32
- package/dist/public/assets/index-KDa5PCoi.js +0 -356
- package/dist/public/assets/pcm-worklet-CxcDy7PB.js +0 -1
|
@@ -1,9 +1,3 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
function attachmentId(sourceId, candidate) {
|
|
3
|
-
return `${sourceId}:${createHash('sha256').update(JSON.stringify({
|
|
4
|
-
paneId: candidate.paneId, process: candidate.process,
|
|
5
|
-
})).digest('hex')}`;
|
|
6
|
-
}
|
|
7
1
|
function operation(sourceId, row) {
|
|
8
2
|
const base = {
|
|
9
3
|
source: { sourceId, cursor: row.cursor },
|
|
@@ -87,9 +81,7 @@ export class NativeInboxCoordinator {
|
|
|
87
81
|
clearTimeout(this.#timer);
|
|
88
82
|
this.#timer = undefined;
|
|
89
83
|
await this.#tail.catch(() => { });
|
|
90
|
-
const tracked = [...this.#tracked.values()];
|
|
91
84
|
this.#tracked.clear();
|
|
92
|
-
await Promise.all(tracked.map((entry) => (this.#context.runControl.revoke(entry.lease, 'adapter_stopped').catch(() => { }))));
|
|
93
85
|
}
|
|
94
86
|
reconcile() {
|
|
95
87
|
const pending = this.#tail.then(() => this.#reconcile());
|
|
@@ -151,8 +143,8 @@ export class NativeInboxCoordinator {
|
|
|
151
143
|
let topologyChanged = !this.#baselineReady;
|
|
152
144
|
const rows = new Map(snapshot.rows.map((row) => [row.paneId, structuredClone(row)]));
|
|
153
145
|
// Root Runtime independently revokes a run when its foreground-process proof changes. A coordinator
|
|
154
|
-
// entry must not outlive that lease: retaining it
|
|
155
|
-
//
|
|
146
|
+
// entry must not outlive that lease: retaining it feeds a revoked run into Inbox.restore() and blocks
|
|
147
|
+
// every healthy pane in the same provider snapshot.
|
|
156
148
|
for (const [paneId, tracked] of [...this.#tracked]) {
|
|
157
149
|
if (!tracked.lease.signal.aborted
|
|
158
150
|
&& this.#context.runs.resolve(tracked.lease.ref) === tracked.lease)
|
|
@@ -160,74 +152,50 @@ export class NativeInboxCoordinator {
|
|
|
160
152
|
this.#tracked.delete(paneId);
|
|
161
153
|
topologyChanged = true;
|
|
162
154
|
}
|
|
163
|
-
for (const [paneId
|
|
155
|
+
for (const [paneId] of [...this.#tracked]) {
|
|
164
156
|
if (rows.has(paneId))
|
|
165
157
|
continue;
|
|
166
158
|
this.#tracked.delete(paneId);
|
|
167
|
-
await this.#context.runControl.revoke(tracked.lease, 'adapter_stopped').catch(() => { });
|
|
168
159
|
topologyChanged = true;
|
|
169
160
|
}
|
|
170
|
-
let
|
|
161
|
+
let associationError;
|
|
171
162
|
for (const row of rows.values()) {
|
|
172
163
|
try {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
...(process.tty === undefined ? (pane.tty === undefined ? {} : { tty: pane.tty }) : { tty: process.tty }),
|
|
184
|
-
},
|
|
185
|
-
};
|
|
186
|
-
const nextAttachmentId = attachmentId(this.#sourceId, candidateBase);
|
|
187
|
-
const candidate = { ...candidateBase, attachmentId: nextAttachmentId };
|
|
188
|
-
let tracked = this.#tracked.get(row.paneId);
|
|
189
|
-
if (!tracked) {
|
|
190
|
-
const lease = await this.#context.runControl.attach(candidate);
|
|
191
|
-
tracked = {
|
|
192
|
-
lease, projector: this.#context.inbox.forRun(lease),
|
|
193
|
-
attachmentId: nextAttachmentId, row,
|
|
194
|
-
};
|
|
195
|
-
this.#tracked.set(row.paneId, tracked);
|
|
164
|
+
let lease = this.#context.currentRunForPane(row.paneId);
|
|
165
|
+
if (!lease || lease.ref.agentId !== this.#agentId
|
|
166
|
+
|| this.#context.runs.resolve(lease.ref) !== lease) {
|
|
167
|
+
// A stale socket/source row is not process ownership evidence. Runtime will retry process
|
|
168
|
+
// discovery independently; until then this row must not create or degrade a run.
|
|
169
|
+
rows.delete(row.paneId);
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
if (row.sessionId !== undefined && lease.ref.sessionId === undefined) {
|
|
173
|
+
await this.#context.runControl.associateSession(lease, row.sessionId);
|
|
196
174
|
topologyChanged = true;
|
|
197
175
|
}
|
|
198
|
-
else if (
|
|
199
|
-
|
|
200
|
-
tracked = {
|
|
201
|
-
lease, projector: this.#context.inbox.forRun(lease),
|
|
202
|
-
attachmentId: nextAttachmentId, row,
|
|
203
|
-
};
|
|
204
|
-
this.#tracked.set(row.paneId, tracked);
|
|
176
|
+
else if (row.sessionId !== undefined && lease.ref.sessionId !== row.sessionId) {
|
|
177
|
+
lease = await this.#context.runControl.replaceSession(lease, row.sessionId);
|
|
205
178
|
topologyChanged = true;
|
|
206
179
|
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
const lease = await this.#context.runControl.replace(tracked.lease, candidate, 'session_replaced');
|
|
180
|
+
let tracked = this.#tracked.get(row.paneId);
|
|
181
|
+
if (!tracked || tracked.lease !== lease) {
|
|
210
182
|
tracked = {
|
|
211
183
|
lease, projector: this.#context.inbox.forRun(lease),
|
|
212
|
-
|
|
184
|
+
row,
|
|
213
185
|
};
|
|
214
186
|
this.#tracked.set(row.paneId, tracked);
|
|
215
187
|
topologyChanged = true;
|
|
216
188
|
}
|
|
217
|
-
else if (tracked.lease.ref.sessionId === undefined && row.sessionId !== undefined) {
|
|
218
|
-
await this.#context.runControl.associateSession(tracked.lease, row.sessionId);
|
|
219
|
-
topologyChanged = true;
|
|
220
|
-
}
|
|
221
189
|
}
|
|
222
190
|
catch (error) {
|
|
223
|
-
// One
|
|
224
|
-
// previously
|
|
191
|
+
// One stale run/session association must not suppress every other pane from the same provider.
|
|
192
|
+
// Keep any previously projected row and retry after Runtime has reconciled process ownership.
|
|
225
193
|
rows.delete(row.paneId);
|
|
226
|
-
|
|
194
|
+
associationError ??= error instanceof Error ? error.message : String(error);
|
|
227
195
|
}
|
|
228
196
|
}
|
|
229
|
-
const effectiveAvailability =
|
|
230
|
-
const effectiveMessage =
|
|
197
|
+
const effectiveAvailability = associationError ? 'degraded' : snapshot.availability;
|
|
198
|
+
const effectiveMessage = associationError ?? snapshot.message;
|
|
231
199
|
const availabilityChanged = this.#availability !== effectiveAvailability
|
|
232
200
|
|| this.#availabilityMessage !== effectiveMessage;
|
|
233
201
|
const restoreNeeded = topologyChanged || availabilityChanged;
|
|
@@ -14,7 +14,13 @@ const isRecord = (value) => value !== null && typeof value === 'object';
|
|
|
14
14
|
// Detection is best-effort and must never throw the whole request.
|
|
15
15
|
export function defaultRun(cmd, args) {
|
|
16
16
|
return new Promise((resolve) => {
|
|
17
|
-
|
|
17
|
+
// These commands are parsed as machine output. Inherited locales can translate fields such as
|
|
18
|
+
// macOS `ps lstart` (for example Chinese weekday/month names), making otherwise stable output
|
|
19
|
+
// unparsable and leaving safety-critical process identities incomplete.
|
|
20
|
+
execFile(cmd, args, {
|
|
21
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
22
|
+
env: { ...process.env, LC_ALL: 'C' },
|
|
23
|
+
}, (err, stdout) => {
|
|
18
24
|
resolve(err ? '' : String(stdout));
|
|
19
25
|
});
|
|
20
26
|
});
|
package/dist/src/apiErrors.js
CHANGED
|
@@ -3,13 +3,19 @@ export class PublicApiError extends Error {
|
|
|
3
3
|
status;
|
|
4
4
|
code;
|
|
5
5
|
retryAfterSeconds;
|
|
6
|
-
|
|
6
|
+
providerRequestId;
|
|
7
|
+
logFields;
|
|
8
|
+
constructor(status, code, message, retryAfterSeconds, diagnostics = {}) {
|
|
7
9
|
super(message);
|
|
8
10
|
this.name = 'PublicApiError';
|
|
9
11
|
this.status = status;
|
|
10
12
|
this.code = code;
|
|
11
13
|
if (retryAfterSeconds !== undefined)
|
|
12
14
|
this.retryAfterSeconds = retryAfterSeconds;
|
|
15
|
+
if (diagnostics.providerRequestId)
|
|
16
|
+
this.providerRequestId = diagnostics.providerRequestId;
|
|
17
|
+
if (diagnostics.logFields)
|
|
18
|
+
this.logFields = diagnostics.logFields;
|
|
13
19
|
}
|
|
14
20
|
}
|
|
15
21
|
function requestIdFor(locals, idFactory) {
|
|
@@ -68,6 +74,7 @@ export function apiErrorBoundary({ idFactory = randomUUID, log = console, } = {}
|
|
|
68
74
|
}
|
|
69
75
|
if (status >= 500) {
|
|
70
76
|
log.error('[handmux] api error', {
|
|
77
|
+
...(error instanceof PublicApiError ? error.logFields : {}),
|
|
71
78
|
requestId,
|
|
72
79
|
method: req.method,
|
|
73
80
|
path: req.originalUrl,
|
|
@@ -80,6 +87,8 @@ export function apiErrorBoundary({ idFactory = randomUUID, log = console, } = {}
|
|
|
80
87
|
const body = {
|
|
81
88
|
error: message, code, requestId,
|
|
82
89
|
...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }),
|
|
90
|
+
...(error instanceof PublicApiError && error.providerRequestId
|
|
91
|
+
? { providerRequestId: error.providerRequestId } : {}),
|
|
83
92
|
};
|
|
84
93
|
res.status(status).json(body);
|
|
85
94
|
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { providerMode, voiceProviderRegistry } from './providerRegistry.js';
|
|
2
|
+
// HANDMUX_ASR_PROVIDER selects exactly one configured provider. Without it, keep legacy XFYUN_* installs
|
|
3
|
+
// working; a Tencent-only environment is also detected for direct server launches.
|
|
4
|
+
export function asrConfigFromRegistry(env, registry) {
|
|
5
|
+
const selected = env.HANDMUX_ASR_PROVIDER;
|
|
6
|
+
if (selected) {
|
|
7
|
+
const adapter = registry.get(selected);
|
|
8
|
+
if (!adapter)
|
|
9
|
+
return null;
|
|
10
|
+
const config = adapter.readConfig(env, providerMode(adapter, env.HANDMUX_ASR_MODE));
|
|
11
|
+
return adapter.isConfigured(config) ? config : null;
|
|
12
|
+
}
|
|
13
|
+
for (const provider of ['xfyun', ...registry.adapters
|
|
14
|
+
.map((adapter) => adapter.id).filter((id) => id !== 'xfyun')]) {
|
|
15
|
+
const adapter = registry.get(provider);
|
|
16
|
+
if (!adapter)
|
|
17
|
+
continue;
|
|
18
|
+
const config = adapter.readConfig(env, adapter.defaultMode);
|
|
19
|
+
if (adapter.isConfigured(config))
|
|
20
|
+
return config;
|
|
21
|
+
}
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
export function asrConfig(env = process.env) {
|
|
25
|
+
return asrConfigFromRegistry(env, voiceProviderRegistry);
|
|
26
|
+
}
|
|
27
|
+
export function isAsrConfigured(env = process.env) {
|
|
28
|
+
return asrConfig(env) !== null;
|
|
29
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { tencentAdapter } from './providers/tencent.js';
|
|
2
|
+
import { xfyunAdapter } from './providers/xfyun.js';
|
|
3
|
+
export function createVoiceProviderRegistry(adapters) {
|
|
4
|
+
const byId = new Map();
|
|
5
|
+
for (const adapter of adapters) {
|
|
6
|
+
if (!adapter.id || byId.has(adapter.id))
|
|
7
|
+
throw new Error(`duplicate voice provider: ${adapter.id}`);
|
|
8
|
+
byId.set(adapter.id, adapter);
|
|
9
|
+
}
|
|
10
|
+
return {
|
|
11
|
+
adapters: [...adapters],
|
|
12
|
+
get: (id) => typeof id === 'string' ? byId.get(id) : undefined,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export const voiceProviderRegistry = createVoiceProviderRegistry([tencentAdapter, xfyunAdapter]);
|
|
16
|
+
export function providerMode(adapter, value) {
|
|
17
|
+
return adapter.profiles.some((profile) => profile.mode === value)
|
|
18
|
+
? value : adapter.defaultMode;
|
|
19
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { recognizeTencentSentence, TencentSentenceEmptyResultError, TencentSentenceError, } from '../tencentSentence.js';
|
|
2
|
+
import { buildTencentAsrSignedUrl } from '../tencentSign.js';
|
|
3
|
+
import { silentPcmProbe, verifyStreamingConnection, VoiceVerificationError, } from '../verify.js';
|
|
4
|
+
const FILTER_MODAL = {
|
|
5
|
+
low: '0', medium: '1', high: '2',
|
|
6
|
+
};
|
|
7
|
+
async function verifyTencentStreamingConfig(config, dependencies) {
|
|
8
|
+
const session = buildTencentAsrSignedUrl({ ...config, filterModal: '1' });
|
|
9
|
+
await verifyStreamingConnection({
|
|
10
|
+
url: session.url,
|
|
11
|
+
sendOnOpen: (socket) => {
|
|
12
|
+
socket.send(silentPcmProbe().subarray(0, 1_280));
|
|
13
|
+
socket.send(JSON.stringify({ type: 'end' }));
|
|
14
|
+
},
|
|
15
|
+
acceptMessage: (message) => {
|
|
16
|
+
let value;
|
|
17
|
+
try {
|
|
18
|
+
value = JSON.parse(message);
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
throw new VoiceVerificationError('invalid_response', 'provider returned an invalid response');
|
|
22
|
+
}
|
|
23
|
+
const root = value && typeof value === 'object' ? value : {};
|
|
24
|
+
if (typeof root.code !== 'number') {
|
|
25
|
+
throw new VoiceVerificationError('invalid_response', 'provider returned an invalid response');
|
|
26
|
+
}
|
|
27
|
+
if (root.code !== 0) {
|
|
28
|
+
throw new VoiceVerificationError(`tencent_${root.code}`, `Tencent Cloud ASR rejected the configuration (${root.code}): ${String(root.message || 'unknown error')}`);
|
|
29
|
+
}
|
|
30
|
+
return true;
|
|
31
|
+
},
|
|
32
|
+
}, dependencies);
|
|
33
|
+
}
|
|
34
|
+
export async function verifyTencentAsr(config, dependencies = {}) {
|
|
35
|
+
const timeoutMs = dependencies.timeoutMs ?? 8_000;
|
|
36
|
+
const startedAt = Date.now();
|
|
37
|
+
// SentenceRecognition authenticates SecretId/SecretKey but has no AppId parameter. Probe the signed
|
|
38
|
+
// streaming endpoint first so a mistyped AppId cannot produce a false-positive sentence verification.
|
|
39
|
+
await verifyTencentStreamingConfig(config, { ...dependencies, timeoutMs });
|
|
40
|
+
if (config.mode === 'streaming') {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
|
44
|
+
if (remainingMs <= 0) {
|
|
45
|
+
throw new VoiceVerificationError('verification_timeout', 'provider verification timed out');
|
|
46
|
+
}
|
|
47
|
+
const controller = new AbortController();
|
|
48
|
+
const timeout = setTimeout(() => controller.abort(), remainingMs);
|
|
49
|
+
try {
|
|
50
|
+
await recognizeTencentSentence(config, silentPcmProbe(), {
|
|
51
|
+
...(dependencies.fetch ? { fetch: dependencies.fetch } : {}),
|
|
52
|
+
filterModal: 1, signal: controller.signal,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
if (controller.signal.aborted) {
|
|
57
|
+
throw new VoiceVerificationError('verification_timeout', 'provider verification timed out');
|
|
58
|
+
}
|
|
59
|
+
// The verification probe is deliberately silent. An empty recognition proves the signed request,
|
|
60
|
+
// credentials, permission, and model were all accepted, so it is a successful configuration check.
|
|
61
|
+
if (error instanceof TencentSentenceEmptyResultError)
|
|
62
|
+
return;
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
clearTimeout(timeout);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
export const tencentAdapter = {
|
|
70
|
+
id: 'tencent',
|
|
71
|
+
defaultMode: 'streaming',
|
|
72
|
+
fields: [
|
|
73
|
+
{ key: 'appId', env: 'TENCENT_ASR_APPID', promptKey: 'setup.voiceTencentAppId' },
|
|
74
|
+
{ key: 'secretId', env: 'TENCENT_ASR_SECRET_ID', promptKey: 'setup.voiceTencentSecretId', secret: true },
|
|
75
|
+
{ key: 'secretKey', env: 'TENCENT_ASR_SECRET_KEY', promptKey: 'setup.voiceTencentSecretKey', secret: true },
|
|
76
|
+
{
|
|
77
|
+
key: 'engineModelType', env: 'TENCENT_ASR_ENGINE_MODEL_TYPE',
|
|
78
|
+
promptKey: 'setup.voiceTencentEngine', defaultValue: '16k_zh',
|
|
79
|
+
},
|
|
80
|
+
],
|
|
81
|
+
profiles: [
|
|
82
|
+
{ mode: 'streaming', labelKey: 'setup.voiceTencent', hintKey: 'setup.voiceTencentHint' },
|
|
83
|
+
{
|
|
84
|
+
mode: 'sentence', labelKey: 'setup.voiceTencentSentence',
|
|
85
|
+
hintKey: 'setup.voiceTencentSentenceHint',
|
|
86
|
+
},
|
|
87
|
+
],
|
|
88
|
+
capabilities: { fillerFilter: true },
|
|
89
|
+
readConfig: (env, mode) => ({
|
|
90
|
+
provider: 'tencent', mode,
|
|
91
|
+
appId: env.TENCENT_ASR_APPID || '',
|
|
92
|
+
secretId: env.TENCENT_ASR_SECRET_ID || '',
|
|
93
|
+
secretKey: env.TENCENT_ASR_SECRET_KEY || '',
|
|
94
|
+
engineModelType: env.TENCENT_ASR_ENGINE_MODEL_TYPE || '16k_zh',
|
|
95
|
+
}),
|
|
96
|
+
isConfigured: (config) => config.provider === 'tencent'
|
|
97
|
+
&& !!(config.appId && config.secretId && config.secretKey && config.engineModelType),
|
|
98
|
+
verify: (config) => {
|
|
99
|
+
if (config.provider !== 'tencent')
|
|
100
|
+
throw new Error('tencent adapter received another provider config');
|
|
101
|
+
return verifyTencentAsr(config);
|
|
102
|
+
},
|
|
103
|
+
createStreamingSession: (config, options) => {
|
|
104
|
+
if (config.provider !== 'tencent')
|
|
105
|
+
throw new Error('tencent adapter received another provider config');
|
|
106
|
+
return {
|
|
107
|
+
provider: 'tencent', protocol: 'tencent-asr-v2',
|
|
108
|
+
...buildTencentAsrSignedUrl({
|
|
109
|
+
...config,
|
|
110
|
+
filterModal: FILTER_MODAL[options.fillerFilter],
|
|
111
|
+
}),
|
|
112
|
+
};
|
|
113
|
+
},
|
|
114
|
+
recognizeSentence: (config, audio, options) => {
|
|
115
|
+
if (config.provider !== 'tencent')
|
|
116
|
+
throw new Error('tencent adapter received another provider config');
|
|
117
|
+
return recognizeTencentSentence(config, audio, {
|
|
118
|
+
filterModal: Number(FILTER_MODAL[options.fillerFilter]),
|
|
119
|
+
});
|
|
120
|
+
},
|
|
121
|
+
publicError: (error) => error instanceof TencentSentenceError ? {
|
|
122
|
+
message: error.message,
|
|
123
|
+
code: error.code,
|
|
124
|
+
...(error.requestId ? { providerRequestId: error.requestId } : {}),
|
|
125
|
+
...(error instanceof TencentSentenceEmptyResultError ? {
|
|
126
|
+
logFields: {
|
|
127
|
+
provider: 'tencent',
|
|
128
|
+
mode: 'sentence',
|
|
129
|
+
...(error.requestId ? { providerRequestId: error.requestId } : {}),
|
|
130
|
+
audioByteLength: error.audioByteLength,
|
|
131
|
+
audioDurationMs: error.audioDurationMs,
|
|
132
|
+
},
|
|
133
|
+
} : {}),
|
|
134
|
+
} : null,
|
|
135
|
+
};
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { buildIatSignedUrl } from '../iflySign.js';
|
|
2
|
+
import { silentPcmProbe, verifyStreamingConnection, VoiceVerificationError, } from '../verify.js';
|
|
3
|
+
const XFYUN_BUSINESS = {
|
|
4
|
+
language: 'zh_cn', domain: 'iat', accent: 'mandarin', vad_eos: 10_000,
|
|
5
|
+
dwa: 'wpgs', ptt: 1, nunum: 1,
|
|
6
|
+
};
|
|
7
|
+
const XFYUN_FORMAT = 'audio/L16;rate=16000';
|
|
8
|
+
export async function verifyXfyunAsr(config, dependencies = {}) {
|
|
9
|
+
const session = buildIatSignedUrl({ ...config, date: new Date().toUTCString() });
|
|
10
|
+
const audio = Buffer.from(silentPcmProbe().subarray(0, 1_280)).toString('base64');
|
|
11
|
+
await verifyStreamingConnection({
|
|
12
|
+
url: session.url,
|
|
13
|
+
sendOnOpen: (socket) => {
|
|
14
|
+
socket.send(JSON.stringify({
|
|
15
|
+
common: { app_id: config.appId },
|
|
16
|
+
business: XFYUN_BUSINESS,
|
|
17
|
+
data: { status: 0, format: XFYUN_FORMAT, encoding: 'raw', audio },
|
|
18
|
+
}));
|
|
19
|
+
socket.send(JSON.stringify({
|
|
20
|
+
data: { status: 2, format: XFYUN_FORMAT, encoding: 'raw', audio: '' },
|
|
21
|
+
}));
|
|
22
|
+
},
|
|
23
|
+
acceptMessage: (message) => {
|
|
24
|
+
let value;
|
|
25
|
+
try {
|
|
26
|
+
value = JSON.parse(message);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
throw new VoiceVerificationError('invalid_response', 'provider returned an invalid response');
|
|
30
|
+
}
|
|
31
|
+
const root = value && typeof value === 'object' ? value : {};
|
|
32
|
+
if (typeof root.code !== 'number') {
|
|
33
|
+
throw new VoiceVerificationError('invalid_response', 'provider returned an invalid response');
|
|
34
|
+
}
|
|
35
|
+
if (root.code !== 0) {
|
|
36
|
+
throw new VoiceVerificationError(`xfyun_${root.code}`, `XFYUN rejected the configuration (${root.code}): ${String(root.message || 'unknown error')}`);
|
|
37
|
+
}
|
|
38
|
+
return true;
|
|
39
|
+
},
|
|
40
|
+
}, dependencies);
|
|
41
|
+
}
|
|
42
|
+
export const xfyunAdapter = {
|
|
43
|
+
id: 'xfyun',
|
|
44
|
+
defaultMode: 'streaming',
|
|
45
|
+
fields: [
|
|
46
|
+
{ key: 'appId', env: 'XFYUN_APPID', promptKey: 'setup.voiceAppId' },
|
|
47
|
+
{ key: 'apiKey', env: 'XFYUN_APIKEY', promptKey: 'setup.voiceApiKey', secret: true },
|
|
48
|
+
{ key: 'apiSecret', env: 'XFYUN_APISECRET', promptKey: 'setup.voiceApiSecret', secret: true },
|
|
49
|
+
],
|
|
50
|
+
profiles: [
|
|
51
|
+
{ mode: 'streaming', labelKey: 'setup.voiceXfyun', hintKey: 'setup.voiceXfyunHint' },
|
|
52
|
+
],
|
|
53
|
+
readConfig: (env) => ({
|
|
54
|
+
provider: 'xfyun', mode: 'streaming',
|
|
55
|
+
appId: env.XFYUN_APPID || '', apiKey: env.XFYUN_APIKEY || '', apiSecret: env.XFYUN_APISECRET || '',
|
|
56
|
+
}),
|
|
57
|
+
isConfigured: (config) => config.provider === 'xfyun'
|
|
58
|
+
&& !!(config.appId && config.apiKey && config.apiSecret),
|
|
59
|
+
verify: (config) => {
|
|
60
|
+
if (config.provider !== 'xfyun')
|
|
61
|
+
throw new Error('xfyun adapter received another provider config');
|
|
62
|
+
return verifyXfyunAsr(config);
|
|
63
|
+
},
|
|
64
|
+
createStreamingSession: (config) => {
|
|
65
|
+
if (config.provider !== 'xfyun')
|
|
66
|
+
throw new Error('xfyun adapter received another provider config');
|
|
67
|
+
const credentials = config;
|
|
68
|
+
return {
|
|
69
|
+
provider: 'xfyun', protocol: 'xfyun-iat-v2',
|
|
70
|
+
...buildIatSignedUrl({ ...credentials, date: new Date().toUTCString() }),
|
|
71
|
+
};
|
|
72
|
+
},
|
|
73
|
+
};
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { createHash, createHmac } from 'node:crypto';
|
|
2
|
+
const HOST = 'asr.tencentcloudapi.com';
|
|
3
|
+
const ENDPOINT = `https://${HOST}`;
|
|
4
|
+
const ACTION = 'SentenceRecognition';
|
|
5
|
+
const VERSION = '2019-06-14';
|
|
6
|
+
const SERVICE = 'asr';
|
|
7
|
+
const ALGORITHM = 'TC3-HMAC-SHA256';
|
|
8
|
+
const CONTENT_TYPE = 'application/json; charset=utf-8';
|
|
9
|
+
export class TencentSentenceError extends Error {
|
|
10
|
+
code;
|
|
11
|
+
requestId;
|
|
12
|
+
constructor(code, message, requestId) {
|
|
13
|
+
super(message);
|
|
14
|
+
this.name = 'TencentSentenceError';
|
|
15
|
+
this.code = code;
|
|
16
|
+
if (requestId)
|
|
17
|
+
this.requestId = requestId;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export class TencentSentenceEmptyResultError extends TencentSentenceError {
|
|
21
|
+
audioByteLength;
|
|
22
|
+
audioDurationMs;
|
|
23
|
+
constructor(audioByteLength, requestId) {
|
|
24
|
+
super('speech_not_recognized', 'Tencent Cloud ASR did not recognize any speech', requestId);
|
|
25
|
+
this.name = 'TencentSentenceEmptyResultError';
|
|
26
|
+
this.audioByteLength = audioByteLength;
|
|
27
|
+
// SentenceRecognition receives 16 kHz, mono, signed 16-bit PCM: 32 bytes per millisecond.
|
|
28
|
+
this.audioDurationMs = audioByteLength / 32;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
const isRecord = (value) => (value !== null && typeof value === 'object' && !Array.isArray(value));
|
|
32
|
+
const sha256 = (value) => createHash('sha256').update(value).digest('hex');
|
|
33
|
+
const hmac = (key, value) => (createHmac('sha256', key).update(value).digest());
|
|
34
|
+
function utcDate(timestamp) {
|
|
35
|
+
return new Date(timestamp * 1000).toISOString().slice(0, 10);
|
|
36
|
+
}
|
|
37
|
+
export function buildTencentSentenceRequest(config, audio, timestamp = Math.floor(Date.now() / 1000), filterModal = 1) {
|
|
38
|
+
const body = JSON.stringify({
|
|
39
|
+
ProjectId: 0,
|
|
40
|
+
SubServiceType: 2,
|
|
41
|
+
EngSerViceType: config.engineModelType,
|
|
42
|
+
SourceType: 1,
|
|
43
|
+
VoiceFormat: 'pcm',
|
|
44
|
+
FilterModal: filterModal,
|
|
45
|
+
Data: Buffer.from(audio).toString('base64'),
|
|
46
|
+
DataLen: audio.byteLength,
|
|
47
|
+
});
|
|
48
|
+
const canonicalHeaders = `content-type:${CONTENT_TYPE}\nhost:${HOST}\n`;
|
|
49
|
+
const signedHeaders = 'content-type;host';
|
|
50
|
+
const canonicalRequest = [
|
|
51
|
+
'POST', '/', '', canonicalHeaders, signedHeaders, sha256(body),
|
|
52
|
+
].join('\n');
|
|
53
|
+
const date = utcDate(timestamp);
|
|
54
|
+
const credentialScope = `${date}/${SERVICE}/tc3_request`;
|
|
55
|
+
const stringToSign = [
|
|
56
|
+
ALGORITHM, String(timestamp), credentialScope, sha256(canonicalRequest),
|
|
57
|
+
].join('\n');
|
|
58
|
+
const secretDate = hmac(`TC3${config.secretKey}`, date);
|
|
59
|
+
const secretService = hmac(secretDate, SERVICE);
|
|
60
|
+
const secretSigning = hmac(secretService, 'tc3_request');
|
|
61
|
+
const signature = createHmac('sha256', secretSigning).update(stringToSign).digest('hex');
|
|
62
|
+
const authorization = `${ALGORITHM} Credential=${config.secretId}/${credentialScope}, `
|
|
63
|
+
+ `SignedHeaders=${signedHeaders}, Signature=${signature}`;
|
|
64
|
+
return {
|
|
65
|
+
url: ENDPOINT,
|
|
66
|
+
headers: {
|
|
67
|
+
Authorization: authorization,
|
|
68
|
+
'Content-Type': CONTENT_TYPE,
|
|
69
|
+
Host: HOST,
|
|
70
|
+
'X-TC-Action': ACTION,
|
|
71
|
+
'X-TC-Timestamp': String(timestamp),
|
|
72
|
+
'X-TC-Version': VERSION,
|
|
73
|
+
},
|
|
74
|
+
body,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
export async function recognizeTencentSentence(config, audio, { fetch: fetchImpl = fetch, timestamp, filterModal = 1, signal } = {}) {
|
|
78
|
+
const request = buildTencentSentenceRequest(config, audio, timestamp, filterModal);
|
|
79
|
+
let response;
|
|
80
|
+
try {
|
|
81
|
+
response = await fetchImpl(request.url, {
|
|
82
|
+
method: 'POST', headers: request.headers, body: request.body,
|
|
83
|
+
...(signal ? { signal } : {}),
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
throw new TencentSentenceError('TencentNetworkError', `Tencent Cloud ASR could not be reached: ${error instanceof Error ? error.message : String(error)}`);
|
|
88
|
+
}
|
|
89
|
+
let value;
|
|
90
|
+
try {
|
|
91
|
+
value = await response.json();
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
throw new TencentSentenceError('TencentInvalidResponse', `Tencent Cloud ASR returned HTTP ${response.status} with an invalid response`);
|
|
95
|
+
}
|
|
96
|
+
const root = isRecord(value) && isRecord(value.Response) ? value.Response : null;
|
|
97
|
+
const requestId = typeof root?.RequestId === 'string' ? root.RequestId : undefined;
|
|
98
|
+
const cloudError = isRecord(root?.Error) ? root.Error : null;
|
|
99
|
+
if (cloudError) {
|
|
100
|
+
const code = typeof cloudError.Code === 'string' ? cloudError.Code : 'TencentCloudError';
|
|
101
|
+
const message = typeof cloudError.Message === 'string' ? cloudError.Message : 'Tencent Cloud ASR rejected the request';
|
|
102
|
+
throw new TencentSentenceError(code, message, requestId);
|
|
103
|
+
}
|
|
104
|
+
if (!response.ok || !root || typeof root.Result !== 'string') {
|
|
105
|
+
throw new TencentSentenceError('TencentInvalidResponse', `Tencent Cloud ASR returned an invalid response (HTTP ${response.status})`, requestId);
|
|
106
|
+
}
|
|
107
|
+
if (root.Result.trim() === '') {
|
|
108
|
+
throw new TencentSentenceEmptyResultError(audio.byteLength, requestId);
|
|
109
|
+
}
|
|
110
|
+
return { text: root.Result, ...(requestId ? { requestId } : {}) };
|
|
111
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { createHmac, randomInt, randomUUID } from 'node:crypto';
|
|
2
|
+
// Tencent Real-time ASR WebSocket 2.0. The signature covers host/path and the key-sorted raw query;
|
|
3
|
+
// only the resulting short-lived URL leaves the server.
|
|
4
|
+
export function buildTencentAsrSignedUrl({ appId, secretId, secretKey, engineModelType = '16k_zh', filterModal = '1', host = 'asr.cloud.tencent.com', timestamp = Math.floor(Date.now() / 1000), expired = timestamp + 300, nonce = randomInt(100000, 1000000), voiceId = randomUUID(), }) {
|
|
5
|
+
const params = {
|
|
6
|
+
engine_model_type: engineModelType,
|
|
7
|
+
expired: String(expired),
|
|
8
|
+
filter_modal: filterModal,
|
|
9
|
+
nonce: String(nonce),
|
|
10
|
+
secretid: secretId,
|
|
11
|
+
speaker_diarization: '0',
|
|
12
|
+
timestamp: String(timestamp),
|
|
13
|
+
voice_format: '1',
|
|
14
|
+
voice_id: voiceId,
|
|
15
|
+
};
|
|
16
|
+
const query = Object.keys(params).sort().map((key) => `${key}=${params[key]}`).join('&');
|
|
17
|
+
const signingText = `${host}/asr/v2/${appId}?${query}`;
|
|
18
|
+
const signature = createHmac('sha1', secretKey).update(signingText).digest('base64');
|
|
19
|
+
return { url: `wss://${signingText}&signature=${encodeURIComponent(signature)}` };
|
|
20
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import WebSocket from 'ws';
|
|
2
|
+
export class VoiceVerificationError extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(code, message) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = 'VoiceVerificationError';
|
|
7
|
+
this.code = code;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
const socketMessage = (data) => {
|
|
11
|
+
if (typeof data === 'string')
|
|
12
|
+
return data;
|
|
13
|
+
if (Buffer.isBuffer(data))
|
|
14
|
+
return data.toString('utf8');
|
|
15
|
+
if (data instanceof ArrayBuffer)
|
|
16
|
+
return Buffer.from(data).toString('utf8');
|
|
17
|
+
if (ArrayBuffer.isView(data)) {
|
|
18
|
+
return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString('utf8');
|
|
19
|
+
}
|
|
20
|
+
if (Array.isArray(data) && data.every(Buffer.isBuffer))
|
|
21
|
+
return Buffer.concat(data).toString('utf8');
|
|
22
|
+
return String(data);
|
|
23
|
+
};
|
|
24
|
+
// Opens the provider's real signed endpoint with a short, hard timeout. Provider adapters own the frames
|
|
25
|
+
// and response semantics; this helper owns only the shared socket lifecycle and never includes the signed URL
|
|
26
|
+
// in an error, so credentials and signed query material cannot leak into setup output.
|
|
27
|
+
export function verifyStreamingConnection(verification, { socketFactory = (url) => new WebSocket(url), timeoutMs = 8_000 } = {}) {
|
|
28
|
+
return new Promise((resolve, reject) => {
|
|
29
|
+
let settled = false;
|
|
30
|
+
let socket;
|
|
31
|
+
const finish = (error) => {
|
|
32
|
+
if (settled)
|
|
33
|
+
return;
|
|
34
|
+
settled = true;
|
|
35
|
+
clearTimeout(timer);
|
|
36
|
+
try {
|
|
37
|
+
socket?.terminate();
|
|
38
|
+
}
|
|
39
|
+
catch { /* already closed */ }
|
|
40
|
+
if (error)
|
|
41
|
+
reject(error);
|
|
42
|
+
else
|
|
43
|
+
resolve();
|
|
44
|
+
};
|
|
45
|
+
const timer = setTimeout(() => finish(new VoiceVerificationError('verification_timeout', 'provider verification timed out')), timeoutMs);
|
|
46
|
+
try {
|
|
47
|
+
socket = socketFactory(verification.url);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
finish(new VoiceVerificationError('connection_failed', 'could not open provider connection'));
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
socket.on('open', () => {
|
|
54
|
+
try {
|
|
55
|
+
verification.sendOnOpen?.(socket);
|
|
56
|
+
if (verification.successOnOpen)
|
|
57
|
+
finish();
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
finish(new VoiceVerificationError('probe_failed', 'could not send provider verification data'));
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
socket.on('message', (data) => {
|
|
64
|
+
try {
|
|
65
|
+
if (verification.acceptMessage?.(socketMessage(data)))
|
|
66
|
+
finish();
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
finish(error instanceof Error ? error
|
|
70
|
+
: new VoiceVerificationError('provider_rejected', 'provider rejected the configuration'));
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
socket.on('unexpected-response', (_request, response) => finish(new VoiceVerificationError('handshake_rejected', `provider rejected the WebSocket handshake${response.statusCode ? ` (HTTP ${response.statusCode})` : ''}`)));
|
|
74
|
+
socket.on('error', () => finish(new VoiceVerificationError('connection_failed', 'could not connect to the provider')));
|
|
75
|
+
socket.on('close', (code) => finish(new VoiceVerificationError('connection_closed', `provider closed the verification connection (code ${code})`)));
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
// One second of 16 kHz mono signed 16-bit PCM. It contains no user audio and forms a valid request body
|
|
79
|
+
// while keeping validation cost and transfer size minimal.
|
|
80
|
+
export const silentPcmProbe = () => new Uint8Array(16_000 * 2);
|