@oadank/dsh-input-tools 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -1
- package/lib/client.js +157 -18
- package/lib/index.js +145 -14
- package/package.json +6 -1
- package/patches/apply-voice-patch.ps1 +112 -0
- package/patches/dsh-voice-rc8.patch +3781 -0
|
@@ -0,0 +1,3781 @@
|
|
|
1
|
+
diff --git a/packages/api/remotes/src/client/index.ts b/packages/api/remotes/src/client/index.ts
|
|
2
|
+
index 6a5164f160..f5a57a5e65 100644
|
|
3
|
+
--- a/packages/api/remotes/src/client/index.ts
|
|
4
|
+
+++ b/packages/api/remotes/src/client/index.ts
|
|
5
|
+
@@ -44,6 +44,7 @@ export type {
|
|
6
|
+
RpcRequest, RpcResponse, RpcResult, SessionId, SessionModels, SessionSearchItem,
|
|
7
|
+
SessionSummary, SettingsNamespaceView, SettingsPathOpView, SkillEntry, StreamChunk,
|
|
8
|
+
SubagentAddress, SubagentCatalog, JobView, ToolCallView, ToolEventView, ToolResultView,
|
|
9
|
+
+ VoiceAttachmentRef, VoiceMediaType,
|
|
10
|
+
WorkspaceId, WorkspaceView,
|
|
11
|
+
} from '@deepseek-ai/dsh-client-connection/client'
|
|
12
|
+
export type {} from '@deepseek-ai/dsh-api-gateway/client'
|
|
13
|
+
diff --git a/packages/attachment/attachment/src/error.ts b/packages/attachment/attachment/src/error.ts
|
|
14
|
+
index 2e2d695dae..c91a4f862f 100644
|
|
15
|
+
--- a/packages/attachment/attachment/src/error.ts
|
|
16
|
+
+++ b/packages/attachment/attachment/src/error.ts
|
|
17
|
+
@@ -23,6 +23,7 @@ export type AttachmentErrorCode =
|
|
18
|
+
| 'ATTACHMENT_WRITE_FAILED'
|
|
19
|
+
| 'ATTACHMENT_NOT_FOUND'
|
|
20
|
+
| 'ATTACHMENT_READ_FAILED'
|
|
21
|
+
+ | 'VOICE_ASR_FAILED'
|
|
22
|
+
|
|
23
|
+
/** Runtime membership for structurally compatible errors crossing package boundaries. */
|
|
24
|
+
const IMAGE_ADMISSION_ERROR_CODE_SET: ReadonlySet<string> = new Set(IMAGE_ADMISSION_ERROR_CODES)
|
|
25
|
+
diff --git a/packages/client/connection/src/api-request-trust.ts b/packages/client/connection/src/api-request-trust.ts
|
|
26
|
+
index ea8914ccc6..1985ca7d08 100644
|
|
27
|
+
--- a/packages/client/connection/src/api-request-trust.ts
|
|
28
|
+
+++ b/packages/client/connection/src/api-request-trust.ts
|
|
29
|
+
@@ -105,6 +105,21 @@ export function isTrustedApiRequest(request: ApiTrustRequest, trustedHosts: read
|
|
30
|
+
if (host === undefined) return false
|
|
31
|
+
const hostUrl = parseAuthority(host)
|
|
32
|
+
if (hostUrl === undefined) return false
|
|
33
|
+
+ // [本地改造 2026-08-14] tailnet 域名(Tailscale *.ts.net)视为可信:
|
|
34
|
+
+ // 本机通过 tailscale serve 转发(https://lecoo.tailb5f10f.ts.net → 127.0.0.1:3080)
|
|
35
|
+
+ // 访问时 Host/Origin 是 *.ts.net,trustedHosts 参数在 nssm 环境下注入失败,
|
|
36
|
+
+ // 直接信任 tailnet 域名(仅本 tailnet 可达,非公开攻击面)。
|
|
37
|
+
+ if (hostUrl.hostname.endsWith('.ts.net')) {
|
|
38
|
+
+ const origin = header(request.headers, 'origin')
|
|
39
|
+
+ if (origin !== undefined) {
|
|
40
|
+
+ try {
|
|
41
|
+
+ if (new URL(origin).hostname.endsWith('.ts.net')) return true
|
|
42
|
+
+ } catch {
|
|
43
|
+
+ return false
|
|
44
|
+
+ }
|
|
45
|
+
+ }
|
|
46
|
+
+ return true
|
|
47
|
+
+ }
|
|
48
|
+
if (!isLoopbackHostname(hostUrl.hostname) && !isTrustedAuthority(hostUrl, trustedHosts)) return false
|
|
49
|
+
// Cross-site fence: modern browsers label the initiator relationship on
|
|
50
|
+
// every fetch; an explicit cross-site marker is refused regardless of Origin.
|
|
51
|
+
diff --git a/packages/client/connection/src/client/api.ts b/packages/client/connection/src/client/api.ts
|
|
52
|
+
index 1b7627b293..c89661c80d 100644
|
|
53
|
+
--- a/packages/client/connection/src/client/api.ts
|
|
54
|
+
+++ b/packages/client/connection/src/client/api.ts
|
|
55
|
+
@@ -16,6 +16,7 @@ export type {
|
|
56
|
+
GoalsApi, GoalRef,
|
|
57
|
+
SettingsApi, SettingsNamespaceView, SettingsPathOpView, SettingsSecretView,
|
|
58
|
+
CredentialsApi, CredentialView, ConfigurableProviderView, DiscoveredModelView, LlmApi,
|
|
59
|
+
+ BalanceApi, BalanceView, VoiceAttachmentRef, VoiceMediaType,
|
|
60
|
+
SubagentsApi, SubagentAddress, SubagentCatalog, SubagentListEntry, SubagentPromptReceipt,
|
|
61
|
+
JobView,
|
|
62
|
+
} from '@deepseek-ai/dsh-host-apiproxy/api'
|
|
63
|
+
diff --git a/packages/client/connection/src/client/index.ts b/packages/client/connection/src/client/index.ts
|
|
64
|
+
index 9847d48cdf..a398bfdafd 100644
|
|
65
|
+
--- a/packages/client/connection/src/client/index.ts
|
|
66
|
+
+++ b/packages/client/connection/src/client/index.ts
|
|
67
|
+
@@ -29,6 +29,7 @@ export type {
|
|
68
|
+
GoalsApi, GoalRef,
|
|
69
|
+
SettingsApi, SettingsNamespaceView, SettingsPathOpView, SettingsSecretView,
|
|
70
|
+
CredentialsApi, CredentialView, ConfigurableProviderView, DiscoveredModelView, LlmApi,
|
|
71
|
+
+ BalanceApi, BalanceView, VoiceAttachmentRef, VoiceMediaType,
|
|
72
|
+
} from './api.ts'
|
|
73
|
+
export {
|
|
74
|
+
RpcId,
|
|
75
|
+
@@ -103,7 +104,12 @@ export function apply(ctx: Context): void {
|
|
76
|
+
}
|
|
77
|
+
const handle: ConnectionHandle = {
|
|
78
|
+
api,
|
|
79
|
+
- isLoopback: pageLocation === undefined || isLoopbackHostname(pageLocation.hostname),
|
|
80
|
+
+ // [本地改造 2026-08-15] tailnet 域名(*.ts.net)与回环同等视为可信持久化来源:
|
|
81
|
+
+ // host 端 fence 已信任 ts.net(见 api-request-trust.ts),此处同步放宽,
|
|
82
|
+
+ // 否则远程浏览器(tailscale serve 访问)settings 走 memory 模式,
|
|
83
|
+
+ // busyEnter 等设置刷新即丢(用户选"插队"后回默认"排队")。
|
|
84
|
+
+ isLoopback: pageLocation === undefined || isLoopbackHostname(pageLocation.hostname)
|
|
85
|
+
+ || (typeof pageLocation.hostname === 'string' && pageLocation.hostname.endsWith('.ts.net')),
|
|
86
|
+
hostDescription: {
|
|
87
|
+
getSnapshot: () => description,
|
|
88
|
+
subscribe: (listener) => {
|
|
89
|
+
diff --git a/packages/client/runtime/src/client/contract/session.ts b/packages/client/runtime/src/client/contract/session.ts
|
|
90
|
+
index e07267d487..c47da28c4a 100644
|
|
91
|
+
--- a/packages/client/runtime/src/client/contract/session.ts
|
|
92
|
+
+++ b/packages/client/runtime/src/client/contract/session.ts
|
|
93
|
+
@@ -9,7 +9,7 @@
|
|
94
|
+
*/
|
|
95
|
+
import type { AttachmentIdType, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
|
96
|
+
import type {
|
|
97
|
+
- MessageId, PromptContentPart, QueueAction, RpcResult, SessionId,
|
|
98
|
+
+ MessageId, PromptContentPart, QueueAction, RpcResult, SessionId, VoiceAttachmentRef,
|
|
99
|
+
} from '@deepseek-ai/dsh-api-remotes/client'
|
|
100
|
+
import type { RemoteResult } from '@deepseek-ai/dsh-typert-protocol'
|
|
101
|
+
import type { ConversationSnapshot } from '../sessions/conversation.ts'
|
|
102
|
+
@@ -51,6 +51,14 @@ export interface ISession {
|
|
103
|
+
readAttachment(
|
|
104
|
+
attachmentId: AttachmentIdType,
|
|
105
|
+
): Promise<RpcResult<{ attachment: ImageAttachmentRef; data: Uint8Array }>>
|
|
106
|
+
+ /**
|
|
107
|
+
+ * Resolve one durable voice object referenced by this session.
|
|
108
|
+
+ * @param voiceId - opaque id found in the folded session log.
|
|
109
|
+
+ * @returns the authenticated reference and decoded bytes.
|
|
110
|
+
+ */
|
|
111
|
+
+ readVoice(
|
|
112
|
+
+ voiceId: string,
|
|
113
|
+
+ ): Promise<RpcResult<{ attachment: VoiceAttachmentRef; data: Uint8Array }>>
|
|
114
|
+
/**
|
|
115
|
+
* Apply one edit, remove, or strict steer action to a still-pending queue occurrence.
|
|
116
|
+
* @param itemId - agent-owned inbox occurrence identity.
|
|
117
|
+
diff --git a/packages/client/runtime/src/client/sessions/session.ts b/packages/client/runtime/src/client/sessions/session.ts
|
|
118
|
+
index 02939cf28f..06b447411e 100644
|
|
119
|
+
--- a/packages/client/runtime/src/client/sessions/session.ts
|
|
120
|
+
+++ b/packages/client/runtime/src/client/sessions/session.ts
|
|
121
|
+
@@ -5,7 +5,7 @@ import type { AttachmentIdType, ImageAttachmentRef } from '@deepseek-ai/dsh-atta
|
|
122
|
+
import type { SessionEvent } from '@deepseek-ai/dsh-session/types'
|
|
123
|
+
import type {
|
|
124
|
+
HistoryEntry, IApiClient, MessageId, MuxFrame, PromptContentPart, QueueAction, RpcError,
|
|
125
|
+
- RpcId, RpcResponse, RpcResult, SessionId, SubagentAddress, ToolEventView,
|
|
126
|
+
+ RpcId, RpcResponse, RpcResult, SessionId, SubagentAddress, ToolEventView, VoiceAttachmentRef,
|
|
127
|
+
} from '@deepseek-ai/dsh-api-remotes/client'
|
|
128
|
+
// Value import from the inline-safe wire layer (not the connection plugin):
|
|
129
|
+
// plugin-to-plugin value imports are a bundle purity error.
|
|
130
|
+
@@ -285,6 +285,28 @@ export class Session implements SessionFace {
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
+ /**
|
|
135
|
+
+ * Resolve one voice object referenced by this session into browser-consumable bytes.
|
|
136
|
+
+ * @param voiceId - opaque id found in the folded session log.
|
|
137
|
+
+ * @returns the authenticated reference and decoded bytes.
|
|
138
|
+
+ */
|
|
139
|
+
+ async readVoice(
|
|
140
|
+
+ voiceId: string,
|
|
141
|
+
+ ): Promise<RpcResult<{ attachment: VoiceAttachmentRef; data: Uint8Array }>> {
|
|
142
|
+
+ try {
|
|
143
|
+
+ const result = (await this.api.sessions.voice({
|
|
144
|
+
+ sessionId: this.sessionId,
|
|
145
|
+
+ voiceId,
|
|
146
|
+
+ })).result
|
|
147
|
+
+ if (!result.ok) return result
|
|
148
|
+
+ const binary = atob(result.value.data)
|
|
149
|
+
+ const data = Uint8Array.from(binary, char => char.charCodeAt(0))
|
|
150
|
+
+ return { ok: true, value: { attachment: result.value.attachment, data } }
|
|
151
|
+
+ } catch (error) {
|
|
152
|
+
+ return transportError(error)
|
|
153
|
+
+ }
|
|
154
|
+
+ }
|
|
155
|
+
+
|
|
156
|
+
/** Apply one operation to a still-pending queue occurrence. */
|
|
157
|
+
async updateQueue(itemId: MessageId, action: QueueAction): Promise<RpcResult<{ accepted: true }>> {
|
|
158
|
+
try {
|
|
159
|
+
diff --git a/packages/client/ui-conversation/src/client/apply.ts b/packages/client/ui-conversation/src/client/apply.ts
|
|
160
|
+
index 294009a09b..8974f4bbeb 100644
|
|
161
|
+
--- a/packages/client/ui-conversation/src/client/apply.ts
|
|
162
|
+
+++ b/packages/client/ui-conversation/src/client/apply.ts
|
|
163
|
+
@@ -248,6 +248,7 @@ export function apply(ctx: Context): void {
|
|
164
|
+
return {
|
|
165
|
+
views,
|
|
166
|
+
releaseSessionImages: (id) => { conversation.releaseSessionImages(id) },
|
|
167
|
+
+ releaseSessionVoices: (id) => { conversation.releaseSessionVoices(id) },
|
|
168
|
+
bindDraftMirror: write => inputHub.shell(sessionId).bindMirror(write),
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
@@ -299,6 +300,10 @@ export function apply(ctx: Context): void {
|
|
172
|
+
toggleCommandMenu: undefined,
|
|
173
|
+
stop: undefined,
|
|
174
|
+
command: undefined,
|
|
175
|
+
+ readBalance: undefined,
|
|
176
|
+
+ sendVoice: undefined,
|
|
177
|
+
+ transcribeVoice: undefined,
|
|
178
|
+
+ synthesizeVoice: undefined,
|
|
179
|
+
hooks: { notices: ABSENT_NOTICES, lexicon: ABSENT_LEXICON, menuLauncher: ABSENT_MENU_LAUNCHER },
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
@@ -354,6 +359,42 @@ export function apply(ctx: Context): void {
|
|
183
|
+
const result = await session.command(line)
|
|
184
|
+
return result.ok && result.value.matched
|
|
185
|
+
},
|
|
186
|
+
+ readBalance: async () => {
|
|
187
|
+
+ const connection = ctx.get('connection') as { api: import('@deepseek-ai/dsh-client-connection/client').IApiClient } | undefined
|
|
188
|
+
+ if (connection === undefined) return { balance: null }
|
|
189
|
+
+ const response = await connection.api.balance.get({ sessionId })
|
|
190
|
+
+ if (!response.result.ok) return { balance: null }
|
|
191
|
+
+ return response.result.value
|
|
192
|
+
+ },
|
|
193
|
+
+ sendVoice: async (part, mode) => {
|
|
194
|
+
+ const session = sessions.binding(sessionId)?.session
|
|
195
|
+
+ if (session === undefined) return false
|
|
196
|
+
+ const result = await session.prompt([part], mode)
|
|
197
|
+
+ return result.ok
|
|
198
|
+
+ },
|
|
199
|
+
+ transcribeVoice: async (part) => {
|
|
200
|
+
+ const connection = ctx.get('connection') as { api: import('@deepseek-ai/dsh-client-connection/client').IApiClient } | undefined
|
|
201
|
+
+ if (connection === undefined) return null
|
|
202
|
+
+ const response = await connection.api.sessions.voiceAsr({
|
|
203
|
+
+ sessionId,
|
|
204
|
+
+ mediaType: part.mediaType,
|
|
205
|
+
+ data: part.data,
|
|
206
|
+
+ ...(part.durationMs === undefined ? {} : { durationMs: part.durationMs }),
|
|
207
|
+
+ })
|
|
208
|
+
+ if (!response.result.ok) return null
|
|
209
|
+
+ return response.result.value.text
|
|
210
|
+
+ },
|
|
211
|
+
+ synthesizeVoice: async (text, provider) => {
|
|
212
|
+
+ const connection = ctx.get('connection') as { api: import('@deepseek-ai/dsh-client-connection/client').IApiClient } | undefined
|
|
213
|
+
+ if (connection === undefined) return null
|
|
214
|
+
+ const response = await connection.api.sessions.voiceTts({
|
|
215
|
+
+ sessionId,
|
|
216
|
+
+ text,
|
|
217
|
+
+ ...(provider === undefined || provider === '' ? {} : { provider }),
|
|
218
|
+
+ })
|
|
219
|
+
+ if (!response.result.ok) return null
|
|
220
|
+
+ return response.result.value
|
|
221
|
+
+ },
|
|
222
|
+
hooks: {
|
|
223
|
+
notices: shell.notices,
|
|
224
|
+
lexicon: shell.lexicon,
|
|
225
|
+
@@ -402,6 +443,18 @@ export function apply(ctx: Context): void {
|
|
226
|
+
},
|
|
227
|
+
loadOlder: () => { void scoped.loadOlder() },
|
|
228
|
+
loadImage: attachment => conversation.resolveImage(sessionId, attachment),
|
|
229
|
+
+ loadVoice: attachment => conversation.resolveVoice(sessionId, attachment),
|
|
230
|
+
+ synthesizeVoice: async (text, provider) => {
|
|
231
|
+
+ const connection = ctx.get('connection') as { api: import('@deepseek-ai/dsh-client-connection/client').IApiClient } | undefined
|
|
232
|
+
+ if (connection === undefined) return null
|
|
233
|
+
+ const response = await connection.api.sessions.voiceTts({
|
|
234
|
+
+ sessionId,
|
|
235
|
+
+ text,
|
|
236
|
+
+ ...(provider === undefined || provider === '' ? {} : { provider }),
|
|
237
|
+
+ })
|
|
238
|
+
+ if (!response.result.ok) return null
|
|
239
|
+
+ return response.result.value
|
|
240
|
+
+ },
|
|
241
|
+
// Unregistered 'trajectory' id is safe: the tab ring falls back to
|
|
242
|
+
// the first view, and the untouched inspect target stays inert.
|
|
243
|
+
inspectCall: (callId) => {
|
|
244
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/ChatNodeSeat.tsx b/packages/client/ui-conversation/src/client/chat/ChatNodeSeat.tsx
|
|
245
|
+
index bc9c96fd41..ba2b5ae040 100644
|
|
246
|
+
--- a/packages/client/ui-conversation/src/client/chat/ChatNodeSeat.tsx
|
|
247
|
+
+++ b/packages/client/ui-conversation/src/client/chat/ChatNodeSeat.tsx
|
|
248
|
+
@@ -18,7 +18,7 @@ type RoutedChatNodeOwner = {
|
|
249
|
+
/** Subscribe and dispatch one stable Context key without observing sibling Nodes. */
|
|
250
|
+
export const ChatNodeSeat = memo(function ChatNodeSeat({
|
|
251
|
+
nodeKey, selectedCallId, cwd, openFile, inspectCall, forkAt,
|
|
252
|
+
- renderMessageImages, fileMentions, useSession, renderSlot, t,
|
|
253
|
+
+ renderMessageImages, loadImage, loadVoice, fileMentions, useSession, renderSlot, t,
|
|
254
|
+
}: ChatNodeSeatProps) {
|
|
255
|
+
const node = useSession(snapshot => snapshot.chat.nodes.get(nodeKey))
|
|
256
|
+
const routedNode = node as ChatNode | undefined
|
|
257
|
+
@@ -31,9 +31,12 @@ export const ChatNodeSeat = memo(function ChatNodeSeat({
|
|
258
|
+
inspectCall,
|
|
259
|
+
forkAt,
|
|
260
|
+
renderMessageImages,
|
|
261
|
+
+ loadImage,
|
|
262
|
+
+ loadVoice,
|
|
263
|
+
fileMentions,
|
|
264
|
+
}, [
|
|
265
|
+
- node, selectedCallId, cwd, openFile, inspectCall, forkAt, renderMessageImages, fileMentions,
|
|
266
|
+
+ node, selectedCallId, cwd, openFile, inspectCall, forkAt, renderMessageImages, loadImage, loadVoice,
|
|
267
|
+
+ fileMentions,
|
|
268
|
+
])
|
|
269
|
+
if (routedNode === undefined || owner === null) return null
|
|
270
|
+
// Runtime dispatch owns the correlation: every Node's discriminant is the
|
|
271
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/ChatView.module.css b/packages/client/ui-conversation/src/client/chat/ChatView.module.css
|
|
272
|
+
index ffee12984f..639b4f97e9 100644
|
|
273
|
+
--- a/packages/client/ui-conversation/src/client/chat/ChatView.module.css
|
|
274
|
+
+++ b/packages/client/ui-conversation/src/client/chat/ChatView.module.css
|
|
275
|
+
@@ -131,6 +131,19 @@
|
|
276
|
+
justify-content: center;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
+/* [本地改造 2026-08-16] 语音铁律回复:左对齐行,位于最后一条助手消息下方。 */
|
|
280
|
+
+.ttsReplyRow {
|
|
281
|
+
+ display: flex;
|
|
282
|
+
+ justify-content: flex-start;
|
|
283
|
+
+ padding: 2px 0;
|
|
284
|
+
+}
|
|
285
|
+
+
|
|
286
|
+
+.ttsReplyPending {
|
|
287
|
+
+ color: var(--dsw-alias-label-tertiary);
|
|
288
|
+
+ font-size: 12px;
|
|
289
|
+
+ line-height: 28px;
|
|
290
|
+
+}
|
|
291
|
+
+
|
|
292
|
+
.older button {
|
|
293
|
+
border: none;
|
|
294
|
+
border-radius: 14px;
|
|
295
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/ChatView.tsx b/packages/client/ui-conversation/src/client/chat/ChatView.tsx
|
|
296
|
+
index 80f9bc2d73..22c3fec2d6 100644
|
|
297
|
+
--- a/packages/client/ui-conversation/src/client/chat/ChatView.tsx
|
|
298
|
+
+++ b/packages/client/ui-conversation/src/client/chat/ChatView.tsx
|
|
299
|
+
@@ -156,7 +156,7 @@ function TurnStatus({ startTime, t }: {
|
|
300
|
+
* ordered business Node crosses the keyed renderer seat.
|
|
301
|
+
*/
|
|
302
|
+
export function ChatView({
|
|
303
|
+
- useSession, useSessions, useStore, renderSlot, sessionId, openFile, loadOlder, loadImage, inspectCall, chatScroll, forkAt,
|
|
304
|
+
+ useSession, useSessions, useStore, renderSlot, sessionId, openFile, loadOlder, loadImage, loadVoice, inspectCall, chatScroll, forkAt,
|
|
305
|
+
fileMentions, t,
|
|
306
|
+
}: ChatViewSlotProps) {
|
|
307
|
+
const order = useSession(s => s.chat.order)
|
|
308
|
+
@@ -440,6 +440,8 @@ export function ChatView({
|
|
309
|
+
inspectCall={inspectCall}
|
|
310
|
+
forkAt={forkAt}
|
|
311
|
+
renderMessageImages={renderMessageImages}
|
|
312
|
+
+ loadImage={loadImage}
|
|
313
|
+
+ loadVoice={loadVoice}
|
|
314
|
+
fileMentions={fileMentions}
|
|
315
|
+
renderSlot={renderSlot}
|
|
316
|
+
t={t}
|
|
317
|
+
@@ -456,6 +458,7 @@ export function ChatView({
|
|
318
|
+
key={item.id}
|
|
319
|
+
content={item.content}
|
|
320
|
+
renderMessageImages={renderMessageImages}
|
|
321
|
+
+ loadVoice={loadVoice}
|
|
322
|
+
t={t}
|
|
323
|
+
/>
|
|
324
|
+
))}
|
|
325
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/MessageItem.module.css b/packages/client/ui-conversation/src/client/chat/MessageItem.module.css
|
|
326
|
+
index 54e5f6f98d..0ef782e759 100644
|
|
327
|
+
--- a/packages/client/ui-conversation/src/client/chat/MessageItem.module.css
|
|
328
|
+
+++ b/packages/client/ui-conversation/src/client/chat/MessageItem.module.css
|
|
329
|
+
@@ -16,6 +16,101 @@
|
|
330
|
+
min-width: 0;
|
|
331
|
+
max-width: min(525px, 82%);
|
|
332
|
+
}
|
|
333
|
+
+
|
|
334
|
+
+/* [本地改造 2026-08-16] 语音消息卡片:播放按钮 + 时长,右对齐气泡风格。 */
|
|
335
|
+
+.voiceCard {
|
|
336
|
+
+ display: inline-flex;
|
|
337
|
+
+ align-items: center;
|
|
338
|
+
+ gap: 8px;
|
|
339
|
+
+ max-width: 100%;
|
|
340
|
+
+ height: 44px;
|
|
341
|
+
+ padding: 0 16px 0 10px;
|
|
342
|
+
+ background: var(--dsw-specific-bubble);
|
|
343
|
+
+ border-radius: 22px;
|
|
344
|
+
+ /* [本地改造 2026-08-16] 无 transcript 的语音按时长定宽,宽度须含 padding。 */
|
|
345
|
+
+ box-sizing: border-box;
|
|
346
|
+
+}
|
|
347
|
+
+
|
|
348
|
+
+.voicePlay {
|
|
349
|
+
+ display: grid;
|
|
350
|
+
+ place-items: center;
|
|
351
|
+
+ flex: none;
|
|
352
|
+
+ width: 28px;
|
|
353
|
+
+ height: 28px;
|
|
354
|
+
+ border: none;
|
|
355
|
+
+ border-radius: 999px;
|
|
356
|
+
+ background: var(--dsw-specific-selector);
|
|
357
|
+
+ color: var(--dsw-alias-label-primary);
|
|
358
|
+
+ cursor: pointer;
|
|
359
|
+
+}
|
|
360
|
+
+
|
|
361
|
+
+.voicePlay:hover:not(:disabled) {
|
|
362
|
+
+ background: var(--dsw-alias-interactive-bg-hover-solid);
|
|
363
|
+
+}
|
|
364
|
+
+
|
|
365
|
+
+.voicePlay:disabled {
|
|
366
|
+
+ opacity: 0.5;
|
|
367
|
+
+ cursor: default;
|
|
368
|
+
+}
|
|
369
|
+
+
|
|
370
|
+
+.voiceDuration {
|
|
371
|
+
+ color: var(--dsw-alias-label-secondary);
|
|
372
|
+
+ font-size: 13px;
|
|
373
|
+
+ line-height: 20px;
|
|
374
|
+
+ font-variant-numeric: tabular-nums;
|
|
375
|
+
+}
|
|
376
|
+
+
|
|
377
|
+
+/* [本地改造 2026-08-16] 语音识别文本(host ASR 写入 attachment.transcript)。 */
|
|
378
|
+
+.voiceTranscript {
|
|
379
|
+
+ display: block;
|
|
380
|
+
+ max-width: 320px;
|
|
381
|
+
+ overflow: hidden;
|
|
382
|
+
+ text-overflow: ellipsis;
|
|
383
|
+
+ white-space: nowrap;
|
|
384
|
+
+ color: var(--dsw-alias-label-secondary);
|
|
385
|
+
+ font-size: 12px;
|
|
386
|
+
+ line-height: 16px;
|
|
387
|
+
+}
|
|
388
|
+
+
|
|
389
|
+
+/* [本地改造 2026-08-21] ASR 失败(无 transcript)时的弱提示。 */
|
|
390
|
+
+.voiceTranscriptFailed {
|
|
391
|
+
+ display: block;
|
|
392
|
+
+ max-width: 320px;
|
|
393
|
+
+ overflow: hidden;
|
|
394
|
+
+ text-overflow: ellipsis;
|
|
395
|
+
+ white-space: nowrap;
|
|
396
|
+
+ color: var(--dsw-alias-label-tertiary);
|
|
397
|
+
+ font-size: 12px;
|
|
398
|
+
+ line-height: 16px;
|
|
399
|
+
+ opacity: 0.85;
|
|
400
|
+
+}
|
|
401
|
+
+
|
|
402
|
+
+/* [本地改造 2026-08-21] 语音条内置复制按钮(对齐系统 .action:28px 圆形透明)。 */
|
|
403
|
+
+.voiceCopy {
|
|
404
|
+
+ display: inline-flex;
|
|
405
|
+
+ align-items: center;
|
|
406
|
+
+ justify-content: center;
|
|
407
|
+
+ width: 28px;
|
|
408
|
+
+ height: 28px;
|
|
409
|
+
+ padding: 6px;
|
|
410
|
+
+ border: none;
|
|
411
|
+
+ border-radius: 28px;
|
|
412
|
+
+ background: transparent;
|
|
413
|
+
+ color: var(--dsw-alias-label-tertiary);
|
|
414
|
+
+ cursor: pointer;
|
|
415
|
+
+ flex-shrink: 0;
|
|
416
|
+
+}
|
|
417
|
+
+.voiceCopy:hover {
|
|
418
|
+
+ background: var(--dsw-alias-interactive-bg-hover);
|
|
419
|
+
+ color: var(--dsw-alias-label-secondary);
|
|
420
|
+
+}
|
|
421
|
+
+
|
|
422
|
+
+/* [本地改造 2026-08-16] 助手语音回复独立横条:左对齐,与用户语音消息同级。 */
|
|
423
|
+
+.voiceReplyRow {
|
|
424
|
+
+ display: flex;
|
|
425
|
+
+ justify-content: flex-start;
|
|
426
|
+
+ padding: 2px 0;
|
|
427
|
+
+}
|
|
428
|
+
.bubble {
|
|
429
|
+
/* 525px cap inside the 736 column; percentage keeps narrow windows sane. */
|
|
430
|
+
max-width: 100%;
|
|
431
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/MessageItem.tsx b/packages/client/ui-conversation/src/client/chat/MessageItem.tsx
|
|
432
|
+
index 8a882a70c5..67aa18ba1a 100644
|
|
433
|
+
--- a/packages/client/ui-conversation/src/client/chat/MessageItem.tsx
|
|
434
|
+
+++ b/packages/client/ui-conversation/src/client/chat/MessageItem.tsx
|
|
435
|
+
@@ -3,11 +3,13 @@
|
|
436
|
+
// assistant answers), pending steering (copy only), context injection,
|
|
437
|
+
// compaction marker, retry disclosure, and unknown-surface JSON rows.
|
|
438
|
+
|
|
439
|
+
-import { memo, useEffect, useMemo, useState } from 'react'
|
|
440
|
+
+import { memo, useEffect, useMemo, useRef, useState } from 'react'
|
|
441
|
+
import type { ReactNode } from 'react'
|
|
442
|
+
import type {
|
|
443
|
+
ModelRetryNode, TurnErrorNode, UserMessageNode,
|
|
444
|
+
} from '@deepseek-ai/dsh-client-runtime/client'
|
|
445
|
+
+import type { VoiceAttachmentRef } from '@deepseek-ai/dsh-client-connection/client'
|
|
446
|
+
+import type { PropsRenderSlots } from '@deepseek-ai/dsh-client-ui-slots'
|
|
447
|
+
import { JsonBlock, MessageText, StateDot } from '@deepseek-ai/dsh-client-ui-primitives'
|
|
448
|
+
import type { ChatNodeOwnerProps, ChatNodeViewProps, ChatViewSlotProps } from '../contract/slots.ts'
|
|
449
|
+
import { ReferenceIcon } from '../reference/ReferenceIcon.tsx'
|
|
450
|
+
@@ -17,14 +19,40 @@ import { MessageIconActions } from './MessageIconActions.tsx'
|
|
451
|
+
import css from './MessageItem.module.css'
|
|
452
|
+
|
|
453
|
+
type UserImage = Extract<UserMessageNode['content'][number], { type: 'image' }>
|
|
454
|
+
+type UserVoice = Extract<UserMessageNode['content'][number], { type: 'voice' }>
|
|
455
|
+
+
|
|
456
|
+
+// [本地改造 2026-08-18] 微信式语音互斥:同一时刻只播一条语音。点新的自动停旧的,
|
|
457
|
+
+// 避免多个 <audio> 同时发声混在一起。模块级单例,跨卡片共享。
|
|
458
|
+
+let activeVoice: { id: string; audio: HTMLAudioElement; setPlaying: (playing: boolean) => void } | null = null
|
|
459
|
+
+
|
|
460
|
+
+function playExclusive(id: string, audio: HTMLAudioElement, setPlaying: (playing: boolean) => void): void {
|
|
461
|
+
+ if (activeVoice !== null && activeVoice.id !== id) {
|
|
462
|
+
+ activeVoice.audio.pause()
|
|
463
|
+
+ activeVoice.setPlaying(false)
|
|
464
|
+
+ }
|
|
465
|
+
+ activeVoice = { id, audio, setPlaying }
|
|
466
|
+
+ setPlaying(true)
|
|
467
|
+
+}
|
|
468
|
+
+
|
|
469
|
+
+// [本地改造 2026-08-18] 录音互斥:InputBar 开始录音时调用,停掉正在播放的语音,
|
|
470
|
+
+// 避免外放声音被麦克风录进新语音(回声)。
|
|
471
|
+
+export function stopVoicePlayback(): void {
|
|
472
|
+
+ if (activeVoice !== null) {
|
|
473
|
+
+ activeVoice.audio.pause()
|
|
474
|
+
+ activeVoice.setPlaying(false)
|
|
475
|
+
+ activeVoice = null
|
|
476
|
+
+ }
|
|
477
|
+
+}
|
|
478
|
+
|
|
479
|
+
function contentParts(content: readonly unknown[]): {
|
|
480
|
+
text: string
|
|
481
|
+
images: { attachment: UserImage['attachment'] }[]
|
|
482
|
+
+ voices: { attachment: UserVoice['attachment'] }[]
|
|
483
|
+
rest: unknown[]
|
|
484
|
+
} {
|
|
485
|
+
const texts: string[] = []
|
|
486
|
+
const images: { attachment: UserImage['attachment'] }[] = []
|
|
487
|
+
+ const voices: { attachment: UserVoice['attachment'] }[] = []
|
|
488
|
+
const rest: unknown[] = []
|
|
489
|
+
for (const block of content) {
|
|
490
|
+
const b = block as { type?: string; text?: string; attachment?: unknown }
|
|
491
|
+
@@ -32,9 +60,12 @@ function contentParts(content: readonly unknown[]): {
|
|
492
|
+
else if (b.type === 'image' && b.attachment !== undefined) {
|
|
493
|
+
images.push({ attachment: (b as UserImage).attachment })
|
|
494
|
+
}
|
|
495
|
+
+ else if (b.type === 'voice' && b.attachment !== undefined) {
|
|
496
|
+
+ voices.push({ attachment: (b as UserVoice).attachment })
|
|
497
|
+
+ }
|
|
498
|
+
else rest.push(block)
|
|
499
|
+
}
|
|
500
|
+
- return { text: texts.join(''), images, rest }
|
|
501
|
+
+ return { text: texts.join(''), images, voices, rest }
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
function retrySeconds(milliseconds: number): number {
|
|
505
|
+
@@ -212,27 +243,198 @@ function projectUserText(text: string, sessionLabels: readonly string[]): ReactN
|
|
506
|
+
return <>{parts}</>
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
+/** [本地改造 2026-08-16] 语音条宽度:4 秒内固定 96px(短语音不显拥挤),超过后每增 1 秒 +4px,上限 320px。 */
|
|
510
|
+
+function voiceCardWidth(seconds: number): number {
|
|
511
|
+
+ if (seconds <= 4) return 96
|
|
512
|
+
+ return Math.min(320, 96 + (seconds - 4) * 4)
|
|
513
|
+
+}
|
|
514
|
+
+
|
|
515
|
+
+/** Right-aligned voice message card: session-authorized playback with duration. */
|
|
516
|
+
+export function VoiceCard({ attachment, load, actions, asrFailedHint = false, t }: {
|
|
517
|
+
+ attachment: VoiceAttachmentRef
|
|
518
|
+
+ load?: (ref: VoiceAttachmentRef) => Promise<string>
|
|
519
|
+
+ /** [本地改造 2026-08-21] Voice-actions slot strip rendered at the card tail. */
|
|
520
|
+
+ actions?: ReactNode
|
|
521
|
+
+ /** [本地改造 2026-08-21] 无转写时是否显示「未能识别」提示:仅用户语音消息
|
|
522
|
+
+ * 开启;助手 TTS 语音回复没有 transcript 概念,不显示。 */
|
|
523
|
+
+ asrFailedHint?: boolean
|
|
524
|
+
+ t: ChatViewSlotProps['t']
|
|
525
|
+
+}) { const [url, setUrl] = useState<string | null>(null)
|
|
526
|
+
+ const [failed, setFailed] = useState(false)
|
|
527
|
+
+ const [playing, setPlaying] = useState(false)
|
|
528
|
+
+ const [copied, setCopied] = useState(false)
|
|
529
|
+
+ const audioRef = useRef<HTMLAudioElement | null>(null)
|
|
530
|
+
+ useEffect(() => {
|
|
531
|
+
+ let cancelled = false
|
|
532
|
+
+ setFailed(false)
|
|
533
|
+
+ setUrl(null)
|
|
534
|
+
+ // A missing loader (deployment without the voice channel) degrades to the
|
|
535
|
+
+ // disabled card rather than crashing the whole message row.
|
|
536
|
+
+ if (load === undefined) {
|
|
537
|
+
+ setFailed(true)
|
|
538
|
+
+ return () => { cancelled = true }
|
|
539
|
+
+ }
|
|
540
|
+
+ load(attachment).then((next) => {
|
|
541
|
+
+ if (!cancelled) setUrl(next)
|
|
542
|
+
+ }, () => {
|
|
543
|
+
+ if (!cancelled) setFailed(true)
|
|
544
|
+
+ })
|
|
545
|
+
+ return () => { cancelled = true }
|
|
546
|
+
+ }, [attachment, load])
|
|
547
|
+
+ const toggle = (): void => {
|
|
548
|
+
+ const audio = audioRef.current
|
|
549
|
+
+ if (audio === null) return
|
|
550
|
+
+ if (playing) {
|
|
551
|
+
+ audio.pause()
|
|
552
|
+
+ if (activeVoice?.id === attachment.voiceId) activeVoice = null
|
|
553
|
+
+ } else {
|
|
554
|
+
+ playExclusive(attachment.voiceId, audio, setPlaying)
|
|
555
|
+
+ void audio.play().catch(() => { setFailed(true) })
|
|
556
|
+
+ }
|
|
557
|
+
+ }
|
|
558
|
+
+ const seconds = attachment.durationMs !== undefined
|
|
559
|
+
+ ? Math.max(1, Math.ceil(attachment.durationMs / 1_000))
|
|
560
|
+
+ : null
|
|
561
|
+
+ // [本地改造 2026-08-16] 语音条宽度按时长变化(微信风格):8 秒以内固定宽度
|
|
562
|
+
+ // (短语音不显拥挤),超过 8 秒按秒数线性增宽,上限 320px;
|
|
563
|
+
+ // 带 transcript 的用户语音由文本自然撑宽(保持现状)。
|
|
564
|
+
+ const hasTranscript = attachment.transcript !== undefined && attachment.transcript !== ''
|
|
565
|
+
+ const durationWidth = !hasTranscript && attachment.durationMs !== undefined
|
|
566
|
+
+ ? { width: voiceCardWidth(seconds ?? 1) }
|
|
567
|
+
+ : undefined
|
|
568
|
+
+ return (
|
|
569
|
+
+ <div className={css.voiceCard} data-voice style={durationWidth}>
|
|
570
|
+
+ <button
|
|
571
|
+
+ type="button"
|
|
572
|
+
+ className={css.voicePlay}
|
|
573
|
+
+ aria-label={playing ? t('voice.pause') : t('voice.play')}
|
|
574
|
+
+ disabled={failed}
|
|
575
|
+
+ onClick={toggle}
|
|
576
|
+
+ >
|
|
577
|
+
+ {playing
|
|
578
|
+
+ ? (
|
|
579
|
+
+ <svg viewBox="0 0 16 16" width="14" height="14" aria-hidden>
|
|
580
|
+
+ <rect x="3.5" y="3.5" width="3" height="9" rx="1" fill="currentColor"/>
|
|
581
|
+
+ <rect x="9.5" y="3.5" width="3" height="9" rx="1" fill="currentColor"/>
|
|
582
|
+
+ </svg>
|
|
583
|
+
+ )
|
|
584
|
+
+ : (
|
|
585
|
+
+ <svg viewBox="0 0 16 16" width="14" height="14" aria-hidden>
|
|
586
|
+
+ <path d="M5 3.5L12.5 8L5 12.5V3.5Z" fill="currentColor"/>
|
|
587
|
+
+ </svg>
|
|
588
|
+
+ )}
|
|
589
|
+
+ </button>
|
|
590
|
+
+ <span className={css.voiceDuration}>{seconds === null ? '' : `${seconds}s`}</span>
|
|
591
|
+
+ {hasTranscript ? (
|
|
592
|
+
+ <span className={css.voiceTranscript} title={t('voice.transcriptLabel')}>
|
|
593
|
+
+ {attachment.transcript}
|
|
594
|
+
+ </span>
|
|
595
|
+
+ ) : asrFailedHint ? (
|
|
596
|
+
+ <span className={css.voiceTranscriptFailed} title={t('voice.asrFailed')}>
|
|
597
|
+
+ {t('voice.asrFailed')}
|
|
598
|
+
+ </span>
|
|
599
|
+
+ ) : null}
|
|
600
|
+
+ {/* [本地改造 2026-08-21] 语音条复制按钮:外部 voice-actions 优先(插件);
|
|
601
|
+
+ 未提供(如 AI 语音回复)且有转写时,用内置按钮兜底,保证所有语音条可复制。 */}
|
|
602
|
+
+ {actions !== undefined
|
|
603
|
+
+ ? actions
|
|
604
|
+
+ : (hasTranscript
|
|
605
|
+
+ ? (
|
|
606
|
+
+ <button
|
|
607
|
+
+ type="button"
|
|
608
|
+
+ className={css.voiceCopy}
|
|
609
|
+
+ aria-label={copied ? t('copied') : t('copy')}
|
|
610
|
+
+ title={copied ? t('copied') : t('copy')}
|
|
611
|
+
+ onClick={() => {
|
|
612
|
+
+ const text = attachment.transcript ?? ''
|
|
613
|
+
+ const done = (): void => {
|
|
614
|
+
+ setCopied(true)
|
|
615
|
+
+ window.setTimeout(() => setCopied(false), 1000)
|
|
616
|
+
+ }
|
|
617
|
+
+ if (typeof navigator !== 'undefined' && navigator.clipboard?.writeText) {
|
|
618
|
+
+ void navigator.clipboard.writeText(text).then(done, done)
|
|
619
|
+
+ } else { done() }
|
|
620
|
+
+ }}
|
|
621
|
+
+ >
|
|
622
|
+
+ {copied
|
|
623
|
+
+ ? (
|
|
624
|
+
+ <svg viewBox="0 0 16 16" width="14" height="14" aria-hidden>
|
|
625
|
+
+ <path d="M3.5 8.5L6.5 11.5L12.5 4.5" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"/>
|
|
626
|
+
+ </svg>
|
|
627
|
+
+ )
|
|
628
|
+
+ : (
|
|
629
|
+
+ <svg viewBox="0 0 16 16" width="14" height="14" aria-hidden>
|
|
630
|
+
+ <rect x="5.5" y="5.5" width="7" height="7" rx="1.2" fill="none" stroke="currentColor" strokeWidth="1.3"/>
|
|
631
|
+
+ <path d="M10.5 5.5V4.5A1 1 0 0 0 9.5 3.5H5A1 1 0 0 0 4 4.5v4.5a1 1 0 0 0 1 1h1" fill="none" stroke="currentColor" strokeWidth="1.3"/>
|
|
632
|
+
+ </svg>
|
|
633
|
+
+ )}
|
|
634
|
+
+ </button>
|
|
635
|
+
+ )
|
|
636
|
+
+ : null)}
|
|
637
|
+
+ {url !== null && (
|
|
638
|
+
+ <audio
|
|
639
|
+
+ ref={audioRef}
|
|
640
|
+
+ src={url}
|
|
641
|
+
+ onPlay={() => { setPlaying(true) }}
|
|
642
|
+
+ onPause={() => { setPlaying(false) }}
|
|
643
|
+
+ onEnded={() => { setPlaying(false) }}
|
|
644
|
+
+ />
|
|
645
|
+
+ )}
|
|
646
|
+
+ </div>
|
|
647
|
+
+ )
|
|
648
|
+
+}
|
|
649
|
+
+
|
|
650
|
+
/** Right-aligned bubble shared by user and steering rows. */
|
|
651
|
+
function UserStyleBubble({
|
|
652
|
+
- content, renderMessageImages, actions, pending = false, referenceLabels = [], t,
|
|
653
|
+
+ content,
|
|
654
|
+
+ renderMessageImages,
|
|
655
|
+
+ voiceLoader,
|
|
656
|
+
+ actions,
|
|
657
|
+
+ renderVoiceActions,
|
|
658
|
+
+ voiceAsrFailedHint = false,
|
|
659
|
+
+ pending = false,
|
|
660
|
+
+ referenceLabels = [],
|
|
661
|
+
+ t,
|
|
662
|
+
}: {
|
|
663
|
+
content: readonly unknown[]
|
|
664
|
+
renderMessageImages: ChatNodeOwnerProps['renderMessageImages']
|
|
665
|
+
+ voiceLoader: (ref: VoiceAttachmentRef) => Promise<string>
|
|
666
|
+
/** Optional IconActions (or similar) below the bubble; receives the joined text. */
|
|
667
|
+
actions?: (text: string) => ReactNode
|
|
668
|
+
+ /** [本地改造 2026-08-21] Voice-actions slot strip, resolved per voice card. */
|
|
669
|
+
+ renderVoiceActions?: (attachment: VoiceAttachmentRef, index: number) => ReactNode
|
|
670
|
+
+ /** [本地改造 2026-08-21] 语音无转写时显示「未能识别」提示(用户消息才开)。 */
|
|
671
|
+
+ voiceAsrFailedHint?: boolean
|
|
672
|
+
/** Whether this is the Host-authoritative pre-admission steering projection. */
|
|
673
|
+
pending?: boolean
|
|
674
|
+
/** Exact session mention labels associated by the adjacent recall node. */
|
|
675
|
+
referenceLabels?: readonly string[]
|
|
676
|
+
t: ChatViewSlotProps['t']
|
|
677
|
+
}): ReactNode {
|
|
678
|
+
- const { text, images, rest } = contentParts(content)
|
|
679
|
+
+ const { text, images, voices, rest } = contentParts(content)
|
|
680
|
+
+ // [本地改造 2026-08-21] 语音消息复制:把每条语音的转写文本并入复制文本。
|
|
681
|
+
+ // 纯语音消息本身没有 text 段,转写只挂在 attachment.transcript 上,否则复制按钮
|
|
682
|
+
+ // 写出的 text 为空,粘贴是空白。有文字时文本在前、转写按语音顺序追加在后。
|
|
683
|
+
+ const voiceTranscripts = voices
|
|
684
|
+
+ .map(v => v.attachment.transcript)
|
|
685
|
+
+ .filter((s): s is string => typeof s === 'string' && s !== '')
|
|
686
|
+
+ const copyText = [text, ...voiceTranscripts].join('\n').trim()
|
|
687
|
+
const truncated = (total: number): string => t('json.truncated', { total })
|
|
688
|
+
const showBubble = text !== '' || rest.length > 0
|
|
689
|
+
return (
|
|
690
|
+
<div className={css.userRow} data-pending-steering={pending || undefined} data-time-hover-root>
|
|
691
|
+
<div className={css.userStack}>
|
|
692
|
+
{renderMessageImages({ images, align: 'end' })}
|
|
693
|
+
+ {voices.map((voice, i) => (
|
|
694
|
+
+ <VoiceCard
|
|
695
|
+
+ key={i}
|
|
696
|
+
+ attachment={voice.attachment}
|
|
697
|
+
+ load={voiceLoader}
|
|
698
|
+
+ actions={renderVoiceActions?.(voice.attachment, i)}
|
|
699
|
+
+ asrFailedHint={voiceAsrFailedHint}
|
|
700
|
+
+ t={t}
|
|
701
|
+
+ />
|
|
702
|
+
+ ))}
|
|
703
|
+
{showBubble && <div className={css.bubble}>
|
|
704
|
+
{projectUserText(text, referenceLabels)}
|
|
705
|
+
{rest.map((block, i) => <JsonBlock key={i} label={t('message.extraBlock')} payload={block} truncatedLabel={truncated} />)}
|
|
706
|
+
@@ -243,7 +445,7 @@ function UserStyleBubble({
|
|
707
|
+
</div>
|
|
708
|
+
)}
|
|
709
|
+
</div>
|
|
710
|
+
- {actions?.(text)}
|
|
711
|
+
+ {actions?.(copyText)}
|
|
712
|
+
</div>
|
|
713
|
+
)
|
|
714
|
+
}
|
|
715
|
+
@@ -254,15 +456,18 @@ function UserStyleBubble({
|
|
716
|
+
* @param props - Pending message content and conversation translator.
|
|
717
|
+
* @returns the pending steering bubble.
|
|
718
|
+
*/
|
|
719
|
+
-export function PendingSteeringBubble({ content, renderMessageImages, t }: {
|
|
720
|
+
+export function PendingSteeringBubble({ content, renderMessageImages, loadVoice, t }: {
|
|
721
|
+
content: readonly unknown[]
|
|
722
|
+
renderMessageImages: ChatNodeOwnerProps['renderMessageImages']
|
|
723
|
+
+ loadVoice?: (ref: VoiceAttachmentRef) => Promise<string>
|
|
724
|
+
t: ChatViewSlotProps['t']
|
|
725
|
+
}): ReactNode {
|
|
726
|
+
+ const voiceLoader = loadVoice ?? (() => Promise.reject(new Error(t('voice.loadFailed'))))
|
|
727
|
+
return (
|
|
728
|
+
<UserStyleBubble
|
|
729
|
+
content={content}
|
|
730
|
+
renderMessageImages={renderMessageImages}
|
|
731
|
+
+ voiceLoader={voiceLoader}
|
|
732
|
+
pending
|
|
733
|
+
t={t}
|
|
734
|
+
actions={text => (
|
|
735
|
+
@@ -278,14 +483,36 @@ export function PendingSteeringBubble({ content, renderMessageImages, t }: {
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/** User and admitted-steering keyed Chat renderer. */
|
|
739
|
+
+export type UserMessageNodeViewProps = ChatNodeViewProps<'user' | 'steering'>
|
|
740
|
+
+ & PropsRenderSlots<'conversation.chat.user-actions' | 'conversation.chat.voice-actions'>
|
|
741
|
+
+
|
|
742
|
+
export const UserMessageNodeView = memo(function UserMessageNodeView({
|
|
743
|
+
- node, renderMessageImages, t,
|
|
744
|
+
-}: ChatNodeViewProps<'user' | 'steering'>) {
|
|
745
|
+
+ node, renderMessageImages, loadVoice, renderSlot, t,
|
|
746
|
+
+}: UserMessageNodeViewProps) {
|
|
747
|
+
const data = node.data
|
|
748
|
+
+ // [本地改造 2026-08-21] 用户消息操作行:语音转写只挂在 attachment.transcript 上、
|
|
749
|
+
+ // 不在 text 段,把转写文本作为 owner currency 交给 user-actions 槽
|
|
750
|
+
+ // (供「复制转写」类按钮直接使用,无需再按 id 回溯消息)。
|
|
751
|
+
+ const voiceTranscripts = data.content
|
|
752
|
+
+ .filter((b): b is UserVoice => b.type === 'voice')
|
|
753
|
+
+ .map(b => b.attachment.transcript)
|
|
754
|
+
+ .filter((s): s is string => typeof s === 'string' && s !== '')
|
|
755
|
+
+ const userActions = renderSlot('conversation.chat.user-actions', { seq: data.seq, voiceTranscripts })
|
|
756
|
+
+ // [本地改造 2026-08-21] 语音条尾部动作:逐条语音卡解析,给「复制转写」按钮
|
|
757
|
+
+ // 放在语音条最后面(卡片内、转写文本之后),而不是消息操作行里。
|
|
758
|
+
+ const renderVoiceActions = (attachment: VoiceAttachmentRef, index: number): ReactNode =>
|
|
759
|
+
+ renderSlot('conversation.chat.voice-actions', {
|
|
760
|
+
+ seq: data.seq,
|
|
761
|
+
+ index,
|
|
762
|
+
+ transcript: attachment.transcript ?? '',
|
|
763
|
+
+ })
|
|
764
|
+
return (
|
|
765
|
+
<UserStyleBubble
|
|
766
|
+
content={data.content}
|
|
767
|
+
renderMessageImages={renderMessageImages}
|
|
768
|
+
+ voiceLoader={loadVoice}
|
|
769
|
+
+ renderVoiceActions={renderVoiceActions}
|
|
770
|
+
+ voiceAsrFailedHint
|
|
771
|
+
{...data.referenceLabels === undefined ? {} : { referenceLabels: data.referenceLabels }}
|
|
772
|
+
t={t}
|
|
773
|
+
actions={text => (
|
|
774
|
+
@@ -294,6 +521,7 @@ export const UserMessageNodeView = memo(function UserMessageNodeView({
|
|
775
|
+
time={data.time}
|
|
776
|
+
clock="start"
|
|
777
|
+
className={css.actions}
|
|
778
|
+
+ extraActions={userActions}
|
|
779
|
+
t={t}
|
|
780
|
+
/>
|
|
781
|
+
)}
|
|
782
|
+
@@ -304,6 +532,10 @@ export const UserMessageNodeView = memo(function UserMessageNodeView({
|
|
783
|
+
/** Injected-context keyed Chat renderer. */
|
|
784
|
+
export const ContextMessageNodeView = memo(function ContextMessageNodeView({ node, t }: ChatNodeViewProps<'context'>) {
|
|
785
|
+
const data = node.data
|
|
786
|
+
+ // [本地改造 2026-08-16] 隐藏 vision-qa 图片识别的注入行:识别在后台完成,用户无感知
|
|
787
|
+
+ // (消息仍在会话日志、模型可见;用户只看到自己的图片消息与助手回复——正常图片交互)
|
|
788
|
+
+ const src = data.source as { kind?: string; plugin?: string } | null
|
|
789
|
+
+ if (src?.kind === 'plugin' && src.plugin === 'vision-qa') return null
|
|
790
|
+
return (
|
|
791
|
+
<ContextInjectionRow
|
|
792
|
+
content={data.content}
|
|
793
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/ReasoningRow.module.css b/packages/client/ui-conversation/src/client/chat/ReasoningRow.module.css
|
|
794
|
+
index 6b2f6c0cd3..d0a54e34a6 100644
|
|
795
|
+
--- a/packages/client/ui-conversation/src/client/chat/ReasoningRow.module.css
|
|
796
|
+
+++ b/packages/client/ui-conversation/src/client/chat/ReasoningRow.module.css
|
|
797
|
+
@@ -66,6 +66,8 @@
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
.thinkBody {
|
|
801
|
+
+ max-height: 240px;
|
|
802
|
+
+ overflow-y: auto;
|
|
803
|
+
padding: 4px 0 4px 22px;
|
|
804
|
+
color: var(--dsw-alias-label-tertiary);
|
|
805
|
+
font-size: 14px;
|
|
806
|
+
@@ -74,6 +76,16 @@
|
|
807
|
+
word-break: break-word;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
+.thinkBody::-webkit-scrollbar-thumb {
|
|
811
|
+
+ border: 2px solid transparent;
|
|
812
|
+
+ background-clip: padding-box;
|
|
813
|
+
+ border-radius: 6px;
|
|
814
|
+
+}
|
|
815
|
+
+
|
|
816
|
+
+.thinkBody::-webkit-scrollbar-track {
|
|
817
|
+
+ margin: 4px 0;
|
|
818
|
+
+}
|
|
819
|
+
+
|
|
820
|
+
@media (prefers-reduced-motion: reduce) {
|
|
821
|
+
.root[data-state='running'] .row::after {
|
|
822
|
+
animation: none;
|
|
823
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/ReasoningRow.tsx b/packages/client/ui-conversation/src/client/chat/ReasoningRow.tsx
|
|
824
|
+
index f8a340d20c..b048e65b08 100644
|
|
825
|
+
--- a/packages/client/ui-conversation/src/client/chat/ReasoningRow.tsx
|
|
826
|
+
+++ b/packages/client/ui-conversation/src/client/chat/ReasoningRow.tsx
|
|
827
|
+
@@ -27,6 +27,7 @@ function latestLine(text: string): string {
|
|
828
|
+
export function ReasoningRow({ text, running, t }: { text: string; running: boolean; t: ChatViewSlotProps['t'] }) {
|
|
829
|
+
const [expanded, setExpanded] = useState(false)
|
|
830
|
+
const summaryRef = useRef<HTMLSpanElement>(null)
|
|
831
|
+
+ const bodyRef = useRef<HTMLDivElement>(null)
|
|
832
|
+
const summary = running ? latestLine(text) : firstLine(text)
|
|
833
|
+
const scheduleSummaryScroll = useThrottledVisualUpdate(() => {
|
|
834
|
+
const element = summaryRef.current
|
|
835
|
+
@@ -36,6 +37,13 @@ export function ReasoningRow({ text, running, t }: { text: string; running: bool
|
|
836
|
+
useEffect(() => {
|
|
837
|
+
scheduleSummaryScroll()
|
|
838
|
+
}, [running, scheduleSummaryScroll, summary])
|
|
839
|
+
+ // The expanded body is height-capped; keep the streamed tail visible while
|
|
840
|
+
+ // the row is open so new reasoning tokens never sit below the fold.
|
|
841
|
+
+ useEffect(() => {
|
|
842
|
+
+ if (!running || !expanded) return
|
|
843
|
+
+ const element = bodyRef.current
|
|
844
|
+
+ if (element !== null) element.scrollTop = element.scrollHeight
|
|
845
|
+
+ }, [running, expanded, text])
|
|
846
|
+
|
|
847
|
+
return (
|
|
848
|
+
<div className={css.root} data-variant="think" data-state={running ? 'running' : 'ok'}>
|
|
849
|
+
@@ -58,7 +66,7 @@ export function ReasoningRow({ text, running, t }: { text: string; running: bool
|
|
850
|
+
</>
|
|
851
|
+
)}
|
|
852
|
+
>
|
|
853
|
+
- <div className={css.thinkBody}>{text}</div>
|
|
854
|
+
+ <div ref={bodyRef} className={css.thinkBody}>{text}</div>
|
|
855
|
+
</DisclosureRow>
|
|
856
|
+
</div>
|
|
857
|
+
)
|
|
858
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/TtsVoiceCard.module.css b/packages/client/ui-conversation/src/client/chat/TtsVoiceCard.module.css
|
|
859
|
+
new file mode 100644
|
|
860
|
+
index 0000000000..d89ae22df0
|
|
861
|
+
--- /dev/null
|
|
862
|
+
+++ b/packages/client/ui-conversation/src/client/chat/TtsVoiceCard.module.css
|
|
863
|
+
@@ -0,0 +1,43 @@
|
|
864
|
+
+/* Synthesized voice-reply pill (voice-iron-rule): left-aligned row under the
|
|
865
|
+
+ last assistant message, mirroring the user voice card's playback chrome. */
|
|
866
|
+
+
|
|
867
|
+
+.card {
|
|
868
|
+
+ display: inline-flex;
|
|
869
|
+
+ align-items: center;
|
|
870
|
+
+ gap: 8px;
|
|
871
|
+
+ height: 44px;
|
|
872
|
+
+ padding: 0 16px 0 10px;
|
|
873
|
+
+ border: 1px solid var(--dsw-alias-border-l1);
|
|
874
|
+
+ border-radius: 22px;
|
|
875
|
+
+ background: var(--dsw-alias-markdown-code-block);
|
|
876
|
+
+}
|
|
877
|
+
+
|
|
878
|
+
+.play {
|
|
879
|
+
+ display: grid;
|
|
880
|
+
+ place-items: center;
|
|
881
|
+
+ flex: none;
|
|
882
|
+
+ width: 28px;
|
|
883
|
+
+ height: 28px;
|
|
884
|
+
+ border: none;
|
|
885
|
+
+ border-radius: 999px;
|
|
886
|
+
+ background: var(--dsw-specific-selector);
|
|
887
|
+
+ color: var(--dsw-alias-label-primary);
|
|
888
|
+
+ cursor: pointer;
|
|
889
|
+
+}
|
|
890
|
+
+
|
|
891
|
+
+.play:hover {
|
|
892
|
+
+ background: var(--dsw-alias-interactive-bg-hover-solid);
|
|
893
|
+
+}
|
|
894
|
+
+
|
|
895
|
+
+.duration {
|
|
896
|
+
+ color: var(--dsw-alias-label-secondary);
|
|
897
|
+
+ font-size: 13px;
|
|
898
|
+
+ line-height: 20px;
|
|
899
|
+
+ font-variant-numeric: tabular-nums;
|
|
900
|
+
+}
|
|
901
|
+
+
|
|
902
|
+
+.label {
|
|
903
|
+
+ color: var(--dsw-alias-label-tertiary);
|
|
904
|
+
+ font-size: 12px;
|
|
905
|
+
+ line-height: 20px;
|
|
906
|
+
+}
|
|
907
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/TtsVoiceCard.tsx b/packages/client/ui-conversation/src/client/chat/TtsVoiceCard.tsx
|
|
908
|
+
new file mode 100644
|
|
909
|
+
index 0000000000..4e409bde8b
|
|
910
|
+
--- /dev/null
|
|
911
|
+
+++ b/packages/client/ui-conversation/src/client/chat/TtsVoiceCard.tsx
|
|
912
|
+
@@ -0,0 +1,67 @@
|
|
913
|
+
+// TtsVoiceCard: locally synthesized reply audio (voice-iron-rule reply). The
|
|
914
|
+
+// bytes come back inline from voice.tts and play straight from an object URL —
|
|
915
|
+
+// nothing is stored in the session log, so the card is pure presentation.
|
|
916
|
+
+
|
|
917
|
+
+import { useEffect, useRef, useState } from 'react'
|
|
918
|
+
+import type { ChatViewSlotProps } from '../contract/slots.ts'
|
|
919
|
+
+import css from './TtsVoiceCard.module.css'
|
|
920
|
+
+
|
|
921
|
+
+/** Decode a base64 payload into a browser Blob URL. */
|
|
922
|
+
+function audioUrlOf(mediaType: string, data: string): string {
|
|
923
|
+
+ const binary = atob(data)
|
|
924
|
+
+ const bytes = new Uint8Array(binary.length)
|
|
925
|
+
+ for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index)
|
|
926
|
+
+ return URL.createObjectURL(new Blob([bytes.buffer], { type: mediaType }))
|
|
927
|
+
+}
|
|
928
|
+
+
|
|
929
|
+
+/** Synthesized voice reply pill: play/pause + estimated duration. */
|
|
930
|
+
+export function TtsVoiceCard({ mediaType, data, durationMs, t }: {
|
|
931
|
+
+ mediaType: string
|
|
932
|
+
+ data: string
|
|
933
|
+
+ durationMs?: number
|
|
934
|
+
+ t: ChatViewSlotProps['t']
|
|
935
|
+
+}) {
|
|
936
|
+
+ const [url] = useState(() => audioUrlOf(mediaType, data))
|
|
937
|
+
+ const [playing, setPlaying] = useState(false)
|
|
938
|
+
+ const audioRef = useRef<HTMLAudioElement | null>(null)
|
|
939
|
+
+ useEffect(() => () => { URL.revokeObjectURL(url) }, [url])
|
|
940
|
+
+ const toggle = (): void => {
|
|
941
|
+
+ const audio = audioRef.current
|
|
942
|
+
+ if (audio === null) return
|
|
943
|
+
+ if (playing) audio.pause()
|
|
944
|
+
+ else void audio.play().catch(() => { setPlaying(false) })
|
|
945
|
+
+ }
|
|
946
|
+
+ const seconds = durationMs === undefined ? null : Math.max(1, Math.ceil(durationMs / 1_000))
|
|
947
|
+
+ return (
|
|
948
|
+
+ <div className={css.card} data-tts-voice>
|
|
949
|
+
+ <button
|
|
950
|
+
+ type="button"
|
|
951
|
+
+ className={css.play}
|
|
952
|
+
+ aria-label={playing ? t('voice.pause') : t('voice.play')}
|
|
953
|
+
+ onClick={toggle}
|
|
954
|
+
+ >
|
|
955
|
+
+ {playing
|
|
956
|
+
+ ? (
|
|
957
|
+
+ <svg viewBox="0 0 16 16" width="14" height="14" aria-hidden>
|
|
958
|
+
+ <rect x="3.5" y="3.5" width="3" height="9" rx="1" fill="currentColor"/>
|
|
959
|
+
+ <rect x="9.5" y="3.5" width="3" height="9" rx="1" fill="currentColor"/>
|
|
960
|
+
+ </svg>
|
|
961
|
+
+ )
|
|
962
|
+
+ : (
|
|
963
|
+
+ <svg viewBox="0 0 16 16" width="14" height="14" aria-hidden>
|
|
964
|
+
+ <path d="M5 3.5L12.5 8L5 12.5V3.5Z" fill="currentColor"/>
|
|
965
|
+
+ </svg>
|
|
966
|
+
+ )}
|
|
967
|
+
+ </button>
|
|
968
|
+
+ <span className={css.duration}>{seconds === null ? '' : `${seconds}s`}</span>
|
|
969
|
+
+ <span className={css.label}>{t('voice.reply')}</span>
|
|
970
|
+
+ <audio
|
|
971
|
+
+ ref={audioRef}
|
|
972
|
+
+ src={url}
|
|
973
|
+
+ onPlay={() => { setPlaying(true) }}
|
|
974
|
+
+ onPause={() => { setPlaying(false) }}
|
|
975
|
+
+ onEnded={() => { setPlaying(false) }}
|
|
976
|
+
+ />
|
|
977
|
+
+ </div>
|
|
978
|
+
+ )
|
|
979
|
+
+}
|
|
980
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/VoiceReplyNodeView.tsx b/packages/client/ui-conversation/src/client/chat/VoiceReplyNodeView.tsx
|
|
981
|
+
new file mode 100644
|
|
982
|
+
index 0000000000..8fd17edbbe
|
|
983
|
+
--- /dev/null
|
|
984
|
+
+++ b/packages/client/ui-conversation/src/client/chat/VoiceReplyNodeView.tsx
|
|
985
|
+
@@ -0,0 +1,20 @@
|
|
986
|
+
+// VoiceReplyNodeView: the assistant's synthesized voice reply as its own
|
|
987
|
+
+// durable chat row — a standalone voice bar (play button + duration), mirroring
|
|
988
|
+
+// the user's voice messages instead of being buried inside the text reply.
|
|
989
|
+
+
|
|
990
|
+
+import { memo } from 'react'
|
|
991
|
+
+import type { ChatNodeViewProps } from '../contract/slots.ts'
|
|
992
|
+
+import { VoiceCard } from './MessageItem.tsx'
|
|
993
|
+
+import css from './MessageItem.module.css'
|
|
994
|
+
+
|
|
995
|
+
+/** Assistant voice-reply keyed Chat renderer: one standalone voice bar. */
|
|
996
|
+
+export const VoiceReplyNodeView = memo(function VoiceReplyNodeView({
|
|
997
|
+
+ node, loadVoice, t,
|
|
998
|
+
+}: ChatNodeViewProps<'voice-reply'>) {
|
|
999
|
+
+ const { voice } = node.data
|
|
1000
|
+
+ return (
|
|
1001
|
+
+ <div className={css.voiceReplyRow} data-voice-reply>
|
|
1002
|
+
+ <VoiceCard attachment={voice} load={loadVoice} t={t} />
|
|
1003
|
+
+ </div>
|
|
1004
|
+
+ )
|
|
1005
|
+
+})
|
|
1006
|
+
diff --git a/packages/client/ui-conversation/src/client/chat/register-node-renderers.ts b/packages/client/ui-conversation/src/client/chat/register-node-renderers.ts
|
|
1007
|
+
index ed311e4b74..ab63cde601 100644
|
|
1008
|
+
--- a/packages/client/ui-conversation/src/client/chat/register-node-renderers.ts
|
|
1009
|
+
+++ b/packages/client/ui-conversation/src/client/chat/register-node-renderers.ts
|
|
1010
|
+
@@ -7,16 +7,31 @@ import {
|
|
1011
|
+
TurnMaxTokensNodeView, UnknownNodeView, UserMessageNodeView,
|
|
1012
|
+
} from './MessageItem.tsx'
|
|
1013
|
+
import { TurnTailNodeView } from './TurnTailNodeView.tsx'
|
|
1014
|
+
+import { VoiceReplyNodeView } from './VoiceReplyNodeView.tsx'
|
|
1015
|
+
|
|
1016
|
+
/**
|
|
1017
|
+
* Register this package's business renderers behind the keyed Chat Node seat.
|
|
1018
|
+
* @param ctx - owning UI Conversation context.
|
|
1019
|
+
*/
|
|
1020
|
+
export function registerChatNodeRenderers(ctx: Context): void {
|
|
1021
|
+
- ctx.slots.inject('conversation.chat.node', () => ctx.slots.register(
|
|
1022
|
+
- { name: 'conversation.chat.node', key: 'user', locale: NS }, UserMessageNodeView))
|
|
1023
|
+
- ctx.slots.inject('conversation.chat.node', () => ctx.slots.register(
|
|
1024
|
+
- { name: 'conversation.chat.node', key: 'steering', locale: NS }, UserMessageNodeView))
|
|
1025
|
+
+ ctx.slots.inject('conversation.chat.node', () => ctx.slots.register({
|
|
1026
|
+
+ name: 'conversation.chat.node',
|
|
1027
|
+
+ key: 'user',
|
|
1028
|
+
+ locale: NS,
|
|
1029
|
+
+ children: {
|
|
1030
|
+
+ 'conversation.chat.user-actions': { kind: 'list', scope: 'session' },
|
|
1031
|
+
+ 'conversation.chat.voice-actions': { kind: 'list', scope: 'session' },
|
|
1032
|
+
+ },
|
|
1033
|
+
+ }, UserMessageNodeView))
|
|
1034
|
+
+ ctx.slots.inject('conversation.chat.node', () => ctx.slots.register({
|
|
1035
|
+
+ name: 'conversation.chat.node',
|
|
1036
|
+
+ key: 'steering',
|
|
1037
|
+
+ locale: NS,
|
|
1038
|
+
+ children: {
|
|
1039
|
+
+ 'conversation.chat.user-actions': { kind: 'list', scope: 'session' },
|
|
1040
|
+
+ 'conversation.chat.voice-actions': { kind: 'list', scope: 'session' },
|
|
1041
|
+
+ },
|
|
1042
|
+
+ }, UserMessageNodeView))
|
|
1043
|
+
ctx.slots.inject('conversation.chat.node', () => ctx.slots.register(
|
|
1044
|
+
{ name: 'conversation.chat.node', key: 'context', locale: NS }, ContextMessageNodeView))
|
|
1045
|
+
ctx.slots.inject('conversation.chat.node', () => ctx.slots.register(
|
|
1046
|
+
@@ -46,6 +61,8 @@ export function registerChatNodeRenderers(ctx: Context): void {
|
|
1047
|
+
'conversation.chat.assistant-actions': { kind: 'list', scope: 'session' },
|
|
1048
|
+
},
|
|
1049
|
+
}, TurnTailNodeView))
|
|
1050
|
+
+ ctx.slots.inject('conversation.chat.node', () => ctx.slots.register(
|
|
1051
|
+
+ { name: 'conversation.chat.node', key: 'voice-reply', locale: NS }, VoiceReplyNodeView))
|
|
1052
|
+
ctx.slots.inject('conversation.chat.node', () => ctx.slots.register(
|
|
1053
|
+
{ name: 'conversation.chat.node', key: 'unknown', locale: NS }, UnknownNodeView))
|
|
1054
|
+
}
|
|
1055
|
+
diff --git a/packages/client/ui-conversation/src/client/contract/slots.ts b/packages/client/ui-conversation/src/client/contract/slots.ts
|
|
1056
|
+
index 4ea0c79402..a7a8e7b60f 100644
|
|
1057
|
+
--- a/packages/client/ui-conversation/src/client/contract/slots.ts
|
|
1058
|
+
+++ b/packages/client/ui-conversation/src/client/contract/slots.ts
|
|
1059
|
+
@@ -11,7 +11,7 @@ import type {
|
|
1060
|
+
TurnLocation, WorkspaceId,
|
|
1061
|
+
} from '@deepseek-ai/dsh-client-runtime/client'
|
|
1062
|
+
import type { MarkdownFileMentions } from '@deepseek-ai/dsh-client-ui-primitives'
|
|
1063
|
+
-import type { MessageId } from '@deepseek-ai/dsh-client-connection/client'
|
|
1064
|
+
+import type { BalanceView, MessageId, PromptContentPart, VoiceAttachmentRef } from '@deepseek-ai/dsh-client-connection/client'
|
|
1065
|
+
import type {} from '@deepseek-ai/dsh-client-ui-layout/client'
|
|
1066
|
+
import type { ComposerBlock } from '../input/blocks.ts'
|
|
1067
|
+
import type {
|
|
1068
|
+
@@ -140,6 +140,32 @@ declare module '@deepseek-ai/dsh-client-ui-slots' {
|
|
1069
|
+
scope: 'session'
|
|
1070
|
+
owner: AssistantActionOwnerProps
|
|
1071
|
+
}
|
|
1072
|
+
+ /**
|
|
1073
|
+
+ * Action strip attached to one finalized user message, rendered inside
|
|
1074
|
+
+ * that message's IconActions row. Mirror of the assistant-actions seat:
|
|
1075
|
+
+ * user messages carry no messageId, so the owner addresses the message by
|
|
1076
|
+
+ * its stable `seq` and hands the voice transcripts (the rendered
|
|
1077
|
+
+ * transcript text lives only on `attachment.transcript`, not in any text
|
|
1078
|
+
+ * block — contributors that copy a voice message need this directly).
|
|
1079
|
+
+ * Entries render by ascending `order`.
|
|
1080
|
+
+ */
|
|
1081
|
+
+ 'conversation.chat.user-actions': {
|
|
1082
|
+
+ kind: 'list'
|
|
1083
|
+
+ scope: 'session'
|
|
1084
|
+
+ owner: UserActionOwnerProps
|
|
1085
|
+
+ }
|
|
1086
|
+
+ /**
|
|
1087
|
+
+ * Actions rendered at the tail of one voice card (inside the card, after
|
|
1088
|
+
+ * the transcript label), addressed by message `seq` + card `index` and
|
|
1089
|
+
+ * handed the single card's transcript. This is where a "copy transcript"
|
|
1090
|
+
+ * affordance belongs — inside the voice bar itself, not in the message
|
|
1091
|
+
+ * action row. Entries render by ascending `order`.
|
|
1092
|
+
+ */
|
|
1093
|
+
+ 'conversation.chat.voice-actions': {
|
|
1094
|
+
+ kind: 'list'
|
|
1095
|
+
+ scope: 'session'
|
|
1096
|
+
+ owner: VoiceActionOwnerProps
|
|
1097
|
+
+ }
|
|
1098
|
+
/**
|
|
1099
|
+
* The body of the details panel for the tool call the user selected —
|
|
1100
|
+
* one occupant, so taking it means rendering every tool's output, not just
|
|
1101
|
+
@@ -380,6 +406,24 @@ export interface AssistantActionOwnerProps {
|
|
1102
|
+
messageId: MessageId
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
+/** Owner currency of one finalized user message's action strip. */
|
|
1106
|
+
+export interface UserActionOwnerProps {
|
|
1107
|
+
+ /** User messages carry no messageId; `seq` is the stable message identity. */
|
|
1108
|
+
+ seq: number
|
|
1109
|
+
+ /** Non-empty voice transcripts on this message, in attachment order. */
|
|
1110
|
+
+ voiceTranscripts: readonly string[]
|
|
1111
|
+
+}
|
|
1112
|
+
+
|
|
1113
|
+
+/** Owner currency of one voice card's tail actions. */
|
|
1114
|
+
+export interface VoiceActionOwnerProps {
|
|
1115
|
+
+ /** Stable message identity of the containing user message. */
|
|
1116
|
+
+ seq: number
|
|
1117
|
+
+ /** Zero-based index of the voice card within the message. */
|
|
1118
|
+
+ index: number
|
|
1119
|
+
+ /** This card's transcript (may be empty when ASR produced none). */
|
|
1120
|
+
+ transcript: string
|
|
1121
|
+
+}
|
|
1122
|
+
+
|
|
1123
|
+
/** Hook constrained to business data published on the current Chat Node's Turn. */
|
|
1124
|
+
export type UseChatNodeTurnData = <Key extends Extract<keyof ConversationTurnDataMap, string>>(
|
|
1125
|
+
key: Key,
|
|
1126
|
+
@@ -403,6 +447,10 @@ export interface ChatNodeOwnerProps {
|
|
1127
|
+
forkAt: (seq: number) => void
|
|
1128
|
+
/** Render a historical image group through the attachment slot. */
|
|
1129
|
+
renderMessageImages: RenderMessageImages
|
|
1130
|
+
+ /** Resolve a session-authorized historical image for inline display. */
|
|
1131
|
+
+ loadImage: (attachment: ImageAttachmentRef) => Promise<string>
|
|
1132
|
+
+ /** Resolve a session-authorized historical voice object for inline playback. */
|
|
1133
|
+
+ loadVoice: (attachment: VoiceAttachmentRef) => Promise<string>
|
|
1134
|
+
fileMentions: (owner: TurnTailOwnerProps) => MarkdownFileMentions | undefined
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
@@ -474,6 +522,8 @@ export interface ConversationSessionInjected {
|
|
1138
|
+
}
|
|
1139
|
+
/** Release historical image URLs when this rendered session scope unmounts. */
|
|
1140
|
+
releaseSessionImages: (sessionId: SessionId) => void
|
|
1141
|
+
+ /** Release historical voice URLs when this rendered session scope unmounts. */
|
|
1142
|
+
+ releaseSessionVoices: (sessionId: SessionId) => void
|
|
1143
|
+
/** Bind the input machine's draft persistence mirror to the session store. */
|
|
1144
|
+
bindDraftMirror: (write: (text: string) => void) => () => void
|
|
1145
|
+
}
|
|
1146
|
+
@@ -556,6 +606,35 @@ export interface ComposerBarInjected {
|
|
1147
|
+
* Resolves admission: false = rejected/unmatched/transport failure.
|
|
1148
|
+
*/
|
|
1149
|
+
command: ((line: string) => Promise<boolean>) | undefined
|
|
1150
|
+
+ /**
|
|
1151
|
+
+ * 查询 DeepSeek 直连账户余额(模型按钮旁的余额指示);非直连/无 key
|
|
1152
|
+
+ * 返回 null。absent 仅当没有 session(组件据此隐藏)。
|
|
1153
|
+
+ */
|
|
1154
|
+
+ readBalance: (() => Promise<{ balance: BalanceView | null }>) | undefined
|
|
1155
|
+
+ /**
|
|
1156
|
+
+ * 发送一条录音语音消息(host 落盘后自动 ASR 转文本,agent 按文本回复);
|
|
1157
|
+
+ * absent 仅当没有 session。
|
|
1158
|
+
+ * Resolves admission: false = rejected/transport failure.
|
|
1159
|
+
+ * @param mode - queue or steer delivery, resolved by composer policy so the
|
|
1160
|
+
+ * user's busy-state preference (设置里的"排队/插队") applies to voice too.
|
|
1161
|
+
+ */
|
|
1162
|
+
+ sendVoice: ((part: Extract<PromptContentPart, { type: 'voice' }>, mode: InputSubmitMode) => Promise<boolean>) | undefined
|
|
1163
|
+
+ /**
|
|
1164
|
+
+ * 即时转写一条录音("说话转文本"手势:上滑右侧松手时调用,结果作为普通
|
|
1165
|
+
+ * 文本消息发送,音频不落库);absent 仅当没有 session。
|
|
1166
|
+
+ * @returns recognized text, or null when recognition failed.
|
|
1167
|
+
+ */
|
|
1168
|
+
+ transcribeVoice: ((part: Extract<PromptContentPart, { type: 'voice' }>) => Promise<string | null>) | undefined
|
|
1169
|
+
+ /**
|
|
1170
|
+
+ * 合成一条语音回复(本地 TTS 引擎),返回可直接播放的音频数据;
|
|
1171
|
+
+ * absent 仅当没有 session。用于"语音铁律"自动回复和手动语音按钮。
|
|
1172
|
+
+ * @returns encoded audio, or null when synthesis failed.
|
|
1173
|
+
+ */
|
|
1174
|
+
+ synthesizeVoice: ((text: string, provider?: string) => Promise<{
|
|
1175
|
+
+ mediaType: string
|
|
1176
|
+
+ data: string
|
|
1177
|
+
+ durationMs?: number
|
|
1178
|
+
+ } | null>) | undefined
|
|
1179
|
+
/**
|
|
1180
|
+
* Registrant hooks compartment: the renderer binds these to
|
|
1181
|
+
* useNotices/useLexicon (static absent sources without a session — hook
|
|
1182
|
+
@@ -736,6 +815,17 @@ export interface ChatViewInjected {
|
|
1183
|
+
loadOlder: () => void
|
|
1184
|
+
/** Resolve a session-authorized historical image for inline display. */
|
|
1185
|
+
loadImage: (attachment: ImageAttachmentRef) => Promise<string>
|
|
1186
|
+
+ /** Resolve a session-authorized historical voice object for inline playback. */
|
|
1187
|
+
+ loadVoice: (attachment: VoiceAttachmentRef) => Promise<string>
|
|
1188
|
+
+ /**
|
|
1189
|
+
+ * 合成语音回复(本地 TTS;"语音铁律":上一条用户消息是语音时,回复完成后
|
|
1190
|
+
+ * 自动调用并播放)。返回可直接播放的音频数据;失败返回 null。
|
|
1191
|
+
+ */
|
|
1192
|
+
+ synthesizeVoice: ((text: string, provider?: string) => Promise<{
|
|
1193
|
+
+ mediaType: string
|
|
1194
|
+
+ data: string
|
|
1195
|
+
+ durationMs?: number
|
|
1196
|
+
+ } | null>) | undefined
|
|
1197
|
+
/** Hand a call off to the trajectory view: write the one-shot inspect target and switch tabs. */
|
|
1198
|
+
inspectCall: (callId: CallId) => void
|
|
1199
|
+
/**
|
|
1200
|
+
diff --git a/packages/client/ui-conversation/src/client/conversation-nodes/register.ts b/packages/client/ui-conversation/src/client/conversation-nodes/register.ts
|
|
1201
|
+
index 5086253e81..79156231cc 100644
|
|
1202
|
+
--- a/packages/client/ui-conversation/src/client/conversation-nodes/register.ts
|
|
1203
|
+
+++ b/packages/client/ui-conversation/src/client/conversation-nodes/register.ts
|
|
1204
|
+
@@ -11,6 +11,7 @@ import { registerToolConversationNode } from './tool.ts'
|
|
1205
|
+
import { registerTurnErrorConversationNode } from './turn-error.ts'
|
|
1206
|
+
import { registerTurnMaxTokensConversationNode } from './turn-max-tokens.ts'
|
|
1207
|
+
import { registerTurnTailConversationNode } from './turn-tail.ts'
|
|
1208
|
+
+import { registerVoiceReplyConversationNode } from './voice-reply.ts'
|
|
1209
|
+
|
|
1210
|
+
/**
|
|
1211
|
+
* Register the Chat business Definitions and target builder contributed by this package.
|
|
1212
|
+
@@ -27,6 +28,7 @@ export function registerConversationNodes(ctx: Context): void {
|
|
1213
|
+
registerTurnErrorConversationNode(ctx)
|
|
1214
|
+
registerTurnMaxTokensConversationNode(ctx)
|
|
1215
|
+
registerTurnTailConversationNode(ctx)
|
|
1216
|
+
+ registerVoiceReplyConversationNode(ctx)
|
|
1217
|
+
registerUnknownConversationFallback(ctx)
|
|
1218
|
+
registerChatConversationView(ctx)
|
|
1219
|
+
}
|
|
1220
|
+
diff --git a/packages/client/ui-conversation/src/client/conversation-nodes/voice-reply.ts b/packages/client/ui-conversation/src/client/conversation-nodes/voice-reply.ts
|
|
1221
|
+
new file mode 100644
|
|
1222
|
+
index 0000000000..e7312b797f
|
|
1223
|
+
--- /dev/null
|
|
1224
|
+
+++ b/packages/client/ui-conversation/src/client/conversation-nodes/voice-reply.ts
|
|
1225
|
+
@@ -0,0 +1,79 @@
|
|
1226
|
+
+import type { Context } from '@deepseek-ai/cordis'
|
|
1227
|
+
+import type {
|
|
1228
|
+
+ ConversationMatch, ConversationNodeDefinition,
|
|
1229
|
+
+} from '@deepseek-ai/dsh-client-runtime/client'
|
|
1230
|
+
+import type { VoiceAttachmentRef } from '@deepseek-ai/dsh-client-connection/client'
|
|
1231
|
+
+import { chatNode } from './common.ts'
|
|
1232
|
+
+
|
|
1233
|
+
+/** One assistant voice-reply row's durable payload (from the voice/reply event). */
|
|
1234
|
+
+export interface VoiceReplyChatData {
|
|
1235
|
+
+ readonly turn: number
|
|
1236
|
+
+ readonly seq: number
|
|
1237
|
+
+ readonly time: number
|
|
1238
|
+
+ readonly voice: VoiceAttachmentRef
|
|
1239
|
+
+}
|
|
1240
|
+
+
|
|
1241
|
+
+declare module '@deepseek-ai/dsh-client-ui-conversation/client' {
|
|
1242
|
+
+ interface ChatNodeDataMap {
|
|
1243
|
+
+ /** Assistant's synthesized voice reply, persisted beside the user's voice messages. */
|
|
1244
|
+
+ 'voice-reply': VoiceReplyChatData
|
|
1245
|
+
+ }
|
|
1246
|
+
+}
|
|
1247
|
+
+
|
|
1248
|
+
+interface VoiceReplyState {
|
|
1249
|
+
+ readonly turn: number
|
|
1250
|
+
+ readonly seq: number
|
|
1251
|
+
+ readonly time: number
|
|
1252
|
+
+ readonly voice: VoiceAttachmentRef
|
|
1253
|
+
+}
|
|
1254
|
+
+
|
|
1255
|
+
+function stateFrom(match: ConversationMatch): VoiceReplyState | undefined {
|
|
1256
|
+
+ if (match.event.type !== 'voice/reply') return undefined
|
|
1257
|
+
+ const { turn, voiceId, mediaType, bytes, durationMs, transcript } = match.event.data
|
|
1258
|
+
+ return {
|
|
1259
|
+
+ turn,
|
|
1260
|
+
+ seq: match.event.seq,
|
|
1261
|
+
+ time: match.event.time,
|
|
1262
|
+
+ voice: {
|
|
1263
|
+
+ voiceId,
|
|
1264
|
+
+ mediaType: mediaType as VoiceAttachmentRef['mediaType'],
|
|
1265
|
+
+ bytes,
|
|
1266
|
+
+ ...(durationMs === undefined ? {} : { durationMs }),
|
|
1267
|
+
+ // [本地改造 2026-08-21] AI 语音回复的转写文本(合成的正文),供语音条显示与复制
|
|
1268
|
+
+ ...(typeof transcript === 'string' && transcript !== '' ? { transcript } : {}),
|
|
1269
|
+
+ },
|
|
1270
|
+
+ }
|
|
1271
|
+
+}
|
|
1272
|
+
+
|
|
1273
|
+
+/** [本地改造 2026-08-16] 助手语音回复独立横条:跟随 voice/reply 事件渲染,
|
|
1274
|
+
+ * 与用户语音消息同级持久化(可回放、可翻查),不混在文字回复里。
|
|
1275
|
+
+ * 同一 turn 可有多条语音回复(自动回复 + 主动发送),match id 用事件 seq
|
|
1276
|
+
+ * 保证每条唯一,避免 assembler "more than one start Match" 崩溃。 */
|
|
1277
|
+
+export const voiceReplyDefinition: ConversationNodeDefinition<VoiceReplyState> = {
|
|
1278
|
+
+ kind: 'voice-reply',
|
|
1279
|
+
+ target: 'chat',
|
|
1280
|
+
+ match: (event) => {
|
|
1281
|
+
+ if (event.type === 'voice/reply') return { id: String(event.seq), role: 'start' }
|
|
1282
|
+
+ return null
|
|
1283
|
+
+ },
|
|
1284
|
+
+ start: (_context, match) => {
|
|
1285
|
+
+ const state = stateFrom(match)
|
|
1286
|
+
+ if (state === undefined) throw new Error('voice-reply start requires a voice/reply event')
|
|
1287
|
+
+ return state
|
|
1288
|
+
+ },
|
|
1289
|
+
+ update: context => context.state,
|
|
1290
|
+
+ buildViewNode: (context) => {
|
|
1291
|
+
+ const state = context.state
|
|
1292
|
+
+ if (state === undefined) return null
|
|
1293
|
+
+ const data: VoiceReplyChatData = { turn: state.turn, seq: state.seq, time: state.time, voice: state.voice }
|
|
1294
|
+
+ return chatNode(context, 'voice-reply', state.seq, data)
|
|
1295
|
+
+ },
|
|
1296
|
+
+}
|
|
1297
|
+
+
|
|
1298
|
+
+/**
|
|
1299
|
+
+ * Register the assistant voice-reply contribution.
|
|
1300
|
+
+ * @param ctx - owning UI Conversation context.
|
|
1301
|
+
+ */
|
|
1302
|
+
+export function registerVoiceReplyConversationNode(ctx: Context): void {
|
|
1303
|
+
+ ctx.conversationEvents.register(voiceReplyDefinition)
|
|
1304
|
+
+}
|
|
1305
|
+
diff --git a/packages/client/ui-conversation/src/client/index.ts b/packages/client/ui-conversation/src/client/index.ts
|
|
1306
|
+
index 814734411b..5d8e0082b3 100644
|
|
1307
|
+
--- a/packages/client/ui-conversation/src/client/index.ts
|
|
1308
|
+
+++ b/packages/client/ui-conversation/src/client/index.ts
|
|
1309
|
+
@@ -13,6 +13,7 @@ export type {} from './conversation-nodes/tool.ts'
|
|
1310
|
+
export type {} from './conversation-nodes/turn-error.ts'
|
|
1311
|
+
export type {} from './conversation-nodes/turn-max-tokens.ts'
|
|
1312
|
+
export type {} from './conversation-nodes/turn-tail.ts'
|
|
1313
|
+
+export type {} from './conversation-nodes/voice-reply.ts'
|
|
1314
|
+
|
|
1315
|
+
export { apply, inject } from './apply.ts'
|
|
1316
|
+
export { ConversationController } from './service.ts'
|
|
1317
|
+
diff --git a/packages/client/ui-conversation/src/client/locales.ts b/packages/client/ui-conversation/src/client/locales.ts
|
|
1318
|
+
index f4e7a7c59a..8e7f378d52 100644
|
|
1319
|
+
--- a/packages/client/ui-conversation/src/client/locales.ts
|
|
1320
|
+
+++ b/packages/client/ui-conversation/src/client/locales.ts
|
|
1321
|
+
@@ -50,6 +50,26 @@ export const zh = {
|
|
1322
|
+
'image.modelUnsupported': '当前模型不支持图片,请切换支持图片的模型',
|
|
1323
|
+
'image.subagentUnsupported': '子智能体会话暂不支持图片',
|
|
1324
|
+
'image.sendFailed': '图片发送失败({reason}),请重新添加图片后再试',
|
|
1325
|
+
+ 'voice.start': '点击开始录音,再点停止发送',
|
|
1326
|
+
+ 'voice.releaseToSend': '松手发送 · 上滑取消 / 转文字',
|
|
1327
|
+
+ 'voice.gestureSend': '松手发送',
|
|
1328
|
+
+ 'voice.gestureCancel': '松手取消',
|
|
1329
|
+
+ 'voice.gestureText': '松手转文字',
|
|
1330
|
+
+ 'voice.recognizing': '语音识别中…',
|
|
1331
|
+
+ 'voice.recognitionFailed': '语音转文字失败,请重试',
|
|
1332
|
+
+ 'voice.stopAndSend': '停止并发送语音',
|
|
1333
|
+
+ 'voice.cancel': '取消录音',
|
|
1334
|
+
+ 'voice.unsupportedFormat': '当前浏览器录音格式不支持,请更换浏览器',
|
|
1335
|
+
+ 'voice.sendFailed': '语音发送失败,请重试',
|
|
1336
|
+
+ 'voice.empty': '未录到声音,请重试',
|
|
1337
|
+
+ 'voice.micUnavailable': '无法访问麦克风,请检查权限设置',
|
|
1338
|
+
+ 'voice.play': '播放语音',
|
|
1339
|
+
+ 'voice.pause': '暂停播放',
|
|
1340
|
+
+ 'voice.transcriptLabel': '语音识别文本',
|
|
1341
|
+
+ 'voice.asrFailed': '未能识别(ASR 未配置或识别失败)',
|
|
1342
|
+
+ 'voice.loadFailed': '语音加载失败',
|
|
1343
|
+
+ 'voice.synthesizing': '正在合成语音回复…',
|
|
1344
|
+
+ 'voice.reply': '语音回复',
|
|
1345
|
+
'context.aria': '上下文已用 {percent}',
|
|
1346
|
+
'context.used': '上下文已用',
|
|
1347
|
+
'context.system': '系统提示词',
|
|
1348
|
+
@@ -227,6 +247,26 @@ export const en = {
|
|
1349
|
+
'image.modelUnsupported': 'The current model does not support images; switch to a model that does',
|
|
1350
|
+
'image.subagentUnsupported': 'Subagent sessions do not support images yet',
|
|
1351
|
+
'image.sendFailed': 'Sending images failed ({reason}); re-add them and try again',
|
|
1352
|
+
+ 'voice.start': 'Click to record; click again to send',
|
|
1353
|
+
+ 'voice.releaseToSend': 'Release to send · slide up to cancel / as text',
|
|
1354
|
+
+ 'voice.gestureSend': 'Release to send',
|
|
1355
|
+
+ 'voice.gestureCancel': 'Release to cancel',
|
|
1356
|
+
+ 'voice.gestureText': 'Release as text',
|
|
1357
|
+
+ 'voice.recognizing': 'Recognizing speech…',
|
|
1358
|
+
+ 'voice.recognitionFailed': 'Speech-to-text failed; try again',
|
|
1359
|
+
+ 'voice.stopAndSend': 'Stop and send voice',
|
|
1360
|
+
+ 'voice.cancel': 'Cancel recording',
|
|
1361
|
+
+ 'voice.unsupportedFormat': 'This browser cannot record supported audio; try another browser',
|
|
1362
|
+
+ 'voice.sendFailed': 'Failed to send voice; try again',
|
|
1363
|
+
+ 'voice.empty': 'No audio captured; try again',
|
|
1364
|
+
+ 'voice.micUnavailable': 'Microphone unavailable; check your permissions',
|
|
1365
|
+
+ 'voice.play': 'Play voice',
|
|
1366
|
+
+ 'voice.pause': 'Pause playback',
|
|
1367
|
+
+ 'voice.transcriptLabel': 'Voice transcript',
|
|
1368
|
+
+ 'voice.asrFailed': 'Recognition failed (ASR not configured or errored)',
|
|
1369
|
+
+ 'voice.loadFailed': 'Failed to load voice',
|
|
1370
|
+
+ 'voice.synthesizing': 'Synthesizing voice reply…',
|
|
1371
|
+
+ 'voice.reply': 'Voice reply',
|
|
1372
|
+
'context.aria': '{percent} of context used',
|
|
1373
|
+
'context.used': 'of context used',
|
|
1374
|
+
'context.system': 'System prompt',
|
|
1375
|
+
diff --git a/packages/client/ui-conversation/src/client/service.ts b/packages/client/ui-conversation/src/client/service.ts
|
|
1376
|
+
index a37816472c..eb8b4dffe1 100644
|
|
1377
|
+
--- a/packages/client/ui-conversation/src/client/service.ts
|
|
1378
|
+
+++ b/packages/client/ui-conversation/src/client/service.ts
|
|
1379
|
+
@@ -15,6 +15,7 @@ import type { Context } from '@deepseek-ai/cordis'
|
|
1380
|
+
import type { ISessions, SessionFace, SessionId } from '@deepseek-ai/dsh-client-runtime/client'
|
|
1381
|
+
import type { SubmitImageAttachment, SubmitOutcome } from '@deepseek-ai/dsh-client-ui-input-trigger/client'
|
|
1382
|
+
import type { ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment'
|
|
1383
|
+
+import type { VoiceAttachmentRef } from '@deepseek-ai/dsh-api-remotes/client'
|
|
1384
|
+
import type { ComposerAttachment } from './contract/slots.ts'
|
|
1385
|
+
import type { QueueAction, QueueItemId } from './contract/queue.ts'
|
|
1386
|
+
import type { ComposerBlocks } from './input/blocks.ts'
|
|
1387
|
+
@@ -75,6 +76,12 @@ interface ImageUrlEntry {
|
|
1388
|
+
readonly pending: Promise<string>
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
+interface VoiceUrlEntry {
|
|
1392
|
+
+ readonly sessionId: SessionId
|
|
1393
|
+
+ readonly generation: number
|
|
1394
|
+
+ readonly pending: Promise<string>
|
|
1395
|
+
+}
|
|
1396
|
+
+
|
|
1397
|
+
/** Unsupported browser-declared image type, localized by the UI boundary. */
|
|
1398
|
+
export class UnsupportedImageMediaTypeError extends Error {
|
|
1399
|
+
/** Browser-declared MIME value, possibly empty. */
|
|
1400
|
+
@@ -98,6 +105,9 @@ export class ConversationController extends Service implements IConversation {
|
|
1401
|
+
private readonly imageUrls = new Map<string, ImageUrlEntry>()
|
|
1402
|
+
private readonly imageGenerations = new Map<SessionId, number>()
|
|
1403
|
+
private readonly createdImageUrls = new Set<string>()
|
|
1404
|
+
+ private readonly voiceUrls = new Map<string, VoiceUrlEntry>()
|
|
1405
|
+
+ private readonly voiceGenerations = new Map<SessionId, number>()
|
|
1406
|
+
+ private readonly createdVoiceUrls = new Set<string>()
|
|
1407
|
+
private disposed = false
|
|
1408
|
+
|
|
1409
|
+
/**
|
|
1410
|
+
@@ -115,9 +125,13 @@ export class ConversationController extends Service implements IConversation {
|
|
1411
|
+
this.disposed = true
|
|
1412
|
+
for (const url of this.createdImageUrls) revokePreview(url)
|
|
1413
|
+
this.createdImageUrls.clear()
|
|
1414
|
+
+ for (const url of this.createdVoiceUrls) revokePreview(url)
|
|
1415
|
+
+ this.createdVoiceUrls.clear()
|
|
1416
|
+
this.draftAttachments.clear()
|
|
1417
|
+
this.imageUrls.clear()
|
|
1418
|
+
this.imageGenerations.clear()
|
|
1419
|
+
+ this.voiceUrls.clear()
|
|
1420
|
+
+ this.voiceGenerations.clear()
|
|
1421
|
+
}, 'conversation attachment URL cache')
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
@@ -282,6 +296,63 @@ export class ConversationController extends Service implements IConversation {
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
+ /**
|
|
1429
|
+
+ * Resolve and cache one session-authorized historical voice URL.
|
|
1430
|
+
+ * @param sessionId - owning session authorization scope.
|
|
1431
|
+
+ * @param attachment - durable voice reference.
|
|
1432
|
+
+ * @returns browser URL valid until its rendered session is released.
|
|
1433
|
+
+ */
|
|
1434
|
+
+ resolveVoice(sessionId: SessionId, attachment: VoiceAttachmentRef): Promise<string> {
|
|
1435
|
+
+ if (this.disposed) return Promise.reject(new Error('conversation.resolveVoice: service is disposed'))
|
|
1436
|
+
+ const key = `${sessionId}:${attachment.voiceId}`
|
|
1437
|
+
+ const cached = this.voiceUrls.get(key)
|
|
1438
|
+
+ if (cached !== undefined) return cached.pending
|
|
1439
|
+
+ const generation = this.voiceGenerations.get(sessionId) ?? 0
|
|
1440
|
+
+ const session = this.requireSessions().binding(sessionId)?.session
|
|
1441
|
+
+ if (session === undefined) {
|
|
1442
|
+
+ return Promise.reject(new Error(`conversation.resolveVoice: unknown session "${sessionId}"`))
|
|
1443
|
+
+ }
|
|
1444
|
+
+ const pending = session.readVoice(attachment.voiceId)
|
|
1445
|
+
+ .then((result) => {
|
|
1446
|
+
+ if (!result.ok) throw new Error(`${result.error.code}: ${result.error.message}`)
|
|
1447
|
+
+ if (this.disposed) throw new Error('conversation.resolveVoice: service was disposed before loading completed')
|
|
1448
|
+
+ if ((this.voiceGenerations.get(sessionId) ?? 0) !== generation) {
|
|
1449
|
+
+ throw new Error('historical voice scope was released before loading completed')
|
|
1450
|
+
+ }
|
|
1451
|
+
+ if (typeof URL.createObjectURL !== 'function') {
|
|
1452
|
+
+ return `data:${result.value.attachment.mediaType};base64,${bytesToBase64(result.value.data)}`
|
|
1453
|
+
+ }
|
|
1454
|
+
+ const bytes = Uint8Array.from(result.value.data)
|
|
1455
|
+
+ const url = URL.createObjectURL(new Blob([bytes.buffer], { type: result.value.attachment.mediaType }))
|
|
1456
|
+
+ this.createdVoiceUrls.add(url)
|
|
1457
|
+
+ return url
|
|
1458
|
+
+ })
|
|
1459
|
+
+ .catch((error: unknown) => {
|
|
1460
|
+
+ if (this.voiceUrls.get(key)?.generation === generation) this.voiceUrls.delete(key)
|
|
1461
|
+
+ throw error
|
|
1462
|
+
+ })
|
|
1463
|
+
+ this.voiceUrls.set(key, { sessionId, generation, pending })
|
|
1464
|
+
+ return pending
|
|
1465
|
+
+ }
|
|
1466
|
+
+
|
|
1467
|
+
+ /**
|
|
1468
|
+
+ * Release every historical voice URL owned by one rendered session.
|
|
1469
|
+
+ * @param sessionId - rendered session scope.
|
|
1470
|
+
+ */
|
|
1471
|
+
+ releaseSessionVoices(sessionId: SessionId): void {
|
|
1472
|
+
+ this.voiceGenerations.set(sessionId, (this.voiceGenerations.get(sessionId) ?? 0) + 1)
|
|
1473
|
+
+ for (const [key, entry] of this.voiceUrls) {
|
|
1474
|
+
+ if (entry.sessionId !== sessionId) continue
|
|
1475
|
+
+ this.voiceUrls.delete(key)
|
|
1476
|
+
+ void entry.pending.then((url) => {
|
|
1477
|
+
+ if (!this.createdVoiceUrls.delete(url)) return
|
|
1478
|
+
+ revokePreview(url)
|
|
1479
|
+
+ }, () => {
|
|
1480
|
+
+ // A failed or invalidated load owns no object URL.
|
|
1481
|
+
+ })
|
|
1482
|
+
+ }
|
|
1483
|
+
+ }
|
|
1484
|
+
+
|
|
1485
|
+
/** Apply one operation to a pending queue occurrence. */
|
|
1486
|
+
async updateQueue(itemId: QueueItemId, action: QueueAction): Promise<void> {
|
|
1487
|
+
const session = this.scopedSession('updateQueue')
|
|
1488
|
+
diff --git a/packages/client/ui-conversation/src/client/skeleton/ConversationSession.tsx b/packages/client/ui-conversation/src/client/skeleton/ConversationSession.tsx
|
|
1489
|
+
index d576aa175d..e7e9b5d503 100644
|
|
1490
|
+
--- a/packages/client/ui-conversation/src/client/skeleton/ConversationSession.tsx
|
|
1491
|
+
+++ b/packages/client/ui-conversation/src/client/skeleton/ConversationSession.tsx
|
|
1492
|
+
@@ -137,7 +137,7 @@ export function ConversationSessionHeader({
|
|
1493
|
+
*/
|
|
1494
|
+
export function ConversationSession({
|
|
1495
|
+
sessionId, useSession, useInput, inputActions, useStore, actions,
|
|
1496
|
+
- renderSlot, views, bindDraftMirror, releaseSessionImages,
|
|
1497
|
+
+ renderSlot, views, bindDraftMirror, releaseSessionImages, releaseSessionVoices,
|
|
1498
|
+
}: ConversationSessionProps) {
|
|
1499
|
+
useSyncExternalStore(views.subscribe, views.version)
|
|
1500
|
+
const tabs = views.list()
|
|
1501
|
+
@@ -160,7 +160,15 @@ export function ConversationSession({
|
|
1502
|
+
|
|
1503
|
+
useEffect(() => () => {
|
|
1504
|
+
releaseSessionImages(sessionId)
|
|
1505
|
+
- }, [releaseSessionImages, sessionId])
|
|
1506
|
+
+ releaseSessionVoices(sessionId)
|
|
1507
|
+
+ // [本地修复 2026-08-16] 依赖只保留 sessionId:inject 的
|
|
1508
|
+
+ // releaseSessionImages/releaseSessionVoices 每次渲染都是新函数引用,
|
|
1509
|
+
+ // 若列入依赖,每次渲染都会先执行本 cleanup → voice/image generation
|
|
1510
|
+
+ // 持续递增 → 加载中的 resolveVoice/resolveImage 被
|
|
1511
|
+
+ // "historical voice scope was released" 拒绝 → 语音回复按钮变灰。
|
|
1512
|
+
+ // 只在切会话/卸载时释放一次即可,与 effect 语义一致。
|
|
1513
|
+
+ // eslint-disable-next-line react-hooks/exhaustive-deps
|
|
1514
|
+
+ }, [sessionId])
|
|
1515
|
+
|
|
1516
|
+
if (blank && composerPhase === 'blank') return null
|
|
1517
|
+
return (
|
|
1518
|
+
diff --git a/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css b/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css
|
|
1519
|
+
index e5b85df3e9..1a5b45deac 100644
|
|
1520
|
+
--- a/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css
|
|
1521
|
+
+++ b/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css
|
|
1522
|
+
@@ -270,7 +270,9 @@
|
|
1523
|
+
flex-wrap: wrap;
|
|
1524
|
+
align-items: center;
|
|
1525
|
+
justify-content: space-between;
|
|
1526
|
+
- gap: 12px;
|
|
1527
|
+
+ /* [本地改造 2026-08-16] 竖屏/窄宽时允许换行:工具按钮与右侧控制不会被挤折叠 */
|
|
1528
|
+
+ flex-wrap: wrap;
|
|
1529
|
+
+ gap: 6px 12px;
|
|
1530
|
+
/* 2px moved from the bottom pad to the top: the whole control row sits 2px
|
|
1531
|
+
lower in the card (it read too high against the textarea) while the card
|
|
1532
|
+
height and the controls' own centering stay untouched. */
|
|
1533
|
+
@@ -302,15 +304,13 @@
|
|
1534
|
+
gap: 12px;
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
+/* [本地改造 2026-08-16] trailing 间距收窄(模型/余额/上下文挤在一起,省垂直空间);
|
|
1538
|
+
+ 窄宽时可换行,避免竖屏折叠。 */
|
|
1539
|
+
.trailing {
|
|
1540
|
+
- flex: none;
|
|
1541
|
+
- /* Wrap keeps the left mode chips and the right controls apart when the card
|
|
1542
|
+
- runs out of row width: the trailing group (model + send) moves to its own
|
|
1543
|
+
- line instead of the left group shrinking until its chip overlaps the
|
|
1544
|
+
- model trigger (external:107). The auto margin re-anchors it right on the
|
|
1545
|
+
- wrapped line; on a single line space-between already pins it right. */
|
|
1546
|
+
- margin-left: auto;
|
|
1547
|
+
- gap: 12px;
|
|
1548
|
+
+ flex: 1 1 auto;
|
|
1549
|
+
+ justify-content: flex-end;
|
|
1550
|
+
+ flex-wrap: wrap;
|
|
1551
|
+
+ gap: 6px;
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
/* Attach circle (figma + control): 28px, selector fill, primary glyph. */
|
|
1555
|
+
@@ -336,6 +336,67 @@
|
|
1556
|
+
cursor: default;
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
+/* [本地改造 2026-08-16] 语音录音中:麦克风按钮换成红点计时(飞书/微信式状态)。 */
|
|
1560
|
+
+.voiceRecording {
|
|
1561
|
+
+ color: var(--dsw-alias-state-error-primary);
|
|
1562
|
+
+}
|
|
1563
|
+
+
|
|
1564
|
+
+.voiceRecording:not(:disabled) {
|
|
1565
|
+
+ background: var(--dsw-alias-interactive-bg-hover-danger);
|
|
1566
|
+
+}
|
|
1567
|
+
+
|
|
1568
|
+
+.voiceTimer {
|
|
1569
|
+
+ display: inline-flex;
|
|
1570
|
+
+ align-items: center;
|
|
1571
|
+
+ gap: 5px;
|
|
1572
|
+
+ font-size: 12px;
|
|
1573
|
+
+ line-height: 1;
|
|
1574
|
+
+ font-variant-numeric: tabular-nums;
|
|
1575
|
+
+}
|
|
1576
|
+
+
|
|
1577
|
+
+.voiceDot {
|
|
1578
|
+
+ width: 7px;
|
|
1579
|
+
+ height: 7px;
|
|
1580
|
+
+ border-radius: 999px;
|
|
1581
|
+
+ background: currentColor;
|
|
1582
|
+
+ animation: dsh-voice-pulse 1.2s ease-in-out infinite;
|
|
1583
|
+
+}
|
|
1584
|
+
+
|
|
1585
|
+
+@keyframes dsh-voice-pulse {
|
|
1586
|
+
+ 0%, 100% { opacity: 1; }
|
|
1587
|
+
+ 50% { opacity: 0.35; }
|
|
1588
|
+
+}
|
|
1589
|
+
+
|
|
1590
|
+
+@media (prefers-reduced-motion: reduce) {
|
|
1591
|
+
+ .voiceDot {
|
|
1592
|
+
+ animation: none;
|
|
1593
|
+
+ }
|
|
1594
|
+
+}
|
|
1595
|
+
+
|
|
1596
|
+
+/* [本地改造 2026-08-16] 录音手势浮层:按住说话时提示当前松手动作。 */
|
|
1597
|
+
+.voiceHint {
|
|
1598
|
+
+ display: inline-flex;
|
|
1599
|
+
+ align-items: center;
|
|
1600
|
+
+ height: 28px;
|
|
1601
|
+
+ padding: 0 10px;
|
|
1602
|
+
+ border-radius: 8px;
|
|
1603
|
+
+ background: var(--dsw-alias-interactive-bg-hover-solid);
|
|
1604
|
+
+ color: var(--dsw-alias-label-secondary);
|
|
1605
|
+
+ font-size: 12px;
|
|
1606
|
+
+ line-height: 1;
|
|
1607
|
+
+ white-space: nowrap;
|
|
1608
|
+
+}
|
|
1609
|
+
+
|
|
1610
|
+
+.voiceHint[data-gesture='cancel'] {
|
|
1611
|
+
+ color: var(--dsw-alias-state-error-primary);
|
|
1612
|
+
+ background: var(--dsw-alias-interactive-bg-hover-danger);
|
|
1613
|
+
+}
|
|
1614
|
+
+
|
|
1615
|
+
+.voiceHint[data-gesture='text'] {
|
|
1616
|
+
+ color: var(--dsw-alias-label-primary);
|
|
1617
|
+
+ background: var(--dsw-alias-interactive-bg-hover-solid);
|
|
1618
|
+
+}
|
|
1619
|
+
+
|
|
1620
|
+
/* Plan / Read-only / model — native <select>, chip-like closed chrome
|
|
1621
|
+
(figma ToggleButton: 13/20 medium secondary, 12px chevron). */
|
|
1622
|
+
.select {
|
|
1623
|
+
diff --git a/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx b/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx
|
|
1624
|
+
index 14feaca291..29c44da070 100644
|
|
1625
|
+
--- a/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx
|
|
1626
|
+
+++ b/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx
|
|
1627
|
+
@@ -12,21 +12,18 @@ import clsx from 'clsx'
|
|
1628
|
+
import {
|
|
1629
|
+
IconPlusOutline16, IconWarningOutline16, Toast, Tooltip,
|
|
1630
|
+
} from '@deepseek-ai/dsh-client-ui-primitives'
|
|
1631
|
+
-// Type-only: the `plan` projection key merge (the TodoDock posture — the
|
|
1632
|
+
-// composer reads a host-computed value; the domain owns the key).
|
|
1633
|
+
+import type { SessionId } from '@deepseek-ai/dsh-client-runtime/client'
|
|
1634
|
+
+import type { Translate } from '@deepseek-ai/dsh-client-locale/client'
|
|
1635
|
+
+// Type-only: pulls the `plan` SessionProjectionMap merge so useProjection('plan') type-checks.
|
|
1636
|
+
import type {} from '@deepseek-ai/dsh-plan-mode/client'
|
|
1637
|
+
-// Type-only: the `goal` projection key merge (hint disambiguation).
|
|
1638
|
+
-import type {} from '@deepseek-ai/dsh-goal/client'
|
|
1639
|
+
-// The `imageLimits` projection key merge (intake pre-check) arrives with the
|
|
1640
|
+
-// wire types: apiproxy's sessions contract declares it, and client-runtime's
|
|
1641
|
+
-// api-remotes import already places it in every client program.
|
|
1642
|
+
-import type { Translate } from '@deepseek-ai/dsh-client-ui-slots'
|
|
1643
|
+
import type { ComposerBarProps } from '../contract/slots.ts'
|
|
1644
|
+
import { deriveDecorations } from '../input/decorations.ts'
|
|
1645
|
+
import type { DraftDecorations } from '../input/decorations.ts'
|
|
1646
|
+
import { attachmentErrorText, imageSizeText } from '../image-labels.ts'
|
|
1647
|
+
import { ReferenceIcon } from '../reference/ReferenceIcon.tsx'
|
|
1648
|
+
import { ContextMeter } from './ContextMeter.tsx'
|
|
1649
|
+
+/* [本地改造 2026-08-20] 余额/图片/语音按钮全部迁移至插件 @oadank/dsh-client-composer;
|
|
1650
|
+
+ 源码 InputBar 不再持有图片/语音 UI,余额挂 conversation.input.right(插件)。 */
|
|
1651
|
+
import { PermissionSelect } from './PermissionSelect.tsx'
|
|
1652
|
+
import { isSafariBrowser, repairSafariTextareaLayout } from './safari.ts'
|
|
1653
|
+
import css from './InputBar.module.css'
|
|
1654
|
+
@@ -201,25 +198,27 @@ export function InputBar({
|
|
1655
|
+
const revealSelectionFocus = (el: HTMLTextAreaElement): void => {
|
|
1656
|
+
// selectionStart/End are number|null in lib.dom; the type-aware lint program narrows them.
|
|
1657
|
+
const caret = el.selectionDirection === 'backward' ? el.selectionStart : el.selectionEnd
|
|
1658
|
+
- // oxlint-disable-next-line typescript/no-unnecessary-condition
|
|
1659
|
+
revealCaret(caret ?? el.value.length)
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
- // Unlock (mount / session switch) returns focus to the box, and owns the
|
|
1663
|
+
- // reveal that comes with it. `preventScroll` because this focus is ours, not
|
|
1664
|
+
- // a gesture: the textarea is as tall as the draft, so the browser's reveal
|
|
1665
|
+
- // would walk up to the conversation scrollport and move the transcript under
|
|
1666
|
+
- // a user who only switched session. That leaves the caret to us — the DOM is
|
|
1667
|
+
- // reused across sessions, so switching to a longer draft keeps the previous
|
|
1668
|
+
- // offset while the value swap puts the caret at the new draft's end, which is
|
|
1669
|
+
- // off screen (measured on all three engines: offset 0 with the caret 940px
|
|
1670
|
+
- // down). Suppress the walk, then reveal in our own box.
|
|
1671
|
+
+ // [本地改造 2026-08-16] 彻底取消"切会话/挂载自动聚焦":移动端/触屏切会话
|
|
1672
|
+
+ // 会弹输入法。聚焦只在"同一会话内解锁边沿"(locked true→false,运行结束
|
|
1673
|
+
+ // 或从禁用态恢复)触发;sessionId 变化(切会话)或首次挂载都不抢焦点。
|
|
1674
|
+
+ const lastLockedRef = useRef<boolean>(locked)
|
|
1675
|
+
+ const lastFocusSessionRef = useRef<SessionId | undefined>(sessionId)
|
|
1676
|
+
useEffect(() => {
|
|
1677
|
+
const el = inputRef.current
|
|
1678
|
+
- if (locked || el === null) return
|
|
1679
|
+
+ const wasLocked = lastLockedRef.current
|
|
1680
|
+
+ lastLockedRef.current = locked
|
|
1681
|
+
+ if (el === null) return
|
|
1682
|
+
+ if (locked || !wasLocked) return
|
|
1683
|
+
+ if (lastFocusSessionRef.current !== sessionId) return
|
|
1684
|
+
el.focus({ preventScroll: true })
|
|
1685
|
+
revealSelectionFocus(el)
|
|
1686
|
+
}, [locked, sessionId])
|
|
1687
|
+
+ useEffect(() => {
|
|
1688
|
+
+ lastFocusSessionRef.current = sessionId
|
|
1689
|
+
+ }, [sessionId])
|
|
1690
|
+
|
|
1691
|
+
// A persisted draft arrives AFTER the unlock effect: ConversationSession
|
|
1692
|
+
// adopts it in its own mount effect, and a parent's mount effect runs after
|
|
1693
|
+
@@ -270,12 +269,10 @@ export function InputBar({
|
|
1694
|
+
}, [])
|
|
1695
|
+
|
|
1696
|
+
// selectionStart/End are number|null in lib.dom; the type-aware lint program narrows them.
|
|
1697
|
+
- /* oxlint-disable typescript/no-unnecessary-condition */
|
|
1698
|
+
const selectionOf = (el: HTMLTextAreaElement) => ({
|
|
1699
|
+
start: el.selectionStart ?? 0,
|
|
1700
|
+
end: el.selectionEnd ?? el.selectionStart ?? 0,
|
|
1701
|
+
})
|
|
1702
|
+
- /* oxlint-enable typescript/no-unnecessary-condition */
|
|
1703
|
+
|
|
1704
|
+
const onKeyDown = (e: KeyboardEvent<HTMLTextAreaElement>): void => {
|
|
1705
|
+
if (workspaceTrigger) {
|
|
1706
|
+
@@ -292,7 +289,6 @@ export function InputBar({
|
|
1707
|
+
// IME guard so a composition-closing Shift+Enter still breaks the line.
|
|
1708
|
+
if (e.key === 'Enter' && e.shiftKey) return
|
|
1709
|
+
// keyCode 229 is the legacy IME-composition signal engines emit without isComposing.
|
|
1710
|
+
- // oxlint-disable-next-line typescript/no-deprecated
|
|
1711
|
+
const composing = composingRef.current || e.nativeEvent.isComposing || e.nativeEvent.keyCode === 229
|
|
1712
|
+
if (!composing && !machineBusy && !locked
|
|
1713
|
+
&& (e.key === 'Backspace' || e.key === 'Delete')) {
|
|
1714
|
+
@@ -374,7 +370,6 @@ export function InputBar({
|
|
1715
|
+
safariNativeShrinkRef.current = safari && next.length < draft.length
|
|
1716
|
+
keyboard.setDraft(next)
|
|
1717
|
+
// selectionStart is number|null in lib.dom; the type-aware lint program narrows it.
|
|
1718
|
+
- // oxlint-disable-next-line typescript/no-unnecessary-condition
|
|
1719
|
+
keyboard.track(next, e.target.selectionStart ?? next.length)
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
@@ -463,7 +458,6 @@ export function InputBar({
|
|
1723
|
+
})()
|
|
1724
|
+
if (rejected !== null) showToast(rejected)
|
|
1725
|
+
}, [addImages, attachments, imageLimits, showToast, t])
|
|
1726
|
+
-
|
|
1727
|
+
const canAcceptDrop = !locked && !machineBusy && addImages !== undefined
|
|
1728
|
+
|
|
1729
|
+
const onSelect = (e: React.SyntheticEvent<HTMLTextAreaElement>): void => {
|
|
1730
|
+
@@ -482,6 +476,13 @@ export function InputBar({
|
|
1731
|
+
inputRef.current?.focus({ preventScroll: true })
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
+ // [本地改造 2026-08-16] 语音按钮/取消按钮:按下时**释放**文本输入框焦点
|
|
1735
|
+
+ // (blur),移动端点语音不会弹输入法;`preventScroll` 同理保持滚动位置。
|
|
1736
|
+
+ const releaseFocus = (e: MouseEvent<HTMLButtonElement>): void => {
|
|
1737
|
+
+ e.preventDefault()
|
|
1738
|
+
+ inputRef.current?.blur()
|
|
1739
|
+
+ }
|
|
1740
|
+
+
|
|
1741
|
+
const onToggleCommandMenu = (): void => {
|
|
1742
|
+
const el = inputRef.current
|
|
1743
|
+
if (el !== null) toggleCommandMenu?.(selectionOf(el))
|
|
1744
|
+
@@ -694,25 +695,27 @@ export function InputBar({
|
|
1745
|
+
</div>
|
|
1746
|
+
<div className={css.row}>
|
|
1747
|
+
<div className={css.tools}>
|
|
1748
|
+
- <Tooltip label={t('input.commands')} side="top" delayMs={500}>
|
|
1749
|
+
- <button
|
|
1750
|
+
- type="button"
|
|
1751
|
+
- className={css.add}
|
|
1752
|
+
- aria-label={t('input.commands')}
|
|
1753
|
+
- aria-haspopup="listbox"
|
|
1754
|
+
- aria-expanded={commandMenuOpen}
|
|
1755
|
+
- disabled={locked || toggleCommandMenu === undefined}
|
|
1756
|
+
- onMouseDown={keepFocus}
|
|
1757
|
+
- onClick={onToggleCommandMenu}
|
|
1758
|
+
- >
|
|
1759
|
+
- <IconPlusOutline16 size={14} />
|
|
1760
|
+
- </button>
|
|
1761
|
+
- </Tooltip>
|
|
1762
|
+
+ {/* [本地改造 2026-08-20] 余额/图片/语音按钮全部由插件 @oadank/dsh-client-composer 通过
|
|
1763
|
+
+ conversation.input.left 提供;插件图片按钮调官方 onAddImages(=intakeImages),
|
|
1764
|
+
+ 图片走官方 draft 链路随文本发送。用户要求按钮顺序:[🖼][🎙][+]——
|
|
1765
|
+
+ 插件按钮在最左(命令 + 按钮之前),+ 命令按钮在右。 */}
|
|
1766
|
+
+ {leftItems}
|
|
1767
|
+
+ <button
|
|
1768
|
+
+ type="button"
|
|
1769
|
+
+ className={css.add}
|
|
1770
|
+
+ aria-label={t('input.commands')}
|
|
1771
|
+
+ aria-haspopup="listbox"
|
|
1772
|
+
+ aria-expanded={commandMenuOpen}
|
|
1773
|
+
+ disabled={locked || toggleCommandMenu === undefined}
|
|
1774
|
+
+ onMouseDown={releaseFocus}
|
|
1775
|
+
+ onClick={onToggleCommandMenu}
|
|
1776
|
+
+ >
|
|
1777
|
+
+ <IconPlusOutline16 size={14} />
|
|
1778
|
+
+ </button>
|
|
1779
|
+
<div className={css.modes}>
|
|
1780
|
+
{accessSelect}
|
|
1781
|
+
{renderSlot('conversation.input.plan', { locked })}
|
|
1782
|
+
</div>
|
|
1783
|
+
- {leftItems}
|
|
1784
|
+
</div>
|
|
1785
|
+
<div className={css.trailing}>
|
|
1786
|
+
{rightItems}
|
|
1787
|
+
@@ -725,7 +728,7 @@ export function InputBar({
|
|
1788
|
+
className={css.primary}
|
|
1789
|
+
aria-label={t('input.stop')}
|
|
1790
|
+
disabled={stop === undefined}
|
|
1791
|
+
- onMouseDown={keepFocus}
|
|
1792
|
+
+ onMouseDown={releaseFocus}
|
|
1793
|
+
onClick={stop}
|
|
1794
|
+
>
|
|
1795
|
+
<svg viewBox="0 0 16 16" width="16" height="16" aria-hidden>
|
|
1796
|
+
diff --git a/packages/core/session/src/known-event-types.ts b/packages/core/session/src/known-event-types.ts
|
|
1797
|
+
index 098743dd8a..ad82a01de7 100644
|
|
1798
|
+
--- a/packages/core/session/src/known-event-types.ts
|
|
1799
|
+
+++ b/packages/core/session/src/known-event-types.ts
|
|
1800
|
+
@@ -64,5 +64,6 @@ export const KNOWN_SESSION_EVENT_TYPES: ReadonlySet<string> = new Set([
|
|
1801
|
+
'turn/end',
|
|
1802
|
+
'turn/start',
|
|
1803
|
+
'user/message',
|
|
1804
|
+
+ 'voice/reply',
|
|
1805
|
+
'web/deepseek-search-llm-request',
|
|
1806
|
+
])
|
|
1807
|
+
diff --git a/packages/core/session/src/types.ts b/packages/core/session/src/types.ts
|
|
1808
|
+
index 31ce28a01b..7f33eb2fb9 100644
|
|
1809
|
+
--- a/packages/core/session/src/types.ts
|
|
1810
|
+
+++ b/packages/core/session/src/types.ts
|
|
1811
|
+
@@ -334,6 +334,25 @@ export interface SessionEventMap {
|
|
1812
|
+
* so tolerating concurrent writers needs a signal beyond the log.
|
|
1813
|
+
*/
|
|
1814
|
+
'session/end-seed': Record<string, never>
|
|
1815
|
+
+ /**
|
|
1816
|
+
+ * [本地改造 2026-08-16] 助手侧语音回复:turn 完成后 host 把最后一条助手文本
|
|
1817
|
+
+ * 合成 TTS 并落盘,作为一条独立持久语音消息(与用户语音消息同级)。前端
|
|
1818
|
+
+ * 渲染为单独语音横条(复用 VoiceCard)。log-only:不进入模型历史重建。
|
|
1819
|
+
+ */
|
|
1820
|
+
+ 'voice/reply': {
|
|
1821
|
+
+ /** The turn whose closing assistant text this reply speaks. */
|
|
1822
|
+
+ turn: number
|
|
1823
|
+
+ /** Opaque storage identifier of the synthesized audio object. */
|
|
1824
|
+
+ voiceId: string
|
|
1825
|
+
+ /** Audio container format of the stored object. */
|
|
1826
|
+
+ mediaType: string
|
|
1827
|
+
+ /** Exact encoded byte length. */
|
|
1828
|
+
+ bytes: number
|
|
1829
|
+
+ /** Recorder-style duration in milliseconds. */
|
|
1830
|
+
+ durationMs?: number
|
|
1831
|
+
+ /** [本地改造 2026-08-21] 合成的正文(转写文本),供语音条显示与复制。 */
|
|
1832
|
+
+ transcript?: string
|
|
1833
|
+
+ }
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
/** The appendable event-type keys of {@link SessionEventMap}, plugin-merged extensions included. */
|
|
1837
|
+
diff --git a/packages/host/apiproxy/package.json b/packages/host/apiproxy/package.json
|
|
1838
|
+
index fba04c4759..6de5aaaa52 100644
|
|
1839
|
+
--- a/packages/host/apiproxy/package.json
|
|
1840
|
+
+++ b/packages/host/apiproxy/package.json
|
|
1841
|
+
@@ -45,15 +45,16 @@
|
|
1842
|
+
],
|
|
1843
|
+
"license": "MIT",
|
|
1844
|
+
"dependencies": {
|
|
1845
|
+
- "@deepseek-ai/dsh-attachment": "workspace:^",
|
|
1846
|
+
"@deepseek-ai/dsh-agent": "workspace:^",
|
|
1847
|
+
"@deepseek-ai/dsh-agent-default-model": "workspace:^",
|
|
1848
|
+
"@deepseek-ai/dsh-api-remotes": "workspace:^",
|
|
1849
|
+
+ "@deepseek-ai/dsh-attachment": "workspace:^",
|
|
1850
|
+
"@deepseek-ai/dsh-brand": "workspace:^",
|
|
1851
|
+
"@deepseek-ai/dsh-commands": "workspace:^",
|
|
1852
|
+
"@deepseek-ai/dsh-credentials": "workspace:^",
|
|
1853
|
+
"@deepseek-ai/dsh-goal": "workspace:^",
|
|
1854
|
+
"@deepseek-ai/dsh-host-directory-picker": "workspace:^",
|
|
1855
|
+
+ "@deepseek-ai/dsh-jobs": "workspace:^",
|
|
1856
|
+
"@deepseek-ai/dsh-llm": "workspace:^",
|
|
1857
|
+
"@deepseek-ai/dsh-native-command": "workspace:^",
|
|
1858
|
+
"@deepseek-ai/dsh-session": "workspace:^",
|
|
1859
|
+
@@ -65,13 +66,13 @@
|
|
1860
|
+
"@deepseek-ai/dsh-settings": "workspace:^",
|
|
1861
|
+
"@deepseek-ai/dsh-skill": "workspace:^",
|
|
1862
|
+
"@deepseek-ai/dsh-subagent": "workspace:^",
|
|
1863
|
+
- "@deepseek-ai/dsh-jobs": "workspace:^",
|
|
1864
|
+
"@deepseek-ai/dsh-tools": "workspace:^",
|
|
1865
|
+
"@deepseek-ai/dsh-user-approval": "workspace:^",
|
|
1866
|
+
"@deepseek-ai/dsh-user-questions": "workspace:^",
|
|
1867
|
+
"@deepseek-ai/dsh-workspace": "workspace:^",
|
|
1868
|
+
"@deepseek-ai/schemastery": "workspace:^",
|
|
1869
|
+
"fflate": "^0.8.2",
|
|
1870
|
+
+ "ws": "^8.21.0",
|
|
1871
|
+
"zod": "^4.4.3"
|
|
1872
|
+
},
|
|
1873
|
+
"peerDependencies": {
|
|
1874
|
+
@@ -89,6 +90,6 @@
|
|
1875
|
+
"@deepseek-ai/dsh-storage-domain": "workspace:^",
|
|
1876
|
+
"@deepseek-ai/dsh-typert-protocol": "workspace:^",
|
|
1877
|
+
"@deepseek-ai/dsh-typert-registry": "workspace:^",
|
|
1878
|
+
- "@deepseek-ai/cordis": "workspace:^"
|
|
1879
|
+
+ "@types/ws": "^8.18.1"
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
diff --git a/packages/host/apiproxy/src/api-proxy.ts b/packages/host/apiproxy/src/api-proxy.ts
|
|
1883
|
+
index ccb2d6d20c..aa6d647c1d 100644
|
|
1884
|
+
--- a/packages/host/apiproxy/src/api-proxy.ts
|
|
1885
|
+
+++ b/packages/host/apiproxy/src/api-proxy.ts
|
|
1886
|
+
@@ -4,16 +4,16 @@
|
|
1887
|
+
*/
|
|
1888
|
+
|
|
1889
|
+
import { randomUUID } from 'node:crypto'
|
|
1890
|
+
-import { mkdir, stat } from 'node:fs/promises'
|
|
1891
|
+
+import { mkdir, rm, stat, writeFile } from 'node:fs/promises'
|
|
1892
|
+
import { homedir } from 'node:os'
|
|
1893
|
+
-import { dirname } from 'node:path'
|
|
1894
|
+
+import { dirname, join } from 'node:path'
|
|
1895
|
+
import type { Context } from '@deepseek-ai/cordis'
|
|
1896
|
+
import { installModelSelection } from '@deepseek-ai/dsh-agent'
|
|
1897
|
+
import type { Agent, ModelSelection, ModelSelectionRef, AgentOptions, AgentStatus } from '@deepseek-ai/dsh-agent'
|
|
1898
|
+
import type {} from '@deepseek-ai/dsh-agent-presets/types'
|
|
1899
|
+
-import { AttachmentError, admitEncodedImages } from '@deepseek-ai/dsh-attachment'
|
|
1900
|
+
+import { AttachmentError } from '@deepseek-ai/dsh-attachment'
|
|
1901
|
+
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
|
1902
|
+
-import { contentHasImage, createUserMessage, freezeMessage, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
|
|
1903
|
+
+import { createUserMessage, freezeMessage, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
|
|
1904
|
+
import { errorChain } from '@deepseek-ai/dsh-llm'
|
|
1905
|
+
import type { ContentBlock, MessageSource } from '@deepseek-ai/dsh-llm'
|
|
1906
|
+
import { isAppendSurfaceEvent, isJsonValue } from '@deepseek-ai/dsh-session'
|
|
1907
|
+
@@ -36,7 +36,7 @@ import {
|
|
1908
|
+
import type { PresetBearingSession } from '@deepseek-ai/dsh-agent-presets'
|
|
1909
|
+
import type {} from '@deepseek-ai/dsh-tools'
|
|
1910
|
+
import type {
|
|
1911
|
+
- ApiProxy, ConfigurableProviderView, CredentialView, GoalRef, HistoryEntry, HostFrame,
|
|
1912
|
+
+ ApiProxy, BalanceView, ConfigurableProviderView, CredentialView, GoalRef, HistoryEntry, HostFrame,
|
|
1913
|
+
ModelCatalogFailure, ModelProviderGroup,
|
|
1914
|
+
ModelReasoning, MuxFrame, PromptContentPart, QuestionResponsePayload, SessionListMetadata, SessionProjectionsBlock, SessionSearchItem,
|
|
1915
|
+
QueuedInboxItem, SessionSummary, SettingsNamespaceView, SubagentAddress, JobView, ToolEventView,
|
|
1916
|
+
@@ -51,6 +51,8 @@ import {
|
|
1917
|
+
type SessionLogExportReady,
|
|
1918
|
+
type SessionLogCompressionLevel,
|
|
1919
|
+
} from './session-export.ts'
|
|
1920
|
+
+import { readVoiceFile, saveVoiceFile, synthesizeReplyVoice, transcribeVoice, voiceObjectPath, voiceStorageRoot } from './voice.ts'
|
|
1921
|
+
+import type { VoiceAttachmentRef } from './api/sessions.ts'
|
|
1922
|
+
import type { SessionRawArtifact } from '@deepseek-ai/dsh-session-persistence'
|
|
1923
|
+
import {
|
|
1924
|
+
SESSION_SEARCH_RESULT_LIMIT,
|
|
1925
|
+
@@ -124,19 +126,150 @@ export const DEFAULT_COLD_BLANK_PROBE_MAX_BYTES = 1024
|
|
1926
|
+
/** Conversation message event types (the pagination counting unit). */
|
|
1927
|
+
const MESSAGE_TYPES = new Set(['user/message', 'assistant/message'])
|
|
1928
|
+
|
|
1929
|
+
+/** Decode the browser payload while rejecting non-canonical base64 forms. */
|
|
1930
|
+
+function decodeBase64(data: string): Uint8Array {
|
|
1931
|
+
+ const decoded = Buffer.from(data, 'base64')
|
|
1932
|
+
+ if (data.length === 0 || decoded.toString('base64') !== data) {
|
|
1933
|
+
+ throw new AttachmentError('Image upload is not canonical base64.', 'INVALID_IMAGE_BASE64')
|
|
1934
|
+
+ }
|
|
1935
|
+
+ return new Uint8Array(decoded)
|
|
1936
|
+
+}
|
|
1937
|
+
+
|
|
1938
|
+
+/** DeepSeek 直连余额缓存(5 秒过期:balance 接口很轻,实时性优先,仅防抖重复轮询)。 */
|
|
1939
|
+
+let deepseekBalanceCache: { value: BalanceView | null; cachedAt: number } | null = null
|
|
1940
|
+
+
|
|
1941
|
+
+/**
|
|
1942
|
+
+ * 查询 DeepSeek 直连账户余额。读取 DEEPSEEK_API_KEY 凭证(dsh-web 直连
|
|
1943
|
+
+ * deepseek 路由的 key),调官方 GET /user/balance;无凭证、非直连部署或
|
|
1944
|
+
+ * 查询失败一律返回 null(余额只是辅助指示,绝不让 UI 报错)。
|
|
1945
|
+
+ * @param ctx - host context(读 credentials 可选服务)。
|
|
1946
|
+
+ * @returns 余额视图;不可用时为 null。
|
|
1947
|
+
+ */
|
|
1948
|
+
+async function readDeepSeekBalance(ctx: Context): Promise<BalanceView | null> {
|
|
1949
|
+
+ const now = Date.now()
|
|
1950
|
+
+ if (deepseekBalanceCache !== null && now - deepseekBalanceCache.cachedAt < 5_000) {
|
|
1951
|
+
+ return deepseekBalanceCache.value
|
|
1952
|
+
+ }
|
|
1953
|
+
+ let apiKey: string | undefined
|
|
1954
|
+
+ const credentials = ctx.get('credentials')
|
|
1955
|
+
+ if (credentials !== undefined) {
|
|
1956
|
+
+ const hit = await credentials.resolve(credentialRef('DEEPSEEK_API_KEY'))
|
|
1957
|
+
+ apiKey = hit?.value
|
|
1958
|
+
+ } else {
|
|
1959
|
+
+ apiKey = process.env.DEEPSEEK_API_KEY
|
|
1960
|
+
+ }
|
|
1961
|
+
+ if (apiKey === undefined || apiKey.length === 0) {
|
|
1962
|
+
+ deepseekBalanceCache = { value: null, cachedAt: now }
|
|
1963
|
+
+ return null
|
|
1964
|
+
+ }
|
|
1965
|
+
+ try {
|
|
1966
|
+
+ const response = await fetch('https://api.deepseek.com/user/balance', {
|
|
1967
|
+
+ headers: { authorization: `Bearer ${apiKey}` },
|
|
1968
|
+
+ signal: AbortSignal.timeout(5000),
|
|
1969
|
+
+ })
|
|
1970
|
+
+ if (!response.ok) {
|
|
1971
|
+
+ deepseekBalanceCache = { value: null, cachedAt: now }
|
|
1972
|
+
+ return null
|
|
1973
|
+
+ }
|
|
1974
|
+
+ const data = await response.json() as {
|
|
1975
|
+
+ balance_infos?: Array<{ currency: string; total_balance: string; granted_balance: string; topped_up_balance: string }>
|
|
1976
|
+
+ }
|
|
1977
|
+
+ const info = data.balance_infos?.[0]
|
|
1978
|
+
+ if (info === undefined) {
|
|
1979
|
+
+ deepseekBalanceCache = { value: null, cachedAt: now }
|
|
1980
|
+
+ return null
|
|
1981
|
+
+ }
|
|
1982
|
+
+ const value: BalanceView = {
|
|
1983
|
+
+ currency: info.currency,
|
|
1984
|
+
+ total: info.total_balance,
|
|
1985
|
+
+ granted: info.granted_balance,
|
|
1986
|
+
+ toppedUp: info.topped_up_balance,
|
|
1987
|
+
+ }
|
|
1988
|
+
+ deepseekBalanceCache = { value, cachedAt: now }
|
|
1989
|
+
+ return value
|
|
1990
|
+
+ } catch {
|
|
1991
|
+
+ // 网络抖动/超时:保留旧缓存值(若有),否则 null——绝不把失败抛给 UI。
|
|
1992
|
+
+ return deepseekBalanceCache?.value ?? null
|
|
1993
|
+
+ }
|
|
1994
|
+
+}
|
|
1995
|
+
+
|
|
1996
|
+
/** Validate one prompt as a batch before publishing any durable image object. */
|
|
1997
|
+
async function durablePromptContent(ctx: Context, content: readonly PromptContentPart[]): Promise<ContentBlock[]> {
|
|
1998
|
+
if (content.every(part => part.type === 'text')) {
|
|
1999
|
+
return content.map(part => ({ type: 'text', text: part.text }))
|
|
2000
|
+
}
|
|
2001
|
+
- const refs = await admitEncodedImages(ctx.attachments, content.filter(part => part.type === 'image'))
|
|
2002
|
+
- let next = 0
|
|
2003
|
+
- return content.map(part => part.type === 'text'
|
|
2004
|
+
- ? { type: 'text', text: part.text }
|
|
2005
|
+
- // admitEncodedImages returns one reference per image part in order.
|
|
2006
|
+
- : { type: 'image', attachment: refs[next++] as ImageAttachmentRef })
|
|
2007
|
+
+ const limits = ctx.attachments.imageLimits
|
|
2008
|
+
+ if (content.filter(part => part.type === 'image').length > limits.maxImagesPerMessage) {
|
|
2009
|
+
+ throw new AttachmentError('Prompt exceeds the configured image-count limit.', 'TOO_MANY_IMAGES')
|
|
2010
|
+
+ }
|
|
2011
|
+
+ const prepared = content.map(part => part.type === 'text'
|
|
2012
|
+
+ ? part
|
|
2013
|
+
+ : { part, data: decodeBase64(part.data) })
|
|
2014
|
+
+ const images = prepared.filter((item): item is { part: Extract<PromptContentPart, { type: 'image' }>; data: Uint8Array } =>
|
|
2015
|
+
+ 'part' in item && item.part.type === 'image')
|
|
2016
|
+
+ const totalBytes = images.reduce((sum, image) => sum + image.data.byteLength, 0)
|
|
2017
|
+
+ if (totalBytes > limits.maxMessageImageBytes) {
|
|
2018
|
+
+ throw new AttachmentError('Prompt exceeds the configured aggregate image-byte limit.', 'IMAGES_TOO_LARGE')
|
|
2019
|
+
+ }
|
|
2020
|
+
+ for (const image of images) {
|
|
2021
|
+
+ await ctx.attachments.validateImage({
|
|
2022
|
+
+ data: image.data,
|
|
2023
|
+
+ mediaType: image.part.mediaType,
|
|
2024
|
+
+ ...image.part.name === undefined ? {} : { name: image.part.name },
|
|
2025
|
+
+ })
|
|
2026
|
+
+ }
|
|
2027
|
+
+ const root = voiceStorageRoot()
|
|
2028
|
+
+ const blocks: ContentBlock[] = []
|
|
2029
|
+
+ for (const item of prepared) {
|
|
2030
|
+
+ if (!('data' in item)) {
|
|
2031
|
+
+ blocks.push({ type: 'text', text: item.text })
|
|
2032
|
+
+ continue
|
|
2033
|
+
+ }
|
|
2034
|
+
+ if (item.part.type === 'voice') {
|
|
2035
|
+
+ // [本地改造 2026-08-16] 语音消息:录音落盘(与图片同池的内容寻址对象)。
|
|
2036
|
+
+ // 落盘后**同步自动 ASR**(本地 sherpa 服务)并把识别文本写入 attachment.transcript——
|
|
2037
|
+
+ // 模型从 serialize 拿到现成识别文本直接回复:自动流程、无需 agent 自觉、
|
|
2038
|
+
+ // 不再二次注入"用户身份"文本;识别失败时 transcript 缺省,serialize 降级为
|
|
2039
|
+
+ // 本地路径文本,agent 可主动调工具补识别。
|
|
2040
|
+
+ let attachment: VoiceAttachmentRef
|
|
2041
|
+
+ try {
|
|
2042
|
+
+ attachment = await saveVoiceFile(
|
|
2043
|
+
+ root, item.data, item.part.mediaType, item.part.durationMs,
|
|
2044
|
+
+ )
|
|
2045
|
+
+ } catch {
|
|
2046
|
+
+ throw new AttachmentError('Unable to persist voice object.', 'ATTACHMENT_WRITE_FAILED')
|
|
2047
|
+
+ }
|
|
2048
|
+
+ // [本地改造 2026-08-21] ASR 识别失败 = 阻断发送:语音块不进模型。
|
|
2049
|
+
+ // 之前识别失败会把无 transcript 的语音块交给模型,serialize 降级为本地
|
|
2050
|
+
+ // 路径文本,模型自己拿语音路径去本地识别,用户不可控。现在失败即抛错,
|
|
2051
|
+
+ // 前端收到 attachment-error + message,提示先配置 ASR 服务。
|
|
2052
|
+
+ const text = await transcribeVoice(voiceObjectPath(root, attachment.voiceId))
|
|
2053
|
+
+ if (text === '') {
|
|
2054
|
+
+ throw new AttachmentError(
|
|
2055
|
+
+ '语音识别失败:ASR 服务未配置或识别出错,请先在「设置 → 语音服务」配置 ASR 后重试。',
|
|
2056
|
+
+ 'VOICE_ASR_FAILED',
|
|
2057
|
+
+ )
|
|
2058
|
+
+ }
|
|
2059
|
+
+ attachment = { ...attachment, transcript: text }
|
|
2060
|
+
+ blocks.push({ type: 'voice', attachment })
|
|
2061
|
+
+ continue
|
|
2062
|
+
+ }
|
|
2063
|
+
+ const attachment = await ctx.attachments.saveImage({
|
|
2064
|
+
+ data: item.data,
|
|
2065
|
+
+ mediaType: item.part.mediaType,
|
|
2066
|
+
+ ...item.part.name === undefined ? {} : { name: item.part.name },
|
|
2067
|
+
+ })
|
|
2068
|
+
+ blocks.push({ type: 'image', attachment })
|
|
2069
|
+
+ }
|
|
2070
|
+
+ return blocks
|
|
2071
|
+
}
|
|
2072
|
+
|
|
2073
|
+
+/**
|
|
2074
|
+
+ * [本地改造 2026-08-16] 语音识别已内置于 durablePromptContent:落盘后同步调本地 ASR,
|
|
2075
|
+
+ * transcript 写入 attachment。serialize.ts 优先输出识别文本;仅当识别失败(transcript
|
|
2076
|
+
+ * 缺省)时才降级为本地路径文本,此时 agent 可用工具补识别。
|
|
2077
|
+
+ */
|
|
2078
|
+
+
|
|
2079
|
+
/** Search durable content for an image reference, including nested tool results. */
|
|
2080
|
+
function imageBlockIn(content: unknown, match: (ref: ImageAttachmentRef) => boolean): ImageAttachmentRef | undefined {
|
|
2081
|
+
if (!Array.isArray(content)) return undefined
|
|
2082
|
+
@@ -181,9 +314,68 @@ function imageInEvent(event: SessionEvent, match: (ref: ImageAttachmentRef) => b
|
|
2083
|
+
return undefined
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
-/** True when the current model-visible surface contains an image. */
|
|
2087
|
+
-function messagesHaveImage(messages: readonly { content: readonly ContentBlock[] }[]): boolean {
|
|
2088
|
+
- return messages.some(message => contentHasImage(message.content))
|
|
2089
|
+
+/** Search durable content for a voice reference, including nested tool results. */
|
|
2090
|
+
+function voiceBlockIn(content: unknown, match: (ref: VoiceAttachmentRef) => boolean): VoiceAttachmentRef | undefined {
|
|
2091
|
+
+ if (!Array.isArray(content)) return undefined
|
|
2092
|
+
+ for (const value of content) {
|
|
2093
|
+
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) continue
|
|
2094
|
+
+ const block = value as { type?: unknown; attachment?: unknown; content?: unknown }
|
|
2095
|
+
+ if (block.type === 'voice' && typeof block.attachment === 'object' && block.attachment !== null) {
|
|
2096
|
+
+ const ref = block.attachment as VoiceAttachmentRef
|
|
2097
|
+
+ if (match(ref)) return ref
|
|
2098
|
+
+ }
|
|
2099
|
+
+ if (block.type === 'tool-result') {
|
|
2100
|
+
+ const nested = voiceBlockIn(block.content, match)
|
|
2101
|
+
+ if (nested !== undefined) return nested
|
|
2102
|
+
+ }
|
|
2103
|
+
+ }
|
|
2104
|
+
+ return undefined
|
|
2105
|
+
+}
|
|
2106
|
+
+
|
|
2107
|
+
+/** Search every durable event carrier that can own model-visible voice content. */
|
|
2108
|
+
+function voiceInEvent(event: SessionEvent, match: (ref: VoiceAttachmentRef) => boolean): VoiceAttachmentRef | undefined {
|
|
2109
|
+
+ // [本地改造 2026-08-16] 助手语音回复事件:payload 直接携带 voiceId(非 voice 块)。
|
|
2110
|
+
+ if (event.type === 'voice/reply') {
|
|
2111
|
+
+ const { voiceId, mediaType, bytes, durationMs } = event.data as {
|
|
2112
|
+
+ voiceId: string
|
|
2113
|
+
+ mediaType: string
|
|
2114
|
+
+ bytes: number
|
|
2115
|
+
+ durationMs?: number
|
|
2116
|
+
+ }
|
|
2117
|
+
+ const ref: VoiceAttachmentRef = { voiceId, mediaType: mediaType as VoiceAttachmentRef['mediaType'], bytes, ...(durationMs === undefined ? {} : { durationMs }) }
|
|
2118
|
+
+ return match(ref) ? ref : undefined
|
|
2119
|
+
+ }
|
|
2120
|
+
+ const data = event.data as {
|
|
2121
|
+
+ content?: unknown
|
|
2122
|
+
+ message?: { content?: unknown }
|
|
2123
|
+
+ inserted?: Array<{ content?: unknown }>
|
|
2124
|
+
+ chunk?: { type?: unknown; block?: unknown }
|
|
2125
|
+
+ }
|
|
2126
|
+
+ const direct = voiceBlockIn(data.content, match)
|
|
2127
|
+
+ if (direct !== undefined) return direct
|
|
2128
|
+
+ if (data.message !== undefined) {
|
|
2129
|
+
+ const wrapped = voiceBlockIn(data.message.content, match)
|
|
2130
|
+
+ if (wrapped !== undefined) return wrapped
|
|
2131
|
+
+ }
|
|
2132
|
+
+ if (data.inserted !== undefined) {
|
|
2133
|
+
+ for (const message of data.inserted) {
|
|
2134
|
+
+ const inserted = voiceBlockIn(message.content, match)
|
|
2135
|
+
+ if (inserted !== undefined) return inserted
|
|
2136
|
+
+ }
|
|
2137
|
+
+ }
|
|
2138
|
+
+ if (event.type === 'assistant/chunk' && data.chunk?.type === 'block-end') {
|
|
2139
|
+
+ return voiceBlockIn([data.chunk.block], match)
|
|
2140
|
+
+ }
|
|
2141
|
+
+ return undefined
|
|
2142
|
+
+}
|
|
2143
|
+
+
|
|
2144
|
+
+/** Resolve the first voice reference matching one opaque id. */
|
|
2145
|
+
+function referencedVoice(events: readonly SessionEvent[], voiceId: string): VoiceAttachmentRef | undefined {
|
|
2146
|
+
+ for (const event of events) {
|
|
2147
|
+
+ const found = voiceInEvent(event, ref => ref.voiceId === voiceId)
|
|
2148
|
+
+ if (found !== undefined) return found
|
|
2149
|
+
+ }
|
|
2150
|
+
+ return undefined
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
/** Resolve the first reference matching one opaque id. */
|
|
2154
|
+
@@ -1299,6 +1491,11 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
|
2155
|
+
broadcast({ type: 'session/queue', sessionId: session.id, items: queueItems(agent, event.data) })
|
|
2156
|
+
})
|
|
2157
|
+
|
|
2158
|
+
+ // [本地改造 2026-08-17] 语音能力已迁移至插件 @anoslide/dsh-host-voice:
|
|
2159
|
+
+ // send_voice 工具 + turn/end 自动语音回复 + TTS 三引擎(默认 auto=小米优先,缺失降级 edge)。
|
|
2160
|
+
+ // 实现见 ~/.dsh/profiles/node_modules/@anoslide/dsh-host-voice/lib/index.js(cordis.patch.yml insert)。
|
|
2161
|
+
+ // voice.ts 仍保留:voiceAsr/voiceTts RPC(本地转写 + 编辑器内合成)与 voiceInEvent 解析共用。
|
|
2162
|
+
+
|
|
2163
|
+
/** Remove a wait before settling it: synchronous deletion makes the first claimant win. */
|
|
2164
|
+
function claimQuestion(pending: PendingQuestion, outcome: 'answered' | 'cancelled'): void {
|
|
2165
|
+
pendingQuestions.delete(pending.rpcId)
|
|
2166
|
+
@@ -2208,18 +2405,11 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
|
2167
|
+
? {}
|
|
2168
|
+
: { reasoningEffort: ReasoningEffortId(reasoningEffort) },
|
|
2169
|
+
})
|
|
2170
|
+
- const pendingImage = [...found.agent.inbox.nextTurn, ...found.agent.inbox.nextStep]
|
|
2171
|
+
- .some(message => contentHasImage(message.content))
|
|
2172
|
+
- if (pendingImage || messagesHaveImage(found.agent.session.deriveMessages())) {
|
|
2173
|
+
- const info = await ctx.llm.resolveModelInfo(resolved.provider, resolved.model)
|
|
2174
|
+
- if (info.inputModalities !== undefined && !info.inputModalities.includes('image')) {
|
|
2175
|
+
- return err(request, {
|
|
2176
|
+
- code: 'model-unavailable',
|
|
2177
|
+
- message: `Model "${resolved.model}" does not accept image input, but this session already contains images; select an image-capable model.`,
|
|
2178
|
+
- details: { provider, model },
|
|
2179
|
+
- })
|
|
2180
|
+
- }
|
|
2181
|
+
- }
|
|
2182
|
+
+ // [本地改造 2026-08-16] 移除官方"模型不支持图片就拒绝切换"的检查:
|
|
2183
|
+
+ // 本部署图片/语音与模型原生能力解耦——llm-deepseek serialize.ts 把
|
|
2184
|
+
+ // image/voice 块转文本(本地路径/ASR 文本),识图走视觉 MCP(look),
|
|
2185
|
+
+ // 图片从不直接进模型。因此 inputModalities 不含 image 的模型
|
|
2186
|
+
+ // (如 deepseek-v4-flash)在含图会话里同样可以正常切换使用。
|
|
2187
|
+
const selected: ModelSelection = {
|
|
2188
|
+
provider: resolved.provider,
|
|
2189
|
+
model: resolved.model,
|
|
2190
|
+
@@ -2396,23 +2586,24 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
|
2191
|
+
...(canonicalTimeZone === undefined ? {} : { clientTimeZone: canonicalTimeZone }),
|
|
2192
|
+
}
|
|
2193
|
+
const hasImage = content.some(part => part.type === 'image')
|
|
2194
|
+
+ const hasVoice = content.some(part => part.type === 'voice')
|
|
2195
|
+
const admit = async (): Promise<RpcResponse<{ accepted: true }>> => {
|
|
2196
|
+
try {
|
|
2197
|
+
- if (hasImage) {
|
|
2198
|
+
- const current = selectionFor(agent).current
|
|
2199
|
+
- const modelInfo = await ctx.llm.resolveModelInfo(current.provider, current.model)
|
|
2200
|
+
- if (modelInfo.inputModalities !== undefined && !modelInfo.inputModalities.includes('image')) {
|
|
2201
|
+
- return err(request, {
|
|
2202
|
+
- code: 'attachment-error',
|
|
2203
|
+
- message: `Model "${current.model}" does not support image input.`,
|
|
2204
|
+
- details: { reason: 'MODEL_DOES_NOT_SUPPORT_IMAGES' },
|
|
2205
|
+
- })
|
|
2206
|
+
- }
|
|
2207
|
+
- }
|
|
2208
|
+
+ // [本地改造 2026-08-16] 完全放行图片(参考 dsh-vscode-layout 补丁):
|
|
2209
|
+
+ // 图片进用户消息正常显示;llm-deepseek 序列化时把图片块转为本地附件路径文本,
|
|
2210
|
+
+ // agent 用视觉 MCP(look)识图——不再按模型是否支持图片拦截。
|
|
2211
|
+
const durable = await durablePromptContent(ctx, content)
|
|
2212
|
+
const message: UserMessage = createUserMessage({ content: durable, source })
|
|
2213
|
+
- if (mode === 'steer') agent.steer(message)
|
|
2214
|
+
- else agent.followup(message)
|
|
2215
|
+
+ if (mode === 'steer') {
|
|
2216
|
+
+ agent.steer(message)
|
|
2217
|
+
+ } else {
|
|
2218
|
+
+ // [本地改造 2026-08-16] 语音消息与文本/图片一样走正常 followup:voice
|
|
2219
|
+
+ // 块入队上屏(前端语音卡片可播放),识别文本已写入 attachment.transcript,
|
|
2220
|
+
+ // 模型从序列化文本直接拿到转写内容。
|
|
2221
|
+
+ // [本地改造 2026-08-21] ASR 识别失败的语音消息在 durablePromptContent
|
|
2222
|
+
+ // 阶段即被阻断(VOICE_ASR_FAILED),不会走到这里。
|
|
2223
|
+
+ agent.followup(message)
|
|
2224
|
+
+ }
|
|
2225
|
+
} catch (error: unknown) {
|
|
2226
|
+
if (error instanceof AttachmentError) {
|
|
2227
|
+
return err(request, {
|
|
2228
|
+
@@ -2429,7 +2620,7 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
|
2229
|
+
}
|
|
2230
|
+
return ok(request, { accepted: true as const })
|
|
2231
|
+
}
|
|
2232
|
+
- return hasImage ? serializeImageAdmission(agent, admit) : admit()
|
|
2233
|
+
+ return hasImage || hasVoice ? serializeImageAdmission(agent, admit) : admit()
|
|
2234
|
+
},
|
|
2235
|
+
|
|
2236
|
+
async attachment(request) {
|
|
2237
|
+
@@ -2481,6 +2672,129 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
|
2238
|
+
}
|
|
2239
|
+
},
|
|
2240
|
+
|
|
2241
|
+
+ async voice(request) {
|
|
2242
|
+
+ const { sessionId, voiceId } = request.payload
|
|
2243
|
+
+ console.error(`[voice-rpc-debug] voice called sessionId=${sessionId} voiceId=${voiceId}`)
|
|
2244
|
+
+ let state: SessionReadState
|
|
2245
|
+
+ try {
|
|
2246
|
+
+ state = await readSessionState(sessionId)
|
|
2247
|
+
+ } catch (error: unknown) {
|
|
2248
|
+
+ if (error instanceof SessionNotFound) {
|
|
2249
|
+
+ return err(request, {
|
|
2250
|
+
+ code: 'session-not-found',
|
|
2251
|
+
+ message: error.message,
|
|
2252
|
+
+ details: { sessionId },
|
|
2253
|
+
+ })
|
|
2254
|
+
+ }
|
|
2255
|
+
+ return err(request, {
|
|
2256
|
+
+ code: 'internal',
|
|
2257
|
+
+ message: `voice authorization unavailable for session "${sessionId}": ${String(error)}`,
|
|
2258
|
+
+ details: {},
|
|
2259
|
+
+ })
|
|
2260
|
+
+ }
|
|
2261
|
+
+ const ref = referencedVoice(state.events, voiceId)
|
|
2262
|
+
+ if (ref === undefined) {
|
|
2263
|
+
+ return err(request, {
|
|
2264
|
+
+ code: 'attachment-error',
|
|
2265
|
+
+ message: 'Voice object is not referenced by this session.',
|
|
2266
|
+
+ details: { reason: 'VOICE_NOT_REFERENCED' },
|
|
2267
|
+
+ })
|
|
2268
|
+
+ }
|
|
2269
|
+
+ try {
|
|
2270
|
+
+ const stored = await readVoiceFile(voiceStorageRoot(), ref)
|
|
2271
|
+
+ return ok(request, {
|
|
2272
|
+
+ attachment: stored.ref,
|
|
2273
|
+
+ data: Buffer.from(stored.data).toString('base64'),
|
|
2274
|
+
+ })
|
|
2275
|
+
+ } catch {
|
|
2276
|
+
+ return err(request, {
|
|
2277
|
+
+ code: 'internal',
|
|
2278
|
+
+ message: 'Unable to read voice object.',
|
|
2279
|
+
+ details: {},
|
|
2280
|
+
+ })
|
|
2281
|
+
+ }
|
|
2282
|
+
+ },
|
|
2283
|
+
+
|
|
2284
|
+
+ async voiceAsr(request) {
|
|
2285
|
+
+ // On-the-fly transcription for the "speak-to-text" gesture: decode the
|
|
2286
|
+
+ // recording to a temp file, transcribe, and remove it — nothing durable.
|
|
2287
|
+
+ const { data } = request.payload
|
|
2288
|
+
+ let tempPath: string | undefined
|
|
2289
|
+
+ try {
|
|
2290
|
+
+ const bytes = decodeBase64(data)
|
|
2291
|
+
+ tempPath = join(process.env.TEMP ?? '/tmp', `dsh-asr-in-${randomUUID()}`)
|
|
2292
|
+
+ await writeFile(tempPath, bytes)
|
|
2293
|
+
+ const text = await transcribeVoice(tempPath)
|
|
2294
|
+
+ return ok(request, { text: text === '' ? null : text })
|
|
2295
|
+
+ } catch {
|
|
2296
|
+
+ return ok(request, { text: null })
|
|
2297
|
+
+ } finally {
|
|
2298
|
+
+ if (tempPath !== undefined) await rm(tempPath, { force: true }).catch(() => {})
|
|
2299
|
+
+ }
|
|
2300
|
+
+ },
|
|
2301
|
+
+
|
|
2302
|
+
+ async voiceTts(request) {
|
|
2303
|
+
+ // Local TTS through the agents-to-im engine: synthesize to mp3 (the
|
|
2304
|
+
+ // browser-universal format), read the bytes back, and return them
|
|
2305
|
+
+ // inline — nothing durable, the reply plays locally.
|
|
2306
|
+
+ const { text, provider } = request.payload
|
|
2307
|
+
+ try {
|
|
2308
|
+
+ const audio = await synthesizeReplyVoice(text, provider)
|
|
2309
|
+
+ if (audio === null) return ok(request, null)
|
|
2310
|
+
+ return ok(request, {
|
|
2311
|
+
+ mediaType: audio.mediaType,
|
|
2312
|
+
+ data: Buffer.from(audio.data).toString('base64'),
|
|
2313
|
+
+ ...(audio.durationMs === undefined ? {} : { durationMs: audio.durationMs }),
|
|
2314
|
+
+ })
|
|
2315
|
+
+ } catch {
|
|
2316
|
+
+ return ok(request, null)
|
|
2317
|
+
+ }
|
|
2318
|
+
+ },
|
|
2319
|
+
+
|
|
2320
|
+
+ async sendVoiceMessage(request) {
|
|
2321
|
+
+ // [本地改造 2026-08-16] 主动发语音消息:合成 TTS → 落盘 → 追加 voice/reply
|
|
2322
|
+
+ // 事件(独立持久语音横条)。agent 通过本 RPC 可以主动给用户发语音。
|
|
2323
|
+
+ const { sessionId, text, provider } = request.payload
|
|
2324
|
+
+ try {
|
|
2325
|
+
+ const audio = await synthesizeReplyVoice(text, provider)
|
|
2326
|
+
+ if (audio === null) {
|
|
2327
|
+
+ return err(request, {
|
|
2328
|
+
+ code: 'internal',
|
|
2329
|
+
+ message: 'voice synthesis failed',
|
|
2330
|
+
+ details: {},
|
|
2331
|
+
+ })
|
|
2332
|
+
+ }
|
|
2333
|
+
+ const attachment = await saveVoiceFile(
|
|
2334
|
+
+ voiceStorageRoot(), audio.data, audio.mediaType as never, audio.durationMs,
|
|
2335
|
+
+ )
|
|
2336
|
+
+ const agent = ctx.agents.get(sessionId)
|
|
2337
|
+
+ if (agent === undefined || agent.session.id !== sessionId) {
|
|
2338
|
+
+ return err(request, {
|
|
2339
|
+
+ code: 'session-not-found',
|
|
2340
|
+
+ message: 'no attached session for sendVoiceMessage',
|
|
2341
|
+
+ details: { sessionId },
|
|
2342
|
+
+ })
|
|
2343
|
+
+ }
|
|
2344
|
+
+ const turn = agent.session.events
|
|
2345
|
+
+ .filter((event): event is SessionEvent & { type: 'turn/start' } => event.type === 'turn/start')
|
|
2346
|
+
+ .at(-1)?.data.turn ?? 0
|
|
2347
|
+
+ agent.session.append('voice/reply', {
|
|
2348
|
+
+ turn,
|
|
2349
|
+
+ voiceId: attachment.voiceId,
|
|
2350
|
+
+ mediaType: attachment.mediaType,
|
|
2351
|
+
+ bytes: attachment.bytes,
|
|
2352
|
+
+ ...(attachment.durationMs === undefined ? {} : { durationMs: attachment.durationMs }),
|
|
2353
|
+
+ })
|
|
2354
|
+
+ return ok(request, { accepted: true as const })
|
|
2355
|
+
+ } catch {
|
|
2356
|
+
+ return err(request, {
|
|
2357
|
+
+ code: 'internal',
|
|
2358
|
+
+ message: 'sendVoiceMessage failed',
|
|
2359
|
+
+ details: {},
|
|
2360
|
+
+ })
|
|
2361
|
+
+ }
|
|
2362
|
+
+ },
|
|
2363
|
+
+
|
|
2364
|
+
updateQueue(request) {
|
|
2365
|
+
const { sessionId, itemId, action } = request.payload
|
|
2366
|
+
if (action.kind === 'edit' && action.content.some(block => block.type !== 'text')) {
|
|
2367
|
+
@@ -3340,6 +3654,25 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
|
2368
|
+
},
|
|
2369
|
+
},
|
|
2370
|
+
|
|
2371
|
+
+ balance: {
|
|
2372
|
+
+ async get(request) {
|
|
2373
|
+
+ // [本地改造 2026-08-16] 余额指示只对 deepseek 直连模型显示:
|
|
2374
|
+
+ // 调用方带上 sessionId 时,检查该会话当前 provider;非
|
|
2375
|
+
+ // deepseek-official(如 qwen 百炼)直接返回 null,前端据此隐藏。
|
|
2376
|
+
+ const { sessionId } = request.payload
|
|
2377
|
+
+ if (sessionId !== undefined) {
|
|
2378
|
+
+ const found = await agentFor(sessionId)
|
|
2379
|
+
+ if (!('error' in found)) {
|
|
2380
|
+
+ const current = selectionFor(found.agent).current
|
|
2381
|
+
+ if (current.provider !== 'deepseek-official') {
|
|
2382
|
+
+ return ok(request, { balance: null })
|
|
2383
|
+
+ }
|
|
2384
|
+
+ }
|
|
2385
|
+
+ }
|
|
2386
|
+
+ return ok(request, { balance: await readDeepSeekBalance(ctx) })
|
|
2387
|
+
+ },
|
|
2388
|
+
+ },
|
|
2389
|
+
+
|
|
2390
|
+
events: {
|
|
2391
|
+
mux(_request, signal) {
|
|
2392
|
+
const queue = new FrameQueue<RpcRequest<MuxFrame>>()
|
|
2393
|
+
diff --git a/packages/host/apiproxy/src/api/balance.schema.ts b/packages/host/apiproxy/src/api/balance.schema.ts
|
|
2394
|
+
new file mode 100644
|
|
2395
|
+
index 0000000000..3a77b47e6c
|
|
2396
|
+
--- /dev/null
|
|
2397
|
+
+++ b/packages/host/apiproxy/src/api/balance.schema.ts
|
|
2398
|
+
@@ -0,0 +1,30 @@
|
|
2399
|
+
+/**
|
|
2400
|
+
+ * balance domain zod schemas (names derived from map keys: balanceGetRequestSchema /
|
|
2401
|
+
+ * balanceGetValueSchema).
|
|
2402
|
+
+ */
|
|
2403
|
+
+
|
|
2404
|
+
+import { z } from 'zod'
|
|
2405
|
+
+import type { RequestPayload, ResponseValue } from './rpc-map.ts'
|
|
2406
|
+
+import type { Wire } from './rpc.schema.ts'
|
|
2407
|
+
+import type { BalanceView } from './balance.ts'
|
|
2408
|
+
+import { sessionIdSchema } from './sessions.schema.ts'
|
|
2409
|
+
+
|
|
2410
|
+
+/** BalanceView row of balance.get. */
|
|
2411
|
+
+export const balanceViewSchema = z.object({
|
|
2412
|
+
+ currency: z.string().min(1),
|
|
2413
|
+
+ total: z.string().min(1),
|
|
2414
|
+
+ granted: z.string().min(1),
|
|
2415
|
+
+ toppedUp: z.string().min(1),
|
|
2416
|
+
+}) satisfies z.ZodType<Wire<BalanceView>>
|
|
2417
|
+
+
|
|
2418
|
+
+/** balance.get request payload. */
|
|
2419
|
+
+export const balanceGetRequestSchema = z.object({
|
|
2420
|
+
+ // [本地改造 2026-08-16] 可选 sessionId:host 据此判断当前会话的模型 provider
|
|
2421
|
+
+ // 是否为 deepseek 直连——非直连(如 qwen 百炼)时余额指示不显示。
|
|
2422
|
+
+ sessionId: sessionIdSchema.optional(),
|
|
2423
|
+
+}) satisfies z.ZodType<Wire<RequestPayload<'balance.get'>>>
|
|
2424
|
+
+
|
|
2425
|
+
+/** balance.get response value. */
|
|
2426
|
+
+export const balanceGetValueSchema = z.object({
|
|
2427
|
+
+ balance: balanceViewSchema.nullable(),
|
|
2428
|
+
+}) satisfies z.ZodType<Wire<ResponseValue<'balance.get'>>>
|
|
2429
|
+
diff --git a/packages/host/apiproxy/src/api/balance.ts b/packages/host/apiproxy/src/api/balance.ts
|
|
2430
|
+
new file mode 100644
|
|
2431
|
+
index 0000000000..ec764e384f
|
|
2432
|
+
--- /dev/null
|
|
2433
|
+
+++ b/packages/host/apiproxy/src/api/balance.ts
|
|
2434
|
+
@@ -0,0 +1,32 @@
|
|
2435
|
+
+/**
|
|
2436
|
+
+ * balance domain contract: DeepSeek 直连账户余额查询。仅当部署配置了
|
|
2437
|
+
+ * DEEPSEEK_API_KEY(直连 deepseek 路由)时有意义;走 LiteLLM 等其他
|
|
2438
|
+
+ * 路由的部署返回 null(前端据此隐藏余额指示)。
|
|
2439
|
+
+ */
|
|
2440
|
+
+
|
|
2441
|
+
+import type { RpcRequest, RpcResponse } from './rpc.ts'
|
|
2442
|
+
+import type { SessionId } from '@deepseek-ai/dsh-session/types'
|
|
2443
|
+
+
|
|
2444
|
+
+/** Wire view of one DeepSeek balance entry. */
|
|
2445
|
+
+export interface BalanceView {
|
|
2446
|
+
+ /** 币种(如 CNY)。 */
|
|
2447
|
+
+ currency: string
|
|
2448
|
+
+ /** 总额。 */
|
|
2449
|
+
+ total: string
|
|
2450
|
+
+ /** 赠送金额。 */
|
|
2451
|
+
+ granted: string
|
|
2452
|
+
+ /** 充值金额。 */
|
|
2453
|
+
+ toppedUp: string
|
|
2454
|
+
+}
|
|
2455
|
+
+
|
|
2456
|
+
+/** Balance-domain unary methods (the map keys balance.* of RpcMethodMap). */
|
|
2457
|
+
+export interface BalanceApi {
|
|
2458
|
+
+ /**
|
|
2459
|
+
+ * 查询 DeepSeek 直连账户余额。无 DEEPSEEK_API_KEY 凭证或查询失败时
|
|
2460
|
+
+ * 返回 null(非直连部署/网络异常均不报错——余额只是辅助指示)。
|
|
2461
|
+
+ * [本地改造 2026-08-16] 可选 sessionId:调用方传入当前会话 id 时,
|
|
2462
|
+
+ * host 仅在该会话的模型 provider 为 deepseek 直连(deepseek-official)
|
|
2463
|
+
+ * 时返回余额,否则返回 null(余额指示只对 deepseek 直连模型显示)。
|
|
2464
|
+
+ */
|
|
2465
|
+
+ get(request: RpcRequest<{ sessionId?: SessionId }>): Promise<RpcResponse<{ balance: BalanceView | null }>>
|
|
2466
|
+
+}
|
|
2467
|
+
diff --git a/packages/host/apiproxy/src/api/index.ts b/packages/host/apiproxy/src/api/index.ts
|
|
2468
|
+
index b5e1d1ffd9..83c0b56c17 100644
|
|
2469
|
+
--- a/packages/host/apiproxy/src/api/index.ts
|
|
2470
|
+
+++ b/packages/host/apiproxy/src/api/index.ts
|
|
2471
|
+
@@ -15,6 +15,7 @@ import type { GoalsApi } from './goals.ts'
|
|
2472
|
+
import type { SettingsApi } from './settings.ts'
|
|
2473
|
+
import type { CredentialsApi } from './credentials.ts'
|
|
2474
|
+
import type { LlmApi } from './llm.ts'
|
|
2475
|
+
+import type { BalanceApi } from './balance.ts'
|
|
2476
|
+
import type { DownloadsApi } from './downloads.ts'
|
|
2477
|
+
import type { ClientResponse, RpcReceipt } from './rpc.ts'
|
|
2478
|
+
|
|
2479
|
+
@@ -31,6 +32,8 @@ export interface ApiProxy {
|
|
2480
|
+
settings: SettingsApi
|
|
2481
|
+
credentials: CredentialsApi
|
|
2482
|
+
llm: LlmApi
|
|
2483
|
+
+ /** DeepSeek 直连账户余额(辅助指示;非直连返回 null)。 */
|
|
2484
|
+
+ balance: BalanceApi
|
|
2485
|
+
/** Host-only download surfaces (GET, no wire envelope); absent from IApiClient. */
|
|
2486
|
+
downloads: DownloadsApi
|
|
2487
|
+
/**
|
|
2488
|
+
@@ -46,6 +49,7 @@ export type {
|
|
2489
|
+
HistoryEntry, ModelCatalogFailure, ModelCatalogModel, ModelProviderGroup, ModelReasoning,
|
|
2490
|
+
ModelReasoningEffort, ModelSelection, PromptContentPart, QueueAction, SessionModels,
|
|
2491
|
+
SessionListMetadata, SessionProjectionsBlock, SessionSearchItem, SessionsApi, SessionSummary,
|
|
2492
|
+
+ VoiceAttachmentRef, VoiceMediaType,
|
|
2493
|
+
} from './sessions.ts'
|
|
2494
|
+
export type { DirectoryEntry, DirectoryListing, HostApi } from './host.ts'
|
|
2495
|
+
export type {
|
|
2496
|
+
@@ -61,6 +65,7 @@ export type { GoalsApi, GoalId, GoalRef } from './goals.ts'
|
|
2497
|
+
export type { SettingsApi, SettingsNamespaceView, SettingsPathOpView, SettingsSecretView } from './settings.ts'
|
|
2498
|
+
export type { CredentialsApi, CredentialView } from './credentials.ts'
|
|
2499
|
+
export type { ConfigurableProviderView, DiscoveredModelView, LlmApi } from './llm.ts'
|
|
2500
|
+
+export type { BalanceApi, BalanceView } from './balance.ts'
|
|
2501
|
+
export type { DownloadsApi } from './downloads.ts'
|
|
2502
|
+
export type { ApprovalResponsePayload } from './approvals.ts'
|
|
2503
|
+
|
|
2504
|
+
diff --git a/packages/host/apiproxy/src/api/rpc-map.ts b/packages/host/apiproxy/src/api/rpc-map.ts
|
|
2505
|
+
index 80dede1799..1f70c4c8a2 100644
|
|
2506
|
+
--- a/packages/host/apiproxy/src/api/rpc-map.ts
|
|
2507
|
+
+++ b/packages/host/apiproxy/src/api/rpc-map.ts
|
|
2508
|
+
@@ -13,6 +13,7 @@ import type { GoalsApi } from './goals.ts'
|
|
2509
|
+
import type { SettingsApi } from './settings.ts'
|
|
2510
|
+
import type { CredentialsApi } from './credentials.ts'
|
|
2511
|
+
import type { LlmApi } from './llm.ts'
|
|
2512
|
+
+import type { BalanceApi } from './balance.ts'
|
|
2513
|
+
import type { SubagentsApi } from './subagents.ts'
|
|
2514
|
+
import type { RpcResponse } from './rpc.ts'
|
|
2515
|
+
|
|
2516
|
+
@@ -32,6 +33,10 @@ export interface RpcMethodMap {
|
|
2517
|
+
'session.fork': SessionsApi['fork']
|
|
2518
|
+
'session.prompt': SessionsApi['prompt']
|
|
2519
|
+
'session.attachment': SessionsApi['attachment']
|
|
2520
|
+
+ 'session.voice': SessionsApi['voice']
|
|
2521
|
+
+ 'session.voiceAsr': SessionsApi['voiceAsr']
|
|
2522
|
+
+ 'session.voiceTts': SessionsApi['voiceTts']
|
|
2523
|
+
+ 'session.sendVoiceMessage': SessionsApi['sendVoiceMessage']
|
|
2524
|
+
'session.updateQueue': SessionsApi['updateQueue']
|
|
2525
|
+
'session.cancel': SessionsApi['cancel']
|
|
2526
|
+
'subagent.list': SubagentsApi['list']
|
|
2527
|
+
@@ -74,6 +79,7 @@ export interface RpcMethodMap {
|
|
2528
|
+
'llm.providers': LlmApi['providers']
|
|
2529
|
+
'llm.models': LlmApi['models']
|
|
2530
|
+
'llm.discoverModels': LlmApi['discoverModels']
|
|
2531
|
+
+ 'balance.get': BalanceApi['get']
|
|
2532
|
+
}
|
|
2533
|
+
|
|
2534
|
+
/** Business request payload of method K (reaches through the RpcRequest narrow form to payload). */
|
|
2535
|
+
diff --git a/packages/host/apiproxy/src/api/sessions.schema.ts b/packages/host/apiproxy/src/api/sessions.schema.ts
|
|
2536
|
+
index c415015776..ba4a8596b4 100644
|
|
2537
|
+
--- a/packages/host/apiproxy/src/api/sessions.schema.ts
|
|
2538
|
+
+++ b/packages/host/apiproxy/src/api/sessions.schema.ts
|
|
2539
|
+
@@ -13,6 +13,7 @@ import type { Wire } from './rpc.schema.ts'
|
|
2540
|
+
import type {
|
|
2541
|
+
HistoryEntry, ModelCatalogFailure, ModelCatalogModel, ModelProviderGroup, ModelReasoning,
|
|
2542
|
+
ModelReasoningEffort, ModelSelection, SessionListMetadata, SessionProjectionsBlock, SessionSearchItem, SessionSummary,
|
|
2543
|
+
+ VoiceAttachmentRef,
|
|
2544
|
+
} from './sessions.ts'
|
|
2545
|
+
import type { ToolEventView } from './events.ts'
|
|
2546
|
+
import type { AttachmentIdType, ImageAttachmentLimits, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
|
2547
|
+
@@ -279,10 +280,21 @@ export const imageMediaTypeSchema = z.union([
|
|
2548
|
+
z.literal('image/gif'),
|
|
2549
|
+
])
|
|
2550
|
+
|
|
2551
|
+
+/** Audio media types accepted by the version-one browser voice wire. */
|
|
2552
|
+
+export const voiceMediaTypeSchema = z.union([
|
|
2553
|
+
+ z.literal('audio/webm'),
|
|
2554
|
+
+ z.literal('audio/ogg'),
|
|
2555
|
+
+ z.literal('audio/mp4'),
|
|
2556
|
+
+ z.literal('audio/wav'),
|
|
2557
|
+
+ // [本地改造 2026-08-16] TTS 语音回复为 MP3(synthesizeReplyVoice 统一转 mp3)。
|
|
2558
|
+
+ z.literal('audio/mpeg'),
|
|
2559
|
+
+])
|
|
2560
|
+
+
|
|
2561
|
+
/** Prompt wire content is intentionally narrower than merge-extensible durable core content. */
|
|
2562
|
+
export const promptContentPartSchema = z.discriminatedUnion('type', [
|
|
2563
|
+
z.object({ type: z.literal('text'), text: z.string() }),
|
|
2564
|
+
z.object({ type: z.literal('image'), mediaType: imageMediaTypeSchema, data: z.string(), name: z.string().optional() }),
|
|
2565
|
+
+ z.object({ type: z.literal('voice'), mediaType: voiceMediaTypeSchema, data: z.string(), durationMs: z.number().int().positive().optional() }),
|
|
2566
|
+
])
|
|
2567
|
+
|
|
2568
|
+
/** session.prompt request payload, including optional browser-local request provenance. */
|
|
2569
|
+
@@ -327,6 +339,66 @@ export const sessionAttachmentValueSchema = z.object({
|
|
2570
|
+
data: z.string(),
|
|
2571
|
+
}) satisfies z.ZodType<Wire<ResponseValue<'session.attachment'>>>
|
|
2572
|
+
|
|
2573
|
+
+/** Durable voice reference returned from the authenticated session lookup. */
|
|
2574
|
+
+export const voiceAttachmentRefSchema = z.object({
|
|
2575
|
+
+ voiceId: z.string().min(1),
|
|
2576
|
+
+ mediaType: voiceMediaTypeSchema,
|
|
2577
|
+
+ bytes: z.number().int().positive(),
|
|
2578
|
+
+ durationMs: z.number().int().positive().optional(),
|
|
2579
|
+
+ transcript: z.string().optional(),
|
|
2580
|
+
+}) as unknown as z.ZodType<VoiceAttachmentRef>
|
|
2581
|
+
+
|
|
2582
|
+
+/** session.voice request payload. */
|
|
2583
|
+
+export const sessionVoiceRequestSchema = z.object({
|
|
2584
|
+
+ sessionId: sessionIdSchema,
|
|
2585
|
+
+ voiceId: z.string().min(1),
|
|
2586
|
+
+}) satisfies z.ZodType<Wire<RequestPayload<'session.voice'>>>
|
|
2587
|
+
+
|
|
2588
|
+
+/** session.voice response value. */
|
|
2589
|
+
+export const sessionVoiceValueSchema = z.object({
|
|
2590
|
+
+ attachment: voiceAttachmentRefSchema,
|
|
2591
|
+
+ data: z.string(),
|
|
2592
|
+
+}) satisfies z.ZodType<Wire<ResponseValue<'session.voice'>>>
|
|
2593
|
+
+
|
|
2594
|
+
+/** session.voiceAsr request payload. */
|
|
2595
|
+
+export const sessionVoiceAsrRequestSchema = z.object({
|
|
2596
|
+
+ sessionId: sessionIdSchema,
|
|
2597
|
+
+ mediaType: voiceMediaTypeSchema,
|
|
2598
|
+
+ data: z.string(),
|
|
2599
|
+
+ durationMs: z.number().int().positive().optional(),
|
|
2600
|
+
+}) satisfies z.ZodType<Wire<RequestPayload<'session.voiceAsr'>>>
|
|
2601
|
+
+
|
|
2602
|
+
+/** session.voiceAsr response value. */
|
|
2603
|
+
+export const sessionVoiceAsrValueSchema = z.object({
|
|
2604
|
+
+ text: z.string().nullable(),
|
|
2605
|
+
+}) satisfies z.ZodType<Wire<ResponseValue<'session.voiceAsr'>>>
|
|
2606
|
+
+
|
|
2607
|
+
+/** session.voiceTts request payload. */
|
|
2608
|
+
+export const sessionVoiceTtsRequestSchema = z.object({
|
|
2609
|
+
+ sessionId: sessionIdSchema,
|
|
2610
|
+
+ text: z.string().min(1),
|
|
2611
|
+
+ provider: z.string().optional(),
|
|
2612
|
+
+}) satisfies z.ZodType<Wire<RequestPayload<'session.voiceTts'>>>
|
|
2613
|
+
+
|
|
2614
|
+
+/** session.voiceTts response value. */
|
|
2615
|
+
+export const sessionVoiceTtsValueSchema = z.object({
|
|
2616
|
+
+ mediaType: z.string(),
|
|
2617
|
+
+ data: z.string(),
|
|
2618
|
+
+ durationMs: z.number().int().positive().optional(),
|
|
2619
|
+
+}).nullable() satisfies z.ZodType<Wire<ResponseValue<'session.voiceTts'>>>
|
|
2620
|
+
+
|
|
2621
|
+
+/** session.sendVoiceMessage request payload. */
|
|
2622
|
+
+export const sessionSendVoiceMessageRequestSchema = z.object({
|
|
2623
|
+
+ sessionId: sessionIdSchema,
|
|
2624
|
+
+ text: z.string().min(1),
|
|
2625
|
+
+ provider: z.string().optional(),
|
|
2626
|
+
+}) satisfies z.ZodType<Wire<RequestPayload<'session.sendVoiceMessage'>>>
|
|
2627
|
+
+
|
|
2628
|
+
+/** session.sendVoiceMessage response value. */
|
|
2629
|
+
+export const sessionSendVoiceMessageValueSchema = z.object({
|
|
2630
|
+
+ accepted: z.literal(true),
|
|
2631
|
+
+}) satisfies z.ZodType<Wire<ResponseValue<'session.sendVoiceMessage'>>>
|
|
2632
|
+
+
|
|
2633
|
+
/** session.updateQueue request payload. */
|
|
2634
|
+
export const sessionUpdateQueueRequestSchema = z.object({
|
|
2635
|
+
sessionId: sessionIdSchema,
|
|
2636
|
+
diff --git a/packages/host/apiproxy/src/api/sessions.ts b/packages/host/apiproxy/src/api/sessions.ts
|
|
2637
|
+
index 2e7c9b22f4..3ace1972e3 100644
|
|
2638
|
+
--- a/packages/host/apiproxy/src/api/sessions.ts
|
|
2639
|
+
+++ b/packages/host/apiproxy/src/api/sessions.ts
|
|
2640
|
+
@@ -87,6 +87,24 @@ export interface SessionProjectionsBlock {
|
|
2641
|
+
export type PromptContentPart =
|
|
2642
|
+
| { type: 'text'; text: string }
|
|
2643
|
+
| { type: 'image'; mediaType: ImageMediaType; data: string; name?: string }
|
|
2644
|
+
+ | { type: 'voice'; mediaType: VoiceMediaType; data: string; durationMs?: number }
|
|
2645
|
+
+
|
|
2646
|
+
+/** Audio formats accepted by the version-one browser voice wire. */
|
|
2647
|
+
+export type VoiceMediaType = 'audio/webm' | 'audio/ogg' | 'audio/mp4' | 'audio/wav' | 'audio/mpeg'
|
|
2648
|
+
+
|
|
2649
|
+
+/** Durable, serializable metadata for one immutable voice object. */
|
|
2650
|
+
+export interface VoiceAttachmentRef {
|
|
2651
|
+
+ /** Opaque storage identifier; never a filesystem path or bearer URL. */
|
|
2652
|
+
+ voiceId: string
|
|
2653
|
+
+ /** Media type of the stored browser-uploaded bytes. */
|
|
2654
|
+
+ mediaType: VoiceMediaType
|
|
2655
|
+
+ /** Exact encoded byte length. */
|
|
2656
|
+
+ bytes: number
|
|
2657
|
+
+ /** Optional recorder-reported length in milliseconds. */
|
|
2658
|
+
+ durationMs?: number
|
|
2659
|
+
+ /** Local ASR transcript; absent when recognition failed or is unavailable. */
|
|
2660
|
+
+ transcript?: string
|
|
2661
|
+
+}
|
|
2662
|
+
|
|
2663
|
+
/** Complete model selection for one session. */
|
|
2664
|
+
export interface ModelSelection {
|
|
2665
|
+
@@ -356,6 +374,49 @@ export interface SessionsApi {
|
|
2666
|
+
attachment(request: RpcRequest<{ sessionId: SessionId; attachmentId: AttachmentIdType }>):
|
|
2667
|
+
Promise<RpcResponse<{ attachment: ImageAttachmentRef; data: string }>>
|
|
2668
|
+
|
|
2669
|
+
+ /** Reads one durable voice object after proving that this session's log references its id. */
|
|
2670
|
+
+ voice(request: RpcRequest<{ sessionId: SessionId; voiceId: string }>):
|
|
2671
|
+
+ Promise<RpcResponse<{ attachment: VoiceAttachmentRef; data: string }>>
|
|
2672
|
+
+
|
|
2673
|
+
+ /**
|
|
2674
|
+
+ * Transcribe one browser recording on the fly without storing it (the
|
|
2675
|
+
+ * "speak-to-text" gesture sends the resulting text as an ordinary message).
|
|
2676
|
+
+ * @param request - temporary voice bytes exactly as the composer captured them.
|
|
2677
|
+
+ * @returns recognized text, or null when recognition failed or is unavailable.
|
|
2678
|
+
+ */
|
|
2679
|
+
+ voiceAsr(request: RpcRequest<{
|
|
2680
|
+
+ sessionId: SessionId
|
|
2681
|
+
+ mediaType: VoiceMediaType
|
|
2682
|
+
+ data: string
|
|
2683
|
+
+ durationMs?: number
|
|
2684
|
+
+ }>): Promise<RpcResponse<{ text: string | null }>>
|
|
2685
|
+
+
|
|
2686
|
+
+ /**
|
|
2687
|
+
+ * Synthesize one reply into speech through the local TTS engines and return
|
|
2688
|
+
+ * the encoded audio directly (nothing durable — the browser plays it inline).
|
|
2689
|
+
+ * @param request - reply text and optional TTS provider override (auto by default).
|
|
2690
|
+
+ * @returns encoded audio and declared media type, or null when synthesis failed.
|
|
2691
|
+
+ */
|
|
2692
|
+
+ voiceTts(request: RpcRequest<{
|
|
2693
|
+
+ sessionId: SessionId
|
|
2694
|
+
+ text: string
|
|
2695
|
+
+ provider?: string
|
|
2696
|
+
+ }>): Promise<RpcResponse<{ mediaType: string; data: string; durationMs?: number } | null>>
|
|
2697
|
+
+
|
|
2698
|
+
+ /**
|
|
2699
|
+
+ * [本地改造 2026-08-16] Send one synthesized voice message into the session:
|
|
2700
|
+
+ * the host speaks `text`, persists it as a `voice/reply` event (an independent
|
|
2701
|
+
+ * durable voice row beside the user's own voice messages), so the agent can
|
|
2702
|
+
+ * actively send a voice message on its own — no user voice needed.
|
|
2703
|
+
+ * @param request - text to speak and optional TTS provider override.
|
|
2704
|
+
+ * @returns accepted once the voice/reply event is logged.
|
|
2705
|
+
+ */
|
|
2706
|
+
+ sendVoiceMessage(request: RpcRequest<{
|
|
2707
|
+
+ sessionId: SessionId
|
|
2708
|
+
+ text: string
|
|
2709
|
+
+ provider?: string
|
|
2710
|
+
+ }>): Promise<RpcResponse<{ accepted: true }>>
|
|
2711
|
+
+
|
|
2712
|
+
/**
|
|
2713
|
+
* Edits, removes, or strictly steers one pending queued occurrence on an ordinary session.
|
|
2714
|
+
* Session-backed subagents reject with `agent-busy`.
|
|
2715
|
+
diff --git a/packages/host/apiproxy/src/edge-tts.ts b/packages/host/apiproxy/src/edge-tts.ts
|
|
2716
|
+
new file mode 100644
|
|
2717
|
+
index 0000000000..6b2b3dcbcc
|
|
2718
|
+
--- /dev/null
|
|
2719
|
+
+++ b/packages/host/apiproxy/src/edge-tts.ts
|
|
2720
|
+
@@ -0,0 +1,119 @@
|
|
2721
|
+
+/**
|
|
2722
|
+
+ * Microsoft Edge TTS — 原生 WebSocket 客户端(免费,无需 API key)。
|
|
2723
|
+
+ * [本地改造 2026-08-16] 从 agents-to-im 移植进 DSH 本体(TTS 独立化,不再依赖外部工具)。
|
|
2724
|
+
+ * 对齐 Python edge-tts v7.2.8 的 DRM + Headers;Sec-MS-GEC 令牌放在 URL 参数里。
|
|
2725
|
+
+ * @module dsh-host-apiproxy/edge-tts
|
|
2726
|
+
+ */
|
|
2727
|
+
+
|
|
2728
|
+
+import WebSocket from 'ws'
|
|
2729
|
+
+import { createHash, randomBytes } from 'node:crypto'
|
|
2730
|
+
+
|
|
2731
|
+
+const TRUSTED_CLIENT_TOKEN = '6A5AA1D4EAFF4E9FB37E23D68491D6F4'
|
|
2732
|
+
+const BASE_URL = 'speech.platform.bing.com/consumer/speech/synthesize/readaloud'
|
|
2733
|
+
+const CHROMIUM_FULL_VERSION = '143.0.3650.75'
|
|
2734
|
+
+const CHROMIUM_MAJOR_VERSION = '143'
|
|
2735
|
+
+const SEC_MS_GEC_VERSION = `1-${CHROMIUM_FULL_VERSION}`
|
|
2736
|
+
+const WIN_EPOCH = 11644473600
|
|
2737
|
+
+const S_TO_NS = 1e9
|
|
2738
|
+
+
|
|
2739
|
+
+function generateSecMsGec(): string {
|
|
2740
|
+
+ let ticks = Date.now() / 1000
|
|
2741
|
+
+ ticks += WIN_EPOCH
|
|
2742
|
+
+ ticks -= ticks % 300
|
|
2743
|
+
+ ticks *= S_TO_NS / 100
|
|
2744
|
+
+ const strToHash = `${Math.floor(ticks)}${TRUSTED_CLIENT_TOKEN}`
|
|
2745
|
+
+ return createHash('sha256').update(strToHash, 'ascii').digest('hex').toUpperCase()
|
|
2746
|
+
+}
|
|
2747
|
+
+
|
|
2748
|
+
+function generateMuid(): string {
|
|
2749
|
+
+ return randomBytes(16).toString('hex').toUpperCase()
|
|
2750
|
+
+}
|
|
2751
|
+
+
|
|
2752
|
+
+function uuid(): string {
|
|
2753
|
+
+ return crypto.randomUUID().replaceAll('-', '')
|
|
2754
|
+
+}
|
|
2755
|
+
+
|
|
2756
|
+
+function escapeXml(s: string): string {
|
|
2757
|
+
+ return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
2758
|
+
+ .replace(/"/g, '"').replace(/'/g, ''')
|
|
2759
|
+
+}
|
|
2760
|
+
+
|
|
2761
|
+
+function getWssUrl(): string {
|
|
2762
|
+
+ return `wss://${BASE_URL}/edge/v1?TrustedClientToken=${TRUSTED_CLIENT_TOKEN}`
|
|
2763
|
+
+ + `&Sec-MS-GEC=${generateSecMsGec()}&Sec-MS-GEC-Version=${SEC_MS_GEC_VERSION}`
|
|
2764
|
+
+}
|
|
2765
|
+
+
|
|
2766
|
+
+function getWssHeaders(): Record<string, string> {
|
|
2767
|
+
+ return {
|
|
2768
|
+
+ 'Pragma': 'no-cache',
|
|
2769
|
+
+ 'Cache-Control': 'no-cache',
|
|
2770
|
+
+ 'Origin': 'chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold',
|
|
2771
|
+
+ 'User-Agent': `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${CHROMIUM_MAJOR_VERSION}.0.0.0 Safari/537.36 Edg/${CHROMIUM_MAJOR_VERSION}.0.0.0`,
|
|
2772
|
+
+ 'Accept-Encoding': 'gzip, deflate, br, zstd',
|
|
2773
|
+
+ 'Accept-Language': 'en-US,en;q=0.9',
|
|
2774
|
+
+ 'Cookie': `muid=${generateMuid()};`,
|
|
2775
|
+
+ }
|
|
2776
|
+
+}
|
|
2777
|
+
+
|
|
2778
|
+
+/**
|
|
2779
|
+
+ * Synthesize speech through the free Microsoft Edge endpoint.
|
|
2780
|
+
+ * @param text - plain text to speak.
|
|
2781
|
+
+ * @param voice - Edge voice name (default zh-CN-XiaoxiaoNeural).
|
|
2782
|
+
+ * @returns MP3 bytes (audio-24khz-48kbitrate-mono-mp3).
|
|
2783
|
+
+ */
|
|
2784
|
+
+export function edgeTts(text: string, voice = 'zh-CN-XiaoxiaoNeural'): Promise<Buffer> {
|
|
2785
|
+
+ return new Promise((resolve, reject) => {
|
|
2786
|
+
+ const ws = new WebSocket(getWssUrl(), { headers: getWssHeaders() })
|
|
2787
|
+
+ const audioData: Buffer[] = []
|
|
2788
|
+
+ let messageTimeout: ReturnType<typeof setTimeout> | undefined
|
|
2789
|
+
+
|
|
2790
|
+
+ const connectTimeout = setTimeout(() => {
|
|
2791
|
+
+ ws.terminate()
|
|
2792
|
+
+ reject(new Error('Edge TTS WebSocket connect timeout (10s)'))
|
|
2793
|
+
+ }, 10_000)
|
|
2794
|
+
+
|
|
2795
|
+
+ ws.on('message', (rawData, isBinary) => {
|
|
2796
|
+
+ // ws 默认 binaryType='nodebuffer':运行时数据就是 Buffer。
|
|
2797
|
+
+ const buf = rawData as Buffer
|
|
2798
|
+
+ if (!isBinary) {
|
|
2799
|
+
+ const str = buf.toString('utf8')
|
|
2800
|
+
+ if (str.includes('turn.end')) {
|
|
2801
|
+
+ if (messageTimeout !== undefined) clearTimeout(messageTimeout)
|
|
2802
|
+
+ resolve(Buffer.concat(audioData))
|
|
2803
|
+
+ ws.close()
|
|
2804
|
+
+ }
|
|
2805
|
+
+ return
|
|
2806
|
+
+ }
|
|
2807
|
+
+ const separator = 'Path:audio\r\n'
|
|
2808
|
+
+ const idx = buf.indexOf(separator)
|
|
2809
|
+
+ if (idx !== -1) audioData.push(buf.subarray(idx + separator.length))
|
|
2810
|
+
+ })
|
|
2811
|
+
+
|
|
2812
|
+
+ ws.on('error', (err) => {
|
|
2813
|
+
+ clearTimeout(connectTimeout)
|
|
2814
|
+
+ reject(err)
|
|
2815
|
+
+ })
|
|
2816
|
+
+
|
|
2817
|
+
+ ws.on('open', () => {
|
|
2818
|
+
+ clearTimeout(connectTimeout)
|
|
2819
|
+
+ messageTimeout = setTimeout(() => {
|
|
2820
|
+
+ ws.close()
|
|
2821
|
+
+ reject(new Error('Edge TTS message timeout (30s)'))
|
|
2822
|
+
+ }, 30_000)
|
|
2823
|
+
+
|
|
2824
|
+
+ const speechConfig = JSON.stringify({
|
|
2825
|
+
+ context: { synthesis: { audio: {
|
|
2826
|
+
+ metadataoptions: { sentenceBoundaryEnabled: false, wordBoundaryEnabled: false },
|
|
2827
|
+
+ outputFormat: 'audio-24khz-48kbitrate-mono-mp3',
|
|
2828
|
+
+ } } },
|
|
2829
|
+
+ })
|
|
2830
|
+
+ const configMsg = `X-Timestamp:${Date()}\r\nContent-Type:application/json; charset=utf-8\r\nPath:speech.config\r\n\r\n${speechConfig}`
|
|
2831
|
+
+ ws.send(configMsg, { compress: true })
|
|
2832
|
+
+
|
|
2833
|
+
+ const ssml = '<speak version=\'1.0\' xmlns=\'http://www.w3.org/2001/10/synthesis\' xml:lang=\'zh-CN\'>'
|
|
2834
|
+
+ + `<voice name='${voice}'><prosody pitch='+0Hz' rate='+0%' volume='+0%'>${escapeXml(text)}</prosody></voice></speak>`
|
|
2835
|
+
+ const ssmlMsg = `X-RequestId:${uuid()}\r\nContent-Type:application/ssml+xml\r\nX-Timestamp:${Date()}Z\r\nPath:ssml\r\n\r\n${ssml}`
|
|
2836
|
+
+ ws.send(ssmlMsg, { compress: true })
|
|
2837
|
+
+ })
|
|
2838
|
+
+ })
|
|
2839
|
+
+}
|
|
2840
|
+
diff --git a/packages/host/apiproxy/src/fetch/client.ts b/packages/host/apiproxy/src/fetch/client.ts
|
|
2841
|
+
index 70e3ece58f..27ab78e6db 100644
|
|
2842
|
+
--- a/packages/host/apiproxy/src/fetch/client.ts
|
|
2843
|
+
+++ b/packages/host/apiproxy/src/fetch/client.ts
|
|
2844
|
+
@@ -30,6 +30,10 @@ import {
|
|
2845
|
+
sessionSearchValueSchema,
|
|
2846
|
+
sessionSelectModelValueSchema,
|
|
2847
|
+
sessionUpdateQueueValueSchema,
|
|
2848
|
+
+ sessionVoiceAsrValueSchema,
|
|
2849
|
+
+ sessionVoiceTtsValueSchema,
|
|
2850
|
+
+ sessionVoiceValueSchema,
|
|
2851
|
+
+ sessionSendVoiceMessageValueSchema,
|
|
2852
|
+
} from '../api/sessions.schema.ts'
|
|
2853
|
+
import {
|
|
2854
|
+
workspaceArchiveSessionValueSchema,
|
|
2855
|
+
@@ -61,6 +65,7 @@ import {
|
|
2856
|
+
credentialsDescribeValueSchema, credentialsSetValueSchema, credentialsUnsetValueSchema,
|
|
2857
|
+
} from '../api/credentials.schema.ts'
|
|
2858
|
+
import { llmDiscoverModelsValueSchema, llmModelsValueSchema, llmProvidersValueSchema } from '../api/llm.schema.ts'
|
|
2859
|
+
+import { balanceGetValueSchema } from '../api/balance.schema.ts'
|
|
2860
|
+
import {
|
|
2861
|
+
subagentHistoryValueSchema,
|
|
2862
|
+
subagentInterruptValueSchema,
|
|
2863
|
+
@@ -96,6 +101,10 @@ export interface IApiClient {
|
|
2864
|
+
fork(payload: RequestPayload<'session.fork'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.fork'>>>
|
|
2865
|
+
prompt(payload: RequestPayload<'session.prompt'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.prompt'>>>
|
|
2866
|
+
attachment(payload: RequestPayload<'session.attachment'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.attachment'>>>
|
|
2867
|
+
+ voice(payload: RequestPayload<'session.voice'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.voice'>>>
|
|
2868
|
+
+ voiceAsr(payload: RequestPayload<'session.voiceAsr'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.voiceAsr'>>>
|
|
2869
|
+
+ voiceTts(payload: RequestPayload<'session.voiceTts'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.voiceTts'>>>
|
|
2870
|
+
+ sendVoiceMessage(payload: RequestPayload<'session.sendVoiceMessage'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.sendVoiceMessage'>>>
|
|
2871
|
+
updateQueue(payload: RequestPayload<'session.updateQueue'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.updateQueue'>>>
|
|
2872
|
+
cancel(payload: RequestPayload<'session.cancel'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'session.cancel'>>>
|
|
2873
|
+
}
|
|
2874
|
+
@@ -161,6 +170,9 @@ export interface IApiClient {
|
|
2875
|
+
models(payload: RequestPayload<'llm.models'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'llm.models'>>>
|
|
2876
|
+
discoverModels(payload: RequestPayload<'llm.discoverModels'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'llm.discoverModels'>>>
|
|
2877
|
+
}
|
|
2878
|
+
+ balance: {
|
|
2879
|
+
+ get(payload: RequestPayload<'balance.get'>, signal?: AbortSignal): Promise<RpcResponse<ResponseValue<'balance.get'>>>
|
|
2880
|
+
+ }
|
|
2881
|
+
/** client-response passthrough (rpcId is a backfill of the server-request's id — never minted here). */
|
|
2882
|
+
respond(message: ClientResponse, signal?: AbortSignal): Promise<RpcReceipt>
|
|
2883
|
+
}
|
|
2884
|
+
@@ -180,6 +192,10 @@ const UNARY_VALUE_SCHEMAS: { [K in keyof RpcMethodMap]: z.ZodType<Wire<ResponseV
|
|
2885
|
+
'session.fork': sessionForkValueSchema,
|
|
2886
|
+
'session.prompt': sessionPromptValueSchema,
|
|
2887
|
+
'session.attachment': sessionAttachmentValueSchema,
|
|
2888
|
+
+ 'session.voice': sessionVoiceValueSchema,
|
|
2889
|
+
+ 'session.voiceAsr': sessionVoiceAsrValueSchema,
|
|
2890
|
+
+ 'session.voiceTts': sessionVoiceTtsValueSchema,
|
|
2891
|
+
+ 'session.sendVoiceMessage': sessionSendVoiceMessageValueSchema,
|
|
2892
|
+
'session.updateQueue': sessionUpdateQueueValueSchema,
|
|
2893
|
+
'session.cancel': sessionCancelValueSchema,
|
|
2894
|
+
'subagent.list': subagentListValueSchema,
|
|
2895
|
+
@@ -222,6 +238,7 @@ const UNARY_VALUE_SCHEMAS: { [K in keyof RpcMethodMap]: z.ZodType<Wire<ResponseV
|
|
2896
|
+
'llm.providers': llmProvidersValueSchema,
|
|
2897
|
+
'llm.models': llmModelsValueSchema,
|
|
2898
|
+
'llm.discoverModels': llmDiscoverModelsValueSchema,
|
|
2899
|
+
+ 'balance.get': balanceGetValueSchema,
|
|
2900
|
+
}
|
|
2901
|
+
|
|
2902
|
+
/** Default timeout for bounded unary calls (rpc-compare 2026-07-19: a hung host must not leave callers pending forever). */
|
|
2903
|
+
@@ -420,6 +437,10 @@ export abstract class AbstractApiClient implements IApiClient {
|
|
2904
|
+
fork: (payload, signal) => this.callUnary('session.fork', payload, signal),
|
|
2905
|
+
prompt: (payload, signal) => this.callUnary('session.prompt', payload, signal),
|
|
2906
|
+
attachment: (payload, signal) => this.callUnary('session.attachment', payload, signal),
|
|
2907
|
+
+ voice: (payload, signal) => this.callUnary('session.voice', payload, signal),
|
|
2908
|
+
+ voiceAsr: (payload, signal) => this.callUnary('session.voiceAsr', payload, signal),
|
|
2909
|
+
+ voiceTts: (payload, signal) => this.callUnary('session.voiceTts', payload, signal),
|
|
2910
|
+
+ sendVoiceMessage: (payload, signal) => this.callUnary('session.sendVoiceMessage', payload, signal),
|
|
2911
|
+
updateQueue: (payload, signal) => this.callUnary('session.updateQueue', payload, signal),
|
|
2912
|
+
cancel: (payload, signal) => this.callUnary('session.cancel', payload, signal),
|
|
2913
|
+
}
|
|
2914
|
+
@@ -500,6 +521,10 @@ export abstract class AbstractApiClient implements IApiClient {
|
|
2915
|
+
discoverModels: (payload, signal) => this.callUnary('llm.discoverModels', payload, signal),
|
|
2916
|
+
}
|
|
2917
|
+
|
|
2918
|
+
+ readonly balance: IApiClient['balance'] = {
|
|
2919
|
+
+ get: (payload, signal) => this.callUnary('balance.get', payload, signal),
|
|
2920
|
+
+ }
|
|
2921
|
+
+
|
|
2922
|
+
readonly events: IApiClient['events'] = {
|
|
2923
|
+
mux: (payload, signal, onOpen) => this.openMux(payload, signal, onOpen),
|
|
2924
|
+
host: (payload, signal, onOpen) => this.openHost(payload, signal, onOpen),
|
|
2925
|
+
diff --git a/packages/host/apiproxy/src/fetch/handler.ts b/packages/host/apiproxy/src/fetch/handler.ts
|
|
2926
|
+
index 697171ec53..e600ea2a07 100644
|
|
2927
|
+
--- a/packages/host/apiproxy/src/fetch/handler.ts
|
|
2928
|
+
+++ b/packages/host/apiproxy/src/fetch/handler.ts
|
|
2929
|
+
@@ -28,6 +28,10 @@ import {
|
|
2930
|
+
sessionSearchRequestSchema,
|
|
2931
|
+
sessionSelectModelRequestSchema,
|
|
2932
|
+
sessionUpdateQueueRequestSchema,
|
|
2933
|
+
+ sessionVoiceAsrRequestSchema,
|
|
2934
|
+
+ sessionVoiceRequestSchema,
|
|
2935
|
+
+ sessionVoiceTtsRequestSchema,
|
|
2936
|
+
+ sessionSendVoiceMessageRequestSchema,
|
|
2937
|
+
} from '../api/sessions.schema.ts'
|
|
2938
|
+
import {
|
|
2939
|
+
hostCreateDirectoryRequestSchema, hostDescribeRequestSchema,
|
|
2940
|
+
@@ -64,6 +68,7 @@ import {
|
|
2941
|
+
credentialsDescribeRequestSchema, credentialsSetRequestSchema, credentialsUnsetRequestSchema,
|
|
2942
|
+
} from '../api/credentials.schema.ts'
|
|
2943
|
+
import { llmDiscoverModelsRequestSchema, llmModelsRequestSchema, llmProvidersRequestSchema } from '../api/llm.schema.ts'
|
|
2944
|
+
+import { balanceGetRequestSchema } from '../api/balance.schema.ts'
|
|
2945
|
+
import {
|
|
2946
|
+
subagentHistoryRequestSchema,
|
|
2947
|
+
subagentInterruptRequestSchema,
|
|
2948
|
+
@@ -98,6 +103,10 @@ const UNARY_ROUTES: UnaryRoutes = {
|
|
2949
|
+
'session.fork': { schema: sessionForkRequestSchema, invoke: (api, r) => api.sessions.fork(r) },
|
|
2950
|
+
'session.prompt': { schema: sessionPromptRequestSchema, invoke: (api, r) => api.sessions.prompt(r) },
|
|
2951
|
+
'session.attachment': { schema: sessionAttachmentRequestSchema, invoke: (api, r) => api.sessions.attachment(r) },
|
|
2952
|
+
+ 'session.voice': { schema: sessionVoiceRequestSchema, invoke: (api, r) => api.sessions.voice(r) },
|
|
2953
|
+
+ 'session.voiceAsr': { schema: sessionVoiceAsrRequestSchema, invoke: (api, r) => api.sessions.voiceAsr(r) },
|
|
2954
|
+
+ 'session.voiceTts': { schema: sessionVoiceTtsRequestSchema, invoke: (api, r) => api.sessions.voiceTts(r) },
|
|
2955
|
+
+ 'session.sendVoiceMessage': { schema: sessionSendVoiceMessageRequestSchema, invoke: (api, r) => api.sessions.sendVoiceMessage(r) },
|
|
2956
|
+
'session.updateQueue': { schema: sessionUpdateQueueRequestSchema, invoke: (api, r) => api.sessions.updateQueue(r) },
|
|
2957
|
+
'session.cancel': { schema: sessionCancelRequestSchema, invoke: (api, r) => api.sessions.cancel(r) },
|
|
2958
|
+
'subagent.list': { schema: subagentListRequestSchema, invoke: (api, r, signal) => api.subagents.list(r, signal) },
|
|
2959
|
+
@@ -140,6 +149,7 @@ const UNARY_ROUTES: UnaryRoutes = {
|
|
2960
|
+
'llm.providers': { schema: llmProvidersRequestSchema, invoke: (api, r) => api.llm.providers(r) },
|
|
2961
|
+
'llm.models': { schema: llmModelsRequestSchema, invoke: (api, r) => api.llm.models(r) },
|
|
2962
|
+
'llm.discoverModels': { schema: llmDiscoverModelsRequestSchema, invoke: (api, r, signal) => api.llm.discoverModels(r, signal) },
|
|
2963
|
+
+ 'balance.get': { schema: balanceGetRequestSchema, invoke: (api, r) => api.balance.get(r) },
|
|
2964
|
+
}
|
|
2965
|
+
|
|
2966
|
+
/** Route lookup that narrows an arbitrary path segment to a map key (single cast point for the string→key refinement). */
|
|
2967
|
+
diff --git a/packages/host/apiproxy/src/index.ts b/packages/host/apiproxy/src/index.ts
|
|
2968
|
+
index ac6c770801..edbebc9539 100644
|
|
2969
|
+
--- a/packages/host/apiproxy/src/index.ts
|
|
2970
|
+
+++ b/packages/host/apiproxy/src/index.ts
|
|
2971
|
+
@@ -89,6 +89,7 @@ export class ApiProxyService extends Service implements ApiProxy {
|
|
2972
|
+
readonly settings: ApiProxy['settings']
|
|
2973
|
+
readonly credentials: ApiProxy['credentials']
|
|
2974
|
+
readonly llm: ApiProxy['llm']
|
|
2975
|
+
+ readonly balance: ApiProxy['balance']
|
|
2976
|
+
readonly events: ApiProxy['events']
|
|
2977
|
+
readonly downloads: ApiProxy['downloads']
|
|
2978
|
+
readonly respond: ApiProxy['respond']
|
|
2979
|
+
@@ -117,6 +118,7 @@ export class ApiProxyService extends Service implements ApiProxy {
|
|
2980
|
+
this.settings = api.settings
|
|
2981
|
+
this.credentials = api.credentials
|
|
2982
|
+
this.llm = api.llm
|
|
2983
|
+
+ this.balance = api.balance
|
|
2984
|
+
this.events = api.events
|
|
2985
|
+
this.downloads = api.downloads
|
|
2986
|
+
// createApiProxy returns closures (no `this` capture), so the bind is
|
|
2987
|
+
diff --git a/packages/host/apiproxy/src/voice.ts b/packages/host/apiproxy/src/voice.ts
|
|
2988
|
+
new file mode 100644
|
|
2989
|
+
index 0000000000..33a3eb7d19
|
|
2990
|
+
--- /dev/null
|
|
2991
|
+
+++ b/packages/host/apiproxy/src/voice.ts
|
|
2992
|
+
@@ -0,0 +1,506 @@
|
|
2993
|
+
+/**
|
|
2994
|
+
+ * Durable voice-object storage and local ASR transcription, mirroring the
|
|
2995
|
+
+ * content-addressed attachment layout below `DSH_HOME/attachments/v1`.
|
|
2996
|
+
+ * [本地改造 2026-08-16] voice message support: browser recordings land in the
|
|
2997
|
+
+ * same objects pool as images (sha256-addressed), then transcode to WAV for
|
|
2998
|
+
+ * the local sherpa-onnx ASR service. A failed transcription never blocks the
|
|
2999
|
+
+ * message — the block keeps no transcript and serialization degrades the copy.
|
|
3000
|
+
+ * @module @deepseek-ai/dsh-host-apiproxy/voice
|
|
3001
|
+
+ */
|
|
3002
|
+
+
|
|
3003
|
+
+import { createHash, randomUUID } from 'node:crypto'
|
|
3004
|
+
+import { mkdir, open, readFile, unlink, writeFile } from 'node:fs/promises'
|
|
3005
|
+
+import { constants, readFileSync } from 'node:fs'
|
|
3006
|
+
+import { homedir } from 'node:os'
|
|
3007
|
+
+import { join, resolve } from 'node:path'
|
|
3008
|
+
+import { spawn, spawnSync, execFileSync } from 'node:child_process'
|
|
3009
|
+
+import type { VoiceAttachmentRef, VoiceMediaType } from './api/sessions.ts'
|
|
3010
|
+
+import { edgeTts } from './edge-tts.ts'
|
|
3011
|
+
+
|
|
3012
|
+
+/** ASR media types accepted from the browser wire. */
|
|
3013
|
+
+export const VOICE_MEDIA_TYPES: readonly VoiceMediaType[] = [
|
|
3014
|
+
+ 'audio/webm', 'audio/ogg', 'audio/mp4', 'audio/wav',
|
|
3015
|
+
+]
|
|
3016
|
+
+
|
|
3017
|
+
+/** Maximum encoded bytes accepted for one voice object. */
|
|
3018
|
+
+export const MAX_VOICE_BYTES = 25 * 1024 * 1024
|
|
3019
|
+
+
|
|
3020
|
+
+/** Absolute versioned storage root (same layout as the attachment backend). */
|
|
3021
|
+
+export function voiceStorageRoot(configuredHome?: string): string {
|
|
3022
|
+
+ const home = configuredHome ?? process.env.DSH_HOME ?? join(homedir(), '.dsh')
|
|
3023
|
+
+ return resolve(join(home, 'attachments', 'v1'))
|
|
3024
|
+
+}
|
|
3025
|
+
+
|
|
3026
|
+
+/** Resolve the absolute object path for one voice reference. */
|
|
3027
|
+
+export function voiceObjectPath(root: string, voiceId: string): string {
|
|
3028
|
+
+ return objectPath(root, voiceId.replace(/^sha256:/, ''))
|
|
3029
|
+
+}
|
|
3030
|
+
+
|
|
3031
|
+
+function objectPath(root: string, sha256: string): string {
|
|
3032
|
+
+ return join(root, 'objects', sha256.slice(0, 2), sha256)
|
|
3033
|
+
+}
|
|
3034
|
+
+
|
|
3035
|
+
+/**
|
|
3036
|
+
+ * Store immutable voice bytes below a versioned root, content-addressed by
|
|
3037
|
+
+ * sha256 exactly like image objects. A concurrent duplicate write resolves to
|
|
3038
|
+
+ * the existing object; a conflicting target with different bytes is an error.
|
|
3039
|
+
+ * @param root - absolute `DSH_HOME/attachments/v1` root.
|
|
3040
|
+
+ * @param data - encoded browser recording bytes.
|
|
3041
|
+
+ * @param mediaType - declared recording container format.
|
|
3042
|
+
+ * @param durationMs - optional recorder-reported length.
|
|
3043
|
+
+ * @returns durable content-addressed reference.
|
|
3044
|
+
+ */
|
|
3045
|
+
+export async function saveVoiceFile(
|
|
3046
|
+
+ root: string,
|
|
3047
|
+
+ data: Uint8Array,
|
|
3048
|
+
+ mediaType: VoiceMediaType,
|
|
3049
|
+
+ durationMs?: number,
|
|
3050
|
+
+): Promise<VoiceAttachmentRef> {
|
|
3051
|
+
+ if (data.byteLength > MAX_VOICE_BYTES) {
|
|
3052
|
+
+ throw new Error(`Voice object exceeds the ${MAX_VOICE_BYTES}-byte limit.`)
|
|
3053
|
+
+ }
|
|
3054
|
+
+ const sha256 = createHash('sha256').update(data).digest('hex')
|
|
3055
|
+
+ const bucket = join(root, 'objects', sha256.slice(0, 2))
|
|
3056
|
+
+ const target = objectPath(root, sha256)
|
|
3057
|
+
+ await mkdir(bucket, { recursive: true, mode: 0o700 })
|
|
3058
|
+
+ let handle
|
|
3059
|
+
+ try {
|
|
3060
|
+
+ handle = await open(target, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600)
|
|
3061
|
+
+ await handle.writeFile(data)
|
|
3062
|
+
+ await handle.close()
|
|
3063
|
+
+ handle = undefined
|
|
3064
|
+
+ } catch (error) {
|
|
3065
|
+
+ if (handle !== undefined) await handle.close().catch(() => {})
|
|
3066
|
+
+ if (!(error instanceof Error && 'code' in error && error.code === 'EEXIST')) {
|
|
3067
|
+
+ throw new Error(`Unable to persist voice object: ${String(error)}`, { cause: error })
|
|
3068
|
+
+ }
|
|
3069
|
+
+ // A duplicate sha256 target already exists; its bytes are identical by
|
|
3070
|
+
+ // construction, so the concurrent writer's object is the same object.
|
|
3071
|
+
+ }
|
|
3072
|
+
+ return {
|
|
3073
|
+
+ voiceId: `sha256:${sha256}`,
|
|
3074
|
+
+ mediaType,
|
|
3075
|
+
+ bytes: data.byteLength,
|
|
3076
|
+
+ ...(durationMs === undefined ? {} : { durationMs }),
|
|
3077
|
+
+ }
|
|
3078
|
+
+}
|
|
3079
|
+
+
|
|
3080
|
+
+/**
|
|
3081
|
+
+ * Read one content-addressed voice object.
|
|
3082
|
+
+ * @param root - absolute `DSH_HOME/attachments/v1` root.
|
|
3083
|
+
+ * @param ref - reference recorded in the session log.
|
|
3084
|
+
+ * @returns stored bytes and the reference.
|
|
3085
|
+
+ */
|
|
3086
|
+
+export async function readVoiceFile(root: string, ref: VoiceAttachmentRef): Promise<{ ref: VoiceAttachmentRef; data: Uint8Array }> {
|
|
3087
|
+
+ const data = new Uint8Array(await readFile(voiceObjectPath(root, ref.voiceId)))
|
|
3088
|
+
+ return { ref, data }
|
|
3089
|
+
+}
|
|
3090
|
+
+
|
|
3091
|
+
+/** Default ffmpeg binary; overridable for non-Windows deployments. */
|
|
3092
|
+
+export const FFMPEG_BIN = process.env.DSH_VOICE_FFMPEG_BIN
|
|
3093
|
+
+ ?? 'C:\\Users\\oadan\\AppData\\Local\\Microsoft\\WinGet\\Links\\ffmpeg.exe'
|
|
3094
|
+
+
|
|
3095
|
+
+/** Default local ASR endpoint (sherpa-onnx SenseVoice service). */
|
|
3096
|
+
+export const ASR_SERVICE_URL = process.env.DSH_ASR_SERVICE_URL ?? 'http://127.0.0.1:18790/transcribe'
|
|
3097
|
+
+
|
|
3098
|
+
+/**
|
|
3099
|
+
+ * 读取插件 dsh-host-voice 写入的 ~/.dsh/voice-config.json 的 ASR 配置,
|
|
3100
|
+
+ * 让设置页的 ASR 模式(service / cmd / api)真正作用于后台自动识别主链路,
|
|
3101
|
+
+ * 而不是永远硬编码打 18790。读取失败或无配置时返回 null,调用方退回默认
|
|
3102
|
+
+ * 常驻服务,保持旧行为。
|
|
3103
|
+
+ */
|
|
3104
|
+
+interface AsrConfig {
|
|
3105
|
+
+ enabled: boolean
|
|
3106
|
+
+ mode: 'service' | 'cmd' | 'api'
|
|
3107
|
+
+ url: string
|
|
3108
|
+
+ cmd: string
|
|
3109
|
+
+ apiKey: string
|
|
3110
|
+
+ apiBaseUrl: string
|
|
3111
|
+
+}
|
|
3112
|
+
+
|
|
3113
|
+
+function loadAsrConfig(): AsrConfig | null {
|
|
3114
|
+
+ const path = join(process.env.DSH_HOME ?? join(homedir(), '.dsh'), 'voice-config.json')
|
|
3115
|
+
+ try {
|
|
3116
|
+
+ const parsed = JSON.parse(readFileSync(path, 'utf8'))
|
|
3117
|
+
+ const asr = parsed?.engines?.asr
|
|
3118
|
+
+ if (asr === undefined || asr === null || typeof asr !== 'object') return null
|
|
3119
|
+
+ return {
|
|
3120
|
+
+ enabled: asr.enabled !== false,
|
|
3121
|
+
+ mode: asr.mode === 'cmd' || asr.mode === 'api' ? asr.mode : 'service',
|
|
3122
|
+
+ url: typeof asr.url === 'string' && asr.url.trim() !== '' ? asr.url : 'http://127.0.0.1:18790',
|
|
3123
|
+
+ cmd: typeof asr.cmd === 'string' ? asr.cmd : '',
|
|
3124
|
+
+ apiKey: typeof asr.apiKey === 'string' ? asr.apiKey : '',
|
|
3125
|
+
+ apiBaseUrl: typeof asr.apiBaseUrl === 'string' && asr.apiBaseUrl.trim() !== ''
|
|
3126
|
+
+ ? asr.apiBaseUrl
|
|
3127
|
+
+ : 'https://api.xiaomimimo.com/v1',
|
|
3128
|
+
+ }
|
|
3129
|
+
+ } catch {
|
|
3130
|
+
+ return null
|
|
3131
|
+
+ }
|
|
3132
|
+
+}
|
|
3133
|
+
+
|
|
3134
|
+
+/**
|
|
3135
|
+
+ * 转写一段录音。路由优先级:
|
|
3136
|
+
+ * 1) 环境变量 DSH_ASR_SERVICE_URL(硬覆盖,保持旧行为,仍只认 service 风格 {audioPath});
|
|
3137
|
+
+ * 2) 读 voice-config.json 的 engines.asr,按 mode 路由:
|
|
3138
|
+
+ * - cmd:本地命令(sherpa-onnx-offline.exe,结果在 stderr,合并双流解析 "text");
|
|
3139
|
+
+ * - api:在线 ASR(小米 mimo-v2.5-asr / OpenAI Whisper 兼容);
|
|
3140
|
+
+ * - service(默认):POST {audioPath} 到 asr.url/transcribe;
|
|
3141
|
+
+ * 3) 无配置 / 未启用:退回默认常驻服务 18790。
|
|
3142
|
+
+ * 浏览器容器(webm/ogg)先转 16kHz 单声道 WAV,sherpa 只认标准 wav。
|
|
3143
|
+
+ * 识别失败绝不抛错,返回 '' 让调用方降级。
|
|
3144
|
+
+ * @param audioPath - 已落盘录音的绝对路径。
|
|
3145
|
+
+ * @returns 识别文本,或 ''(服务不可用 / 失败)。
|
|
3146
|
+
+ */
|
|
3147
|
+
+export async function transcribeVoice(audioPath: string): Promise<string> {
|
|
3148
|
+
+ let wavPath: string | undefined
|
|
3149
|
+
+ const controller = new AbortController()
|
|
3150
|
+
+ const timer = setTimeout(() => { controller.abort() }, 35_000)
|
|
3151
|
+
+ try {
|
|
3152
|
+
+ wavPath = await transcodeToWav(audioPath)
|
|
3153
|
+
+ // 1) 环境变量硬覆盖(保持旧行为)
|
|
3154
|
+
+ const envUrl = process.env.DSH_ASR_SERVICE_URL
|
|
3155
|
+
+ if (envUrl !== undefined && envUrl.trim() !== '') {
|
|
3156
|
+
+ const response = await fetch(envUrl, {
|
|
3157
|
+
+ method: 'POST',
|
|
3158
|
+
+ headers: { 'Content-Type': 'application/json' },
|
|
3159
|
+
+ body: JSON.stringify({ audioPath: wavPath }),
|
|
3160
|
+
+ signal: controller.signal,
|
|
3161
|
+
+ })
|
|
3162
|
+
+ if (!response.ok) return ''
|
|
3163
|
+
+ const payload = await response.json() as { text?: unknown }
|
|
3164
|
+
+ return typeof payload.text === 'string' ? payload.text : ''
|
|
3165
|
+
+ }
|
|
3166
|
+
+ // 2) 读插件配置,按 asr.mode 路由
|
|
3167
|
+
+ const asr = loadAsrConfig()
|
|
3168
|
+
+ if (asr === null || !asr.enabled) {
|
|
3169
|
+
+ const response = await fetch(ASR_SERVICE_URL, {
|
|
3170
|
+
+ method: 'POST',
|
|
3171
|
+
+ headers: { 'Content-Type': 'application/json' },
|
|
3172
|
+
+ body: JSON.stringify({ audioPath: wavPath }),
|
|
3173
|
+
+ signal: controller.signal,
|
|
3174
|
+
+ })
|
|
3175
|
+
+ if (!response.ok) return ''
|
|
3176
|
+
+ const payload = await response.json() as { text?: unknown }
|
|
3177
|
+
+ return typeof payload.text === 'string' ? payload.text : ''
|
|
3178
|
+
+ }
|
|
3179
|
+
+ if (asr.mode === 'cmd') {
|
|
3180
|
+
+ // [本地改造 2026-08-21] cmd 模式缺命令配置 = 明确失败,绝不降级到常驻服务
|
|
3181
|
+
+ if (asr.cmd.trim() === '') {
|
|
3182
|
+
+ console.error('[asr] cmd 模式但未配置本地命令,识别失败(不降级常驻服务)')
|
|
3183
|
+
+ return ''
|
|
3184
|
+
+ }
|
|
3185
|
+
+ const parts = asr.cmd.trim().split(/\s+/)
|
|
3186
|
+
+ const bin = parts[0]
|
|
3187
|
+
+ if (bin !== undefined) {
|
|
3188
|
+
+ const result = spawnSync(bin, [...parts.slice(1), wavPath], {
|
|
3189
|
+
+ windowsHide: true,
|
|
3190
|
+
+ encoding: 'utf-8',
|
|
3191
|
+
+ timeout: 60_000,
|
|
3192
|
+
+ stdio: ['ignore', 'pipe', 'pipe'],
|
|
3193
|
+
+ })
|
|
3194
|
+
+ const all = (result.stdout ?? '') + '\n' + (result.stderr ?? '')
|
|
3195
|
+
+ const m = all.match(/"text"\s*:\s*"([^"]*)"/)
|
|
3196
|
+
+ return (m?.[1] ?? '').trim()
|
|
3197
|
+
+ }
|
|
3198
|
+
+ return ''
|
|
3199
|
+
+ }
|
|
3200
|
+
+ if (asr.mode === 'api') {
|
|
3201
|
+
+ // [本地改造 2026-08-21] api 模式缺 key = 明确失败,绝不降级到常驻服务
|
|
3202
|
+
+ if (asr.apiKey.trim() === '') {
|
|
3203
|
+
+ console.error('[asr] api 模式但未配置 API Key,识别失败(不降级常驻服务)')
|
|
3204
|
+
+ return ''
|
|
3205
|
+
+ }
|
|
3206
|
+
+ const apiKey = asr.apiKey.trim()
|
|
3207
|
+
+ const baseUrl = asr.apiBaseUrl.replace(/\/+$/, '')
|
|
3208
|
+
+ const audioBase64 = (await readFile(audioPath)).toString('base64')
|
|
3209
|
+
+ if (baseUrl.includes('openai')) {
|
|
3210
|
+
+ const form = new FormData()
|
|
3211
|
+
+ const blob = new Blob([Buffer.from(audioBase64, 'base64')], { type: 'audio/wav' })
|
|
3212
|
+
+ form.append('file', blob, 'audio.wav')
|
|
3213
|
+
+ form.append('model', 'whisper-1')
|
|
3214
|
+
+ const response = await fetch(`${baseUrl}/audio/transcriptions`, {
|
|
3215
|
+
+ method: 'POST',
|
|
3216
|
+
+ headers: { Authorization: `Bearer ${apiKey}` },
|
|
3217
|
+
+ body: form,
|
|
3218
|
+
+ signal: controller.signal,
|
|
3219
|
+
+ })
|
|
3220
|
+
+ if (!response.ok) return ''
|
|
3221
|
+
+ const payload = await response.json() as { text?: unknown }
|
|
3222
|
+
+ return typeof payload.text === 'string' ? payload.text.trim() : ''
|
|
3223
|
+
+ }
|
|
3224
|
+
+ const response = await fetch(`${baseUrl}/chat/completions`, {
|
|
3225
|
+
+ method: 'POST',
|
|
3226
|
+
+ headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
|
|
3227
|
+
+ body: JSON.stringify({
|
|
3228
|
+
+ model: 'mimo-v2.5-asr',
|
|
3229
|
+
+ messages: [
|
|
3230
|
+
+ {
|
|
3231
|
+
+ role: 'user',
|
|
3232
|
+
+ content: [
|
|
3233
|
+
+ { type: 'input_audio', input_audio: { data: `data:audio/wav;base64,${audioBase64}` } },
|
|
3234
|
+
+ ],
|
|
3235
|
+
+ },
|
|
3236
|
+
+ ],
|
|
3237
|
+
+ extra_body: { asr_options: { language: 'auto' } },
|
|
3238
|
+
+ }),
|
|
3239
|
+
+ signal: controller.signal,
|
|
3240
|
+
+ })
|
|
3241
|
+
+ if (!response.ok) return ''
|
|
3242
|
+
+ const payload = await response.json() as { choices?: Array<{ message?: { content?: unknown } }> }
|
|
3243
|
+
+ const text = typeof payload.choices?.[0]?.message?.content === 'string'
|
|
3244
|
+
+ ? payload.choices[0].message.content.trim()
|
|
3245
|
+
+ : ''
|
|
3246
|
+
+ return text
|
|
3247
|
+
+ }
|
|
3248
|
+
+ // 3) service 模式(默认):POST {audioPath} 到 asr.url/transcribe
|
|
3249
|
+
+ const baseUrl = asr.url.trim().replace(/\/+$/, '')
|
|
3250
|
+
+ const response = await fetch(`${baseUrl}/transcribe`, {
|
|
3251
|
+
+ method: 'POST',
|
|
3252
|
+
+ headers: { 'Content-Type': 'application/json' },
|
|
3253
|
+
+ body: JSON.stringify({ audioPath: wavPath }),
|
|
3254
|
+
+ signal: controller.signal,
|
|
3255
|
+
+ })
|
|
3256
|
+
+ if (!response.ok) return ''
|
|
3257
|
+
+ const payload = await response.json() as { text?: unknown }
|
|
3258
|
+
+ return typeof payload.text === 'string' ? payload.text : ''
|
|
3259
|
+
+ } catch {
|
|
3260
|
+
+ return ''
|
|
3261
|
+
+ } finally {
|
|
3262
|
+
+ clearTimeout(timer)
|
|
3263
|
+
+ if (wavPath !== undefined) await unlink(wavPath).catch(() => {})
|
|
3264
|
+
+ }
|
|
3265
|
+
+}
|
|
3266
|
+
+
|
|
3267
|
+
+/** Transcode any container ffmpeg decodes to 16 kHz mono PCM WAV in the temp dir. */
|
|
3268
|
+
+function transcodeToWav(inputPath: string): Promise<string> {
|
|
3269
|
+
+ const wavPath = join(process.env.TEMP ?? '/tmp', `dsh-asr-${randomUUID()}.wav`)
|
|
3270
|
+
+ return new Promise((resolveWav, reject) => {
|
|
3271
|
+
+ const child = spawn(FFMPEG_BIN, [
|
|
3272
|
+
+ '-i', inputPath,
|
|
3273
|
+
+ '-ar', '16000', '-ac', '1',
|
|
3274
|
+
+ '-c:a', 'pcm_s16le',
|
|
3275
|
+
+ '-y', wavPath,
|
|
3276
|
+
+ ], { windowsHide: true, stdio: 'ignore' })
|
|
3277
|
+
+ const timer = setTimeout(() => {
|
|
3278
|
+
+ child.kill()
|
|
3279
|
+
+ reject(new Error('ffmpeg transcode timed out.'))
|
|
3280
|
+
+ }, 30_000)
|
|
3281
|
+
+ child.once('error', (error) => {
|
|
3282
|
+
+ clearTimeout(timer)
|
|
3283
|
+
+ reject(error)
|
|
3284
|
+
+ })
|
|
3285
|
+
+ child.once('close', (code) => {
|
|
3286
|
+
+ clearTimeout(timer)
|
|
3287
|
+
+ if (code === 0) resolveWav(wavPath)
|
|
3288
|
+
+ else reject(new Error(`ffmpeg exited with code ${code ?? 'null'}.`))
|
|
3289
|
+
+ })
|
|
3290
|
+
+ })
|
|
3291
|
+
+}
|
|
3292
|
+
+
|
|
3293
|
+
+export interface SynthesizedVoice {
|
|
3294
|
+
+ mediaType: string
|
|
3295
|
+
+ data: Uint8Array
|
|
3296
|
+
+ durationMs?: number
|
|
3297
|
+
+}
|
|
3298
|
+
+
|
|
3299
|
+
+/**
|
|
3300
|
+
+ * Synthesize reply voice through self-contained engines(TTS 独立化,不依赖 agents-to-im):
|
|
3301
|
+
+ * - auto/edge:微软 Edge TTS(免费无 key,WebSocket,开箱即用)
|
|
3302
|
+
+ * - xiaomi:小米 MiMo TTS(HTTP 直连,需 TTS_XIAOMI_KEY 环境变量;文本 (唱歌) 标签触发唱歌)
|
|
3303
|
+
+ * - local:本地 TTS 命令(DSH_LOCAL_TTS_CMD,文本作末参,stdout 输出音频)
|
|
3304
|
+
+ * 输出统一转 mp3(浏览器全兼容)。Never throws — returns null on failure.
|
|
3305
|
+
+ * @param text - reply text to speak.
|
|
3306
|
+
+ * @param provider - engine override (auto/edge/xiaomi/local; unknown falls back to edge).
|
|
3307
|
+
+ * @returns encoded audio, or null when synthesis or reading failed.
|
|
3308
|
+
+ */
|
|
3309
|
+
+export async function synthesizeReplyVoice(text: string, provider?: string): Promise<SynthesizedVoice | null> {
|
|
3310
|
+
+ const speak = stripMarkdown(text)
|
|
3311
|
+
+ const engine = provider ?? 'auto'
|
|
3312
|
+
+ try {
|
|
3313
|
+
+ if (engine === 'xiaomi') return await synthesizeXiaomiVoice(speak)
|
|
3314
|
+
+ if (engine === 'local') return await synthesizeLocalVoice(speak)
|
|
3315
|
+
+ // [本地改造 2026-08-17] 默认(auto)优先小米 MiMo(用户配置),key 缺失/合成失败时降级微软 edge
|
|
3316
|
+
+ if (engine === 'auto') {
|
|
3317
|
+
+ const xiaomi = await synthesizeXiaomiVoice(speak)
|
|
3318
|
+
+ if (xiaomi !== null) return xiaomi
|
|
3319
|
+
+ }
|
|
3320
|
+
+ return await synthesizeEdgeVoice(speak)
|
|
3321
|
+
+ } catch {
|
|
3322
|
+
+ return null
|
|
3323
|
+
+ }
|
|
3324
|
+
+}
|
|
3325
|
+
+
|
|
3326
|
+
+/** 微软 Edge TTS(免费):输出即 mp3,无需转码。 */
|
|
3327
|
+
+async function synthesizeEdgeVoice(text: string): Promise<SynthesizedVoice | null> {
|
|
3328
|
+
+ const voice = process.env.TTS_EDGE_VOICE ?? 'zh-CN-XiaoxiaoNeural'
|
|
3329
|
+
+ const mp3 = await edgeTts(text, voice)
|
|
3330
|
+
+ return toMp3(new Uint8Array(mp3), 'audio/mpeg')
|
|
3331
|
+
+}
|
|
3332
|
+
+
|
|
3333
|
+
+/** 小米 MiMo TTS:HTTP 直连 api.xiaomimimo.com(key/音色从环境变量读,不碰外部配置文件)。 */
|
|
3334
|
+
+async function synthesizeXiaomiVoice(text: string): Promise<SynthesizedVoice | null> {
|
|
3335
|
+
+ const apiKey = process.env.TTS_XIAOMI_KEY ?? ''
|
|
3336
|
+
+ // [tts-debug] 定位小米失败:打印 key 是否存在 + HTTP 状态
|
|
3337
|
+
+ console.error(`[tts-debug] xiaomi key=${apiKey === '' ? 'MISSING' : 'present'} env=${Object.keys(process.env).filter(k => /TTS|XIAOMI/i.test(k)).join(',') || 'none'}`)
|
|
3338
|
+
+ if (apiKey === '') return null
|
|
3339
|
+
+ const baseUrl = process.env.TTS_XIAOMI_BASE_URL ?? 'https://api.xiaomimimo.com/v1'
|
|
3340
|
+
+ const voice = process.env.TTS_XIAOMI_VOICE ?? 'mimo_default'
|
|
3341
|
+
+ const response = await fetch(`${baseUrl}/chat/completions`, {
|
|
3342
|
+
+ method: 'POST',
|
|
3343
|
+
+ headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
|
|
3344
|
+
+ body: JSON.stringify({
|
|
3345
|
+
+ model: 'mimo-v2.5-tts',
|
|
3346
|
+
+ messages: [
|
|
3347
|
+
+ { role: 'user', content: '把下面的文字转成语音' },
|
|
3348
|
+
+ { role: 'assistant', content: text },
|
|
3349
|
+
+ ],
|
|
3350
|
+
+ max_tokens: 8192,
|
|
3351
|
+
+ speed: 1.0,
|
|
3352
|
+
+ voice,
|
|
3353
|
+
+ audio: { format: 'wav' },
|
|
3354
|
+
+ }),
|
|
3355
|
+
+ })
|
|
3356
|
+
+ console.error(`[tts-debug] xiaomi http=${response.status}`)
|
|
3357
|
+
+ if (!response.ok) return null
|
|
3358
|
+
+ const payload = await response.json() as { choices?: Array<{ message?: { audio?: { data?: unknown } } }> }
|
|
3359
|
+
+ const data = payload.choices?.[0]?.message?.audio?.data
|
|
3360
|
+
+ if (typeof data !== 'string' || data.length < 100) return null
|
|
3361
|
+
+ return toMp3(new Uint8Array(Buffer.from(data, 'base64')), 'audio/wav')
|
|
3362
|
+
+}
|
|
3363
|
+
+
|
|
3364
|
+
+/** 本地 TTS 命令(可插拔):spawn DSH_LOCAL_TTS_CMD,文本作末参,stdout 收音频字节。 */
|
|
3365
|
+
+async function synthesizeLocalVoice(text: string): Promise<SynthesizedVoice | null> {
|
|
3366
|
+
+ const command = process.env.DSH_LOCAL_TTS_CMD ?? ''
|
|
3367
|
+
+ if (command === '') return null
|
|
3368
|
+
+ const parts = command.split(/\s+/)
|
|
3369
|
+
+ const bin = parts[0]
|
|
3370
|
+
+ if (bin === undefined) return null
|
|
3371
|
+
+ const rest = parts.slice(1)
|
|
3372
|
+
+ const audio = execFileSync(bin, [...rest, text], {
|
|
3373
|
+
+ windowsHide: true,
|
|
3374
|
+
+ encoding: 'buffer',
|
|
3375
|
+
+ timeout: 60_000,
|
|
3376
|
+
+ }) as Buffer
|
|
3377
|
+
+ return toMp3(new Uint8Array(audio), 'audio/mpeg')
|
|
3378
|
+
+}
|
|
3379
|
+
+
|
|
3380
|
+
+/** 统一转 mp3:已是 mp3 直接返回;wav/其他容器用 ffmpeg 转(失败保留原格式)。 */
|
|
3381
|
+
+async function toMp3(data: Uint8Array, declared: string): Promise<SynthesizedVoice | null> {
|
|
3382
|
+
+ const isMp3 = data.length > 2 && data[0] === 0xFF && ((data[1] ?? 0) & 0xE0) === 0xE0
|
|
3383
|
+
+ let finalData = data
|
|
3384
|
+
+ let mediaType = declared
|
|
3385
|
+
+ if (!isMp3) {
|
|
3386
|
+
+ const tmpIn = join(process.env.TEMP ?? '/tmp', `dsh-tts-in-${randomUUID()}.wav`)
|
|
3387
|
+
+ const mp3Path = join(process.env.TEMP ?? '/tmp', `dsh-tts-${randomUUID()}.mp3`)
|
|
3388
|
+
+ await writeFile(tmpIn, data)
|
|
3389
|
+
+ try {
|
|
3390
|
+
+ execFileSync(FFMPEG_BIN, ['-y', '-i', tmpIn, '-c:a', 'libmp3lame', '-b:a', '128k', mp3Path], {
|
|
3391
|
+
+ windowsHide: true, stdio: 'ignore', timeout: 30_000,
|
|
3392
|
+
+ })
|
|
3393
|
+
+ finalData = new Uint8Array(await readFile(mp3Path))
|
|
3394
|
+
+ mediaType = 'audio/mpeg'
|
|
3395
|
+
+ } catch {
|
|
3396
|
+
+ // 转码失败保留原容器(部分浏览器仍可播)。
|
|
3397
|
+
+ } finally {
|
|
3398
|
+
+ await unlink(tmpIn).catch(() => {})
|
|
3399
|
+
+ await unlink(mp3Path).catch(() => {})
|
|
3400
|
+
+ }
|
|
3401
|
+
+ }
|
|
3402
|
+
+ const durationMs = estimateAudioDurationMs(finalData)
|
|
3403
|
+
+ return {
|
|
3404
|
+
+ mediaType,
|
|
3405
|
+
+ data: finalData,
|
|
3406
|
+
+ ...(durationMs === undefined ? {} : { durationMs }),
|
|
3407
|
+
+ }
|
|
3408
|
+
+}
|
|
3409
|
+
+
|
|
3410
|
+
+/** True when the leading bytes are an Ogg container (OggS magic). */
|
|
3411
|
+
+function looksLikeOgg(data: Uint8Array): boolean {
|
|
3412
|
+
+ return data.length >= 4
|
|
3413
|
+
+ && data[0] === 0x4F && data[1] === 0x67 && data[2] === 0x67 && data[3] === 0x53 // 'OggS'
|
|
3414
|
+
+}
|
|
3415
|
+
+
|
|
3416
|
+
+/**
|
|
3417
|
+
+ * [本地改造 2026-08-16] Strip Markdown syntax for TTS reading: headings, bold/
|
|
3418
|
+
+ * italic markers, inline code, links, tables, lists, and dividers become plain
|
|
3419
|
+
+ * readable text. Multi-line output is joined with spaces so the engine speaks
|
|
3420
|
+
+ * it fluently.
|
|
3421
|
+
+ */
|
|
3422
|
+
+function stripMarkdown(text: string): string {
|
|
3423
|
+
+ return text
|
|
3424
|
+
+ .replace(/```[\s\S]*?```/g, ' ') // fenced code blocks
|
|
3425
|
+
+ .replace(/`([^`]+)`/g, '$1') // inline code
|
|
3426
|
+
+ .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') // links: keep label
|
|
3427
|
+
+ .replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1') // images: keep alt
|
|
3428
|
+
+ .replace(/^#{1,6}\s*/gm, '') // ATX headings
|
|
3429
|
+
+ .replace(/^>+\s*/gm, '') // blockquotes
|
|
3430
|
+
+ .replace(/^\s*[-*+]\s+/gm, '') // list bullets
|
|
3431
|
+
+ .replace(/^\s*\d+[.)]\s+/gm, '') // numbered lists
|
|
3432
|
+
+ .replace(/^\s*\|?[\s:|-]+\|?\s*$/gm, '') // table separator rows (| --- | --- |)
|
|
3433
|
+
+ .replace(/^[-*_]{3,}\s*$/gm, '') // horizontal rules
|
|
3434
|
+
+ .replace(/\|/g, ' ') // table pipes
|
|
3435
|
+
+ .replace(/\*\*([^*]+)\*\*/g, '$1') // bold
|
|
3436
|
+
+ .replace(/\*([^*]+)\*/g, '$1') // italic
|
|
3437
|
+
+ .replace(/__([^_]+)__/g, '$1') // bold underscore
|
|
3438
|
+
+ .replace(/_([^_]+)_/g, '$1') // italic underscore
|
|
3439
|
+
+ .replace(/~~([^~]+)~~/g, '$1') // strikethrough
|
|
3440
|
+
+ .replace(/^\s*[-*_]\s*$/gm, '') // lone dash rows
|
|
3441
|
+
+ .replace(/\s*\n\s*/g, ' ') // newlines → space (fluent speech)
|
|
3442
|
+
+ .replace(/\s{2,}/g, ' ')
|
|
3443
|
+
+ .trim()
|
|
3444
|
+
+}
|
|
3445
|
+
+
|
|
3446
|
+
+/**
|
|
3447
|
+
+ * Approximate audio duration for the reply pill. ffprobe is authoritative when
|
|
3448
|
+
+ * available (any container); otherwise falls back to a container-aware byte
|
|
3449
|
+
+ * estimate. Exact decode is overkill here — a second count is enough.
|
|
3450
|
+
+ * @param data - encoded audio bytes.
|
|
3451
|
+
+ * @param path - absolute produced-file path (ffprobe input).
|
|
3452
|
+
+ * @returns estimated duration in ms, or undefined when unreadable.
|
|
3453
|
+
+ */
|
|
3454
|
+
+function estimateAudioDurationMs(data: Uint8Array, path?: string): number | undefined {
|
|
3455
|
+
+ if (path !== undefined) {
|
|
3456
|
+
+ try {
|
|
3457
|
+
+ const ffprobe = process.env.DSH_VOICE_FFPROBE_BIN
|
|
3458
|
+
+ ?? 'C:\\Users\\oadan\\AppData\\Local\\Microsoft\\WinGet\\Links\\ffprobe.exe'
|
|
3459
|
+
+ const out = execFileSync(ffprobe, [
|
|
3460
|
+
+ '-v', 'error', '-show_entries', 'format=duration',
|
|
3461
|
+
+ '-of', 'default=noprint_wrappers=1:nokey=1', path,
|
|
3462
|
+
+ ], { encoding: 'utf8', windowsHide: true, timeout: 10_000 }).trim()
|
|
3463
|
+
+ const seconds = Number.parseFloat(out)
|
|
3464
|
+
+ if (Number.isFinite(seconds) && seconds > 0) return Math.round(seconds * 1000)
|
|
3465
|
+
+ } catch {
|
|
3466
|
+
+ // ffprobe unavailable or failed; fall through to the byte estimate.
|
|
3467
|
+
+ }
|
|
3468
|
+
+ }
|
|
3469
|
+
+ if (looksLikeOgg(data)) {
|
|
3470
|
+
+ // Ogg/Opus at the standard 48 kHz; a 12-byte frame carries 20ms (i.e. 600
|
|
3471
|
+
+ // bytes/s) — the bitrate heuristic below would grossly overestimate, so
|
|
3472
|
+
+ // give a rough constant bitrate guess instead.
|
|
3473
|
+
+ const kbps = 48
|
|
3474
|
+
+ return Math.round(data.length / (kbps * 1000 / 8) * 1000)
|
|
3475
|
+
+ }
|
|
3476
|
+
+ // MP3: skip a leading ID3v2 tag (its binary metadata can fake a frame sync),
|
|
3477
|
+
+ // then use the first real frame's bitrate.
|
|
3478
|
+
+ let offset = 0
|
|
3479
|
+
+ if (data.length >= 10 && (data[0] ?? 0) === 0x49 && (data[1] ?? 0) === 0x44 && (data[2] ?? 0) === 0x33 // 'ID3'
|
|
3480
|
+
+ && ((data[3] ?? 0) & 0xFF) < 0xFF && ((data[4] ?? 0) & 0xFF) < 0xFF) {
|
|
3481
|
+
+ const size = (((data[6] ?? 0) & 0x7F) << 21) | (((data[7] ?? 0) & 0x7F) << 14)
|
|
3482
|
+
+ | (((data[8] ?? 0) & 0x7F) << 7) | ((data[9] ?? 0) & 0x7F)
|
|
3483
|
+
+ offset = 10 + size
|
|
3484
|
+
+ }
|
|
3485
|
+
+ while (offset + 4 <= data.length) {
|
|
3486
|
+
+ const sync = ((data[offset] ?? 0) << 8) | (data[offset + 1] ?? 0)
|
|
3487
|
+
+ if ((sync & 0xFFE0) === 0xFFE0) {
|
|
3488
|
+
+ const bitrateIndex = ((data[offset + 2] ?? 0) >>> 4) & 0x0F
|
|
3489
|
+
+ const sampleRateIndex = ((data[offset + 2] ?? 0) >>> 2) & 0x03
|
|
3490
|
+
+ if (bitrateIndex === 0 || bitrateIndex === 15 || sampleRateIndex === 3) return undefined
|
|
3491
|
+
+ const bitrates = [32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320]
|
|
3492
|
+
+ const kbps = bitrates[bitrateIndex - 1] ?? 128
|
|
3493
|
+
+ return Math.round((data.length - offset) / (kbps * 1000 / 8) * 1000)
|
|
3494
|
+
+ }
|
|
3495
|
+
+ offset += 1
|
|
3496
|
+
+ }
|
|
3497
|
+
+ return undefined
|
|
3498
|
+
+}
|
|
3499
|
+
diff --git a/packages/host/apiproxy/tests/api-proxy-blank.spec.ts b/packages/host/apiproxy/tests/api-proxy-blank.spec.ts
|
|
3500
|
+
index 1ee3a5100f..579c259f02 100644
|
|
3501
|
+
--- a/packages/host/apiproxy/tests/api-proxy-blank.spec.ts
|
|
3502
|
+
+++ b/packages/host/apiproxy/tests/api-proxy-blank.spec.ts
|
|
3503
|
+
@@ -48,7 +48,7 @@ function appendStandalone(session: Session): void {
|
|
3504
|
+
commandId: CommandId('blank-cmd-1'), name: 'plan', args: '', source: { kind: 'user' },
|
|
3505
|
+
})
|
|
3506
|
+
session.append('plan/mode', { active: true })
|
|
3507
|
+
- session.append('command/done', { commandId: CommandId('blank-cmd-1'), kind: 'success', text: 'Plan mode on.' })
|
|
3508
|
+
+ session.append('command/done', { commandId: CommandId('blank-cmd-1'), kind: 'success', text: '计划模式已开启。' })
|
|
3509
|
+
session.append('session/title', {
|
|
3510
|
+
title: 'standalone title', messageSeqs: [], source: { kind: 'fallback' },
|
|
3511
|
+
})
|
|
3512
|
+
diff --git a/packages/host/apiproxy/tests/client-handler.spec.ts b/packages/host/apiproxy/tests/client-handler.spec.ts
|
|
3513
|
+
index de9d4ddab0..9bc48d8ae1 100644
|
|
3514
|
+
--- a/packages/host/apiproxy/tests/client-handler.spec.ts
|
|
3515
|
+
+++ b/packages/host/apiproxy/tests/client-handler.spec.ts
|
|
3516
|
+
@@ -28,6 +28,7 @@ function scriptedApi(overrides: {
|
|
3517
|
+
settings?: Partial<ApiProxy['settings']>
|
|
3518
|
+
credentials?: Partial<ApiProxy['credentials']>
|
|
3519
|
+
llm?: Partial<ApiProxy['llm']>
|
|
3520
|
+
+ balance?: Partial<ApiProxy['balance']>
|
|
3521
|
+
respond?: ApiProxy['respond']
|
|
3522
|
+
} = {}): ApiProxy {
|
|
3523
|
+
async function *empty<F>(): AsyncGenerator<RpcRequest<F>> { /* no frames */ }
|
|
3524
|
+
@@ -59,6 +60,13 @@ function scriptedApi(overrides: {
|
|
3525
|
+
attachment: { attachmentId: 'a' as never, mediaType: 'image/png', bytes: 1, width: 1, height: 1 },
|
|
3526
|
+
data: 'AA==',
|
|
3527
|
+
}),
|
|
3528
|
+
+ voice: r => ok(r, {
|
|
3529
|
+
+ attachment: { voiceId: 'v', mediaType: 'audio/webm', bytes: 1 },
|
|
3530
|
+
+ data: 'AA==',
|
|
3531
|
+
+ }),
|
|
3532
|
+
+ voiceAsr: r => ok(r, { text: 'stub transcription' }),
|
|
3533
|
+
+ voiceTts: r => ok(r, { mediaType: 'audio/mpeg', data: 'SUQz', durationMs: 800 }),
|
|
3534
|
+
+ sendVoiceMessage: r => ok(r, { accepted: true as const }),
|
|
3535
|
+
updateQueue: r => ok(r, { accepted: true as const }),
|
|
3536
|
+
cancel: r => ok(r, { accepted: true as const }),
|
|
3537
|
+
...overrides.sessions,
|
|
3538
|
+
@@ -128,6 +136,9 @@ function scriptedApi(overrides: {
|
|
3539
|
+
discoverModels: err,
|
|
3540
|
+
...overrides.llm,
|
|
3541
|
+
},
|
|
3542
|
+
+ balance: {
|
|
3543
|
+
+ get: r => ok(r, { balance: null }),
|
|
3544
|
+
+ },
|
|
3545
|
+
events: { mux: () => empty<MuxFrame>(), host: () => empty<HostFrame>(), ...overrides.events },
|
|
3546
|
+
respond: overrides.respond ?? (() => Promise.resolve({ accepted: false as const, reason: 'not-pending' as const })),
|
|
3547
|
+
downloads: { sessionLog: async () => new Response('stub', { status: 404 }) },
|
|
3548
|
+
diff --git a/packages/host/apiproxy/tests/fetch-carrier.spec.ts b/packages/host/apiproxy/tests/fetch-carrier.spec.ts
|
|
3549
|
+
index 77432c55af..6e2bfb5da4 100644
|
|
3550
|
+
--- a/packages/host/apiproxy/tests/fetch-carrier.spec.ts
|
|
3551
|
+
+++ b/packages/host/apiproxy/tests/fetch-carrier.spec.ts
|
|
3552
|
+
@@ -102,6 +102,21 @@ function fakeApi(overrides: Partial<{ muxFrames: MuxFrame[]; hostFrames: HostFra
|
|
3553
|
+
result: { ok: true, value: { attachment: { attachmentId: 'a' as never, mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1 }, data: 'AA==' } },
|
|
3554
|
+
}
|
|
3555
|
+
},
|
|
3556
|
+
+ async voice(request) {
|
|
3557
|
+
+ return {
|
|
3558
|
+
+ rpcId: request.rpcId,
|
|
3559
|
+
+ result: { ok: true, value: { attachment: { voiceId: 'v', mediaType: 'audio/webm' as const, bytes: 1 }, data: 'AA==' } },
|
|
3560
|
+
+ }
|
|
3561
|
+
+ },
|
|
3562
|
+
+ async voiceAsr(request) {
|
|
3563
|
+
+ return { rpcId: request.rpcId, result: { ok: true, value: { text: 'stub transcription' } } }
|
|
3564
|
+
+ },
|
|
3565
|
+
+ async voiceTts(request) {
|
|
3566
|
+
+ return { rpcId: request.rpcId, result: { ok: true, value: { mediaType: 'audio/mpeg', data: 'SUQz', durationMs: 800 } } }
|
|
3567
|
+
+ },
|
|
3568
|
+
+ async sendVoiceMessage(request) {
|
|
3569
|
+
+ return { rpcId: request.rpcId, result: { ok: true, value: { accepted: true as const } } }
|
|
3570
|
+
+ },
|
|
3571
|
+
async updateQueue(request) {
|
|
3572
|
+
return { rpcId: request.rpcId, result: { ok: true, value: { accepted: true as const } } }
|
|
3573
|
+
},
|
|
3574
|
+
@@ -282,6 +297,11 @@ function fakeApi(overrides: Partial<{ muxFrames: MuxFrame[]; hostFrames: HostFra
|
|
3575
|
+
return { rpcId: request.rpcId, result: { ok: true, value: { models: [] } } }
|
|
3576
|
+
},
|
|
3577
|
+
},
|
|
3578
|
+
+ balance: {
|
|
3579
|
+
+ async get(request) {
|
|
3580
|
+
+ return { rpcId: request.rpcId, result: { ok: true, value: { balance: null } } }
|
|
3581
|
+
+ },
|
|
3582
|
+
+ },
|
|
3583
|
+
events: {
|
|
3584
|
+
mux: (_request, signal) => stream(muxFrames, signal),
|
|
3585
|
+
host: (_request, signal) => stream(hostFrames, signal),
|
|
3586
|
+
diff --git a/packages/host/directory-picker-auto/src/resolve.ts b/packages/host/directory-picker-auto/src/resolve.ts
|
|
3587
|
+
index 395e2da55f..d1548d4961 100644
|
|
3588
|
+
--- a/packages/host/directory-picker-auto/src/resolve.ts
|
|
3589
|
+
+++ b/packages/host/directory-picker-auto/src/resolve.ts
|
|
3590
|
+
@@ -13,7 +13,7 @@ export type DirectoryPickerBackendKind = 'native' | 'browse'
|
|
3591
|
+
|
|
3592
|
+
/** Environment keys the resolution reads (a `process.env` subset). */
|
|
3593
|
+
export type DirectoryPickerEnv = Readonly<
|
|
3594
|
+
- Partial<Record<'SSH_CONNECTION' | 'SSH_TTY' | 'DISPLAY' | 'WAYLAND_DISPLAY', string>>
|
|
3595
|
+
+ Partial<Record<'SSH_CONNECTION' | 'SSH_TTY' | 'DISPLAY' | 'WAYLAND_DISPLAY' | 'DSH_FORCE_BROWSE_PICKER', string>>
|
|
3596
|
+
>
|
|
3597
|
+
|
|
3598
|
+
/** Host facts the backend choice is a pure function of, sampled once at boot. */
|
|
3599
|
+
@@ -45,6 +45,10 @@ const present = (value: string | undefined): boolean => value !== undefined && v
|
|
3600
|
+
* @returns the backend kind to mount.
|
|
3601
|
+
*/
|
|
3602
|
+
export function resolveDirectoryPickerBackend(facts: DirectoryPickerHostFacts): DirectoryPickerBackendKind {
|
|
3603
|
+
+ // [本地改造 2026-08-13] DSH_FORCE_BROWSE_PICKER=1 → 强制 browse(浏览器目录树):
|
|
3604
|
+
+ // Windows nssm 服务跑在 session 0,原生 IFileOpenDialog COM 对话框无交互桌面
|
|
3605
|
+
+ // 弹不出(pick 卡死)。browse 是纯 HTTP 目录列表,session 0 完全可用。
|
|
3606
|
+
+ if (facts.env.DSH_FORCE_BROWSE_PICKER === '1') return 'browse'
|
|
3607
|
+
if (facts.bindHost !== '127.0.0.1') return 'browse'
|
|
3608
|
+
if (present(facts.env.SSH_CONNECTION) || present(facts.env.SSH_TTY)) return 'browse'
|
|
3609
|
+
if (facts.platform === 'darwin' || facts.platform === 'win32') return 'native'
|
|
3610
|
+
diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts
|
|
3611
|
+
index 498b3fb2f7..a5e8458366 100644
|
|
3612
|
+
--- a/packages/llm/llm-deepseek/src/serialize.ts
|
|
3613
|
+
+++ b/packages/llm/llm-deepseek/src/serialize.ts
|
|
3614
|
+
@@ -17,6 +17,7 @@ import type {
|
|
3615
|
+
WireTool,
|
|
3616
|
+
WireUserContentPart,
|
|
3617
|
+
} from './types.ts'
|
|
3618
|
+
+import { join } from 'node:path'
|
|
3619
|
+
|
|
3620
|
+
/** Adapter-level request defaults (from plugin config). */
|
|
3621
|
+
export interface RequestDefaults {
|
|
3622
|
+
@@ -79,11 +80,58 @@ function flattenText(blocks: ContentBlock[]): string {
|
|
3623
|
+
.join('')
|
|
3624
|
+
}
|
|
3625
|
+
|
|
3626
|
+
-/** Reject core image content before any text-flattening path can silently erase it. */
|
|
3627
|
+
-function assertTextOnly(blocks: readonly ContentBlock[]): void {
|
|
3628
|
+
- if (contentHasImage(blocks)) {
|
|
3629
|
+
- throw new LlmError('The DeepSeek chat-completions adapter does not support image content.', 'UNSUPPORTED_CONTENT')
|
|
3630
|
+
+/** [本地改造 2026-08-16] 把 image 块转成含本地附件路径的文本(参考 dsh-vscode-layout 补丁):
|
|
3631
|
+
+ * 文本模型收到路径后,必须通过视觉 MCP(mcp__visionqa__look / mcp__zai-vision__analyze_image)
|
|
3632
|
+
+ * 识图;该文件无扩展名,read_image 等按扩展名校验的工具会拒绝,禁止使用。 */
|
|
3633
|
+
+function imageAsText(block: ContentBlock): ContentBlock {
|
|
3634
|
+
+ const ref = (block as { attachment?: { attachmentId?: unknown; name?: string; mediaType?: string } }).attachment
|
|
3635
|
+
+ const rawId = typeof ref?.attachmentId === 'string' ? ref.attachmentId : ''
|
|
3636
|
+
+ const hex = rawId.startsWith('sha256:') ? rawId.slice('sha256:'.length) : rawId
|
|
3637
|
+
+ const name = typeof ref?.name === 'string' && ref.name.length > 0 ? ref.name : 'image'
|
|
3638
|
+
+ const mediaType = ref?.mediaType ?? 'image/jpeg'
|
|
3639
|
+
+ const home = process.env.DSH_HOME ?? ''
|
|
3640
|
+
+ const path = hex.length > 0 && home !== ''
|
|
3641
|
+
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex)
|
|
3642
|
+
+ : '(unknown)'
|
|
3643
|
+
+ return { type: 'text', text: `[用户发送了一张图片,名称 "${name}",类型 ${mediaType}。请用视觉 MCP 工具识图(mcp__visionqa__look 或 mcp__zai-vision__analyze_image,传入 image_path),不要用 read_image(该文件无扩展名,read_image 会拒绝):${path}]` }
|
|
3644
|
+
+}
|
|
3645
|
+
+
|
|
3646
|
+
+function imagesAsText(blocks: readonly ContentBlock[]): ContentBlock[] {
|
|
3647
|
+
+ return blocks.map((block) => {
|
|
3648
|
+
+ if (block.type === 'image') return imageAsText(block)
|
|
3649
|
+
+ if (block.type === 'tool-result') return { ...block, content: imagesAsText(block.content) }
|
|
3650
|
+
+ return block
|
|
3651
|
+
+ })
|
|
3652
|
+
+}
|
|
3653
|
+
+
|
|
3654
|
+
+/** [本地改造 2026-08-16] 把 voice 块转成文本:attachment.transcript 存在时直接给出
|
|
3655
|
+
+ * 识别文本(旧宿主链路兼容);否则输出本地语音文件路径——agent 收到路径后主动调
|
|
3656
|
+
+ * 本地 ASR 服务识别(与图片走视觉 MCP 同一模式),识别结果显示在助手侧。 */
|
|
3657
|
+
+function voiceAsText(block: ContentBlock): ContentBlock {
|
|
3658
|
+
+ const ref = (block as { attachment?: { voiceId?: unknown; durationMs?: unknown; transcript?: unknown } }).attachment
|
|
3659
|
+
+ const rawId = typeof ref?.voiceId === 'string' ? ref.voiceId : ''
|
|
3660
|
+
+ const hex = rawId.startsWith('sha256:') ? rawId.slice('sha256:'.length) : rawId
|
|
3661
|
+
+ const transcript = typeof ref?.transcript === 'string' && ref.transcript.length > 0
|
|
3662
|
+
+ ? ref.transcript
|
|
3663
|
+
+ : null
|
|
3664
|
+
+ const durationMs = typeof ref?.durationMs === 'number' ? ref.durationMs : null
|
|
3665
|
+
+ const duration = durationMs === null ? '' : `(时长 ${Math.round(durationMs / 1000)} 秒)`
|
|
3666
|
+
+ if (transcript !== null) {
|
|
3667
|
+
+ return { type: 'text', text: `[用户发送了一条语音${duration},识别内容:${transcript}]` }
|
|
3668
|
+
}
|
|
3669
|
+
+ const home = process.env.DSH_HOME ?? ''
|
|
3670
|
+
+ const path = hex.length > 0 && home !== ''
|
|
3671
|
+
+ ? join(home, 'attachments', 'v1', 'objects', hex.slice(0, 2), hex)
|
|
3672
|
+
+ : '(unknown)'
|
|
3673
|
+
+ return { type: 'text', text: `[用户发送了一条语音${duration},本地语音文件路径: ${path}]` }
|
|
3674
|
+
+}
|
|
3675
|
+
+
|
|
3676
|
+
+function voicesAsText(blocks: readonly ContentBlock[]): ContentBlock[] {
|
|
3677
|
+
+ return blocks.map((block) => {
|
|
3678
|
+
+ if (block.type === 'voice') return voiceAsText(block)
|
|
3679
|
+
+ if (block.type === 'tool-result') return { ...block, content: voicesAsText(block.content) }
|
|
3680
|
+
+ return block
|
|
3681
|
+
+ })
|
|
3682
|
+
}
|
|
3683
|
+
|
|
3684
|
+
/** Reject roles whose DeepSeek history format cannot carry image input. */
|
|
3685
|
+
@@ -203,19 +251,22 @@ function serializeAssistant(message: Message): WireMessage {
|
|
3686
|
+
export function serializeMessages(messages: Message[]): WireMessage[] {
|
|
3687
|
+
const wire: WireMessage[] = []
|
|
3688
|
+
for (const message of messages) {
|
|
3689
|
+
- assertTextOnly(message.content)
|
|
3690
|
+
+ // [本地改造 2026-08-16] 图片块先转本地路径文本(imagesAsText),agent 用视觉 MCP 识图;
|
|
3691
|
+
+ // 语音块同样转本地路径文本(voicesAsText),agent 用本地 ASR 服务识别——识别结果
|
|
3692
|
+
+ // 以工具输出显示在助手侧(与识图同一模式),host 不再二次注入识别文本。
|
|
3693
|
+
+ const content = voicesAsText(imagesAsText(message.content))
|
|
3694
|
+
if (message.role === 'system') {
|
|
3695
|
+
- wire.push({ role: 'system', content: flattenText(message.content) })
|
|
3696
|
+
+ wire.push({ role: 'system', content: flattenText(content) })
|
|
3697
|
+
continue
|
|
3698
|
+
}
|
|
3699
|
+
if (message.role === 'assistant') {
|
|
3700
|
+
- wire.push(serializeAssistant(message))
|
|
3701
|
+
+ wire.push(serializeAssistant({ ...message, content }))
|
|
3702
|
+
continue
|
|
3703
|
+
}
|
|
3704
|
+
// user role: tool results ride in user messages in the harness
|
|
3705
|
+
// vocabulary, but DeepSeek wants them as role:'tool' messages.
|
|
3706
|
+
- const toolResults = message.content.filter(block => block.type === 'tool-result')
|
|
3707
|
+
- const text = flattenText(message.content)
|
|
3708
|
+
+ const toolResults = content.filter(block => block.type === 'tool-result')
|
|
3709
|
+
+ const text = flattenText(content)
|
|
3710
|
+
if (text.length > 0 || toolResults.length === 0) {
|
|
3711
|
+
wire.push({ role: 'user', content: text })
|
|
3712
|
+
}
|
|
3713
|
+
diff --git a/packages/llm/llm/src/types.ts b/packages/llm/llm/src/types.ts
|
|
3714
|
+
index 8c5be187dd..750496f164 100644
|
|
3715
|
+
--- a/packages/llm/llm/src/types.ts
|
|
3716
|
+
+++ b/packages/llm/llm/src/types.ts
|
|
3717
|
+
@@ -74,6 +74,29 @@ export interface ImageBlock {
|
|
3718
|
+
attachment: ImageAttachmentRef
|
|
3719
|
+
}
|
|
3720
|
+
|
|
3721
|
+
+/**
|
|
3722
|
+
+ * A durable voice recording reference, valid in user content today. The host
|
|
3723
|
+
+ * persists the browser-uploaded bytes content-addressed beside image objects
|
|
3724
|
+
+ * and transcribes them through the local ASR service; the transcript rides the
|
|
3725
|
+
+ * reference so serialization can degrade cleanly when recognition fails.
|
|
3726
|
+
+ */
|
|
3727
|
+
+export interface VoiceBlock {
|
|
3728
|
+
+ type: 'voice'
|
|
3729
|
+
+ /** Immutable bytes and recognition metadata for one recording. */
|
|
3730
|
+
+ attachment: {
|
|
3731
|
+
+ /** Opaque storage identifier; never a filesystem path or bearer URL. */
|
|
3732
|
+
+ voiceId: string
|
|
3733
|
+
+ /** Recording container format from the browser wire (audio/mpeg added for TTS replies). */
|
|
3734
|
+
+ mediaType: 'audio/webm' | 'audio/ogg' | 'audio/mp4' | 'audio/wav' | 'audio/mpeg'
|
|
3735
|
+
+ /** Exact encoded byte length. */
|
|
3736
|
+
+ bytes: number
|
|
3737
|
+
+ /** Recorder-reported length in milliseconds. */
|
|
3738
|
+
+ durationMs?: number
|
|
3739
|
+
+ /** Local ASR transcript; absent when recognition failed or is unavailable. */
|
|
3740
|
+
+ transcript?: string
|
|
3741
|
+
+ }
|
|
3742
|
+
+}
|
|
3743
|
+
+
|
|
3744
|
+
/** A tool invocation requested by the model. */
|
|
3745
|
+
export interface ToolCallBlock {
|
|
3746
|
+
type: 'tool-call'
|
|
3747
|
+
@@ -100,6 +123,7 @@ export interface ContentBlockMap {
|
|
3748
|
+
'text': TextBlock
|
|
3749
|
+
'reasoning': ReasoningBlock
|
|
3750
|
+
'image': ImageBlock
|
|
3751
|
+
+ 'voice': VoiceBlock
|
|
3752
|
+
'tool-call': ToolCallBlock
|
|
3753
|
+
'tool-result': ToolResultBlock
|
|
3754
|
+
}
|
|
3755
|
+
diff --git a/packages/subprocess/subprocess-local/src/spawn.ts b/packages/subprocess/subprocess-local/src/spawn.ts
|
|
3756
|
+
index 433ba01791..328f987893 100644
|
|
3757
|
+
--- a/packages/subprocess/subprocess-local/src/spawn.ts
|
|
3758
|
+
+++ b/packages/subprocess/subprocess-local/src/spawn.ts
|
|
3759
|
+
@@ -275,10 +275,18 @@ export function killGroup(pid: number, sig: NodeJS.Signals): void {
|
|
3760
|
+
*/
|
|
3761
|
+
export function taskkillProcessTree(pid: number): void {
|
|
3762
|
+
if (pid <= 0) return
|
|
3763
|
+
- // Outcome deliberately unchecked: an already-absent tree (status 128), exit
|
|
3764
|
+
- // races, and a missing taskkill binary (spawnSync reports, never throws) are
|
|
3765
|
+
- // as tolerable here as ESRCH is for a POSIX group signal.
|
|
3766
|
+
- spawnSync('taskkill', ['/PID', String(pid), '/T', '/F'], { stdio: 'ignore' })
|
|
3767
|
+
+ // [本地改造 2026-08-16] taskkill 优先用 System32 全路径:nssm 服务的 PATH 快照
|
|
3768
|
+
+ // 曾缺 System32,导致 spawnSync('taskkill') ENOENT 静默失败、挂起进程永远杀不掉
|
|
3769
|
+
+ // (超时/停止全部失效)。全路径 + PATH 兜底双保险。
|
|
3770
|
+
+ const root = process.env.SystemRoot ?? 'C:\\Windows'
|
|
3771
|
+
+ const candidates = [join(root, 'System32', 'taskkill.exe'), 'taskkill']
|
|
3772
|
+
+ for (const command of candidates) {
|
|
3773
|
+
+ // Outcome deliberately unchecked: an already-absent tree (status 128), exit
|
|
3774
|
+
+ // races, and a missing taskkill binary (spawnSync reports, never throws) are
|
|
3775
|
+
+ // as tolerable here as ESRCH is for a POSIX group signal.
|
|
3776
|
+
+ const result = spawnSync(command, ['/PID', String(pid), '/T', '/F'], { stdio: 'ignore' })
|
|
3777
|
+
+ if (result.error === undefined || (result.error as NodeJS.ErrnoException).code !== 'ENOENT') break
|
|
3778
|
+
+ }
|
|
3779
|
+
}
|
|
3780
|
+
|
|
3781
|
+
/**
|