dsh-plugin-mobile-gateway 0.10.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PROTOCOL.md +125 -7
- package/README.md +23 -1
- package/docs/dsh-rc2-mobile-integration.md +9 -1
- package/docs/file-cards-mobile-integration.md +111 -0
- package/docs/voice-input-hold-thresholds.md +35 -0
- package/docs/voice-input-implementation.md +275 -0
- package/docs/voice-input-ipa-analysis.md +150 -0
- package/docs/voice-input-ipa-evidence/deepseek-asr-constructor.asm +338 -0
- package/docs/voice-input-ipa-evidence/deepseek-extraction.json +5154 -0
- package/docs/voice-input-ipa-evidence/deepseek-focused-context.txt +210 -0
- package/docs/voice-input-ipa-evidence/deepseek-localization.json +29 -0
- package/docs/voice-input-ipa-evidence/deepseek-voice-strings.txt +206 -0
- package/docs/voice-input-ipa-evidence/doubao/address-map.txt +11 -0
- package/docs/voice-input-ipa-evidence/doubao/code-string-refs.txt +22 -0
- package/docs/voice-input-ipa-evidence/doubao/end-longpress-disasm.txt +149 -0
- package/docs/voice-input-ipa-evidence/doubao/focused-context.txt +271 -0
- package/docs/voice-input-ipa-evidence/doubao/locate_string_refs.py +42 -0
- package/docs/voice-input-ipa-evidence/doubao/network-candidates.txt +39 -0
- package/docs/voice-input-ipa-evidence/doubao/objc-msgsend-stub.txt +1 -0
- package/docs/voice-input-ipa-evidence/doubao/ranges.txt +858 -0
- package/docs/voice-input-ipa-evidence/doubao/recognizer-recorder.txt +134 -0
- package/docs/voice-input-ipa-evidence/doubao/result-disasm.txt +189 -0
- package/docs/voice-input-ipa-evidence/doubao/settext-selector.txt +2 -0
- package/docs/voice-input-ipa-evidence/doubao/start-stop-disasm.txt +358 -0
- package/docs/voice-input-ipa-evidence/doubao/strings.txt +3536 -0
- package/docs/voice-input-ipa-evidence/doubao-extraction.json +7418 -0
- package/docs/voice-input-ipa-evidence/doubao-findings.md +52 -0
- package/docs/voice-input-ipa-evidence/hold-threshold-rea-evidence.json +1 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-hold-callback-refs.txt +1 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-input-area-metadata.txt +31 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-input-gesture-init.asm +267 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-static-summary.json +42 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-voice-gesture-constructor.asm +371 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/doubao-findings.md +31 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/doubao-text-mask-objc-metadata.txt +119 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/doubao-text-mask-threshold-evidence.asm +156 -0
- package/docs/voice-input-ipa-evidence/hold-thresholds/doubao-text-mask-voice-view-getter.asm +14 -0
- package/docs/voice-input-ipa-evidence/rea-evidence.json +1 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/README.md +33 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-objc.txt +222957 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-preconfigure-annotated.asm +90 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-preconfigure.asm +90 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-session-configure-annotated.asm +131 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-session-configure.asm +131 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-string-reference-candidates.json +20 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-touch-began.asm +133 -0
- package/docs/voice-input-ipa-evidence/startup-20261014/rea-macho-summary.json +2435 -0
- package/lib/client.js +6 -1
- package/lib/dsh-host-adapter.mjs +10 -0
- package/lib/index.mjs +90 -14
- package/lib/session-event-details.mjs +53 -0
- package/lib/speech-input.mjs +192 -0
- package/package.json +3 -3
package/lib/client.js
CHANGED
|
@@ -686,7 +686,6 @@ window.__ModuleLoader__.load({
|
|
|
686
686
|
React.createElement('strong', null, name), ':', description)
|
|
687
687
|
|
|
688
688
|
return React.createElement('div', { style: styles.backdrop, onMouseDown: () => setOpen(false) },
|
|
689
|
-
React.createElement('style', null, panelCss),
|
|
690
689
|
qr ? React.createElement('div', { style: { ...styles.backdrop, zIndex: 1201 },
|
|
691
690
|
onMouseDown: (event) => { event.stopPropagation(); closePairingDialog() } },
|
|
692
691
|
React.createElement('div', { className: 'mgw-pairing-dialog', 'data-closing': pairingClosing, ref: pairingDialogRef, role: 'dialog', 'aria-modal': true, 'aria-label': '设备配对二维码', tabIndex: -1,
|
|
@@ -1033,6 +1032,12 @@ window.__ModuleLoader__.load({
|
|
|
1033
1032
|
|
|
1034
1033
|
const inject = ['slots']
|
|
1035
1034
|
function apply(ctx) {
|
|
1035
|
+
ctx.effect(() => {
|
|
1036
|
+
const style = document.createElement('style')
|
|
1037
|
+
style.textContent = panelCss
|
|
1038
|
+
document.head.appendChild(style)
|
|
1039
|
+
return () => style.remove()
|
|
1040
|
+
}, 'mobile-gateway: panel styles')
|
|
1036
1041
|
ctx.slots.inject('sidebar.footer.action', () => ctx.slots.register(
|
|
1037
1042
|
{ name: 'sidebar.footer.action', id: 'mobile-gateway-devices', order: 80, label: '移动设备' },
|
|
1038
1043
|
(props) => React.createElement(FooterButton, props),
|
package/lib/dsh-host-adapter.mjs
CHANGED
|
@@ -198,6 +198,16 @@ export function createDshHostAdapter(typertGateway) {
|
|
|
198
198
|
}
|
|
199
199
|
|
|
200
200
|
return {
|
|
201
|
+
speech: {
|
|
202
|
+
catalog: (signal) => invoke('speech', 'catalog', {}, signal),
|
|
203
|
+
follow: (signal) => typertGateway.stream({ namespace: 'speech', method: 'follow', args: {}, signal }),
|
|
204
|
+
configure: (patch, signal) => invoke('speech', 'configure', { patch }, signal),
|
|
205
|
+
prepare: (providerId, options, signal) => invoke('speech', 'prepare', {
|
|
206
|
+
providerId, ...(options === undefined ? {} : { options }),
|
|
207
|
+
}, signal),
|
|
208
|
+
cancelPreparation: (providerId, signal) => invoke('speech', 'cancelPreparation', { providerId }, signal),
|
|
209
|
+
transcribe: (request, signal) => invoke('speech', 'transcribe', { request }, signal),
|
|
210
|
+
},
|
|
201
211
|
sessions: {
|
|
202
212
|
list: (payload = {}, signal) => invoke('session', 'list', sessionListArgs(payload), signal),
|
|
203
213
|
search: (payload, signal) => invoke('session', 'search', requestArgs(payload), signal),
|
package/lib/index.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { stringifyWireFrame } from './wire-json.mjs'
|
|
2
|
+
import { assistantFinish, failureDetails, toolResultDetails, turnReasonDetails } from './session-event-details.mjs'
|
|
2
3
|
// dsh-plugin-mobile-gateway — persistent Host plugin.
|
|
3
4
|
//
|
|
4
5
|
// Two directions:
|
|
@@ -35,6 +36,7 @@ import { stringifyWireFrame } from './wire-json.mjs'
|
|
|
35
36
|
// { "type": "attachment", "sessionId", "attachmentId" }
|
|
36
37
|
// -> verified image metadata + canonical base64 bytes
|
|
37
38
|
// { "type": "file-list", "requestId"?, "sessionId", "path"? }
|
|
39
|
+
// { "type": "file-resolve", "requestId", "sessionId", "path" }
|
|
38
40
|
// { "type": "file-download-open", "requestId", "sessionId", "path" }
|
|
39
41
|
// { "type": "file-download-read", "transferId", "offset" }
|
|
40
42
|
// { "type": "file-download-cancel", "transferId" }
|
|
@@ -93,6 +95,7 @@ import { stringifyWireFrame } from './wire-json.mjs'
|
|
|
93
95
|
// { "kind": "command-selected", "sessionId", "command", "selected" }
|
|
94
96
|
// { "kind": "attachment", "sessionId", "attachment", "data" }
|
|
95
97
|
// { "kind": "file-list", "requestId"?, "sessionId", "path", "entries" }
|
|
98
|
+
// { "kind": "file-resolved", "requestId", "sessionId", "path", "name", "mediaType", "size", "modifiedAt" }
|
|
96
99
|
// { "kind": "file-download-opened", "requestId", "transferId", "sessionId",
|
|
97
100
|
// "path", "name", "mediaType", "size", "chunkBytes" }
|
|
98
101
|
// { "kind": "file-download-chunk", "transferId", "offset", "data", "eof", "sha256"? }
|
|
@@ -132,6 +135,7 @@ import Schema from '@deepseek-ai/schemastery'
|
|
|
132
135
|
import { WebSocketServer } from 'ws'
|
|
133
136
|
import devicesModule from './devices.js'
|
|
134
137
|
import { createDshHostAdapter, DSH_VERSION, SESSION_FORMAT_VERSION } from './dsh-host-adapter.mjs'
|
|
138
|
+
import { createSpeechInputManager, SPEECH_REQUEST_TYPES } from './speech-input.mjs'
|
|
135
139
|
import { handlePluginQuery, PLUGIN_REQUEST_TYPES } from './plugin-management.mjs'
|
|
136
140
|
import { createSessionFollower } from './session-follower.mjs'
|
|
137
141
|
import { createGatewayState, GATEWAY_MODES } from './gateway-state.mjs'
|
|
@@ -307,6 +311,7 @@ function buildWireEvent(session, event) {
|
|
|
307
311
|
}
|
|
308
312
|
return Object.assign(base, {
|
|
309
313
|
event: { type: 'assistant/message', turn: d.turn, step: d.step, text, reasoning, toolCalls,
|
|
314
|
+
...(assistantFinish(d) ? { finish: assistantFinish(d) } : {}),
|
|
310
315
|
...(d.interrupted === true ? { interrupted: true } : {}),
|
|
311
316
|
...(d.usage === undefined ? {} : { usage: d.usage }),
|
|
312
317
|
...(images.length ? { images } : {}),
|
|
@@ -314,7 +319,15 @@ function buildWireEvent(session, event) {
|
|
|
314
319
|
})
|
|
315
320
|
}
|
|
316
321
|
case 'assistant/attempt':
|
|
317
|
-
return Object.assign(base, { event: { type: event.type, turn: d.turn, step: d.step, stream: d.stream
|
|
322
|
+
return Object.assign(base, { event: { type: event.type, turn: d.turn, step: d.step, stream: d.stream,
|
|
323
|
+
...(assistantFinish(d) ? { finish: assistantFinish(d) } : {}) } })
|
|
324
|
+
case 'llm/retry':
|
|
325
|
+
case 'llm/retry-started':
|
|
326
|
+
return Object.assign(base, { event: { type: event.type, turn: d.turn, step: d.step,
|
|
327
|
+
retryId: d.retryId, retry: d.retry,
|
|
328
|
+
...(event.type === 'llm/retry' ? { provider: d.provider, mode: d.mode,
|
|
329
|
+
maxRetries: d.maxRetries, delayMs: d.delayMs, failure: failureDetails(d.failure) } : {}),
|
|
330
|
+
} })
|
|
318
331
|
case 'session/title':
|
|
319
332
|
return Object.assign(base, {
|
|
320
333
|
event: {
|
|
@@ -325,6 +338,16 @@ function buildWireEvent(session, event) {
|
|
|
325
338
|
})
|
|
326
339
|
case 'agent-preset/selected':
|
|
327
340
|
return Object.assign(base, { event: { type: event.type, agentPreset: d.agentPreset } })
|
|
341
|
+
case 'deliverables/presented':
|
|
342
|
+
return Object.assign(base, {
|
|
343
|
+
event: { type: event.type, turn: d.turn, callId: d.callId,
|
|
344
|
+
files: (Array.isArray(d.files) ? d.files : [])
|
|
345
|
+
.filter(file => file && typeof file.path === 'string' && file.path.trim())
|
|
346
|
+
.map(file => ({ path: file.path,
|
|
347
|
+
...(typeof file.description === 'string' ? { description: file.description } : {}),
|
|
348
|
+
})),
|
|
349
|
+
},
|
|
350
|
+
})
|
|
328
351
|
case 'tool/call':
|
|
329
352
|
return Object.assign(base, {
|
|
330
353
|
event: { type: 'tool/call', turn: d.turn, step: d.step, callId: d.callId, name: d.name, arguments: d.arguments },
|
|
@@ -346,6 +369,7 @@ function buildWireEvent(session, event) {
|
|
|
346
369
|
callId: d.message?.toolCallId ?? d.message?.source?.callId,
|
|
347
370
|
isError: d.message?.isError === true || !!d.error || ((d.message && d.message.content) || []).some(block => block?.type === 'tool-result' && block.isError === true),
|
|
348
371
|
preview,
|
|
372
|
+
...toolResultDetails(d),
|
|
349
373
|
...(d.error ? { error: d.error } : {}),
|
|
350
374
|
},
|
|
351
375
|
})
|
|
@@ -404,7 +428,8 @@ function buildWireEvent(session, event) {
|
|
|
404
428
|
case 'step/start':
|
|
405
429
|
case 'step/end':
|
|
406
430
|
return Object.assign(base, {
|
|
407
|
-
event: { type: event.type, turn: d.turn, step: d.step, reason: d.reason && d.reason.kind
|
|
431
|
+
event: { type: event.type, turn: d.turn, step: d.step, reason: d.reason && d.reason.kind,
|
|
432
|
+
...(turnReasonDetails(d.reason) ? { reasonDetails: turnReasonDetails(d.reason) } : {}) },
|
|
408
433
|
})
|
|
409
434
|
default:
|
|
410
435
|
return Object.assign(base, { event: { type: event.type } })
|
|
@@ -673,20 +698,31 @@ function createFileTransferManager(api, options) {
|
|
|
673
698
|
if (!stat.isDirectory()) {
|
|
674
699
|
return fileTransferError('file-workspace-unavailable', 'the session working directory is unavailable', requestType, sessionId)
|
|
675
700
|
}
|
|
676
|
-
return { root }
|
|
701
|
+
return { root, inputRoot: path.resolve(item.cwd) }
|
|
677
702
|
} catch {
|
|
678
703
|
return fileTransferError('file-workspace-unavailable', 'the session working directory is unavailable', requestType, sessionId)
|
|
679
704
|
}
|
|
680
705
|
}
|
|
681
706
|
|
|
682
707
|
const resolveWorkspaceTarget = async (sessionId, rawPath, requestType, inputOptions) => {
|
|
683
|
-
|
|
708
|
+
// Only the resolver accepts native absolute paths from recorded deliveries
|
|
709
|
+
// or Markdown links. Download requests remain workspace-relative.
|
|
710
|
+
const absoluteInput = inputOptions?.allowAbsolute === true
|
|
711
|
+
&& typeof rawPath === 'string' && path.isAbsolute(rawPath.trim())
|
|
712
|
+
const requested = absoluteInput
|
|
713
|
+
? (rawPath.includes('\0') || rawPath.trim().split(/[\\/]+/).includes('..') ? null : rawPath.trim())
|
|
714
|
+
: normalizedWorkspaceInput(rawPath, inputOptions)
|
|
684
715
|
if (!requested) {
|
|
685
|
-
return fileTransferError('bad-request', `${requestType} requires a relative workspace path`, requestType, sessionId)
|
|
716
|
+
return fileTransferError('bad-request', `${requestType} requires a valid ${inputOptions?.allowAbsolute ? '' : 'relative '}workspace path`, requestType, sessionId)
|
|
686
717
|
}
|
|
687
718
|
const rootResult = await resolveSessionRoot(sessionId, requestType)
|
|
688
719
|
if (rootResult.kind === 'error') return rootResult
|
|
689
|
-
|
|
720
|
+
// macOS commonly exposes one workspace as both /var/... and /private/var/...
|
|
721
|
+
// (and cwd itself may be a symlink). Map only the known cwd alias before
|
|
722
|
+
// enforcing the canonical root, then still verify the target's realpath.
|
|
723
|
+
const candidate = absoluteInput && isInsideWorkspace(rootResult.inputRoot, requested)
|
|
724
|
+
? path.resolve(rootResult.root, path.relative(rootResult.inputRoot, requested))
|
|
725
|
+
: path.resolve(rootResult.root, requested)
|
|
690
726
|
if (!isInsideWorkspace(rootResult.root, candidate)) {
|
|
691
727
|
return fileTransferError('file-not-allowed', 'path must stay inside the session working directory', requestType, sessionId)
|
|
692
728
|
}
|
|
@@ -705,6 +741,28 @@ function createFileTransferManager(api, options) {
|
|
|
705
741
|
}
|
|
706
742
|
}
|
|
707
743
|
|
|
744
|
+
const resolveFile = async (msg) => {
|
|
745
|
+
const session = requireSessionId(msg)
|
|
746
|
+
if (session.error) return session.error
|
|
747
|
+
if (typeof msg.requestId !== 'string' || !msg.requestId.trim()) {
|
|
748
|
+
return fileTransferError('bad-request', 'file-resolve requires a requestId', 'file-resolve', session.value)
|
|
749
|
+
}
|
|
750
|
+
const resolved = await resolveWorkspaceTarget(session.value, msg.path, 'file-resolve', { allowAbsolute: true })
|
|
751
|
+
if (resolved.kind === 'error') return resolved
|
|
752
|
+
if (!resolved.stat.isFile()) {
|
|
753
|
+
return fileTransferError('file-not-regular', 'only regular files can be previewed', 'file-resolve', session.value)
|
|
754
|
+
}
|
|
755
|
+
if (resolved.stat.size > options.fileDownloadMaxBytes) {
|
|
756
|
+
return fileTransferError('file-too-large', 'file exceeds the configured download limit', 'file-resolve', session.value)
|
|
757
|
+
}
|
|
758
|
+
return {
|
|
759
|
+
kind: 'file-resolved', requestId: msg.requestId, sessionId: session.value,
|
|
760
|
+
path: workspaceRelativePath(resolved.root, resolved.target),
|
|
761
|
+
name: path.basename(resolved.target), mediaType: fileMediaType(resolved.target),
|
|
762
|
+
size: resolved.stat.size, modifiedAt: resolved.stat.mtimeMs,
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
708
766
|
const fileList = async (msg) => {
|
|
709
767
|
const session = requireSessionId(msg)
|
|
710
768
|
if (session.error) return session.error
|
|
@@ -884,9 +942,11 @@ function createFileTransferManager(api, options) {
|
|
|
884
942
|
|
|
885
943
|
return {
|
|
886
944
|
async handle(client, msg) {
|
|
945
|
+
const correlation = typeof msg.requestId === 'string' ? { requestId: msg.requestId } : {}
|
|
887
946
|
if (!options.fileDownloadsEnabled) {
|
|
888
|
-
return fileTransferError('file-download-disabled', 'file downloads are disabled by gateway configuration', msg.type)
|
|
947
|
+
return { ...fileTransferError('file-download-disabled', 'file downloads are disabled by gateway configuration', msg.type), ...correlation }
|
|
889
948
|
}
|
|
949
|
+
if (msg.type === 'file-resolve') return { ...await resolveFile(msg), ...correlation }
|
|
890
950
|
if (msg.type === 'file-list') return fileList(msg)
|
|
891
951
|
if (msg.type === 'file-download-open') return open(client, msg)
|
|
892
952
|
if (msg.type === 'file-download-read') return read(client, msg)
|
|
@@ -1077,12 +1137,14 @@ function trimConversationEvent(event) {
|
|
|
1077
1137
|
case 'assistant/message':
|
|
1078
1138
|
case 'assistant/attempt': {
|
|
1079
1139
|
const { stream, ...data } = event.data || {}
|
|
1080
|
-
|
|
1140
|
+
const finish = assistantFinish(event.data || {})
|
|
1141
|
+
return { ...event, data: { ...data, ...(finish ? { finish } : {}) } }
|
|
1081
1142
|
}
|
|
1082
1143
|
case 'tool/result': {
|
|
1083
1144
|
const d = event.data || {}
|
|
1084
1145
|
const message = d.message
|
|
1085
|
-
|
|
1146
|
+
const details = toolResultDetails(d)
|
|
1147
|
+
if (!message || !Array.isArray(message.content)) return { ...event, data: { ...d, ...details } }
|
|
1086
1148
|
let changed = false
|
|
1087
1149
|
const truncateBlock = (block) => {
|
|
1088
1150
|
if (!block || typeof block !== 'object') return block
|
|
@@ -1096,8 +1158,7 @@ function trimConversationEvent(event) {
|
|
|
1096
1158
|
return block
|
|
1097
1159
|
}
|
|
1098
1160
|
const content = message.content.map(truncateBlock)
|
|
1099
|
-
|
|
1100
|
-
return { ...event, data: { ...d, message: { ...message, content } } }
|
|
1161
|
+
return { ...event, data: { ...d, ...details, ...(changed ? { message: { ...message, content } } : {}) } }
|
|
1101
1162
|
}
|
|
1102
1163
|
default:
|
|
1103
1164
|
return event
|
|
@@ -2212,6 +2273,7 @@ const plugin = {
|
|
|
2212
2273
|
const pendingQuestions = new Map()
|
|
2213
2274
|
const pendingApprovals = new Map()
|
|
2214
2275
|
const fileTransfers = createFileTransferManager(api, options)
|
|
2276
|
+
const speechInput = createSpeechInputManager(api, options)
|
|
2215
2277
|
const fileTransferExpiryTimer = setInterval(() => {
|
|
2216
2278
|
fileTransfers.expire().catch((error) => {
|
|
2217
2279
|
log(`file download expiry failed: ${error && error.message ? error.message : String(error)}`)
|
|
@@ -2907,6 +2969,14 @@ const plugin = {
|
|
|
2907
2969
|
|
|
2908
2970
|
if (msg.type === 'ping') {
|
|
2909
2971
|
ws.send(stringifyWireFrame({ kind: 'pong', at: Date.now() }))
|
|
2972
|
+
} else if (SPEECH_REQUEST_TYPES.has(msg.type)) {
|
|
2973
|
+
if (!ws.deviceId) {
|
|
2974
|
+
sendFrame({ kind: 'error', code: 'authentication-required', requestType: msg.type,
|
|
2975
|
+
...(typeof msg.requestId === 'string' ? { requestId: msg.requestId } : {}),
|
|
2976
|
+
message: 'Speech input requires a paired device' })
|
|
2977
|
+
return
|
|
2978
|
+
}
|
|
2979
|
+
void speechInput.handle(ws, msg, sendFrame).then(sendFrame)
|
|
2910
2980
|
} else if (msg.type === 'subscribe') {
|
|
2911
2981
|
if (msg.assistantStream !== undefined && typeof msg.assistantStream !== 'boolean') {
|
|
2912
2982
|
sendFrame({ kind: 'error', code: 'bad-request', requestType: 'subscribe', message: 'assistantStream must be a boolean' })
|
|
@@ -2942,7 +3012,7 @@ const plugin = {
|
|
|
2942
3012
|
respondToApproval(msg).then((frame) => {
|
|
2943
3013
|
if (frame && ws.readyState === 1) ws.send(stringifyWireFrame(frame))
|
|
2944
3014
|
})
|
|
2945
|
-
} else if (msg.type === 'file-list' || msg.type === 'file-download-open' ||
|
|
3015
|
+
} else if (msg.type === 'file-list' || msg.type === 'file-resolve' || msg.type === 'file-download-open' ||
|
|
2946
3016
|
msg.type === 'file-download-read' || msg.type === 'file-download-cancel') {
|
|
2947
3017
|
fileTransfers.handle(ws, msg).then((frame) => {
|
|
2948
3018
|
if (frame && ws.readyState === 1) ws.send(stringifyWireFrame(frame))
|
|
@@ -2989,6 +3059,7 @@ const plugin = {
|
|
|
2989
3059
|
})
|
|
2990
3060
|
|
|
2991
3061
|
ws.on('close', () => {
|
|
3062
|
+
speechInput.closeClient(ws)
|
|
2992
3063
|
ws.sessionFollower.stop()
|
|
2993
3064
|
clients.delete(ws)
|
|
2994
3065
|
if (ws.deviceId) registry.disconnected(ws.deviceId)
|
|
@@ -3027,11 +3098,12 @@ const plugin = {
|
|
|
3027
3098
|
'goals',
|
|
3028
3099
|
'schedule-management',
|
|
3029
3100
|
'plugin-management-v1',
|
|
3101
|
+
...(device ? ['speech-input-v1'] : []),
|
|
3030
3102
|
'session-cancel',
|
|
3031
3103
|
'queue-control',
|
|
3032
3104
|
'session-archive',
|
|
3033
3105
|
'session-rename',
|
|
3034
|
-
...(options.fileDownloadsEnabled ? ['file-downloads'] : []),
|
|
3106
|
+
...(options.fileDownloadsEnabled ? ['file-downloads', 'file-references-v1'] : []),
|
|
3035
3107
|
],
|
|
3036
3108
|
port: transport.port || webServer.port,
|
|
3037
3109
|
clients: clients.size,
|
|
@@ -3201,7 +3273,7 @@ const plugin = {
|
|
|
3201
3273
|
|
|
3202
3274
|
const disposeApprovals = ctx.on('approval/request', (request, next) => {
|
|
3203
3275
|
const sessionId = request?.agent?.id === undefined ? null : String(request.agent.id)
|
|
3204
|
-
if (!sessionId || typeof request.toolName !== 'string' || !
|
|
3276
|
+
if (!sessionId || typeof request.toolName !== 'string' || !gatewayEnabled) return next()
|
|
3205
3277
|
const rpcId = crypto.randomUUID()
|
|
3206
3278
|
const approvalId = crypto.randomUUID()
|
|
3207
3279
|
return new Promise((resolve, reject) => {
|
|
@@ -3231,7 +3303,10 @@ const plugin = {
|
|
|
3231
3303
|
}
|
|
3232
3304
|
pendingApprovals.set(rpcId, pending)
|
|
3233
3305
|
broadcastInteractionFrame(approvalFrameFor(rpcId, pending))
|
|
3306
|
+
// 控制连接重连期间仍登记待审批,桌面端继续处理同一个请求。
|
|
3307
|
+
// 若桌面端仍在等待,后续连接可以重放;无可用答复时保持原有回退行为。
|
|
3234
3308
|
startDownstreamApproval(pending)
|
|
3309
|
+
if (!hasInteractionClient(sessionId)) fallbackApproval(pending)
|
|
3235
3310
|
log(`approval requested: rpcId=${rpcId} approvalId=${approvalId} session=${sessionId} tool=${request.toolName}`)
|
|
3236
3311
|
})
|
|
3237
3312
|
}, { prepend: true })
|
|
@@ -3396,6 +3471,7 @@ const plugin = {
|
|
|
3396
3471
|
if (waitTimer) clearTimeout(waitTimer)
|
|
3397
3472
|
clearInterval(fileTransferExpiryTimer)
|
|
3398
3473
|
backgroundStreamAbort.abort()
|
|
3474
|
+
speechInput.dispose()
|
|
3399
3475
|
for (const pending of [...pendingQuestions.values()]) fallbackQuestion(pending)
|
|
3400
3476
|
for (const pending of [...pendingApprovals.values()]) fallbackApproval(pending)
|
|
3401
3477
|
fileTransfers.dispose().catch((error) => {
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// Keep diagnostic facts when token replay and tool bodies are trimmed for mobile.
|
|
2
|
+
const TOOL_OUTPUT_LIMIT = 16_000
|
|
3
|
+
|
|
4
|
+
export function failureDetails(value) {
|
|
5
|
+
if (!value || typeof value !== 'object') return undefined
|
|
6
|
+
const result = {}
|
|
7
|
+
for (const key of ['message', 'code', 'name', 'reason', 'requestId']) {
|
|
8
|
+
if (typeof value[key] === 'string') result[key] = value[key]
|
|
9
|
+
}
|
|
10
|
+
for (const key of ['status', 'providerRetryAfterMs']) {
|
|
11
|
+
if (Number.isFinite(value[key])) result[key] = value[key]
|
|
12
|
+
}
|
|
13
|
+
return Object.keys(result).length ? result : undefined
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function assistantFinish(data) {
|
|
17
|
+
for (let i = (data.stream?.length || 0) - 1; i >= 0; i--) {
|
|
18
|
+
const record = data.stream[i]
|
|
19
|
+
if (record?.type !== 'chunk' || record.chunk?.type !== 'finish') continue
|
|
20
|
+
const reason = record.chunk.reason
|
|
21
|
+
if (typeof reason?.kind !== 'string') return undefined
|
|
22
|
+
const failure = failureDetails(reason.failure ?? reason.error)
|
|
23
|
+
return { kind: reason.kind, ...(failure ? { failure } : {}) }
|
|
24
|
+
}
|
|
25
|
+
return data.finish
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function toolResultDetails(data) {
|
|
29
|
+
const texts = []
|
|
30
|
+
const visit = block => {
|
|
31
|
+
if (block?.type === 'text' && typeof block.text === 'string') texts.push(block.text)
|
|
32
|
+
else if (block?.type === 'tool-result' && Array.isArray(block.content)) block.content.forEach(visit)
|
|
33
|
+
}
|
|
34
|
+
for (const block of data.message?.content || []) visit(block)
|
|
35
|
+
const full = texts.join('')
|
|
36
|
+
const outputTruncated = full.length > TOOL_OUTPUT_LIMIT
|
|
37
|
+
// Retain the terminal status and sandbox hints at the end as well as the start.
|
|
38
|
+
const output = outputTruncated
|
|
39
|
+
? `${full.slice(0, TOOL_OUTPUT_LIMIT / 2)}\n[…]\n${full.slice(-TOOL_OUTPUT_LIMIT / 2)}`
|
|
40
|
+
: full
|
|
41
|
+
return { output, outputTruncated }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function turnReasonDetails(reason) {
|
|
45
|
+
if (!reason || typeof reason.kind !== 'string') return undefined
|
|
46
|
+
const error = failureDetails(reason.error ?? reason.failure)
|
|
47
|
+
return { kind: reason.kind, ...(error ? { error } : {}),
|
|
48
|
+
...(typeof reason.reason?.kind === 'string' ? { reason: {
|
|
49
|
+
kind: reason.reason.kind,
|
|
50
|
+
...(typeof reason.reason.reason === 'string' ? { reason: reason.reason.reason } : {}),
|
|
51
|
+
} } : {}),
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
// 移动连接只代理官方 speech Remote,不持有识别模型或写入会话历史。
|
|
2
|
+
export const SPEECH_REQUEST_TYPES = new Set([
|
|
3
|
+
'speech-catalog', 'speech-subscribe', 'speech-unsubscribe', 'speech-configure',
|
|
4
|
+
'speech-prepare', 'speech-cancel-preparation', 'speech-transcribe', 'speech-cancel',
|
|
5
|
+
])
|
|
6
|
+
|
|
7
|
+
const MAX_AUDIO_BYTES = 4 * 1024 * 1024
|
|
8
|
+
const UNAVAILABLE_CODES = new Set(['gateway/invocation-unavailable', 'gateway/definition-unavailable', 'gateway/service-unavailable'])
|
|
9
|
+
|
|
10
|
+
function failure(code, message) {
|
|
11
|
+
return Object.assign(new Error(message), { code })
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function identifier(value, name) {
|
|
15
|
+
if (typeof value !== 'string' || !value.trim() || value.length > 128) {
|
|
16
|
+
throw failure('bad-request', `${name} must be a nonempty string of at most 128 characters`)
|
|
17
|
+
}
|
|
18
|
+
return value.trim()
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function errorFrame(msg, error) {
|
|
22
|
+
return { kind: 'error', requestType: msg.type,
|
|
23
|
+
...(typeof msg.requestId === 'string' ? { requestId: msg.requestId } : {}),
|
|
24
|
+
code: UNAVAILABLE_CODES.has(error?.code) ? 'speech/unavailable' : error?.code || 'speech/failed',
|
|
25
|
+
message: UNAVAILABLE_CODES.has(error?.code)
|
|
26
|
+
? 'Enable the official voice input bundle in DSH plugin management' : error?.message || 'Speech operation failed' }
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function createSpeechInputManager(api, { maxPayloadBytes = 64 * 1024 * 1024,
|
|
30
|
+
transcriptionTimeoutMs = 180_000 } = {}) {
|
|
31
|
+
const clients = new Map()
|
|
32
|
+
let activeTranscriptions = 0
|
|
33
|
+
let disposed = false
|
|
34
|
+
// 为 JSON 信封留出空间;实际限制还须服从官方插件的目录。
|
|
35
|
+
const audioLimit = Math.min(MAX_AUDIO_BYTES, Math.max(0, Math.floor((maxPayloadBytes - 4096) / 4) * 3))
|
|
36
|
+
const stateFor = client => {
|
|
37
|
+
if (disposed) throw failure('speech/unavailable', 'Speech gateway has been disposed')
|
|
38
|
+
let state = clients.get(client)
|
|
39
|
+
if (!state) {
|
|
40
|
+
state = { operations: new Map(), follow: null, closed: false }
|
|
41
|
+
clients.set(client, state)
|
|
42
|
+
}
|
|
43
|
+
return state
|
|
44
|
+
}
|
|
45
|
+
const normalizeCatalog = value => {
|
|
46
|
+
if (!value || !Array.isArray(value.providers) || !value.selection
|
|
47
|
+
|| !Number.isSafeInteger(value.maxAudioBytes) || value.maxAudioBytes < 46
|
|
48
|
+
|| !Number.isFinite(value.maxDurationSeconds) || value.maxDurationSeconds < 1) {
|
|
49
|
+
throw failure('speech/invalid-response', 'Official speech catalog is invalid')
|
|
50
|
+
}
|
|
51
|
+
return { ...value, maxAudioBytes: Math.min(audioLimit, value.maxAudioBytes),
|
|
52
|
+
maxDurationSeconds: value.maxDurationSeconds, available: true }
|
|
53
|
+
}
|
|
54
|
+
const stopFollow = state => {
|
|
55
|
+
state.follow?.abort.abort()
|
|
56
|
+
state.follow = null
|
|
57
|
+
}
|
|
58
|
+
const catalog = async signal => normalizeCatalog(await api.speech.catalog(signal))
|
|
59
|
+
|
|
60
|
+
return {
|
|
61
|
+
async handle(client, msg, send) {
|
|
62
|
+
try {
|
|
63
|
+
const requestId = identifier(msg.requestId, 'requestId')
|
|
64
|
+
const state = stateFor(client)
|
|
65
|
+
if (msg.type === 'speech-cancel') {
|
|
66
|
+
const targetRequestId = identifier(msg.targetRequestId, 'targetRequestId')
|
|
67
|
+
const task = state.operations.get(targetRequestId)
|
|
68
|
+
if (!task || task.type !== 'speech-transcribe') throw failure('speech/request-not-found', 'Transcription is not active on this connection')
|
|
69
|
+
task.abort.abort(failure('speech/cancelled', 'Transcription cancelled'))
|
|
70
|
+
return { kind: 'speech-cancelled', requestId, targetRequestId }
|
|
71
|
+
}
|
|
72
|
+
if (msg.type === 'speech-unsubscribe') {
|
|
73
|
+
stopFollow(state)
|
|
74
|
+
return { kind: 'speech-unsubscribed', requestId }
|
|
75
|
+
}
|
|
76
|
+
if (state.operations.has(requestId) || state.follow?.requestId === requestId) {
|
|
77
|
+
throw failure('speech/request-in-progress', 'requestId is already active on this connection')
|
|
78
|
+
}
|
|
79
|
+
if (state.operations.size >= 8) throw failure('speech/busy', 'Too many pending speech requests')
|
|
80
|
+
if (msg.type === 'speech-subscribe') {
|
|
81
|
+
stopFollow(state)
|
|
82
|
+
const follow = { abort: new AbortController(), requestId }
|
|
83
|
+
state.follow = follow
|
|
84
|
+
// 先回执,再按订阅 requestId 推送完整快照。Host 的准备任务不属于此观察连接。
|
|
85
|
+
queueMicrotask(async () => {
|
|
86
|
+
try {
|
|
87
|
+
const stream = await api.speech.follow(follow.abort.signal)
|
|
88
|
+
for await (const snapshot of stream) {
|
|
89
|
+
if (state.closed || state.follow !== follow || follow.abort.signal.aborted) break
|
|
90
|
+
send({ kind: 'speech-state', requestId, ...normalizeCatalog(snapshot) })
|
|
91
|
+
}
|
|
92
|
+
if (!follow.abort.signal.aborted) throw failure('speech/stream-ended', 'Speech readiness stream ended; subscribe again')
|
|
93
|
+
} catch (error) {
|
|
94
|
+
if (!state.closed && state.follow === follow && !follow.abort.signal.aborted) send(errorFrame(msg, error))
|
|
95
|
+
} finally {
|
|
96
|
+
if (state.follow === follow) state.follow = null
|
|
97
|
+
}
|
|
98
|
+
})
|
|
99
|
+
return { kind: 'speech-subscribed', requestId }
|
|
100
|
+
}
|
|
101
|
+
const abort = new AbortController()
|
|
102
|
+
const task = { abort, type: msg.type }
|
|
103
|
+
let timer
|
|
104
|
+
let admitted = false
|
|
105
|
+
state.operations.set(requestId, task)
|
|
106
|
+
try {
|
|
107
|
+
if (msg.type === 'speech-catalog') {
|
|
108
|
+
try { return { kind: 'speech-catalog', requestId, ...await catalog(abort.signal) } }
|
|
109
|
+
catch (error) {
|
|
110
|
+
if (UNAVAILABLE_CODES.has(error?.code)) return { kind: 'speech-catalog', requestId,
|
|
111
|
+
available: false, providers: [], code: 'speech/unavailable',
|
|
112
|
+
message: 'Enable the official voice input bundle in DSH plugin management' }
|
|
113
|
+
throw error
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (msg.type === 'speech-transcribe') {
|
|
117
|
+
if ([...state.operations.values()].some(value => value !== task && value.type === msg.type)
|
|
118
|
+
|| activeTranscriptions >= 4) throw failure('speech/busy', 'Too many active transcriptions')
|
|
119
|
+
const encoded = msg.audioBase64
|
|
120
|
+
if (typeof encoded !== 'string' || !encoded.length || encoded.length > Math.ceil(audioLimit / 3) * 4) {
|
|
121
|
+
throw failure('speech/invalid-audio', 'Audio is empty or exceeds the gateway byte limit')
|
|
122
|
+
}
|
|
123
|
+
const audio = Buffer.from(encoded, 'base64')
|
|
124
|
+
if (audio.length > audioLimit || audio.toString('base64') !== encoded) {
|
|
125
|
+
throw failure('speech/invalid-audio', 'Audio must use canonical base64 within the gateway byte limit')
|
|
126
|
+
}
|
|
127
|
+
const request = { audioBase64: encoded,
|
|
128
|
+
...(msg.providerId === undefined ? {} : { providerId: identifier(msg.providerId, 'providerId') }),
|
|
129
|
+
...(msg.language === undefined ? {} : { language: identifier(msg.language, 'language') }) }
|
|
130
|
+
activeTranscriptions++
|
|
131
|
+
admitted = true
|
|
132
|
+
timer = setTimeout(() => abort.abort(failure('speech/timeout', 'Transcription timed out')), transcriptionTimeoutMs)
|
|
133
|
+
timer.unref?.()
|
|
134
|
+
const limits = await catalog(abort.signal)
|
|
135
|
+
abort.signal.throwIfAborted()
|
|
136
|
+
if (audio.length > limits.maxAudioBytes) throw failure('speech/invalid-audio', 'Audio exceeds the official plugin byte limit')
|
|
137
|
+
// 官方 Remote 负责 WAV、采样率、语言、模型就绪与录音时长校验。
|
|
138
|
+
const result = await api.speech.transcribe(request, abort.signal)
|
|
139
|
+
abort.signal.throwIfAborted()
|
|
140
|
+
if (typeof result?.text !== 'string' || !Number.isFinite(result.audioSeconds) || result.audioSeconds < 0
|
|
141
|
+
|| !Number.isFinite(result.inferenceSeconds) || result.inferenceSeconds < 0) {
|
|
142
|
+
throw failure('speech/invalid-response', 'Official speech transcript is invalid')
|
|
143
|
+
}
|
|
144
|
+
return { kind: 'speech-transcript', requestId, text: result.text,
|
|
145
|
+
audioSeconds: result.audioSeconds, inferenceSeconds: result.inferenceSeconds }
|
|
146
|
+
}
|
|
147
|
+
if (msg.type === 'speech-configure') {
|
|
148
|
+
const patch = {
|
|
149
|
+
...(msg.providerId === undefined ? {} : { providerId: identifier(msg.providerId, 'providerId') }),
|
|
150
|
+
...(msg.language === undefined ? {} : { language: identifier(msg.language, 'language') }),
|
|
151
|
+
}
|
|
152
|
+
if (!Object.keys(patch).length) throw failure('bad-request', 'Provide providerId or language')
|
|
153
|
+
await api.speech.configure(patch, abort.signal)
|
|
154
|
+
return { kind: 'speech-configured', requestId }
|
|
155
|
+
}
|
|
156
|
+
const providerId = identifier(msg.providerId, 'providerId')
|
|
157
|
+
if (msg.type === 'speech-prepare') {
|
|
158
|
+
const options = msg.downloadSource === undefined ? undefined : { downloadSource: msg.downloadSource }
|
|
159
|
+
if (options && (typeof options.downloadSource !== 'string' || !options.downloadSource.trim() || options.downloadSource.length > 2048)) {
|
|
160
|
+
throw failure('bad-request', 'downloadSource must be an advertised nonempty origin')
|
|
161
|
+
}
|
|
162
|
+
await api.speech.prepare(providerId, options, abort.signal)
|
|
163
|
+
return { kind: 'speech-preparing', requestId, providerId }
|
|
164
|
+
}
|
|
165
|
+
if (msg.type === 'speech-cancel-preparation') {
|
|
166
|
+
await api.speech.cancelPreparation(providerId, abort.signal)
|
|
167
|
+
return { kind: 'speech-preparation-cancelled', requestId, providerId }
|
|
168
|
+
}
|
|
169
|
+
throw failure('bad-request', 'Unknown speech request')
|
|
170
|
+
} catch (error) {
|
|
171
|
+
throw abort.signal.aborted ? abort.signal.reason : error
|
|
172
|
+
} finally {
|
|
173
|
+
clearTimeout(timer)
|
|
174
|
+
state.operations.delete(requestId)
|
|
175
|
+
if (admitted) activeTranscriptions--
|
|
176
|
+
}
|
|
177
|
+
} catch (error) { return errorFrame(msg, error) }
|
|
178
|
+
},
|
|
179
|
+
closeClient(client) {
|
|
180
|
+
const state = clients.get(client)
|
|
181
|
+
if (!state) return
|
|
182
|
+
state.closed = true
|
|
183
|
+
stopFollow(state)
|
|
184
|
+
for (const task of state.operations.values()) task.abort.abort(failure('speech/cancelled', 'Mobile connection closed'))
|
|
185
|
+
clients.delete(client)
|
|
186
|
+
},
|
|
187
|
+
dispose() {
|
|
188
|
+
disposed = true
|
|
189
|
+
for (const client of clients.keys()) this.closeClient(client)
|
|
190
|
+
},
|
|
191
|
+
}
|
|
192
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-plugin-mobile-gateway",
|
|
3
|
-
"version": "0.10.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.10.2",
|
|
4
|
+
"description": "DeepSeek Harness 移动网关,支持安全设备配对、实时会话同步、插件管理、官方语音输入、局域网及 Cloudflare Tunnel,以及自适应深浅色控制面板。0.10.2 新增文件卡片事件透传与安全文件引用解析,复用分块下载为移动端文件卡片和正文文件链接预览提供协议支持。",
|
|
5
5
|
"main": "lib/index.mjs",
|
|
6
6
|
"files": [
|
|
7
7
|
"bin",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
},
|
|
27
27
|
"bin": "bin/setup-ip.mjs",
|
|
28
28
|
"scripts": {
|
|
29
|
-
"test": "node test/wire-json.test.mjs && node test/setup-ip.test.mjs && node test/host-adapter.test.mjs && node test/plugin-management.test.mjs && node test/session-preset.test.mjs && node test/session-follower.test.mjs && node test/gateway.test.mjs && node test/auth.test.mjs && node test/lan.test.mjs && node test/cloudflared-binary.test.mjs && node test/cloudflare-tunnel.test.mjs && node --test test/public-endpoint-check.test.mjs && node test/gateway-tools.test.mjs && node test/multi-gateway.test.mjs"
|
|
29
|
+
"test": "node test/wire-json.test.mjs && node test/setup-ip.test.mjs && node test/host-adapter.test.mjs && node test/speech-input.test.mjs && node test/plugin-management.test.mjs && node test/session-preset.test.mjs && node test/session-follower.test.mjs && node test/session-event-details.test.mjs && node test/gateway.test.mjs && node --test test/approval-replay.test.mjs && node test/auth.test.mjs && node test/lan.test.mjs && node test/cloudflared-binary.test.mjs && node test/cloudflare-tunnel.test.mjs && node --test test/public-endpoint-check.test.mjs && node test/gateway-tools.test.mjs && node test/multi-gateway.test.mjs"
|
|
30
30
|
},
|
|
31
31
|
"exports": {
|
|
32
32
|
".": "./lib/index.mjs",
|