dsh-plugin-mobile-gateway 0.10.0 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/PROTOCOL.md +125 -7
  2. package/README.md +23 -1
  3. package/docs/dsh-rc2-mobile-integration.md +9 -1
  4. package/docs/file-cards-mobile-integration.md +111 -0
  5. package/docs/voice-input-hold-thresholds.md +35 -0
  6. package/docs/voice-input-implementation.md +275 -0
  7. package/docs/voice-input-ipa-analysis.md +150 -0
  8. package/docs/voice-input-ipa-evidence/deepseek-asr-constructor.asm +338 -0
  9. package/docs/voice-input-ipa-evidence/deepseek-extraction.json +5154 -0
  10. package/docs/voice-input-ipa-evidence/deepseek-focused-context.txt +210 -0
  11. package/docs/voice-input-ipa-evidence/deepseek-localization.json +29 -0
  12. package/docs/voice-input-ipa-evidence/deepseek-voice-strings.txt +206 -0
  13. package/docs/voice-input-ipa-evidence/doubao/address-map.txt +11 -0
  14. package/docs/voice-input-ipa-evidence/doubao/code-string-refs.txt +22 -0
  15. package/docs/voice-input-ipa-evidence/doubao/end-longpress-disasm.txt +149 -0
  16. package/docs/voice-input-ipa-evidence/doubao/focused-context.txt +271 -0
  17. package/docs/voice-input-ipa-evidence/doubao/locate_string_refs.py +42 -0
  18. package/docs/voice-input-ipa-evidence/doubao/network-candidates.txt +39 -0
  19. package/docs/voice-input-ipa-evidence/doubao/objc-msgsend-stub.txt +1 -0
  20. package/docs/voice-input-ipa-evidence/doubao/ranges.txt +858 -0
  21. package/docs/voice-input-ipa-evidence/doubao/recognizer-recorder.txt +134 -0
  22. package/docs/voice-input-ipa-evidence/doubao/result-disasm.txt +189 -0
  23. package/docs/voice-input-ipa-evidence/doubao/settext-selector.txt +2 -0
  24. package/docs/voice-input-ipa-evidence/doubao/start-stop-disasm.txt +358 -0
  25. package/docs/voice-input-ipa-evidence/doubao/strings.txt +3536 -0
  26. package/docs/voice-input-ipa-evidence/doubao-extraction.json +7418 -0
  27. package/docs/voice-input-ipa-evidence/doubao-findings.md +52 -0
  28. package/docs/voice-input-ipa-evidence/hold-threshold-rea-evidence.json +1 -0
  29. package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-hold-callback-refs.txt +1 -0
  30. package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-input-area-metadata.txt +31 -0
  31. package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-input-gesture-init.asm +267 -0
  32. package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-static-summary.json +42 -0
  33. package/docs/voice-input-ipa-evidence/hold-thresholds/deepseek-voice-gesture-constructor.asm +371 -0
  34. package/docs/voice-input-ipa-evidence/hold-thresholds/doubao-findings.md +31 -0
  35. package/docs/voice-input-ipa-evidence/hold-thresholds/doubao-text-mask-objc-metadata.txt +119 -0
  36. package/docs/voice-input-ipa-evidence/hold-thresholds/doubao-text-mask-threshold-evidence.asm +156 -0
  37. package/docs/voice-input-ipa-evidence/hold-thresholds/doubao-text-mask-voice-view-getter.asm +14 -0
  38. package/docs/voice-input-ipa-evidence/rea-evidence.json +1 -0
  39. package/docs/voice-input-ipa-evidence/startup-20261014/README.md +33 -0
  40. package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-objc.txt +222957 -0
  41. package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-preconfigure-annotated.asm +90 -0
  42. package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-preconfigure.asm +90 -0
  43. package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-session-configure-annotated.asm +131 -0
  44. package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-session-configure.asm +131 -0
  45. package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-string-reference-candidates.json +20 -0
  46. package/docs/voice-input-ipa-evidence/startup-20261014/deepseek-touch-began.asm +133 -0
  47. package/docs/voice-input-ipa-evidence/startup-20261014/rea-macho-summary.json +2435 -0
  48. package/lib/client.js +6 -1
  49. package/lib/dsh-host-adapter.mjs +10 -0
  50. package/lib/index.mjs +90 -14
  51. package/lib/session-event-details.mjs +53 -0
  52. package/lib/speech-input.mjs +192 -0
  53. package/package.json +3 -3
package/lib/client.js CHANGED
@@ -686,7 +686,6 @@ window.__ModuleLoader__.load({
686
686
  React.createElement('strong', null, name), ':', description)
687
687
 
688
688
  return React.createElement('div', { style: styles.backdrop, onMouseDown: () => setOpen(false) },
689
- React.createElement('style', null, panelCss),
690
689
  qr ? React.createElement('div', { style: { ...styles.backdrop, zIndex: 1201 },
691
690
  onMouseDown: (event) => { event.stopPropagation(); closePairingDialog() } },
692
691
  React.createElement('div', { className: 'mgw-pairing-dialog', 'data-closing': pairingClosing, ref: pairingDialogRef, role: 'dialog', 'aria-modal': true, 'aria-label': '设备配对二维码', tabIndex: -1,
@@ -1033,6 +1032,12 @@ window.__ModuleLoader__.load({
1033
1032
 
1034
1033
  const inject = ['slots']
1035
1034
  function apply(ctx) {
1035
+ ctx.effect(() => {
1036
+ const style = document.createElement('style')
1037
+ style.textContent = panelCss
1038
+ document.head.appendChild(style)
1039
+ return () => style.remove()
1040
+ }, 'mobile-gateway: panel styles')
1036
1041
  ctx.slots.inject('sidebar.footer.action', () => ctx.slots.register(
1037
1042
  { name: 'sidebar.footer.action', id: 'mobile-gateway-devices', order: 80, label: '移动设备' },
1038
1043
  (props) => React.createElement(FooterButton, props),
@@ -198,6 +198,16 @@ export function createDshHostAdapter(typertGateway) {
198
198
  }
199
199
 
200
200
  return {
201
+ speech: {
202
+ catalog: (signal) => invoke('speech', 'catalog', {}, signal),
203
+ follow: (signal) => typertGateway.stream({ namespace: 'speech', method: 'follow', args: {}, signal }),
204
+ configure: (patch, signal) => invoke('speech', 'configure', { patch }, signal),
205
+ prepare: (providerId, options, signal) => invoke('speech', 'prepare', {
206
+ providerId, ...(options === undefined ? {} : { options }),
207
+ }, signal),
208
+ cancelPreparation: (providerId, signal) => invoke('speech', 'cancelPreparation', { providerId }, signal),
209
+ transcribe: (request, signal) => invoke('speech', 'transcribe', { request }, signal),
210
+ },
201
211
  sessions: {
202
212
  list: (payload = {}, signal) => invoke('session', 'list', sessionListArgs(payload), signal),
203
213
  search: (payload, signal) => invoke('session', 'search', requestArgs(payload), signal),
package/lib/index.mjs CHANGED
@@ -1,4 +1,5 @@
1
1
  import { stringifyWireFrame } from './wire-json.mjs'
2
+ import { assistantFinish, failureDetails, toolResultDetails, turnReasonDetails } from './session-event-details.mjs'
2
3
  // dsh-plugin-mobile-gateway — persistent Host plugin.
3
4
  //
4
5
  // Two directions:
@@ -35,6 +36,7 @@ import { stringifyWireFrame } from './wire-json.mjs'
35
36
  // { "type": "attachment", "sessionId", "attachmentId" }
36
37
  // -> verified image metadata + canonical base64 bytes
37
38
  // { "type": "file-list", "requestId"?, "sessionId", "path"? }
39
+ // { "type": "file-resolve", "requestId", "sessionId", "path" }
38
40
  // { "type": "file-download-open", "requestId", "sessionId", "path" }
39
41
  // { "type": "file-download-read", "transferId", "offset" }
40
42
  // { "type": "file-download-cancel", "transferId" }
@@ -93,6 +95,7 @@ import { stringifyWireFrame } from './wire-json.mjs'
93
95
  // { "kind": "command-selected", "sessionId", "command", "selected" }
94
96
  // { "kind": "attachment", "sessionId", "attachment", "data" }
95
97
  // { "kind": "file-list", "requestId"?, "sessionId", "path", "entries" }
98
+ // { "kind": "file-resolved", "requestId", "sessionId", "path", "name", "mediaType", "size", "modifiedAt" }
96
99
  // { "kind": "file-download-opened", "requestId", "transferId", "sessionId",
97
100
  // "path", "name", "mediaType", "size", "chunkBytes" }
98
101
  // { "kind": "file-download-chunk", "transferId", "offset", "data", "eof", "sha256"? }
@@ -132,6 +135,7 @@ import Schema from '@deepseek-ai/schemastery'
132
135
  import { WebSocketServer } from 'ws'
133
136
  import devicesModule from './devices.js'
134
137
  import { createDshHostAdapter, DSH_VERSION, SESSION_FORMAT_VERSION } from './dsh-host-adapter.mjs'
138
+ import { createSpeechInputManager, SPEECH_REQUEST_TYPES } from './speech-input.mjs'
135
139
  import { handlePluginQuery, PLUGIN_REQUEST_TYPES } from './plugin-management.mjs'
136
140
  import { createSessionFollower } from './session-follower.mjs'
137
141
  import { createGatewayState, GATEWAY_MODES } from './gateway-state.mjs'
@@ -307,6 +311,7 @@ function buildWireEvent(session, event) {
307
311
  }
308
312
  return Object.assign(base, {
309
313
  event: { type: 'assistant/message', turn: d.turn, step: d.step, text, reasoning, toolCalls,
314
+ ...(assistantFinish(d) ? { finish: assistantFinish(d) } : {}),
310
315
  ...(d.interrupted === true ? { interrupted: true } : {}),
311
316
  ...(d.usage === undefined ? {} : { usage: d.usage }),
312
317
  ...(images.length ? { images } : {}),
@@ -314,7 +319,15 @@ function buildWireEvent(session, event) {
314
319
  })
315
320
  }
316
321
  case 'assistant/attempt':
317
- return Object.assign(base, { event: { type: event.type, turn: d.turn, step: d.step, stream: d.stream } })
322
+ return Object.assign(base, { event: { type: event.type, turn: d.turn, step: d.step, stream: d.stream,
323
+ ...(assistantFinish(d) ? { finish: assistantFinish(d) } : {}) } })
324
+ case 'llm/retry':
325
+ case 'llm/retry-started':
326
+ return Object.assign(base, { event: { type: event.type, turn: d.turn, step: d.step,
327
+ retryId: d.retryId, retry: d.retry,
328
+ ...(event.type === 'llm/retry' ? { provider: d.provider, mode: d.mode,
329
+ maxRetries: d.maxRetries, delayMs: d.delayMs, failure: failureDetails(d.failure) } : {}),
330
+ } })
318
331
  case 'session/title':
319
332
  return Object.assign(base, {
320
333
  event: {
@@ -325,6 +338,16 @@ function buildWireEvent(session, event) {
325
338
  })
326
339
  case 'agent-preset/selected':
327
340
  return Object.assign(base, { event: { type: event.type, agentPreset: d.agentPreset } })
341
+ case 'deliverables/presented':
342
+ return Object.assign(base, {
343
+ event: { type: event.type, turn: d.turn, callId: d.callId,
344
+ files: (Array.isArray(d.files) ? d.files : [])
345
+ .filter(file => file && typeof file.path === 'string' && file.path.trim())
346
+ .map(file => ({ path: file.path,
347
+ ...(typeof file.description === 'string' ? { description: file.description } : {}),
348
+ })),
349
+ },
350
+ })
328
351
  case 'tool/call':
329
352
  return Object.assign(base, {
330
353
  event: { type: 'tool/call', turn: d.turn, step: d.step, callId: d.callId, name: d.name, arguments: d.arguments },
@@ -346,6 +369,7 @@ function buildWireEvent(session, event) {
346
369
  callId: d.message?.toolCallId ?? d.message?.source?.callId,
347
370
  isError: d.message?.isError === true || !!d.error || ((d.message && d.message.content) || []).some(block => block?.type === 'tool-result' && block.isError === true),
348
371
  preview,
372
+ ...toolResultDetails(d),
349
373
  ...(d.error ? { error: d.error } : {}),
350
374
  },
351
375
  })
@@ -404,7 +428,8 @@ function buildWireEvent(session, event) {
404
428
  case 'step/start':
405
429
  case 'step/end':
406
430
  return Object.assign(base, {
407
- event: { type: event.type, turn: d.turn, step: d.step, reason: d.reason && d.reason.kind },
431
+ event: { type: event.type, turn: d.turn, step: d.step, reason: d.reason && d.reason.kind,
432
+ ...(turnReasonDetails(d.reason) ? { reasonDetails: turnReasonDetails(d.reason) } : {}) },
408
433
  })
409
434
  default:
410
435
  return Object.assign(base, { event: { type: event.type } })
@@ -673,20 +698,31 @@ function createFileTransferManager(api, options) {
673
698
  if (!stat.isDirectory()) {
674
699
  return fileTransferError('file-workspace-unavailable', 'the session working directory is unavailable', requestType, sessionId)
675
700
  }
676
- return { root }
701
+ return { root, inputRoot: path.resolve(item.cwd) }
677
702
  } catch {
678
703
  return fileTransferError('file-workspace-unavailable', 'the session working directory is unavailable', requestType, sessionId)
679
704
  }
680
705
  }
681
706
 
682
707
  const resolveWorkspaceTarget = async (sessionId, rawPath, requestType, inputOptions) => {
683
- const requested = normalizedWorkspaceInput(rawPath, inputOptions)
708
+ // Only the resolver accepts native absolute paths from recorded deliveries
709
+ // or Markdown links. Download requests remain workspace-relative.
710
+ const absoluteInput = inputOptions?.allowAbsolute === true
711
+ && typeof rawPath === 'string' && path.isAbsolute(rawPath.trim())
712
+ const requested = absoluteInput
713
+ ? (rawPath.includes('\0') || rawPath.trim().split(/[\\/]+/).includes('..') ? null : rawPath.trim())
714
+ : normalizedWorkspaceInput(rawPath, inputOptions)
684
715
  if (!requested) {
685
- return fileTransferError('bad-request', `${requestType} requires a relative workspace path`, requestType, sessionId)
716
+ return fileTransferError('bad-request', `${requestType} requires a valid ${inputOptions?.allowAbsolute ? '' : 'relative '}workspace path`, requestType, sessionId)
686
717
  }
687
718
  const rootResult = await resolveSessionRoot(sessionId, requestType)
688
719
  if (rootResult.kind === 'error') return rootResult
689
- const candidate = path.resolve(rootResult.root, requested)
720
+ // macOS commonly exposes one workspace as both /var/... and /private/var/...
721
+ // (and cwd itself may be a symlink). Map only the known cwd alias before
722
+ // enforcing the canonical root, then still verify the target's realpath.
723
+ const candidate = absoluteInput && isInsideWorkspace(rootResult.inputRoot, requested)
724
+ ? path.resolve(rootResult.root, path.relative(rootResult.inputRoot, requested))
725
+ : path.resolve(rootResult.root, requested)
690
726
  if (!isInsideWorkspace(rootResult.root, candidate)) {
691
727
  return fileTransferError('file-not-allowed', 'path must stay inside the session working directory', requestType, sessionId)
692
728
  }
@@ -705,6 +741,28 @@ function createFileTransferManager(api, options) {
705
741
  }
706
742
  }
707
743
 
744
+ const resolveFile = async (msg) => {
745
+ const session = requireSessionId(msg)
746
+ if (session.error) return session.error
747
+ if (typeof msg.requestId !== 'string' || !msg.requestId.trim()) {
748
+ return fileTransferError('bad-request', 'file-resolve requires a requestId', 'file-resolve', session.value)
749
+ }
750
+ const resolved = await resolveWorkspaceTarget(session.value, msg.path, 'file-resolve', { allowAbsolute: true })
751
+ if (resolved.kind === 'error') return resolved
752
+ if (!resolved.stat.isFile()) {
753
+ return fileTransferError('file-not-regular', 'only regular files can be previewed', 'file-resolve', session.value)
754
+ }
755
+ if (resolved.stat.size > options.fileDownloadMaxBytes) {
756
+ return fileTransferError('file-too-large', 'file exceeds the configured download limit', 'file-resolve', session.value)
757
+ }
758
+ return {
759
+ kind: 'file-resolved', requestId: msg.requestId, sessionId: session.value,
760
+ path: workspaceRelativePath(resolved.root, resolved.target),
761
+ name: path.basename(resolved.target), mediaType: fileMediaType(resolved.target),
762
+ size: resolved.stat.size, modifiedAt: resolved.stat.mtimeMs,
763
+ }
764
+ }
765
+
708
766
  const fileList = async (msg) => {
709
767
  const session = requireSessionId(msg)
710
768
  if (session.error) return session.error
@@ -884,9 +942,11 @@ function createFileTransferManager(api, options) {
884
942
 
885
943
  return {
886
944
  async handle(client, msg) {
945
+ const correlation = typeof msg.requestId === 'string' ? { requestId: msg.requestId } : {}
887
946
  if (!options.fileDownloadsEnabled) {
888
- return fileTransferError('file-download-disabled', 'file downloads are disabled by gateway configuration', msg.type)
947
+ return { ...fileTransferError('file-download-disabled', 'file downloads are disabled by gateway configuration', msg.type), ...correlation }
889
948
  }
949
+ if (msg.type === 'file-resolve') return { ...await resolveFile(msg), ...correlation }
890
950
  if (msg.type === 'file-list') return fileList(msg)
891
951
  if (msg.type === 'file-download-open') return open(client, msg)
892
952
  if (msg.type === 'file-download-read') return read(client, msg)
@@ -1077,12 +1137,14 @@ function trimConversationEvent(event) {
1077
1137
  case 'assistant/message':
1078
1138
  case 'assistant/attempt': {
1079
1139
  const { stream, ...data } = event.data || {}
1080
- return { ...event, data }
1140
+ const finish = assistantFinish(event.data || {})
1141
+ return { ...event, data: { ...data, ...(finish ? { finish } : {}) } }
1081
1142
  }
1082
1143
  case 'tool/result': {
1083
1144
  const d = event.data || {}
1084
1145
  const message = d.message
1085
- if (!message || !Array.isArray(message.content)) return event
1146
+ const details = toolResultDetails(d)
1147
+ if (!message || !Array.isArray(message.content)) return { ...event, data: { ...d, ...details } }
1086
1148
  let changed = false
1087
1149
  const truncateBlock = (block) => {
1088
1150
  if (!block || typeof block !== 'object') return block
@@ -1096,8 +1158,7 @@ function trimConversationEvent(event) {
1096
1158
  return block
1097
1159
  }
1098
1160
  const content = message.content.map(truncateBlock)
1099
- if (!changed) return event
1100
- return { ...event, data: { ...d, message: { ...message, content } } }
1161
+ return { ...event, data: { ...d, ...details, ...(changed ? { message: { ...message, content } } : {}) } }
1101
1162
  }
1102
1163
  default:
1103
1164
  return event
@@ -2212,6 +2273,7 @@ const plugin = {
2212
2273
  const pendingQuestions = new Map()
2213
2274
  const pendingApprovals = new Map()
2214
2275
  const fileTransfers = createFileTransferManager(api, options)
2276
+ const speechInput = createSpeechInputManager(api, options)
2215
2277
  const fileTransferExpiryTimer = setInterval(() => {
2216
2278
  fileTransfers.expire().catch((error) => {
2217
2279
  log(`file download expiry failed: ${error && error.message ? error.message : String(error)}`)
@@ -2907,6 +2969,14 @@ const plugin = {
2907
2969
 
2908
2970
  if (msg.type === 'ping') {
2909
2971
  ws.send(stringifyWireFrame({ kind: 'pong', at: Date.now() }))
2972
+ } else if (SPEECH_REQUEST_TYPES.has(msg.type)) {
2973
+ if (!ws.deviceId) {
2974
+ sendFrame({ kind: 'error', code: 'authentication-required', requestType: msg.type,
2975
+ ...(typeof msg.requestId === 'string' ? { requestId: msg.requestId } : {}),
2976
+ message: 'Speech input requires a paired device' })
2977
+ return
2978
+ }
2979
+ void speechInput.handle(ws, msg, sendFrame).then(sendFrame)
2910
2980
  } else if (msg.type === 'subscribe') {
2911
2981
  if (msg.assistantStream !== undefined && typeof msg.assistantStream !== 'boolean') {
2912
2982
  sendFrame({ kind: 'error', code: 'bad-request', requestType: 'subscribe', message: 'assistantStream must be a boolean' })
@@ -2942,7 +3012,7 @@ const plugin = {
2942
3012
  respondToApproval(msg).then((frame) => {
2943
3013
  if (frame && ws.readyState === 1) ws.send(stringifyWireFrame(frame))
2944
3014
  })
2945
- } else if (msg.type === 'file-list' || msg.type === 'file-download-open' ||
3015
+ } else if (msg.type === 'file-list' || msg.type === 'file-resolve' || msg.type === 'file-download-open' ||
2946
3016
  msg.type === 'file-download-read' || msg.type === 'file-download-cancel') {
2947
3017
  fileTransfers.handle(ws, msg).then((frame) => {
2948
3018
  if (frame && ws.readyState === 1) ws.send(stringifyWireFrame(frame))
@@ -2989,6 +3059,7 @@ const plugin = {
2989
3059
  })
2990
3060
 
2991
3061
  ws.on('close', () => {
3062
+ speechInput.closeClient(ws)
2992
3063
  ws.sessionFollower.stop()
2993
3064
  clients.delete(ws)
2994
3065
  if (ws.deviceId) registry.disconnected(ws.deviceId)
@@ -3027,11 +3098,12 @@ const plugin = {
3027
3098
  'goals',
3028
3099
  'schedule-management',
3029
3100
  'plugin-management-v1',
3101
+ ...(device ? ['speech-input-v1'] : []),
3030
3102
  'session-cancel',
3031
3103
  'queue-control',
3032
3104
  'session-archive',
3033
3105
  'session-rename',
3034
- ...(options.fileDownloadsEnabled ? ['file-downloads'] : []),
3106
+ ...(options.fileDownloadsEnabled ? ['file-downloads', 'file-references-v1'] : []),
3035
3107
  ],
3036
3108
  port: transport.port || webServer.port,
3037
3109
  clients: clients.size,
@@ -3201,7 +3273,7 @@ const plugin = {
3201
3273
 
3202
3274
  const disposeApprovals = ctx.on('approval/request', (request, next) => {
3203
3275
  const sessionId = request?.agent?.id === undefined ? null : String(request.agent.id)
3204
- if (!sessionId || typeof request.toolName !== 'string' || !hasInteractionClient(sessionId)) return next()
3276
+ if (!sessionId || typeof request.toolName !== 'string' || !gatewayEnabled) return next()
3205
3277
  const rpcId = crypto.randomUUID()
3206
3278
  const approvalId = crypto.randomUUID()
3207
3279
  return new Promise((resolve, reject) => {
@@ -3231,7 +3303,10 @@ const plugin = {
3231
3303
  }
3232
3304
  pendingApprovals.set(rpcId, pending)
3233
3305
  broadcastInteractionFrame(approvalFrameFor(rpcId, pending))
3306
+ // 控制连接重连期间仍登记待审批,桌面端继续处理同一个请求。
3307
+ // 若桌面端仍在等待,后续连接可以重放;无可用答复时保持原有回退行为。
3234
3308
  startDownstreamApproval(pending)
3309
+ if (!hasInteractionClient(sessionId)) fallbackApproval(pending)
3235
3310
  log(`approval requested: rpcId=${rpcId} approvalId=${approvalId} session=${sessionId} tool=${request.toolName}`)
3236
3311
  })
3237
3312
  }, { prepend: true })
@@ -3396,6 +3471,7 @@ const plugin = {
3396
3471
  if (waitTimer) clearTimeout(waitTimer)
3397
3472
  clearInterval(fileTransferExpiryTimer)
3398
3473
  backgroundStreamAbort.abort()
3474
+ speechInput.dispose()
3399
3475
  for (const pending of [...pendingQuestions.values()]) fallbackQuestion(pending)
3400
3476
  for (const pending of [...pendingApprovals.values()]) fallbackApproval(pending)
3401
3477
  fileTransfers.dispose().catch((error) => {
@@ -0,0 +1,53 @@
1
+ // Keep diagnostic facts when token replay and tool bodies are trimmed for mobile.
2
+ const TOOL_OUTPUT_LIMIT = 16_000
3
+
4
+ export function failureDetails(value) {
5
+ if (!value || typeof value !== 'object') return undefined
6
+ const result = {}
7
+ for (const key of ['message', 'code', 'name', 'reason', 'requestId']) {
8
+ if (typeof value[key] === 'string') result[key] = value[key]
9
+ }
10
+ for (const key of ['status', 'providerRetryAfterMs']) {
11
+ if (Number.isFinite(value[key])) result[key] = value[key]
12
+ }
13
+ return Object.keys(result).length ? result : undefined
14
+ }
15
+
16
+ export function assistantFinish(data) {
17
+ for (let i = (data.stream?.length || 0) - 1; i >= 0; i--) {
18
+ const record = data.stream[i]
19
+ if (record?.type !== 'chunk' || record.chunk?.type !== 'finish') continue
20
+ const reason = record.chunk.reason
21
+ if (typeof reason?.kind !== 'string') return undefined
22
+ const failure = failureDetails(reason.failure ?? reason.error)
23
+ return { kind: reason.kind, ...(failure ? { failure } : {}) }
24
+ }
25
+ return data.finish
26
+ }
27
+
28
+ export function toolResultDetails(data) {
29
+ const texts = []
30
+ const visit = block => {
31
+ if (block?.type === 'text' && typeof block.text === 'string') texts.push(block.text)
32
+ else if (block?.type === 'tool-result' && Array.isArray(block.content)) block.content.forEach(visit)
33
+ }
34
+ for (const block of data.message?.content || []) visit(block)
35
+ const full = texts.join('')
36
+ const outputTruncated = full.length > TOOL_OUTPUT_LIMIT
37
+ // Retain the terminal status and sandbox hints at the end as well as the start.
38
+ const output = outputTruncated
39
+ ? `${full.slice(0, TOOL_OUTPUT_LIMIT / 2)}\n[…]\n${full.slice(-TOOL_OUTPUT_LIMIT / 2)}`
40
+ : full
41
+ return { output, outputTruncated }
42
+ }
43
+
44
+ export function turnReasonDetails(reason) {
45
+ if (!reason || typeof reason.kind !== 'string') return undefined
46
+ const error = failureDetails(reason.error ?? reason.failure)
47
+ return { kind: reason.kind, ...(error ? { error } : {}),
48
+ ...(typeof reason.reason?.kind === 'string' ? { reason: {
49
+ kind: reason.reason.kind,
50
+ ...(typeof reason.reason.reason === 'string' ? { reason: reason.reason.reason } : {}),
51
+ } } : {}),
52
+ }
53
+ }
@@ -0,0 +1,192 @@
1
+ // 移动连接只代理官方 speech Remote,不持有识别模型或写入会话历史。
2
+ export const SPEECH_REQUEST_TYPES = new Set([
3
+ 'speech-catalog', 'speech-subscribe', 'speech-unsubscribe', 'speech-configure',
4
+ 'speech-prepare', 'speech-cancel-preparation', 'speech-transcribe', 'speech-cancel',
5
+ ])
6
+
7
+ const MAX_AUDIO_BYTES = 4 * 1024 * 1024
8
+ const UNAVAILABLE_CODES = new Set(['gateway/invocation-unavailable', 'gateway/definition-unavailable', 'gateway/service-unavailable'])
9
+
10
+ function failure(code, message) {
11
+ return Object.assign(new Error(message), { code })
12
+ }
13
+
14
+ function identifier(value, name) {
15
+ if (typeof value !== 'string' || !value.trim() || value.length > 128) {
16
+ throw failure('bad-request', `${name} must be a nonempty string of at most 128 characters`)
17
+ }
18
+ return value.trim()
19
+ }
20
+
21
+ function errorFrame(msg, error) {
22
+ return { kind: 'error', requestType: msg.type,
23
+ ...(typeof msg.requestId === 'string' ? { requestId: msg.requestId } : {}),
24
+ code: UNAVAILABLE_CODES.has(error?.code) ? 'speech/unavailable' : error?.code || 'speech/failed',
25
+ message: UNAVAILABLE_CODES.has(error?.code)
26
+ ? 'Enable the official voice input bundle in DSH plugin management' : error?.message || 'Speech operation failed' }
27
+ }
28
+
29
+ export function createSpeechInputManager(api, { maxPayloadBytes = 64 * 1024 * 1024,
30
+ transcriptionTimeoutMs = 180_000 } = {}) {
31
+ const clients = new Map()
32
+ let activeTranscriptions = 0
33
+ let disposed = false
34
+ // 为 JSON 信封留出空间;实际限制还须服从官方插件的目录。
35
+ const audioLimit = Math.min(MAX_AUDIO_BYTES, Math.max(0, Math.floor((maxPayloadBytes - 4096) / 4) * 3))
36
+ const stateFor = client => {
37
+ if (disposed) throw failure('speech/unavailable', 'Speech gateway has been disposed')
38
+ let state = clients.get(client)
39
+ if (!state) {
40
+ state = { operations: new Map(), follow: null, closed: false }
41
+ clients.set(client, state)
42
+ }
43
+ return state
44
+ }
45
+ const normalizeCatalog = value => {
46
+ if (!value || !Array.isArray(value.providers) || !value.selection
47
+ || !Number.isSafeInteger(value.maxAudioBytes) || value.maxAudioBytes < 46
48
+ || !Number.isFinite(value.maxDurationSeconds) || value.maxDurationSeconds < 1) {
49
+ throw failure('speech/invalid-response', 'Official speech catalog is invalid')
50
+ }
51
+ return { ...value, maxAudioBytes: Math.min(audioLimit, value.maxAudioBytes),
52
+ maxDurationSeconds: value.maxDurationSeconds, available: true }
53
+ }
54
+ const stopFollow = state => {
55
+ state.follow?.abort.abort()
56
+ state.follow = null
57
+ }
58
+ const catalog = async signal => normalizeCatalog(await api.speech.catalog(signal))
59
+
60
+ return {
61
+ async handle(client, msg, send) {
62
+ try {
63
+ const requestId = identifier(msg.requestId, 'requestId')
64
+ const state = stateFor(client)
65
+ if (msg.type === 'speech-cancel') {
66
+ const targetRequestId = identifier(msg.targetRequestId, 'targetRequestId')
67
+ const task = state.operations.get(targetRequestId)
68
+ if (!task || task.type !== 'speech-transcribe') throw failure('speech/request-not-found', 'Transcription is not active on this connection')
69
+ task.abort.abort(failure('speech/cancelled', 'Transcription cancelled'))
70
+ return { kind: 'speech-cancelled', requestId, targetRequestId }
71
+ }
72
+ if (msg.type === 'speech-unsubscribe') {
73
+ stopFollow(state)
74
+ return { kind: 'speech-unsubscribed', requestId }
75
+ }
76
+ if (state.operations.has(requestId) || state.follow?.requestId === requestId) {
77
+ throw failure('speech/request-in-progress', 'requestId is already active on this connection')
78
+ }
79
+ if (state.operations.size >= 8) throw failure('speech/busy', 'Too many pending speech requests')
80
+ if (msg.type === 'speech-subscribe') {
81
+ stopFollow(state)
82
+ const follow = { abort: new AbortController(), requestId }
83
+ state.follow = follow
84
+ // 先回执,再按订阅 requestId 推送完整快照。Host 的准备任务不属于此观察连接。
85
+ queueMicrotask(async () => {
86
+ try {
87
+ const stream = await api.speech.follow(follow.abort.signal)
88
+ for await (const snapshot of stream) {
89
+ if (state.closed || state.follow !== follow || follow.abort.signal.aborted) break
90
+ send({ kind: 'speech-state', requestId, ...normalizeCatalog(snapshot) })
91
+ }
92
+ if (!follow.abort.signal.aborted) throw failure('speech/stream-ended', 'Speech readiness stream ended; subscribe again')
93
+ } catch (error) {
94
+ if (!state.closed && state.follow === follow && !follow.abort.signal.aborted) send(errorFrame(msg, error))
95
+ } finally {
96
+ if (state.follow === follow) state.follow = null
97
+ }
98
+ })
99
+ return { kind: 'speech-subscribed', requestId }
100
+ }
101
+ const abort = new AbortController()
102
+ const task = { abort, type: msg.type }
103
+ let timer
104
+ let admitted = false
105
+ state.operations.set(requestId, task)
106
+ try {
107
+ if (msg.type === 'speech-catalog') {
108
+ try { return { kind: 'speech-catalog', requestId, ...await catalog(abort.signal) } }
109
+ catch (error) {
110
+ if (UNAVAILABLE_CODES.has(error?.code)) return { kind: 'speech-catalog', requestId,
111
+ available: false, providers: [], code: 'speech/unavailable',
112
+ message: 'Enable the official voice input bundle in DSH plugin management' }
113
+ throw error
114
+ }
115
+ }
116
+ if (msg.type === 'speech-transcribe') {
117
+ if ([...state.operations.values()].some(value => value !== task && value.type === msg.type)
118
+ || activeTranscriptions >= 4) throw failure('speech/busy', 'Too many active transcriptions')
119
+ const encoded = msg.audioBase64
120
+ if (typeof encoded !== 'string' || !encoded.length || encoded.length > Math.ceil(audioLimit / 3) * 4) {
121
+ throw failure('speech/invalid-audio', 'Audio is empty or exceeds the gateway byte limit')
122
+ }
123
+ const audio = Buffer.from(encoded, 'base64')
124
+ if (audio.length > audioLimit || audio.toString('base64') !== encoded) {
125
+ throw failure('speech/invalid-audio', 'Audio must use canonical base64 within the gateway byte limit')
126
+ }
127
+ const request = { audioBase64: encoded,
128
+ ...(msg.providerId === undefined ? {} : { providerId: identifier(msg.providerId, 'providerId') }),
129
+ ...(msg.language === undefined ? {} : { language: identifier(msg.language, 'language') }) }
130
+ activeTranscriptions++
131
+ admitted = true
132
+ timer = setTimeout(() => abort.abort(failure('speech/timeout', 'Transcription timed out')), transcriptionTimeoutMs)
133
+ timer.unref?.()
134
+ const limits = await catalog(abort.signal)
135
+ abort.signal.throwIfAborted()
136
+ if (audio.length > limits.maxAudioBytes) throw failure('speech/invalid-audio', 'Audio exceeds the official plugin byte limit')
137
+ // 官方 Remote 负责 WAV、采样率、语言、模型就绪与录音时长校验。
138
+ const result = await api.speech.transcribe(request, abort.signal)
139
+ abort.signal.throwIfAborted()
140
+ if (typeof result?.text !== 'string' || !Number.isFinite(result.audioSeconds) || result.audioSeconds < 0
141
+ || !Number.isFinite(result.inferenceSeconds) || result.inferenceSeconds < 0) {
142
+ throw failure('speech/invalid-response', 'Official speech transcript is invalid')
143
+ }
144
+ return { kind: 'speech-transcript', requestId, text: result.text,
145
+ audioSeconds: result.audioSeconds, inferenceSeconds: result.inferenceSeconds }
146
+ }
147
+ if (msg.type === 'speech-configure') {
148
+ const patch = {
149
+ ...(msg.providerId === undefined ? {} : { providerId: identifier(msg.providerId, 'providerId') }),
150
+ ...(msg.language === undefined ? {} : { language: identifier(msg.language, 'language') }),
151
+ }
152
+ if (!Object.keys(patch).length) throw failure('bad-request', 'Provide providerId or language')
153
+ await api.speech.configure(patch, abort.signal)
154
+ return { kind: 'speech-configured', requestId }
155
+ }
156
+ const providerId = identifier(msg.providerId, 'providerId')
157
+ if (msg.type === 'speech-prepare') {
158
+ const options = msg.downloadSource === undefined ? undefined : { downloadSource: msg.downloadSource }
159
+ if (options && (typeof options.downloadSource !== 'string' || !options.downloadSource.trim() || options.downloadSource.length > 2048)) {
160
+ throw failure('bad-request', 'downloadSource must be an advertised nonempty origin')
161
+ }
162
+ await api.speech.prepare(providerId, options, abort.signal)
163
+ return { kind: 'speech-preparing', requestId, providerId }
164
+ }
165
+ if (msg.type === 'speech-cancel-preparation') {
166
+ await api.speech.cancelPreparation(providerId, abort.signal)
167
+ return { kind: 'speech-preparation-cancelled', requestId, providerId }
168
+ }
169
+ throw failure('bad-request', 'Unknown speech request')
170
+ } catch (error) {
171
+ throw abort.signal.aborted ? abort.signal.reason : error
172
+ } finally {
173
+ clearTimeout(timer)
174
+ state.operations.delete(requestId)
175
+ if (admitted) activeTranscriptions--
176
+ }
177
+ } catch (error) { return errorFrame(msg, error) }
178
+ },
179
+ closeClient(client) {
180
+ const state = clients.get(client)
181
+ if (!state) return
182
+ state.closed = true
183
+ stopFollow(state)
184
+ for (const task of state.operations.values()) task.abort.abort(failure('speech/cancelled', 'Mobile connection closed'))
185
+ clients.delete(client)
186
+ },
187
+ dispose() {
188
+ disposed = true
189
+ for (const client of clients.keys()) this.closeClient(client)
190
+ },
191
+ }
192
+ }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "dsh-plugin-mobile-gateway",
3
- "version": "0.10.0",
4
- "description": "Mobile gateway for DeepSeek Harness with secure device pairing, realtime session sync, plugin management, LAN and Cloudflare Tunnel access, and a responsive theme-aware control panel",
3
+ "version": "0.10.2",
4
+ "description": "DeepSeek Harness 移动网关,支持安全设备配对、实时会话同步、插件管理、官方语音输入、局域网及 Cloudflare Tunnel,以及自适应深浅色控制面板。0.10.2 新增文件卡片事件透传与安全文件引用解析,复用分块下载为移动端文件卡片和正文文件链接预览提供协议支持。",
5
5
  "main": "lib/index.mjs",
6
6
  "files": [
7
7
  "bin",
@@ -26,7 +26,7 @@
26
26
  },
27
27
  "bin": "bin/setup-ip.mjs",
28
28
  "scripts": {
29
- "test": "node test/wire-json.test.mjs && node test/setup-ip.test.mjs && node test/host-adapter.test.mjs && node test/plugin-management.test.mjs && node test/session-preset.test.mjs && node test/session-follower.test.mjs && node test/gateway.test.mjs && node test/auth.test.mjs && node test/lan.test.mjs && node test/cloudflared-binary.test.mjs && node test/cloudflare-tunnel.test.mjs && node --test test/public-endpoint-check.test.mjs && node test/gateway-tools.test.mjs && node test/multi-gateway.test.mjs"
29
+ "test": "node test/wire-json.test.mjs && node test/setup-ip.test.mjs && node test/host-adapter.test.mjs && node test/speech-input.test.mjs && node test/plugin-management.test.mjs && node test/session-preset.test.mjs && node test/session-follower.test.mjs && node test/session-event-details.test.mjs && node test/gateway.test.mjs && node --test test/approval-replay.test.mjs && node test/auth.test.mjs && node test/lan.test.mjs && node test/cloudflared-binary.test.mjs && node test/cloudflare-tunnel.test.mjs && node --test test/public-endpoint-check.test.mjs && node test/gateway-tools.test.mjs && node test/multi-gateway.test.mjs"
30
30
  },
31
31
  "exports": {
32
32
  ".": "./lib/index.mjs",