@ljwei-stak/dsh-model-route-bot 0.66.3 → 0.67.0

package/apps/integration-client/desktop-pane.jsx

@@ -4,11 +4,14 @@ import { connectDesktopStream } from './desktop-stream.mjs'
 import { createAcknowledgedRoutineRecorder } from './routine-recorder.mjs'
 import { RoutineCaptureControls } from './routines-panel.jsx'
 import { UiIcon } from './ui-icons.jsx'
+import { DesktopTools } from './desktop-tools.jsx'
+import { sendDesktopKeys, sendDesktopText } from './desktop-keyboard.mjs'
+import { createDesktopTouchTracker } from './desktop-touch.mjs'
 
 const rootOf = machine => typeof machine?.roots?.[0] === 'string' ? machine.roots[0] : machine?.rootId ?? machine?.roots?.[0]?.id ?? ''
 const STATUS = { idle: '未连接', connecting: '连接中', authenticating: '验证中', connected: '已连接', releasing: '归还控制中', disconnected: '已断开', error: '连接异常' }
 
-export function DesktopViewer({ botId, machines, openSession, renewSession, closeSession, saveRoutine }) {
+export function DesktopViewer({ botId, machines, openSession, renewSession, closeSession, saveRoutine, transport }) {
   const [machineId, setMachineId] = useState(machines[0]?.machineId ?? machines[0]?.id ?? '')
   const [displayId, setDisplayId] = useState('')
   const [displays, setDisplays] = useState([])
@@ -16,31 +19,44 @@ export function DesktopViewer({ botId, machines, openSession, renewSession, clos
   const [expanded, setExpanded] = useState(false)
   const [stats, setStats] = useState({ fps: 0, width: 0, height: 0 })
   const [text, setText] = useState('')
+  const [toolsOpen,setToolsOpen]=useState(false),[keyboardBusy,setKeyboardBusy]=useState(false),[keyboardError,setKeyboardError]=useState('')
+  const [fit,setFit]=useState(true),[fullscreen,setFullscreen]=useState(false)
   const [metadata, setMetadata] = useState({})
   const recorderRef = useRef(null)
   if (!recorderRef.current) recorderRef.current = createAcknowledgedRoutineRecorder()
   const recorder = recorderRef.current
   const canvasRef = useRef(null), largeRef = useRef(null), streamRef = useRef(null), ticketRef = useRef(null)
+  const pendingStop=useRef(null),stopAttempt=useRef(null)
+  const connecting=useRef(false)
   const generation = useRef(0), pendingFrame = useRef(null), drawing = useRef(false)
   const surface = useRef(null)
   const size = useRef({ width: 0, height: 0 }), rate = useRef({ at: performance.now(), count: 0 })
   const renewTimer = useRef(null), moveTimer = useRef(null), pendingMove = useRef(null)
   const [canControl, setCanControl] = useState(false)
+  const touches=useRef(null);if(!touches.current)touches.current=createDesktopTouchTracker()
+  const viewerRef=useRef(null),dialogRef=useRef(null),expandButton=useRef(null),toolsButton=useRef(null),keyboardPending=useRef(false),stateRef=useRef(state)
+  stateRef.current=state
   const closeRef = useRef(closeSession)
   closeRef.current = closeSession
   useEffect(() => { if (!machineId && machines[0]) setMachineId(machines[0].machineId ?? machines[0].id) }, [machines, machineId])
-  const release = useCallback(() => { pendingMove.current = null; void streamRef.current?.releaseInput()?.catch(() => {}) }, [])
+  const release = useCallback((options={}) => {if(!options.keepTouches)touches.current.clear(); pendingMove.current = null; void streamRef.current?.releaseInput()?.catch(() => {}) }, [])
   const stop = useCallback(async () => {
     recorder.invalidate('ROUTINE_DESKTOP_DISCONNECTED')
     generation.current++; clearTimeout(renewTimer.current); renewTimer.current = null
+    setKeyboardBusy(false);keyboardPending.current=false
     if (moveTimer.current !== null) cancelAnimationFrame(moveTimer.current)
     moveTimer.current = null; pendingMove.current = null; pendingFrame.current = null
+    touches.current.clear()
     size.current = { width: 0, height: 0 }
     streamRef.current?.close(); streamRef.current = null
     const ticket = ticketRef.current; ticketRef.current = null
-    if (ticket) {
-      try { return await closeRef.current(botId, ticket.sessionId) }
-      catch { return { stopConfirmed: false } }
+    if(ticket)pendingStop.current={botId,sessionId:ticket.sessionId,close:closeRef.current}
+    if(pendingStop.current){
+      if(stopAttempt.current)return stopAttempt.current
+      const target=pendingStop.current
+      const attempt=(async()=>{try{const result=await target.close(target.botId,target.sessionId);if(result?.stopConfirmed===true&&pendingStop.current===target)pendingStop.current=null;return result}catch{return {stopConfirmed:false}}})()
+      stopAttempt.current=attempt
+      try{return await attempt}finally{if(stopAttempt.current===attempt)stopAttempt.current=null}
     }
     return { stopConfirmed: true }
   }, [botId])
@@ -88,18 +104,25 @@ export function DesktopViewer({ botId, machines, openSession, renewSession, clos
     finally { drawing.current = false }
   }, [])
   const connect = useCallback(async () => {
-    await stop()
+    if(connecting.current)return
+    connecting.current=true
+    try{
+    const stopped=await stop()
+    if(stopped?.stopConfirmed!==true){setState({status:'error',mode:'watch',error:'上一桌面尚未确认释放输入。请点击“断开”核对原会话，再连接电脑。'});return}
     const active = generation.current
     setState({ status: 'connecting', mode: 'watch' }); setStats({ fps: 0, width: 0, height: 0 })
     try {
       const ticket = await openSession(botId, machineId, { ...(displayId ? { displayId } : {}), mode: 'watch', fps: 15 })
-      if (active !== generation.current) { await closeSession(botId, ticket.sessionId); return }
+      if (active !== generation.current) {pendingStop.current={botId,sessionId:ticket.sessionId,close:closeSession};const closed=await stop();if(closed?.stopConfirmed!==true)setState({status:'error',mode:'watch',error:'已取消连接，远端释放尚未确认，请核对原会话。'});return }
       ticketRef.current = ticket; setCanControl(ticket.canControl === true)
       if (ticket.displays?.length) setDisplays(ticket.displays)
-      streamRef.current = connectDesktopStream({ ticket, onFrame: draw,
+      streamRef.current = connectDesktopStream({ ticket, botId, transport, onFrame: draw,
         onBeforeInput: (event, meta) => recorder.inputSent(event, meta),
         onInputAcknowledged: (event, meta) => recorder.inputAcknowledged(event, meta),
         onInputFailure: (event, meta) => recorder.inputFailed(event, meta),
+        onBeforeRelease: meta=>recorder.releaseSent(meta),
+        onReleaseAcknowledged: meta=>recorder.releaseAcknowledged(meta),
+        onReleaseFailure: meta=>recorder.releaseFailed(meta),
         onMetadata: value => {
           if (active !== generation.current) return
           if (value.displays) setDisplays(value.displays)
@@ -121,13 +144,34 @@ export function DesktopViewer({ botId, machines, openSession, renewSession, clos
       }
       renew()
     } catch (failure) { if (active === generation.current) setState({ status: 'error', mode: 'watch', error: String(failure?.message ?? failure) }) }
-  }, [stop, botId, machineId, displayId, openSession, closeSession, renewSession, draw])
+    }finally{connecting.current=false}
+  }, [stop, botId, machineId, displayId, openSession, closeSession, renewSession, draw, transport])
   useEffect(() => { return () => { void stop() } }, [stop])
+  useEffect(()=>{const update=()=>setFullscreen(Boolean(document.fullscreenElement&&viewerRef.current?.contains(document.fullscreenElement)));document.addEventListener('fullscreenchange',update);return()=>document.removeEventListener('fullscreenchange',update)},[])
+  useEffect(()=>{
+    if(!expanded)return
+    if(surface.current&&largeRef.current){largeRef.current.width=surface.current.width;largeRef.current.height=surface.current.height;largeRef.current.getContext('2d')?.drawImage(surface.current,0,0)}
+    const previous=document.activeElement,overflow=document.body.style.overflow
+    document.body.style.overflow='hidden';dialogRef.current?.querySelector('button')?.focus()
+    return()=>{if(document.body.style.overflow==='hidden')document.body.style.overflow=overflow;if(previous?.isConnected)previous.focus()}
+  },[expanded])
   useEffect(() => {
     const lost = () => release()
     window.addEventListener('blur', lost); document.addEventListener('visibilitychange', lost)
     return () => { window.removeEventListener('blur', lost); document.removeEventListener('visibilitychange', lost) }
   }, [release])
+  useEffect(()=>{
+    const wheel=event=>{
+      if(stateRef.current.mode!=='control')return
+      event.preventDefault()
+      const scale=event.deltaMode===1?16:event.deltaMode===2?window.innerHeight:1
+      const deltaX=Math.max(-4800,Math.min(4800,Math.round(event.deltaX*scale))),deltaY=Math.max(-4800,Math.min(4800,Math.round(event.deltaY*scale)))
+      if(deltaX||deltaY)streamRef.current?.input({type:'wheel',deltaX,deltaY})
+    }
+    const canvases=[canvasRef.current,largeRef.current].filter(Boolean)
+    for(const canvas of canvases)canvas.addEventListener('wheel',wheel,{passive:false})
+    return()=>{for(const canvas of canvases)canvas.removeEventListener('wheel',wheel)}
+  },[expanded])
   const point = event => {
     const rect = event.currentTarget.getBoundingClientRect()
     try { return framePoint({ clientX: event.clientX - rect.left, clientY: event.clientY - rect.top, rect, frame: size.current }) }
@@ -135,53 +179,78 @@ export function DesktopViewer({ botId, machines, openSession, renewSession, clos
   }
   const pointer = (event, action) => {
     if (state.mode !== 'control') return
-    const position = point(event); if (!position) return
+    if(action==='down'&&event.pointerType==='touch')try{event.currentTarget.setPointerCapture?.(event.pointerId)}catch{}
+    const gesture=touches.current.handle(action,event)
+    if(gesture.handled){event.preventDefault();if(gesture.release)release({keepTouches:true});if(gesture.wheel)streamRef.current?.input(gesture.wheel);return}
+    const position = point(event); if (!position) {if(action==='up'){event.preventDefault();release()};return}
     event.preventDefault()
     const packet = { type: 'pointer', action, ...position, button: ['left', 'middle', 'right'][event.button] ?? 'left' }
     if (action === 'move') {
       pendingMove.current = packet
       if (moveTimer.current === null) moveTimer.current = requestAnimationFrame(() => { moveTimer.current = null; if (pendingMove.current) streamRef.current?.input(pendingMove.current); pendingMove.current = null })
     } else {
-      if (action === 'down') { event.currentTarget.focus({ preventScroll: true }); event.currentTarget.setPointerCapture?.(event.pointerId) }
+      if(moveTimer.current!==null){cancelAnimationFrame(moveTimer.current);moveTimer.current=null;if(pendingMove.current)streamRef.current?.input(pendingMove.current);pendingMove.current=null}
+      if (action === 'down') { event.currentTarget.focus({ preventScroll: true });try{event.currentTarget.setPointerCapture?.(event.pointerId)}catch{} }
       streamRef.current?.input(packet)
     }
   }
   const keyboard = (event, action) => {
-    if (state.mode !== 'control' || event.isComposing) return
+    if (state.mode !== 'control' || event.isComposing || event.nativeEvent?.isComposing || event.keyCode===229) return
+    if((event.ctrlKey||event.metaKey)&&event.key.toLowerCase()==='v'){event.stopPropagation();return}
     event.preventDefault(); event.stopPropagation()
     const input = desktopKeyboardInput(event, action)
     if (input) streamRef.current?.input(input)
   }
-  const screen = (ref, big = false) => <canvas ref={ref} tabIndex={0} aria-label="远程桌面，取得控制权后可用鼠标键盘操作" className={big ? 'bp-stream-canvas bp-stream-large' : 'bp-stream-canvas'}
+  const keyboardAction=async work=>{
+    if(keyboardPending.current||stateRef.current.status!=='connected'||stateRef.current.mode!=='control')return
+    const stream=streamRef.current,active=generation.current;keyboardPending.current=true;setKeyboardBusy(true);setKeyboardError('')
+    const current=()=>active===generation.current&&stream===streamRef.current&&stateRef.current.mode==='control'
+    try{await work(stream,current)}catch(failure){if(active===generation.current)setKeyboardError(`远端尚未确认这次输入：${failure.message}。请检查电脑，不会自动重发。`)}
+    finally{if(active===generation.current){keyboardPending.current=false;setKeyboardBusy(false)}}
+  }
+  const submitText=()=>{const value=text;void keyboardAction(async(stream,current)=>{await sendDesktopText(stream,value,current);if(current())setText(before=>before===value?'':before)})}
+  const localPaste=async()=>{const active=generation.current;try{if(!navigator.clipboard?.readText)throw new Error('请直接粘贴到上方文本框。');const value=await navigator.clipboard.readText();if(active!==generation.current)return;if(value.length>8192)throw new Error('本机文字超过 8192 字，请分段粘贴。');setText(value);setKeyboardError('')}catch(failure){if(active===generation.current)setKeyboardError(`无法读取本机剪贴板：${failure.message}`)}}
+  const closeTools=()=>{setToolsOpen(false);toolsButton.current?.focus()}
+  const toggleFullscreen=async()=>{try{if(document.fullscreenElement)await document.exitFullscreen();else await (dialogRef.current??viewerRef.current).requestFullscreen()}catch{setKeyboardError('无法进入系统全屏，请使用放大视图。');setToolsOpen(true)}}
+  const tools=<DesktopTools open={toolsOpen} onClose={closeTools} connected={state.status==='connected'} canInput={state.mode==='control'} text={text} onText={setText} onSend={submitText} onPaste={()=>{void localPaste()}}
+    onShortcut={keys=>{void keyboardAction((stream,current)=>sendDesktopKeys(stream,keys,current))}} working={keyboardBusy} error={keyboardError}/>
+  const screen = (ref, big = false) => <canvas ref={ref} tabIndex={0} aria-label="远程桌面，取得控制权后可用鼠标键盘操作" data-control={state.mode==='control'} className={big ? 'bp-stream-canvas bp-stream-large' : 'bp-stream-canvas'}
+    style={!fit&&size.current.width?{width:size.current.width,maxWidth:'none',maxHeight:'none'}:undefined}
     onPointerMove={event => pointer(event, 'move')} onPointerDown={event => pointer(event, 'down')} onPointerUp={event => pointer(event, 'up')}
     onPointerCancel={release} onLostPointerCapture={event => { if (event.buttons) release() }} onBlur={release} onContextMenu={event => event.preventDefault()}
+    onPointerLeave={event=>{if(event.buttons&&!event.currentTarget.hasPointerCapture?.(event.pointerId))release()}}
     onKeyDown={event => keyboard(event, 'down')} onKeyUp={event => keyboard(event, 'up')}
-    onPaste={event => { if (state.mode === 'control') { event.preventDefault(); streamRef.current?.input({ type: 'text', text: event.clipboardData.getData('text/plain') }) } }}
-    onWheel={event => { if (state.mode === 'control') { event.preventDefault(); streamRef.current?.input({ type: 'wheel', deltaX: Math.round(event.deltaX), deltaY: Math.round(event.deltaY) }) } }} />
+    onPaste={event => { if (state.mode === 'control') { event.preventDefault();const value=event.clipboardData.getData('text/plain');void keyboardAction((stream,current)=>sendDesktopText(stream,value,current)) } }}
+    />
   const controls = <div className="bp-actions">
     <button disabled={state.status !== 'connected' || state.mode === 'watch'} onClick={() => { release(); streamRef.current?.setMode('watch') }}>仅观看</button>
-    <button disabled={state.status !== 'connected' || state.mode === 'control' || !canControl} onClick={() => streamRef.current?.setMode('control')}>取得控制权</button>
-    <span role="status">{state.mode === 'control' ? '你正在控制这台电脑' : '仅观看'}</span></div>
-  return <section className="bp-desktop-viewer"><div className="bp-card-title"><h3><UiIcon name="computer" size={16} /> 远程桌面</h3><span className="bp-desktop-status" data-connected={state.status === 'connected'}>{STATUS[state.status] ?? state.status}</span></div>
-    <div className="bp-actions"><select aria-label="远程桌面电脑" value={machineId} onChange={event => { void stop(); setMachineId(event.target.value); setDisplayId(''); setState({ status: 'idle', mode: 'watch' }) }}>
+    <button className={state.mode==='control'?'':'bp-button-primary'} disabled={state.status !== 'connected' || state.mode === 'control' || !canControl} onClick={() => streamRef.current?.setMode('control')}>取得控制权</button>
+    <span className="bp-desktop-control-status" data-mode={state.mode} role="status">{state.mode === 'control' ? '你正在控制这台电脑' : '仅观看'}</span></div>
+  const toolbar=<div className="bp-desktop-toolbar">{controls}<div className="bp-desktop-view-tools"><button ref={toolsButton} type="button" aria-label="键盘与文字" aria-expanded={toolsOpen} onClick={()=>{release();setToolsOpen(value=>!value)}}><UiIcon name="keyboard"/></button>
+    <button type="button" aria-label={fit?'原始尺寸':'适应窗口'} title={fit?'原始尺寸':'适应窗口'} aria-pressed={!fit} onClick={()=>setFit(value=>!value)}><UiIcon name="fit"/></button>
+    {typeof document.documentElement.requestFullscreen==='function'&&<button type="button" aria-label={fullscreen?'退出系统全屏':'系统全屏'} title={fullscreen?'退出系统全屏':'系统全屏'} onClick={()=>{void toggleFullscreen()}}><UiIcon name="expand"/></button>}</div></div>
+  return <section ref={viewerRef} className="bp-desktop-viewer"><div className="bp-card-title"><h3><UiIcon name="computer" size={16} /> 远程桌面</h3><span className="bp-desktop-status" data-connected={state.status === 'connected'}>{STATUS[state.status] ?? state.status}</span></div>
+    <div className="bp-actions"><select aria-label="远程桌面电脑" value={machineId} onChange={event => { void stop(); setMachineId(event.target.value);setText('');setToolsOpen(false);setKeyboardError(''); setDisplayId(''); setState({ status: 'idle', mode: 'watch' }) }}>
       {machines.map(item => <option key={item.machineId ?? item.id} value={item.machineId ?? item.id}>{item.label ?? item.name ?? item.machineId ?? item.id}</option>)}</select>
       {displays.length > 1 && <select aria-label="远程显示器" value={displayId} onChange={event => { void stop(); setDisplayId(event.target.value); setState({ status: 'idle', mode: 'watch' }) }}>
         <option value="">主显示器</option>{displays.map(display => <option key={display.id ?? display.displayId} value={display.id ?? display.displayId}>{display.name ?? display.id ?? display.displayId}</option>)}</select>}
       <button className="bp-button-primary" disabled={!machineId || !botId || ['connecting', 'authenticating'].includes(state.status)} onClick={() => { void connect() }}>{state.status === 'connected' ? '重新连接' : '连接桌面'}</button>
-      <button disabled={!ticketRef.current} onClick={() => { void stop().then(result => setState(result?.stopConfirmed === true ? { status: 'idle', mode: 'watch' }
+      <button disabled={!ticketRef.current&&!pendingStop.current} onClick={() => { void stop().then(result => setState(result?.stopConfirmed === true ? { status: 'idle', mode: 'watch' }
         : { status: 'error', mode: 'watch', error: '画面连接已关闭，但远端尚未确认释放输入，请检查远程电脑。' })) }}>断开</button></div>
-    {state.error && <p className="bp-error" role="alert">{state.error}</p>}{controls}
-    <div className="bp-stream-screen" data-connected={state.status === 'connected'} onDoubleClick={() => { if (state.mode === 'watch') setExpanded(true) }}>{screen(canvasRef)}
+    {state.error && <p className="bp-error" role="alert">{state.error}</p>}{!expanded&&toolbar}
+    <div className="bp-stream-screen" data-fit={fit} data-connected={state.status === 'connected'} onDoubleClick={() => { if (state.mode === 'watch') setExpanded(true) }}>{screen(canvasRef)}
       {state.status !== 'connected' && <div className="bp-desktop-idle"><UiIcon name="computer" size={34} /><strong>{['connecting', 'authenticating'].includes(state.status) ? '正在连接你的电脑…' : '你的电脑，就在这里'}</strong><span>连接后查看实时桌面，或取得鼠标键盘控制权</span></div>}
     </div>
-    <div className="bp-actions"><small>{stats.width ? `${stats.width} × ${stats.height} · ${stats.fps} 帧/秒${stats.frameWidth && stats.frameWidth !== stats.width ? ` · 传输 ${stats.frameWidth} × ${stats.frameHeight}` : ''}` : '连接后显示整台电脑的桌面。'}</small><button onClick={() => setExpanded(true)}>放大</button></div>
-    {state.mode === 'control' && <div className="bp-actions"><input aria-label="发送到远程电脑的文字" value={text} onChange={event => setText(event.target.value)} placeholder="中文或多行文字" />
-      <button disabled={!text} onClick={() => { if (streamRef.current?.input({ type: 'text', text })) setText('') }}>发送文字</button></div>}
+    <div className="bp-actions"><small>{stats.width ? `${stats.width} × ${stats.height} · ${stats.fps} 帧/秒${stats.frameWidth && stats.frameWidth !== stats.width ? ` · 传输 ${stats.frameWidth} × ${stats.frameHeight}` : ''}` : '连接后显示整台电脑的桌面。'}</small><button ref={expandButton} onClick={() => setExpanded(true)}><UiIcon name="expand" size={15}/>放大</button></div>
+    {state.mode==='control'&&<p className="bp-desktop-touch-hint">手机单指操作，双指滚动；离开画面会释放按键和鼠标。</p>}
+    {!expanded&&tools}
     {typeof saveRoutine === 'function' && <RoutineCaptureControls recorder={recorder} sessionId={ticketRef.current?.sessionId} metadata={metadata}
       mode={state.mode} connected={state.status === 'connected'} drainInputs={() => streamRef.current?.drain()}
       releaseInput={() => streamRef.current?.releaseInput()} saveRoutine={routine => saveRoutine(botId, routine)} />}
-    {expanded && <div className="bp-stream-overlay" role="dialog" aria-modal="true" aria-label="远程桌面放大"><div className="bp-stream-dialog"><div className="bp-actions">{controls}
-      <button onClick={() => { release(); setExpanded(false) }}>关闭放大</button></div>{screen(largeRef, true)}</div></div>}
+    {expanded && <div className="bp-stream-overlay" role="dialog" aria-modal="true" aria-label="远程桌面放大" ref={dialogRef} onKeyDown={event=>{
+      if(event.key==='Escape'&&!event.nativeEvent.isComposing){event.preventDefault();event.stopPropagation();release();setToolsOpen(false);setExpanded(false)}
+      if(event.key==='Tab'){const focusable=[...dialogRef.current.querySelectorAll('button:not(:disabled),textarea,canvas,[tabindex="0"]')];const first=focusable[0],last=focusable.at(-1);if(event.shiftKey&&document.activeElement===first){event.preventDefault();last?.focus()}else if(!event.shiftKey&&document.activeElement===last){event.preventDefault();first?.focus()}}
+    }}><div className="bp-stream-dialog"><header className="bp-desktop-expanded-header"><strong><UiIcon name="computer"/>{machines.find(item=>(item.machineId??item.id)===machineId)?.label??'远程电脑'}</strong><span className="bp-desktop-status" data-connected={state.status==='connected'}>{STATUS[state.status]??state.status}</span><button onClick={() => { release();setToolsOpen(false);setExpanded(false) }}>关闭放大</button></header>{toolbar}<div className="bp-desktop-expanded-stage" data-fit={fit}>{screen(largeRef, true)}{tools}</div></div></div>}
   </section>
 }
 
@@ -192,10 +261,10 @@ export function DesktopPane({ bot }) {
   if (!api?.open) return <p className="bp-muted">桌面通道尚未就绪。</p>
   if (error) return <p role="alert">{error}</p>
   if (!machines.length) return <p className="bp-muted">在“远程电脑”中配对电脑并为这个 Bot 授权。</p>
-  return <DesktopViewer key={bot.id} botId={bot.id} machines={machines} openSession={api.open} renewSession={api.renew} closeSession={api.close} saveRoutine={api.saveRoutine} />
+  return <DesktopViewer key={bot.id} botId={bot.id} machines={machines} openSession={api.open} renewSession={api.renew} closeSession={api.close} saveRoutine={api.saveRoutine} transport={api.transport} />
 }
 
-export function DesktopWorkbench({ snapshot, dispatch }) {
+export function DesktopWorkbench({ snapshot, dispatch, transport }) {
   const bots = snapshot.bots ?? [], [botId, setBotId] = useState(bots[0]?.id ?? '')
   const machines = (snapshot.machines?.items ?? []).filter(item => item.status !== 'revoked')
   const current = useRef({ dispatch, machines }); current.current = { dispatch, machines }
@@ -205,5 +274,5 @@ export function DesktopWorkbench({ snapshot, dispatch }) {
   const saveRoutine = useCallback((id, routine) => current.current.dispatch('routine.save', { botId: id, routine }), [])
   if (!machines.length) return <section className="bp-card"><div className="bp-empty-state"><span className="bp-empty-icon"><UiIcon name="computer" size={29} /></span><strong>连接你的第一台电脑</strong><p>在自己的 Windows 或云电脑上运行 Agent，再导入配对文件，就能在这里查看和接管桌面。</p><div className="bp-actions"><a className="bp-download-link" href="https://github.com/Alice-Marx/dsh-model-router/releases/download/v0.63.0/dsh-remote-agent-0.1.0-windows-x64.zip" target="_blank" rel="noopener noreferrer"><UiIcon name="arrow" size={15} />下载 Windows Agent</a><button className="bp-button-primary" onClick={() => { const setup = document.querySelector('.bp-machine-setup'); if (setup) { setup.open = true; setup.scrollIntoView({ block: 'start', behavior: 'smooth' }) } }}>导入配对文件</button></div></div></section>
   return <section className="bp-card"><div className="bp-actions" style={{ marginBottom: 20 }}><label>连接身份<select value={botId} onChange={event => setBotId(event.target.value)}>{bots.map(bot => <option key={bot.id} value={bot.id}>{bot.name ?? bot.id}</option>)}</select></label></div>
-    <DesktopViewer key={botId} botId={botId} machines={machines} openSession={openSession} renewSession={renewSession} closeSession={closeSession} saveRoutine={saveRoutine} /></section>
+    <DesktopViewer key={botId} botId={botId} machines={machines} openSession={openSession} renewSession={renewSession} closeSession={closeSession} saveRoutine={saveRoutine} transport={transport} /></section>
 }

package/apps/integration-client/desktop-stream.mjs

@@ -1,8 +1,14 @@
 /** The viewer never receives a pairing secret. Its ticket belongs to one authenticated desktop session. */
-export function connectDesktopStream({ ticket, onFrame, onState, onMetadata, onBeforeInput, onInputAcknowledged, onInputFailure,
+export function connectDesktopStream({ ticket, botId, transport, onFrame, onState, onMetadata, onBeforeInput, onInputAcknowledged, onInputFailure,onBeforeRelease,onReleaseAcknowledged,onReleaseFailure,
   WebSocketImpl = globalThis.WebSocket, ackTimeoutMs = 10000, now = Date.now }) {
-  if (!ticket?.websocketUrl || typeof WebSocketImpl !== 'function') throw new Error('桌面连接不可用。')
-  const socket = new WebSocketImpl(ticket.websocketUrl)
+  let websocketUrl=ticket?.websocketUrl
+  if(ticket?.websocketPath){
+    if(ticket.websocketPath!=='/bot-product/desktop/v1/socket'||!['http:','https:'].includes(globalThis.location?.protocol))throw new Error('此桌面连接需要已登录的网页宿主。')
+    const endpoint=new URL(ticket.websocketPath,globalThis.location.href);endpoint.protocol=endpoint.protocol==='https:'?'wss:':'ws:';websocketUrl=endpoint.href
+  }
+  if (ticket?.carrier!=='owner-http'&&(!websocketUrl || typeof WebSocketImpl !== 'function')) throw new Error('桌面连接不可用。')
+  if(ticket?.carrier==='owner-http'&&typeof transport!=='function')throw new Error('宿主桌面数据通道尚未就绪。')
+  const socket = ticket?.carrier==='owner-http'?createOwnerDesktopSocket({ticket,botId,dispatch:transport}):new WebSocketImpl(websocketUrl)
   socket.binaryType = 'arraybuffer'
   let closed = false, mode = 'watch', desiredMode = ticket.mode === 'control' ? 'control' : 'watch', nextId = 0
   let lastError = ''
@@ -20,6 +26,7 @@ export function connectDesktopStream({ ticket, onFrame, onState, onMetadata, onB
     if (!pending.delete(row.id)) return
     clearTimeout(row.timer)
     if (row.event) onInputFailure?.(row.event, { ...row.meta, failure })
+    else onReleaseFailure?.({...row.meta,failure})
     row.reject(Object.assign(new Error(failure.code), failure))
   }
   const failPending = code => { for (const row of [...pending.values()]) failed(row, { code, possiblyApplied: true }) }
@@ -36,6 +43,7 @@ export function connectDesktopStream({ ticket, onFrame, onState, onMetadata, onB
     pending.set(id, row)
     try {
       if (event) onBeforeInput?.(event, meta)
+      else onBeforeRelease?.(meta)
       if (!send({ type, id, ...(event ? { event } : {}) })) throw Object.assign(new Error('DESKTOP_DISCONNECTED'), { code: 'DESKTOP_DISCONNECTED' })
     } catch (error) { failed(row, { code: error.code ?? 'DESKTOP_DISCONNECTED', possiblyApplied: false }); return null }
     return promise
@@ -55,6 +63,7 @@ export function connectDesktopStream({ ticket, onFrame, onState, onMetadata, onB
       }
       pending.delete(row.id); clearTimeout(row.timer)
       if (row.event) onInputAcknowledged?.(row.event, { ...row.meta, acknowledgedAt: now() })
+      else onReleaseAcknowledged?.({...row.meta,acknowledgedAt:now(),ok:true})
       row.resolve({ ok: true, id: row.id }); return
     }
     if (value.type === 'ready') {
@@ -86,3 +95,4 @@ export function connectDesktopStream({ ticket, onFrame, onState, onMetadata, onB
     close() { if (closed) return; send({ type: 'close' }); closed = true; mode = 'watch'; failPending('DESKTOP_DISCONNECTED'); socket.close() },
   })
 }
+import { createOwnerDesktopSocket } from './desktop-http-socket.mjs'

package/apps/integration-client/desktop-tools.jsx

@@ -0,0 +1,19 @@
+import React,{useEffect,useRef} from 'react'
+import {DESKTOP_SHORTCUTS} from './desktop-keyboard.mjs'
+import {UiIcon} from './ui-icons.jsx'
+
+export function DesktopTools({open,onClose,connected,canInput,text,onText,onSend,onPaste,onShortcut,working,error}){
+  const panel=useRef(null)
+  useEffect(()=>{if(open)panel.current?.querySelector('textarea')?.focus()},[open])
+  if(!open)return null
+  return <section className="bp-desktop-tools-panel" ref={panel} aria-label="远程键盘与文字" onKeyDown={event=>{
+    if(event.key==='Escape'&&!event.nativeEvent.isComposing&&event.nativeEvent.keyCode!==229){event.preventDefault();event.stopPropagation();onClose()}
+  }}><header><strong><UiIcon name="keyboard" size={17}/>键盘与文字</strong><button type="button" onClick={onClose} aria-label="关闭键盘面板"><UiIcon name="close"/></button></header>
+    <label>发送到远程电脑<textarea rows={4} maxLength={8192} value={text} onChange={event=>onText(event.target.value)} placeholder="中文、多行文字或本机复制的内容" autoComplete="off" spellCheck={false}/></label>
+    <div className="bp-actions"><button type="button" className="bp-button-primary" disabled={!connected||!canInput||!text||working} onClick={onSend}>{working?'等待远端确认…':'发送文字'}</button><button type="button" disabled={working} onClick={onPaste}>读取本机剪贴板</button></div>
+    {!canInput&&<p className="bp-muted">取得控制权后可以发送文字和快捷键。</p>}
+    {error&&<p className="bp-error" role="alert">{error}</p>}
+    <div className="bp-desktop-shortcuts" role="group" aria-label="远程快捷键">{DESKTOP_SHORTCUTS.map(row=><button key={row.label} type="button" disabled={!connected||!canInput||working} onClick={()=>onShortcut(row.keys)}>{row.label}</button>)}</div>
+    <p className="bp-muted">本机文字会以键盘输入发送；快捷键“复制 / 粘贴”操作的是远端剪贴板。</p>
+  </section>
+}

package/apps/integration-client/desktop-touch.mjs

@@ -0,0 +1,27 @@
+/** Once a gesture has two fingers, none of its remaining contacts may resume a held mouse drag. */
+export function createDesktopTouchTracker(){
+  const points=new Map();let multiple=false,center=null
+  const average=()=>{const values=[...points.values()];return {x:values.reduce((sum,item)=>sum+item.x,0)/values.length,y:values.reduce((sum,item)=>sum+item.y,0)/values.length}}
+  const clear=()=>{points.clear();multiple=false;center=null}
+  return Object.freeze({clear,handle(action,event){
+    if(event.pointerType!=='touch')return {handled:false}
+    if(!Number.isFinite(event.clientX)||!Number.isFinite(event.clientY)||!Number.isSafeInteger(event.pointerId))return {handled:true,release:true}
+    if(action==='down'){
+      points.set(event.pointerId,{x:event.clientX,y:event.clientY})
+      if(points.size<2&&!multiple)return {handled:false}
+      const release=!multiple;multiple=true;center=average();return {handled:true,release}
+    }
+    if(action==='move'){
+      if(!points.has(event.pointerId))return {handled:multiple}
+      points.set(event.pointerId,{x:event.clientX,y:event.clientY})
+      if(!multiple)return {handled:false}
+      if(points.size<2)return {handled:true}
+      const next=average(),previous=center??next;center=next
+      const deltaX=Math.max(-480,Math.min(480,Math.round(previous.x-next.x))),deltaY=Math.max(-480,Math.min(480,Math.round(previous.y-next.y)))
+      return {handled:true,...(deltaX||deltaY?{wheel:{type:'wheel',deltaX,deltaY}}:{})}
+    }
+    const handled=multiple;points.delete(event.pointerId);center=points.size>=2?average():null
+    if(!points.size){multiple=false;center=null}
+    return {handled,release:handled&&!points.size}
+  }})
+}

package/apps/integration-client/desktop.css

@@ -0,0 +1,41 @@
+.bp-desktop-toolbar { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 10px; padding: 9px; border: 1px solid var(--bp-border); border-radius: 12px; background: var(--bp-subtle); }
+.bp-desktop-toolbar > .bp-actions { margin: 0; gap: 7px; flex: 1; }
+.bp-desktop-view-tools { display: flex; flex: none; gap: 4px; }
+.bp-desktop-viewer .bp-desktop-view-tools button { width: 36px; min-width: 36px; height: 36px; padding: 7px; }
+.bp-desktop-view-tools button[aria-pressed=true],.bp-desktop-view-tools button[aria-expanded=true] { border-color: var(--bp-solid); color: var(--bp-solid); background: var(--bp-hover); }
+.bp-desktop-control-status { display: inline-flex; align-items: center; gap: 6px; margin-left: 4px; font-size: 11px; color: var(--bp-muted); }
+.bp-desktop-control-status[data-mode=control] { color: var(--bp-solid); }
+.bp-desktop-control-status[data-mode=control]::before { content: ''; width: 6px; height: 6px; border-radius: 50%; background: currentColor; }
+.bp-desktop-touch-hint { display: none; margin: 0; color: var(--bp-muted); font-size: 11px; }
+@media(pointer:coarse) { .bp-desktop-touch-hint { display: block; } }
+.bp-stream-canvas { touch-action: pan-y; }
+.bp-stream-canvas[data-control=true] { touch-action: none; }
+.bp-stream-screen[data-fit=false],.bp-desktop-expanded-stage[data-fit=false] { overflow: auto; align-items: flex-start; justify-content: flex-start; }
+.bp-stream-screen[data-fit=false] .bp-stream-canvas { flex: none; }
+.bp-desktop-tools-panel { min-width: 0; padding: 16px; border: 1px solid var(--bp-border); border-radius: 14px; background: var(--bp-surface); box-shadow: 0 12px 32px #00000014; }
+.bp-desktop-tools-panel > header { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-bottom: 12px; }
+.bp-desktop-tools-panel strong { display: flex; align-items: center; gap: 8px; }
+.bp-desktop-tools-panel > label { display: grid; gap: 8px; margin-bottom: 12px; font-size: 12px; color: var(--bp-muted); }
+.bp-desktop-viewer .bp-desktop-tools-panel textarea { width: 100%; min-height: 100px; max-height: 220px; min-width: 0; padding: 10px; resize: vertical; border: 1px solid var(--bp-border); border-radius: 9px; background: var(--bp-subtle); color: var(--bp-ink); font: inherit; }
+.bp-desktop-shortcuts { display: grid; grid-template-columns: repeat(4,minmax(0,1fr)); gap: 7px; margin-top: 16px; padding-top: 14px; border-top: 1px solid var(--bp-border); }
+.bp-desktop-shortcuts button { overflow-wrap: anywhere; }
+.bp-stream-overlay { padding: 16px; background: #101714; }
+.bp-desktop-viewer :is(button,input,textarea,select,canvas) { -webkit-app-region: no-drag; }
+.bp-stream-dialog { height: calc(100dvh - 32px); width: 100%; max-width: 1500px; min-height: 0; --bp-subtle: #1b2420; --bp-surface: #242e28; --bp-ink: #eef4ee; --bp-muted: #abb8ad; --bp-border: #425147; --bp-hover: #35483b; --bp-solid: #9acda7; --bp-on-solid: #142319; }
+.bp-desktop-expanded-header { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; min-height: 40px; }
+.bp-desktop-expanded-header > strong { display: flex; align-items: center; gap: 9px; min-width: 0; margin-right: auto; }
+.bp-desktop-expanded-stage { position: relative; flex: 1; min-height: 0; min-width: 0; display: flex; align-items: center; justify-content: center; overflow: hidden; border: 1px solid var(--bp-border); border-radius: 12px; background: #101612; }
+.bp-desktop-expanded-stage .bp-stream-large { width: 100%; height: 100%; max-height: none; object-fit: contain; flex: none; }
+.bp-desktop-expanded-stage > .bp-desktop-tools-panel { position: absolute; z-index: 1; left: 16px; top: 16px; width: min(420px,calc(100% - 32px)); max-height: calc(100% - 32px); overflow: auto; }
+.bp-stream-overlay:fullscreen { padding: 12px; }
+.bp-stream-overlay:fullscreen .bp-stream-dialog { height: calc(100dvh - 24px); max-width: none; }
+@media(max-width:600px){
+  .bp-desktop-toolbar { gap: 7px; }
+  .bp-desktop-toolbar > .bp-actions { flex-wrap: wrap; }
+  .bp-desktop-control-status { width: 100%; margin: 2px 4px; }
+  .bp-desktop-shortcuts { grid-template-columns: repeat(2,minmax(0,1fr)); }
+  .bp-stream-overlay { padding: 8px; }
+  .bp-stream-dialog { height: calc(100dvh - 16px); max-width: 100%; gap: 8px; }
+  .bp-desktop-viewer button { min-height: 40px; }
+  .bp-desktop-expanded-stage > .bp-desktop-tools-panel { left: 8px; top: 8px; width: calc(100% - 16px); max-height: calc(100% - 16px); }
+}