dsh-plugin-mobile-gateway 0.4.2 → 0.6.0

package/lib/client.js

@@ -37,13 +37,14 @@ window.__ModuleLoader__.load({
       },
       panel: {
         width: 380, maxWidth: 'calc(100vw - 24px)', height: '100%', overflowY: 'auto',
+        display: 'flex', flexDirection: 'column',
         background: colors.panel, color: colors.text, borderLeft: `1px solid ${colors.border}`,
         boxShadow: '-12px 0 40px rgba(0,0,0,.22)', padding: 20,
         fontFamily: 'system-ui, sans-serif', fontSize: 13, boxSizing: 'border-box',
       },
-      header: { display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: 18 },
+      header: { display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: 18, flexShrink: 0 },
       title: { margin: 0, fontSize: 18, fontWeight: 650 },
-      card: { background: colors.card, border: `1px solid ${colors.border}`, borderRadius: 12, padding: 14, marginBottom: 12 },
+      card: { background: colors.card, border: `1px solid ${colors.border}`, borderRadius: 12, padding: 14, marginBottom: 12, flexShrink: 0 },
       label: { display: 'block', color: colors.muted, fontSize: 12, marginBottom: 6 },
       input: { width: '100%', boxSizing: 'border-box', border: `1px solid ${colors.border}`, borderRadius: 8, padding: '8px 10px', marginBottom: 10, background: 'transparent', color: 'inherit', outline: 'none' },
       pairingText: { width: '100%', minHeight: 86, boxSizing: 'border-box', resize: 'vertical', border: `1px solid ${colors.border}`, borderRadius: 8, padding: '9px 10px', margin: '10px 0', background: 'transparent', color: 'inherit', outline: 'none', fontFamily: 'ui-monospace, SFMono-Regular, Menlo, monospace', fontSize: 11, lineHeight: 1.45, overflowWrap: 'anywhere' },
@@ -59,6 +60,7 @@ window.__ModuleLoader__.load({
       switchInput: { position: 'absolute', opacity: 0, width: 1, height: 1 },
       switchTrack: { position: 'absolute', inset: 0, borderRadius: 99, transition: 'background .18s ease', cursor: 'pointer' },
       switchKnob: { position: 'absolute', top: 3, width: 18, height: 18, borderRadius: 99, background: '#fff', boxShadow: '0 1px 3px rgba(0,0,0,.35)', transition: 'left .18s ease', pointerEvents: 'none' },
+      version: { flexShrink: 0, marginTop: 'auto', padding: '18px 2px 0', color: colors.muted, fontSize: 11, lineHeight: 1.4, textAlign: 'right', letterSpacing: '.02em' },
     }
 
     async function request(path, options) {
@@ -373,7 +375,10 @@ window.__ModuleLoader__.load({
                     device,
                     onRevoke: revoke,
                     revoking: revokingIds.has(device.id),
-                  })))))
+                  }))),
+          status && status.version
+            ? React.createElement('div', { style: styles.version, 'aria-label': `插件版本 ${status.version}` }, `v${status.version}`)
+            : null))
     }
 
     function FooterButton(props) {

package/lib/index.mjs

@@ -18,8 +18,9 @@
 //   client -> server: { "type": "ping" }
 //                     { "type": "subscribe", "sessionId": "..." }  (optional filter)
 //                     { "type": "unsubscribe" }
-//                     { "type": "message", "sessionId"?, "text", "mode"?: "queue"|"steer",
-//                       "workspaceId"?, "cwd"? }
+//                     { "type": "message", "sessionId"?, "text"?, "images"?:
+//                       [{ "mediaType", "data", "name"? }], "mode"?: "queue"|"steer",
+//                       "workspaceId"?, "cwd"?, "clientTimeZone"? }
 //                        sessionId omitted -> a new session is created first; the
 //                        new session can be placed in a workspace via workspaceId
 //                        (or cwd; at most one, workspaceId wins)
@@ -30,6 +31,8 @@
 //                        -> raw SessionEvent page (scheme A), capped at maxBytes
 //                           (default 4 MiB); hasMore + nextBeforeSeq page backward;
 //                           view:"conversation" trims chunk/header events and tool output
+//                     { "type": "attachment", "sessionId", "attachmentId" }
+//                        -> verified image metadata + canonical base64 bytes
 //                     { "type": "search", "query" }                -> session search
 //                     { "type": "host" }                           -> host.describe snapshot (incl. default provider/model)
 //                     { "type": "default-model" }                   -> agentDefaultModel.currentSelection()
@@ -48,13 +51,20 @@
 //                     { "type": "agent-presets" }                     -> preset roster (+ isDefault)
 //                     { "type": "defaults" }                          -> default agent preset + default permission
 //                     { "type": "set-default", "target": "agent-preset"|"permission", "value" }
-//   server -> client: { "kind": "hello", "protocol": 2, "authenticated", "device"?, "port", "clients" }
+//                     { "type": "question-answer", "rpcId", "sessionId", "answers": [...] }
+//                     { "type": "question-cancel", "rpcId", "sessionId" }
+//   server -> client: { "kind": "hello", "protocol": 3, "capabilities": ["images"],
+//                       "authenticated", "device"?, "port", "clients" }
 //                     { "kind": "pong", "at" }
 //                     { "kind": "subscribed", "sessionId" }
 //                     { "kind": "sent", "sessionId", "mode", "command"? }
 //                     { "kind": "workspaces" | "sessions" | "history" | "search", ...data }
+//                     { "kind": "attachment", "sessionId", "attachment", "data" }
 //                     { "kind": "error", "code", "message", "requestType"?, "sessionId"? }
 //                     { "kind": "event", "sessionId", "seq", "time", "event": { ... } }
+//                     { "kind": "question-requested", "rpcId", "sessionId", "questions", "replay"? }
+//                     { "kind": "question-response", "rpcId", "sessionId", "action", "accepted", "reason"? }
+//                     { "kind": "question-resolved", "rpcId", "sessionId", "outcome" }
 import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import http from 'node:http'
@@ -68,12 +78,24 @@ import QRCode from 'qrcode'
 
 const { createRegistry } = devicesModule
 
+const PLUGIN_VERSION = (() => {
+  try {
+    return JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version
+  } catch {
+    return 'unknown'
+  }
+})()
 const MAX_PREVIEW = 400
 const LOG_FILE = '/tmp/mobile-gateway.log'
 const DEFAULT_WS_PATH = '/ws/mobile'
 const DEFAULT_PAIRING_TTL_MS = 5 * 60 * 1000
 const DEFAULT_GATEWAY_WAIT_TIMEOUT_MS = 5 * 60 * 1000
+// DSH 0.1.1 allows up to 100 MiB of decoded images in one prompt. Base64 plus
+// the JSON envelope needs roughly 4/3 of that on the wire. Keep this separately
+// configurable so deployments may choose a lower transport ceiling.
+const DEFAULT_MAX_WS_PAYLOAD_BYTES = 144 * 1024 * 1024
 const MAX_MANAGEMENT_BODY_BYTES = 16 * 1024
+const IMAGE_MEDIA_TYPES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif'])
 
 // Cordis validates this schema at plugin load and fills these defaults. Keep
 // the defaults conservative: installing the bundle must never create an
@@ -83,6 +105,7 @@ const Config = Schema.object({
   requireAuth: Schema.boolean().default(true),
   gatewayEnabled: Schema.boolean().default(false),
   gatewayWaitTimeoutMs: Schema.natural().min(30_000).max(30 * 60 * 1000).default(DEFAULT_GATEWAY_WAIT_TIMEOUT_MS),
+  maxPayloadBytes: Schema.natural().min(1024 * 1024).max(160 * 1024 * 1024).default(DEFAULT_MAX_WS_PAYLOAD_BYTES),
   adminLoopbackOnly: Schema.boolean().default(true),
   publicUrl: Schema.string().default(''),
   deviceFile: Schema.string().default(''),
@@ -114,20 +137,40 @@ function textOf(blocks) {
   return text
 }
 
+function imagesOf(blocks) {
+  const images = []
+  for (const block of blocks) {
+    if (!block || block.type !== 'image' || !block.attachment) continue
+    const attachment = block.attachment
+    images.push({
+      attachmentId: attachment.attachmentId,
+      mediaType: attachment.mediaType,
+      bytes: attachment.bytes,
+      width: attachment.width,
+      height: attachment.height,
+      ...(attachment.name ? { name: attachment.name } : {}),
+    })
+  }
+  return images
+}
+
 // Build the small, owned JSON wire record for one session event. Reads only
 // leaf fields of the live SessionEvent — never serializes live objects.
 function buildWireEvent(session, event) {
-  const base = { sessionId: String(session.id), seq: event.seq, time: event.time }
+  const base = { kind: 'event', sessionId: String(session.id), seq: event.seq, time: event.time }
   const d = event.data || {}
   switch (event.type) {
-    case 'user/message':
+    case 'user/message': {
+      const images = imagesOf(d.content || [])
       return Object.assign(base, {
         event: {
           type: 'user/message',
           text: textOf(d.content || []),
           source: d.source && d.source.kind,
+          ...(images.length ? { images } : {}),
         },
       })
+    }
     case 'assistant/chunk': {
       const chunk = d.chunk || {}
       const ev = { type: 'assistant/chunk', turn: d.turn, step: d.step, chunkType: chunk.type }
@@ -142,6 +185,7 @@ function buildWireEvent(session, event) {
       let text = ''
       let reasoning = ''
       const toolCalls = []
+      const images = imagesOf(blocks)
       for (const block of blocks) {
         if (!block) continue
         if (block.type === 'text' && typeof block.text === 'string') text += block.text
@@ -149,7 +193,7 @@ function buildWireEvent(session, event) {
         else if (block.type === 'tool-call') toolCalls.push({ id: block.id, name: block.name, arguments: block.arguments })
       }
       return Object.assign(base, {
-        event: { type: 'assistant/message', turn: d.turn, step: d.step, text, reasoning, toolCalls },
+        event: { type: 'assistant/message', turn: d.turn, step: d.step, text, reasoning, toolCalls, ...(images.length ? { images } : {}) },
       })
     }
     case 'tool/call':
@@ -191,8 +235,37 @@ function buildWireEvent(session, event) {
 // the wire frame to send back, or null when nothing should be sent.
 async function admitMessage(api, msg) {
   const text = typeof msg.text === 'string' ? msg.text.trim() : ''
-  if (!text) {
-    return { kind: 'error', code: 'bad-request', message: 'text must be a non-empty string' }
+  const images = msg.images === undefined ? [] : msg.images
+  if (!Array.isArray(images)) {
+    return { kind: 'error', code: 'bad-request', message: 'images must be an array' }
+  }
+  if (images.length > 20) {
+    return { kind: 'error', code: 'bad-request', message: 'a message can contain at most 20 images' }
+  }
+  const imageParts = []
+  for (let index = 0; index < images.length; index++) {
+    const image = images[index]
+    if (!image || typeof image !== 'object') {
+      return { kind: 'error', code: 'bad-request', message: `images[${index}] must be an object` }
+    }
+    if (!IMAGE_MEDIA_TYPES.has(image.mediaType)) {
+      return { kind: 'error', code: 'bad-request', message: `images[${index}].mediaType is unsupported` }
+    }
+    if (typeof image.data !== 'string' || image.data.length === 0) {
+      return { kind: 'error', code: 'bad-request', message: `images[${index}].data must be a non-empty base64 string` }
+    }
+    if (image.name !== undefined && (typeof image.name !== 'string' || image.name.length > 255)) {
+      return { kind: 'error', code: 'bad-request', message: `images[${index}].name must be a string of at most 255 characters` }
+    }
+    imageParts.push({
+      type: 'image',
+      mediaType: image.mediaType,
+      data: image.data,
+      ...(image.name ? { name: image.name } : {}),
+    })
+  }
+  if (!text && imageParts.length === 0) {
+    return { kind: 'error', code: 'bad-request', message: 'message requires non-empty text or at least one image' }
   }
   const mode = msg.mode === 'steer' ? 'steer' : 'queue'
 
@@ -219,10 +292,16 @@ async function admitMessage(api, msg) {
 
     const resp = await api.sessions.prompt({
       rpcId: crypto.randomUUID(),
-      payload: { sessionId, mode, content: [{ type: 'text', text }] },
+      payload: {
+        sessionId,
+        mode,
+        // Match the official WebUI ordering: images first, optional text last.
+        content: [...imageParts, ...(text ? [{ type: 'text', text }] : [])],
+        ...(typeof msg.clientTimeZone === 'string' && msg.clientTimeZone.trim() ? { clientTimeZone: msg.clientTimeZone.trim() } : {}),
+      },
     })
     if (resp.result.ok) {
-      log(`mobile message accepted: session=${sessionId} mode=${mode} text="${text.slice(0, 60)}"`)
+      log(`mobile message accepted: session=${sessionId} mode=${mode} images=${imageParts.length} text="${text.slice(0, 60)}"`)
       return {
         kind: 'sent',
         sessionId,
@@ -426,6 +505,20 @@ async function handleQuery(api, typertGateway, agentDefaultModel, msg) {
     }
     return frame
   }
+  if (msg.type === 'attachment') {
+    const sessionId = requireSessionId(msg)
+    if (sessionId.error) return sessionId.error
+    const attachmentId = typeof msg.attachmentId === 'string' && msg.attachmentId.trim() !== '' ? msg.attachmentId.trim() : null
+    if (!attachmentId) {
+      return { kind: 'error', code: 'bad-request', message: 'attachment requires an attachmentId', requestType: 'attachment', sessionId: sessionId.value }
+    }
+    const frame = await proxyQuery(api, 'attachment', api.sessions.attachment.bind(api.sessions), {
+      sessionId: sessionId.value,
+      attachmentId,
+    })
+    if (frame.kind === 'attachment') frame.sessionId = sessionId.value
+    return frame
+  }
   if (msg.type === 'search') {
     const query = typeof msg.query === 'string' ? msg.query.trim() : ''
     if (!query) {
@@ -815,6 +908,7 @@ const plugin = {
       requireAuth: true,
       gatewayEnabled: false,
       gatewayWaitTimeoutMs: DEFAULT_GATEWAY_WAIT_TIMEOUT_MS,
+      maxPayloadBytes: DEFAULT_MAX_WS_PAYLOAD_BYTES,
       adminLoopbackOnly: true,
       publicUrl: '',
       deviceFile: '',
@@ -843,7 +937,7 @@ const plugin = {
     const registry = createRegistry(deviceFile, { pairingTtlMs: options.pairingTtlMs })
     const wss = new WebSocketServer({
       noServer: true,
-      maxPayload: 1024 * 1024,
+      maxPayload: options.maxPayloadBytes,
       // Authentication data may ride in requested subprotocols, but a secret
       // must never be echoed as the negotiated protocol.
       handleProtocols(protocols) {
@@ -851,6 +945,8 @@ const plugin = {
       },
     })
     const clients = new Set()
+    const pendingQuestions = new Map()
+    const questionMuxAbort = new AbortController()
     let counter = 0
     let gatewayEnabled = options.gatewayEnabled === true
     let waitExpiresAt = null
@@ -912,6 +1008,72 @@ const plugin = {
       }
     }
 
+    const questionFrameFor = (rpcId, payload, replay = false) => ({
+      kind: 'question-requested',
+      rpcId: String(rpcId),
+      sessionId: String(payload.sessionId),
+      questions: payload.questions,
+      ...(replay ? { replay: true } : {}),
+    })
+
+    const broadcastQuestionFrame = (frame) => {
+      const wire = JSON.stringify(frame)
+      for (const client of clients) {
+        if (client.filterSessionId && client.filterSessionId !== frame.sessionId) continue
+        if (client.readyState === 1) client.send(wire)
+      }
+    }
+
+    const respondToQuestion = async (msg, cancel = false) => {
+      const rpcId = typeof msg.rpcId === 'string' && msg.rpcId.trim() ? msg.rpcId.trim() : null
+      const sessionId = typeof msg.sessionId === 'string' && msg.sessionId.trim() ? msg.sessionId.trim() : null
+      if (!rpcId || !sessionId) {
+        return { kind: 'error', code: 'bad-request', message: `${msg.type} requires rpcId and sessionId`, requestType: msg.type }
+      }
+
+      if (cancel) {
+        const pending = pendingQuestions.get(rpcId)
+        if (pending && pending.sessionId !== sessionId) {
+          return { kind: 'error', code: 'bad-request', message: 'sessionId does not match the pending question', requestType: msg.type, sessionId }
+        }
+      } else if (!Array.isArray(msg.answers)) {
+        return { kind: 'error', code: 'bad-request', message: 'question-answer requires an answers array', requestType: msg.type, sessionId }
+      }
+
+      const result = cancel
+        ? { ok: false, error: { code: 'cancelled', message: 'question cancelled by mobile user', details: {} } }
+        : {
+            ok: true,
+            value: {
+              sessionId,
+              answer: {
+                answers: msg.answers.map((answer) => ({
+                  id: answer && answer.id,
+                  selected: answer && answer.selected,
+                  ...(answer && answer.custom !== undefined ? { custom: answer.custom } : {}),
+                })),
+              },
+            },
+          }
+
+      try {
+        const receipt = await api.respond({ type: 'client-response', rpcId, result })
+        log(`question response: rpcId=${rpcId} session=${sessionId} action=${cancel ? 'cancel' : 'answer'} accepted=${receipt.accepted}${receipt.accepted ? '' : ` reason=${receipt.reason}`}`)
+        return {
+          kind: 'question-response',
+          rpcId,
+          sessionId,
+          action: cancel ? 'cancel' : 'answer',
+          accepted: receipt.accepted,
+          ...(!receipt.accepted ? { reason: receipt.reason } : {}),
+        }
+      } catch (error) {
+        const message = error && error.message ? error.message : String(error)
+        log(`question response failed: rpcId=${rpcId} ${message}`)
+        return { kind: 'error', code: 'internal', message, requestType: msg.type, sessionId }
+      }
+    }
+
     if (gatewayEnabled) setGatewayEnabled(true, 'enabled by startup config')
 
     log(`applying: path=${wsPath}, webServer.port=${webServer.port}, gatewayEnabled=${gatewayEnabled}, requireAuth=${requireAuth}, devices=${registry.count()}`)
@@ -934,6 +1096,7 @@ const plugin = {
           const p = url.pathname
           if (req.method === 'GET' && p === '/mgw/status') {
             sendJson(res, 200, {
+              version: PLUGIN_VERSION,
               requireAuth,
               gatewayEnabled,
               waitExpiresAt,
@@ -1097,11 +1260,15 @@ const plugin = {
             } else if (msg.type === 'unsubscribe') {
               ws.filterSessionId = undefined
               ws.send(JSON.stringify({ kind: 'subscribed', sessionId: null }))
+            } else if (msg.type === 'question-answer' || msg.type === 'question-cancel') {
+              respondToQuestion(msg, msg.type === 'question-cancel').then((frame) => {
+                if (frame && ws.readyState === 1) ws.send(JSON.stringify(frame))
+              })
             } else if (msg.type === 'message') {
               admitMessage(api, msg).then((frame) => {
                 if (frame && ws.readyState === 1) ws.send(JSON.stringify(frame))
               })
-            } else if (msg.type === 'workspaces' || msg.type === 'sessions' || msg.type === 'history' ||
+            } else if (msg.type === 'workspaces' || msg.type === 'sessions' || msg.type === 'history' || msg.type === 'attachment' ||
                        msg.type === 'search' || msg.type === 'host' || msg.type === 'directories' ||
                        msg.type === 'workspace-create' || msg.type === 'models' || msg.type === 'select-model' ||
                        msg.type === 'permission-options' || msg.type === 'permission' || msg.type === 'context-usage' ||
@@ -1132,12 +1299,17 @@ const plugin = {
           }
           ws.send(JSON.stringify({
             kind: 'hello',
-            protocol: 2,
+            protocol: 3,
+            capabilities: ['images'],
             port: transport.port || webServer.port,
             clients: clients.size,
             authenticated: !!device,
             ...(device ? { device: { id: device.id, name: device.name } } : {}),
           }))
+          for (const pending of pendingQuestions.values()) {
+            if (ws.filterSessionId && ws.filterSessionId !== pending.sessionId) continue
+            ws.send(JSON.stringify(questionFrameFor(pending.rpcId, pending, true)))
+          }
         })
     }
 
@@ -1202,8 +1374,51 @@ const plugin = {
     })
     log('session/event listener attached')
 
+    const questionMuxTask = api.events && typeof api.events.mux === 'function' && typeof api.respond === 'function'
+      ? (async () => {
+          try {
+            for await (const envelope of api.events.mux(
+              { rpcId: crypto.randomUUID(), payload: {} },
+              questionMuxAbort.signal,
+            )) {
+              const payload = envelope && envelope.payload
+              if (!payload || typeof payload.type !== 'string') continue
+              if (payload.type === 'question/requested') {
+                const rpcId = String(envelope.rpcId)
+                const pending = {
+                  rpcId,
+                  sessionId: String(payload.sessionId),
+                  questions: payload.questions,
+                }
+                pendingQuestions.set(rpcId, pending)
+                broadcastQuestionFrame(questionFrameFor(rpcId, pending))
+                log(`question requested: rpcId=${rpcId} session=${pending.sessionId} questions=${Array.isArray(pending.questions) ? pending.questions.length : 0}`)
+              } else if (payload.type === 'question/resolved') {
+                const questionRpcId = String(payload.questionRpcId)
+                pendingQuestions.delete(questionRpcId)
+                broadcastQuestionFrame({
+                  kind: 'question-resolved',
+                  rpcId: questionRpcId,
+                  sessionId: String(payload.sessionId),
+                  outcome: payload.outcome,
+                })
+                log(`question resolved: rpcId=${questionRpcId} session=${payload.sessionId} outcome=${payload.outcome}`)
+              }
+            }
+          } catch (error) {
+            if (!questionMuxAbort.signal.aborted) {
+              log(`question mux failed: ${error && error.message ? error.message : String(error)}`)
+            }
+          }
+        })()
+      : null
+    if (questionMuxTask) log('api gateway question mux attached')
+    else log('api gateway question mux unavailable; human-in-the-loop disabled')
+
     ctx.effect(() => () => {
       if (waitTimer) clearTimeout(waitTimer)
+      questionMuxAbort.abort()
+      pendingQuestions.clear()
       disposeUpgrade()
       disposeMgmt()
       disposeEvents()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-plugin-mobile-gateway",
-  "version": "0.4.2",
+  "version": "0.6.0",
   "description": "Persistent WebSocket gateway for dsh: exposes /ws/mobile on the web server and forwards agent session output to connected mobile clients",
   "main": "lib/index.mjs",
   "files": [