dsh-mobilecode 0.3.0 → 0.4.0

package/README.md

@@ -115,6 +115,17 @@ The Devices pane shows a real-time mirror of the attached device. It is produced
   key (`~/.dsh/mobilecode/stream-access.key`, `0600`), expiring within 10 minutes
   and re-minted automatically. Coordinates are normalized 0..1 of the streamed
   frame, so one mapping serves every rotation.
+
+**Multimodal screenshots**
+
+When the routed model declares image input, `device_screen` delivers the
+screenshot **as an image block** — the model literally sees the screen instead of
+reading a file path. This mirrors the in-tree `read_image` tool: the PNG is
+committed to DSH's durable attachment store (`ctx.get('attachments').saveImage`)
+and returned as a `{type:'image', attachment}` content block, gated on
+`llm.resolveModelInfo(...).inputModalities`. It **degrades, never refuses**: a
+text-only route, a headless profile, or a host without the attachment store keeps
+the plain JSON summary (path + UI tree + OCR) with no new error.
 - `device_log` — device logs: logcat `main`/`crash`/`events`/`kernel` buffers
   (kernel = dmesg, needs adb root — works on emulators) with an optional
   case-insensitive substring filter, capped line count.

package/lib/index.js

@@ -21,6 +21,7 @@ import * as UiTree from './uitree.js'
 import * as FrameSource from './frame-source.js'
 import * as StreamAccess from './stream-access.js'
 import { AndroidStreamHost, ROTATION_CYCLE } from './android-stream.js'
+import * as Vision from './vision.js'
 import { DevicePreviewEngine } from './device-preview.js'
 import * as Setup from './setup.js'
 import { registerMobileSkill } from './skill.js'
@@ -1120,7 +1121,7 @@ function deviceStreamTool(host, access) {
   })
 }
 
-function deviceScreenTool(engine) {
+function deviceScreenTool(engine, vision) {
   return defineTool({
     name: 'device_screen',
     description: 'See what is on an attached Android device right now: captures the screen as a PNG file, dumps the UI ' +
@@ -1177,6 +1178,7 @@ function deviceScreenTool(engine) {
             },
           },
           ocrError: { type: 'string' },
+          image: Vision.IMAGE_REF_SCHEMA,
         },
       },
       render: (_args, value) => {
@@ -1200,10 +1202,13 @@ function deviceScreenTool(engine) {
           if (v.ocr.length > 40) lines.push(`  … and ${v.ocr.length - 40} more`)
         }
         if (!v.ui?.length && !v.ocr?.length) lines.push('No text found on screen.')
-        return [{ type: 'text', text: lines.join('\n') }]
+        const blocks = [{ type: 'text', text: lines.join('\n') }]
+        // When the routed model accepts images, the screenshot rides along as a
+        // real image block so the model SEES the screen (see lib/vision.js).
+        return Vision.appendImageBlock(blocks, v)
       },
     },
-    async execute(args) {
+    async execute(args, exec) {
       const serial = await requireAndroidDevice(args.serial)
       const png = await DeviceBuild.screenCapture(serial, args.directory)
       const [ui, foreground, size] = await Promise.all([
@@ -1227,6 +1232,8 @@ function deviceScreenTool(engine) {
           else out.ocrError = 'PaddleOCR returned no text (or failed silently)'
         }
       }
+      const image = await Vision.maybeAttachScreenshot(vision, png, exec)
+      if (image !== undefined) out.image = image
       return out
     },
   })
@@ -1604,6 +1611,7 @@ export function apply(ctx, config) {
   const engine = new DevicePreviewEngine()
   const streamHost = new AndroidStreamHost()
   const streamAccess = new StreamAccess.StreamAccessController()
+  const vision = Vision.resolveVisionServices(ctx)
   const handle = {
     engine,
     stream: streamHost,
@@ -1644,7 +1652,7 @@ export function apply(ctx, config) {
       const disposers = [
         deviceRunTool(engine, config),
         deviceDetectTool(engine, config),
-        deviceScreenTool(engine),
+        deviceScreenTool(engine, vision),
         deviceUiTreeTool(),
         deviceTapElementTool(),
         deviceWaitForTool(),

package/lib/vision.js

@@ -0,0 +1,130 @@
+/**
+ * dsh-mobilecode — native multimodal delivery.
+ *
+ * When the routed model declares image input, the capture tools hand the model
+ * the screenshot ITSELF (a `{type:'image', attachment}` block) instead of only a
+ * file path it would have to open. DSH 0.1.1 carries images end to end: tool
+ * results may contain image blocks, bytes live in the durable attachment store
+ * (`ctx.get('attachments')`), and `llm.resolveModelInfo(...).inputModalities`
+ * says whether the routed model accepts images. This mirrors the in-tree
+ * `read_image` tool in dsh-tool-fs.
+ *
+ * The deliberate difference from `read_image`: where that tool REFUSES on a
+ * text-only route (the image is its whole point), the capture tools here
+ * DEGRADE. The primary output is always the JSON summary; the image block is an
+ * enhancement added only when (a) the attachment store is mounted, (b) the
+ * calling route's resolved model declares `image` input, and (c) admission
+ * succeeds. Any failure in that chain silently keeps the text-only behavior, so
+ * text-only routes, headless profiles, and older hosts never see a new error.
+ *
+ * Everything is typed structurally — the plugin is plain JS and must not depend
+ * on the host's attachment type exports.
+ * @module vision
+ */
+
+import { readFile } from 'node:fs/promises'
+import path from 'node:path'
+
+/**
+ * Resolve the optional vision services from the plugin context. Both come back
+ * absent on hosts that do not mount them; every consumer treats that as
+ * "stay text-only".
+ */
+export function resolveVisionServices(ctx) {
+  const get = typeof ctx?.get === 'function' ? ctx.get.bind(ctx) : undefined
+  if (get === undefined) return {}
+  const attachments = get('attachments')
+  const llm = get('llm')
+  return {
+    ...(attachments !== undefined && typeof attachments.saveImage === 'function' ? { attachments } : {}),
+    ...(llm !== undefined && typeof llm.resolveModelInfo === 'function' ? { llm } : {}),
+  }
+}
+
+/**
+ * True when the calling route's resolved model declares `image` input. Mirrors
+ * `read_image`'s gate (request-header config first, then agent options) but
+ * answers false instead of throwing: a tool result that enters durable history
+ * must not carry an image its route cannot replay.
+ */
+export async function imageInputActive(services, exec) {
+  if (services.llm === undefined || services.attachments === undefined) return false
+  try {
+    const routed = exec?.agent?.session?.requestHeader?.()?.config
+    const provider = routed?.provider ?? exec?.agent?.options?.provider
+    const model = routed?.model ?? exec?.agent?.options?.model
+    if (provider === undefined || model === undefined) return false
+    const info = await services.llm.resolveModelInfo(provider, model, exec?.signal)
+    return info?.inputModalities?.includes('image') === true
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Durably commit one screenshot PNG and return the plain reference for the
+ * result value, or undefined when the store is absent or admission fails
+ * (oversized, malformed) — never an error, per the degrade-not-refuse rule.
+ */
+export async function saveScreenshotAttachment(services, png, name) {
+  const attachments = services.attachments
+  if (attachments === undefined) return undefined
+  try {
+    const ref = await attachments.saveImage({ data: png, mediaType: 'image/png', name })
+    if (typeof ref?.attachmentId !== 'string' || ref.attachmentId === '') return undefined
+    return {
+      attachmentId: ref.attachmentId,
+      mediaType: ref.mediaType,
+      bytes: ref.bytes,
+      width: ref.width,
+      height: ref.height,
+      ...(ref.name === undefined ? {} : { name: ref.name }),
+    }
+  } catch {
+    return undefined
+  }
+}
+
+/**
+ * Convenience for the capture tools: gate on the route, read the file, and save
+ * the attachment — returning undefined (degrade) on any miss. Never throws.
+ */
+export async function maybeAttachScreenshot(services, filePath, exec) {
+  if (services.attachments === undefined || typeof filePath !== 'string' || filePath === '') return undefined
+  if (!(await imageInputActive(services, exec))) return undefined
+  try {
+    const data = await readFile(filePath)
+    return await saveScreenshotAttachment(services, data, path.basename(filePath))
+  } catch {
+    return undefined
+  }
+}
+
+/** Output-schema fragment for the optional `image` result field. */
+export const IMAGE_REF_SCHEMA = {
+  type: 'object',
+  additionalProperties: false,
+  description: 'Durable attachment reference for the screenshot delivered to the model as an image block '
+    + '(present only when the routed model declares image input).',
+  properties: {
+    attachmentId: { type: 'string', required: true },
+    mediaType: { type: 'string', required: true },
+    bytes: { type: 'number', required: true },
+    width: { type: 'number', required: true },
+    height: { type: 'number', required: true },
+    name: { type: 'string' },
+  },
+}
+
+/**
+ * Append the image block to a render's content blocks when the value carries an
+ * `image` ref — so an image-capable model SEES the screen. Returns the same
+ * array for chaining.
+ */
+export function appendImageBlock(blocks, value) {
+  const image = value?.image
+  if (image !== undefined && typeof image.attachmentId === 'string') {
+    blocks.push({ type: 'image', attachment: image })
+  }
+  return blocks
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-mobilecode",
   "description": "MobileCode for the dsh web GUI: detect iOS/Android projects, run serve-sim / serve-avd preview servers, and build-install-launch the app on the simulator or emulator from the session — plus agent tools (device_run, device_detect). Hot-pluggable — mounted via the profile bundle list + cordis.patch.yml, no dsh source changes.",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "type": "module",
   "packageManager": "pnpm@11.22.0",
   "engines": {