dsh-phone-control 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 dsh-phone-control contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,61 @@
-# Temporary Holding Version
+# dsh-phone-control
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+A DeepSeek Harness plugin that drives a real Android phone: read the screen as an accessibility **node tree**,
+tap by **text**, type into fields, screenshot only when it helps.
+
+Docs: this file (English) · [简体中文](../README.zh-CN.md)
+
+## Why
+
+A 1080×2400 screenshot is 1–2 MB of PNG. The node tree for the same screen is 1–3 KB of JSON with labels,
+view ids and clickable flags — usually a few hundred tokens instead of tens of thousands.
+
+## Install
+
+```bash
+npm install dsh-phone-control --prefix ~/.dsh/profiles/desktop
+```
+
+Add `"dsh-phone-control"` to `dsh.profile.bundles` in that profile's `package.json`, then restart DSH.
+The settings panel appears under *Settings → Phone bridge*.
+
+## Pair in five steps
+
+1. Install the Android app from the repository's releases page.
+2. Enable its **Accessibility service**.
+3. Open *Settings → Phone bridge* in DSH and read the 8-digit code.
+4. Type the code into the app and tap **Connect**.
+5. The panel turns green; the phone keeps a 30-day token and reconnects by itself afterwards.
+
+## Tools
+
+| Tool | What it does |
+| --- | --- |
+| `phone_ui_dump` | Accessibility tree as JSON (`maxDepth`, `maxNodes`, `includeInvisible`) |
+| `phone_tap_text` | Click the widget matching text / content-description / view-id |
+| `phone_set_text` | Write into a field located by hint or view-id |
+| `phone_screenshot` | PNG of the current screen |
+| `phone_tap` / `phone_long_press` / `phone_swipe` | Coordinate input |
+| `phone_input` / `phone_key` | Type text; send `back` / `home` / `recents` / `notifications` |
+| `phone_open_app` / `phone_current_app` / `phone_list_apps` | Launch, identify, enumerate apps |
+| `phone_status` / `phone_pair_code` / `phone_disconnect` | Listener state, pairing code, drop session |
+
+## Settings panel
+
+Pairing code with countdown and a *New code* button, listening address/port (`0` = auto), bind address,
+paired devices, open sessions, failed attempts, and a *Disconnect* button. Port changes rebind immediately and
+persist to `<profile>/phone-bridge.json`.
+
+## Protocol
+
+Binary frames (`DSHP`, 19-byte header, 8 MB cap). Handshake: `HELLO` → `CHALLENGE` → `AUTH` → `OK`, session key
+from ephemeral ECDH (prime256v1) + PBKDF2-stretched pairing code through HKDF, HMAC proofs both ways, then
+AES-256-GCM with the sequence number in the nonce. Only the SHA-256 of the pairing token is stored.
+
+## Node version
+
+Requires Node.js 20 or newer. Runtime dependency: `ws`.
+
+## License
+
+MIT

package/cordis.patch.yml

@@ -0,0 +1,10 @@
+- insert:
+    - id: phone-control
+      name: dsh-phone-control
+      config:
+        # 0 = 启动时自动挑一个空闲端口（8000–9999），挑中的端口会写入
+        # <profile>/phone-bridge.json 并在之后的启动里沿用。
+        # 想固定端口就改成具体数字，例如 8788。
+        port: 0
+        bindHost: "0.0.0.0"
+        autoCode: true

package/index.js

@@ -0,0 +1,379 @@
+import { readFile } from 'node:fs/promises'
+import { fileURLToPath } from 'node:url'
+import { SpikeService } from './lib/service.js'
+import { BASE_PATH, createHandler } from './lib/routes.js'
+
+export const name = 'dsh-phone-control'
+export const inject = ['tools', 'skills']
+
+const OUTPUT = {
+  schema: { type: 'object', additionalProperties: true },
+  render: (_args, value) => {
+    const blocks = [{ type: 'text', text: JSON.stringify(value, null, 2) }]
+    const image = value?.image
+    if (image !== undefined && image !== null && typeof image.attachmentId === 'string') {
+      blocks.push({ type: 'image', attachment: image })
+    }
+    return blocks
+  },
+}
+
+const SKILL_DIR = new URL('./skills/phone-control/', import.meta.url)
+const SKILL_PATH = new URL('./skills/phone-control/SKILL.md', import.meta.url)
+
+const CANDIDATE = {
+  name: 'phone-control',
+  description: '通过 DSH 手机桥操控已配对的 Android 手机：截图、坐标点击、长按、滑动、输入文本、返回/桌面/最近任务、打开应用。',
+  invocation: {
+    modelInvocable: true,
+    userInvocable: true,
+  },
+  provider: 'dsh-phone-control',
+  source: 'bundled',
+  resourceBase: { kind: 'directory', path: fileURLToPath(SKILL_DIR) },
+  rank: 50,
+  locator: fileURLToPath(SKILL_PATH),
+}
+
+const provider = {
+  name: 'dsh-phone-control',
+  async list() {
+    return [CANDIDATE]
+  },
+  async get(candidate) {
+    if (candidate?.name !== 'phone-control') return undefined
+    const { rank: _rank, locator: entryPath, ...summary } = candidate
+    const raw = await readFile(entryPath, 'utf8')
+    const body = raw.replace(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/u, '').trim()
+    return {
+      ...summary,
+      content: body,
+    }
+  },
+}
+
+const NUMBER = { type: 'number' }
+
+function makeTool({ name, description, properties = {}, required = [], timeoutMs = 20000, present, run }) {
+  return {
+    name,
+    description,
+    parameters: {
+      type: 'object',
+      properties,
+      required,
+      additionalProperties: false,
+    },
+    output: OUTPUT,
+    isConcurrencySafe: () => false,
+    timeoutMs,
+    async execute(args) {
+      return run(args ?? {})
+    },
+    presentCall() {
+      return present
+    },
+  }
+}
+
+async function imageResult(ctx, raw) {
+  const encoded = String(raw?.imageB64 ?? '')
+  if (encoded.length === 0) throw new Error('PHONE_IMAGE: 手机没有返回截图数据')
+  const data = Buffer.from(encoded, 'base64')
+  if (data.length === 0) throw new Error('PHONE_IMAGE: 截图数据为空')
+  let attachments
+  try {
+    attachments = ctx.get('attachments')
+  } catch {
+    attachments = undefined
+  }
+  if (attachments === undefined || typeof attachments.saveImage !== 'function') {
+    throw new Error('ATTACHMENTS_UNAVAILABLE: DSH 附件服务不可用')
+  }
+  const ref = await attachments.saveImage({
+    data,
+    mediaType: 'image/png',
+    name: `phone-${Date.now()}.png`,
+  })
+  return {
+    ok: true,
+    width: raw.width ?? ref.width,
+    height: raw.height ?? ref.height,
+    bytes: data.length,
+    capturedAt: raw.capturedAt ?? Date.now(),
+    image: {
+      attachmentId: ref.attachmentId,
+      mediaType: ref.mediaType,
+      bytes: ref.bytes,
+      width: raw.width ?? ref.width,
+      height: raw.height ?? ref.height,
+      ...(ref.name === undefined ? {} : { name: ref.name }),
+    },
+  }
+}
+
+export function apply(ctx, config) {
+  const service = new SpikeService({ ctx, config })
+  ctx.provide('phoneBridgeSpike', service)
+
+  // ---- settings page transport -------------------------------------------
+  // Acquired through ctx.inject rather than a module-level `inject` entry: a
+  // profile without a web server (a TUI profile) must still load this plugin,
+  // and ctx.inject parks only this child fiber instead of the whole plugin.
+  try {
+    ctx.inject(['webServer'], (webCtx) => {
+      webCtx.effect(
+        () => webCtx.webServer.register({
+          kind: 'prefix',
+          path: BASE_PATH,
+          handler: createHandler(service, { logger: ctx.logger }),
+        }),
+        'dsh-phone-control: settings API',
+      )
+    })
+  } catch (error) {
+    ctx.logger?.warn?.(
+      `[phone-control] 没有可用的 web 服务，设置页面已禁用: ${error?.message ?? String(error)}`,
+    )
+  }
+
+  const definitions = [
+    makeTool({
+      name: 'phone_status',
+      description: '查看手机桥状态：是否监听、手机是否已连接、配对码、端口和待处理请求数。',
+      timeoutMs: 10000,
+      present: { card: 'generic', title: '手机桥状态', kind: 'read' },
+      async run() {
+        return service.status()
+      },
+    }),
+    makeTool({
+      name: 'phone_pair_code',
+      description: '查看当前 8 位配对码、监听端口和绑定地址；在手机 App 的“配对码”中输入这 8 位数字。传 refresh=true 可强制换一个新码（并解除错误次数锁定）。',
+      properties: {
+        refresh: { type: 'boolean', description: '强制生成新的 8 位配对码并解除锁定，默认 false' },
+      },
+      timeoutMs: 10000,
+      present: { card: 'generic', title: '配对码', kind: 'read' },
+      async run(args) {
+        if (args?.refresh === true) service.refreshCode(true)
+        let s = service.status()
+        if (args?.refresh !== true && Date.now() > s.codeExpiresAt && s.lockedUntil === null) {
+          service.refreshCode()
+          s = service.status()
+        }
+        return {
+          code: s.pairingCode,
+          port: s.port,
+          bindHost: s.bindHost,
+          paired: s.paired,
+          connected: s.connected,
+          expiresAt: s.codeExpiresAt,
+          ttlSec: Math.max(0, Math.round((s.codeExpiresAt - Date.now()) / 1000)),
+          failedAttempts: s.failedAttempts,
+          lockedUntil: s.lockedUntil,
+          lockedRemainSec: s.lockedUntil === null ? 0 : Math.max(0, Math.round((s.lockedUntil - Date.now()) / 1000)),
+        }
+      },
+    }),
+    makeTool({
+      name: 'phone_screenshot',
+      description: '截取手机当前屏幕并返回 PNG 图片。坐标类工具应使用这张图片的像素坐标。',
+      timeoutMs: 30000,
+      present: { card: 'generic', title: '手机截图', kind: 'read' },
+      async run() {
+        const raw = await service.request('screenshot', {}, 25000)
+        return imageResult(ctx, raw)
+      },
+    }),
+    makeTool({
+      name: 'phone_ui_dump',
+      description:
+        '读取手机当前屏幕的控件树（无障碍节点）：text / contentDescription / view-id / bounds / 是否可点击可编辑。比截图小 1–2 个数量级、且不受分辨率与动画影响；想「看屏幕上有什么」优先用它，只有节点树为空（游戏、Canvas、部分 Flutter 界面）时才回退 phone_screenshot。',
+      properties: {
+        maxDepth: { type: 'number', description: '最大深度，默认 30' },
+        maxNodes: { type: 'number', description: '最多返回多少个节点，默认 400' },
+        includeInvisible: { type: 'boolean', description: '是否包含不可见节点，默认 false' },
+      },
+      timeoutMs: 20000,
+      present: { card: 'generic', title: '手机控件树', kind: 'read' },
+      async run(args) {
+        return service.request(
+          'ui_dump',
+          {
+            maxDepth: Number(args?.maxDepth ?? 30),
+            maxNodes: Number(args?.maxNodes ?? 400),
+            includeInvisible: args?.includeInvisible === true,
+          },
+          15000,
+        )
+      },
+    }),
+    makeTool({
+      name: 'phone_tap_text',
+      description:
+        '按文字点击手机屏幕上的控件（优先无障碍 ACTION_CLICK，自绘控件退回按 bounds 中心手势点击）。比「先截图再换算坐标」更稳更省，是点击按钮/列表项的首选。',
+      properties: {
+        text: { type: 'string', description: '要点击的文字 / contentDescription / view-id 片段' },
+        exact: { type: 'boolean', description: '是否要求完全相等，默认 false（包含匹配）' },
+        index: { type: 'number', description: '命中多个时取第几个（从 0 开始），默认 0，可点击的优先' },
+      },
+      required: ['text'],
+      timeoutMs: 15000,
+      present: { card: 'generic', title: '按文字点击', kind: 'action' },
+      async run(args) {
+        return service.request(
+          'tap_text',
+          { text: String(args.text ?? ''), exact: args?.exact === true, index: Number(args?.index ?? 0) },
+          10000,
+        )
+      },
+    }),
+    makeTool({
+      name: 'phone_set_text',
+      description:
+        '把文字直接写入输入框（无障碍 ACTION_SET_TEXT，不经键盘）：不传 target 就写进当前聚焦的输入框，传 target 则先按文字定位那个输入框。失败会返回 NOT_EDITABLE，此时改用 phone_input。',
+      properties: {
+        text: { type: 'string', description: '要写入的文字（会替换原内容）' },
+        target: { type: 'string', description: '可选：输入框的提示文字 / view-id 片段，用于定位' },
+      },
+      required: ['text'],
+      timeoutMs: 20000,
+      present: { card: 'generic', title: '写入输入框', kind: 'action' },
+      async run(args) {
+        return service.request(
+          'set_text',
+          { text: String(args.text ?? ''), ...(args?.target === undefined ? {} : { target: String(args.target) }) },
+          15000,
+        )
+      },
+    }),
+    makeTool({
+      name: 'phone_tap',
+      description: '点击手机屏幕上的一个坐标。坐标系与 phone_screenshot 返回图片的像素坐标一致。',
+      properties: { x: NUMBER, y: NUMBER },
+      required: ['x', 'y'],
+      timeoutMs: 15000,
+      present: { card: 'generic', title: '手机点击', kind: 'action' },
+      async run(args) {
+        return service.request('tap', { x: Number(args.x), y: Number(args.y), durationMs: 50 }, 10000)
+      },
+    }),
+    makeTool({
+      name: 'phone_long_press',
+      description: '在手机屏幕坐标上长按，默认 800ms。',
+      properties: { x: NUMBER, y: NUMBER, durationMs: { type: 'number', minimum: 100, maximum: 10000 } },
+      required: ['x', 'y'],
+      timeoutMs: 15000,
+      present: { card: 'generic', title: '手机长按', kind: 'action' },
+      async run(args) {
+        return service.request(
+          'long_press',
+          { x: Number(args.x), y: Number(args.y), durationMs: Number(args.durationMs ?? 800) },
+          10000,
+        )
+      },
+    }),
+    makeTool({
+      name: 'phone_swipe',
+      description: '从 (x1,y1) 滑动到 (x2,y2)，默认 300ms。',
+      properties: {
+        x1: NUMBER,
+        y1: NUMBER,
+        x2: NUMBER,
+        y2: NUMBER,
+        durationMs: { type: 'number', minimum: 50, maximum: 10000 },
+      },
+      required: ['x1', 'y1', 'x2', 'y2'],
+      timeoutMs: 15000,
+      present: { card: 'generic', title: '手机滑动', kind: 'action' },
+      async run(args) {
+        return service.request(
+          'swipe',
+          {
+            x1: Number(args.x1),
+            y1: Number(args.y1),
+            x2: Number(args.x2),
+            y2: Number(args.y2),
+            durationMs: Number(args.durationMs ?? 300),
+          },
+          10000,
+        )
+      },
+    }),
+    makeTool({
+      name: 'phone_input',
+      description: '向手机当前获得焦点的输入框写入文本（替换原内容）。失败时返回 INPUT_UNSUPPORTED。',
+      properties: { text: { type: 'string' } },
+      required: ['text'],
+      timeoutMs: 20000,
+      present: { card: 'generic', title: '手机输入', kind: 'action' },
+      async run(args) {
+        return service.request('input', { text: String(args.text ?? '') }, 15000)
+      },
+    }),
+    makeTool({
+      name: 'phone_key',
+      description: '发送全局按键：back（返回）、home（桌面）、recents（最近任务）、notifications（通知栏）。',
+      properties: { key: { type: 'string', enum: ['back', 'home', 'recents', 'notifications'] } },
+      required: ['key'],
+      timeoutMs: 10000,
+      present: { card: 'generic', title: '手机按键', kind: 'action' },
+      async run(args) {
+        return service.request('key', { key: String(args.key) }, 8000)
+      },
+    }),
+    makeTool({
+      name: 'phone_open_app',
+      description: '通过包名打开手机上的应用。',
+      properties: { package: { type: 'string' } },
+      required: ['package'],
+      timeoutMs: 15000,
+      present: { card: 'generic', title: '打开手机应用', kind: 'action' },
+      async run(args) {
+        return service.request('open_app', { package: String(args.package) }, 10000)
+      },
+    }),
+    makeTool({
+      name: 'phone_current_app',
+      description: '返回手机当前前台应用的包名和界面标题。',
+      timeoutMs: 10000,
+      present: { card: 'generic', title: '手机前台应用', kind: 'read' },
+      async run() {
+        return service.request('current_app', {}, 8000)
+      },
+    }),
+    makeTool({
+      name: 'phone_list_apps',
+      description: '列出手机已安装且可启动的应用（包名与名称）。',
+      timeoutMs: 15000,
+      present: { card: 'generic', title: '手机应用列表', kind: 'read' },
+      async run() {
+        return service.request('list_apps', {}, 12000)
+      },
+    }),
+    makeTool({
+      name: 'phone_disconnect',
+      description: '从电脑端断开当前手机连接；手机通知会变为未连接。',
+      timeoutMs: 10000,
+      present: { card: 'generic', title: '断开手机桥', kind: 'action' },
+      async run() {
+        const closed = service.disconnectAll()
+        return { ok: true, closed, status: service.status() }
+      },
+    }),
+  ]
+
+  for (const definition of definitions) {
+    ctx.effect(() => ctx.tools.register(definition), `dsh-phone-control: ${definition.name}`)
+  }
+
+  ctx.skills.registerProvider(() => provider)
+
+  ctx.effect(() => {
+    void service.start().catch((error) => {
+      ctx.logger?.warn?.(`[phone-control] start failed: ${error?.message ?? String(error)}`)
+    })
+    return () => service.dispose()
+  }, 'dsh-phone-control: tcp listener')
+}