@openbrt/audioctl 0.1.3 → 0.1.4

package/README.md

@@ -1,11 +1,13 @@
 # @openbrt/audioctl
 
-Generic WeClawBot audio-output control CLI and AI-agent plugin.
+Generic WeClawBot audio/voice endpoint control CLI and AI-agent plugin.
 
 It is meant for user-owned agents such as Codex, Claude Code, OpenClaw, Hermes,
 WorkBuddy, Gemini CLI, or any shell-capable agent runtime. It is not tied to one
-speaker brand. Any ESP32/Linux audio device that implements the WeClawBot MQTT
-audio command envelope can use the same package.
+speaker brand. Any ESP32/Linux audio or voice endpoint that implements the
+WeClawBot MQTT/app envelope can use the same package. Music playback is one app
+on top of the endpoint; the same VM/capability model can later host TTS, voice
+remote control, smart-home control, and voice interaction apps.
 
 ## Tell your agent
 
@@ -15,11 +17,17 @@ audio command envelope can use the same package.
 
 Agent contract:
 
-- If already bound, run the high-level task directly with `audioctl workbgm --play --json`.
+- If already bound, inspect the device and run the high-level task directly.
 - If not bound, ask the user only for the short binding code, then bind and continue.
-- If NetEase Music authorization is expired, tell the user to reopen the speaker
-  provisioning/authorization entry on their phone and complete NetEase Music
-  authorization. After the user says “已授权”, retry the task.
+- Preferred BYOA flow: use a legal music platform API in the agent environment
+  to build a playable queue, then push it with `audioctl queue QUEUE.json --play --json`.
+  NetEase Cloud Music Open Platform can be used this way to request the highest
+  available quality URLs. Spotify should be driven through official Spotify
+  Connect/Web Playback SDK or existing-user playback control; do not assume the
+  Spotify Web API gives full-track direct audio URLs.
+- If music-platform authorization is missing, guide the user through the
+  platform authorization entry available to the agent/phone, then continue after
+  the user says “已授权”.
 - Never ask the user to paste AppSecret, PrivateKey, accessToken, refreshToken,
   Wi-Fi passwords, MQTT credentials, or other secrets into chat.
 
@@ -53,6 +61,7 @@ audioctl volumeup
 audioctl volumedown
 audioctl status
 audioctl inspect --json
+audioctl queue ./work-bgm.queue.json --play
 audioctl workbgm --play
 audioctl rule ./my-playlist-rule.json
 audioctl deepnight
@@ -64,16 +73,24 @@ existing MQTT channel. Agent scripts can read Wi-Fi connection state, BLE
 provisioning state, wake-word capability, playback, VM/app status, and health
 without receiving Wi-Fi passwords, MQTT credentials, or music-platform secrets.
 
+`queue --play` is the generic agent-built music path. The agent uses the user's
+authorized music provider outside the speaker, writes a
+`weclawbot.playable_queue.v1`/`schemaVersion: 1` queue containing only safe media
+URLs and redacted metadata, then pushes it to the device. The queue must not
+contain platform private keys, OAuth tokens, cookies, custom request headers, or
+Wi-Fi/MQTT credentials. Expiring media URLs are allowed; refreshing them is the
+agent/app's job.
+
 `workbgm --play` installs a persistent `work-bgm-autopilot` VM app on the audio
 device, writes the bundled 工作 BGM playlist rule through the device-side music
 adapter, requests a refresh, and starts playback once. The app expresses a
 continuous BGM intent and respects later user-initiated pause/resume controls.
-If the device reports `netease_auth_refresh_failed`, the agent should stop
-retrying and tell the user to complete the NetEase Music authorization flow from
-the phone provisioning/authorization entry. The user only needs to come back and
-say “已授权”; the agent then retries `workbgm --play`. The agent must not ask
-the user to paste AppSecret, PrivateKey, accessToken, refreshToken, Wi-Fi
-passwords, or MQTT credentials into chat.
+If the device reports `netease_auth_refresh_failed`, that only means the legacy
+device-side NetEase runtime cannot refresh itself. The better agent behavior is
+to switch to the BYOA queue/app path: obtain the user's music-platform identity
+in the agent environment, build a fresh queue, and push it with `audioctl queue`.
+Only if the user explicitly wants the device-side NetEase runtime should the
+agent route them to the phone authorization entry.
 
 `deepnight` pushes the bundled “深夜工作” playlist rule and asks the device to
 refresh/play according to its local runtime. The package never contains music

package/agent-plugin.json

@@ -4,11 +4,14 @@
   "name": "audioctl",
   "package": "@openbrt/audioctl",
   "bin": "audioctl",
-  "description": "Bind and control WeClawBot-compatible audio output devices from user-owned AI agents.",
+  "description": "Bind and drive WeClawBot-compatible audio, voice, and ambient I/O devices from user-owned AI agents.",
   "transport": "weclawbot.link/mqtt",
-  "user_contract": "用户只需要给出设备身份和目标；agent 负责安装工具、绑定、执行、诊断和报告结果。",
+  "user_contract": "用户只需要给出设备身份和目标；agent 负责选择合法平台 API、完成授权引导、构建最小权限脚本/队列、推送到设备 VM、诊断和报告结果。",
   "device_classes": [
+    "audio_io",
     "audio_output",
+    "voice_endpoint",
+    "smart_home_endpoint",
     "music_player"
   ],
   "agents": [
@@ -28,9 +31,15 @@
     "audio.previous",
     "audio.seek",
     "audio.set_volume",
+    "audio.queue.write",
+    "audio.tts",
+    "voice.wakeword",
+    "voice.capture",
+    "iot.control",
     "device.inspect",
     "music.autopilot",
     "music.set_rule",
+    "music.set_queue",
     "music.work_bgm",
     "music.status",
     "app.install"
@@ -44,10 +53,18 @@
   "natural_language_tasks": [
     {
       "intent": "帮我在音箱建立一个工作 BGM 歌单，持续播放",
-      "primary_command": "audioctl workbgm --play --json",
+      "primary_command": "audioctl queue QUEUE.json --play --json",
+      "agent_strategy": "先运行 audioctl inspect --json。优先在 agent 侧使用用户授权的音乐平台 API 生成 QUEUE.json，再运行 audioctl queue QUEUE.json --play --json；网易云开放平台可生成最高可用音质播放 URL。Spotify 必须走官方 Spotify Connect/Web Playback SDK 或控制用户已有 Spotify 设备，不能假设 Web API 能提供完整歌曲直链。如果 agent 当前没有音乐平台身份，只要求用户在 agent/手机授权入口完成对应平台授权。",
+      "fallback_command": "audioctl workbgm --play --json",
       "if_unbound": "只向用户索要绑定码，然后运行 audioctl bind CODE --name AGENT；不要要求用户理解 MQTT、topic 或凭据文件。",
-      "if_netease_auth_failed": "告诉用户：请在手机上的音箱配网/授权入口重新完成网易云音乐授权，完成后回复“已授权”。不要要求用户粘贴 AppSecret、PrivateKey、accessToken、refreshToken、Wi-Fi 密码或 MQTT 凭据。",
-      "after_user_confirms_authorized": "重新运行 audioctl workbgm --play --json，然后报告是否 active/degraded。"
+      "if_music_platform_auth_missing": "告诉用户：我需要你给这个 agent 一个音乐平台身份。请在 agent 支持的网易云开放平台或 Spotify 官方授权入口完成授权；完成后只需回复“已授权”。不要把 AppSecret、PrivateKey、accessToken、refreshToken、Wi-Fi 密码或 MQTT 凭据粘贴到聊天里。",
+      "if_device_side_netease_auth_failed": "这是设备内置网易运行时的旧授权失效。优先不要让用户折腾原厂 App；改由 agent 侧使用用户授权的音乐平台 API 生成队列并推送。只有当用户明确要继续使用设备侧网易运行时时，才引导手机端重新授权。",
+      "after_user_confirms_authorized": "agent 使用已授权的平台 API 生成 QUEUE.json，运行 audioctl queue QUEUE.json --play --json，然后报告是否 active/degraded。"
+    },
+    {
+      "intent": "让音箱作为语音遥控、家庭智能或语音互动端点",
+      "primary_command": "audioctl app install MANIFEST.json APP.wasm",
+      "agent_strategy": "不要把设备假定为音乐播放器。先运行 audioctl inspect --json 读取设备能力，再生成声明最小 capabilities 的 VM app。语音、TTS、家居控制等能力必须通过 manifest capabilities 显式声明；设备侧只执行授权范围内的 I/O。"
     }
   ],
   "commands": [
@@ -75,6 +92,10 @@
       "name": "rule",
       "usage": "audioctl rule RULE.json"
     },
+    {
+      "name": "queue",
+      "usage": "audioctl queue QUEUE.json [--play]"
+    },
     {
       "name": "app install",
       "usage": "audioctl app install MANIFEST.json APP.wasm"

package/bin/audioctl.mjs

@@ -13,6 +13,7 @@ import {
   buildAppInstallControl,
   buildAgentPrompt,
   buildAudioControl,
+  buildQueueControl,
   buildWorkBgmAppInstallControl,
   expandPath,
   normalizeCredentials,
@@ -45,6 +46,7 @@ async function run(name, values) {
   if (name === "prompt") return commandPrompt(values);
   if (name === "manifest") return commandManifest();
   if (name === "app") return commandApp(values);
+  if (name === "queue") return commandQueue(values);
   if (name === "rule") return commandRule(values);
   if (name === "workbgm") return commandWorkBgm(values);
   if (name === "deepnight") return commandDeepnight(values);
@@ -63,6 +65,7 @@ function usage(exitCode = 0) {
   ${COMMAND_NAME} volume 0..100
   ${COMMAND_NAME} seek MILLISECONDS
   ${COMMAND_NAME} rule RULE.json
+  ${COMMAND_NAME} queue QUEUE.json [--play]
   ${COMMAND_NAME} app install MANIFEST.json APP.wasm
   ${COMMAND_NAME} workbgm [--play]
   ${COMMAND_NAME} deepnight
@@ -202,6 +205,30 @@ async function commandRule(values) {
   await send("set_rule", options, { rule });
 }
 
+async function commandQueue(values) {
+  const options = parseOptions(values, {
+    credentials: credentialsPath(),
+    timeout: 12,
+    json: false,
+    nowait: false,
+    play: false,
+  });
+  const file = String(options._[0] || "");
+  if (!file) throw new Error("queue_requires_file");
+  const queue = JSON.parse(await fs.readFile(file, "utf8"));
+  const credentials = await readCredentials(options.credentials);
+  const delivery = await publishAudioControl(
+    credentials,
+    buildQueueControl({ queue, play: options.play }),
+    {
+      timeoutMs: Number(options.timeout) * 1000,
+      wait: !options.nowait,
+    },
+  );
+  print(delivery.status || delivery, options.json);
+  if (delivery.status?.kind === "rejected") process.exitCode = 1;
+}
+
 async function commandApp(values) {
   const subcommand = String(values[0] || "");
   if (subcommand !== "install") {

package/lib/index.mjs

@@ -45,6 +45,8 @@ export const WORKBGM_RULE = Object.freeze({
   refreshHintSeconds: 21600,
 });
 
+export const PLAYABLE_QUEUE_SCHEMA = "weclawbot.playable_queue.v1";
+
 const WORKBGM_APP_WASM_BASE64 =
   "AGFzbQEAAAABBAFgAAADAgEABwgBBG1haW4AAAoEAQIACw==";
 
@@ -174,6 +176,15 @@ export function buildAudioControl(command, options = {}) {
   };
 }
 
+export function buildQueueControl(options = {}) {
+  return buildAudioControl("set_queue", {
+    id: options.id,
+    queue: options.queue,
+    play: options.play,
+    autoplay: options.autoplay,
+  });
+}
+
 export function buildAppInstallControl(options = {}) {
   const wasm =
     Buffer.isBuffer(options.wasm) || options.wasm instanceof Uint8Array
@@ -270,6 +281,7 @@ export function normalizeAudioCommand(command, options = {}) {
     "reload",
     "status",
     "set_rule",
+    "set_queue",
   ]);
   if (!supported.has(normalized)) throw new Error(`command_unsupported: ${command}`);
 
@@ -291,9 +303,79 @@ export function normalizeAudioCommand(command, options = {}) {
   if (normalized === "set_rule") {
     music.rule = validateRule(options.rule);
   }
+  if (normalized === "set_queue") {
+    music.queue = validatePlayableQueue(options.queue);
+    music.play = options.play === true || options.autoplay === true;
+  }
   return music;
 }
 
+export function validatePlayableQueue(queue) {
+  if (!queue || typeof queue !== "object") {
+    throw new Error("queue_invalid");
+  }
+  const schemaOk =
+    queue.schema === PLAYABLE_QUEUE_SCHEMA ||
+    queue.schemaVersion === 1;
+  if (!schemaOk || !Array.isArray(queue.tracks) || queue.tracks.length < 1) {
+    throw new Error("queue_invalid");
+  }
+  if (queue.tracks.length > 80) {
+    throw new Error("queue_too_large");
+  }
+  const normalizedTracks = queue.tracks.map((track, index) => {
+    if (!track || typeof track !== "object") {
+      throw new Error(`queue_track_invalid:${index + 1}`);
+    }
+    const forbiddenKeys = [
+      "headers",
+      "cookies",
+      "cookie",
+      "authorization",
+      "accessToken",
+      "access_token",
+      "refreshToken",
+      "refresh_token",
+      "privateKey",
+      "private_key",
+      "appSecret",
+      "app_secret",
+    ];
+    for (const key of forbiddenKeys) {
+      if (Object.hasOwn(track, key)) {
+        throw new Error(`queue_track_secret_field_blocked:${key}`);
+      }
+    }
+    const url = normalizeMediaUrl(track.url, index + 1);
+    return {
+      title: string(track.title || `Track ${index + 1}`).slice(0, 160),
+      artist: string(track.artist).slice(0, 160),
+      originalId: string(track.originalId || track.original_id || track.id || index + 1).slice(0, 120),
+      encryptedId: string(track.encryptedId || track.encrypted_id).slice(0, 160),
+      provider: string(track.provider || queue.provider || "agent").slice(0, 64),
+      durationMs: clamp(Number(track.durationMs || track.duration_ms) || 0, 0, 24 * 60 * 60 * 1000),
+      bitrate: clamp(Number(track.bitrate) || 0, 0, 999_999),
+      format: string(track.format).slice(0, 32),
+      url,
+    };
+  });
+  return {
+    schema: PLAYABLE_QUEUE_SCHEMA,
+    schemaVersion: 1,
+    title: string(queue.title || "Agent playlist").slice(0, 120),
+    source: string(queue.source || "agent-built playable queue").slice(0, 240),
+    provider: string(queue.provider || "agent").slice(0, 64),
+    generatedBy: string(queue.generatedBy || queue.generated_by || "audioctl-agent").slice(0, 120),
+    generatedAt: string(queue.generatedAt || queue.generated_at) || new Date().toISOString(),
+    expiresAt: string(queue.expiresAt || queue.expires_at).slice(0, 80),
+    expiresAtEpoch: Math.max(0, Math.floor(Number(queue.expiresAtEpoch || queue.expires_at_epoch) || 0)),
+    refreshHintSeconds: Math.max(0, Math.floor(Number(queue.refreshHintSeconds || queue.refresh_hint_seconds) || 0)),
+    loop: queue.loop !== false,
+    respectUserPause: queue.respectUserPause !== false && queue.respect_user_pause !== false,
+    tracks: normalizedTracks,
+  };
+}
+
 export function validateRule(rule) {
   if (
     !rule ||
@@ -339,7 +421,15 @@ export function normalizeAppManifest(manifest, artifact = {}) {
     "log.emit",
     "music.state.read",
     "music.rule.write",
+    "music.queue.write",
     "music.playback.write",
+    "audio.queue.write",
+    "audio.playback.write",
+    "audio.output.write",
+    "audio.tts",
+    "voice.wakeword",
+    "voice.capture",
+    "iot.control",
   ]);
   const capabilities = Array.isArray(manifest.capabilities)
     ? manifest.capabilities.map((capability) => string(capability)).filter(Boolean)
@@ -463,15 +553,17 @@ export function remediationForStatus(status) {
       user_action_required: true,
       safe_to_retry_without_user: false,
       summary:
-        "设备内的网易云用户授权已过期，refresh token 没有续上；agent 不能凭空恢复用户授权。",
+        "设备内置网易运行时的用户授权已过期。优先改走 BYOA：由 agent 使用用户授权的音乐平台 API 生成播放队列并推送到设备。",
       user_message:
-        "请在手机上打开音箱配网/授权入口，重新完成网易云音乐授权。完成后回到 agent 说“已授权”。不要发送任何密钥、token 或密码。",
-      agent_after_user_confirms: "audioctl workbgm --play --json",
+        "我需要一个可用的音乐平台身份来生成 BGM 队列。请在 agent 支持的网易云开放平台或 Spotify 官方授权入口完成授权；完成后回到 agent 说“已授权”。不要发送任何密钥、token 或密码。",
+      agent_after_user_confirms:
+        "generate QUEUE.json with the authorized music platform API, then run audioctl queue QUEUE.json --play --json",
       steps: [
-        "告诉用户：请在手机上的音箱配网/授权入口重新完成网易云音乐授权。",
-        "告诉用户：授权完成后只需要回复“已授权”。",
+        "优先使用 agent 侧 BYOA 音乐驱动：让用户在 agent 支持的网易云开放平台或 Spotify 官方授权入口完成授权。",
+        "网易云开放平台可由 agent 请求最高可用音质 URL，生成 weclawbot.playable_queue.v1 队列，然后运行 audioctl queue QUEUE.json --play --json。",
+        "Spotify 必须走官方 Spotify Connect/Web Playback SDK 或控制用户已有 Spotify 设备；不要声称 Spotify Web API 能提供完整歌曲直链。",
         "不要要求用户理解内部授权包，也不要让用户粘贴 AppSecret、PrivateKey、accessToken、refreshToken、Wi-Fi 密码或 MQTT 凭据。",
-        "用户回复已授权后，agent 自动重试 audioctl workbgm --play --json，并报告结果。",
+        "只有用户明确要求继续使用设备侧网易运行时时，才引导用户去手机端配网/授权入口重新授权，然后重试 audioctl workbgm --play --json。",
       ],
     });
   }
@@ -503,18 +595,20 @@ export function buildAgentPrompt(options = {}) {
   const expires = Number(options.expiresAt || options.expires_at || 0);
   const expiresLine = expires > 0 ? `绑定码过期时间：${new Date(expires).toISOString()}` : "";
   return [
-    `请作为我的 AI agent 绑定并控制一台 WeClawBot 音频输出设备。`,
+    `请作为我的 AI agent 绑定并控制一台 WeClawBot 音频/语音/环境 I/O 设备。`,
     `设备别名：${alias}`,
     `绑定码：${code}`,
     expiresLine,
     `请在你的运行环境安装 npm 包 ${PACKAGE_NAME}，然后执行：`,
     `npm exec --package ${PACKAGE_NAME} -- ${COMMAND_NAME} bind ${code} --name ${shellToken(agentName)}`,
     `npm exec --package ${PACKAGE_NAME} -- ${COMMAND_NAME} doctor --online`,
-    `绑定成功后，把它作为默认 BGM/音乐输出设备。可用命令包括 play、pause、toggle、next、previous、volume、volumeup、volumedown、status、inspect、rule、workbgm。`,
-    `如果我要建立“工作 BGM 歌单，持续播放”，请优先执行：npm exec --package ${PACKAGE_NAME} -- ${COMMAND_NAME} workbgm --play --json`,
-    `如果返回 health.problems 包含 netease_auth_refresh_failed，请只告诉我：请在手机上的音箱配网/授权入口重新完成网易云音乐授权，完成后回复“已授权”；不要反复重试，不要让我理解内部授权包，也不要要求我把 AppSecret、PrivateKey、accessToken 或 refreshToken 发到聊天里。`,
+    `绑定成功后，把它作为默认音频/语音/环境 I/O 端点；不要把它假定成只会播放音乐。可用命令包括 play、pause、toggle、next、previous、volume、volumeup、volumedown、status、inspect、queue、rule、workbgm、app install。`,
+    `如果我要建立“工作 BGM 歌单，持续播放”，优先由你在 agent 侧使用用户授权的音乐平台 API 生成 QUEUE.json，再执行：npm exec --package ${PACKAGE_NAME} -- ${COMMAND_NAME} queue QUEUE.json --play --json。网易云开放平台可请求最高可用音质 URL。`,
+    `如果用户选择 Spotify，必须走 Spotify 官方 Spotify Connect/Web Playback SDK 或控制用户已有 Spotify 播放设备；不要假设 Spotify Web API 能提供完整歌曲直链。`,
+    `如果你当前没有音乐平台身份，请只告诉我：我需要你在 agent 支持的网易云开放平台或 Spotify 官方授权入口完成授权；完成后回复“已授权”。不要反复重试，不要让我理解内部授权包，也不要要求我把 AppSecret、PrivateKey、accessToken 或 refreshToken 发到聊天里。`,
+    `如果返回 health.problems 包含 netease_auth_refresh_failed，说明设备侧内置网易运行时授权失效；优先改走 agent 侧 queue/app 驱动。只有我明确要求继续使用设备侧网易运行时时，才引导手机端重新授权并重试 workbgm。`,
     `如果我要播放「${profile}」，也可以执行：npm exec --package ${PACKAGE_NAME} -- ${COMMAND_NAME} deepnight`,
-    `不要向设备写入音乐平台私钥；音乐平台授权、歌单刷新和长期规则由设备侧已配置的安全运行时处理。`,
+    `不要向设备写入全局音乐平台私钥；平台授权优先保存在 agent 运行环境。只有在我明确授权时，才把最小范围、可轮换、可撤销的 app-scoped 凭据随 VM app 写入设备。`,
   ].filter(Boolean).join("\n");
 }
 
@@ -566,6 +660,55 @@ export function summarizeInspect(inspect) {
   };
 }
 
+function normalizeMediaUrl(value, trackNumber) {
+  const raw = string(value);
+  let parsed;
+  try {
+    parsed = new URL(raw);
+  } catch {
+    throw new Error(`queue_track_url_invalid:${trackNumber}`);
+  }
+  const protocol = parsed.protocol.toLowerCase();
+  const host = parsed.hostname.toLowerCase();
+  if (!new Set(["http:", "https:"]).has(protocol) || !host) {
+    throw new Error(`queue_track_url_invalid:${trackNumber}`);
+  }
+  if (!isPublicMediaHost(host)) {
+    throw new Error(`queue_track_url_host_blocked:${trackNumber}`);
+  }
+  if (protocol === "http:" && !isNeteaseMediaHost(host)) {
+    throw new Error(`queue_track_url_requires_https:${trackNumber}`);
+  }
+  return parsed.toString();
+}
+
+function isNeteaseMediaHost(host) {
+  return host === "music.126.net" || host.endsWith(".music.126.net");
+}
+
+function isPublicMediaHost(host) {
+  if (
+    host === "localhost" ||
+    host.endsWith(".localhost") ||
+    host.endsWith(".local") ||
+    host.endsWith(".lan")
+  ) {
+    return false;
+  }
+  const match = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/u);
+  if (!match) return true;
+  const parts = match.slice(1).map((part) => Number(part));
+  if (parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
+    return false;
+  }
+  const [a, b] = parts;
+  if (a === 0 || a === 10 || a === 127 || (a === 169 && b === 254)) return false;
+  if (a === 172 && b >= 16 && b <= 31) return false;
+  if (a === 192 && b === 168) return false;
+  if (a >= 224) return false;
+  return true;
+}
+
 function connectMqtt(profile, options = {}) {
   const timeoutMs = Math.max(1000, Number(options.timeoutMs) || 12_000);
   return mqtt.connect(profile.url, {

package/package.json

@@ -1,15 +1,19 @@
 {
   "name": "@openbrt/audioctl",
-  "version": "0.1.3",
-  "description": "Generic WeClawBot audio-output control CLI and AI-agent plugin.",
+  "version": "0.1.4",
+  "description": "Generic WeClawBot audio/voice endpoint control CLI and AI-agent plugin.",
   "type": "module",
   "license": "MIT",
   "keywords": [
     "weclawbot",
     "audio",
+    "voice",
     "music",
+    "tts",
+    "smart-home",
     "mqtt",
     "esp32",
+    "audio-vm",
     "agent",
     "byoa",
     "openclaw",