dsh-whale-live 0.2.0 → 0.2.2

package/CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog
+
+## 0.2.2 — 2026-11-12
+
+### Added
+- **Real-time watching.** `whale_live action=watch` starts a 4-second danmaku poll; when new messages arrive the plugin calls the Agent's `followup()` to wake the model, so it can answer while the stream is live. `watch step=stop` (or closing the session) stops it.
+- The plugin now captures the calling Agent from tool execution so it can be woken later.
+
+### Notes
+- If `followup` is unavailable in a future DSH version, the plugin falls back to `inject` (context only, no wake) instead of failing the loop.
+## 0.2.1 — 2026-11-11
+
+### Fixed
+- **Mouth sync failed on non-ASCII data paths.** `miniaudio` could not decode the synthesized mp3 when the audio folder contained Chinese characters (`failed to decode file, -7`), so speech fell back to silent, mouth-less playback. The mp3 is now copied to an ASCII temp path before decoding.
+- `whale_say` now surfaces the mouth-sync error instead of quietly omitting the "mouth moved" marker.
+
+### Added
+- `whale_live action=vts`: trigger VTube Studio hotkeys / toggle expressions / set the mouth while live, so the model can act on its own judgement.
+
+### Known issues in 0.2.0
+- Mouth movement during plugin speech could silently fail whenever `dataDir` contained non-ASCII characters.
+- No plugin-side way to trigger expressions or hotkeys during a stream.

package/index.js

@@ -15,6 +15,7 @@ import { createStore } from './lib/store.js';
 import { createDanmaku } from './lib/bili.js';
 import { createVoice } from './lib/voice.js';
 import { createAvatar } from './lib/avatar.js';
+import { createVts } from './lib/vts.js';
 import { createStage } from './lib/stage.js';
 import { diagnose } from './lib/diagnostics.js';
 import { createTools } from './lib/tools.js';
@@ -53,9 +54,10 @@ export async function apply(ctx, config = {}) {
   store.ensure();
   const danmaku = createDanmaku(settings, ctx.logger);
   const avatar = createAvatar(settings, ctx.logger);
+  const vts = createVts(settings, ctx.logger);
   const stage = createStage(settings, avatar, ctx.logger);
   const voice = createVoice(settings, store, ctx.logger, avatar);
-  const tools = createTools({ ctx, store, danmaku, voice, avatar, stage, diagnose, config: settings });
+  const tools = createTools({ ctx, store, danmaku, voice, avatar, vts, stage, diagnose, config: settings });
 
   ctx.effect(() => {
     const disposers = [];

package/lib/tools.js

@@ -7,6 +7,9 @@
  * 这里只写直播那个“她”的东西；cortex 由本鲸自己动手写。
  */
 
+import fs from 'node:fs';
+import path from 'node:path';
+
 const TEXT_OUTPUT = Object.freeze({
   schema: {
     type: 'object',
@@ -31,7 +34,84 @@ function timeOf(iso) {
   }
 }
 
-export function createTools({ ctx, store, danmaku, voice, avatar, stage, diagnose, config }) {
+export function createTools({ ctx, store, danmaku, voice, avatar, vts, stage, diagnose, config }) {
+  let lastAgent = null;
+  let watchTimer = null;
+  let watchBusy = false;
+
+  function stopWatch() {
+    if (watchTimer) {
+      clearInterval(watchTimer);
+      watchTimer = null;
+    }
+  }
+
+  /** 叫醒本鲸：优先 followup；签名不对就换一种；都不行就 inject（只进上下文不唤醒）。 */
+  function wakeAgent(text) {
+    const agent = lastAgent;
+    if (!agent) return false;
+    try {
+      agent.followup(text);
+      return true;
+    } catch {
+      try {
+        agent.followup({ role: 'user', content: text });
+        return true;
+      } catch {
+        try {
+          agent.inject(text);
+        } catch {
+          /* 叫不醒也不能影响轮询 */
+        }
+        return false;
+      }
+    }
+  }
+
+  async function watchTick() {
+    if (watchBusy || !store.state().session) return;
+    watchBusy = true;
+    try {
+      const { messages } = await danmaku.pollOnce();
+      if (!messages.length) return;
+      for (const message of messages) {
+        store.appendEvent({ kind: 'danmaku', user: message.user, text: message.text, self: message.self === true });
+      }
+      const lines = messages.map((message) => `${message.user}：${message.text}`).join('\n');
+      wakeAgent(`[直播间新弹幕]\n${lines}\n\n（去看看，想接就接；不想接也可以记一笔。）`);
+    } catch {
+      /* 轮询失败下一轮再来 */
+    } finally {
+      watchBusy = false;
+    }
+  }
+
+  /** 探测 Agent 上有什么方法（只记录名字和类型，不调用）。 */
+  function probeAgent(agent) {
+    if (!agent) return;
+    lastAgent = agent;
+    try {
+      const own = Object.getOwnPropertyNames(agent);
+      const proto = Object.getPrototypeOf(agent);
+      const protoNames = proto ? Object.getOwnPropertyNames(proto) : [];
+      const types = {};
+      for (const name of [...new Set([...own, ...protoNames])]) {
+        try {
+          types[name] = typeof agent[name];
+        } catch {
+          types[name] = 'unreadable';
+        }
+      }
+      fs.writeFileSync(
+        path.join(store.paths.dataDir, 'agent-probe.json'),
+        JSON.stringify({ when: new Date().toISOString(), own, proto: protoNames, types }, null, 2),
+        'utf8',
+      );
+    } catch {
+      /* 探测失败不影响正常使用 */
+    }
+  }
+
   const roomName = config.roomName || '直播间';
   const roomId = config.roomId;
 
@@ -84,7 +164,7 @@ export function createTools({ ctx, store, danmaku, voice, avatar, stage, diagnos
       properties: {
         action: {
           type: 'string',
-          enum: ['open', 'status', 'read', 'note', 'listen', 'close', 'propose', 'memory_list', 'commit', 'reject', 'avatar', 'stage', 'diagnose'],
+          enum: ['open', 'status', 'read', 'note', 'listen', 'close', 'propose', 'memory_list', 'commit', 'reject', 'avatar', 'vts', 'stage', 'diagnose', 'watch'],
           description: '要做什么。',
         },
         kind: {
@@ -143,6 +223,22 @@ export function createTools({ ctx, store, danmaku, voice, avatar, stage, diagnos
           enum: ['status', 'open', 'wait'],
           description: 'avatar：看状态 / 让 Steam 拉起 VTS / 等 8001 就绪。',
         },
+        hotkey: {
+          type: 'string',
+          description: 'vts：触发的热键名（例如 脸红、爱心眼、流泪、＞＜、前倾、尾巴开关）。',
+        },
+        expression: {
+          type: 'string',
+          description: 'vts：要开关的表情文件名（例如 脸红、黑脸）。',
+        },
+        expressionOn: {
+          type: 'boolean',
+          description: 'vts：表情开（默认）还是关。',
+        },
+        mouth: {
+          type: 'number',
+          description: 'vts：直接把 MouthOpen 设成 0~1（调试用）。',
+        },
       },
     },
     output: TEXT_OUTPUT,
@@ -150,7 +246,8 @@ export function createTools({ ctx, store, danmaku, voice, avatar, stage, diagnos
     timeoutMs: 130000,
     presentCall: () => ({ card: 'generic', title: '鲸鱼直播台', kind: 'other' }),
 
-    async execute(args = {}) {
+    async execute(args = {}, exec = {}) {
+      probeAgent(exec?.agent);
       const action = args.action;
 
       if (action === 'open') {
@@ -248,6 +345,22 @@ export function createTools({ ctx, store, danmaku, voice, avatar, stage, diagnos
         return { text: `收到 ${result.messages.length} 条：\n${lines.join('\n')}\n\n接下来看你怎么接。` };
       }
 
+      if (action === 'watch') {
+        const step = args.step || 'start';
+        if (step === 'stop') {
+          stopWatch();
+          return { text: 'watch 已停。' };
+        }
+        if (step === 'test') {
+          const ok = wakeAgent('[测试] 本鲸被叫醒了。');
+          return { text: ok ? '叫醒测试已发出，看看本鲸会不会醒。' : '叫醒失败：没有 agent 或 followup 不可用。' };
+        }
+        if (!store.state().session) return { text: '先 action=open 开一场再 watch。' };
+        stopWatch();
+        watchTimer = setInterval(watchTick, 4000);
+        return { text: 'watch 已开：每 4 秒看一次弹幕，有新话就叫醒本鲸；下播/stop 会停。' };
+      }
+
       if (action === 'close') {
         const meta = store.closeSession({ text: args.text });
         if (!meta) return { text: '现在没有开着的场。' };
@@ -305,6 +418,27 @@ export function createTools({ ctx, store, danmaku, voice, avatar, stage, diagnos
         }
       }
 
+      if (action === 'vts') {
+        try {
+          if (args.hotkey) {
+            const result = await vts.hotkey(args.hotkey);
+            return { text: result.ok ? `热键已触发：${args.hotkey}` : `热键没触发：${result.error || result.err || '未知'}` };
+          }
+          if (args.expression) {
+            const on = args.expressionOn !== false;
+            const result = await vts.expression(args.expression, on);
+            return { text: result.ok ? `表情 ${args.expression} 已${on ? '开' : '关'}` : `表情失败：${result.error || result.err || '未知'}` };
+          }
+          if (args.mouth !== undefined && args.mouth !== null) {
+            const result = await vts.mouth(args.mouth);
+            return { text: result.ok ? `嘴型已设为 ${Number(args.mouth) || 0}` : `嘴型失败：${result.error || result.err || '未知'}` };
+          }
+          return { text: '要给 hotkey / expression / mouth 其中一个。' };
+        } catch (error) {
+          return { text: `VTS 操作出错：${error instanceof Error ? error.message : String(error)}` };
+        }
+      }
+
       if (action === 'stage') {
         const result = await stage.stage();
         const lines = result.steps.map((step) => `${step.ok ? '✓' : step.optional ? '·' : '✗'} ${step.text}`);
@@ -377,7 +511,10 @@ export function createTools({ ctx, store, danmaku, voice, avatar, stage, diagnos
       if (!result.ok) return { text: `没念出来：${result.error}` };
       if (session) store.appendEvent({ kind: 'say', text });
       const seconds = Math.round((result.ms ?? 0) / 100) / 10;
-      return { text: `${result.interrupted ? '（上一句被掐断）' : ''}念了 ${seconds} 秒${result.avatar ? '（皮套跟着动了嘴）' : ''}：${text}${session ? `\n（已记进本场，第 ${store.state().session?.says ?? '?'} 句）` : ''}` };
+      const mouth = result.avatar
+        ? '（皮套跟着动了嘴）'
+        : `（嘴型没连上${result.vtsError ? `：${result.vtsError.replace(/\s+/g, ' ').slice(0, 160)}` : ''}）`;
+      return { text: `${result.interrupted ? '（上一句被掐断）' : ''}念了 ${seconds} 秒${mouth}：${text}${session ? `\n（已记进本场，第 ${store.state().session?.says ?? '?'} 句）` : ''}` };
     },
   };
 

package/lib/voice.js

@@ -79,15 +79,8 @@ export function createVoice(config = {}, store = null, logger = null, avatar = n
       return { ok: false, busy: true, error: '上一句还在说（要打断请带 interrupt）' };
     }
 
-    let useVts = false;
-    if (vtubeDir) {
-      try {
-        const status = avatar ? await avatar.status() : { up: false };
-        useVts = status.up === true;
-      } catch {
-        useVts = false;
-      }
-    }
+    // 不靠端口探测决定：只要配了 vtubeDir 就带上，say.py 自己连不上会安静回退。
+    const useVts = Boolean(vtubeDir) && fs.existsSync(path.join(vtubeDir, 'vts_speak.py'));
 
     cleanup();
     fs.mkdirSync(audioDir, { recursive: true });
@@ -164,6 +157,7 @@ export function createVoice(config = {}, store = null, logger = null, avatar = n
             ms: payload.ms ?? Date.now() - started,
             interrupted: payload.interrupted === true,
             avatar: payload.avatar === true,
+            vtsError: payload.avatar === true ? '' : stderr.trim().slice(-240),
             text,
           });
           return;

package/lib/vts.js

@@ -0,0 +1,65 @@
+/**
+ * VTube Studio 的“手”：热键、表情、嘴型。
+ * 走 vtube-bridge 里现成的 vts_client.py，不重复造轮子。
+ */
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+export function createVts(config = {}, logger = null) {
+  const dir = String(config.vtubeDir || '');
+  const pythonPath = String(config.pythonPath || '');
+
+  function runCli(args, timeoutMs = 10000) {
+    return new Promise((resolve) => {
+      const script = dir ? path.join(dir, 'vts_client.py') : '';
+      if (!dir || !pythonPath || !fs.existsSync(script)) {
+        resolve({ ok: false, error: '没配 vtubeDir / pythonPath，或找不到 vts_client.py' });
+        return;
+      }
+      const child = spawn(pythonPath, [script, ...args], {
+        cwd: dir,
+        windowsHide: true,
+        env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' },
+      });
+      let out = '';
+      let err = '';
+      child.stdout?.setEncoding('utf8');
+      child.stderr?.setEncoding('utf8');
+      child.stdout?.on('data', (chunk) => { out += chunk; });
+      child.stderr?.on('data', (chunk) => { err += chunk; });
+      const killer = setTimeout(() => {
+        try {
+          child.kill();
+        } catch {
+          /* 已退出 */
+        }
+      }, timeoutMs);
+      child.on('error', (error) => {
+        clearTimeout(killer);
+        resolve({ ok: false, error: error.message });
+      });
+      child.on('close', (code) => {
+        clearTimeout(killer);
+        resolve({ ok: code === 0, out: out.trim().slice(-600), err: err.trim().slice(-300) });
+      });
+    });
+  }
+
+  return {
+    async hotkey(name) {
+      const result = await runCli(['hotkey', String(name ?? '')]);
+      logger?.info?.(`whale-live: VTS hotkey ${name} ok=${result.ok}`);
+      return { ...result, action: `hotkey:${name}` };
+    },
+    async expression(name, on = true) {
+      const result = await runCli(['expr', String(name ?? ''), on ? 'on' : 'off']);
+      logger?.info?.(`whale-live: VTS expr ${name} ${on ? 'on' : 'off'} ok=${result.ok}`);
+      return { ...result, action: `expr:${name}:${on ? 'on' : 'off'}` };
+    },
+    async mouth(value) {
+      const result = await runCli(['mouth', String(Number(value) || 0)]);
+      return { ...result, action: `mouth:${Number(value) || 0}` };
+    },
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name":  "dsh-whale-live",
-    "version":  "0.2.0",
+    "version":  "0.2.2",
     "description":  "Left-sidebar live console for DeepSeek Harness: persona script, stream log, Bilibili danmaku, edge-tts speech, optional VTube Studio/OBS/直播姬 launch helpers and a memory inbox. Windows only.",
     "type":  "module",
     "main":  "./index.js",
@@ -21,7 +21,8 @@
                   "locale/*.json",
                   "icon.svg",
                   "README.md",
-                  "LICENSE"
+                  "LICENSE",
+                  "CHANGELOG.md"
               ],
     "dsh":  {
                 "manifestVersion":  1,

package/runtime/say.py

@@ -195,11 +195,24 @@ def main():
 
     mouth = None
     if args.vts_dir:
+        # miniaudio 在 Windows 上打不开含中文的路径（error -7），
+        # 解码前先复制到英文临时文件，播放也用这份。
+        import shutil
+        import tempfile
+
+        ascii_path = os.path.join(tempfile.gettempdir(), "dsh-whale-say-%d.mp3" % os.getpid())
+        try:
+            shutil.copyfile(path, ascii_path)
+            probe_path = ascii_path
+        except Exception:
+            probe_path = path
         mouth = Mouth(args.vts_dir)
         if mouth.vts is not None:
-            if not mouth.load(path):
+            if not mouth.load(probe_path):
                 print("[vts] 包络算不出来，只出声：%s" % mouth.error, file=sys.stderr)
                 mouth = None
+            else:
+                path = probe_path
         else:
             print("[vts] 皮套没连上，只出声：%s" % mouth.error, file=sys.stderr)
             mouth = None