agentus 0.0.0-stage → 0.1.0

package/packages/server/src/dashscope.ts

@@ -0,0 +1,235 @@
+// DashScope (阿里百炼) voice adapter.
+//
+// Shapes verified against the operator's own MaaS endpoint on 2026-10-02 (probe scripts
+// in ~/.hermes/cache/agentus/night):
+//   TTS   POST {root}/api/v1/services/audio/tts/SpeechSynthesizer
+//         {model:"qwen-audio-3.0-tts-flash", input:{text, voice:"longanhuan_v3.6",
+//         format:"wav", sample_rate}} → output.audio.url (fetch server-side, return bytes)
+//   ASR   POST {compat}/chat/completions with an input_audio base64 data URL
+//         (qwen3-asr-flash) → choices[0].message.content; hotwords ride in a system
+//         message (the compat surface has no vocabulary parameter)
+//   ASR流  WSS {root}/api-ws/v1/inference, run-task/finish-task duplex events, binary
+//         PCM frames after task-started, result-generated carries partial + final
+//         sentences. Supports instant hotwords (parameters.vocabulary) and context.
+//
+// The base URL is the operator's — every Bailian workspace id is different, and the
+// whole point is that this is NOT the coding-plan endpoint. No URLs or keys are baked in.
+import WebSocket from "ws";
+import { getSettings, dashscopeUrls } from "./settings.js";
+
+export class DashscopeError extends Error {
+  constructor(message: string, readonly code: "not_configured" | "upstream" | "timeout" | "bad_audio") {
+    super(message);
+  }
+}
+
+const TIMEOUT_MS = 45_000;
+
+function creds(): { key: string; root: string; compat: string; ws: string } {
+  const v = getSettings().voice;
+  if (v.provider !== "dashscope" || !v.baseUrl || !v.apiKey) {
+    throw new DashscopeError("dashscope voice is not configured (settings page: endpoint + API key)", "not_configured");
+  }
+  const urls = dashscopeUrls(v.baseUrl);
+  return { key: v.apiKey, ...urls };
+}
+
+async function fetchJson(url: string, init: RequestInit, label: string): Promise<Response> {
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), TIMEOUT_MS);
+  try {
+    const res = await fetch(url, { ...init, signal: ac.signal });
+    if (!res.ok) {
+      const detail = await res.text().catch(() => "");
+      throw new DashscopeError(`${label} ${res.status}: ${detail.slice(0, 300) || res.statusText}`, "upstream");
+    }
+    return res;
+  } catch (e) {
+    if ((e as Error).name === "AbortError") throw new DashscopeError(`${label} timed out after ${TIMEOUT_MS / 1000}s`, "timeout");
+    throw e;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** Text → audio bytes. Non-streaming: the response carries a 24h OSS URL; we fetch it
+ *  server-side so the browser only ever talks to Agentus (and the key never pairs with
+ *  a cross-origin request from the client). */
+export async function dashscopeTts(text: string, opts: { voice?: string; speed?: number } = {}): Promise<{
+  contentType: string;
+  audio: Buffer;
+}> {
+  const { key, root } = creds();
+  const v = getSettings().voice;
+  const rate = Math.min(2, Math.max(0.5, opts.speed ?? 1));
+  const res = await fetchJson(`${root}/api/v1/services/audio/tts/SpeechSynthesizer`, {
+    method: "POST",
+    headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+    body: JSON.stringify({
+      model: v.ttsModel,
+      input: {
+        text,
+        voice: opts.voice || v.ttsVoice,
+        format: v.ttsFormat,
+        sample_rate: v.ttsFormat === "wav" ? 24000 : undefined,
+        ...(rate !== 1 ? { rate } : {}),
+      },
+    }),
+  }, "dashscope tts");
+  const data = (await res.json()) as { output?: { audio?: { url?: string; data?: string } } };
+  const audio = data?.output?.audio;
+  if (audio?.data) {
+    return { contentType: mimeOf(v.ttsFormat), audio: Buffer.from(audio.data, "base64") };
+  }
+  if (!audio?.url) throw new DashscopeError(`dashscope tts returned no audio: ${JSON.stringify(data).slice(0, 200)}`, "upstream");
+  // the url is http:// on some tenants; fetch as-is (same cloud, short-lived, signed)
+  const dl = await fetchJson(audio.url, { method: "GET" }, "dashscope tts download");
+  return { contentType: dl.headers.get("content-type") ?? mimeOf(v.ttsFormat), audio: Buffer.from(await dl.arrayBuffer()) };
+}
+
+function mimeOf(format: string): string {
+  return format === "wav" ? "audio/wav" : format === "opus" ? "audio/ogg" : "audio/mpeg";
+}
+
+/** Hotwords for the compat route: a compact system message. The docs describe it as
+ *  "背景文本和实体词表" — exactly the context-enhancement channel, and it is what makes
+ *  "Agentus"/"网关" survive instead of "安逸 slot"/"万关". */
+export function hotwordContext(words: string[]): string {
+  if (!words.length) return "";
+  return `本次语音可能涉及的词汇表（识别时可优先匹配）：${words.slice(0, 60).join("、")}。`;
+}
+
+/** One-shot transcription (the stop-and-finalise path, and the batch model).
+ *  `audio` is raw bytes; `mime` decides the data URL prefix the model is told. */
+export async function dashscopeAsr(audio: Buffer, mime: string, opts: { contextWords?: string[] } = {}): Promise<string> {
+  const { key, compat } = creds();
+  const v = getSettings().voice;
+  const dataUrl = `data:${mime || "audio/wav"};base64,${audio.toString("base64")}`;
+  const messages: unknown[] = [];
+  const ctx = hotwordContext(opts.contextWords ?? []);
+  if (ctx) messages.push({ role: "system", content: [{ type: "text", text: ctx }] });
+  messages.push({ role: "user", content: [{ type: "input_audio", input_audio: { data: dataUrl } }] });
+  const res = await fetchJson(`${compat}/chat/completions`, {
+    method: "POST",
+    headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+    body: JSON.stringify({ model: v.asrBatchModel, messages, asr_options: { enable_itn: true } }),
+  }, "dashscope asr");
+  const data = (await res.json()) as { choices?: { message?: { content?: string } }[] };
+  return String(data?.choices?.[0]?.message?.content ?? "").trim();
+}
+
+/**
+ * A live streaming recognition task over the DashScope WebSocket.
+ *
+ * The browser cannot talk to this endpoint itself: the auth header on a WebSocket
+ * handshake is forbidden in browsers, and the raw key must not leave the server anyway.
+ * So the cockpit server owns the upstream socket and the browser relays PCM through our
+ * own /ws/asr (see index.ts). Events:
+ *   onPartial(text)  — running sentence, replaces the previous partial
+ *   onFinal(text)    — a completed sentence (sentence_end)
+ */
+export function openDashscopeStream(opts: {
+  /** instant hotwords WITH weights: {"词": 1..5}, 50 = super-hotword */
+  vocabulary?: Record<string, number>;
+  /** plain word list, used when no weighted vocabulary is available */
+  contextWords?: string[];
+  /** context enhancement: recent lines of the conversation being dictated into */
+  context?: { role: "user" | "assistant"; text: string }[];
+  onPartial: (text: string) => void;
+  onFinal: (text: string) => void;
+  onError: (message: string) => void;
+  onClose: () => void;
+}): { sendAudio: (chunk: Buffer) => void; stop: () => void; started: Promise<void> } {
+  const { key, ws } = creds();
+  const v = getSettings().voice;
+  const socket = new WebSocket(ws, { headers: { authorization: `Bearer ${key}` } });
+  const taskId = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 10)}`;
+  let startedResolve: (() => void) | null = null;
+  let startedReject: ((e: Error) => void) | null = null;
+  const started = new Promise<void>((resolve, reject) => { startedResolve = resolve; startedReject = reject; });
+  let closing = false;
+  // "done" must be exactly once and must not depend on WHO closed the socket: the first
+  // relay had the finalise path suppressed (closing=true swallowed the close), so the
+  // browser sat waiting for an end that never came.
+  let done = false;
+  const notifyDone = (): void => {
+    if (done) return;
+    done = true;
+    opts.onClose();
+  };
+  const vocab: Record<string, number> = { ...(opts.vocabulary ?? {}) };
+  for (const w of opts.contextWords ?? []) if (!(w in vocab)) vocab[w] = 4;
+
+  socket.on("open", () => {
+    socket.send(JSON.stringify({
+      header: { action: "run-task", task_id: taskId, streaming: "duplex" },
+      payload: {
+        task_group: "audio", task: "asr", function: "recognition", model: v.asrModel,
+        parameters: { format: "pcm", sample_rate: 16000, ...(Object.keys(vocab).length ? { vocabulary: vocab } : {}) },
+        // context enhancement: what the operator has been talking about. Only sent when
+        // we have it: an empty array is not the same request as "no context".
+        input: opts.context?.length ? { context: opts.context.slice(0, 5) } : {},
+      },
+    }));
+  });
+  socket.on("message", (raw: Buffer | string) => {
+    let msg: { header?: { event?: string; error_message?: string }; payload?: { output?: { sentence?: { text?: string; sentence_end?: boolean }; text?: string } } };
+    try {
+      msg = JSON.parse(String(raw)) as typeof msg;
+    } catch {
+      return;
+    }
+    const event = msg?.header?.event ?? "";
+    if (event === "task-started") { startedResolve?.(); startedResolve = null; return; }
+    if (event === "task-failed") {
+      const m = `dashscope asr: ${msg?.header?.error_message ?? "task failed"}`;
+      startedReject?.(new Error(m)); startedReject = null;
+      opts.onError(m);
+      return;
+    }
+    if (event === "result-generated") {
+      const sent = msg?.payload?.output?.sentence;
+      const text = String(sent?.text ?? msg?.payload?.output?.text ?? "");
+      if (!text) return;
+      if (sent?.sentence_end) opts.onFinal(text);
+      else opts.onPartial(text);
+      return;
+    }
+    if (event === "task-finished") {
+      // the API's own end-of-task event: report it now rather than waiting for the socket
+      startedReject?.(new Error("task finished before start"));
+      startedReject = null;
+      notifyDone();
+      return;
+    }
+  });
+  socket.on("error", (e: Error) => {
+    startedReject?.(e); startedReject = null;
+    opts.onError(`dashscope asr socket: ${e.message}`);
+  });
+  socket.on("close", notifyDone);
+
+  const timeout = setTimeout(() => {
+    startedReject?.(new Error("dashscope asr: task-started timed out"));
+    startedReject = null;
+  }, 10_000);
+  started.catch(() => { /* surfaced via onError/stop below */ }).finally(() => void 0);
+  // clear the timer once started succeeds; reject path already rejected the promise
+  void started.then(() => clearTimeout(timeout), () => { clearTimeout(timeout); try { socket.close(); } catch { /* gone */ } });
+
+  return {
+    sendAudio(chunk: Buffer): void {
+      if (socket.readyState === socket.OPEN) socket.send(chunk, { binary: true });
+    },
+    stop(): void {
+      closing = true;
+      try {
+        if (socket.readyState === socket.OPEN) {
+          socket.send(JSON.stringify({ header: { action: "finish-task", task_id: taskId, streaming: "duplex" }, payload: { input: {} } }));
+          setTimeout(() => { try { socket.close(); } catch { /* already */ } }, 2000);
+        }
+      } catch { /* already closed */ }
+    },
+    started,
+  };
+}

package/packages/server/src/fs.ts

@@ -0,0 +1,165 @@
+// Directory browsing for the new-slot workspace picker.
+//
+// The operator already has the right to point an agent at any directory (that is the
+// product), so this adds no new power — it just makes the choice discoverable instead
+// of demanding a path typed from memory. Listing is deliberately narrow: directories
+// only, one level at a time, capped, and never followed through symlinks we cannot stat.
+import fs from "node:fs";
+import { homedir } from "node:os";
+import path from "node:path";
+
+export interface DirEntry {
+  name: string;
+  path: string;
+  /** "dir" for directories, "file" for everything else the panel can show */
+  kind: "dir" | "file";
+  /** bytes, files only (the panel prints a human size) */
+  size?: number;
+}
+
+export interface DirListing {
+  /** the directory that was listed (absolute, symlink-resolved) */
+  path: string;
+  /** parent directory, or null at the root */
+  parent: string | null;
+  home: string;
+  entries: DirEntry[];
+  /** hit the entry cap — the UI says so instead of pretending the list is complete */
+  truncated: boolean;
+}
+
+const MAX_ENTRIES = 300;
+
+/** `~`, `~/x`, relative and absolute all land on an absolute path. */
+export function expandPath(input: string | null | undefined): string {
+  const raw = String(input ?? "").trim();
+  if (!raw) return homedir();
+  if (raw === "~") return homedir();
+  if (raw.startsWith("~/")) return path.resolve(path.join(homedir(), raw.slice(2)));
+  return path.resolve(raw);
+}
+
+export class FsError extends Error {
+  constructor(message: string, readonly code: "not_found" | "not_a_directory" | "not_a_file" | "permission_denied" | "bad_path") {
+    super(message);
+  }
+}
+
+export function listDirs(input: string | null | undefined, opts: { includeFiles?: boolean } = {}): DirListing {
+  const target = expandPath(input);
+  let st: fs.Stats;
+  try {
+    st = fs.statSync(target); // stat (not lstat): a symlinked workspace should work
+  } catch (e) {
+    const code = (e as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") throw new FsError(`no such directory: ${target}`, "not_found");
+    if (code === "EACCES") throw new FsError(`permission denied: ${target}`, "permission_denied");
+    throw new FsError(`cannot read ${target}: ${code ?? String(e)}`, "bad_path");
+  }
+  if (!st.isDirectory()) throw new FsError(`not a directory: ${target}`, "not_a_directory");
+
+  let names: fs.Dirent[] = [];
+  try {
+    names = fs.readdirSync(target, { withFileTypes: true });
+  } catch (e) {
+    const code = (e as NodeJS.ErrnoException).code;
+    if (code === "EACCES") throw new FsError(`permission denied: ${target}`, "permission_denied");
+    throw new FsError(`cannot list ${target}: ${code ?? String(e)}`, "bad_path");
+  }
+
+  const entries: DirEntry[] = [];
+  for (const d of names) {
+    // Hidden directories are the bulk of a home directory and almost never the answer;
+    // the path input still accepts them when someone really wants one.
+    if (d.name.startsWith(".")) continue;
+    const full = path.join(target, d.name);
+    let isDir = d.isDirectory();
+    // a symlink to a directory is a normal way to organise work: stat it
+    if (!isDir && d.isSymbolicLink()) {
+      try {
+        isDir = fs.statSync(full).isDirectory();
+      } catch {
+        continue; // dangling symlink or unreadable target: skip
+      }
+    }
+    if (isDir) {
+      entries.push({ name: d.name, path: full, kind: "dir" });
+      continue;
+    }
+    // The file panel (head ▤) wants files too; the picker (new slot) must not
+    // offer one as a working directory, so it stays opt-in.
+    if (opts.includeFiles && !d.isSymbolicLink()) {
+      let size: number | undefined;
+      try {
+        size = fs.statSync(full).size;
+      } catch {
+        /* unreadable: still list it, size unknown */
+      }
+      entries.push({ name: d.name, path: full, kind: "file", size });
+    }
+  }
+  entries.sort((a, b) => (
+    a.kind === b.kind
+      ? a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: "base" })
+      : a.kind === "dir" ? -1 : 1  // directories first, like every file manager
+  ));
+
+  const truncated = entries.length > MAX_ENTRIES;
+  const trimmed = truncated ? entries.slice(0, MAX_ENTRIES) : entries;
+  const parent = path.dirname(target);
+  return {
+    path: target,
+    parent: parent === target ? null : parent,
+    home: homedir(),
+    entries: trimmed,
+    truncated,
+  };
+}
+
+export interface FilePreview {
+  path: string;
+  name: string;
+  size: number;
+  /** decoded utf-8 text, cut at maxBytes (the tail is reported, never faked) */
+  content: string;
+  truncated: boolean;
+  /** looks like binary: the panel shows a "cannot preview" note instead of mojibake */
+  binary: boolean;
+}
+
+const DEFAULT_PREVIEW_BYTES = 256 * 1024;
+
+/** Read a file for the workspace panel. Read-only on purpose: this panel is a
+ *  browser, not an editor — writing files is the agent's job, not the cockpit's. */
+export function readTextFile(input: string, opts: { maxBytes?: number } = {}): FilePreview {
+  const target = expandPath(input);
+  const maxBytes = Math.max(1024, opts.maxBytes ?? DEFAULT_PREVIEW_BYTES);
+  let st: fs.Stats;
+  try {
+    st = fs.statSync(target);
+  } catch (e) {
+    const code = (e as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") throw new FsError(`no such file: ${target}`, "not_found");
+    if (code === "EACCES") throw new FsError(`permission denied: ${target}`, "permission_denied");
+    throw new FsError(`cannot read ${target}: ${code ?? String(e)}`, "bad_path");
+  }
+  if (st.isDirectory()) throw new FsError(`not a file: ${target}`, "not_a_file");
+  const fd = fs.openSync(target, "r");
+  try {
+    const len = Math.min(st.size, maxBytes);
+    const buf = Buffer.alloc(len);
+    fs.readSync(fd, buf, 0, len, 0);
+    // A NUL byte in the first 8KB is the classic binary tell.
+    const binary = buf.subarray(0, Math.min(buf.length, 8192)).includes(0);
+    return {
+      path: target,
+      name: path.basename(target),
+      size: st.size,
+      content: binary ? "" : buf.toString("utf8"),
+      truncated: st.size > len,
+      binary,
+    };
+  } finally {
+    fs.closeSync(fd);
+  }
+}

package/packages/server/src/hotwords.ts

@@ -0,0 +1,132 @@
+// Hotwords: fixed + dynamic, both feeding the same place (the ASR request).
+//
+// DashScope gives two channels and we use them where each is strongest:
+//  * instant `vocabulary` ({"词": 权重}) on the streaming WebSocket — decoded-side bias,
+//    exactly the mechanism the operator named ("固定热词");
+//  * a system-message word list on the OpenAI-compatible batch route — the docs'
+//    "实体词表" context channel, because that surface has no vocabulary parameter.
+//
+// "根据上下文动态生成高频热词": the cockpit knows the real context — the session's own
+// transcript (and the workspace names). We rank recent message text by term frequency
+// with a cheap CJK+latin tokenizer, keep plausible terms (length, noise words, code
+// identifiers), and merge them under the fixed list (fixed wins, and fixed entries may
+// carry =weight, including DashScope's weight-50 super-hotwords).
+import { getSettings } from "./settings.js";
+import type { Store } from "./store/store.js";
+
+export interface Hotword {
+  word: string;
+  weight: number;
+  origin: "fixed" | "dynamic";
+}
+
+/** Words that are frequent but useless as hotwords: they bias toward text that is
+ *  already the default output. */
+const STOP = new Set([
+  "的", "了", "是", "在", "我", "你", "他", "她", "它", "我们", "你们", "他们", "这", "那", "有", "没有",
+  "一个", "可以", "什么", "怎么", "这个", "那个", "就是", "然后", "因为", "所以", "如果", "但是",
+  "the", "and", "for", "with", "that", "this", "from", "have", "will", "your", "you", "are", "was",
+  "not", "but", "all", "can", "get", "out", "its", "into", "then", "than", "also", "use", "used",
+  "using", "useful", "here", "there", "when", "what", "which", "while", "please", "thanks",
+]);
+
+/** Parse "词" or "词=4" (weight 1..5, or the super-hotword 50). */
+export function parseHotword(raw: string): Hotword | null {
+  const t = raw.trim();
+  if (!t) return null;
+  const m = /^(.*?)(?:\s*=\s*(\d+))?$/s.exec(t);
+  const word = (m?.[1] ?? t).trim();
+  if (!word) return null;
+  let weight = Number(m?.[2] ?? 4);
+  if (!Number.isFinite(weight)) weight = 4;
+  weight = weight >= 50 ? 50 : Math.min(5, Math.max(1, Math.round(weight)));
+  return { word, weight, origin: "fixed" };
+}
+
+const WORD_RE = /[A-Za-z][A-Za-z0-9_+.-]{2,}|[一-龥]{2,8}/g;
+
+/** Term-frequency over the operator's recent text, filtered and ranked. Returns at most
+ *  `limit` terms; fixed hotwords always survive the cap separately. */
+export function extractDynamicHotwords(texts: string[], limit = 20): Hotword[] {
+  const counts = new Map<string, { n: number; sample: string }>();
+  for (const text of texts) {
+    for (const m of text.matchAll(WORD_RE)) {
+      const raw = m[0];
+      const lower = raw.toLowerCase();
+      if (STOP.has(lower) || STOP.has(raw)) continue;
+      // ascii runs: require a bit of substance (a path segment, an identifier), not noise
+      if (/^[A-Za-z]/.test(raw)) {
+        if (raw.length > 24) continue;
+        // skip obvious code-only tokens like "return", file suffixes
+        if (/^[\w.-]*\.(ts|js|py|md|json|yaml|yml|sh|tsx|jsx)$/.test(lower)) continue;
+      } else if (raw.length > 8) {
+        continue;
+      }
+      const prev = counts.get(raw);
+      if (prev) prev.n += 1;
+      else counts.set(raw, { n: 1, sample: raw });
+    }
+  }
+  const ranked = [...counts.entries()]
+    .filter(([, v]) => v.n >= 2)
+    .sort((a, b) => b[1].n - a[1].n || a[0].localeCompare(b[0]))
+    .slice(0, limit);
+  return ranked.map(([word, v]) => ({
+    word,
+    weight: v.n >= 5 ? 5 : 4,
+    origin: "dynamic" as const,
+  }));
+}
+
+/** Recent transcript text to mine hotwords from: the given session first (its messages
+ *  are the conversation the operator is dictating INTO), then the other sessions so a
+ *  term used across the cockpit today also surfaces. */
+export function contextTexts(store: Store, sessionId?: string, budgetChars = 24_000): string[] {
+  const out: string[] = [];
+  let used = 0;
+  const take = (rows: { kind: string; payload: unknown }[], cap: number): void => {
+    let n = 0;
+    for (const r of rows) {
+      if (n >= cap || used > budgetChars) break;
+      const p = r.payload as { text?: string; title?: string; content?: string };
+      const t = String(p?.text ?? p?.title ?? p?.content ?? "");
+      if (t.length > 1 && t.length < 4000) {
+        out.push(t);
+        used += t.length;
+        n += 1;
+      }
+    }
+  };
+  if (sessionId) take(store.messagesTail(sessionId, 60).messages, 40);
+  for (const s of store.listSessions(false)) {
+    if (s.id === sessionId) continue;
+    take(store.messagesTail(s.id, 12).messages, 8);
+  }
+  return out;
+}
+
+/** The full list an ASR request should carry: fixed first (they may use =50), dynamic
+ *  filling the rest up to the configured limit. */
+export function hotwordsFor(store: Store, sessionId?: string): Hotword[] {
+  const v = getSettings().voice;
+  const fixed = v.hotwords.map(parseHotword).filter((h): h is Hotword => Boolean(h));
+  const seen = new Set(fixed.map((h) => h.word));
+  const limit = Math.max(0, Math.min(200, v.hotwordLimit));
+  const out: Hotword[] = fixed.slice(0, limit);
+  if (v.dynamicHotwords) {
+    const dyn = extractDynamicHotwords(contextTexts(store, sessionId), Math.max(0, limit - out.length));
+    for (const h of dyn) {
+      if (seen.has(h.word)) continue;
+      seen.add(h.word);
+      out.push(h);
+    }
+  }
+  return out;
+}
+
+/** DashScope's instant-hotword payload: {词: 权重}. */
+export function vocabularyOf(words: Hotword[]): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const h of words) out[h.word] = h.weight;
+  return out;
+}