@downcity/plugins 1.0.56 → 1.0.59

package/src/tts/runtime/Synthesizer.ts

@@ -1,480 +0,0 @@
-/**
- * TTS 语音合成 runtime。
- *
- * 关键点（中文）
- * - 读取本地模型目录，不再依赖 console 模型池。
- * - 根据模型族选择对应 Python runner，并把输出落到本地文件。
- */
-
-import { execFile as execFileCb } from "node:child_process";
-import path from "node:path";
-import { promisify } from "node:util";
-import fs from "fs-extra";
-import type { TtsPluginConfig, TtsSynthesizeInput } from "@/tts/types/TtsPlugin.js";
-import type { TtsAudioFormat, TtsModelId } from "@/tts/types/Tts.js";
-import type { PluginCommandContext } from "@downcity/agent/internal/plugin/types/Plugin.js";
-import { renderChatMessageFileTag } from "@downcity/agent/internal/executor/messages/ChatMessageMarkup.js";
-import { getTtsModelCatalogItem, resolveTtsModelId } from "@/tts/runtime/Catalog.js";
-import { resolveTtsModelsRootDir } from "@/tts/runtime/Paths.js";
-
-const execFileAsync = promisify(execFileCb);
-const DEFAULT_TTS_TIMEOUT_MS = 300_000;
-
-function normalizeText(value: unknown): string {
-  return String(value || "").trim();
-}
-
-function normalizeFormat(value: unknown): TtsAudioFormat {
-  return normalizeText(value).toLowerCase() === "flac" ? "flac" : "wav";
-}
-
-function normalizeSpeed(value: unknown): number {
-  if (typeof value !== "number" || !Number.isFinite(value) || Number.isNaN(value)) {
-    return 1;
-  }
-  const next = Math.max(0.5, Math.min(2, value));
-  return Number(next.toFixed(2));
-}
-
-function normalizeTimeoutMs(value: unknown): number {
-  if (typeof value !== "number" || !Number.isFinite(value) || Number.isNaN(value)) {
-    return DEFAULT_TTS_TIMEOUT_MS;
-  }
-  if (value < 5_000) return 5_000;
-  if (value > 900_000) return 900_000;
-  return Math.floor(value);
-}
-
-function sanitizeFileStem(value: string): string {
-  return value
-    .replace(/[^A-Za-z0-9_-]+/g, "-")
-    .replace(/-+/g, "-")
-    .replace(/^-+|-+$/g, "")
-    .slice(0, 48) || "tts";
-}
-
-function toProjectRelativePath(projectRoot: string, targetPath: string): string | null {
-  const relative = path.relative(projectRoot, targetPath);
-  if (!relative) return null;
-  if (relative.startsWith("..")) return null;
-  if (path.isAbsolute(relative)) return null;
-  return relative.split(path.sep).join("/");
-}
-
-function resolveOutputTarget(params: {
-  context: PluginCommandContext;
-  format: TtsAudioFormat;
-  output?: string;
-  modelId: string;
-}): { absPath: string; relativePath: string } {
-  const output = normalizeText(params.output);
-  const defaultDir = path.join(params.context.paths.getCacheDirPath(), "tts");
-  const target = output
-    ? (path.isAbsolute(output)
-        ? path.normalize(output)
-        : path.resolve(params.context.rootPath, output))
-    : defaultDir;
-
-  const ext = `.${params.format}`;
-  const hasExplicitFile = Boolean(path.extname(target));
-  const filePath = hasExplicitFile
-    ? target
-    : path.join(
-        target,
-        `${Date.now()}-${sanitizeFileStem(params.modelId)}${ext}`,
-      );
-  const relativePath = toProjectRelativePath(params.context.rootPath, filePath);
-  if (!relativePath) {
-    throw new Error(`TTS output must stay inside project root: ${filePath}`);
-  }
-  return {
-    absPath: filePath,
-    relativePath,
-  };
-}
-
-function pickLastNonEmptyLine(value: string): string {
-  const lines = String(value || "")
-    .split(/\r?\n/)
-    .map((line) => line.trim())
-    .filter(Boolean);
-  return lines.length > 0 ? lines[lines.length - 1] : "";
-}
-
-/**
- * 判断一条 stderr 是否属于可忽略的 Python 推理提示。
- *
- * 关键点（中文）
- * - 某些第三方库会把 warning / info 打到 stderr，但并不影响音频文件生成。
- * - 这里只过滤已知的非致命提示，避免把真实错误静默吞掉。
- */
-function isIgnorablePythonStderrLine(value: string): boolean {
-  const line = normalizeText(value);
-  if (!line) return true;
-  if (/^Setting `pad_token_id` to `eos_token_id`/u.test(line)) return true;
-  if (/^`torch_dtype` is deprecated!/u.test(line)) return true;
-  if (/^Warning: flash-attn is not installed\./u.test(line)) return true;
-  return false;
-}
-
-/**
- * 提取 stderr 中真正需要上抛的错误文本。
- *
- * 关键点（中文）
- * - 去掉空行与已知 warning 后，只要还有剩余内容，就按真实错误处理。
- * - 返回最后一条非 warning 行，方便 CLI 给出更稳定的错误摘要。
- */
-function pickUnexpectedPythonStderr(value: string): string {
-  const lines = String(value || "")
-    .split(/\r?\n/)
-    .map((line) => line.trim())
-    .filter((line) => Boolean(line) && !isIgnorablePythonStderrLine(line));
-  return lines.length > 0 ? lines[lines.length - 1] : "";
-}
-
-/**
- * 归一化 Python stderr 摘要。
- *
- * 关键点（中文）
- * - 某些 runner 会把非致命提示打印到 stderr。
- * - 这里保留最后一条真正有意义的内容，供上层作为说明返回。
- */
-function normalizePythonStderrSummary(value: string): string | undefined {
-  const lines = String(value || "")
-    .split(/\r?\n/)
-    .map((line) => line.trim())
-    .filter(Boolean);
-  if (lines.length === 0) {
-    return undefined;
-  }
-  return lines[lines.length - 1];
-}
-
-function detectLanguageHint(input: string): "zh" | "en" {
-  return /[\u3400-\u9fff]/u.test(input) ? "zh" : "en";
-}
-
-function resolveKokoroVoicePath(params: {
-  modelDir: string;
-  voice?: string;
-  language: string;
-}): string {
-  const voicesDir = path.join(params.modelDir, "voices");
-  const requested = normalizeText(params.voice);
-  if (requested) {
-    const requestedPath = requested.endsWith(".pt")
-      ? path.resolve(voicesDir, requested)
-      : path.resolve(voicesDir, `${requested}.pt`);
-    if (fs.existsSync(requestedPath)) {
-      return requestedPath;
-    }
-  }
-
-  const preferred =
-    params.language === "zh"
-      ? ["zf_xiaoni.pt", "af_heart.pt"]
-      : ["af_heart.pt", "zf_xiaoni.pt"];
-  for (const fileName of preferred) {
-    const candidate = path.join(voicesDir, fileName);
-    if (fs.existsSync(candidate)) {
-      return candidate;
-    }
-  }
-
-  const entries = fs.readdirSync(voicesDir).filter((item) => item.endsWith(".pt"));
-  if (entries.length === 0) {
-    throw new Error(`kokoro voice assets are missing: ${voicesDir}`);
-  }
-  return path.join(voicesDir, entries[0]);
-}
-
-function buildPythonExecEnv(pythonBin: string): NodeJS.ProcessEnv {
-  const env: NodeJS.ProcessEnv = { ...process.env };
-  const raw = String(pythonBin || "").trim();
-  if (!raw || (!raw.includes("/") && !raw.includes("\\"))) {
-    return env;
-  }
-  const pythonDir = path.dirname(path.resolve(raw));
-  const currentPath = String(env.PATH || "");
-  const segments = currentPath
-    .split(path.delimiter)
-    .map((item) => item.trim())
-    .filter(Boolean);
-  if (!segments.includes(pythonDir)) {
-    env.PATH = [pythonDir, ...segments].join(path.delimiter);
-  }
-  return env;
-}
-
-async function runPythonInline(params: {
-  pythonBin: string;
-  script: string;
-  args: string[];
-  timeoutMs: number;
-}): Promise<{
-  /**
-   * Python runner stderr 摘要（可选）。
-   */
-  stderrSummary?: string;
-}> {
-  let stdout = "";
-  let stderr = "";
-  try {
-    const output = await execFileAsync(
-      params.pythonBin,
-      ["-c", params.script, ...params.args],
-      {
-        timeout: params.timeoutMs,
-        maxBuffer: 8 * 1024 * 1024,
-        env: buildPythonExecEnv(params.pythonBin),
-      },
-    );
-    stdout = String(output.stdout || "");
-    stderr = String(output.stderr || "");
-  } catch (error) {
-    const errorLike = error as {
-      stdout?: string;
-      stderr?: string;
-      message?: string;
-    };
-    stdout = String(errorLike.stdout || "");
-    stderr = String(errorLike.stderr || "");
-    const err = pickLastNonEmptyLine(stderr) || pickLastNonEmptyLine(stdout);
-    if (err) {
-      throw new Error(`python runner failed: ${err}`);
-    }
-    throw new Error(`python runner failed: ${String(errorLike.message || error)}`);
-  }
-  const err = pickUnexpectedPythonStderr(stderr);
-  return {
-    stderrSummary: err || normalizePythonStderrSummary(stderr),
-  };
-}
-
-const KOKORO_INLINE_SCRIPT = [
-  "import numpy as np",
-  "import soundfile as sf",
-  "import torch",
-  "from kokoro import KModel, KPipeline",
-  "config_path = __import__('sys').argv[1]",
-  "model_path = __import__('sys').argv[2]",
-  "voice_path = __import__('sys').argv[3]",
-  "lang_code = __import__('sys').argv[4]",
-  "text = __import__('sys').argv[5]",
-  "output_path = __import__('sys').argv[6]",
-  "speed = float(__import__('sys').argv[7])",
-  "device = 'cuda' if torch.cuda.is_available() else 'cpu'",
-  "model = KModel(config=config_path, model=model_path).to(device).eval()",
-  "pipeline = KPipeline(lang_code=lang_code, model=model, device=device)",
-  "chunks = []",
-  "for _, _, audio in pipeline(text, voice=voice_path, speed=speed):",
-  "    if audio is None:",
-  "        continue",
-  "    chunks.append(audio.cpu().numpy() if hasattr(audio, 'cpu') else np.asarray(audio))",
-  "if not chunks:",
-  "    raise RuntimeError('kokoro returned empty audio')",
-  "wave = np.concatenate(chunks)",
-  "sf.write(output_path, wave, 24000)",
-  "print(output_path)",
-].join("\n");
-
-const QWEN3_INLINE_SCRIPT = [
-  "import soundfile as sf",
-  "import torch",
-  "from qwen_tts import Qwen3TTSModel",
-  "model_path = __import__('sys').argv[1]",
-  "text = __import__('sys').argv[2]",
-  "voice = __import__('sys').argv[3]",
-  "language = __import__('sys').argv[4]",
-  "output_path = __import__('sys').argv[5]",
-  "speed = float(__import__('sys').argv[6])",
-  "device = 'cuda' if torch.cuda.is_available() else 'cpu'",
-  "dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32",
-  "model = Qwen3TTSModel.from_pretrained(model_path, device_map=device, dtype=dtype)",
-  "speakers = model.get_supported_speakers() or []",
-  "speaker = voice if voice and voice in speakers else (speakers[0] if speakers else voice)",
-  "if not speaker:",
-  "    raise RuntimeError('qwen3 supported speaker list is empty')",
-  "languages = model.get_supported_languages() or []",
-  "resolved_language = language if language and language in languages else ('Auto' if 'Auto' in languages else (languages[0] if languages else 'Auto'))",
-  "wavs, sample_rate = model.generate_custom_voice(text=text, speaker=speaker, language=resolved_language, non_streaming_mode=True, do_sample=True, top_p=0.9, temperature=0.7, repetition_penalty=1.05)",
-  "wave = wavs[0] if isinstance(wavs, list) else wavs",
-  "sf.write(output_path, wave, sample_rate)",
-  "print(output_path)",
-].join("\n");
-
-async function runKokoroSynthesizer(params: {
-  pythonBin: string;
-  modelDir: string;
-  text: string;
-  voice?: string;
-  language: string;
-  outputPath: string;
-  speed: number;
-  timeoutMs: number;
-}): Promise<{
-  /**
-   * Python runner stderr 摘要（可选）。
-   */
-  stderrSummary?: string;
-}> {
-  const voicePath = resolveKokoroVoicePath({
-    modelDir: params.modelDir,
-    voice: params.voice,
-    language: params.language,
-  });
-  const configPath = path.join(params.modelDir, "config.json");
-  const modelPath = path.join(params.modelDir, "kokoro-v1_0.pth");
-  return runPythonInline({
-    pythonBin: params.pythonBin,
-    script: KOKORO_INLINE_SCRIPT,
-    args: [
-      configPath,
-      modelPath,
-      voicePath,
-      params.language === "zh" ? "z" : "a",
-      params.text,
-      params.outputPath,
-      String(params.speed),
-    ],
-    timeoutMs: params.timeoutMs,
-  });
-}
-
-async function runQwen3Synthesizer(params: {
-  pythonBin: string;
-  modelDir: string;
-  text: string;
-  voice?: string;
-  language: string;
-  outputPath: string;
-  speed: number;
-  timeoutMs: number;
-}): Promise<{
-  /**
-   * Python runner stderr 摘要（可选）。
-   */
-  stderrSummary?: string;
-}> {
-  return runPythonInline({
-    pythonBin: params.pythonBin,
-    script: QWEN3_INLINE_SCRIPT,
-    args: [
-      params.modelDir,
-      params.text,
-      normalizeText(params.voice),
-      params.language === "zh" ? "Chinese" : params.language === "en" ? "English" : "Auto",
-      params.outputPath,
-      String(params.speed),
-    ],
-    timeoutMs: params.timeoutMs,
-  });
-}
-
-/**
- * 执行一次 TTS 合成，并写出音频文件。
- */
-export async function synthesizeSpeechFile(params: {
-  /**
-   * 当前执行上下文。
-   */
-  context: PluginCommandContext;
-  /**
-   * 当前 plugin 配置。
-   */
-  config: TtsPluginConfig;
-  /**
-   * 本次合成输入。
-   */
-  input: TtsSynthesizeInput;
-}): Promise<{
-  /**
-   * 输出相对路径。
-   */
-  outputPath: string;
-  /**
-   * 可发送文件标签。
-   */
-  fileTag: string;
-  /**
-   * 文件字节数。
-   */
-  bytes: number;
-  /**
-   * Python runner stderr 摘要（可选）。
-   */
-  stderrSummary?: string;
-}> {
-  const text = normalizeText(params.input.text);
-  if (!text) {
-    throw new Error("tts synthesize requires text");
-  }
-
-  const modelId = resolveTtsModelId(
-    normalizeText(params.input.modelId || params.config.modelId),
-  );
-  if (!modelId) {
-    throw new Error("tts modelId is missing");
-  }
-
-  const model = getTtsModelCatalogItem(modelId);
-  if (!model) {
-    throw new Error(`Unsupported tts model: ${modelId}`);
-  }
-
-  const format = normalizeFormat(params.input.format || params.config.format);
-  const speed = normalizeSpeed(
-    typeof params.input.speed === "number" ? params.input.speed : params.config.speed,
-  );
-  const language =
-    normalizeText(params.input.language || params.config.language) ||
-    detectLanguageHint(text);
-  const pythonBin = normalizeText(params.config.pythonBin) || "python3";
-  const timeoutMs = normalizeTimeoutMs(params.config.timeoutMs);
-  const modelsRootDir = resolveTtsModelsRootDir({
-    projectRoot: params.context.rootPath,
-    modelsDir: params.config.modelsDir,
-  });
-  const modelDir = path.join(modelsRootDir, modelId);
-  const output = resolveOutputTarget({
-    context: params.context,
-    format,
-    output: normalizeText(params.input.output || params.config.outputDir),
-    modelId,
-  });
-
-  await fs.ensureDir(path.dirname(output.absPath));
-  const runResult =
-    model.family === "kokoro"
-      ? await runKokoroSynthesizer({
-        pythonBin,
-        modelDir,
-        text,
-        voice: params.input.voice || params.config.voice,
-        language,
-        outputPath: output.absPath,
-        speed,
-        timeoutMs,
-      })
-      : await runQwen3Synthesizer({
-        pythonBin,
-        modelDir,
-        text,
-        voice: params.input.voice || params.config.voice,
-        language,
-        outputPath: output.absPath,
-        speed,
-        timeoutMs,
-      });
-
-  const stats = await fs.stat(output.absPath);
-  return {
-    outputPath: output.relativePath,
-    fileTag: renderChatMessageFileTag({
-      type: "audio",
-      path: output.relativePath,
-    }),
-    bytes: stats.size,
-    ...(runResult.stderrSummary ? { stderrSummary: runResult.stderrSummary } : {}),
-  };
-}

package/src/tts/types/Tts.ts

@@ -1,99 +0,0 @@
-/**
- * TTS 模型目录与推理族类型定义。
- *
- * 设计目标（中文）
- * - 统一描述 TTS 领域里的稳定模型 ID、模型族与下载清单。
- * - 让 plugin / runtime / console 共用同一套目录元数据。
- */
-
-/**
- * TTS provider 类型。
- *
- * 说明（中文）
- * - 当前仅保留 `local`，统一走本地模型目录与 Python runner。
- */
-export type TtsProvider = "local";
-
-/**
- * 内置可选 TTS 模型 ID。
- *
- * 说明（中文）
- * - 保持稳定字符串，避免后续改名破坏用户配置。
- */
-export type TtsModelId =
-  | "qwen3-tts-0.6b"
-  | "kokoro-82m"
-  | "qwen3-tts-1.7b";
-
-/**
- * TTS 本地推理实现族。
- *
- * 说明（中文）
- * - `qwen3`：Qwen3-TTS 系列。
- * - `kokoro`：Kokoro 系列。
- */
-export type TtsRuntimeFamily = "qwen3" | "kokoro";
-
-/**
- * TTS 输出音频格式。
- *
- * 说明（中文）
- * - 当前仅保留本地 runner 稳定支持的格式。
- */
-export type TtsAudioFormat = "wav" | "flac";
-
-/**
- * 单个 HuggingFace 下载资源定义。
- */
-export interface TtsModelAsset {
-  /**
-   * 资源仓库 ID（owner/repo）。
-   */
-  repoId: string;
-  /**
-   * 下载 revision（通常为 main）。
-   */
-  revision: string;
-  /**
-   * 仅下载这些文件（可选）。
-   *
-   * 说明（中文）
-   * - 为空时表示下载该 repo 的全部文件。
-   * - 用于像 Kokoro 这样只需要少量核心文件的场景。
-   */
-  files?: string[];
-  /**
-   * 下载到模型目录内的目标子目录（可选）。
-   */
-  targetSubdir?: string;
-}
-
-/**
- * TTS 内置模型目录条目。
- */
-export interface TtsModelCatalogItem {
-  /**
-   * 模型稳定 ID（配置与命令唯一键）。
-   */
-  id: TtsModelId;
-  /**
-   * Console / CLI 展示名称。
-   */
-  label: string;
-  /**
-   * 面向用户的简要说明。
-   */
-  description: string;
-  /**
-   * 本地推理所对应的实现族。
-   */
-  family: TtsRuntimeFamily;
-  /**
-   * 是否作为优先推荐项展示。
-   */
-  recommended: boolean;
-  /**
-   * 模型安装时需要下载的资源列表。
-   */
-  assets: TtsModelAsset[];
-}

package/src/voice/Config.ts

@@ -1,135 +0,0 @@
-/**
- * Voice plugin 配置读写工具。
- *
- * 关键点（中文）
- * - 行为配置与转写依赖配置统一收敛到 `plugins.voice`。
- * - 写入前将输入规范化为 JSON object，避免 undefined 进入项目配置。
- */
-
-import type { VoicePluginConfig } from "@/voice/types/VoicePlugin.js";
-import type { AgentPluginConfigRuntime } from "@downcity/agent/internal/types/agent/AgentRuntimeAssembly.js";
-import type { JsonObject, JsonValue } from "@downcity/agent/internal/types/common/Json.js";
-
-/**
- * 将普通对象转为可持久化 JSON object。
- */
-export function toJsonObject(
-  input: Record<string, unknown> | null | undefined,
-): JsonObject | null {
-  if (!input) return null;
-  const out: JsonObject = {};
-  for (const [key, value] of Object.entries(input)) {
-    if (value === undefined) continue;
-    if (
-      value === null ||
-      typeof value === "string" ||
-      typeof value === "number" ||
-      typeof value === "boolean"
-    ) {
-      out[key] = value;
-      continue;
-    }
-    if (Array.isArray(value)) {
-      out[key] = value
-        .filter((item) => item !== undefined)
-        .map((item) => item as JsonValue);
-      continue;
-    }
-    if (typeof value === "object") {
-      out[key] = toJsonObject(value as Record<string, unknown>) || {};
-    }
-  }
-  return out;
-}
-
-/**
- * 读取字符串选项。
- */
-export function getStringOpt(
-  opts: Record<string, JsonValue>,
-  key: string,
-): string | undefined {
-  const value = opts[key];
-  if (typeof value !== "string") return undefined;
-  const trimmed = value.trim();
-  return trimmed || undefined;
-}
-
-/**
- * 读取布尔选项。
- */
-export function getBooleanOpt(
-  opts: Record<string, JsonValue>,
-  key: string,
-  defaultValue: boolean,
-): boolean {
-  const value = opts[key];
-  if (typeof value === "boolean") return value;
-  if (typeof value === "string") {
-    const normalized = value.trim().toLowerCase();
-    if (["1", "true", "yes", "y", "on"].includes(normalized)) return true;
-    if (["0", "false", "no", "n", "off"].includes(normalized)) return false;
-  }
-  return defaultValue;
-}
-
-/**
- * 读取 Voice Plugin 配置。
- */
-export function readVoicePluginConfig(runtime: {
-  config: {
-    plugins?: Record<string, unknown>;
-  };
-}): VoicePluginConfig {
-  const current = runtime.config.plugins?.voice;
-  const normalized =
-    current && typeof current === "object" && !Array.isArray(current)
-      ? (current as VoicePluginConfig)
-      : {};
-  return {
-    enabled: normalized.enabled === true,
-    injectPrompt:
-      typeof normalized.injectPrompt === "boolean"
-        ? normalized.injectPrompt
-        : true,
-    augmentMessage:
-      typeof normalized.augmentMessage === "boolean"
-        ? normalized.augmentMessage
-        : true,
-    provider:
-      normalized.provider === "command" || normalized.provider === "local"
-        ? normalized.provider
-        : "local",
-    ...(typeof normalized.modelId === "string" ? { modelId: normalized.modelId } : {}),
-    ...(typeof normalized.modelsDir === "string" ? { modelsDir: normalized.modelsDir } : {}),
-    ...(typeof normalized.pythonBin === "string" ? { pythonBin: normalized.pythonBin } : {}),
-    ...(typeof normalized.command === "string" ? { command: normalized.command } : {}),
-    ...(typeof normalized.language === "string" ? { language: normalized.language } : {}),
-    ...(typeof normalized.timeoutMs === "number" ? { timeoutMs: normalized.timeoutMs } : {}),
-    ...(typeof normalized.strategy === "string" ? { strategy: normalized.strategy } : {}),
-    ...(Array.isArray(normalized.installedModels)
-      ? { installedModels: normalized.installedModels }
-      : {}),
-  };
-}
-
-/**
- * 写入完整 voice plugin 配置。
- */
-export async function writeVoicePluginConfig(params: {
-  agentState: {
-    config: {
-      plugins?: Record<string, unknown>;
-    };
-    pluginConfig: AgentPluginConfigRuntime;
-  };
-  value: VoicePluginConfig;
-}): Promise<void> {
-  if (!params.agentState.config.plugins) {
-    params.agentState.config.plugins = {};
-  }
-  params.agentState.config.plugins.voice = (toJsonObject(params.value) || {}) as JsonObject;
-  await params.agentState.pluginConfig.persistProjectPlugins(
-    params.agentState.config.plugins as Record<string, JsonObject>,
-  );
-}