@downcity/plugins 1.0.4

package/src/tts/runtime/DependencyInstaller.ts

@@ -0,0 +1,436 @@
+/**
+ * TTS 依赖安装器。
+ *
+ * 关键点（中文）
+ * - 统一管理 `city tts` 的 Python 依赖安装逻辑。
+ * - 根据模型自动推导 runner（qwen3 / kokoro）并执行 `python -m pip install`。
+ */
+
+import { execFile as execFileCb } from "node:child_process";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import fs from "fs-extra";
+import type { TtsModelId, TtsRuntimeFamily } from "@/tts/types/Tts.js";
+
+const execFileAsync = promisify(execFileCb);
+
+const DEFAULT_PIP_TIMEOUT_MS = 300_000;
+const OUTPUT_TAIL_LIMIT = 1200;
+const DEFAULT_VENV_DIR = path.join(os.homedir(), ".downcity", "venvs", "tts");
+
+/**
+ * TTS 依赖 runner 类型。
+ */
+export type TtsDependencyRunner = TtsRuntimeFamily;
+
+/**
+ * 单个 runner 的安装摘要。
+ */
+export interface TtsDependencyInstallItem {
+  /**
+   * 当前安装 runner。
+   */
+  runner: TtsDependencyRunner;
+  /**
+   * Python 可执行文件。
+   */
+  pythonBin: string;
+  /**
+   * 执行参数（不含 pythonBin）。
+   */
+  args: string[];
+  /**
+   * 安装包列表。
+   */
+  packages: string[];
+  /**
+   * 实际执行的命令行。
+   */
+  command: string;
+  /**
+   * 执行耗时（毫秒）。
+   */
+  elapsedMs: number;
+  /**
+   * 是否跳过安装。
+   */
+  skipped?: boolean;
+  /**
+   * 跳过原因（可选）。
+   */
+  skipReason?: string;
+  /**
+   * stdout 尾部（可选）。
+   */
+  stdoutTail?: string;
+  /**
+   * stderr 尾部（可选）。
+   */
+  stderrTail?: string;
+}
+
+/**
+ * 依赖安装结果。
+ */
+export interface TtsDependencyInstallResult {
+  /**
+   * 本次使用的 Python 可执行文件。
+   */
+  pythonBin: string;
+  /**
+   * 本次处理的 runner 列表。
+   */
+  runners: TtsDependencyRunner[];
+  /**
+   * 各 runner 的安装摘要。
+   */
+  items: TtsDependencyInstallItem[];
+  /**
+   * 是否使用了虚拟环境安装。
+   */
+  usedVirtualEnv: boolean;
+  /**
+   * 虚拟环境目录（仅 `usedVirtualEnv=true` 时存在）。
+   */
+  venvDir?: string;
+  /**
+   * 初始尝试使用的 Python（仅进入虚拟环境回退时存在）。
+   */
+  basePythonBin?: string;
+}
+
+/**
+ * 依赖安装输入参数。
+ */
+export interface TtsDependencyInstallInput {
+  /**
+   * Python 可执行文件（默认 `python3`）。
+   */
+  pythonBin?: string;
+  /**
+   * 需要安装的 runner 列表。
+   */
+  runners: TtsDependencyRunner[];
+  /**
+   * 是否使用 `pip -U` 升级。
+   */
+  upgrade?: boolean;
+  /**
+   * pip 安装超时（毫秒）。
+   */
+  timeoutMs?: number;
+  /**
+   * 虚拟环境目录（可选）。
+   */
+  venvDir?: string;
+}
+
+function normalizePythonBin(input?: string): string {
+  const text = String(input || "").trim();
+  return text || "python3";
+}
+
+function normalizeTimeoutMs(value?: number): number {
+  if (!Number.isFinite(value as number)) return DEFAULT_PIP_TIMEOUT_MS;
+  const ms = Math.floor(Number(value));
+  if (ms < 5_000) return 5_000;
+  if (ms > 1_800_000) return 1_800_000;
+  return ms;
+}
+
+function normalizeVenvDir(input?: string): string {
+  const text = String(input || "").trim();
+  if (!text) return DEFAULT_VENV_DIR;
+  return path.resolve(text);
+}
+
+function tailText(value: string): string | undefined {
+  const text = String(value || "").trim();
+  if (!text) return undefined;
+  if (text.length <= OUTPUT_TAIL_LIMIT) return text;
+  return text.slice(text.length - OUTPUT_TAIL_LIMIT);
+}
+
+export function resolveTtsVenvPythonBin(venvDir: string): string {
+  return path.join(venvDir, "bin", "python");
+}
+
+/**
+ * 返回 TTS 默认虚拟环境目录。
+ */
+export function resolveDefaultTtsVenvDir(): string {
+  return DEFAULT_VENV_DIR;
+}
+
+/**
+ * 返回 TTS 默认虚拟环境 Python 路径。
+ */
+export function resolveDefaultTtsVenvPythonBin(): string {
+  return resolveTtsVenvPythonBin(DEFAULT_VENV_DIR);
+}
+
+/**
+ * 判断是否为 PEP 668（externally-managed-environment）错误。
+ */
+export function isPep668InstallError(text: string): boolean {
+  const msg = String(text || "").toLowerCase();
+  if (!msg) return false;
+  return (
+    msg.includes("externally-managed-environment") ||
+    msg.includes("this environment is externally managed") ||
+    msg.includes("pep 668") ||
+    msg.includes("--break-system-packages")
+  );
+}
+
+function dedupeRunners(runners: TtsDependencyRunner[]): TtsDependencyRunner[] {
+  const seen = new Set<TtsDependencyRunner>();
+  const out: TtsDependencyRunner[] = [];
+  for (const runner of runners) {
+    if (seen.has(runner)) continue;
+    seen.add(runner);
+    out.push(runner);
+  }
+  return out;
+}
+
+function getRunnerPackages(runner: TtsDependencyRunner): string[] {
+  if (runner === "qwen3") {
+    return ["qwen-tts", "soundfile", "torch", "torchaudio", "transformers", "accelerate"];
+  }
+  return ["kokoro==0.7.16", "soundfile", "misaki[zh]", "torch"];
+}
+
+async function checkRunnerPackagesInstalled(params: {
+  pythonBin: string;
+  packages: string[];
+  timeoutMs: number;
+}): Promise<{
+  installed: boolean;
+  args: string[];
+  command: string;
+}> {
+  const args = ["-m", "pip", "show", ...params.packages.map((item) => item.replace(/\[.*\]$/, "").replace(/==.+$/, ""))];
+  const command = [params.pythonBin, ...args].join(" ");
+  try {
+    await execFileAsync(params.pythonBin, args, {
+      timeout: params.timeoutMs,
+      maxBuffer: 8 * 1024 * 1024,
+    });
+    return {
+      installed: true,
+      args,
+      command,
+    };
+  } catch {
+    return {
+      installed: false,
+      args,
+      command,
+    };
+  }
+}
+
+type RunnerInstallFailure = {
+  runner: TtsDependencyRunner;
+  command: string;
+  reason: string;
+};
+
+type RunnerInstallAttempt =
+  | {
+      success: true;
+      item: TtsDependencyInstallItem;
+    }
+  | {
+      success: false;
+      failure: RunnerInstallFailure;
+    };
+
+async function installRunnerWithPython(params: {
+  pythonBin: string;
+  runner: TtsDependencyRunner;
+  upgrade: boolean;
+  timeoutMs: number;
+}): Promise<RunnerInstallAttempt> {
+  const packages = getRunnerPackages(params.runner);
+  const checkStartedAt = Date.now();
+  const checkResult = await checkRunnerPackagesInstalled({
+    pythonBin: params.pythonBin,
+    packages,
+    timeoutMs: params.timeoutMs,
+  });
+  if (checkResult.installed) {
+    return {
+      success: true,
+      item: {
+        runner: params.runner,
+        pythonBin: params.pythonBin,
+        args: checkResult.args,
+        packages,
+        command: checkResult.command,
+        elapsedMs: Date.now() - checkStartedAt,
+        skipped: true,
+        skipReason: "already-installed",
+      },
+    };
+  }
+
+  const args = [
+    "-m",
+    "pip",
+    "install",
+    ...(params.upgrade ? ["-U"] : []),
+    ...packages,
+  ];
+  const command = [params.pythonBin, ...args].join(" ");
+  const startedAt = Date.now();
+  try {
+    const { stdout, stderr } = await execFileAsync(params.pythonBin, args, {
+      timeout: params.timeoutMs,
+      maxBuffer: 8 * 1024 * 1024,
+    });
+    return {
+      success: true,
+      item: {
+        runner: params.runner,
+        pythonBin: params.pythonBin,
+        args,
+        packages,
+        command,
+        elapsedMs: Date.now() - startedAt,
+        stdoutTail: tailText(String(stdout || "")),
+        stderrTail: tailText(String(stderr || "")),
+      },
+    };
+  } catch (error) {
+    const errorLike = error as {
+      stdout?: string;
+      stderr?: string;
+      message?: string;
+    };
+    const reason =
+      tailText(String(errorLike.stderr || "")) ||
+      tailText(String(errorLike.stdout || "")) ||
+      String(errorLike.message || error);
+    return {
+      success: false,
+      failure: {
+        runner: params.runner,
+        command,
+        reason,
+      },
+    };
+  }
+}
+
+export async function ensureTtsVirtualEnv(params: {
+  /**
+   * 虚拟环境目录。
+   */
+  venvDir?: string;
+  /**
+   * 用于创建 venv 的基础 Python。
+   */
+  basePythonBin?: string;
+}): Promise<string> {
+  const venvDir = normalizeVenvDir(params.venvDir);
+  const basePythonBin = normalizePythonBin(params.basePythonBin);
+  const pythonBin = resolveTtsVenvPythonBin(venvDir);
+  if (await fs.pathExists(pythonBin)) {
+    return pythonBin;
+  }
+  await fs.ensureDir(venvDir);
+  await execFileAsync(basePythonBin, ["-m", "venv", venvDir], {
+    timeout: DEFAULT_PIP_TIMEOUT_MS,
+    maxBuffer: 8 * 1024 * 1024,
+  });
+  return pythonBin;
+}
+
+async function installAllRunners(params: {
+  pythonBin: string;
+  runners: TtsDependencyRunner[];
+  upgrade: boolean;
+  timeoutMs: number;
+}): Promise<{
+  success: boolean;
+  items: TtsDependencyInstallItem[];
+  failures: RunnerInstallFailure[];
+}> {
+  const items: TtsDependencyInstallItem[] = [];
+  const failures: RunnerInstallFailure[] = [];
+  for (const runner of params.runners) {
+    const result = await installRunnerWithPython({
+      pythonBin: params.pythonBin,
+      runner,
+      upgrade: params.upgrade,
+      timeoutMs: params.timeoutMs,
+    });
+    if (result.success) {
+      items.push(result.item);
+    } else {
+      failures.push(result.failure);
+    }
+  }
+  return {
+    success: failures.length === 0,
+    items,
+    failures,
+  };
+}
+
+/**
+ * 安装 TTS Python 依赖。
+ */
+export async function installTtsDependencies(
+  input: TtsDependencyInstallInput,
+): Promise<TtsDependencyInstallResult> {
+  const runners = dedupeRunners(input.runners);
+  const basePythonBin = normalizePythonBin(input.pythonBin);
+  const timeoutMs = normalizeTimeoutMs(input.timeoutMs);
+  const upgrade = input.upgrade === true;
+  const venvDir = normalizeVenvDir(input.venvDir);
+  const venvPythonBin = await ensureTtsVirtualEnv({
+    venvDir,
+    basePythonBin,
+  });
+  const venvAttempt = await installAllRunners({
+    pythonBin: venvPythonBin,
+    runners,
+    upgrade,
+    timeoutMs,
+  });
+  if (!venvAttempt.success) {
+    throw new Error(
+      venvAttempt.failures
+        .map((item) => `${item.runner}: ${item.reason}`)
+        .join("\n"),
+    );
+  }
+
+  return {
+    pythonBin: venvPythonBin,
+    runners,
+    items: venvAttempt.items,
+    usedVirtualEnv: true,
+    venvDir,
+    basePythonBin,
+  };
+}
+
+/**
+ * 根据模型列表推导需要安装的 runner 集合。
+ */
+export function resolveTtsRunnersByModels(modelIds: TtsModelId[]): TtsDependencyRunner[] {
+  const runners = new Set<TtsDependencyRunner>();
+  for (const modelId of modelIds) {
+    if (modelId === "qwen3-tts-0.6b") {
+      runners.add("qwen3");
+      continue;
+    }
+    runners.add("kokoro");
+  }
+  return [...runners];
+}

package/src/tts/runtime/Installer.ts

@@ -0,0 +1,297 @@
+/**
+ * TTS 模型安装器。
+ *
+ * 关键点（中文）
+ * - 统一把 HuggingFace 资源下载到本地模型目录。
+ * - 支持“整仓下载”与“仅下载指定文件”两种模式。
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import fsExtra from "fs-extra";
+import type { TtsModelCatalogItem, TtsModelId } from "@/tts/types/Tts.js";
+
+const HUGGINGFACE_HOST = "https://huggingface.co";
+const TTS_INSTALL_MANIFEST = "downcity.tts.install.json";
+
+type HuggingFaceModelApiResponse = {
+  siblings?: Array<{
+    rfilename?: string;
+  }>;
+};
+
+/**
+ * TTS 模型安装结果。
+ */
+export interface TtsModelInstallResult {
+  /**
+   * 模型目录（绝对路径）。
+   */
+  modelDir: string;
+  /**
+   * 本次下载文件数。
+   */
+  downloadedFiles: number;
+  /**
+   * 本次跳过文件数（已存在且未强制覆盖）。
+   */
+  skippedFiles: number;
+  /**
+   * 本次实际下载的文件路径（相对模型目录）。
+   */
+  downloadedFilePaths: string[];
+  /**
+   * 本次跳过的文件路径（相对模型目录）。
+   */
+  skippedFilePaths: string[];
+}
+
+function encodePathSegments(input: string): string {
+  return input
+    .split("/")
+    .map((segment) => encodeURIComponent(segment))
+    .join("/");
+}
+
+function buildAuthHeaders(token?: string): Record<string, string> | undefined {
+  const t = String(token || "").trim();
+  if (!t) return undefined;
+  return { Authorization: `Bearer ${t}` };
+}
+
+function resolveSafeTargetPath(baseDir: string, relativePath: string): string {
+  const normalized = relativePath.replace(/\\/g, "/");
+  const target = path.resolve(baseDir, normalized);
+  const base = path.resolve(baseDir);
+  if (target !== base && !target.startsWith(`${base}${path.sep}`)) {
+    throw new Error(`Unsafe file path from HuggingFace: ${relativePath}`);
+  }
+  return target;
+}
+
+async function readRepoFiles(input: {
+  repoId: string;
+  hfToken?: string;
+}): Promise<string[]> {
+  const repoUrlPath = encodePathSegments(input.repoId);
+  const metadataUrl = `${HUGGINGFACE_HOST}/api/models/${repoUrlPath}`;
+  const response = await fetch(metadataUrl, {
+    headers: {
+      Accept: "application/json",
+      ...(buildAuthHeaders(input.hfToken) || {}),
+    },
+  });
+  if (!response.ok) {
+    const responseText = await response.text().catch(() => "");
+    throw new Error(
+      `Failed to fetch model metadata (${input.repoId}): HTTP ${response.status}${responseText ? ` ${responseText.slice(0, 300)}` : ""}`,
+    );
+  }
+  const payload = (await response.json()) as HuggingFaceModelApiResponse;
+  const files = (Array.isArray(payload.siblings) ? payload.siblings : [])
+    .map((item) => String(item?.rfilename || "").trim())
+    .filter(Boolean)
+    .filter((item) => item !== ".gitattributes");
+  if (files.length === 0) {
+    throw new Error(`No downloadable files found in repo: ${input.repoId}`);
+  }
+  return files;
+}
+
+async function downloadModelFile(input: {
+  repoId: string;
+  revision: string;
+  filePath: string;
+  targetPath: string;
+  hfToken?: string;
+}): Promise<void> {
+  const repoPath = encodePathSegments(input.repoId);
+  const revision = encodeURIComponent(input.revision);
+  const filePath = encodePathSegments(input.filePath);
+  const url = `${HUGGINGFACE_HOST}/${repoPath}/resolve/${revision}/${filePath}?download=1`;
+  const response = await fetch(url, {
+    headers: {
+      ...(buildAuthHeaders(input.hfToken) || {}),
+    },
+  });
+  if (!response.ok) {
+    const responseText = await response.text().catch(() => "");
+    throw new Error(
+      `Failed to download ${input.filePath}: HTTP ${response.status}${responseText ? ` ${responseText.slice(0, 300)}` : ""}`,
+    );
+  }
+  if (!response.body) {
+    throw new Error(`Empty response body for file: ${input.filePath}`);
+  }
+
+  const tempPath = `${input.targetPath}.downloading`;
+  await fsExtra.ensureDir(path.dirname(input.targetPath));
+  try {
+    const body = Readable.fromWeb(
+      response.body as unknown as globalThis.ReadableStream<Uint8Array>,
+    );
+    await pipeline(body, fs.createWriteStream(tempPath));
+    await fsExtra.move(tempPath, input.targetPath, { overwrite: true });
+  } catch (error) {
+    await fsExtra.remove(tempPath).catch(() => undefined);
+    throw error;
+  }
+}
+
+/**
+ * 判断模型在本地是否已安装。
+ *
+ * 关键点（中文）
+ * - 优先识别安装清单 `downcity.tts.install.json`。
+ * - 若无清单但目录非空，也视为已存在模型，避免重复下载。
+ */
+export async function detectLocalTtsModelInstallState(input: {
+  /**
+   * 模型 ID。
+   */
+  modelId: TtsModelId;
+  /**
+   * TTS 模型根目录。
+   */
+  modelsRootDir: string;
+}): Promise<{
+  /**
+   * 模型目录（绝对路径）。
+   */
+  modelDir: string;
+  /**
+   * 是否判断为“已安装”。
+   */
+  installed: boolean;
+  /**
+   * 已安装来源。
+   */
+  source?: "manifest" | "directory";
+}> {
+  const modelDir = path.resolve(input.modelsRootDir, input.modelId);
+  const manifestPath = path.join(modelDir, TTS_INSTALL_MANIFEST);
+  const hasManifest = await fsExtra.pathExists(manifestPath);
+  if (hasManifest) {
+    return {
+      modelDir,
+      installed: true,
+      source: "manifest",
+    };
+  }
+
+  const hasModelDir = await fsExtra.pathExists(modelDir);
+  if (!hasModelDir) {
+    return {
+      modelDir,
+      installed: false,
+    };
+  }
+
+  const entries = await fsExtra.readdir(modelDir).catch(() => []);
+  const hasPersistedFiles = entries.some(
+    (entry) => !String(entry).endsWith(".downloading"),
+  );
+  if (!hasPersistedFiles) {
+    return {
+      modelDir,
+      installed: false,
+    };
+  }
+
+  return {
+    modelDir,
+    installed: true,
+    source: "directory",
+  };
+}
+
+/**
+ * 安装指定 TTS 模型到本地目录。
+ *
+ * 关键点（中文）
+ * - 使用 HTTP 文件级下载，不依赖 git-lfs。
+ * - 默认复用已有文件；`force=true` 时覆盖重下。
+ */
+export async function installTtsModelFromHuggingFace(input: {
+  /**
+   * 目标模型目录项。
+   */
+  model: TtsModelCatalogItem;
+  /**
+   * TTS 模型根目录。
+   */
+  modelsRootDir: string;
+  /**
+   * 是否强制覆盖。
+   */
+  force?: boolean;
+  /**
+   * HuggingFace Token（可选）。
+   */
+  hfToken?: string;
+}): Promise<TtsModelInstallResult> {
+  const modelDir = path.resolve(input.modelsRootDir, input.model.id);
+  await fsExtra.ensureDir(modelDir);
+
+  let downloadedFiles = 0;
+  let skippedFiles = 0;
+  const downloadedFilePaths: string[] = [];
+  const skippedFilePaths: string[] = [];
+
+  for (const asset of input.model.assets) {
+    const assetFiles =
+      Array.isArray(asset.files) && asset.files.length > 0
+        ? asset.files
+        : await readRepoFiles({
+            repoId: asset.repoId,
+            hfToken: input.hfToken,
+          });
+    const assetBaseDir = asset.targetSubdir
+      ? path.resolve(modelDir, asset.targetSubdir)
+      : modelDir;
+    await fsExtra.ensureDir(assetBaseDir);
+
+    for (const filePath of assetFiles) {
+      const relativeToModel = asset.targetSubdir
+        ? path.posix.join(asset.targetSubdir.replace(/\\/g, "/"), filePath)
+        : filePath;
+      const targetPath = resolveSafeTargetPath(assetBaseDir, filePath);
+      const exists = await fsExtra.pathExists(targetPath);
+      if (exists && input.force !== true) {
+        skippedFiles += 1;
+        skippedFilePaths.push(relativeToModel);
+        continue;
+      }
+      await downloadModelFile({
+        repoId: asset.repoId,
+        revision: asset.revision,
+        filePath,
+        targetPath,
+        hfToken: input.hfToken,
+      });
+      downloadedFiles += 1;
+      downloadedFilePaths.push(relativeToModel);
+    }
+  }
+
+  const manifestPath = path.join(modelDir, TTS_INSTALL_MANIFEST);
+  await fsExtra.writeJson(
+    manifestPath,
+    {
+      modelId: input.model.id,
+      installedAt: new Date().toISOString(),
+      assets: input.model.assets,
+    },
+    { spaces: 2 },
+  );
+
+  return {
+    modelDir,
+    downloadedFiles,
+    skippedFiles,
+    downloadedFilePaths,
+    skippedFilePaths,
+  };
+}