@downcity/agent 1.1.1

package/src/plugins/voice/runtime/Transcriber.ts

@@ -0,0 +1,467 @@
+import path from "node:path";
+import { exec as execWithShell, execFile as execFileCb } from "node:child_process";
+import { promisify } from "node:util";
+import fs from "fs-extra";
+import type {
+  VoiceModelId,
+  VoiceProvider,
+  VoiceTranscribeStrategy,
+} from "@/shared/types/Voice.js";
+import { resolveVoiceModelsRootDir } from "./Paths.js";
+import type { VoicePluginConfig } from "@/shared/types/VoicePlugin.js";
+import type { PluginCommandContext } from "@/shared/types/Plugin.js";
+
+const execShellAsync = promisify(execWithShell);
+const execFileAsync = promisify(execFileCb);
+const DEFAULT_TRANSCRIBE_TIMEOUT_MS = 120_000;
+
+/**
+ * Voice 转写输入参数。
+ */
+export interface VoiceTranscribeInput {
+  /**
+   * 运行时上下文（用于读取配置/根目录/日志）。
+   */
+  context: PluginCommandContext;
+  /**
+   * 待转写音频路径（相对项目根目录或绝对路径）。
+   */
+  audioPath: string;
+  /**
+   * 语言提示（可选）。
+   */
+  language?: string;
+}
+
+/**
+ * Voice 转写输出结果。
+ */
+export interface VoiceTranscribeResult {
+  /**
+   * 转写文本。
+   */
+  text: string;
+  /**
+   * 归一化后的音频绝对路径。
+   */
+  audioPath: string;
+  /**
+   * 本次使用的模型 ID。
+   */
+  modelId: VoiceModelId;
+  /**
+   * 本次使用的 provider。
+   */
+  provider: VoiceProvider;
+  /**
+   * 本次转写耗时（毫秒）。
+   */
+  elapsedMs: number;
+  /**
+   * 实际执行器标识。
+   */
+  runner: "funasr" | "transformers-whisper" | "command";
+}
+
+type VoiceConfigResolved = {
+  config: VoicePluginConfig;
+  modelId: VoiceModelId;
+  provider: VoiceProvider;
+  modelDir: string;
+  audioPath: string;
+  timeoutMs: number;
+  language: string;
+  pythonBin: string;
+  commandTemplate?: string;
+  strategy: VoiceTranscribeStrategy;
+};
+
+function normalizeLanguage(input?: string): string {
+  const text = String(input || "").trim();
+  return text || "zh";
+}
+
+function normalizeTimeoutMs(value?: number): number {
+  if (!Number.isFinite(value as number)) return DEFAULT_TRANSCRIBE_TIMEOUT_MS;
+  const ms = Number(value);
+  if (ms < 1_000) return 1_000;
+  if (ms > 600_000) return 600_000;
+  return Math.floor(ms);
+}
+
+function normalizePythonBin(value?: string): string {
+  const text = String(value || "").trim();
+  return text || "python3";
+}
+
+function normalizeStrategy(strategy?: VoiceTranscribeStrategy): VoiceTranscribeStrategy {
+  if (strategy === "funasr") return "funasr";
+  if (strategy === "transformers-whisper") return "transformers-whisper";
+  if (strategy === "command") return "command";
+  return "auto";
+}
+
+function toAbsoluteAudioPath(context: PluginCommandContext, input: string): string {
+  const raw = String(input || "").trim();
+  if (!raw) {
+    throw new Error("voice transcribe requires audioPath");
+  }
+  if (path.isAbsolute(raw)) return path.resolve(raw);
+  return path.resolve(context.rootPath, raw);
+}
+
+function pickLastNonEmptyLine(value: string): string {
+  const lines = String(value || "")
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+  return lines.length > 0 ? lines[lines.length - 1] : "";
+}
+
+function shellEscapeSingle(value: string): string {
+  return `'${String(value).replace(/'/g, `'"'"'`)}'`;
+}
+
+/**
+ * 为 python 执行构建环境变量。
+ *
+ * 关键点（中文）
+ * - 当 `pythonBin` 指向 venv 内解释器时，把其 `bin` 目录前置到 PATH，
+ *   以便 python 子进程内调用 `pip`（FunASR 远程代码会用到）能正确命中同一 venv。
+ */
+function buildPythonExecEnv(pythonBin: string): NodeJS.ProcessEnv {
+  const env: NodeJS.ProcessEnv = { ...process.env };
+  const raw = String(pythonBin || "").trim();
+  if (!raw || (!raw.includes("/") && !raw.includes("\\"))) {
+    return env;
+  }
+  const pythonDir = path.dirname(path.resolve(raw));
+  const currentPath = String(env.PATH || "");
+  const segments = currentPath
+    .split(path.delimiter)
+    .map((item) => item.trim())
+    .filter(Boolean);
+  if (!segments.includes(pythonDir)) {
+    env.PATH = [pythonDir, ...segments].join(path.delimiter);
+  }
+  return env;
+}
+
+function renderCommandTemplate(template: string, values: Record<string, string>): string {
+  return template.replace(/\{(audioPath|modelDir|modelId|language)\}/g, (_, key) => {
+    const value = values[key] || "";
+    return shellEscapeSingle(value);
+  });
+}
+
+async function runCustomCommand(params: {
+  template: string;
+  values: Record<string, string>;
+  timeoutMs: number;
+}): Promise<string> {
+  const command = renderCommandTemplate(params.template, params.values);
+  const { stdout, stderr } = await execShellAsync(command, {
+    timeout: params.timeoutMs,
+    maxBuffer: 8 * 1024 * 1024,
+  });
+  const text = pickLastNonEmptyLine(String(stdout || ""));
+  if (text) return text;
+  const err = pickLastNonEmptyLine(String(stderr || ""));
+  if (err) {
+    throw new Error(`voice custom command produced no transcript: ${err}`);
+  }
+  throw new Error("voice custom command produced empty transcript");
+}
+
+async function runPythonInline(params: {
+  pythonBin: string;
+  script: string;
+  args: string[];
+  timeoutMs: number;
+}): Promise<string> {
+  let stdout = "";
+  let stderr = "";
+  try {
+    const output = await execFileAsync(
+      params.pythonBin,
+      ["-c", params.script, ...params.args],
+      {
+        timeout: params.timeoutMs,
+        maxBuffer: 8 * 1024 * 1024,
+        env: buildPythonExecEnv(params.pythonBin),
+      },
+    );
+    stdout = String(output.stdout || "");
+    stderr = String(output.stderr || "");
+  } catch (error) {
+    const errorLike = error as {
+      stdout?: string;
+      stderr?: string;
+      message?: string;
+    };
+    stdout = String(errorLike.stdout || "");
+    stderr = String(errorLike.stderr || "");
+    const err = pickLastNonEmptyLine(stderr) || pickLastNonEmptyLine(stdout);
+    if (err) {
+      throw new Error(`python runner failed: ${err}`);
+    }
+    throw new Error(`python runner failed: ${String(errorLike.message || error)}`);
+  }
+  const text = pickLastNonEmptyLine(stdout);
+  if (text) return text;
+  const err = pickLastNonEmptyLine(stderr);
+  if (err) {
+    throw new Error(`python runner produced no transcript: ${err}`);
+  }
+  throw new Error("python runner produced empty transcript");
+}
+
+const FUNASR_INLINE_SCRIPT = [
+  "from funasr import AutoModel",
+  "import sys",
+  "model_dir = sys.argv[1]",
+  "audio_path = sys.argv[2]",
+  "model = AutoModel(model=model_dir, trust_remote_code=True, disable_update=True)",
+  "result = model.generate(input=audio_path)",
+  "text = ''",
+  "if isinstance(result, list) and len(result) > 0:",
+  "    first = result[0]",
+  "    if isinstance(first, dict):",
+  "        text = str(first.get('text') or '')",
+  "elif isinstance(result, dict):",
+  "    text = str(result.get('text') or '')",
+  "print(text.strip())",
+].join("\n");
+
+const TRANSFORMERS_WHISPER_INLINE_SCRIPT = [
+  "from transformers import pipeline",
+  "import sys",
+  "model_dir = sys.argv[1]",
+  "audio_path = sys.argv[2]",
+  "language = sys.argv[3] if len(sys.argv) > 3 else ''",
+  "pipe = pipeline('automatic-speech-recognition', model=model_dir)",
+  "kwargs = {}",
+  "if language:",
+  "    kwargs['generate_kwargs'] = {'language': language}",
+  "result = pipe(audio_path, **kwargs)",
+  "text = ''",
+  "if isinstance(result, dict):",
+  "    text = str(result.get('text') or '')",
+  "else:",
+  "    text = str(result)",
+  "print(text.strip())",
+].join("\n");
+
+async function runFunasrRunner(params: {
+  pythonBin: string;
+  modelDir: string;
+  audioPath: string;
+  timeoutMs: number;
+}): Promise<string> {
+  return runPythonInline({
+    pythonBin: params.pythonBin,
+    script: FUNASR_INLINE_SCRIPT,
+    args: [params.modelDir, params.audioPath],
+    timeoutMs: params.timeoutMs,
+  });
+}
+
+async function runTransformersWhisperRunner(params: {
+  pythonBin: string;
+  modelDir: string;
+  audioPath: string;
+  language: string;
+  timeoutMs: number;
+}): Promise<string> {
+  return runPythonInline({
+    pythonBin: params.pythonBin,
+    script: TRANSFORMERS_WHISPER_INLINE_SCRIPT,
+    args: [params.modelDir, params.audioPath, params.language],
+    timeoutMs: params.timeoutMs,
+  });
+}
+
+async function resolveVoiceConfig(input: VoiceTranscribeInput): Promise<VoiceConfigResolved> {
+  const pluginConfig =
+    input.context.config.plugins?.asr &&
+    typeof input.context.config.plugins.asr === "object" &&
+    !Array.isArray(input.context.config.plugins.asr)
+      ? input.context.config.plugins.asr
+      : null;
+
+  const enabled =
+    pluginConfig && (pluginConfig as { enabled?: unknown }).enabled === true;
+  if (!enabled) {
+    throw new Error("ASR plugin is disabled. Run `city asr on` first.");
+  }
+
+  const provider = String(
+    (pluginConfig as { provider?: unknown } | null)?.provider || "local",
+  ) as VoiceProvider;
+  if (!["local", "command"].includes(provider)) {
+    throw new Error(`Unsupported asr provider: ${provider}`);
+  }
+
+  const modelId =
+    String(
+      (pluginConfig as { modelId?: unknown } | null)?.modelId || "SenseVoiceSmall",
+    ).trim() || "SenseVoiceSmall";
+  if (!modelId && provider === "local") {
+    throw new Error("ASR active model is not configured. Run `city asr use <modelId>`.");
+  }
+
+  const modelsRootDir = resolveVoiceModelsRootDir({
+    projectRoot: input.context.rootPath,
+    modelsDir: String(
+      (pluginConfig as { modelsDir?: unknown } | null)?.modelsDir || "",
+    ).trim(),
+  });
+  const modelDir = path.resolve(modelsRootDir, modelId);
+  if (provider === "local") {
+    const modelDirExists = await fs.pathExists(modelDir);
+    if (!modelDirExists) {
+      throw new Error(`Voice model directory does not exist: ${modelDir}`);
+    }
+  }
+
+  const audioPath = toAbsoluteAudioPath(input.context, input.audioPath);
+  const audioExists = await fs.pathExists(audioPath);
+  if (!audioExists) {
+    throw new Error(`Audio file does not exist: ${audioPath}`);
+  }
+
+  const commandTemplate = String(
+    (pluginConfig as { command?: unknown } | null)?.command || "",
+  ).trim();
+  const strategy = normalizeStrategy(
+    ((pluginConfig as { strategy?: unknown } | null)?.strategy as
+      | VoiceTranscribeStrategy
+      | undefined) ||
+      (provider === "command" ? "command" : "auto"),
+  );
+
+  return {
+    config: (pluginConfig as VoicePluginConfig | null) || {
+      provider: "local",
+    },
+    modelId: modelId as VoiceModelId,
+    provider,
+    modelDir,
+    audioPath,
+    timeoutMs: normalizeTimeoutMs(
+      (pluginConfig as { timeoutMs?: unknown } | null)?.timeoutMs as number | undefined,
+    ),
+    language: normalizeLanguage(
+      input.language ||
+        String((pluginConfig as { language?: unknown } | null)?.language || ""),
+    ),
+    pythonBin: normalizePythonBin(
+      String(
+        (pluginConfig as { pythonBin?: unknown } | null)?.pythonBin || "",
+      ),
+    ),
+    commandTemplate: commandTemplate || undefined,
+    strategy,
+  };
+}
+
+function resolveAutoRunnerOrder(modelId: VoiceModelId): Array<"funasr" | "transformers-whisper"> {
+  if (modelId === "whisper-large-v3-turbo") {
+    return ["transformers-whisper", "funasr"];
+  }
+  return ["funasr", "transformers-whisper"];
+}
+
+/**
+ * 执行 ASR 音频转写。
+ *
+ * 关键点（中文）
+ * - 该函数是 ASR plugin 对 chat 等 service 暴露的核心能力。
+ * - 内置 runner 失败时会返回清晰报错，调用方可降级为附件流程。
+ */
+export async function transcribeVoiceAudio(
+  input: VoiceTranscribeInput,
+): Promise<VoiceTranscribeResult> {
+  const startedAt = Date.now();
+  const resolved = await resolveVoiceConfig(input);
+
+  const runnerFailures: string[] = [];
+
+  const tryRunner = async (
+    runner: "funasr" | "transformers-whisper" | "command",
+  ): Promise<VoiceTranscribeResult | null> => {
+    try {
+      let text = "";
+      if (runner === "command") {
+        const template = String(resolved.commandTemplate || "").trim();
+        if (!template) {
+          throw new Error(
+            "ASR transcribe strategy=command requires plugins.asr.command",
+          );
+        }
+        text = await runCustomCommand({
+          template,
+          timeoutMs: resolved.timeoutMs,
+          values: {
+            audioPath: resolved.audioPath,
+            modelDir: resolved.modelDir,
+            modelId: resolved.modelId,
+            language: resolved.language,
+          },
+        });
+      } else if (runner === "funasr") {
+        text = await runFunasrRunner({
+          pythonBin: resolved.pythonBin,
+          modelDir: resolved.modelDir,
+          audioPath: resolved.audioPath,
+          timeoutMs: resolved.timeoutMs,
+        });
+      } else {
+        text = await runTransformersWhisperRunner({
+          pythonBin: resolved.pythonBin,
+          modelDir: resolved.modelDir,
+          audioPath: resolved.audioPath,
+          language: resolved.language,
+          timeoutMs: resolved.timeoutMs,
+        });
+      }
+
+      const normalized = String(text || "").trim();
+      if (!normalized) {
+        throw new Error("transcript is empty");
+      }
+
+      return {
+        text: normalized,
+        audioPath: resolved.audioPath,
+        modelId: resolved.modelId,
+        provider: resolved.provider,
+        elapsedMs: Date.now() - startedAt,
+        runner,
+      };
+    } catch (error) {
+      runnerFailures.push(`${runner}: ${String(error)}`);
+      return null;
+    }
+  };
+
+  if (resolved.strategy === "command") {
+    const result = await tryRunner("command");
+    if (result) return result;
+  } else if (resolved.strategy === "funasr") {
+    const result = await tryRunner("funasr");
+    if (result) return result;
+  } else if (resolved.strategy === "transformers-whisper") {
+    const result = await tryRunner("transformers-whisper");
+    if (result) return result;
+  } else {
+    for (const runner of resolveAutoRunnerOrder(resolved.modelId)) {
+      const result = await tryRunner(runner);
+      if (result) return result;
+    }
+  }
+
+  throw new Error(
+    `Voice transcription failed for model "${resolved.modelId}". ${runnerFailures.join(" | ")}`,
+  );
+}

package/src/plugins/web/Dependency.ts

@@ -0,0 +1,17 @@
+/**
+ * Web Plugin 兼容导出层。
+ *
+ * 关键点（中文）
+ * - `web` plugin 只保留薄适配层。
+ * - 真正的联网/浏览器能力由外部 provider 项目负责。
+ */
+
+export {
+  readWebPluginConfig,
+  writeWebPluginConfig,
+} from "@/plugins/web/runtime/Config.js";
+export {
+  inspectWebPluginDependency,
+  installWebPluginDependency,
+  doctorWebPluginDependency,
+} from "@/plugins/web/runtime/Source.js";

package/src/plugins/web/PROMPT.agent-browser.txt

@@ -0,0 +1,17 @@
+# Web Plugin / agent-browser
+
+当前 provider 是 `agent-browser`。
+
+你应直接使用 `agent-browser` 项目的浏览器自动化方法，而不是在本项目里自己实现浏览器交互。
+
+工作原则：
+
+- 浏览器交互、打开页面、点击、填写、截图、滚动等操作优先交给 `agent-browser`
+- 严格遵循 `agent-browser` 的基本工作流：
+  1. `open`
+  2. `snapshot -i`
+  3. 使用元素引用交互
+  4. 页面变化后重新 snapshot
+- 如果 `agent-browser` 命令不可用，应先报告依赖缺失
+
+你在处理需要浏览器操作的任务时，应以外部 `agent-browser` 项目为准。

package/src/plugins/web/PROMPT.txt

@@ -0,0 +1,33 @@
+# Web Plugin
+
+此 plugin 已内化 `web-access` 的核心联网策略。只要 plugin 启用，就应把联网任务当作一等能力处理。
+
+## 总原则
+
+- 目标驱动：像人一样思考，先定义成功标准，再选择最可能直达的起点。
+- 过程校验：每一步结果都是证据。路径不推进时，及时切换方法，不在错误方式上反复重试。
+- 一手来源优先：搜索引擎用于发现线索，结论尽量回到官网、官方平台、原始页面、官方文档或源码。
+- 完成即停止：达到目标后不要为了“看起来更完整”继续消耗额外步骤。
+
+## 工具选择
+
+- 需要发现信息来源、查关键词脉络：优先 `WebSearch`
+- 已知 URL，需要定向提取页面正文或答案：优先 `WebFetch`
+- 已知 URL，需要原始 HTML、meta、JSON-LD 等结构化内容：优先 `curl`
+- 需要登录态、动态渲染、站内导航、复杂交互或反爬站点：直接使用浏览器/CDP 能力
+- 页面正文导向内容可优先考虑 Jina 类 markdown 预处理，以节省 token；但数据面板、商品页、复杂布局页面不要盲信预处理结果
+
+## 浏览器策略
+
+- 进入浏览器层后，先理解页面结构，再决定下一步动作
+- 程序化手段快但更容易触发反爬；GUI 交互更慢但确定性更高。程序化受阻时，回退到 GUI
+- 优先使用页面自身生成的链接，不要手工猜测或裁剪站内 URL 参数
+- 只操作自己创建的标签页，不打扰用户已有标签页；任务结束后关闭自己创建的标签页
+- 遇到登录墙时，先判断它是否真的阻挡目标；只有目标确实拿不到时再要求用户登录
+
+## 并行与核实
+
+- 多个独立调研目标可并行拆分；有依赖关系的目标不要盲目并行
+- 核实类任务要回到一手来源，不要用多个二手报道互相循环印证
+
+来源说明：该内化策略参考 `https://github.com/eze-is/web-access`，并按 Downcity plugin 体系进行了收敛与改写。

package/src/plugins/web/PROMPT.web-access.txt

@@ -0,0 +1,13 @@
+# Web Plugin / web-access
+
+当前 provider 是 `web-access`。
+
+你应直接遵循已安装的 `web-access` skill/project 逻辑，而不是在本项目里重新实现一套联网框架。
+
+工作原则：
+
+- 优先沿用 `web-access` 自己的联网策略、工具选择和浏览哲学
+- 把 Downcity 的 `web` plugin 视为 provider 选择器与接入层
+- 如果 `web-access` 不可用，先指出依赖缺失，而不是自行退回到本项目里的临时实现
+
+你在处理联网任务时，应以外部 `web-access` 项目为准。