@downcity/plugins 1.0.56 → 1.0.59

package/src/asr/Plugin.ts

@@ -1,509 +1,290 @@
 /**
- * ASR Plugin。
+ * AsrPlugin：语音转写插件。
  *
  * 关键点（中文）
- * - ASR Plugin 现在自己内聚转写依赖，不再暴露独立依赖端口心智。
- * - 具体模型、依赖安装、命令模板等实现细节全部收敛到 plugin 内部 dependency helper。
- * - 当前作为 chat 入站消息增强中间件接入语音转写。
+ * - ASR 能力通过 constructor 注入，推荐传入 `city.ai.asr`。
+ * - plugin 不负责本地模型、Python 依赖、provider 或项目配置。
+ * - `auto: true` 时会在 chat 入站阶段自动转写 voice/audio 附件，并把结果写入正文。
  */
 
+import path from "node:path";
 import { BasePlugin } from "@downcity/agent/internal/plugin/core/BasePlugin.js";
-import type { Plugin } from "@downcity/agent/internal/plugin/types/Plugin.js";
-import type { VoicePluginConfig } from "@/voice/types/VoicePlugin.js";
+import type { AgentContext } from "@downcity/agent/internal/types/runtime/agent/AgentContext.js";
+import type {
+  JsonObject,
+  JsonValue,
+} from "@downcity/agent/internal/types/common/Json.js";
 import { CHAT_PLUGIN_POINTS } from "@/chat/runtime/PluginPoints.js";
-import { isPluginEnabled } from "@downcity/agent/internal/plugin/core/Activation.js";
-import { writeProjectPluginEnabled } from "@downcity/agent/internal/plugin/core/ProjectConfigStore.js";
-import {
-  listVoiceModels,
-  resolveVoicePluginModelId,
-} from "@/asr/ModelCatalog.js";
-import {
-  getBooleanOpt,
-  getStringOpt,
-  readVoicePluginConfig,
-  toJsonObject,
-  writeVoicePluginConfig,
-} from "@/asr/Config.js";
-import { augmentAsrInboundMessage } from "@/asr/InboundAugment.js";
-import {
-  checkVoiceTranscriber,
-  installVoiceTranscriber,
-  readVoiceTranscriberConfig,
-  transcribeWithVoiceDependency,
-  writeVoiceTranscriberConfig,
-} from "@/asr/Dependency.js";
+import type {
+  ChatInboundAugmentInput,
+  ChatPluginAttachment,
+} from "@/chat/types/ChatPlugin.js";
+import type {
+  AsrPluginInput,
+  AsrPluginOptions,
+  AsrPluginResult,
+} from "@/asr/types/AsrPlugin.js";
 
-function createAsrPluginDefinition(plugin: Plugin): Plugin {
+const DEFAULT_ASR_PLUGIN_NAME = "asr";
+const DEFAULT_ASR_PLUGIN_TITLE = "ASR";
+const DEFAULT_ASR_PLUGIN_DESCRIPTION =
+  "Transcribe voice and audio attachments through an injected ASR function.";
+
+/**
+ * 判断值是否为普通对象。
+ */
+function to_record(value: unknown): Record<string, unknown> | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  return value as Record<string, unknown>;
+}
+
+/**
+ * XML 文本转义。
+ */
+function escape_xml_text(value: string): string {
+  return String(value || "")
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;");
+}
+
+/**
+ * XML 属性转义。
+ */
+function escape_xml_attr(value: string): string {
+  return escape_xml_text(value).replace(/"/g, "&quot;");
+}
+
+/**
+ * 归一化 action payload。
+ */
+function normalize_asr_payload(payload: JsonValue | undefined): AsrPluginInput {
+  const record = to_record(payload ?? {});
+  if (!record) {
+    throw new TypeError("AsrPlugin.transcribe payload must be an object");
+  }
+  const audio_path =
+    typeof record.audio_path === "string"
+      ? record.audio_path.trim()
+      : "";
+  const url = typeof record.url === "string" ? record.url.trim() : "";
+  const data_url = typeof record.data_url === "string" ? record.data_url.trim() : "";
+  if (!audio_path && !url && !data_url) {
+    throw new Error("AsrPlugin.transcribe requires audio_path, url, or data_url");
+  }
   return {
-    name: "asr",
-  title: "Automatic Speech Recognition",
-  description:
-    "Detects voice and audio attachments in inbound chat messages, runs transcription through the configured ASR dependency, and appends readable text so the agent can work with spoken input like normal message content.",
-  config: {
-    plugin: "asr",
-    scope: "project",
-    defaultValue: {
-      injectPrompt: true,
-      augmentMessage: true,
-      provider: "local",
-      modelId: "SenseVoiceSmall",
-    },
-  },
-  setup: {
-    mode: "install-configure",
-    title: "安装语音识别",
-    description: "选择模型后即可完成依赖安装与当前配置写入。",
-    fields: [
-      {
-        key: "modelId",
-        label: "模型",
-        type: "select",
-        required: true,
-        sourceAction: "models",
-      },
-      {
-        key: "installDeps",
-        label: "安装 Python 依赖",
-        type: "checkbox",
-      },
-    ],
-    primaryAction: "install",
-    statusAction: "status",
-  },
-  usage: {
-    title: "配置语音识别",
-    description: "设置当前 agent 如何把语音内容转成文本并注入对话链路。",
-    fields: [
-      {
-        key: "injectPrompt",
-        label: "注入 Prompt 提示",
-        type: "boolean",
-        trueLabel: "开启",
-        falseLabel: "关闭",
-        description: "开启后会把 ASR 使用约束注入到 agent 的系统提示词。",
-      },
-      {
-        key: "augmentMessage",
-        label: "自动增强消息",
-        type: "boolean",
-        trueLabel: "开启",
-        falseLabel: "关闭",
-        description: "开启后，入站语音消息会先自动转写再交给 agent。",
-      },
-      {
-        key: "modelId",
-        label: "默认模型",
-        type: "select",
-        sourceAction: "models",
-      },
-      {
-        key: "language",
-        label: "默认语言提示",
-        type: "string",
-        placeholder: "auto / zh / en",
-      },
-      {
-        key: "timeoutMs",
-        label: "超时（毫秒）",
-        type: "number",
-        placeholder: "60000",
-      },
-    ],
-    saveAction: "configure",
-    statusAction: "status",
-  },
-  async availability(context) {
-    if (!isPluginEnabled({ plugin, context })) {
-      return {
-        enabled: false,
-        available: false,
-        reasons: ["asr plugin disabled in project config"],
-      };
+    ...(record as AsrPluginInput),
+    ...(audio_path ? { audio_path } : {}),
+    ...(url ? { url } : {}),
+    ...(data_url ? { data_url } : {}),
+  };
+}
+
+/**
+ * 校验 ASR 返回结果。
+ */
+function normalize_asr_result(result: AsrPluginResult): AsrPluginResult {
+  const record = to_record(result);
+  if (!record || typeof record.text !== "string") {
+    throw new TypeError("AsrPlugin asr function must return { text: string }");
+  }
+  return {
+    ...record,
+    text: record.text.trim(),
+  } as AsrPluginResult;
+}
+
+/**
+ * 生成相对展示路径。
+ */
+function to_display_src(root_path: string, attachment: ChatPluginAttachment): string {
+  const raw =
+    typeof attachment.path === "string" && attachment.path.trim()
+      ? attachment.path.trim()
+      : attachment.fileName || attachment.attachmentId || attachment.kind;
+  const normalized_root = path.resolve(root_path);
+  const normalized_raw = path.isAbsolute(raw) ? path.resolve(raw) : raw;
+  if (path.isAbsolute(normalized_raw) && normalized_raw.startsWith(`${normalized_root}${path.sep}`)) {
+    return normalized_raw.slice(normalized_root.length + 1);
+  }
+  return raw;
+}
+
+/**
+ * 把语音转写结果写入 bodyText。
+ */
+function append_voice_text(input: ChatInboundAugmentInput, voice_blocks: string[]): ChatInboundAugmentInput {
+  const current = String(input.bodyText || "").trim();
+  const addition = voice_blocks.map((item) => item.trim()).filter(Boolean).join("\n\n");
+  if (!addition) return input;
+  return {
+    ...input,
+    bodyText: [current, addition].filter(Boolean).join("\n\n"),
+  };
+}
+
+/**
+ * Agent ASR 插件。
+ */
+export class AsrPlugin extends BasePlugin {
+  /**
+   * 当前 plugin 稳定名称。
+   */
+  readonly name: string;
+
+  /**
+   * 插件标题。
+   */
+  readonly title: string;
+
+  /**
+   * 插件说明。
+   */
+  readonly description: string;
+
+  private readonly asr: AsrPluginOptions["asr"];
+  private readonly auto: boolean;
+  private readonly language?: string;
+
+  constructor(options: AsrPluginOptions) {
+    super();
+    const name = String(options.name || DEFAULT_ASR_PLUGIN_NAME).trim();
+    if (!name) {
+      throw new Error("AsrPlugin requires a non-empty name");
+    }
+    if (typeof options.asr !== "function") {
+      throw new Error("AsrPlugin requires an asr function");
     }
-    const dependencyStatus = await checkVoiceTranscriber(context);
-    return {
-      enabled: true,
-      available: dependencyStatus.available,
-      reasons: dependencyStatus.reasons,
-    };
-  },
-  hooks: {
+    this.name = name;
+    this.title = String(options.title || DEFAULT_ASR_PLUGIN_TITLE).trim();
+    this.description = String(
+      options.description || DEFAULT_ASR_PLUGIN_DESCRIPTION,
+    ).trim();
+    this.asr = options.asr;
+    this.auto = options.auto === true;
+    this.language =
+      typeof options.language === "string" && options.language.trim()
+        ? options.language.trim()
+        : undefined;
+  }
+
+  /**
+   * ASR 插件给模型的使用说明。
+   */
+  system(_context: AgentContext): string {
+    return [
+      "# ASR Plugin",
+      "",
+      "Use this plugin when the user asks to transcribe, recognize, or understand voice/audio content.",
+      this.auto
+        ? "Inbound voice/audio chat attachments are automatically transcribed into `<voice src=\"...\">...</voice>` blocks in the user text."
+        : "Automatic inbound voice transcription is disabled; call the plugin explicitly when transcription is needed.",
+      "",
+      "Call through `plugin_call`:",
+      "",
+      "```ts",
+      "plugin_call({",
+      `  plugin: "${this.name}",`,
+      '  action: "transcribe",',
+      "  payload: {",
+      '    audio_path: "...",',
+      "  },",
+      "});",
+      "```",
+      "",
+      "Payload rules:",
+      "- Provide one of `audio_path`, `url`, or `data_url`.",
+      "- Optional fields: `language`, `media_type`, `file_name`, `provider_options`.",
+      "- Do not invent transcript text. If transcription fails, report the failure clearly.",
+    ].join("\n");
+  }
+
+  /**
+   * 执行一次 ASR 转写。
+   */
+  private async transcribe(input: AsrPluginInput): Promise<AsrPluginResult> {
+    const result = await this.asr({
+      ...(this.language ? { language: this.language } : {}),
+      ...input,
+    });
+    return normalize_asr_result(result);
+  }
+
+  /**
+   * 自动增强 chat 入站消息。
+   */
+  private async auto_transcribe_inbound(params: {
+    context: AgentContext;
+    value: JsonValue;
+  }): Promise<JsonValue> {
+    if (!this.auto) return params.value;
+    const input = params.value as unknown as ChatInboundAugmentInput;
+    const voice_attachments = (Array.isArray(input.attachments) ? input.attachments : []).filter(
+      (item) =>
+        (item.kind === "voice" || item.kind === "audio") &&
+        typeof item.path === "string" &&
+        item.path.trim(),
+    );
+    if (voice_attachments.length === 0) {
+      return input as unknown as JsonValue;
+    }
+
+    const voice_blocks: string[] = [];
+    for (const attachment of voice_attachments) {
+      try {
+        const result = await this.transcribe({
+          audio_path: String(attachment.path || "").trim(),
+          ...(attachment.contentType ? { media_type: attachment.contentType } : {}),
+          ...(attachment.fileName ? { file_name: attachment.fileName } : {}),
+        });
+        if (!result.text) continue;
+        const src = to_display_src(params.context.rootPath, attachment);
+        voice_blocks.push(
+          `<voice src="${escape_xml_attr(src)}">${escape_xml_text(result.text)}</voice>`,
+        );
+      } catch {
+        // 关键点（中文）：自动转写失败不阻塞主消息链路。
+      }
+    }
+
+    return append_voice_text(input, voice_blocks) as unknown as JsonValue;
+  }
+
+  /**
+   * pipeline / action 扩展点。
+   */
+  readonly hooks = {
     pipeline: {
       [CHAT_PLUGIN_POINTS.augmentInbound]: [
-        async ({ context, value }) => {
-          return await augmentAsrInboundMessage({ context, value });
+        async ({ context, value }: { context: AgentContext; value: JsonValue }) => {
+          return await this.auto_transcribe_inbound({ context, value });
         },
       ],
     },
-  },
-  actions: {
-    status: {
-      allowWhenDisabled: true,
-      command: {
-        description: "查看 asr plugin 当前状态",
-        mapInput() {
-          return {};
-        },
-      },
-      execute: async ({ context }) => {
-        const config = readVoicePluginConfig(context);
-        const availability = await plugin.availability!(context);
-        const transcriberConfig = readVoiceTranscriberConfig(context);
-        return {
-          success: true,
-          data: {
-            plugin: toJsonObject(config) || {},
-            availability: {
-              enabled: availability.enabled,
-              available: availability.available,
-              reasons: availability.reasons,
-            },
-            transcriber:
-              toJsonObject((transcriberConfig || null) as Record<string, unknown> | null),
-          },
-        };
-      },
-    },
-    install: {
-      allowWhenDisabled: true,
-      command: {
-        description: "安装 asr 转写依赖",
-        configure(command) {
-          command
-            .argument("[models...]")
-            .option("--active-model <modelId>", "安装完成后设为当前模型")
-            .option("--models-dir <path>", "模型目录（可选）")
-            .option("--python <bin>", "Python 可执行文件（默认 python3）")
-            .option("--no-install-deps", "跳过依赖安装")
-            .option("--force", "强制覆盖已存在资源")
-            .option("--hf-token <token>", "HuggingFace token（可选）");
-        },
-        mapInput({ args, opts }) {
-          return {
-            modelIds: args,
-            ...(getStringOpt(opts, "activeModel")
-              ? { activeModel: getStringOpt(opts, "activeModel") }
-              : {}),
-            ...(getStringOpt(opts, "modelsDir")
-              ? { modelsDir: getStringOpt(opts, "modelsDir") }
-              : {}),
-            ...(getStringOpt(opts, "python")
-              ? { pythonBin: getStringOpt(opts, "python") }
-              : {}),
-            installDeps: getBooleanOpt(opts, "installDeps", true),
-            force: getBooleanOpt(opts, "force", false),
-            ...(getStringOpt(opts, "hfToken")
-              ? { hfToken: getStringOpt(opts, "hfToken") }
-              : {}),
-          };
-        },
-      },
-      execute: async ({ context, payload }) => {
-        const result = await installVoiceTranscriber({
-          context,
-          input:
-            payload && typeof payload === "object" && !Array.isArray(payload)
-              ? payload
-              : undefined,
-        });
-        return {
-          success: result.success,
-          ...(result.message ? { message: result.message } : {}),
-          ...(result.details !== undefined ? { data: result.details } : {}),
-          ...(result.success ? {} : { error: result.message || "install failed" }),
-        };
-      },
-    },
-    configure: {
-      allowWhenDisabled: true,
-      execute: async ({ context, payload }) => {
-        const payloadObject =
-          payload && typeof payload === "object" && !Array.isArray(payload)
-            ? (payload as Partial<VoicePluginConfig> & Record<string, unknown>)
-            : {};
-        const { enabled: _ignoredEnabled, ...patch } = payloadObject;
-        const current = readVoicePluginConfig(context);
-        const next = {
-          ...current,
-          ...(patch as Partial<VoicePluginConfig>),
-        } satisfies VoicePluginConfig;
-        await writeVoicePluginConfig({
-          agentState: context,
-          value: next,
-        });
-        return {
-          success: true,
-          data: {
-            plugin: toJsonObject(next) || {},
-          },
-        };
-      },
-    },
-    on: {
-      allowWhenDisabled: true,
-      command: {
-        description: "启用 asr plugin，并可选安装转写依赖",
-        configure(command) {
-          command
-            .argument("[models...]")
-            .option("--active-model <modelId>", "安装完成后设为当前模型")
-            .option("--models-dir <path>", "模型目录（可选）")
-            .option("--python <bin>", "Python 可执行文件（默认 python3）")
-            .option("--no-install", "仅启用 plugin，不安装依赖")
-            .option("--no-install-deps", "跳过依赖安装")
-            .option("--force", "强制覆盖已存在资源")
-            .option("--hf-token <token>", "HuggingFace token（可选）")
-            .option("--no-inject-prompt", "关闭 prompt 注入")
-            .option("--no-augment-message", "关闭消息增强");
-        },
-        mapInput({ args, opts }) {
-          return {
-            modelIds: args,
-            ...(getStringOpt(opts, "activeModel")
-              ? { activeModel: getStringOpt(opts, "activeModel") }
-              : {}),
-            ...(getStringOpt(opts, "modelsDir")
-              ? { modelsDir: getStringOpt(opts, "modelsDir") }
-              : {}),
-            ...(getStringOpt(opts, "python")
-              ? { pythonBin: getStringOpt(opts, "python") }
-              : {}),
-            install: getBooleanOpt(opts, "install", true),
-            installDeps: getBooleanOpt(opts, "installDeps", true),
-            force: getBooleanOpt(opts, "force", false),
-            injectPrompt: getBooleanOpt(opts, "injectPrompt", true),
-            augmentMessage: getBooleanOpt(opts, "augmentMessage", true),
-            ...(getStringOpt(opts, "hfToken")
-              ? { hfToken: getStringOpt(opts, "hfToken") }
-              : {}),
-          };
-        },
-      },
-      execute: async ({ context, payload }) => {
-        await writeProjectPluginEnabled({
-          pluginName: "asr",
-          enabled: true,
-          context,
-        });
-        const nextConfig = {
-          ...readVoicePluginConfig(context),
-          injectPrompt:
-            typeof (payload as { injectPrompt?: unknown }).injectPrompt === "boolean"
-              ? ((payload as { injectPrompt?: boolean }).injectPrompt as boolean)
-              : true,
-          augmentMessage:
-            typeof (payload as { augmentMessage?: unknown }).augmentMessage === "boolean"
-              ? ((payload as { augmentMessage?: boolean }).augmentMessage as boolean)
-              : true,
-        };
-        await writeVoicePluginConfig({
-          agentState: context,
-          value: nextConfig,
-        });
-        if ((payload as { install?: unknown }).install !== false) {
-          const installResult = await installVoiceTranscriber({
-            context,
-            input:
-              payload && typeof payload === "object" && !Array.isArray(payload)
-                ? payload
-                : undefined,
-          });
-          if (!installResult.success) {
-            return {
-              success: false,
-              error: installResult.message || "asr dependency install failed",
-              message: installResult.message || "asr dependency install failed",
-            };
-          }
-        }
-        return {
-          success: true,
-          data: {
-            plugin: toJsonObject(nextConfig) || {},
-            transcriber: toJsonObject(readVoiceTranscriberConfig(context)) || {},
-          },
-        };
-      },
-    },
-    off: {
-      command: {
-        description: "关闭 asr plugin",
-        mapInput() {
-          return {};
-        },
-      },
-      execute: async ({ context }) => {
-        await writeProjectPluginEnabled({
-          pluginName: "asr",
-          enabled: false,
-          context,
-        });
-        return {
-          success: true,
-          data: {
-            plugin: toJsonObject(readVoicePluginConfig(context) as Record<string, unknown>) || {},
-          },
-        };
-      },
-    },
-    use: {
-      command: {
-        description: "切换 asr 当前转写模型",
-        configure(command) {
-          command.argument("<modelId>");
-        },
-        mapInput({ args }) {
-          const modelId = String(args[0] || "").trim();
-          if (!modelId) {
-            throw new Error("modelId is required");
-          }
-          return {
-            modelId,
-          };
-        },
-      },
-      execute: async ({ context, payload }) => {
-        const modelId = String((payload as { modelId?: unknown }).modelId || "").trim();
-        const resolvedModelId = resolveVoicePluginModelId(modelId);
-        if (!resolvedModelId) {
-          return {
-            success: false,
-            error: `Unsupported asr model: ${modelId}`,
-            message: `Unsupported asr model: ${modelId}`,
-          };
-        }
-        const transcriberConfig = await writeVoiceTranscriberConfig({
-          context,
-          value: {
-            ...readVoiceTranscriberConfig(context),
-            modelId: resolvedModelId,
-          },
-        });
-        return {
-          success: true,
-          data: {
-            transcriber: toJsonObject(transcriberConfig as Record<string, unknown>) || {},
-          },
-        };
-      },
-    },
+  };
+
+  /**
+   * 显式 action 集合。
+   */
+  readonly actions = {
     transcribe: {
-      command: {
-        description: "转写本地音频文件",
-        configure(command) {
-          command
-            .argument("<audioPath>")
-            .option("--language <code>", "语言提示（可选，例如 zh / en）");
-        },
-        mapInput({ args, opts }) {
-          const audioPath = String(args[0] || "").trim();
-          if (!audioPath) {
-            throw new Error("audioPath is required");
-          }
+      execute: async ({ payload }: { payload: JsonValue }) => {
+        try {
+          const input = normalize_asr_payload(payload);
+          const result = await this.transcribe(input);
           return {
-            audioPath,
-            ...(getStringOpt(opts, "language")
-              ? { language: getStringOpt(opts, "language") }
-              : {}),
+            success: true,
+            data: result as unknown as JsonObject,
+            message: "audio transcribed",
           };
-        },
-      },
-      execute: async ({ context, payload }) => {
-        const pluginStatus = await plugin.availability!(context);
-        if (!pluginStatus.enabled || !pluginStatus.available) {
+        } catch (error) {
           return {
             success: false,
-            error: pluginStatus.reasons.join("; "),
-            message: pluginStatus.reasons.join("; "),
+            error: String(error),
+            message: String(error),
           };
         }
-        const result = await transcribeWithVoiceDependency({
-          context,
-          audioPath: String((payload as { audioPath?: unknown }).audioPath || ""),
-          language:
-            typeof (payload as { language?: unknown }).language === "string"
-              ? String((payload as { language?: unknown }).language || "")
-              : undefined,
-        });
-        return {
-          success: result.success,
-          ...(result !== undefined ? { data: result } : {}),
-          ...(result.success ? {} : { error: result.error || "transcribe failed" }),
-          ...(result.success ? {} : { message: result.error || "transcribe failed" }),
-        };
-      },
-    },
-    models: {
-      allowWhenDisabled: true,
-      command: {
-        description: "列出内置 asr 支持的模型目录",
-        mapInput() {
-          return {};
-        },
-      },
-      execute: async () => {
-        return {
-          success: true,
-          data: {
-            models: listVoiceModels(),
-          },
-        };
-      },
-    },
-    doctor: {
-      command: {
-        description: "检查 asr plugin 与转写依赖可用性",
-        mapInput() {
-          return {};
-        },
-      },
-      execute: async ({ context }) => {
-        const availability = await plugin.availability!(context);
-        const dependencyStatus = await checkVoiceTranscriber(context);
-        return {
-          success: true,
-          data: {
-            availability: {
-              enabled: availability.enabled,
-              available: availability.available,
-              reasons: availability.reasons,
-            },
-            transcriberStatus: {
-              available: dependencyStatus.available,
-              reasons: dependencyStatus.reasons,
-              ...(dependencyStatus.details !== undefined
-                ? { details: dependencyStatus.details }
-                : {}),
-            },
-          },
-          message: availability.available
-            ? "asr plugin is available"
-            : availability.reasons.join("; ") || "asr plugin is not available",
-        };
       },
     },
-  },
-  system(context) {
-    const config = readVoicePluginConfig(context);
-    if (!isPluginEnabled({ plugin, context }) || config.injectPrompt !== true) {
-      return "";
-    }
-    return [
-      "# ASR Plugin",
-      "Audio attachments may be transcribed before agent execution.",
-    ].join("\n");
-  },
   };
 }
-
-/**
- * AsrPlugin：语音识别插件。
- */
-export class AsrPlugin extends BasePlugin {
-  readonly name = "asr";
-
-  constructor() {
-    super();
-    Object.assign(this, createAsrPluginDefinition(this));
-  }
-}