@downcity/plugins 1.0.174 → 1.0.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/BuiltinPlugins.d.ts +4 -9
- package/bin/BuiltinPlugins.d.ts.map +1 -1
- package/bin/BuiltinPlugins.js +4 -8
- package/bin/BuiltinPlugins.js.map +1 -1
- package/bin/chat/Action.d.ts +8 -2
- package/bin/chat/Action.d.ts.map +1 -1
- package/bin/chat/Action.js +9 -5
- package/bin/chat/Action.js.map +1 -1
- package/bin/chat/ChatPlugin.d.ts +2 -22
- package/bin/chat/ChatPlugin.d.ts.map +1 -1
- package/bin/chat/ChatPlugin.js +2 -21
- package/bin/chat/ChatPlugin.js.map +1 -1
- package/bin/chat/Index.d.ts +1 -1
- package/bin/chat/Index.d.ts.map +1 -1
- package/bin/chat/channels/BaseChatChannel.js +1 -1
- package/bin/chat/channels/BaseChatChannel.js.map +1 -1
- package/bin/chat/channels/Configuration.d.ts +1 -1
- package/bin/chat/channels/Configuration.js +1 -1
- package/bin/chat/channels/RuntimeChannel.d.ts +2 -3
- package/bin/chat/channels/RuntimeChannel.d.ts.map +1 -1
- package/bin/chat/channels/RuntimeChannel.js +0 -8
- package/bin/chat/channels/RuntimeChannel.js.map +1 -1
- package/bin/chat/channels/feishu/Feishu.d.ts.map +1 -1
- package/bin/chat/channels/feishu/Feishu.js +2 -1
- package/bin/chat/channels/feishu/Feishu.js.map +1 -1
- package/bin/chat/runtime/ChannelContextStore.d.ts.map +1 -1
- package/bin/chat/runtime/ChannelContextStore.js +13 -12
- package/bin/chat/runtime/ChannelContextStore.js.map +1 -1
- package/bin/chat/runtime/ChatActionExecution.d.ts +26 -1
- package/bin/chat/runtime/ChatActionExecution.d.ts.map +1 -1
- package/bin/chat/runtime/ChatActionExecution.js +31 -3
- package/bin/chat/runtime/ChatActionExecution.js.map +1 -1
- package/bin/chat/runtime/ChatActionInput.d.ts +1 -2
- package/bin/chat/runtime/ChatActionInput.d.ts.map +1 -1
- package/bin/chat/runtime/ChatActionInput.js +1 -27
- package/bin/chat/runtime/ChatActionInput.js.map +1 -1
- package/bin/chat/runtime/ChatChannelActions.d.ts +2 -66
- package/bin/chat/runtime/ChatChannelActions.d.ts.map +1 -1
- package/bin/chat/runtime/ChatChannelActions.js +3 -195
- package/bin/chat/runtime/ChatChannelActions.js.map +1 -1
- package/bin/chat/runtime/ChatChannelConfig.d.ts +2 -23
- package/bin/chat/runtime/ChatChannelConfig.d.ts.map +1 -1
- package/bin/chat/runtime/ChatChannelConfig.js +3 -116
- package/bin/chat/runtime/ChatChannelConfig.js.map +1 -1
- package/bin/chat/runtime/ChatChannelCore.d.ts +0 -14
- package/bin/chat/runtime/ChatChannelCore.d.ts.map +1 -1
- package/bin/chat/runtime/ChatChannelCore.js +1 -1
- package/bin/chat/runtime/ChatChannelCore.js.map +1 -1
- package/bin/chat/runtime/ChatChannelFacade.d.ts +1 -1
- package/bin/chat/runtime/ChatChannelFacade.d.ts.map +1 -1
- package/bin/chat/runtime/ChatChannelFacade.js +1 -1
- package/bin/chat/runtime/ChatChannelFacade.js.map +1 -1
- package/bin/chat/runtime/ChatHistoryStore.d.ts.map +1 -1
- package/bin/chat/runtime/ChatHistoryStore.js +4 -3
- package/bin/chat/runtime/ChatHistoryStore.js.map +1 -1
- package/bin/chat/runtime/ChatPluginActionRegistry.d.ts.map +1 -1
- package/bin/chat/runtime/ChatPluginActionRegistry.js +27 -89
- package/bin/chat/runtime/ChatPluginActionRegistry.js.map +1 -1
- package/bin/chat/runtime/ChatPluginSystem.d.ts +3 -2
- package/bin/chat/runtime/ChatPluginSystem.d.ts.map +1 -1
- package/bin/chat/runtime/ChatPluginSystem.js +5 -5
- package/bin/chat/runtime/ChatPluginSystem.js.map +1 -1
- package/bin/chat/runtime/ChatQueueStore.d.ts +1 -1
- package/bin/chat/runtime/ChatQueueStore.js +2 -2
- package/bin/chat/runtime/ChatQueueStore.js.map +1 -1
- package/bin/chat/runtime/ChatQueueWorker.js +1 -1
- package/bin/chat/runtime/ChatQueueWorker.js.map +1 -1
- package/bin/chat/runtime/ChatSessionDelete.d.ts.map +1 -1
- package/bin/chat/runtime/ChatSessionDelete.js +11 -20
- package/bin/chat/runtime/ChatSessionDelete.js.map +1 -1
- package/bin/chat/runtime/ChatStorage.d.ts +32 -0
- package/bin/chat/runtime/ChatStorage.d.ts.map +1 -0
- package/bin/chat/runtime/ChatStorage.js +158 -0
- package/bin/chat/runtime/ChatStorage.js.map +1 -0
- package/bin/chat/runtime/SystemPrompt.d.ts +3 -2
- package/bin/chat/runtime/SystemPrompt.d.ts.map +1 -1
- package/bin/chat/runtime/SystemPrompt.js +4 -5
- package/bin/chat/runtime/SystemPrompt.js.map +1 -1
- package/bin/chat/types/ChannelConfiguration.d.ts +2 -2
- package/bin/chat/types/ChatPluginActionPayload.d.ts +7 -45
- package/bin/chat/types/ChatPluginActionPayload.d.ts.map +1 -1
- package/bin/chat/types/ChatPluginOptions.d.ts +0 -21
- package/bin/chat/types/ChatPluginOptions.d.ts.map +1 -1
- package/bin/chat/types/ChatStorage.d.ts +36 -0
- package/bin/chat/types/ChatStorage.d.ts.map +1 -0
- package/bin/chat/types/ChatStorage.js +9 -0
- package/bin/chat/types/ChatStorage.js.map +1 -0
- package/bin/contact/Action.d.ts +3 -2
- package/bin/contact/Action.d.ts.map +1 -1
- package/bin/contact/Action.js +2 -2
- package/bin/contact/Action.js.map +1 -1
- package/bin/contact/ContactPlugin.d.ts.map +1 -1
- package/bin/contact/ContactPlugin.js +17 -14
- package/bin/contact/ContactPlugin.js.map +1 -1
- package/bin/contact/runtime/ChatRuntime.js +1 -1
- package/bin/contact/runtime/ChatRuntime.js.map +1 -1
- package/bin/contact/types/ContactPluginOptions.d.ts +4 -0
- package/bin/contact/types/ContactPluginOptions.d.ts.map +1 -1
- package/bin/index.d.ts +5 -5
- package/bin/index.d.ts.map +1 -1
- package/bin/index.js +2 -2
- package/bin/index.js.map +1 -1
- package/bin/memory/Action.d.ts.map +1 -1
- package/bin/memory/Action.js +16 -4
- package/bin/memory/Action.js.map +1 -1
- package/bin/memory/runtime/Store.d.ts.map +1 -1
- package/bin/memory/runtime/Store.js +0 -22
- package/bin/memory/runtime/Store.js.map +1 -1
- package/bin/memory/runtime/Writer.d.ts +1 -1
- package/bin/memory/runtime/Writer.d.ts.map +1 -1
- package/bin/memory/runtime/Writer.js +2 -4
- package/bin/memory/runtime/Writer.js.map +1 -1
- package/bin/skill/Command.d.ts +3 -11
- package/bin/skill/Command.d.ts.map +1 -1
- package/bin/skill/Command.js +3 -31
- package/bin/skill/Command.js.map +1 -1
- package/bin/skill/PROMPT.d.ts +1 -1
- package/bin/skill/PROMPT.d.ts.map +1 -1
- package/bin/skill/PROMPT.js +1 -1
- package/bin/skill/PROMPT.js.map +1 -1
- package/bin/skill/Plugin.d.ts +3 -2
- package/bin/skill/Plugin.d.ts.map +1 -1
- package/bin/skill/Plugin.js +37 -129
- package/bin/skill/Plugin.js.map +1 -1
- package/bin/skill/runtime/Prompt.d.ts +11 -2
- package/bin/skill/runtime/Prompt.d.ts.map +1 -1
- package/bin/skill/runtime/Prompt.js +93 -8
- package/bin/skill/runtime/Prompt.js.map +1 -1
- package/bin/skill/runtime/SystemProvider.d.ts +2 -1
- package/bin/skill/runtime/SystemProvider.d.ts.map +1 -1
- package/bin/skill/runtime/SystemProvider.js +4 -9
- package/bin/skill/runtime/SystemProvider.js.map +1 -1
- package/bin/skill/types/SkillPlugin.d.ts +6 -18
- package/bin/skill/types/SkillPlugin.d.ts.map +1 -1
- package/bin/skill/types/SkillPlugin.js +2 -2
- package/bin/sound/Plugin.d.ts +69 -0
- package/bin/sound/Plugin.d.ts.map +1 -0
- package/bin/sound/Plugin.js +623 -0
- package/bin/sound/Plugin.js.map +1 -0
- package/bin/sound/types/SoundPlugin.d.ts +150 -0
- package/bin/sound/types/SoundPlugin.d.ts.map +1 -0
- package/bin/sound/types/SoundPlugin.js +10 -0
- package/bin/sound/types/SoundPlugin.js.map +1 -0
- package/bin/task/Action.d.ts +2 -0
- package/bin/task/Action.d.ts.map +1 -1
- package/bin/task/Action.js +6 -0
- package/bin/task/Action.js.map +1 -1
- package/bin/task/runtime/Runner.d.ts +4 -0
- package/bin/task/runtime/Runner.d.ts.map +1 -1
- package/bin/task/runtime/Runner.js +15 -11
- package/bin/task/runtime/Runner.js.map +1 -1
- package/bin/task/runtime/TaskActionExecution.d.ts +2 -0
- package/bin/task/runtime/TaskActionExecution.d.ts.map +1 -1
- package/bin/task/runtime/TaskActionExecution.js +1 -0
- package/bin/task/runtime/TaskActionExecution.js.map +1 -1
- package/bin/task/runtime/TaskPluginActionRegistry.d.ts.map +1 -1
- package/bin/task/runtime/TaskPluginActionRegistry.js +1 -0
- package/bin/task/runtime/TaskPluginActionRegistry.js.map +1 -1
- package/bin/task/runtime/TaskRunChatDispatch.js +1 -1
- package/bin/task/runtime/TaskRunChatDispatch.js.map +1 -1
- package/bin/task/runtime/TaskRunnerRound.d.ts.map +1 -1
- package/bin/task/runtime/TaskRunnerRound.js +19 -26
- package/bin/task/runtime/TaskRunnerRound.js.map +1 -1
- package/bin/task/runtime/TaskRunnerSession.d.ts +6 -2
- package/bin/task/runtime/TaskRunnerSession.d.ts.map +1 -1
- package/bin/task/runtime/TaskRunnerSession.js +93 -72
- package/bin/task/runtime/TaskRunnerSession.js.map +1 -1
- package/bin/task/runtime/TaskRunnerTypes.d.ts +4 -4
- package/bin/task/runtime/TaskRunnerTypes.d.ts.map +1 -1
- package/bin/task/types/TaskPluginOptions.d.ts.map +1 -1
- package/bin/web/PROMPT.d.ts +1 -1
- package/bin/web/PROMPT.d.ts.map +1 -1
- package/bin/web/PROMPT.js +2 -2
- package/bin/web/PROMPT.js.map +1 -1
- package/bin/web/Plugin.d.ts +1 -21
- package/bin/web/Plugin.d.ts.map +1 -1
- package/bin/web/Plugin.js +14 -79
- package/bin/web/Plugin.js.map +1 -1
- package/bin/web/runtime/Prompt.d.ts +14 -0
- package/bin/web/runtime/Prompt.d.ts.map +1 -0
- package/bin/web/runtime/Prompt.js +77 -0
- package/bin/web/runtime/Prompt.js.map +1 -0
- package/bin/web/types/WebPlugin.d.ts +40 -29
- package/bin/web/types/WebPlugin.d.ts.map +1 -1
- package/bin/web/types/WebPlugin.js +11 -3
- package/bin/web/types/WebPlugin.js.map +1 -1
- package/bin/workboard/runtime/Collector.js +1 -1
- package/bin/workboard/runtime/Collector.js.map +1 -1
- package/bin/workboard/runtime/Normalizer.js +1 -1
- package/bin/workboard/runtime/Normalizer.js.map +1 -1
- package/bin/workboard/runtime/SessionSummary.d.ts.map +1 -1
- package/bin/workboard/runtime/SessionSummary.js +11 -38
- package/bin/workboard/runtime/SessionSummary.js.map +1 -1
- package/package.json +8 -4
- package/scripts/chat-access.test.mjs +1 -1
- package/scripts/chat-channel-config.test.mjs +15 -64
- package/scripts/chat-storage.test.mjs +92 -0
- package/scripts/skill-plugin.test.mjs +93 -0
- package/scripts/sound-plugin.test.mjs +250 -0
- package/scripts/web-plugin.test.mjs +85 -0
- package/src/BuiltinPlugins.ts +7 -17
- package/src/chat/Action.ts +16 -5
- package/src/chat/ChatPlugin.ts +6 -35
- package/src/chat/Index.ts +0 -1
- package/src/chat/channels/BaseChatChannel.ts +1 -1
- package/src/chat/channels/Configuration.ts +1 -1
- package/src/chat/channels/RuntimeChannel.ts +2 -14
- package/src/chat/channels/feishu/Feishu.ts +2 -1
- package/src/chat/runtime/ChannelContextStore.ts +16 -12
- package/src/chat/runtime/ChatActionExecution.ts +48 -3
- package/src/chat/runtime/ChatActionInput.ts +0 -31
- package/src/chat/runtime/ChatChannelActions.ts +3 -254
- package/src/chat/runtime/ChatChannelConfig.ts +4 -133
- package/src/chat/runtime/ChatChannelCore.ts +1 -15
- package/src/chat/runtime/ChatChannelFacade.ts +0 -4
- package/src/chat/runtime/ChatHistoryStore.ts +4 -3
- package/src/chat/runtime/ChatPluginActionRegistry.ts +26 -98
- package/src/chat/runtime/ChatPluginSystem.ts +9 -3
- package/src/chat/runtime/ChatQueueStore.ts +2 -2
- package/src/chat/runtime/ChatQueueWorker.ts +1 -1
- package/src/chat/runtime/ChatSessionDelete.ts +11 -21
- package/src/chat/runtime/ChatStorage.ts +203 -0
- package/src/chat/runtime/SystemPrompt.ts +8 -3
- package/src/chat/types/ChannelConfiguration.ts +2 -2
- package/src/chat/types/ChatPluginActionPayload.ts +8 -49
- package/src/chat/types/ChatPluginOptions.ts +0 -22
- package/src/chat/types/ChatStorage.ts +37 -0
- package/src/contact/Action.ts +9 -1
- package/src/contact/ContactPlugin.ts +38 -14
- package/src/contact/runtime/ChatRuntime.ts +1 -1
- package/src/contact/types/ContactPluginOptions.ts +6 -0
- package/src/index.ts +16 -14
- package/src/memory/Action.ts +19 -4
- package/src/memory/runtime/Store.ts +0 -27
- package/src/memory/runtime/Writer.ts +2 -5
- package/src/skill/Command.ts +3 -33
- package/src/skill/PROMPT.ts +1 -1
- package/src/skill/PROMPT.ts.txt +5 -1
- package/src/skill/Plugin.ts +49 -156
- package/src/skill/runtime/Prompt.ts +109 -9
- package/src/skill/runtime/SystemProvider.ts +5 -9
- package/src/skill/types/SkillPlugin.ts +6 -18
- package/src/sound/Plugin.ts +699 -0
- package/src/sound/types/SoundPlugin.ts +163 -0
- package/src/task/Action.ts +8 -0
- package/src/task/runtime/Runner.ts +19 -13
- package/src/task/runtime/TaskActionExecution.ts +3 -0
- package/src/task/runtime/TaskPluginActionRegistry.ts +1 -0
- package/src/task/runtime/TaskRunChatDispatch.ts +1 -1
- package/src/task/runtime/TaskRunnerRound.ts +20 -38
- package/src/task/runtime/TaskRunnerSession.ts +108 -72
- package/src/task/runtime/TaskRunnerTypes.ts +4 -4
- package/src/task/types/TaskPluginOptions.ts +1 -0
- package/src/web/PROMPT.ts +2 -2
- package/src/web/PROMPT.ts.txt +4 -2
- package/src/web/Plugin.ts +20 -90
- package/src/web/runtime/Prompt.ts +99 -0
- package/src/web/types/WebPlugin.ts +42 -29
- package/src/workboard/runtime/Collector.ts +1 -1
- package/src/workboard/runtime/Normalizer.ts +1 -1
- package/src/workboard/runtime/SessionSummary.ts +11 -42
- package/bin/asr/Plugin.d.ts +0 -63
- package/bin/asr/Plugin.d.ts.map +0 -1
- package/bin/asr/Plugin.js +0 -280
- package/bin/asr/Plugin.js.map +0 -1
- package/bin/asr/types/AsrPlugin.d.ts +0 -114
- package/bin/asr/types/AsrPlugin.d.ts.map +0 -1
- package/bin/asr/types/AsrPlugin.js +0 -10
- package/bin/asr/types/AsrPlugin.js.map +0 -1
- package/bin/chat/runtime/ChatSession.d.ts +0 -97
- package/bin/chat/runtime/ChatSession.d.ts.map +0 -1
- package/bin/chat/runtime/ChatSession.js +0 -91
- package/bin/chat/runtime/ChatSession.js.map +0 -1
- package/bin/chat/runtime/ChatSessionContextComposer.d.ts +0 -45
- package/bin/chat/runtime/ChatSessionContextComposer.d.ts.map +0 -1
- package/bin/chat/runtime/ChatSessionContextComposer.js +0 -60
- package/bin/chat/runtime/ChatSessionContextComposer.js.map +0 -1
- package/bin/chat/runtime/ChatSessionTypes.d.ts +0 -30
- package/bin/chat/runtime/ChatSessionTypes.d.ts.map +0 -1
- package/bin/chat/runtime/ChatSessionTypes.js +0 -9
- package/bin/chat/runtime/ChatSessionTypes.js.map +0 -1
- package/bin/tts/Plugin.d.ts +0 -49
- package/bin/tts/Plugin.d.ts.map +0 -1
- package/bin/tts/Plugin.js +0 -247
- package/bin/tts/Plugin.js.map +0 -1
- package/bin/tts/types/TtsPlugin.d.ts +0 -122
- package/bin/tts/types/TtsPlugin.d.ts.map +0 -1
- package/bin/tts/types/TtsPlugin.js +0 -10
- package/bin/tts/types/TtsPlugin.js.map +0 -1
- package/bin/web/runtime/Install.d.ts +0 -19
- package/bin/web/runtime/Install.d.ts.map +0 -1
- package/bin/web/runtime/Install.js +0 -178
- package/bin/web/runtime/Install.js.map +0 -1
- package/src/asr/Plugin.ts +0 -322
- package/src/asr/types/AsrPlugin.ts +0 -118
- package/src/chat/runtime/ChatSession.ts +0 -168
- package/src/chat/runtime/ChatSessionContextComposer.ts +0 -106
- package/src/chat/runtime/ChatSessionTypes.ts +0 -32
- package/src/tts/Plugin.ts +0 -284
- package/src/tts/types/TtsPlugin.ts +0 -129
- package/src/web/runtime/Install.ts +0 -241
|
@@ -0,0 +1,699 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SoundPlugin:统一的语音识别与语音合成插件。
|
|
3
|
+
*
|
|
4
|
+
* 关键点(中文)
|
|
5
|
+
* - 对 Agent 暴露 `models`、`asr`、`tts` 三个 action。
|
|
6
|
+
* - 模型目录与真实 ASR/TTS 能力全部由 FED/City constructor 回调注入。
|
|
7
|
+
* - 本地音频只负责读取并转换为 data URL,不加载或运行任何本地语音模型。
|
|
8
|
+
* - TTS 返回 AI SDK UIMessage,音频 file part 由 agent 统一落盘。
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import fs from "node:fs/promises";
|
|
12
|
+
import path from "node:path";
|
|
13
|
+
import { BasePlugin, createAction } from "@downcity/agent";
|
|
14
|
+
import { z } from "zod";
|
|
15
|
+
import type { AgentContext, JsonObject, JsonValue } from "@downcity/agent";
|
|
16
|
+
import { CHAT_PLUGIN_POINTS } from "@/chat/runtime/PluginPoints.js";
|
|
17
|
+
import type {
|
|
18
|
+
ChatInboundAugmentInput,
|
|
19
|
+
ChatPluginAttachment,
|
|
20
|
+
} from "@/chat/types/ChatPlugin.js";
|
|
21
|
+
import type {
|
|
22
|
+
SoundPluginAsrInput,
|
|
23
|
+
SoundPluginAsrResult,
|
|
24
|
+
SoundPluginAsrSegment,
|
|
25
|
+
SoundPluginCapability,
|
|
26
|
+
SoundPluginModel,
|
|
27
|
+
SoundPluginModelsResult,
|
|
28
|
+
SoundPluginOptions,
|
|
29
|
+
SoundPluginTtsInput,
|
|
30
|
+
SoundPluginTtsResult,
|
|
31
|
+
} from "@/sound/types/SoundPlugin.js";
|
|
32
|
+
|
|
33
|
+
const DEFAULT_SOUND_PLUGIN_NAME = "sound";
|
|
34
|
+
const DEFAULT_SOUND_PLUGIN_TITLE = "Sound";
|
|
35
|
+
const DEFAULT_SOUND_PLUGIN_DESCRIPTION =
|
|
36
|
+
"Transcribe audio and synthesize speech through FED-provided models.";
|
|
37
|
+
const DEFAULT_AUDIO_MEDIA_TYPE = "audio/mpeg";
|
|
38
|
+
|
|
39
|
+
const AUDIO_MEDIA_TYPES: Record<string, string> = {
|
|
40
|
+
".aac": "audio/aac",
|
|
41
|
+
".flac": "audio/flac",
|
|
42
|
+
".m4a": "audio/mp4",
|
|
43
|
+
".mp3": "audio/mpeg",
|
|
44
|
+
".oga": "audio/ogg",
|
|
45
|
+
".ogg": "audio/ogg",
|
|
46
|
+
".opus": "audio/opus",
|
|
47
|
+
".wav": "audio/wav",
|
|
48
|
+
".webm": "audio/webm",
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const SOUND_MODELS_INPUT_SCHEMA = z.object({
|
|
52
|
+
capability: z.enum(["asr", "tts"]).optional(),
|
|
53
|
+
}).passthrough();
|
|
54
|
+
|
|
55
|
+
const SOUND_ASR_INPUT_SCHEMA = z.object({
|
|
56
|
+
model: z.string().optional(),
|
|
57
|
+
audio_path: z.string().optional(),
|
|
58
|
+
url: z.string().optional(),
|
|
59
|
+
data_url: z.string().optional(),
|
|
60
|
+
language: z.string().optional(),
|
|
61
|
+
media_type: z.string().optional(),
|
|
62
|
+
filename: z.string().optional(),
|
|
63
|
+
provider_options: z.record(z.string(), z.unknown()).optional(),
|
|
64
|
+
}).passthrough();
|
|
65
|
+
|
|
66
|
+
const SOUND_TTS_INPUT_SCHEMA = z.object({
|
|
67
|
+
model: z.string().optional(),
|
|
68
|
+
text: z.string(),
|
|
69
|
+
language: z.string().optional(),
|
|
70
|
+
voice: z.string().optional(),
|
|
71
|
+
format: z.string().optional(),
|
|
72
|
+
speed: z.number().optional(),
|
|
73
|
+
instructions: z.string().optional(),
|
|
74
|
+
provider_options: z.record(z.string(), z.unknown()).optional(),
|
|
75
|
+
}).passthrough();
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* 判断值是否为普通对象。
|
|
79
|
+
*/
|
|
80
|
+
function to_record(value: unknown): Record<string, unknown> | null {
|
|
81
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
82
|
+
return value as Record<string, unknown>;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* 把异常及 cause 链整理为可读文本。
|
|
87
|
+
*/
|
|
88
|
+
function describe_error(error: unknown): string {
|
|
89
|
+
if (!(error instanceof Error)) return String(error);
|
|
90
|
+
const parts = [error.message || error.name || "Error"];
|
|
91
|
+
let current: unknown = error.cause;
|
|
92
|
+
let depth = 0;
|
|
93
|
+
while (current && depth < 3) {
|
|
94
|
+
if (!(current instanceof Error)) {
|
|
95
|
+
parts.push(String(current));
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
const code = (current as { code?: unknown }).code;
|
|
99
|
+
const code_text = typeof code === "string" && code ? `[${code}] ` : "";
|
|
100
|
+
parts.push(`${code_text}${current.message || current.name}`.trim());
|
|
101
|
+
current = current.cause;
|
|
102
|
+
depth += 1;
|
|
103
|
+
}
|
|
104
|
+
return parts.filter(Boolean).join(" :: ");
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* 归一化可选字符串。
|
|
109
|
+
*/
|
|
110
|
+
function normalize_optional_string(value: unknown): string | undefined {
|
|
111
|
+
const normalized = typeof value === "string" ? value.trim() : "";
|
|
112
|
+
return normalized || undefined;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* XML 文本转义。
|
|
117
|
+
*/
|
|
118
|
+
function escape_xml_text(value: string): string {
|
|
119
|
+
return value
|
|
120
|
+
.replace(/&/g, "&")
|
|
121
|
+
.replace(/</g, "<")
|
|
122
|
+
.replace(/>/g, ">");
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* XML 属性转义。
|
|
127
|
+
*/
|
|
128
|
+
function escape_xml_attr(value: string): string {
|
|
129
|
+
return escape_xml_text(value).replace(/"/g, """);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* 归一化模型筛选 action 的 payload。
|
|
134
|
+
*/
|
|
135
|
+
function normalize_models_capability(
|
|
136
|
+
payload: JsonValue | undefined,
|
|
137
|
+
): SoundPluginCapability | undefined {
|
|
138
|
+
const record = to_record(payload ?? {});
|
|
139
|
+
if (!record) {
|
|
140
|
+
throw new TypeError("SoundPlugin.models payload must be an object");
|
|
141
|
+
}
|
|
142
|
+
const capability = normalize_optional_string(record.capability);
|
|
143
|
+
if (!capability) return undefined;
|
|
144
|
+
if (capability !== "asr" && capability !== "tts") {
|
|
145
|
+
throw new TypeError("SoundPlugin.models capability must be asr or tts");
|
|
146
|
+
}
|
|
147
|
+
return capability;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* 归一化 ASR action payload。
|
|
152
|
+
*/
|
|
153
|
+
function normalize_asr_payload(payload: JsonValue | undefined): SoundPluginAsrInput {
|
|
154
|
+
const record = to_record(payload ?? {});
|
|
155
|
+
if (!record) {
|
|
156
|
+
throw new TypeError("SoundPlugin.asr payload must be an object");
|
|
157
|
+
}
|
|
158
|
+
const audio_path = normalize_optional_string(record.audio_path);
|
|
159
|
+
const url = normalize_optional_string(record.url);
|
|
160
|
+
const data_url = normalize_optional_string(record.data_url);
|
|
161
|
+
const source_count = [audio_path, url, data_url].filter(Boolean).length;
|
|
162
|
+
if (source_count !== 1) {
|
|
163
|
+
throw new TypeError("SoundPlugin.asr requires exactly one of audio_path, url, or data_url");
|
|
164
|
+
}
|
|
165
|
+
return {
|
|
166
|
+
...(record as SoundPluginAsrInput),
|
|
167
|
+
...(audio_path ? { audio_path } : {}),
|
|
168
|
+
...(url ? { url } : {}),
|
|
169
|
+
...(data_url ? { data_url } : {}),
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* 归一化 TTS action payload。
|
|
175
|
+
*/
|
|
176
|
+
function normalize_tts_payload(payload: JsonValue | undefined): SoundPluginTtsInput {
|
|
177
|
+
const record = to_record(payload ?? {});
|
|
178
|
+
if (!record) {
|
|
179
|
+
throw new TypeError("SoundPlugin.tts payload must be an object");
|
|
180
|
+
}
|
|
181
|
+
const text = normalize_optional_string(record.text);
|
|
182
|
+
if (!text) {
|
|
183
|
+
throw new TypeError("SoundPlugin.tts requires text");
|
|
184
|
+
}
|
|
185
|
+
return {
|
|
186
|
+
...(record as SoundPluginTtsInput),
|
|
187
|
+
text,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* 推断本地音频 MIME 类型。
|
|
193
|
+
*/
|
|
194
|
+
function infer_audio_media_type(file_path: string, fallback?: string): string {
|
|
195
|
+
const normalized_fallback = normalize_optional_string(fallback);
|
|
196
|
+
if (normalized_fallback) return normalized_fallback;
|
|
197
|
+
return AUDIO_MEDIA_TYPES[path.extname(file_path).toLowerCase()] ?? DEFAULT_AUDIO_MEDIA_TYPE;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* 把本地音频读取为 FED 可直接接收的 data URL。
|
|
202
|
+
*/
|
|
203
|
+
async function local_audio_to_data_url(input: {
|
|
204
|
+
/** 当前 Agent 项目根目录。 */
|
|
205
|
+
root_path: string;
|
|
206
|
+
/** 音频绝对路径或相对项目根目录的路径。 */
|
|
207
|
+
audio_path: string;
|
|
208
|
+
/** 调用方显式提供的 MIME 类型。 */
|
|
209
|
+
media_type?: string;
|
|
210
|
+
}): Promise<{ data_url: string; media_type: string; filename: string }> {
|
|
211
|
+
const file_path = path.isAbsolute(input.audio_path)
|
|
212
|
+
? path.resolve(input.audio_path)
|
|
213
|
+
: path.resolve(input.root_path, input.audio_path);
|
|
214
|
+
const media_type = infer_audio_media_type(file_path, input.media_type);
|
|
215
|
+
const bytes = await fs.readFile(file_path);
|
|
216
|
+
return {
|
|
217
|
+
data_url: `data:${media_type};base64,${bytes.toString("base64")}`,
|
|
218
|
+
media_type,
|
|
219
|
+
filename: path.basename(file_path),
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* 把 Agent 公开 ASR 输入解析为可直接发送给 FED 的输入。
|
|
225
|
+
*/
|
|
226
|
+
async function resolve_asr_input(
|
|
227
|
+
context: AgentContext,
|
|
228
|
+
input: SoundPluginAsrInput,
|
|
229
|
+
): Promise<SoundPluginAsrInput> {
|
|
230
|
+
if (!input.audio_path) return input;
|
|
231
|
+
const local = await local_audio_to_data_url({
|
|
232
|
+
root_path: context.rootPath,
|
|
233
|
+
audio_path: input.audio_path,
|
|
234
|
+
media_type: input.media_type,
|
|
235
|
+
});
|
|
236
|
+
const { audio_path: _audio_path, ...rest } = input;
|
|
237
|
+
return {
|
|
238
|
+
...rest,
|
|
239
|
+
data_url: local.data_url,
|
|
240
|
+
media_type: local.media_type,
|
|
241
|
+
filename: normalize_optional_string(input.filename) ?? local.filename,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* 为指定能力解析模型 ID。
|
|
247
|
+
*/
|
|
248
|
+
function resolve_model_id(
|
|
249
|
+
capability: SoundPluginCapability,
|
|
250
|
+
input_model: unknown,
|
|
251
|
+
default_model: string | undefined,
|
|
252
|
+
): string {
|
|
253
|
+
const model = normalize_optional_string(input_model) ?? default_model;
|
|
254
|
+
if (model) return model;
|
|
255
|
+
throw new TypeError(
|
|
256
|
+
`SoundPlugin.${capability} requires a model id; call sound.models first or configure default_${capability}_model`,
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* 归一化 JSON 对象。
|
|
262
|
+
*/
|
|
263
|
+
function normalize_json_object(value: unknown): JsonObject | undefined {
|
|
264
|
+
const record = to_record(value);
|
|
265
|
+
return record ? record as JsonObject : undefined;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* 归一化单个 FED 语音模型。
|
|
270
|
+
*/
|
|
271
|
+
function normalize_sound_model(value: SoundPluginModel): SoundPluginModel | null {
|
|
272
|
+
const record = to_record(value);
|
|
273
|
+
if (!record) return null;
|
|
274
|
+
const id = normalize_optional_string(record.id);
|
|
275
|
+
if (!id) return null;
|
|
276
|
+
const modalities = Array.isArray(record.modalities)
|
|
277
|
+
? record.modalities
|
|
278
|
+
.map((item) => normalize_optional_string(item))
|
|
279
|
+
.filter((item): item is string => Boolean(item))
|
|
280
|
+
: [];
|
|
281
|
+
if (!modalities.includes("asr") && !modalities.includes("tts")) return null;
|
|
282
|
+
const tags = Array.isArray(record.tags)
|
|
283
|
+
? record.tags
|
|
284
|
+
.map((item) => normalize_optional_string(item))
|
|
285
|
+
.filter((item): item is string => Boolean(item))
|
|
286
|
+
: undefined;
|
|
287
|
+
const meta = normalize_json_object(record.meta);
|
|
288
|
+
return {
|
|
289
|
+
id,
|
|
290
|
+
name: normalize_optional_string(record.name) ?? id,
|
|
291
|
+
...(typeof record.description === "string"
|
|
292
|
+
? { description: record.description.trim() }
|
|
293
|
+
: {}),
|
|
294
|
+
modalities,
|
|
295
|
+
...(tags?.length ? { tags } : {}),
|
|
296
|
+
...(meta ? { meta } : {}),
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* 归一化并筛选 FED 语音模型列表。
|
|
302
|
+
*/
|
|
303
|
+
function normalize_sound_models(
|
|
304
|
+
values: SoundPluginModel[],
|
|
305
|
+
capability?: SoundPluginCapability,
|
|
306
|
+
): SoundPluginModelsResult {
|
|
307
|
+
const items = values
|
|
308
|
+
.map((item) => normalize_sound_model(item))
|
|
309
|
+
.filter((item): item is SoundPluginModel => item !== null)
|
|
310
|
+
.filter((item) => !capability || item.modalities.includes(capability));
|
|
311
|
+
return { items };
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* 校验并归一化单个 ASR 分段。
|
|
316
|
+
*/
|
|
317
|
+
function normalize_asr_segment(value: unknown): SoundPluginAsrSegment | null {
|
|
318
|
+
const record = to_record(value);
|
|
319
|
+
if (!record) return null;
|
|
320
|
+
const text = normalize_optional_string(record.text);
|
|
321
|
+
if (!text) return null;
|
|
322
|
+
const start_second = Number(record.startSecond);
|
|
323
|
+
const end_second = Number(record.endSecond);
|
|
324
|
+
if (!Number.isFinite(start_second) || !Number.isFinite(end_second)) return null;
|
|
325
|
+
return {
|
|
326
|
+
text,
|
|
327
|
+
startSecond: start_second,
|
|
328
|
+
endSecond: end_second,
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* 校验并归一化 ASR 返回结果。
|
|
334
|
+
*/
|
|
335
|
+
function normalize_asr_result(result: SoundPluginAsrResult): SoundPluginAsrResult {
|
|
336
|
+
const record = to_record(result);
|
|
337
|
+
const text = normalize_optional_string(record?.text);
|
|
338
|
+
if (!record || !text) {
|
|
339
|
+
throw new TypeError("SoundPlugin asr function must return transcription text");
|
|
340
|
+
}
|
|
341
|
+
const segments = Array.isArray(record.segments)
|
|
342
|
+
? record.segments
|
|
343
|
+
.map((item) => normalize_asr_segment(item))
|
|
344
|
+
.filter((item): item is SoundPluginAsrSegment => item !== null)
|
|
345
|
+
: undefined;
|
|
346
|
+
const language = normalize_optional_string(record.language);
|
|
347
|
+
const duration_in_seconds = Number(record.durationInSeconds);
|
|
348
|
+
return {
|
|
349
|
+
text,
|
|
350
|
+
...(segments ? { segments } : {}),
|
|
351
|
+
...(language ? { language } : {}),
|
|
352
|
+
...(Number.isFinite(duration_in_seconds) && duration_in_seconds >= 0
|
|
353
|
+
? { durationInSeconds: duration_in_seconds }
|
|
354
|
+
: {}),
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* 校验 TTS 返回的 AI SDK UIMessage。
|
|
360
|
+
*/
|
|
361
|
+
function normalize_tts_result(result: SoundPluginTtsResult): SoundPluginTtsResult {
|
|
362
|
+
const record = to_record(result);
|
|
363
|
+
if (!record || !Array.isArray(record.parts)) {
|
|
364
|
+
throw new TypeError("SoundPlugin tts function must return an AI SDK UIMessage");
|
|
365
|
+
}
|
|
366
|
+
const has_audio_file = record.parts.some((part) => {
|
|
367
|
+
const part_record = to_record(part);
|
|
368
|
+
return part_record?.type === "file"
|
|
369
|
+
&& typeof part_record.mediaType === "string"
|
|
370
|
+
&& part_record.mediaType.startsWith("audio/");
|
|
371
|
+
});
|
|
372
|
+
if (!has_audio_file) {
|
|
373
|
+
throw new TypeError("SoundPlugin tts UIMessage must contain an audio file part");
|
|
374
|
+
}
|
|
375
|
+
return result;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* 生成 chat 入站附件的展示路径。
|
|
380
|
+
*/
|
|
381
|
+
function to_display_src(root_path: string, attachment: ChatPluginAttachment): string {
|
|
382
|
+
const raw = normalize_optional_string(attachment.path)
|
|
383
|
+
?? attachment.fileName
|
|
384
|
+
?? attachment.attachmentId
|
|
385
|
+
?? attachment.kind;
|
|
386
|
+
const normalized_root = path.resolve(root_path);
|
|
387
|
+
const normalized_raw = path.isAbsolute(raw) ? path.resolve(raw) : raw;
|
|
388
|
+
if (
|
|
389
|
+
path.isAbsolute(normalized_raw)
|
|
390
|
+
&& normalized_raw.startsWith(`${normalized_root}${path.sep}`)
|
|
391
|
+
) {
|
|
392
|
+
return normalized_raw.slice(normalized_root.length + 1);
|
|
393
|
+
}
|
|
394
|
+
return raw;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* 把自动转写结果追加到 chat 正文。
|
|
399
|
+
*/
|
|
400
|
+
function append_voice_text(
|
|
401
|
+
input: ChatInboundAugmentInput,
|
|
402
|
+
voice_blocks: string[],
|
|
403
|
+
): ChatInboundAugmentInput {
|
|
404
|
+
const current = String(input.bodyText || "").trim();
|
|
405
|
+
const addition = voice_blocks.map((item) => item.trim()).filter(Boolean).join("\n\n");
|
|
406
|
+
if (!addition) return input;
|
|
407
|
+
return {
|
|
408
|
+
...input,
|
|
409
|
+
bodyText: [current, addition].filter(Boolean).join("\n\n"),
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* Agent 统一语音插件。
|
|
415
|
+
*/
|
|
416
|
+
export class SoundPlugin extends BasePlugin {
|
|
417
|
+
/** 当前 plugin 稳定名称。 */
|
|
418
|
+
readonly name: string;
|
|
419
|
+
|
|
420
|
+
/** Plugin 展示标题。 */
|
|
421
|
+
readonly title: string;
|
|
422
|
+
|
|
423
|
+
/** Plugin 用途说明。 */
|
|
424
|
+
readonly description: string;
|
|
425
|
+
|
|
426
|
+
private readonly asr_handler: SoundPluginOptions["asr"];
|
|
427
|
+
private readonly tts_handler: SoundPluginOptions["tts"];
|
|
428
|
+
private readonly list_models?: SoundPluginOptions["list_models"];
|
|
429
|
+
private readonly default_asr_model?: string;
|
|
430
|
+
private readonly default_tts_model?: string;
|
|
431
|
+
private readonly auto_asr: boolean;
|
|
432
|
+
private readonly language?: string;
|
|
433
|
+
private readonly voice?: string;
|
|
434
|
+
private readonly format?: string;
|
|
435
|
+
|
|
436
|
+
constructor(options: SoundPluginOptions) {
|
|
437
|
+
super();
|
|
438
|
+
const name = normalize_optional_string(options.name) ?? DEFAULT_SOUND_PLUGIN_NAME;
|
|
439
|
+
if (typeof options.asr !== "function") {
|
|
440
|
+
throw new TypeError("SoundPlugin requires an asr function");
|
|
441
|
+
}
|
|
442
|
+
if (typeof options.tts !== "function") {
|
|
443
|
+
throw new TypeError("SoundPlugin requires a tts function");
|
|
444
|
+
}
|
|
445
|
+
const default_asr_model = normalize_optional_string(options.default_asr_model);
|
|
446
|
+
if (options.auto_asr === true && !default_asr_model) {
|
|
447
|
+
throw new TypeError("SoundPlugin auto_asr requires default_asr_model");
|
|
448
|
+
}
|
|
449
|
+
this.name = name;
|
|
450
|
+
this.title = normalize_optional_string(options.title) ?? DEFAULT_SOUND_PLUGIN_TITLE;
|
|
451
|
+
this.description = normalize_optional_string(options.description)
|
|
452
|
+
?? DEFAULT_SOUND_PLUGIN_DESCRIPTION;
|
|
453
|
+
this.asr_handler = options.asr;
|
|
454
|
+
this.tts_handler = options.tts;
|
|
455
|
+
this.list_models = options.list_models;
|
|
456
|
+
this.default_asr_model = default_asr_model;
|
|
457
|
+
this.default_tts_model = normalize_optional_string(options.default_tts_model);
|
|
458
|
+
this.auto_asr = options.auto_asr === true;
|
|
459
|
+
this.language = normalize_optional_string(options.language);
|
|
460
|
+
this.voice = normalize_optional_string(options.voice);
|
|
461
|
+
this.format = normalize_optional_string(options.format);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/**
|
|
465
|
+
* SoundPlugin 给 Agent 的使用说明。
|
|
466
|
+
*/
|
|
467
|
+
system(_context: AgentContext): string {
|
|
468
|
+
return [
|
|
469
|
+
"# Sound Plugin",
|
|
470
|
+
"",
|
|
471
|
+
"Use this plugin for speech recognition (ASR) and text-to-speech (TTS).",
|
|
472
|
+
"Do not call TTS for ordinary text replies unless the user explicitly requests audio.",
|
|
473
|
+
this.auto_asr
|
|
474
|
+
? "Inbound voice/audio chat attachments are automatically transcribed into `<voice src=\"...\">...</voice>` blocks."
|
|
475
|
+
: "Automatic inbound transcription is disabled; call `asr` explicitly when needed.",
|
|
476
|
+
"",
|
|
477
|
+
"## Actions",
|
|
478
|
+
"",
|
|
479
|
+
"- `models`: list FED models whose modalities include `asr` or `tts`. Pass `capability` to filter the list.",
|
|
480
|
+
"- `asr`: transcribe audio. Provide one of `audio_path`, `url`, or `data_url`.",
|
|
481
|
+
"- `tts`: synthesize speech from required `text` and return an audio file part.",
|
|
482
|
+
"",
|
|
483
|
+
"## Model selection",
|
|
484
|
+
"",
|
|
485
|
+
"Pass the selected FED model ID in `model`. When no plugin default is configured, call `models` first with the required capability.",
|
|
486
|
+
"Never use an ASR-only model for TTS or a TTS-only model for ASR.",
|
|
487
|
+
"",
|
|
488
|
+
"## Results",
|
|
489
|
+
"",
|
|
490
|
+
"ASR returns transcript text and may include timed segments, language, and duration.",
|
|
491
|
+
"TTS returns an AI SDK UIMessage. Its audio file part is saved under project resources and attached to the assistant response automatically.",
|
|
492
|
+
"Do not invent a transcript or audio result when a FED call fails.",
|
|
493
|
+
"",
|
|
494
|
+
`When unsure, use \`plugin_read { plugin: \"${this.name}\", action: \"...\" }\` to inspect the complete schema.`,
|
|
495
|
+
].join("\n");
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* 执行一次 ASR 转写。
|
|
500
|
+
*/
|
|
501
|
+
private async transcribe(
|
|
502
|
+
context: AgentContext,
|
|
503
|
+
input: SoundPluginAsrInput,
|
|
504
|
+
): Promise<SoundPluginAsrResult> {
|
|
505
|
+
const model = resolve_model_id("asr", input.model, this.default_asr_model);
|
|
506
|
+
const resolved_input = await resolve_asr_input(context, {
|
|
507
|
+
...(this.language ? { language: this.language } : {}),
|
|
508
|
+
...input,
|
|
509
|
+
model,
|
|
510
|
+
});
|
|
511
|
+
return normalize_asr_result(await this.asr_handler(resolved_input));
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* 执行一次 TTS 合成。
|
|
516
|
+
*/
|
|
517
|
+
private async synthesize(input: SoundPluginTtsInput): Promise<SoundPluginTtsResult> {
|
|
518
|
+
const model = resolve_model_id("tts", input.model, this.default_tts_model);
|
|
519
|
+
const result = await this.tts_handler({
|
|
520
|
+
...(this.language ? { language: this.language } : {}),
|
|
521
|
+
...(this.voice ? { voice: this.voice } : {}),
|
|
522
|
+
...(this.format ? { format: this.format } : {}),
|
|
523
|
+
...input,
|
|
524
|
+
model,
|
|
525
|
+
});
|
|
526
|
+
return normalize_tts_result(result);
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* 自动转写 chat 入站语音附件。
|
|
531
|
+
*/
|
|
532
|
+
private async auto_transcribe_inbound(input: {
|
|
533
|
+
/** 当前 Agent 上下文。 */
|
|
534
|
+
context: AgentContext;
|
|
535
|
+
/** chat 入站管道值。 */
|
|
536
|
+
value: JsonValue;
|
|
537
|
+
}): Promise<JsonValue> {
|
|
538
|
+
if (!this.auto_asr) return input.value;
|
|
539
|
+
const inbound = input.value as unknown as ChatInboundAugmentInput;
|
|
540
|
+
const voice_attachments = (Array.isArray(inbound.attachments) ? inbound.attachments : [])
|
|
541
|
+
.filter((item) =>
|
|
542
|
+
(item.kind === "voice" || item.kind === "audio")
|
|
543
|
+
&& Boolean(normalize_optional_string(item.path))
|
|
544
|
+
);
|
|
545
|
+
if (voice_attachments.length === 0) return input.value;
|
|
546
|
+
|
|
547
|
+
const voice_blocks: string[] = [];
|
|
548
|
+
for (const attachment of voice_attachments) {
|
|
549
|
+
try {
|
|
550
|
+
const result = await this.transcribe(input.context, {
|
|
551
|
+
audio_path: String(attachment.path || "").trim(),
|
|
552
|
+
...(attachment.contentType ? { media_type: attachment.contentType } : {}),
|
|
553
|
+
...(attachment.fileName ? { filename: attachment.fileName } : {}),
|
|
554
|
+
});
|
|
555
|
+
const src = to_display_src(input.context.rootPath, attachment);
|
|
556
|
+
voice_blocks.push(
|
|
557
|
+
`<voice src="${escape_xml_attr(src)}">${escape_xml_text(result.text)}</voice>`,
|
|
558
|
+
);
|
|
559
|
+
} catch {
|
|
560
|
+
// 关键点(中文):自动转写失败不阻塞 chat 主消息链路。
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
return append_voice_text(inbound, voice_blocks) as unknown as JsonValue;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/**
|
|
567
|
+
* chat pipeline 扩展点。
|
|
568
|
+
*/
|
|
569
|
+
readonly hooks = {
|
|
570
|
+
pipeline: {
|
|
571
|
+
[CHAT_PLUGIN_POINTS.augmentInbound]: [
|
|
572
|
+
async ({ context, value }: { context: AgentContext; value: JsonValue }) =>
|
|
573
|
+
await this.auto_transcribe_inbound({ context, value }),
|
|
574
|
+
],
|
|
575
|
+
},
|
|
576
|
+
};
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* 显式 action 集合。
|
|
580
|
+
*/
|
|
581
|
+
readonly actions = {
|
|
582
|
+
models: createAction({
|
|
583
|
+
description: "List FED models that support ASR or TTS.",
|
|
584
|
+
input_schema: {
|
|
585
|
+
zod: SOUND_MODELS_INPUT_SCHEMA,
|
|
586
|
+
json_schema: {
|
|
587
|
+
type: "object",
|
|
588
|
+
properties: {
|
|
589
|
+
capability: {
|
|
590
|
+
type: "string",
|
|
591
|
+
enum: ["asr", "tts"],
|
|
592
|
+
description: "Optional sound capability filter.",
|
|
593
|
+
},
|
|
594
|
+
},
|
|
595
|
+
},
|
|
596
|
+
},
|
|
597
|
+
examples: [
|
|
598
|
+
{ title: "All sound models", payload: {} },
|
|
599
|
+
{ title: "ASR models", payload: { capability: "asr" } },
|
|
600
|
+
{ title: "TTS models", payload: { capability: "tts" } },
|
|
601
|
+
],
|
|
602
|
+
execute: async ({ input }: { input: JsonValue }) => {
|
|
603
|
+
try {
|
|
604
|
+
if (!this.list_models) {
|
|
605
|
+
throw new TypeError("SoundPlugin list_models is not configured");
|
|
606
|
+
}
|
|
607
|
+
const capability = normalize_models_capability(input);
|
|
608
|
+
const result = normalize_sound_models(await this.list_models(), capability);
|
|
609
|
+
return {
|
|
610
|
+
success: true,
|
|
611
|
+
data: result as unknown as JsonObject,
|
|
612
|
+
message: "sound models listed",
|
|
613
|
+
};
|
|
614
|
+
} catch (error) {
|
|
615
|
+
const message = describe_error(error);
|
|
616
|
+
return { success: false, error: message, message };
|
|
617
|
+
}
|
|
618
|
+
},
|
|
619
|
+
}),
|
|
620
|
+
asr: createAction({
|
|
621
|
+
description:
|
|
622
|
+
"Transcribe audio with a FED ASR model. Local audio paths are converted to data URLs before the FED call.",
|
|
623
|
+
input_schema: {
|
|
624
|
+
zod: SOUND_ASR_INPUT_SCHEMA,
|
|
625
|
+
json_schema: {
|
|
626
|
+
type: "object",
|
|
627
|
+
additionalProperties: true,
|
|
628
|
+
properties: {
|
|
629
|
+
model: { type: "string", description: "FED model ID supporting ASR." },
|
|
630
|
+
audio_path: { type: "string", description: "Absolute or project-relative local audio path." },
|
|
631
|
+
url: { type: "string", description: "Remote audio URL." },
|
|
632
|
+
data_url: { type: "string", description: "Audio data URL." },
|
|
633
|
+
language: { type: "string", description: "Optional language hint." },
|
|
634
|
+
media_type: { type: "string", description: "Optional audio MIME type." },
|
|
635
|
+
filename: { type: "string", description: "Optional original file name." },
|
|
636
|
+
},
|
|
637
|
+
},
|
|
638
|
+
},
|
|
639
|
+
examples: [
|
|
640
|
+
{ title: "Local audio", payload: { model: "asr-model-id", audio_path: "./input.wav" } },
|
|
641
|
+
{ title: "Remote audio", payload: { model: "asr-model-id", url: "https://example.com/audio.mp3" } },
|
|
642
|
+
],
|
|
643
|
+
execute: async ({ context, input }: { context: AgentContext; input: JsonValue }) => {
|
|
644
|
+
try {
|
|
645
|
+
const result = await this.transcribe(context, normalize_asr_payload(input));
|
|
646
|
+
return {
|
|
647
|
+
success: true,
|
|
648
|
+
data: result as unknown as JsonObject,
|
|
649
|
+
message: "audio transcribed",
|
|
650
|
+
};
|
|
651
|
+
} catch (error) {
|
|
652
|
+
const message = describe_error(error);
|
|
653
|
+
return { success: false, error: message, message };
|
|
654
|
+
}
|
|
655
|
+
},
|
|
656
|
+
}),
|
|
657
|
+
tts: createAction({
|
|
658
|
+
description:
|
|
659
|
+
"Synthesize speech with a FED TTS model and return an AI SDK UIMessage containing audio.",
|
|
660
|
+
input_schema: {
|
|
661
|
+
zod: SOUND_TTS_INPUT_SCHEMA,
|
|
662
|
+
json_schema: {
|
|
663
|
+
type: "object",
|
|
664
|
+
additionalProperties: true,
|
|
665
|
+
required: ["text"],
|
|
666
|
+
properties: {
|
|
667
|
+
model: { type: "string", description: "FED model ID supporting TTS." },
|
|
668
|
+
text: { type: "string", description: "Text to synthesize." },
|
|
669
|
+
language: { type: "string", description: "Optional language hint." },
|
|
670
|
+
voice: { type: "string", description: "Optional voice ID." },
|
|
671
|
+
format: { type: "string", description: "Optional audio format." },
|
|
672
|
+
speed: { type: "number", description: "Optional speech speed." },
|
|
673
|
+
instructions: { type: "string", description: "Optional voice style instructions." },
|
|
674
|
+
},
|
|
675
|
+
},
|
|
676
|
+
},
|
|
677
|
+
examples: [
|
|
678
|
+
{ title: "Default voice", payload: { model: "tts-model-id", text: "Hello, world" } },
|
|
679
|
+
{
|
|
680
|
+
title: "Specific voice",
|
|
681
|
+
payload: { model: "tts-model-id", text: "Welcome back", voice: "alloy", format: "mp3" },
|
|
682
|
+
},
|
|
683
|
+
],
|
|
684
|
+
execute: async ({ input }: { input: JsonValue }) => {
|
|
685
|
+
try {
|
|
686
|
+
const result = await this.synthesize(normalize_tts_payload(input));
|
|
687
|
+
return {
|
|
688
|
+
success: true,
|
|
689
|
+
data: result as unknown as JsonObject,
|
|
690
|
+
message: "speech synthesized",
|
|
691
|
+
};
|
|
692
|
+
} catch (error) {
|
|
693
|
+
const message = describe_error(error);
|
|
694
|
+
return { success: false, error: message, message };
|
|
695
|
+
}
|
|
696
|
+
},
|
|
697
|
+
}),
|
|
698
|
+
};
|
|
699
|
+
}
|