dsh-audiogen 0.1.0

package/package.json

@@ -0,0 +1,93 @@
+{
+  "name": "dsh-audiogen",
+  "description": "AI audio generation plugin for the dsh web GUI: multi-vendor TTS/music/sound-effect channels (OpenAI-compatible, ElevenLabs, MiniMax, Stability AI and custom), per-channel model/voice catalogs, Agent tool and a sidebar AI 音频 panel.",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "lib/index.js",
+  "exports": {
+    ".": "./lib/index.js",
+    "./client": "./lib/client.js",
+    "./package.json": "./package.json"
+  },
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    },
+    "client": {
+      "inject": [
+        "@deepseek-ai/dsh-client-runtime",
+        "@deepseek-ai/dsh-client-connection",
+        "@deepseek-ai/dsh-client-ui-settings",
+        "@deepseek-ai/dsh-client-ui-tool"
+      ],
+      "platform": "web"
+    }
+  },
+  "dependencies": {
+    "schemastery": "^3.18.0"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-client-connection": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-locale": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-runtime": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-ui-primitives": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-ui-settings": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-client-ui-slots": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-host-webserver": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-llm": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-settings": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-system-prompt": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-tools": "0.1.0-rc.7",
+    "@types/node": "^22.20.0",
+    "@types/react": "~18.3.1",
+    "@types/react-dom": "^18.3.5",
+    "jsdom": "25.0.1",
+    "lightningcss": "1.32.0",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "schemastery": "^3.18.0",
+    "tsdown": "0.22.2",
+    "typescript": "~5.7.2"
+  },
+  "files": [
+    "lib",
+    "src",
+    "skills",
+    "cordis.patch.yml",
+    "README.md"
+  ],
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/shimingming520/dsh-audiogen.git"
+  },
+  "homepage": "https://github.com/shimingming520/dsh-audiogen",
+  "bugs": {
+    "url": "https://github.com/shimingming520/dsh-audiogen/issues"
+  },
+  "keywords": [
+    "dsh",
+    "deepseek-harness",
+    "plugin",
+    "audio",
+    "tts",
+    "music",
+    "sfx",
+    "elevenlabs",
+    "minimax",
+    "stability"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "watch": "tsdown --watch",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "pnpm run typecheck && pnpm run build"
+  }
+}

package/skills/design/SKILL.md

@@ -0,0 +1,13 @@
+# 音色/音效设计
+
+## 触发
+- `/audio:design <描述>`
+
+## 用途
+- 帮助用户把音色或音效需求细化成可复用的描述，供后续 TTS/music/sfx 生成。
+- 输出不建议直接生成音频，而是给出音色参数建议和可用的模型/音色清单。
+
+## 流程
+1. 询问目标风格、音域、情绪、适用场景。
+2. 生成结构化的音色描述（如温暖复古合成器、未来感 UI 提示音）。
+3. 在后续生成中复用该描述，必要时通过 `generate_audio` 试听。

package/skills/music/SKILL.md

@@ -0,0 +1,16 @@
+# 音乐生成
+
+## 触发
+- `/audio:music <描述>`
+- 用户说“生成一段音乐 / 配乐 / BGM”
+
+## 参数
+- prompt: 必填，风格/情绪/乐器/时长描述
+- model: 可选，已配置的音频模型
+- duration: 可选，秒数
+- format: 可选，mp3 / wav
+
+## 流程
+1. 确认已配置支持音乐生成的渠道（如 Stability Audio / 自定义）。
+2. 调用 `generate_audio`，mode=music。
+3. 将生成的音频 URL 返回给用户。

package/skills/sfx/SKILL.md

@@ -0,0 +1,16 @@
+# 音效生成
+
+## 触发
+- `/audio:sfx <描述>`
+- 用户说“生成一个音效 / 提示音 / 環境音”
+
+## 参数
+- prompt: 必填，音效描述
+- model: 可选，已配置模型
+- duration: 可选，秒数
+- format: 可选
+
+## 流程
+1. 确认已配置音效生成渠道。
+2. 调用 `generate_audio`，mode=sfx。
+3. 将音频 URL 返回。

package/skills/tts/SKILL.md

@@ -0,0 +1,18 @@
+# TTS 文本转语音
+
+## 触发
+- `/audio:tts <文本>`
+- 用户说“朗读 / 配音 / 语音生成”
+
+## 参数
+- text: 必填，要朗读的文本
+- model: 可选，已配置的模型/音色
+- voice: 可选，音色
+- speed: 可选，语速倍率
+- format: 可选，mp3 / wav 等
+
+## 流程
+1. 确认已配置音频渠道（设置 → 插件 → AI 音频）。
+2. 若用户未指定模型且有多个，先询问。
+3. 调用 `generate_audio` 工具，mode=tts。
+4. 把返回的音频 URL 提供给用户，可播放/下载。

package/src/agent-audio-tools.ts

@@ -0,0 +1,190 @@
+/**
+ * Agent-facing audio-generation tool backed by the same host engine as the
+ * panel. Returns a compact JSON text result with same-origin audio URLs so the
+ * model can reference the generated audio without receiving binary payloads.
+ */
+
+import type { Context } from '@deepseek-ai/cordis'
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import { randomUUID } from 'node:crypto'
+import type { AudioChannel } from './audio-engine.ts'
+import { generateAudio, AudioGenError } from './audio-engine.ts'
+import { appendHistory, saveAudioFile } from './audio-store.ts'
+import type { AudioMode, GenerateAudioRequest } from './protocol.ts'
+
+export interface AgentAudioToolConfig {
+  enabled: boolean
+  allowAgentAudioGeneration: boolean
+  channels: AudioChannel[]
+  defaultChannelId: string
+}
+
+interface AgentAudioRef {
+  id: string
+  url: string
+  mime: string
+  bytes: number
+}
+
+interface AgentAudioResult {
+  status: string
+  message: string
+  mode: AudioMode
+  model: string
+  audio: AgentAudioRef[]
+  error?: string
+}
+
+const audioRefSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    id: { type: 'string', required: true },
+    url: { type: 'string', required: true },
+    mime: { type: 'string', required: true },
+    bytes: { type: 'integer', required: true },
+  },
+} as const
+
+const resultSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    status: { type: 'string', required: true },
+    message: { type: 'string', required: true },
+    mode: { type: 'string', required: true, enum: ['tts', 'music', 'sfx'] },
+    model: { type: 'string', required: true },
+    audio: { type: 'array', required: true, items: audioRefSchema },
+    error: { type: 'string' },
+  },
+} as const
+
+function renderResult(value: AgentAudioResult): Array<{ type: 'text'; text: string }> {
+  return [{ type: 'text', text: JSON.stringify(value) }]
+}
+
+function resolveModel(config: AgentAudioToolConfig, requested: unknown): {
+  channel: AudioChannel
+  alias: string
+  upstream: string
+} {
+  const entries = config.channels.flatMap(channel => channel.models.map(model => ({ channel, alias: model.alias, upstream: model.id })))
+  if (entries.length === 0) {
+    throw new AudioGenError('No audio models/voices are configured. Open Settings > Plugins > AI Audio and add at least one.', 'no-models-configured')
+  }
+  const wanted = typeof requested === 'string' && requested.trim() !== '' ? requested.trim() : ''
+  if (wanted === '') {
+    if (entries.length === 1) return entries[0]!
+    const options = entries.map(entry => `"${entry.channel.name} · ${entry.alias}"`).join(', ')
+    throw new AudioGenError(`Multiple audio models/voices are available — ask the user which channel and model to use, then call again. Options: ${options}.`, 'model-choice-required')
+  }
+  const hosting = entries.filter(entry => entry.alias === wanted)
+  if (hosting.length === 0) {
+    const available = [...new Set(entries.map(entry => entry.alias))].join(', ')
+    throw new AudioGenError(`Audio model/voice "${wanted}" is not configured. Choose one of: ${available}.`, 'audio-model-not-configured')
+  }
+  const preferred = hosting.find(entry => entry.channel.id === config.defaultChannelId)
+  return preferred ?? hosting[0]!
+}
+
+function ensureConfigured(config: AgentAudioToolConfig): void {
+  if (!config.enabled) throw new AudioGenError('AI audio generation is disabled. Open Settings > Plugins > AI Audio and enable it.', 'plugin-disabled')
+  if (!config.allowAgentAudioGeneration) throw new AudioGenError('Agent audio generation is disabled in Settings > Plugins > AI Audio.', 'agent-generation-disabled')
+  const usable = config.channels.some(channel => channel.apiUrl.trim() !== '' && channel.apiKey.trim() !== '')
+  if (!usable) throw new AudioGenError('Audio API credentials are not configured. Open Settings > Plugins > AI Audio, add a channel and fill its API URL and API key.', 'audio-api-not-configured')
+}
+
+/** Register the Agent audio tool. */
+export function registerAgentAudioTools(ctx: Context, resolve: () => AgentAudioToolConfig): () => void {
+  const disposer = ctx.tools.register(defineTool({
+    name: 'generate_audio',
+    description: 'Generate audio with the configured audio provider. Supports text-to-speech, music generation and sound effects. The tool call waits for the upstream result and returns same-origin audio URLs; pass those URLs to the user for playback or download. If multiple models are configured, first ask the user which one to use or pass model explicitly.',
+    parameters: {
+      prompt: { type: 'string', required: true, description: 'For tts, the text to speak. For music/sfx, a descriptive prompt.' },
+      mode: { type: 'string', enum: ['tts', 'music', 'sfx'], description: 'Generation mode. Defaults to tts.' },
+      model: { type: 'string', description: 'One of the configured audio models/voices. Defaults to the first configured model.' },
+      voice: { type: 'string', description: 'Optional voice id/name for TTS providers.' },
+      speed: { type: 'number', description: 'Optional speaking rate / speed multiplier where supported.' },
+      duration: { type: 'number', description: 'Requested duration in seconds for music/sfx.' },
+      format: { type: 'string', description: 'Output format such as mp3 or wav.' },
+    },
+    output: {
+      schema: resultSchema,
+      render: (_args, value) => renderResult(value),
+    },
+    timeoutMs: 300_000,
+    isConcurrencySafe: () => true,
+    async execute(args, exec) {
+      const config = resolve()
+      ensureConfigured(config)
+      const picked = resolveModel(config, args.model)
+      const request: GenerateAudioRequest = {
+        mode: args.mode === 'music' ? 'music' : args.mode === 'sfx' ? 'sfx' : 'tts',
+        model: picked.alias,
+        upstream: picked.upstream,
+        channelId: picked.channel.id,
+        channel: picked.channel.name,
+        prompt: args.prompt.trim(),
+        ...(typeof args.voice === 'string' && args.voice.trim() !== '' ? { voice: args.voice.trim() } : {}),
+        ...(typeof args.speed === 'number' ? { speed: args.speed } : {}),
+        ...(typeof args.duration === 'number' ? { duration: args.duration } : {}),
+        ...(typeof args.format === 'string' && args.format.trim() !== '' ? { format: args.format.trim() } : {}),
+      }
+      try {
+        const outputs = await generateAudio(picked.channel, request, exec.signal)
+        const audio: AgentAudioRef[] = []
+        for (const [index, output] of outputs.entries()) {
+          const saved = await saveAudioFile(output.data, output.mime, `generated-${index + 1}`)
+          audio.push({
+            id: saved.id,
+            url: `/api/dsh-audiogen/audio/${encodeURIComponent(saved.file)}`,
+            mime: saved.mime,
+            bytes: saved.bytes,
+          })
+        }
+        try {
+          await appendHistory({
+            id: randomUUID(),
+            createdAt: Date.now(),
+            mode: request.mode,
+            model: picked.alias,
+            prompt: request.prompt,
+            ...(request.voice === undefined ? {} : { voice: request.voice }),
+            ...(request.speed === undefined ? {} : { speed: request.speed }),
+            ...(request.duration === undefined ? {} : { duration: request.duration }),
+            ...(request.format === undefined ? {} : { format: request.format }),
+            audio: outputs.map((output, index) => ({
+              id: audio[index]!.id,
+              b64: Buffer.from(output.data).toString('base64'),
+              mime: audio[index]!.mime,
+              bytes: audio[index]!.bytes,
+              url: audio[index]!.url,
+            })),
+            channelId: picked.channel.id,
+            channel: picked.channel.name,
+          })
+        } catch {
+          // History is best-effort and must not fail the agent tool.
+        }
+        return {
+          status: 'completed',
+          message: 'Audio generation completed. The audio files can be played/downloaded from the returned URLs.',
+          mode: request.mode,
+          model: picked.alias,
+          audio,
+        }
+      } catch (error) {
+        if (exec.signal?.aborted === true) throw error
+        return {
+          status: 'failed',
+          message: 'Audio generation failed.',
+          mode: request.mode,
+          model: picked.alias,
+          audio: [],
+          error: error instanceof Error ? error.message : String(error),
+        }
+      }
+    },
+  }))
+  return disposer
+}