@soffits/dsh-deltachat-bot 0.0.0-stage → 0.2.0

package/src/voice.js

@@ -0,0 +1,126 @@
+/** Voice messages and audio files: decoding and transcription. */
+import { spawn } from 'node:child_process'
+
+/**
+ * Wrap raw 16 kHz mono 16-bit PCM in the canonical 44-byte WAV header that
+ * DSH speech providers accept.
+ * @param pcm - little-endian samples.
+ * @returns WAV bytes.
+ */
+export function pcmToWave(pcm) {
+  const data = pcm.length % 2 === 0 ? pcm : pcm.subarray(0, pcm.length - 1)
+  const header = Buffer.alloc(44)
+  header.write('RIFF', 0, 'ascii')
+  header.writeUInt32LE(36 + data.length, 4)
+  header.write('WAVE', 8, 'ascii')
+  header.write('fmt ', 12, 'ascii')
+  header.writeUInt32LE(16, 16)
+  header.writeUInt16LE(1, 20)
+  header.writeUInt16LE(1, 22)
+  header.writeUInt32LE(16000, 24)
+  header.writeUInt32LE(32000, 28)
+  header.writeUInt16LE(2, 32)
+  header.writeUInt16LE(16, 34)
+  header.write('data', 36, 'ascii')
+  header.writeUInt32LE(data.length, 40)
+  return Buffer.concat([header, data])
+}
+
+/**
+ * Run one program and collect its output.
+ * @param argv - program and arguments; no shell is involved.
+ * @param signal - cancellation.
+ * @param timeoutMs - upper bound for the run.
+ * @returns stdout bytes; rejects with stderr on a non-zero exit.
+ */
+export function runProgram(argv, signal, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(signal.reason ?? new Error('aborted'))
+      return
+    }
+    const child = spawn(argv[0], argv.slice(1), { stdio: ['ignore', 'pipe', 'pipe'] })
+    const out = []
+    const err = []
+    let settled = false
+    const finish = (error, value) => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      signal?.removeEventListener('abort', abort)
+      if (error) reject(error)
+      else resolve(value)
+    }
+    const abort = () => {
+      child.kill('SIGKILL')
+      finish(new Error(`${argv[0]} was cancelled`))
+    }
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL')
+      finish(new Error(`${argv[0]} took longer than ${Math.round(timeoutMs / 1000)} s`))
+    }, timeoutMs)
+    signal?.addEventListener('abort', abort, { once: true })
+    child.stdout.on('data', (chunk) => out.push(chunk))
+    child.stderr.on('data', (chunk) => err.push(chunk))
+    child.on('error', (error) => finish(new Error(`${argv[0]} could not start: ${error.message}`)))
+    child.on('close', (code) => {
+      if (code === 0) finish(undefined, Buffer.concat(out))
+      else finish(new Error(`${argv[0]} exited with ${code}: ${Buffer.concat(err).toString('utf8').trim().slice(-400)}`))
+    })
+  })
+}
+
+/**
+ * Transcribe one recording.
+ * @param options - `{ mode, path, language, command, ffmpeg, speech, signal, timeoutMs }`:
+ *   `mode` is `dsh` (the DSH speech-to-text service, `speech`), `command`
+ *   (`command` argv, `{file}` replaced by `path`, transcript on stdout) or `off`.
+ * @returns `{ text, by }`; rejects when transcription fails.
+ */
+export async function transcribe(options) {
+  const { mode, path, signal } = options
+  const timeoutMs = options.timeoutMs ?? 180_000
+  if (mode === 'command') {
+    const command = Array.isArray(options.command) ? options.command.filter((part) => typeof part === 'string') : []
+    if (command.length === 0) throw new Error('no transcription command is configured')
+    const argv = command.some((part) => part.includes('{file}'))
+      ? command.map((part) => part.replaceAll('{file}', path))
+      : [...command, path]
+    const text = (await runProgram(argv, signal, timeoutMs)).toString('utf8').trim()
+    return { text, by: 'the configured transcription command' }
+  }
+  if (mode !== 'dsh') throw new Error('transcription is turned off')
+  const speech = options.speech
+  if (speech === undefined || typeof speech.resolve !== 'function') throw new Error('the DSH speech-to-text service is not enabled')
+  const pcm = await runProgram(
+    [options.ffmpeg || 'ffmpeg', '-nostdin', '-hide_banner', '-loglevel', 'error', '-i', path, '-vn', '-ac', '1', '-ar', '16000', '-f', 's16le', 'pipe:1'],
+    signal,
+    timeoutMs,
+  )
+  if (pcm.length === 0) throw new Error('the recording has no audio')
+  const spec = speech.resolve({
+    audio: pcmToWave(pcm),
+    ...(typeof options.language === 'string' && options.language.length > 0 ? { language: options.language } : {}),
+  })
+  const result = await speech.transcribe(spec, signal ?? new AbortController().signal)
+  const provider = spec.provider?.info?.name ?? spec.provider?.info?.id
+  return { text: String(result?.text ?? '').trim(), by: `DSH speech-to-text${provider ? ` (${provider})` : ''}` }
+}
+
+/**
+ * The text the agent reads next to a spoken recording.
+ * @param media - `{ kind, name, seconds }`.
+ * @param outcome - `{ path, text, by, error }`.
+ * @returns prompt text.
+ */
+export function transcriptNote(media, outcome) {
+  const length = media.seconds === undefined ? '' : `, ${Math.floor(media.seconds / 60)}:${String(media.seconds % 60).padStart(2, '0')}`
+  const where = typeof outcome.path === 'string' ? `the file is attached and saved at ${outcome.path}` : 'the file is attached'
+  if (outcome.error !== undefined) {
+    return `[Delta Chat ${media.kind} "${media.name}"${length}: not transcribed (${outcome.error}); ${where}]`
+  }
+  if (outcome.text.length === 0) {
+    return `[Delta Chat ${media.kind} "${media.name}"${length}: ${outcome.by} heard no speech; ${where}]`
+  }
+  return `[Delta Chat ${media.kind} "${media.name}"${length}, transcribed by ${outcome.by}; the transcript may contain mistakes, and ${where} to check it]\n${outcome.text}`
+}

package/tools.js

@@ -0,0 +1,60 @@
+/**
+ * Model-facing tool for the Delta Chat bot. A separate plugin entry from the
+ * bot (index.js): the bot is looked up as the `deltachatChats` service at call
+ * time, so reloading or restarting the bot never unregisters the tool.
+ */
+import z from '@deepseek-ai/schemastery'
+import { SERVICE_NAME } from './src/constants.js'
+
+export const name = 'deltachat-bot-tools'
+export const inject = ['tools']
+export const Config = z.object({})
+
+/** JSON Schema of the send tool's arguments. */
+export const SEND_SCHEMA = {
+  type: 'object',
+  properties: {
+    chat: { type: 'string', description: 'Numeric Delta Chat chat id; each message from a chat names it in its sender line, and /id in the chat shows it.' },
+    text: { type: 'string', description: 'Message text; Markdown is turned into plain text.' },
+    files: { type: 'array', items: { type: 'string' }, description: 'Absolute paths of files to send.' },
+    reply_to: { type: 'string', description: 'Message id to quote (or to react to, with reaction).' },
+    reaction: { type: 'string', description: 'An emoji to react to reply_to with instead of sending a message; an empty string removes the reaction.' },
+  },
+  required: ['chat'],
+  additionalProperties: false,
+}
+
+/**
+ * The send tool.
+ * @param getService - returns the running bot's service, or undefined.
+ * @returns a tool definition.
+ */
+export function createSendTool(getService) {
+  return {
+    name: 'deltachat_send',
+    description: 'Send a message, files or a reaction into a Delta Chat chat of the DSH Delta Chat bot, addressed by its numeric chat id. Not for answering in the chat this session is bound to: the final answer of the turn is delivered there by itself, so text-only sends into that chat are refused. Use it to reach another chat, to send files, or to react to a message. The message is also recorded in the session bound to that chat; no turn is started.',
+    parameters: SEND_SCHEMA,
+    output: { schema: { type: 'object' }, render: (_args, value) => [{ type: 'text', text: JSON.stringify(value, null, 2) }] },
+    async execute(args, exec) {
+      const service = getService()
+      if (service === undefined) return { code: 'unavailable', message: 'The Delta Chat bot is not running.' }
+      const sessionId = exec?.agent?.session?.id ?? exec?.agent?.id
+      try {
+        return await service.send({
+          to: args?.chat,
+          text: typeof args?.text === 'string' ? args.text : '',
+          files: Array.isArray(args?.files) ? args.files : [],
+          ...(args?.reply_to === undefined ? {} : { replyTo: args.reply_to }),
+          ...(typeof args?.reaction === 'string' ? { reaction: args.reaction } : {}),
+          from: typeof sessionId === 'string' ? { sessionId, label: `session ${sessionId}` } : undefined,
+        })
+      } catch (error) {
+        return { code: 'failed', message: error?.message ?? String(error) }
+      }
+    },
+  }
+}
+
+export function apply(ctx) {
+  ctx.tools.register(createSendTool(() => ctx.get(SERVICE_NAME)))
+}