dsh-her 0.0.1 → 0.1.0

package/lib/types/voice/cosyvoice.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Alibaba Cloud Model Studio (百炼) CosyVoice over its WebSocket API: one
+ * connection, kept open and reused, one task per sentence (run-task →
+ * continue-task → finish-task; audio arrives as binary frames, then
+ * task-finished). Tasks run one at a time, in order; the speech queue is
+ * serial anyway, and a preview simply waits its turn.
+ *
+ * Audio is requested as raw PCM16 and wrapped in WAV here, so its duration
+ * is exact. Docs: https://help.aliyun.com/zh/model-studio/cosyvoice-websocket-api
+ */
+/** The account cannot be used: no key or a refused one (`key`), or no quota left (`quota`). Retrying soon will not help. */
+export declare class CloudAccountError extends Error {
+    readonly reason: 'key' | 'quota';
+    constructor(message: string, reason?: 'key' | 'quota');
+}
+export interface CloudAudio {
+    /** WAV bytes. */
+    audio: Uint8Array;
+    durationMs: number;
+    /** From sending the task to the last audio frame. */
+    ms: number;
+}
+export declare class CosyVoice {
+    private socket;
+    private opening;
+    private task;
+    private chain;
+    private readonly model;
+    private readonly getKey;
+    private readonly endpoints;
+    /** The endpoint that last accepted the key; tried first. */
+    private endpoint;
+    constructor(model: string, getKey: () => Promise<string | undefined>, endpoints?: readonly string[]);
+    /** Open the connection ahead of time (when you start talking), so her first sentence does not pay for it. */
+    warm(): void;
+    /** One sentence. Rejects with CloudAccountError when there is no key or it is refused. */
+    synthesize(text: string, voice: string, rate: number): Promise<CloudAudio>;
+    close(): void;
+    private run;
+    private once;
+    private connect;
+    private open;
+    private dial;
+    private onMessage;
+    private settle;
+    private fail;
+}

package/lib/types/voice/edge.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Microsoft Edge "Read aloud" voices (晓晓, 晓伊 …) through the open-source
+ * command-line tool edge-tts (https://github.com/rany2/edge-tts), the same
+ * way dsh-audiobook does it: we never talk to the service ourselves.
+ *
+ * Where the tool comes from, first found wins:
+ *   1. one already installed by dsh-her (`<home>/edge-tts`), dsh-audiobook or
+ *      dsh-tingdong (`~/.dsh/<plugin>/edge-tts`), or on the PATH via Homebrew;
+ *   2. otherwise it is installed in the background into `<home>/edge-tts`,
+ *      a Python virtual environment of its own, built with the computer's
+ *      Python 3.8+ if it has one;
+ *   3. without one (a new Mac has only a /usr/bin/python3 that asks to install
+ *      the developer tools), uv (https://github.com/astral-sh/uv, one pinned,
+ *      checksummed download) fetches a Python of its own into `<home>/uv`
+ *      and builds the environment with it. No dialogs, nothing system-wide.
+ * Deleting `<home>/edge-tts` (and `<home>/uv`) uninstalls it.
+ *
+ * One process per sentence, so each pays for a fresh connection (about 2.5 s
+ * measured on 2026-10-29); the speech queue synthesizes ahead to hide it.
+ * Needs the internet; not an official Microsoft API, so it may stop working.
+ */
+export interface EdgeAudio {
+    /** MP3 bytes. */
+    audio: Uint8Array;
+    durationMs: number;
+    ms: number;
+}
+export type EdgeStatus = 'missing' | 'installing' | 'ready' | 'failed';
+/** `--rate` for a speed factor: 1.2 → "+20%". */
+export declare function edgeRate(speed: number): string;
+export declare class EdgeVoice {
+    private readonly ownDir;
+    private installing;
+    private state;
+    private error;
+    private readonly viaUv;
+    /** `ownDir`: where dsh-her installs its own copy (`<home>/edge-tts`). `viaUv`: skip the computer's Python (tests). */
+    constructor(ownDir: string, options?: {
+        viaUv?: boolean;
+    });
+    get status(): {
+        status: EdgeStatus;
+        error?: string;
+    };
+    /** The edge-tts command, or undefined when there is none yet. */
+    bin(): string | undefined;
+    /** Make sure the tool is there, installing it once if needed. */
+    ensure(): Promise<void>;
+    /** One sentence as MP3. */
+    synthesize(text: string, voice: string, speed: number): Promise<EdgeAudio>;
+    private install;
+}

package/lib/types/voice/engine.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Parent side of the local speech recognition child process. Requests are answered in order of completion,
+ * matched by id. If the child dies, pending requests fail and the next call
+ * starts a fresh one.
+ *
+ * Inside the DSH desktop app `process.execPath` is Electron, so the child is
+ * started with ELECTRON_RUN_AS_NODE. The sherpa native library directory is
+ * put on the loader path, as sherpa-onnx documents.
+ */
+import type { WorkerInit } from './worker.ts';
+export declare class LocalVoice {
+    private child;
+    private nextId;
+    private readonly pending;
+    private ready;
+    private readonly init;
+    constructor(init: WorkerInit);
+    /** Load the models (once); resolves with how long loading took. */
+    start(): Promise<{
+        loadMs: number;
+    }>;
+    recognize(samples: Float32Array, sampleRate: number): Promise<{
+        text: string;
+        lang: string;
+        ms: number;
+    }>;
+    stop(): void;
+    private spawn;
+    private request;
+}

package/lib/types/voice/models.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Local speech recognition models (sherpa-onnx). They are downloaded on first
+ * use into `<home>/models`, never shipped in the package. Her own voice is in
+ * the cloud (cosyvoice.ts). Design: docs/语音方案.md.
+ */
+export interface ModelSpec {
+    id: 'asr' | 'vad';
+    /** Directory (or file, for a bare .onnx) it unpacks to under the models dir. */
+    name: string;
+    url: string;
+    bytes: number;
+    archive: boolean;
+}
+export declare const MODELS: readonly ModelSpec[];
+export declare function modelPath(modelsDir: string, spec: ModelSpec): string;
+export declare function missingModels(modelsDir: string): ModelSpec[];
+/**
+ * Download and unpack one model. Writes to a temporary name first, so an
+ * interrupted download never looks installed.
+ */
+export declare function fetchModel(modelsDir: string, spec: ModelSpec, onProgress?: (received: number, total: number) => void): Promise<void>;

package/lib/types/voice/speech-queue.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Turns her replies into audio sentence by sentence, in order.
+ *
+ * The next few sentences are synthesized at once (AHEAD), so while one plays
+ * the following ones are already on their way; a slow voice service only
+ * makes the first sentence of a reply wait. Sentences are started and
+ * emitted strictly in order, across replies, so the first sentence of a reply
+ * is never queued behind a later one. Each result is emitted as soon as it
+ * and everything before it are ready (the body queues playback), and a reply's
+ * end is emitted after its last sentence. Cancelling a reply drops what has
+ * not been emitted yet; synthesis already running for it is thrown away.
+ *
+ * A sentence whose synthesis fails is still emitted, without audio, so the
+ * subtitle shows and the reply goes on.
+ */
+import type { Cue } from '../protocol/index.ts';
+export interface Synthesized {
+    audio: Uint8Array;
+    durationMs: number;
+    format: 'wav' | 'mp3';
+}
+export interface SpeechOut {
+    sentence(uid: string, seq: number, text: string, audio: Synthesized | undefined, cues: Cue[]): void;
+    end(uid: string): void;
+}
+/** What a reply has sent so far: enough to say how much of it was heard when it is cut off. */
+export interface SentSentence {
+    seq: number;
+    text: string;
+    durationMs: number;
+}
+export declare class SpeechQueue {
+    private readonly jobs;
+    private running;
+    private readonly cancelled;
+    private readonly sent;
+    private readonly synth;
+    private readonly out;
+    constructor(synth: (text: string) => Promise<Synthesized | undefined>, out: SpeechOut);
+    say(uid: string, seq: number, text: string, cues?: Cue[]): void;
+    end(uid: string): void;
+    cancel(uid: string): void;
+    /** Sentences of a reply that went out, in order. */
+    sentOf(uid: string): SentSentence[];
+    /** Resolves when nothing is queued or being synthesized. */
+    idle(): Promise<void>;
+    /** Start synthesizing the first AHEAD sentences still to emit. */
+    private startAhead;
+    private pump;
+}
+/**
+ * How much of a reply was heard when it was cut off: every sentence before
+ * the one playing, plus the share of that one its played time covers.
+ */
+export declare function heardText(sent: SentSentence[], seq: number, playedMs: number): string;

package/lib/types/voice/voice.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Her voice for the brain.
+ *
+ * - Listening: local speech recognition (sherpa-onnx SenseVoice in a child
+ *   process, engine.ts). Its model is downloaded in the background on first
+ *   use; until then she only reads what you type.
+ * - Speaking: the service of the voice she uses (core/voices.ts): Microsoft
+ *   Edge voices through the edge-tts tool (edge.ts, the default), or Alibaba
+ *   Cloud Model Studio CosyVoice (cosyvoice.ts). When the service cannot
+ *   speak (tool not installed yet, no key, no quota, offline), a sentence goes
+ *   out without audio and shows as text; a key added later is picked up
+ *   without restarting.
+ * - Cache: short sentences ("嗯。" "好呀。") are kept, so the most common
+ *   answers start at once.
+ */
+import { type VoiceService } from '../core/index.ts';
+import type { Synthesized } from './speech-queue.ts';
+export interface VoiceConfig {
+    /** false: she only writes (text only). */
+    enabled: boolean;
+    /** Voice id (src/core/voices.ts); its service speaks. */
+    voice: string;
+    speed: number;
+    /** CPU threads for local speech recognition. */
+    threads: number;
+}
+/** Listening (local recognition). */
+export type VoiceStatus = 'off' | 'downloading' | 'loading' | 'ready' | 'failed';
+/** Speaking, with the service of her current voice. */
+export type SpeakingStatus = 'off' | 'checking' | 'installing' | 'ready' | 'no-key' | 'no-quota' | 'failed';
+type Logger = {
+    info(message: string): void;
+    warn(message: string): void;
+};
+export declare class Voice {
+    private asr;
+    private listening;
+    private speakingState;
+    private speakingError;
+    private keyCheckedAt;
+    private readonly cloud;
+    private readonly edge;
+    private readonly cache;
+    private readonly modelsDir;
+    private config;
+    private readonly logger;
+    /** `edgeDir`: where dsh-her installs its own edge-tts if none is found. */
+    constructor(modelsDir: string, edgeDir: string, config: VoiceConfig, logger: Logger);
+    /** Listening. */
+    get status(): VoiceStatus;
+    get speaking(): {
+        status: SpeakingStatus;
+        service: VoiceService;
+        error?: string;
+    };
+    /** Check the cloud voice and get recognition ready. Never throws; the statuses say how it went. */
+    start(): Promise<void>;
+    /** You started talking: open the cloud connection now, while your words are recognized. */
+    warm(): void;
+    /** Change her voice and speed (from the "她" page); takes effect with the next sentence. */
+    setVoice(voice: string, speed: number): void;
+    /** One sentence in a given voice, for trying voices; not cached. Throws with a message for the page. */
+    preview(text: string, voice: string, speed: number): Promise<Synthesized>;
+    /** Her sentence as audio, or undefined when she cannot speak right now (it shows as text). */
+    synthesize(text: string): Promise<Synthesized | undefined>;
+    /** What you said, from 16-bit mono PCM. Empty when nothing was understood. */
+    recognize(pcm16: Uint8Array, sampleRate: number): Promise<{
+        text: string;
+        ms: number;
+    }>;
+    stop(): void;
+    private speak;
+    private mark;
+}
+export {};

package/lib/types/voice/worker.d.ts

@@ -0,0 +1,37 @@
+/**
+ * The local voice child process: sherpa-onnx speech recognition (SenseVoice),
+ * on CPU. It runs apart from DSH so a native crash or
+ * a long synthesis never stalls or kills the host. The parent (engine.ts)
+ * forks it with advanced IPC serialization, so audio travels as typed arrays.
+ */
+export interface WorkerInit {
+    asrDir: string;
+    threads: number;
+}
+export type WorkerRequest = {
+    id: number;
+    op: 'init';
+    init: WorkerInit;
+} | {
+    id: number;
+    op: 'asr';
+    samples: Float32Array;
+    sampleRate: number;
+};
+export type WorkerReply = {
+    id: number;
+    ok: true;
+    op: 'init';
+    loadMs: number;
+} | {
+    id: number;
+    ok: true;
+    op: 'asr';
+    text: string;
+    lang: string;
+    ms: number;
+} | {
+    id: number;
+    ok: false;
+    error: string;
+};

package/lib/voice/cloud-key.js

@@ -0,0 +1,23 @@
+/**
+ * The Model Studio (百炼) API key, never kept in a file: the environment
+ * variable `DASHSCOPE_API_KEY`, or on macOS the keychain item the user adds with
+ *
+ *   security add-generic-password -a dsh-her -s dsh-her-dashscope -w
+ *
+ * Read on every connection, so a key added later works without restarting DSH.
+ */
+import { execFile } from 'node:child_process';
+export const KEYCHAIN_SERVICE = 'dsh-her-dashscope';
+export function readCloudKey() {
+    const fromEnv = process.env.DASHSCOPE_API_KEY?.trim();
+    if (fromEnv)
+        return Promise.resolve(fromEnv);
+    if (process.platform !== 'darwin')
+        return Promise.resolve(undefined);
+    return new Promise(resolve => {
+        execFile('security', ['find-generic-password', '-s', KEYCHAIN_SERVICE, '-w'], { timeout: 10_000 }, (error, stdout) => {
+            const key = stdout.trim();
+            resolve(error === null && key !== '' ? key : undefined);
+        });
+    });
+}

package/lib/voice/cosyvoice.js

@@ -0,0 +1,194 @@
+/**
+ * Alibaba Cloud Model Studio (百炼) CosyVoice over its WebSocket API: one
+ * connection, kept open and reused, one task per sentence (run-task →
+ * continue-task → finish-task; audio arrives as binary frames, then
+ * task-finished). Tasks run one at a time, in order; the speech queue is
+ * serial anyway, and a preview simply waits its turn.
+ *
+ * Audio is requested as raw PCM16 and wrapped in WAV here, so its duration
+ * is exact. Docs: https://help.aliyun.com/zh/model-studio/cosyvoice-websocket-api
+ */
+import { randomUUID } from 'node:crypto';
+import WebSocket from 'ws';
+import { wavFromPcm16 } from "../core/index.js";
+/**
+ * Mainland China (Beijing) and international (Singapore): a key works in the
+ * region it was made in only, so a refused key is tried on the other one, and
+ * the one that took it is kept.
+ */
+const ENDPOINTS = ['wss://dashscope.aliyuncs.com/api-ws/v1/inference', 'wss://dashscope-intl.aliyuncs.com/api-ws/v1/inference'];
+const SAMPLE_RATE = 24000;
+const TASK_TIMEOUT_MS = 20_000;
+const CONNECT_TIMEOUT_MS = 8_000;
+/** The account cannot be used: no key or a refused one (`key`), or no quota left (`quota`). Retrying soon will not help. */
+export class CloudAccountError extends Error {
+    reason;
+    constructor(message, reason = 'key') {
+        super(message);
+        this.reason = reason;
+    }
+}
+/** The cloud answered and refused this sentence: retrying it would not help. */
+class CloudTaskError extends Error {
+}
+export class CosyVoice {
+    socket;
+    opening;
+    task;
+    chain = Promise.resolve();
+    model;
+    getKey;
+    endpoints;
+    /** The endpoint that last accepted the key; tried first. */
+    endpoint;
+    constructor(model, getKey, endpoints = ENDPOINTS) {
+        this.model = model;
+        this.getKey = getKey;
+        this.endpoints = endpoints;
+        this.endpoint = endpoints[0];
+    }
+    /** Open the connection ahead of time (when you start talking), so her first sentence does not pay for it. */
+    warm() {
+        this.connect().catch(() => undefined);
+    }
+    /** One sentence. Rejects with CloudAccountError when there is no key or it is refused. */
+    synthesize(text, voice, rate) {
+        const run = this.chain.then(() => this.run(text, voice, rate));
+        this.chain = run.catch(() => undefined);
+        return run;
+    }
+    close() {
+        this.socket?.close();
+        this.socket = undefined;
+        this.opening = undefined;
+    }
+    async run(text, voice, rate) {
+        const started = Date.now();
+        let pcm;
+        try {
+            pcm = await this.once(await this.connect(), text, voice, rate);
+        }
+        catch (error) {
+            if (error instanceof CloudAccountError || error instanceof CloudTaskError)
+                throw error;
+            // A connection the server had already closed: try once more on a fresh one.
+            this.close();
+            pcm = await this.once(await this.connect(), text, voice, rate);
+        }
+        return { audio: wavFromPcm16(pcm, SAMPLE_RATE), durationMs: Math.round((pcm.byteLength / 2 / SAMPLE_RATE) * 1000), ms: Date.now() - started };
+    }
+    once(socket, text, voice, rate) {
+        return new Promise((resolve, reject) => {
+            const id = randomUUID().replaceAll('-', '');
+            const timer = setTimeout(() => this.fail(new Error('云端语音超时')), TASK_TIMEOUT_MS);
+            this.task = { id, text, chunks: [], resolve, reject, timer };
+            socket.send(JSON.stringify({
+                header: { action: 'run-task', task_id: id, streaming: 'duplex' },
+                payload: {
+                    task_group: 'audio', task: 'tts', function: 'SpeechSynthesizer', model: this.model,
+                    parameters: { text_type: 'PlainText', voice, format: 'pcm', sample_rate: SAMPLE_RATE, volume: 50, rate, pitch: 1 },
+                    input: {},
+                },
+            }));
+        });
+    }
+    connect() {
+        if (this.socket?.readyState === WebSocket.OPEN)
+            return Promise.resolve(this.socket);
+        this.opening ??= this.open().finally(() => { this.opening = undefined; });
+        return this.opening;
+    }
+    async open() {
+        const key = await this.getKey();
+        if (key === undefined)
+            throw new CloudAccountError('还没有百炼的 API key');
+        let refused;
+        for (const endpoint of [this.endpoint, ...this.endpoints.filter(e => e !== this.endpoint)]) {
+            try {
+                const socket = await this.dial(endpoint, key);
+                this.endpoint = endpoint;
+                return socket;
+            }
+            catch (error) {
+                if (!(error instanceof CloudAccountError))
+                    throw error;
+                refused = error;
+            }
+        }
+        throw refused;
+    }
+    dial(endpoint, key) {
+        return new Promise((resolve, reject) => {
+            const socket = new WebSocket(endpoint, { headers: { Authorization: `bearer ${key}` }, handshakeTimeout: CONNECT_TIMEOUT_MS });
+            socket.on('unexpected-response', (_request, response) => {
+                const status = response.statusCode ?? 0;
+                reject(status === 401 || status === 403 ? new CloudAccountError(`百炼拒绝了这个 key（HTTP ${status}）`) : new Error(`连不上百炼（HTTP ${status}）`));
+                socket.terminate();
+            });
+            socket.on('open', () => {
+                this.socket = socket;
+                resolve(socket);
+            });
+            socket.on('message', (data, isBinary) => this.onMessage(socket, data, isBinary));
+            socket.on('error', error => {
+                reject(error);
+                if (this.socket === socket)
+                    this.fail(error);
+            });
+            socket.on('close', () => {
+                if (this.socket === socket) {
+                    this.socket = undefined;
+                    this.fail(new Error('百炼的连接断了'));
+                }
+            });
+        });
+    }
+    onMessage(socket, data, isBinary) {
+        const task = this.task;
+        if (task === undefined)
+            return;
+        if (isBinary) {
+            task.chunks.push(data);
+            return;
+        }
+        let event;
+        try {
+            event = JSON.parse(data.toString('utf8'));
+        }
+        catch {
+            return;
+        }
+        const header = event.header ?? {};
+        if (header.task_id !== undefined && header.task_id !== task.id)
+            return;
+        switch (header.event) {
+            case 'task-started':
+                socket.send(JSON.stringify({ header: { action: 'continue-task', task_id: task.id, streaming: 'duplex' }, payload: { input: { text: task.text } } }));
+                socket.send(JSON.stringify({ header: { action: 'finish-task', task_id: task.id, streaming: 'duplex' }, payload: { input: {} } }));
+                break;
+            case 'task-finished':
+                this.settle();
+                task.resolve(Buffer.concat(task.chunks));
+                break;
+            case 'task-failed': {
+                const message = `${header.error_code ?? ''} ${header.error_message ?? ''}`.trim();
+                this.fail(/InvalidApiKey|Unauthorized|AccessDenied/i.test(message)
+                    ? new CloudAccountError(`百炼拒绝了这个 key（${message}）`)
+                    : /AllocationQuota|FreeTierOnly|Arrearage|Throttling\.AllocationQuota/i.test(message)
+                        ? new CloudAccountError(`百炼的额度用完了（${message}）`, 'quota')
+                        : new CloudTaskError(`云端语音失败：${message || '未知原因'}`));
+                break;
+            }
+        }
+    }
+    settle() {
+        if (this.task !== undefined)
+            clearTimeout(this.task.timer);
+        this.task = undefined;
+    }
+    fail(error) {
+        const task = this.task;
+        this.settle();
+        task?.reject(error);
+    }
+}