@moxxy/plugin-stt-local 0.28.0

package/src/host-client.ts

@@ -0,0 +1,200 @@
+/**
+ * Parent-side manager for the sherpa sidecar: lazy-spawn on first transcribe,
+ * a correlated request/reply protocol over the fork IPC channel, restart-once
+ * on a crash, and a clean shutdown. The `fork` itself is injectable (a
+ * {@link ForkLike}) so tests drive the whole protocol against a fake child with
+ * no real process.
+ */
+
+import { fork } from 'node:child_process';
+
+import type { HostReply, HostRequest, TranscribeResult } from './host-protocol.js';
+
+/** The minimal child-process surface the client drives — satisfied by a real
+ *  `ChildProcess` and by a test fake. */
+export interface ChildHandle {
+  send(message: unknown): boolean;
+  on(event: 'message', listener: (message: unknown) => void): this;
+  on(event: 'exit', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this;
+  on(event: 'error', listener: (err: Error) => void): this;
+  kill(signal?: NodeJS.Signals): boolean;
+}
+
+/** Fork the sidecar with the given env overrides. Injectable for tests. */
+export type ForkLike = (modulePath: string, env: NodeJS.ProcessEnv) => ChildHandle;
+
+/** Default fork: advanced serialization (Float32Array round-trip), an IPC
+ *  channel, and inherited std streams so sherpa's own diagnostics reach the
+ *  runner log. `env` already carries the platform loader-path var. */
+export const defaultFork: ForkLike = (modulePath, env) =>
+  fork(modulePath, [], {
+    serialization: 'advanced',
+    env,
+    execArgv: [],
+    stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
+  }) as unknown as ChildHandle;
+
+export interface HostClientOptions {
+  /** Absolute path to the built sidecar (`dist/sidecar.js`). */
+  readonly hostPath: string;
+  /** Env the child is forked with (platform loader path merged over process.env). */
+  readonly env: NodeJS.ProcessEnv;
+  /** Injected fork (tests). Defaults to {@link defaultFork}. */
+  readonly forkImpl?: ForkLike;
+  /** Per-request deadline. Default 300s (a long clip on a big model on CPU). */
+  readonly requestTimeoutMs?: number;
+  /** Optional diagnostic sink. */
+  readonly log?: (msg: string) => void;
+}
+
+/** The slice of a host the transcriber depends on — lets tests swap a fake. */
+export interface HostClientLike {
+  transcribe(req: Omit<HostRequest, 'id' | 'type'>): Promise<TranscribeResult>;
+  shutdown(): void;
+}
+
+/** Raised when the sidecar dies with a request in flight; drives the one retry. */
+export class HostCrashError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'HostCrashError';
+  }
+}
+
+interface Pending {
+  readonly resolve: (r: TranscribeResult) => void;
+  readonly reject: (err: Error) => void;
+  timer: ReturnType<typeof setTimeout> | null;
+}
+
+const DEFAULT_TIMEOUT_MS = 300_000;
+
+export class HostClient implements HostClientLike {
+  private readonly hostPath: string;
+  private readonly env: NodeJS.ProcessEnv;
+  private readonly forkImpl: ForkLike;
+  private readonly timeoutMs: number;
+  private readonly log: (msg: string) => void;
+
+  private child: ChildHandle | null = null;
+  private readonly pending = new Map<number, Pending>();
+  private nextId = 1;
+  private disposed = false;
+
+  constructor(opts: HostClientOptions) {
+    this.hostPath = opts.hostPath;
+    this.env = opts.env;
+    this.forkImpl = opts.forkImpl ?? defaultFork;
+    this.timeoutMs = opts.requestTimeoutMs ?? DEFAULT_TIMEOUT_MS;
+    this.log = opts.log ?? (() => {});
+  }
+
+  /** Transcribe, restarting the sidecar ONCE if it crashes mid-request. */
+  async transcribe(req: Omit<HostRequest, 'id' | 'type'>): Promise<TranscribeResult> {
+    try {
+      return await this.send(req);
+    } catch (err) {
+      if (err instanceof HostCrashError && !this.disposed) {
+        this.log(`stt-local: sherpa sidecar crashed (${err.message}) — restarting once`);
+        return await this.send(req); // a second crash propagates
+      }
+      throw err;
+    }
+  }
+
+  private send(req: Omit<HostRequest, 'id' | 'type'>): Promise<TranscribeResult> {
+    if (this.disposed) return Promise.reject(new Error('stt-local host is shut down'));
+    const child = this.ensureChild();
+    const id = this.nextId++;
+    const message: HostRequest = { ...req, id, type: 'transcribe' };
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id);
+        // A hung decode is unrecoverable for this child — reset it so the next
+        // call starts fresh.
+        this.killChild();
+        reject(new Error(`stt-local transcription timed out after ${this.timeoutMs} ms`));
+      }, this.timeoutMs);
+      timer.unref?.();
+      this.pending.set(id, { resolve, reject, timer });
+      try {
+        child.send(message);
+      } catch (err) {
+        this.settle(id, () => reject(err instanceof Error ? err : new Error(String(err))));
+      }
+    });
+  }
+
+  private ensureChild(): ChildHandle {
+    if (this.child) return this.child;
+    const child = this.forkImpl(this.hostPath, this.env);
+    this.child = child;
+    child.on('message', (msg: unknown) => this.onMessage(msg));
+    child.on('exit', (code, signal) => this.onExit(code, signal));
+    child.on('error', (err) => this.onError(err));
+    return child;
+  }
+
+  private onMessage(msg: unknown): void {
+    const reply = msg as HostReply;
+    if (!reply || typeof reply !== 'object' || typeof reply.id !== 'number') return;
+    const p = this.pending.get(reply.id);
+    if (!p) return;
+    this.settle(reply.id, () => {
+      if (reply.ok) {
+        p.resolve(reply.language !== undefined ? { text: reply.text, language: reply.language } : { text: reply.text });
+      } else {
+        p.reject(new Error(`stt-local sidecar ${reply.error.kind} error: ${reply.error.message}`));
+      }
+    });
+  }
+
+  private onExit(code: number | null, signal: NodeJS.Signals | null): void {
+    this.child = null;
+    if (this.pending.size === 0) return;
+    const why = `sidecar exited (code=${code ?? 'null'}, signal=${signal ?? 'null'})`;
+    this.rejectAll(new HostCrashError(why));
+  }
+
+  private onError(err: Error): void {
+    this.child = null;
+    this.rejectAll(new HostCrashError(`sidecar spawn/runtime error: ${err.message}`));
+  }
+
+  /** Resolve/reject one pending request, clearing its timer and map entry. */
+  private settle(id: number, run: () => void): void {
+    const p = this.pending.get(id);
+    if (!p) return;
+    if (p.timer) clearTimeout(p.timer);
+    this.pending.delete(id);
+    run();
+  }
+
+  private rejectAll(err: Error): void {
+    for (const [id, p] of this.pending) {
+      if (p.timer) clearTimeout(p.timer);
+      this.pending.delete(id);
+      p.reject(err);
+    }
+  }
+
+  private killChild(): void {
+    const child = this.child;
+    this.child = null;
+    if (child) {
+      try {
+        child.kill('SIGKILL');
+      } catch {
+        /* already gone */
+      }
+    }
+  }
+
+  /** Kill the sidecar and fail any in-flight requests. Idempotent. */
+  shutdown(): void {
+    if (this.disposed) return;
+    this.disposed = true;
+    this.rejectAll(new Error('stt-local host shut down'));
+    this.killChild();
+  }
+}

package/src/host-protocol.test.ts

@@ -0,0 +1,171 @@
+import { describe, expect, it } from 'vitest';
+import {
+  createMessageHandler,
+  type HostRequest,
+  type SherpaModule,
+  type SherpaOfflineRecognizer,
+  type SherpaOfflineRecognizerResult,
+  type SherpaOfflineStream,
+} from './host-protocol.js';
+
+function req(over: Partial<HostRequest> = {}): HostRequest {
+  return {
+    id: 1,
+    type: 'transcribe',
+    modelKey: '/models/base/base-encoder.onnx',
+    encoder: '/models/base/base-encoder.onnx',
+    decoder: '/models/base/base-decoder.onnx',
+    tokens: '/models/base/base-tokens.txt',
+    numThreads: 2,
+    provider: 'cpu',
+    language: '',
+    task: 'transcribe',
+    samples: new Float32Array([0.1, -0.1, 0.2]),
+    sampleRate: 16_000,
+    ...over,
+  };
+}
+
+/** A fake sherpa module recording constructions + decode calls, with
+ *  configurable text/behaviour. */
+function fakeSherpa(
+  opts: {
+    text?: string;
+    lang?: string;
+    throwOnConstruct?: boolean;
+    throwOnDecode?: boolean;
+  } = {},
+): {
+  module: SherpaModule;
+  constructed: number;
+  decoded: number;
+  lastConfig: unknown;
+  lastSamples: Float32Array | null;
+} {
+  const state = {
+    constructed: 0,
+    decoded: 0,
+    lastConfig: undefined as unknown,
+    lastSamples: null as Float32Array | null,
+  };
+  class FakeStream implements SherpaOfflineStream {
+    accepted: Float32Array | null = null;
+    acceptWaveform(args: { sampleRate: number; samples: Float32Array }): void {
+      this.accepted = args.samples;
+      state.lastSamples = args.samples;
+    }
+  }
+  class FakeRecognizer implements SherpaOfflineRecognizer {
+    constructor(config: unknown) {
+      if (opts.throwOnConstruct) throw new Error('addon failed to load libonnxruntime.dylib');
+      state.constructed += 1;
+      state.lastConfig = config;
+    }
+    createStream(): SherpaOfflineStream {
+      return new FakeStream();
+    }
+    decode(): void {
+      state.decoded += 1;
+      if (opts.throwOnDecode) throw new Error('decode blew up');
+    }
+    getResult(): SherpaOfflineRecognizerResult {
+      return opts.lang !== undefined
+        ? { text: opts.text ?? 'hello world', lang: opts.lang }
+        : { text: opts.text ?? 'hello world' };
+    }
+  }
+  return {
+    module: { OfflineRecognizer: FakeRecognizer as unknown as SherpaModule['OfflineRecognizer'] },
+    get constructed() {
+      return state.constructed;
+    },
+    get decoded() {
+      return state.decoded;
+    },
+    get lastConfig() {
+      return state.lastConfig;
+    },
+    get lastSamples() {
+      return state.lastSamples;
+    },
+  };
+}
+
+describe('createMessageHandler', () => {
+  it('transcribes and returns the text', async () => {
+    const s = fakeSherpa({ text: 'the quick brown fox' });
+    const handle = createMessageHandler(() => s.module);
+    const reply = await handle(req({ id: 7 }));
+    expect(reply).toMatchObject({ id: 7, ok: true, text: 'the quick brown fox' });
+    expect(s.lastSamples).toEqual(new Float32Array([0.1, -0.1, 0.2]));
+  });
+
+  it('passes the whisper model config (encoder/decoder/tokens/language/task) through', async () => {
+    const s = fakeSherpa();
+    const handle = createMessageHandler(() => s.module);
+    await handle(req({ language: 'pl', task: 'transcribe' }));
+    expect(s.lastConfig).toMatchObject({
+      featConfig: { sampleRate: 16_000, featureDim: 80 },
+      modelConfig: {
+        whisper: {
+          encoder: '/models/base/base-encoder.onnx',
+          decoder: '/models/base/base-decoder.onnx',
+          language: 'pl',
+          task: 'transcribe',
+        },
+        tokens: '/models/base/base-tokens.txt',
+        numThreads: 2,
+        provider: 'cpu',
+      },
+    });
+  });
+
+  it('surfaces a detected language when the native result reports one', async () => {
+    const s = fakeSherpa({ text: 'cześć', lang: 'pl' });
+    const handle = createMessageHandler(() => s.module);
+    const reply = await handle(req());
+    expect(reply).toMatchObject({ ok: true, text: 'cześć', language: 'pl' });
+  });
+
+  it('caches one OfflineRecognizer per modelKey (loads the model once)', async () => {
+    const s = fakeSherpa();
+    let loads = 0;
+    const handle = createMessageHandler(() => {
+      loads += 1;
+      return s.module;
+    });
+    await handle(req({ id: 1 }));
+    await handle(req({ id: 2 }));
+    await handle(
+      req({ id: 3, modelKey: '/models/small/small-encoder.onnx', encoder: '/models/small/small-encoder.onnx' }),
+    );
+    expect(loads).toBe(1); // module loaded once
+    expect(s.constructed).toBe(2); // one recognizer per distinct modelKey
+    expect(s.decoded).toBe(3);
+  });
+
+  it('classifies a construct failure as an init error', async () => {
+    const s = fakeSherpa({ throwOnConstruct: true });
+    const handle = createMessageHandler(() => s.module);
+    const reply = await handle(req());
+    expect(reply.ok).toBe(false);
+    if (!reply.ok) expect(reply.error.kind).toBe('init');
+  });
+
+  it('classifies a load-time throw as an init error', async () => {
+    const handle = createMessageHandler(() => {
+      throw new Error('cannot find module sherpa-onnx-node');
+    });
+    const reply = await handle(req());
+    expect(reply.ok).toBe(false);
+    if (!reply.ok) expect(reply.error.kind).toBe('init');
+  });
+
+  it('classifies a decode failure as a runtime error', async () => {
+    const s = fakeSherpa({ throwOnDecode: true });
+    const handle = createMessageHandler(() => s.module);
+    const reply = await handle(req());
+    expect(reply.ok).toBe(false);
+    if (!reply.ok) expect(reply.error.kind).toBe('runtime');
+  });
+});

package/src/host-protocol.ts

@@ -0,0 +1,152 @@
+/**
+ * The tiny message protocol spoken between the parent (host-client) and the
+ * forked sherpa sidecar, plus the pure per-message handler the sidecar runs.
+ * Kept free of `node:child_process` so BOTH sides can import it (the parent for
+ * the types, the child for the handler) without dragging fork plumbing into
+ * either bundle.
+ *
+ * Wire shape (over the fork IPC channel, `serialization: 'advanced'` so the
+ * `Float32Array` of samples round-trips intact):
+ *   parent → child:  TranscribeRequest  (carries the decoded 16 kHz mono PCM)
+ *   child  → parent: HostReply          (the transcript text)
+ */
+
+/** One transcription request. Carries the full model config so the sidecar is
+ *  stateless-per-message except for a recognizer cache keyed by `modelKey`. The
+ *  audio is already decoded to Float32 mono @ 16 kHz by the parent — the sidecar
+ *  only owns the native recognition. */
+export interface TranscribeRequest {
+  readonly id: number;
+  readonly type: 'transcribe';
+  /** Cache key for the loaded recognizer (the absolute encoder path). */
+  readonly modelKey: string;
+  /** Absolute path to the Whisper encoder `.onnx`. */
+  readonly encoder: string;
+  /** Absolute path to the Whisper decoder `.onnx`. */
+  readonly decoder: string;
+  /** Absolute path to `<id>-tokens.txt`. */
+  readonly tokens: string;
+  /** Inference thread count. */
+  readonly numThreads: number;
+  /** Compute provider (`cpu`). */
+  readonly provider: string;
+  /** Best-effort Whisper language tag; empty string ⇒ auto-detect. */
+  readonly language: string;
+  /** Whisper task: `transcribe` (same-language) or `translate` (→ English). */
+  readonly task: string;
+  /** Mono PCM in [-1, 1] at `sampleRate`; structured-cloned intact over IPC. */
+  readonly samples: Float32Array;
+  /** Sample rate of `samples` (16000 — the parent resamples before sending). */
+  readonly sampleRate: number;
+}
+
+export type HostRequest = TranscribeRequest;
+
+export type HostErrorKind =
+  /** Recognizer failed to load (bad/absent files, unsupported arch, dlopen). */
+  | 'init'
+  /** Decoding itself threw. */
+  | 'runtime';
+
+export interface TranscribeResult {
+  readonly text: string;
+  /** Whisper's detected language, when the native result reports one. */
+  readonly language?: string;
+}
+
+export type HostReply =
+  | ({ readonly id: number; readonly ok: true } & TranscribeResult)
+  | {
+      readonly id: number;
+      readonly ok: false;
+      readonly error: { readonly message: string; readonly kind: HostErrorKind };
+    };
+
+/** The subset of the sherpa-onnx-node surface the sidecar uses. Declared here
+ *  (rather than leaning on the addon's ambient types) so the handler is
+ *  unit-testable with a fake module and typechecks without the native addon. */
+export interface SherpaOfflineStream {
+  acceptWaveform(args: { sampleRate: number; samples: Float32Array }): void;
+}
+
+export interface SherpaOfflineRecognizerResult {
+  readonly text: string;
+  /** Whisper detected-language field on some builds; read defensively. */
+  readonly lang?: string;
+}
+
+export interface SherpaOfflineRecognizer {
+  createStream(): SherpaOfflineStream;
+  decode(stream: SherpaOfflineStream): void;
+  getResult(stream: SherpaOfflineStream): SherpaOfflineRecognizerResult;
+}
+
+export interface SherpaModule {
+  OfflineRecognizer: new (config: unknown) => SherpaOfflineRecognizer;
+}
+
+/** Lazily load the native sherpa module (so a dlopen failure surfaces as a
+ *  classified reply, not a boot crash). */
+export type LoadSherpa = () => SherpaModule;
+
+export type HostMessageHandler = (req: HostRequest) => Promise<HostReply>;
+
+/**
+ * Build the sidecar's message handler. Loads the sherpa module on first use and
+ * caches one `OfflineRecognizer` per `modelKey` so repeated calls on the same
+ * model reuse the loaded weights. Every failure becomes a typed `ok:false`
+ * reply — nothing throws out of `handle`.
+ */
+export function createMessageHandler(loadSherpa: LoadSherpa): HostMessageHandler {
+  const cache = new Map<string, SherpaOfflineRecognizer>();
+  let mod: SherpaModule | null = null;
+
+  return async function handle(req: HostRequest): Promise<HostReply> {
+    let recognizer = cache.get(req.modelKey);
+    if (!recognizer) {
+      try {
+        mod ??= loadSherpa();
+        recognizer = new mod.OfflineRecognizer({
+          featConfig: { sampleRate: 16_000, featureDim: 80 },
+          modelConfig: {
+            whisper: {
+              encoder: req.encoder,
+              decoder: req.decoder,
+              // `language`/`task` are present in the OfflineWhisperModelConfig
+              // types but exercised by no upstream example — pass them through
+              // best-effort (empty language ⇒ Whisper auto-detects anyway).
+              language: req.language,
+              task: req.task,
+            },
+            tokens: req.tokens,
+            numThreads: req.numThreads,
+            provider: req.provider,
+            debug: false,
+          },
+        });
+        cache.set(req.modelKey, recognizer);
+      } catch (err) {
+        return { id: req.id, ok: false, error: { message: errMsg(err), kind: 'init' } };
+      }
+    }
+
+    try {
+      const stream = recognizer.createStream();
+      stream.acceptWaveform({ sampleRate: req.sampleRate, samples: req.samples });
+      recognizer.decode(stream);
+      const result = recognizer.getResult(stream);
+      const text = typeof result?.text === 'string' ? result.text : '';
+      const reply: { id: number; ok: true } & TranscribeResult =
+        typeof result?.lang === 'string' && result.lang
+          ? { id: req.id, ok: true, text, language: result.lang }
+          : { id: req.id, ok: true, text };
+      return reply;
+    } catch (err) {
+      return { id: req.id, ok: false, error: { message: errMsg(err), kind: 'runtime' } };
+    }
+  };
+}
+
+function errMsg(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}

package/src/index.ts

@@ -0,0 +1,108 @@
+import { definePlugin, defineTranscriber, type Plugin } from '@moxxy/sdk';
+
+import {
+  createLocalWhisperTranscriber,
+  LOCAL_WHISPER_TRANSCRIBER_NAME,
+  shutdownLocalStt,
+  type LocalWhisperOptions,
+} from './local-stt.js';
+
+export {
+  LocalWhisperTranscriber,
+  createLocalWhisperTranscriber,
+  shutdownLocalStt,
+  LOCAL_WHISPER_TRANSCRIBER_NAME,
+  type LocalWhisperOptions,
+  type WhisperModelPaths,
+} from './local-stt.js';
+export {
+  MODEL_CATALOG,
+  DEFAULT_MODEL_ID,
+  modelIds,
+  findModel,
+  requireModel,
+  type WhisperModelEntry,
+  type WhisperModelId,
+} from './models.js';
+export {
+  pcm16ToFloat32,
+  downmixToMono,
+  resampleLinear,
+  parseWav,
+  isRiffWave,
+  TARGET_SAMPLE_RATE,
+  type ParsedWav,
+} from './audio.js';
+export { decodeToMono16k, type DecodeOptions } from './decode.js';
+export {
+  decodeViaFfmpeg,
+  missingFfmpegError,
+  __resetFfmpegProbeForTest,
+} from './ffmpeg.js';
+export { HostClient, type HostClientLike, type ForkLike, type ChildHandle } from './host-client.js';
+export {
+  sherpaPlatformPackage,
+  resolveSherpaLibDir,
+  sherpaEnv,
+  libraryPathVar,
+} from './platform.js';
+
+export interface BuildLocalSttPluginOptions {
+  /** Build-time defaults / test seams for the transcriber (injected
+   *  `ensureModelImpl`, `hostFactory`, `fetchImpl`, `modelsDir`, `log`, …). */
+  readonly defaults?: LocalWhisperOptions;
+}
+
+/**
+ * Build the @moxxy/plugin-stt-local plugin. Registers exactly one transcriber,
+ * `local-whisper`, backed by sherpa-onnx Whisper models running in a forked
+ * sidecar. Side-effect free at load: no model is downloaded and no process is
+ * spawned until the first `transcribe`. Per-activation config
+ * (`session.transcribers.setActive('local-whisper', { model, language,
+ * numThreads })`) overrides the build-time defaults.
+ *
+ * Like the OpenAI STT sibling, the plugin does NOT set itself active —
+ * registering it is side-effect free (no auto-adopt in TranscriberRegistry).
+ * The host/user activates it explicitly, so a local-only setup never
+ * surprises anyone with a network call (there isn't one here) and a mixed
+ * setup keeps whatever STT backend was chosen.
+ */
+export function buildLocalSttPlugin(opts: BuildLocalSttPluginOptions = {}): Plugin {
+  const defaults = opts.defaults ?? {};
+  return definePlugin({
+    name: '@moxxy/plugin-stt-local',
+    version: '0.0.0',
+    transcribers: [
+      defineTranscriber({
+        name: LOCAL_WHISPER_TRANSCRIBER_NAME,
+        displayName: 'Local Whisper (offline, multilingual)',
+        createClient: (config: Record<string, unknown>) =>
+          createLocalWhisperTranscriber({ ...defaults, ...configToOptions(config) }),
+      }),
+    ],
+    // Kill every spawned sidecar when the session/runner shuts down.
+    hooks: { onShutdown: () => shutdownLocalStt() },
+  });
+}
+
+/** Narrow the untrusted per-activation config into typed options — only the
+ *  known string `model` / `language` / `provider` and a positive-integer
+ *  `numThreads` are honored so a malformed config can't break `createClient` or
+ *  route to a bad model (unknown ids are caught by `requireModel`). */
+function configToOptions(config: Record<string, unknown>): Partial<LocalWhisperOptions> {
+  const out: { -readonly [K in keyof LocalWhisperOptions]?: LocalWhisperOptions[K] } = {};
+  if (typeof config.model === 'string' && config.model) out.model = config.model;
+  if (typeof config.language === 'string') out.language = config.language;
+  if (typeof config.provider === 'string' && config.provider) out.provider = config.provider;
+  if (
+    typeof config.numThreads === 'number' &&
+    Number.isInteger(config.numThreads) &&
+    config.numThreads > 0
+  ) {
+    out.numThreads = config.numThreads;
+  }
+  return out;
+}
+
+// Discovery entry: `createPluginLoader` requires a default Plugin export.
+export default buildLocalSttPlugin();