@tychenjiajun/pi-voice 0.0.0-stage → 0.2.0

package/src/catalog.ts

@@ -0,0 +1,89 @@
+import { CATALOG_MODELS_GENERATED } from "./catalog.generated.js";
+import { canonicalLanguage, languageIdentity, resolveModelLanguage } from "./languages.js";
+export { canonicalLanguage, languageIdentity, resolveModelLanguage } from "./languages.js";
+
+export type CatalogModel = {
+  readonly id: string;
+  readonly name: string;
+  readonly description: string;
+  readonly repository: string;
+  readonly revision: string;
+  readonly license: string;
+  readonly family: string;
+  readonly parameters: string | null;
+  readonly languages: readonly string[];
+  readonly capabilities: {
+    readonly streaming: boolean;
+    readonly translate: boolean;
+    readonly languageDetection: boolean;
+  };
+  readonly quant: string;
+  readonly filename: string;
+  readonly size: number;
+  readonly sha256: string;
+};
+
+export const CATALOG_MODELS: readonly CatalogModel[] = CATALOG_MODELS_GENERATED;
+
+const languageNames = new Intl.DisplayNames(["en"], { type: "language" });
+
+export function displayLanguage(language: string): string {
+  // ASR catalogs conventionally use zh for Mandarin and list Cantonese as yue.
+  // Make that spoken-language distinction explicit without losing the familiar
+  // umbrella term users look for.
+  if (canonicalLanguage(language) === "zh") return "Mandarin (Chinese)";
+  try {
+    return languageNames.of(language) ?? language;
+  } catch {
+    return language;
+  }
+}
+
+export function modelMatchesLanguage(model: CatalogModel, language: string): boolean {
+  return resolveModelLanguage(model, language) !== undefined;
+}
+
+function preferredLanguageMatchCount(
+  model: CatalogModel,
+  preferredLanguages: readonly string[],
+): number {
+  return [...new Set(preferredLanguages.map(languageIdentity))].filter((language) =>
+    modelMatchesLanguage(model, language),
+  ).length;
+}
+
+// The catalog carries no editorial rank or score: the pickers order measured
+// models by benchmark, so names only settle models without one.
+export function rankCatalogModels(
+  models: readonly CatalogModel[],
+  preferredLanguages: readonly string[] = [],
+  isDownloaded: (model: CatalogModel) => boolean = () => false,
+): CatalogModel[] {
+  return [...models].sort(
+    (left, right) =>
+      preferredLanguageMatchCount(right, preferredLanguages) -
+        preferredLanguageMatchCount(left, preferredLanguages) ||
+      Number(isDownloaded(right)) - Number(isDownloaded(left)) ||
+      left.name.localeCompare(right.name),
+  );
+}
+
+/**
+ * What a catalog search runs against. Languages and capabilities are left
+ * out on purpose: the pickers already scope and grade by language, and a
+ * long haystack made short queries match nearly everything.
+ */
+export function catalogModelSearchText(model: CatalogModel): string {
+  return [model.id, model.name, model.family, model.parameters ?? ""].join(" ").toLowerCase();
+}
+
+export function getCatalogModel(id: string): CatalogModel | undefined {
+  return CATALOG_MODELS.find((model) => model.id === id);
+}
+
+export function formatBinarySize(bytes: number): string {
+  if (bytes >= 1024 ** 3) {
+    return `${(bytes / 1024 ** 3).toFixed(bytes >= 10 * 1024 ** 3 ? 0 : 1)} GiB`;
+  }
+  return `${Math.round(bytes / (1024 * 1024))} MiB`;
+}

package/src/chinese.ts

@@ -0,0 +1,52 @@
+import type { ChineseOutput } from "./settings.js";
+
+type Converter = (text: string) => string;
+
+const converters = new Map<ChineseOutput, Promise<Converter>>();
+
+async function createConverter(output: ChineseOutput): Promise<Converter> {
+  const { default: OpenCC } = await import("opencc-js");
+  switch (output) {
+    case "simplified":
+      return OpenCC.Converter({ from: "t", to: "cn" });
+    case "traditional-taiwan":
+      return OpenCC.Converter({ from: "cn", to: "tw" });
+    case "traditional-hong-kong":
+      return OpenCC.Converter({ from: "cn", to: "hk" });
+  }
+}
+
+function converterFor(output: ChineseOutput): Promise<Converter> {
+  const existing = converters.get(output);
+  if (existing) return existing;
+
+  const loading = createConverter(output);
+  converters.set(output, loading);
+  void loading.catch(() => {
+    if (converters.get(output) === loading) converters.delete(output);
+  });
+  return loading;
+}
+
+export function isChineseLanguage(language: string): boolean {
+  const base = language.toLowerCase().split("-", 1)[0];
+  return base === "zh" || base === "yue";
+}
+
+export async function convertChineseOutput(
+  text: string,
+  output: ChineseOutput,
+): Promise<string> {
+  return (await converterFor(output))(text);
+}
+
+export function chineseOutputSummary(output: ChineseOutput): string {
+  switch (output) {
+    case "simplified":
+      return "Simplified";
+    case "traditional-taiwan":
+      return "Traditional (Taiwan)";
+    case "traditional-hong-kong":
+      return "Traditional (Hong Kong)";
+  }
+}

package/src/deferred.ts

@@ -0,0 +1,30 @@
+/** A promise with external settle controls. Settling is idempotent. */
+export class Deferred<T = void> {
+  readonly promise: Promise<T>;
+  private resolvePromise!: (value: T) => void;
+  private rejectPromise!: (error: unknown) => void;
+  private done = false;
+
+  constructor() {
+    this.promise = new Promise<T>((resolve, reject) => {
+      this.resolvePromise = resolve;
+      this.rejectPromise = reject;
+    });
+  }
+
+  get settled(): boolean {
+    return this.done;
+  }
+
+  resolve(value: T): void {
+    if (this.done) return;
+    this.done = true;
+    this.resolvePromise(value);
+  }
+
+  reject(error: unknown): void {
+    if (this.done) return;
+    this.done = true;
+    this.rejectPromise(error);
+  }
+}

package/src/dictation-controller.ts

@@ -0,0 +1,315 @@
+import { CAPTURE_SAMPLE_RATE } from "./audio-constants.js";
+import { describeError, log, logStep, sinceKeyPress } from "./log.js";
+import { PcmChunker } from "./pcm-chunker.js";
+import type { MicrophoneSetting, TranscribeSettings } from "./settings.js";
+import type { DictationReservation, TranscriptionService } from "./transcription-service.js";
+
+/** What a recording captured. `interruption` is why it ended early, if it did. */
+export type CapturedAudio = { pcm: Float32Array; interruption?: Error; droppedFrames?: number };
+/** A running recording. `stop` ends it and releases the device. */
+export type Capture = { stop(): Promise<CapturedAudio> };
+/** Where a recording delivers audio, in order, until its `stop` resolves. */
+export type CaptureSinks = {
+  onAudio(chunk: Float32Array): void;
+  onFailure(error: Error): void;
+};
+/** Opens a microphone and starts recording; resolves once audio is flowing. */
+export type OpenCapture = (microphone: MicrophoneSetting, sinks: CaptureSinks) => Promise<Capture>;
+export type DictationResult = {
+  text: string;
+  speechSeconds: number;
+  transcribeSeconds: number;
+  /** The microphone failed mid-recording; `text` covers the audio before it. */
+  interruption?: Error;
+  /** Audio lost inside the recording because it wasn't read in time. */
+  droppedFrames?: number;
+};
+export type DictationState =
+  | { phase: "idle" | "ready" | "starting" | "listening" | "transcribing" | "cancelling" | "disposed" }
+  | { phase: "result"; result: DictationResult }
+  | { phase: "error"; stage: "model" | "capture" | "transcription"; cause: unknown };
+export type DictationControllerOptions = {
+  openCapture: OpenCapture;
+  now?: () => number;
+  onChange?: (state: DictationState) => void;
+  onAudio?: (chunk: Float32Array) => void;
+};
+type Take = {
+  settings: TranscribeSettings;
+  reservation: DictationReservation;
+  abort: AbortController;
+  chunker: PcmChunker;
+  capture?: Capture;
+  interruption?: Error;
+  stopping?: Promise<CapturedAudio>;
+  submission?: Promise<DictationResult | undefined>;
+};
+
+/** Owns one capture/reservation lifecycle, never the injected service itself. */
+export class DictationController {
+  private current: DictationState = { phase: "idle" };
+  private take: Take | undefined;
+  private disposed = false;
+  private cleanup: Promise<void> = Promise.resolve();
+  private starting: Promise<void> | undefined;
+  /** Bumped by cancel, so a start still opening the microphone knows it lost. */
+  private generation = 0;
+  private opening = false;
+  private startedAt = 0;
+  private readonly now: () => number;
+  private readiness: "loading" | "ready" | "failed" = "loading";
+
+  constructor(
+    private readonly service: Pick<TranscriptionService, "reserveDictation">,
+    private readonly options: DictationControllerOptions,
+  ) {
+    this.now = options.now ?? (() => performance.now());
+  }
+
+  get state(): DictationState { return this.current; }
+  get modelState(): "loading" | "ready" | "failed" { return this.readiness; }
+  get elapsedMs(): number { return Math.max(0, this.now() - this.startedAt); }
+  /** Set when the microphone failed during the current recording. */
+  get interruption(): Error | undefined { return this.take?.interruption; }
+
+  private notify(): void {
+    if (this.disposed) return;
+    // Presentation must not strand the reservation or drop recorded audio.
+    try { this.options.onChange?.(this.current); } catch { /* UI owns rendering errors. */ }
+  }
+  private setState(state: DictationState): void {
+    if (this.disposed) return;
+    this.current = state;
+    if (state.phase === "error") {
+      log.error(`${state.stage} failed: ${describeError(state.cause)}`);
+      logStep("idle");
+    }
+    this.notify();
+  }
+
+  /** Reserves the model for a take; on failure the state says why. */
+  private reserve(settings: TranscribeSettings): Take | undefined {
+    this.readiness = "loading";
+    logStep("reserving model", settings.model.id);
+    const reservedAt = this.now();
+    let reservation: DictationReservation;
+    try {
+      reservation = this.service.reserveDictation(settings);
+    } catch (cause) {
+      this.readiness = "failed";
+      this.setState({ phase: "error", stage: "model", cause });
+      return undefined;
+    }
+    const take: Take = {
+      settings, reservation, abort: new AbortController(),
+      chunker: new PcmChunker((chunk) => reservation.feed(chunk)),
+    };
+    this.take = take;
+    void reservation.ready.then(
+      () => {
+        if (this.disposed || this.take !== take) return;
+        log.debug(`model ready ${Math.round(this.now() - reservedAt)} ms after reserving`);
+        this.readiness = "ready";
+        this.notify();
+      },
+      (cause: unknown) => {
+        if (this.disposed || this.take !== take) return;
+        this.readiness = "failed";
+        if (this.current.phase === "ready") this.setState({ phase: "error", stage: "model", cause });
+        else {
+          log.error(`model failed to load: ${describeError(cause)}`);
+          this.notify(); // Keep capturing; submission will report the error.
+        }
+      },
+    );
+    return take;
+  }
+
+  /** Optional prewarming. Model preparation overlaps with reading or recording. */
+  prepare(settings: TranscribeSettings): void {
+    if (this.disposed || ["starting", "listening", "transcribing", "cancelling"].includes(this.current.phase)) return;
+    if (this.take?.settings === settings && this.readiness !== "failed") return;
+    this.take?.reservation.cancel();
+    this.take = undefined;
+    if (this.reserve(settings)) this.setState({ phase: "ready" });
+  }
+
+  /**
+   * Starts recording. Without a prewarmed take, the model is reserved once
+   * the microphone is listening: loading it runs synchronous native setup,
+   * which must not hold up the start of the recording.
+   */
+  start(settings: TranscribeSettings): Promise<void> {
+    if (this.disposed) return Promise.resolve();
+    if (this.current.phase === "starting") return this.starting ?? Promise.resolve();
+    if (["listening", "transcribing", "cancelling"].includes(this.current.phase)) return Promise.resolve();
+    let take = this.take?.settings === settings && this.readiness !== "failed" ? this.take : undefined;
+    if (!take) {
+      this.take?.reservation.cancel();
+      this.take = undefined;
+    }
+    const generation = this.generation;
+    this.opening = true;
+    this.setState({ phase: "starting" });
+    const current = () => !this.disposed && this.generation === generation && (!take || this.take === take);
+    const work = this.cleanup.then(async () => {
+      if (!current()) return;
+      const openedAt = this.now();
+      logStep("opening microphone");
+      // Audio that arrives before the take exists; delivered to it in order.
+      const early: Float32Array[] = [];
+      let earlyFailure: Error | undefined;
+      let capture: Capture;
+      try {
+        capture = await this.options.openCapture(settings.microphone, {
+          onAudio: (chunk) => {
+            if (!current()) return;
+            if (take) this.feed(take, chunk);
+            else early.push(chunk);
+            try { this.options.onAudio?.(chunk); } catch { /* Audio is already fed. */ }
+          },
+          onFailure: (error) => {
+            if (!current()) return;
+            if (take) take.interruption = error;
+            else earlyFailure = error;
+            this.notify();
+          },
+        });
+      } catch (cause) {
+        if (!current()) return; // Cancelled meanwhile; cancel() released the take.
+        if (take) {
+          this.take = undefined;
+          take.chunker.discard();
+          take.reservation.cancel();
+        }
+        this.setState({ phase: "error", stage: "capture", cause });
+        return;
+      }
+      const opened = this.now() - openedAt;
+      if (current() && !take) {
+        take = this.reserve(settings);
+        if (take) {
+          for (const chunk of early) this.feed(take, chunk);
+          take.interruption = earlyFailure;
+        }
+      }
+      if (!take || !current()) {
+        // Cancelled or disposed while opening (cancel() waits on this
+        // release), or the model couldn't be reserved (a retry waits on it).
+        log.debug(`microphone opened in ${Math.round(opened)} ms, then released`);
+        const release = capture.stop().then(() => undefined, () => undefined);
+        if (!take && current()) this.cleanup = this.cleanup.then(() => release);
+        await release;
+        return;
+      }
+      take.capture = capture;
+      this.startedAt = this.now();
+      const sincePress = sinceKeyPress();
+      const press = sincePress === undefined ? "" : `, ${Math.round(sincePress)} ms after the key press`;
+      log.info(`listening: microphone opened in ${Math.round(opened)} ms${press}; model ${this.readiness}`);
+      logStep("listening");
+      this.setState({ phase: "listening" });
+    }).finally(() => {
+      if (this.generation === generation) this.opening = false;
+    });
+    this.starting = work;
+    return work;
+  }
+
+  private feed(take: Take, chunk: Float32Array): void {
+    // A failed stream feed falls back to submitting the whole recording.
+    try { take.chunker.push(chunk); } catch { /* The recording keeps the audio. */ }
+  }
+
+  private stopCapture(take: Take): Promise<CapturedAudio> {
+    if (take.stopping) return take.stopping;
+    const capture = take.capture;
+    take.capture = undefined;
+    if (!capture) return Promise.resolve({ pcm: new Float32Array() });
+    // Normalize synchronous failures too; native implementations normally reject.
+    try { take.stopping = capture.stop(); }
+    catch (error) { take.stopping = Promise.reject(error); }
+    return take.stopping;
+  }
+
+  stop(): Promise<DictationResult | undefined> {
+    const take = this.take;
+    if (!take || this.disposed) return Promise.resolve(undefined);
+    if (take.submission) return take.submission;
+    if (this.current.phase !== "listening") return Promise.resolve(undefined);
+    const stoppedAt = this.now();
+    logStep("stopping microphone");
+    this.setState({ phase: "transcribing" });
+    let stage: "capture" | "transcription" = "capture";
+    take.submission = this.stopCapture(take).then(async (captured) => {
+      // Cancellation while the native microphone is stopping must never submit.
+      if (this.take !== take || take.abort.signal.aborted) return undefined;
+      const { pcm } = captured;
+      const interruption = captured.interruption ?? take.interruption;
+      if (interruption && pcm.length === 0) throw interruption;
+      take.chunker.flush();
+      stage = "transcription";
+      logStep("transcribing", `${(pcm.length / CAPTURE_SAMPLE_RATE).toFixed(1)} s of audio`);
+      const text = await take.reservation.submit(pcm, take.abort.signal);
+      if (this.disposed || this.take !== take || take.abort.signal.aborted) return undefined;
+      const result = {
+        text,
+        speechSeconds: pcm.length / CAPTURE_SAMPLE_RATE,
+        transcribeSeconds: Math.max(0, (this.now() - stoppedAt) / 1000),
+        ...(interruption ? { interruption } : {}),
+        ...(captured.droppedFrames ? { droppedFrames: captured.droppedFrames } : {}),
+      };
+      this.take = undefined;
+      const empty = text ? "" : "; no speech detected";
+      log.info(
+        `transcribed ${result.speechSeconds.toFixed(1)} s of audio in ${result.transcribeSeconds.toFixed(1)} s${empty}`,
+      );
+      logStep("idle");
+      this.setState({ phase: "result", result });
+      return result;
+    }).catch((cause: unknown) => {
+      take.reservation.cancel(); // Releases the lane if stop failed before submit.
+      if (this.disposed || this.take !== take || take.abort.signal.aborted) return undefined;
+      this.take = undefined;
+      this.setState({ phase: "error", stage, cause });
+      return undefined;
+    });
+    return take.submission;
+  }
+
+  /** Cancel is serialized with capture teardown, so retries never overlap devices. */
+  cancel(): Promise<void> {
+    const take = this.take;
+    this.take = undefined; // Invalidate callbacks before touching native resources.
+    this.generation++;
+    const opening = this.opening;
+    this.opening = false;
+    if (!take && !opening) return this.cleanup;
+    if (take) {
+      take.abort.abort();
+      take.chunker.discard();
+      take.reservation.cancel();
+    }
+    if (opening) log.info("start cancelled");
+    else if (take?.submission) log.info("transcription cancelled");
+    else if (take?.capture) log.info("recording discarded");
+    logStep("idle");
+    if (!this.disposed) this.setState({ phase: "cancelling" });
+    const cleanup = Promise.all([
+      this.cleanup,
+      take && this.stopCapture(take).catch(() => undefined),
+      take?.submission,
+      this.starting,
+    ]).then(() => {
+      if (this.cleanup === cleanup) this.setState({ phase: "idle" });
+    });
+    this.cleanup = cleanup;
+    return cleanup;
+  }
+
+  dispose(): Promise<void> {
+    this.disposed = true;
+    this.current = { phase: "disposed" };
+    return this.cancel();
+  }
+}

package/src/file-audio.ts

@@ -0,0 +1,164 @@
+import { spawn } from "node:child_process";
+
+export const FILE_SAMPLE_RATE = 16_000;
+
+const BYTES_PER_SAMPLE = Float32Array.BYTES_PER_ELEMENT;
+const MAX_DECODED_BYTES = 128 * 1024 * 1024;
+const MAX_STDERR_CHARS = 8 * 1024;
+
+type FfmpegConfiguration = {
+  executable: string;
+  variable?: "PI_VOICE_FFMPEG_PATH" | "PI_TRANSCRIBE_FFMPEG_PATH";
+};
+
+function ffmpegConfiguration(): FfmpegConfiguration {
+  const current = process.env.PI_VOICE_FFMPEG_PATH?.trim();
+  if (current) return { executable: current, variable: "PI_VOICE_FFMPEG_PATH" };
+  const legacy = process.env.PI_TRANSCRIBE_FFMPEG_PATH?.trim();
+  if (legacy) return { executable: legacy, variable: "PI_TRANSCRIBE_FFMPEG_PATH" };
+  return { executable: "ffmpeg" };
+}
+
+function installHint(): string {
+  switch (process.platform) {
+    case "darwin":
+      return "On macOS with Homebrew: brew install ffmpeg";
+    case "win32":
+      return "On Windows: winget install Gyan.FFmpeg";
+    default:
+      return "On Debian/Ubuntu: sudo apt install ffmpeg (use your distribution's package manager elsewhere)";
+  }
+}
+
+function missingFfmpegError(configuration: FfmpegConfiguration): Error {
+  const locationHelp = configuration.variable
+    ? `The configured ${configuration.variable} (${configuration.executable}) could not be found. Correct it or unset it to use PATH.`
+    : "The ffmpeg executable was not found on PATH.";
+  return new Error(
+    [
+      "File transcription requires FFmpeg.",
+      locationHelp,
+      installHint(),
+      "Explain this requirement to the user and ask permission before installing system software, then retry transcribe_file.",
+    ].join(" "),
+  );
+}
+
+function abortReason(signal: AbortSignal): Error {
+  return signal.reason instanceof Error
+    ? signal.reason
+    : new Error("File audio decoding was cancelled");
+}
+
+export type DecodedFileAudio = {
+  pcm: Float32Array;
+  seconds: number;
+};
+
+/** Decode the first audio stream in a local media file to transcribe.cpp PCM. */
+export async function decodeFileAudio(
+  path: string,
+  signal?: AbortSignal,
+): Promise<DecodedFileAudio> {
+  signal?.throwIfAborted();
+  const configuration = ffmpegConfiguration();
+  const executable = configuration.executable;
+  const child = spawn(
+    executable,
+    [
+      "-hide_banner",
+      "-loglevel",
+      "error",
+      "-nostdin",
+      "-protocol_whitelist",
+      "file,pipe",
+      "-i",
+      path,
+      "-map",
+      "0:a:0",
+      "-vn",
+      "-sn",
+      "-dn",
+      "-ac",
+      "1",
+      "-ar",
+      String(FILE_SAMPLE_RATE),
+      "-acodec",
+      "pcm_f32le",
+      "-f",
+      "f32le",
+      "pipe:1",
+    ],
+    { stdio: ["ignore", "pipe", "pipe"], windowsHide: true },
+  );
+
+  return new Promise<DecodedFileAudio>((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    let byteLength = 0;
+    let stderr = "";
+    let terminalError: Error | undefined;
+
+    const onAbort = (): void => {
+      terminalError = abortReason(signal!);
+      child.kill("SIGTERM");
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+
+    child.stdout.on("data", (chunk: Buffer) => {
+      if (terminalError) return;
+      byteLength += chunk.length;
+      if (byteLength > MAX_DECODED_BYTES) {
+        terminalError = new Error(
+          "Decoded audio exceeds the 128 MiB safety limit (about 35 minutes at 16 kHz mono). Split the media file into smaller parts and retry.",
+        );
+        child.kill("SIGTERM");
+        return;
+      }
+      chunks.push(chunk);
+    });
+
+    child.stderr.on("data", (chunk: Buffer) => {
+      stderr += chunk.toString("utf8");
+      if (stderr.length > MAX_STDERR_CHARS) stderr = stderr.slice(-MAX_STDERR_CHARS);
+    });
+
+    child.once("error", (error: NodeJS.ErrnoException) => {
+      terminalError =
+        error.code === "ENOENT"
+          ? missingFfmpegError(configuration)
+          : new Error(`Could not start FFmpeg (${executable}): ${error.message}`);
+    });
+
+    child.once("close", (code, closeSignal) => {
+      signal?.removeEventListener("abort", onAbort);
+      if (terminalError) {
+        reject(terminalError);
+        return;
+      }
+      if (code !== 0) {
+        const detail = stderr.trim() || `process exited with code ${code ?? "unknown"}`;
+        reject(
+          new Error(
+            `FFmpeg could not decode the file${closeSignal ? ` (signal ${closeSignal})` : ""}: ${detail}`,
+          ),
+        );
+        return;
+      }
+      if (byteLength === 0) {
+        reject(new Error("FFmpeg decoded no audio samples; the file may not contain an audio stream"));
+        return;
+      }
+      if (byteLength % BYTES_PER_SAMPLE !== 0) {
+        reject(new Error(`FFmpeg returned ${byteLength} bytes of incomplete float32 PCM`));
+        return;
+      }
+
+      const output = Buffer.concat(chunks, byteLength);
+      const pcm =
+        output.byteOffset % BYTES_PER_SAMPLE === 0
+          ? new Float32Array(output.buffer, output.byteOffset, output.byteLength / BYTES_PER_SAMPLE)
+          : new Float32Array(Uint8Array.from(output).buffer);
+      resolve({ pcm, seconds: pcm.length / FILE_SAMPLE_RATE });
+    });
+  });
+}