@tychenjiajun/pi-voice 0.0.0-stage → 0.2.0

package/src/onboarding.ts

@@ -0,0 +1,319 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import {
+  chooseCatalogModel,
+  chooseLanguages,
+  defaultSpokenLanguages,
+  type CatalogModelPostActivation,
+} from "./model-picker.js";
+import { createModelActivation } from "./model-activation.js";
+import { microphonePermission } from "./audio.js";
+import { chooseMicrophone, microphonesEqual } from "./microphone-picker.js";
+import {
+  chooseRecommendedModel,
+  hasRecommendedAlternatives,
+} from "./recommendation-picker.js";
+import { CATALOG_MODELS } from "./catalog.js";
+import { recommendModels } from "./recommendations.js";
+import { createShortcutPicker } from "./shortcuts.js";
+import { tryVoice } from "./try-it.js";
+import {
+  DEFAULT_MICROPHONE,
+  settingsForModel,
+  writeSettings,
+  type BackendSettings,
+  type ChineseOutput,
+  type MicrophoneSetting,
+  type TranscribeSettings,
+  type TranscriptionLanguage,
+} from "./settings.js";
+import { DEFAULT_SHORTCUT } from "./shortcut-core.js";
+
+function requireTui(ctx: ExtensionContext): boolean {
+  if (ctx.mode === "tui") return true;
+  ctx.ui.notify("Pi Voice configuration requires the interactive TUI", "error");
+  return false;
+}
+
+type SettingsOptions = NonNullable<Parameters<typeof settingsForModel>[2]>;
+
+/** Build the shared download-and-save pipeline from the flow's live settings. */
+function createSettingsActivation(
+  options: () => SettingsOptions,
+  onCommitted: (settings: TranscribeSettings) => void,
+) {
+  return createModelActivation({
+    buildSettings: (model, path) => settingsForModel(model.id, path, options()),
+    onCommitted,
+  });
+}
+
+type ModelSelectionOptions = {
+  shortcut?: string;
+  preferredLanguages?: readonly string[];
+  transcriptionLanguage?: TranscriptionLanguage;
+  chineseOutput?: ChineseOutput;
+  currentModelId?: string;
+  microphone?: MicrophoneSetting;
+  /** Preserved across the model re-pick so remote config survives. */
+  backend?: BackendSettings;
+  /** Persists language changes made before this flow activates a model. */
+  onPreferredLanguagesChange?: (languages: string[]) => Promise<void>;
+  postActivation?: CatalogModelPostActivation;
+};
+
+export async function runModelSelection(
+  ctx: ExtensionContext,
+  options: ModelSelectionOptions = {},
+): Promise<TranscribeSettings | undefined> {
+  if (!requireTui(ctx)) return undefined;
+
+  let preferredLanguages = [
+    ...(options.preferredLanguages ?? defaultSpokenLanguages()),
+  ];
+  let currentModelId = options.currentModelId;
+  let configured: TranscribeSettings | undefined;
+  const { activate, waitForCommits } = createSettingsActivation(
+    () => ({
+      shortcut: configured?.shortcut ?? options.shortcut ?? DEFAULT_SHORTCUT,
+      preferredLanguages,
+      transcriptionLanguage:
+        configured?.transcriptionLanguage ?? options.transcriptionLanguage,
+      chineseOutput: configured?.chineseOutput ?? options.chineseOutput,
+      microphone: configured?.microphone ?? options.microphone ?? DEFAULT_MICROPHONE,
+      backend: configured?.backend ?? options.backend,
+    }),
+    (settings) => {
+      configured = settings;
+      currentModelId = settings.model.id;
+    },
+  );
+
+  // One pane switches between the models on disk and downloads new ones.
+  while (true) {
+    const selection = await chooseCatalogModel(ctx, preferredLanguages, currentModelId, {
+      postActivation: options.postActivation,
+      // Keep the post-selection state when the picker reopens after a
+      // round-trip through the language step.
+      activatedInFlow: configured !== undefined,
+      onActivate: activate,
+      cancelLabel: "close",
+    });
+    // The picker can close while its last commit is still in flight; wait so
+    // configured reflects every selection that will land on disk.
+    await waitForCommits();
+
+    if (!selection || selection.type === "complete") return configured;
+
+    // Esc and Continue both keep the selection here; the picker edits live
+    // state rather than gating it behind a confirm.
+    const changed = await chooseLanguages(ctx, preferredLanguages);
+    if (changed) {
+      try {
+        if (configured) {
+          const updated = { ...configured, preferredLanguages: changed.languages };
+          await writeSettings(updated);
+          configured = updated;
+        } else {
+          await options.onPreferredLanguagesChange?.(changed.languages);
+        }
+        preferredLanguages = changed.languages;
+      } catch (error) {
+        ctx.ui.notify(
+          `Could not save preferred languages: ${error instanceof Error ? error.message : String(error)}`,
+          "error",
+        );
+      }
+    }
+
+    // Reaching the language pane after a commit takes a Tab that landed while
+    // the save was in flight. Esc there closes the flow instead of bouncing
+    // back to a model pane that no longer offers the language step.
+    if (!changed?.confirmed && configured) return configured;
+  }
+}
+
+async function chooseOnboardingShortcut(
+  ctx: ExtensionContext,
+  current: string,
+): Promise<string | undefined> {
+  return ctx.ui.custom<string | undefined>((tui, theme, keybindings, done) =>
+    createShortcutPicker(tui, theme, keybindings, current, done),
+  );
+}
+
+async function saveOnboardingSettings(
+  ctx: ExtensionContext,
+  settings: TranscribeSettings,
+): Promise<boolean> {
+  try {
+    await writeSettings(settings);
+    return true;
+  } catch (error) {
+    ctx.ui.notify(
+      `Could not save settings: ${error instanceof Error ? error.message : String(error)}`,
+      "error",
+    );
+    return false;
+  }
+}
+
+/** The Try it step, with its shortcut, microphone, and model adjustments looping back into it. */
+async function finishOnboarding(
+  ctx: ExtensionContext,
+  initial: TranscribeSettings,
+  changeModel: (current: TranscribeSettings) => Promise<TranscribeSettings | undefined>,
+): Promise<TranscribeSettings> {
+  let configured = initial;
+  while (true) {
+    const result = await tryVoice(ctx, configured);
+    if (result?.action === "model") {
+      configured = (await changeModel(configured)) ?? configured;
+      continue;
+    }
+    if (result?.action === "shortcut") {
+      const shortcut = await chooseOnboardingShortcut(ctx, configured.shortcut);
+      if (!shortcut || shortcut === configured.shortcut) continue;
+      const updated = { ...configured, shortcut };
+      if (await saveOnboardingSettings(ctx, updated)) configured = updated;
+      continue;
+    }
+    if (result?.action === "microphone") {
+      const permission = microphonePermission();
+      const microphone = await chooseMicrophone(ctx, configured.microphone, permission);
+      if (!microphone || microphonesEqual(microphone, configured.microphone)) continue;
+      const updated = { ...configured, microphone };
+      if (await saveOnboardingSettings(ctx, updated)) configured = updated;
+      continue;
+    }
+    return configured;
+  }
+}
+
+/** Change the model from Try it, including a round trip through languages. */
+export async function changeOnboardingModel(
+  ctx: ExtensionContext,
+  current: TranscribeSettings,
+): Promise<TranscribeSettings | undefined> {
+  let languages = [...current.preferredLanguages];
+  let picks = recommendModels(CATALOG_MODELS, languages);
+  // With nothing to recommend besides the current model, the catalog is the
+  // whole flow. Otherwise the recommendation pane fronts it, and Esc from
+  // the catalog returns there rather than to Try it.
+  let recommending = hasRecommendedAlternatives(picks);
+  let pane: "recommended" | "browse" = recommending ? "recommended" : "browse";
+  let chosen: TranscribeSettings | undefined;
+  const activation = createSettingsActivation(
+    () => ({
+      shortcut: current.shortcut,
+      preferredLanguages: languages,
+      microphone: current.microphone,
+      chineseOutput: current.chineseOutput,
+      backend: current.backend,
+    }),
+    (settings) => {
+      chosen = settings;
+    },
+  );
+
+  while (true) {
+    const result = pane === "recommended"
+      ? await chooseRecommendedModel(ctx, languages, picks, activation.activate, {
+          title: "Change model",
+          expanded: true,
+          onboardingStep: 3,
+        })
+      : await chooseCatalogModel(ctx, languages, chosen?.model.id ?? current.model.id, {
+          postActivation: "advance",
+          onActivate: activation.activate,
+          cancelLabel: "back",
+          onboardingStep: 3,
+          title: "Browse all models",
+        });
+    await activation.waitForCommits();
+    if (!result && pane === "browse" && recommending) {
+      pane = "recommended";
+      continue;
+    }
+    if (!result || result.type === "complete" || result.type === "back") return chosen;
+    if (result.type === "other-models") {
+      pane = "browse";
+      continue;
+    }
+
+    const changed = await chooseLanguages(ctx, languages, {
+      cancelLabel: "back",
+      onboardingStep: 3,
+    });
+    if (!changed?.confirmed) continue;
+    // Language edits remain a draft until a model is selected. Esc back to
+    // Try it must not silently alter the settings it is about to test.
+    languages = changed.languages;
+    picks = recommendModels(CATALOG_MODELS, languages);
+    // Even a single pick deserves its recommendation after languages change.
+    recommending = true;
+    pane = "recommended";
+  }
+}
+
+export async function runOnboarding(
+  ctx: ExtensionContext,
+  shortcut = DEFAULT_SHORTCUT,
+): Promise<TranscribeSettings | undefined> {
+  if (!requireTui(ctx)) return undefined;
+
+  let languages = defaultSpokenLanguages();
+  // Navigation cannot undo a completed settings commit. Keep it across a
+  // return to languages, even if the user then exits without another choice.
+  let configured: TranscribeSettings | undefined;
+  while (true) {
+    const chosen = await chooseLanguages(ctx, languages, {
+      cancelLabel: "exit",
+      onboardingStep: 1,
+    });
+    if (!chosen?.confirmed) return configured;
+    languages = chosen.languages;
+    // The picks are judged on the benchmark rig; the Try it step measures
+    // the real wait on this machine.
+    const picks = recommendModels(CATALOG_MODELS, languages);
+    const { activate, waitForCommits } = createSettingsActivation(
+      () => ({ shortcut, preferredLanguages: languages }),
+      (settings) => {
+        configured = settings;
+      },
+    );
+
+    let changeLanguages = false;
+    while (!changeLanguages) {
+      const recommendation = await chooseRecommendedModel(
+        ctx,
+        languages,
+        picks,
+        activate,
+        { onboardingStep: 2 },
+      );
+      await waitForCommits();
+      if (!recommendation) return configured;
+      if (recommendation.type === "complete" && configured) {
+        return finishOnboarding(ctx, configured, (current) => changeOnboardingModel(ctx, current));
+      }
+      if (recommendation.type === "change-languages" || recommendation.type === "back") {
+        changeLanguages = true;
+        continue;
+      }
+
+      const selection = await chooseCatalogModel(ctx, languages, configured?.model.id, {
+        postActivation: "advance",
+        onActivate: activate,
+        cancelLabel: "back",
+        onboardingStep: 2,
+        title: "Browse all models",
+      });
+      await waitForCommits();
+      if (selection?.type === "complete" && configured) {
+        return finishOnboarding(ctx, configured, (current) => changeOnboardingModel(ctx, current));
+      }
+      if (selection?.type === "change-languages") changeLanguages = true;
+      // Esc from the complete selector returns to the recommendation pane.
+    }
+  }
+}

package/src/openai-asr.ts

@@ -0,0 +1,247 @@
+import OpenAI from "openai";
+import { CAPTURE_SAMPLE_RATE } from "./audio-constants.js";
+import { convertChineseOutput, isChineseLanguage } from "./chinese.js";
+import { describeError, writeLog } from "./log.js";
+import type { BackendSettings, ChineseOutput } from "./settings.js";
+import type {
+  DictationStream,
+  TranscribeCppBackend,
+  TranscriptionOptions,
+} from "./transcription.js";
+
+type OpenAIAsrSettings = Extract<BackendSettings, { type: "openai-asr" }>;
+
+/** The slice of the local backend this module needs for offline fallback. */
+export type LocalFallback = Pick<
+  TranscribeCppBackend,
+  "prepare" | "transcribe" | "dispose"
+>;
+
+export type AsrRequest = {
+  /** `data:audio/wav;base64,…` of the whole recording. */
+  dataUrl: string;
+};
+
+/** Uploads the recording and returns the transcript text. Injectable for tests. */
+export type AsrTransport = (
+  request: AsrRequest,
+  signal?: AbortSignal,
+) => Promise<string>;
+
+const SAMPLE_RATE = CAPTURE_SAMPLE_RATE;
+const REQUEST_TIMEOUT_MS = 60_000;
+const MAX_RETRIES = 1;
+
+/** 16-bit mono PCM WAV from float samples in [-1, 1]. */
+export function encodeWav(pcm: Float32Array, sampleRate: number): Buffer {
+  const dataSize = pcm.length * 2;
+  const buffer = Buffer.alloc(44 + dataSize);
+  buffer.write("RIFF", 0, "ascii");
+  buffer.writeUInt32LE(36 + dataSize, 4);
+  buffer.write("WAVE", 8, "ascii");
+  buffer.write("fmt ", 12, "ascii");
+  buffer.writeUInt32LE(16, 16); // PCM fmt chunk size
+  buffer.writeUInt16LE(1, 20); // format: PCM
+  buffer.writeUInt16LE(1, 22); // channels: mono
+  buffer.writeUInt32LE(sampleRate, 24);
+  buffer.writeUInt32LE(sampleRate * 2, 28); // byte rate
+  buffer.writeUInt16LE(2, 32); // block align
+  buffer.writeUInt16LE(16, 34); // bits per sample
+  buffer.write("data", 36, "ascii");
+  buffer.writeUInt32LE(dataSize, 40);
+  for (let i = 0; i < pcm.length; i++) {
+    const sample = Math.max(-1, Math.min(1, pcm[i]!));
+    buffer.writeInt16LE(Math.round(sample * 32767), 44 + i * 2);
+  }
+  return buffer;
+}
+
+export function toDataUrl(pcm: Float32Array, sampleRate: number): string {
+  return `data:audio/wav;base64,${encodeWav(pcm, sampleRate).toString("base64")}`;
+}
+
+/**
+ * Chat-completions body with an `input_audio` content block carrying the WAV
+ * recording as a base64 data URL. Generic OpenAI-compatible shape — no
+ * provider-specific extensions are attached.
+ */
+export function buildChatBody(
+  model: string,
+  dataUrl: string,
+): Record<string, unknown> {
+  return {
+    model,
+    messages: [
+      {
+        role: "user",
+        content: [{ type: "input_audio", input_audio: { data: dataUrl } }],
+      },
+    ],
+    stream: false,
+  };
+}
+
+type ChatCompletionLike = {
+  choices?: Array<{ message?: { content?: unknown } }>;
+};
+
+function createOpenAITransport(config: OpenAIAsrSettings): {
+  transport: AsrTransport;
+  dispose: () => void;
+} {
+  const apiKey = process.env[config.apiKeyEnv]?.trim();
+  if (!apiKey) {
+    throw new Error(
+      `${config.apiKeyEnv} is not set. Export your ASR API key (e.g. in your shell profile) and restart Pi.`,
+    );
+  }
+  const client = new OpenAI({
+    apiKey,
+    baseURL: config.baseUrl,
+    timeout: REQUEST_TIMEOUT_MS,
+    maxRetries: MAX_RETRIES,
+  });
+  return {
+    transport: async (request, signal) => {
+      const body = buildChatBody(config.model, request.dataUrl);
+      const completion = (await client.chat.completions.create(
+        body as never,
+        { signal } as never,
+      )) as ChatCompletionLike;
+      const content = completion.choices?.[0]?.message?.content;
+      if (typeof content !== "string" || !content.trim()) {
+        throw new Error("the ASR endpoint returned an empty transcript");
+      }
+      return content;
+    },
+    // openai v7 is fetch-based and keeps no sockets open between requests.
+    dispose: () => undefined,
+  };
+}
+
+/**
+ * Trims, then applies the Chinese script preference exactly like the local
+ * backend. The chat-completions response carries no detected language, so
+ * this mirrors the local `finishTranscript` with detectedLanguage = "" —
+ * conversion runs only when the configured language is Chinese.
+ */
+export async function finishRemoteTranscript(
+  text: string,
+  options: TranscriptionOptions,
+): Promise<string> {
+  const trimmed = text.trim();
+  return isChineseLanguage(options.language ?? "")
+    ? convertChineseOutput(trimmed, options.chineseOutput ?? "simplified")
+    : trimmed;
+}
+
+export type OpenAIAsrBackendDeps = {
+  /** Creates the local model used when a remote call fails. */
+  createFallback?: () => Promise<LocalFallback>;
+  /** Overrides the network transport (tests). */
+  transport?: AsrTransport;
+};
+
+/**
+ * Batch-only remote backend: `startStream` resolves to undefined so the
+ * service records the whole clip and transcribes it after stop, exactly like
+ * a non-streaming local model. Remote failures fall back to the local model;
+ * aborts are never retried locally.
+ */
+export class OpenAIAsrBackend {
+  private client: { transport: AsrTransport; dispose: () => void } | undefined;
+  private fallback: Promise<LocalFallback> | undefined;
+  private disposed = false;
+  private readonly injected: AsrTransport | undefined;
+
+  constructor(
+    private readonly config: OpenAIAsrSettings,
+    private readonly deps: OpenAIAsrBackendDeps = {},
+  ) {
+    this.injected = deps.transport;
+  }
+
+  async prepare(): Promise<void> {
+    if (this.disposed) throw new Error("Remote ASR backend has been disposed");
+    // The transport is created lazily in transcribe() so that a missing API
+    // key (or any construction failure) falls back to the local model instead
+    // of failing prepare() and blocking every subsequent transcription.
+  }
+
+  /** Batch-only: the dictation service drops live chunks and transcribes on stop. */
+  async startStream(): Promise<DictationStream | undefined> {
+    return undefined;
+  }
+
+  async transcribe(
+    pcm: Float32Array,
+    options: TranscriptionOptions = {},
+  ): Promise<string> {
+    if (pcm.length === 0) throw new Error("No audio samples were provided");
+    const signal = options.signal;
+    try {
+      let transport = this.injected;
+      if (!transport) {
+        if (this.disposed) throw new Error("Remote ASR backend has been disposed");
+        // Lazily build the transport inside the try so a missing API key or
+        // any construction failure falls back to the local model. Success is
+        // cached; a failure is not, so a later call retries once the env is
+        // fixed.
+        const client = this.client ?? createOpenAITransport(this.config);
+        this.client = client;
+        transport = client.transport;
+      }
+      signal?.throwIfAborted();
+      const text = await transport(
+        { dataUrl: toDataUrl(pcm, SAMPLE_RATE) },
+        signal,
+      );
+      signal?.throwIfAborted();
+      writeLog(
+        "debug",
+        "voice",
+        `remote ASR transcribed ${(pcm.length / SAMPLE_RATE).toFixed(1)}s`,
+      );
+      return await finishRemoteTranscript(text, options);
+    } catch (error) {
+      if (signal?.aborted || this.disposed) throw error;
+      if (!this.deps.createFallback) throw error;
+      writeLog(
+        "warn",
+        "voice",
+        `remote ASR failed (${describeError(error)}); falling back to the local model`,
+      );
+      try {
+        const local = await this.fallbackBackend();
+        return await local.transcribe(pcm, options);
+      } catch (fallbackError) {
+        throw new Error(
+          `Remote ASR failed (${describeError(error)}) and the local fallback failed (${describeError(fallbackError)})`,
+        );
+      }
+    }
+  }
+
+  async dispose(): Promise<void> {
+    if (this.disposed) return;
+    this.disposed = true;
+    this.client?.dispose();
+    this.client = undefined;
+    if (this.fallback) {
+      const local = await this.fallback.catch(() => undefined);
+      await local?.dispose().catch(() => undefined);
+    }
+  }
+
+  private fallbackBackend(): Promise<LocalFallback> {
+    if (!this.fallback) {
+      // A rejected fallback is not cached: a later attempt can re-create the
+      // local model instead of re-throwing a stale failure.
+      this.fallback = this.deps.createFallback!().catch((error) => {
+        this.fallback = undefined;
+        throw error;
+      });
+    }
+    return this.fallback;
+  }
+}

package/src/pcm-chunker.ts

@@ -0,0 +1,46 @@
+import { CAPTURE_SAMPLE_RATE } from "./audio-constants.js";
+
+const STREAM_CHUNK_SAMPLES = CAPTURE_SAMPLE_RATE / 2;
+
+/** Coalesces recorder chunks into fresh Float32 PCM chunks without timers. */
+export class PcmChunker {
+  private pieces: Float32Array[] = [];
+  private sampleCount = 0;
+
+  constructor(
+    private readonly onChunk: (chunk: Float32Array) => void,
+    private readonly targetSamples = STREAM_CHUNK_SAMPLES,
+  ) {
+    if (!Number.isInteger(targetSamples) || targetSamples <= 0) {
+      throw new Error("PCM chunk size must be a positive integer");
+    }
+  }
+
+  push(samples: Float32Array): void {
+    if (samples.length === 0) return;
+    this.pieces.push(samples);
+    this.sampleCount += samples.length;
+    if (this.sampleCount >= this.targetSamples) this.emit();
+  }
+
+  flush(): void {
+    if (this.sampleCount > 0) this.emit();
+  }
+
+  discard(): void {
+    this.pieces = [];
+    this.sampleCount = 0;
+  }
+
+  private emit(): void {
+    const pcm = new Float32Array(this.sampleCount);
+    let offset = 0;
+    for (const piece of this.pieces) {
+      pcm.set(piece, offset);
+      offset += piece.length;
+    }
+    this.pieces = [];
+    this.sampleCount = 0;
+    this.onChunk(pcm);
+  }
+}