@wirecat/cli-core 0.498.0 → 0.499.0

package/dist/speech/models.d.ts

@@ -24,6 +24,8 @@ export interface SpeechModel {
     featureDim: number;
 }
 export declare const MODELS: SpeechModel[];
+/** OpenAI's recommended transcription model (speech-to-text guide, read 2026-10-31). */
+export declare const OPENAI_TRANSCRIBE_MODEL = "gpt-transcribe";
 /** Cuts speech at pauses. Every model needs it: five minutes in one piece crash the WebAssembly build. */
 export declare const VAD: ModelFile;
 /**

package/dist/speech/models.js

@@ -105,6 +105,8 @@ export const MODELS = [
         }),
     },
 ];
+/** OpenAI's recommended transcription model (speech-to-text guide, read 2026-10-31). */
+export const OPENAI_TRANSCRIBE_MODEL = "gpt-transcribe";
 /** Cuts speech at pauses. Every model needs it: five minutes in one piece crash the WebAssembly build. */
 export const VAD = {
     name: "silero_vad.onnx",

package/dist/speech/openai.d.ts

@@ -0,0 +1,54 @@
+import { type RetryConfig } from "@wirecat/cli-base";
+import { type Fetch } from "../embeddings/remote.js";
+import type { Pcm } from "./decode.js";
+import { type SpeechCutter, type TimedText } from "./recognize.js";
+/**
+ * Ten minutes of 16 kHz PCM16 is 19.2 MB: under OpenAI's 25 MB per file (speech-to-text guide, read 2026-10-31)
+ * whichever megabyte it means, with room for the multipart body around it.
+ */
+export declare const LONGEST_REQUEST_SECONDS = 600;
+export interface OpenAITranscription {
+    apiKey: string;
+    /** `gpt-transcribe` when not given. */
+    model?: string;
+    /** ISO-639-1 codes the speech may be in. Sent as `languages`; the singular `language` is never sent with it. */
+    languages?: readonly string[];
+    prompt?: string;
+    /** Literal terms expected in the speech; OpenAI refuses the whole request if one holds `<`, `>` or a line break. */
+    keywords?: readonly string[];
+    baseUrl?: string;
+    signal?: AbortSignal;
+    fetch?: Fetch;
+    /**
+     * Fewer, longer requests for a caller that wants the text and not the turns: a recording of up to ten minutes in
+     * one request, a longer one in pieces of up to ten minutes joined across pauses. Each piece then spans the silence
+     * between its utterances, so two channels heard this way do not merge into a dialogue.
+     */
+    bundle?: boolean;
+    /** The voice detector model; the shared models directory's when not given. */
+    vadModel?: string;
+    env?: NodeJS.ProcessEnv;
+    /** Seams for tests: the detector, the retry policy and the clock. */
+    cutter?: (vadModel: string) => SpeechCutter;
+    retry?: RetryConfig;
+    sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+    random?: () => number;
+}
+/**
+ * Each stretch of speech the detector found is a piece of its own: on one channel of a call, a pause is often the
+ * other person talking. Two stretches the detector split without a pause stay one. With `limit`, neighbours are also
+ * joined across pauses while the piece stays within it.
+ */
+export declare const piecesOf: (spans: readonly {
+    start: number;
+    end: number;
+}[], limit?: number) => {
+    start: number;
+    end: number;
+}[];
+/**
+ * Speech to text by OpenAI's `/audio/transcriptions`, from one channel's samples: one piece per stretch of speech the
+ * voice detector found, each sent alone, with where it lies in seconds. Two channels heard this way merge into a
+ * dialogue by `start`. `gpt-transcribe` gives no timestamps of its own, so the times are the detector's.
+ */
+export declare const transcribeOpenAI: (audio: Pcm, options: OpenAITranscription) => Promise<TimedText[]>;

package/dist/speech/openai.js

@@ -0,0 +1,199 @@
+import { setTimeout as wait } from "node:timers/promises";
+import { backoffMs, CliError, DEFAULT_RETRY } from "@wirecat/cli-base";
+import { providerWaitMs } from "@wirecat/cli-base/http";
+import { OPENAI_URL } from "../embeddings/remote.js";
+import { filesPresent, modelsDirectory, vadPath } from "./install.js";
+import { OPENAI_TRANSCRIBE_MODEL, VAD } from "./models.js";
+import { SAMPLE_RATE, toModelRate } from "./recognize.js";
+import { encodeWav } from "./wav.js";
+/**
+ * Ten minutes of 16 kHz PCM16 is 19.2 MB: under OpenAI's 25 MB per file (speech-to-text guide, read 2026-10-31)
+ * whichever megabyte it means, with room for the multipart body around it.
+ */
+export const LONGEST_REQUEST_SECONDS = 600;
+/**
+ * The detector cuts speech that runs past 20 seconds with no pause; on real speech that cut left 0.074 s between the
+ * two stretches, and the shortest real pause it found was 0.43 s. Below this the two are one stretch, cut mid-word.
+ */
+const JOINED_SECONDS = 0.2;
+/** Kept around each piece so its first and last sounds are not clipped; never reaches into the next piece. */
+const PAD_SECONDS = 0.2;
+const LEAD = SAMPLE_RATE / 2;
+/** OpenAI's codes for money or limits run out (error-codes guide, read 2026-10-31): waiting does not help. */
+const EXHAUSTED = new Set([
+    "insufficient_quota",
+    "credit_balance_exhausted",
+    "organization_spend_limit_exceeded",
+    "project_spend_limit_exceeded",
+    "organization_usage_limit_exceeded",
+]);
+const RETRY = { ...DEFAULT_RETRY, retries: 4 };
+const interrupted = (signal) => {
+    if (!signal?.aborted)
+        return;
+    if (signal.reason instanceof CliError)
+        throw signal.reason;
+    const timeout = signal.reason instanceof Error && signal.reason.name === "TimeoutError";
+    throw new CliError(timeout ? "timeout" : "cancelled", "the transcription was interrupted");
+};
+const sleeping = (ms, signal) => wait(ms, undefined, signal ? { signal } : {}).then(() => undefined, (error) => {
+    interrupted(signal);
+    throw error;
+});
+const formOf = (wav, options) => {
+    const form = new FormData();
+    form.append("file", new Blob([wav], { type: "audio/wav" }), "audio.wav");
+    form.append("model", options.model ?? OPENAI_TRANSCRIBE_MODEL);
+    // Repeated `name[]` fields: how OpenAI's own SDK sends an array in a multipart body.
+    for (const language of options.languages ?? [])
+        form.append("languages[]", language);
+    for (const keyword of options.keywords ?? [])
+        form.append("keywords[]", keyword);
+    if (options.prompt)
+        form.append("prompt", options.prompt);
+    return form;
+};
+const safe = (value) => typeof value === "string" && /^[\w.-]{1,64}$/.test(value) ? value : undefined;
+/** OpenAI's error code and type, never its message: a message can quote the request. */
+const failureOf = async (response) => {
+    try {
+        const body = (await response.json());
+        const code = safe(body.error?.code);
+        const type = safe(body.error?.type);
+        return { ...(code ? { code } : {}), ...(type ? { type } : {}) };
+    }
+    catch {
+        return {};
+    }
+};
+/** One file, one request. 429, 408 and server errors are tried again; the provider's `Retry-After` beats our backoff. */
+const send = async (wav, options) => {
+    const post = options.fetch ?? fetch;
+    const retry = options.retry ?? RETRY;
+    const sleep = options.sleep ?? sleeping;
+    const random = options.random ?? Math.random;
+    const url = `${(options.baseUrl ?? OPENAI_URL).replace(/\/+$/, "")}/audio/transcriptions`;
+    for (let attempt = 1;; attempt++) {
+        interrupted(options.signal);
+        let response;
+        try {
+            response = await post(url, {
+                method: "POST",
+                headers: { authorization: `Bearer ${options.apiKey}` },
+                body: formOf(wav, options),
+                ...(options.signal ? { signal: options.signal } : {}),
+            });
+        }
+        catch {
+            interrupted(options.signal);
+            if (attempt > retry.retries)
+                throw new CliError("network_error", "OpenAI could not be reached", { reason: "network.failed" });
+            await sleep(backoffMs(attempt, retry, random), options.signal);
+            continue;
+        }
+        if (response.ok) {
+            const body = (await response.json().catch(() => interrupted(options.signal)));
+            if (typeof body?.text !== "string")
+                throw new CliError("invalid_response", "OpenAI answered without a transcript", {
+                    reason: "provider.invalid-response",
+                });
+            return body.text.trim();
+        }
+        const { code, type } = await failureOf(response);
+        const status = response.status;
+        const said = `HTTP ${status}${code ? `, ${code}` : type ? `, ${type}` : ""}`;
+        if (status === 401)
+            throw new CliError("authentication_error", `OpenAI refused the key (${said}) — set it again with \`models keys set openai\``, { reason: "auth.failed" });
+        if (status === 403)
+            throw new CliError("permission_error", `OpenAI refused the request (${said}) — it answers this when the country or region is not supported, or the key's project may not use the model`, { reason: "access.denied" });
+        if (status === 413)
+            throw new CliError("validation_error", `OpenAI refused the audio as too large (${said})`, {
+                reason: "transcription.too-large",
+            });
+        if (EXHAUSTED.has(code ?? "") || EXHAUSTED.has(type ?? ""))
+            throw new CliError("permission_error", `OpenAI refused: the account's credit, spend limit or usage limit is used up (${said}) — see the billing page`, { reason: "transcription.quota-exhausted" });
+        const again = status === 408 || status === 429 || status >= 500;
+        const asked = providerWaitMs(response.headers, () => new Date());
+        if (again && attempt <= retry.retries && (asked === undefined || asked <= retry.maxRetryAfterMs)) {
+            await sleep(asked ?? backoffMs(attempt, retry, random), options.signal);
+            continue;
+        }
+        if (status === 429)
+            throw new CliError("rate_limited", `OpenAI answered ${said}`, {
+                reason: "provider.rate-limited",
+                ...(asked === undefined ? {} : { retry_after_ms: asked }),
+            });
+        if (status >= 500)
+            throw new CliError([502, 503, 504].includes(status) ? "provider_unavailable" : "provider_error", `OpenAI answered ${said}`, { reason: [502, 503, 504].includes(status) ? "provider.unavailable" : "provider.refused" });
+        throw new CliError("provider_error", `OpenAI answered ${said}`, { reason: "provider.refused" });
+    }
+};
+/**
+ * Each stretch of speech the detector found is a piece of its own: on one channel of a call, a pause is often the
+ * other person talking. Two stretches the detector split without a pause stay one. With `limit`, neighbours are also
+ * joined across pauses while the piece stays within it.
+ */
+export const piecesOf = (spans, limit) => {
+    const pieces = [];
+    for (const span of spans) {
+        const last = pieces.at(-1);
+        const joined = last !== undefined && span.start - last.end < JOINED_SECONDS;
+        const fits = last !== undefined && limit !== undefined && span.end - last.start <= limit;
+        if (last && (fits || (joined && span.end - last.start <= LONGEST_REQUEST_SECONDS)))
+            last.end = span.end;
+        else
+            pieces.push({ start: span.start, end: span.end });
+    }
+    return pieces;
+};
+const openCutter = async (vadModel) => {
+    const { openSpeechCutter } = await import("./recognize.js");
+    return openSpeechCutter(vadModel);
+};
+const checkKeywords = (keywords = []) => {
+    const bad = keywords.find((keyword) => typeof keyword !== "string" || !keyword.trim() || /[<>\r\n]/.test(keyword));
+    if (bad !== undefined)
+        throw new CliError("validation_error", "a keyword must be a non-empty single line without < or > — OpenAI refuses the whole request otherwise");
+};
+/**
+ * Speech to text by OpenAI's `/audio/transcriptions`, from one channel's samples: one piece per stretch of speech the
+ * voice detector found, each sent alone, with where it lies in seconds. Two channels heard this way merge into a
+ * dialogue by `start`. `gpt-transcribe` gives no timestamps of its own, so the times are the detector's.
+ */
+export const transcribeOpenAI = async (audio, options) => {
+    if (!options.apiKey)
+        throw new CliError("authentication_error", "no OpenAI key — `models keys set openai`, or OPENAI_API_KEY", {
+            reason: "transcription.no-key",
+        });
+    checkKeywords(options.keywords);
+    interrupted(options.signal);
+    const pcm = toModelRate(audio.samples, audio.rate);
+    const duration = (pcm.length - LEAD) / SAMPLE_RATE;
+    if (options.bundle && duration <= LONGEST_REQUEST_SECONDS) {
+        const text = await send(encodeWav(pcm.subarray(LEAD)), options);
+        return text ? [{ start: 0, end: duration, text }] : [];
+    }
+    const vadModel = options.vadModel ?? vadPath(modelsDirectory(options.env));
+    if (!options.cutter && !filesPresent([[VAD, vadModel]]))
+        throw new CliError("not_found", "the voice detector that finds the speech is not on this machine — it comes with any speech model: `models download`", { reason: "model.not-downloaded", model: "silero_vad" });
+    const cutter = await (options.cutter ?? openCutter)(vadModel);
+    let spans;
+    try {
+        spans = cutter.cut(pcm);
+    }
+    finally {
+        cutter.free();
+    }
+    const pieces = piecesOf(spans, options.bundle ? LONGEST_REQUEST_SECONDS : undefined);
+    const heard = [];
+    for (const [index, piece] of pieces.entries()) {
+        const end = Math.min(piece.end, duration);
+        const from = Math.max(piece.start - PAD_SECONDS, pieces[index - 1]?.end ?? 0);
+        const to = Math.min(end + PAD_SECONDS, pieces[index + 1]?.start ?? duration);
+        const samples = pcm.subarray(LEAD + Math.floor(from * SAMPLE_RATE), LEAD + Math.ceil(to * SAMPLE_RATE));
+        const text = await send(encodeWav(samples), options);
+        if (text)
+            heard.push({ start: Math.min(piece.start, end), end, text });
+    }
+    return heard;
+};

package/dist/speech/recognize.d.ts

@@ -1,7 +1,15 @@
 import type { SpeechModel } from "./models.js";
+/** Seconds from the start of the recording. */
+export interface TimedText {
+    start: number;
+    end: number;
+    text: string;
+}
 /** What transcription needs from a speech engine — small enough for a test to stand in for it. */
 export interface Recognizer {
     recognize(pcm: Float32Array): string;
+    /** Each stretch of speech the voice detector cut, with where it lies; the same input as `recognize`. */
+    pieces?(pcm: Float32Array): TimedText[];
     free(): void;
 }
 export declare const SAMPLE_RATE = 16000;
@@ -13,7 +21,7 @@ export declare const toModelRate: (samples: Float32Array, rate: number) => Float
 export declare const decodeOgg: (bytes: Uint8Array) => Promise<{
     samples: Float32Array;
     rate: number;
-}>;
+}[]>;
 interface SherpaStream {
     acceptWaveform(rate: number, samples: Float32Array): void;
     free(): void;
@@ -31,6 +39,7 @@ interface Sherpa {
         acceptWaveform(samples: Float32Array): void;
         isEmpty(): boolean;
         front(): {
+            start: number;
             samples: Float32Array;
         };
         pop(): void;
@@ -55,6 +64,21 @@ export declare const detector: (vadModel: string) => {
     debug: number;
     bufferSizeInSeconds: number;
 };
+export interface SpeechSpan {
+    start: number;
+    end: number;
+    samples: Float32Array;
+}
+export interface SpeechCutter {
+    /** Where the speech is, in seconds of the recording; `pcm` as `toModelRate` made it. */
+    cut(pcm: Float32Array): {
+        start: number;
+        end: number;
+    }[];
+    free(): void;
+}
+/** The voice detector alone, for a recognizer that is not on this machine. */
+export declare const openSpeechCutter: (vadModel: string) => SpeechCutter;
 /**
  * The engine's WebAssembly writes its logs (a growing buffer says "Overflow! … No data loss!") with
  * `printf`, which its file system sends straight to file descriptor 1 and so into `--json` output.

package/dist/speech/recognize.js

@@ -31,10 +31,9 @@ export const decodeOgg = async (bytes) => {
     await decoder.ready;
     try {
         const { channelData, sampleRate, samplesDecoded } = await decoder.decodeFile(bytes);
-        const samples = channelData[0];
-        if (!samples || samplesDecoded === 0)
+        if (!channelData[0] || samplesDecoded === 0)
             throw new Error("no Ogg Opus audio in it");
-        return { samples, rate: sampleRate };
+        return channelData.map((samples) => ({ samples, rate: sampleRate }));
     }
     finally {
         decoder.free();
@@ -59,6 +58,43 @@ export const detector = (vadModel) => ({
     debug: 0,
     bufferSizeInSeconds: 30,
 });
+/**
+ * `pcm` as `toModelRate` made it: the detector's sample offsets count its half-second lead, the recording does not.
+ * Each stretch goes to `each` as it is cut, so an hour of speech is never held twice.
+ */
+const cut = (vad, pcm, each) => {
+    // One detector can hear several recordings in a row; nothing of the last may leak into this one.
+    vad.reset();
+    const last = Math.max(0, pcm.length - LEAD);
+    const drain = () => {
+        for (; !vad.isEmpty(); vad.pop()) {
+            const { start, samples } = vad.front();
+            each({
+                start: Math.min(last, Math.max(0, start - LEAD)) / SAMPLE_RATE,
+                end: Math.min(last, Math.max(0, start + samples.length - LEAD)) / SAMPLE_RATE,
+                samples,
+            });
+        }
+    };
+    for (let i = 0; i + WINDOW <= pcm.length; i += WINDOW) {
+        vad.acceptWaveform(pcm.subarray(i, i + WINDOW));
+        drain();
+    }
+    vad.flush();
+    drain();
+};
+/** The voice detector alone, for a recognizer that is not on this machine. */
+export const openSpeechCutter = (vadModel) => {
+    const vad = loadSherpa().createVad(detector(vadModel));
+    return {
+        cut: (pcm) => {
+            const spans = [];
+            cut(vad, pcm, ({ start, end }) => spans.push({ start, end }));
+            return spans;
+        },
+        free: () => vad.free(),
+    };
+};
 /**
  * The model, and a voice detector that cuts the audio at pauses into pieces of at most 20 seconds.
  * The cutting is not optional: five minutes in one piece crash the WebAssembly build (measured).
@@ -107,25 +143,20 @@ export const openRecognizer = (model, path, vadModel) => {
             stream.free();
         }
     };
+    const pieces = (pcm) => {
+        const heard = [];
+        cut(vad, pcm, ({ start, end, samples }) => {
+            const text = piece(samples);
+            if (text)
+                heard.push({ start, end, text });
+        });
+        return heard;
+    };
     return {
-        recognize: (pcm) => {
-            // One recognizer can hear several recordings in a row; nothing of the last may leak into this one.
-            vad.reset();
-            const parts = [];
-            const drain = () => {
-                while (!vad.isEmpty()) {
-                    parts.push(piece(vad.front().samples));
-                    vad.pop();
-                }
-            };
-            for (let i = 0; i + WINDOW <= pcm.length; i += WINDOW) {
-                vad.acceptWaveform(pcm.subarray(i, i + WINDOW));
-                drain();
-            }
-            vad.flush();
-            drain();
-            return parts.filter(Boolean).join(" ");
-        },
+        recognize: (pcm) => pieces(pcm)
+            .map(({ text }) => text)
+            .join(" "),
+        pieces,
         free: () => {
             vad.free();
             engine.free();

package/dist/speech/transcribe.d.ts

@@ -1,18 +1,21 @@
-import { CliError } from "@wirecat/cli-base";
+import { CliError, type KeyringStore } from "@wirecat/cli-base";
 import type { Messenger } from "../cli/messenger/context.js";
 import { type Settings } from "../cli/settings.js";
 import { type MessengerAdapter, type RemoteFile, type Transcript } from "../ports/messenger.js";
 import type { Ffmpeg } from "./decode.js";
 import { type SpeechModel } from "./models.js";
 import type { Recognizer } from "./recognize.js";
-/** `messenger`: its own recognition (Telegram Premium). `local`: a model on this machine. `auto`: the first, then the second. */
-export declare const TRANSCRIBE_WITH: readonly ["auto", "messenger", "local"];
+/**
+ * `messenger`: its own recognition (Telegram Premium). `local`: a model on this machine. `auto`: the first, then the
+ * second. `openai`: OpenAI's API, only when asked by name — `auto` never sends audio off the machine.
+ */
+export declare const TRANSCRIBE_WITH: readonly ["auto", "messenger", "local", "openai"];
 export type TranscribeWith = (typeof TRANSCRIBE_WITH)[number];
 export interface Heard extends Transcript {
     messageId: string;
-    /** The messenger's provider, or `local`. */
+    /** The messenger's provider, `local` or `openai`. */
     via: string;
-    /** The local model, when it was one. */
+    /** The local or OpenAI model, when it was one. */
     model?: string;
 }
 export type OpenRecognizer = (model: SpeechModel, path: (name: string) => string, vadModel: string) => Recognizer;
@@ -24,17 +27,26 @@ export interface Choice {
     /** The local recognizer, where a test hands one in. */
     open?: OpenRecognizer;
     ffmpeg?: Ffmpeg;
+    /** The profile's `transcribeModel` for `openai`, else `gpt-transcribe`. */
+    transcribeModel?: string;
+    apiKey?: string;
+    /** Why nothing can be heard: an unknown `transcribeWith`, or for `openai` no key or a read-only process. */
+    refusal?: CliError;
+    fetch?: (url: string, init: RequestInit) => Promise<Response>;
+    /** The command's own cancel and deadline, for a request that leaves the machine. */
+    signal?: AbortSignal;
 }
 /** A voice note, or an audio file someone sent — an iPhone recording reaches WhatsApp as `audio/mp4`. */
 export declare const isAudioFile: ({ kind, mime }: {
     kind: string;
     mime?: string | undefined;
 }) => boolean;
-/** The flag, then the profile's `transcribeWith` and `speechModel`, then `auto` and the CLI's first model. */
+/** The flag, then the profile's `transcribeWith`, `speechModel` and `transcribeModel`, then `auto` and the CLI's first model. */
 export declare const choose: (messenger: Messenger, settings: Pick<Settings, "configured" | "shared">, flags: {
     local?: boolean;
     model?: string;
-}, env?: NodeJS.ProcessEnv) => Choice;
+    with?: TranscribeWith;
+}, env?: NodeJS.ProcessEnv, keyring?: KeyringStore) => Choice;
 /** Never a download: a model is hundreds of MB, and whether to fetch one is the owner's call. */
 export declare const notDownloaded: (messenger: Messenger, model: SpeechModel) => CliError;
 /**
@@ -43,5 +55,9 @@ export declare const notDownloaded: (messenger: Messenger, model: SpeechModel) =
  * `read` gets the recording once its bytes are all in, for a reference the messenger renewed while they were read.
  */
 export declare const hearOnline: (messenger: Messenger, connection: MessengerAdapter, chat: string, messageId: string, choice: Choice, read?: (file: RemoteFile) => Promise<void>) => Promise<Heard | Uint8Array>;
-/** The local model, on the bytes `hearOnline` fetched. */
-export declare const hearLocally: (bytes: Uint8Array, messageId: string, { model, directory, open, ffmpeg }: Choice) => Promise<Heard>;
+/** The local model, or OpenAI when chosen, on the bytes `hearOnline` fetched. */
+export declare const hearLocally: (bytes: Uint8Array, messageId: string, choice: Choice) => Promise<Heard>;
+/** Where a kept transcript came from, as `Kept` stores it: the messenger's provider, a local model, or `openai:<model>`. */
+export declare const keptSource: (heard: Heard) => string;
+/** The reverse of `keptSource`. */
+export declare const heardVia: (source: string, provider: string) => Pick<Heard, "via" | "model">;

package/dist/speech/transcribe.js

@@ -1,26 +1,52 @@
 import { CliError, isCliError } from "@wirecat/cli-base";
 import { fromFile } from "../cli/settings.js";
+import { embeddingKeys } from "../embeddings/keys.js";
+import { readOnlyLockOf, readOnlyRefusal } from "../ports/app.js";
 import { capability } from "../ports/messenger.js";
 import { typedCommand } from "../runs/scope.js";
 import { installedBytes, isInstalled, megabytes, modelPath, modelsDirectory, vadPath } from "./install.js";
-import { orderedModels, speechModel } from "./models.js";
-/** `messenger`: its own recognition (Telegram Premium). `local`: a model on this machine. `auto`: the first, then the second. */
-export const TRANSCRIBE_WITH = ["auto", "messenger", "local"];
+import { OPENAI_TRANSCRIBE_MODEL, orderedModels, speechModel } from "./models.js";
+/**
+ * `messenger`: its own recognition (Telegram Premium). `local`: a model on this machine. `auto`: the first, then the
+ * second. `openai`: OpenAI's API, only when asked by name — `auto` never sends audio off the machine.
+ */
+export const TRANSCRIBE_WITH = ["auto", "messenger", "local", "openai"];
 /** A recording is read into memory whole; minutes of Opus are a few megabytes. */
 const LARGEST_VOICE = 32 * 1024 * 1024;
 /** A voice note, or an audio file someone sent — an iPhone recording reaches WhatsApp as `audio/mp4`. */
 export const isAudioFile = ({ kind, mime }) => kind === "voice" || kind === "audio" || (mime?.startsWith("audio/") ?? false);
-/** The flag, then the profile's `transcribeWith` and `speechModel`, then `auto` and the CLI's first model. */
-export const choose = (messenger, settings, flags, env = process.env) => {
+/** The flag, then the profile's `transcribeWith`, `speechModel` and `transcribeModel`, then `auto` and the CLI's first model. */
+export const choose = (messenger, settings, flags, env = process.env, keyring) => {
     const model = flags.model ?? fromFile(settings, "speechModel", undefined).value;
+    const chosen = flags.local || flags.model !== undefined
+        ? "local"
+        : (flags.with ?? fromFile(settings, "transcribeWith", "auto").value);
+    // Refused only where something is heard: `models list` reads the same profile for its model alone.
+    const known = TRANSCRIBE_WITH.includes(chosen);
     return {
         directory: modelsDirectory(env),
-        with: flags.local || flags.model !== undefined
-            ? "local"
-            : fromFile(settings, "transcribeWith", "auto").value,
+        with: known ? chosen : "auto",
+        ...(known
+            ? {}
+            : {
+                refusal: new CliError("validation_error", `the profile's transcribeWith is one of ${TRANSCRIBE_WITH.join(", ")}`),
+            }),
         model: model === undefined ? orderedModels(messenger.speechModels)[0] : speechModel(model),
+        transcribeModel: fromFile(settings, "transcribeModel", OPENAI_TRANSCRIBE_MODEL).value,
+        ...(known && chosen === "openai" ? openaiAccess(messenger, env, keyring) : {}),
     };
 };
+const openaiAccess = (messenger, env, keyring) => {
+    const lock = readOnlyLockOf(messenger.app, env);
+    if (lock)
+        return { refusal: readOnlyRefusal(lock, "send audio to OpenAI") };
+    const apiKey = embeddingKeys(messenger.app, env, keyring).read("openai")?.key;
+    return apiKey
+        ? { apiKey }
+        : {
+            refusal: new CliError("authentication_error", `no OpenAI key — \`${typedCommand(messenger.app.command, "models keys set openai")}\`, or OPENAI_API_KEY`, { reason: "transcription.no-key" }),
+        };
+};
 /** Never a download: a model is hundreds of MB, and whether to fetch one is the owner's call. */
 export const notDownloaded = (messenger, model) => new CliError("not_found", `the speech model ${model.id} is not on this machine — the owner downloads it once with ` +
     `\`${typedCommand(messenger.app.command, `models download ${model.id}`)}\` (${megabytes(installedBytes(model))})`, { reason: "model.not-downloaded", model: model.id });
@@ -30,7 +56,9 @@ export const notDownloaded = (messenger, model) => new CliError("not_found", `th
  * `read` gets the recording once its bytes are all in, for a reference the messenger renewed while they were read.
  */
 export const hearOnline = async (messenger, connection, chat, messageId, choice, read) => {
-    if (choice.with !== "local") {
+    if (choice.refusal)
+        throw choice.refusal;
+    if (choice.with === "auto" || choice.with === "messenger") {
         const canAsk = typeof connection.transcribe === "function";
         if (choice.with === "messenger" || canAsk) {
             try {
@@ -44,7 +72,7 @@ export const hearOnline = async (messenger, connection, chat, messageId, choice,
             }
         }
     }
-    if (!isInstalled(choice.model, choice.directory))
+    if (choice.with !== "openai" && !isInstalled(choice.model, choice.directory))
         throw notDownloaded(messenger, choice.model);
     const { files } = await capability(connection, "download", "download attachments")(chat, messageId);
     const voice = files.find(isAudioFile);
@@ -54,11 +82,34 @@ export const hearOnline = async (messenger, connection, chat, messageId, choice,
     await read?.(voice);
     return bytes;
 };
-/** The local model, on the bytes `hearOnline` fetched. */
-export const hearLocally = async (bytes, messageId, { model, directory, open, ffmpeg }) => {
-    const { openRecognizer, toModelRate } = await import("./recognize.js");
+/** The local model, or OpenAI when chosen, on the bytes `hearOnline` fetched. */
+export const hearLocally = async (bytes, messageId, choice) => {
+    const { model, directory, open, ffmpeg } = choice;
     const { decodeAudio } = await import("./decode.js");
     const { samples, rate } = await decodeAudio(bytes, ffmpeg);
+    if (choice.with === "openai") {
+        if (choice.refusal)
+            throw choice.refusal;
+        const { transcribeOpenAI } = await import("./openai.js");
+        const transcribeModel = choice.transcribeModel ?? OPENAI_TRANSCRIBE_MODEL;
+        const pieces = await transcribeOpenAI({ samples, rate }, {
+            apiKey: choice.apiKey ?? "",
+            model: transcribeModel,
+            vadModel: vadPath(directory),
+            // A voice message wants its text, not its turns: one request, and no voice detector needed.
+            bundle: true,
+            ...(choice.fetch ? { fetch: choice.fetch } : {}),
+            ...(choice.signal ? { signal: choice.signal } : {}),
+        });
+        return {
+            messageId,
+            text: pieces.map(({ text }) => text).join(" "),
+            pending: false,
+            via: "openai",
+            model: transcribeModel,
+        };
+    }
+    const { openRecognizer, toModelRate } = await import("./recognize.js");
     const recognizer = (open ?? openRecognizer)(model, modelPath(directory, model), vadPath(directory));
     try {
         return {
@@ -73,6 +124,14 @@ export const hearLocally = async (bytes, messageId, { model, directory, open, ff
         recognizer.free();
     }
 };
+/** Where a kept transcript came from, as `Kept` stores it: the messenger's provider, a local model, or `openai:<model>`. */
+export const keptSource = (heard) => heard.via === "openai" ? `openai:${heard.model ?? OPENAI_TRANSCRIBE_MODEL}` : (heard.model ?? heard.via);
+/** The reverse of `keptSource`. */
+export const heardVia = (source, provider) => source === provider
+    ? { via: source }
+    : source.startsWith("openai:")
+        ? { via: "openai", model: source.slice("openai:".length) }
+        : { via: "local", model: source };
 const readAll = async (file) => {
     const chunks = [];
     let total = 0;

package/dist/speech/wav.d.ts

@@ -25,3 +25,5 @@ export declare const openWavAudio: (plan: WavAudioPlan, input: WavAudioOptions)
     windows(): AsyncIterable<WavPcmWindow>;
     close(): Promise<void>;
 }>;
+/** PCM16 mono WAV of samples in [-1, 1], for a recognizer that takes a file. */
+export declare const encodeWav: (samples: Float32Array, sampleRate?: number) => Uint8Array;

package/dist/speech/wav.js

@@ -196,3 +196,23 @@ export const openWavAudio = async (plan, input) => {
         throw error;
     }
 };
+/** PCM16 mono WAV of samples in [-1, 1], for a recognizer that takes a file. */
+export const encodeWav = (samples, sampleRate = 16000) => {
+    const bytes = Buffer.alloc(44 + samples.length * 2);
+    bytes.write("RIFF", 0, "ascii");
+    bytes.writeUInt32LE(bytes.length - 8, 4);
+    bytes.write("WAVEfmt ", 8, "ascii");
+    bytes.writeUInt32LE(16, 16);
+    bytes.writeUInt16LE(1, 20);
+    bytes.writeUInt16LE(1, 22);
+    bytes.writeUInt32LE(sampleRate, 24);
+    bytes.writeUInt32LE(sampleRate * 2, 28);
+    bytes.writeUInt16LE(2, 32);
+    bytes.writeUInt16LE(16, 34);
+    bytes.write("data", 36, "ascii");
+    bytes.writeUInt32LE(samples.length * 2, 40);
+    samples.forEach((sample, index) => {
+        bytes.writeInt16LE(Math.round(Math.max(-1, Math.min(1, sample || 0)) * 32767), 44 + index * 2);
+    });
+    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.length);
+};