@wirecat/cli-core 0.497.0 → 0.499.0

package/dist/speech/transcribe.js

@@ -1,26 +1,52 @@
 import { CliError, isCliError } from "@wirecat/cli-base";
 import { fromFile } from "../cli/settings.js";
+import { embeddingKeys } from "../embeddings/keys.js";
+import { readOnlyLockOf, readOnlyRefusal } from "../ports/app.js";
 import { capability } from "../ports/messenger.js";
 import { typedCommand } from "../runs/scope.js";
 import { installedBytes, isInstalled, megabytes, modelPath, modelsDirectory, vadPath } from "./install.js";
-import { orderedModels, speechModel } from "./models.js";
-/** `messenger`: its own recognition (Telegram Premium). `local`: a model on this machine. `auto`: the first, then the second. */
-export const TRANSCRIBE_WITH = ["auto", "messenger", "local"];
+import { OPENAI_TRANSCRIBE_MODEL, orderedModels, speechModel } from "./models.js";
+/**
+ * `messenger`: its own recognition (Telegram Premium). `local`: a model on this machine. `auto`: the first, then the
+ * second. `openai`: OpenAI's API, only when asked by name — `auto` never sends audio off the machine.
+ */
+export const TRANSCRIBE_WITH = ["auto", "messenger", "local", "openai"];
 /** A recording is read into memory whole; minutes of Opus are a few megabytes. */
 const LARGEST_VOICE = 32 * 1024 * 1024;
 /** A voice note, or an audio file someone sent — an iPhone recording reaches WhatsApp as `audio/mp4`. */
 export const isAudioFile = ({ kind, mime }) => kind === "voice" || kind === "audio" || (mime?.startsWith("audio/") ?? false);
-/** The flag, then the profile's `transcribeWith` and `speechModel`, then `auto` and the CLI's first model. */
-export const choose = (messenger, settings, flags, env = process.env) => {
+/** The flag, then the profile's `transcribeWith`, `speechModel` and `transcribeModel`, then `auto` and the CLI's first model. */
+export const choose = (messenger, settings, flags, env = process.env, keyring) => {
     const model = flags.model ?? fromFile(settings, "speechModel", undefined).value;
+    const chosen = flags.local || flags.model !== undefined
+        ? "local"
+        : (flags.with ?? fromFile(settings, "transcribeWith", "auto").value);
+    // Refused only where something is heard: `models list` reads the same profile for its model alone.
+    const known = TRANSCRIBE_WITH.includes(chosen);
     return {
         directory: modelsDirectory(env),
-        with: flags.local || flags.model !== undefined
-            ? "local"
-            : fromFile(settings, "transcribeWith", "auto").value,
+        with: known ? chosen : "auto",
+        ...(known
+            ? {}
+            : {
+                refusal: new CliError("validation_error", `the profile's transcribeWith is one of ${TRANSCRIBE_WITH.join(", ")}`),
+            }),
         model: model === undefined ? orderedModels(messenger.speechModels)[0] : speechModel(model),
+        transcribeModel: fromFile(settings, "transcribeModel", OPENAI_TRANSCRIBE_MODEL).value,
+        ...(known && chosen === "openai" ? openaiAccess(messenger, env, keyring) : {}),
     };
 };
+const openaiAccess = (messenger, env, keyring) => {
+    const lock = readOnlyLockOf(messenger.app, env);
+    if (lock)
+        return { refusal: readOnlyRefusal(lock, "send audio to OpenAI") };
+    const apiKey = embeddingKeys(messenger.app, env, keyring).read("openai")?.key;
+    return apiKey
+        ? { apiKey }
+        : {
+            refusal: new CliError("authentication_error", `no OpenAI key — \`${typedCommand(messenger.app.command, "models keys set openai")}\`, or OPENAI_API_KEY`, { reason: "transcription.no-key" }),
+        };
+};
 /** Never a download: a model is hundreds of MB, and whether to fetch one is the owner's call. */
 export const notDownloaded = (messenger, model) => new CliError("not_found", `the speech model ${model.id} is not on this machine — the owner downloads it once with ` +
     `\`${typedCommand(messenger.app.command, `models download ${model.id}`)}\` (${megabytes(installedBytes(model))})`, { reason: "model.not-downloaded", model: model.id });
@@ -30,7 +56,9 @@ export const notDownloaded = (messenger, model) => new CliError("not_found", `th
  * `read` gets the recording once its bytes are all in, for a reference the messenger renewed while they were read.
  */
 export const hearOnline = async (messenger, connection, chat, messageId, choice, read) => {
-    if (choice.with !== "local") {
+    if (choice.refusal)
+        throw choice.refusal;
+    if (choice.with === "auto" || choice.with === "messenger") {
         const canAsk = typeof connection.transcribe === "function";
         if (choice.with === "messenger" || canAsk) {
             try {
@@ -44,7 +72,7 @@ export const hearOnline = async (messenger, connection, chat, messageId, choice,
             }
         }
     }
-    if (!isInstalled(choice.model, choice.directory))
+    if (choice.with !== "openai" && !isInstalled(choice.model, choice.directory))
         throw notDownloaded(messenger, choice.model);
     const { files } = await capability(connection, "download", "download attachments")(chat, messageId);
     const voice = files.find(isAudioFile);
@@ -54,11 +82,34 @@ export const hearOnline = async (messenger, connection, chat, messageId, choice,
     await read?.(voice);
     return bytes;
 };
-/** The local model, on the bytes `hearOnline` fetched. */
-export const hearLocally = async (bytes, messageId, { model, directory, open, ffmpeg }) => {
-    const { openRecognizer, toModelRate } = await import("./recognize.js");
+/** The local model, or OpenAI when chosen, on the bytes `hearOnline` fetched. */
+export const hearLocally = async (bytes, messageId, choice) => {
+    const { model, directory, open, ffmpeg } = choice;
     const { decodeAudio } = await import("./decode.js");
     const { samples, rate } = await decodeAudio(bytes, ffmpeg);
+    if (choice.with === "openai") {
+        if (choice.refusal)
+            throw choice.refusal;
+        const { transcribeOpenAI } = await import("./openai.js");
+        const transcribeModel = choice.transcribeModel ?? OPENAI_TRANSCRIBE_MODEL;
+        const pieces = await transcribeOpenAI({ samples, rate }, {
+            apiKey: choice.apiKey ?? "",
+            model: transcribeModel,
+            vadModel: vadPath(directory),
+            // A voice message wants its text, not its turns: one request, and no voice detector needed.
+            bundle: true,
+            ...(choice.fetch ? { fetch: choice.fetch } : {}),
+            ...(choice.signal ? { signal: choice.signal } : {}),
+        });
+        return {
+            messageId,
+            text: pieces.map(({ text }) => text).join(" "),
+            pending: false,
+            via: "openai",
+            model: transcribeModel,
+        };
+    }
+    const { openRecognizer, toModelRate } = await import("./recognize.js");
     const recognizer = (open ?? openRecognizer)(model, modelPath(directory, model), vadPath(directory));
     try {
         return {
@@ -73,6 +124,14 @@ export const hearLocally = async (bytes, messageId, { model, directory, open, ff
         recognizer.free();
     }
 };
+/** Where a kept transcript came from, as `Kept` stores it: the messenger's provider, a local model, or `openai:<model>`. */
+export const keptSource = (heard) => heard.via === "openai" ? `openai:${heard.model ?? OPENAI_TRANSCRIBE_MODEL}` : (heard.model ?? heard.via);
+/** The reverse of `keptSource`. */
+export const heardVia = (source, provider) => source === provider
+    ? { via: source }
+    : source.startsWith("openai:")
+        ? { via: "openai", model: source.slice("openai:".length) }
+        : { via: "local", model: source };
 const readAll = async (file) => {
     const chunks = [];
     let total = 0;

package/dist/speech/wav.d.ts

@@ -25,3 +25,5 @@ export declare const openWavAudio: (plan: WavAudioPlan, input: WavAudioOptions)
     windows(): AsyncIterable<WavPcmWindow>;
     close(): Promise<void>;
 }>;
+/** PCM16 mono WAV of samples in [-1, 1], for a recognizer that takes a file. */
+export declare const encodeWav: (samples: Float32Array, sampleRate?: number) => Uint8Array;

package/dist/speech/wav.js

@@ -196,3 +196,23 @@ export const openWavAudio = async (plan, input) => {
         throw error;
     }
 };
+/** PCM16 mono WAV of samples in [-1, 1], for a recognizer that takes a file. */
+export const encodeWav = (samples, sampleRate = 16000) => {
+    const bytes = Buffer.alloc(44 + samples.length * 2);
+    bytes.write("RIFF", 0, "ascii");
+    bytes.writeUInt32LE(bytes.length - 8, 4);
+    bytes.write("WAVEfmt ", 8, "ascii");
+    bytes.writeUInt32LE(16, 16);
+    bytes.writeUInt16LE(1, 20);
+    bytes.writeUInt16LE(1, 22);
+    bytes.writeUInt32LE(sampleRate, 24);
+    bytes.writeUInt32LE(sampleRate * 2, 28);
+    bytes.writeUInt16LE(2, 32);
+    bytes.writeUInt16LE(16, 34);
+    bytes.write("data", 36, "ascii");
+    bytes.writeUInt32LE(samples.length * 2, 40);
+    samples.forEach((sample, index) => {
+        bytes.writeInt16LE(Math.round(Math.max(-1, Math.min(1, sample || 0)) * 32767), 44 + index * 2);
+    });
+    return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.length);
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.497.0",
+  "version": "0.499.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",