@markusylisiurunen/tau 0.2.123 → 0.2.124

package/dist/core/utils/gemini_transcription.js

@@ -0,0 +1,136 @@
+import { z } from "zod";
+const GEMINI_GENERATE_CONTENT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/models";
+const DEFAULT_GEMINI_TRANSCRIPTION_MODEL = "gemini-3.5-flash";
+const DEFAULT_GEMINI_TRANSCRIPTION_THINKING_LEVEL = "minimal";
+const DEFAULT_GEMINI_AUDIO_MIME_TYPE = "audio/wav";
+const errorPayloadSchema = z.object({
+    error: z
+        .object({
+        message: z.string().trim().min(1).optional(),
+        status: z.string().trim().min(1).optional(),
+        code: z.number().int().optional(),
+    })
+        .optional(),
+});
+const GEMINI_TRANSCRIPTION_RESPONSE_SCHEMA = {
+    type: "OBJECT",
+    properties: {
+        transcription: { type: "STRING" },
+    },
+    required: ["transcription"],
+};
+const textPartSchema = z.object({ text: z.string() });
+const apiResponseSchema = z.object({
+    candidates: z
+        .array(z.object({
+        content: z.object({
+            parts: z.array(z.unknown()).optional(),
+        }),
+    }))
+        .optional(),
+});
+const transcriptionResultSchema = z.object({
+    transcription: z.string(),
+});
+export async function transcribeGeminiAudio(options) {
+    const apiKey = options.apiKey.trim();
+    if (!apiKey) {
+        throw new Error("missing Gemini API key");
+    }
+    const fetchFn = options.fetchImpl ?? fetch;
+    const model = options.model ?? DEFAULT_GEMINI_TRANSCRIPTION_MODEL;
+    const response = await fetchFn(`${GEMINI_GENERATE_CONTENT_BASE_URL}/${encodeURIComponent(model)}:generateContent`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            "x-goog-api-key": apiKey,
+        },
+        body: JSON.stringify({
+            systemInstruction: {
+                parts: [
+                    {
+                        text: buildTranscriptionSystemInstruction(),
+                    },
+                ],
+            },
+            contents: [
+                {
+                    parts: [
+                        {
+                            text: buildTranscriptionPrompt(),
+                        },
+                        {
+                            inlineData: {
+                                mimeType: options.mimeType ?? DEFAULT_GEMINI_AUDIO_MIME_TYPE,
+                                data: options.audio.toString("base64"),
+                            },
+                        },
+                    ],
+                },
+            ],
+            generationConfig: {
+                responseMimeType: "application/json",
+                responseSchema: GEMINI_TRANSCRIPTION_RESPONSE_SCHEMA,
+                thinkingConfig: {
+                    thinkingLevel: DEFAULT_GEMINI_TRANSCRIPTION_THINKING_LEVEL,
+                },
+            },
+        }),
+    });
+    const responseText = await response.text();
+    let payload;
+    try {
+        payload = responseText ? JSON.parse(responseText) : undefined;
+    }
+    catch {
+        payload = undefined;
+    }
+    if (!response.ok) {
+        const parsed = errorPayloadSchema.safeParse(payload);
+        const fallbackMessage = responseText.trim() || `HTTP ${response.status}`;
+        throw new Error(parsed.success ? (parsed.data.error?.message ?? fallbackMessage) : fallbackMessage);
+    }
+    return extractGeminiText(payload).trim();
+}
+function buildTranscriptionSystemInstruction() {
+    return [
+        "You are a speech-to-text engine.",
+        "Transcribe the speaker's intended message for insertion into a chat input.",
+        "Detect the speaker's language and transcribe in that same language; never translate unless the speaker explicitly asks for translation.",
+        "Preserve the speaker's wording and register as spoken, including colloquial forms, dialect, and informal language; do not normalize informal speech into formal standard language.",
+        "Use natural punctuation and capitalization where helpful, without changing the speaker's wording or register.",
+        "Lightly clean only speech artifacts that do not affect meaning, such as filler words, repeated stutters, obvious false starts, and unintelligible mumbling.",
+        "Do not rewrite, paraphrase, summarize, answer the speaker, add labels, add timestamps, or describe background sounds.",
+    ].join("\n");
+}
+function buildTranscriptionPrompt() {
+    return [
+        "Transcribe the attached audio into the transcription field.",
+        "Return only the lightly cleaned transcript text, with no timestamps or commentary.",
+    ].join("\n");
+}
+function extractGeminiText(payload) {
+    const parsed = apiResponseSchema.safeParse(payload);
+    if (!parsed.success) {
+        return "";
+    }
+    const responseText = (parsed.data.candidates?.[0]?.content.parts ?? [])
+        .map((part) => {
+        const parsedPart = textPartSchema.safeParse(part);
+        return parsedPart.success ? parsedPart.data.text : "";
+    })
+        .join("");
+    let transcriptionPayload;
+    try {
+        transcriptionPayload = responseText ? JSON.parse(responseText) : undefined;
+    }
+    catch {
+        return "";
+    }
+    const transcription = transcriptionResultSchema.safeParse(transcriptionPayload);
+    if (!transcription.success) {
+        return "";
+    }
+    return transcription.data.transcription;
+}
+//# sourceMappingURL=gemini_transcription.js.map

package/dist/core/utils/gemini_transcription.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gemini_transcription.js","sourceRoot":"","sources":["../../../src/core/utils/gemini_transcription.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,gCAAgC,GAAG,yDAAyD,CAAC;AACnG,MAAM,kCAAkC,GAAG,kBAAkB,CAAC;AAC9D,MAAM,2CAA2C,GAAG,SAAS,CAAC;AAC9D,MAAM,8BAA8B,GAAG,WAAW,CAAC;AAEnD,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC;SACL,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC5C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC3C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;KAClC,CAAC;SACD,QAAQ,EAAE;CACd,CAAC,CAAC;AAEH,MAAM,oCAAoC,GAAG;IAC3C,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE;QACV,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;KAClC;IACD,QAAQ,EAAE,CAAC,eAAe,CAAC;CAC5B,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACtD,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,UAAU,EAAE,CAAC;SACV,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;SACvC,CAAC;KACH,CAAC,CACH;SACA,QAAQ,EAAE;CACd,CAAC,CAAC;AACH,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;CAC1B,CAAC,CAAC;AAUH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAAmC;IAC7E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACrC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,kCAAkC,CAAC;IAClE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAC5B,GAAG,gCAAgC,IAAI,kBAAkB,CAAC,KAAK,CAAC,kBAAkB,EAClF;QACE,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,gBAAgB,EAAE,MAAM;SACzB;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,iBAAiB,EAAE;gBACjB,KAAK,EAAE;oBACL;wBACE,IAAI,EAAE,mCAAmC,EAAE;qBAC5C;iBACF;aACF;YACD,QAAQ,EAAE;gBACR;oBACE,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,wBAAwB,EAAE;yBACjC;wBACD;4BACE,UAAU,EAAE;gCACV,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,8BAA8B;gCAC5D,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;6BACvC;yBACF;qBACF;iBACF;aACF;YACD,gBAAgB,EAAE;gBAChB,gBAAgB,EAAE,kBAAkB;gBACpC,cAAc,EAAE,oCAAoC;gBACpD,cAAc,EAAE;oBACd,aAAa,EAAE,2CAA2C;iBAC3D;aACF;SACF,CAAC;KACH,CACF,CAAC;IAEF,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,OAAgB,CAAC;IACrB,IAAI,CAAC;QACH,OAAO,GAAG,YAAY,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAa,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,SAAS,CAAC;IACtB,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CACb,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CACnF,CAAC;IACJ,CAAC;IAED,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,mCAAmC;IAC1C,OAAO;QACL,kCAAkC;QAClC,4EAA4E;QAC5E,yIAAyI;QACzI,oLAAoL;QACpL,+GAA+G;QAC/G,6JAA6J;QAC7J,uHAAuH;KACxH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,wBAAwB;IAC/B,OAAO;QACL,6DAA6D;QAC7D,oFAAoF;KACrF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAgB;IACzC,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;SACpE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,UAAU,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACxD,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;IAEZ,IAAI,oBAA6B,CAAC;IAClC,IAAI,CAAC;QACH,oBAAoB,GAAG,YAAY,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAa,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,aAAa,GAAG,yBAAyB,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IAChF,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC;AAC1C,CAAC"}

package/dist/core/utils/speech_to_text.js

@@ -0,0 +1,23 @@
+import { transcribeGeminiAudio } from "./gemini_transcription.js";
+import { transcribeMistralAudio } from "./mistral_transcription.js";
+export async function transcribeAudio(options) {
+    switch (options.provider) {
+        case "gemini":
+            return await transcribeGeminiAudio({
+                apiKey: options.apiKey,
+                audio: options.audio,
+                mimeType: options.mimeType,
+                fetchImpl: options.fetchImpl,
+            });
+        case "mistral":
+            return await transcribeMistralAudio({
+                apiKey: options.apiKey,
+                audio: options.audio,
+                mimeType: options.mimeType,
+                fileName: options.fileName,
+                language: options.language,
+                fetchImpl: options.fetchImpl,
+            });
+    }
+}
+//# sourceMappingURL=speech_to_text.js.map

package/dist/core/utils/speech_to_text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"speech_to_text.js","sourceRoot":"","sources":["../../../src/core/utils/speech_to_text.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AAYpE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA4B;IAChE,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;QACzB,KAAK,QAAQ;YACX,OAAO,MAAM,qBAAqB,CAAC;gBACjC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAC;QACL,KAAK,SAAS;YACZ,OAAO,MAAM,sBAAsB,CAAC;gBAClC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAC;IACP,CAAC;AACH,CAAC"}

package/dist/core/version.js

@@ -1,2 +1,2 @@
-export const APP_VERSION = "0.2.123";
+export const APP_VERSION = "0.2.124";
 //# sourceMappingURL=version.js.map

package/dist/tui/chat_controller.js

@@ -9,7 +9,7 @@ import { getAuthPath } from "../core/auth/auth_paths.js";
 import { AuthStorage } from "../core/auth/auth_storage.js";
 import { createCredentialResolver, } from "../core/auth/credential_resolver.js";
 import { createCommandRegistry, getRiskLevelDescription, } from "../core/commands/index.js";
-import { createDefaultConfigDeps, getMistralApiKey, loadRuntimeConfig, } from "../core/config/index.js";
+import { createDefaultConfigDeps, getGoogleApiKey, getMistralApiKey, loadRuntimeConfig, } from "../core/config/index.js";
 import { startDiffReviewSession as startCoreDiffReviewSession } from "../core/diff_review/index.js";
 import { ChatRuntime } from "../core/runtime/chat_runtime.js";
 import { createDefaultCoreDeps } from "../core/runtime/deps.js";
@@ -24,9 +24,9 @@ import { formatCwdChangeNotice, formatProjectContextChangeNotice, formatRiskLeve
 import { formatAdaptiveNumber, formatCwd, formatPathForDisplay, formatTokenWindow, } from "../core/utils/format.js";
 import { streamGeminiSpeechAudio } from "../core/utils/gemini_speech.js";
 import { extractAllFencedCodeBlocks, extractAssistantText } from "../core/utils/messages.js";
-import { transcribeMistralAudio } from "../core/utils/mistral_transcription.js";
 import { streamModel } from "../core/utils/model_stream.js";
 import { listProjectFilesAsync } from "../core/utils/project_files.js";
+import { transcribeAudio } from "../core/utils/speech_to_text.js";
 import { getAutoCompactionMetadataFromMessage, hasAutoCompactionContinuationMetadata, stripTauUserMetadata, } from "../core/utils/user_metadata.js";
 import { APP_VERSION } from "../core/version.js";
 import { DiffReviewService, } from "./chat_controller/diff_review_service.js";
@@ -1266,9 +1266,9 @@ export class ChatController {
             this.view.addSystemMessage("/listen is currently supported only on macOS.", "warn");
             return;
         }
-        const apiKey = getMistralApiKey(this.config, this.deps.env.env());
+        const apiKey = this.getSpeechToTextApiKey();
         if (!apiKey) {
-            this.view.addSystemMessage("set MISTRAL_API_KEY or apiKeys.mistral to use /listen", "error");
+            this.view.addSystemMessage(this.getSpeechToTextApiKeyErrorMessage("use /listen"), "error");
             return;
         }
         let audioPath;
@@ -1433,12 +1433,29 @@ export class ChatController {
         }
         return path;
     }
+    getSpeechToTextProvider() {
+        return this.config.speechToText?.provider ?? "mistral";
+    }
+    getSpeechToTextApiKey() {
+        const provider = this.getSpeechToTextProvider();
+        return provider === "gemini"
+            ? getGoogleApiKey(this.config, this.deps.env.env())
+            : getMistralApiKey(this.config, this.deps.env.env());
+    }
+    getSpeechToTextApiKeyErrorMessage(action) {
+        const provider = this.getSpeechToTextProvider();
+        return provider === "gemini"
+            ? `set GEMINI_API_KEY or apiKeys.google to ${action}`
+            : `set MISTRAL_API_KEY or apiKeys.mistral to ${action}`;
+    }
     async transcribeListenAudio(audio) {
-        const apiKey = getMistralApiKey(this.config, this.deps.env.env());
+        const provider = this.getSpeechToTextProvider();
+        const apiKey = this.getSpeechToTextApiKey();
         if (!apiKey) {
-            throw new Error("missing MISTRAL_API_KEY or apiKeys.mistral");
+            throw new Error(this.getSpeechToTextApiKeyErrorMessage("transcribe speech"));
         }
-        return await transcribeMistralAudio({
+        return await transcribeAudio({
+            provider,
             apiKey,
             audio,
             mimeType: "audio/wav",