@gobing-ai/knowledge-kit 0.0.21 → 0.0.23

package/plugins/generations/gemini-tts/package.json

@@ -0,0 +1,16 @@
+{
+    "name": "@gobing-ai/gemini-tts",
+    "type": "module",
+    "private": true,
+    "scripts": {
+        "typecheck": "tsc --noEmit"
+    },
+    "dependencies": {
+        "@gobing-ai/ts-utils": "catalog:",
+        "@gobing-ai/voice-gen": "workspace:*"
+    },
+    "devDependencies": {
+        "@gobing-ai/kk-core": "workspace:*",
+        "@types/bun": "1.3.14"
+    }
+}

package/plugins/generations/gemini-tts/plugin.json

@@ -0,0 +1,6 @@
+{
+    "name": "gemini-tts",
+    "kind": "generator",
+    "entry": "./dist/index.js",
+    "version": "1.0.0"
+}

package/plugins/generations/gemini-tts/profiles.json

@@ -0,0 +1,3 @@
+{
+    "robin-news": { "voice": "voice_REPLACE_ME" }
+}

package/plugins/generations/gemini-tts/src/gemini-client.ts

@@ -0,0 +1,190 @@
+import { parseWav, type VoiceboxClient, type VoiceboxGenerateBody, type VoiceboxProfile } from '@gobing-ai/voice-gen';
+
+export const GEMINI_TTS_MODELS = ['gemini-3.8-flash-tts', 'gemini-3.8-flash-lite-tts'] as const;
+export const GEMINI_TTS_DEFAULT_MODEL = GEMINI_TTS_MODELS[0];
+export const GEMINI_ASR_DEFAULT_MODEL = 'gemini-3.8-flash';
+const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com';
+const DEFAULT_PROFILES_PATH = new URL('../profiles.json', import.meta.url).pathname;
+const TRANSCRIBE_PROMPT = 'Transcribe this audio verbatim. Output only the transcript, no commentary.';
+const PLACEHOLDER_VOICE = 'voice_REPLACE_ME';
+
+type Env = Record<string, string | undefined>;
+
+export interface GeminiVoiceClientOptions {
+    env?: Env;
+    fetch?: typeof fetch;
+    sleep?: (ms: number) => Promise<void>;
+}
+
+interface InteractionContent {
+    type?: string;
+    data?: string;
+    text?: string;
+}
+
+interface InteractionResponse {
+    id?: string;
+    steps?: { type?: string; content?: InteractionContent[] }[];
+    error?: { message?: string };
+}
+
+/** First `model_output` content part of the given type (observed live shape, 2026-09-24). */
+function findOutput(json: InteractionResponse, type: string): InteractionContent | undefined {
+    for (const step of json.steps ?? []) {
+        if (step.type !== 'model_output') continue;
+        const hit = step.content?.find((c) => c.type === type);
+        if (hit) return hit;
+    }
+    return undefined;
+}
+
+function positiveInt(raw: string | undefined, fallback: number): number {
+    const n = raw ? Number.parseInt(raw, 10) : Number.NaN;
+    return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+
+function excerpt(text: string): string {
+    return text.length > 30 ? `${text.slice(0, 30)}…` : text;
+}
+
+/**
+ * VoiceboxClient over the Gemini 3.8 TTS REST API, so voice-gen's `processGeneratorIO` pipeline
+ * (verify-retry, speed, concat, QC, MP3) runs unchanged against a hosted backend (ADR-023).
+ * Synthesis is unary: `generate` buffers the WAV and `waitUntilDone` completes immediately.
+ */
+export function createGeminiVoiceClient(options: GeminiVoiceClientOptions = {}): VoiceboxClient {
+    const env = options.env ?? process.env;
+    const doFetch = options.fetch ?? fetch;
+    const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
+    const baseUrl = (env.GOOGLE_BASE_URL || DEFAULT_BASE_URL).replace(/\/+$/, '');
+    const model = env.GEMINI_TTS_MODEL || GEMINI_TTS_DEFAULT_MODEL;
+    const maxAttempts = positiveInt(env.GEMINI_TTS_MAX_ATTEMPTS, 4);
+    const timeoutMs = positiveInt(env.GEMINI_TTS_TIMEOUT_MS, 120_000);
+    const buffered = new Map<string, Uint8Array>();
+    let profiles: Record<string, { voice?: string }> | undefined;
+    let seq = 0;
+
+    async function post(body: unknown, label: string): Promise<InteractionResponse> {
+        const apiKey = env.GEMINI_API_KEY ?? '';
+        for (let attempt = 1; ; attempt++) {
+            let res: Response;
+            try {
+                res = await doFetch(`${baseUrl}/v1beta/interactions`, {
+                    method: 'POST',
+                    headers: { 'content-type': 'application/json', 'x-goog-api-key': apiKey },
+                    body: JSON.stringify(body),
+                    signal: AbortSignal.timeout(timeoutMs),
+                });
+            } catch (err: unknown) {
+                const msg = err instanceof Error ? err.message : String(err);
+                if (attempt >= maxAttempts)
+                    throw new Error(`${label}: request failed after ${attempt} attempts: ${msg}`);
+                await sleep(2000 * 2 ** (attempt - 1));
+                continue;
+            }
+            if (res.ok) return (await res.json()) as InteractionResponse;
+            const raw = await res.text();
+            let apiMessage = raw.slice(0, 300);
+            try {
+                apiMessage = (JSON.parse(raw) as InteractionResponse).error?.message ?? apiMessage;
+            } catch {
+                // non-JSON error body — keep the raw excerpt
+            }
+            const retryable = res.status === 429 || res.status >= 500;
+            if (!retryable || attempt >= maxAttempts) {
+                throw new Error(`${label}: HTTP ${res.status} after ${attempt} attempt(s): ${apiMessage}`);
+            }
+            const retryAfter = Number(res.headers.get('retry-after'));
+            await sleep(retryAfter > 0 ? retryAfter * 1000 : 2000 * 2 ** (attempt - 1));
+        }
+    }
+
+    async function loadProfiles(): Promise<Record<string, { voice?: string }>> {
+        if (profiles) return profiles;
+        const path = env.GEMINI_TTS_PROFILES || DEFAULT_PROFILES_PATH;
+        const file = Bun.file(path);
+        if (!(await file.exists())) throw new Error(`gemini-tts: profile map not found: ${path}`);
+        profiles = (await file.json()) as Record<string, { voice?: string }>;
+        return profiles;
+    }
+
+    const client: VoiceboxClient = {
+        url: baseUrl,
+
+        async health(): Promise<void> {
+            if (!env.GEMINI_API_KEY) throw new Error('gemini-tts: GEMINI_API_KEY is required');
+            if (!(GEMINI_TTS_MODELS as readonly string[]).includes(model)) {
+                throw new Error(
+                    `gemini-tts: unsupported GEMINI_TTS_MODEL "${model}" (allowed: ${GEMINI_TTS_MODELS.join(', ')})`,
+                );
+            }
+        },
+
+        async resolveProfile(nameOrId: string): Promise<VoiceboxProfile> {
+            if (/^voice(key)?_/.test(nameOrId) && nameOrId !== PLACEHOLDER_VOICE) {
+                return { id: nameOrId, name: nameOrId };
+            }
+            const voice = (await loadProfiles())[nameOrId]?.voice;
+            if (!voice) throw new Error(`gemini-tts: unknown voice profile "${nameOrId}"`);
+            if (voice === PLACEHOLDER_VOICE) {
+                throw new Error(
+                    `gemini-tts: profile "${nameOrId}" still holds the placeholder voice id — create the replicated voice (see plugins/generations/gemini-tts/README.md) and set it in GEMINI_TTS_PROFILES`,
+                );
+            }
+            return { id: voice, name: nameOrId };
+        },
+
+        async generate(body: VoiceboxGenerateBody): Promise<{ id: string }> {
+            const part: Record<string, unknown> = { type: 'text', text: body.text };
+            if (body.instruct) part.annotations = [{ type: 'speech_metadata', style: body.instruct }];
+            const json = await post(
+                {
+                    model,
+                    input: [{ type: 'user_input', content: [part] }],
+                    response_format: { type: 'audio', mime_type: 'audio/wav', sample_rate: 24000 },
+                    generation_config: { speech_config: [{ voice: body.profile_id }] },
+                },
+                `gemini-tts generate ("${excerpt(body.text)}")`,
+            );
+            const audio = findOutput(json, 'audio');
+            if (!audio?.data) throw new Error(`gemini-tts generate ("${excerpt(body.text)}"): response had no audio`);
+            seq += 1;
+            const id = `${json.id ?? 'gemini'}#${seq}`;
+            buffered.set(id, new Uint8Array(Buffer.from(audio.data, 'base64')));
+            return { id };
+        },
+
+        async waitUntilDone(id: string) {
+            const bytes = buffered.get(id);
+            if (!bytes) return { status: 'failed' as const, error: `unknown generation id ${id}` };
+            const wav = parseWav(bytes);
+            return { status: 'completed' as const, duration: wav.dataBytes.length / wav.byteRate };
+        },
+
+        async downloadAudio(id: string): Promise<Uint8Array> {
+            const bytes = buffered.get(id);
+            if (!bytes) throw new Error(`gemini-tts: unknown generation id ${id}`);
+            buffered.delete(id);
+            return bytes;
+        },
+    };
+
+    if (env.GEMINI_TTS_ASR !== 'off') {
+        const asrModel = env.GEMINI_TTS_ASR_MODEL || GEMINI_ASR_DEFAULT_MODEL;
+        client.transcribe = async (audio: Uint8Array) => {
+            const json = await post(
+                {
+                    model: asrModel,
+                    input: [
+                        { type: 'text', text: TRANSCRIBE_PROMPT },
+                        { type: 'audio', data: Buffer.from(audio).toString('base64'), mime_type: 'audio/wav' },
+                    ],
+                },
+                'gemini-tts transcribe',
+            );
+            return { text: findOutput(json, 'text')?.text?.trim() ?? '' };
+        };
+    }
+
+    return client;
+}

package/plugins/generations/gemini-tts/src/index.ts

@@ -0,0 +1,33 @@
+import { parseArgs } from 'node:util';
+import { echoError } from '@gobing-ai/ts-utils';
+import { processGeneratorIO } from '@gobing-ai/voice-gen';
+import { createGeminiVoiceClient } from './gemini-client';
+
+export * from './gemini-client';
+
+export async function main(argv: string[] = process.argv.slice(2)): Promise<number> {
+    const { values } = parseArgs({
+        args: argv,
+        options: {
+            in: { type: 'string' },
+            out: { type: 'string' },
+        },
+    });
+
+    if (!values.in || !values.out) {
+        echoError('gemini-tts failed: Missing required arguments: --in and --out');
+        return 1;
+    }
+
+    try {
+        await processGeneratorIO(values.in, values.out, createGeminiVoiceClient(), undefined, 'kk:gemini-tts');
+        return 0;
+    } catch (err: unknown) {
+        echoError(`gemini-tts failed: ${err instanceof Error ? err.message : String(err)}`);
+        return 1;
+    }
+}
+
+if (import.meta.main) {
+    process.exit(await main());
+}

package/plugins/generations/gemini-tts/src/voice-create.ts

@@ -0,0 +1,180 @@
+import { parseArgs } from 'node:util';
+import { echo, echoError } from '@gobing-ai/ts-utils';
+import { parseWav } from '@gobing-ai/voice-gen';
+import { GEMINI_TTS_DEFAULT_MODEL } from './gemini-client';
+
+const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com';
+const VOICE_ID = /^(?:voice_|voicekey_)/;
+
+export interface CreateReplicatedVoiceOptions {
+    source: Uint8Array;
+    consent: Uint8Array;
+    name: string;
+    store: boolean;
+    model: string;
+    env?: Record<string, string | undefined>;
+    fetch?: typeof fetch;
+}
+
+/**
+ * Reject a clip that is not 24 kHz mono 16-bit PCM WAV. Source clips must also be 10–30 s.
+ * The error names the measured rate, channels, bits, and duration.
+ */
+export function validateVoiceWav(bytes: Uint8Array, role: 'source' | 'consent'): void {
+    let wav: ReturnType<typeof parseWav>;
+    try {
+        wav = parseWav(bytes);
+    } catch (err: unknown) {
+        const detail = err instanceof Error ? err.message : String(err);
+        throw new Error(
+            `gemini-tts voice-create: ${role} is not a RIFF WAV (${detail}); need 24000 Hz, 1 ch, 16-bit PCM`,
+        );
+    }
+    const duration = wav.byteRate > 0 ? wav.dataBytes.length / wav.byteRate : 0;
+    const problems: string[] = [];
+    if (wav.audioFormat !== 1) problems.push(`audioFormat ${wav.audioFormat} (need PCM 1)`);
+    if (wav.sampleRate !== 24000) problems.push(`${wav.sampleRate} Hz (need 24000)`);
+    if (wav.numChannels !== 1) problems.push(`${wav.numChannels} ch (need 1)`);
+    if (wav.bitsPerSample !== 16) problems.push(`${wav.bitsPerSample}-bit (need 16)`);
+    if (role === 'source' && (duration < 10 || duration > 30)) {
+        problems.push(`duration ${duration.toFixed(2)}s (need 10–30s)`);
+    }
+    if (problems.length > 0) {
+        throw new Error(
+            `gemini-tts voice-create: ${role} is ${wav.sampleRate} Hz, ${wav.numChannels} ch, ${wav.bitsPerSample}-bit, ${duration.toFixed(2)}s — ${problems.join('; ')}`,
+        );
+    }
+}
+
+function pickVoiceId(json: Record<string, unknown>): string | undefined {
+    const candidates: unknown[] = [json.name, json.id, json.key];
+    const nested = json.voice;
+    if (nested && typeof nested === 'object') {
+        const voice = nested as Record<string, unknown>;
+        candidates.push(voice.name, voice.id, voice.key);
+    }
+    for (const candidate of candidates) {
+        if (typeof candidate === 'string' && VOICE_ID.test(candidate)) return candidate;
+    }
+    return undefined;
+}
+
+function redact(text: string, secret: string): string {
+    return secret ? text.split(secret).join('[redacted]') : text;
+}
+
+/** POST /v1beta/voices and return the stored or stateless voice id. Never includes the API key in errors. */
+export async function createReplicatedVoice(opts: CreateReplicatedVoiceOptions): Promise<{ id: string }> {
+    const env = opts.env ?? process.env;
+    const apiKey = env.GEMINI_API_KEY ?? '';
+    if (!apiKey) throw new Error('gemini-tts voice-create: GEMINI_API_KEY is required');
+    const baseUrl = (env.GOOGLE_BASE_URL || DEFAULT_BASE_URL).replace(/\/+$/, '');
+    const doFetch = opts.fetch ?? fetch;
+    let res: Response;
+    try {
+        res = await doFetch(`${baseUrl}/v1beta/voices`, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json', 'x-goog-api-key': apiKey },
+            body: JSON.stringify({
+                store: opts.store,
+                voice: {
+                    model: opts.model,
+                    type: 'replicated',
+                    display_name: opts.name,
+                    replicated: {
+                        source_audio: { mime_type: 'audio/wav', data: Buffer.from(opts.source).toString('base64') },
+                        consent_audio: { mime_type: 'audio/wav', data: Buffer.from(opts.consent).toString('base64') },
+                    },
+                },
+            }),
+            signal: AbortSignal.timeout(120_000),
+        });
+    } catch (err: unknown) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(`gemini-tts voice-create: request failed: ${redact(msg, apiKey)}`);
+    }
+    const raw = await res.text();
+    if (!res.ok) {
+        let apiMessage = raw.slice(0, 300);
+        try {
+            const parsed = JSON.parse(raw) as { error?: { message?: string } };
+            apiMessage = parsed.error?.message ?? apiMessage;
+        } catch {
+            // non-JSON error body — keep the raw excerpt
+        }
+        throw new Error(`gemini-tts voice-create: HTTP ${res.status}: ${redact(apiMessage, apiKey)}`);
+    }
+    let json: Record<string, unknown>;
+    try {
+        json = JSON.parse(raw) as Record<string, unknown>;
+    } catch {
+        throw new Error('gemini-tts voice-create: response was not JSON');
+    }
+    const id = pickVoiceId(json);
+    if (!id) {
+        throw new Error(`gemini-tts voice-create: response had no voice id (keys: ${Object.keys(json).join(', ')})`);
+    }
+    return { id };
+}
+
+async function readWav(path: string, role: 'source' | 'consent'): Promise<Uint8Array> {
+    const file = Bun.file(path);
+    if (!(await file.exists())) throw new Error(`gemini-tts voice-create: ${role} not found: ${path}`);
+    return new Uint8Array(await file.arrayBuffer());
+}
+
+export async function main(
+    argv: string[] = process.argv.slice(2),
+    deps: { env?: Record<string, string | undefined>; fetch?: typeof fetch } = {},
+): Promise<number> {
+    let values: { source?: string; consent?: string; name?: string; store?: string; model?: string };
+    try {
+        ({ values } = parseArgs({
+            args: argv,
+            options: {
+                source: { type: 'string' },
+                consent: { type: 'string' },
+                name: { type: 'string' },
+                store: { type: 'string' },
+                model: { type: 'string' },
+            },
+        }));
+    } catch (err: unknown) {
+        echoError(`gemini-tts voice-create: ${err instanceof Error ? err.message : String(err)}`);
+        return 1;
+    }
+    if (!values.source || !values.consent || !values.name) {
+        echoError('gemini-tts voice-create: missing required arguments: --source, --consent, and --name');
+        return 1;
+    }
+    const storeRaw = values.store ?? 'true';
+    if (storeRaw !== 'true' && storeRaw !== 'false') {
+        echoError(`gemini-tts voice-create: --store must be true or false (got "${storeRaw}")`);
+        return 1;
+    }
+    try {
+        const source = await readWav(values.source, 'source');
+        const consent = await readWav(values.consent, 'consent');
+        validateVoiceWav(source, 'source');
+        validateVoiceWav(consent, 'consent');
+        const env = deps.env ?? process.env;
+        const { id } = await createReplicatedVoice({
+            source,
+            consent,
+            name: values.name,
+            store: storeRaw === 'true',
+            model: values.model || env.GEMINI_TTS_MODEL || GEMINI_TTS_DEFAULT_MODEL,
+            env,
+            fetch: deps.fetch,
+        });
+        echo(id);
+        return 0;
+    } catch (err: unknown) {
+        echoError(err instanceof Error ? err.message : String(err));
+        return 1;
+    }
+}
+
+if (import.meta.main) {
+    process.exit(await main());
+}

package/plugins/generations/gemini-tts/tsconfig.json

@@ -0,0 +1,8 @@
+{
+    "extends": "../../../tooling/typescript/base.json",
+    "compilerOptions": {
+        "rootDir": "./src",
+        "outDir": "./dist"
+    },
+    "include": ["src/**/*"]
+}