@kognitivedev/backend-cloud 0.2.29

package/src/cloud-voice-runtime-service.ts

@@ -0,0 +1,465 @@
+import type {
+    CloudVoiceAgentConfig,
+    CloudVoiceChannel,
+    CloudVoiceConversationProfile,
+    CloudVoiceHumanizationConfig,
+    CloudVoicePipelineConfig,
+    CloudVoiceProvider,
+    CloudVoiceRuntimeCapabilities,
+    CloudVoiceSpeechConfig,
+} from "./types";
+import { getRecord, getString } from "./utils";
+
+export const DEFAULT_CLOUD_VOICE_HUMANIZATION: Required<CloudVoiceHumanizationConfig> = {
+    enabled: true,
+    openingMode: "auto",
+    openingStyle: "brief",
+    fillerStyle: "light",
+    backchannelFrequency: "low",
+    disfluency: "rare",
+    toolLatencyFillerMs: 700,
+    conversationProfile: {
+        personality: "warm",
+        tone: "professional",
+        pacing: "concise",
+        unclearAudio: "ask_repeat",
+        confirmation: "critical_fields",
+        escalation: "when_blocked",
+        numberReadback: true,
+    },
+};
+
+export const CLOUD_VOICE_CONFIG_VERSION = 2 as const;
+export const CLOUD_VOICE_PROMPT_COMPILER_VERSION = "cloud-voice-provider-native-v2";
+
+export const CLOUD_VOICE_PROVIDER_CAPABILITIES: Record<CloudVoiceProvider, CloudVoiceRuntimeCapabilities> = {
+    "openai-realtime": {
+        canUpdateInstructionsLive: true,
+        supportsSessionResume: false,
+        supportsToolCalling: true,
+        supportsOutputAudioTranscripts: true,
+        supportsServerVadConfig: true,
+        supportsSemanticVad: true,
+    },
+    "gemini-live": {
+        canUpdateInstructionsLive: false,
+        supportsSessionResume: true,
+        supportsToolCalling: true,
+        supportsOutputAudioTranscripts: true,
+        supportsServerVadConfig: true,
+        supportsNativeAudioOptions: true,
+        supportsAffectiveDialog: true,
+        supportsProactiveAudio: true,
+    },
+    "kognitive-voice": {
+        canUpdateInstructionsLive: true,
+        supportsSessionResume: true,
+        supportsToolCalling: true,
+        supportsOutputAudioTranscripts: true,
+        supportsServerVadConfig: false,
+        supportsPipelineEotConfig: true,
+        supportsCartesiaTtsControls: true,
+    },
+    "xai-realtime": {
+        canUpdateInstructionsLive: true,
+        supportsSessionResume: false,
+        supportsToolCalling: true,
+        supportsOutputAudioTranscripts: true,
+        supportsServerVadConfig: true,
+    },
+};
+
+const DEFAULT_KOGNITIVE_PIPELINE: CloudVoicePipelineConfig = {
+    transport: { type: "websocket", provider: "kognitive-websocket" },
+    stt: { provider: "deepgram", model: "nova-3", language: "en" },
+    llm: { provider: "openai", model: "gpt-4o-mini" },
+    tts: { provider: "cartesia", model: "sonic-3", voice: "a0e99841-438c-4a64-b679-ae501e7d6091" },
+    turn: {
+        interruptResponse: true,
+        createResponse: true,
+        prefixPaddingMs: 300,
+        silenceDurationMs: 650,
+    },
+};
+
+const SPEECH_LANGUAGE_INSTRUCTIONS: Record<string, string> = {
+    en: "English",
+    "en-US": "English",
+    "en-GB": "English",
+    tr: "Turkish",
+    "tr-TR": "Turkish",
+    de: "German",
+    "de-DE": "German",
+    fr: "French",
+    "fr-FR": "French",
+    es: "Spanish",
+    "es-ES": "Spanish",
+    it: "Italian",
+    "it-IT": "Italian",
+    pt: "Portuguese",
+    "pt-PT": "Portuguese",
+    "pt-BR": "Portuguese",
+    nl: "Dutch",
+    "nl-NL": "Dutch",
+    sv: "Swedish",
+    "sv-SE": "Swedish",
+    no: "Norwegian",
+    "nb-NO": "Norwegian",
+    da: "Danish",
+    "da-DK": "Danish",
+    fi: "Finnish",
+    "fi-FI": "Finnish",
+    pl: "Polish",
+    "pl-PL": "Polish",
+    cs: "Czech",
+    "cs-CZ": "Czech",
+    ro: "Romanian",
+    "ro-RO": "Romanian",
+    el: "Greek",
+    "el-GR": "Greek",
+    ru: "Russian",
+    "ru-RU": "Russian",
+    uk: "Ukrainian",
+    "uk-UA": "Ukrainian",
+};
+
+const SPEECH_LANGUAGE_CODES: Record<string, string> = {
+    en: "en-US",
+    tr: "tr-TR",
+    de: "de-DE",
+    fr: "fr-FR",
+    es: "es-ES",
+    it: "it-IT",
+    pt: "pt-PT",
+    nl: "nl-NL",
+    sv: "sv-SE",
+    no: "nb-NO",
+    da: "da-DK",
+    fi: "fi-FI",
+    pl: "pl-PL",
+    cs: "cs-CZ",
+    ro: "ro-RO",
+    el: "el-GR",
+    ru: "ru-RU",
+    uk: "uk-UA",
+};
+
+const SPEECH_ACCENT_LANGUAGE_CODES: Record<string, string> = {
+    "neutral english": "en-US",
+    "british english": "en-GB",
+    "irish english": "en-IE",
+    "scottish english": "en-GB",
+    "standard istanbul turkish": "tr-TR",
+    "aegean turkish": "tr-TR",
+    "anatolian turkish": "tr-TR",
+    "standard german": "de-DE",
+    "austrian german": "de-AT",
+    "swiss german influenced standard german": "de-CH",
+    "standard french from france": "fr-FR",
+    "belgian french": "fr-BE",
+    "swiss french": "fr-CH",
+    "castilian spanish": "es-ES",
+    "andalusian spanish": "es-ES",
+    "canarian spanish": "es-ES",
+};
+
+function isPhoneChannel(channel?: CloudVoiceChannel) {
+    return channel === "phone" || channel === "sip" || channel === "outbound";
+}
+
+export function normalizeCloudVoiceHumanizationConfig(value: unknown): Required<CloudVoiceHumanizationConfig> {
+    const record = getRecord(value);
+    const openingMode = record.openingMode === "wait" || record.openingMode === "auto"
+        ? record.openingMode
+        : DEFAULT_CLOUD_VOICE_HUMANIZATION.openingMode;
+    const openingStyle = record.openingStyle === "warm" || record.openingStyle === "professional" || record.openingStyle === "brief"
+        ? record.openingStyle
+        : DEFAULT_CLOUD_VOICE_HUMANIZATION.openingStyle;
+    const fillerStyle = record.fillerStyle === "off" || record.fillerStyle === "natural" || record.fillerStyle === "light"
+        ? record.fillerStyle
+        : DEFAULT_CLOUD_VOICE_HUMANIZATION.fillerStyle;
+    const backchannelFrequency = record.backchannelFrequency === "off" || record.backchannelFrequency === "medium" || record.backchannelFrequency === "low"
+        ? record.backchannelFrequency
+        : DEFAULT_CLOUD_VOICE_HUMANIZATION.backchannelFrequency;
+    const disfluency = record.disfluency === "off" || record.disfluency === "rare"
+        ? record.disfluency
+        : DEFAULT_CLOUD_VOICE_HUMANIZATION.disfluency;
+    const toolLatencyFillerMs = typeof record.toolLatencyFillerMs === "number" && Number.isFinite(record.toolLatencyFillerMs)
+        ? Math.max(0, Math.round(record.toolLatencyFillerMs))
+        : DEFAULT_CLOUD_VOICE_HUMANIZATION.toolLatencyFillerMs;
+    return {
+        enabled: typeof record.enabled === "boolean" ? record.enabled : DEFAULT_CLOUD_VOICE_HUMANIZATION.enabled,
+        openingMode,
+        openingStyle,
+        fillerStyle,
+        backchannelFrequency,
+        disfluency,
+        toolLatencyFillerMs,
+        conversationProfile: normalizeCloudVoiceConversationProfile(record.conversationProfile ?? record.profile, record),
+    };
+}
+
+export function normalizeCloudVoiceConversationProfile(value: unknown, legacy: Record<string, unknown> = {}): CloudVoiceConversationProfile {
+    const record = getRecord(value);
+    const openingStyle = getString(legacy.openingStyle, "");
+    const fillerStyle = getString(legacy.fillerStyle, "");
+    const backchannelFrequency = getString(legacy.backchannelFrequency, "");
+    const defaults = DEFAULT_CLOUD_VOICE_HUMANIZATION.conversationProfile;
+    const personality: CloudVoiceConversationProfile["personality"] = record.personality === "neutral" || record.personality === "warm" || record.personality === "expert" || record.personality === "concierge"
+        ? record.personality
+        : openingStyle === "professional"
+            ? "expert"
+            : "warm";
+    return {
+        personality,
+        tone: record.tone === "casual" || record.tone === "professional" || record.tone === "empathetic" || record.tone === "polished"
+            ? record.tone
+            : openingStyle === "warm"
+                ? "empathetic"
+                : defaults.tone,
+        pacing: record.pacing === "concise" || record.pacing === "measured" || record.pacing === "deliberate" || record.pacing === "energetic"
+            ? record.pacing
+            : fillerStyle === "natural" || backchannelFrequency === "medium"
+                ? "measured"
+                : defaults.pacing,
+        unclearAudio: record.unclearAudio === "ask_repeat" || record.unclearAudio === "confirm_best_guess"
+            ? record.unclearAudio
+            : defaults.unclearAudio,
+        confirmation: record.confirmation === "critical_fields" || record.confirmation === "all_actions" || record.confirmation === "minimal"
+            ? record.confirmation
+            : defaults.confirmation,
+        escalation: record.escalation === "when_blocked" || record.escalation === "on_request" || record.escalation === "never"
+            ? record.escalation
+            : defaults.escalation,
+        numberReadback: typeof record.numberReadback === "boolean" ? record.numberReadback : defaults.numberReadback,
+    };
+}
+
+export function getCloudVoiceProviderCapabilities(provider: CloudVoiceProvider): CloudVoiceRuntimeCapabilities {
+    return CLOUD_VOICE_PROVIDER_CAPABILITIES[provider] ?? CLOUD_VOICE_PROVIDER_CAPABILITIES["openai-realtime"];
+}
+
+export function normalizeSpeechConfig(config: Pick<CloudVoiceAgentConfig, "metadata">): CloudVoiceSpeechConfig | undefined {
+    const speech = getRecord(getRecord(config.metadata).speech);
+    const normalized: CloudVoiceSpeechConfig = {};
+    const knownKeys = ["language", "accent", "style", "pace", "emotion"] as const;
+    for (const key of knownKeys) {
+        const value = speech[key];
+        if (typeof value === "string" && value.trim()) normalized[key] = value.trim();
+    }
+    for (const [key, value] of Object.entries(speech)) {
+        if ((knownKeys as readonly string[]).includes(key) || value === undefined) continue;
+        normalized[key] = value;
+    }
+    return Object.keys(normalized).length > 0 ? normalized : undefined;
+}
+
+export function toLanguageInstruction(value: string | undefined) {
+    return value ? SPEECH_LANGUAGE_INSTRUCTIONS[value] ?? SPEECH_LANGUAGE_INSTRUCTIONS[value.toLowerCase()] ?? value : undefined;
+}
+
+export function resolveCloudVoiceSpeechLanguageCode(config: Pick<CloudVoiceAgentConfig, "metadata">) {
+    const speech = normalizeSpeechConfig(config);
+    const accentCode = speech?.accent ? SPEECH_ACCENT_LANGUAGE_CODES[speech.accent.toLowerCase()] : undefined;
+    if (accentCode) return accentCode;
+    const language = speech?.language;
+    if (!language) return undefined;
+    if (/^[a-z]{2,3}-[A-Z]{2}$/i.test(language)) return language;
+    return SPEECH_LANGUAGE_CODES[language] ?? SPEECH_LANGUAGE_CODES[language.toLowerCase()] ?? language;
+}
+
+function resolveCloudVoiceTranscriptionLanguage(config: Pick<CloudVoiceAgentConfig, "metadata">) {
+    const speech = normalizeSpeechConfig(config);
+    const language = speech?.language;
+    if (!language) return undefined;
+    return language.includes("-") ? language.split("-")[0] : language;
+}
+
+function normalizePipelineTransport(config: CloudVoiceAgentConfig): CloudVoicePipelineConfig["transport"] {
+    const transport = getRecord(config.pipeline?.transport);
+    const type = transport.type === "webrtc" || transport.type === "websocket"
+        ? transport.type
+        : config.transport;
+    const provider = transport.provider === "daily" || transport.provider === "kognitive-websocket"
+        ? transport.provider
+        : type === "webrtc"
+            ? "daily"
+            : "kognitive-websocket";
+    return { type, provider };
+}
+
+export function normalizeCloudVoicePipelineConfig(config: CloudVoiceAgentConfig): CloudVoicePipelineConfig {
+    const pipeline = getRecord(config.pipeline);
+    const stt = getRecord(pipeline.stt);
+    const llm = getRecord(pipeline.llm);
+    const tts = getRecord(pipeline.tts);
+    const turn = getRecord(pipeline.turn);
+    const backgroundAudio = getRecord(pipeline.backgroundAudio);
+    const sttLanguage = typeof stt.language === "string" && stt.language.trim()
+        ? stt.language.trim()
+        : resolveCloudVoiceTranscriptionLanguage(config);
+
+    return {
+        transport: normalizePipelineTransport(config),
+        stt: {
+            provider: getString(stt.provider, DEFAULT_KOGNITIVE_PIPELINE.stt.provider),
+            model: getString(stt.model, DEFAULT_KOGNITIVE_PIPELINE.stt.model),
+            ...(sttLanguage ? { language: sttLanguage } : {}),
+        },
+        llm: {
+            provider: getString(llm.provider, DEFAULT_KOGNITIVE_PIPELINE.llm.provider),
+            model: getString(llm.model, config.model || DEFAULT_KOGNITIVE_PIPELINE.llm.model),
+        },
+        tts: {
+            provider: getString(tts.provider, DEFAULT_KOGNITIVE_PIPELINE.tts.provider),
+            model: getString(tts.model, DEFAULT_KOGNITIVE_PIPELINE.tts.model),
+            voice: getString(tts.voice, config.voice && config.voice !== "alloy" ? config.voice : DEFAULT_KOGNITIVE_PIPELINE.tts.voice),
+            ...(Object.keys(getRecord(tts.options)).length > 0 ? { options: getRecord(tts.options) } : {}),
+        },
+        ...(Object.keys(turn).length > 0
+            ? { turn: { ...DEFAULT_KOGNITIVE_PIPELINE.turn, ...turn } }
+            : { turn: DEFAULT_KOGNITIVE_PIPELINE.turn }),
+        ...(Object.keys(backgroundAudio).length > 0 ? { backgroundAudio } : {}),
+    };
+}
+
+export function toPreparedTranscription(config: Pick<CloudVoiceAgentConfig, "metadata" | "transcription">) {
+    if (config.transcription === null) return null;
+    const transcription = getRecord(config.transcription);
+    const language = resolveCloudVoiceTranscriptionLanguage(config);
+    return language
+        ? { ...transcription, language }
+        : transcription;
+}
+
+export function toOpenAITurnDetection(value: unknown) {
+    if (value === null) return null;
+    const turnDetection = getRecord(value);
+    const type = typeof turnDetection.type === "string" ? turnDetection.type : "";
+    if (!type) return undefined;
+
+    const config: Record<string, unknown> = { type };
+    if (typeof turnDetection.create_response === "boolean") {
+        config.create_response = turnDetection.create_response;
+    } else if (typeof turnDetection.createResponse === "boolean") {
+        config.create_response = turnDetection.createResponse;
+    }
+    if (typeof turnDetection.interrupt_response === "boolean") {
+        config.interrupt_response = turnDetection.interrupt_response;
+    } else if (typeof turnDetection.interruptResponse === "boolean") {
+        config.interrupt_response = turnDetection.interruptResponse;
+    }
+    if (typeof turnDetection.prefix_padding_ms === "number") {
+        config.prefix_padding_ms = turnDetection.prefix_padding_ms;
+    } else if (typeof turnDetection.prefixPaddingMs === "number") {
+        config.prefix_padding_ms = turnDetection.prefixPaddingMs;
+    }
+    if (typeof turnDetection.silence_duration_ms === "number") {
+        config.silence_duration_ms = turnDetection.silence_duration_ms;
+    } else if (typeof turnDetection.silenceDurationMs === "number") {
+        config.silence_duration_ms = turnDetection.silenceDurationMs;
+    }
+    if (typeof turnDetection.threshold === "number") {
+        config.threshold = turnDetection.threshold;
+    }
+    if (typeof turnDetection.eagerness === "string") {
+        config.eagerness = turnDetection.eagerness;
+    }
+    return config;
+}
+
+export function normalizeCloudVoiceTurnDetection(
+    provider: CloudVoiceProvider,
+    channel: CloudVoiceChannel,
+    value: unknown,
+) {
+    if (value === null) return null;
+    const record = getRecord(value);
+    const isPhone = isPhoneChannel(channel);
+    if (provider === "openai-realtime" && !isPhone) {
+        const type = getString(record.type, "semantic_vad") === "off" ? "off" : getString(record.type, "semantic_vad");
+        if (type === "off") return null;
+        if (type === "semantic_vad") {
+            return {
+                type: "semantic_vad",
+                createResponse: typeof record.createResponse === "boolean" ? record.createResponse : true,
+                interruptResponse: typeof record.interruptResponse === "boolean" ? record.interruptResponse : true,
+                eagerness: getString(record.eagerness, "low") || "low",
+            };
+        }
+    }
+    if (provider === "gemini-live") {
+        return {
+            type: "server_vad",
+            createResponse: typeof record.createResponse === "boolean" ? record.createResponse : true,
+            interruptResponse: typeof record.interruptResponse === "boolean" ? record.interruptResponse : true,
+            prefixPaddingMs: typeof record.prefixPaddingMs === "number" ? record.prefixPaddingMs : 120,
+            silenceDurationMs: typeof record.silenceDurationMs === "number" ? record.silenceDurationMs : 700,
+            startOfSpeechSensitivity: getString(record.startOfSpeechSensitivity, "START_SENSITIVITY_LOW"),
+            endOfSpeechSensitivity: getString(record.endOfSpeechSensitivity, "END_SENSITIVITY_LOW"),
+            disabled: typeof record.disabled === "boolean" ? record.disabled : false,
+        };
+    }
+    if (provider === "kognitive-voice") {
+        return {
+            type: "flux_eot",
+            eager_eot_threshold: typeof record.eager_eot_threshold === "number" ? record.eager_eot_threshold : 0.85,
+            eot_threshold: typeof record.eot_threshold === "number" ? record.eot_threshold : 0.65,
+            eot_timeout_ms: typeof record.eot_timeout_ms === "number" ? record.eot_timeout_ms : 900,
+        };
+    }
+    return {
+        type: getString(record.type, "server_vad") || "server_vad",
+        createResponse: typeof record.createResponse === "boolean" ? record.createResponse : true,
+        interruptResponse: typeof record.interruptResponse === "boolean" ? record.interruptResponse : true,
+        threshold: typeof record.threshold === "number" ? record.threshold : 0.6,
+        prefixPaddingMs: typeof record.prefixPaddingMs === "number" ? record.prefixPaddingMs : 300,
+        silenceDurationMs: typeof record.silenceDurationMs === "number" ? record.silenceDurationMs : 700,
+    };
+}
+
+export function normalizeCloudVoiceInputNoiseReduction(
+    provider: CloudVoiceProvider,
+    channel: CloudVoiceChannel,
+    value: unknown,
+) {
+    if (value === null) return null;
+    const record = getRecord(value);
+    if (Object.keys(record).length > 0) return record;
+
+    if (provider === "openai-realtime" && isPhoneChannel(channel)) {
+        return { type: "near_field" };
+    }
+
+    return null;
+}
+
+export function compileCartesiaTtsOptions(config: Pick<CloudVoiceAgentConfig, "voice" | "metadata" | "providerOptions" | "pipeline">) {
+    const providerOptions = getRecord(config.providerOptions);
+    const pipeline = getRecord(config.pipeline ?? providerOptions.pipeline);
+    const tts = getRecord(pipeline.tts);
+    const cartesia = getRecord(tts.options ?? tts.providerOptions ?? providerOptions.cartesia);
+    const speech = normalizeSpeechConfig(config);
+    const options: Record<string, unknown> = {
+        modelId: getString(tts.model, getString(cartesia.modelId, "sonic-3")),
+        voice: getString(tts.voice, config.voice && config.voice !== "alloy" ? config.voice : DEFAULT_KOGNITIVE_PIPELINE.tts.voice),
+    };
+    const pace = getString(speech?.pace, "").toLowerCase();
+    const speed = typeof cartesia.speed === "number" || cartesia.speed === "slow" || cartesia.speed === "normal" || cartesia.speed === "fast"
+        ? cartesia.speed
+        : pace.includes("slow")
+            ? "slow"
+            : pace.includes("fast") || pace.includes("energetic")
+                ? "fast"
+                : "normal";
+    options.speed = speed;
+    const emotion = getString(speech?.emotion, "");
+    if (emotion) options.emotion = [emotion];
+    if (typeof cartesia.pronunciationDictId === "string" && cartesia.pronunciationDictId.trim()) {
+        options.pronunciationDictId = cartesia.pronunciationDictId.trim();
+    }
+    if (cartesia.contextMode === "reset" || cartesia.contextMode === "continue") {
+        options.contextMode = cartesia.contextMode;
+    }
+    return options;
+}