@drax/ai-back 3.41.0 → 3.43.0

package/src/index.ts

@@ -2,35 +2,46 @@ import {OpenAiConfig} from "./config/OpenAiConfig.js";
 import {GoogleAiConfig} from "./config/GoogleAiConfig.js";
 import {OllamaAiConfig} from "./config/OllamaAiConfig.js";
 import {DeepSeekConfig} from "./config/DeepSeekConfig.js";
+import {ElevenLabsTTSConfig} from "./config/ElevenLabsTTSConfig.js";
 import {AILogSchema, AILogBaseSchema} from "./schemas/AILogSchema.js";
+import {TTSRequestSchema, TTSVoiceSettingsSchema} from "./schemas/TTSRequestSchema.js";
 import AILogModel from "./models/AILogModel.js";
 import AILogMongoRepository from "./repository/mongo/AILogMongoRepository.js";
 import AILogSqliteRepository from "./repository/sqlite/AILogSqliteRepository.js";
-import {OpenAiProviderFactory} from "./factory/OpenAiProviderFactory.js";
-import {GoogleAiProviderFactory} from "./factory/GoogleAiProviderFactory.js";
-import {OllamaAiProviderFactory} from "./factory/OllamaAiProviderFactory.js";
-import {DeepSeekProviderFactory} from "./factory/DeepSeekProviderFactory.js";
-import {AiProviderFactory} from "./factory/AiProviderFactory.js";
+import {OpenAiProviderFactory} from "./factory/ai/OpenAiProviderFactory.js";
+import {GoogleAiProviderFactory} from "./factory/ai/GoogleAiProviderFactory.js";
+import {OllamaAiProviderFactory} from "./factory/ai/OllamaAiProviderFactory.js";
+import {DeepSeekAiProviderFactory} from "./factory/ai/DeepSeekAiProviderFactory.js";
+import {AiProviderFactory} from "./factory/ai/AiProviderFactory.js";
+import {ElevenLabsTTSProviderFactory} from "./factory/tts/ElevenLabsTTSProviderFactory.js";
+import {TTSProviderFactory} from "./factory/tts/TTSProviderFactory.js";
+import type {TTSProviderInfo} from "./factory/tts/TTSProviderFactory.js";
 import {DraxAgentFactory} from "./factory/DraxAgentFactory.js";
 import AILogServiceFactory from "./factory/services/AILogServiceFactory.js";
-import {OpenAiProvider} from "./providers/OpenAiProvider.js";
-import {GoogleAiProvider} from "./providers/GoogleAiProvider.js";
-import {OllamaAiProvider} from "./providers/OllamaAiProvider.js";
-import {DeepSeekProvider} from "./providers/DeepSeekProvider.js";
+import {OpenAiProvider} from "./providers/ai/OpenAiProvider.js";
+import {GoogleAiProvider} from "./providers/ai/GoogleAiProvider.js";
+import {OllamaAiProvider} from "./providers/ai/OllamaAiProvider.js";
+import {DeepSeekAiProvider} from "./providers/ai/DeepSeekAiProvider.js";
+import {ElevenLabsTTSProvider} from "./providers/tts/ElevenLabsTTSProvider.js";
 import {BuilderTool} from "./tools/BuilderTool.js";
 import {KnowledgeService} from "./services/KnowledgeService.js";
 import {AILogService} from "./services/AILogService.js";
+import {TTSGenericService} from "./services/TTSGenericService.js";
+import {PromptAudioService} from "./services/PromptAudioService.js";
 import AILogPermissions from "./permissions/AILogPermissions.js";
 import AgentPermissions from "./permissions/AgentPermissions.js";
 import AgentSessionPermissions from "./permissions/AgentSessionPermissions.js";
 import AIPermissions from "./permissions/AIPermissions.js";
+import TTSPermissions from "./permissions/TTSPermissions.js";
 import AILogController from "./controllers/AILogController.js";
 import AICrudController from "./controllers/AICrudController.js";
 import AIGenericController from "./controllers/AIGenericController.js";
+import TTSGenericController from "./controllers/TTSGenericController.js";
 import DraxAgentController from "./controllers/DraxAgentController.js";
 import AgentSessionController from "./controllers/AgentSessionController.js";
 import AILogRoutes from "./routes/AILogRoutes.js";
 import AIRoutes from "./routes/AIRoutes.js";
+import TTSRoutes from "./routes/TTSRoutes.js";
 import DraxAgentRoutes from "./routes/DraxAgentRoutes.js";
 import AgentSessionRoutes from "./routes/AgentSessionRoutes.js";
 import {DraxAgent} from "./agents/DraxAgent.js";
@@ -40,6 +51,9 @@ import type {
     IPromptContentPart,
     IPromptContentPartImage,
     IPromptContentPartText,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptImage,
     IPromptImageDetail,
     IPromptMessage,
@@ -48,6 +62,19 @@ import type {
     IPromptResponse,
     IPromptTool
 } from "./interfaces/IAIProvider.js";
+import type {
+    ITTSParams,
+    ITTSProvider,
+    ITTSResponse,
+    ITTSVoiceSettings,
+} from "./interfaces/ITTSProvider.js";
+import type {
+    TTSRequest,
+    TTSVoiceSettings,
+} from "./schemas/TTSRequestSchema.js";
+import type {
+    TTSRequestContext,
+} from "./services/TTSGenericService.js";
 import type {
     ToolBuilderMethod,
     ToolBuilderOptions,
@@ -85,7 +112,18 @@ export type {
     IPromptContentPart,
     IPromptContentPartImage,
     IPromptContentPartText,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptResponse,
+    ITTSProvider,
+    ITTSParams,
+    ITTSResponse,
+    ITTSVoiceSettings,
+    TTSRequest,
+    TTSVoiceSettings,
+    TTSRequestContext,
+    TTSProviderInfo,
     ToolBuilderMethod,
     ToolBuilderOptions,
     ToolBuilderService,
@@ -108,40 +146,51 @@ export {
     GoogleAiConfig,
     OllamaAiConfig,
     DeepSeekConfig,
+    ElevenLabsTTSConfig,
     AILogSchema,
     AILogBaseSchema,
+    TTSRequestSchema,
+    TTSVoiceSettingsSchema,
     AILogModel,
     AILogMongoRepository,
     AILogSqliteRepository,
     OpenAiProviderFactory,
     GoogleAiProviderFactory,
     OllamaAiProviderFactory,
-    DeepSeekProviderFactory,
+    DeepSeekAiProviderFactory,
     AiProviderFactory,
+    ElevenLabsTTSProviderFactory,
+    TTSProviderFactory,
     DraxAgentFactory,
     AILogServiceFactory,
     OpenAiProvider,
     GoogleAiProvider,
     OllamaAiProvider,
-    DeepSeekProvider,
+    DeepSeekAiProvider,
+    ElevenLabsTTSProvider,
     BuilderTool,
     //Service
     KnowledgeService,
     AILogService,
+    TTSGenericService,
+    PromptAudioService,
     //Permissions
     AILogPermissions,
     AgentPermissions,
     AIPermissions,
+    TTSPermissions,
     AgentSessionPermissions,
     //Controllers
     AILogController,
     AICrudController,
     AIGenericController,
+    TTSGenericController,
     DraxAgentController,
     AgentSessionController,
     DraxAgent,
     AILogRoutes,
     AIRoutes,
+    TTSRoutes,
     DraxAgentRoutes,
     AgentSessionRoutes
 }

package/src/interfaces/IAIProvider.ts

@@ -1,4 +1,5 @@
 import { ZodSchema } from 'zod'
+import type {ITTSVoiceSettings} from "./ITTSProvider.js";
 
 type Role = 'user' | 'assistant' | 'system';
 
@@ -74,6 +75,38 @@ interface IPromptParams {
     userAgent?: string,
     tenant?: string | null,
     user?: string | null,
+    audioResponse?: boolean | IPromptAudioParams,
+}
+
+interface IPromptAudioParams {
+    enabled?: boolean,
+    provider?: string,
+    voiceId?: string,
+    model?: string,
+    outputFormat?: string,
+    voiceSettings?: ITTSVoiceSettings,
+    previousText?: string,
+    nextText?: string,
+    languageCode?: string,
+    seed?: number,
+    operationTitle?: string,
+    operationGroup?: string,
+}
+
+interface IPromptAudioResponseMeta {
+    provider: string,
+    model: string,
+    voiceId: string,
+    outputFormat?: string,
+    size: number,
+    time: number,
+}
+
+interface IPromptAudioResponse {
+    audio: string,
+    contentType: string,
+    encoding: 'base64',
+    meta: IPromptAudioResponseMeta,
 }
 
 interface IPromptResponse {
@@ -81,7 +114,8 @@ interface IPromptResponse {
     tokens: number,
     inputTokens: number,
     outputTokens: number,
-    time: number
+    time: number,
+    audio?: IPromptAudioResponse,
 }
 
 interface IAIProvider {
@@ -92,6 +126,9 @@ export type {
     IAIProvider,
     IPromptParams,
     IPromptResponse,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptMessage,
     IPromptMemory,
     IPromptTool,

package/src/interfaces/IDraxAgent.ts

@@ -1,5 +1,7 @@
 import type {
     IAIProvider,
+    IPromptAudioParams,
+    IPromptAudioResponse,
     IPromptContentPart,
     IPromptImage,
     IPromptMemory,
@@ -67,6 +69,7 @@ interface DraxAgentMessageInput extends DraxAgentSessionInput {
     toolMaxIterations?: number;
     operationTitle?: string;
     operationGroup?: string;
+    audioResponse?: boolean | IPromptAudioParams;
 }
 
 interface DraxAgentMessageOutput {
@@ -79,6 +82,7 @@ interface DraxAgentMessageOutput {
     inputTokens: number;
     outputTokens: number;
     time: number;
+    audio?: IPromptAudioResponse;
 }
 
 interface DraxAgentSession {

package/src/interfaces/ITTSProvider.ts

@@ -0,0 +1,47 @@
+interface ITTSVoiceSettings {
+    stability?: number;
+    similarityBoost?: number;
+    style?: number;
+    useSpeakerBoost?: boolean;
+    speed?: number;
+}
+
+interface ITTSParams {
+    text: string;
+    voiceId?: string;
+    model?: string;
+    outputFormat?: string;
+    voiceSettings?: ITTSVoiceSettings;
+    previousText?: string;
+    nextText?: string;
+    languageCode?: string;
+    seed?: number;
+    operationTitle?: string;
+    operationGroup?: string;
+    ip?: string;
+    userAgent?: string;
+    tenant?: string | null;
+    user?: string | null;
+}
+
+interface ITTSResponse {
+    audio: Buffer;
+    contentType: string;
+    size: number;
+    time: number;
+    provider: string;
+    model: string;
+    voiceId: string;
+    outputFormat?: string;
+}
+
+interface ITTSProvider {
+    textToSpeech(input: ITTSParams): Promise<ITTSResponse>
+}
+
+export type {
+    ITTSProvider,
+    ITTSParams,
+    ITTSResponse,
+    ITTSVoiceSettings,
+}

package/src/permissions/AIPermissions.ts

@@ -8,4 +8,3 @@ enum AIPermissions {
 
 export { AIPermissions };
 export default AIPermissions;
-

package/src/permissions/TTSPermissions.ts

@@ -0,0 +1,8 @@
+enum TTSPermissions {
+
+    TextToSpeech = "tts:textToSpeech",
+
+}
+
+export { TTSPermissions };
+export default TTSPermissions;

package/src/providers/{DeepSeekProvider.ts → ai/DeepSeekAiProvider.ts}

@@ -1,10 +1,10 @@
 import OpenAI from "openai";
 import type {IAILogBase} from "@drax/ai-share";
-import type {IPromptParams} from "../interfaces/IAIProvider.js";
-import type {AILogService} from "../services/AILogService.js";
+import type {IPromptParams} from "../../interfaces/IAIProvider.js";
+import type {AILogService} from "../../services/AILogService.js";
 import OpenAiProvider from "./OpenAiProvider.js";
 
-class DeepSeekProvider extends OpenAiProvider{
+class DeepSeekAiProvider extends OpenAiProvider{
     protected _baseUrl: string
 
     constructor(apiKey: string, model: string, baseUrl: string = "https://api.deepseek.com", visionModel?: string, aiLogService?: AILogService) {
@@ -54,5 +54,5 @@ class DeepSeekProvider extends OpenAiProvider{
     }
 }
 
-export default DeepSeekProvider
-export {DeepSeekProvider}
+export default DeepSeekAiProvider
+export {DeepSeekAiProvider}

package/src/providers/{GoogleAiProvider.ts → ai/GoogleAiProvider.ts}

@@ -14,9 +14,10 @@ import type {
     IPromptParams,
     IPromptResponse,
     IPromptTool
-} from "../interfaces/IAIProvider";
-import type {AILogService} from "../services/AILogService";
+} from "../../interfaces/IAIProvider.js";
+import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 class GoogleAiProvider implements IAIProvider{
     protected _apiKey: string
@@ -446,6 +447,7 @@ class GoogleAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -464,7 +466,8 @@ class GoogleAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/providers/{OllamaAiProvider.ts → ai/OllamaAiProvider.ts}

@@ -6,9 +6,10 @@ import type {
     IPromptParams,
     IPromptResponse,
     IPromptTool
-} from "../interfaces/IAIProvider";
-import type {AILogService} from "../services/AILogService";
+} from "../../interfaces/IAIProvider.js";
+import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 type OllamaMessage = {
     role: "system" | "user" | "assistant" | "tool",
@@ -426,6 +427,7 @@ class OllamaAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -444,7 +446,8 @@ class OllamaAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/providers/{OpenAiProvider.ts → ai/OpenAiProvider.ts}

@@ -7,9 +7,10 @@ import type {
     IPromptParams,
     IPromptResponse,
     IPromptTool
-} from "../interfaces/IAIProvider";
-import type {AILogService} from "../services/AILogService";
+} from "../../interfaces/IAIProvider.js";
+import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 class OpenAiProvider implements IAIProvider{
     protected _apiKey: string
@@ -358,6 +359,7 @@ class OpenAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -376,7 +378,8 @@ class OpenAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/providers/tts/ElevenLabsTTSProvider.ts

@@ -0,0 +1,132 @@
+import type {ITTSParams, ITTSProvider, ITTSResponse} from "../../interfaces/ITTSProvider.js";
+
+class ElevenLabsTTSProvider implements ITTSProvider {
+    protected _apiKey: string
+    protected _baseUrl: string
+    protected _model: string
+    protected _voiceId: string
+    protected _outputFormat?: string
+
+    constructor(apiKey: string, model: string, voiceId: string, baseUrl: string = "https://api.elevenlabs.io", outputFormat?: string) {
+        if (!apiKey) {
+            throw new Error("ElevenLabs apiKey required")
+        }
+        if (!model) {
+            throw new Error("ElevenLabs model required")
+        }
+        if (!voiceId) {
+            throw new Error("ElevenLabs voiceId required")
+        }
+
+        this._apiKey = apiKey
+        this._model = model
+        this._voiceId = voiceId
+        this._baseUrl = baseUrl.replace(/\/+$/, "")
+        this._outputFormat = outputFormat
+    }
+
+    get model() {
+        if (!this._model) {
+            throw new Error("ElevenLabs model not found")
+        }
+        return this._model
+    }
+
+    get voiceId() {
+        if (!this._voiceId) {
+            throw new Error("ElevenLabs voiceId not found")
+        }
+        return this._voiceId
+    }
+
+    protected mapContentType(outputFormat?: string) {
+        if (!outputFormat) {
+            return "audio/mpeg"
+        }
+
+        if (outputFormat.startsWith("mp3")) {
+            return "audio/mpeg"
+        }
+        if (outputFormat.startsWith("opus")) {
+            return "audio/ogg"
+        }
+        if (outputFormat.startsWith("pcm")) {
+            return "audio/wav"
+        }
+        if (outputFormat.startsWith("ulaw") || outputFormat.startsWith("alaw")) {
+            return "audio/basic"
+        }
+
+        return "application/octet-stream"
+    }
+
+    protected mapVoiceSettings(voiceSettings: ITTSParams["voiceSettings"]) {
+        if (!voiceSettings) {
+            return undefined
+        }
+
+        return {
+            stability: voiceSettings.stability,
+            similarity_boost: voiceSettings.similarityBoost,
+            style: voiceSettings.style,
+            use_speaker_boost: voiceSettings.useSpeakerBoost,
+            speed: voiceSettings.speed,
+        }
+    }
+
+    protected buildBody(input: ITTSParams, model: string) {
+        return {
+            text: input.text,
+            model_id: model,
+            ...(input.voiceSettings ? {voice_settings: this.mapVoiceSettings(input.voiceSettings)} : {}),
+            ...(input.previousText ? {previous_text: input.previousText} : {}),
+            ...(input.nextText ? {next_text: input.nextText} : {}),
+            ...(input.languageCode ? {language_code: input.languageCode} : {}),
+            ...(input.seed !== undefined ? {seed: input.seed} : {}),
+        }
+    }
+
+    async textToSpeech(input: ITTSParams): Promise<ITTSResponse> {
+        const startedAt = Date.now()
+        const model = input.model ?? this.model
+        const voiceId = input.voiceId ?? this.voiceId
+        const outputFormat = input.outputFormat ?? this._outputFormat
+        const url = new URL(`${this._baseUrl}/v1/text-to-speech/${encodeURIComponent(voiceId)}`)
+
+        if (outputFormat) {
+            url.searchParams.set("output_format", outputFormat)
+        }
+
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Accept": this.mapContentType(outputFormat),
+                "Content-Type": "application/json",
+                "xi-api-key": this._apiKey,
+            },
+            body: JSON.stringify(this.buildBody(input, model)),
+        })
+
+        if (!response.ok) {
+            const errorText = await response.text()
+            throw new Error(`ElevenLabs TTS request failed (${response.status}): ${errorText}`)
+        }
+
+        const audio = Buffer.from(await response.arrayBuffer())
+        const contentType = response.headers.get("content-type") ?? this.mapContentType(outputFormat)
+
+        return {
+            audio,
+            contentType,
+            size: audio.byteLength,
+            time: Date.now() - startedAt,
+            provider: "elevenlabs",
+            model,
+            voiceId,
+            outputFormat,
+        }
+    }
+}
+
+export default ElevenLabsTTSProvider;
+export {ElevenLabsTTSProvider}

package/src/routes/TTSRoutes.ts

@@ -0,0 +1,13 @@
+import TTSGenericController from "../controllers/TTSGenericController.js";
+
+async function TTSFastifyRoutes(fastify, options) {
+
+    const genericController: TTSGenericController = new TTSGenericController()
+
+    fastify.get('/api/tts/providers', (req,rep) => genericController.availableProviders(req,rep))
+    fastify.post('/api/tts', (req,rep) => genericController.textToSpeech(req,rep))
+
+}
+
+export default TTSFastifyRoutes;
+export {TTSFastifyRoutes}

package/src/schemas/TTSRequestSchema.ts

@@ -0,0 +1,38 @@
+import {z} from "zod";
+
+const TTSVoiceSettingsSchema = z.object({
+    stability: z.number().min(0).max(1).optional(),
+    similarityBoost: z.number().min(0).max(1).optional(),
+    style: z.number().min(0).max(1).optional(),
+    useSpeakerBoost: z.boolean().optional(),
+    speed: z.number().positive().optional(),
+})
+
+const TTSRequestSchema = z.object({
+    text: z.string().min(1),
+    provider: z.string().default("ElevenLabs"),
+    voiceId: z.string().optional(),
+    model: z.string().optional(),
+    outputFormat: z.string().optional(),
+    voiceSettings: TTSVoiceSettingsSchema.optional(),
+    previousText: z.string().optional(),
+    nextText: z.string().optional(),
+    languageCode: z.string().optional(),
+    seed: z.number().int().optional(),
+    responseFormat: z.enum(["audio", "base64"]).default("audio"),
+    operationTitle: z.string().optional(),
+    operationGroup: z.string().optional(),
+})
+
+type TTSRequest = z.infer<typeof TTSRequestSchema>
+type TTSVoiceSettings = z.infer<typeof TTSVoiceSettingsSchema>
+
+export {
+    TTSRequestSchema,
+    TTSVoiceSettingsSchema,
+}
+
+export type {
+    TTSRequest,
+    TTSVoiceSettings,
+}