@drax/ai-back 3.42.0 → 3.43.0

package/dist/agents/DraxAgent.js

@@ -78,6 +78,7 @@ class DraxAgent {
             userAgent: input.userAgent,
             tenant: input.tenantId ?? session.tenantId ?? null,
             user: input.userId ?? session.userId ?? null,
+            audioResponse: input.audioResponse,
         });
         const assistantMessage = this.normalizeOutput(response.output);
         const now = new Date();
@@ -100,6 +101,7 @@ class DraxAgent {
             inputTokens: response.inputTokens,
             outputTokens: response.outputTokens,
             time: response.time,
+            ...(response.audio ? { audio: response.audio } : {}),
         };
     }
     async createSession(input = {}) {

package/dist/controllers/AIGenericController.js

@@ -35,6 +35,27 @@ const PromptInputFileSchema = z.object({
     mimetype: z.string().optional(),
     url: z.string().optional(),
 });
+const PromptAudioVoiceSettingsSchema = z.object({
+    stability: z.number().min(0).max(1).optional(),
+    similarityBoost: z.number().min(0).max(1).optional(),
+    style: z.number().min(0).max(1).optional(),
+    useSpeakerBoost: z.boolean().optional(),
+    speed: z.number().positive().optional(),
+});
+const PromptAudioParamsSchema = z.object({
+    enabled: z.boolean().optional(),
+    provider: z.string().optional(),
+    voiceId: z.string().optional(),
+    model: z.string().optional(),
+    outputFormat: z.string().optional(),
+    voiceSettings: PromptAudioVoiceSettingsSchema.optional(),
+    previousText: z.string().optional(),
+    nextText: z.string().optional(),
+    languageCode: z.string().optional(),
+    seed: z.number().int().optional(),
+    operationTitle: z.string().optional(),
+    operationGroup: z.string().optional(),
+});
 const GenericPromptRequestSchema = z.object({
     systemPrompt: z.string().min(1),
     userInput: z.string().optional(),
@@ -50,6 +71,7 @@ const GenericPromptRequestSchema = z.object({
     model: z.string().optional(),
     operationTitle: z.string().optional(),
     operationGroup: z.string().optional(),
+    audioResponse: z.union([z.boolean(), PromptAudioParamsSchema]).optional(),
 });
 class AIGenericController extends CommonController {
     async prompt(request, reply) {

package/dist/controllers/DraxAgentController.js

@@ -28,6 +28,27 @@ const PromptInputFileSchema = z.object({
     mimetype: z.string().optional(),
     url: z.string().optional(),
 });
+const PromptAudioVoiceSettingsSchema = z.object({
+    stability: z.number().min(0).max(1).optional(),
+    similarityBoost: z.number().min(0).max(1).optional(),
+    style: z.number().min(0).max(1).optional(),
+    useSpeakerBoost: z.boolean().optional(),
+    speed: z.number().positive().optional(),
+});
+const PromptAudioParamsSchema = z.object({
+    enabled: z.boolean().optional(),
+    provider: z.string().optional(),
+    voiceId: z.string().optional(),
+    model: z.string().optional(),
+    outputFormat: z.string().optional(),
+    voiceSettings: PromptAudioVoiceSettingsSchema.optional(),
+    previousText: z.string().optional(),
+    nextText: z.string().optional(),
+    languageCode: z.string().optional(),
+    seed: z.number().int().optional(),
+    operationTitle: z.string().optional(),
+    operationGroup: z.string().optional(),
+});
 const AgentSessionRequestSchema = z.object({
     identifier: z.string().min(1).optional(),
     sessionId: z.string().optional(),
@@ -47,6 +68,7 @@ const AgentMessageRequestSchema = AgentSessionRequestSchema.extend({
     toolMaxIterations: z.number().optional(),
     operationTitle: z.string().optional(),
     operationGroup: z.string().optional(),
+    audioResponse: z.union([z.boolean(), PromptAudioParamsSchema]).optional(),
 });
 class DraxAgentController extends CommonController {
     constructor(options = {}) {

package/dist/index.js

@@ -26,6 +26,7 @@ import { BuilderTool } from "./tools/BuilderTool.js";
 import { KnowledgeService } from "./services/KnowledgeService.js";
 import { AILogService } from "./services/AILogService.js";
 import { TTSGenericService } from "./services/TTSGenericService.js";
+import { PromptAudioService } from "./services/PromptAudioService.js";
 import AILogPermissions from "./permissions/AILogPermissions.js";
 import AgentPermissions from "./permissions/AgentPermissions.js";
 import AgentSessionPermissions from "./permissions/AgentSessionPermissions.js";
@@ -45,7 +46,7 @@ import AgentSessionRoutes from "./routes/AgentSessionRoutes.js";
 import { DraxAgent } from "./agents/DraxAgent.js";
 export { OpenAiConfig, GoogleAiConfig, OllamaAiConfig, DeepSeekConfig, ElevenLabsTTSConfig, AILogSchema, AILogBaseSchema, TTSRequestSchema, TTSVoiceSettingsSchema, AILogModel, AILogMongoRepository, AILogSqliteRepository, OpenAiProviderFactory, GoogleAiProviderFactory, OllamaAiProviderFactory, DeepSeekAiProviderFactory, AiProviderFactory, ElevenLabsTTSProviderFactory, TTSProviderFactory, DraxAgentFactory, AILogServiceFactory, OpenAiProvider, GoogleAiProvider, OllamaAiProvider, DeepSeekAiProvider, ElevenLabsTTSProvider, BuilderTool, 
 //Service
-KnowledgeService, AILogService, TTSGenericService, 
+KnowledgeService, AILogService, TTSGenericService, PromptAudioService, 
 //Permissions
 AILogPermissions, AgentPermissions, AIPermissions, TTSPermissions, AgentSessionPermissions, 
 //Controllers

package/dist/providers/ai/GoogleAiProvider.js

@@ -1,5 +1,6 @@
 import { GoogleGenAI } from "@google/genai";
 import { toJSONSchema } from "zod";
+import PromptAudioService from "../../services/PromptAudioService.js";
 class GoogleAiProvider {
     constructor(apiKey, model, visionModel, aiLogService) {
         if (!apiKey) {
@@ -330,6 +331,7 @@ class GoogleAiProvider {
             const endTime = performance.now();
             const time = endTime - startTime;
             const endedAt = new Date();
+            const audio = await PromptAudioService.build(input, output);
             await this.registerPromptLog(input, {
                 model,
                 systemPrompt,
@@ -346,7 +348,8 @@ class GoogleAiProvider {
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? { audio } : {}),
             };
         }
         catch (e) {

package/dist/providers/ai/OllamaAiProvider.js

@@ -1,4 +1,5 @@
 import { toJSONSchema } from "zod";
+import PromptAudioService from "../../services/PromptAudioService.js";
 class OllamaAiProvider {
     constructor(baseUrl, model, visionModel, embeddingModel, aiLogService) {
         if (!baseUrl) {
@@ -305,6 +306,7 @@ class OllamaAiProvider {
             const endTime = performance.now();
             const time = endTime - startTime;
             const endedAt = new Date();
+            const audio = await PromptAudioService.build(input, output);
             await this.registerPromptLog(input, {
                 model,
                 systemPrompt,
@@ -321,7 +323,8 @@ class OllamaAiProvider {
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? { audio } : {}),
             };
         }
         catch (e) {

package/dist/providers/ai/OpenAiProvider.js

@@ -1,5 +1,6 @@
 import OpenAI from "openai";
 import { zodResponseFormat } from "openai/helpers/zod";
+import PromptAudioService from "../../services/PromptAudioService.js";
 class OpenAiProvider {
     constructor(apiKey, model, visionModel, aiLogService) {
         if (!apiKey) {
@@ -265,6 +266,7 @@ class OpenAiProvider {
             const endTime = performance.now();
             const time = endTime - startTime;
             const endedAt = new Date();
+            const audio = await PromptAudioService.build(input, output);
             await this.registerPromptLog(input, {
                 model,
                 systemPrompt,
@@ -281,7 +283,8 @@ class OpenAiProvider {
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? { audio } : {}),
             };
         }
         catch (e) {

package/dist/services/PromptAudioService.js

@@ -0,0 +1,68 @@
+import TTSProviderFactory from "../factory/tts/TTSProviderFactory.js";
+class PromptAudioService {
+    static audioParams(input) {
+        if (!input.audioResponse) {
+            return null;
+        }
+        if (input.audioResponse === true) {
+            return {};
+        }
+        if (input.audioResponse.enabled === false) {
+            return null;
+        }
+        return input.audioResponse;
+    }
+    static outputToText(output) {
+        if (typeof output === "string") {
+            return output;
+        }
+        if (output === null || output === undefined) {
+            return "";
+        }
+        return JSON.stringify(output);
+    }
+    static async build(input, output) {
+        const audioParams = PromptAudioService.audioParams(input);
+        if (!audioParams) {
+            return undefined;
+        }
+        const text = PromptAudioService.outputToText(output).trim();
+        if (!text) {
+            return undefined;
+        }
+        const providerName = audioParams.provider ?? "ElevenLabs";
+        const ttsProvider = TTSProviderFactory.instance(providerName);
+        const response = await ttsProvider.textToSpeech({
+            text,
+            voiceId: audioParams.voiceId,
+            model: audioParams.model,
+            outputFormat: audioParams.outputFormat,
+            voiceSettings: audioParams.voiceSettings,
+            previousText: audioParams.previousText,
+            nextText: audioParams.nextText,
+            languageCode: audioParams.languageCode,
+            seed: audioParams.seed,
+            operationTitle: audioParams.operationTitle ?? input.operationTitle,
+            operationGroup: audioParams.operationGroup ?? input.operationGroup,
+            ip: input.ip,
+            userAgent: input.userAgent,
+            tenant: input.tenant,
+            user: input.user,
+        });
+        return {
+            audio: response.audio.toString("base64"),
+            contentType: response.contentType,
+            encoding: "base64",
+            meta: {
+                provider: response.provider,
+                model: response.model,
+                voiceId: response.voiceId,
+                outputFormat: response.outputFormat,
+                size: response.size,
+                time: response.time,
+            },
+        };
+    }
+}
+export default PromptAudioService;
+export { PromptAudioService, };

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "3.42.0",
+  "version": "3.43.0",
   "description": "Ai utils",
   "main": "dist/index.js",
   "types": "types/index.d.ts",
@@ -18,7 +18,7 @@
   "author": "Cristian Incarnato & Drax Team",
   "license": "ISC",
   "dependencies": {
-    "@drax/ai-share": "^3.31.0",
+    "@drax/ai-share": "^3.43.0",
     "@drax/crud-back": "^3.39.0",
     "mongoose": "^8.23.0",
     "mongoose-paginate-v2": "^1.8.3"
@@ -46,5 +46,5 @@
     "typescript": "^5.9.3",
     "vitest": "^3.0.8"
   },
-  "gitHead": "30b8ca8c49adb16b4617e61c497b554389c49dd1"
+  "gitHead": "9cb1cab8a6fe2a6c574d08596bc6287f5a5311a6"
 }

package/src/agents/DraxAgent.ts

@@ -110,6 +110,7 @@ class DraxAgent {
             userAgent: input.userAgent,
             tenant: input.tenantId ?? session.tenantId ?? null,
             user: input.userId ?? session.userId ?? null,
+            audioResponse: input.audioResponse,
         });
 
         const assistantMessage = this.normalizeOutput(response.output);
@@ -134,6 +135,7 @@ class DraxAgent {
             inputTokens: response.inputTokens,
             outputTokens: response.outputTokens,
             time: response.time,
+            ...(response.audio ? {audio: response.audio} : {}),
         };
     }
 

package/src/controllers/AIGenericController.ts

@@ -42,6 +42,29 @@ const PromptInputFileSchema = z.object({
     url: z.string().optional(),
 })
 
+const PromptAudioVoiceSettingsSchema = z.object({
+    stability: z.number().min(0).max(1).optional(),
+    similarityBoost: z.number().min(0).max(1).optional(),
+    style: z.number().min(0).max(1).optional(),
+    useSpeakerBoost: z.boolean().optional(),
+    speed: z.number().positive().optional(),
+})
+
+const PromptAudioParamsSchema = z.object({
+    enabled: z.boolean().optional(),
+    provider: z.string().optional(),
+    voiceId: z.string().optional(),
+    model: z.string().optional(),
+    outputFormat: z.string().optional(),
+    voiceSettings: PromptAudioVoiceSettingsSchema.optional(),
+    previousText: z.string().optional(),
+    nextText: z.string().optional(),
+    languageCode: z.string().optional(),
+    seed: z.number().int().optional(),
+    operationTitle: z.string().optional(),
+    operationGroup: z.string().optional(),
+})
+
 const GenericPromptRequestSchema = z.object({
     systemPrompt: z.string().min(1),
     userInput: z.string().optional(),
@@ -57,6 +80,7 @@ const GenericPromptRequestSchema = z.object({
     model: z.string().optional(),
     operationTitle: z.string().optional(),
     operationGroup: z.string().optional(),
+    audioResponse: z.union([z.boolean(), PromptAudioParamsSchema]).optional(),
 })
 
 class AIGenericController extends CommonController {

package/src/controllers/DraxAgentController.ts

@@ -35,6 +35,29 @@ const PromptInputFileSchema = z.object({
     url: z.string().optional(),
 });
 
+const PromptAudioVoiceSettingsSchema = z.object({
+    stability: z.number().min(0).max(1).optional(),
+    similarityBoost: z.number().min(0).max(1).optional(),
+    style: z.number().min(0).max(1).optional(),
+    useSpeakerBoost: z.boolean().optional(),
+    speed: z.number().positive().optional(),
+});
+
+const PromptAudioParamsSchema = z.object({
+    enabled: z.boolean().optional(),
+    provider: z.string().optional(),
+    voiceId: z.string().optional(),
+    model: z.string().optional(),
+    outputFormat: z.string().optional(),
+    voiceSettings: PromptAudioVoiceSettingsSchema.optional(),
+    previousText: z.string().optional(),
+    nextText: z.string().optional(),
+    languageCode: z.string().optional(),
+    seed: z.number().int().optional(),
+    operationTitle: z.string().optional(),
+    operationGroup: z.string().optional(),
+});
+
 const AgentSessionRequestSchema = z.object({
     identifier: z.string().min(1).optional(),
     sessionId: z.string().optional(),
@@ -55,6 +78,7 @@ const AgentMessageRequestSchema = AgentSessionRequestSchema.extend({
     toolMaxIterations: z.number().optional(),
     operationTitle: z.string().optional(),
     operationGroup: z.string().optional(),
+    audioResponse: z.union([z.boolean(), PromptAudioParamsSchema]).optional(),
 });
 
 class DraxAgentController extends CommonController {

package/src/index.ts

@@ -27,6 +27,7 @@ import {BuilderTool} from "./tools/BuilderTool.js";
 import {KnowledgeService} from "./services/KnowledgeService.js";
 import {AILogService} from "./services/AILogService.js";
 import {TTSGenericService} from "./services/TTSGenericService.js";
+import {PromptAudioService} from "./services/PromptAudioService.js";
 import AILogPermissions from "./permissions/AILogPermissions.js";
 import AgentPermissions from "./permissions/AgentPermissions.js";
 import AgentSessionPermissions from "./permissions/AgentSessionPermissions.js";
@@ -50,6 +51,9 @@ import type {
     IPromptContentPart,
     IPromptContentPartImage,
     IPromptContentPartText,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptImage,
     IPromptImageDetail,
     IPromptMessage,
@@ -108,6 +112,9 @@ export type {
     IPromptContentPart,
     IPromptContentPartImage,
     IPromptContentPartText,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptResponse,
     ITTSProvider,
     ITTSParams,
@@ -166,6 +173,7 @@ export {
     KnowledgeService,
     AILogService,
     TTSGenericService,
+    PromptAudioService,
     //Permissions
     AILogPermissions,
     AgentPermissions,

package/src/interfaces/IAIProvider.ts

@@ -1,4 +1,5 @@
 import { ZodSchema } from 'zod'
+import type {ITTSVoiceSettings} from "./ITTSProvider.js";
 
 type Role = 'user' | 'assistant' | 'system';
 
@@ -74,6 +75,38 @@ interface IPromptParams {
     userAgent?: string,
     tenant?: string | null,
     user?: string | null,
+    audioResponse?: boolean | IPromptAudioParams,
+}
+
+interface IPromptAudioParams {
+    enabled?: boolean,
+    provider?: string,
+    voiceId?: string,
+    model?: string,
+    outputFormat?: string,
+    voiceSettings?: ITTSVoiceSettings,
+    previousText?: string,
+    nextText?: string,
+    languageCode?: string,
+    seed?: number,
+    operationTitle?: string,
+    operationGroup?: string,
+}
+
+interface IPromptAudioResponseMeta {
+    provider: string,
+    model: string,
+    voiceId: string,
+    outputFormat?: string,
+    size: number,
+    time: number,
+}
+
+interface IPromptAudioResponse {
+    audio: string,
+    contentType: string,
+    encoding: 'base64',
+    meta: IPromptAudioResponseMeta,
 }
 
 interface IPromptResponse {
@@ -81,7 +114,8 @@ interface IPromptResponse {
     tokens: number,
     inputTokens: number,
     outputTokens: number,
-    time: number
+    time: number,
+    audio?: IPromptAudioResponse,
 }
 
 interface IAIProvider {
@@ -92,6 +126,9 @@ export type {
     IAIProvider,
     IPromptParams,
     IPromptResponse,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptMessage,
     IPromptMemory,
     IPromptTool,

package/src/interfaces/IDraxAgent.ts

@@ -1,5 +1,7 @@
 import type {
     IAIProvider,
+    IPromptAudioParams,
+    IPromptAudioResponse,
     IPromptContentPart,
     IPromptImage,
     IPromptMemory,
@@ -67,6 +69,7 @@ interface DraxAgentMessageInput extends DraxAgentSessionInput {
     toolMaxIterations?: number;
     operationTitle?: string;
     operationGroup?: string;
+    audioResponse?: boolean | IPromptAudioParams;
 }
 
 interface DraxAgentMessageOutput {
@@ -79,6 +82,7 @@ interface DraxAgentMessageOutput {
     inputTokens: number;
     outputTokens: number;
     time: number;
+    audio?: IPromptAudioResponse;
 }
 
 interface DraxAgentSession {

package/src/providers/ai/GoogleAiProvider.ts

@@ -17,6 +17,7 @@ import type {
 } from "../../interfaces/IAIProvider.js";
 import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 class GoogleAiProvider implements IAIProvider{
     protected _apiKey: string
@@ -446,6 +447,7 @@ class GoogleAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -464,7 +466,8 @@ class GoogleAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/providers/ai/OllamaAiProvider.ts

@@ -9,6 +9,7 @@ import type {
 } from "../../interfaces/IAIProvider.js";
 import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 type OllamaMessage = {
     role: "system" | "user" | "assistant" | "tool",
@@ -426,6 +427,7 @@ class OllamaAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -444,7 +446,8 @@ class OllamaAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/providers/ai/OpenAiProvider.ts

@@ -10,6 +10,7 @@ import type {
 } from "../../interfaces/IAIProvider.js";
 import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 class OpenAiProvider implements IAIProvider{
     protected _apiKey: string
@@ -358,6 +359,7 @@ class OpenAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -376,7 +378,8 @@ class OpenAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/services/PromptAudioService.ts

@@ -0,0 +1,87 @@
+import TTSProviderFactory from "../factory/tts/TTSProviderFactory.js";
+import type {IPromptAudioParams, IPromptAudioResponse, IPromptParams} from "../interfaces/IAIProvider.js";
+
+class PromptAudioService {
+
+    static audioParams(input: IPromptParams): IPromptAudioParams | null {
+        if(!input.audioResponse){
+            return null
+        }
+
+        if(input.audioResponse === true){
+            return {}
+        }
+
+        if(input.audioResponse.enabled === false){
+            return null
+        }
+
+        return input.audioResponse
+    }
+
+    static outputToText(output: unknown): string {
+        if(typeof output === "string"){
+            return output
+        }
+
+        if(output === null || output === undefined){
+            return ""
+        }
+
+        return JSON.stringify(output)
+    }
+
+    static async build(input: IPromptParams, output: unknown): Promise<IPromptAudioResponse | undefined> {
+        const audioParams = PromptAudioService.audioParams(input)
+
+        if(!audioParams){
+            return undefined
+        }
+
+        const text = PromptAudioService.outputToText(output).trim()
+
+        if(!text){
+            return undefined
+        }
+
+        const providerName = audioParams.provider ?? "ElevenLabs"
+        const ttsProvider = TTSProviderFactory.instance(providerName)
+        const response = await ttsProvider.textToSpeech({
+            text,
+            voiceId: audioParams.voiceId,
+            model: audioParams.model,
+            outputFormat: audioParams.outputFormat,
+            voiceSettings: audioParams.voiceSettings,
+            previousText: audioParams.previousText,
+            nextText: audioParams.nextText,
+            languageCode: audioParams.languageCode,
+            seed: audioParams.seed,
+            operationTitle: audioParams.operationTitle ?? input.operationTitle,
+            operationGroup: audioParams.operationGroup ?? input.operationGroup,
+            ip: input.ip,
+            userAgent: input.userAgent,
+            tenant: input.tenant,
+            user: input.user,
+        })
+
+        return {
+            audio: response.audio.toString("base64"),
+            contentType: response.contentType,
+            encoding: "base64",
+            meta: {
+                provider: response.provider,
+                model: response.model,
+                voiceId: response.voiceId,
+                outputFormat: response.outputFormat,
+                size: response.size,
+                time: response.time,
+            },
+        }
+    }
+
+}
+
+export default PromptAudioService
+export {
+    PromptAudioService,
+}