@drax/ai-back 3.42.0 → 3.44.0

package/src/index.ts

@@ -27,6 +27,7 @@ import {BuilderTool} from "./tools/BuilderTool.js";
 import {KnowledgeService} from "./services/KnowledgeService.js";
 import {AILogService} from "./services/AILogService.js";
 import {TTSGenericService} from "./services/TTSGenericService.js";
+import {PromptAudioService} from "./services/PromptAudioService.js";
 import AILogPermissions from "./permissions/AILogPermissions.js";
 import AgentPermissions from "./permissions/AgentPermissions.js";
 import AgentSessionPermissions from "./permissions/AgentSessionPermissions.js";
@@ -44,12 +45,16 @@ import TTSRoutes from "./routes/TTSRoutes.js";
 import DraxAgentRoutes from "./routes/DraxAgentRoutes.js";
 import AgentSessionRoutes from "./routes/AgentSessionRoutes.js";
 import {DraxAgent} from "./agents/DraxAgent.js";
+import {BuildContextTool} from "./tools/BuildContextTool.js";
 import type {IAILogRepository} from "./interfaces/IAILogRepository.js";
 import type {
     IAIProvider,
     IPromptContentPart,
     IPromptContentPartImage,
     IPromptContentPartText,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptImage,
     IPromptImageDetail,
     IPromptMessage,
@@ -76,6 +81,10 @@ import type {
     ToolBuilderOptions,
     ToolBuilderService
 } from "./interfaces/IBuilderTool.js";
+import type {
+    BuildContextToolContext,
+    BuildContextToolOptions,
+} from "./tools/BuildContextTool.js";
 import type {
     DraxAgentControllerOptions
 } from "./interfaces/IDraxAgentController.js";
@@ -108,6 +117,9 @@ export type {
     IPromptContentPart,
     IPromptContentPartImage,
     IPromptContentPartText,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptResponse,
     ITTSProvider,
     ITTSParams,
@@ -120,6 +132,8 @@ export type {
     ToolBuilderMethod,
     ToolBuilderOptions,
     ToolBuilderService,
+    BuildContextToolContext,
+    BuildContextToolOptions,
     DraxAgentControllerOptions,
     DraxAgentFastifyRoutesOptions,
     DraxAgentConfig,
@@ -162,10 +176,12 @@ export {
     DeepSeekAiProvider,
     ElevenLabsTTSProvider,
     BuilderTool,
+    BuildContextTool,
     //Service
     KnowledgeService,
     AILogService,
     TTSGenericService,
+    PromptAudioService,
     //Permissions
     AILogPermissions,
     AgentPermissions,

package/src/interfaces/IAIProvider.ts

@@ -1,4 +1,5 @@
 import { ZodSchema } from 'zod'
+import type {ITTSVoiceSettings} from "./ITTSProvider.js";
 
 type Role = 'user' | 'assistant' | 'system';
 
@@ -74,6 +75,38 @@ interface IPromptParams {
     userAgent?: string,
     tenant?: string | null,
     user?: string | null,
+    audioResponse?: boolean | IPromptAudioParams,
+}
+
+interface IPromptAudioParams {
+    enabled?: boolean,
+    provider?: string,
+    voiceId?: string,
+    model?: string,
+    outputFormat?: string,
+    voiceSettings?: ITTSVoiceSettings,
+    previousText?: string,
+    nextText?: string,
+    languageCode?: string,
+    seed?: number,
+    operationTitle?: string,
+    operationGroup?: string,
+}
+
+interface IPromptAudioResponseMeta {
+    provider: string,
+    model: string,
+    voiceId: string,
+    outputFormat?: string,
+    size: number,
+    time: number,
+}
+
+interface IPromptAudioResponse {
+    audio: string,
+    contentType: string,
+    encoding: 'base64',
+    meta: IPromptAudioResponseMeta,
 }
 
 interface IPromptResponse {
@@ -81,7 +114,8 @@ interface IPromptResponse {
     tokens: number,
     inputTokens: number,
     outputTokens: number,
-    time: number
+    time: number,
+    audio?: IPromptAudioResponse,
 }
 
 interface IAIProvider {
@@ -92,6 +126,9 @@ export type {
     IAIProvider,
     IPromptParams,
     IPromptResponse,
+    IPromptAudioParams,
+    IPromptAudioResponse,
+    IPromptAudioResponseMeta,
     IPromptMessage,
     IPromptMemory,
     IPromptTool,

package/src/interfaces/IDraxAgent.ts

@@ -1,5 +1,7 @@
 import type {
     IAIProvider,
+    IPromptAudioParams,
+    IPromptAudioResponse,
     IPromptContentPart,
     IPromptImage,
     IPromptMemory,
@@ -67,6 +69,7 @@ interface DraxAgentMessageInput extends DraxAgentSessionInput {
     toolMaxIterations?: number;
     operationTitle?: string;
     operationGroup?: string;
+    audioResponse?: boolean | IPromptAudioParams;
 }
 
 interface DraxAgentMessageOutput {
@@ -79,6 +82,7 @@ interface DraxAgentMessageOutput {
     inputTokens: number;
     outputTokens: number;
     time: number;
+    audio?: IPromptAudioResponse;
 }
 
 interface DraxAgentSession {

package/src/providers/ai/GoogleAiProvider.ts

@@ -17,6 +17,7 @@ import type {
 } from "../../interfaces/IAIProvider.js";
 import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 class GoogleAiProvider implements IAIProvider{
     protected _apiKey: string
@@ -446,6 +447,7 @@ class GoogleAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -464,7 +466,8 @@ class GoogleAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/providers/ai/OllamaAiProvider.ts

@@ -9,6 +9,7 @@ import type {
 } from "../../interfaces/IAIProvider.js";
 import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 type OllamaMessage = {
     role: "system" | "user" | "assistant" | "tool",
@@ -426,6 +427,7 @@ class OllamaAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -444,7 +446,8 @@ class OllamaAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/providers/ai/OpenAiProvider.ts

@@ -10,6 +10,7 @@ import type {
 } from "../../interfaces/IAIProvider.js";
 import type {AILogService} from "../../services/AILogService.js";
 import type {IAILogBase} from "@drax/ai-share";
+import PromptAudioService from "../../services/PromptAudioService.js";
 
 class OpenAiProvider implements IAIProvider{
     protected _apiKey: string
@@ -358,6 +359,7 @@ class OpenAiProvider implements IAIProvider{
             const endTime = performance.now()
             const time = endTime - startTime
             const endedAt = new Date()
+            const audio = await PromptAudioService.build(input, output)
 
             await this.registerPromptLog(input, {
                 model,
@@ -376,7 +378,8 @@ class OpenAiProvider implements IAIProvider{
                 tokens,
                 inputTokens,
                 outputTokens,
-                time
+                time,
+                ...(audio ? {audio} : {}),
             }
         } catch (e: any) {
             const endedAt = new Date()

package/src/services/PromptAudioService.ts

@@ -0,0 +1,87 @@
+import TTSProviderFactory from "../factory/tts/TTSProviderFactory.js";
+import type {IPromptAudioParams, IPromptAudioResponse, IPromptParams} from "../interfaces/IAIProvider.js";
+
+class PromptAudioService {
+
+    static audioParams(input: IPromptParams): IPromptAudioParams | null {
+        if(!input.audioResponse){
+            return null
+        }
+
+        if(input.audioResponse === true){
+            return {}
+        }
+
+        if(input.audioResponse.enabled === false){
+            return null
+        }
+
+        return input.audioResponse
+    }
+
+    static outputToText(output: unknown): string {
+        if(typeof output === "string"){
+            return output
+        }
+
+        if(output === null || output === undefined){
+            return ""
+        }
+
+        return JSON.stringify(output)
+    }
+
+    static async build(input: IPromptParams, output: unknown): Promise<IPromptAudioResponse | undefined> {
+        const audioParams = PromptAudioService.audioParams(input)
+
+        if(!audioParams){
+            return undefined
+        }
+
+        const text = PromptAudioService.outputToText(output).trim()
+
+        if(!text){
+            return undefined
+        }
+
+        const providerName = audioParams.provider ?? "ElevenLabs"
+        const ttsProvider = TTSProviderFactory.instance(providerName)
+        const response = await ttsProvider.textToSpeech({
+            text,
+            voiceId: audioParams.voiceId,
+            model: audioParams.model,
+            outputFormat: audioParams.outputFormat,
+            voiceSettings: audioParams.voiceSettings,
+            previousText: audioParams.previousText,
+            nextText: audioParams.nextText,
+            languageCode: audioParams.languageCode,
+            seed: audioParams.seed,
+            operationTitle: audioParams.operationTitle ?? input.operationTitle,
+            operationGroup: audioParams.operationGroup ?? input.operationGroup,
+            ip: input.ip,
+            userAgent: input.userAgent,
+            tenant: input.tenant,
+            user: input.user,
+        })
+
+        return {
+            audio: response.audio.toString("base64"),
+            contentType: response.contentType,
+            encoding: "base64",
+            meta: {
+                provider: response.provider,
+                model: response.model,
+                voiceId: response.voiceId,
+                outputFormat: response.outputFormat,
+                size: response.size,
+                time: response.time,
+            },
+        }
+    }
+
+}
+
+export default PromptAudioService
+export {
+    PromptAudioService,
+}