@drax/ai-back 3.0.0 → 3.17.0

package/src/providers/OpenAiProvider.ts

@@ -1,14 +1,23 @@
 import OpenAI from "openai";
-import type {ZodTypeAny} from "zod"
 import { zodResponseFormat } from "openai/helpers/zod";
-import type {IAIProvider, IPromptParams, IPromptResponse} from "../interfaces/IAIProvider";
+import type {
+    IAIProvider,
+    IPromptContentPart,
+    IPromptMessage,
+    IPromptParams,
+    IPromptResponse
+} from "../interfaces/IAIProvider";
+import type {AILogService} from "../services/AILogService";
+import type {IAILogBase} from "@drax/ai-share";
 
 class OpenAiProvider implements IAIProvider{
     protected _apiKey: string
     protected _model: any
+    protected _visionModel?: string
     protected _client: any
+    protected _aiLogService?: AILogService
 
-    constructor(apiKey: string, model: string) {
+    constructor(apiKey: string, model: string, visionModel?: string, aiLogService?: AILogService) {
 
         if (!apiKey) {
             throw new Error("OpenAI apiKey required")
@@ -19,6 +28,8 @@ class OpenAiProvider implements IAIProvider{
 
         this._apiKey = apiKey
         this._model = model
+        this._visionModel = visionModel
+        this._aiLogService = aiLogService
     }
 
     get model(){
@@ -38,6 +49,170 @@ class OpenAiProvider implements IAIProvider{
         return this._client
     }
 
+    protected get visionModel(){
+        return this._visionModel
+    }
+
+    protected buildUserContent(input: IPromptParams): string | Array<{type: 'text', text: string} | {type: 'image_url', image_url: {url: string, detail?: 'auto' | 'low' | 'high'}}> {
+        if(input.userContent && input.userContent.length > 0){
+            return this.mapContentParts(input.userContent)
+        }
+
+        if(input.userImages && input.userImages.length > 0){
+            const content: Array<{type: 'text', text: string} | {type: 'image_url', image_url: {url: string, detail?: 'auto' | 'low' | 'high'}}> = []
+
+            if(input.userInput){
+                content.push({type: 'text', text: input.userInput})
+            }
+
+            content.push(...input.userImages.map(image => ({
+                type: 'image_url' as const,
+                image_url: {
+                    url: image.url,
+                    ...(image.detail ? {detail: image.detail} : {}),
+                }
+            })))
+
+            return content
+        }
+
+        return input.userInput ?? ""
+    }
+
+    protected mapContentParts(content: IPromptContentPart[]){
+        return content.map(part => {
+            if(part.type === 'text'){
+                return {
+                    type: 'text' as const,
+                    text: part.text
+                }
+            }
+
+            return {
+                type: 'image_url' as const,
+                image_url: {
+                    url: part.imageUrl,
+                    ...(part.detail ? {detail: part.detail} : {}),
+                }
+            }
+        })
+    }
+
+    protected mapHistory(history: IPromptMessage[] = []){
+        return history.map(message => ({
+            role: message.role,
+            content: typeof message.content === 'string'
+                ? message.content
+                : this.mapContentParts(message.content)
+        }))
+    }
+
+    protected hasImageInput(input: IPromptParams){
+        if(input.userImages && input.userImages.length > 0){
+            return true
+        }
+
+        if(input.userContent?.some(part => part.type === 'image')){
+            return true
+        }
+
+        return input.history?.some(message =>
+            Array.isArray(message.content) && message.content.some(part => part.type === 'image')
+        ) ?? false
+    }
+
+    protected serializePromptInput(input: IPromptParams, systemPrompt: string){
+        return JSON.stringify({
+            systemPrompt,
+            history: input.history,
+            userInput: input.userInput,
+            userContent: input.userContent,
+            memory: input.memory,
+            knowledgeBase: input.knowledgeBase,
+        })
+    }
+
+    protected serializePromptOutput(output: unknown){
+        if (typeof output === "string") {
+            return output
+        }
+
+        if (output === null || output === undefined) {
+            return undefined
+        }
+
+        return JSON.stringify(output)
+    }
+
+    protected buildLogPayload(input: IPromptParams, params: {
+        model: string,
+        systemPrompt: string,
+        startedAt: Date,
+        endedAt?: Date,
+        inputTokens?: number,
+        outputTokens?: number,
+        tokens?: number,
+        output?: unknown,
+        success: boolean,
+        errorMessage?: string,
+    }): IAILogBase {
+        return {
+            provider: "openai",
+            model: params.model,
+            operationTitle: input.operationTitle,
+            operationGroup: input.operationGroup,
+            ip: input.ip,
+            userAgent: input.userAgent,
+            input: this.serializePromptInput(input, params.systemPrompt),
+            inputImages: input.userImages?.map(image => ({
+                url: image.url,
+            })) ?? input.userContent
+                ?.filter(part => part.type === "image")
+                .map(part => ({
+                    url: part.imageUrl,
+                })),
+            inputFiles: input.inputFiles,
+            inputTokens: params.inputTokens,
+            outputTokens: params.outputTokens,
+            tokens: params.tokens,
+            startedAt: params.startedAt,
+            endedAt: params.endedAt,
+            responseTime: params.endedAt ? `${params.endedAt.getTime() - params.startedAt.getTime()}ms` : undefined,
+            output: this.serializePromptOutput(params.output),
+            success: params.success,
+            errorMessage: params.errorMessage,
+            tenant: input.tenant,
+            user: input.user,
+        }
+    }
+
+    protected async registerPromptLog(input: IPromptParams, params: {
+        model: string,
+        systemPrompt: string,
+        startedAt: Date,
+        endedAt?: Date,
+        inputTokens?: number,
+        outputTokens?: number,
+        tokens?: number,
+        output?: unknown,
+        success: boolean,
+        errorMessage?: string,
+    }){
+        if(!this._aiLogService){
+            return
+        }
+
+        try{
+            await this._aiLogService.create(this.buildLogPayload(input, params))
+        }catch(e: any){
+            console.error("Error registerPromptLog", {
+                name: e?.name,
+                message: e?.message,
+                stack: e?.stack,
+            })
+        }
+    }
+
     async generateEmbedding({text, model="text-embedding-ada-002"}: {text:string,model:string }): Promise<number[]> {
         const response = await this.client.embeddings.create({
             model: model,
@@ -52,8 +227,6 @@ class OpenAiProvider implements IAIProvider{
             throw new Error("systemPrompt required")
         }
 
-        const model = input.model ?? this.model
-
         let systemPrompt = input.systemPrompt
 
         if(input.memory && input.memory.length > 0){
@@ -65,37 +238,66 @@ class OpenAiProvider implements IAIProvider{
         }
 
 
-        let userInput = input.userInput
-
+        const userInput = this.buildUserContent(input)
+        const model = input.model ?? (this.hasImageInput(input) ? this.visionModel ?? this.model : this.model)
+        const startedAt = new Date()
         const startTime = performance.now()
 
-        const chatCompletion = await this.client.chat.completions.create({
-            messages: [
-                {role: 'system', content: systemPrompt},
-                ...(input.history ? input.history : []),
-                {role: 'user', content: userInput},
-            ],
+        try {
+            const chatCompletion = await this.client.chat.completions.create({
+                messages: [
+                    {role: 'system', content: systemPrompt},
+                    ...this.mapHistory(input.history),
+                    {role: 'user', content: userInput},
+                ],
 
-            ...(input.zodSchema ? {response_format: zodResponseFormat(input.zodSchema, "event")} : {}),
-            ...(input.jsonSchema ? {response_format: input.jsonSchema} : {}),
-            model: model,
-        });
+                ...(input.zodSchema ? {response_format: zodResponseFormat(input.zodSchema, "event")} : {}),
+                ...(input.jsonSchema ? {response_format: input.jsonSchema} : {}),
+                model: model,
+            });
 
 
-        const output = chatCompletion.choices[0].message.content
-        const tokens = chatCompletion.usage.total_tokens
-        const inputTokens = chatCompletion.usage.prompt_tokens
-        const outputTokens = chatCompletion.usage.completion_tokens
+            const output = chatCompletion.choices[0].message.content
+            const tokens = chatCompletion.usage.total_tokens
+            const inputTokens = chatCompletion.usage.prompt_tokens
+            const outputTokens = chatCompletion.usage.completion_tokens
 
-        const endTime = performance.now()
-        const time = endTime - startTime
+            const endTime = performance.now()
+            const time = endTime - startTime
+            const endedAt = new Date()
 
-        return {
-            output,
-            tokens,
-            inputTokens,
-            outputTokens,
-            time
+            await this.registerPromptLog(input, {
+                model,
+                systemPrompt,
+                startedAt,
+                endedAt,
+                inputTokens,
+                outputTokens,
+                tokens,
+                output,
+                success: true,
+            })
+
+            return {
+                output,
+                tokens,
+                inputTokens,
+                outputTokens,
+                time
+            }
+        } catch (e: any) {
+            const endedAt = new Date()
+
+            await this.registerPromptLog(input, {
+                model,
+                systemPrompt,
+                startedAt,
+                endedAt,
+                success: false,
+                errorMessage: e?.message,
+            })
+
+            throw e
         }
     }
 

package/src/repository/mongo/AILogMongoRepository.ts

@@ -0,0 +1,22 @@
+
+import {AbstractMongoRepository} from "@drax/crud-back";
+import {AILogModel} from "../../models/AILogModel.js";
+import type {IAILogRepository} from '../../interfaces/IAILogRepository'
+import type {IAILog, IAILogBase} from "@drax/ai-share";
+
+
+class AILogMongoRepository extends AbstractMongoRepository<IAILog, IAILogBase, IAILogBase> implements IAILogRepository {
+
+    constructor() {
+        super();
+        this._model = AILogModel;
+        this._searchFields = ['provider', 'model', 'operationTitle', 'operationGroup', 'ip', 'userAgent', 'input', 'output', 'errorMessage'];
+        this._populateFields = ['tenant', 'user'];
+        this._lean = true
+    }
+
+}
+
+export default AILogMongoRepository
+export {AILogMongoRepository}
+

package/src/repository/sqlite/AILogSqliteRepository.ts

@@ -0,0 +1,53 @@
+
+import {AbstractSqliteRepository} from "@drax/crud-back";
+import type {IAILogRepository} from '../../interfaces/IAILogRepository'
+import type {IAILog, IAILogBase} from "@drax/ai-share";
+import {SqliteTableField} from "@drax/common-back";
+
+class AILogSqliteRepository extends AbstractSqliteRepository<IAILog, IAILogBase, IAILogBase> implements IAILogRepository {
+
+    protected db: any;
+    protected tableName: string = 'AILog';
+    protected dataBaseFile: string;
+    protected searchFields: string[] = ['provider', 'model', 'operationTitle', 'operationGroup', 'ip', 'userAgent', 'input', 'output', 'errorMessage'];
+    protected booleanFields: string[] = ['success'];
+    protected jsonFields: string[] = ['inputImages', 'inputFiles'];
+    protected identifier: string = '_id';
+    protected populateFields = [
+        { field: 'tenant', table: 'tenant', identifier: '_id' },
+{ field: 'user', table: 'user', identifier: '_id' }
+    ]
+    protected verbose: boolean = false;
+    protected tableFields: SqliteTableField[] = [
+        {name: "provider", type: "TEXT", unique: false, primary: false},
+{name: "model", type: "TEXT", unique: false, primary: false},
+{name: "operationTitle", type: "TEXT", unique: false, primary: false},
+{name: "operationGroup", type: "TEXT", unique: false, primary: false},
+{name: "ip", type: "TEXT", unique: false, primary: false},
+{name: "userAgent", type: "TEXT", unique: false, primary: false},
+{name: "input", type: "TEXT", unique: false, primary: false},
+{name: "inputImages", type: "TEXT", unique: false, primary: false},
+{name: "inputFiles", type: "TEXT", unique: false, primary: false},
+{name: "inputTokens", type: "INTEGER", unique: false, primary: false},
+{name: "inputTokens", type: "TEXT", unique: false, primary: false},
+{name: "outputTokens", type: "INTEGER", unique: false, primary: false},
+{name: "outputTokens", type: "TEXT", unique: false, primary: false},
+{name: "tokens", type: "INTEGER", unique: false, primary: false},
+{name: "tokens", type: "TEXT", unique: false, primary: false},
+{name: "startedAt", type: "TEXT", unique: false, primary: false},
+{name: "endedAt", type: "TEXT", unique: false, primary: false},
+{name: "responseTime", type: "TEXT", unique: false, primary: false},
+{name: "output", type: "TEXT", unique: false, primary: false},
+{name: "success", type: "TEXT", unique: false, primary: false},
+{name: "statusCode", type: "INTEGER", unique: false, primary: false},
+{name: "statusCode", type: "TEXT", unique: false, primary: false},
+{name: "errorMessage", type: "TEXT", unique: false, primary: false},
+{name: "tenant", type: "TEXT", unique: false, primary: false},
+{name: "user", type: "TEXT", unique: false, primary: false}
+    ]
+
+}
+
+export default AILogSqliteRepository
+export {AILogSqliteRepository}
+

package/src/routes/AILogRoutes.ts

@@ -0,0 +1,38 @@
+
+import AILogController from "../controllers/AILogController.js";
+import {CrudSchemaBuilder} from "@drax/crud-back";
+import {AILogSchema, AILogBaseSchema} from '../schemas/AILogSchema.js'
+
+async function AILogFastifyRoutes(fastify, options) {
+
+    const controller: AILogController = new AILogController()
+    const schemas = new CrudSchemaBuilder(AILogSchema, AILogBaseSchema,AILogBaseSchema, 'AILog', 'openapi-3.0', ['ai']);
+
+    fastify.get('/api/ailog', {schema: schemas.paginateSchema}, (req,rep) => controller.paginate(req,rep))
+    
+    fastify.get('/api/ailog/find', {schema: schemas.findSchema}, (req,rep) => controller.find(req,rep))
+    
+    fastify.get('/api/ailog/search', {schema: schemas.searchSchema}, (req,rep) => controller.search(req,rep))
+    
+    fastify.get('/api/ailog/:id', {schema: schemas.findByIdSchema}, (req,rep) => controller.findById(req,rep))
+    
+    fastify.get('/api/ailog/find-one', {schema: schemas.findOneSchema}, (req,rep) => controller.findOne(req,rep))
+    
+    fastify.get('/api/ailog/group-by', {schema: schemas.groupBySchema}, (req,rep) => controller.groupBy(req,rep))
+
+    fastify.post('/api/ailog', {schema: schemas.createSchema}, (req,rep) =>controller.create(req,rep))
+
+    fastify.put('/api/ailog/:id', {schema: schemas.updateSchema}, (req,rep) =>controller.update(req,rep))
+    
+    fastify.patch('/api/ailog/:id', {schema: schemas.updateSchema}, (req,rep) =>controller.updatePartial(req,rep))
+
+    fastify.delete('/api/ailog/:id', {schema: schemas.deleteSchema}, (req,rep) =>controller.delete(req,rep))
+    
+    fastify.get('/api/ailog/export', (req,rep) =>controller.export(req,rep))
+    
+    fastify.post('/api/ailog/import', (req,rep) => controller.import(req,rep))
+    
+}
+
+export default AILogFastifyRoutes;
+export {AILogFastifyRoutes}

package/src/routes/AIRoutes.ts

@@ -0,0 +1,18 @@
+
+import AICrudController from "../controllers/AICrudController.js";
+import AIGenericController from "../controllers/AIGenericController.js";
+
+
+async function AiFastifyRoutes(fastify, options) {
+
+    const crudController: AICrudController = new AICrudController()
+    const genericController: AIGenericController = new AIGenericController()
+
+    fastify.post('/api/ai/prompt/crud', (req,rep) => crudController.prompt(req,rep))
+    fastify.post('/api/ai/prompt/generic', (req,rep) => genericController.prompt(req,rep))
+
+
+}
+
+export default AiFastifyRoutes;
+export {AiFastifyRoutes}

package/src/schemas/AILogSchema.ts

@@ -0,0 +1,52 @@
+import {z} from 'zod';
+
+
+const AILogBaseSchema = z.object({
+    provider: z.string().optional(),
+    model: z.string().optional(),
+    operationTitle: z.string().optional(),
+    operationGroup: z.string().optional(),
+    ip: z.string().optional(),
+    userAgent: z.string().optional(),
+    input: z.string().optional(),
+    inputImages: z.array(
+        z.object({
+            filename: z.string().optional(),
+            filepath: z.string().optional(),
+            size: z.number().nullable().optional(),
+            mimetype: z.string().optional(),
+            url: z.string().optional()
+        })
+    ).optional(),
+    inputFiles: z.array(
+        z.object({
+            filename: z.string().optional(),
+            filepath: z.string().optional(),
+            size: z.number().nullable().optional(),
+            mimetype: z.string().optional(),
+            url: z.string().optional()
+        })
+    ).optional(),
+    inputTokens: z.number().nullable().optional(),
+    outputTokens: z.number().nullable().optional(),
+    tokens: z.number().nullable().optional(),
+    startedAt: z.coerce.date().nullable().optional(),
+    endedAt: z.coerce.date().nullable().optional(),
+    responseTime: z.string().optional(),
+    output: z.string().optional(),
+    success: z.boolean().optional(),
+    statusCode: z.number().nullable().optional(),
+    errorMessage: z.string().optional(),
+    tenant: z.coerce.string().optional().nullable(),
+    user: z.coerce.string().optional().nullable()
+});
+
+const AILogSchema = AILogBaseSchema
+    .extend({
+        _id: z.coerce.string(),
+        tenant: z.object({_id: z.coerce.string(), name: z.string()}).nullable().optional(),
+        user: z.object({_id: z.coerce.string(), username: z.string()}).nullable().optional()
+    })
+
+export default AILogSchema;
+export {AILogSchema, AILogBaseSchema}

package/src/services/AILogService.ts

@@ -0,0 +1,20 @@
+
+import type{IAILogRepository} from "../interfaces/IAILogRepository";
+import type {IAILogBase, IAILog} from "@drax/ai-share";
+import {AbstractService} from "@drax/crud-back";
+import type {ZodObject, ZodRawShape} from "zod";
+
+class AILogService extends AbstractService<IAILog, IAILogBase, IAILogBase> {
+
+
+    constructor(AILogRepository: IAILogRepository, baseSchema?: ZodObject<ZodRawShape>, fullSchema?: ZodObject<ZodRawShape>) {
+        super(AILogRepository, baseSchema, fullSchema);
+
+        this._validateOutput = true
+
+    }
+
+}
+
+export default AILogService
+export {AILogService}

package/test/OpenAiProvider.test.ts

@@ -6,6 +6,97 @@ import {IPromptMemory, IPromptMessage} from "../src/interfaces/IAIProvider";
 
 describe('OpenAi Test', () => {
 
+    test('OpenAi prompt supports image inputs and vision model fallback', async () => {
+        let request: any
+
+        class MockedOpenAiProvider extends OpenAiProvider {
+            constructor() {
+                super('test-key', 'gpt-4.1-mini', 'gpt-4o-mini')
+                this._client = {
+                    chat: {
+                        completions: {
+                            create: async (payload: any) => {
+                                request = payload
+                                return {
+                                    choices: [{message: {content: '{"name":"invoice"}'}}],
+                                    usage: {
+                                        total_tokens: 10,
+                                        prompt_tokens: 7,
+                                        completion_tokens: 3
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        const openAi = new MockedOpenAiProvider()
+
+        const r = await openAi.prompt({
+            systemPrompt: 'Extract invoice data',
+            userInput: 'Read this invoice',
+            userImages: [{url: 'data:image/png;base64,abc123', detail: 'high'}]
+        })
+
+        expect(r.output).toBe('{"name":"invoice"}')
+        expect(request.model).toBe('gpt-4o-mini')
+        expect(request.messages[1]).toEqual({
+            role: 'user',
+            content: [
+                {type: 'text', text: 'Read this invoice'},
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: 'data:image/png;base64,abc123',
+                        detail: 'high'
+                    }
+                }
+            ]
+        })
+    })
+
+    test('OpenAi prompt keeps default model for text-only inputs', async () => {
+        let request: any
+
+        class MockedOpenAiProvider extends OpenAiProvider {
+            constructor() {
+                super('test-key', 'gpt-4.1-mini', 'gpt-4o-mini')
+                this._client = {
+                    chat: {
+                        completions: {
+                            create: async (payload: any) => {
+                                request = payload
+                                return {
+                                    choices: [{message: {content: 'Buenos Aires'}}],
+                                    usage: {
+                                        total_tokens: 8,
+                                        prompt_tokens: 6,
+                                        completion_tokens: 2
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        const openAi = new MockedOpenAiProvider()
+
+        await openAi.prompt({
+            systemPrompt: 'You are an AI assistant.',
+            userInput: 'What is the capital of Argentina?'
+        })
+
+        expect(request.model).toBe('gpt-4.1-mini')
+        expect(request.messages[1]).toEqual({
+            role: 'user',
+            content: 'What is the capital of Argentina?'
+        })
+    })
+
     test('OpenAi Factory', () => {
         const openAi = OpenAiProviderFactory.instance()
         expect(openAi).toBeInstanceOf(OpenAiProvider)