@drax/ai-back 2.0.0 → 3.16.0

package/src/factory/services/AILogServiceFactory.ts

@@ -0,0 +1,41 @@
+
+import AILogMongoRepository from '../../repository/mongo/AILogMongoRepository.js'
+import AILogSqliteRepository from '../../repository/sqlite/AILogSqliteRepository.js'
+import type {IAILogRepository} from "../../interfaces/IAILogRepository";
+import {AILogService} from '../../services/AILogService.js'
+import {AILogBaseSchema, AILogSchema} from "../../schemas/AILogSchema.js";
+import {COMMON, CommonConfig, DraxConfig} from "@drax/common-back";
+
+class AILogServiceFactory {
+    private static service: AILogService;
+
+    public static get instance(): AILogService {
+        if (!AILogServiceFactory.service) {
+            
+            let repository: IAILogRepository
+            switch (DraxConfig.getOrLoad(CommonConfig.DbEngine)) {
+                case COMMON.DB_ENGINES.MONGODB:
+                    repository = new AILogMongoRepository()
+                    break;
+                case COMMON.DB_ENGINES.SQLITE:
+                    const dbFile = DraxConfig.getOrLoad(CommonConfig.SqliteDbFile)
+                    repository = new AILogSqliteRepository(dbFile, false)
+                    repository.build()
+                    break;
+                default:
+                    throw new Error("DraxConfig.DB_ENGINE must be one of " + Object.values(COMMON.DB_ENGINES).join(", "));
+            }
+            
+            const baseSchema = AILogBaseSchema;
+            const fullSchema = AILogSchema;
+            AILogServiceFactory.service = new AILogService(repository, baseSchema, fullSchema);
+        }
+        return AILogServiceFactory.service;
+    }
+}
+
+export default AILogServiceFactory
+export {
+    AILogServiceFactory
+}
+

package/src/index.ts

@@ -1,20 +1,58 @@
 import {OpenAiConfig} from "./config/OpenAiConfig.js";
+import {AILogSchema, AILogBaseSchema} from "./schemas/AILogSchema.js";
+import AILogModel from "./models/AILogModel.js";
+import AILogMongoRepository from "./repository/mongo/AILogMongoRepository.js";
+import AILogSqliteRepository from "./repository/sqlite/AILogSqliteRepository.js";
 import {OpenAiProviderFactory} from "./factory/OpenAiProviderFactory.js";
+import AILogServiceFactory from "./factory/services/AILogServiceFactory.js";
 import {OpenAiProvider} from "./providers/OpenAiProvider.js";
 import {KnowledgeService} from "./services/KnowledgeService.js";
-import type {IAIProvider, IPromptResponse, IPromptParams, IPromptMessage, IPromptMemory} from "./interfaces/IAIProvider.js";
+import {AILogService} from "./services/AILogService.js";
+import AILogPermissions from "./permissions/AILogPermissions.js";
+import AILogController from "./controllers/AILogController.js";
+import AILogRoutes from "./routes/AILogRoutes.js";
+import type {IAILogRepository} from "./interfaces/IAILogRepository.js";
+import type {
+    IAIProvider,
+    IPromptContentPart,
+    IPromptContentPartImage,
+    IPromptContentPartText,
+    IPromptImage,
+    IPromptImageDetail,
+    IPromptMessage,
+    IPromptMemory,
+    IPromptParams,
+    IPromptResponse
+} from "./interfaces/IAIProvider.js";
 
 export type {
+
+    IAILogRepository,
     IAIProvider,
     IPromptParams,
     IPromptMessage,
     IPromptMemory,
+    IPromptImage,
+    IPromptImageDetail,
+    IPromptContentPart,
+    IPromptContentPartImage,
+    IPromptContentPartText,
     IPromptResponse,
 }
 
 export {
     OpenAiConfig,
+    AILogSchema,
+    AILogBaseSchema,
+    AILogModel,
+    AILogMongoRepository,
+    AILogSqliteRepository,
     OpenAiProviderFactory,
+    AILogServiceFactory,
     OpenAiProvider,
     KnowledgeService,
+    AILogService,
+    AILogPermissions,
+    AILogController,
+    AILogRoutes,
 }

package/src/interfaces/IAILogRepository.ts

@@ -0,0 +1,11 @@
+
+import type {IAILog, IAILogBase} from '@drax/ai-share'
+import {IDraxCrudRepository} from "@drax/crud-share";
+
+interface IAILogRepository extends IDraxCrudRepository<IAILog, IAILogBase, IAILogBase>{
+
+}
+
+export {IAILogRepository}
+
+

package/src/interfaces/IAIProvider.ts

@@ -2,9 +2,29 @@ import { ZodSchema } from 'zod'
 
 type Role = 'user' | 'assistant' | 'system';
 
+type IPromptImageDetail = 'auto' | 'low' | 'high';
+
+interface IPromptImage {
+    url: string;
+    detail?: IPromptImageDetail;
+}
+
+interface IPromptContentPartText {
+    type: 'text';
+    text: string;
+}
+
+interface IPromptContentPartImage {
+    type: 'image';
+    imageUrl: string;
+    detail?: IPromptImageDetail;
+}
+
+type IPromptContentPart = IPromptContentPartText | IPromptContentPartImage;
+
 interface IPromptMessage {
     role: Role;
-    content: string;
+    content: string | IPromptContentPart[];
 }
 
 interface IPromptMemory {
@@ -15,6 +35,15 @@ interface IPromptMemory {
 interface IPromptParams {
     systemPrompt: string,
     userInput?: string,
+    userImages?: IPromptImage[],
+    inputFiles?: Array<{
+        filename?: string,
+        filepath?: string,
+        size?: number | null,
+        mimetype?: string,
+        url?: string
+    }>,
+    userContent?: IPromptContentPart[],
     history?: IPromptMessage[],
     memory?: IPromptMemory[],
     memoryHeader?: string | '[MEMORY]' | '[MEMORIA]'
@@ -23,6 +52,12 @@ interface IPromptParams {
     zodSchema?: ZodSchema<any>,
     jsonSchema?: object,
     model?: string,
+    operationTitle?: string,
+    operationGroup?: string,
+    ip?: string,
+    userAgent?: string,
+    tenant?: string | null,
+    user?: string | null,
 }
 
 interface IPromptResponse {
@@ -37,4 +72,15 @@ interface IAIProvider {
     prompt(input: IPromptParams): Promise<IPromptResponse>
 }
 
-export type {IAIProvider, IPromptParams, IPromptResponse, IPromptMessage, IPromptMemory}
+export type {
+    IAIProvider,
+    IPromptParams,
+    IPromptResponse,
+    IPromptMessage,
+    IPromptMemory,
+    IPromptImage,
+    IPromptImageDetail,
+    IPromptContentPart,
+    IPromptContentPartImage,
+    IPromptContentPartText,
+}

package/src/models/AILogModel.ts

@@ -0,0 +1,65 @@
+
+import {mongoose} from '@drax/common-back';
+import {PaginateModel} from "mongoose";
+import uniqueValidator from 'mongoose-unique-validator';
+import mongoosePaginate from 'mongoose-paginate-v2'
+import type {IAILog} from '@drax/ai-share'
+
+const AILogSchema = new mongoose.Schema<IAILog>({
+            provider: {type: String,   required: false, index: true, unique: false },
+            model: {type: String,   required: false, index: true, unique: false },
+            operationTitle: {type: String,   required: false, index: false, unique: false },
+            operationGroup: {type: String,   required: false, index: false, unique: false },
+            ip: {type: String,   required: false, index: false, unique: false },
+            userAgent: {type: String,   required: false, index: false, unique: false },
+            input: {type: String,   required: false, index: false, unique: false },
+            inputImages: [{
+                        filename: {type: String,   required: false, index: false, unique: false },
+            filepath: {type: String,   required: false, index: false, unique: false },
+            size: {type: Number,   required: false, index: false, unique: false },
+            mimetype: {type: String,   required: false, index: false, unique: false },
+            url: {type: String,   required: false, index: false, unique: false }
+            }],
+            inputFiles: [{
+                        filename: {type: String,   required: false, index: false, unique: false },
+            filepath: {type: String,   required: false, index: false, unique: false },
+            size: {type: Number,   required: false, index: false, unique: false },
+            mimetype: {type: String,   required: false, index: false, unique: false },
+            url: {type: String,   required: false, index: false, unique: false }
+            }],
+            inputTokens: {type: Number,   required: false, index: false, unique: false },
+            outputTokens: {type: Number,   required: false, index: false, unique: false },
+            tokens: {type: Number,   required: false, index: false, unique: false },
+            startedAt: {type: Date,   required: false, index: false, unique: false },
+            endedAt: {type: Date,   required: false, index: false, unique: false },
+            responseTime: {type: String,   required: false, index: false, unique: false },
+            output: {type: String,   required: false, index: false, unique: false },
+            success: {type: Boolean,   required: false, index: false, unique: false },
+            statusCode: {type: Number,   required: false, index: false, unique: false },
+            errorMessage: {type: String,   required: false, index: false, unique: false },
+            tenant: {type: mongoose.Schema.Types.ObjectId, ref: 'Tenant',  required: false, index: false, unique: false },
+            user: {type: mongoose.Schema.Types.ObjectId, ref: 'User',  required: false, index: false, unique: false }
+}, {timestamps: true});
+
+AILogSchema.plugin(uniqueValidator, {message: 'validation.unique'});
+AILogSchema.plugin(mongoosePaginate);
+
+AILogSchema.virtual("id").get(function () {
+    return this._id.toString();
+});
+
+
+AILogSchema.set('toJSON', {getters: true, virtuals: true});
+
+AILogSchema.set('toObject', {getters: true, virtuals: true});
+
+const MODEL_NAME = 'AILog';
+const COLLECTION_NAME = 'AILog';
+const AILogModel = mongoose.model<IAILog, PaginateModel<IAILog>>(MODEL_NAME, AILogSchema,COLLECTION_NAME);
+
+export {
+    AILogSchema,
+    AILogModel
+}
+
+export default AILogModel

package/src/permissions/AILogPermissions.ts

@@ -0,0 +1,14 @@
+
+enum AILogPermissions {
+
+    Create = "ailog:create",
+    Update = "ailog:update",
+    Delete = "ailog:delete",
+    View = "ailog:view",
+    Manage = "ailog:manage"
+
+}
+
+export { AILogPermissions };
+export default AILogPermissions;
+

package/src/providers/OpenAiProvider.ts

@@ -1,14 +1,23 @@
 import OpenAI from "openai";
-import type {ZodTypeAny} from "zod"
 import { zodResponseFormat } from "openai/helpers/zod";
-import type {IAIProvider, IPromptParams, IPromptResponse} from "../interfaces/IAIProvider";
+import type {
+    IAIProvider,
+    IPromptContentPart,
+    IPromptMessage,
+    IPromptParams,
+    IPromptResponse
+} from "../interfaces/IAIProvider";
+import type {AILogService} from "../services/AILogService";
+import type {IAILogBase} from "@drax/ai-share";
 
 class OpenAiProvider implements IAIProvider{
     protected _apiKey: string
     protected _model: any
+    protected _visionModel?: string
     protected _client: any
+    protected _aiLogService?: AILogService
 
-    constructor(apiKey: string, model: string) {
+    constructor(apiKey: string, model: string, visionModel?: string, aiLogService?: AILogService) {
 
         if (!apiKey) {
             throw new Error("OpenAI apiKey required")
@@ -19,6 +28,8 @@ class OpenAiProvider implements IAIProvider{
 
         this._apiKey = apiKey
         this._model = model
+        this._visionModel = visionModel
+        this._aiLogService = aiLogService
     }
 
     get model(){
@@ -38,6 +49,170 @@ class OpenAiProvider implements IAIProvider{
         return this._client
     }
 
+    protected get visionModel(){
+        return this._visionModel
+    }
+
+    protected buildUserContent(input: IPromptParams): string | Array<{type: 'text', text: string} | {type: 'image_url', image_url: {url: string, detail?: 'auto' | 'low' | 'high'}}> {
+        if(input.userContent && input.userContent.length > 0){
+            return this.mapContentParts(input.userContent)
+        }
+
+        if(input.userImages && input.userImages.length > 0){
+            const content: Array<{type: 'text', text: string} | {type: 'image_url', image_url: {url: string, detail?: 'auto' | 'low' | 'high'}}> = []
+
+            if(input.userInput){
+                content.push({type: 'text', text: input.userInput})
+            }
+
+            content.push(...input.userImages.map(image => ({
+                type: 'image_url' as const,
+                image_url: {
+                    url: image.url,
+                    ...(image.detail ? {detail: image.detail} : {}),
+                }
+            })))
+
+            return content
+        }
+
+        return input.userInput ?? ""
+    }
+
+    protected mapContentParts(content: IPromptContentPart[]){
+        return content.map(part => {
+            if(part.type === 'text'){
+                return {
+                    type: 'text' as const,
+                    text: part.text
+                }
+            }
+
+            return {
+                type: 'image_url' as const,
+                image_url: {
+                    url: part.imageUrl,
+                    ...(part.detail ? {detail: part.detail} : {}),
+                }
+            }
+        })
+    }
+
+    protected mapHistory(history: IPromptMessage[] = []){
+        return history.map(message => ({
+            role: message.role,
+            content: typeof message.content === 'string'
+                ? message.content
+                : this.mapContentParts(message.content)
+        }))
+    }
+
+    protected hasImageInput(input: IPromptParams){
+        if(input.userImages && input.userImages.length > 0){
+            return true
+        }
+
+        if(input.userContent?.some(part => part.type === 'image')){
+            return true
+        }
+
+        return input.history?.some(message =>
+            Array.isArray(message.content) && message.content.some(part => part.type === 'image')
+        ) ?? false
+    }
+
+    protected serializePromptInput(input: IPromptParams, systemPrompt: string){
+        return JSON.stringify({
+            systemPrompt,
+            history: input.history,
+            userInput: input.userInput,
+            userContent: input.userContent,
+            memory: input.memory,
+            knowledgeBase: input.knowledgeBase,
+        })
+    }
+
+    protected serializePromptOutput(output: unknown){
+        if (typeof output === "string") {
+            return output
+        }
+
+        if (output === null || output === undefined) {
+            return undefined
+        }
+
+        return JSON.stringify(output)
+    }
+
+    protected buildLogPayload(input: IPromptParams, params: {
+        model: string,
+        systemPrompt: string,
+        startedAt: Date,
+        endedAt?: Date,
+        inputTokens?: number,
+        outputTokens?: number,
+        tokens?: number,
+        output?: unknown,
+        success: boolean,
+        errorMessage?: string,
+    }): IAILogBase {
+        return {
+            provider: "openai",
+            model: params.model,
+            operationTitle: input.operationTitle,
+            operationGroup: input.operationGroup,
+            ip: input.ip,
+            userAgent: input.userAgent,
+            input: this.serializePromptInput(input, params.systemPrompt),
+            inputImages: input.userImages?.map(image => ({
+                url: image.url,
+            })) ?? input.userContent
+                ?.filter(part => part.type === "image")
+                .map(part => ({
+                    url: part.imageUrl,
+                })),
+            inputFiles: input.inputFiles,
+            inputTokens: params.inputTokens,
+            outputTokens: params.outputTokens,
+            tokens: params.tokens,
+            startedAt: params.startedAt,
+            endedAt: params.endedAt,
+            responseTime: params.endedAt ? `${params.endedAt.getTime() - params.startedAt.getTime()}ms` : undefined,
+            output: this.serializePromptOutput(params.output),
+            success: params.success,
+            errorMessage: params.errorMessage,
+            tenant: input.tenant,
+            user: input.user,
+        }
+    }
+
+    protected async registerPromptLog(input: IPromptParams, params: {
+        model: string,
+        systemPrompt: string,
+        startedAt: Date,
+        endedAt?: Date,
+        inputTokens?: number,
+        outputTokens?: number,
+        tokens?: number,
+        output?: unknown,
+        success: boolean,
+        errorMessage?: string,
+    }){
+        if(!this._aiLogService){
+            return
+        }
+
+        try{
+            await this._aiLogService.create(this.buildLogPayload(input, params))
+        }catch(e: any){
+            console.error("Error registerPromptLog", {
+                name: e?.name,
+                message: e?.message,
+                stack: e?.stack,
+            })
+        }
+    }
+
     async generateEmbedding({text, model="text-embedding-ada-002"}: {text:string,model:string }): Promise<number[]> {
         const response = await this.client.embeddings.create({
             model: model,
@@ -52,8 +227,6 @@ class OpenAiProvider implements IAIProvider{
             throw new Error("systemPrompt required")
         }
 
-        const model = input.model ?? this.model
-
         let systemPrompt = input.systemPrompt
 
         if(input.memory && input.memory.length > 0){
@@ -65,37 +238,66 @@ class OpenAiProvider implements IAIProvider{
         }
 
 
-        let userInput = input.userInput
-
+        const userInput = this.buildUserContent(input)
+        const model = input.model ?? (this.hasImageInput(input) ? this.visionModel ?? this.model : this.model)
+        const startedAt = new Date()
         const startTime = performance.now()
 
-        const chatCompletion = await this.client.chat.completions.create({
-            messages: [
-                {role: 'system', content: systemPrompt},
-                ...(input.history ? input.history : []),
-                {role: 'user', content: userInput},
-            ],
+        try {
+            const chatCompletion = await this.client.chat.completions.create({
+                messages: [
+                    {role: 'system', content: systemPrompt},
+                    ...this.mapHistory(input.history),
+                    {role: 'user', content: userInput},
+                ],
 
-            ...(input.zodSchema ? {response_format: zodResponseFormat(input.zodSchema, "event")} : {}),
-            ...(input.jsonSchema ? {response_format: input.jsonSchema} : {}),
-            model: model,
-        });
+                ...(input.zodSchema ? {response_format: zodResponseFormat(input.zodSchema, "event")} : {}),
+                ...(input.jsonSchema ? {response_format: input.jsonSchema} : {}),
+                model: model,
+            });
 
 
-        const output = chatCompletion.choices[0].message.content
-        const tokens = chatCompletion.usage.total_tokens
-        const inputTokens = chatCompletion.usage.prompt_tokens
-        const outputTokens = chatCompletion.usage.completion_tokens
+            const output = chatCompletion.choices[0].message.content
+            const tokens = chatCompletion.usage.total_tokens
+            const inputTokens = chatCompletion.usage.prompt_tokens
+            const outputTokens = chatCompletion.usage.completion_tokens
 
-        const endTime = performance.now()
-        const time = endTime - startTime
+            const endTime = performance.now()
+            const time = endTime - startTime
+            const endedAt = new Date()
 
-        return {
-            output,
-            tokens,
-            inputTokens,
-            outputTokens,
-            time
+            await this.registerPromptLog(input, {
+                model,
+                systemPrompt,
+                startedAt,
+                endedAt,
+                inputTokens,
+                outputTokens,
+                tokens,
+                output,
+                success: true,
+            })
+
+            return {
+                output,
+                tokens,
+                inputTokens,
+                outputTokens,
+                time
+            }
+        } catch (e: any) {
+            const endedAt = new Date()
+
+            await this.registerPromptLog(input, {
+                model,
+                systemPrompt,
+                startedAt,
+                endedAt,
+                success: false,
+                errorMessage: e?.message,
+            })
+
+            throw e
         }
     }
 

package/src/repository/mongo/AILogMongoRepository.ts

@@ -0,0 +1,22 @@
+
+import {AbstractMongoRepository} from "@drax/crud-back";
+import {AILogModel} from "../../models/AILogModel.js";
+import type {IAILogRepository} from '../../interfaces/IAILogRepository'
+import type {IAILog, IAILogBase} from "@drax/ai-share";
+
+
+class AILogMongoRepository extends AbstractMongoRepository<IAILog, IAILogBase, IAILogBase> implements IAILogRepository {
+
+    constructor() {
+        super();
+        this._model = AILogModel;
+        this._searchFields = ['provider', 'model', 'operationTitle', 'operationGroup', 'ip', 'userAgent', 'input', 'output', 'errorMessage'];
+        this._populateFields = ['tenant', 'user'];
+        this._lean = true
+    }
+
+}
+
+export default AILogMongoRepository
+export {AILogMongoRepository}
+

package/src/repository/sqlite/AILogSqliteRepository.ts

@@ -0,0 +1,53 @@
+
+import {AbstractSqliteRepository} from "@drax/crud-back";
+import type {IAILogRepository} from '../../interfaces/IAILogRepository'
+import type {IAILog, IAILogBase} from "@drax/ai-share";
+import {SqliteTableField} from "@drax/common-back";
+
+class AILogSqliteRepository extends AbstractSqliteRepository<IAILog, IAILogBase, IAILogBase> implements IAILogRepository {
+
+    protected db: any;
+    protected tableName: string = 'AILog';
+    protected dataBaseFile: string;
+    protected searchFields: string[] = ['provider', 'model', 'operationTitle', 'operationGroup', 'ip', 'userAgent', 'input', 'output', 'errorMessage'];
+    protected booleanFields: string[] = ['success'];
+    protected jsonFields: string[] = ['inputImages', 'inputFiles'];
+    protected identifier: string = '_id';
+    protected populateFields = [
+        { field: 'tenant', table: 'tenant', identifier: '_id' },
+{ field: 'user', table: 'user', identifier: '_id' }
+    ]
+    protected verbose: boolean = false;
+    protected tableFields: SqliteTableField[] = [
+        {name: "provider", type: "TEXT", unique: false, primary: false},
+{name: "model", type: "TEXT", unique: false, primary: false},
+{name: "operationTitle", type: "TEXT", unique: false, primary: false},
+{name: "operationGroup", type: "TEXT", unique: false, primary: false},
+{name: "ip", type: "TEXT", unique: false, primary: false},
+{name: "userAgent", type: "TEXT", unique: false, primary: false},
+{name: "input", type: "TEXT", unique: false, primary: false},
+{name: "inputImages", type: "TEXT", unique: false, primary: false},
+{name: "inputFiles", type: "TEXT", unique: false, primary: false},
+{name: "inputTokens", type: "INTEGER", unique: false, primary: false},
+{name: "inputTokens", type: "TEXT", unique: false, primary: false},
+{name: "outputTokens", type: "INTEGER", unique: false, primary: false},
+{name: "outputTokens", type: "TEXT", unique: false, primary: false},
+{name: "tokens", type: "INTEGER", unique: false, primary: false},
+{name: "tokens", type: "TEXT", unique: false, primary: false},
+{name: "startedAt", type: "TEXT", unique: false, primary: false},
+{name: "endedAt", type: "TEXT", unique: false, primary: false},
+{name: "responseTime", type: "TEXT", unique: false, primary: false},
+{name: "output", type: "TEXT", unique: false, primary: false},
+{name: "success", type: "TEXT", unique: false, primary: false},
+{name: "statusCode", type: "INTEGER", unique: false, primary: false},
+{name: "statusCode", type: "TEXT", unique: false, primary: false},
+{name: "errorMessage", type: "TEXT", unique: false, primary: false},
+{name: "tenant", type: "TEXT", unique: false, primary: false},
+{name: "user", type: "TEXT", unique: false, primary: false}
+    ]
+
+}
+
+export default AILogSqliteRepository
+export {AILogSqliteRepository}
+