@lobehub/chat 1.15.6 → 1.15.8

package/src/database/server/migrations/meta/_journal.json

@@ -56,6 +56,13 @@
       "when": 1724254147447,
       "tag": "0007_fix_embedding_table",
       "breakpoints": true
+    },
+    {
+      "idx": 8,
+      "version": "7",
+      "when": 1725366565650,
+      "tag": "0008_add_rag_evals",
+      "breakpoints": true
     }
   ],
   "version": "6"

package/src/database/server/models/file.ts

@@ -1,5 +1,5 @@
-import { asc, count, eq, ilike, inArray, notExists, sum } from 'drizzle-orm';
-import { and, desc } from 'drizzle-orm/expressions';
+import { asc, count, eq, ilike, inArray, notExists, or, sum } from 'drizzle-orm';
+import { and, desc, like } from 'drizzle-orm/expressions';
 
 import { serverDBEnv } from '@/config/db';
 import { serverDB } from '@/database/server/core/db';
@@ -280,4 +280,13 @@ export class FileModel {
       }
     }
   };
+
+  async findByNames(fileNames: string[]) {
+    return serverDB.query.files.findMany({
+      where: and(
+        or(...fileNames.map((name) => like(files.name, `${name}%`))),
+        eq(files.userId, this.userId),
+      ),
+    });
+  }
 }

package/src/database/server/models/ragEval/dataset.ts

@@ -0,0 +1,59 @@
+import { and, desc, eq } from 'drizzle-orm';
+
+import { serverDB } from '@/database/server';
+import { NewEvalDatasetsItem, evalDatasets } from '@/database/server/schemas/lobechat';
+import { RAGEvalDataSetItem } from '@/types/eval';
+
+export class EvalDatasetModel {
+  private userId: string;
+
+  constructor(userId: string) {
+    this.userId = userId;
+  }
+
+  create = async (params: NewEvalDatasetsItem) => {
+    const [result] = await serverDB
+      .insert(evalDatasets)
+      .values({ ...params, userId: this.userId })
+      .returning();
+    return result;
+  };
+
+  delete = async (id: number) => {
+    return serverDB
+      .delete(evalDatasets)
+      .where(and(eq(evalDatasets.id, id), eq(evalDatasets.userId, this.userId)));
+  };
+
+  query = async (knowledgeBaseId: string): Promise<RAGEvalDataSetItem[]> => {
+    return serverDB
+      .select({
+        createdAt: evalDatasets.createdAt,
+        description: evalDatasets.description,
+        id: evalDatasets.id,
+        name: evalDatasets.name,
+        updatedAt: evalDatasets.updatedAt,
+      })
+      .from(evalDatasets)
+      .where(
+        and(
+          eq(evalDatasets.userId, this.userId),
+          eq(evalDatasets.knowledgeBaseId, knowledgeBaseId),
+        ),
+      )
+      .orderBy(desc(evalDatasets.createdAt));
+  };
+
+  findById = async (id: number) => {
+    return serverDB.query.evalDatasets.findFirst({
+      where: and(eq(evalDatasets.id, id), eq(evalDatasets.userId, this.userId)),
+    });
+  };
+
+  update = async (id: number, value: Partial<NewEvalDatasetsItem>) => {
+    return serverDB
+      .update(evalDatasets)
+      .set({ ...value, updatedAt: new Date() })
+      .where(and(eq(evalDatasets.id, id), eq(evalDatasets.userId, this.userId)));
+  };
+}

package/src/database/server/models/ragEval/datasetRecord.ts

@@ -0,0 +1,87 @@
+import { and, eq, inArray } from 'drizzle-orm';
+
+import { serverDB } from '@/database/server';
+import {
+  NewEvalDatasetRecordsItem,
+  evalDatasetRecords,
+  files,
+} from '@/database/server/schemas/lobechat';
+import { EvalDatasetRecordRefFile } from '@/types/eval';
+
+export class EvalDatasetRecordModel {
+  private userId: string;
+
+  constructor(userId: string) {
+    this.userId = userId;
+  }
+
+  create = async (params: NewEvalDatasetRecordsItem) => {
+    const [result] = await serverDB
+      .insert(evalDatasetRecords)
+      .values({ ...params, userId: this.userId })
+      .returning();
+    return result;
+  };
+
+  batchCreate = async (params: NewEvalDatasetRecordsItem[]) => {
+    const [result] = await serverDB
+      .insert(evalDatasetRecords)
+      .values(params.map((item) => ({ ...item, userId: this.userId })))
+      .returning();
+
+    return result;
+  };
+
+  delete = async (id: number) => {
+    return serverDB
+      .delete(evalDatasetRecords)
+      .where(and(eq(evalDatasetRecords.id, id), eq(evalDatasetRecords.userId, this.userId)));
+  };
+
+  query = async (datasetId: number) => {
+    const list = await serverDB.query.evalDatasetRecords.findMany({
+      where: and(
+        eq(evalDatasetRecords.datasetId, datasetId),
+        eq(evalDatasetRecords.userId, this.userId),
+      ),
+    });
+    const fileList = list.flatMap((item) => item.referenceFiles).filter(Boolean) as string[];
+
+    const fileItems = await serverDB
+      .select({ fileType: files.fileType, id: files.id, name: files.name })
+      .from(files)
+      .where(and(inArray(files.id, fileList), eq(files.userId, this.userId)))
+      .execute();
+
+    return list.map((item) => {
+      return {
+        ...item,
+        referenceFiles: (item.referenceFiles?.map((fileId) => {
+          return fileItems.find((file) => file.id === fileId);
+        }) || []) as EvalDatasetRecordRefFile[],
+      };
+    });
+  };
+
+  findByDatasetId = async (datasetId: number) => {
+    return serverDB.query.evalDatasetRecords.findMany({
+      where: and(
+        eq(evalDatasetRecords.datasetId, datasetId),
+        eq(evalDatasetRecords.userId, this.userId),
+      ),
+    });
+  };
+
+  findById = async (id: number) => {
+    return serverDB.query.evalDatasetRecords.findFirst({
+      where: and(eq(evalDatasetRecords.id, id), eq(evalDatasetRecords.userId, this.userId)),
+    });
+  };
+
+  update = async (id: number, value: Partial<NewEvalDatasetRecordsItem>) => {
+    return serverDB
+      .update(evalDatasetRecords)
+      .set(value)
+      .where(and(eq(evalDatasetRecords.id, id), eq(evalDatasetRecords.userId, this.userId)));
+  };
+}

package/src/database/server/models/ragEval/evaluation.ts

@@ -0,0 +1,96 @@
+import { SQL, and, count, desc, eq, inArray } from 'drizzle-orm';
+
+import { serverDB } from '@/database/server';
+import {
+  NewEvalEvaluationItem,
+  evalDatasets,
+  evalEvaluation,
+  evaluationRecords,
+} from '@/database/server/schemas/lobechat';
+import { EvalEvaluationStatus, RAGEvalEvaluationItem } from '@/types/eval';
+
+export class EvalEvaluationModel {
+  private userId: string;
+
+  constructor(userId: string) {
+    this.userId = userId;
+  }
+
+  create = async (params: NewEvalEvaluationItem) => {
+    const [result] = await serverDB
+      .insert(evalEvaluation)
+      .values({ ...params, userId: this.userId })
+      .returning();
+    return result;
+  };
+
+  delete = async (id: number) => {
+    return serverDB
+      .delete(evalEvaluation)
+      .where(and(eq(evalEvaluation.id, id), eq(evalEvaluation.userId, this.userId)));
+  };
+
+  queryByKnowledgeBaseId = async (knowledgeBaseId: string) => {
+    const evaluations = await serverDB
+      .select({
+        createdAt: evalEvaluation.createdAt,
+        dataset: {
+          id: evalDatasets.id,
+          name: evalDatasets.name,
+        },
+        evalRecordsUrl: evalEvaluation.evalRecordsUrl,
+        id: evalEvaluation.id,
+        name: evalEvaluation.name,
+        status: evalEvaluation.status,
+        updatedAt: evalEvaluation.updatedAt,
+      })
+      .from(evalEvaluation)
+      .leftJoin(evalDatasets, eq(evalDatasets.id, evalEvaluation.datasetId))
+      .orderBy(desc(evalEvaluation.createdAt))
+      .where(
+        and(
+          eq(evalEvaluation.userId, this.userId),
+          eq(evalEvaluation.knowledgeBaseId, knowledgeBaseId),
+        ),
+      );
+
+    // 然后查询每个评估的记录统计
+    const evaluationIds = evaluations.map((evals) => evals.id);
+
+    const recordStats = await serverDB
+      .select({
+        evaluationId: evaluationRecords.evaluationId,
+        success: count(evaluationRecords.status).if(
+          eq(evaluationRecords.status, EvalEvaluationStatus.Success),
+        ) as SQL<number>,
+        total: count(),
+      })
+      .from(evaluationRecords)
+      .where(inArray(evaluationRecords.evaluationId, evaluationIds))
+      .groupBy(evaluationRecords.evaluationId);
+
+    return evaluations.map((evaluation) => {
+      const stats = recordStats.find((stat) => stat.evaluationId === evaluation.id);
+
+      return {
+        ...evaluation,
+        recordsStats: stats
+          ? { success: Number(stats.success), total: Number(stats.total) }
+          : { success: 0, total: 0 },
+      } as RAGEvalEvaluationItem;
+    });
+  };
+
+  findById = async (id: number) => {
+    return serverDB.query.evalEvaluation.findFirst({
+      where: and(eq(evalEvaluation.id, id), eq(evalEvaluation.userId, this.userId)),
+    });
+  };
+
+  update = async (id: number, value: Partial<NewEvalEvaluationItem>) => {
+    return serverDB
+      .update(evalEvaluation)
+      .set(value)
+      .where(and(eq(evalEvaluation.id, id), eq(evalEvaluation.userId, this.userId)));
+  };
+}

package/src/database/server/models/ragEval/evaluationRecord.ts

@@ -0,0 +1,64 @@
+import { and, eq } from 'drizzle-orm';
+
+import { serverDB } from '@/database/server';
+import { NewEvaluationRecordsItem, evaluationRecords } from '@/database/server/schemas/lobechat';
+
+export class EvaluationRecordModel {
+  private userId: string;
+
+  constructor(userId: string) {
+    this.userId = userId;
+  }
+
+  create = async (params: NewEvaluationRecordsItem) => {
+    const [result] = await serverDB
+      .insert(evaluationRecords)
+      .values({ ...params, userId: this.userId })
+      .returning();
+    return result;
+  };
+
+  batchCreate = async (params: NewEvaluationRecordsItem[]) => {
+    return serverDB
+      .insert(evaluationRecords)
+      .values(params.map((item) => ({ ...item, userId: this.userId })))
+      .returning();
+  };
+
+  delete = async (id: number) => {
+    return serverDB
+      .delete(evaluationRecords)
+      .where(and(eq(evaluationRecords.id, id), eq(evaluationRecords.userId, this.userId)));
+  };
+
+  query = async (reportId: number) => {
+    return serverDB.query.evaluationRecords.findMany({
+      where: and(
+        eq(evaluationRecords.evaluationId, reportId),
+        eq(evaluationRecords.userId, this.userId),
+      ),
+    });
+  };
+
+  findById = async (id: number) => {
+    return serverDB.query.evaluationRecords.findFirst({
+      where: and(eq(evaluationRecords.id, id), eq(evaluationRecords.userId, this.userId)),
+    });
+  };
+
+  findByEvaluationId = async (evaluationId: number) => {
+    return serverDB.query.evaluationRecords.findMany({
+      where: and(
+        eq(evaluationRecords.evaluationId, evaluationId),
+        eq(evaluationRecords.userId, this.userId),
+      ),
+    });
+  };
+
+  update = async (id: number, value: Partial<NewEvaluationRecordsItem>) => {
+    return serverDB
+      .update(evaluationRecords)
+      .set(value)
+      .where(and(eq(evaluationRecords.id, id), eq(evaluationRecords.userId, this.userId)));
+  };
+}

package/src/database/server/models/ragEval/index.ts

@@ -0,0 +1,4 @@
+export * from './dataset';
+export * from './datasetRecord';
+export * from './evaluation';
+export * from './evaluationRecord';

package/src/database/server/schemas/lobechat/asyncTask.ts

@@ -0,0 +1,24 @@
+/* eslint-disable sort-keys-fix/sort-keys-fix  */
+import { integer, jsonb, pgTable, text, uuid } from 'drizzle-orm/pg-core';
+
+import { createdAt, updatedAt } from './_helpers';
+import { users } from './user';
+
+export const asyncTasks = pgTable('async_tasks', {
+  id: uuid('id').defaultRandom().primaryKey(),
+  type: text('type'),
+
+  status: text('status'),
+  error: jsonb('error'),
+
+  userId: text('user_id')
+    .references(() => users.id, { onDelete: 'cascade' })
+    .notNull(),
+  duration: integer('duration'),
+
+  createdAt: createdAt(),
+  updatedAt: updatedAt(),
+});
+
+export type NewAsyncTaskItem = typeof asyncTasks.$inferInsert;
+export type AsyncTaskSelectItem = typeof asyncTasks.$inferSelect;

package/src/database/server/schemas/lobechat/file.ts

@@ -11,28 +11,12 @@ import {
 } from 'drizzle-orm/pg-core';
 import { createInsertSchema } from 'drizzle-zod';
 
-import { chunks } from '@/database/server/schemas/lobechat/rag';
-
 import { idGenerator } from '../../utils/idGenerator';
 import { createdAt, updatedAt } from './_helpers';
+import { asyncTasks } from './asyncTask';
+import { chunks } from './rag';
 import { users } from './user';
 
-export const asyncTasks = pgTable('async_tasks', {
-  id: uuid('id').defaultRandom().primaryKey(),
-  type: text('type'),
-  status: text('status'),
-  error: jsonb('error'),
-  userId: text('user_id')
-    .references(() => users.id, { onDelete: 'cascade' })
-    .notNull(),
-  duration: integer('duration'),
-  createdAt: createdAt(),
-  updatedAt: updatedAt(),
-});
-
-export type NewAsyncTaskItem = typeof asyncTasks.$inferInsert;
-export type AsyncTaskSelectItem = typeof asyncTasks.$inferSelect;
-
 export const globalFiles = pgTable('global_files', {
   hashId: varchar('hash_id', { length: 64 }).primaryKey(),
   fileType: varchar('file_type', { length: 255 }).notNull(),

package/src/database/server/schemas/lobechat/index.ts

@@ -1,9 +1,11 @@
 export * from './agent';
+export * from './asyncTask';
 export * from './discover';
 export * from './file';
 export * from './message';
 export * from './nextauth';
 export * from './rag';
+export * from './ragEvals';
 export * from './relations';
 export * from './session';
 export * from './topic';

package/src/database/server/schemas/lobechat/ragEvals.ts

@@ -0,0 +1,105 @@
+/* eslint-disable sort-keys-fix/sort-keys-fix  */
+import { integer, jsonb, pgTable, text, uuid } from 'drizzle-orm/pg-core';
+
+import { DEFAULT_EMBEDDING_MODEL, DEFAULT_MODEL } from '@/const/settings';
+import { EvalEvaluationStatus } from '@/types/eval';
+
+import { createdAt, updatedAt } from './_helpers';
+import { knowledgeBases } from './file';
+import { embeddings } from './rag';
+import { users } from './user';
+
+export const evalDatasets = pgTable('rag_eval_datasets', {
+  id: integer('id').generatedAlwaysAsIdentity({ startWith: 30_000 }).primaryKey(),
+
+  description: text('description'),
+  name: text('name').notNull(),
+
+  knowledgeBaseId: text('knowledge_base_id').references(() => knowledgeBases.id, {
+    onDelete: 'cascade',
+  }),
+  userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
+
+  updatedAt: updatedAt(),
+  createdAt: createdAt(),
+});
+
+export type NewEvalDatasetsItem = typeof evalDatasets.$inferInsert;
+export type EvalDatasetsSelectItem = typeof evalDatasets.$inferSelect;
+
+export const evalDatasetRecords = pgTable('rag_eval_dataset_records', {
+  id: integer('id').generatedAlwaysAsIdentity().primaryKey(),
+  datasetId: integer('dataset_id')
+    .references(() => evalDatasets.id, { onDelete: 'cascade' })
+    .notNull(),
+
+  ideal: text('ideal'),
+  question: text('question'),
+  referenceFiles: text('reference_files').array(),
+  metadata: jsonb('metadata'),
+
+  userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
+  createdAt: createdAt(),
+});
+
+export type NewEvalDatasetRecordsItem = typeof evalDatasetRecords.$inferInsert;
+export type EvalDatasetRecordsSelectItem = typeof evalDatasetRecords.$inferSelect;
+
+export const evalEvaluation = pgTable('rag_eval_evaluations', {
+  id: integer('id').generatedAlwaysAsIdentity().primaryKey(),
+  name: text('name').notNull(),
+  description: text('description'),
+
+  evalRecordsUrl: text('eval_records_url'),
+  status: text('status').$defaultFn(() => EvalEvaluationStatus.Pending),
+  error: jsonb('error'),
+
+  datasetId: integer('dataset_id')
+    .references(() => evalDatasets.id, { onDelete: 'cascade' })
+    .notNull(),
+  knowledgeBaseId: text('knowledge_base_id').references(() => knowledgeBases.id, {
+    onDelete: 'cascade',
+  }),
+  languageModel: text('language_model').$defaultFn(() => DEFAULT_MODEL),
+  embeddingModel: text('embedding_model').$defaultFn(() => DEFAULT_EMBEDDING_MODEL),
+
+  userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
+  createdAt: createdAt(),
+  updatedAt: updatedAt(),
+});
+
+export type NewEvalEvaluationItem = typeof evalEvaluation.$inferInsert;
+export type EvalEvaluationSelectItem = typeof evalEvaluation.$inferSelect;
+
+export const evaluationRecords = pgTable('rag_eval_evaluation_records', {
+  id: integer('id').generatedAlwaysAsIdentity().primaryKey(),
+
+  question: text('question').notNull(),
+  answer: text('answer'),
+  context: text('context').array(),
+  ideal: text('ideal'),
+
+  status: text('status').$defaultFn(() => EvalEvaluationStatus.Pending),
+  error: jsonb('error'),
+
+  languageModel: text('language_model'),
+  embeddingModel: text('embedding_model'),
+
+  questionEmbeddingId: uuid('question_embedding_id').references(() => embeddings.id, {
+    onDelete: 'set null',
+  }),
+
+  duration: integer('duration'),
+  datasetRecordId: integer('dataset_record_id')
+    .references(() => evalDatasetRecords.id, { onDelete: 'cascade' })
+    .notNull(),
+  evaluationId: integer('evaluation_id')
+    .references(() => evalEvaluation.id, { onDelete: 'cascade' })
+    .notNull(),
+
+  userId: text('user_id').references(() => users.id, { onDelete: 'cascade' }),
+  createdAt: createdAt(),
+});
+
+export type NewEvaluationRecordsItem = typeof evaluationRecords.$inferInsert;
+export type EvaluationRecordsSelectItem = typeof evaluationRecords.$inferSelect;

package/src/database/server/schemas/lobechat/relations.ts

@@ -3,8 +3,9 @@ import { relations } from 'drizzle-orm';
 import { pgTable, primaryKey, text } from 'drizzle-orm/pg-core';
 
 import { agents, agentsFiles, agentsKnowledgeBases } from './agent';
+import { asyncTasks } from './asyncTask';
 import { agentsTags, plugins, pluginsTags, tags } from './discover';
-import { asyncTasks, files, knowledgeBases } from './file';
+import { files, knowledgeBases } from './file';
 import { messages, messagesFiles } from './message';
 import { unstructuredChunks } from './rag';
 import { sessionGroups, sessions } from './session';

package/src/libs/agent-runtime/types/chat.ts

@@ -72,6 +72,8 @@ export interface ChatStreamPayload {
    * @default openai
    */
   provider?: string;
+
+  responseMode?: 'streamText' | 'json';
   /**
    * @title 是否开启流式请求
    * @default true
@@ -84,6 +86,7 @@ export interface ChatStreamPayload {
   temperature: number;
   tool_choice?: string;
   tools?: ChatCompletionTool[];
+
   /**
    * @title 控制生成文本中最高概率的单个令牌
    * @default 1

package/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts

@@ -149,7 +149,7 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
       this.baseURL = this.client.baseURL;
     }
 
-    async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
+    async chat({ responseMode, ...payload }: ChatStreamPayload, options?: ChatCompetitionOptions) {
       try {
         const postPayload = chatCompletion?.handlePayload
           ? chatCompletion.handlePayload(payload, this._options)
@@ -186,6 +186,8 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
           debugResponse(response);
         }
 
+        if (responseMode === 'json') return Response.json(response);
+
         const stream = transformResponseToStream(response as unknown as OpenAI.ChatCompletion);
 
         return StreamingResponse(OpenAIStream(stream, options?.callback), {

package/src/libs/langchain/loaders/index.ts

@@ -27,7 +27,7 @@ export class ChunkingLoader {
       const fileBlob = new Blob([Buffer.from(content)]);
       const txt = this.uint8ArrayToString(content);
 
-      const type = this.getType(filename);
+      const type = this.getType(filename?.toLowerCase());
 
       switch (type) {
         case 'code': {

package/src/locales/default/index.ts

@@ -13,6 +13,7 @@ import migration from './migration';
 import modelProvider from './modelProvider';
 import plugin from './plugin';
 import portal from './portal';
+import ragEval from './ragEval';
 import setting from './setting';
 import welcome from './welcome';
 
@@ -31,6 +32,7 @@ const resources = {
   modelProvider,
   plugin,
   portal,
+  ragEval,
   setting,
   tool,
   welcome,

package/src/locales/default/knowledgeBase.ts

@@ -23,6 +23,7 @@ export default {
     title: '新建知识库',
   },
   tab: {
+    evals: '评测',
     files: '文档',
     settings: '设置',
     testing: '召回测试',