@lobehub/lobehub 2.0.0-next.127 → 2.0.0-next.129

package/packages/database/src/repositories/dataImporter/index.ts

@@ -67,7 +67,7 @@ const IMPORT_TABLE_CONFIG: TableImportConfig[] = [
   },
   {
     conflictStrategy: 'skip',
-    preserveId: true, // 需要保留原始ID
+    preserveId: true, // Need to preserve original ID
     relations: [
       {
         field: 'providerId',
@@ -89,7 +89,7 @@ const IMPORT_TABLE_CONFIG: TableImportConfig[] = [
     uniqueConstraints: ['slug'],
   },
   {
-    // 对slug字段进行特殊处理
+    // Special processing for slug field
     fieldProcessors: {
       slug: (value) => `${value}-${uuid().slice(0, 8)}`,
     },
@@ -113,7 +113,7 @@ const IMPORT_TABLE_CONFIG: TableImportConfig[] = [
   },
   {
     conflictStrategy: 'skip',
-    isCompositeKey: true, // 使用复合主键 [agentId, sessionId]
+    isCompositeKey: true, // Uses composite primary key [agentId, sessionId]
     relations: [
       {
         field: 'agentId',
@@ -172,7 +172,7 @@ const IMPORT_TABLE_CONFIG: TableImportConfig[] = [
   },
   {
     conflictStrategy: 'skip',
-    preserveId: true, // 使用消息ID作为主键
+    preserveId: true, // Uses message ID as primary key
     relations: [
       {
         field: 'id',
@@ -182,7 +182,7 @@ const IMPORT_TABLE_CONFIG: TableImportConfig[] = [
     table: 'messagePlugins',
   },
   {
-    isCompositeKey: true, // 使用复合主键 [messageId, chunkId]
+    isCompositeKey: true, // Uses composite primary key [messageId, chunkId]
     relations: [
       {
         field: 'messageId',
@@ -196,7 +196,7 @@ const IMPORT_TABLE_CONFIG: TableImportConfig[] = [
     table: 'messageChunks',
   },
   {
-    isCompositeKey: true, // 使用复合主键 [id, queryId, chunkId]
+    isCompositeKey: true, // Uses composite primary key [id, queryId, chunkId]
     relations: [
       {
         field: 'id',
@@ -228,7 +228,7 @@ const IMPORT_TABLE_CONFIG: TableImportConfig[] = [
   // },
   {
     conflictStrategy: 'skip',
-    preserveId: true, // 使用消息ID作为主键
+    preserveId: true, // Uses message ID as primary key
     relations: [
       {
         field: 'id',
@@ -239,7 +239,7 @@ const IMPORT_TABLE_CONFIG: TableImportConfig[] = [
   },
   // {
   //   conflictStrategy: 'skip',
-  //   preserveId: true, // 使用消息ID作为主键
+  //   preserveId: true, // Uses message ID as primary key
   //   relations: [
   //     {
   //       field: 'id',
@@ -273,7 +273,7 @@ export class DataImporterRepos {
   };
 
   /**
-   * 导入PostgreSQL数据
+   * Import PostgreSQL data
    */
   async importPgData(
     dbData: ImportPgDataStructure,
@@ -282,13 +282,13 @@ export class DataImporterRepos {
     const results: Record<string, ImportResult> = {};
     const { data } = dbData;
 
-    // 初始化ID映射表和冲突记录
+    // Initialize ID mapping table and conflict records
     this.idMaps = {};
     this.conflictRecords = {};
 
     try {
       await this.db.transaction(async (trx) => {
-        // 按配置顺序导入表
+        // Import tables in configuration order
         for (const config of IMPORT_TABLE_CONFIG) {
           const { table: tableName } = config;
 
@@ -299,7 +299,7 @@ export class DataImporterRepos {
             continue;
           }
 
-          // 使用统一的导入方法
+          // Use unified import method
           const result = await this.importTableData(trx, config, tableData, conflictStrategy);
           console.log(`imported table: ${tableName}, records: ${tableData.length}`);
 
@@ -325,11 +325,11 @@ export class DataImporterRepos {
   }
 
   /**
-   * 从错误中提取详细信息
+   * Extract detailed information from error
    */
   private extractErrorDetails(error: any) {
     if (error.code === '23505') {
-      // PostgreSQL 唯一约束错误码
+      // PostgreSQL unique constraint error code
       const match = error.detail?.match(/Key \((.+?)\)=\((.+?)\) already exists/);
       if (match) {
         return {
@@ -344,7 +344,7 @@ export class DataImporterRepos {
   }
 
   /**
-   * 统一的表数据导入函数 - 处理所有类型的表
+   * Unified table data import function - Handles all types of tables
    */
   private async importTableData(
     trx: any,
@@ -368,13 +368,13 @@ export class DataImporterRepos {
     const table = EXPORT_TABLES[tableName];
     const result: ImportResult = { added: 0, errors: 0, skips: 0, updated: 0 };
 
-    // 初始化该表的ID映射
+    // Initialize ID mapping for this table
     if (!this.idMaps[tableName]) {
       this.idMaps[tableName] = {};
     }
 
     try {
-      // 1. 查找已存在的记录（基于clientId和userId）
+      // 1. Find existing records (based on clientId and userId)
       let existingRecords: any[] = [];
 
       if ('clientId' in table && 'userId' in table) {
@@ -387,7 +387,7 @@ export class DataImporterRepos {
         }
       }
 
-      // 如果需要保留原始ID，还需要检查ID是否已存在
+      // If need to preserve original ID, also check if ID already exists
       if (preserveId && !isCompositeKey) {
         const ids = tableData.map((item) => item.id).filter(Boolean);
         if (ids.length > 0) {
@@ -395,7 +395,7 @@ export class DataImporterRepos {
             where: inArray(table.id, ids),
           });
 
-          // 合并到已存在记录集合中
+          // Merge into existing records set
           existingRecords = [
             ...existingRecords,
             ...idExistingRecords.filter(
@@ -407,28 +407,28 @@ export class DataImporterRepos {
 
       result.skips = existingRecords.length;
 
-      // 2. 为已存在的记录建立ID映射
+      // 2. Establish ID mapping for existing records
       for (const record of existingRecords) {
-        // 只有非复合主键表才需要ID映射
+        // Only non-composite key tables need ID mapping
         if (!isCompositeKey) {
           this.idMaps[tableName][record.id] = record.id;
           if (record.clientId) {
             this.idMaps[tableName][record.clientId] = record.id;
           }
 
-          // 记录中可能使用的任何其他ID标识符
+          // Any other ID identifiers that may be used in records
           const originalRecord = tableData.find(
             (item) => item.id === record.id || item.clientId === record.clientId,
           );
 
           if (originalRecord) {
-            // 确保原始记录ID也映射到数据库记录ID
+            // Ensure original record ID also maps to database record ID
             this.idMaps[tableName][originalRecord.id] = record.id;
           }
         }
       }
 
-      // 3. 筛选出需要插入的记录
+      // 3. Filter out records that need to be inserted
       const recordsToInsert = tableData.filter(
         (item) =>
           !existingRecords.some(
@@ -442,22 +442,22 @@ export class DataImporterRepos {
         return result;
       }
 
-      // 4. 准备导入数据
+      // 4. Prepare import data
       const preparedData = recordsToInsert.map((item) => {
         const originalId = item.id;
 
-        // 处理日期字段
+        // Process date fields
         const dateFields: any = {};
         if (item.createdAt) dateFields.createdAt = new Date(item.createdAt);
         if (item.updatedAt) dateFields.updatedAt = new Date(item.updatedAt);
         if (item.accessedAt) dateFields.accessedAt = new Date(item.accessedAt);
 
-        // 创建新记录对象
+        // Create new record object
         let newRecord: any = {};
 
-        // 根据是否复合主键和是否保留ID决定如何处理
+        // Decide how to process based on whether it's composite key and whether to preserve ID
         if (isCompositeKey) {
-          // 对于复合主键表，不包含id字段
+          // For composite key tables, don't include id field
           // eslint-disable-next-line @typescript-eslint/no-unused-vars
           const { id: _, ...rest } = item;
           newRecord = {
@@ -467,7 +467,7 @@ export class DataImporterRepos {
             userId: this.userId,
           };
         } else {
-          // 非复合主键表处理
+          // Non-composite key table processing
           newRecord = {
             ...(preserveId ? item : { ...item, id: undefined }),
             ...dateFields,
@@ -476,19 +476,19 @@ export class DataImporterRepos {
           };
         }
 
-        // 应用字段处理器
+        // Apply field processors
         for (const field in fieldProcessors) {
           if (newRecord[field] !== undefined) {
             newRecord[field] = fieldProcessors[field](newRecord[field]);
           }
         }
 
-        // 特殊表处理
+        // Special table processing
         if (tableName === 'userSettings') {
           newRecord.id = this.userId;
         }
 
-        // 处理关系字段（外键引用）
+        // Process relation fields (foreign key references)
         for (const relation of relations) {
           const { field, sourceTable } = relation;
 
@@ -498,7 +498,7 @@ export class DataImporterRepos {
             if (mappedId) {
               newRecord[field] = mappedId;
             } else {
-              // 找不到映射，设为null
+              // Cannot find mapping, set to null
               console.warn(
                 `Could not find mapped ID for ${field}=${newRecord[field]} in table ${sourceTable}`,
               );
@@ -507,7 +507,7 @@ export class DataImporterRepos {
           }
         }
 
-        // 简化处理自引用字段 - 直接设为null
+        // Simplified processing of self-reference fields - directly set to null
         for (const selfRef of selfReferences) {
           const { field } = selfRef;
           if (newRecord[field] !== undefined) {
@@ -518,15 +518,15 @@ export class DataImporterRepos {
         return { newRecord, originalId };
       });
 
-      // 5. 检查唯一约束并应用冲突策略
+      // 5. Check unique constraints and apply conflict strategy
       for (const record of preparedData) {
         if (isCompositeKey && uniqueConstraints.length > 0) {
-          // 对于复合主键表，将所有唯一约束字段作为一个组合条件
+          // For composite key tables, treat all unique constraint fields as a combined condition
           const whereConditions = uniqueConstraints
             .filter((field) => record.newRecord[field] !== undefined)
             .map((field) => eq(table[field], record.newRecord[field]));
 
-          // 添加userId条件（如果表有userId字段）
+          // Add userId condition (if table has userId field)
           if ('userId' in table) {
             whereConditions.push(eq(table.userId, this.userId));
           }
@@ -537,7 +537,7 @@ export class DataImporterRepos {
             });
 
             if (exists) {
-              // 记录冲突
+              // Record conflict
               if (!this.conflictRecords[tableName]) this.conflictRecords[tableName] = [];
               this.conflictRecords[tableName].push({
                 field: uniqueConstraints.join(','),
@@ -546,13 +546,13 @@ export class DataImporterRepos {
                   .join(','),
               });
 
-              // 应用冲突策略
+              // Apply conflict strategy
               switch (conflictStrategy) {
                 case 'skip': {
                   record.newRecord._skip = true;
                   result.skips++;
 
-                  // 关键改进：即使跳过，也建立ID映射关系
+                  // Key improvement: establish ID mapping even if skipped
                   if (!isCompositeKey) {
                     this.idMaps[tableName][record.originalId] = exists.id;
                     if (record.newRecord.clientId) {
@@ -562,11 +562,11 @@ export class DataImporterRepos {
                   break;
                 }
                 case 'override': {
-                  // 不需要额外操作，插入时会覆盖
+                  // No additional operation needed, will be overridden on insert
                   break;
                 }
                 case 'merge': {
-                  // 合并数据
+                  // Merge data
                   await trx
                     .update(table)
                     .set(record.newRecord)
@@ -582,30 +582,30 @@ export class DataImporterRepos {
             }
           }
         } else {
-          // 处理唯一约束
+          // Process unique constraints
           for (const field of uniqueConstraints) {
             if (!record.newRecord[field]) continue;
 
-            // 检查字段值是否已存在
+            // Check if field value already exists
             const exists = await trx.query[tableName].findFirst({
               where: eq(table[field], record.newRecord[field]),
             });
 
             if (exists) {
-              // 记录冲突
+              // Record conflict
               if (!this.conflictRecords[tableName]) this.conflictRecords[tableName] = [];
               this.conflictRecords[tableName].push({
                 field,
                 value: record.newRecord[field],
               });
 
-              // 应用冲突策略
+              // Apply conflict strategy
               switch (conflictStrategy) {
                 case 'skip': {
                   record.newRecord._skip = true;
                   result.skips++;
 
-                  // 关键改进：即使跳过，也建立ID映射关系
+                  // Key improvement: establish ID mapping even if skipped
                   if (!isCompositeKey) {
                     this.idMaps[tableName][record.originalId] = exists.id;
                     if (record.newRecord.clientId) {
@@ -615,7 +615,7 @@ export class DataImporterRepos {
                   break;
                 }
                 case 'override': {
-                  // 应用字段处理器
+                  // Apply field processor
                   if (field in fieldProcessors) {
                     record.newRecord[field] = fieldProcessors[field](record.newRecord[field]);
                   }
@@ -623,7 +623,7 @@ export class DataImporterRepos {
                 }
 
                 case 'merge': {
-                  // 合并数据
+                  // Merge data
                   await trx
                     .update(table)
                     .set(record.newRecord)
@@ -641,13 +641,13 @@ export class DataImporterRepos {
         }
       }
 
-      // 过滤掉标记为跳过的记录
+      // Filter out records marked to be skipped
       const filteredData = preparedData.filter((record) => !record.newRecord._skip);
 
-      // 清除临时标记
+      // Clear temporary markers
       filteredData.forEach((record) => delete record.newRecord._skip);
 
-      // 6. 批量插入数据
+      // 6. Batch insert data
       const BATCH_SIZE = 100;
 
       for (let i = 0; i < filteredData.length; i += BATCH_SIZE) {
@@ -657,12 +657,12 @@ export class DataImporterRepos {
         const originalIds = batch.map((item) => item.originalId);
 
         try {
-          // 插入并返回结果
+          // Insert and return result
           const insertQuery = trx.insert(table).values(itemsToInsert);
 
           let insertResult;
 
-          // 只对非复合主键表需要返回ID
+          // Only non-composite key tables need to return ID
           if (!isCompositeKey) {
             const res = await insertQuery.returning();
             insertResult = res.map((item: any) => ({
@@ -671,18 +671,18 @@ export class DataImporterRepos {
             }));
           } else {
             await insertQuery;
-            insertResult = itemsToInsert.map(() => ({})); // 创建空结果以维持计数
+            insertResult = itemsToInsert.map(() => ({})); // Create empty result to maintain count
           }
 
           result.added += insertResult.length;
 
-          // 建立ID映射关系 (只对非复合主键表)
+          // Establish ID mapping relationship (only for non-composite key tables)
           if (!isCompositeKey) {
             for (const [j, newRecord] of insertResult.entries()) {
               const originalId = originalIds[j];
               this.idMaps[tableName][originalId] = newRecord.id;
 
-              // 同时确保clientId也能映射到正确的ID
+              // Also ensure clientId can map to the correct ID
               const originalRecord = tableData.find((item) => item.id === originalId);
               if (originalRecord && originalRecord.clientId) {
                 this.idMaps[tableName][originalRecord.clientId] = newRecord.id;
@@ -692,7 +692,7 @@ export class DataImporterRepos {
         } catch (error) {
           console.error(`Error batch inserting ${tableName}:`, error);
 
-          // 处理错误并记录
+          // Handle error and record
           if ((error as any).code === '23505') {
             const match = (error as any).detail?.match(/Key \((.+?)\)=\((.+?)\) already exists/);
             if (match) {

package/packages/database/src/repositories/knowledge/index.test.ts

@@ -146,17 +146,29 @@ describe('KnowledgeRepo', () => {
       expect(regularFile?.sourceType).toBe('file');
     });
 
-    it('should show all documents in All category (no filtering)', async () => {
+    it('should exclude sourceType=file documents from All category', async () => {
       const results = await knowledgeRepo.query({ category: FilesTabs.All });
 
-      // All category should include everything
-      expect(results.length).toBeGreaterThanOrEqual(6); // 2 files + 4 documents
+      // All category should include files from files table + documents with sourceType != 'file'
+      // 2 files + 2 documents (api-pdf and web-doc)
+      // Excluded: uploaded-pdf and editor-doc both have sourceType='file'
+      expect(results.length).toBe(4);
 
+      // Should NOT include documents with sourceType='file' (globally excluded now)
       const editorDoc = results.find((item) => item.name === 'editor-doc.md');
       const uploadedPdf = results.find((item) => item.name === 'uploaded-pdf.pdf');
+      expect(editorDoc).toBeUndefined();
+      expect(uploadedPdf).toBeUndefined();
 
-      expect(editorDoc).toBeDefined();
-      expect(uploadedPdf).toBeDefined();
+      // Should include documents with sourceType != 'file'
+      const apiPdf = results.find((item) => item.name === 'api-pdf.pdf');
+      const webDoc = results.find((item) => item.name === 'web-doc.txt');
+      expect(apiPdf).toBeDefined();
+      expect(webDoc).toBeDefined();
+
+      // Should include files from files table
+      const regularFile = results.find((item) => item.name === 'regular-pdf-file.pdf');
+      expect(regularFile).toBeDefined();
     });
 
     it('should apply both filters together in Documents category', async () => {

package/packages/database/src/repositories/knowledge/index.ts

@@ -281,7 +281,10 @@ export class KnowledgeRepo {
     q,
     knowledgeBaseId,
   }: QueryFileListParams = {}): ReturnType<typeof sql> {
-    let whereConditions: any[] = [sql`${documents.userId} = ${this.userId}`];
+    let whereConditions: any[] = [
+      sql`${documents.userId} = ${this.userId}`,
+      sql`${documents.sourceType} != ${'file'}`,
+    ];
 
     // Search filter
     if (q) {
@@ -300,12 +303,9 @@ export class KnowledgeRepo {
         );
         whereConditions.push(sql`(${sql.join(orConditions, sql` OR `)})`);
 
-        // Exclude custom/document and source_type='file' from Documents category
+        // Exclude custom/document from Documents category
         if (category === FilesTabs.Documents) {
-          whereConditions.push(
-            sql`${documents.fileType} != ${'custom/document'}`,
-            sql`${documents.sourceType} != ${'file'}`,
-          );
+          whereConditions.push(sql`${documents.fileType} != ${'custom/document'}`);
         }
       } else if (fileTypePrefix) {
         whereConditions.push(sql`${documents.fileType} ILIKE ${`${fileTypePrefix}%`}`);

package/packages/database/src/schemas/generation.ts

@@ -10,7 +10,7 @@ import { files } from './file';
 import { users } from './user';
 
 /**
- * 生成主题表 - 用于组织和管理 AI 生成内容的主题
+ * Generation topics table - Used to organize and manage AI-generated content topics
  */
 export const generationTopics = pgTable('generation_topics', {
   id: text('id')
@@ -22,10 +22,10 @@ export const generationTopics = pgTable('generation_topics', {
     .references(() => users.id, { onDelete: 'cascade' })
     .notNull(),
 
-  /** 简要描述主题内容，由 LLM 生成 */
+  /** Brief description of topic content, generated by LLM */
   title: text('title'),
 
-  /** 主题封面图片 URL */
+  /** Topic cover image URL */
   coverUrl: text('cover_url'),
 
   ...timestamps,
@@ -37,7 +37,7 @@ export type NewGenerationTopic = typeof generationTopics.$inferInsert;
 export type GenerationTopicItem = typeof generationTopics.$inferSelect;
 
 /**
- * 生成批次表 - 存储一次生成请求的配置信息
+ * Generation batches table - Stores configuration information for a single generation request
  */
 export const generationBatches = pgTable('generation_batches', {
   id: text('id')
@@ -53,25 +53,25 @@ export const generationBatches = pgTable('generation_batches', {
     .notNull()
     .references(() => generationTopics.id, { onDelete: 'cascade' }),
 
-  /** 服务商名称 */
+  /** Provider name */
   provider: text('provider').notNull(),
 
-  /** 模型名称 */
+  /** Model name */
   model: text('model').notNull(),
 
-  /** 生成提示词 */
+  /** Generation prompt */
   prompt: text('prompt').notNull(),
 
-  /** 图片宽度 */
+  /** Image width */
   width: integer('width'),
 
-  /** 图片高度 */
+  /** Image height */
   height: integer('height'),
 
-  /** 图片比例 */
+  /** Image aspect ratio */
   ratio: varchar('ratio', { length: 64 }),
 
-  /** 存储生成批次的配置，存放不需要建立索引的公共配置 */
+  /** Stores generation batch configuration for common settings that don't need indexing */
   config: jsonb('config'),
 
   ...timestamps,
@@ -86,7 +86,7 @@ export type GenerationBatchWithGenerations = GenerationBatchItem & {
 };
 
 /**
- *  存储单个 AI 生成信息
+ * Stores individual AI generation information
  */
 export const generations = pgTable('generations', {
   id: text('id')
@@ -102,18 +102,18 @@ export const generations = pgTable('generations', {
     .notNull()
     .references(() => generationBatches.id, { onDelete: 'cascade' }),
 
-  /** 关联的异步任务 ID */
+  /** Associated async task ID */
   asyncTaskId: uuid('async_task_id').references(() => asyncTasks.id, {
     onDelete: 'set null',
   }),
 
-  /** 关联的生成文件 ID，删除文件时连带删除生成记录 */
+  /** Associated generated file ID, deletes generation record when file is deleted */
   fileId: text('file_id').references(() => files.id, { onDelete: 'cascade' }),
 
-  /** 生成种子值 */
+  /** Generation seed value */
   seed: integer('seed'),
 
-  /** 生成的资源信息，包含存储在 s3 上的 key, 图片实际宽高，缩略图 key 等 */
+  /** Generated asset information, including S3 storage key, actual image dimensions, thumbnail key, etc. */
   asset: jsonb('asset').$type<ImageGenerationAsset>(),
 
   ...timestamps,

package/packages/database/src/schemas/nextauth.ts

@@ -20,7 +20,7 @@ export const nextauthAccounts = pgTable(
     session_state: text('session_state'),
     token_type: text('token_type'),
     type: text('type').$type<AdapterAccount>().notNull(),
-    userId: text('userId')
+    userId: text('user_id')
       .notNull()
       .references(() => users.id, { onDelete: 'cascade' }),
   },
@@ -40,7 +40,7 @@ export const nextauthAccounts = pgTable(
 export const nextauthSessions = pgTable(`nextauth_sessions`, {
   expires: timestamp('expires', { mode: 'date' }).notNull(),
   sessionToken: text('sessionToken').primaryKey(),
-  userId: text('userId')
+  userId: text('user_id')
     .notNull()
     .references(() => users.id, { onDelete: 'cascade' }),
 });
@@ -77,7 +77,7 @@ export const nextauthAuthenticators = pgTable(
     credentialPublicKey: text('credentialPublicKey').notNull(),
     providerAccountId: text('providerAccountId').notNull(),
     transports: text('transports'),
-    userId: text('userId')
+    userId: text('user_id')
       .notNull()
       .references(() => users.id, { onDelete: 'cascade' }),
   },