@lobehub/lobehub 2.0.0-next.169 → 2.0.0-next.170

package/packages/database/migrations/meta/_journal.json

@@ -427,6 +427,13 @@
       "when": 1765437218969,
       "tag": "0060_add_user_last_active_at",
       "breakpoints": true
+    },
+    {
+      "idx": 61,
+      "version": "7",
+      "when": 1765540257262,
+      "tag": "0061_add_document_and_memory_index",
+      "breakpoints": true
     }
   ],
   "version": "6"

package/packages/database/src/core/migrations.json

@@ -946,5 +946,31 @@
     "bps": true,
     "folderMillis": 1765437218969,
     "hash": "004923916a27bc61294eeb20738f2884f1ca43978a03aa9f0216fc849c98465b"
+  },
+  {
+    "sql": [
+      "ALTER TABLE \"user_memories\" ADD COLUMN \"captured_at\" timestamp with time zone DEFAULT now() NOT NULL;",
+      "\nALTER TABLE \"user_memories_contexts\" ADD COLUMN \"captured_at\" timestamp with time zone DEFAULT now() NOT NULL;",
+      "\nALTER TABLE \"user_memories_experiences\" ADD COLUMN \"captured_at\" timestamp with time zone DEFAULT now() NOT NULL;",
+      "\nALTER TABLE \"user_memories_identities\" ADD COLUMN \"captured_at\" timestamp with time zone DEFAULT now() NOT NULL;",
+      "\nALTER TABLE \"user_memories_preferences\" ADD COLUMN \"captured_at\" timestamp with time zone DEFAULT now() NOT NULL;",
+      "\nCREATE INDEX \"agents_user_id_idx\" ON \"agents\" USING btree (\"user_id\");",
+      "\nCREATE INDEX \"documents_source_type_idx\" ON \"documents\" USING btree (\"source_type\");",
+      "\nCREATE INDEX \"documents_user_id_idx\" ON \"documents\" USING btree (\"user_id\");",
+      "\nCREATE INDEX \"files_user_id_idx\" ON \"files\" USING btree (\"user_id\");",
+      "\nCREATE INDEX \"users_created_at_idx\" ON \"users\" USING btree (\"created_at\");",
+      "\nCREATE INDEX \"users_banned_true_created_at_idx\" ON \"users\" USING btree (\"created_at\") WHERE \"users\".\"banned\" = true;",
+      "\nCREATE INDEX \"user_memories_user_id_index\" ON \"user_memories\" USING btree (\"user_id\");",
+      "\nCREATE INDEX \"user_memories_contexts_user_id_index\" ON \"user_memories_contexts\" USING btree (\"user_id\");",
+      "\nCREATE INDEX \"user_memories_experiences_user_id_index\" ON \"user_memories_experiences\" USING btree (\"user_id\");",
+      "\nCREATE INDEX \"user_memories_experiences_user_memory_id_index\" ON \"user_memories_experiences\" USING btree (\"user_memory_id\");",
+      "\nCREATE INDEX \"user_memories_identities_user_id_index\" ON \"user_memories_identities\" USING btree (\"user_id\");",
+      "\nCREATE INDEX \"user_memories_identities_user_memory_id_index\" ON \"user_memories_identities\" USING btree (\"user_memory_id\");",
+      "\nCREATE INDEX \"user_memories_preferences_user_id_index\" ON \"user_memories_preferences\" USING btree (\"user_id\");",
+      "\nCREATE INDEX \"user_memories_preferences_user_memory_id_index\" ON \"user_memories_preferences\" USING btree (\"user_memory_id\");"
+    ],
+    "bps": true,
+    "folderMillis": 1765540257262,
+    "hash": "2cc13eba2a174ebf18ac32a3ae673338a3dd46ec37a22108a7783a80f44a5286"
   }
 ]

package/packages/database/src/models/__tests__/document.test.ts

@@ -115,8 +115,10 @@ describe('DocumentModel', () => {
       const userDocs = await documentModel.query();
       const otherUserDocs = await documentModel2.query();
 
-      expect(userDocs).toHaveLength(0);
-      expect(otherUserDocs).toHaveLength(1);
+      expect(userDocs.items).toHaveLength(0);
+      expect(userDocs.total).toBe(0);
+      expect(otherUserDocs.items).toHaveLength(1);
+      expect(otherUserDocs.total).toBe(1);
     });
   });
 
@@ -127,7 +129,8 @@ describe('DocumentModel', () => {
 
       const result = await documentModel.query();
 
-      expect(result).toHaveLength(2);
+      expect(result.items).toHaveLength(2);
+      expect(result.total).toBe(2);
     });
 
     it('should only return documents for the current user', async () => {
@@ -136,8 +139,9 @@ describe('DocumentModel', () => {
 
       const result = await documentModel.query();
 
-      expect(result).toHaveLength(1);
-      expect(result[0].content).toBe('User 1 document');
+      expect(result.items).toHaveLength(1);
+      expect(result.total).toBe(1);
+      expect(result.items[0].content).toBe(null); // content is excluded in query
     });
 
     it('should return documents ordered by updatedAt desc', async () => {
@@ -160,8 +164,8 @@ describe('DocumentModel', () => {
 
       const result = await documentModel.query();
 
-      expect(result[0].id).toBe(doc1Id);
-      expect(result[1].id).toBe(doc2Id);
+      expect(result.items[0].id).toBe(doc1Id);
+      expect(result.items[1].id).toBe(doc2Id);
     });
   });
 

package/packages/database/src/models/__tests__/userMemories.test.ts

@@ -109,6 +109,7 @@ function generateRandomCreateUserMemoryExperienceParams() {
       situationVector: generateRandomEmbedding(),
       tags: [],
       type: 'learning',
+      capturedAt: new Date(),
     },
   } as CreateUserMemoryExperienceParams;
 }
@@ -127,6 +128,7 @@ function generateRandomCreateUserMemoryIdentityParams() {
       role: 'role ' + nanoid(),
       tags: [],
       type: 'personal',
+      capturedAt: new Date(),
     },
   } as CreateUserMemoryIdentityParams;
 }
@@ -148,6 +150,7 @@ function generateRandomCreateUserMemoryContextParams() {
       titleVector: generateRandomEmbedding(),
       type: 'environment',
       userMemoryIds: [],
+      capturedAt: new Date(),
     },
   } as CreateUserMemoryContextParams;
 }
@@ -163,6 +166,7 @@ function generateRandomCreateUserMemoryPreferenceParams() {
       suggestions: 'suggestions ' + nanoid(),
       tags: [],
       type: 'choice',
+      capturedAt: new Date(),
     },
   } as CreateUserMemoryPreferenceParams;
 }

package/packages/database/src/models/document.ts

@@ -1,8 +1,15 @@
-import { and, desc, eq } from 'drizzle-orm';
+import { and, count, desc, eq, inArray } from 'drizzle-orm';
 
 import { DocumentItem, NewDocument, documents } from '../schemas';
 import { LobeChatDatabase } from '../type';
 
+export interface QueryDocumentParams {
+  current?: number;
+  fileTypes?: string[];
+  pageSize?: number;
+  sourceTypes?: string[];
+}
+
 export class DocumentModel {
   private userId: string;
   private db: LobeChatDatabase;
@@ -31,11 +38,72 @@ export class DocumentModel {
     return this.db.delete(documents).where(eq(documents.userId, this.userId));
   };
 
-  query = async (): Promise<DocumentItem[]> => {
-    return this.db.query.documents.findMany({
-      orderBy: [desc(documents.updatedAt)],
-      where: eq(documents.userId, this.userId),
-    });
+  query = async ({
+    current = 0,
+    pageSize = 9999,
+    fileTypes,
+    sourceTypes,
+  }: QueryDocumentParams = {}): Promise<{
+    items: DocumentItem[];
+    total: number;
+  }> => {
+    const offset = current * pageSize;
+    const conditions = [eq(documents.userId, this.userId)];
+
+    if (fileTypes?.length) {
+      conditions.push(inArray(documents.fileType, fileTypes));
+    }
+
+    if (sourceTypes?.length) {
+      conditions.push(inArray(documents.sourceType, sourceTypes as ('file' | 'web' | 'api')[]));
+    }
+
+    const whereCondition = and(...conditions);
+
+    // Fetch items and total count in parallel
+    // Optimize: Exclude large JSONB fields (content, pages, editorData) for better performance
+    const [rawItems, totalResult] = await Promise.all([
+      this.db
+        .select({
+          accessedAt: documents.accessedAt,
+          clientId: documents.clientId,
+          createdAt: documents.createdAt,
+          fileId: documents.fileId,
+          fileType: documents.fileType,
+          filename: documents.filename,
+          id: documents.id,
+          metadata: documents.metadata,
+          parentId: documents.parentId,
+          slug: documents.slug,
+          source: documents.source,
+          sourceType: documents.sourceType,
+          title: documents.title,
+          totalCharCount: documents.totalCharCount,
+          totalLineCount: documents.totalLineCount,
+          updatedAt: documents.updatedAt,
+          userId: documents.userId,
+          // Exclude large fields: content, pages, editorData
+        })
+        .from(documents)
+        .where(whereCondition)
+        .orderBy(desc(documents.updatedAt))
+        .limit(pageSize)
+        .offset(offset),
+      this.db
+        .select({ count: count(documents.id) })
+        .from(documents)
+        .where(whereCondition),
+    ]);
+
+    // Map to DocumentItem type with excluded fields as null
+    const items = rawItems.map((item) => ({
+      ...item,
+      content: null,
+      editorData: null,
+      pages: null,
+    })) as DocumentItem[];
+
+    return { items, total: totalResult[0].count };
   };
 
   findById = async (id: string): Promise<DocumentItem | undefined> => {

package/packages/database/src/models/userMemory.ts

@@ -753,6 +753,7 @@ export class UserMemoryModel {
         accessedAt: userMemoriesContexts.accessedAt,
         associatedObjects: userMemoriesContexts.associatedObjects,
         associatedSubjects: userMemoriesContexts.associatedSubjects,
+        capturedAt: userMemoriesContexts.capturedAt,
         createdAt: userMemoriesContexts.createdAt,
         currentStatus: userMemoriesContexts.currentStatus,
         description: userMemoriesContexts.description,
@@ -803,6 +804,7 @@ export class UserMemoryModel {
       .select({
         accessedAt: userMemoriesExperiences.accessedAt,
         action: userMemoriesExperiences.action,
+        capturedAt: userMemoriesExperiences.capturedAt,
         createdAt: userMemoriesExperiences.createdAt,
         id: userMemoriesExperiences.id,
         keyLearning: userMemoriesExperiences.keyLearning,
@@ -852,6 +854,7 @@ export class UserMemoryModel {
     let query = this.db
       .select({
         accessedAt: userMemoriesPreferences.accessedAt,
+        capturedAt: userMemoriesPreferences.capturedAt,
         conclusionDirectives: userMemoriesPreferences.conclusionDirectives,
         createdAt: userMemoriesPreferences.createdAt,
         id: userMemoriesPreferences.id,

package/packages/database/src/schemas/agent.ts

@@ -64,6 +64,7 @@ export const agents = pgTable(
   (t) => [
     uniqueIndex('client_id_user_id_unique').on(t.clientId, t.userId),
     uniqueIndex('agents_slug_user_id_unique').on(t.slug, t.userId),
+    index('agents_user_id_idx').on(t.userId),
     index('agents_title_idx').on(t.title),
     index('agents_description_idx').on(t.description),
   ],

package/packages/database/src/schemas/file.ts

@@ -97,6 +97,8 @@ export const documents = pgTable(
   (table) => [
     index('documents_source_idx').on(table.source),
     index('documents_file_type_idx').on(table.fileType),
+    index('documents_source_type_idx').on(table.sourceType),
+    index('documents_user_id_idx').on(table.userId),
     index('documents_file_id_idx').on(table.fileId),
     index('documents_parent_id_idx').on(table.parentId),
     uniqueIndex('documents_client_id_user_id_unique').on(table.clientId, table.userId),
@@ -152,6 +154,7 @@ export const files = pgTable(
   (table) => {
     return {
       fileHashIdx: index('file_hash_idx').on(table.fileHash),
+      userIdIdx: index('files_user_id_idx').on(table.userId),
       parentIdIdx: index('files_parent_id_idx').on(table.parentId),
       clientIdUnique: uniqueIndex('files_client_id_user_id_unique').on(
         table.clientId,

package/packages/database/src/schemas/user.ts

@@ -2,6 +2,7 @@
 import { DEFAULT_PREFERENCE } from '@lobechat/const';
 import type { CustomPluginParams } from '@lobechat/types';
 import { LobeChatPluginManifest } from '@lobehub/chat-plugin-sdk';
+import { sql } from 'drizzle-orm';
 import { boolean, index, jsonb, pgTable, primaryKey, text } from 'drizzle-orm/pg-core';
 
 import { timestamps, timestamptz, varchar255 } from './_helpers';
@@ -46,10 +47,18 @@ export const users = pgTable(
 
     ...timestamps,
   },
-  (table) => ({
-    emailIdx: index('users_email_idx').on(table.email),
-    usernameIdx: index('users_username_idx').on(table.username),
-  }),
+  (table) => [
+    index('users_email_idx').on(table.email),
+    index('users_username_idx').on(table.username),
+    index('users_created_at_idx').on(table.createdAt),
+    /**
+     * Partial index to speed up admin queries on banned users.
+     * Only rows with banned=true are indexed.
+     */
+    index('users_banned_true_created_at_idx')
+      .on(table.createdAt)
+      .where(sql`${table.banned} = true`),
+  ],
 );
 
 export type NewUser = typeof users.$inferInsert;

package/packages/database/src/schemas/userMemories.ts

@@ -30,6 +30,7 @@ export const userMemories = pgTable(
 
     accessedCount: bigint('accessed_count', { mode: 'number' }).default(0),
     lastAccessedAt: timestamptz('last_accessed_at').notNull(),
+    capturedAt: timestamptz('captured_at').notNull().defaultNow(),
 
     ...timestamps,
   },
@@ -42,6 +43,7 @@ export const userMemories = pgTable(
       'hnsw',
       table.detailsVector1024.op('vector_cosine_ops'),
     ),
+    index('user_memories_user_id_index').on(table.userId),
   ],
 );
 
@@ -72,6 +74,8 @@ export const userMemoriesContexts = pgTable(
     scoreImpact: numeric('score_impact', { mode: 'number' }).default(0),
     scoreUrgency: numeric('score_urgency', { mode: 'number' }).default(0),
 
+    capturedAt: timestamptz('captured_at').notNull().defaultNow(),
+
     ...timestamps,
   },
   (table) => [
@@ -84,6 +88,7 @@ export const userMemoriesContexts = pgTable(
       table.descriptionVector.op('vector_cosine_ops'),
     ),
     index('user_memories_contexts_type_index').on(table.type),
+    index('user_memories_contexts_user_id_index').on(table.userId),
   ],
 );
 
@@ -110,6 +115,8 @@ export const userMemoriesPreferences = pgTable(
 
     scorePriority: numeric('score_priority', { mode: 'number' }).default(0),
 
+    capturedAt: timestamptz('captured_at').notNull().defaultNow(),
+
     ...timestamps,
   },
   (table) => [
@@ -117,6 +124,8 @@ export const userMemoriesPreferences = pgTable(
       'hnsw',
       table.conclusionDirectivesVector.op('vector_cosine_ops'),
     ),
+    index('user_memories_preferences_user_id_index').on(table.userId),
+    index('user_memories_preferences_user_memory_id_index').on(table.userMemoryId),
   ],
 );
 
@@ -142,6 +151,8 @@ export const userMemoriesIdentities = pgTable(
     relationship: varchar255('relationship'),
     role: text('role'),
 
+    capturedAt: timestamptz('captured_at').notNull().defaultNow(),
+
     ...timestamps,
   },
   (table) => [
@@ -150,6 +161,8 @@ export const userMemoriesIdentities = pgTable(
       table.descriptionVector.op('vector_cosine_ops'),
     ),
     index('user_memories_identities_type_index').on(table.type),
+    index('user_memories_identities_user_id_index').on(table.userId),
+    index('user_memories_identities_user_memory_id_index').on(table.userMemoryId),
   ],
 );
 
@@ -180,6 +193,8 @@ export const userMemoriesExperiences = pgTable(
 
     scoreConfidence: real('score_confidence').default(0),
 
+    capturedAt: timestamptz('captured_at').notNull().defaultNow(),
+
     ...timestamps,
   },
   (table) => [
@@ -196,6 +211,8 @@ export const userMemoriesExperiences = pgTable(
       table.keyLearningVector.op('vector_cosine_ops'),
     ),
     index('user_memories_experiences_type_index').on(table.type),
+    index('user_memories_experiences_user_id_index').on(table.userId),
+    index('user_memories_experiences_user_memory_id_index').on(table.userMemoryId),
   ],
 );
 

package/src/services/document/index.ts

@@ -24,7 +24,7 @@ export class DocumentService {
     return lambdaClient.document.createDocument.mutate(params);
   }
 
-  async queryDocuments(): Promise<DocumentItem[]> {
+  async queryDocuments(): Promise<{ items: DocumentItem[]; total: number }> {
     return lambdaClient.document.queryDocuments.query();
   }
 

package/src/store/file/slices/document/action.ts

@@ -189,7 +189,7 @@ export const createDocumentSlice: StateCreator<
     set({ isDocumentListLoading: true }, false, n('fetchDocuments/start'));
 
     try {
-      const documentItems = await documentService.queryDocuments();
+      const { items: documentItems } = await documentService.queryDocuments();
       const documents = documentItems.filter(isAllowedDocument).map((doc) => ({
         ...doc,
         filename: doc.filename ?? doc.title ?? 'Untitled',