@lobehub/chat 1.84.27 → 1.85.0

package/packages/file-loaders/src/loaders/pdf/index.ts

@@ -1,37 +1,54 @@
+import debug from 'debug';
 import { readFile } from 'node:fs/promises';
-import * as pdfjsLib from 'pdfjs-dist';
-import type { PDFDocumentProxy, PDFPageProxy, TextContent } from 'pdfjs-dist/types/src/display/api';
+import type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist';
+import { getDocument, version } from 'pdfjs-dist/legacy/build/pdf.mjs';
+// @ts-ignore
+import * as _pdfjsWorker from 'pdfjs-dist/legacy/build/pdf.worker.mjs';
+import type { TextContent } from 'pdfjs-dist/types/src/display/api';
 
 import type { DocumentPage, FileLoaderInterface } from '../../types';
 
+const log = debug('file-loaders:pdf');
+
 /**
  * Loads PDF files page by page using the official pdfjs-dist library.
  */
 export class PdfLoader implements FileLoaderInterface {
   private pdfInstance: PDFDocumentProxy | null = null;
+  private pdfjsWorker = _pdfjsWorker;
 
   private async getPDFFile(filePath: string) {
+    // GlobalWorkerOptions.workerSrc should have been set at the module level.
+    // We are now relying on pdfjs-dist to use this path when it creates a worker.
+
+    log('Reading PDF file:', filePath);
     const dataBuffer = await readFile(filePath);
+    log('PDF file read successfully, size:', dataBuffer.length, 'bytes');
 
-    const loadingTask = pdfjsLib.getDocument({
+    const loadingTask = getDocument({
       data: new Uint8Array(dataBuffer.buffer, dataBuffer.byteOffset, dataBuffer.length),
       useSystemFonts: true,
-      // Explicitly disable worker thread
-      worker: undefined, // Attempt to use system fonts
     });
 
-    return await loadingTask.promise;
+    log('PDF document loading task created');
+    const pdf = await loadingTask.promise;
+    log('PDF document loaded successfully, pages:', pdf.numPages);
+    return pdf;
   }
 
   async loadPages(filePath: string): Promise<DocumentPage[]> {
+    log('Starting to load PDF pages from:', filePath);
     try {
       const pdf: PDFDocumentProxy = await this.getPDFFile(filePath);
 
       const pages: DocumentPage[] = [];
+      log(`Processing ${pdf.numPages} PDF pages`);
 
       for (let i = 1; i <= pdf.numPages; i += 1) {
+        log(`Loading page ${i}/${pdf.numPages}`);
         const page: PDFPageProxy = await pdf.getPage(i);
         const content: TextContent = await page.getTextContent();
+        log(`Page ${i} text content retrieved, items:`, content.items.length);
 
         // --- Revert to EXACT Simple Langchain PDFLoader Logic ---
         let lastY;
@@ -61,6 +78,7 @@ export class PdfLoader implements FileLoaderInterface {
         const pageLines = cleanedPageContent.split('\n');
         const lineCount = pageLines.length;
         const charCount = cleanedPageContent.length;
+        log(`Page ${i} processed, lines: ${lineCount}, chars: ${charCount}`);
 
         pages.push({
           charCount,
@@ -70,15 +88,19 @@ export class PdfLoader implements FileLoaderInterface {
         });
 
         // Clean up page resources
+        log(`Cleaning up page ${i} resources`);
         page.cleanup();
       }
 
       // Clean up document resources
+      log('Cleaning up PDF document resources');
       await pdf.destroy();
 
+      log(`PDF loading completed for ${filePath}, total pages:`, pages.length);
       return pages;
     } catch (e) {
       const error = e as Error;
+      log('Error encountered while loading PDF file');
       console.error(
         `Error loading PDF file ${filePath} using pdfjs-dist: ${error.message}`,
         error.stack,
@@ -92,6 +114,7 @@ export class PdfLoader implements FileLoaderInterface {
         },
         pageContent: '',
       };
+      log('Created error page for failed PDF loading');
       return [errorPage];
     }
   }
@@ -103,25 +126,42 @@ export class PdfLoader implements FileLoaderInterface {
    * @returns Aggregated content as a string.
    */
   async aggregateContent(pages: DocumentPage[]): Promise<string> {
-    return pages
-      .filter((page) => !page.metadata.error)
-      .map((page) => page.pageContent)
-      .join('\n\n');
+    log('Aggregating content from', pages.length, 'PDF pages');
+    const validPages = pages.filter((page) => !page.metadata.error);
+    log(
+      `Found ${validPages.length} valid pages for aggregation (${pages.length - validPages.length} pages with errors filtered out)`,
+    );
+
+    const result = validPages.map((page) => page.pageContent).join('\n\n');
+    log('PDF content aggregated successfully, length:', result.length);
+    return result;
   }
 
   async attachDocumentMetadata(filePath: string): Promise<any> {
+    log('Attaching document metadata for PDF:', filePath);
     const pdf: PDFDocumentProxy = await this.getPDFFile(filePath);
 
-    const pdfMetadata = (await pdf.getMetadata().catch(() => null)) ?? null;
+    log('Getting PDF metadata');
+    const pdfMetadata =
+      (await pdf.getMetadata().catch((err) => {
+        log('Error retrieving PDF metadata');
+        console.error(`Error getting PDF metadata: ${err.message}`);
+        return null;
+      })) ?? null;
+
     const pdfInfo = pdfMetadata?.info ?? {};
     const metadata = pdfMetadata?.metadata ?? null;
+    log('PDF metadata retrieved:', {
+      hasInfo: !!Object.keys(pdfInfo).length,
+      hasMetadata: !!metadata,
+    });
 
     return {
       pdfInfo: pdfInfo,
       // PDF info (Author, Title, etc.)
       pdfMetadata: metadata,
       // PDF metadata
-      pdfVersion: pdfjsLib.version,
+      pdfVersion: version,
     };
   }
 }

package/packages/file-loaders/src/loaders/pptx/index.ts

@@ -1,8 +1,11 @@
+import debug from 'debug';
 import path from 'node:path';
 
 import type { DocumentPage, FileLoaderInterface } from '../../types';
 import { type ExtractedFile, extractFiles, parseString } from '../../utils/parser-utils';
 
+const log = debug('file-loaders:pptx');
+
 /**
  * Represents a loader for PPTX files using extracted utility functions.
  *
@@ -19,20 +22,25 @@ export class PptxLoader implements FileLoaderInterface {
    *          `DocumentPage` object with error information in its metadata.
    */
   async loadPages(filePath: string): Promise<DocumentPage[]> {
+    log('Loading PPTX file:', filePath);
     const sourceFileName = path.basename(filePath);
+    log('Source file name:', sourceFileName);
 
     try {
       // --- File Extraction Step ---
       const slidesRegex = /ppt\/slides\/slide\d+\.xml/g;
       const slideNumberRegex = /slide(\d+)\.xml/;
 
+      log('Extracting slide XML files from PPTX');
       // Extract only slide XML files
       const slideFiles: ExtractedFile[] = await extractFiles(filePath, (fileName) =>
         slidesRegex.test(fileName),
       );
+      log('Extracted slide files:', slideFiles.length);
 
       // --- Validation Step ---
       if (slideFiles.length === 0) {
+        log('No slide XML files found in the PPTX file');
         console.warn(`No slide XML files found in ${sourceFileName}. May be corrupted or empty.`);
         return [
           this.createErrorPage(
@@ -43,6 +51,7 @@ export class PptxLoader implements FileLoaderInterface {
       }
 
       // --- Sorting Step ---
+      log('Sorting slide files by slide number');
       // Sort files based on the slide number extracted from the path
       slideFiles.sort((a, b) => {
         const matchA = a.path.match(slideNumberRegex);
@@ -51,13 +60,17 @@ export class PptxLoader implements FileLoaderInterface {
         const numB = matchB ? parseInt(matchB[1], 10) : Infinity;
         return numA - numB;
       });
+      log('Slide files sorted');
 
       // --- Page Creation Step ---
+      log('Creating document pages from slide files');
       const pages: DocumentPage[] = slideFiles
         .map((slideFile, index) => {
           try {
+            log(`Processing slide ${index + 1}/${slideFiles.length}, path: ${slideFile.path}`);
             const xmlDoc = parseString(slideFile.content);
             const paragraphNodes = xmlDoc.getElementsByTagName('a:p');
+            log(`Found ${paragraphNodes.length} paragraph nodes in slide ${index + 1}`);
 
             const slideText = Array.from(paragraphNodes)
               .map((pNode) => {
@@ -72,6 +85,9 @@ export class PptxLoader implements FileLoaderInterface {
             const lines = slideText.split('\n');
             const slideNumberMatch = slideFile.path.match(slideNumberRegex);
             const slideNumber = slideNumberMatch ? parseInt(slideNumberMatch[1], 10) : index + 1; // Fallback to index if regex fails
+            log(
+              `Slide ${index + 1} text extracted, lines: ${lines.length}, characters: ${slideText.length}`,
+            );
 
             const metadata = {
               pageCount: slideFiles.length, // Total number of slides found
@@ -86,6 +102,7 @@ export class PptxLoader implements FileLoaderInterface {
               pageContent: slideText.trim(), // Trim final content
             };
           } catch (parseError) {
+            log(`Error parsing slide ${slideFile.path}`);
             console.error(
               `Failed to parse XML for slide ${slideFile.path} in ${sourceFileName}: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
             );
@@ -101,9 +118,11 @@ export class PptxLoader implements FileLoaderInterface {
         })
         // Filter out any potential nulls if we change the error handling above
         .filter((page): page is DocumentPage => page !== null);
+      log(`Created ${pages.length} document pages from slides`);
 
       if (pages.length === 0) {
         // This case might happen if all slides failed to parse
+        log('Parsing resulted in zero valid pages');
         console.warn(`Parsing resulted in zero valid pages for ${sourceFileName}`);
         return [this.createErrorPage('Parsing resulted in zero valid pages.', sourceFileName)];
       }
@@ -112,15 +131,18 @@ export class PptxLoader implements FileLoaderInterface {
       const allErrored = pages.every((page) => page.metadata?.error);
       if (allErrored) {
         // If all pages resulted in errors, perhaps return a single summary error
+        log('All slides failed to parse');
         console.warn(`All slides failed to parse for ${sourceFileName}`);
         return [this.createErrorPage('All slides failed to parse correctly.', sourceFileName)];
         // Or return all the individual error pages: return pages;
       }
 
+      log('PPTX loading completed successfully');
       return pages;
     } catch (error) {
       // --- Error Handling Step ---
       // This catches errors from extractFiles or other unexpected issues
+      log('Error loading or processing PPTX file');
       const errorMessage = `Failed to load or process PPTX file: ${error instanceof Error ? error.message : String(error)}`;
       console.error(errorMessage, { filePath });
       return [this.createErrorPage(errorMessage, sourceFileName)];
@@ -137,16 +159,21 @@ export class PptxLoader implements FileLoaderInterface {
    * @returns A Promise resolving to the aggregated content string.
    */
   async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    log('Aggregating content from', pages.length, 'PPTX pages');
     // Ensure pages array is valid and non-empty before proceeding
     // Filter out error pages before aggregation unless we want to include error messages
     const validPages = pages.filter((page) => !page.metadata?.error);
+    log(
+      `Found ${validPages.length} valid pages for aggregation (${pages.length - validPages.length} error pages filtered out)`,
+    );
 
     if (validPages.length === 0) {
       // If only error pages existed, return empty or a summary error message
+      log('No valid pages found, returning content of first page (may be error page)');
       return pages[0]?.pageContent || ''; // Return content of the first page (might be an error page)
     }
 
-    return validPages
+    const result = validPages
       .map((page) => {
         const slideNumber = page.metadata?.slideNumber;
         // Use Markdown H2 for slide headers
@@ -156,6 +183,9 @@ ${page.pageContent}
 </slide_page>`;
       })
       .join('\n\n'); // Use Markdown horizontal rule as separator
+
+    log('PPTX content aggregated successfully, length:', result.length);
+    return result;
   }
 
   /**
@@ -171,6 +201,7 @@ ${page.pageContent}
     sourceFileName: string,
     sourceFilePath?: string,
   ): DocumentPage {
+    log('Creating error page:', errorInfo);
     return {
       charCount: 0,
       lineCount: 0,

package/packages/file-loaders/src/loaders/text/index.test.ts

@@ -1,5 +1,5 @@
 import path from 'node:path';
-import { beforeEach } from 'vitest';
+import { beforeEach, describe, expect, it } from 'vitest';
 
 import type { FileLoaderInterface } from '../../types';
 import { TextLoader } from './index';

package/packages/file-loaders/src/loaders/text/index.ts

@@ -1,17 +1,23 @@
+import debug from 'debug';
 import { readFile } from 'node:fs/promises';
 
 import type { DocumentPage, FileLoaderInterface } from '../../types';
 
+const log = debug('file-loaders:text');
+
 /**
  * 用于加载纯文本文件的加载器。
  */
 export class TextLoader implements FileLoaderInterface {
   async loadPages(filePath: string): Promise<DocumentPage[]> {
+    log('Loading text file:', filePath);
     try {
       const fileContent = await readFile(filePath, 'utf8');
+      log('Text file loaded successfully, size:', fileContent.length, 'bytes');
       const lines = fileContent.split('\n');
       const lineCount = lines.length;
       const charCount = fileContent.length;
+      log('Text file stats:', { charCount, lineCount });
 
       const page: DocumentPage = {
         charCount,
@@ -23,9 +29,11 @@ export class TextLoader implements FileLoaderInterface {
         pageContent: fileContent,
       };
 
+      log('Text page created successfully');
       return [page];
     } catch (e) {
       const error = e as Error;
+      log('Error encountered while loading text file');
       console.error(`Error loading text file ${filePath}: ${error.message}`);
       // 如果读取失败，返回一个包含错误信息的 Page
       const errorPage: DocumentPage = {
@@ -36,6 +44,7 @@ export class TextLoader implements FileLoaderInterface {
         },
         pageContent: '',
       };
+      log('Created error page for failed text file loading');
       return [errorPage];
     }
   }
@@ -47,7 +56,10 @@ export class TextLoader implements FileLoaderInterface {
    * @returns 聚合后的内容
    */
   async aggregateContent(pages: DocumentPage[]): Promise<string> {
+    log('Aggregating content from', pages.length, 'text pages');
     // 默认使用换行符连接，可以根据需要调整或使其可配置
-    return pages.map((page) => page.pageContent).join('\n');
+    const result = pages.map((page) => page.pageContent).join('\n');
+    log('Content aggregated successfully, length:', result.length);
+    return result;
   }
 }

package/packages/file-loaders/test/__snapshots__/loaders.test.ts.snap

@@ -1,5 +1,46 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`loadFile Integration Tests > PDF Handling > should load content from a pdf file using filePath 1`] = `
+{
+  "content": "123",
+  "fileType": "pdf",
+  "filename": "test.pdf",
+  "metadata": {
+    "loaderSpecific": {
+      "pdfInfo": {
+        "CreationDate": "D:20250419143655Z00'00'",
+        "Creator": "Pages文稿",
+        "EncryptFilterName": null,
+        "IsAcroFormPresent": false,
+        "IsCollectionPresent": false,
+        "IsLinearized": false,
+        "IsSignaturesPresent": false,
+        "IsXFAPresent": false,
+        "Language": null,
+        "ModDate": "D:20250419143655Z00'00'",
+        "PDFFormatVersion": "1.3",
+        "Producer": "macOS 版本15.3.2（版号24D81） Quartz PDFContext",
+        "Title": "test",
+      },
+      "pdfMetadata": null,
+      "pdfVersion": "4.10.38",
+    },
+  },
+  "pages": [
+    {
+      "charCount": 3,
+      "lineCount": 1,
+      "metadata": {
+        "pageNumber": 1,
+      },
+      "pageContent": "123",
+    },
+  ],
+  "totalCharCount": 3,
+  "totalLineCount": 1,
+}
+`;
+
 exports[`loadFile Integration Tests > Text Handling (.txt, .csv, .md, etc.) > should load content from a test.csv file using filePath 1`] = `
 {
   "content": "ID,Name,Value

package/packages/file-loaders/test/loaders.test.ts

@@ -36,4 +36,24 @@ describe('loadFile Integration Tests', () => {
       testPureTextFile(file);
     });
   });
+
+  describe('PDF Handling', () => {
+    it(`should load content from a pdf file using filePath`, async () => {
+      const filePath = getFixturePath('test.pdf');
+
+      // Pass filePath directly to loadFile
+      const docs = await loadFile(filePath);
+
+      expect(docs.content).toEqual('123');
+      expect(docs.source).toEqual(filePath);
+
+      // @ts-expect-error
+      delete docs.source;
+      // @ts-expect-error
+      delete docs.createdTime;
+      // @ts-expect-error
+      delete docs.modifiedTime;
+      expect(docs).toMatchSnapshot();
+    });
+  });
 });

package/packages/file-loaders/test/setup.ts

@@ -0,0 +1,17 @@
+// Polyfill DOMMatrix for pdfjs-dist in Node.js environment
+import { DOMMatrix } from 'canvas';
+
+if (typeof global.DOMMatrix === 'undefined') {
+  // @ts-ignore
+  global.DOMMatrix = DOMMatrix;
+}
+
+// Polyfill URL.createObjectURL and URL.revokeObjectURL for pdfjs-dist
+if (typeof global.URL.createObjectURL === 'undefined') {
+  global.URL.createObjectURL = () => 'blob:http://localhost/fake-blob-url';
+}
+if (typeof global.URL.revokeObjectURL === 'undefined') {
+  global.URL.revokeObjectURL = () => {
+    /* no-op */
+  };
+}

package/packages/file-loaders/vitest.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    // coverage: {
+    //   all: false,
+    //   provider: 'v8',
+    //   reporter: ['text', 'json', 'lcov', 'text-summary'],
+    //   reportsDirectory: './coverage/app',
+    // },
+    environment: 'happy-dom',
+    // setupFiles: join(__dirname, './test/setup.ts'),
+  },
+});

package/src/const/file.ts

@@ -1 +1,8 @@
-export const FILE_UPLOAD_BLACKLIST = ['.DS_Store'];
+export const FILE_UPLOAD_BLACKLIST = [
+  '.DS_Store',
+  'Thumbs.db',
+  'desktop.ini',
+  '.localized',
+  'ehthumbs.db',
+  'ehthumbs_vista.db',
+];

package/src/database/client/migrations.json

@@ -456,6 +456,28 @@
     ],
     "bps": true,
     "folderMillis": 1744602998656,
-    "hash": "9a32c373461472a4afdb45e690c3009a0db0eaae81dcf6c8d05277a48f3a5e85"
+    "hash": "fdbac49ffdbe759234e760d0d48cdc1854028ea70d756a12b72f24305b4f3072"
+  },
+  {
+    "sql": [
+      "CREATE TABLE IF NOT EXISTS \"document_chunks\" (\n\t\"document_id\" varchar(30) NOT NULL,\n\t\"chunk_id\" uuid NOT NULL,\n\t\"page_index\" integer,\n\t\"user_id\" text NOT NULL,\n\t\"created_at\" timestamp with time zone DEFAULT now() NOT NULL,\n\tCONSTRAINT \"document_chunks_document_id_chunk_id_pk\" PRIMARY KEY(\"document_id\",\"chunk_id\")\n);\n",
+      "\nCREATE TABLE IF NOT EXISTS \"documents\" (\n\t\"id\" varchar(30) PRIMARY KEY NOT NULL,\n\t\"title\" text,\n\t\"content\" text,\n\t\"file_type\" varchar(255) NOT NULL,\n\t\"filename\" text,\n\t\"total_char_count\" integer NOT NULL,\n\t\"total_line_count\" integer NOT NULL,\n\t\"metadata\" jsonb,\n\t\"pages\" jsonb,\n\t\"source_type\" text NOT NULL,\n\t\"source\" text NOT NULL,\n\t\"file_id\" text,\n\t\"user_id\" text NOT NULL,\n\t\"client_id\" text,\n\t\"accessed_at\" timestamp with time zone DEFAULT now() NOT NULL,\n\t\"created_at\" timestamp with time zone DEFAULT now() NOT NULL,\n\t\"updated_at\" timestamp with time zone DEFAULT now() NOT NULL\n);\n",
+      "\nCREATE TABLE IF NOT EXISTS \"topic_documents\" (\n\t\"document_id\" text NOT NULL,\n\t\"topic_id\" text NOT NULL,\n\t\"user_id\" text NOT NULL,\n\t\"created_at\" timestamp with time zone DEFAULT now() NOT NULL,\n\tCONSTRAINT \"topic_documents_document_id_topic_id_pk\" PRIMARY KEY(\"document_id\",\"topic_id\")\n);\n",
+      "\nALTER TABLE \"document_chunks\" ADD CONSTRAINT \"document_chunks_document_id_documents_id_fk\" FOREIGN KEY (\"document_id\") REFERENCES \"public\".\"documents\"(\"id\") ON DELETE cascade ON UPDATE no action;",
+      "\nALTER TABLE \"document_chunks\" ADD CONSTRAINT \"document_chunks_chunk_id_chunks_id_fk\" FOREIGN KEY (\"chunk_id\") REFERENCES \"public\".\"chunks\"(\"id\") ON DELETE cascade ON UPDATE no action;",
+      "\nALTER TABLE \"document_chunks\" ADD CONSTRAINT \"document_chunks_user_id_users_id_fk\" FOREIGN KEY (\"user_id\") REFERENCES \"public\".\"users\"(\"id\") ON DELETE cascade ON UPDATE no action;",
+      "\nALTER TABLE \"documents\" ADD CONSTRAINT \"documents_file_id_files_id_fk\" FOREIGN KEY (\"file_id\") REFERENCES \"public\".\"files\"(\"id\") ON DELETE set null ON UPDATE no action;",
+      "\nALTER TABLE \"documents\" ADD CONSTRAINT \"documents_user_id_users_id_fk\" FOREIGN KEY (\"user_id\") REFERENCES \"public\".\"users\"(\"id\") ON DELETE cascade ON UPDATE no action;",
+      "\nALTER TABLE \"topic_documents\" ADD CONSTRAINT \"topic_documents_document_id_documents_id_fk\" FOREIGN KEY (\"document_id\") REFERENCES \"public\".\"documents\"(\"id\") ON DELETE cascade ON UPDATE no action;",
+      "\nALTER TABLE \"topic_documents\" ADD CONSTRAINT \"topic_documents_topic_id_topics_id_fk\" FOREIGN KEY (\"topic_id\") REFERENCES \"public\".\"topics\"(\"id\") ON DELETE cascade ON UPDATE no action;",
+      "\nALTER TABLE \"topic_documents\" ADD CONSTRAINT \"topic_documents_user_id_users_id_fk\" FOREIGN KEY (\"user_id\") REFERENCES \"public\".\"users\"(\"id\") ON DELETE cascade ON UPDATE no action;",
+      "\nCREATE INDEX \"documents_source_idx\" ON \"documents\" USING btree (\"source\");",
+      "\nCREATE INDEX \"documents_file_type_idx\" ON \"documents\" USING btree (\"file_type\");",
+      "\nCREATE INDEX \"documents_file_id_idx\" ON \"documents\" USING btree (\"file_id\");",
+      "\nCREATE UNIQUE INDEX \"documents_client_id_user_id_unique\" ON \"documents\" USING btree (\"client_id\",\"user_id\");\n"
+    ],
+    "bps": true,
+    "folderMillis": 1746724476380,
+    "hash": "0518cd9882f7ea38eb498b31c8dda73fb56bbc3aa55445ecbc7a9e716631d047"
   }
 ]

package/src/database/migrations/0022_add_documents.sql

@@ -0,0 +1,49 @@
+CREATE TABLE IF NOT EXISTS "document_chunks" (
+	"document_id" varchar(30) NOT NULL,
+	"chunk_id" uuid NOT NULL,
+	"page_index" integer,
+	"user_id" text NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "document_chunks_document_id_chunk_id_pk" PRIMARY KEY("document_id","chunk_id")
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "documents" (
+	"id" varchar(30) PRIMARY KEY NOT NULL,
+	"title" text,
+	"content" text,
+	"file_type" varchar(255) NOT NULL,
+	"filename" text,
+	"total_char_count" integer NOT NULL,
+	"total_line_count" integer NOT NULL,
+	"metadata" jsonb,
+	"pages" jsonb,
+	"source_type" text NOT NULL,
+	"source" text NOT NULL,
+	"file_id" text,
+	"user_id" text NOT NULL,
+	"client_id" text,
+	"accessed_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "topic_documents" (
+	"document_id" text NOT NULL,
+	"topic_id" text NOT NULL,
+	"user_id" text NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "topic_documents_document_id_topic_id_pk" PRIMARY KEY("document_id","topic_id")
+);
+--> statement-breakpoint
+ALTER TABLE "document_chunks" ADD CONSTRAINT "document_chunks_document_id_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "document_chunks" ADD CONSTRAINT "document_chunks_chunk_id_chunks_id_fk" FOREIGN KEY ("chunk_id") REFERENCES "public"."chunks"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "document_chunks" ADD CONSTRAINT "document_chunks_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "documents" ADD CONSTRAINT "documents_file_id_files_id_fk" FOREIGN KEY ("file_id") REFERENCES "public"."files"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "documents" ADD CONSTRAINT "documents_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "topic_documents" ADD CONSTRAINT "topic_documents_document_id_documents_id_fk" FOREIGN KEY ("document_id") REFERENCES "public"."documents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "topic_documents" ADD CONSTRAINT "topic_documents_topic_id_topics_id_fk" FOREIGN KEY ("topic_id") REFERENCES "public"."topics"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "topic_documents" ADD CONSTRAINT "topic_documents_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "documents_source_idx" ON "documents" USING btree ("source");--> statement-breakpoint
+CREATE INDEX "documents_file_type_idx" ON "documents" USING btree ("file_type");--> statement-breakpoint
+CREATE INDEX "documents_file_id_idx" ON "documents" USING btree ("file_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "documents_client_id_user_id_unique" ON "documents" USING btree ("client_id","user_id");