@lobehub/chat 1.81.3 → 1.81.5

package/packages/file-loaders/src/types.ts

@@ -0,0 +1,200 @@
+// Define supported file types - consider using an enum or const assertion
+export type SupportedFileType = 'pdf' | 'docx' | 'txt' | 'excel' | 'pptx'; // | 'pptx' | 'latex' | 'epub' | 'code' | 'markdown';
+
+/**
+ * 代表一个完整的已加载文件，包含文件级信息和其所有页面/块。
+ */
+export interface FileDocument {
+  /**
+   * 文件内容
+   */
+  content: string;
+
+  /**
+   * 文件创建时间戳。
+   */
+  createdTime: Date;
+
+  /**
+   * 文件类型或扩展名。
+   */
+  fileType: string;
+
+  /**
+   * 原始文件名。
+   */
+  filename: string;
+
+  /**
+   * 文件级别的元数据。
+   * 例如从文件属性中提取的标题、作者，或整个文件加载失败时的错误。
+   */
+  metadata: {
+    /**
+     * 允许添加其他文件级别的元数据。
+     */
+    [key: string]: any;
+    /**
+     * 文档作者 (如果可用)。
+     */
+    author?: string;
+    /**
+     * 如果整个文件加载失败，记录错误信息。
+     */
+    error?: string;
+    /**
+     * 文档标题 (如果可用)。
+     */
+    title?: string;
+  };
+
+  /**
+   * 文件最后修改时间戳。
+   */
+  modifiedTime: Date;
+
+  /**
+   * 包含文档中所有逻辑页面/块的数组。
+   * 顺序通常对应文件中的自然顺序。
+   */
+  pages?: DocumentPage[];
+
+  /**
+   * 原始文件的完整路径。
+   */
+  source: string;
+
+  /**
+   * 整个文档的总字符数 (所有 Page 的 charCount 之和)。
+   * 需要在所有 Page 加载和计算后得出。
+   */
+  totalCharCount: number;
+
+  /**
+   * 整个文档的总行数 (所有 Page 的 lineCount 之和)。
+   * 需要在所有 Page 加载和计算后得出。
+   */
+  totalLineCount: number;
+}
+
+/**
+ * 代表文件中的一个逻辑单元/页面/块。
+ */
+export interface DocumentPage {
+  /**
+   * 此页/块内容的字符数。
+   */
+  charCount: number;
+
+  /**
+   * 此页/块内容的行数。
+   */
+  lineCount: number;
+
+  /**
+   * 与此页/块相关的元数据。
+   */
+  metadata: {
+    /**
+     * 允许添加其他特定于页/块的元数据。
+     */
+    [key: string]: any;
+
+    /**
+     * 如果原始文件单元被进一步分割成块，这是当前块的索引。
+     */
+    chunkIndex?: number;
+
+    /**
+     * 处理此页/块时发生的错误。
+     */
+    error?: string;
+
+    /**
+     * 此页/块在原始文件中的结束行号。
+     */
+    lineNumberEnd?: number;
+
+    /**
+     * 此页/块在原始文件中的起始行号。
+     */
+    lineNumberStart?: number;
+
+    /**
+     * 页码 (适用于 PDF, DOCX)。
+     */
+    pageNumber?: number;
+
+    /**
+     * 与此页/块相关的章节标题。
+     */
+    sectionTitle?: string;
+
+    /**
+     * 工作表名称 (适用于 XLSX)。
+     */
+    sheetName?: string;
+
+    /**
+     * 幻灯片编号 (适用于 PPTX)。
+     */
+    slideNumber?: number;
+
+    /**
+     * 如果原始文件单元被进一步分割成块，这是该单元的总块数。
+     */
+    totalChunks?: number;
+  };
+
+  /**
+   * 此页/块的核心文本内容。
+   */
+  pageContent: string;
+}
+
+/**
+ * 可选的文件元数据，用于覆盖从文件系统读取的信息。
+ */
+export interface FileMetadata {
+  /**
+   * 文件创建时间戳。
+   */
+  createdTime?: Date;
+  /**
+   * 文件类型或扩展名。
+   */
+  fileType?: string;
+  /**
+   * 文件名。
+   */
+  filename?: string;
+  /**
+   * 文件最后修改时间戳。
+   */
+  modifiedTime?: Date;
+  /**
+   * 文件来源标识 (例如 S3 URL 或原始路径)。
+   */
+  source?: string;
+}
+
+/**
+ * 定义所有文件加载器类必须实现的接口。
+ */
+export interface FileLoaderInterface {
+  /**
+   * 将从 loadPages 获取的页面内容聚合成单一的字符串。
+   * @param pages DocumentPage 对象的数组。
+   * @returns 返回聚合后的文本内容的 Promise。
+   */
+  aggregateContent(pages: DocumentPage[]): Promise<string>;
+
+  attachDocumentMetadata?(filePath: string): Promise<Record<string, any>>;
+
+  /**
+   * 根据文件路径加载文件内容，并将其分割为逻辑页面/块。
+   * @param filePath 文件的完整路径。
+   * @returns 返回包含 DocumentPage 对象的数组的 Promise。
+   */
+  loadPages(filePath: string): Promise<DocumentPage[]>;
+}

package/packages/file-loaders/src/utils/isTextReadableFile.ts

@@ -0,0 +1,68 @@
+export const TEXT_READABLE_FILE_TYPES = [
+  // Plain Text & Markup
+  'txt',
+  'md',
+  'markdown',
+  'mdx',
+
+  // Configuration & Data
+  'json',
+  'xml',
+  'yaml',
+  'yml',
+  'toml',
+  'ini',
+  'cfg',
+  'conf',
+  'csv',
+
+  // Web Development
+  'html',
+  'htm',
+  'css',
+  'scss',
+  'less',
+  'js',
+  'jsx',
+  'ts',
+  'tsx',
+  'mjs',
+  'vue',
+  'svelte',
+  'svg',
+
+  // Scripting & Programming
+  'php',
+  'py',
+  'rb',
+  'java',
+  'c',
+  'cpp',
+  'h',
+  'hpp',
+  'cs',
+  'go',
+  'rs',
+  'swift',
+  'kt',
+  'sh',
+  'bash',
+  'bat',
+  'ps1',
+
+  // Other
+  'log',
+  'sql',
+  'patch',
+  'diff',
+  'db', // Often text-based, like SQLite journals
+];
+
+/**
+ * Determine if a file can be read as text based on its extension.
+ * @param fileType File extension (without the leading dot)
+ * @returns Whether the file is likely text-readable
+ */
+export function isTextReadableFile(fileType: string): boolean {
+  return TEXT_READABLE_FILE_TYPES.includes(fileType.toLowerCase());
+}

package/packages/file-loaders/src/utils/parser-utils.ts

@@ -0,0 +1,112 @@
+import { DOMParser } from '@xmldom/xmldom';
+import concat from 'concat-stream';
+import { Buffer } from 'node:buffer';
+import yauzl from 'yauzl';
+
+// Define basic error messages
+const ERRORMSG = {
+  fileCorrupted: (filepath: string | Buffer) =>
+    `[OfficeParser]: Your file ${typeof filepath === 'string' ? filepath : 'Buffer'} seems to be corrupted. If you are sure it is fine, please create a ticket.`,
+  invalidInput: `[OfficeParser]: Invalid input type: Expected a Buffer or a valid file path`,
+};
+
+/** Returns parsed xml document for a given xml text.
+ * @param {string} xml The xml string from the doc file
+ * @returns {XMLDocument}
+ */
+export const parseString = (xml: string) => {
+  const parser = new DOMParser();
+  return parser.parseFromString(xml, 'text/xml') as unknown as XMLDocument;
+};
+
+export interface ExtractedFile {
+  content: string;
+  path: string;
+}
+
+/** Extract specific files from either a ZIP file buffer or file path based on a filter function.
+ * @param {Buffer|string}          zipInput ZIP file input, either a Buffer or a file path (string).
+ * @param {(fileName: string) => boolean} filterFn A function that receives the entry file name and returns true if the file should be extracted.
+ * @returns {Promise<ExtractedFile[]>} Resolves to an array of object containing file path and content.
+ */
+export function extractFiles(
+  zipInput: Buffer | string,
+  filterFn: (fileName: string) => boolean,
+): Promise<ExtractedFile[]> {
+  return new Promise((resolve, reject) => {
+    /** Processes zip file and resolves with the path of file and their content.
+     * @param {yauzl.ZipFile} zipfile
+     */
+    const processZipfile = (zipfile: yauzl.ZipFile) => {
+      const extractedFiles: ExtractedFile[] = [];
+      zipfile.readEntry(); // Start reading entries
+
+      zipfile.on('entry', (entry: yauzl.Entry) => {
+        // Directory entries end with a forward slash
+        if (entry.fileName.endsWith('/')) {
+          zipfile.readEntry(); // Skip directories
+          return;
+        }
+
+        // Use the filter function to determine if the file should be extracted
+        if (filterFn(entry.fileName)) {
+          zipfile.openReadStream(entry, (err, readStream) => {
+            if (err) {
+              zipfile.close(); // Ensure zipfile is closed on error
+              return reject(err);
+            }
+            if (!readStream) {
+              zipfile.close();
+              return reject(new Error(`Could not open read stream for ${entry.fileName}`));
+            }
+
+            // Use concat-stream to collect the data into a single Buffer
+            readStream.pipe(
+              concat((data) => {
+                extractedFiles.push({
+                  content: data.toString('utf8'),
+                  path: entry.fileName, // Specify encoding
+                });
+                zipfile.readEntry(); // Continue reading entries
+              }),
+            );
+            readStream.on('error', (streamErr) => {
+              zipfile.close();
+              reject(streamErr);
+            });
+          });
+        } else {
+          zipfile.readEntry(); // Skip entries that don't match the filter
+        }
+      });
+
+      zipfile.on('end', () => {
+        resolve(extractedFiles);
+        zipfile.close(); // Close the zipfile when done reading entries
+      });
+
+      zipfile.on('error', (err) => {
+        zipfile.close();
+        reject(err);
+      });
+    };
+
+    // Determine whether the input is a buffer or file path
+    if (Buffer.isBuffer(zipInput)) {
+      // Process ZIP from Buffer
+      yauzl.fromBuffer(zipInput, { lazyEntries: true }, (err, zipfile) => {
+        if (err || !zipfile) return reject(err || new Error('Failed to open zip from buffer'));
+        processZipfile(zipfile);
+      });
+    } else if (typeof zipInput === 'string') {
+      // Process ZIP from File Path
+      yauzl.open(zipInput, { lazyEntries: true }, (err, zipfile) => {
+        if (err || !zipfile)
+          return reject(err || new Error(`Failed to open zip file: ${zipInput}`));
+        processZipfile(zipfile);
+      });
+    } else {
+      reject(new Error(ERRORMSG.invalidInput));
+    }
+  });
+}

package/packages/file-loaders/test/__snapshots__/loaders.test.ts.snap

@@ -0,0 +1,93 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`loadFile Integration Tests > Text Handling (.txt, .csv, .md, etc.) > should load content from a test.csv file using filePath 1`] = `
+{
+  "content": "ID,Name,Value
+1,Alpha,100
+2,Beta,200
+3,Gamma,300
+",
+  "fileType": "csv",
+  "filename": "test.csv",
+  "metadata": {
+    "loaderSpecific": undefined,
+  },
+  "pages": [
+    {
+      "charCount": 49,
+      "lineCount": 5,
+      "metadata": {
+        "lineNumberEnd": 5,
+        "lineNumberStart": 1,
+      },
+      "pageContent": "ID,Name,Value
+1,Alpha,100
+2,Beta,200
+3,Gamma,300
+",
+    },
+  ],
+  "totalCharCount": 49,
+  "totalLineCount": 5,
+}
+`;
+
+exports[`loadFile Integration Tests > Text Handling (.txt, .csv, .md, etc.) > should load content from a test.md file using filePath 1`] = `
+{
+  "content": "# Markdown Test
+
+This is a test.
+",
+  "fileType": "md",
+  "filename": "test.md",
+  "metadata": {
+    "loaderSpecific": undefined,
+  },
+  "pages": [
+    {
+      "charCount": 33,
+      "lineCount": 4,
+      "metadata": {
+        "lineNumberEnd": 4,
+        "lineNumberStart": 1,
+      },
+      "pageContent": "# Markdown Test
+
+This is a test.
+",
+    },
+  ],
+  "totalCharCount": 33,
+  "totalLineCount": 4,
+}
+`;
+
+exports[`loadFile Integration Tests > Text Handling (.txt, .csv, .md, etc.) > should load content from a test.txt file using filePath 1`] = `
+{
+  "content": "This is line 1.
+This is line 2.
+End of text file.
+",
+  "fileType": "txt",
+  "filename": "test.txt",
+  "metadata": {
+    "loaderSpecific": undefined,
+  },
+  "pages": [
+    {
+      "charCount": 50,
+      "lineCount": 4,
+      "metadata": {
+        "lineNumberEnd": 4,
+        "lineNumberStart": 1,
+      },
+      "pageContent": "This is line 1.
+This is line 2.
+End of text file.
+",
+    },
+  ],
+  "totalCharCount": 50,
+  "totalLineCount": 4,
+}
+`;

package/packages/file-loaders/test/fixtures/test.csv

@@ -0,0 +1,4 @@
+ID,Name,Value
+1,Alpha,100
+2,Beta,200
+3,Gamma,300

package/packages/file-loaders/test/fixtures/test.md

@@ -0,0 +1,3 @@
+# Markdown Test
+
+This is a test.

package/packages/file-loaders/test/fixtures/test.txt

@@ -0,0 +1,3 @@
+This is line 1.
+This is line 2.
+End of text file.

package/packages/file-loaders/test/loaders.test.ts

@@ -0,0 +1,39 @@
+// @vitest-environment node
+import fs from 'node:fs';
+import path from 'node:path';
+import { describe, expect, it } from 'vitest';
+
+import { loadFile } from '../src';
+
+const getFixturePath = (filename: string) => path.join(__dirname, 'fixtures', filename);
+
+const TEXT_FILES = ['test.txt', 'test.csv', 'test.md'];
+
+describe('loadFile Integration Tests', () => {
+  describe('Text Handling (.txt, .csv, .md, etc.)', () => {
+    const testPureTextFile = (fileName: string) => {
+      it(`should load content from a ${fileName} file using filePath`, async () => {
+        const filePath = getFixturePath(fileName);
+        const expectedContent = fs.readFileSync(filePath, 'utf-8');
+
+        // Pass filePath directly to loadFile
+        const docs = await loadFile(filePath);
+
+        expect(docs.content).toEqual(expectedContent);
+        expect(docs.source).toEqual(filePath);
+
+        // @ts-expect-error
+        delete docs.source;
+        // @ts-expect-error
+        delete docs.createdTime;
+        // @ts-expect-error
+        delete docs.modifiedTime;
+        expect(docs).toMatchSnapshot();
+      });
+    };
+
+    TEXT_FILES.forEach((file) => {
+      testPureTextFile(file);
+    });
+  });
+});

package/scripts/prebuild.mts

@@ -25,12 +25,16 @@ const partialBuildPages = [
     disabled: isDesktop,
     paths: ['src/app/[variants]/(main)/(mobile)'],
   },
+  {
+    name: 'oauth',
+    disabled: isDesktop,
+    paths: ['src/app/[variants]/oauth', 'src/app/(backend)/oidc'],
+  },
   {
     name: 'api-webhooks',
     disabled: isDesktop,
     paths: ['src/app/(backend)/api/webhooks'],
   },
-
   // no need for web
   {
     name: 'desktop-devtools',

package/src/app/(backend)/trpc/desktop/[trpc]/route.ts

@@ -0,0 +1,26 @@
+import { fetchRequestHandler } from '@trpc/server/adapters/fetch';
+import type { NextRequest } from 'next/server';
+
+import { pino } from '@/libs/logger';
+import { createLambdaContext } from '@/libs/trpc/lambda/context';
+import { desktopRouter } from '@/server/routers/desktop';
+
+const handler = (req: NextRequest) =>
+  fetchRequestHandler({
+    /**
+     * @link https://trpc.io/docs/v11/context
+     */
+    createContext: () => createLambdaContext(req),
+
+    endpoint: '/trpc/desktop',
+
+    onError: ({ error, path, type }) => {
+      pino.info(`Error in tRPC handler (tools) on path: ${path}, type: ${type}`);
+      console.error(error);
+    },
+
+    req,
+    router: desktopRouter,
+  });
+
+export { handler as GET, handler as POST };

package/src/features/Conversation/Messages/Assistant/Tool/Render/Arguments/ObjectEntity.tsx

@@ -0,0 +1,81 @@
+import { createStyles } from 'antd-style';
+import { memo } from 'react';
+
+import { useIsMobile } from '@/hooks/useIsMobile';
+import { shinyTextStylish } from '@/styles/loading';
+
+import ValueCell from './ValueCell';
+
+const useStyles = createStyles(({ css, token }) => ({
+  arrayRow: css`
+    &:not(:first-child) {
+      border-block-start: 1px dotted ${token.colorBorderSecondary};
+    }
+  `,
+  colon: css`
+    color: ${token.colorTextTertiary};
+  `,
+  key: css`
+    color: ${token.colorTextTertiary};
+  `,
+  row: css`
+    display: flex;
+    align-items: baseline;
+
+    &:not(:first-child) {
+      border-block-start: 1px dotted ${token.colorBorderSecondary};
+    }
+  `,
+  shineText: shinyTextStylish(token),
+  value: css`
+    color: ${token.colorTextSecondary};
+  `,
+}));
+
+const formatValue = (value: any): string | string[] => {
+  if (Array.isArray(value)) {
+    return value.map((v) => (typeof v === 'object' ? JSON.stringify(v) : v));
+  }
+
+  if (typeof value === 'object' && value !== null) {
+    return Object.entries(value)
+      .map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : v}`)
+      .join(', ');
+  }
+  return String(value);
+};
+
+interface ObjectEntityProps {
+  editable?: boolean;
+  hasMinWidth: boolean;
+  objectKey: string;
+  shine?: boolean;
+  value: any;
+}
+
+const ObjectEntity = memo<ObjectEntityProps>(({ hasMinWidth, shine, value, objectKey }) => {
+  const { styles, cx } = useStyles();
+  const isMobile = useIsMobile();
+  const formatedValue = formatValue(value);
+
+  return (
+    <div className={styles.row}>
+      <span
+        className={styles.key}
+        style={{ minWidth: hasMinWidth ? (isMobile ? 60 : 140) : undefined }}
+      >
+        {objectKey}
+      </span>
+      <span className={styles.colon}>:</span>
+      <div className={cx(shine ? styles.shineText : styles.value)} style={{ width: '100%' }}>
+        {typeof formatedValue === 'string' ? (
+          <ValueCell value={formatedValue} />
+        ) : (
+          formatedValue.map((v, i) => <ValueCell key={i + v} value={v} />)
+        )}
+      </div>
+    </div>
+  );
+});
+
+export default ObjectEntity;