@nocobase/ai 2.2.0-alpha.1 → 2.2.0-alpha.10

package/src/document-loader/index.ts

@@ -11,13 +11,22 @@ import { Document } from '@langchain/core/documents';
 import { Worker } from 'node:worker_threads';
 import path from 'node:path';
 
-export const loadByWorker = async (extname: string, blob: Blob): Promise<Document[]> => {
-  const buffer = Buffer.from(await blob.arrayBuffer());
+export type DocumentLoaderWorkerOptions = {
+  filePath: string;
+  mimeType?: string;
+  /** Timeout in milliseconds for the worker to complete. Defaults to 5 minutes. */
+  timeout?: number;
+};
+
+const DEFAULT_WORKER_TIMEOUT = 5 * 60 * 1000;
+
+export const loadByWorker = async (extname: string, options: DocumentLoaderWorkerOptions): Promise<Document[]> => {
   const isTsRuntime = __filename.endsWith('.ts');
   const workerPath = path.join(__dirname, `loader.worker.${isTsRuntime ? 'ts' : 'js'}`);
   const worker = new Worker(workerPath, {
     execArgv: isTsRuntime ? ['--require', 'tsx/cjs'] : undefined,
   });
+  const timeout = options.timeout ?? DEFAULT_WORKER_TIMEOUT;
   return new Promise<Document[]>((resolve, reject) => {
     let settled = false;
     const close = (error?: Error, result?: Document[]) => {
@@ -25,6 +34,7 @@ export const loadByWorker = async (extname: string, blob: Blob): Promise<Documen
         return;
       }
       settled = true;
+      clearTimeout(timer);
       if (error) {
         reject(error);
         return;
@@ -32,6 +42,10 @@ export const loadByWorker = async (extname: string, blob: Blob): Promise<Documen
       resolve(result || []);
     };
 
+    const timer = setTimeout(() => {
+      close(new Error(`Document loading timed out after ${Math.round(timeout / 1000)}s`));
+    }, timeout);
+
     worker.once('message', (payload: { documents?: Document[]; error?: string }) => {
       if (payload?.error) {
         close(new Error(payload.error));
@@ -48,8 +62,8 @@ export const loadByWorker = async (extname: string, blob: Blob): Promise<Documen
 
     worker.postMessage({
       extname,
-      mimeType: blob.type,
-      buffer: Uint8Array.from(buffer),
+      filePath: options.filePath,
+      mimeType: options.mimeType,
     });
   }).finally(() => {
     worker.terminate().catch(() => undefined);

package/src/document-loader/loader.worker.ts

@@ -19,7 +19,7 @@ import { loadXlsx } from './xlsx';
 type ParsePayload = {
   extname: string;
   mimeType?: string;
-  buffer: Uint8Array;
+  filePath: string;
 };
 
 type WorkerResponse = {
@@ -27,54 +27,60 @@ type WorkerResponse = {
   error?: string;
 };
 
-const loadPdf = async (blob: Blob): Promise<Document[]> => {
-  const loader = new PDFLoader(blob);
-  return loader.load();
+const loadPdf = async (filePath: string): Promise<Document[]> => {
+  const loader = new PDFLoader(filePath);
+  try {
+    return await loader.load();
+  } catch (error) {
+    const err = error as Error;
+    if (err?.name === 'PasswordException' || /password/i.test(err?.message)) {
+      throw new Error('The PDF file is password-protected and cannot be parsed. Please upload an unlocked version.');
+    }
+    throw error;
+  }
 };
 
-const loadDoc = async (blob: Blob, type: 'docx' | 'doc'): Promise<Document[]> => {
-  const loader = new DocxLoader(blob, { type });
+const loadDoc = async (filePath: string, type: 'docx' | 'doc'): Promise<Document[]> => {
+  const loader = new DocxLoader(filePath, { type });
   return loader.load();
 };
 
-const loadPpt = async (blob: Blob): Promise<Document[]> => {
-  const loader = new PPTXLoader(blob);
+const loadPpt = async (filePath: string): Promise<Document[]> => {
+  const loader = new PPTXLoader(filePath);
   return loader.load();
 };
 
-const loadTxt = async (blob: Blob): Promise<Document[]> => {
-  const loader = new TextLoader(blob);
+const loadTxt = async (filePath: string): Promise<Document[]> => {
+  const loader = new TextLoader(filePath);
   return loader.load();
 };
 
-const loadCsv = async (blob: Blob): Promise<Document[]> => {
-  const loader = new CSVLoader(blob);
+const loadCsv = async (filePath: string): Promise<Document[]> => {
+  const loader = new CSVLoader(filePath);
   return loader.load();
 };
 
 const loadByExtname = async (payload: ParsePayload): Promise<Document[]> => {
-  // @ts-ignore
-  const blob = new Blob([Buffer.from(payload.buffer)], { type: payload.mimeType ?? 'application/octet-stream' });
-
   switch (payload.extname) {
     case '.pdf':
-      return loadPdf(blob);
+      return loadPdf(payload.filePath);
     case '.ppt':
     case '.pptx':
-      return loadPpt(blob);
+      return loadPpt(payload.filePath);
     case '.doc':
-      return loadDoc(blob, 'doc');
+      return loadDoc(payload.filePath, 'doc');
     case '.docx':
-      return loadDoc(blob, 'docx');
+      return loadDoc(payload.filePath, 'docx');
     case '.csv':
-      return loadCsv(blob);
+      return loadCsv(payload.filePath);
     case '.xls':
     case '.xlsx':
-      return loadXlsx(blob);
+    case '.xlsm':
+      return loadXlsx(payload.filePath, payload.mimeType);
     case '.json':
     case '.md':
     case '.txt':
-      return loadTxt(blob);
+      return loadTxt(payload.filePath);
     default:
       return [];
   }
@@ -92,8 +98,9 @@ parentPort?.on('message', async (payload: ParsePayload) => {
     };
     parentPort?.postMessage(response);
   } catch (error) {
+    const err = error as Error;
     const response: WorkerResponse = {
-      error: String(error?.stack || error),
+      error: err?.message || String(error),
     };
     parentPort?.postMessage(response);
   }

package/src/document-loader/xlsx.ts

@@ -8,6 +8,7 @@
  */
 
 import { Document } from '@langchain/core/documents';
+import { readFile } from 'node:fs/promises';
 import * as XLSX from 'xlsx';
 
 const normalizeCellValue = (value: unknown): string => {
@@ -43,10 +44,10 @@ const sheetToLines = (sheet: XLSX.WorkSheet): string[] => {
     .filter((line) => line.trim().length > 0);
 };
 
-export const loadXlsx = async (blob: Blob): Promise<Document[]> => {
-  const buffer = await blob.arrayBuffer();
+export const loadXlsx = async (filePath: string, mimeType?: string): Promise<Document[]> => {
+  const buffer = await readFile(filePath);
   const workbook = XLSX.read(buffer, {
-    type: 'array',
+    type: 'buffer',
     cellText: true,
   });
 
@@ -69,8 +70,8 @@ export const loadXlsx = async (blob: Blob): Promise<Document[]> => {
       new Document({
         pageContent: [`Sheet: ${sheetName}`, ...lines].join('\n'),
         metadata: {
-          source: 'blob',
-          blobType: blob.type,
+          source: filePath,
+          blobType: mimeType,
           sheetName,
           sheetIndex: index,
         },

package/src/mcp-manager/index.ts

@@ -15,6 +15,8 @@ import { StructuredToolInterface } from '@langchain/core/tools';
 import { MCPEntry, MCPFilter, MCPManager, MCPOptions, MCPTestResult, MCPToolEntry } from './types';
 import type { DynamicToolsProvider, Permission, ToolsRegistration, ToolsOptions } from '../tools-manager/types';
 import type { Context } from '@nocobase/actions';
+import { normalizeMCPOptions, renderMCPOptions } from './options-renderer';
+import { UserContextMCPClientManager } from './user-context-client-manager';
 
 export class DefaultMCPManager implements MCPManager {
   private readonly mcpRegistry = new Registry<MCPEntry>();
@@ -23,9 +25,15 @@ export class DefaultMCPManager implements MCPManager {
   private client: MultiServerMCPClient | null = null;
   private toolsMap: Record<string, StructuredToolInterface[]> = {};
   private toolsPermissionMap: Record<string, Permission> = {};
+  private readonly userContextClientManager: UserContextMCPClientManager;
 
   constructor(private readonly app: any) {
     this.provideCollectionManager = () => app.mainDataSource;
+    this.userContextClientManager = new UserContextMCPClientManager({
+      app,
+      listEntries: () => this.listMCP({ enabled: true, useUserContext: true }),
+      buildConnection: (options) => this.buildMCPConnection(options),
+    });
   }
 
   async init() {
@@ -72,6 +80,14 @@ export class DefaultMCPManager implements MCPManager {
     if (filter.transport) {
       where['transport'] = filter.transport;
     }
+    if (filter.useUserContext != null) {
+      where['useUserContext'] =
+        filter.useUserContext === true
+          ? true
+          : {
+              [Op.or]: [false, null],
+            };
+    }
     return (await this.aiMcpClientsModel?.findAll({ where }))?.map((item) => item.toJSON() as MCPEntry) ?? [];
   }
 
@@ -88,7 +104,7 @@ export class DefaultMCPManager implements MCPManager {
     }
 
     // Get all enabled MCP entries
-    const entries = await this.listMCP({ enabled: true });
+    const entries = await this.listMCP({ enabled: true, useUserContext: false });
 
     if (entries.length === 0) {
       return;
@@ -97,7 +113,7 @@ export class DefaultMCPManager implements MCPManager {
     // Build connections object
     const connections: Record<string, StdioConnection | StreamableHTTPConnection> = {};
     for (const entry of entries) {
-      connections[entry.name] = this.buildMCPConnection(entry);
+      connections[entry.name] = this.buildMCPConnection(await renderMCPOptions(entry, this.app));
     }
 
     // Create new client and initialize connections
@@ -121,48 +137,82 @@ export class DefaultMCPManager implements MCPManager {
   }
 
   getMCPToolsProvider(): DynamicToolsProvider {
-    return async (register: ToolsRegistration): Promise<void> => {
+    return async (register: ToolsRegistration, filter): Promise<void> => {
       // Use cached tools from rebuildClient
       for (const [serverName, tools] of Object.entries(this.toolsMap)) {
-        for (const tool of tools) {
-          const toolName = `mcp-${serverName}-${tool.name}`;
-          const toolOptions: ToolsOptions = {
-            scope: 'GENERAL',
-            from: 'mcp',
-            defaultPermission: this.toolsPermissionMap[toolName],
-            introduction: {
-              title: tool.name,
-              about: tool.description,
-            },
-            definition: {
-              name: toolName,
-              description: tool.description || `MCP tool: ${tool.name} from ${serverName}`,
-              schema: tool.schema,
-            },
-            invoke: async (_ctx: Context, args: any) => {
-              try {
-                const result = await tool.invoke(args);
-                return result;
-              } catch (error: any) {
-                return {
-                  status: 'error' as const,
-                  content: error?.message || 'Tool invocation failed',
-                };
-              }
-            },
-          };
-          register.registerTools(toolOptions);
-        }
+        this.registerToolsFromMap(register, serverName, tools);
       }
+
+      if (!filter?.ctx) {
+        return;
+      }
+
+      const userToolsMap = await this.userContextClientManager.getToolsMap(filter.ctx);
+      for (const [serverName, tools] of Object.entries(userToolsMap)) {
+        this.registerToolsFromMap(register, serverName, tools);
+      }
+    };
+  }
+
+  private registerToolsFromMap(
+    register: ToolsRegistration,
+    serverName: string,
+    tools: StructuredToolInterface[],
+  ): void {
+    for (const tool of tools) {
+      const toolName = `mcp-${serverName}-${tool.name}`;
+      this.ensureToolPermission(toolName, tool.name);
+      const toolOptions: ToolsOptions = {
+        scope: 'GENERAL',
+        from: 'mcp',
+        defaultPermission: this.toolsPermissionMap[toolName],
+        introduction: {
+          title: tool.name,
+          about: tool.description,
+        },
+        definition: {
+          name: toolName,
+          description: tool.description || `MCP tool: ${tool.name} from ${serverName}`,
+          schema: tool.schema,
+        },
+        invoke: async (_ctx: Context, args: any) => {
+          try {
+            const result = await tool.invoke(args);
+            return result;
+          } catch (error: any) {
+            return {
+              status: 'error' as const,
+              content: error?.message || 'Tool invocation failed',
+            };
+          }
+        },
+      };
+      register.registerTools(toolOptions);
+    }
+  }
+
+  private ensureToolPermission(toolName: string, rawToolName: string) {
+    if (!(toolName in this.toolsPermissionMap)) {
+      this.toolsPermissionMap[toolName] = rawToolName.startsWith('get') ? 'ALLOW' : 'ASK';
+    }
+  }
+
+  async listMCPTools(ctx?: Context): Promise<Record<string, MCPToolEntry[]>> {
+    const toolsMap = {
+      ...this.toolsMap,
+      ...(ctx ? await this.userContextClientManager.getToolsMap(ctx) : {}),
     };
+
+    return this.formatMCPTools(toolsMap);
   }
 
-  async listMCPTools(): Promise<Record<string, MCPToolEntry[]>> {
+  private formatMCPTools(toolsMap: Record<string, StructuredToolInterface[]>): Record<string, MCPToolEntry[]> {
     return Object.fromEntries(
-      Object.entries(this.toolsMap).map(([serverName, tools]) => [
+      Object.entries(toolsMap).map(([serverName, tools]) => [
         serverName,
         tools.map((tool) => {
           const toolName = `mcp-${serverName}-${tool.name}`;
+          this.ensureToolPermission(toolName, tool.name);
           return {
             name: toolName,
             title: tool.name,
@@ -179,8 +229,13 @@ export class DefaultMCPManager implements MCPManager {
     this.toolsPermissionMap[toolName] = permission;
   }
 
-  async testConnection(options: MCPOptions): Promise<MCPTestResult> {
-    const { transport } = options;
+  async clearUserContextCache(): Promise<void> {
+    await this.userContextClientManager.clear();
+  }
+
+  async testConnection(options: MCPOptions, ctx?: Context): Promise<MCPTestResult> {
+    const renderedOptions = await renderMCPOptions(normalizeMCPOptions(options), this.app, ctx);
+    const { transport } = renderedOptions;
 
     // Validate required fields
     if (!transport) {
@@ -190,14 +245,14 @@ export class DefaultMCPManager implements MCPManager {
       };
     }
 
-    if (transport === 'stdio' && !options.command) {
+    if (transport === 'stdio' && !renderedOptions.command) {
       return {
         success: false,
         error: 'Command is required for stdio transport',
       };
     }
 
-    if ((transport === 'http' || transport === 'sse') && !options.url) {
+    if ((transport === 'http' || transport === 'sse') && !renderedOptions.url) {
       return {
         success: false,
         error: 'URL is required for HTTP/SSE transport',
@@ -207,7 +262,7 @@ export class DefaultMCPManager implements MCPManager {
     let client: MultiServerMCPClient | null = null;
 
     try {
-      const connection = this.buildMCPConnection(options);
+      const connection = this.buildMCPConnection(renderedOptions);
       const serverName = 'test-server';
 
       client = new MultiServerMCPClient({
@@ -303,18 +358,20 @@ export class DefaultMCPManager implements MCPManager {
   }
 
   private async persistenceEntry(entry: MCPEntry): Promise<void> {
+    const normalizedEntry = normalizeMCPOptions(entry) as MCPEntry;
     await this.sequelize.transaction(async (transaction) => {
-      const existed = await this.aiMcpClientsModel.findOne({ where: { name: entry.name }, transaction });
+      const existed = await this.aiMcpClientsModel.findOne({ where: { name: normalizedEntry.name }, transaction });
       if (existed) {
         await existed.update(
           {
-            transport: entry.transport,
-            command: entry.command,
-            args: entry.args,
-            env: entry.env,
-            url: entry.url,
-            headers: entry.headers,
-            restart: entry.restart,
+            transport: normalizedEntry.transport,
+            command: normalizedEntry.command,
+            args: normalizedEntry.args,
+            env: normalizedEntry.env,
+            url: normalizedEntry.url,
+            headers: normalizedEntry.headers,
+            restart: normalizedEntry.restart,
+            useUserContext: normalizedEntry.useUserContext,
           },
           { transaction },
         );
@@ -323,7 +380,7 @@ export class DefaultMCPManager implements MCPManager {
 
       await this.aiMcpClientsModel.create(
         {
-          ...entry,
+          ...normalizedEntry,
         },
         { transaction },
       );
@@ -331,13 +388,15 @@ export class DefaultMCPManager implements MCPManager {
   }
 
   private normalizeEntry(name: string, options: MCPOptions): MCPEntry {
-    return {
+    const entry: MCPEntry = {
       name,
       enabled: true,
       ...options,
       args: options.args ?? [],
       env: options.env ?? {},
+      useUserContext: options.useUserContext === true,
     };
+    return normalizeMCPOptions(entry) as MCPEntry;
   }
 
   private get aiMcpClientsCollection() {

package/src/mcp-manager/options-renderer.ts

@@ -0,0 +1,132 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import type { Context } from '@nocobase/actions';
+import { parse } from '@nocobase/utils';
+import type { MCPOptions } from './types';
+
+const unsafePathSegments = new Set(['__proto__', 'prototype', 'constructor']);
+const currentUserVariableRegExp = /{{\s*(?:(ctx)\.)?(currentUser|\$user)(?:\.([^}]+))?\s*}}/g;
+
+const hasUnsafePathSegment = (path: string) => {
+  return path
+    .split(/[.[\]]+/)
+    .filter(Boolean)
+    .some((segment) => unsafePathSegments.has(segment));
+};
+
+const toPlainObject = (value: any) => {
+  if (!value) {
+    return value;
+  }
+  if (typeof value.toJSON === 'function') {
+    return value.toJSON();
+  }
+  return value;
+};
+
+const getCurrentUserReferencePaths = (value: unknown) => {
+  const text = typeof value === 'string' ? value : JSON.stringify(value ?? {});
+  return Array.from(text.matchAll(currentUserVariableRegExp))
+    .map((match) => match[3]?.trim())
+    .filter((path): path is string => !!path && !hasUnsafePathSegment(path));
+};
+
+const getCurrentUserAppends = (paths: string[], currentUser: any) => {
+  return Array.from(
+    new Set(
+      paths.map((path) => path.split('.')[0]).filter((append) => append && !Reflect.has(currentUser || {}, append)),
+    ),
+  );
+};
+
+async function getCurrentUser(ctx?: Context, template?: unknown) {
+  const currentUser = ctx?.state?.currentUser ?? ctx?.auth?.user;
+  if (!ctx || !currentUser) {
+    return toPlainObject(currentUser);
+  }
+
+  const appends = getCurrentUserAppends(getCurrentUserReferencePaths(template), currentUser);
+  if (!appends.length) {
+    return toPlainObject(currentUser);
+  }
+
+  const user = await ctx.db.getRepository('users').findOne({
+    filterByTk: currentUser.id,
+    appends,
+  });
+  return toPlainObject(user) || toPlainObject(currentUser);
+}
+
+const stringifyRecord = (value: unknown): Record<string, string> => {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return {};
+  }
+  return Object.entries(value as Record<string, unknown>).reduce<Record<string, string>>((result, [key, item]) => {
+    if (!key || item == null) {
+      return result;
+    }
+    result[key] = String(item);
+    return result;
+  }, {});
+};
+
+const stringifyArray = (value: unknown): string[] => {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return value.filter((item) => item != null).map((item) => String(item));
+};
+
+const getRequestVariables = (ctx?: Context) => ({
+  headers: ctx?.request?.headers ?? {},
+  token: ctx?.getBearerToken?.() ?? '',
+});
+
+const emptyRequestVariables = {
+  headers: {},
+  token: '',
+};
+
+export const normalizeMCPOptions = (options: MCPOptions): MCPOptions => {
+  const normalized: MCPOptions = {
+    ...options,
+    args: stringifyArray(options.args),
+    env: stringifyRecord(options.env),
+    headers: stringifyRecord(options.headers),
+    useUserContext: options.transport === 'stdio' ? false : options.useUserContext === true,
+  };
+
+  if (normalized.transport === 'stdio') {
+    normalized.url = undefined;
+    normalized.headers = {};
+  }
+  return normalized;
+};
+
+export async function renderMCPOptions(
+  options: MCPOptions,
+  app: { environment?: { getVariables?: () => Record<string, unknown> } },
+  ctx?: Context,
+): Promise<MCPOptions> {
+  const currentUser = options.useUserContext ? await getCurrentUser(ctx, options) : undefined;
+  const request = options.useUserContext ? getRequestVariables(ctx) : emptyRequestVariables;
+  const variables = {
+    $env: app.environment?.getVariables?.() ?? {},
+    currentUser,
+    $user: currentUser,
+    request,
+    ctx: {
+      currentUser,
+      request,
+    },
+  };
+
+  return normalizeMCPOptions(parse(options)(variables) as MCPOptions);
+}

package/src/mcp-manager/types.ts

@@ -8,18 +8,20 @@
  */
 
 import type { MultiServerMCPClient } from '@langchain/mcp-adapters';
+import type { Context } from '@nocobase/actions';
 import type { DynamicToolsProvider, Permission } from '../tools-manager/types';
 
 export interface MCPManager extends MCPRegistration {
   init(): Promise<void>;
   getMCP(name: string): Promise<MCPEntry>;
   listMCP(filter: MCPFilter): Promise<MCPEntry[]>;
-  testConnection(options: MCPOptions): Promise<MCPTestResult>;
+  testConnection(options: MCPOptions, ctx?: Context): Promise<MCPTestResult>;
   rebuildClient(): Promise<void>;
   getClient(): MultiServerMCPClient | null;
   getMCPToolsProvider(): DynamicToolsProvider;
-  listMCPTools(): Promise<Record<string, MCPToolEntry[]>>;
+  listMCPTools(ctx?: Context): Promise<Record<string, MCPToolEntry[]>>;
   updateMCPToolPermission(toolName: string, permission: Permission): Promise<void>;
+  clearUserContextCache(): Promise<void>;
 }
 
 export interface MCPRegistration {
@@ -34,6 +36,7 @@ export type MCPOptions = {
   url?: string;
   headers?: Record<string, string>;
   restart?: Record<string, any>;
+  useUserContext?: boolean;
 };
 
 export type MCPEntry = MCPOptions & {
@@ -45,6 +48,7 @@ export type MCPFilter = {
   name?: string;
   enabled?: boolean;
   transport?: MCPTransport;
+  useUserContext?: boolean;
 };
 
 export type MCPTransport = 'stdio' | 'sse' | 'http';