@nocobase/ai 2.1.0-alpha.2 → 2.1.0-alpha.20

package/lib/loader/employee.js

@@ -0,0 +1,207 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var employee_exports = {};
+__export(employee_exports, {
+  AIEmployeeLoader: () => AIEmployeeLoader,
+  normalizeAIEmployeeToolSetting: () => normalizeAIEmployeeToolSetting
+});
+module.exports = __toCommonJS(employee_exports);
+var import_utils = require("@nocobase/utils");
+var import_scanner = require("./scanner");
+var import_fs = require("fs");
+var import_promises = require("fs/promises");
+var import_types = require("./types");
+var import_path = __toESM(require("path"));
+const _AIEmployeeLoader = class _AIEmployeeLoader extends import_types.LoadAndRegister {
+  constructor(ai, options) {
+    super(ai, options);
+    this.ai = ai;
+    this.options = options;
+    this.log = options.log;
+    this.scanner = new import_scanner.DirectoryScanner(this.options.scan);
+  }
+  scanner;
+  files = [];
+  employeeDescriptors = [];
+  log;
+  async scan() {
+    this.files = await this.scanner.scan();
+  }
+  async import() {
+    if (!this.files.length) {
+      return;
+    }
+    const grouped = /* @__PURE__ */ new Map();
+    for (const fd of this.files) {
+      const employeeRoot = getEmployeeRoot(fd);
+      const group = grouped.get(employeeRoot) ?? [];
+      group.push(fd);
+      grouped.set(employeeRoot, group);
+    }
+    const descriptors = await Promise.all(
+      Array.from(grouped.entries()).map(async ([employeeRoot, fds]) => {
+        var _a, _b, _c;
+        const file = fds.find((fd) => fd.extname === ".ts" || fd.extname === ".js");
+        if (!file || !(0, import_fs.existsSync)(file.path)) {
+          return null;
+        }
+        const promptFile = fds.find((fd) => fd.basename === "prompt.md");
+        const name = import_path.default.basename(employeeRoot);
+        try {
+          const imported = await (0, import_utils.importModule)(file.path);
+          const mod = (imported == null ? void 0 : imported.default) ?? imported;
+          const employeeOptions = typeof mod === "function" ? mod() : mod;
+          if (!employeeOptions || !employeeOptions.username) {
+            (_a = this.log) == null ? void 0 : _a.warn(`ai employee [${name}] register ignored: invalid definition at ${file.path}`);
+            return null;
+          }
+          const { skills = [], tools = [] } = employeeOptions;
+          if (promptFile && (0, import_fs.existsSync)(promptFile.path)) {
+            try {
+              employeeOptions.systemPrompt = await (0, import_promises.readFile)(promptFile.path, "utf-8");
+            } catch (e) {
+              (_b = this.log) == null ? void 0 : _b.error(
+                `ai employee [${name}] load fail: error occur when reading prompt.md at ${promptFile.path}`,
+                e
+              );
+              return null;
+            }
+          }
+          return {
+            name,
+            employeeRoot,
+            file,
+            options: {
+              ...employeeOptions,
+              skills: [...skills, ...await discoverSkills(employeeRoot)],
+              tools: [...tools, ...await discoverTools(employeeRoot)]
+            }
+          };
+        } catch (e) {
+          (_c = this.log) == null ? void 0 : _c.error(`ai employee [${name}] load fail: error occur when import ${file.path}`, e);
+          return null;
+        }
+      })
+    );
+    this.employeeDescriptors = descriptors.filter((item) => Boolean(item));
+  }
+  async register() {
+    if (!this.employeeDescriptors.length) {
+      return;
+    }
+    const { employeeManager } = this.ai;
+    for (const descriptor of this.employeeDescriptors) {
+      await employeeManager.registerEmployee(descriptor.options);
+    }
+  }
+};
+__name(_AIEmployeeLoader, "AIEmployeeLoader");
+let AIEmployeeLoader = _AIEmployeeLoader;
+function getEmployeeRoot(fd) {
+  if (fd.basename === "index.ts" || fd.basename === "index.js" || fd.basename === "prompt.md") {
+    return import_path.default.dirname(fd.path);
+  }
+  return fd.path;
+}
+__name(getEmployeeRoot, "getEmployeeRoot");
+async function discoverSkills(employeeRoot) {
+  const skillsDir = import_path.default.join(employeeRoot, "skills");
+  if (!(0, import_fs.existsSync)(skillsDir)) {
+    return [];
+  }
+  const scanner = new import_scanner.DirectoryScanner({
+    basePath: employeeRoot,
+    pattern: ["skills/**/SKILLS.md"]
+  });
+  const skillFiles = await scanner.scan();
+  return uniq(skillFiles.map((it) => it.directory).filter(Boolean));
+}
+__name(discoverSkills, "discoverSkills");
+async function discoverTools(employeeRoot) {
+  const toolsDir = import_path.default.join(employeeRoot, "tools");
+  if (!(0, import_fs.existsSync)(toolsDir)) {
+    return [];
+  }
+  const scanner = new import_scanner.DirectoryScanner({
+    basePath: employeeRoot,
+    pattern: ["tools/**/*.ts", "tools/**/*.js", "!tools/**/*.d.ts"]
+  });
+  const toolFiles = await scanner.scan();
+  return uniqBy(
+    toolFiles.map(
+      (it) => normalizeAIEmployeeToolSetting(
+        it.basename === "index.ts" || it.basename === "index.js" ? it.directory : it.name
+      )
+    ).filter((tool) => Boolean(tool == null ? void 0 : tool.name)),
+    "name"
+  );
+}
+__name(discoverTools, "discoverTools");
+function uniq(values) {
+  return [...new Set(values)];
+}
+__name(uniq, "uniq");
+function uniqBy(values, key) {
+  const valueMap = /* @__PURE__ */ new Map();
+  for (const value of values) {
+    valueMap.set(value[key], value);
+  }
+  return [...valueMap.values()];
+}
+__name(uniqBy, "uniqBy");
+function normalizeAIEmployeeToolSetting(tool) {
+  if (!tool) {
+    return;
+  }
+  if (typeof tool === "string") {
+    return { name: tool };
+  }
+  if (!tool.name) {
+    return;
+  }
+  return {
+    name: tool.name,
+    ...tool.autoCall != null ? { autoCall: tool.autoCall } : {}
+  };
+}
+__name(normalizeAIEmployeeToolSetting, "normalizeAIEmployeeToolSetting");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AIEmployeeLoader,
+  normalizeAIEmployeeToolSetting
+});

package/lib/loader/index.d.ts

@@ -9,3 +9,6 @@
 export * from './types';
 export * from './scanner';
 export * from './tools';
+export * from './skills';
+export * from './mcp';
+export * from './employee';

package/lib/loader/index.js

@@ -26,9 +26,15 @@ module.exports = __toCommonJS(loader_exports);
 __reExport(loader_exports, require("./types"), module.exports);
 __reExport(loader_exports, require("./scanner"), module.exports);
 __reExport(loader_exports, require("./tools"), module.exports);
+__reExport(loader_exports, require("./skills"), module.exports);
+__reExport(loader_exports, require("./mcp"), module.exports);
+__reExport(loader_exports, require("./employee"), module.exports);
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   ...require("./types"),
   ...require("./scanner"),
-  ...require("./tools")
+  ...require("./tools"),
+  ...require("./skills"),
+  ...require("./mcp"),
+  ...require("./employee")
 });

package/lib/loader/mcp.d.ts

@@ -0,0 +1,34 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { Logger } from '@nocobase/logger';
+import { AIManager } from '../ai-manager';
+import { MCPOptions } from '../mcp-manager';
+import { LoadAndRegister } from './types';
+import { DirectoryScanner, DirectoryScannerOptions, FileDescriptor } from './scanner';
+export type MCPLoaderOptions = {
+    scan: DirectoryScannerOptions;
+    log?: Logger;
+};
+export declare class MCPLoader extends LoadAndRegister<MCPLoaderOptions> {
+    protected readonly ai: AIManager;
+    protected readonly options: MCPLoaderOptions;
+    protected readonly scanner: DirectoryScanner;
+    protected files: FileDescriptor[];
+    protected mcpDescriptors: MCPDescriptor[];
+    protected log: Logger;
+    constructor(ai: AIManager, options: MCPLoaderOptions);
+    protected scan(): Promise<void>;
+    protected import(): Promise<void>;
+    protected register(): Promise<void>;
+}
+export type MCPDescriptor = {
+    name: string;
+    file: FileDescriptor;
+    options: MCPOptions;
+};

package/lib/loader/mcp.js

@@ -0,0 +1,108 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var mcp_exports = {};
+__export(mcp_exports, {
+  MCPLoader: () => MCPLoader
+});
+module.exports = __toCommonJS(mcp_exports);
+var import_utils = require("@nocobase/utils");
+var import_fs = require("fs");
+var import_types = require("./types");
+var import_scanner = require("./scanner");
+var import_utils2 = require("./utils");
+const _MCPLoader = class _MCPLoader extends import_types.LoadAndRegister {
+  constructor(ai, options) {
+    super(ai, options);
+    this.ai = ai;
+    this.options = options;
+    this.log = options.log;
+    this.scanner = new import_scanner.DirectoryScanner(this.options.scan);
+  }
+  scanner;
+  files = [];
+  mcpDescriptors = [];
+  log;
+  async scan() {
+    this.files = await this.scanner.scan();
+  }
+  async import() {
+    if (!this.files.length) {
+      return;
+    }
+    const descriptors = await Promise.all(
+      this.files.map(async (file) => {
+        var _a, _b, _c;
+        const name = file.name;
+        if (!(0, import_fs.existsSync)(file.path)) {
+          (_a = this.log) == null ? void 0 : _a.error(`mcp [${name}] ignored: can not find definition file at ${file.path}`);
+          return null;
+        }
+        try {
+          const imported = await (0, import_utils.importModule)(file.path);
+          const mod = (imported == null ? void 0 : imported.default) ?? imported;
+          const options = typeof mod === "function" ? mod() : mod;
+          if (!(0, import_utils2.isNonEmptyObject)(options)) {
+            (_b = this.log) == null ? void 0 : _b.warn(`mcp [${name}] register ignored: invalid definition at ${file.path}`);
+            return null;
+          }
+          return {
+            name,
+            file,
+            options
+          };
+        } catch (e) {
+          (_c = this.log) == null ? void 0 : _c.error(`mcp [${name}] load fail: error occur when import ${file.path}`, e);
+          return null;
+        }
+      })
+    );
+    this.mcpDescriptors = descriptors.filter((item) => Boolean(item));
+  }
+  async register() {
+    var _a;
+    if (!this.mcpDescriptors.length) {
+      return;
+    }
+    const { mcpManager } = this.ai;
+    for (const descriptor of this.mcpDescriptors) {
+      try {
+        await mcpManager.registerMCP({
+          [descriptor.name]: descriptor.options
+        });
+      } catch (e) {
+        (_a = this.log) == null ? void 0 : _a.error(`mcp [${descriptor.name}] register ignored: error occur when invoke registerMCP`, e);
+      }
+    }
+  }
+};
+__name(_MCPLoader, "MCPLoader");
+let MCPLoader = _MCPLoader;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  MCPLoader
+});

package/lib/loader/skills.d.ts

@@ -0,0 +1,42 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { DirectoryScanner, DirectoryScannerOptions, FileDescriptor } from './scanner';
+import { AIManager } from '../ai-manager';
+import { LoadAndRegister } from './types';
+import { Logger } from '@nocobase/logger';
+import { SkillsScope } from '../skills-manager';
+export type SkillsLoaderOptions = {
+    scan: DirectoryScannerOptions;
+    log?: Logger;
+};
+export declare class SkillsLoader extends LoadAndRegister<SkillsLoaderOptions> {
+    protected readonly ai: AIManager;
+    protected readonly options: SkillsLoaderOptions;
+    protected readonly scanner: DirectoryScanner;
+    protected files: FileDescriptor[];
+    protected skillsDescriptors: SkillsDescriptor[];
+    protected log: Logger;
+    constructor(ai: AIManager, options: SkillsLoaderOptions);
+    protected scan(): Promise<void>;
+    protected import(): Promise<void>;
+    protected register(): Promise<void>;
+}
+export type SkillsDescriptor = {
+    scope: SkillsScope;
+    name: string;
+    description: string;
+    content: string;
+    skillsFile: FileDescriptor;
+    skillsDir: FileDescriptor;
+    tools?: string[];
+    introduction?: {
+        title: string;
+        about?: string;
+    };
+};

package/lib/loader/skills.js

@@ -0,0 +1,138 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var skills_exports = {};
+__export(skills_exports, {
+  SkillsLoader: () => SkillsLoader
+});
+module.exports = __toCommonJS(skills_exports);
+var import_scanner = require("./scanner");
+var import_promises = require("fs/promises");
+var import_fs = require("fs");
+var import_types = require("./types");
+var import_gray_matter = __toESM(require("gray-matter"));
+var import_path = __toESM(require("path"));
+const _SkillsLoader = class _SkillsLoader extends import_types.LoadAndRegister {
+  constructor(ai, options) {
+    super(ai, options);
+    this.ai = ai;
+    this.options = options;
+    this.log = options.log;
+    this.scanner = new import_scanner.DirectoryScanner(this.options.scan);
+  }
+  scanner;
+  files = [];
+  skillsDescriptors = [];
+  log;
+  async scan() {
+    this.files = await this.scanner.scan();
+  }
+  async import() {
+    if (!this.files.length) {
+      return;
+    }
+    this.skillsDescriptors = await Promise.all(
+      this.files.map(async (skillsFile) => {
+        var _a, _b, _c;
+        if (skillsFile.basename !== "SKILLS.md") {
+          return null;
+        }
+        if (!(0, import_fs.existsSync)(skillsFile.path)) {
+          (_a = this.log) == null ? void 0 : _a.error(`skills [${skillsFile.directory}] ignored: can not find SKILLS.md at ${skillsFile.path}`);
+          return null;
+        }
+        const skillsDir = new import_scanner.FileDescriptor(import_path.default.dirname(skillsFile.path));
+        const name = skillsFile.directory;
+        const entry = { name, skillsFile, skillsDir };
+        try {
+          const skills = await (0, import_promises.readFile)(skillsFile.path, "utf-8");
+          const { data, content } = (0, import_gray_matter.default)(skills);
+          entry.scope = data["scope"] ?? "SPECIFIED";
+          entry.name = data["name"];
+          entry.description = data["description"];
+          entry.content = content;
+          entry.introduction = data["introduction"];
+          entry.tools = data["tools"] ?? [];
+        } catch (e) {
+          (_b = this.log) == null ? void 0 : _b.error(`skills [${name}] load fail: error occur when reading SKILLS.md at ${skillsFile.path}`, e);
+          return null;
+        }
+        try {
+          const toolsScanner = new import_scanner.DirectoryScanner({
+            basePath: skillsDir.path,
+            pattern: ["tools/**/*.ts", "tools/**/*.js", "!tools/**/*.d.ts"]
+          });
+          const toolsFiles = await toolsScanner.scan();
+          entry.tools = Array.from(
+            /* @__PURE__ */ new Set([
+              ...entry.tools,
+              ...toolsFiles.map(
+                (it) => it.basename === "index.ts" || it.basename === "index.js" ? it.directory : it.name
+              )
+            ])
+          );
+        } catch (e) {
+          (_c = this.log) == null ? void 0 : _c.error(`skills [${name}] load fail: error occur when loading tools at ${skillsDir.path}`, e);
+          return null;
+        }
+        return entry;
+      }).filter((it) => it != null)
+    );
+  }
+  async register() {
+    if (!this.skillsDescriptors.length) {
+      return;
+    }
+    const { skillsManager } = this.ai;
+    for (const descriptor of this.skillsDescriptors) {
+      await skillsManager.registerSkills({
+        scope: descriptor.scope,
+        name: descriptor.name,
+        description: descriptor.description,
+        content: descriptor.content,
+        tools: descriptor.tools,
+        introduction: descriptor.introduction
+      });
+    }
+  }
+};
+__name(_SkillsLoader, "SkillsLoader");
+let SkillsLoader = _SkillsLoader;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SkillsLoader
+});

package/lib/mcp-manager/index.d.ts

@@ -0,0 +1,43 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { MultiServerMCPClient } from '@langchain/mcp-adapters';
+import { MCPEntry, MCPFilter, MCPManager, MCPOptions, MCPTestResult, MCPToolEntry } from './types';
+import type { DynamicToolsProvider, Permission } from '../tools-manager/types';
+export declare class DefaultMCPManager implements MCPManager {
+    private readonly app;
+    private readonly mcpRegistry;
+    private readonly provideCollectionManager;
+    private mode;
+    private client;
+    private toolsMap;
+    private toolsPermissionMap;
+    constructor(app: any);
+    init(): Promise<void>;
+    registerMCP(registration: {
+        [key: string | symbol]: MCPOptions;
+    }): Promise<void>;
+    getMCP(name: string): Promise<MCPEntry>;
+    listMCP(filter?: MCPFilter): Promise<MCPEntry[]>;
+    rebuildClient(): Promise<void>;
+    getClient(): MultiServerMCPClient | null;
+    getMCPToolsProvider(): DynamicToolsProvider;
+    listMCPTools(): Promise<Record<string, MCPToolEntry[]>>;
+    updateMCPToolPermission(toolName: string, permission: Permission): Promise<void>;
+    testConnection(options: MCPOptions): Promise<MCPTestResult>;
+    private buildMCPConnection;
+    persistence(): Promise<void>;
+    private persistenceEntry;
+    private normalizeEntry;
+    private get aiMcpClientsCollection();
+    private get aiMcpClientsModel();
+    private get sequelize();
+    private get collectionManager();
+}
+export declare function defineMCP(options: MCPOptions): MCPOptions;
+export * from './types';