@nocobase/ai 2.1.0-beta.2 → 2.1.0-beta.20

package/src/loader/employee.ts

@@ -0,0 +1,194 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { importModule } from '@nocobase/utils';
+import { DirectoryScanner, DirectoryScannerOptions, FileDescriptor } from './scanner';
+import { existsSync } from 'fs';
+import { readFile } from 'fs/promises';
+import { AIManager } from '../ai-manager';
+import { LoadAndRegister } from './types';
+import { Logger } from '@nocobase/logger';
+import path from 'path';
+import { AIEmployeeOptions, AIEmployeeToolSetting } from '../ai-employee-manager';
+
+export type AIEmployeeLoaderOptions = { pluginName: string; scan: DirectoryScannerOptions; log?: Logger };
+
+export class AIEmployeeLoader extends LoadAndRegister<AIEmployeeLoaderOptions> {
+  protected readonly scanner: DirectoryScanner;
+
+  protected files: FileDescriptor[] = [];
+  protected employeeDescriptors: AIEmployeeDescriptor[] = [];
+  protected log: Logger;
+
+  constructor(
+    protected readonly ai: AIManager,
+    protected readonly options: AIEmployeeLoaderOptions,
+  ) {
+    super(ai, options);
+    this.log = options.log;
+    this.scanner = new DirectoryScanner(this.options.scan);
+  }
+
+  protected async scan(): Promise<void> {
+    this.files = await this.scanner.scan();
+  }
+
+  protected async import(): Promise<void> {
+    if (!this.files.length) {
+      return;
+    }
+
+    const grouped = new Map<string, FileDescriptor[]>();
+    for (const fd of this.files) {
+      const employeeRoot = getEmployeeRoot(fd);
+      const group = grouped.get(employeeRoot) ?? [];
+      group.push(fd);
+      grouped.set(employeeRoot, group);
+    }
+
+    const descriptors = await Promise.all(
+      Array.from(grouped.entries()).map(async ([employeeRoot, fds]) => {
+        const file = fds.find((fd) => fd.extname === '.ts' || fd.extname === '.js');
+        if (!file || !existsSync(file.path)) {
+          return null;
+        }
+        const promptFile = fds.find((fd) => fd.basename === 'prompt.md');
+
+        const name = path.basename(employeeRoot);
+        try {
+          const imported = await importModule(file.path);
+          const mod = imported?.default ?? imported;
+          const employeeOptions = typeof mod === 'function' ? mod() : mod;
+          if (!employeeOptions || !employeeOptions.username) {
+            this.log?.warn(`ai employee [${name}] register ignored: invalid definition at ${file.path}`);
+            return null;
+          }
+          const { skills = [], tools = [] } = employeeOptions;
+
+          if (promptFile && existsSync(promptFile.path)) {
+            try {
+              employeeOptions.systemPrompt = await readFile(promptFile.path, 'utf-8');
+            } catch (e) {
+              this.log?.error(
+                `ai employee [${name}] load fail: error occur when reading prompt.md at ${promptFile.path}`,
+                e,
+              );
+              return null;
+            }
+          }
+
+          return {
+            name,
+            employeeRoot,
+            file,
+            options: {
+              ...employeeOptions,
+              skills: [...skills, ...(await discoverSkills(employeeRoot))],
+              tools: [...tools, ...(await discoverTools(employeeRoot))],
+            } as AIEmployeeOptions,
+          } satisfies AIEmployeeDescriptor;
+        } catch (e) {
+          this.log?.error(`ai employee [${name}] load fail: error occur when import ${file.path}`, e);
+          return null;
+        }
+      }),
+    );
+
+    this.employeeDescriptors = descriptors.filter((item): item is AIEmployeeDescriptor => Boolean(item));
+  }
+
+  protected async register(): Promise<void> {
+    if (!this.employeeDescriptors.length) {
+      return;
+    }
+    const { employeeManager } = this.ai;
+    for (const descriptor of this.employeeDescriptors) {
+      await employeeManager.registerEmployee(descriptor.options);
+    }
+  }
+}
+
+export type AIEmployeeDescriptor = {
+  name: string;
+  employeeRoot: string;
+  file: FileDescriptor;
+  options: AIEmployeeOptions;
+};
+
+function getEmployeeRoot(fd: FileDescriptor) {
+  if (fd.basename === 'index.ts' || fd.basename === 'index.js' || fd.basename === 'prompt.md') {
+    return path.dirname(fd.path);
+  }
+  return fd.path;
+}
+
+async function discoverSkills(employeeRoot: string): Promise<string[]> {
+  const skillsDir = path.join(employeeRoot, 'skills');
+  if (!existsSync(skillsDir)) {
+    return [];
+  }
+  const scanner = new DirectoryScanner({
+    basePath: employeeRoot,
+    pattern: ['skills/**/SKILLS.md'],
+  });
+  const skillFiles = await scanner.scan();
+  return uniq(skillFiles.map((it) => it.directory).filter(Boolean));
+}
+
+async function discoverTools(employeeRoot: string): Promise<AIEmployeeToolSetting[]> {
+  const toolsDir = path.join(employeeRoot, 'tools');
+  if (!existsSync(toolsDir)) {
+    return [];
+  }
+  const scanner = new DirectoryScanner({
+    basePath: employeeRoot,
+    pattern: ['tools/**/*.ts', 'tools/**/*.js', '!tools/**/*.d.ts'],
+  });
+  const toolFiles = await scanner.scan();
+  return uniqBy(
+    toolFiles
+      .map((it) =>
+        normalizeAIEmployeeToolSetting(
+          it.basename === 'index.ts' || it.basename === 'index.js' ? it.directory : it.name,
+        ),
+      )
+      .filter((tool): tool is AIEmployeeToolSetting => Boolean(tool?.name)),
+    'name',
+  );
+}
+
+function uniq<T>(values: T[]) {
+  return [...new Set(values)];
+}
+
+function uniqBy<T extends Record<string, unknown>>(values: T[], key: keyof T) {
+  const valueMap = new Map<unknown, T>();
+  for (const value of values) {
+    valueMap.set(value[key], value);
+  }
+  return [...valueMap.values()];
+}
+
+export function normalizeAIEmployeeToolSetting(
+  tool?: string | AIEmployeeToolSetting,
+): AIEmployeeToolSetting | undefined {
+  if (!tool) {
+    return;
+  }
+  if (typeof tool === 'string') {
+    return { name: tool };
+  }
+  if (!tool.name) {
+    return;
+  }
+  return {
+    name: tool.name,
+    ...(tool.autoCall != null ? { autoCall: tool.autoCall } : {}),
+  };
+}

package/src/loader/index.ts

@@ -10,3 +10,6 @@
 export * from './types';
 export * from './scanner';
 export * from './tools';
+export * from './skills';
+export * from './mcp';
+export * from './employee';

package/src/loader/mcp.ts

@@ -0,0 +1,101 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { importModule } from '@nocobase/utils';
+import { existsSync } from 'fs';
+import { Logger } from '@nocobase/logger';
+import { AIManager } from '../ai-manager';
+import { MCPOptions } from '../mcp-manager';
+import { LoadAndRegister } from './types';
+import { DirectoryScanner, DirectoryScannerOptions, FileDescriptor } from './scanner';
+import { isNonEmptyObject } from './utils';
+
+export type MCPLoaderOptions = { pluginName: string; scan: DirectoryScannerOptions; log?: Logger };
+
+export class MCPLoader extends LoadAndRegister<MCPLoaderOptions> {
+  protected readonly scanner: DirectoryScanner;
+
+  protected files: FileDescriptor[] = [];
+  protected mcpDescriptors: MCPDescriptor[] = [];
+  protected log: Logger;
+
+  constructor(
+    protected readonly ai: AIManager,
+    protected readonly options: MCPLoaderOptions,
+  ) {
+    super(ai, options);
+    this.log = options.log;
+    this.scanner = new DirectoryScanner(this.options.scan);
+  }
+
+  protected async scan(): Promise<void> {
+    this.files = await this.scanner.scan();
+  }
+
+  protected async import(): Promise<void> {
+    if (!this.files.length) {
+      return;
+    }
+
+    const descriptors = await Promise.all(
+      this.files.map(async (file) => {
+        const name = file.name;
+        if (!existsSync(file.path)) {
+          this.log?.error(`mcp [${name}] ignored: can not find definition file at ${file.path}`);
+          return null;
+        }
+
+        try {
+          const imported = await importModule(file.path);
+          const mod = imported?.default ?? imported;
+          const options = typeof mod === 'function' ? mod() : mod;
+
+          if (!isNonEmptyObject(options)) {
+            this.log?.warn(`mcp [${name}] register ignored: invalid definition at ${file.path}`);
+            return null;
+          }
+
+          return {
+            name,
+            file,
+            options: options as MCPOptions,
+          } satisfies MCPDescriptor;
+        } catch (e) {
+          this.log?.error(`mcp [${name}] load fail: error occur when import ${file.path}`, e);
+          return null;
+        }
+      }),
+    );
+
+    this.mcpDescriptors = descriptors.filter((item): item is MCPDescriptor => Boolean(item));
+  }
+
+  protected async register(): Promise<void> {
+    if (!this.mcpDescriptors.length) {
+      return;
+    }
+
+    const { mcpManager } = this.ai;
+    for (const descriptor of this.mcpDescriptors) {
+      try {
+        await mcpManager.registerMCP({
+          [descriptor.name]: descriptor.options,
+        });
+      } catch (e) {
+        this.log?.error(`mcp [${descriptor.name}] register ignored: error occur when invoke registerMCP`, e);
+      }
+    }
+  }
+}
+
+export type MCPDescriptor = {
+  name: string;
+  file: FileDescriptor;
+  options: MCPOptions;
+};

package/src/loader/skills.ts

@@ -0,0 +1,129 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { DirectoryScanner, DirectoryScannerOptions, FileDescriptor } from './scanner';
+import { readFile } from 'fs/promises';
+import _ from 'lodash';
+import { existsSync } from 'fs';
+import { AIManager } from '../ai-manager';
+import { LoadAndRegister } from './types';
+import { Logger } from '@nocobase/logger';
+import matter from 'gray-matter';
+import path from 'path';
+import { SkillsScope } from '../skills-manager';
+
+export type SkillsLoaderOptions = { pluginName: string; scan: DirectoryScannerOptions; log?: Logger };
+export class SkillsLoader extends LoadAndRegister<SkillsLoaderOptions> {
+  protected readonly scanner: DirectoryScanner;
+
+  protected files: FileDescriptor[] = [];
+  protected skillsDescriptors: SkillsDescriptor[] = [];
+  protected log: Logger;
+
+  constructor(
+    protected readonly ai: AIManager,
+    protected readonly options: SkillsLoaderOptions,
+  ) {
+    super(ai, options);
+    this.log = options.log;
+    this.scanner = new DirectoryScanner(this.options.scan);
+  }
+
+  protected async scan(): Promise<void> {
+    this.files = await this.scanner.scan();
+  }
+
+  protected async import(): Promise<void> {
+    if (!this.files.length) {
+      return;
+    }
+
+    this.skillsDescriptors = await Promise.all(
+      this.files
+        .map(async (skillsFile) => {
+          if (skillsFile.basename !== 'SKILLS.md') {
+            return null;
+          }
+          if (!existsSync(skillsFile.path)) {
+            this.log?.error(`skills [${skillsFile.directory}] ignored: can not find SKILLS.md at ${skillsFile.path}`);
+            return null;
+          }
+          const skillsDir = new FileDescriptor(path.dirname(skillsFile.path));
+          const name = skillsFile.directory;
+          const entry: Partial<SkillsDescriptor> = { name, skillsFile, skillsDir };
+          try {
+            const skills = await readFile(skillsFile.path, 'utf-8');
+            const { data, content } = matter(skills);
+            entry.scope = data['scope'] ?? 'SPECIFIED';
+            entry.name = data['name'];
+            entry.description = data['description'];
+            entry.content = content;
+            entry.introduction = data['introduction'];
+            entry.tools = data['tools'] ?? [];
+          } catch (e) {
+            this.log?.error(`skills [${name}] load fail: error occur when reading SKILLS.md at ${skillsFile.path}`, e);
+            return null;
+          }
+
+          try {
+            const toolsScanner = new DirectoryScanner({
+              basePath: skillsDir.path,
+              pattern: ['tools/**/*.ts', 'tools/**/*.js', '!tools/**/*.d.ts'],
+            });
+            const toolsFiles = await toolsScanner.scan();
+            entry.tools = Array.from(
+              new Set([
+                ...entry.tools,
+                ...toolsFiles.map((it) =>
+                  it.basename === 'index.ts' || it.basename === 'index.js' ? it.directory : it.name,
+                ),
+              ]),
+            );
+          } catch (e) {
+            this.log?.error(`skills [${name}] load fail: error occur when loading tools at ${skillsDir.path}`, e);
+            return null;
+          }
+
+          return entry as SkillsDescriptor;
+        })
+        .filter((it) => it != null),
+    );
+  }
+
+  protected async register(): Promise<void> {
+    if (!this.skillsDescriptors.length) {
+      return;
+    }
+    const { skillsManager } = this.ai;
+    for (const descriptor of this.skillsDescriptors) {
+      await skillsManager.registerSkills({
+        scope: descriptor.scope,
+        name: descriptor.name,
+        description: descriptor.description,
+        content: descriptor.content,
+        tools: descriptor.tools,
+        introduction: descriptor.introduction,
+      });
+    }
+  }
+}
+
+export type SkillsDescriptor = {
+  scope: SkillsScope;
+  name: string;
+  description: string;
+  content: string;
+  skillsFile: FileDescriptor;
+  skillsDir: FileDescriptor;
+  tools?: string[];
+  introduction?: {
+    title: string;
+    about?: string;
+  };
+};

package/src/loader/tools.ts

@@ -18,7 +18,7 @@ import { LoadAndRegister } from './types';
 import { Logger } from '@nocobase/logger';
 import { isNonEmptyObject } from './utils';
 
-export type ToolsLoaderOptions = { scan: DirectoryScannerOptions; log?: Logger };
+export type ToolsLoaderOptions = { pluginName: string; scan: DirectoryScannerOptions; log?: Logger };
 export class ToolsLoader extends LoadAndRegister<ToolsLoaderOptions> {
   protected readonly scanner: DirectoryScanner;
 
@@ -106,7 +106,7 @@ export class ToolsLoader extends LoadAndRegister<ToolsLoaderOptions> {
         continue;
       }
       const { name, toolsOptions, description } = descriptor;
-      if (await toolsManager.getTools(name)) {
+      if (toolsManager.isToolsExisted(name)) {
         this.log?.warn(`tools [${descriptor.name}] register ignored: duplicate register for tools`);
         continue;
       }
@@ -118,6 +118,7 @@ export class ToolsLoader extends LoadAndRegister<ToolsLoaderOptions> {
       }
       try {
         toolsManager.registerTools(toolsOptions);
+        this.log?.info(`tools [${toolsOptions.definition.name}] registered from plugin [${this.options.pluginName}]`);
       } catch (e) {
         this.log?.error(`tools [${descriptor.name}] register ignored: error occur when invoke registerTools`, e);
         continue;