@nocobase/ai 2.0.0

package/src/__tests__/resource/ai/tools/group/group3/index.ts

@@ -0,0 +1,23 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Context } from '@nocobase/actions';
+import { defineTools } from '@nocobase/ai';
+
+export default defineTools({
+  scope: 'GENERAL',
+  definition: {
+    name: 'group3',
+    description: 'hallow group3',
+    schema: null,
+  },
+  invoke: async (ctx: Context, args: any, id: string) => {
+    return { status: 'success' };
+  },
+});

package/src/__tests__/resource/ai/tools/hallow/index.ts

@@ -0,0 +1,23 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Context } from '@nocobase/actions';
+import { defineTools } from '@nocobase/ai';
+
+export default defineTools({
+  scope: 'GENERAL',
+  definition: {
+    name: 'hallow',
+    description: 'hallow tools',
+    schema: null,
+  },
+  invoke: async (ctx: Context, args: any, id: string) => {
+    return { status: 'success' };
+  },
+});

package/src/__tests__/resource/ai/tools/ignored.ts

@@ -0,0 +1,13 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Context } from '@nocobase/actions';
+import { defineTools } from '@nocobase/ai';
+
+export const IGNORE = true;

package/src/__tests__/resource/ai/tools/jsfiles/formFiller.js

@@ -0,0 +1,61 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all) __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if ((from && typeof from === 'object') || typeof from === 'function') {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, {
+          get: () => from[key],
+          enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable,
+        });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, '__esModule', { value: true }), mod);
+var formFiller_exports = {};
+__export(formFiller_exports, {
+  default: () => formFiller_default,
+});
+module.exports = __toCommonJS(formFiller_exports);
+var import_ai = require('@nocobase/ai');
+var import_zod = require('zod');
+var formFiller_default = (0, import_ai.defineTools)({
+  scope: 'GENERAL',
+  defaultPermission: 'ALLOW',
+  execution: 'frontend',
+  introduction: {
+    title: '{{t("Form filler")}}',
+    about: '{{t("Fill the form with the given content")}}',
+  },
+  definition: {
+    name: 'formFiller',
+    description: 'Fill the form with the given content',
+    schema: import_zod.z.object({
+      form: import_zod.z.string().describe('The UI Schema ID of the target form to be filled.'),
+      data: import_zod.z.object({}).catchall(import_zod.z.any()).describe(
+        `Structured key-value pairs matching the form's JSON Schema,
+       to be assigned to form.values.
+       Example: { "username": "alice", "email": "alice@example.com", "age": 30 }`,
+      ),
+    }),
+  },
+  invoke: async () => {
+    return {
+      status: 'success',
+    };
+  },
+});

package/src/__tests__/resource/ai/tools/jsfiles/formFiller2/index.js

@@ -0,0 +1,61 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all) __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if ((from && typeof from === 'object') || typeof from === 'function') {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, {
+          get: () => from[key],
+          enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable,
+        });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, '__esModule', { value: true }), mod);
+var formFiller_exports = {};
+__export(formFiller_exports, {
+  default: () => formFiller_default,
+});
+module.exports = __toCommonJS(formFiller_exports);
+var import_ai = require('@nocobase/ai');
+var import_zod = require('zod');
+var formFiller_default = (0, import_ai.defineTools)({
+  scope: 'GENERAL',
+  defaultPermission: 'ALLOW',
+  execution: 'frontend',
+  introduction: {
+    title: '{{t("Form filler")}}',
+    about: '{{t("Fill the form with the given content")}}',
+  },
+  definition: {
+    name: 'formFiller',
+    description: 'Fill the form with the given content',
+    schema: import_zod.z.object({
+      form: import_zod.z.string().describe('The UI Schema ID of the target form to be filled.'),
+      data: import_zod.z.object({}).catchall(import_zod.z.any()).describe(
+        `Structured key-value pairs matching the form's JSON Schema,
+       to be assigned to form.values.
+       Example: { "username": "alice", "email": "alice@example.com", "age": 30 }`,
+      ),
+    }),
+  },
+  invoke: async () => {
+    return {
+      status: 'success',
+    };
+  },
+});

package/src/__tests__/resource/ai/tools/print.ts

@@ -0,0 +1,23 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Context } from '@nocobase/actions';
+import { defineTools } from '@nocobase/ai';
+
+export default defineTools({
+  scope: 'GENERAL',
+  definition: {
+    name: 'print',
+    description: 'print tools',
+    schema: null,
+  },
+  invoke: async (ctx: Context, args: any, id: string) => {
+    return { status: 'success' };
+  },
+});

package/src/__tests__/tools.test.ts

@@ -0,0 +1,133 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { createMockServer, MockServer } from '@nocobase/test';
+import { ToolsLoader } from '../loader';
+import path from 'path';
+import { AIManager } from '../ai-manager';
+import { ToolsManager } from '../tools-manager';
+
+describe('Tools loader test cases', () => {
+  const basePath = path.join(process.cwd(), 'packages/core/ai/src/__tests__/resource/ai');
+  let app: MockServer;
+  let aiManager: AIManager;
+  let toolsManager: ToolsManager;
+  let loader: ToolsLoader;
+
+  beforeEach(async () => {
+    app = await createMockServer();
+    aiManager = app.aiManager;
+    toolsManager = aiManager.toolsManager;
+    loader = new ToolsLoader(aiManager, {
+      scan: {
+        basePath,
+        pattern: ['**/tools/**/*.ts', '**/tools/**/*.js', '!**/tools/**/*.d.ts', '**/tools/**/*/description.md'],
+      },
+    });
+  });
+
+  afterEach(async () => {
+    await app.destroy();
+  });
+
+  it('should load tools file in tools root directory ', async () => {
+    await loader.load();
+    const tools = await toolsManager.getTools('print');
+    expect(tools).toBeDefined();
+    expect(tools.definition.name).eq('print');
+    expect(tools.definition.description).eq('print tools');
+    expect(await tools.invoke(null, null, null)).toEqual({ status: 'success' });
+  });
+
+  it('should load tools directory in tools root directory ', async () => {
+    await loader.load();
+    const tools = await toolsManager.getTools('hallow');
+    expect(tools).toBeDefined();
+    expect(tools.definition.name).eq('hallow');
+    expect(tools.definition.description).eq('hallow tools');
+    expect(await tools.invoke(null, null, null)).toEqual({ status: 'success' });
+  });
+
+  it('should load description markdown file in tools directory ', async () => {
+    await loader.load();
+    const tools = await toolsManager.getTools('desc');
+    expect(tools).toBeDefined();
+    expect(tools.definition.name).eq('desc');
+    expect(tools.definition.description).eq('# DESC\n');
+    expect(await tools.invoke(null, null, null)).toEqual({ status: 'success' });
+    expect(await tools.invoke(null, null, null)).toEqual({ status: 'success' });
+  });
+
+  it('should load tools file in skills directory ', async () => {
+    await loader.load();
+    const tools = await toolsManager.getTools('read');
+    expect(tools).toBeDefined();
+    expect(tools.definition.name).eq('read');
+    expect(tools.definition.description).eq('read document');
+    expect(await tools.invoke(null, null, null)).toEqual({ status: 'success' });
+  });
+
+  it('should load tools directory in skills directory ', async () => {
+    await loader.load();
+    const tools = await toolsManager.getTools('search');
+    expect(tools).toBeDefined();
+    expect(tools.definition.name).eq('search');
+    expect(tools.definition.description).eq('# SEARCH\n');
+    expect(await tools.invoke(null, null, null)).toEqual({ status: 'success' });
+  });
+
+  it('should load tools when tools in directory for grouping', async () => {
+    await loader.load();
+    const group1 = await toolsManager.getTools('group1');
+    expect(group1).toBeDefined();
+    expect(group1.definition.name).eq('group1');
+    expect(group1.definition.description).eq('hallow group1');
+    expect(await group1.invoke(null, null, null)).toEqual({ status: 'success' });
+
+    const group2 = await toolsManager.getTools('group2');
+    expect(group2).toBeDefined();
+    expect(group2.definition.name).eq('group2');
+    expect(group2.definition.description).eq('hallow group2');
+    expect(await group2.invoke(null, null, null)).toEqual({ status: 'success' });
+
+    const group3 = await toolsManager.getTools('group3');
+    expect(group3).toBeDefined();
+    expect(group3.definition.name).eq('group3');
+    expect(group3.definition.description).eq('# GROUP3\n');
+    expect(await group3.invoke(null, null, null)).toEqual({ status: 'success' });
+  });
+
+  it('should ignore tools which not export default', async () => {
+    await loader.load();
+    const tools = await toolsManager.getTools('ignored');
+    expect(tools).toBeUndefined();
+  });
+
+  it.skip('should load .js file', async () => {
+    await loader.load();
+    const toolsNames = ['formFiller', 'formFiller2'];
+    for (const toolsName of toolsNames) {
+      const tools = await toolsManager.getTools(toolsName);
+      expect(tools).toBeDefined();
+      expect(tools.definition.name).eq(toolsName);
+      expect(tools.definition.description).eq('Fill the form with the given content');
+      expect(await tools.invoke(null, null, null)).toEqual({ status: 'success' });
+    }
+  });
+
+  it('should ignore empty file', async () => {
+    const toolsLoader = new ToolsLoader(aiManager, {
+      scan: { basePath, pattern: ['tools/empty.ts'] },
+    });
+    // const saved = process.env.VITEST;
+    delete process.env.VITEST;
+    await expect(toolsLoader.load()).resolves.not.toThrow();
+    // process.env.VITEST = saved;
+  });
+});

package/src/ai-manager.ts

@@ -0,0 +1,21 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { DocumentManager } from './document-manager';
+import { DefaultToolsManager, ToolsManager } from './tools-manager';
+
+export class AIManager {
+  documentManager: DocumentManager;
+  toolsManager: ToolsManager;
+
+  constructor(protected readonly app: any) {
+    this.documentManager = new DocumentManager();
+    this.toolsManager = new DefaultToolsManager();
+  }
+}

package/src/document-manager/document.ts

@@ -0,0 +1,9 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+

package/src/document-manager/id-mapper.ts

@@ -0,0 +1,56 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+export class IdMapper {
+  private nextId = 1;
+  private ext2num = new Map<string, number>();
+  private num2ext = new Map<number, string>();
+
+  toNumeric(id: string | number): number {
+    if (typeof id === 'number') {
+      return id;
+    }
+
+    let num = this.ext2num.get(id);
+    if (!num) {
+      num = this.nextId++;
+      this.ext2num.set(id, num);
+      this.num2ext.set(num, id);
+    }
+    return num;
+  }
+
+  getNumeric(id: string | number): number | undefined {
+    if (typeof id === 'number') {
+      return id;
+    }
+
+    return this.ext2num.get(id);
+  }
+
+  toExternal(id: number): string | number {
+    return this.num2ext.get(id) ?? id;
+  }
+
+  remove(id: string | number) {
+    if (typeof id === 'number') {
+      const ext = this.num2ext.get(id);
+      if (ext) {
+        this.num2ext.delete(id);
+        this.ext2num.delete(ext);
+      }
+    } else {
+      const num = this.ext2num.get(id);
+      if (num) {
+        this.ext2num.delete(id);
+        this.num2ext.delete(num);
+      }
+    }
+  }
+}

package/src/document-manager/index.ts

@@ -0,0 +1,38 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Document, DocumentOptions, IndexOptions } from 'flexsearch';
+import { Index } from './search-index';
+
+export class DocumentManager {
+  indexes: Map<string, Index> = new Map();
+  documents: Map<string, Document> = new Map();
+
+  addIndex(name: string, options?: IndexOptions) {
+    const index = new Index(options);
+    this.indexes.set(name, index);
+    return index;
+  }
+
+  getIndex(name: string): Index | undefined {
+    return this.indexes.get(name);
+  }
+
+  addDocument(name: string, options?: DocumentOptions) {
+    const doc = new Document(options);
+    this.documents.set(name, doc);
+    return doc;
+  }
+
+  getDocument(name: string): Document | undefined {
+    return this.documents.get(name);
+  }
+}
+
+export { Index as FlexSearchIndex } from 'flexsearch';

package/src/document-manager/search-index.ts

@@ -0,0 +1,48 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Index as BaseIndex, ExportHandler, IndexOptions } from 'flexsearch';
+import { IdMapper } from './id-mapper';
+
+export class Index {
+  private index: BaseIndex;
+  private ids = new IdMapper();
+
+  constructor(options?: IndexOptions) {
+    this.index = new BaseIndex(options);
+  }
+
+  async add(id: string | number, text: string) {
+    const numericId = this.ids.toNumeric(id);
+    return this.index.addAsync(numericId, text);
+  }
+
+  async remove(id: string | number) {
+    const numericId = this.ids.getNumeric(id);
+    if (numericId === undefined) {
+      return;
+    }
+
+    await this.index.removeAsync(numericId);
+    this.ids.remove(id);
+  }
+
+  async search(query: string, options?: any): Promise<(string | number)[]> {
+    const result = (await this.index.searchAsync(query, options)) as number[];
+    return result.map((id) => this.ids.toExternal(id));
+  }
+
+  export(handler: ExportHandler) {
+    return this.index.export(handler);
+  }
+
+  import(key: string, data: string) {
+    return this.index.import(key, data);
+  }
+}

package/src/index.ts

@@ -0,0 +1,13 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+export * from './ai-manager';
+export * from './document-manager';
+export * from './tools-manager';
+export * from './loader';

package/src/loader/index.ts

@@ -0,0 +1,12 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+export * from './types';
+export * from './scanner';
+export * from './tools';

package/src/loader/scanner.ts

@@ -0,0 +1,54 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import fg from 'fast-glob';
+import path from 'path';
+
+export type DirectoryScannerOptions = {
+  basePath: string;
+  pattern: string[];
+};
+
+export class DirectoryScanner {
+  protected readonly files: FileDescriptor[] = [];
+
+  constructor(private readonly options: DirectoryScannerOptions) {}
+
+  async scan(): Promise<FileDescriptor[]> {
+    const { basePath, pattern } = this.options;
+    const filePaths = await fg(pattern, {
+      cwd: basePath,
+      absolute: true,
+      onlyFiles: true,
+      dot: false,
+      followSymbolicLinks: false,
+    });
+
+    return filePaths.map((f) => new FileDescriptor(f));
+  }
+}
+
+export class FileDescriptor {
+  constructor(private readonly filePath: string) {}
+  get name() {
+    return path.parse(this.filePath).name;
+  }
+  get directory() {
+    return path.basename(path.dirname(this.filePath));
+  }
+  get path() {
+    return this.filePath;
+  }
+  get extname() {
+    return path.extname(this.filePath);
+  }
+  get basename() {
+    return path.basename(this.filePath);
+  }
+}