@xpert-ai/plugin-sdk 3.6.0-beta.6 → 3.6.0

package/jest.config.ts

@@ -0,0 +1,28 @@
+/* eslint-disable */
+import { readFileSync } from 'fs'
+
+// Reading the SWC compilation config and remove the "exclude"
+// for the test files to be compiled by SWC
+const { exclude: _, ...swcJestConfig } = JSON.parse(readFileSync(`${__dirname}/.swcrc`, 'utf-8'))
+
+// disable .swcrc look-up by SWC core because we're passing in swcJestConfig ourselves.
+// If we do not disable this, SWC Core will read .swcrc and won't transform our test files due to "exclude"
+if (swcJestConfig.swcrc === undefined) {
+  swcJestConfig.swcrc = false
+}
+
+// Uncomment if using global setup/teardown files being transformed via swc
+// https://nx.dev/packages/jest/documents/overview#global-setup/teardown-with-nx-libraries
+// jest needs EsModule Interop to find the default exported setup/teardown functions
+// swcJestConfig.module.noInterop = false;
+
+export default {
+  displayName: 'plugin-sdk',
+  preset: '../../jest.preset.js',
+  transform: {
+    '^.+\\.[tj]s$': ['@swc/jest', swcJestConfig]
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  testEnvironment: 'node',
+  coverageDirectory: '../../coverage/packages/plugin-sdk'
+}

package/package.json

@@ -1,11 +1,24 @@
 {
   "name": "@xpert-ai/plugin-sdk",
-  "version": "3.6.0-beta.6",
+  "version": "3.6.0",
   "license": "AGPL-3.0",
   "dependencies": {},
-  "main": "./index.cjs.js",
-  "module": "./index.esm.js",
+  "main": "./index.cjs",
+  "module": "./index.js",
   "publishConfig": {
     "access": "public"
+  },
+  "peerDependencies": {
+    "@langchain/core": "*",
+    "@metad/contracts": "*",
+    "@nestjs/common": "*",
+    "@nestjs/core": "*",
+    "@nestjs/microservices": "*",
+    "lodash-es": "*",
+    "i18next-fs-backend": "*",
+    "i18next": "*",
+    "zod": "*",
+    "fs": "*",
+    "path": "*"
   }
-}
+}

package/project.json

@@ -0,0 +1,59 @@
+{
+  "name": "plugin-sdk",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "packages/plugin-sdk/src",
+  "projectType": "library",
+  "tags": [],
+  "targets": {
+    "build": {
+      "executor": "@nx/rollup:rollup",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "outputPath": "packages/plugin-sdk/dist",
+        "main": "packages/plugin-sdk/src/index.ts",
+        "tsConfig": "packages/plugin-sdk/tsconfig.lib.json",
+        "assets": [
+          {
+            "glob": "*.md",
+            "input": "packages/plugin-sdk/",
+            "output": "."
+          }
+        ],
+        "project": "packages/plugin-sdk/package.json",
+        "compiler": "swc",
+        "format": ["cjs", "esm"],
+        "external": [
+          "@langchain/core",
+          "@metad/contracts",
+          "@nestjs/common",
+          "@nestjs/core",
+          "@nestjs/microservices",
+          "lodash-es",
+          "i18next-fs-backend",
+          "i18next",
+          "kafkajs",
+          "amqplib",
+          "mqtt",
+          "nats",
+          "zod",
+          "fs",
+          "path"
+        ]
+      }
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "options": {
+        "jestConfig": "packages/plugin-sdk/jest.config.ts"
+      }
+    },
+    "nx-release-publish": {
+      "executor": "@nx/js:release-publish",
+      "options": {
+        "packageRoot": "packages/plugin-sdk/dist",
+        "access": "public"
+      }
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,11 @@
+export * from './lib/plugin'
+export * from './lib/plugin.interface'
+export * from './lib/plugin-metadata'
+export * from './lib/types'
+export * from './lib/logger'
+export * from './lib/integration/index'
+export * from './lib/workflow/index'
+export * from './lib/vectorstore/index'
+export * from './lib/rag/index'
+export * from './lib/toolset/index'
+export * from './lib/core/index'

package/src/lib/core/file-system.ts

@@ -0,0 +1,113 @@
+import fsPromises from 'fs/promises'
+import path from 'path'
+import { FileSystemPermission } from './permissions'
+
+
+/**
+ * Restricted FileSystem based on granted permissions
+ */
+export class XpFileSystem {
+  private allowedOps: Set<'read' | 'write' | 'delete' | 'list'>
+  private scope: string[] | undefined
+
+  constructor(permission: FileSystemPermission, private basePath: string, private baseUrl: string) {
+    this.allowedOps = new Set(permission.operations)
+    this.scope = permission.scope
+  }
+
+  /**
+   * Check if operation is allowed
+   */
+  private ensureAllowed(op: 'read' | 'write' | 'delete' | 'list') {
+    if (!this.allowedOps.has(op)) {
+      throw new Error(`Permission denied: ${op} operation not allowed`)
+    }
+  }
+
+  /**
+   * Check if path is within scope
+   */
+  private ensureInScope(targetPath: string) {
+    if (!this.scope || this.scope.length === 0) return
+    const resolved = path.resolve(targetPath)
+    for (const s of this.scope) {
+      const absScope = path.resolve(s)
+      if (resolved.startsWith(absScope)) return
+    }
+    throw new Error(`Permission denied: path "${targetPath}" is out of scope`)
+  }
+
+  /**
+   * Get the absolute path of file in the file system.
+   * 
+   * @param filePath Relative file path
+   * @returns Absolute file path
+   */
+  fullPath(filePath: string): string {
+    return path.join(this.basePath, filePath)
+  }
+
+  /**
+   * Get web url for a given file path in the file system.
+   * 
+   * @param filePath Relative file path
+   * @returns Web URL of file
+   */
+  fullUrl(filePath: string): string {
+    const url = new URL(filePath, this.baseUrl)
+    return url.href
+  }
+
+  /**
+   * Read file contents
+   */
+  async readFile(filePath: string, encoding: BufferEncoding = 'utf-8') {
+    this.ensureAllowed('read')
+    const fullPath = this.fullPath(filePath)
+    this.ensureInScope(fullPath)
+    return await fsPromises.readFile(fullPath)
+  }
+
+  /**
+   * Write file contents
+   */
+  async writeFile(filePath: string, content: string | Buffer): Promise<string> {
+    this.ensureAllowed('write')
+    const fullPath = this.fullPath(filePath)
+    this.ensureInScope(fullPath)
+    await fsPromises.mkdir(path.dirname(fullPath), { recursive: true })
+    await fsPromises.writeFile(fullPath, content)
+    const url = new URL(filePath, this.baseUrl)
+    return url.href
+  }
+
+  /**
+   * Delete a file
+   */
+  async deleteFile(filePath: string): Promise<void> {
+    this.ensureAllowed('delete')
+    this.ensureInScope(filePath)
+    await fsPromises.unlink(filePath)
+  }
+
+  /**
+   * List directory contents
+   */
+  async listDir(dirPath: string): Promise<string[]> {
+    this.ensureAllowed('list')
+    this.ensureInScope(dirPath)
+    return fsPromises.readdir(dirPath)
+  }
+
+  /**
+   * Utility: check if a file or directory exists
+   */
+  async exists(targetPath: string): Promise<boolean> {
+    try {
+      await fsPromises.access(targetPath)
+      return true
+    } catch {
+      return false
+    }
+  }
+}

package/src/lib/core/i18n.ts

@@ -0,0 +1,31 @@
+import { createInstance, i18n as I18nInstance } from 'i18next'
+import FsBackend from 'i18next-fs-backend'
+import path from 'path'
+import fs from 'fs'
+
+export async function createI18nInstance(pluginDir: string, language?: string): Promise<I18nInstance> {
+  const instance = createInstance()
+  const i18nDir = path.join(pluginDir, 'i18n')
+
+  // detect available languages dynamically
+  const lngs = fs
+    .readdirSync(i18nDir)
+    .filter((f) => f.endsWith('.json'))
+    .map((f) => f.replace('.json', ''))
+
+  await instance
+    .use(FsBackend)
+    .init({
+      lng: language,
+      fallbackLng: 'en',
+      preload: lngs,
+      ns: ['default'],
+      defaultNS: 'default',
+      backend: {
+        loadPath: path.join(i18nDir, '{{lng}}.json'),
+      },
+      interpolation: { escapeValue: false },
+    })
+
+  return instance
+}

package/src/lib/core/index.ts

@@ -0,0 +1,4 @@
+export * from './file-system'
+export * from './permissions'
+export * from './schema'
+export * from './i18n'

package/src/lib/core/permissions.ts

@@ -0,0 +1,97 @@
+/**
+ * ===============================
+ * Unified Permissions Definition
+ * ===============================
+ * Used by Agent / Plugin developers to declare required capabilities.
+ * Core system will check and inject allowed resources accordingly.
+ */
+
+/**
+ * Base Permission type
+ */
+export interface BasePermission {
+  type: string; // Discriminator
+  description?: string; // Optional description for UI
+}
+
+/**
+ * 1. LLM Permission
+ * Example: { type: 'llm', provider: 'openai', capability: 'vision' }
+ */
+export interface LLMPermission extends BasePermission {
+  type: 'llm';
+  provider?: string; // e.g. "openai", "anthropic", "azure", "ollama"
+  capability: 'text' | 'chat' | 'vision' | 'embedding';
+  scope?: string[]; // Allowed model names, e.g. ["gpt-4", "gpt-4-vision-preview"]
+  maxTokens?: number; // Maximum output tokens allowed
+  rateLimit?: { rps: number }; // Rate limit per second
+}
+
+/**
+ * 2. Vector Store Permission
+ * Example: { type: 'vectorstore', provider: 'pinecone', operations: ['insert', 'query'] }
+ */
+export interface VectorStorePermission extends BasePermission {
+  type: 'vectorstore';
+  provider: string; // "pinecone" | "milvus" | "chromadb" | ...
+  operations: Array<'insert' | 'query' | 'delete'>;
+  scope?: string[]; // Restrict to index / collection
+}
+
+/**
+ * 3. Knowledge Base Permission
+ * Example: { type: 'knowledge', operations: ['read', 'write'], scope: ['kb_123'] }
+ */
+export interface KnowledgePermission extends BasePermission {
+  type: 'knowledge';
+  operations: Array<'read' | 'write' | 'update' | 'delete'>;
+  scope?: string[]; // Restrict to certain KB IDs
+}
+
+/**
+ * 4. File System Permission
+ * Example: { type: 'filesystem', operations: ['read', 'write'], scope: ['/documents', '/images'] }
+ */
+export interface FileSystemPermission extends BasePermission {
+  type: 'filesystem';
+  operations: Array<'read' | 'write' | 'delete' | 'list'>;
+  scope?: string[]; // Restrict to certain directories or file types
+}
+
+/**
+ * 5. Integration Permission
+ * Example: { type: 'integration', service: 'feishu', operations: ['read', 'write'] }
+ */
+export interface IntegrationPermission extends BasePermission {
+  type: 'integration';
+  service: string; // e.g. 'slack', 'feishu', 'jira', 'sap', etc.
+  operations?: Array<'read' | 'write' | 'update' | 'delete'>;
+  scope?: string[];
+}
+
+// /**
+//  * 4. Document Permission
+//  * Example: { type: 'document', formats: ['pdf'], operations: ['load', 'transform'] }
+//  */
+// export interface DocumentPermission extends BasePermission {
+//   type: 'document';
+//   formats: string[]; // ['pdf', 'pptx', 'docx', 'image', 'html', 'md']
+//   operations: Array<'load' | 'transform' | 'ocr' | 'imageUnderstanding'>;
+// }
+
+/**
+ * Union type for all permissions
+ */
+export type Permission =
+  | LLMPermission
+  | VectorStorePermission
+  | KnowledgePermission
+  | FileSystemPermission
+  | IntegrationPermission
+//   | DocumentPermission
+//   | ExternalPermission;
+
+/**
+ * Permissions array type
+ */
+export type Permissions = Permission[];

package/src/lib/core/schema.ts

@@ -0,0 +1,23 @@
+/**
+ * Common UI Schema field definitions
+ */
+export interface ISchemaUIBase {
+  component: string;       // UI component type
+  label?: string;          // Field label
+  description?: string;    // Field description
+  placeholder?: string;    // Input placeholder
+  order?: number;          // UI display order
+  required?: boolean;      // Whether the field is required
+  visibleWhen?: Record<string, any>; // Conditional rendering
+  enabledWhen?: Record<string, any>; // Conditional enabling
+}
+
+/**
+ * Secret field extensions
+ */
+export interface ISchemaSecretField extends ISchemaUIBase {
+  component: 'secretInput';   // Fixed component type
+  revealable?: boolean;       // Whether plaintext display is allowed (👁 button)
+  maskSymbol?: string;        // Mask symbol (default *)
+  persist?: boolean;          // Whether to persist, false means only used at runtime
+}

package/src/lib/integration/index.ts

@@ -0,0 +1,3 @@
+export * from './strategy.decorator'
+export * from './strategy.interface'
+export * from './strategy.registry'

package/src/lib/integration/strategy.decorator.ts

@@ -0,0 +1,6 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const INTEGRATION_STRATEGY = 'INTEGRATION_STRATEGY';
+
+export const IntegrationStrategyKey = (provider: string) =>
+  SetMetadata(INTEGRATION_STRATEGY, provider);

package/src/lib/integration/strategy.interface.ts

@@ -0,0 +1,11 @@
+import { IIntegration, TIntegrationProvider } from '@metad/contracts'
+
+export type TIntegrationStrategyParams = {
+  query: string
+}
+
+export interface IntegrationStrategy<T = unknown> {
+  meta: TIntegrationProvider
+  execute(integration: IIntegration<T>, payload: TIntegrationStrategyParams): Promise<any>
+  validateConfig?(config: T): Promise<void>
+}

package/src/lib/integration/strategy.registry.ts

@@ -0,0 +1,12 @@
+import { Injectable } from '@nestjs/common'
+import { DiscoveryService, Reflector } from '@nestjs/core'
+import { BaseStrategyRegistry } from '../strategy'
+import { INTEGRATION_STRATEGY } from './strategy.decorator'
+import { IntegrationStrategy } from './strategy.interface'
+
+@Injectable()
+export class IntegrationStrategyRegistry extends BaseStrategyRegistry<IntegrationStrategy> {
+  constructor(discoveryService: DiscoveryService, reflector: Reflector) {
+    super(INTEGRATION_STRATEGY, discoveryService, reflector)
+  }
+}

package/src/lib/logger.ts

@@ -0,0 +1,18 @@
+import { Logger } from '@nestjs/common';
+import type { PluginLogger } from './types';
+
+export function createPluginLogger(scope: string, baseMeta: Record<string, any> = {}): PluginLogger {
+  const nestLogger = new Logger(scope);
+  const wrap = (level: keyof Logger, msg: string, meta?: any) => {
+    const payload = meta ? { ...baseMeta, ...meta } : baseMeta;
+    // 保持与 Nest Logger 接口对齐
+    (nestLogger as any)[level]?.(msg + (Object.keys(payload).length ? ` ${JSON.stringify(payload)}` : ''));
+  };
+  return {
+    child(meta) { return createPluginLogger(scope, { ...baseMeta, ...meta }); },
+    debug: (msg, meta) => wrap('debug' as any, msg, meta),
+    log:   (msg, meta) => wrap('log', msg, meta),
+    warn:  (msg, meta) => wrap('warn', msg, meta),
+    error: (msg, meta) => wrap('error', msg, meta),
+  };
+}

package/src/lib/plugin-metadata.ts

@@ -0,0 +1,20 @@
+/**
+ * Metadata keys used in plugins for defining various aspects like entities, subscribers, and configurations.
+ */
+export const PLUGIN_METADATA = {
+	/**
+	 * Key representing the entities registered within the plugin.
+	 */
+	ENTITIES: 'entities',
+
+	/**
+	 * Key representing event subscribers within the plugin.
+	 */
+	SUBSCRIBERS: 'subscribers',
+
+} as const;
+
+/**
+ * Type definition for valid plugin metadata keys.
+ */
+export type PluginMetadataKey = (typeof PLUGIN_METADATA)[keyof typeof PLUGIN_METADATA];

package/src/lib/{plugin.hook.d.ts → plugin.hook.ts}

@@ -8,6 +8,7 @@ export interface IOnPluginBootstrap {
      */
     onPluginBootstrap(): void | Promise<void>;
 }
+
 /**
  * Interface for plugins with a destroy lifecycle method.
  */
@@ -18,6 +19,7 @@ export interface IOnPluginDestroy {
      */
     onPluginDestroy(): void | Promise<void>;
 }
+
 /**
  * Interface for plugins supporting basic seed operations.
  */
@@ -28,6 +30,7 @@ export interface IOnPluginWithBasicSeed {
      */
     onPluginBasicSeed(): void | Promise<void>;
 }
+
 /**
  * Interface for plugins supporting default seed operations.
  */
@@ -38,6 +41,7 @@ export interface IOnPluginWithDefaultSeed {
      */
     onPluginDefaultSeed(): void | Promise<void>;
 }
+
 /**
  * Interface for plugins supporting random seed operations.
  */

package/src/lib/plugin.interface.ts

@@ -0,0 +1,67 @@
+import { ModuleMetadata, Type } from '@nestjs/common';
+
+/**
+ * Metadata definition for a plugin in NestJS.
+ */
+export interface PluginMetadata extends ModuleMetadata {
+	/**
+	 * List of entities injected by the plugin.
+	 */
+	entities?: Array<Type<any>> | (() => Array<Type<any>>);
+
+	/**
+	 * List of subscribers injected by the plugin.
+	 */
+	subscribers?: Array<Type<any>> | (() => Array<Type<any>>);
+}
+
+/**
+ * Interface for plugins with a bootstrap lifecycle method.
+ */
+export interface IOnPluginBootstrap {
+	/**
+	 * Called when the plugin is being initialized.
+	 * @returns A void or a Promise representing the completion of the operation.
+	 */
+	onPluginBootstrap(): void | Promise<void>;
+}
+
+/**
+ * Interface for plugins with a destroy lifecycle method.
+ */
+export interface IOnPluginDestroy {
+	/**
+	 * Called when the plugin is being destroyed.
+	 * @returns A void or a Promise representing the completion of the operation.
+	 */
+	onPluginDestroy(): void | Promise<void>;
+}
+
+/**
+ * Interface for plugins supporting various seed operations.
+ */
+export interface IOnPluginSeedable {
+	/**
+	 * Invoked when seeding basic plugin data.
+	 * @returns A void or a Promise representing the completion of the operation.
+	 */
+	onPluginBasicSeed?(): void | Promise<void>;
+
+	/**
+	 * Invoked when seeding default plugin data.
+	 * @returns A void or a Promise representing the completion of the operation.
+	 */
+	onPluginDefaultSeed?(): void | Promise<void>;
+
+	/**
+	 * Invoked when seeding random plugin data.
+	 * @returns A void or a Promise representing the completion of the operation.
+	 */
+	onPluginRandomSeed?(): void | Promise<void>;
+}
+
+/**
+ * Represents the combined lifecycle methods for a plugin.
+ * This type combines interfaces for initializing and destroying a plugin.
+ */
+export type PluginLifecycleMethods = IOnPluginBootstrap & IOnPluginDestroy & IOnPluginSeedable;

package/src/lib/plugin.ts

@@ -0,0 +1,32 @@
+import { Module } from '@nestjs/common';
+import { MODULE_METADATA } from '@nestjs/common/constants';
+import { pick } from 'lodash-es';
+import { PluginMetadata } from './plugin.interface';
+import { PLUGIN_METADATA } from './plugin-metadata';
+
+/**
+ * Decorator function for extending NestJS features with additional metadata.
+ *
+ * @param pluginMetadata Metadata to be applied to the target class.
+ * @returns Class decorator function.
+ */
+export function XpertServerPlugin(pluginMetadata: PluginMetadata): ClassDecorator {
+	return (targetClass) => {
+		// Iterate over properties in PLUGIN_METADATA
+		for (const metadataProperty of Object.values(PLUGIN_METADATA)) {
+			const property = metadataProperty as keyof PluginMetadata;
+
+			// Check if the property exists in pluginMetadata and is not undefined
+			if (property in pluginMetadata && pluginMetadata[property] !== undefined) {
+				// Set metadata on the target class using Reflect
+				Reflect.defineMetadata(property, pluginMetadata[property] || [], targetClass);
+			}
+		}
+
+		// Pick relevant metadata from pluginMetadata based on MODULE_METADATA values
+		const metadata = pick(pluginMetadata, Object.values(MODULE_METADATA) as string[]);
+
+		// Apply the Module decorator with the picked metadata
+		Module(metadata)(targetClass);
+	};
+}

package/src/lib/rag/image/index.ts

@@ -0,0 +1,3 @@
+export * from './strategy.decorator'
+export * from './strategy.interface'
+export * from './strategy.registry'

package/src/lib/rag/image/strategy.decorator.ts

@@ -0,0 +1,9 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const IMAGE_UNDERSTANDING_STRATEGY = 'IMAGE_UNDERSTANDING_STRATEGY';
+
+/**
+ * Decorator to mark a provider as an Image Understanding Strategy
+ */
+export const ImageUnderstandingStrategy = (provider: string) =>
+  SetMetadata(IMAGE_UNDERSTANDING_STRATEGY, provider);

package/src/lib/rag/image/strategy.interface.ts

@@ -0,0 +1,37 @@
+import { DocumentInterface } from '@langchain/core/documents'
+import { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { IDocumentUnderstandingProvider, IKnowledgeDocument } from '@metad/contracts'
+import { Permissions, XpFileSystem } from '../../core/index'
+import { ChunkMetadata } from '../types'
+
+export type TImageUnderstandingConfig = {
+  stage: 'test' | 'prod'
+  visionModel: BaseChatModel
+  permissions?: {
+    fileSystem?: XpFileSystem
+  }
+}
+
+export type TImageUnderstandingResult = {
+  chunks: DocumentInterface<Partial<ChunkMetadata>>[]
+  // pages?: Document<Partial<ChunkMetadata>>[]
+  metadata: any // Additional metadata (e.g. model name, processing time)
+}
+
+export interface IImageUnderstandingStrategy<TConfig extends TImageUnderstandingConfig = TImageUnderstandingConfig> {
+  readonly permissions: Permissions
+  /**
+   * Metadata about this strategy
+   */
+  readonly meta: IDocumentUnderstandingProvider
+
+  /**
+   * Validate the configuration
+   */
+  validateConfig(config: TConfig): Promise<void>
+
+  /**
+   * Understand image files (e.g., OCR, VLM, Chart Parsing)
+   */
+  understandImages(doc: IKnowledgeDocument<ChunkMetadata>, config: TConfig): Promise<TImageUnderstandingResult>
+}