@xpert-ai/plugin-sdk 0.0.1

package/src/lib/rag/source/index.ts

@@ -0,0 +1,3 @@
+export * from './strategy.decorator'
+export * from './strategy.interface'
+export * from './strategy.registry'

package/src/lib/rag/source/strategy.decorator.ts

@@ -0,0 +1,9 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const DOCUMENT_SOURCE_STRATEGY = 'DOCUMENT_SOURCE_STRATEGY';
+
+/**
+ * Decorator to mark a provider as a Document Source Strategy
+ */
+export const DocumentSourceStrategy = (provider: string) =>
+  SetMetadata(DOCUMENT_SOURCE_STRATEGY, provider);

package/src/lib/rag/source/strategy.interface.ts

@@ -0,0 +1,28 @@
+import { IDocumentSourceProvider, IIntegration } from '@metad/contracts';
+import { Document } from 'langchain/document';
+import { Permissions } from '../../core';
+
+export interface IDocumentSourceStrategy<TConfig = any> {
+  readonly permissions: Permissions
+  /**
+   * Metadata about this document source
+   */
+  readonly meta: IDocumentSourceProvider
+
+  /**
+   * Validate the configuration
+   */
+  validateConfig(config: TConfig): Promise<void>;
+
+  /**
+   * Test the connection to the source
+   * 
+   * @param config 
+   */
+  test(config: TConfig): Promise<any>;
+
+  /**
+   * Load documents from the source
+   */
+  loadDocuments(config: TConfig, context?: {integration?: IIntegration}): Promise<Document[]>;
+}

package/src/lib/rag/source/strategy.registry.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@nestjs/common';
+import { DiscoveryService, Reflector } from '@nestjs/core';
+import { BaseStrategyRegistry } from '../../strategy';
+import { DOCUMENT_SOURCE_STRATEGY } from './strategy.decorator';
+import { IDocumentSourceStrategy } from './strategy.interface';
+
+@Injectable()
+export class DocumentSourceRegistry<TConfig = any>
+  extends BaseStrategyRegistry<IDocumentSourceStrategy<TConfig>> 
+{
+  constructor(
+    discoveryService: DiscoveryService,
+    reflector: Reflector
+  ) {
+    super(DOCUMENT_SOURCE_STRATEGY, discoveryService, reflector);
+  }
+}

package/src/lib/rag/textsplitter/index.ts

@@ -0,0 +1,3 @@
+export * from './strategy.decorator'
+export * from './strategy.interface'
+export * from './strategy.registry'

package/src/lib/rag/textsplitter/strategy.decorator.ts

@@ -0,0 +1,6 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const TEXT_SPLITTER_STRATEGY = 'TEXT_SPLITTER_STRATEGY';
+
+export const TextSplitterStrategy = (provider: string) =>
+  SetMetadata(TEXT_SPLITTER_STRATEGY, provider);

package/src/lib/rag/textsplitter/strategy.interface.ts

@@ -0,0 +1,28 @@
+import { IDocumentChunkerProvider, KnowledgeStructureEnum } from '@metad/contracts'
+import { Document } from 'langchain/document'
+import { ChunkMetadata } from '../types'
+
+/**
+ * Split text content into chunks for embedding and retrieval
+ */
+export interface ITextSplitterStrategy<TConfig = any> {
+  /**
+   * Metadata about this splitter
+   */
+  readonly meta: IDocumentChunkerProvider
+
+  readonly structure: KnowledgeStructureEnum
+
+  /**
+   * Validate the configuration
+   */
+  validateConfig(config: TConfig): Promise<void>
+
+  /**
+   * Split a text into chunks and pages (if applicable)
+   */
+  splitDocuments(
+    documents: Document[],
+    options?: TConfig
+  ): Promise<{ chunks: Document<ChunkMetadata>[]; pages?: Document<ChunkMetadata>[] }>
+}

package/src/lib/rag/textsplitter/strategy.registry.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@nestjs/common';
+import { DiscoveryService, Reflector } from '@nestjs/core';
+import { BaseStrategyRegistry } from '../../strategy';
+import { TEXT_SPLITTER_STRATEGY } from './strategy.decorator';
+import { ITextSplitterStrategy } from './strategy.interface';
+
+@Injectable()
+export class TextSplitterRegistry<TConfig = any>
+  extends BaseStrategyRegistry<ITextSplitterStrategy<TConfig>> 
+{
+  constructor(
+    discoveryService: DiscoveryService,
+    reflector: Reflector
+  ) {
+    super(TEXT_SPLITTER_STRATEGY, discoveryService, reflector);
+  }
+}

package/src/lib/rag/transformer/index.ts

@@ -0,0 +1,3 @@
+export * from './strategy.decorator'
+export * from './strategy.interface'
+export * from './strategy.registry'

package/src/lib/rag/transformer/strategy.decorator.ts

@@ -0,0 +1,9 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const DOCUMENT_TRANSFORMER_STRATEGY = 'DOCUMENT_TRANSFORMER_STRATEGY';
+
+/**
+ * Decorator to mark a provider as a Document Transformer Strategy
+ */
+export const DocumentTransformerStrategy = (provider: string) =>
+  SetMetadata(DOCUMENT_TRANSFORMER_STRATEGY, provider);

package/src/lib/rag/transformer/strategy.interface.ts

@@ -0,0 +1,48 @@
+import { DocumentInterface } from '@langchain/core/documents'
+import { _TFile, IDocumentProcessorProvider, IIntegration } from '@metad/contracts'
+import { Permissions, XpFileSystem } from '../../core/index'
+import { ChunkMetadata, TDocumentAsset } from '../types'
+
+export type TDocumentTransformerConfig = {
+  stage: 'test' | 'prod'
+  tempDir?: string
+  permissions?: {
+    fileSystem?: XpFileSystem
+    integration?: IIntegration
+  }
+}
+
+export type TDocumentTransformerFile = _TFile & {
+  id?: string
+  filename: string
+  extension: string | undefined
+}
+
+export type TDocumentTransformerInput = TDocumentTransformerFile[] | string | string[]
+
+export type TDocumentTransformerResult = {
+  id?: string
+  chunks: DocumentInterface<ChunkMetadata>[]
+  metadata: {
+    assets?: TDocumentAsset[]
+  }
+}
+
+export interface IDocumentTransformerStrategy<TConfig extends TDocumentTransformerConfig = TDocumentTransformerConfig> {
+  /**
+   * Metadata about this transformer
+   */
+  readonly meta: IDocumentProcessorProvider
+
+  readonly permissions: Permissions
+
+  /**
+   * Validate the configuration
+   */
+  validateConfig(config: TConfig): Promise<void>
+
+  /**
+   * Transform documents (e.g., extract, OCR, normalize, enrich metadata)
+   */
+  transformDocuments(files: TDocumentTransformerInput, config: TConfig): Promise<TDocumentTransformerResult[]>
+}

package/src/lib/rag/transformer/strategy.registry.ts

@@ -0,0 +1,14 @@
+import { Injectable } from '@nestjs/common'
+import { DiscoveryService, Reflector } from '@nestjs/core'
+import { BaseStrategyRegistry } from '../../strategy'
+import { DOCUMENT_TRANSFORMER_STRATEGY } from './strategy.decorator'
+import { IDocumentTransformerStrategy, TDocumentTransformerConfig } from './strategy.interface'
+
+@Injectable()
+export class DocumentTransformerRegistry<
+  TConfig extends TDocumentTransformerConfig = TDocumentTransformerConfig
+> extends BaseStrategyRegistry<IDocumentTransformerStrategy<TConfig>> {
+  constructor(discoveryService: DiscoveryService, reflector: Reflector) {
+    super(DOCUMENT_TRANSFORMER_STRATEGY, discoveryService, reflector)
+  }
+}

package/src/lib/rag/types.ts

@@ -0,0 +1,86 @@
+import { DocumentInterface } from '@langchain/core/documents'
+import fs from 'fs'
+import http from 'http'
+import https from 'https'
+
+export type TDocumentAsset = {
+  type: 'image' | 'video' | 'audio' | 'file'
+  url: string // remote url
+  filePath: string // local path
+}
+
+export interface ChunkMetadata {
+  documentId?: string // 原始文档 ID
+  pageId?: string // 如果有分页，引用页 ID
+  chunkId: string // 当前块的唯一 ID
+  parentId?: string // 如果是子块，引用父块 ID
+  chunkIndex?: number // 在文档内或在父块内的序号
+  startOffset?: number // 原始文本起始位置
+  endOffset?: number // 原始文本结束位置
+  type?: 'parent' | 'child' // 分块类型
+  children?: DocumentInterface<ChunkMetadata>[]
+  assets?: TDocumentAsset[]
+  [key: string]: any // 允许插件扩展
+}
+
+/**
+ * Merge parent chunks with their child chunks based on metadata (parentId and chunkId)
+ * @param chunks
+ * @param children
+ * @returns
+ */
+export function mergeParentChildChunks(
+  chunks: DocumentInterface<ChunkMetadata>[], // Parent chunks
+  children: DocumentInterface<ChunkMetadata>[] // Child chunks
+): DocumentInterface<ChunkMetadata>[] {
+  const chunkMap = new Map<string, DocumentInterface<ChunkMetadata>>()
+  for (const chunk of chunks) {
+    chunkMap.set(chunk.metadata.chunkId, chunk)
+  }
+  for (const child of children) {
+    if (!child.metadata.parentId) {
+      if (chunkMap.has(child.metadata.chunkId)) {
+        console.warn(`Duplicate chunkId found: ${child.metadata.chunkId}, skipping...`)
+        continue
+      }
+      chunkMap.set(child.metadata.chunkId, child)
+      continue
+    }
+    const parent = chunkMap.get(child.metadata.parentId)
+    if (parent) {
+      if (!parent.metadata.children) {
+        parent.metadata.children = []
+      }
+      parent.metadata.children.push(child)
+    }
+  }
+  return Array.from(chunkMap.values())
+}
+
+export function isRemoteFile(path: string): boolean {
+  return path.startsWith('http://') || path.startsWith('https://') || path.startsWith('ftp://')
+}
+
+export async function downloadRemoteFile(url: string, dest: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const file = fs.createWriteStream(dest)
+    const client = url.startsWith('https') ? https : http
+    const request = client.get(url, (response) => {
+      if (response.statusCode !== 200) {
+        reject(new Error(`Failed to get '${url}' (${response.statusCode})`))
+        return
+      }
+      response.pipe(file)
+    })
+    file.on('finish', () => {
+      file.close()
+      resolve(dest)
+    })
+    request.on('error', (err) => {
+      fs.unlink(dest, () => reject(err))
+    })
+    file.on('error', (err) => {
+      fs.unlink(dest, () => reject(err))
+    })
+  })
+}

package/src/lib/strategy.ts

@@ -0,0 +1,37 @@
+import { OnModuleInit } from "@nestjs/common"
+import { DiscoveryService, Reflector } from "@nestjs/core"
+
+export class BaseStrategyRegistry<S> implements OnModuleInit {
+    protected strategies = new Map<string, S>()
+
+    constructor(
+        protected readonly strategyKey: string,
+        protected discoveryService: DiscoveryService,
+        protected reflector: Reflector
+    ) {}
+
+    onModuleInit() {
+        const providers = this.discoveryService.getProviders()
+        for (const wrapper of providers) {
+            const { instance } = wrapper
+            if (!instance) continue
+
+            const type = this.reflector.get<string>(this.strategyKey, instance.constructor)
+            if (type) {
+                this.strategies.set(type, instance as S)
+            }
+        }
+    }
+
+    get(type: string): S {
+        const strategy = this.strategies.get(type)
+        if (!strategy) {
+            throw new Error(`No strategy found for type ${type}`)
+        }
+        return strategy
+    }
+
+    list(): S[] {
+        return Array.from(this.strategies.values())
+    }
+}

package/src/lib/toolset/builtin.ts

@@ -0,0 +1,111 @@
+import { StructuredToolInterface } from '@langchain/core/tools'
+import { BaseStore } from '@langchain/langgraph'
+import {
+	I18nObject,
+	IBuiltinTool,
+	IXpertToolset,
+	ToolProviderCredentials,
+	TToolCredentials,
+	TToolsetParams,
+	XpertToolsetCategoryEnum
+} from '@metad/contracts'
+import { Logger } from '@nestjs/common'
+import { CommandBus, QueryBus } from '@nestjs/cqrs'
+import { BaseToolset } from './toolset'
+
+/**
+ * The context params of creating toolset
+ */
+export type TBuiltinToolsetParams = TToolsetParams & {
+	commandBus: CommandBus
+	queryBus: QueryBus
+	store?: BaseStore
+}
+
+export interface IBuiltinToolset {
+	validateCredentials(credentials: TToolCredentials): Promise<void>
+}
+
+export abstract class BuiltinToolset<T extends StructuredToolInterface = StructuredToolInterface, C = TToolCredentials>
+	extends BaseToolset<T>
+	implements IBuiltinToolset
+{
+	static provider = ''
+	protected logger = new Logger(this.constructor.name)
+
+	providerType: XpertToolsetCategoryEnum.BUILTIN
+
+	credentialsSchema?: { [key: string]: ToolProviderCredentials }
+
+	get tenantId() {
+		return this.params?.tenantId
+	}
+	get organizationId() {
+		return this.params?.organizationId
+	}
+	get commandBus() {
+		return this.params?.commandBus
+	}
+	get queryBus() {
+		return this.params?.queryBus
+	}
+
+	get xpertId() {
+		return this.params?.xpertId
+	}
+
+	constructor(
+		public providerName: string,
+		protected toolset?: IXpertToolset,
+		protected override params?: TBuiltinToolsetParams
+	) {
+		super(params)
+	}
+
+	async validateCredentials(credentials: C): Promise<void> {
+		await this._validateCredentials(credentials)
+	}
+
+	async _validateCredentials(credentials: C) {
+		throw new Error('Method not implemented.')
+	}
+
+	getId() {
+		return this.toolset?.id
+	}
+
+	getCredentials() {
+		return this.toolset?.credentials as C
+	}
+
+	getToolTitle(name: string): string | I18nObject {
+		const tool = this.toolset?.tools?.find((tool) => tool.name === name)
+		const identity = (<IBuiltinTool>tool?.schema)?.identity
+		if (identity) {
+			return identity.label
+		}
+		return null
+	}
+
+	/**
+	 * Get credentials schema
+	 *
+	 * @returns Credentials schema
+	 */
+	getCredentialsSchema(): { [key: string]: ToolProviderCredentials } {
+		return { ...this.credentialsSchema }
+	}
+
+	/**
+	 * Get toolset entity
+	 *
+	 * @returns XpertToolset
+	 */
+	getToolset() {
+		return this.toolset
+	}
+
+	getName() {
+		return this.getToolset()?.name
+	}
+}

package/src/lib/toolset/index.ts

@@ -0,0 +1,5 @@
+export * from './strategy.decorator'
+export * from './strategy.interface'
+export * from './strategy.registry'
+export * from './builtin'
+export * from './toolset'

package/src/lib/toolset/strategy.decorator.ts

@@ -0,0 +1,9 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const TOOLSET_STRATEGY = 'TOOLSET_STRATEGY';
+
+/**
+ * Decorator to mark a provider as a Toolset Strategy
+ */
+export const ToolsetStrategy = (provider: string) =>
+  SetMetadata(TOOLSET_STRATEGY, provider);

package/src/lib/toolset/strategy.interface.ts

@@ -0,0 +1,32 @@
+import { DynamicStructuredTool } from '@langchain/core/tools'
+import { I18nObject } from '@metad/contracts'
+import { ZodSchema } from 'zod'
+import { BuiltinToolset } from './builtin'
+
+export interface IToolsetStrategy<TConfig = any> {
+  /**
+   * Metadata about this toolset
+   */
+  readonly meta: {
+    author: string // author name
+    tags: string[] // tags for categorization
+    name: string // unique key
+    label: I18nObject // i18n label
+    description?: I18nObject // optional i18n description
+    configSchema: any // JSON schema for config validation
+    icon?: {
+      svg?: string
+      png?: string // base64 encoded png image
+      color?: string
+    }
+  }
+
+  /**
+   * Validate the configuration
+   */
+  validateConfig(config: TConfig): Promise<void>
+
+  create(config: TConfig): Promise<BuiltinToolset>
+
+  createTools(): DynamicStructuredTool<ZodSchema>[]
+}

package/src/lib/toolset/strategy.registry.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@nestjs/common';
+import { DiscoveryService, Reflector } from '@nestjs/core';
+import { TOOLSET_STRATEGY } from './strategy.decorator';
+import { IToolsetStrategy } from './strategy.interface';
+import { BaseStrategyRegistry } from '../strategy';
+
+@Injectable()
+export class ToolsetRegistry<TConfig = any>
+  extends BaseStrategyRegistry<IToolsetStrategy<TConfig>> 
+{
+  constructor(
+    discoveryService: DiscoveryService,
+    reflector: Reflector
+  ) {
+    super(TOOLSET_STRATEGY, discoveryService, reflector);
+  }
+}

package/src/lib/toolset/toolset.ts

@@ -0,0 +1,76 @@
+import { BaseToolkit, StructuredTool, StructuredToolInterface } from "@langchain/core/tools"
+import { I18nObject, TStateVariable, TToolsetParams } from "@metad/contracts"
+import { z } from 'zod'
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type ZodObjectAny = z.ZodObject<any, any, any, any>;
+
+/**
+ * Base ability for all toolsets
+ */
+export abstract class BaseToolset<T extends StructuredToolInterface = StructuredToolInterface> extends BaseToolkit {
+	abstract providerName: string
+	// For Langchain
+	tools: T[]
+	// For Langgraph
+	stateVariables: TStateVariable[]
+
+	constructor(protected params?: TToolsetParams) {
+		super()
+	}
+
+	/**
+	 * Async init tools
+	 *
+	 * @returns
+	 */
+	async initTools() {
+		return this.tools
+	}
+
+	/**
+	 * Get ID of the toolset
+	 */
+	abstract getId(): string
+	/**
+	 * Get name of the toolset
+	 */
+	abstract getName(): string
+
+	/**
+	 * Get one tool
+	 *
+	 * @param toolName
+	 * @returns
+	 */
+	getTool(toolName: string) {
+		return this.getTools()?.find((tool) => tool.name === toolName)
+	}
+
+	/**
+	 * Get state variables config
+	 *
+	 * @returns State variables
+	 */
+	async getVariables(): Promise<TStateVariable[]> {
+		return null
+	}
+
+	/**
+	 * Get title of tool
+	 */
+	abstract getToolTitle(name: string): string | I18nObject
+
+	/**
+     * Close all (connections).
+     */
+    async close(): Promise<void> {
+		//
+	}
+}
+
+export abstract class BaseTool<T extends ZodObjectAny = ZodObjectAny> extends StructuredTool<T> {
+	schema: any = z
+		.object({ input: z.string().optional() })
+		.transform((obj) => obj.input)
+}

package/src/lib/types.ts

@@ -0,0 +1,47 @@
+import { PluginMeta } from '@metad/contracts';
+import type { DynamicModule, INestApplicationContext } from '@nestjs/common';
+import type { ZodSchema } from 'zod';
+
+export interface PluginLifecycle {
+  /** 在模块注册完成但应用启动前调用 */
+  onInit?(ctx: PluginContext): Promise<void> | void;
+  /** 在应用启动后调用（可开始对外服务） */
+  onStart?(ctx: PluginContext): Promise<void> | void;
+  /** 优雅停机时调用 */
+  onStop?(ctx: PluginContext): Promise<void> | void;
+}
+
+export interface PluginHealth {
+  /** 返回健康状态与可选的依赖检查详情 */
+  checkHealth?(ctx: PluginContext): Promise<{ status: 'up' | 'down'; details?: any }> | { status: 'up' | 'down'; details?: any };
+}
+
+export interface PluginConfigSpec<T extends object = any> {
+  /** 插件级配置的 zod 校验 schema（可选） */
+  schema?: ZodSchema<T>;
+  /** 默认配置 */
+  defaults?: Partial<T>;
+}
+
+export interface XpertPlugin<TConfig extends object = any> extends PluginLifecycle, PluginHealth {
+  meta: PluginMeta;
+  config?: PluginConfigSpec<TConfig>;
+  /** 返回要挂载到主应用的 DynamicModule（可设为 global） */
+  register(ctx: PluginContext<TConfig>): DynamicModule;
+}
+
+export interface PluginContext<TConfig extends object = any> {
+  app: INestApplicationContext;      // Nest 运行时上下文（启动后注入）
+  logger: PluginLogger;              // SDK 提供的 Logger 包装
+  config: TConfig;                   // 校验合并后的最终配置
+  /** 访问容器中其他 Provider 的辅助方法 */
+  resolve<TInput = any, TResult = TInput>(token: any): TResult;
+}
+
+export interface PluginLogger {
+  child(meta: Record<string, any>): PluginLogger;
+  debug(msg: string, meta?: any): void;
+  log(msg: string, meta?: any): void;
+  warn(msg: string, meta?: any): void;
+  error(msg: string, meta?: any): void;
+}

package/src/lib/vectorstore/index.ts

@@ -0,0 +1,3 @@
+export * from './strategy.decorator'
+export * from './strategy.interface'
+export * from './strategy.registry'

package/src/lib/vectorstore/strategy.decorator.ts

@@ -0,0 +1,6 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const VECTOR_STORE_STRATEGY = 'VECTOR_STORE_STRATEGY';
+
+export const VectorStoreStrategy = (provider: string) =>
+  SetMetadata(VECTOR_STORE_STRATEGY, provider);

package/src/lib/vectorstore/strategy.interface.ts

@@ -0,0 +1,25 @@
+import { EmbeddingsInterface } from '@langchain/core/embeddings'
+import { VectorStore } from '@langchain/core/vectorstores'
+
+/**
+ * Interface for VectorStore strategy。
+ * 
+ * 
+ */
+export interface IVectorStoreStrategy<TConfig extends {collectionName?: string}> {
+  /**
+   * Metadata about the strategy
+   */
+  readonly name: string;
+  readonly description?: string;
+
+  /**
+   * Validate configuration for VectorStore
+   */
+  validateConfig(config: TConfig): Promise<void>;
+
+  /**
+   * Create a VectorStore with given config
+   */
+  createStore(embeddings: EmbeddingsInterface, config: TConfig): Promise<VectorStore>;
+}