@xpert-ai/plugin-sdk 0.0.1 → 0.0.2

package/src/lib/knowledge/knowledge-strategy.registry.ts

@@ -1,36 +0,0 @@
-import { KnowledgeProviderEnum } from '@metad/contracts'
-import { Injectable, OnModuleInit } from '@nestjs/common'
-import { DiscoveryService, Reflector } from '@nestjs/core'
-import { KNOWLEDGE_STRATEGY } from './knowledge-strategy.decorator'
-import { KnowledgeStrategy } from './knowledge-strategy.interface'
-
-@Injectable()
-export class KnowledgeStrategyRegistry implements OnModuleInit {
-	private strategies = new Map<KnowledgeProviderEnum, KnowledgeStrategy>()
-
-	constructor(
-		private discoveryService: DiscoveryService,
-		private reflector: Reflector
-	) {}
-
-	onModuleInit() {
-		const providers = this.discoveryService.getProviders()
-		for (const wrapper of providers) {
-			const { instance } = wrapper
-			if (!instance) continue
-
-			const type = this.reflector.get<KnowledgeProviderEnum>(KNOWLEDGE_STRATEGY, instance.constructor)
-			if (type) {
-				this.strategies.set(type, instance as KnowledgeStrategy)
-			}
-		}
-	}
-
-	get(type: KnowledgeProviderEnum): KnowledgeStrategy {
-		const strategy = this.strategies.get(type)
-		if (!strategy) {
-			throw new Error(`No strategy found for type ${type}`)
-		}
-		return strategy
-	}
-}

package/src/lib/logger.ts

@@ -1,18 +0,0 @@
-import { Logger } from '@nestjs/common';
-import type { PluginLogger } from './types';
-
-export function createPluginLogger(scope: string, baseMeta: Record<string, any> = {}): PluginLogger {
-  const nestLogger = new Logger(scope);
-  const wrap = (level: keyof Logger, msg: string, meta?: any) => {
-    const payload = meta ? { ...baseMeta, ...meta } : baseMeta;
-    // 保持与 Nest Logger 接口对齐
-    (nestLogger as any)[level]?.(msg + (Object.keys(payload).length ? ` ${JSON.stringify(payload)}` : ''));
-  };
-  return {
-    child(meta) { return createPluginLogger(scope, { ...baseMeta, ...meta }); },
-    debug: (msg, meta) => wrap('debug' as any, msg, meta),
-    log:   (msg, meta) => wrap('log', msg, meta),
-    warn:  (msg, meta) => wrap('warn', msg, meta),
-    error: (msg, meta) => wrap('error', msg, meta),
-  };
-}

package/src/lib/plugin-metadata.ts

@@ -1,20 +0,0 @@
-/**
- * Metadata keys used in plugins for defining various aspects like entities, subscribers, and configurations.
- */
-export const PLUGIN_METADATA = {
-	/**
-	 * Key representing the entities registered within the plugin.
-	 */
-	ENTITIES: 'entities',
-
-	/**
-	 * Key representing event subscribers within the plugin.
-	 */
-	SUBSCRIBERS: 'subscribers',
-
-} as const;
-
-/**
- * Type definition for valid plugin metadata keys.
- */
-export type PluginMetadataKey = (typeof PLUGIN_METADATA)[keyof typeof PLUGIN_METADATA];

package/src/lib/plugin.interface.ts

@@ -1,67 +0,0 @@
-import { ModuleMetadata, Type } from '@nestjs/common';
-
-/**
- * Metadata definition for a plugin in NestJS.
- */
-export interface PluginMetadata extends ModuleMetadata {
-	/**
-	 * List of entities injected by the plugin.
-	 */
-	entities?: Array<Type<any>> | (() => Array<Type<any>>);
-
-	/**
-	 * List of subscribers injected by the plugin.
-	 */
-	subscribers?: Array<Type<any>> | (() => Array<Type<any>>);
-}
-
-/**
- * Interface for plugins with a bootstrap lifecycle method.
- */
-export interface IOnPluginBootstrap {
-	/**
-	 * Called when the plugin is being initialized.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginBootstrap(): void | Promise<void>;
-}
-
-/**
- * Interface for plugins with a destroy lifecycle method.
- */
-export interface IOnPluginDestroy {
-	/**
-	 * Called when the plugin is being destroyed.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginDestroy(): void | Promise<void>;
-}
-
-/**
- * Interface for plugins supporting various seed operations.
- */
-export interface IOnPluginSeedable {
-	/**
-	 * Invoked when seeding basic plugin data.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginBasicSeed?(): void | Promise<void>;
-
-	/**
-	 * Invoked when seeding default plugin data.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginDefaultSeed?(): void | Promise<void>;
-
-	/**
-	 * Invoked when seeding random plugin data.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginRandomSeed?(): void | Promise<void>;
-}
-
-/**
- * Represents the combined lifecycle methods for a plugin.
- * This type combines interfaces for initializing and destroying a plugin.
- */
-export type PluginLifecycleMethods = IOnPluginBootstrap & IOnPluginDestroy & IOnPluginSeedable;

package/src/lib/plugin.ts

@@ -1,32 +0,0 @@
-import { Module } from '@nestjs/common';
-import { MODULE_METADATA } from '@nestjs/common/constants';
-import { pick } from 'underscore';
-import { PluginMetadata } from './plugin.interface';
-import { PLUGIN_METADATA } from './plugin-metadata';
-
-/**
- * Decorator function for extending NestJS features with additional metadata.
- *
- * @param pluginMetadata Metadata to be applied to the target class.
- * @returns Class decorator function.
- */
-export function XpertServerPlugin(pluginMetadata: PluginMetadata): ClassDecorator {
-	return (targetClass) => {
-		// Iterate over properties in PLUGIN_METADATA
-		for (const metadataProperty of Object.values(PLUGIN_METADATA)) {
-			const property = metadataProperty as keyof PluginMetadata;
-
-			// Check if the property exists in pluginMetadata and is not undefined
-			if (property in pluginMetadata && pluginMetadata[property] !== undefined) {
-				// Set metadata on the target class using Reflect
-				Reflect.defineMetadata(property, pluginMetadata[property] || [], targetClass);
-			}
-		}
-
-		// Pick relevant metadata from pluginMetadata based on MODULE_METADATA values
-		const metadata = pick(pluginMetadata, Object.values(MODULE_METADATA) as string[]);
-
-		// Apply the Module decorator with the picked metadata
-		Module(metadata)(targetClass);
-	};
-}

package/src/lib/rag/image/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/image/strategy.decorator.ts

@@ -1,9 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const IMAGE_UNDERSTANDING_STRATEGY = 'IMAGE_UNDERSTANDING_STRATEGY';
-
-/**
- * Decorator to mark a provider as an Image Understanding Strategy
- */
-export const ImageUnderstandingStrategy = (provider: string) =>
-  SetMetadata(IMAGE_UNDERSTANDING_STRATEGY, provider);

package/src/lib/rag/image/strategy.interface.ts

@@ -1,42 +0,0 @@
-import { BaseChatModel } from '@langchain/core/language_models/chat_models'
-import { IDocumentUnderstandingProvider } from '@metad/contracts'
-import { Document } from 'langchain/document'
-import { Permissions, XpFileSystem } from '../../core/index'
-import { ChunkMetadata, TDocumentAsset } from '../types'
-
-export type TImageUnderstandingConfig = {
-  stage: 'test' | 'prod'
-  visionModel: BaseChatModel
-  permissions?: {
-    fileSystem?: XpFileSystem
-  }
-}
-
-export type TImageUnderstandingInput = {
-  chunks: Document<ChunkMetadata>[] // 来自 Loader 的初始文档块
-  files: TDocumentAsset[] // 需要处理的图像文件
-}
-
-export type TImageUnderstandingResult = {
-  chunks: Document<Partial<ChunkMetadata>>[]
-  pages?: Document<Partial<ChunkMetadata>>[]
-  metadata: any // 额外的元数据（例如模型名称、处理耗时）
-}
-
-export interface IImageUnderstandingStrategy<TConfig extends TImageUnderstandingConfig = TImageUnderstandingConfig> {
-  readonly permissions: Permissions
-  /**
-   * Metadata about this strategy
-   */
-  readonly meta: IDocumentUnderstandingProvider
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>
-
-  /**
-   * Understand image files (e.g., OCR, VLM, Chart Parsing)
-   */
-  understandImages(params: TImageUnderstandingInput, config: TConfig): Promise<TImageUnderstandingResult>
-}

package/src/lib/rag/image/strategy.registry.ts

@@ -1,17 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { DiscoveryService, Reflector } from '@nestjs/core';
-import { BaseStrategyRegistry } from '../../strategy';
-import { IMAGE_UNDERSTANDING_STRATEGY } from './strategy.decorator';
-import { IImageUnderstandingStrategy, TImageUnderstandingConfig } from './strategy.interface';
-
-@Injectable()
-export class ImageUnderstandingRegistry<TConfig extends TImageUnderstandingConfig = TImageUnderstandingConfig>
-  extends BaseStrategyRegistry<IImageUnderstandingStrategy<TConfig>> 
-{
-  constructor(
-    discoveryService: DiscoveryService,
-    reflector: Reflector
-  ) {
-    super(IMAGE_UNDERSTANDING_STRATEGY, discoveryService, reflector);
-  }
-}

package/src/lib/rag/index.ts

@@ -1,6 +0,0 @@
-export * from './textsplitter/index'
-export * from './source/index'
-export * from './transformer/index'
-export * from './retriever/index'
-export * from './types'
-export * from './image/index'

package/src/lib/rag/retriever/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/retriever/strategy.decorator.ts

@@ -1,9 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const RETRIEVER_STRATEGY = 'RETRIEVER_STRATEGY';
-
-/**
- * Decorator to mark a provider as a Retriever Strategy
- */
-export const RetrieverStrategy = (provider: string) =>
-  SetMetadata(RETRIEVER_STRATEGY, provider);

package/src/lib/rag/retriever/strategy.interface.ts

@@ -1,32 +0,0 @@
-import { VectorStore } from '@langchain/core/vectorstores'
-import { I18nObject } from '@metad/contracts'
-import { Document } from 'langchain/document'
-
-export type TRetrieverConfig = {
-  vectorStore: VectorStore
-}
-
-export interface IRetrieverStrategy<TConfig extends TRetrieverConfig = TRetrieverConfig> {
-  /**
-   * Metadata about this retriever
-   */
-  readonly meta: {
-    name: string
-    label: I18nObject
-    configSchema: any
-    icon: {
-      svg?: string
-      color?: string
-    }
-  }
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>;
-
-  /**
-   * Retrieve relevant documents for a given query
-   */
-  retrieve(query: string, options?: TConfig): Promise<{ documents: Document[] }>;
-}

package/src/lib/rag/retriever/strategy.registry.ts

@@ -1,12 +0,0 @@
-import { Injectable } from '@nestjs/common'
-import { DiscoveryService, Reflector } from '@nestjs/core'
-import { BaseStrategyRegistry } from '../../strategy'
-import { RETRIEVER_STRATEGY } from './strategy.decorator'
-import { IRetrieverStrategy } from './strategy.interface'
-
-@Injectable()
-export class RetrieverRegistry extends BaseStrategyRegistry<IRetrieverStrategy> {
-  constructor(discoveryService: DiscoveryService, reflector: Reflector) {
-    super(RETRIEVER_STRATEGY, discoveryService, reflector)
-  }
-}

package/src/lib/rag/source/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/source/strategy.decorator.ts

@@ -1,9 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const DOCUMENT_SOURCE_STRATEGY = 'DOCUMENT_SOURCE_STRATEGY';
-
-/**
- * Decorator to mark a provider as a Document Source Strategy
- */
-export const DocumentSourceStrategy = (provider: string) =>
-  SetMetadata(DOCUMENT_SOURCE_STRATEGY, provider);

package/src/lib/rag/source/strategy.interface.ts

@@ -1,28 +0,0 @@
-import { IDocumentSourceProvider, IIntegration } from '@metad/contracts';
-import { Document } from 'langchain/document';
-import { Permissions } from '../../core';
-
-export interface IDocumentSourceStrategy<TConfig = any> {
-  readonly permissions: Permissions
-  /**
-   * Metadata about this document source
-   */
-  readonly meta: IDocumentSourceProvider
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>;
-
-  /**
-   * Test the connection to the source
-   * 
-   * @param config 
-   */
-  test(config: TConfig): Promise<any>;
-
-  /**
-   * Load documents from the source
-   */
-  loadDocuments(config: TConfig, context?: {integration?: IIntegration}): Promise<Document[]>;
-}

package/src/lib/rag/source/strategy.registry.ts

@@ -1,17 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { DiscoveryService, Reflector } from '@nestjs/core';
-import { BaseStrategyRegistry } from '../../strategy';
-import { DOCUMENT_SOURCE_STRATEGY } from './strategy.decorator';
-import { IDocumentSourceStrategy } from './strategy.interface';
-
-@Injectable()
-export class DocumentSourceRegistry<TConfig = any>
-  extends BaseStrategyRegistry<IDocumentSourceStrategy<TConfig>> 
-{
-  constructor(
-    discoveryService: DiscoveryService,
-    reflector: Reflector
-  ) {
-    super(DOCUMENT_SOURCE_STRATEGY, discoveryService, reflector);
-  }
-}

package/src/lib/rag/textsplitter/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/textsplitter/strategy.decorator.ts

@@ -1,6 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const TEXT_SPLITTER_STRATEGY = 'TEXT_SPLITTER_STRATEGY';
-
-export const TextSplitterStrategy = (provider: string) =>
-  SetMetadata(TEXT_SPLITTER_STRATEGY, provider);

package/src/lib/rag/textsplitter/strategy.interface.ts

@@ -1,28 +0,0 @@
-import { IDocumentChunkerProvider, KnowledgeStructureEnum } from '@metad/contracts'
-import { Document } from 'langchain/document'
-import { ChunkMetadata } from '../types'
-
-/**
- * Split text content into chunks for embedding and retrieval
- */
-export interface ITextSplitterStrategy<TConfig = any> {
-  /**
-   * Metadata about this splitter
-   */
-  readonly meta: IDocumentChunkerProvider
-
-  readonly structure: KnowledgeStructureEnum
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>
-
-  /**
-   * Split a text into chunks and pages (if applicable)
-   */
-  splitDocuments(
-    documents: Document[],
-    options?: TConfig
-  ): Promise<{ chunks: Document<ChunkMetadata>[]; pages?: Document<ChunkMetadata>[] }>
-}

package/src/lib/rag/textsplitter/strategy.registry.ts

@@ -1,17 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { DiscoveryService, Reflector } from '@nestjs/core';
-import { BaseStrategyRegistry } from '../../strategy';
-import { TEXT_SPLITTER_STRATEGY } from './strategy.decorator';
-import { ITextSplitterStrategy } from './strategy.interface';
-
-@Injectable()
-export class TextSplitterRegistry<TConfig = any>
-  extends BaseStrategyRegistry<ITextSplitterStrategy<TConfig>> 
-{
-  constructor(
-    discoveryService: DiscoveryService,
-    reflector: Reflector
-  ) {
-    super(TEXT_SPLITTER_STRATEGY, discoveryService, reflector);
-  }
-}

package/src/lib/rag/transformer/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/transformer/strategy.decorator.ts

@@ -1,9 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const DOCUMENT_TRANSFORMER_STRATEGY = 'DOCUMENT_TRANSFORMER_STRATEGY';
-
-/**
- * Decorator to mark a provider as a Document Transformer Strategy
- */
-export const DocumentTransformerStrategy = (provider: string) =>
-  SetMetadata(DOCUMENT_TRANSFORMER_STRATEGY, provider);

package/src/lib/rag/transformer/strategy.interface.ts

@@ -1,48 +0,0 @@
-import { DocumentInterface } from '@langchain/core/documents'
-import { _TFile, IDocumentProcessorProvider, IIntegration } from '@metad/contracts'
-import { Permissions, XpFileSystem } from '../../core/index'
-import { ChunkMetadata, TDocumentAsset } from '../types'
-
-export type TDocumentTransformerConfig = {
-  stage: 'test' | 'prod'
-  tempDir?: string
-  permissions?: {
-    fileSystem?: XpFileSystem
-    integration?: IIntegration
-  }
-}
-
-export type TDocumentTransformerFile = _TFile & {
-  id?: string
-  filename: string
-  extension: string | undefined
-}
-
-export type TDocumentTransformerInput = TDocumentTransformerFile[] | string | string[]
-
-export type TDocumentTransformerResult = {
-  id?: string
-  chunks: DocumentInterface<ChunkMetadata>[]
-  metadata: {
-    assets?: TDocumentAsset[]
-  }
-}
-
-export interface IDocumentTransformerStrategy<TConfig extends TDocumentTransformerConfig = TDocumentTransformerConfig> {
-  /**
-   * Metadata about this transformer
-   */
-  readonly meta: IDocumentProcessorProvider
-
-  readonly permissions: Permissions
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>
-
-  /**
-   * Transform documents (e.g., extract, OCR, normalize, enrich metadata)
-   */
-  transformDocuments(files: TDocumentTransformerInput, config: TConfig): Promise<TDocumentTransformerResult[]>
-}

package/src/lib/rag/transformer/strategy.registry.ts

@@ -1,14 +0,0 @@
-import { Injectable } from '@nestjs/common'
-import { DiscoveryService, Reflector } from '@nestjs/core'
-import { BaseStrategyRegistry } from '../../strategy'
-import { DOCUMENT_TRANSFORMER_STRATEGY } from './strategy.decorator'
-import { IDocumentTransformerStrategy, TDocumentTransformerConfig } from './strategy.interface'
-
-@Injectable()
-export class DocumentTransformerRegistry<
-  TConfig extends TDocumentTransformerConfig = TDocumentTransformerConfig
-> extends BaseStrategyRegistry<IDocumentTransformerStrategy<TConfig>> {
-  constructor(discoveryService: DiscoveryService, reflector: Reflector) {
-    super(DOCUMENT_TRANSFORMER_STRATEGY, discoveryService, reflector)
-  }
-}

package/src/lib/rag/types.ts

@@ -1,86 +0,0 @@
-import { DocumentInterface } from '@langchain/core/documents'
-import fs from 'fs'
-import http from 'http'
-import https from 'https'
-
-export type TDocumentAsset = {
-  type: 'image' | 'video' | 'audio' | 'file'
-  url: string // remote url
-  filePath: string // local path
-}
-
-export interface ChunkMetadata {
-  documentId?: string // 原始文档 ID
-  pageId?: string // 如果有分页，引用页 ID
-  chunkId: string // 当前块的唯一 ID
-  parentId?: string // 如果是子块，引用父块 ID
-  chunkIndex?: number // 在文档内或在父块内的序号
-  startOffset?: number // 原始文本起始位置
-  endOffset?: number // 原始文本结束位置
-  type?: 'parent' | 'child' // 分块类型
-  children?: DocumentInterface<ChunkMetadata>[]
-  assets?: TDocumentAsset[]
-  [key: string]: any // 允许插件扩展
-}
-
-/**
- * Merge parent chunks with their child chunks based on metadata (parentId and chunkId)
- * @param chunks
- * @param children
- * @returns
- */
-export function mergeParentChildChunks(
-  chunks: DocumentInterface<ChunkMetadata>[], // Parent chunks
-  children: DocumentInterface<ChunkMetadata>[] // Child chunks
-): DocumentInterface<ChunkMetadata>[] {
-  const chunkMap = new Map<string, DocumentInterface<ChunkMetadata>>()
-  for (const chunk of chunks) {
-    chunkMap.set(chunk.metadata.chunkId, chunk)
-  }
-  for (const child of children) {
-    if (!child.metadata.parentId) {
-      if (chunkMap.has(child.metadata.chunkId)) {
-        console.warn(`Duplicate chunkId found: ${child.metadata.chunkId}, skipping...`)
-        continue
-      }
-      chunkMap.set(child.metadata.chunkId, child)
-      continue
-    }
-    const parent = chunkMap.get(child.metadata.parentId)
-    if (parent) {
-      if (!parent.metadata.children) {
-        parent.metadata.children = []
-      }
-      parent.metadata.children.push(child)
-    }
-  }
-  return Array.from(chunkMap.values())
-}
-
-export function isRemoteFile(path: string): boolean {
-  return path.startsWith('http://') || path.startsWith('https://') || path.startsWith('ftp://')
-}
-
-export async function downloadRemoteFile(url: string, dest: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const file = fs.createWriteStream(dest)
-    const client = url.startsWith('https') ? https : http
-    const request = client.get(url, (response) => {
-      if (response.statusCode !== 200) {
-        reject(new Error(`Failed to get '${url}' (${response.statusCode})`))
-        return
-      }
-      response.pipe(file)
-    })
-    file.on('finish', () => {
-      file.close()
-      resolve(dest)
-    })
-    request.on('error', (err) => {
-      fs.unlink(dest, () => reject(err))
-    })
-    file.on('error', (err) => {
-      fs.unlink(dest, () => reject(err))
-    })
-  })
-}

package/src/lib/strategy.ts

@@ -1,37 +0,0 @@
-import { OnModuleInit } from "@nestjs/common"
-import { DiscoveryService, Reflector } from "@nestjs/core"
-
-export class BaseStrategyRegistry<S> implements OnModuleInit {
-    protected strategies = new Map<string, S>()
-
-    constructor(
-        protected readonly strategyKey: string,
-        protected discoveryService: DiscoveryService,
-        protected reflector: Reflector
-    ) {}
-
-    onModuleInit() {
-        const providers = this.discoveryService.getProviders()
-        for (const wrapper of providers) {
-            const { instance } = wrapper
-            if (!instance) continue
-
-            const type = this.reflector.get<string>(this.strategyKey, instance.constructor)
-            if (type) {
-                this.strategies.set(type, instance as S)
-            }
-        }
-    }
-
-    get(type: string): S {
-        const strategy = this.strategies.get(type)
-        if (!strategy) {
-            throw new Error(`No strategy found for type ${type}`)
-        }
-        return strategy
-    }
-
-    list(): S[] {
-        return Array.from(this.strategies.values())
-    }
-}