@xpert-ai/plugin-sdk 3.6.0 → 3.6.1

package/src/lib/plugin.interface.ts

@@ -1,67 +0,0 @@
-import { ModuleMetadata, Type } from '@nestjs/common';
-
-/**
- * Metadata definition for a plugin in NestJS.
- */
-export interface PluginMetadata extends ModuleMetadata {
-	/**
-	 * List of entities injected by the plugin.
-	 */
-	entities?: Array<Type<any>> | (() => Array<Type<any>>);
-
-	/**
-	 * List of subscribers injected by the plugin.
-	 */
-	subscribers?: Array<Type<any>> | (() => Array<Type<any>>);
-}
-
-/**
- * Interface for plugins with a bootstrap lifecycle method.
- */
-export interface IOnPluginBootstrap {
-	/**
-	 * Called when the plugin is being initialized.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginBootstrap(): void | Promise<void>;
-}
-
-/**
- * Interface for plugins with a destroy lifecycle method.
- */
-export interface IOnPluginDestroy {
-	/**
-	 * Called when the plugin is being destroyed.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginDestroy(): void | Promise<void>;
-}
-
-/**
- * Interface for plugins supporting various seed operations.
- */
-export interface IOnPluginSeedable {
-	/**
-	 * Invoked when seeding basic plugin data.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginBasicSeed?(): void | Promise<void>;
-
-	/**
-	 * Invoked when seeding default plugin data.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginDefaultSeed?(): void | Promise<void>;
-
-	/**
-	 * Invoked when seeding random plugin data.
-	 * @returns A void or a Promise representing the completion of the operation.
-	 */
-	onPluginRandomSeed?(): void | Promise<void>;
-}
-
-/**
- * Represents the combined lifecycle methods for a plugin.
- * This type combines interfaces for initializing and destroying a plugin.
- */
-export type PluginLifecycleMethods = IOnPluginBootstrap & IOnPluginDestroy & IOnPluginSeedable;

package/src/lib/plugin.ts

@@ -1,32 +0,0 @@
-import { Module } from '@nestjs/common';
-import { MODULE_METADATA } from '@nestjs/common/constants';
-import { pick } from 'lodash-es';
-import { PluginMetadata } from './plugin.interface';
-import { PLUGIN_METADATA } from './plugin-metadata';
-
-/**
- * Decorator function for extending NestJS features with additional metadata.
- *
- * @param pluginMetadata Metadata to be applied to the target class.
- * @returns Class decorator function.
- */
-export function XpertServerPlugin(pluginMetadata: PluginMetadata): ClassDecorator {
-	return (targetClass) => {
-		// Iterate over properties in PLUGIN_METADATA
-		for (const metadataProperty of Object.values(PLUGIN_METADATA)) {
-			const property = metadataProperty as keyof PluginMetadata;
-
-			// Check if the property exists in pluginMetadata and is not undefined
-			if (property in pluginMetadata && pluginMetadata[property] !== undefined) {
-				// Set metadata on the target class using Reflect
-				Reflect.defineMetadata(property, pluginMetadata[property] || [], targetClass);
-			}
-		}
-
-		// Pick relevant metadata from pluginMetadata based on MODULE_METADATA values
-		const metadata = pick(pluginMetadata, Object.values(MODULE_METADATA) as string[]);
-
-		// Apply the Module decorator with the picked metadata
-		Module(metadata)(targetClass);
-	};
-}

package/src/lib/rag/image/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/image/strategy.decorator.ts

@@ -1,9 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const IMAGE_UNDERSTANDING_STRATEGY = 'IMAGE_UNDERSTANDING_STRATEGY';
-
-/**
- * Decorator to mark a provider as an Image Understanding Strategy
- */
-export const ImageUnderstandingStrategy = (provider: string) =>
-  SetMetadata(IMAGE_UNDERSTANDING_STRATEGY, provider);

package/src/lib/rag/image/strategy.interface.ts

@@ -1,37 +0,0 @@
-import { DocumentInterface } from '@langchain/core/documents'
-import { BaseChatModel } from '@langchain/core/language_models/chat_models'
-import { IDocumentUnderstandingProvider, IKnowledgeDocument } from '@metad/contracts'
-import { Permissions, XpFileSystem } from '../../core/index'
-import { ChunkMetadata } from '../types'
-
-export type TImageUnderstandingConfig = {
-  stage: 'test' | 'prod'
-  visionModel: BaseChatModel
-  permissions?: {
-    fileSystem?: XpFileSystem
-  }
-}
-
-export type TImageUnderstandingResult = {
-  chunks: DocumentInterface<Partial<ChunkMetadata>>[]
-  // pages?: Document<Partial<ChunkMetadata>>[]
-  metadata: any // Additional metadata (e.g. model name, processing time)
-}
-
-export interface IImageUnderstandingStrategy<TConfig extends TImageUnderstandingConfig = TImageUnderstandingConfig> {
-  readonly permissions: Permissions
-  /**
-   * Metadata about this strategy
-   */
-  readonly meta: IDocumentUnderstandingProvider
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>
-
-  /**
-   * Understand image files (e.g., OCR, VLM, Chart Parsing)
-   */
-  understandImages(doc: IKnowledgeDocument<ChunkMetadata>, config: TConfig): Promise<TImageUnderstandingResult>
-}

package/src/lib/rag/image/strategy.registry.ts

@@ -1,17 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { DiscoveryService, Reflector } from '@nestjs/core';
-import { BaseStrategyRegistry } from '../../strategy';
-import { IMAGE_UNDERSTANDING_STRATEGY } from './strategy.decorator';
-import { IImageUnderstandingStrategy, TImageUnderstandingConfig } from './strategy.interface';
-
-@Injectable()
-export class ImageUnderstandingRegistry<TConfig extends TImageUnderstandingConfig = TImageUnderstandingConfig>
-  extends BaseStrategyRegistry<IImageUnderstandingStrategy<TConfig>> 
-{
-  constructor(
-    discoveryService: DiscoveryService,
-    reflector: Reflector
-  ) {
-    super(IMAGE_UNDERSTANDING_STRATEGY, discoveryService, reflector);
-  }
-}

package/src/lib/rag/index.ts

@@ -1,7 +0,0 @@
-export * from './textsplitter/index'
-export * from './source/index'
-export * from './transformer/index'
-export * from './retriever/index'
-export * from './types'
-export * from './image/index'
-export * from './knowledge/index'

package/src/lib/rag/knowledge/index.ts

@@ -1,3 +0,0 @@
-export * from './knowledge-strategy.decorator'
-export * from './knowledge-strategy.interface'
-export * from './knowledge-strategy.registry'

package/src/lib/rag/knowledge/knowledge-strategy.decorator.ts

@@ -1,6 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const KNOWLEDGE_STRATEGY = 'KNOWLEDGE_STRATEGY';
-
-export const KnowledgeStrategyKey = (provider: string) =>
-  SetMetadata(KNOWLEDGE_STRATEGY, provider);

package/src/lib/rag/knowledge/knowledge-strategy.interface.ts

@@ -1,15 +0,0 @@
-import { IIntegration } from '@metad/contracts'
-import { Document } from '@langchain/core/documents'
-
-export type TKnowledgeStrategyParams = {
-	query: string;
-	k: number;
-	filter: Record<string, any>
-	options: {
-		knowledgebaseId: string
-	}
-}
-
-export interface KnowledgeStrategy {
-	execute(integration: IIntegration, payload: TKnowledgeStrategyParams): Promise<{ chunks: [Document, number][] }>
-}

package/src/lib/rag/knowledge/knowledge-strategy.registry.ts

@@ -1,12 +0,0 @@
-import { Injectable } from '@nestjs/common'
-import { DiscoveryService, Reflector } from '@nestjs/core'
-import { BaseStrategyRegistry } from '../../strategy'
-import { KNOWLEDGE_STRATEGY } from './knowledge-strategy.decorator'
-import { KnowledgeStrategy } from './knowledge-strategy.interface'
-
-@Injectable()
-export class KnowledgeStrategyRegistry extends BaseStrategyRegistry<KnowledgeStrategy> {
-  constructor(discoveryService: DiscoveryService, reflector: Reflector) {
-    super(KNOWLEDGE_STRATEGY, discoveryService, reflector)
-  }
-}

package/src/lib/rag/retriever/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/retriever/strategy.decorator.ts

@@ -1,9 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const RETRIEVER_STRATEGY = 'RETRIEVER_STRATEGY';
-
-/**
- * Decorator to mark a provider as a Retriever Strategy
- */
-export const RetrieverStrategy = (provider: string) =>
-  SetMetadata(RETRIEVER_STRATEGY, provider);

package/src/lib/rag/retriever/strategy.interface.ts

@@ -1,32 +0,0 @@
-import { VectorStore } from '@langchain/core/vectorstores'
-import { I18nObject } from '@metad/contracts'
-import { Document } from '@langchain/core/documents'
-
-export type TRetrieverConfig = {
-  vectorStore: VectorStore
-}
-
-export interface IRetrieverStrategy<TConfig extends TRetrieverConfig = TRetrieverConfig> {
-  /**
-   * Metadata about this retriever
-   */
-  readonly meta: {
-    name: string
-    label: I18nObject
-    configSchema: any
-    icon: {
-      svg?: string
-      color?: string
-    }
-  }
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>;
-
-  /**
-   * Retrieve relevant documents for a given query
-   */
-  retrieve(query: string, options?: TConfig): Promise<{ documents: Document[] }>;
-}

package/src/lib/rag/retriever/strategy.registry.ts

@@ -1,12 +0,0 @@
-import { Injectable } from '@nestjs/common'
-import { DiscoveryService, Reflector } from '@nestjs/core'
-import { BaseStrategyRegistry } from '../../strategy'
-import { RETRIEVER_STRATEGY } from './strategy.decorator'
-import { IRetrieverStrategy } from './strategy.interface'
-
-@Injectable()
-export class RetrieverRegistry extends BaseStrategyRegistry<IRetrieverStrategy> {
-  constructor(discoveryService: DiscoveryService, reflector: Reflector) {
-    super(RETRIEVER_STRATEGY, discoveryService, reflector)
-  }
-}

package/src/lib/rag/source/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/source/strategy.decorator.ts

@@ -1,9 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const DOCUMENT_SOURCE_STRATEGY = 'DOCUMENT_SOURCE_STRATEGY';
-
-/**
- * Decorator to mark a provider as a Document Source Strategy
- */
-export const DocumentSourceStrategy = (provider: string) =>
-  SetMetadata(DOCUMENT_SOURCE_STRATEGY, provider);

package/src/lib/rag/source/strategy.interface.ts

@@ -1,36 +0,0 @@
-import { IDocumentSourceProvider, IIntegration } from '@metad/contracts';
-import { Document } from '@langchain/core/documents';
-import { Permissions } from '../../core';
-
-export interface IDocumentSourceStrategy<TConfig = any> {
-  readonly permissions: Permissions
-  /**
-   * Metadata about this document source
-   */
-  readonly meta: IDocumentSourceProvider
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>;
-
-  /**
-   * Test the connection to the source
-   * 
-   * @param config 
-   */
-  test(config: TConfig): Promise<any>;
-
-  /**
-   * Load documents from the source
-   */
-  loadDocuments(config: TConfig, context?: {integration?: IIntegration}): Promise<Document[]>;
-  /**
-   * Load a single document by its ID
-   * 
-   * @deprecated Planning
-   * @param documentId 
-   * @param context 
-   */
-  loadDocument?(document: Document, context: {integration?: IIntegration}): Promise<Document>;
-}

package/src/lib/rag/source/strategy.registry.ts

@@ -1,17 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { DiscoveryService, Reflector } from '@nestjs/core';
-import { BaseStrategyRegistry } from '../../strategy';
-import { DOCUMENT_SOURCE_STRATEGY } from './strategy.decorator';
-import { IDocumentSourceStrategy } from './strategy.interface';
-
-@Injectable()
-export class DocumentSourceRegistry<TConfig = any>
-  extends BaseStrategyRegistry<IDocumentSourceStrategy<TConfig>> 
-{
-  constructor(
-    discoveryService: DiscoveryService,
-    reflector: Reflector
-  ) {
-    super(DOCUMENT_SOURCE_STRATEGY, discoveryService, reflector);
-  }
-}

package/src/lib/rag/textsplitter/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/textsplitter/strategy.decorator.ts

@@ -1,6 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const TEXT_SPLITTER_STRATEGY = 'TEXT_SPLITTER_STRATEGY';
-
-export const TextSplitterStrategy = (provider: string) =>
-  SetMetadata(TEXT_SPLITTER_STRATEGY, provider);

package/src/lib/rag/textsplitter/strategy.interface.ts

@@ -1,28 +0,0 @@
-import { IDocumentChunkerProvider, KnowledgeStructureEnum } from '@metad/contracts'
-import { DocumentInterface } from '@langchain/core/documents'
-import { ChunkMetadata } from '../types'
-
-/**
- * Split text content into chunks for embedding and retrieval
- */
-export interface ITextSplitterStrategy<TConfig = any> {
-  /**
-   * Metadata about this splitter
-   */
-  readonly meta: IDocumentChunkerProvider
-
-  readonly structure: KnowledgeStructureEnum
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>
-
-  /**
-   * Split a text into chunks and pages (if applicable)
-   */
-  splitDocuments(
-    documents: DocumentInterface[],
-    options?: TConfig
-  ): Promise<{ chunks: DocumentInterface<ChunkMetadata>[] }>
-}

package/src/lib/rag/textsplitter/strategy.registry.ts

@@ -1,17 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { DiscoveryService, Reflector } from '@nestjs/core';
-import { BaseStrategyRegistry } from '../../strategy';
-import { TEXT_SPLITTER_STRATEGY } from './strategy.decorator';
-import { ITextSplitterStrategy } from './strategy.interface';
-
-@Injectable()
-export class TextSplitterRegistry<TConfig = any>
-  extends BaseStrategyRegistry<ITextSplitterStrategy<TConfig>> 
-{
-  constructor(
-    discoveryService: DiscoveryService,
-    reflector: Reflector
-  ) {
-    super(TEXT_SPLITTER_STRATEGY, discoveryService, reflector);
-  }
-}

package/src/lib/rag/transformer/index.ts

@@ -1,3 +0,0 @@
-export * from './strategy.decorator'
-export * from './strategy.interface'
-export * from './strategy.registry'

package/src/lib/rag/transformer/strategy.decorator.ts

@@ -1,9 +0,0 @@
-import { SetMetadata } from '@nestjs/common';
-
-export const DOCUMENT_TRANSFORMER_STRATEGY = 'DOCUMENT_TRANSFORMER_STRATEGY';
-
-/**
- * Decorator to mark a provider as a Document Transformer Strategy
- */
-export const DocumentTransformerStrategy = (provider: string) =>
-  SetMetadata(DOCUMENT_TRANSFORMER_STRATEGY, provider);

package/src/lib/rag/transformer/strategy.interface.ts

@@ -1,34 +0,0 @@
-import { IDocumentProcessorProvider, IIntegration, IKnowledgeDocument } from '@metad/contracts'
-import { Permissions, XpFileSystem } from '../../core/index'
-import { ChunkMetadata } from '../types'
-
-export type TDocumentTransformerConfig = {
-  stage: 'test' | 'prod'
-  tempDir?: string
-  permissions?: {
-    fileSystem?: XpFileSystem
-    integration?: IIntegration
-  }
-}
-
-export interface IDocumentTransformerStrategy<TConfig extends TDocumentTransformerConfig = TDocumentTransformerConfig> {
-  /**
-   * Metadata about this transformer
-   */
-  readonly meta: IDocumentProcessorProvider
-
-  readonly permissions: Permissions
-
-  /**
-   * Validate the configuration
-   */
-  validateConfig(config: TConfig): Promise<void>
-
-  /**
-   * Transform documents (e.g., extract, OCR, normalize, enrich metadata)
-   */
-  transformDocuments(
-    files: Partial<IKnowledgeDocument>[],
-    config: TConfig
-  ): Promise<Partial<IKnowledgeDocument<ChunkMetadata>>[]>
-}

package/src/lib/rag/transformer/strategy.registry.ts

@@ -1,14 +0,0 @@
-import { Injectable } from '@nestjs/common'
-import { DiscoveryService, Reflector } from '@nestjs/core'
-import { BaseStrategyRegistry } from '../../strategy'
-import { DOCUMENT_TRANSFORMER_STRATEGY } from './strategy.decorator'
-import { IDocumentTransformerStrategy, TDocumentTransformerConfig } from './strategy.interface'
-
-@Injectable()
-export class DocumentTransformerRegistry<
-  TConfig extends TDocumentTransformerConfig = TDocumentTransformerConfig
-> extends BaseStrategyRegistry<IDocumentTransformerStrategy<TConfig>> {
-  constructor(discoveryService: DiscoveryService, reflector: Reflector) {
-    super(DOCUMENT_TRANSFORMER_STRATEGY, discoveryService, reflector)
-  }
-}

package/src/lib/rag/types.ts

@@ -1,76 +0,0 @@
-import { DocumentInterface } from '@langchain/core/documents'
-import { IDocChunkMetadata } from '@metad/contracts'
-import fs from 'fs'
-import http from 'http'
-import https from 'https'
-
-export { TDocumentAsset } from '@metad/contracts'
-
-export interface ChunkMetadata extends IDocChunkMetadata{
-  startOffset?: number // Start position in the original text
-  endOffset?: number // End position in the original text
-  type?: 'parent' | 'child' // Chunk type
-  
-  [key: string]: any // Allow plugin extensions
-}
-
-/**
- * Merge parent chunks with their child chunks based on metadata (parentId and chunkId)
- * 
- * @deprecated use buildChunkTreeAndFindLeaves instead
- */
-export function mergeParentChildChunks(
-  chunks: DocumentInterface<ChunkMetadata>[], // Parent chunks
-  children: DocumentInterface<ChunkMetadata>[] // Child chunks
-): DocumentInterface<ChunkMetadata>[] {
-  const chunkMap = new Map<string, DocumentInterface<ChunkMetadata>>()
-  for (const chunk of chunks) {
-    chunkMap.set(chunk.metadata.chunkId, chunk)
-  }
-  for (const child of children) {
-    if (!child.metadata.parentId) {
-      if (chunkMap.has(child.metadata.chunkId)) {
-        console.warn(`Duplicate chunkId found: ${child.metadata.chunkId}, skipping...`)
-        continue
-      }
-      chunkMap.set(child.metadata.chunkId, child)
-      continue
-    }
-    const parent = chunkMap.get(child.metadata.parentId)
-    if (parent) {
-      if (!parent.metadata.children) {
-        parent.metadata.children = []
-      }
-      parent.metadata.children.push(child)
-    }
-  }
-  return Array.from(chunkMap.values())
-}
-
-export function isRemoteFile(path: string): boolean {
-  return path.startsWith('http://') || path.startsWith('https://') || path.startsWith('ftp://')
-}
-
-export async function downloadRemoteFile(url: string, dest: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const file = fs.createWriteStream(dest)
-    const client = url.startsWith('https') ? https : http
-    const request = client.get(url, (response) => {
-      if (response.statusCode !== 200) {
-        reject(new Error(`Failed to get '${url}' (${response.statusCode})`))
-        return
-      }
-      response.pipe(file)
-    })
-    file.on('finish', () => {
-      file.close()
-      resolve(dest)
-    })
-    request.on('error', (err) => {
-      fs.unlink(dest, () => reject(err))
-    })
-    file.on('error', (err) => {
-      fs.unlink(dest, () => reject(err))
-    })
-  })
-}

package/src/lib/strategy.ts

@@ -1,37 +0,0 @@
-import { OnModuleInit } from "@nestjs/common"
-import { DiscoveryService, Reflector } from "@nestjs/core"
-
-export class BaseStrategyRegistry<S> implements OnModuleInit {
-    protected strategies = new Map<string, S>()
-
-    constructor(
-        protected readonly strategyKey: string,
-        protected discoveryService: DiscoveryService,
-        protected reflector: Reflector
-    ) {}
-
-    onModuleInit() {
-        const providers = this.discoveryService.getProviders()
-        for (const wrapper of providers) {
-            const { instance } = wrapper
-            if (!instance) continue
-
-            const type = this.reflector.get<string>(this.strategyKey, instance.constructor)
-            if (type) {
-                this.strategies.set(type, instance as S)
-            }
-        }
-    }
-
-    get(type: string): S {
-        const strategy = this.strategies.get(type)
-        if (!strategy) {
-            throw new Error(`No strategy found for type ${type}`)
-        }
-        return strategy
-    }
-
-    list(): S[] {
-        return Array.from(this.strategies.values())
-    }
-}