@metad/contracts 0.0.1 → 3.6.0-beta.0

package/src/ai/copilot-provider.model.ts

@@ -0,0 +1,46 @@
+import { AiModelTypeEnum } from '../agent'
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { IAiProviderEntity } from './ai-model.model'
+import { ICopilot } from './copilot.model'
+
+export interface ICopilotProvider extends IBasePerTenantAndOrganizationEntityModel {
+  providerName?: string
+  providerType?: string
+  credentials?: Record<string, any>
+  // encryptedConfig?: Record<string, any>
+  isValid?: boolean
+  lastUsed?: Date
+  quotaType?: string
+  quotaLimit?: number
+  quotaUsed?: number
+
+  /**
+   */
+  options?: any
+
+  copilot?: ICopilot
+  copilotId?: string
+
+  models?: ICopilotProviderModel[]
+
+  // Temporary properties
+  provider?: IAiProviderEntity
+}
+
+
+export interface ICopilotProviderModel extends IBasePerTenantAndOrganizationEntityModel {
+  providerName?: string
+  modelName?: string
+  modelType?: AiModelTypeEnum
+  // encryptedConfig?: Record<string, any>
+  modelProperties?: Record<string, any>
+  isValid?: boolean
+
+  // Many to one
+  provider?: ICopilotProvider
+  providerId?: string
+
+  // Temporary properties
+}
+
+export type TCopilotProviderPublicDto = Omit<ICopilotProvider, 'credentials'>

package/src/ai/copilot-role.model.ts

@@ -0,0 +1,48 @@
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { IXpertToolset } from './xpert-toolset.model'
+import { IKnowledgebase } from './knowledgebase.model'
+
+/**
+ * @deprecated use Xpert
+ * 
+ * Copilot role, business role for the copilot
+ */
+export interface ICopilotRole extends IBasePerTenantAndOrganizationEntityModel {
+  name: string
+  title?: string
+  titleCN?: string
+  description?: string
+  /**
+   * 系统提示语
+   */
+  prompt?: string
+  /**
+   * 对话开场白
+   */
+  starters?: string[]
+  active?: boolean
+  avatar?: string
+
+  toolsets?: IXpertToolset[]
+  /**
+   * More configuration
+   */
+  options?: TCopilotRoleOptions
+
+  // Many to many relationship
+  knowledgebases?: IKnowledgebase[]
+}
+
+/**
+ * @deprecated use XpertRole
+ */
+export type TCopilotRoleOptions = {
+  context?: Record<string, any>
+  toolsets: {
+    [id: string]: {
+      [tool: string]: {
+        defaultArgs: Record<string, any>
+      }
+    }
+  }
+}

package/src/ai/copilot-store.model.ts

@@ -0,0 +1,34 @@
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+
+export interface ICopilotStore extends TCopilotStore, IBasePerTenantAndOrganizationEntityModel {}
+
+export interface ICopilotStoreVector extends TCopilotStoreVector, IBasePerTenantAndOrganizationEntityModel {
+}
+
+export type TCopilotStore = {
+  prefix: string
+  key?: string
+  value: any
+}
+
+export type TCopilotStoreVector = {
+  prefix: string
+  key: string
+  field_name: any
+  embedding: number[]
+}
+
+export const MEMORY_QA_PROMPT = `Summarize the experience of the above conversation and output a short question and answer.`
+export const MEMORY_PROFILE_PROMPT = `Extract new user profile information from the above conversation in one short sentence. If no new information is available, return nothing.`
+
+export type TMemory = {
+  memoryId?: string
+}
+export type TMemoryUserProfile = TMemory & {
+  profile: string
+  context?: string
+}
+export type TMemoryQA = TMemory & {
+  question: string
+  answer: string
+}

package/src/ai/copilot-user.model.ts

@@ -0,0 +1,24 @@
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { IOrganization } from '../organization.model'
+import { IUser } from '../user.model'
+import { AiProvider } from './ai.model'
+import { ICopilot, TCopilotTokenUsage } from './copilot.model'
+
+/**
+ * 
+ */
+export interface ICopilotUser extends IBasePerTenantAndOrganizationEntityModel, TCopilotTokenUsage {
+  orgId?: string
+  org?: IOrganization
+  copilotId?: string
+  copilot?: ICopilot
+  userId?: string
+  user?: IUser
+  // Associated AI Model provider
+  provider?: AiProvider | string
+  model?: string
+
+  // Total tokens used historically
+  tokenTotalUsed?: number
+  priceTotalUsed?: number
+}

package/src/ai/copilot.model.ts

@@ -0,0 +1,47 @@
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { ICopilotModel } from './copilot-model.model'
+import { ICopilotProvider } from './copilot-provider.model'
+
+export interface ICopilot extends IBasePerTenantAndOrganizationEntityModel {
+  name?: string
+  role: AiProviderRole
+  enabled?: boolean
+
+  showTokenizer?: boolean
+  /**
+   * Balance of Token 
+   */
+  tokenBalance?: number
+
+  /**
+   * Details config for openai api
+   */
+  options?: any
+
+  modelProvider?: ICopilotProvider
+  copilotModel?: ICopilotModel
+
+  // Temporary properties
+  usage?: TCopilotTokenUsage
+}
+
+/**
+ * The order of priority is: `Embedding`, `Secondary`, `Primary`
+ */
+export enum AiProviderRole {
+  Primary = 'primary',
+  Secondary = 'secondary',
+  Embedding = 'embedding',
+  Reasoning = 'reasoning'
+}
+
+export type TCopilotTokenUsage = {
+  // Token limit for the current period
+  tokenLimit?: number
+  priceLimit?: number
+  // Tokens used in the current period
+  tokenUsed?: number
+  priceUsed?: number
+
+  currency?: string
+}

package/src/ai/environment.model.ts

@@ -0,0 +1,18 @@
+import { IBasePerWorkspaceEntityModel } from "./xpert-workspace.model"
+
+export type TEnvironmentVariable = {
+    name: string
+    value: string
+    type: 'default' | 'secret'
+    /**
+     * Has owner, private variable
+     */
+    owner?: string
+}
+
+export interface IEnvironment extends IBasePerWorkspaceEntityModel {
+    name: string
+    isDefault?: boolean
+    isArchived?: boolean
+    variables: TEnvironmentVariable[]
+}

package/src/ai/feature.model.ts

@@ -0,0 +1,6 @@
+export enum AiFeatureEnum {
+  FEATURE_COPILOT = 'FEATURE_COPILOT',
+  FEATURE_COPILOT_KNOWLEDGEBASE = 'FEATURE_COPILOT_KNOWLEDGEBASE',
+  FEATURE_COPILOT_CHAT = 'FEATURE_COPILOT_CHAT',
+  FEATURE_XPERT = 'FEATURE_XPERT',
+}

package/src/ai/index.ts

@@ -0,0 +1,37 @@
+export * from './ai-model.model'
+export * from './ai.model'
+export * from './chat.model'
+export * from './chat-message.model'
+export * from './chat-message-feedback.model'
+export * from './copilot-checkpoint.model'
+export * from './copilot-example.model'
+export * from './copilot-model.model'
+export * from './copilot-organization.model'
+export * from './copilot-provider.model'
+export * from './copilot-store.model'
+export * from './copilot-user.model'
+export * from './copilot.model'
+export * from './feature.model'
+export * from './rag'
+export * from './rag-web'
+export * from './knowledgebase.model'
+export * from './knowledge-doc.model'
+export * from './knowledge-doc-page.model'
+export * from './role-permissions'
+export * from './xpert-agent-execution.model'
+export * from './xpert-agent.model'
+export * from './xpert-tool.model'
+export * from './xpert-toolset.model'
+export * from './xpert-workspace.model'
+export * from './xpert.model'
+export * from './types'
+export * from './xpert-template.model'
+export * from './xpert-task.model'
+export * from './xpert-workflow.model'
+export * from './xpert-workflow-task.prompt'
+export * from './xpert-tool-mcp.model'
+export * from './xpert-project.model'
+export * from './environment.model'
+export * from './knowledgebase-task.model'
+export * from './knowledge-pipeline'
+export * from './knowledge-retrieval-log.model'

package/src/ai/knowledge-doc-page.model.ts

@@ -0,0 +1,20 @@
+import { Document } from '@langchain/core/documents'
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { IKnowledgeDocument } from './knowledge-doc.model'
+import { DocumentMetadata, IKnowledgebase } from './knowledgebase.model'
+
+
+/**
+ * Segmented page of a knowledge document
+ */
+export interface IKnowledgeDocumentPage<Metadata extends DocumentMetadata = DocumentMetadata>
+  extends Document,
+    IBasePerTenantAndOrganizationEntityModel {
+  documentId?: string
+  document?: IKnowledgeDocument
+  knowledgebaseId?: string
+  knowledgebase?: IKnowledgebase
+  
+  metadata: Metadata
+  pageContent: string
+}

package/src/ai/knowledge-doc.model.ts

@@ -0,0 +1,217 @@
+import { DocumentInterface } from '@langchain/core/documents'
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { IIntegration } from '../integration.model'
+import { IStorageFile } from '../storage-file.model'
+import { IKnowledgeDocumentPage } from './knowledge-doc-page.model'
+import { IKnowledgebaseTask } from './knowledgebase-task.model'
+import { DocumentMetadata, IKnowledgebase } from './knowledgebase.model'
+import { TRagWebOptions } from './rag-web'
+
+
+export type DocumentParserConfig = {
+  pages?: number[][]
+  replaceWhitespace?: boolean
+  removeSensitive?: boolean
+  textSplitterType?: string
+  textSplitter?: {
+    [key: string]: unknown
+  }
+  transformerType?: string
+  transformer?: {
+    [key: string]: unknown
+  }
+  imageUnderstandingType?: string
+  imageUnderstanding?: {
+    [key: string]: unknown
+  }
+}
+
+export type DocumentTextParserConfig = DocumentParserConfig & {
+  delimiter?: string
+  chunkSize?: number | null
+  chunkOverlap?: number | null
+}
+
+export type DocumentSheetParserConfig = DocumentParserConfig & {
+  fields?: string[]
+  indexedFields?: string[]
+}
+
+/**
+ * Import Type:
+ * - file: local file
+ * - web: web document
+ * - feishu
+ * - wechat
+ * - notion
+ * ...
+ */
+export enum KDocumentSourceType {
+  /**
+   * Local files
+   */
+  FILE = 'file',
+  /**
+   * Remote files (FTP, SFTP, etc.)
+   */
+  REMOTE_FILE = 'remote-file',
+  /**
+   * Web documents
+   */
+  WEB = 'web',
+  /**
+   * Folder, parent of other documents
+   */
+  FOLDER = 'folder',
+}
+
+/**
+ * Document type category, determine how to process the document.
+ */
+export enum KBDocumentCategoryEnum {
+  Text = 'text',
+  Image = 'image',
+  Audio = 'audio',
+  Video = 'video',
+  Sheet = 'sheet',
+  Other = 'other'
+}
+
+export enum KBDocumentStatusEnum {
+  WAITING = 'waiting',
+  VALIDATE = 'validate',
+  RUNNING = 'running',
+  CANCEL = 'cancel',
+  FINISH = 'finish',
+  ERROR = 'error'
+}
+
+export type TDocumentWebOptions = TRagWebOptions & {
+  //
+}
+
+export type TDocSourceConfig = {
+  key?: string
+}
+
+export type TKnowledgeDocument = {
+  disabled?: boolean
+  
+  knowledgebaseId?: string
+  knowledgebase?: IKnowledgebase
+
+  /**
+   * @deprecated use fileUrl instead
+   */
+  storageFileId?: string
+  /**
+   * @deprecated use fileUrl instead
+   */
+  storageFile?: IStorageFile
+
+  /**
+   * thumbnail base64 string
+   */
+  thumbnail?: string
+
+  /**
+   * default parser ID
+   */
+  parserId: string
+  parserConfig: DocumentTextParserConfig
+  /**
+   * where dose this document come from
+   */
+  sourceType?: KDocumentSourceType | null
+  sourceConfig?: TDocSourceConfig
+  /**
+   * document type category
+   */
+  category?: KBDocumentCategoryEnum | null
+  /**
+   * Local file extension or Web doc provider
+   */
+  type: string
+  /**
+   * file name or web url
+   */
+  name: string
+  /**
+   * where does it store
+   */
+  filePath: string
+  fileUrl?: string
+
+  size: string
+
+  tokenNum?: number | null
+  chunkNum?: number | null
+
+  progress?: number | null
+  /**
+   * process message
+   */
+  processMsg?: string | null
+
+  processBeginAt?: Date | null
+
+  processDuation?: number | null
+
+  /**
+   * is it validate (0: wasted，1: validate)
+   */
+  status?: KBDocumentStatusEnum
+  /**
+   * The background job id
+   */
+  jobId?: string
+
+  options?: TDocumentWebOptions
+
+  integrationId?: string
+  integration?: IIntegration
+
+  pages?: IKnowledgeDocumentPage[]
+  tasks?: IKnowledgebaseTask[]
+}
+
+/**
+ * Document, include file, web pages, folder, virtual, etc.
+ */
+export interface IKnowledgeDocument extends TKnowledgeDocument, IBasePerTenantAndOrganizationEntityModel {
+  // parentId?: string | null
+  parent?: IKnowledgeDocument | null
+  children?: IKnowledgeDocument[]
+
+  // Temp
+  chunks?: DocumentInterface[]
+  metadata?: Metadata
+}
+
+export interface IDocumentChunk<Metadata = DocumentMetadata> {
+  id: string
+  pageContent: string
+  metadata: Metadata & {
+    knowledgeId?: string
+  }
+  collection_id: string
+}
+
+export type Metadata = Record<string, any>
+
+export interface IKnowledgeDocumentCreateInput
+	extends IKnowledgeDocument, IBasePerTenantAndOrganizationEntityModel {}
+
+export interface IKnowledgeDocumentUpdateInput
+	extends Partial<IKnowledgeDocumentCreateInput> {
+	id?: string;
+}
+
+export interface IKnowledgeDocumentFindInput
+	extends IBasePerTenantAndOrganizationEntityModel,
+		IKnowledgeDocument {}
+
+
+export function isDocumentSheet(type: string): boolean {
+  return ['csv', 'xls', 'xlsx', 'ods'].includes(type)
+}

package/src/ai/knowledge-pipeline.ts

@@ -0,0 +1,106 @@
+import { I18nObject, letterStartSUID } from '../types'
+import { ICopilotModel } from './copilot-model.model'
+import { KnowledgeStructureEnum } from './knowledgebase.model'
+import { IWorkflowNode, WorkflowNodeTypeEnum } from './xpert-workflow.model'
+
+export interface IDocumentNodeProvider {
+  name: string
+  label: I18nObject
+  icon?: {
+    svg?: string
+    color?: string
+  }
+  description?: I18nObject
+  helpUrl?: string
+  configSchema: any;
+}
+
+export enum DocumentSourceProviderCategoryEnum {
+  LocalFile = 'local-file',
+  RemoteFile = 'remote-file',
+  WebCrawl = 'web-crawl',
+  Database = 'database'
+}
+
+export interface IDocumentSourceProvider extends IDocumentNodeProvider {
+  category: DocumentSourceProviderCategoryEnum
+}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDocumentProcessorProvider extends IDocumentNodeProvider {}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDocumentChunkerProvider extends IDocumentNodeProvider {}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDocumentUnderstandingProvider extends IDocumentNodeProvider {
+
+}
+
+
+/**
+ * Knowledge Pipeline Source Node
+ */
+export interface IWFNSource extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.SOURCE,
+  provider: string
+  config: any;
+  integrationId?: string
+}
+
+export interface IWFNProcessor extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.PROCESSOR,
+  provider: string
+  config: any;
+  input: string
+  integrationId?: string
+}
+
+export interface IWFNChunker extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.CHUNKER,
+  provider: string
+  config: any;
+  input: string
+}
+
+export interface IWFNUnderstanding extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.UNDERSTANDING,
+  provider: string
+  config: any;
+  input: string
+  visionModel?: ICopilotModel
+}
+
+export interface IWFNKnowledgeBase extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.KNOWLEDGE_BASE,
+  structure?: KnowledgeStructureEnum
+  /**
+   * Documents input variables
+   */
+  inputs?: string[]
+  /**
+   * Embedding model
+   */
+  copilotModel?: ICopilotModel
+  /**
+   * (optional) Rerank model
+   */
+  rerankModel?: ICopilotModel
+  documents?: string[]
+}
+
+export function genPipelineSourceKey() {
+  return letterStartSUID('Source_')
+}
+export function genPipelineProcessorKey() {
+  return letterStartSUID('Processor_')
+}
+export function genPipelineChunkerKey() {
+  return letterStartSUID('Chunker_')
+}
+export function genPipelineUnderstandingKey() {
+  return letterStartSUID('Understanding_')
+}
+export function genPipelineKnowledgeBaseKey() {
+  return letterStartSUID('KnowledgeBase_')
+}

package/src/ai/knowledge-retrieval-log.model.ts

@@ -0,0 +1,19 @@
+import { IBasePerTenantAndOrganizationEntityModel } from "../base-entity.model"
+import { IKnowledgebase } from "./knowledgebase.model"
+
+export interface IKnowledgeRetrievalLog extends IBasePerTenantAndOrganizationEntityModel {
+  // 检索触发的查询内容
+  query: string
+  // 检索来源，比如：ChatBI、ChatDB、API
+  source: string;
+
+  // 命中分段数
+  hitCount: number;
+
+  // 请求 ID（用于一次对话追踪）
+  requestId: string;
+  
+  knowledgebaseId?: string
+   // 与知识库关联
+  knowledgebase?: IKnowledgebase
+}

package/src/ai/knowledgebase-task.model.ts

@@ -0,0 +1,42 @@
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { IChatConversation } from './chat.model';
+import { IKnowledgeDocument } from './knowledge-doc.model';
+import { IKnowledgebase } from './knowledgebase.model'
+
+/**
+ * Task executions of a knowledgebase
+ */
+export interface IKnowledgebaseTask extends IBasePerTenantAndOrganizationEntityModel {
+  knowledgebaseId?: string
+  knowledgebase?: IKnowledgebase
+
+  conversationId?: string;
+  conversation?: IChatConversation
+
+  taskType: string; // preprocess / re-embed / cleanup ...
+  status?: 'pending' | 'running' | 'success' | 'failed' | 'cancelled';
+  steps: TaskStep[];
+  error?: string;
+
+  /**
+   * Temporary content: Documents not actually processed in the task yet
+   */
+  context?: {
+		documents?: Partial<IKnowledgeDocument>[]
+	}
+
+  /**
+   * Many to Many of documents in task
+   */
+  documents?: IKnowledgeDocument[]
+}
+
+export interface TaskStep {
+  name: string;               // step name: load, preprocess, split, embed, store
+  status: 'pending' | 'running' | 'success' | 'failed';
+  progress: number;           // 0 - 100
+  log?: string;               // optional logs
+  error_message?: string;     // optional error
+  started_at?: Date;
+  finished_at?: Date;
+}