@metad/contracts 3.6.2 → 3.6.7

package/src/ai/knowledge-doc.model.ts

@@ -0,0 +1,364 @@
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { IIntegration } from '../integration.model'
+import { IStorageFile } from '../storage-file.model'
+import { IKnowledgeDocumentPage } from './knowledge-doc-page.model'
+import { IKnowledgebaseTask } from './knowledgebase-task.model'
+import { IKnowledgebase, KBMetadataFieldDef } from './knowledgebase.model'
+import { TRagWebOptions } from './rag-web'
+import { IKnowledgeDocumentChunk } from './knowledge-doc-chunk.model'
+import { DocumentSourceProviderCategoryEnum } from './knowledge-pipeline'
+import { TCopilotModel } from './copilot-model.model'
+import { I18nObject } from '../types'
+
+
+export type DocumentParserConfig = {
+  pages?: number[][]
+  replaceWhitespace?: boolean
+  removeSensitive?: boolean
+  textSplitterType?: string
+  textSplitter?: {
+    [key: string]: unknown
+  }
+  transformerType?: string
+  transformerIntegration?: string
+  transformer?: {
+    [key: string]: unknown
+  }
+  imageUnderstandingType?: string
+  imageUnderstandingIntegration?: string
+  imageUnderstanding?: {
+    [key: string]: unknown
+  }
+  imageUnderstandingModel?: TCopilotModel
+}
+
+export type DocumentTextParserConfig = DocumentParserConfig & {
+  delimiter?: string
+  chunkSize?: number | null
+  chunkOverlap?: number | null
+}
+
+export type DocumentSheetParserConfig = DocumentParserConfig & {
+  fields?: string[]
+  indexedFields?: string[]
+}
+
+/**
+ * Import Type:
+ * - file: local file
+ * - web: web document
+ * ...
+ */
+export enum DocumentTypeEnum {
+  /**
+   * Folder, parent of other documents
+   */
+  FOLDER = 'folder',
+
+  /**
+   * Local files
+   * @deprecated use DocumentSourceProviderCategoryEnum local file type instead
+   */
+  FILE = 'file'
+}
+export const KDocumentSourceType = {
+  ...DocumentSourceProviderCategoryEnum,
+  ...DocumentTypeEnum
+}
+
+/**
+ * Document type category, determine how to process the document.
+ */
+export enum KBDocumentCategoryEnum {
+  Text = 'text',
+  Image = 'image',
+  Audio = 'audio',
+  Video = 'video',
+  Sheet = 'sheet',
+  Other = 'other'
+}
+
+export enum KBDocumentStatusEnum {
+  WAITING = 'waiting',
+  VALIDATE = 'validate',
+  RUNNING = 'running',
+  TRANSFORMED = 'transformed',
+  SPLITTED = 'splitted',
+  UNDERSTOOD = 'understood',
+  EMBEDDING = 'embedding',
+  CANCEL = 'cancel',
+  FINISH = 'finish',
+  ERROR = 'error'
+}
+
+export type TDocumentWebOptions = TRagWebOptions & {
+  //
+}
+
+export type TDocSourceConfig = {
+  key?: string
+}
+
+export type TKnowledgeDocument = {
+  disabled?: boolean
+  
+  knowledgebaseId?: string
+
+  /**
+   * @deprecated use fileUrl instead
+   */
+  storageFileId?: string
+  /**
+   * @deprecated use fileUrl instead
+   */
+  storageFile?: IStorageFile
+
+  /**
+   * thumbnail base64 string
+   */
+  thumbnail?: string
+
+  /**
+   * default parser ID
+   * @deprecated unused
+   */
+  parserId: string
+  parserConfig: DocumentTextParserConfig
+  /**
+   * where dose this document come from
+   */
+  sourceType?: DocumentSourceProviderCategoryEnum | DocumentTypeEnum
+  sourceConfig?: TDocSourceConfig
+  /**
+   * document type category
+   */
+  category?: KBDocumentCategoryEnum | null
+  /**
+   * Local file extension or Web doc provider
+   */
+  type: string
+  /**
+   * file name or web url
+   */
+  name: string
+  /**
+   * where does it store
+   */
+  filePath: string
+  fileUrl?: string
+  /**
+   * Folder path in server for this document file.
+   * Init it in creating document entity.
+   */
+  folder?: string
+
+  size?: string
+  mimeType?: string
+
+  tokenNum?: number | null
+  chunkNum?: number | null
+
+  progress?: number | null
+  /**
+   * process message
+   */
+  processMsg?: string | null
+
+  processBeginAt?: Date | null
+
+  processDuation?: number | null
+
+  /**
+   * is it validate (0: wasted，1: validate)
+   */
+  status?: KBDocumentStatusEnum
+  /**
+   * The background job id
+   */
+  jobId?: string
+
+  options?: TDocumentWebOptions
+
+  integrationId?: string
+  integration?: IIntegration
+
+  /**
+	 * @deprecated use chunks instead
+	 */
+  pages?: IKnowledgeDocumentPage[]
+
+  chunks?: IKnowledgeDocumentChunk[]
+  tasks?: IKnowledgebaseTask[]
+}
+
+/**
+ * Document, include file, web pages, folder, virtual, etc.
+ */
+export interface IKnowledgeDocument<T extends KnowledgeDocumentMetadata = KnowledgeDocumentMetadata> extends TKnowledgeDocument, IBasePerTenantAndOrganizationEntityModel {
+  parent?: IKnowledgeDocument | null
+  children?: IKnowledgeDocument[]
+  knowledgebase?: IKnowledgebase
+
+  draft?: TKnowledgeDocument
+  
+  metadata?: T
+}
+
+/**
+ * System built-in standard document Metadata structure
+ */
+export interface StandardDocumentMetadata {
+  // ---- Document Info ----
+  originalFileName?: string;          // Original file name, e.g. "Complex_SQL_QA.markdown"
+  originalFileSize?: string | null;   // Original file size (if missing, display as "-")
+  uploadTime?: string;                // Upload time in ISO format
+  lastUpdatedTime?: string;           // Last updated time
+  source?: string;                    // Source, e.g. "Local File" / "Web Import" / "API"
+  
+  // ---- Technical Parameters ----
+  segmentRule?: string;               // Segmentation rule, e.g. "General"
+  segmentLength?: number;             // Maximum segment length (token or char)
+  averageSegmentLength?: string;      // Average segment length
+  segmentCount?: number;              // Number of segments
+  recallRate?: string;                // Recall count statistics, e.g. "0.00% (0/11)"
+  embedTime?: string;                 // Embedding time, e.g. "1.99 sec"
+  embedCost?: string | null;          // Embedding cost (if none, display as "-")
+  tokens?: number;                    // Number of tokens in the document
+}
+
+export interface KnowledgeDocumentMetadata extends StandardDocumentMetadata {
+  [key: string]: any
+}
+
+
+// export type Metadata = any
+
+export interface IKnowledgeDocumentCreateInput
+	extends IKnowledgeDocument, IBasePerTenantAndOrganizationEntityModel {}
+
+export interface IKnowledgeDocumentUpdateInput
+	extends Partial<IKnowledgeDocumentCreateInput> {
+	id?: string;
+}
+
+export interface IKnowledgeDocumentFindInput
+	extends IBasePerTenantAndOrganizationEntityModel,
+		IKnowledgeDocument {}
+
+
+export function isDocumentSheet(type: string): boolean {
+  return ['csv', 'xls', 'xlsx', 'ods', 'vnd.openxmlformats-officedocument.spreadsheetml.sheet'].includes(type)
+}
+
+export function isImageType(type: string): boolean {
+  return ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'svg', 'webp'].includes(type)
+}
+
+export function isVideoType(type: string): boolean {
+  return ['mp4', 'avi', 'mov', 'wmv', 'flv', 'mkv', 'webm'].includes(type)
+}
+
+export function isAudioType(type: string): boolean {
+  return ['mp3', 'wav', 'aac', 'flac', 'ogg', 'm4a'].includes(type)
+}
+
+export function classificateDocumentCategory(entity: Partial<IKnowledgeDocument>): KBDocumentCategoryEnum {
+  return isDocumentSheet(entity.type) ? KBDocumentCategoryEnum.Sheet : 
+									isImageType(entity.type) ? KBDocumentCategoryEnum.Image : 
+									isVideoType(entity.type) ? KBDocumentCategoryEnum.Video :
+									isAudioType(entity.type) ? KBDocumentCategoryEnum.Audio :
+									KBDocumentCategoryEnum.Text
+}
+
+/**
+ * System standard Metadata field definition constants
+ * It can be used for rendering, validation, sorting, displaying descriptions, etc.
+ */
+export const STANDARD_METADATA_FIELDS: { group: I18nObject; fields: KBMetadataFieldDef[] }[] = [
+  // Document Info
+  {
+    group: {
+      en_US: 'Document Info',
+      zh_Hans: '文档信息'
+    },
+    fields: [
+      {
+        key: 'title',
+        label: { en_US: 'Original File Name', zh_Hans: '原始文件名称' },
+        type: 'string'
+      },
+      // {
+      //   key: 'originalFileSize',
+      //   label: { en_US: 'Original File Size', zh_Hans: '原始文件大小' },
+      //   type: 'string'
+      // },
+      // {
+      //   key: 'uploadTime',
+      //   label: { en_US: 'Upload Time', zh_Hans: '上传日期' },
+      //   type: 'datetime'
+      // },
+      // {
+      //   key: 'lastUpdatedTime',
+      //   label: { en_US: 'Last Updated Time', zh_Hans: '最后更新时间' },
+      //   type: 'datetime'
+      // },
+      // {
+      //   key: 'source',
+      //   label: { en_US: 'Source', zh_Hans: '来源' },
+      //   type: 'string'
+      // }
+    ]
+  },
+  // Technical Parameters
+  {
+    group: {
+      en_US: 'Technical Parameters',
+      zh_Hans: '技术参数'
+    },
+    fields: [
+       {
+        key: 'tokens',
+        label: { en_US: 'Tokens', zh_Hans: '词元' },
+        type: 'number'
+      },
+      // {
+      //   key: 'segmentRule',
+      //   label: { en_US: 'Segmentation Rule', zh_Hans: '分段规则' },
+      //   type: 'string'
+      // },
+  //     {
+  //       key: 'segmentLength',
+  //       label: { en_US: 'Segment Length', zh_Hans: '段落长度' },
+  //       type: 'number'
+  //     },
+  //     {
+  //       key: 'averageSegmentLength',
+  //       label: { en_US: 'Average Segment Length', zh_Hans: '平均段落长度' },
+  //       type: 'string'
+  //     },
+  //     {
+  //       key: 'segmentCount',
+  //       label: { en_US: 'Segment Count', zh_Hans: '段落数量' },
+  //       type: 'number'
+  //     },
+  //     {
+  //       key: 'recallRate',
+  //       label: { en_US: 'Recall Count', zh_Hans: '召回次数' },
+  //       type: 'string'
+  //     },
+  //     {
+  //       key: 'embedTime',
+  //       label: { en_US: 'Embedding Time', zh_Hans: '嵌入时间' },
+  //       type: 'string'
+  //     },
+  //     {
+  //       key: 'embedCost',
+  //       label: { en_US: 'Embedding Cost', zh_Hans: '嵌入花费' },
+  //       type: 'string'
+  //     }
+    ]
+  }
+] as const;
+
+
+export type StandardMetadataFieldKey = typeof STANDARD_METADATA_FIELDS[number]['fields'][number]['key'];

package/src/ai/knowledge-pipeline.ts

@@ -0,0 +1,138 @@
+import { I18nObject, IconDefinition, letterStartSUID } from '../types'
+import { ICopilotModel } from './copilot-model.model'
+import { KnowledgeStructureEnum } from './knowledgebase.model'
+import { IWorkflowNode, WorkflowNodeTypeEnum } from './xpert-workflow.model'
+import { TXpertParameter } from './xpert.model'
+
+export interface IDocumentNodeProvider {
+  name: string
+  label: I18nObject
+  icon?: IconDefinition
+  description?: I18nObject
+  helpUrl?: string
+  configSchema: any;
+}
+
+/**
+ * Category of document source provider
+ */
+export enum DocumentSourceProviderCategoryEnum {
+  /**
+   * Local files uploaded directly to the system
+   */
+  LocalFile = 'local-file',
+
+  /**
+   * Remote file systems, e.g. S3, FTP, etc.
+   */
+  FileSystem = 'file-system',
+
+  /**
+   * Online documents, e.g. public URLs, Google Docs, etc.
+   */
+  OnlineDocument = 'online-document',
+
+  /**
+   * Web crawling from public websites
+   */
+  WebCrawl = 'web-crawl',
+
+  /**
+   * Database connections, e.g. MySQL, PostgreSQL, etc.
+   * @deprecated Planning
+   */
+  Database = 'database'
+}
+
+export interface IDocumentSourceProvider extends IDocumentNodeProvider {
+  category: DocumentSourceProviderCategoryEnum
+}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDocumentProcessorProvider extends IDocumentNodeProvider {}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDocumentChunkerProvider extends IDocumentNodeProvider {}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDocumentUnderstandingProvider extends IDocumentNodeProvider {
+
+}
+
+
+/**
+ * Knowledge Pipeline Source Node
+ */
+export interface IWFNSource<T = any> extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.SOURCE,
+  provider: string
+  parameters?: TXpertParameter[];
+  config: T;
+  integrationId?: string
+}
+
+export interface IWFNProcessor extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.PROCESSOR,
+  provider: string
+  config: any;
+  input: string
+  integrationId?: string
+}
+
+export interface IWFNChunker extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.CHUNKER,
+  provider: string
+  config: any;
+  input: string
+}
+
+export interface IWFNUnderstanding extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.UNDERSTANDING,
+  provider: string
+  config: any;
+  input: string
+  visionModel?: ICopilotModel
+}
+
+export interface IWFNKnowledgeBase extends IWorkflowNode {
+  type: WorkflowNodeTypeEnum.KNOWLEDGE_BASE,
+  /**
+   * @deprecated
+   */
+  structure?: KnowledgeStructureEnum
+  /**
+   * Documents input variables
+   */
+  inputs?: string[]
+  /**
+   * Embedding model
+   */
+  copilotModel?: ICopilotModel
+  /**
+   * (optional) Rerank model
+   */
+  rerankModel?: ICopilotModel
+
+  /**
+   * (optional) Vision model
+   */
+  visionModel?: ICopilotModel
+
+  documents?: string[]
+}
+
+export function genPipelineSourceKey() {
+  return letterStartSUID('Source_')
+}
+export function genPipelineProcessorKey() {
+  return letterStartSUID('Processor_')
+}
+export function genPipelineChunkerKey() {
+  return letterStartSUID('Chunker_')
+}
+export function genPipelineUnderstandingKey() {
+  return letterStartSUID('Understanding_')
+}
+export function genPipelineKnowledgeBaseKey() {
+  return letterStartSUID('KnowledgeBase_')
+}

package/src/ai/knowledge-retrieval-log.model.ts

@@ -0,0 +1,19 @@
+import { IBasePerTenantAndOrganizationEntityModel } from "../base-entity.model"
+import { IKnowledgebase } from "./knowledgebase.model"
+
+export interface IKnowledgeRetrievalLog extends IBasePerTenantAndOrganizationEntityModel {
+  // 检索触发的查询内容
+  query: string
+  // 检索来源，比如：ChatBI、ChatDB、API
+  source: string;
+
+  // 命中分段数
+  hitCount: number;
+
+  // 请求 ID（用于一次对话追踪）
+  requestId: string;
+  
+  knowledgebaseId?: string
+   // 与知识库关联
+  knowledgebase?: IKnowledgebase
+}

package/src/ai/knowledgebase-task.model.ts

@@ -0,0 +1,46 @@
+import { IBasePerTenantAndOrganizationEntityModel } from '../base-entity.model'
+import { IChatConversation } from './chat.model';
+import { IKnowledgeDocument } from './knowledge-doc.model';
+import { IKnowledgebase } from './knowledgebase.model'
+import { IXpertAgentExecution } from './xpert-agent-execution.model';
+
+/**
+ * Task executions of a knowledgebase
+ */
+export interface IKnowledgebaseTask extends IBasePerTenantAndOrganizationEntityModel {
+  knowledgebaseId?: string
+  knowledgebase?: IKnowledgebase
+
+  conversationId?: string;
+  conversation?: IChatConversation
+
+  executionId?: string;
+  execution?: IXpertAgentExecution;
+
+  taskType: string; // preprocess / re-embed / cleanup ...
+  status?: 'pending' | 'running' | 'success' | 'failed' | 'cancelled';
+  steps: TaskStep[];
+  error?: string;
+
+  /**
+   * Temporary content: Documents not actually processed in the task yet
+   */
+  context?: {
+		documents?: Partial<IKnowledgeDocument>[]
+	}
+
+  /**
+   * Many to Many of documents in task
+   */
+  documents?: IKnowledgeDocument[]
+}
+
+export interface TaskStep {
+  name: string;               // step name: load, preprocess, split, embed, store
+  status: 'pending' | 'running' | 'success' | 'failed';
+  progress: number;           // 0 - 100
+  log?: string;               // optional logs
+  error_message?: string;     // optional error
+  started_at?: Date;
+  finished_at?: Date;
+}