@aloudata/ink-lineage 0.0.1-beta.1

package/src/manager/DataProcessor.ts

@@ -0,0 +1,782 @@
+import {
+  getNodeHeight,
+  groupNodesByEdgesAndDirection,
+} from '@aloudata/ink-graph-new';
+import {
+  EEntityType,
+  EDirection,
+  IEntity,
+  IBseRelation,
+  INodeConfig,
+  IColumnConfig,
+  IEdgeConfig,
+  ILineageData,
+  IColResBase,
+  TDataBase,
+  EDashType,
+} from '../types';
+import _ from 'lodash';
+import { BaseManager } from './BaseManager';
+import {
+  COL_HEIGHT,
+  CUSTOM_NODE,
+  LINEAGE_EDGE,
+  NODE_WIDTH,
+  TABLE_HEADER_HEIGHT,
+  TABLE_NODE,
+} from '../constant';
+import { InkLineageManager } from './LineageManager';
+
+export interface IParsedGraphData<
+  TData extends TDataBase<TColumn>,
+  TColumn extends IColResBase
+> {
+  nodes: INodeConfig<TData, TColumn>[];
+  edges: IEdgeConfig[];
+}
+
+export class DataProcessor<
+  TData extends TDataBase<TColumn>,
+  TColumn extends IColResBase
+> extends BaseManager<TData, TColumn> {
+  static startNodes: string[] = [];
+
+  constructor(lineageManager: InkLineageManager<TData, TColumn>) {
+    super(lineageManager);
+  }
+
+  static parseData<
+    TData extends TDataBase<TColumn>,
+    TColumn extends IColResBase,
+  >(
+    data: ILineageData<TData, TColumn>,
+    startNodes: string[],
+  ): IParsedGraphData<TData, TColumn> {
+    DataProcessor.startNodes = startNodes;
+    //  parse data here
+    const { entities, relations } = data;
+    const nodes = DataProcessor.parseEntities<TData, TColumn>(
+      entities,
+      startNodes,
+    );
+    const edges = DataProcessor.parseRelations(relations);
+
+    const res = {
+      nodes,
+      edges,
+    };
+
+    return res;
+  }
+
+  static parseEntities<
+    TData extends TDataBase<TColumn>,
+    TColumn extends IColResBase,
+  >(entities: IEntity<TData>[], startNodes: string[]) {
+    const nodesId: string[] = [];
+    const nodes: INodeConfig<TData, TColumn>[] = [];
+
+    entities.forEach((entity) => {
+      if (nodesId.includes(entity.guid)) return;
+
+      const isStartNode = startNodes.includes(entity.guid);
+      let node = null;
+
+      switch (entity.typeCode) {
+        case 'Table':
+        case 'View':
+          node = DataProcessor.parseTableEntity<TData, TColumn>(
+            entity,
+            isStartNode,
+          );
+          break;
+        default:
+          node = DataProcessor.parseCustomEntity<TData, TColumn>(
+            entity,
+            isStartNode,
+          );
+          break;
+      }
+
+      if (node) {
+        nodes.push(node);
+        nodesId.push(node.id);
+      }
+    });
+
+    return nodes;
+  }
+
+  static parseRelations(relations: IBseRelation[]) {
+    const existedEdgesMap = new Map<string, string>();
+    const edges: IEdgeConfig[] = [];
+
+    relations.forEach((relation) => {
+      const edgeId = `${relation.srcGuid}___${relation.dstGuid}`;
+
+      if (existedEdgesMap.has(edgeId)) {
+        if (existedEdgesMap.get(edgeId) !== relation.relationTypeCode) {
+          const edge = edges.find((e) => e.id === edgeId);
+          if (edge) {
+            edge.relationType = EDashType.ALL;
+            edge.style.strokeDasharray = [0];
+          }
+        }
+
+        return;
+      }
+
+      const isIndirect = relation.relationTypeCode.includes('Indirect');
+
+      const edge = {
+        id: edgeId,
+        type: LINEAGE_EDGE,
+        data: {
+          ...relation,
+        },
+        source: relation.srcGuid,
+        target: relation.dstGuid,
+        visible: true,
+        isActive: false,
+        isFolded: false,
+        canBeFold: false,
+        relationType: isIndirect ? EDashType.INDIRECT : EDashType.DIRECT,
+        style: {
+          stroke: '#B1B1B1',
+          strokeDasharray: isIndirect ? [2] : [0],
+          lineWidth: 1,
+        },
+        points: [],
+      } as IEdgeConfig;
+
+      edges.push(edge);
+      existedEdgesMap.set(edge.id, edge.data.relationTypeCode);
+    });
+
+    return edges;
+  }
+
+  static parseTableEntity<
+    TData extends TDataBase<TColumn>,
+    TColumn extends IColResBase
+  >(entity: IEntity<TData>, isStartNode: boolean) {
+    const cols = Array.isArray(entity.properties?.columns)
+      ? entity.properties?.columns?.map((col) => {
+        return DataProcessor.parseColumn<TColumn>(
+          col,
+          isStartNode,
+          entity.guid,
+          entity.properties?.name,
+        );
+      }) || []
+      : [];
+
+    // 只有起始资产的relatedCols一开始有元素
+    const relatedCols = Array.isArray(entity.properties?.columns)
+      ? entity.properties?.columns?.map((col) => {
+        return DataProcessor.parseRelatedColumn<TColumn>(
+          col,
+          isStartNode,
+          entity.guid,
+          entity.properties?.name,
+        );
+      }) || []
+      : [];
+
+    const data = {
+      id: entity.guid,
+      type: TABLE_NODE,
+      data: {
+        ...entity.properties,
+        type: (entity.properties?.type as EEntityType) || EEntityType.TABLE,
+        guid: entity.guid,
+        typeCode: entity.typeCode,
+      },
+      style: {
+        width: NODE_WIDTH + 2,
+        height: TABLE_HEADER_HEIGHT,
+        headerHeight: TABLE_HEADER_HEIGHT,
+      },
+      position: {
+        x: 0,
+        y: 0,
+      },
+      isStartNode,
+      isOpen: isStartNode,
+      visible: true,
+      isRelated: false,
+      isFolded: false,
+      isExpandOutput: false,
+      isExpandInput: false,
+      canExpandInput: false,
+      canExpandOutput: false,
+      canFoldInput: false,
+      canFoldOutput: false,
+      hasMoreInput: true,
+      hasMoreOutput: true,
+      children: cols,
+      pagination: {
+        pageNum: 1,
+        pageSize: 10,
+        total: 0,
+        totalPage: 0,
+      },
+      relatedColumns: relatedCols,
+    } as unknown as INodeConfig<TData, TColumn>;
+
+    data.style.height = getNodeHeight(data);
+
+    return data;
+  }
+
+  static parseColumn<TColumn extends IColResBase>(
+    column: TColumn,
+    isStartNode: boolean,
+    tableId: string,
+    tableName: string,
+  ) {
+    return {
+      data: {
+        ...column,
+      },
+      type: '',
+      style: {
+        width: NODE_WIDTH + 2,
+        height: COL_HEIGHT,
+        headerHeight: COL_HEIGHT,
+      },
+      position: { x: 8, y: 0 },
+      id: column.guid,
+      tableId,
+      tableName: column.tableName || tableName,
+      visible: true,
+      isFolded: false,
+      isStartNode,
+      isRelated: false,
+      canExpandInput: false,
+      canExpandOutput: false,
+      canFoldInput: false,
+      canFoldOutput: false,
+      hasMoreInput: true,
+      hasMoreOutput: true,
+    } as IColumnConfig<TColumn>;
+  }
+
+  static parseRelatedColumn<TColumn extends IColResBase>(
+    column: TColumn,
+    isStartNode: boolean,
+    tableId: string,
+    tableName: string,
+  ) {
+    return {
+      data: {
+        ...column,
+      },
+      type: '',
+      style: {
+        width: NODE_WIDTH + 2,
+        height: COL_HEIGHT,
+        headerHeight: COL_HEIGHT,
+      },
+      position: { x: 0, y: 0 },
+      id: column.guid,
+      tableId,
+      tableName: column.tableName || tableName,
+      visible: true,
+      isFolded: false,
+      isStartNode,
+      isRelated: true,
+      isRelatedColumn: true,
+      canExpandInput: false,
+      canExpandOutput: false,
+      canFoldInput: false,
+      canFoldOutput: false,
+      hasMoreInput: true,
+      hasMoreOutput: true,
+    } as IColumnConfig<TColumn>;
+  }
+
+  static parseCustomEntity<
+    TData extends TDataBase<TColumn>,
+    TColumn extends IColResBase
+  >(entity: IEntity<TData>, isStartNode: boolean) {
+    const data = {
+      id: entity.guid,
+      type: CUSTOM_NODE,
+      data: {
+        ...entity.properties,
+        type: (entity.properties?.type as EEntityType) || EEntityType.TABLE,
+        guid: entity.guid,
+        typeCode: entity.typeCode,
+        typeCodeName: entity.typeCodeName,
+      },
+      style: {
+        width: NODE_WIDTH + 2,
+        height: TABLE_HEADER_HEIGHT,
+        headerHeight: TABLE_HEADER_HEIGHT,
+      },
+      position: {
+        x: 0,
+        y: 0,
+      },
+      isStartNode,
+      isOpen: isStartNode,
+      visible: true,
+      isRelated: false,
+      canExpandInput: false,
+      canExpandOutput: false,
+      canFoldInput: false,
+      canFoldOutput: false,
+      hasMoreInput: true,
+      hasMoreOutput: true,
+      children: Array.isArray(entity.properties?.columns)
+        ? entity.properties?.columns?.map((col) => {
+          return DataProcessor.parseColumn<TColumn>(
+            col,
+            isStartNode,
+            entity.guid,
+            entity.properties?.name,
+          );
+        }) || []
+        : [],
+      pagination: {
+        pageNum: 1,
+        pageSize: 10,
+        total: 0,
+        totalPage: 0,
+      },
+      relatedColumns: [],
+    } as unknown as INodeConfig<TData, TColumn>;
+
+    data.style.height = getNodeHeight(data);
+
+    return data;
+  }
+
+  updateSideIconVisible(
+    data: IParsedGraphData<TData, TColumn>,
+  ) {
+    if (_.isNil(data)) return;
+
+    const nodesConfig = data.nodes.filter((cfg) => cfg.visible);
+    const edgesConfig = data.edges.filter((cfg) => cfg.visible);
+    if (!nodesConfig) return;
+
+    const startNodeIndex = nodesConfig.findIndex((node) => node.isStartNode);
+    if (startNodeIndex < 0) return;
+
+    const startNodeCfg = nodesConfig[startNodeIndex];
+    if (!startNodeCfg) return;
+
+    const nodesConfigMap = new Map<
+      string,
+      INodeConfig<TData, TColumn> | IColumnConfig<TColumn>
+    >();
+    nodesConfig?.forEach((n) => {
+      nodesConfigMap.set(n.id, n);
+      n.children?.forEach((child) => {
+        nodesConfigMap.set(child.id, child);
+      });
+    });
+
+    const inputGroupedData = groupNodesByEdgesAndDirection(
+      startNodeCfg,
+      nodesConfigMap,
+      edgesConfig,
+      EDirection.INPUT,
+    );
+
+    const outputGroupedData = groupNodesByEdgesAndDirection(
+      startNodeCfg,
+      nodesConfigMap,
+      edgesConfig,
+      EDirection.OUTPUT,
+    );
+
+    if (!inputGroupedData && !outputGroupedData) return;
+
+    const { groupedNodes: inputGroupNodes } = inputGroupedData;
+    const { groupedNodes: outputGroupNodes } = outputGroupedData;
+    // 两个map合并
+    const groupedNodes = new Map();
+    if (inputGroupNodes) {
+      for (const [key, value] of inputGroupNodes) {
+        if (groupedNodes.has(key)) continue;
+        groupedNodes.set(key, value);
+      }
+    }
+
+    if (outputGroupNodes) {
+      for (const [key, value] of outputGroupNodes) {
+        if (groupedNodes.has(key)) continue;
+        groupedNodes.set(key, value);
+      }
+    }
+
+    const dstNodeIds = edgesConfig.map((edge) => edge.target);
+    const srcNodeIds = edgesConfig.map((edge) => edge.source);
+
+    // table
+    nodesConfig?.forEach((n) => {
+      n.canExpandInput = false;
+      n.canExpandOutput = false;
+      n.canFoldInput = false;
+      n.canFoldOutput = false;
+
+      const currNodeLevel = groupedNodes.get(n.id);
+
+      if (_.isNil(currNodeLevel)) {
+        console.warn(`get node level error '${n.id}'`);
+        return;
+      }
+
+      let isInputLeaf = false;
+      let isOutputLeaf = false;
+      if (currNodeLevel === 0) {
+        isInputLeaf = !dstNodeIds.includes(n.id);
+        isOutputLeaf = !srcNodeIds.includes(n.id);
+      } else if (currNodeLevel < 0) {
+        isInputLeaf = !dstNodeIds.includes(n.id);
+      } else {
+        isOutputLeaf = !srcNodeIds.includes(n.id);
+      }
+
+      // input node exist & left of start node
+      if (currNodeLevel <= 0 && !isInputLeaf) {
+        n.canFoldInput = true;
+      }
+
+      // input node not exist & left of start node
+      if (currNodeLevel <= 0 && isInputLeaf && n.hasMoreInput) {
+        n.canExpandInput = true;
+      }
+
+      // output node exist & right of start node
+      if (currNodeLevel >= 0 && !isOutputLeaf) {
+        n.canFoldOutput = true;
+      }
+
+      // output node not exist & right of start node
+      if (currNodeLevel >= 0 && isOutputLeaf && n.hasMoreOutput) {
+        n.canExpandOutput = true;
+      }
+
+      // column:  always true if isStartNode === true
+      const columnRefreshList = n.isStartNode
+        ? [...n.children, ...n.relatedColumns]
+        : n.relatedColumns;
+
+      columnRefreshList?.forEach((col) => {
+        col.canExpandInput = false;
+        col.canExpandOutput = false;
+        col.canFoldInput = false;
+        col.canFoldOutput = false;
+
+        if (!col.isStartNode && !col.isRelatedColumn) return;
+
+        let isInputLeaf_col = false;
+        let isOutputLeaf_col = false;
+        if (currNodeLevel === 0) {
+          isInputLeaf_col = !dstNodeIds.includes(col.id);
+          isOutputLeaf_col = !srcNodeIds.includes(col.id);
+        } else if (currNodeLevel < 0) {
+          isInputLeaf_col = !dstNodeIds.includes(col.id);
+        } else {
+          isOutputLeaf_col = !srcNodeIds.includes(col.id);
+        }
+
+        // input node exist & left of start node
+        if (currNodeLevel <= 0 && !isInputLeaf_col) {
+          col.canFoldInput = true;
+        }
+
+        // input node not exist & left of start node
+        if (currNodeLevel <= 0 && isInputLeaf_col && col.hasMoreInput) {
+          col.canExpandInput = true;
+        }
+
+        // output node exist & right of start node
+        if (currNodeLevel >= 0 && !isOutputLeaf_col) {
+          col.canFoldOutput = true;
+        }
+
+        // output node not exist & right of start node
+        if (currNodeLevel >= 0 && isOutputLeaf_col && col.hasMoreOutput) {
+          col.canExpandOutput = true;
+        }
+      });
+    });
+  }
+
+  getNodeType(typeCode: string) {
+    return typeCode === 'Table' ? TABLE_NODE : CUSTOM_NODE;
+  }
+
+  updateNodesVisible(
+    data: IParsedGraphData<TData, TColumn>,
+    edgeDashVisible: EDashType,
+    onlyShowRelated: boolean,
+    isFoldChecked: boolean,
+  ) {
+    if (_.isNil(data)) return;
+
+    data.nodes.forEach((node) => {
+      node.visible = this.getNodeVisible(
+        node,
+        data,
+        edgeDashVisible,
+        onlyShowRelated,
+        isFoldChecked,
+      );
+
+      node.relatedColumns?.forEach((col) => {
+        col.visible = this.getRelatedNodeVisible(col, data, edgeDashVisible);
+      });
+
+      node.children?.forEach((child) => {
+        child.visible = this.getNodeVisible(
+          child,
+          data,
+          edgeDashVisible,
+          onlyShowRelated,
+          isFoldChecked,
+        );
+      });
+    });
+  }
+
+  updateEdgesVisible(
+    data: IParsedGraphData<TData, TColumn>,
+    edgeDashVisible: EDashType,
+    onlyShowRelated: boolean,
+    isFoldChecked: boolean,
+  ) {
+    if (_.isNil(data)) return;
+
+    data.edges.forEach((edge) => {
+      edge.visible = this.getEdgeVisible(
+        edge,
+        edgeDashVisible,
+        onlyShowRelated,
+        isFoldChecked,
+      );
+    });
+  }
+
+  getNodeVisible(
+    nodeConfig: INodeConfig<TData, TColumn> | IColumnConfig<TColumn>,
+    data: IParsedGraphData<TData, TColumn>,
+    edgeDashVisible: EDashType,
+    onlyShowRelated: boolean,
+    isFoldChecked: boolean,
+  ) {
+    if (nodeConfig.isStartNode) return true;
+
+    if (nodeConfig.isFolded) return false;
+
+    // 只显示相关节点 & 该节点不是相关节点 & 该节点是表，则该节点不可见
+    if (onlyShowRelated && !nodeConfig.isRelated && !nodeConfig.tableId)
+      return false;
+
+    if (nodeConfig.canBeFold && isFoldChecked) return false;
+
+    // 找出所有target、source为该节点的边
+    const relatedEdges = data.edges.filter((edge) => {
+      return (
+        // 上游表-当前表、当前表-下游表
+        edge.target === nodeConfig.id ||
+        edge.source === nodeConfig.id ||
+        // 上游列-当前列、当前列-下游列
+        (nodeConfig as INodeConfig<TData, TColumn>)?.relatedColumns?.some(
+          (col) => {
+            return edge.target === col.id || edge.source === col.id;
+          },
+        )
+      );
+    });
+
+    if (!relatedEdges.length) return false;
+
+    const allRelatedNodeInvisible = relatedEdges.every((edgeCfg) => {
+      const srcNode = data.nodes.find((node) => node.id === edgeCfg.source);
+      const tgtNode = data.nodes.find((node) => node.id === edgeCfg.target);
+
+      if ((srcNode?.canBeFold || tgtNode?.canBeFold) && isFoldChecked)
+        return true;
+    });
+
+    if (allRelatedNodeInvisible) return false;
+
+    const isAllIndirectEdges = relatedEdges.every(
+      (edge) => edge.relationType === EDashType.INDIRECT,
+    );
+
+    // 开关只有true false，直接、间接
+    if (edgeDashVisible === EDashType.INDIRECT) {
+      return true;
+    } else {
+      // 如果边都是间接边，那么该节点就隐藏
+      if (isAllIndirectEdges) return false;
+
+      return true;
+    }
+  }
+
+  getRelatedNodeVisible(
+    nodeConfig: INodeConfig<TData, TColumn> | IColumnConfig<TColumn>,
+    data: IParsedGraphData<TData, TColumn>,
+    edgeDashVisible: EDashType,
+  ) {
+    // 仅相关开启的情况下，如果高亮链路是表链路，隐藏非相关的列
+    if (this.lineageManager?.rightKeyMenuManager?.onlyShowRelated) {
+      if (!this.lineageManager?.activeNodes?.[0]?.tableName) {
+        if (!_.some(this.lineageManager?.relatedNodes, { id: nodeConfig.id }))
+          return false;
+      }
+    }
+
+    // 找出所有target、source为该节点的边
+    const relatedEdges = data.edges.filter(
+      (edge) =>
+        (edge.target === nodeConfig.id || edge.source === nodeConfig.id) &&
+        edge.visible,
+    );
+
+    if (!relatedEdges.length) return false;
+
+    const isAllIndirectEdges = relatedEdges.every(
+      (edge) => edge.relationType === EDashType.INDIRECT,
+    );
+
+    // 开关只有true false，直接、间接
+    if (edgeDashVisible === EDashType.INDIRECT) {
+      return true;
+    } else {
+      // 如果边都是间接边，那么该节点就隐藏
+      if (isAllIndirectEdges) return false;
+
+      return true;
+    }
+  }
+
+  getEdgeVisible(
+    edgeCfg: IEdgeConfig,
+    edgeDashVisible: EDashType,
+    onlyShowRelated: boolean,
+    isFoldChecked: boolean,
+  ) {
+    // 如果边被折叠了，那么该边就隐藏
+    if (edgeCfg.isFolded) return false;
+
+    let visible = false;
+
+    if (isFoldChecked) {
+      const foldStatus =
+        this.lineageManager?.foldLineageManager?.foldStatusMap?.get(
+          edgeCfg.foldTableId,
+        );
+
+      if (foldStatus) {
+        visible = !!edgeCfg.isExtraFoldEdge;
+      }
+    }
+
+    switch (edgeDashVisible) {
+      case EDashType.DIRECT:
+        visible =
+          edgeCfg.relationType === EDashType.DIRECT ||
+          edgeCfg.relationType === EDashType.ALL;
+        break;
+      case EDashType.INDIRECT:
+        visible = true;
+        break;
+    }
+
+    return visible;
+  }
+
+  updateNodesAndEdgesVisible(
+    data: IParsedGraphData<TData, TColumn>,
+    edgeDashVisible: EDashType,
+    onlyShowRelated: boolean,
+    isFoldChecked: boolean,
+  ) {
+    this.updateEdgesVisible(
+      data,
+      edgeDashVisible,
+      onlyShowRelated,
+      isFoldChecked,
+    );
+
+    // 更新节点本身、列、节点children的可见性
+    this.updateNodesVisible(
+      data,
+      edgeDashVisible,
+      onlyShowRelated,
+      isFoldChecked,
+    );
+  }
+
+  refreshRelatedNodesAndEdges(
+    relatedNodes: (INodeConfig<TData, TColumn> | IColumnConfig<TColumn>)[],
+    relatedEdges: IEdgeConfig[],
+  ) {
+    relatedNodes.forEach((n) => {
+      if (!n) return;
+
+      const nodeConfig = this.lineageManager.getNodeConfigById(n.id);
+      if (!nodeConfig) return;
+
+      nodeConfig.isRelated = true;
+
+      nodeConfig.style.stroke = '#3271C9';
+
+      if (!nodeConfig.isActive) {
+        nodeConfig.style.fill = '#E4EFFF';
+      }
+
+      nodeConfig.style.zIndex = 3;
+    });
+
+    relatedEdges.forEach((e) => {
+      if (!e) return;
+
+      const edge = this.lineageManager.getEdgeConfigById(e.id);
+      if (!edge) return;
+      if (
+        (edge.inFoldLink &&
+          this.lineageManager.foldLineageManager.isFoldChecked) ||
+        edge.isExtraFoldEdge
+      )
+        return;
+
+      edge.style.stroke = '#3271C9';
+    });
+  }
+
+  updateNodeHeight(data: IParsedGraphData<TData, TColumn>) {
+    data.nodes.forEach((node) => {
+      node.style.height = this.calcNodeHeight(node);
+    });
+  }
+
+  updateColumnPosition(data: IParsedGraphData<TData, TColumn>) {
+    data.nodes.forEach((node) => {
+      const visibleRelatedColumns = node.relatedColumns?.filter(
+        (col) => col.visible,
+      );
+      visibleRelatedColumns?.forEach((col, i) => {
+        col.position.y = i * COL_HEIGHT;
+      });
+
+      const visibleColumns = node.children?.filter((col) => col.visible);
+      visibleColumns?.forEach((col, i) => {
+        col.position.y = node.position.y + COL_HEIGHT * i;
+      });
+    });
+  }
+
+  calcNodeHeight(node: INodeConfig<TData, TColumn>) {
+    return getNodeHeight(node);
+  }
+}