@truedat/dd 8.11.6 → 8.11.7

package/src/utils/lineageGraphIndex.js

@@ -0,0 +1,296 @@
+const appendToMap = (map, key, value) => {
+  if (!key) return;
+
+  const values = map.get(key) || [];
+  values.push(value);
+  map.set(key, values);
+};
+
+const pathParts = (id) => {
+  try {
+    return decodeURIComponent(id).split(/[\/#]/).filter(Boolean);
+  } catch {
+    return id.split(/[\/#]/).filter(Boolean);
+  }
+};
+
+const glueScheme = (id) => {
+  if (/^https:\/\/glue\.[^/]+\.amazonaws\.com\/#\//.test(id)) return "https";
+  if (id.startsWith("glue:/Glue/")) return "glue";
+  return undefined;
+};
+
+export const prepareLineageTopology = (topology) => {
+  if (
+    !Array.isArray(topology?.nodes) ||
+    !Array.isArray(topology?.contains) ||
+    !Array.isArray(topology?.depends)
+  )
+    return undefined;
+
+  const nodeById = new Map(
+    topology.nodes.map((node) => [
+      String(node.id),
+      { ...node, id: String(node.id) },
+    ]),
+  );
+  const parentById = new Map();
+  const childrenById = new Map();
+  const ordered = [...nodeById.values()].sort((a, b) =>
+    a.id.localeCompare(b.id),
+  );
+  const contains = [...topology.contains].sort(
+    (a, b) =>
+      a.source.localeCompare(b.source) || a.target.localeCompare(b.target),
+  );
+  for (const { source, target } of contains) {
+    if (!nodeById.has(source) || !nodeById.has(target) || source === target)
+      continue;
+    if (parentById.has(target) && parentById.get(target) !== source)
+      return undefined;
+    const followsAncestors = (id) =>
+      !id || id === target
+        ? id === target
+        : followsAncestors(parentById.get(id));
+    if (followsAncestors(source)) return undefined;
+    parentById.set(target, source);
+  }
+  parentById.forEach((parent, child) =>
+    appendToMap(childrenById, parent, child),
+  );
+  const isGroup = (node) => String(node.class).toLowerCase() === "group";
+  const fieldIds = new Set(
+    ordered
+      .filter((node) => {
+        const parent = nodeById.get(parentById.get(node.id));
+        if (!parent || isGroup(node) || childrenById.has(node.id)) return false;
+        const type = String(node.entity_type || node.type || "").toLowerCase();
+        if (["column", "field", "attribute"].includes(type)) return true;
+        if (
+          [
+            "table",
+            "view",
+            "report",
+            "dashboard",
+            "database",
+            "schema",
+            "system",
+            "process",
+          ].includes(type)
+        )
+          return false;
+        const parts = pathParts(node.id);
+        const parentParts = pathParts(parent.id);
+        if (parts.length < 2) return false;
+        if (parts.at(-2) === parentParts.at(-1)) return true;
+        const siblings = childrenById.get(parent.id) || [];
+        const qualifier = parts.slice(0, -1).join("/");
+        return (
+          siblings.length > 1 &&
+          siblings.every((id) => {
+            const sibling = nodeById.get(id);
+            const siblingParts = pathParts(id);
+            return (
+              !isGroup(sibling) &&
+              !childrenById.has(id) &&
+              siblingParts.length > 1 &&
+              siblingParts.slice(0, -1).join("/") === qualifier
+            );
+          })
+        );
+      })
+      .map(({ id }) => id),
+  );
+
+  const aliasCandidates = new Map();
+  ordered.forEach((node) => {
+    if (!fieldIds.has(node.id) || !glueScheme(node.id)) return;
+    const key = JSON.stringify([
+      parentById.get(node.id),
+      (node.name || pathParts(node.id).at(-1)).trim(),
+    ]);
+    appendToMap(aliasCandidates, key, node);
+  });
+  const originalsByVisibleId = new Map(
+    ordered.map((node) => [node.id, [node]]),
+  );
+  const originalToVisibleId = new Map(ordered.map(({ id }) => [id, id]));
+  aliasCandidates.forEach((members) => {
+    if (
+      members.length !== 2 ||
+      new Set(members.map(({ id }) => glueScheme(id))).size !== 2
+    )
+      return;
+    const visibleId = members[0].id;
+    originalsByVisibleId.set(visibleId, members);
+    members.forEach(({ id }) => {
+      originalToVisibleId.set(id, visibleId);
+      if (id !== visibleId) originalsByVisibleId.delete(id);
+    });
+  });
+  const nodes = ordered
+    .filter(({ id }) => originalsByVisibleId.has(id))
+    .map((node) => {
+      const parentId = parentById.get(node.id);
+      const depthOf = (id) => (id ? 1 + depthOf(parentById.get(id)) : 0);
+      const z = depthOf(parentId);
+      return {
+        ...node,
+        name: node.name || pathParts(node.id).at(-1) || node.id,
+        parentId,
+        z,
+        topology: true,
+        field: fieldIds.has(node.id),
+        container: isGroup(node) || childrenById.has(node.id),
+        resourceContainer: (childrenById.get(node.id) || []).some((id) =>
+          fieldIds.has(id),
+        ),
+        originals: originalsByVisibleId.get(node.id),
+      };
+    });
+  const edges = topology.depends.map((edge, index) => ({
+    ...edge,
+    id: edge.id || `topology-depends-${index}`,
+    originalSource: edge.source,
+    originalTarget: edge.target,
+    source: originalToVisibleId.get(edge.source),
+    target: originalToVisibleId.get(edge.target),
+    normalizedDirection: true,
+  }));
+  if (edges.some(({ source, target }) => !source || !target)) return undefined;
+  return {
+    nodes,
+    edges,
+    nodeById,
+    parentById,
+    originalToVisibleId,
+    originalsByVisibleId,
+  };
+};
+
+export const createLineageGraphIndex = (nodes = [], edges = []) => {
+  const nodeById = new Map();
+  const parentById = new Map();
+  const childrenByParentId = new Map();
+  const incomingByNode = new Map();
+  const outgoingByNode = new Map();
+
+  nodes.forEach((node) => {
+    if (!node?.id) return;
+    nodeById.set(node.id, node);
+    if (node.parentId) {
+      parentById.set(node.id, node.parentId);
+      appendToMap(childrenByParentId, node.parentId, node.id);
+    }
+  });
+
+  edges.forEach((edge) => {
+    if (!edge?.id) return;
+    appendToMap(outgoingByNode, edge.source, edge);
+    appendToMap(incomingByNode, edge.target, edge);
+  });
+
+  const descendantCache = new Map();
+  const descendantIds = (rootId) => {
+    if (descendantCache.has(rootId)) return descendantCache.get(rootId);
+
+    const descendants = new Set();
+    const queue = [...(childrenByParentId.get(rootId) || [])];
+
+    const bfsCollect = ([head, ...tail]) => {
+      if (head === undefined) return;
+      if (descendants.has(head)) {
+        bfsCollect(tail);
+        return;
+      }
+      descendants.add(head);
+      bfsCollect([...tail, ...(childrenByParentId.get(head) || [])]);
+    };
+    bfsCollect(queue);
+
+    descendantCache.set(rootId, descendants);
+    return descendants;
+  };
+
+  return {
+    descendantIds,
+    incomingByNode,
+    nodeById,
+    outgoingByNode,
+    parentById,
+  };
+};
+
+const walkEdges = (
+  seedIds,
+  edgesByNode,
+  endpoint,
+  focusedNodeIds,
+  focusedEdgeIds,
+) => {
+  const visited = new Set(seedIds);
+  const queue = [...seedIds];
+
+  const bfsWalk = ([nodeId, ...remaining]) => {
+    if (nodeId === undefined) return;
+    (edgesByNode.get(nodeId) || []).forEach((edge) => {
+      focusedEdgeIds.add(edge.id);
+      focusedNodeIds.add(edge.source);
+      focusedNodeIds.add(edge.target);
+      const nextId = edge[endpoint];
+      if (!visited.has(nextId)) {
+        visited.add(nextId);
+        remaining.push(nextId);
+      }
+    });
+    bfsWalk(remaining);
+  };
+  bfsWalk(queue);
+};
+
+export const focusLineageFromSeeds = (seedIds, index) => {
+  const seeds = seedIds instanceof Set ? seedIds : new Set(seedIds || []);
+  const focusedNodeIds = new Set(seeds);
+  const focusedEdgeIds = new Set();
+
+  walkEdges(
+    seeds,
+    index.incomingByNode,
+    "source",
+    focusedNodeIds,
+    focusedEdgeIds,
+  );
+  walkEdges(
+    seeds,
+    index.outgoingByNode,
+    "target",
+    focusedNodeIds,
+    focusedEdgeIds,
+  );
+
+  return { focusedEdgeIds, focusedNodeIds };
+};
+
+export const projectLineageNodeIds = (
+  nodeIds,
+  parentById,
+  visibleNodeIds,
+  aliases = new Map(),
+) => {
+  const projected = new Set();
+
+  nodeIds.forEach((nodeId) => {
+    const findVisible = (id, seen) => {
+      if (!id || seen.has(id)) return;
+      const visibleId = aliases.get(id) || id;
+      if (visibleNodeIds.has(visibleId)) {
+        projected.add(visibleId);
+        return;
+      }
+      findVisible(parentById.get(id), new Set([...seen, id]));
+    };
+    findVisible(nodeId, new Set());
+  });
+
+  return projected;
+};