@eqtylab/explorer 0.0.0

package/src/explorer-component/utils/generateGraph.ts

@@ -0,0 +1,271 @@
+import {
+    LineageEdge,
+    LineageNode,
+    ComputeNodeSidebarData,
+    DataNodeSidebarData,
+    createEdge,
+    createDefaultNode,
+    IrohData,
+    DataNodeRegistration,
+    MetadataLineage,
+    StorageLineage,
+    Subgraph,
+    SubgraphInfo,
+    GraphManifest,
+} from "@/explorer-component/components/nodes/types";
+import { resolveIrohCollections } from "./resolveIrohBlobs";
+import { stripCidPrefix } from "./stripPrefix.ts";
+
+import { layoutNodes as layoutNodes } from "./layoutGraph.ts";
+import { sortStatements, indexStatements } from "./statementHelpers.ts";
+import {
+    Anchor,
+    ComputationStatement,
+    ComputationStatementFromJSON,
+    DataStatement,
+    DataStatementFromJSON,
+    VcStatement,
+    RegistrationTypes,
+    GovernanceStatement,
+    SigstoreBundle,
+} from "@/explorer-component/integrityTypes/types.ts";
+import { ExplorerConfig } from "./config.ts";
+
+/// Iterate over graphs and their statements to create the ReactFlow nodes and edges
+/// Handles the new manifest format with graphs containing statements
+export function generateGraphFromGraphs(
+    config: ExplorerConfig,
+    manifest: GraphManifest,
+    customIcons?: Record<string, string>,
+): [LineageNode[], LineageEdge[]] {
+    const allStatements: Record<string, object> = {};
+
+    for (const graph of manifest.graphs) {
+        for (const statement of graph.statements) {
+            const statementId = (statement as any)["@id"];
+            if (statementId) {
+                const graphInfo: SubgraphInfo = {
+                    name: graph.name,
+                    id: graph.id,
+                    parent: graph.parent,
+                };
+                (statement as any)["graph"] = graphInfo;
+                allStatements[statementId] = statement;
+            }
+        }
+    }
+
+    // Use the existing function with the flattened statements
+    return generateGraphFromStatements(config, allStatements, manifest.blobs, manifest.anchors, customIcons);
+}
+
+/// Iterate over all the statements to create the ReactFlow nodes and edges
+/// related info (VCs, metadata, etc) is attached to the ReactFlow node
+export function generateGraphFromStatements(
+    config: ExplorerConfig,
+    statements: Record<string, object>,
+    blobs: Record<string, string>,
+    anchors?: Anchor[],
+    customIcons?: Record<string, string>,
+): [LineageNode[], LineageEdge[]] {
+    // The Nodes to diplay in the graph
+    const nodesMap: Map<string, LineageNode> = new Map();
+    // The edges in the graph
+    let edgesMap: LineageEdge[] = [];
+
+    const irohCollections = resolveIrohCollections(blobs);
+
+    const sortedStatements = sortStatements(statements);
+
+    // Map of metadata that has already been resolved. Key is metadataSubjectID.
+    const { metadataLineage, vcs, actors, storage, governance, sigstoreBundles } = indexStatements(
+        sortedStatements,
+        blobs,
+    );
+
+    for (const [cid, statement] of sortedStatements) {
+        try {
+            // @ts-ignore
+            switch (statement["@type"]) {
+                case RegistrationTypes.Computation: {
+                    const computeStatement: ComputationStatement = ComputationStatementFromJSON(statement);
+
+                    const [computeNodes, computeEdges] = createComputeNode(
+                        config,
+                        computeStatement,
+                        metadataLineage,
+                        vcs,
+                        actors,
+                        irohCollections,
+                        anchors,
+                        customIcons,
+                        governance,
+                        (statement as any)["graph"],
+                    );
+                    edgesMap = [...edgesMap, ...computeEdges];
+                    for (const node of computeNodes) {
+                        if (!nodesMap.has(node.id)) {
+                            // fail safe to create the data node for inputs or outputs that don't have a data registration statement
+                            nodesMap.set(node.id, node);
+                        }
+                    }
+                    break;
+                }
+                case RegistrationTypes.Data: {
+                    const dataStatement: DataStatement = DataStatementFromJSON(statement);
+
+                    if (nodesMap.has(dataStatement.data)) {
+                        const registeredByActor = actors.get(dataStatement.registeredBy);
+                        const vcStatement = vcs.get(dataStatement.id);
+                        const anchor = anchors?.find((anchor) => anchor.statements.includes(dataStatement.id));
+
+                        const registration: DataNodeRegistration = {
+                            statement: dataStatement,
+                            registeredByActor,
+                            anchors: anchor,
+                            vcs: vcStatement,
+                        };
+
+                        const existingNode = nodesMap.get(dataStatement.data);
+                        const sidebarData = existingNode?.data.sidebarData as DataNodeSidebarData;
+                        if (sidebarData && "registrations" in sidebarData) {
+                            // Node already has a DataRegistration statement. Append this one to that list.
+                            const existingRegistrations = sidebarData.registrations ?? [];
+                            sidebarData.registrations = [...existingRegistrations, registration];
+                        }
+                    } else {
+                        const irohCollection = irohCollections.get(stripCidPrefix(dataStatement.data)) || null;
+                        const sourceData = irohCollection ? irohCollection : blobs[stripCidPrefix(dataStatement.data)];
+
+                        const dataNode = createDataNode(
+                            config,
+                            dataStatement,
+                            metadataLineage,
+                            vcs,
+                            actors,
+                            storage,
+                            sourceData,
+                            anchors,
+                            customIcons,
+                            governance,
+                            sigstoreBundles,
+                            (statement as any)["graph"],
+                        );
+                        nodesMap.set(dataNode.id, dataNode);
+                    }
+                    break;
+                }
+                default:
+                    break;
+            }
+        } catch (error) {
+            console.error(`Failed to parse statement for ID ${cid}:`, error);
+        }
+    }
+
+    const nodes: LineageNode[] = Array.from(nodesMap.values());
+    return layoutNodes(nodes, edgesMap);
+}
+
+/// Creates a single data node. If the nodeID already exists, it is updated with this DataStatement data.
+function createDataNode(
+    config: ExplorerConfig,
+    statement: DataStatement,
+    // metadataJsonMap: Map<string, any[]>,
+    metadataLineage: Map<string, MetadataLineage[]>,
+    vcs: Map<string, VcStatement[]>,
+    actors: Map<string, any>,
+    storage: Map<string, StorageLineage>,
+    sourceData?: IrohData[] | string,
+    anchors?: Anchor[],
+    customIcons?: Record<string, string>,
+    governance?: Map<string, GovernanceStatement>,
+    sigstoreBundles?: Map<string, SigstoreBundle[]>,
+    graph?: Subgraph,
+): LineageNode {
+    // Use the first metadata info for the node display
+    const metadata = metadataLineage.get(statement.data)?.[0].metadata as { assetType: string; name: string };
+    const nodeType = metadata["assetType"];
+    const nodeLabel = metadata["name"] ? metadata["name"] : statement.data.slice(-6);
+
+    const nodeConfig = config.nodes.get(nodeType) || config.fallbackConfig;
+    if (nodeConfig == config.fallbackConfig && nodeType) {
+        nodeConfig.displayType = nodeType;
+    }
+    const node = createDefaultNode(statement.data, nodeConfig, nodeLabel, customIcons, graph);
+
+    const registeredByActor = actors.get(statement.registeredBy);
+    const vcStatement = vcs.get(statement.id);
+    const anchor = anchors?.find((anchor) => anchor.statements.includes(statement.id));
+
+    const registration: DataNodeRegistration = { statement, registeredByActor, anchors: anchor, vcs: vcStatement };
+    const sidebarData: DataNodeSidebarData = {
+        registrations: [registration],
+        metadataLineage: metadataLineage.get(statement.data),
+        sourceData: sourceData,
+        storage: storage.get(statement.data),
+        governance: governance?.get(statement.data),
+        sigstoreBundles: sigstoreBundles?.get(statement.data),
+    };
+
+    node.data.sidebarData = sidebarData;
+    return node;
+}
+
+/// Creates a Compute node (and all edges) and generic Data nodes for all the inputs and outputs.
+/// Returns the created Nodes and Edges
+function createComputeNode(
+    config: ExplorerConfig,
+    statement: ComputationStatement,
+    metadataLineage: Map<string, MetadataLineage[]>,
+    vcs: Map<string, VcStatement[]>,
+    actors: Map<string, any>,
+    irohCollections: Map<string, IrohData[]>,
+    anchors?: Anchor[],
+    customIcons?: Record<string, string>,
+    governance?: Map<string, GovernanceStatement>,
+    graph?: Subgraph,
+): [LineageNode[], LineageEdge[]] {
+    const nodeConfig = config.nodes.get("Computation") || config.fallbackConfig;
+    const nodes: LineageNode[] = [];
+    const computeNode = createDefaultNode(statement.id, nodeConfig, "compute", customIcons, graph);
+    const sourceData = statement.computation
+        ? irohCollections.get(stripCidPrefix(statement.computation)) || undefined
+        : undefined;
+
+    const sideBarData: ComputeNodeSidebarData = {
+        statement: statement,
+        metadataLineage: metadataLineage.get(statement.id), // can only be one metadata for a compute
+        vcStatement: vcs.get(statement.id),
+        registeredByActor: actors.get(statement.registeredBy),
+        executedOnActor: statement.executedOn && actors.get(statement.executedOn),
+        operatedByActor: statement.operatedBy && actors.get(statement.operatedBy),
+        sourceData: sourceData,
+        anchor: anchors?.find((anchor) => anchor.statements.includes(statement.id)),
+        governance: governance?.get(statement.id),
+    };
+    computeNode.data.sidebarData = sideBarData;
+
+    nodes.push(computeNode);
+
+    const inputs = Array.isArray(statement.input) ? statement.input : [statement.input];
+    const outputs = Array.isArray(statement.output) ? statement.output : [statement.output];
+
+    const defaultDataNodeConfig = config.nodes.get("Data") || config.fallbackConfig;
+    const inputEdges: LineageEdge[] = inputs.map((input: string) => {
+        // Create default input nodes in case they don't have a DataRegistration
+        const inputNode = createDefaultNode(input, defaultDataNodeConfig, "data");
+        nodes.push(inputNode);
+        return createEdge(input, statement.id);
+    });
+
+    const outputEdges = outputs.map((output: string) => {
+        const outputNode = createDefaultNode(output, defaultDataNodeConfig, "data");
+        nodes.push(outputNode);
+        return createEdge(statement.id, output);
+    });
+
+    const edges = [...inputEdges, ...outputEdges];
+
+    return [nodes, edges];
+}

package/src/explorer-component/utils/layoutGraph.ts

@@ -0,0 +1,148 @@
+import { LineageEdge, LineageNode } from "../components/nodes/types";
+import dagre from "@dagrejs/dagre";
+
+/// Positions all the nodes and edges
+export const layoutNodes = (nodes: LineageNode[], edges: LineageEdge[]): [LineageNode[], LineageEdge[]] => {
+    const verticalGapBetweenNodes = 30;
+    const horizontalGapBetweenColumns = 180;
+
+    const dagreGraph = initializeGraph(nodes, edges);
+
+    // Map of column number and Nodes in that column
+    const columnMap = getNodesByColumn(dagreGraph, nodes);
+
+    const { columnHeights, columnWidths, maxHeight } = sizeGraphColumns(dagreGraph, columnMap, verticalGapBetweenNodes);
+
+    const positionedNodes = setNodePositions(
+        dagreGraph,
+        columnMap,
+        columnHeights,
+        columnWidths,
+        verticalGapBetweenNodes,
+        horizontalGapBetweenColumns,
+        maxHeight,
+    );
+    return [positionedNodes, edges];
+};
+
+// Initializes the dagre Graph with all the nodes and edges
+function initializeGraph(nodes: LineageNode[], edges: LineageEdge[]): dagre.graphlib.Graph {
+    const dagreGraph = new dagre.graphlib.Graph().setDefaultEdgeLabel(() => ({}));
+
+    const direction = "LR";
+    dagreGraph.setGraph({
+        rankdir: direction,
+        nodesep: 30,
+        ranksep: 300,
+        align: "UL",
+    });
+
+    dagreGraph.setGraph({ rankdir: direction });
+
+    for (const node of nodes) {
+        dagreGraph.setNode(node.id, {
+            ...node.data.size,
+        });
+    }
+
+    edges.forEach((edge) => {
+        dagreGraph.setEdge(edge.source, edge.target);
+    });
+
+    dagre.layout(dagreGraph);
+
+    return dagreGraph;
+}
+
+// Returns a MAP of each "column" (aka Rank) and the nodes in the column
+function getNodesByColumn(graph: dagre.graphlib.Graph, nodes: LineageNode[]): Map<number, LineageNode[]> {
+    const columnMap = new Map<number, LineageNode[]>();
+    for (const node of nodes) {
+        const dNode = graph.node(node.id) as { rank: number } | undefined;
+
+        if (dNode && typeof dNode.rank === "number") {
+            const rankNodes = columnMap.get(dNode.rank) ?? [];
+            rankNodes.push(node);
+            columnMap.set(dNode.rank, rankNodes);
+        }
+    }
+
+    return columnMap;
+}
+
+// Calculates the column heights and width for each column in the graph based off the number and size of the nodes in each column
+// Also returns the height of the tallest column
+function sizeGraphColumns(
+    graph: dagre.graphlib.Graph,
+    columnMap: Map<number, LineageNode[]>,
+    verticalGapBetweenNodes: number,
+): { columnHeights: Map<number, number>; columnWidths: Map<number, number>; maxHeight: number } {
+    const columnHeights = new Map<number, number>();
+    const columnWidths = new Map<number, number>();
+    let maxHeight = 0;
+    for (const [rank, nodes] of columnMap) {
+        nodes.sort((a, b) => {
+            const da = graph.node(a.id);
+            const db = graph.node(b.id);
+            return da.y - db.y;
+        });
+        let rankHeight = 0;
+        let maxWidth = 0;
+        for (const node of nodes) {
+            const graphNode = graph.node(node.id);
+            rankHeight += graphNode.height;
+            maxWidth = Math.max(maxWidth, graphNode.width);
+        }
+        rankHeight += (nodes.length - 1) * verticalGapBetweenNodes;
+
+        maxHeight = Math.max(maxHeight, rankHeight);
+        columnHeights.set(rank, rankHeight);
+        columnWidths.set(rank, maxWidth);
+    }
+    return { columnHeights: columnHeights, columnWidths: columnWidths, maxHeight };
+}
+
+// Sets the position of each node in the graph, and returns the nodes with updated positions
+function setNodePositions(
+    graph: dagre.graphlib.Graph,
+    columnMap: Map<number, LineageNode[]>,
+    columnHeights: Map<number, number>,
+    columnWidths: Map<number, number>,
+    verticalGapBetweenNodes: number,
+    horizontalGapBetweenNodes: number,
+    maxColumnHeight: number,
+): LineageNode[] {
+    const positionedNodes: LineageNode[] = [];
+    let xPosition = 0;
+    const rankEntries = Array.from(columnMap.entries());
+    rankEntries.sort((a, b) => {
+        return a[0] - b[0];
+    });
+
+    for (const [rank, nodes] of rankEntries) {
+        const rankHeight = columnHeights.get(rank) || 0;
+        const rankWidth = columnWidths.get(rank) || 0;
+        const rankOffset = (maxColumnHeight - rankHeight) / 2;
+        let yPosition = 0;
+
+        for (const [, node] of nodes.entries()) {
+            const graphNode = graph.node(node.id);
+            node.position = {
+                x: xPosition + rankWidth / 2 - graphNode.width / 2,
+                y: yPosition + rankOffset,
+            };
+
+            const predecessors = graph.predecessors(node.id);
+            node.data.hasPrecedingNode = (predecessors && predecessors.length > 0) || false;
+
+            const successors = graph.successors(node.id);
+            node.data.hasSuccessorNode = (successors && successors.length > 0) || false;
+
+            positionedNodes.push(node);
+            yPosition += graphNode.height + verticalGapBetweenNodes;
+        }
+        xPosition += rankWidth + horizontalGapBetweenNodes;
+    }
+
+    return positionedNodes;
+}

package/src/explorer-component/utils/resolveIrohBlobs.ts

@@ -0,0 +1,117 @@
+import { CID } from "multiformats/cid";
+import { create as createMultihash } from "multiformats/hashes/digest";
+import { IrohData } from "../components/nodes/types";
+
+const RAW_BINARY = 0x55;
+const BLAKE3_HASHSEQ = 0x80;
+const BLAKE3_256 = 0x1e;
+
+export function getIrohMetaCid(codec: number, collectionBlob: Uint8Array): string {
+    const first32 = collectionBlob.slice(0, 32);
+    const multihash = createMultihash(BLAKE3_256, first32); // 0x1e == BLAKE3-256 multihash code
+    const cid = CID.createV1(codec, multihash);
+    return cid.toString();
+}
+
+export function resolveIrohCollections(blobs: Record<string, string>): Map<string, IrohData[]> {
+    // Map<cid, bytes> of possible iroh collections from the Blob
+    const irohCandidateBlobs: Record<string, Uint8Array> = {};
+    for (const [cid, blob] of Object.entries(blobs)) {
+        try {
+            const decoded = Uint8Array.from(atob(blob), (c) => c.charCodeAt(0));
+
+            if (decoded.length >= 64) {
+                irohCandidateBlobs[cid] = decoded;
+            } else {
+                //console.log(`Skipping blob '${cid}' — too short (${decoded.length} bytes)`);
+            }
+        } catch (e) {
+            console.error(`Failed to decode blob for CID '${cid}':`, e);
+            throw e;
+        }
+    }
+
+    const irohCollections = new Map();
+
+    for (const [cid, blob] of Object.entries(irohCandidateBlobs)) {
+        const metaCidLegacy = getIrohMetaCid(RAW_BINARY, blob);
+        const meta_cid = getIrohMetaCid(BLAKE3_HASHSEQ, blob);
+        if (metaCidLegacy in blobs) {
+            const metaBlob = blobs[metaCidLegacy];
+            const collection_blob = blobs[cid];
+            const nameCidMap = extractCollectionNames(blobs, metaBlob, collection_blob);
+            irohCollections.set(cid, nameCidMap);
+        } else if (meta_cid in blobs) {
+            const metaBlob = blobs[metaCidLegacy];
+            const collection_blob = blobs[cid];
+            const nameCidMap = extractCollectionNames(blobs, metaBlob, collection_blob);
+            irohCollections.set(cid, nameCidMap);
+        }
+    }
+
+    return irohCollections;
+}
+
+/**
+ * Decodes a base64 collection blob and extracts file names after CollectionV0. or CollectionV1.
+ * Handles both versions and multiple names.
+ * @param metaBlob - The base64-encoded meta blob that has the names of the items in the collection.
+ * @param collectionBlob - The base64-encoded collection blob of the iroh collection.
+ * @returns Map of FileName -> CID in the iroh collection.
+ */
+function extractCollectionNames(blobs: Record<string, string>, metaBlob: string, collectionBlob: string): IrohData[] {
+    // collection blob contains the hash sequence.
+    const collectionBlobStr = atob(collectionBlob);
+    const hashSeq = Uint8Array.from(collectionBlobStr, (c) => c.charCodeAt(0));
+
+    const hashes = parseHashSeq(hashSeq).slice(1); // drop meta hash
+
+    // meta blob is encoded with `postcard` encoding, which is a length-prefixed format.
+    const names = [];
+
+    const decodedMetaBlob = atob(metaBlob);
+    const metaBytes = Uint8Array.from(decodedMetaBlob, (c) => c.charCodeAt(0));
+
+    const headerLen = 13; // fixed length u8 array, no length prefix
+    const namesBytes = metaBytes.slice(headerLen); // drop header
+
+    let unprocessedNamesBytes = namesBytes.slice(1); // drop array length byte
+    while (unprocessedNamesBytes.length > 0) {
+        const nameLen = unprocessedNamesBytes[0];
+        unprocessedNamesBytes = unprocessedNamesBytes.slice(1); // drop length byte
+
+        const nameBytes = unprocessedNamesBytes.slice(0, nameLen);
+        unprocessedNamesBytes = unprocessedNamesBytes.slice(nameLen); // drop name bytes
+
+        const name = new TextDecoder().decode(nameBytes);
+        names.push(name);
+    }
+
+    if (names.length !== hashes.length) {
+        console.error(`Names and hashes length mismatch: ${names.length} != ${hashes.length}. Names: '${names}'`);
+        return [];
+    }
+
+    const irohDataList: IrohData[] = [];
+    for (let i = 0; i < names.length; i++) {
+        const blob = blobs[hashes[i]];
+        const data = blob ? atob(blob) : null;
+        irohDataList.push({
+            path: names[i],
+            cid: hashes[i],
+            data: data,
+        });
+    }
+    return irohDataList;
+}
+
+function parseHashSeq(bytes: Uint8Array): string[] {
+    if (bytes.length % 32 !== 0) throw new Error("Invalid hash sequence length");
+    const hashes: string[] = [];
+    for (let i = 0; i < bytes.length; i += 32) {
+        const chunk = bytes.slice(i, i + 32);
+        const hashCid = getIrohMetaCid(RAW_BINARY, chunk);
+        hashes.push(hashCid);
+    }
+    return hashes;
+}