@eventcatalog/core 3.7.1 → 3.8.0

package/eventcatalog/src/utils/node-graphs/data-products-node-graph.ts

@@ -0,0 +1,225 @@
+import { getCollection, type CollectionEntry } from 'astro:content';
+import dagre from 'dagre';
+import {
+  createDagreGraph,
+  generateIdForNode,
+  generatedIdForEdge,
+  calculatedNodes,
+  createEdge,
+  getColorFromString,
+} from '@utils/node-graphs/utils/utils';
+
+import { findInMap, createVersionedMap, mergeMaps, collectionToResourceMap } from '@utils/collections/util';
+import { MarkerType } from '@xyflow/react';
+import { getMessages, isCollectionAMessage } from '@utils/collections/messages';
+import { getProducersOfMessage } from '@utils/collections/services';
+import type { CollectionMessageTypes } from '@types';
+import { getNodesAndEdgesForProducedMessage } from './message-node-graph';
+
+type DagreGraph = any;
+
+interface Props {
+  id: string;
+  version: string;
+  defaultFlow?: DagreGraph;
+  mode?: 'simple' | 'full';
+}
+
+const getNodePropertyFromCollectionType = (type: string) => {
+  if (isCollectionAMessage(type)) return 'message';
+  if (type === 'containers') return 'data';
+  return collectionToResourceMap[type as keyof typeof collectionToResourceMap];
+};
+
+export const getNodesAndEdges = async ({ id, defaultFlow, version, mode = 'simple' }: Props) => {
+  const flow = defaultFlow || createDagreGraph({ ranksep: 300, nodesep: 50 });
+  let nodes = [] as any,
+    edges = [] as any;
+
+  const [dataProducts, containers, services, events, queries, commands, channels] = await Promise.all([
+    getCollection('data-products'),
+    getCollection('containers'),
+    getCollection('services'),
+    getCollection('events'),
+    getCollection('queries'),
+    getCollection('commands'),
+    getCollection('channels'),
+  ]);
+
+  const dataProduct = dataProducts.find((dp) => dp.data.id === id && dp.data.version === version);
+
+  // Nothing found...
+  if (!dataProduct) {
+    return {
+      nodes: [],
+      edges: [],
+    };
+  }
+
+  // Build maps for O(1) lookups
+  const messages = [...events, ...commands, ...queries];
+
+  const messageMap = createVersionedMap(messages);
+  const containerMap = createVersionedMap(containers);
+  const serviceMap = createVersionedMap(services);
+  const channelMap = createVersionedMap(channels);
+
+  const inputsRaw = dataProduct?.data.inputs || [];
+  const outputsRaw = dataProduct?.data.outputs || [];
+
+  const resourceMap = mergeMaps<
+    | CollectionEntry<CollectionMessageTypes>
+    | CollectionEntry<'services'>
+    | CollectionEntry<'containers'>
+    | CollectionEntry<'channels'>
+  >(messageMap, serviceMap, containerMap, channelMap);
+
+  // Process inputs - messages, containers, services, channels (etc)
+  inputsRaw.forEach((inputConfig) => {
+    let inputResource = findInMap(resourceMap, inputConfig.id, inputConfig.version) as
+      | CollectionEntry<CollectionMessageTypes>
+      | CollectionEntry<'services'>
+      | CollectionEntry<'containers'>
+      | CollectionEntry<'channels'>;
+
+    const existingNode = nodes.find((n: any) => n.id === generateIdForNode(inputResource));
+
+    if (!existingNode) {
+      const nodeDataKey = getNodePropertyFromCollectionType(inputResource?.collection);
+
+      nodes.push({
+        id: generateIdForNode(inputResource),
+        sourcePosition: 'right',
+        targetPosition: 'left',
+        data: { mode, [nodeDataKey]: { ...inputResource?.data } },
+        type: (inputResource?.collection as any) === 'containers' ? 'data' : inputResource?.collection,
+      });
+    }
+
+    // If collection is a message we render the producers of the message
+    if (isCollectionAMessage(inputResource?.collection)) {
+      const producersOfMessage = getProducersOfMessage(
+        services as CollectionEntry<'services'>[],
+        inputResource as CollectionEntry<CollectionMessageTypes>
+      ) as CollectionEntry<'services'>[];
+      for (const producer of producersOfMessage) {
+        const { nodes: producerNodes, edges: producerEdges } = getNodesAndEdgesForProducedMessage({
+          message: inputResource as CollectionEntry<CollectionMessageTypes>,
+          // We dont render any other services that consume this event for now
+          services: [],
+          // We dont render channels on this view for now...
+          channels: [],
+          currentNodes: nodes,
+          currentEdges: edges,
+          source: producer,
+          mode,
+        });
+
+        nodes.push(...producerNodes);
+        edges.push(...producerEdges);
+      }
+    }
+
+    // Add edge from resource to data product
+    edges.push(
+      createEdge({
+        id: generatedIdForEdge(inputResource, dataProduct),
+        source: generateIdForNode(inputResource),
+        target: generateIdForNode(dataProduct),
+        label: 'input',
+        type: 'animated',
+        data: {
+          customColor: getColorFromString(inputResource.data.id),
+          rootSourceAndTarget: { source: inputResource, target: dataProduct },
+        },
+        markerEnd: {
+          type: MarkerType.ArrowClosed,
+          color: '#666',
+          width: 40,
+          height: 40,
+        },
+      })
+    );
+  });
+
+  // The data product itself
+  nodes.push({
+    id: generateIdForNode(dataProduct),
+    sourcePosition: 'right',
+    targetPosition: 'left',
+    data: { mode, dataProduct: { ...dataProduct.data } },
+    type: 'data-products',
+  });
+
+  // Process outputs - messages, services, containers, channels that the data product produces
+  outputsRaw.forEach((outputConfig) => {
+    // Find the output resource (can be message, service, container, or channel)
+    const outputResource = findInMap(resourceMap, outputConfig.id, outputConfig.version) as
+      | CollectionEntry<CollectionMessageTypes>
+      | CollectionEntry<'services'>
+      | CollectionEntry<'containers'>
+      | CollectionEntry<'channels'>;
+
+    if (!outputResource) return;
+
+    // Add the node if it doesn't exist
+    const existingNode = nodes.find((n: any) => n.id === generateIdForNode(outputResource));
+    if (!existingNode) {
+      const nodeDataKey = getNodePropertyFromCollectionType(outputResource?.collection);
+
+      nodes.push({
+        id: generateIdForNode(outputResource),
+        sourcePosition: 'right',
+        targetPosition: 'left',
+        data: { mode, [nodeDataKey]: { ...outputResource?.data } },
+        type: (outputResource?.collection as any) === 'containers' ? 'data' : outputResource?.collection,
+      });
+    }
+
+    // Add edge from data product to the output resource
+    edges.push(
+      createEdge({
+        id: generatedIdForEdge(dataProduct, outputResource),
+        source: generateIdForNode(dataProduct),
+        target: generateIdForNode(outputResource),
+        label: 'output',
+        type: 'animated',
+        data: {
+          customColor: getColorFromString(outputResource.data.id),
+          rootSourceAndTarget: { source: dataProduct, target: outputResource },
+        },
+        markerEnd: {
+          type: MarkerType.ArrowClosed,
+          color: '#666',
+          width: 40,
+          height: 40,
+        },
+      })
+    );
+  });
+
+  nodes.forEach((node: any) => {
+    flow.setNode(node.id, { width: 150, height: 100 });
+  });
+
+  edges.forEach((edge: any) => {
+    flow.setEdge(edge.source, edge.target);
+  });
+
+  // Render the diagram in memory getting the X and Y
+  dagre.layout(flow);
+
+  // Find any duplicated edges, and merge them into one edge
+  const uniqueEdges = edges.reduce((acc: any[], edge: any) => {
+    const existingEdge = acc.find((e: any) => e.id === edge.id);
+    if (!existingEdge) {
+      acc.push(edge);
+    }
+    return acc;
+  }, []);
+
+  return {
+    nodes: calculatedNodes(flow, nodes),
+    edges: uniqueEdges,
+  };
+};

package/eventcatalog/src/utils/node-graphs/domains-node-graph.ts

@@ -8,6 +8,7 @@ import {
   createEdge,
 } from '@utils/node-graphs/utils/utils';
 import { getNodesAndEdges as getServicesNodeAndEdges } from './services-node-graph';
+import { getNodesAndEdges as getDataProductsNodeAndEdges } from './data-products-node-graph';
 import merge from 'lodash.merge';
 import { createVersionedMap, findInMap } from '@utils/collections/util';
 import type { Node } from '@xyflow/react';
@@ -199,7 +200,11 @@ export const getNodesAndEdges = async ({
     edges = new Map();
 
   // 1. Parallel Fetching
-  const [domains, services] = await Promise.all([getCollection('domains'), getCollection('services')]);
+  const [domains, services, dataProducts] = await Promise.all([
+    getCollection('domains'),
+    getCollection('services'),
+    getCollection('data-products'),
+  ]);
 
   const domain = domains.find((service) => service.data.id === id && service.data.version === version);
 
@@ -214,9 +219,11 @@ export const getNodesAndEdges = async ({
   // 2. Build optimized maps
   const serviceMap = createVersionedMap(services);
   const domainMap = createVersionedMap(domains);
+  const dataProductMap = createVersionedMap(dataProducts);
 
   const rawServices = domain?.data.services || [];
   const rawSubDomains = domain?.data.domains || [];
+  const rawDataProducts = (domain?.data as any)['data-products'] || [];
 
   // Optimized hydration
   const domainServicesWithVersion = rawServices
@@ -229,7 +236,12 @@ export const getNodesAndEdges = async ({
     .filter((d): d is any => !!d)
     .map((svc) => ({ id: svc.data.id, version: svc.data.version }));
 
-  // Get all the nodes for everyhing
+  const domainDataProductsWithVersion = rawDataProducts
+    .map((dataProduct: any) => findInMap(dataProductMap, dataProduct.id, dataProduct.version))
+    .filter((dp: any): dp is any => !!dp)
+    .map((dp: any) => ({ id: dp.data.id, version: dp.data.version }));
+
+  // Get all the nodes for everything
 
   for (const service of domainServicesWithVersion) {
     const { nodes: serviceNodes, edges: serviceEdges } = await getServicesNodeAndEdges({
@@ -255,6 +267,20 @@ export const getNodesAndEdges = async ({
     serviceEdges.forEach((e) => edges.set(e.id, e));
   }
 
+  for (const dataProduct of domainDataProductsWithVersion) {
+    const { nodes: dataProductNodes, edges: dataProductEdges } = await getDataProductsNodeAndEdges({
+      id: dataProduct.id,
+      version: dataProduct.version,
+      defaultFlow: flow,
+      mode,
+    });
+    dataProductNodes.forEach((n: any) => {
+      nodes.set(n.id, nodes.has(n.id) ? merge(nodes.get(n.id), n) : n);
+    });
+    // @ts-ignore
+    dataProductEdges.forEach((e) => edges.set(e.id, e));
+  }
+
   for (const subDomain of domainSubDomainsWithVersion) {
     const { nodes: subDomainNodes, edges: subDomainEdges } = await getNodesAndEdges({
       id: subDomain.id,

package/eventcatalog/src/utils/node-graphs/message-node-graph.ts

@@ -84,25 +84,55 @@ const getNodesAndEdges = async ({
     type: message.collection,
   });
 
-  const producers = (message.data.producers as CollectionEntry<'services'>[]) || [];
-  const consumers = (message.data.consumers as CollectionEntry<'services'>[]) || [];
+  const producers = (message.data.producers as (CollectionEntry<'services'> | CollectionEntry<'data-products'>)[]) || [];
+  const consumers = (message.data.consumers as (CollectionEntry<'services'> | CollectionEntry<'data-products'>)[]) || [];
 
-  // Track nodes that are both sent and received
-  const bothSentAndReceived = findMatchingNodes(producers, consumers);
+  // Track nodes that are both sent and received (only for services)
+  const serviceProducers = producers.filter((p) => p.collection === 'services') as CollectionEntry<'services'>[];
+  const serviceConsumers = consumers.filter((c) => c.collection === 'services') as CollectionEntry<'services'>[];
+  const bothSentAndReceived = findMatchingNodes(serviceProducers, serviceConsumers);
 
   for (const producer of producers) {
-    // Create the producer node
+    const isDataProduct = producer.collection === 'data-products';
+
+    // Create the producer node with appropriate data structure
     nodes.push({
       id: generateIdForNode(producer),
-      type: producer?.collection,
+      type: isDataProduct ? 'data-products' : producer?.collection,
       sourcePosition: 'right',
       targetPosition: 'left',
-      data: { mode, service: { ...producer.data } },
+      data: isDataProduct ? { mode, dataProduct: { ...producer.data } } : { mode, service: { ...producer.data } },
       position: { x: 250, y: 0 },
     });
 
+    // Data products don't have channel configuration, so connect directly to the message
+    if (isDataProduct) {
+      const rootSourceAndTarget = {
+        source: { id: generateIdForNode(producer), collection: producer.collection },
+        target: { id: generateIdForNode(message), collection: message.collection },
+      };
+
+      edges.push({
+        id: generatedIdForEdge(producer, message),
+        source: generateIdForNode(producer),
+        target: generateIdForNode(message),
+        label: 'produces',
+        data: { customColor: getColorFromString(message.data.id), rootSourceAndTarget },
+        animated: false,
+        markerEnd: {
+          type: MarkerType.ArrowClosed,
+          width: 40,
+          height: 40,
+        },
+      });
+      continue;
+    }
+
+    // Service-specific channel handling
+    const serviceProducer = producer as CollectionEntry<'services'>;
+
     // Is the producer sending this message to a channel?
-    const producerConfigurationForMessage = producer.data.sends?.find((send) => send.id === message.data.id);
+    const producerConfigurationForMessage = serviceProducer.data.sends?.find((send) => send.id === message.data.id);
     const producerChannelConfiguration = producerConfigurationForMessage?.to ?? [];
 
     const producerHasChannels = producerChannelConfiguration?.length > 0;
@@ -184,18 +214,44 @@ const getNodesAndEdges = async ({
 
   // The messages the service sends
   for (const consumer of consumers) {
-    // Render the consumer node
+    const isDataProduct = consumer.collection === 'data-products';
+
+    // Render the consumer node with appropriate data structure
     nodes.push({
       id: generateIdForNode(consumer),
       sourcePosition: 'right',
       targetPosition: 'left',
-      data: { title: consumer?.data.id, mode, service: { ...consumer.data } },
+      data: isDataProduct
+        ? { title: consumer?.data.id, mode, dataProduct: { ...consumer.data } }
+        : { title: consumer?.data.id, mode, service: { ...consumer.data } },
       position: { x: 0, y: 0 },
-      type: consumer?.collection,
+      type: isDataProduct ? 'data-products' : consumer?.collection,
     });
 
+    // Data products don't have channel configuration, so connect directly from the message
+    if (isDataProduct) {
+      const rootSourceAndTarget = {
+        source: { id: generateIdForNode(message), collection: message.collection },
+        target: { id: generateIdForNode(consumer), collection: consumer.collection },
+      };
+
+      edges.push(
+        createEdge({
+          id: generatedIdForEdge(message, consumer),
+          source: generateIdForNode(message),
+          target: generateIdForNode(consumer),
+          label: 'consumed by',
+          data: { customColor: getColorFromString(message.data.id), rootSourceAndTarget },
+        })
+      );
+      continue;
+    }
+
+    // Service-specific channel handling
+    const serviceConsumer = consumer as CollectionEntry<'services'>;
+
     // Is the consumer receiving this message from a channel?
-    const consumerConfigurationForMessage = consumer.data.receives?.find((receive) => receive.id === message.data.id);
+    const consumerConfigurationForMessage = serviceConsumer.data.receives?.find((receive) => receive.id === message.data.id);
     const consumerChannelConfiguration = consumerConfigurationForMessage?.from ?? [];
 
     const consumerHasChannels = consumerChannelConfiguration.length > 0;
@@ -237,10 +293,11 @@ const getNodesAndEdges = async ({
       }
 
       // Can any of the consumer channels be linked to any of the producer channels?
-      const producerChannels = producers
+      // Only consider service producers for channel linking (data products don't have sends/receives)
+      const producerChannels = serviceProducers
         .map((producer) => producer.data.sends?.find((send) => send.id === message.data.id)?.to ?? [])
         .flat();
-      const consumerChannels = consumer.data.receives?.find((receive) => receive.id === message.data.id)?.from ?? [];
+      const consumerChannels = serviceConsumer.data.receives?.find((receive) => receive.id === message.data.id)?.from ?? [];
 
       for (const producerChannel of producerChannels) {
         const producerChannelValue = findInMap(
@@ -376,8 +433,7 @@ export const getNodesAndEdgesForQueries = async ({
   mode = 'simple',
   channelRenderMode = 'flat',
 }: Props) => {
-  const queries = await getQueries();
-  const channels = await getChannels();
+  const [queries, channels] = await Promise.all([getQueries(), getChannels()]);
   return getNodesAndEdges({ id, version, defaultFlow, mode, channelRenderMode, collection: queries, channels });
 };
 
@@ -388,8 +444,7 @@ export const getNodesAndEdgesForCommands = async ({
   mode = 'simple',
   channelRenderMode = 'flat',
 }: Props) => {
-  const commands = await getCommands();
-  const channels = await getChannels();
+  const [commands, channels] = await Promise.all([getCommands(), getChannels()]);
   return getNodesAndEdges({ id, version, defaultFlow, mode, channelRenderMode, collection: commands, channels });
 };
 
@@ -400,8 +455,7 @@ export const getNodesAndEdgesForEvents = async ({
   mode = 'simple',
   channelRenderMode = 'flat',
 }: Props) => {
-  const events = await getEvents();
-  const channels = await getChannels();
+  const [events, channels] = await Promise.all([getEvents(), getChannels()]);
   return getNodesAndEdges({ id, version, defaultFlow, mode, channelRenderMode, collection: events, channels });
 };
 

package/eventcatalog/src/utils/page-loaders/page-data-loader.ts

@@ -9,6 +9,7 @@ import { getEntities } from '@utils/collections/entities';
 import { getContainers } from '@utils/collections/containers';
 import { getDiagrams } from '@utils/collections/diagrams';
 import type { CollectionEntry } from 'astro:content';
+import { getDataProducts } from '@utils/collections/data-products';
 
 export const pageDataLoader: Record<PageTypes, () => Promise<CollectionEntry<CollectionTypes>[]>> = {
   events: getEvents,
@@ -21,4 +22,5 @@ export const pageDataLoader: Record<PageTypes, () => Promise<CollectionEntry<Col
   entities: getEntities,
   containers: getContainers,
   diagrams: getDiagrams,
+  'data-products': getDataProducts,
 };

package/package.json

@@ -6,7 +6,7 @@
     "url": "https://github.com/event-catalog/eventcatalog.git"
   },
   "type": "module",
-  "version": "3.7.1",
+  "version": "3.8.0",
   "publishConfig": {
     "access": "public"
   },
@@ -34,7 +34,7 @@
     "@eventcatalog/generator-ai": "^1.1.0",
     "@eventcatalog/license": "^0.0.7",
     "@eventcatalog/linter": "^0.0.2",
-    "@eventcatalog/sdk": "^2.10.0",
+    "@eventcatalog/sdk": "^2.11.0",
     "@eventcatalog/visualizer": "^0.0.6",
     "@fontsource/inter": "^5.2.5",
     "@headlessui/react": "^2.0.3",