@eventcatalog/core 3.7.2 → 3.8.1

package/eventcatalog/src/utils/collections/messages.ts

@@ -3,6 +3,7 @@ import { getCommands } from '@utils/collections/commands';
 import { getEvents } from '@utils/collections/events';
 import { getQueries } from './queries';
 import type { CollectionEntry } from 'astro:content';
+import { satisfies } from './util';
 export { getCommands } from '@utils/collections/commands';
 export { getEvents } from '@utils/collections/events';
 
@@ -11,6 +12,69 @@ interface Props {
   hydrateServices?: boolean;
 }
 
+interface HydrateProducersAndConsumersProps {
+  message: {
+    data: {
+      id: string;
+      version: string;
+      latestVersion?: string;
+    };
+  };
+  services: CollectionEntry<'services'>[];
+  dataProducts: CollectionEntry<'data-products'>[];
+  hydrate?: boolean;
+}
+
+/**
+ * Hydrates producers and consumers for a message (event, command, or query).
+ * Finds services and data products that produce or consume the given message.
+ */
+export const hydrateProducersAndConsumers = ({
+  message,
+  services = [],
+  dataProducts = [],
+  hydrate = true,
+}: HydrateProducersAndConsumersProps) => {
+  const { id: messageId, version: messageVersion, latestVersion = messageVersion } = message.data;
+
+  const matchesVersion = (pointerVersion: string | undefined) => {
+    if (pointerVersion === 'latest' || pointerVersion === undefined) {
+      return messageVersion === latestVersion;
+    }
+    return satisfies(messageVersion, pointerVersion);
+  };
+
+  const toResult = <T extends CollectionEntry<'services'> | CollectionEntry<'data-products'>>(resource: T) => {
+    if (!hydrate) return { id: resource.data.id, version: resource.data.version };
+    return resource;
+  };
+
+  // Services that send this message (producers)
+  const serviceProducers = services
+    .filter((s) => s.data.sends?.some((p) => p.id === messageId && matchesVersion(p.version)))
+    .map(toResult);
+
+  // Services that receive this message (consumers)
+  const serviceConsumers = services
+    .filter((s) => s.data.receives?.some((p) => p.id === messageId && matchesVersion(p.version)))
+    .map(toResult);
+
+  // Data products that output this message (producers)
+  const dataProductProducers = dataProducts
+    .filter((dp) => dp.data.outputs?.some((p) => p.id === messageId && matchesVersion(p.version)))
+    .map(toResult);
+
+  // Data products that input this message (consumers)
+  const dataProductConsumers = dataProducts
+    .filter((dp) => dp.data.inputs?.some((p) => p.id === messageId && matchesVersion(p.version)))
+    .map(toResult);
+
+  return {
+    producers: [...serviceProducers, ...dataProductProducers],
+    consumers: [...serviceConsumers, ...dataProductConsumers],
+  };
+};
+
 type Messages = {
   commands: CollectionEntry<'commands'>[];
   events: CollectionEntry<'events'>[];
@@ -40,3 +104,7 @@ export const getMessages = async ({ getAllVersions = true, hydrateServices = tru
     queries,
   };
 };
+
+export const isCollectionAMessage = (collection: string): boolean => {
+  return ['events', 'commands', 'queries'].includes(collection);
+};

package/eventcatalog/src/utils/collections/queries.ts

@@ -2,7 +2,8 @@ import { getCollection } from 'astro:content';
 import type { CollectionEntry } from 'astro:content';
 import path from 'path';
 import utils from '@eventcatalog/sdk';
-import { createVersionedMap, satisfies } from './util';
+import { createVersionedMap } from './util';
+import { hydrateProducersAndConsumers } from './messages';
 
 const PROJECT_DIR = process.env.PROJECT_DIR || process.cwd();
 const CACHE_ENABLED = process.env.DISABLE_EVENTCATALOG_CACHE !== 'true';
@@ -34,10 +35,11 @@ export const getQueries = async ({ getAllVersions = true, hydrateServices = true
   }
 
   // 1. Fetch collections in parallel
-  const [allQueries, allServices, allChannels] = await Promise.all([
+  const [allQueries, allServices, allChannels, allDataProducts] = await Promise.all([
     getCollection('queries'),
     getCollection('services'),
     getCollection('channels'),
+    getCollection('data-products'),
   ]);
 
   // 2. Build optimized maps
@@ -60,33 +62,13 @@ export const getQueries = async ({ getAllVersions = true, hydrateServices = true
       const latestVersion = queryVersions[0]?.data.version || query.data.version;
       const versions = queryVersions.map((e) => e.data.version);
 
-      // Find Producers (Services that send this query)
-      const producers = allServices
-        .filter((service) =>
-          service.data.sends?.some((item) => {
-            if (item.id !== query.data.id) return false;
-            if (item.version === 'latest' || item.version === undefined) return query.data.version === latestVersion;
-            return satisfies(query.data.version, item.version);
-          })
-        )
-        .map((service) => {
-          if (!hydrateServices) return { id: service.data.id, version: service.data.version };
-          return service;
-        });
-
-      // Find Consumers (Services that receive this query)
-      const consumers = allServices
-        .filter((service) =>
-          service.data.receives?.some((item) => {
-            if (item.id !== query.data.id) return false;
-            if (item.version === 'latest' || item.version === undefined) return query.data.version === latestVersion;
-            return satisfies(query.data.version, item.version);
-          })
-        )
-        .map((service) => {
-          if (!hydrateServices) return { id: service.data.id, version: service.data.version };
-          return service;
-        });
+      // Find producers and consumers (services + data products)
+      const { producers, consumers } = hydrateProducersAndConsumers({
+        message: { data: { ...query.data, latestVersion } },
+        services: allServices,
+        dataProducts: allDataProducts,
+        hydrate: hydrateServices,
+      });
 
       // Find Channels
       const messageChannels = query.data.channels || [];

package/eventcatalog/src/utils/collections/util.ts

@@ -158,8 +158,8 @@ export const getItemsFromCollectionByIdAndSemverOrLatest = <T extends { data: {
 };
 
 export const findMatchingNodes = (
-  nodesA: CollectionEntry<'events' | 'commands' | 'queries' | 'services' | 'containers'>[],
-  nodesB: CollectionEntry<'events' | 'commands' | 'queries' | 'services' | 'containers'>[]
+  nodesA: CollectionEntry<'events' | 'commands' | 'queries' | 'services' | 'containers' | 'data-products'>[],
+  nodesB: CollectionEntry<'events' | 'commands' | 'queries' | 'services' | 'containers' | 'data-products'>[]
 ) => {
   // Track messages that are both sent and received
   return nodesA.filter((nodeA) => {
@@ -182,6 +182,7 @@ export const resourceToCollectionMap = {
   container: 'containers',
   entity: 'entities',
   diagram: 'diagrams',
+  'data-product': 'data-products',
 } as const;
 
 export const collectionToResourceMap = {
@@ -197,6 +198,7 @@ export const collectionToResourceMap = {
   containers: 'container',
   entities: 'entity',
   diagrams: 'diagram',
+  'data-products': 'data-product',
 } as const;
 
 export const getDeprecatedDetails = (item: CollectionEntry<CollectionTypes>) => {
@@ -261,6 +263,13 @@ export const createVersionedMap = <T extends { data: { id: string; version?: str
   return map;
 };
 
+// Merge as many given maps as you want
+export const mergeMaps = <T>(...maps: Map<string, T[]>[]): Map<string, T[]> => {
+  return maps.reduce((acc, map) => {
+    return new Map([...acc, ...map]);
+  }, new Map<string, T[]>());
+};
+
 /**
  * Fast lookup helper.
  * If version is provided, find it. If not, return the first (latest) item.

package/eventcatalog/src/utils/node-graphs/container-node-graph.ts

@@ -37,9 +37,14 @@ export const getNodesAndEdges = async ({ id, version, defaultFlow, mode = 'simpl
 
   const servicesThatWriteToContainer = (container.data.servicesThatWriteToContainer as CollectionEntry<'services'>[]) || [];
   const servicesThatReadFromContainer = (container.data.servicesThatReadFromContainer as CollectionEntry<'services'>[]) || [];
+  const dataProductsThatWriteToContainer =
+    (container.data.dataProductsThatWriteToContainer as CollectionEntry<'data-products'>[]) || [];
+  const dataProductsThatReadFromContainer =
+    (container.data.dataProductsThatReadFromContainer as CollectionEntry<'data-products'>[]) || [];
 
-  // Track nodes that are bth sent and received
+  // Track nodes that are both sent and received
   const bothSentAndReceived = findMatchingNodes(servicesThatWriteToContainer, servicesThatReadFromContainer);
+  const dataProductsBothSentAndReceived = findMatchingNodes(dataProductsThatWriteToContainer, dataProductsThatReadFromContainer);
 
   servicesThatWriteToContainer.forEach((service) => {
     nodes.push({
@@ -67,7 +72,34 @@ export const getNodesAndEdges = async ({ id, version, defaultFlow, mode = 'simpl
     }
   });
 
-  // The message itself
+  // Data products that write to the container
+  dataProductsThatWriteToContainer.forEach((dataProduct) => {
+    nodes.push({
+      id: generateIdForNode(dataProduct),
+      type: 'data-products',
+      sourcePosition: 'right',
+      targetPosition: 'left',
+      data: { mode, dataProduct: { ...dataProduct.data } },
+      position: { x: 250, y: 0 },
+    });
+
+    if (!dataProductsBothSentAndReceived.includes(dataProduct)) {
+      edges.push({
+        id: generatedIdForEdge(dataProduct, container),
+        source: generateIdForNode(dataProduct),
+        target: generateIdForNode(container),
+        label: 'writes to',
+        data: { dataProduct },
+        animated: false,
+        type: 'default',
+        style: {
+          strokeWidth: 1,
+        },
+      });
+    }
+  });
+
+  // The container itself
   nodes.push({
     id: generateIdForNode(container),
     sourcePosition: 'right',
@@ -114,7 +146,38 @@ export const getNodesAndEdges = async ({ id, version, defaultFlow, mode = 'simpl
     }
   });
 
-  // Handle messages that are both sent and received
+  // Data products that read from the container
+  dataProductsThatReadFromContainer.forEach((dataProduct) => {
+    nodes.push({
+      id: generateIdForNode(dataProduct),
+      sourcePosition: 'left',
+      targetPosition: 'right',
+      data: { title: dataProduct?.data.id, mode, dataProduct: { ...dataProduct.data } },
+      position: { x: 0, y: 0 },
+      type: 'data-products',
+    });
+
+    if (!dataProductsBothSentAndReceived.includes(dataProduct)) {
+      edges.push(
+        createEdge({
+          id: generatedIdForEdge(dataProduct, container),
+          source: generateIdForNode(container),
+          target: generateIdForNode(dataProduct),
+          label: `reads from \n (${container.data.technology})`,
+          data: { dataProduct },
+          type: 'multiline',
+          markerStart: {
+            type: MarkerType.ArrowClosed,
+            width: 40,
+            height: 40,
+          },
+          markerEnd: undefined,
+        })
+      );
+    }
+  });
+
+  // Handle services that are both sent and received
   bothSentAndReceived.forEach((_service) => {
     if (container) {
       edges.push(
@@ -139,6 +202,31 @@ export const getNodesAndEdges = async ({ id, version, defaultFlow, mode = 'simpl
     }
   });
 
+  // Handle data products that both read and write
+  dataProductsBothSentAndReceived.forEach((_dataProduct) => {
+    if (container) {
+      edges.push(
+        createEdge({
+          id: generatedIdForEdge(container, _dataProduct) + '-both',
+          source: generateIdForNode(_dataProduct),
+          target: generateIdForNode(container),
+          label: `read and writes to \n (${container.data.technology})`,
+          type: 'multiline',
+          markerStart: {
+            type: MarkerType.ArrowClosed,
+            width: 40,
+            height: 40,
+          },
+          markerEnd: {
+            type: MarkerType.ArrowClosed,
+            width: 40,
+            height: 40,
+          },
+        })
+      );
+    }
+  });
+
   nodes.forEach((node: any) => {
     flow.setNode(node.id, { width: 150, height: 100 });
   });

package/eventcatalog/src/utils/node-graphs/data-products-node-graph.ts

@@ -0,0 +1,225 @@
+import { getCollection, type CollectionEntry } from 'astro:content';
+import dagre from 'dagre';
+import {
+  createDagreGraph,
+  generateIdForNode,
+  generatedIdForEdge,
+  calculatedNodes,
+  createEdge,
+  getColorFromString,
+} from '@utils/node-graphs/utils/utils';
+
+import { findInMap, createVersionedMap, mergeMaps, collectionToResourceMap } from '@utils/collections/util';
+import { MarkerType } from '@xyflow/react';
+import { getMessages, isCollectionAMessage } from '@utils/collections/messages';
+import { getProducersOfMessage } from '@utils/collections/services';
+import type { CollectionMessageTypes } from '@types';
+import { getNodesAndEdgesForProducedMessage } from './message-node-graph';
+
+type DagreGraph = any;
+
+interface Props {
+  id: string;
+  version: string;
+  defaultFlow?: DagreGraph;
+  mode?: 'simple' | 'full';
+}
+
+const getNodePropertyFromCollectionType = (type: string) => {
+  if (isCollectionAMessage(type)) return 'message';
+  if (type === 'containers') return 'data';
+  return collectionToResourceMap[type as keyof typeof collectionToResourceMap];
+};
+
+export const getNodesAndEdges = async ({ id, defaultFlow, version, mode = 'simple' }: Props) => {
+  const flow = defaultFlow || createDagreGraph({ ranksep: 300, nodesep: 50 });
+  let nodes = [] as any,
+    edges = [] as any;
+
+  const [dataProducts, containers, services, events, queries, commands, channels] = await Promise.all([
+    getCollection('data-products'),
+    getCollection('containers'),
+    getCollection('services'),
+    getCollection('events'),
+    getCollection('queries'),
+    getCollection('commands'),
+    getCollection('channels'),
+  ]);
+
+  const dataProduct = dataProducts.find((dp) => dp.data.id === id && dp.data.version === version);
+
+  // Nothing found...
+  if (!dataProduct) {
+    return {
+      nodes: [],
+      edges: [],
+    };
+  }
+
+  // Build maps for O(1) lookups
+  const messages = [...events, ...commands, ...queries];
+
+  const messageMap = createVersionedMap(messages);
+  const containerMap = createVersionedMap(containers);
+  const serviceMap = createVersionedMap(services);
+  const channelMap = createVersionedMap(channels);
+
+  const inputsRaw = dataProduct?.data.inputs || [];
+  const outputsRaw = dataProduct?.data.outputs || [];
+
+  const resourceMap = mergeMaps<
+    | CollectionEntry<CollectionMessageTypes>
+    | CollectionEntry<'services'>
+    | CollectionEntry<'containers'>
+    | CollectionEntry<'channels'>
+  >(messageMap, serviceMap, containerMap, channelMap);
+
+  // Process inputs - messages, containers, services, channels (etc)
+  inputsRaw.forEach((inputConfig) => {
+    let inputResource = findInMap(resourceMap, inputConfig.id, inputConfig.version) as
+      | CollectionEntry<CollectionMessageTypes>
+      | CollectionEntry<'services'>
+      | CollectionEntry<'containers'>
+      | CollectionEntry<'channels'>;
+
+    const existingNode = nodes.find((n: any) => n.id === generateIdForNode(inputResource));
+
+    if (!existingNode) {
+      const nodeDataKey = getNodePropertyFromCollectionType(inputResource?.collection);
+
+      nodes.push({
+        id: generateIdForNode(inputResource),
+        sourcePosition: 'right',
+        targetPosition: 'left',
+        data: { mode, [nodeDataKey]: { ...inputResource?.data } },
+        type: (inputResource?.collection as any) === 'containers' ? 'data' : inputResource?.collection,
+      });
+    }
+
+    // If collection is a message we render the producers of the message
+    if (isCollectionAMessage(inputResource?.collection)) {
+      const producersOfMessage = getProducersOfMessage(
+        services as CollectionEntry<'services'>[],
+        inputResource as CollectionEntry<CollectionMessageTypes>
+      ) as CollectionEntry<'services'>[];
+      for (const producer of producersOfMessage) {
+        const { nodes: producerNodes, edges: producerEdges } = getNodesAndEdgesForProducedMessage({
+          message: inputResource as CollectionEntry<CollectionMessageTypes>,
+          // We dont render any other services that consume this event for now
+          services: [],
+          // We dont render channels on this view for now...
+          channels: [],
+          currentNodes: nodes,
+          currentEdges: edges,
+          source: producer,
+          mode,
+        });
+
+        nodes.push(...producerNodes);
+        edges.push(...producerEdges);
+      }
+    }
+
+    // Add edge from resource to data product
+    edges.push(
+      createEdge({
+        id: generatedIdForEdge(inputResource, dataProduct),
+        source: generateIdForNode(inputResource),
+        target: generateIdForNode(dataProduct),
+        label: 'input',
+        type: 'animated',
+        data: {
+          customColor: getColorFromString(inputResource.data.id),
+          rootSourceAndTarget: { source: inputResource, target: dataProduct },
+        },
+        markerEnd: {
+          type: MarkerType.ArrowClosed,
+          color: '#666',
+          width: 40,
+          height: 40,
+        },
+      })
+    );
+  });
+
+  // The data product itself
+  nodes.push({
+    id: generateIdForNode(dataProduct),
+    sourcePosition: 'right',
+    targetPosition: 'left',
+    data: { mode, dataProduct: { ...dataProduct.data } },
+    type: 'data-products',
+  });
+
+  // Process outputs - messages, services, containers, channels that the data product produces
+  outputsRaw.forEach((outputConfig) => {
+    // Find the output resource (can be message, service, container, or channel)
+    const outputResource = findInMap(resourceMap, outputConfig.id, outputConfig.version) as
+      | CollectionEntry<CollectionMessageTypes>
+      | CollectionEntry<'services'>
+      | CollectionEntry<'containers'>
+      | CollectionEntry<'channels'>;
+
+    if (!outputResource) return;
+
+    // Add the node if it doesn't exist
+    const existingNode = nodes.find((n: any) => n.id === generateIdForNode(outputResource));
+    if (!existingNode) {
+      const nodeDataKey = getNodePropertyFromCollectionType(outputResource?.collection);
+
+      nodes.push({
+        id: generateIdForNode(outputResource),
+        sourcePosition: 'right',
+        targetPosition: 'left',
+        data: { mode, [nodeDataKey]: { ...outputResource?.data } },
+        type: (outputResource?.collection as any) === 'containers' ? 'data' : outputResource?.collection,
+      });
+    }
+
+    // Add edge from data product to the output resource
+    edges.push(
+      createEdge({
+        id: generatedIdForEdge(dataProduct, outputResource),
+        source: generateIdForNode(dataProduct),
+        target: generateIdForNode(outputResource),
+        label: 'output',
+        type: 'animated',
+        data: {
+          customColor: getColorFromString(outputResource.data.id),
+          rootSourceAndTarget: { source: dataProduct, target: outputResource },
+        },
+        markerEnd: {
+          type: MarkerType.ArrowClosed,
+          color: '#666',
+          width: 40,
+          height: 40,
+        },
+      })
+    );
+  });
+
+  nodes.forEach((node: any) => {
+    flow.setNode(node.id, { width: 150, height: 100 });
+  });
+
+  edges.forEach((edge: any) => {
+    flow.setEdge(edge.source, edge.target);
+  });
+
+  // Render the diagram in memory getting the X and Y
+  dagre.layout(flow);
+
+  // Find any duplicated edges, and merge them into one edge
+  const uniqueEdges = edges.reduce((acc: any[], edge: any) => {
+    const existingEdge = acc.find((e: any) => e.id === edge.id);
+    if (!existingEdge) {
+      acc.push(edge);
+    }
+    return acc;
+  }, []);
+
+  return {
+    nodes: calculatedNodes(flow, nodes),
+    edges: uniqueEdges,
+  };
+};

package/eventcatalog/src/utils/node-graphs/domains-node-graph.ts

@@ -8,6 +8,7 @@ import {
   createEdge,
 } from '@utils/node-graphs/utils/utils';
 import { getNodesAndEdges as getServicesNodeAndEdges } from './services-node-graph';
+import { getNodesAndEdges as getDataProductsNodeAndEdges } from './data-products-node-graph';
 import merge from 'lodash.merge';
 import { createVersionedMap, findInMap } from '@utils/collections/util';
 import type { Node } from '@xyflow/react';
@@ -199,7 +200,11 @@ export const getNodesAndEdges = async ({
     edges = new Map();
 
   // 1. Parallel Fetching
-  const [domains, services] = await Promise.all([getCollection('domains'), getCollection('services')]);
+  const [domains, services, dataProducts] = await Promise.all([
+    getCollection('domains'),
+    getCollection('services'),
+    getCollection('data-products'),
+  ]);
 
   const domain = domains.find((service) => service.data.id === id && service.data.version === version);
 
@@ -214,9 +219,11 @@ export const getNodesAndEdges = async ({
   // 2. Build optimized maps
   const serviceMap = createVersionedMap(services);
   const domainMap = createVersionedMap(domains);
+  const dataProductMap = createVersionedMap(dataProducts);
 
   const rawServices = domain?.data.services || [];
   const rawSubDomains = domain?.data.domains || [];
+  const rawDataProducts = (domain?.data as any)['data-products'] || [];
 
   // Optimized hydration
   const domainServicesWithVersion = rawServices
@@ -229,7 +236,12 @@ export const getNodesAndEdges = async ({
     .filter((d): d is any => !!d)
     .map((svc) => ({ id: svc.data.id, version: svc.data.version }));
 
-  // Get all the nodes for everyhing
+  const domainDataProductsWithVersion = rawDataProducts
+    .map((dataProduct: any) => findInMap(dataProductMap, dataProduct.id, dataProduct.version))
+    .filter((dp: any): dp is any => !!dp)
+    .map((dp: any) => ({ id: dp.data.id, version: dp.data.version }));
+
+  // Get all the nodes for everything
 
   for (const service of domainServicesWithVersion) {
     const { nodes: serviceNodes, edges: serviceEdges } = await getServicesNodeAndEdges({
@@ -255,6 +267,20 @@ export const getNodesAndEdges = async ({
     serviceEdges.forEach((e) => edges.set(e.id, e));
   }
 
+  for (const dataProduct of domainDataProductsWithVersion) {
+    const { nodes: dataProductNodes, edges: dataProductEdges } = await getDataProductsNodeAndEdges({
+      id: dataProduct.id,
+      version: dataProduct.version,
+      defaultFlow: flow,
+      mode,
+    });
+    dataProductNodes.forEach((n: any) => {
+      nodes.set(n.id, nodes.has(n.id) ? merge(nodes.get(n.id), n) : n);
+    });
+    // @ts-ignore
+    dataProductEdges.forEach((e) => edges.set(e.id, e));
+  }
+
   for (const subDomain of domainSubDomainsWithVersion) {
     const { nodes: subDomainNodes, edges: subDomainEdges } = await getNodesAndEdges({
       id: subDomain.id,