@auto-engineer/pipeline 0.0.1 → 0.15.0

package/src/server/pipeline-server.ts

@@ -6,31 +6,35 @@ import {
   type Event,
   type MessageBus,
 } from '@auto-engineer/message-bus';
-import { type ILocalMessageStore, MemoryMessageStore } from '@auto-engineer/message-store';
 import cors from 'cors';
 import express from 'express';
 import getPort from 'get-port';
 import { nanoid } from 'nanoid';
 import type { Pipeline } from '../builder/define';
-import type { GraphIR } from '../graph/types';
+import { filterGraph } from '../graph/filter-graph';
+import type { FilterOptions, GraphIR, GraphNode, NodeStatus, NodeType } from '../graph/types';
 import type { PipelineContext } from '../runtime/context';
+import type { EventDefinition } from '../runtime/event-command-map';
 import { EventCommandMapper } from '../runtime/event-command-map';
 import { PhasedExecutor } from '../runtime/phased-executor';
 import { PipelineRuntime } from '../runtime/pipeline-runtime';
 import { SettledTracker } from '../runtime/settled-tracker';
+import { createPipelineEventStore, type PipelineEventStoreContext } from '../store/pipeline-event-store';
 import { SSEManager } from './sse-manager';
 
+export type { EventDefinition };
+
 export interface CommandHandlerWithMetadata extends CommandHandler {
   alias?: string;
   description?: string;
+  displayName?: string;
   fields?: Record<string, unknown>;
   examples?: unknown[];
-  events?: string[];
+  events?: EventDefinition[];
 }
 
 export interface PipelineServerConfig {
   port: number;
-  messageStore?: ILocalMessageStore;
 }
 
 interface EventWithCorrelation extends Event {
@@ -41,17 +45,17 @@ export class PipelineServer {
   private app: express.Application;
   private httpServer: HttpServer;
   private messageBus: MessageBus;
-  private messageStore: ILocalMessageStore;
   private readonly commandHandlers: Map<string, CommandHandlerWithMetadata> = new Map();
   private readonly pipelines: Map<string, Pipeline> = new Map();
   private readonly runtimes: Map<string, PipelineRuntime> = new Map();
   private actualPort: number;
   private readonly requestedPort: number;
-  private currentSessionId?: string;
   private readonly settledTracker: SettledTracker;
   private readonly eventCommandMapper: EventCommandMapper;
   private readonly phasedExecutor: PhasedExecutor;
   private readonly sseManager: SSEManager;
+  private readonly eventStoreContext: PipelineEventStoreContext;
+  private readonly itemKeyExtractors = new Map<string, (data: unknown) => string | undefined>();
 
   constructor(config: PipelineServerConfig) {
     this.requestedPort = config.port;
@@ -61,20 +65,32 @@ export class PipelineServer {
     this.app.use(express.json());
     this.httpServer = createServer(this.app);
     this.messageBus = createMessageBus();
-    this.messageStore = config.messageStore ?? new MemoryMessageStore();
+    this.eventStoreContext = createPipelineEventStore();
     this.eventCommandMapper = new EventCommandMapper([]);
     this.settledTracker = new SettledTracker({
+      readModel: this.eventStoreContext.readModel,
       onDispatch: (commandType, data, correlationId) => {
         void this.dispatchFromSettled(commandType, data, correlationId);
       },
+      onEventEmit: async (event) => {
+        const correlationId = event.data.correlationId;
+        await this.eventStoreContext.eventStore.appendToStream(`pipeline-${correlationId}`, [event]);
+      },
     });
     this.phasedExecutor = new PhasedExecutor({
+      readModel: this.eventStoreContext.readModel,
       onDispatch: (commandType, data, correlationId) => {
         void this.dispatchFromSettled(commandType, data, correlationId);
       },
       onComplete: (event, correlationId) => {
         void this.handlePhasedComplete(event, correlationId);
       },
+      onEventEmit: async (event) => {
+        const data = event.data as Record<string, unknown>;
+        const correlationId =
+          (data.correlationId as string) ?? (data.executionId as string)?.split('-')[1] ?? 'default';
+        await this.eventStoreContext.eventStore.appendToStream(`pipeline-${correlationId}`, [event]);
+      },
     });
     this.sseManager = new SSEManager();
     this.setupRoutes();
@@ -96,6 +112,10 @@ export class PipelineServer {
     return Array.from(this.commandHandlers.keys());
   }
 
+  registerItemKeyExtractor(commandType: string, extractor: (data: unknown) => string | undefined): void {
+    this.itemKeyExtractors.set(commandType, extractor);
+  }
+
   registerPipeline(pipeline: Pipeline): void {
     this.pipelines.set(pipeline.descriptor.name, pipeline);
     this.runtimes.set(pipeline.descriptor.name, new PipelineRuntime(pipeline.descriptor));
@@ -119,8 +139,6 @@ export class PipelineServer {
       this.actualPort = await getPort();
     }
 
-    this.currentSessionId = await this.messageStore.createSession();
-
     await new Promise<void>((resolve) => {
       this.httpServer.listen(this.actualPort, () => {
         resolve();
@@ -130,9 +148,6 @@ export class PipelineServer {
 
   async stop(): Promise<void> {
     this.sseManager.closeAll();
-    if (this.currentSessionId !== undefined) {
-      await this.messageStore.endSession(this.currentSessionId);
-    }
     await new Promise<void>((resolve) => {
       this.httpServer.close(() => resolve());
     });
@@ -176,27 +191,37 @@ export class PipelineServer {
       });
     });
 
-    this.app.get('/pipeline', (_req, res) => {
-      const combinedGraph = this.buildCombinedGraph();
-      const commandToEvents = this.buildCommandToEvents();
-      const eventToCommand = this.buildEventToCommand();
-      const pipelineNodes = this.buildPipelineNodes();
-
-      res.json({
-        nodes: [...combinedGraph.nodes, ...pipelineNodes],
-        edges: combinedGraph.edges,
-        commandToEvents,
-        eventToCommand,
-      });
+    this.app.get('/pipeline', (req, res) => {
+      void (async () => {
+        const commandToEvents = this.buildCommandToEvents();
+        const rawGraph = this.buildCombinedGraph();
+        const pipelineEvents = this.extractPipelineEvents(rawGraph, commandToEvents);
+        const graphWithEvents = this.addCommandEventEdgesToGraph(rawGraph, commandToEvents, pipelineEvents);
+        const graphWithEnrichedEvents = this.enrichEventLabels(graphWithEvents);
+        const completeGraph = this.markBackLinks(graphWithEnrichedEvents);
+        const filterOptions = this.parseFilterOptions(req.query);
+        const filteredGraph = filterGraph(completeGraph, filterOptions);
+        const correlationId = req.query.correlationId as string | undefined;
+        const graphWithStatus = await this.addStatusToCommandNodes(filteredGraph, correlationId);
+
+        const latestRun = await this.eventStoreContext.readModel.getLatestCorrelationId();
+        res.json({
+          nodes: graphWithStatus.nodes,
+          edges: graphWithStatus.edges,
+          latestRun,
+        });
+      })();
     });
 
-    this.app.get('/pipeline/mermaid', (_req, res) => {
-      const mermaid = this.buildMermaidDiagram();
+    this.app.get('/pipeline/mermaid', (req, res) => {
+      const filterOptions = this.parseFilterOptions(req.query);
+      const mermaid = this.buildMermaidDiagram(filterOptions);
       res.type('text/plain').send(mermaid);
     });
 
-    this.app.get('/pipeline/diagram', (_req, res) => {
-      const mermaidDefinition = this.buildMermaidDiagram();
+    this.app.get('/pipeline/diagram', (req, res) => {
+      const filterOptions = this.parseFilterOptions(req.query);
+      const mermaidDefinition = this.buildMermaidDiagram(filterOptions);
       const html = this.buildDiagramHtml(mermaidDefinition);
       res.type('text/html').send(html);
     });
@@ -221,13 +246,14 @@ export class PipelineServer {
           correlationId,
         };
 
-        await this.messageStore.saveMessage('$all', commandWithIds, undefined, 'command');
+        await this.emitCommandDispatched(correlationId, requestId, commandWithIds.type, commandWithIds.data);
 
         void this.processCommand(commandWithIds);
 
         res.json({
           status: 'ack',
           commandId: commandWithIds.requestId,
+          correlationId: commandWithIds.correlationId,
           timestamp: new Date().toISOString(),
         });
       })();
@@ -235,11 +261,17 @@ export class PipelineServer {
 
     this.app.get('/messages', (_req, res) => {
       void (async () => {
-        const messages = await this.messageStore.getMessages('$all');
-        const serialized = messages.map((m) => ({
-          ...m,
-          revision: m.revision.toString(),
-          position: m.position.toString(),
+        const messages = await this.eventStoreContext.readModel.getMessages();
+        const serialized = messages.map((m, index) => ({
+          message: {
+            type: m.messageName,
+            data: m.messageData,
+            correlationId: m.correlationId,
+            requestId: m.requestId,
+          },
+          messageType: m.messageType,
+          revision: String(index),
+          position: String(index),
         }));
         res.json(serialized);
       })();
@@ -247,18 +279,11 @@ export class PipelineServer {
 
     this.app.get('/stats', (_req, res) => {
       void (async () => {
-        const stats = await this.messageStore.getStats();
+        const stats = await this.eventStoreContext.readModel.getStats();
         res.json(stats);
       })();
     });
 
-    this.app.get('/sessions', (_req, res) => {
-      void (async () => {
-        const sessions = await this.messageStore.getSessions();
-        res.json(sessions);
-      })();
-    });
-
     this.app.get('/events', (req, res) => {
       const clientId = `sse-${nanoid()}`;
       const correlationIdFilter = req.query.correlationId as string | undefined;
@@ -281,47 +306,330 @@ export class PipelineServer {
       combinedGraph.edges.push(...graph.edges);
     }
 
-    return combinedGraph;
+    return this.enrichCommandLabels(combinedGraph);
+  }
+
+  private enrichCommandLabels(graph: GraphIR): GraphIR {
+    return {
+      nodes: graph.nodes.map((node) => {
+        if (node.type !== 'command') {
+          return node;
+        }
+        const handler = this.commandHandlers.get(node.label);
+        if (handler?.displayName === undefined) {
+          return node;
+        }
+        return { ...node, label: handler.displayName };
+      }),
+      edges: graph.edges,
+    };
+  }
+
+  private enrichEventLabels(graph: GraphIR): GraphIR {
+    const eventDisplayNames = this.buildEventDisplayNames();
+    return {
+      nodes: graph.nodes.map((node) => {
+        if (node.type !== 'event') {
+          return node;
+        }
+        const displayName = eventDisplayNames.get(node.label);
+        if (displayName === undefined) {
+          return node;
+        }
+        return { ...node, label: displayName };
+      }),
+      edges: graph.edges,
+    };
+  }
+
+  private async addStatusToCommandNodes(graph: GraphIR, correlationId?: string): Promise<GraphIR> {
+    const nodesWithStatus = await Promise.all(
+      graph.nodes.map(async (node) => {
+        if (node.type === 'command') {
+          return this.addStatusToCommandNode(node, correlationId);
+        }
+        if (node.type === 'settled') {
+          return this.addStatusToSettledNode(node, correlationId);
+        }
+        return node;
+      }),
+    );
+    return {
+      nodes: nodesWithStatus,
+      edges: graph.edges,
+    };
+  }
+
+  private async addStatusToCommandNode(node: GraphNode, correlationId?: string): Promise<GraphNode> {
+    const commandName = node.id.replace(/^cmd:/, '');
+    if (correlationId === undefined) {
+      return { ...node, status: 'idle' as NodeStatus, pendingCount: 0, endedCount: 0 };
+    }
+    const stats = await this.computeCommandStats(correlationId, commandName);
+    return {
+      ...node,
+      status: stats.aggregateStatus,
+      pendingCount: stats.pendingCount,
+      endedCount: stats.endedCount,
+    };
+  }
+
+  private async addStatusToSettledNode(node: GraphNode, correlationId?: string): Promise<GraphNode> {
+    if (correlationId === undefined) {
+      return { ...node, status: 'idle' as NodeStatus, pendingCount: 0, endedCount: 0 };
+    }
+    const commandTypes = node.id.replace(/^settled:/, '');
+    const templateId = `template-${commandTypes}`;
+    const stats = await this.eventStoreContext.readModel.computeSettledStats(correlationId, templateId);
+    return {
+      ...node,
+      status: stats.status,
+      pendingCount: stats.pendingCount,
+      endedCount: stats.endedCount,
+    };
+  }
+
+  private async emitItemStatusChanged(
+    correlationId: string,
+    commandType: string,
+    itemKey: string,
+    requestId: string,
+    status: 'running' | 'success' | 'error',
+    attemptCount: number,
+  ): Promise<void> {
+    await this.eventStoreContext.eventStore.appendToStream(`pipeline-${correlationId}`, [
+      {
+        type: 'ItemStatusChanged',
+        data: {
+          correlationId,
+          commandType,
+          itemKey,
+          requestId,
+          status,
+          attemptCount,
+        },
+      },
+    ]);
+  }
+
+  private async emitNodeStatusChanged(
+    correlationId: string,
+    commandName: string,
+    status: NodeStatus,
+    previousStatus: NodeStatus,
+  ): Promise<void> {
+    const stats = await this.computeCommandStats(correlationId, commandName);
+    await this.eventStoreContext.eventStore.appendToStream(`pipeline-${correlationId}`, [
+      {
+        type: 'NodeStatusChanged',
+        data: {
+          correlationId,
+          commandName,
+          nodeId: `cmd:${commandName}`,
+          status,
+          previousStatus,
+          pendingCount: stats.pendingCount,
+          endedCount: stats.endedCount,
+        },
+      },
+    ]);
+  }
+
+  private async emitCommandDispatched(
+    correlationId: string,
+    requestId: string,
+    commandType: string,
+    commandData: Record<string, unknown>,
+  ): Promise<void> {
+    await this.eventStoreContext.eventStore.appendToStream(`pipeline-${correlationId}`, [
+      {
+        type: 'CommandDispatched',
+        data: {
+          correlationId,
+          requestId,
+          commandType,
+          commandData,
+          timestamp: new Date(),
+        },
+      },
+    ]);
+  }
+
+  private async emitDomainEventEmitted(
+    correlationId: string,
+    requestId: string,
+    eventType: string,
+    eventData: Record<string, unknown>,
+  ): Promise<void> {
+    await this.eventStoreContext.eventStore.appendToStream(`pipeline-${correlationId}`, [
+      {
+        type: 'DomainEventEmitted',
+        data: {
+          correlationId,
+          requestId,
+          eventType,
+          eventData,
+          timestamp: new Date(),
+        },
+      },
+    ]);
+  }
+
+  private async emitPipelineRunStarted(correlationId: string, triggerCommand: string): Promise<void> {
+    await this.eventStoreContext.eventStore.appendToStream(`pipeline-${correlationId}`, [
+      {
+        type: 'PipelineRunStarted',
+        data: {
+          correlationId,
+          triggerCommand,
+        },
+      },
+    ]);
+  }
+
+  private async updateNodeStatus(correlationId: string, commandName: string, status: NodeStatus): Promise<void> {
+    const existing = await this.eventStoreContext.readModel.getNodeStatus(correlationId, commandName);
+    const previousStatus: NodeStatus = existing?.status ?? 'idle';
+    await this.emitNodeStatusChanged(correlationId, commandName, status, previousStatus);
+    await this.broadcastNodeStatusChanged(correlationId, commandName, status, previousStatus);
+  }
+
+  private async broadcastNodeStatusChanged(
+    correlationId: string,
+    commandName: string,
+    status: NodeStatus,
+    previousStatus: NodeStatus,
+  ): Promise<void> {
+    const stats = await this.computeCommandStats(correlationId, commandName);
+    const event: Event & { correlationId: string } = {
+      type: 'NodeStatusChanged',
+      data: {
+        nodeId: `cmd:${commandName}`,
+        status,
+        previousStatus,
+        pendingCount: stats.pendingCount,
+        endedCount: stats.endedCount,
+      },
+      correlationId,
+    };
+    this.sseManager.broadcast(event);
+  }
+
+  private async broadcastPipelineRunStarted(correlationId: string, triggerCommand: string): Promise<void> {
+    const event: Event & { correlationId: string } = {
+      type: 'PipelineRunStarted',
+      data: { correlationId, triggerCommand },
+      correlationId,
+    };
+    this.sseManager.broadcast(event);
+    await this.emitPipelineRunStarted(correlationId, triggerCommand);
+  }
+
+  private extractItemKey(commandType: string, data: unknown, requestId: string): string {
+    const extractor = this.itemKeyExtractors.get(commandType);
+    if (extractor !== undefined) {
+      const key = extractor(data);
+      if (key !== undefined) return key;
+    }
+    return requestId;
+  }
+
+  private async getOrCreateItemStatus(
+    correlationId: string,
+    commandType: string,
+    itemKey: string,
+    requestId: string,
+  ): Promise<{ attemptCount: number }> {
+    const existing = await this.eventStoreContext.readModel.getItemStatus(correlationId, commandType, itemKey);
+    const attemptCount = (existing?.attemptCount ?? 0) + 1;
+
+    await this.emitItemStatusChanged(correlationId, commandType, itemKey, requestId, 'running', attemptCount);
+
+    return { attemptCount };
+  }
+
+  private async updateItemStatus(
+    correlationId: string,
+    commandType: string,
+    itemKey: string,
+    status: 'running' | 'success' | 'error',
+  ): Promise<void> {
+    const existing = await this.eventStoreContext.readModel.getItemStatus(correlationId, commandType, itemKey);
+    if (existing !== null) {
+      await this.emitItemStatusChanged(
+        correlationId,
+        commandType,
+        itemKey,
+        existing.currentRequestId,
+        status,
+        existing.attemptCount,
+      );
+    }
+  }
+
+  private async computeCommandStats(
+    correlationId: string,
+    commandType: string,
+  ): Promise<{ pendingCount: number; endedCount: number; aggregateStatus: NodeStatus }> {
+    return this.eventStoreContext.readModel.computeCommandStats(correlationId, commandType);
+  }
+
+  private getEventName(event: EventDefinition): string {
+    return typeof event === 'string' ? event : event.name;
   }
 
   private buildCommandToEvents(): Record<string, string[]> {
     const commandToEvents: Record<string, string[]> = {};
     for (const [name, handler] of this.commandHandlers.entries()) {
       if (handler.events !== undefined && Array.isArray(handler.events)) {
-        commandToEvents[name] = handler.events;
+        commandToEvents[name] = handler.events.map((e) => this.getEventName(e));
       }
     }
     return commandToEvents;
   }
 
-  private buildEventToCommand(): Record<string, string> {
-    const eventToCommand: Record<string, string> = {};
-    for (const pipeline of this.pipelines.values()) {
-      for (const handler of pipeline.descriptor.handlers) {
-        if (handler.type === 'emit') {
-          for (const cmd of handler.commands) {
-            eventToCommand[handler.eventType] = cmd.commandType;
-          }
+  private buildEventDisplayNames(): Map<string, string> {
+    const eventDisplayNames = new Map<string, string>();
+    for (const handler of this.commandHandlers.values()) {
+      if (handler.events === undefined) {
+        continue;
+      }
+      for (const event of handler.events) {
+        if (typeof event !== 'string' && event.displayName !== undefined) {
+          eventDisplayNames.set(event.name, event.displayName);
         }
       }
     }
-    return eventToCommand;
+    return eventDisplayNames;
   }
 
-  private buildPipelineNodes(): Array<{ id: string; name: string; title: string; alias?: string; status: 'None' }> {
-    return Array.from(this.commandHandlers.entries()).map(([name, handler]) => ({
-      id: name,
-      name,
-      title: handler.description ?? '',
-      alias: handler.alias,
-      status: 'None' as const,
-    }));
+  private parseFilterOptions(query: Record<string, unknown>): FilterOptions {
+    const excludeTypesParam = query.excludeTypes;
+    const maintainEdgesParam = query.maintainEdges;
+
+    const excludeTypes: NodeType[] = [];
+    if (typeof excludeTypesParam === 'string' && excludeTypesParam.length > 0) {
+      const types = excludeTypesParam.split(',');
+      for (const t of types) {
+        if (t === 'event' || t === 'command' || t === 'settled') {
+          excludeTypes.push(t);
+        }
+      }
+    }
+
+    const maintainEdges = maintainEdgesParam === 'true';
+
+    return { excludeTypes, maintainEdges };
   }
 
-  private buildMermaidDiagram(): string {
-    const graph = this.buildCombinedGraph();
+  private buildMermaidDiagram(filterOptions?: FilterOptions): string {
     const commandToEvents = this.buildCommandToEvents();
-    const pipelineEvents = this.extractPipelineEvents(graph, commandToEvents);
+    const rawGraph = this.buildCombinedGraph();
+    const pipelineEvents = this.extractPipelineEvents(rawGraph, commandToEvents);
+    const graphWithEvents = this.addCommandEventEdgesToGraph(rawGraph, commandToEvents, pipelineEvents);
+    const graphWithEnrichedEvents = this.enrichEventLabels(graphWithEvents);
+    const completeGraph = this.markBackLinks(graphWithEnrichedEvents);
+    const graph = filterOptions ? filterGraph(completeGraph, filterOptions) : completeGraph;
     const lines: string[] = ['flowchart LR'];
 
     const eventNodes = new Set<string>();
@@ -330,15 +638,101 @@ export class PipelineServer {
     const edgeContext = { index: 0, backLinkIndices: [] as number[] };
 
     this.addGraphNodesToMermaid(graph, lines, eventNodes, commandNodes, settledNodes);
-    const pipelineCommands = new Set(commandNodes);
-    this.addCommandEventNodesToMermaid(commandToEvents, pipelineCommands, pipelineEvents, lines, eventNodes);
-    this.addGraphEdgesToMermaid(graph, commandToEvents, lines, edgeContext);
-    this.addCommandEventEdgesToMermaid(commandToEvents, pipelineCommands, pipelineEvents, lines, edgeContext);
+    this.addGraphEdgesToMermaid(graph, lines, edgeContext);
     this.addMermaidStyles(lines, eventNodes, commandNodes, settledNodes, edgeContext.backLinkIndices);
 
     return lines.join('\n');
   }
 
+  private addCommandEventEdgesToGraph(
+    graph: GraphIR,
+    commandToEvents: Record<string, string[]>,
+    pipelineEvents: Set<string>,
+  ): GraphIR {
+    const commandNodes = new Set(graph.nodes.filter((n) => n.type === 'command').map((n) => n.id.replace('cmd:', '')));
+    const existingEventIds = new Set(graph.nodes.filter((n) => n.type === 'event').map((n) => n.id));
+    const newNodes = [...graph.nodes];
+    const newEdges = [...graph.edges];
+
+    for (const [commandName, events] of Object.entries(commandToEvents)) {
+      if (!commandNodes.has(commandName)) {
+        continue;
+      }
+      for (const eventName of events) {
+        if (!pipelineEvents.has(eventName)) {
+          continue;
+        }
+        const eventId = `evt:${eventName}`;
+        if (!existingEventIds.has(eventId)) {
+          newNodes.push({ id: eventId, type: 'event', label: eventName });
+          existingEventIds.add(eventId);
+        }
+        newEdges.push({ from: `cmd:${commandName}`, to: eventId });
+      }
+    }
+
+    return { nodes: newNodes, edges: newEdges };
+  }
+
+  private markBackLinks(graph: GraphIR): GraphIR {
+    const outgoingEdgesWithBackLink = new Map<string, Array<{ to: string; isBackLink: boolean }>>();
+    for (const edge of graph.edges) {
+      const existing = outgoingEdgesWithBackLink.get(edge.from) ?? [];
+      existing.push({ to: edge.to, isBackLink: edge.backLink === true });
+      outgoingEdgesWithBackLink.set(edge.from, existing);
+    }
+
+    const markedEdges = graph.edges.map((edge) => {
+      if (edge.backLink === true) {
+        return edge;
+      }
+      if (edge.from.startsWith('evt:') && edge.to.startsWith('cmd:')) {
+        const createsBackLink = this.hasPathToExcludingBackLinks(edge.to, edge.from, outgoingEdgesWithBackLink);
+        if (createsBackLink) {
+          return { ...edge, backLink: true };
+        }
+      }
+      return edge;
+    });
+
+    return { nodes: graph.nodes, edges: markedEdges };
+  }
+
+  private hasPathToExcludingBackLinks(
+    from: string,
+    target: string,
+    outgoingEdges: Map<string, Array<{ to: string; isBackLink: boolean }>>,
+  ): boolean {
+    const visited = new Set<string>();
+    const queue = [from];
+
+    while (queue.length > 0) {
+      const current = queue.shift();
+      if (current === undefined) {
+        break;
+      }
+      if (current === target) {
+        return true;
+      }
+      if (visited.has(current)) {
+        continue;
+      }
+      visited.add(current);
+
+      const neighbors = outgoingEdges.get(current) ?? [];
+      for (const neighbor of neighbors) {
+        if (neighbor.isBackLink) {
+          continue;
+        }
+        if (!visited.has(neighbor.to)) {
+          queue.push(neighbor.to);
+        }
+      }
+    }
+
+    return false;
+  }
+
   private extractPipelineEvents(graph: GraphIR, commandToEvents: Record<string, string[]>): Set<string> {
     const pipelineEvents = new Set<string>();
 
@@ -374,11 +768,11 @@ export class PipelineServer {
         const eventName = node.id.replace('evt:', '');
         const safeId = `evt_${eventName}`;
         eventNodes.add(safeId);
-        lines.push(`  ${safeId}([${eventName}])`);
+        lines.push(`  ${safeId}([${node.label}])`);
       } else if (node.id.startsWith('cmd:')) {
         const commandName = node.id.replace('cmd:', '');
         commandNodes.add(commandName);
-        lines.push(`  ${commandName}[${commandName}]`);
+        lines.push(`  ${commandName}[${node.label}]`);
       } else if (node.id.startsWith('settled:')) {
         const commandTypes = node.id.replace('settled:', '').split(',');
         const safeId = `settled_${commandTypes.join('_')}`;
@@ -388,55 +782,19 @@ export class PipelineServer {
     }
   }
 
-  private addCommandEventNodesToMermaid(
-    commandToEvents: Record<string, string[]>,
-    pipelineCommands: Set<string>,
-    pipelineEvents: Set<string>,
-    lines: string[],
-    eventNodes: Set<string>,
-  ): void {
-    for (const [commandName, events] of Object.entries(commandToEvents)) {
-      if (!pipelineCommands.has(commandName)) {
-        continue;
-      }
-      for (const eventName of events) {
-        if (!pipelineEvents.has(eventName)) {
-          continue;
-        }
-        const safeEventId = `evt_${eventName}`;
-        if (!eventNodes.has(safeEventId)) {
-          eventNodes.add(safeEventId);
-          lines.push(`  ${safeEventId}([${eventName}])`);
-        }
-      }
-    }
-  }
-
   private addGraphEdgesToMermaid(
     graph: GraphIR,
-    commandToEvents: Record<string, string[]>,
     lines: string[],
     edgeContext: { index: number; backLinkIndices: number[] },
   ): void {
     for (const edge of graph.edges) {
-      if (edge.from.startsWith('cmd:') && edge.to.startsWith('settled:')) {
-        const commandType = edge.from.replace('cmd:', '');
-        const commandTypes = edge.to.replace('settled:', '').split(',');
-        const settledId = `settled_${commandTypes.join('_')}`;
-        const events = commandToEvents[commandType];
-        if (events !== undefined) {
-          for (const eventName of events) {
-            lines.push(`  evt_${eventName} --> ${settledId}`);
-            edgeContext.index++;
-          }
-        }
-        continue;
-      }
       const from = this.normalizeNodeId(edge.from);
       const to = this.normalizeNodeId(edge.to);
-      lines.push(`  ${from} --> ${to}`);
       if (edge.backLink === true) {
+        lines.push(`  ${from} -.->|retry| ${to}`);
         edgeContext.backLinkIndices.push(edgeContext.index);
+      } else {
+        lines.push(`  ${from} --> ${to}`);
       }
       edgeContext.index++;
     }
@@ -453,27 +811,6 @@ export class PipelineServer {
     return `settled_${commandTypes.join('_')}`;
   }
 
-  private addCommandEventEdgesToMermaid(
-    commandToEvents: Record<string, string[]>,
-    pipelineCommands: Set<string>,
-    pipelineEvents: Set<string>,
-    lines: string[],
-    edgeContext: { index: number; backLinkIndices: number[] },
-  ): void {
-    for (const [commandName, events] of Object.entries(commandToEvents)) {
-      if (!pipelineCommands.has(commandName)) {
-        continue;
-      }
-      for (const eventName of events) {
-        if (!pipelineEvents.has(eventName)) {
-          continue;
-        }
-        lines.push(`  ${commandName} --> evt_${eventName}`);
-        edgeContext.index++;
-      }
-    }
-  }
-
   private addMermaidStyles(
     lines: string[],
     eventNodes: Set<string>,
@@ -511,97 +848,126 @@ export class PipelineServer {
     const handler = this.commandHandlers.get(command.type);
     if (!handler) return;
 
-    this.settledTracker.onCommandStarted(command);
+    const isNewCorrelationId = !(await this.eventStoreContext.readModel.hasCorrelation(command.correlationId));
+    if (isNewCorrelationId) {
+      await this.broadcastPipelineRunStarted(command.correlationId, command.type);
+    }
+
+    const itemKey = this.extractItemKey(command.type, command.data, command.requestId);
+    await this.getOrCreateItemStatus(command.correlationId, command.type, itemKey, command.requestId);
+
+    await this.updateNodeStatus(command.correlationId, command.type, 'running');
+    await this.settledTracker.onCommandStarted(command);
 
     const resultEvent = await handler.handle(command);
     const events = Array.isArray(resultEvent) ? resultEvent : [resultEvent];
 
-    for (const event of events) {
-      const eventWithIds: EventWithCorrelation = {
-        ...event,
-        correlationId: command.correlationId,
-        requestId: command.requestId,
-      };
+    const finalStatus = this.getStatusFromEvents(events);
+    const itemFinalStatus = finalStatus === 'idle' ? 'success' : finalStatus;
+    await this.updateItemStatus(command.correlationId, command.type, itemKey, itemFinalStatus);
+    await this.updateNodeStatus(command.correlationId, command.type, finalStatus);
+
+    const eventsWithIds: EventWithCorrelation[] = events.map((event) => ({
+      ...event,
+      correlationId: command.correlationId,
+      requestId: command.requestId,
+    }));
 
-      await this.messageStore.saveMessage('$all', eventWithIds, undefined, 'event');
+    await Promise.all(
+      eventsWithIds.map((e) =>
+        this.emitDomainEventEmitted(e.correlationId, command.requestId, e.type, e.data as Record<string, unknown>),
+      ),
+    );
 
+    for (const eventWithIds of eventsWithIds) {
       this.sseManager.broadcast(eventWithIds);
 
-      const sourceCommand = this.eventCommandMapper.getSourceCommand(event.type);
+      const sourceCommand = this.eventCommandMapper.getSourceCommand(eventWithIds.type);
       if (sourceCommand !== undefined) {
-        this.settledTracker.onEventReceived(eventWithIds, sourceCommand);
+        await this.settledTracker.onEventReceived(eventWithIds, sourceCommand);
       }
 
-      this.routeEventToPhasedExecutor(eventWithIds);
+      await this.routeEventToPhasedExecutor(eventWithIds);
+    }
+
+    await Promise.all(eventsWithIds.map((e) => this.routeEventToPipelines(e)));
+  }
 
-      await this.routeEventToPipelines(eventWithIds);
+  private getStatusFromEvents(events: Event[]): NodeStatus {
+    for (const event of events) {
+      if (event.type.includes('Failed')) {
+        return 'error';
+      }
     }
+    return 'success';
   }
 
   private async dispatchFromSettled(commandType: string, data: unknown, correlationId: string): Promise<void> {
+    const requestId = `req-${nanoid()}`;
     const command: Command & { correlationId: string; requestId: string } = {
       type: commandType,
       data: data as Record<string, unknown>,
       correlationId,
-      requestId: `req-${nanoid()}`,
+      requestId,
     };
-    await this.messageStore.saveMessage('$all', command, undefined, 'command');
+    await this.emitCommandDispatched(correlationId, requestId, commandType, data as Record<string, unknown>);
     await this.processCommand(command);
   }
 
   private async handlePhasedComplete(event: Event, correlationId: string): Promise<void> {
+    const requestId = `req-${nanoid()}`;
     const eventWithIds: EventWithCorrelation = {
       ...event,
       correlationId,
     };
-    await this.messageStore.saveMessage('$all', eventWithIds, undefined, 'event');
+    await this.emitDomainEventEmitted(correlationId, requestId, event.type, event.data as Record<string, unknown>);
     this.sseManager.broadcast(eventWithIds);
     await this.routeEventToPipelines(eventWithIds);
   }
 
   private async routeEventToPipelines(event: EventWithCorrelation): Promise<void> {
     const ctx = this.createContext(event.correlationId);
-
-    for (const runtime of this.runtimes.values()) {
-      await runtime.handleEvent(event, ctx);
-    }
+    const runtimes = Array.from(this.runtimes.values());
+    await Promise.all(runtimes.map((runtime) => runtime.handleEvent(event, ctx)));
   }
 
   private createContext(correlationId: string): PipelineContext {
     return {
       correlationId,
       emit: async (type: string, data: unknown) => {
+        const requestId = `req-${nanoid()}`;
         const event: EventWithCorrelation = {
           type,
           data: data as Record<string, unknown>,
           correlationId,
         };
-        await this.messageStore.saveMessage('$all', event, undefined, 'event');
+        await this.emitDomainEventEmitted(correlationId, requestId, type, data as Record<string, unknown>);
         this.sseManager.broadcast(event);
         await this.routeEventToPipelines(event);
       },
       sendCommand: async (type: string, data: unknown) => {
+        const requestId = `req-${nanoid()}`;
         const command: Command & { correlationId: string; requestId: string } = {
           type,
           data: data as Record<string, unknown>,
           correlationId,
-          requestId: `req-${nanoid()}`,
+          requestId,
         };
-        await this.messageStore.saveMessage('$all', command, undefined, 'command');
+        await this.emitCommandDispatched(correlationId, requestId, type, data as Record<string, unknown>);
         await this.processCommand(command);
       },
-      startPhased: (handler, event) => {
-        this.phasedExecutor.startPhased(handler, event, correlationId);
+      startPhased: async (handler, event) => {
+        await this.phasedExecutor.startPhased(handler, event, correlationId);
       },
     };
   }
 
-  private routeEventToPhasedExecutor(event: EventWithCorrelation): void {
+  private async routeEventToPhasedExecutor(event: EventWithCorrelation): Promise<void> {
     for (const pipeline of this.pipelines.values()) {
       for (const handler of pipeline.descriptor.handlers) {
         if (handler.type === 'foreach-phased') {
           const itemKey = handler.completion.itemKey(event);
-          this.phasedExecutor.onEventReceived(event, itemKey);
+          await this.phasedExecutor.onEventReceived(event, itemKey);
         }
       }
     }