@auto-engineer/pipeline 0.0.1 → 0.15.0

package/package.json

@@ -4,13 +4,17 @@
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
   "dependencies": {
+    "@event-driven-io/emmett": "^0.38.2",
+    "@event-driven-io/emmett-sqlite": "^0.38.5",
+    "chokidar": "^3.6.0",
     "cors": "^2.8.5",
     "dotenv": "^16.4.5",
     "express": "^4.18.0",
     "get-port": "^7.1.0",
+    "jose": "^5.9.6",
     "nanoid": "^5.0.0",
-    "@auto-engineer/message-store": "0.13.3",
-    "@auto-engineer/message-bus": "0.13.3"
+    "@auto-engineer/file-store": "0.15.0",
+    "@auto-engineer/message-bus": "0.15.0"
   },
   "devDependencies": {
     "@types/cors": "^2.8.17",
@@ -19,7 +23,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.0.1",
+  "version": "0.15.0",
   "scripts": {
     "build": "tsc && tsx ../../scripts/fix-esm-imports.ts",
     "test": "vitest run --reporter=dot",

package/src/builder/define.specs.ts

@@ -148,7 +148,7 @@ describe('settled()', () => {
     const graph = pipeline.toGraph();
     const settledNode = graph.nodes.find((n) => n.id.startsWith('settled:'));
     expect(settledNode).toBeDefined();
-    expect(settledNode?.label).toBe('settled(CheckTests, CheckTypes)');
+    expect(settledNode?.label).toBe('Settled');
   });
 
   it('should accept options-first dispatch with dispatches array', () => {
@@ -426,8 +426,8 @@ describe('forEach() and groupInto() - Phased Execution', () => {
       .build();
 
     const handler = pipeline.descriptor.handlers[0] as ForEachPhasedDescriptor;
-    expect(handler.completion.successEvent).toBe('AllItemsProcessed');
-    expect(handler.completion.failureEvent).toBe('ProcessingFailed');
+    expect(handler.completion.successEvent.name).toBe('AllItemsProcessed');
+    expect(handler.completion.failureEvent.name).toBe('ProcessingFailed');
   });
 
   it('should chain on() from PhasedTerminal', () => {

package/src/builder/define.ts

@@ -73,9 +73,11 @@ export interface PhasedBuilder<T> {
   process(commandType: string, dataFactory: (item: T) => Record<string, unknown>): PhasedChain<T>;
 }
 
+export type CompletionEventConfig = string | { name: string; displayName: string };
+
 export interface CompletionConfig {
-  success: string;
-  failure: string;
+  success: CompletionEventConfig;
+  failure: CompletionEventConfig;
   itemKey: (event: Event) => string;
 }
 
@@ -132,6 +134,13 @@ export interface HandleChain {
   build(): Pipeline;
 }
 
+function normalizeCompletionEvent(config: CompletionEventConfig): { name: string; displayName?: string } {
+  if (typeof config === 'string') {
+    return { name: config };
+  }
+  return config;
+}
+
 class PipelineBuilderImpl implements PipelineBuilder {
   private readonly name: string;
   private versionValue?: string;
@@ -225,10 +234,14 @@ function processForEachPhasedHandler(ctx: GraphBuilderContext, handler: ForEachP
   const sampleCmd = handler.emitFactory({}, '', { type: '', data: {} });
   addNode(ctx, `cmd:${sampleCmd.commandType}`, 'command', sampleCmd.commandType);
   ctx.edges.push({ from: `evt:${handler.eventType}`, to: `cmd:${sampleCmd.commandType}` });
-  addNode(ctx, `evt:${handler.completion.successEvent}`, 'event', handler.completion.successEvent);
-  addNode(ctx, `evt:${handler.completion.failureEvent}`, 'event', handler.completion.failureEvent);
-  ctx.edges.push({ from: `cmd:${sampleCmd.commandType}`, to: `evt:${handler.completion.successEvent}` });
-  ctx.edges.push({ from: `cmd:${sampleCmd.commandType}`, to: `evt:${handler.completion.failureEvent}` });
+  const successEvent = handler.completion.successEvent;
+  const failureEvent = handler.completion.failureEvent;
+  const successLabel = successEvent.displayName ?? successEvent.name;
+  const failureLabel = failureEvent.displayName ?? failureEvent.name;
+  addNode(ctx, `evt:${successEvent.name}`, 'event', successLabel);
+  addNode(ctx, `evt:${failureEvent.name}`, 'event', failureLabel);
+  ctx.edges.push({ from: `cmd:${sampleCmd.commandType}`, to: `evt:${successEvent.name}` });
+  ctx.edges.push({ from: `cmd:${sampleCmd.commandType}`, to: `evt:${failureEvent.name}` });
 }
 
 function processCustomHandler(ctx: GraphBuilderContext, handler: CustomHandlerDescriptor): void {
@@ -243,7 +256,7 @@ function processCustomHandler(ctx: GraphBuilderContext, handler: CustomHandlerDe
 
 function processSettledHandler(ctx: GraphBuilderContext, handler: SettledHandlerDescriptor): void {
   const settledNodeId = `settled:${handler.commandTypes.join(',')}`;
-  addNode(ctx, settledNodeId, 'settled', `settled(${handler.commandTypes.join(', ')})`);
+  addNode(ctx, settledNodeId, 'settled', 'Settled');
 
   for (const commandType of handler.commandTypes) {
     addNode(ctx, `cmd:${commandType}`, 'command', commandType);
@@ -571,8 +584,8 @@ class PhasedBuilderImpl<T> implements PhasedBuilder<T> {
 class PhasedChainImpl<T> implements PhasedChain<T> {
   private stopOnFailureFlag = false;
   private completionConfig?: {
-    successEvent: string;
-    failureEvent: string;
+    successEvent: { name: string; displayName?: string };
+    failureEvent: { name: string; displayName?: string };
     itemKey: KeyExtractor;
   };
 
@@ -594,8 +607,8 @@ class PhasedChainImpl<T> implements PhasedChain<T> {
 
   onComplete(config: CompletionConfig): PhasedTerminal {
     this.completionConfig = {
-      successEvent: config.success,
-      failureEvent: config.failure,
+      successEvent: normalizeCompletionEvent(config.success),
+      failureEvent: normalizeCompletionEvent(config.failure),
       itemKey: config.itemKey as KeyExtractor,
     };
     return new PhasedTerminalImpl(this, this.parent);

package/src/core/descriptors.ts

@@ -44,6 +44,11 @@ export interface RunAwaitHandlerDescriptor {
   onFailure?: GatherEventConfig<FailureContext>;
 }
 
+export type CompletionEventDescriptor = {
+  name: string;
+  displayName?: string;
+};
+
 export interface ForEachPhasedDescriptor {
   type: 'foreach-phased';
   eventType: string;
@@ -54,8 +59,8 @@ export interface ForEachPhasedDescriptor {
   stopOnFailure: boolean;
   emitFactory: (item: unknown, phase: string, event: Event) => CommandDispatch;
   completion: {
-    successEvent: string;
-    failureEvent: string;
+    successEvent: CompletionEventDescriptor;
+    failureEvent: CompletionEventDescriptor;
     itemKey: KeyExtractor;
   };
 }

package/src/graph/filter-graph.specs.ts

@@ -0,0 +1,241 @@
+import { describe, expect, it } from 'vitest';
+
+import { filterGraph } from './filter-graph';
+import type { GraphIR } from './types';
+
+describe('filterGraph', () => {
+  describe('P1: filter nodes by type', () => {
+    it('should remove nodes of excluded type', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'evt:Start', type: 'event', label: 'Start' },
+          { id: 'cmd:Process', type: 'command', label: 'Process' },
+        ],
+        edges: [{ from: 'evt:Start', to: 'cmd:Process' }],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['event'], maintainEdges: false });
+
+      expect(result).toEqual({
+        nodes: [{ id: 'cmd:Process', type: 'command', label: 'Process' }],
+        edges: [],
+      });
+    });
+  });
+
+  describe('P2: dangling edge removal', () => {
+    it('should remove edges referencing filtered nodes in a chain', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'cmd:B', type: 'command', label: 'B' },
+          { id: 'evt:C', type: 'event', label: 'C' },
+        ],
+        edges: [
+          { from: 'evt:A', to: 'cmd:B' },
+          { from: 'cmd:B', to: 'evt:C' },
+        ],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['event'], maintainEdges: false });
+
+      expect(result).toEqual({
+        nodes: [{ id: 'cmd:B', type: 'command', label: 'B' }],
+        edges: [],
+      });
+    });
+  });
+
+  describe('P3: single-hop edge maintenance', () => {
+    it('should reconnect edges through single filtered node', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'cmd:B', type: 'command', label: 'B' },
+          { id: 'evt:C', type: 'event', label: 'C' },
+        ],
+        edges: [
+          { from: 'evt:A', to: 'cmd:B' },
+          { from: 'cmd:B', to: 'evt:C' },
+        ],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['command'], maintainEdges: true });
+
+      expect(result).toEqual({
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'evt:C', type: 'event', label: 'C' },
+        ],
+        edges: [{ from: 'evt:A', to: 'evt:C' }],
+      });
+    });
+  });
+
+  describe('P4: multi-hop edge maintenance', () => {
+    it('should reconnect edges through multiple consecutive filtered nodes', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'cmd:B1', type: 'command', label: 'B1' },
+          { id: 'cmd:B2', type: 'command', label: 'B2' },
+          { id: 'evt:C', type: 'event', label: 'C' },
+        ],
+        edges: [
+          { from: 'evt:A', to: 'cmd:B1' },
+          { from: 'cmd:B1', to: 'cmd:B2' },
+          { from: 'cmd:B2', to: 'evt:C' },
+        ],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['command'], maintainEdges: true });
+
+      expect(result).toEqual({
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'evt:C', type: 'event', label: 'C' },
+        ],
+        edges: [{ from: 'evt:A', to: 'evt:C' }],
+      });
+    });
+  });
+
+  describe('P5: self-loop preservation', () => {
+    it('should preserve self-loops created by edge reconnection and mark as backLink', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'cmd:A', type: 'command', label: 'A' },
+          { id: 'evt:B', type: 'event', label: 'B' },
+        ],
+        edges: [
+          { from: 'cmd:A', to: 'evt:B' },
+          { from: 'evt:B', to: 'cmd:A' },
+        ],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['event'], maintainEdges: true });
+
+      expect(result).toEqual({
+        nodes: [{ id: 'cmd:A', type: 'command', label: 'A' }],
+        edges: [{ from: 'cmd:A', to: 'cmd:A', backLink: true }],
+      });
+    });
+  });
+
+  describe('P6: preserve edge properties', () => {
+    it('should preserve edge labels through reconnection', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'cmd:B', type: 'command', label: 'B' },
+          { id: 'evt:C', type: 'event', label: 'C' },
+        ],
+        edges: [
+          { from: 'evt:A', to: 'cmd:B', label: 'triggers' },
+          { from: 'cmd:B', to: 'evt:C' },
+        ],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['command'], maintainEdges: true });
+
+      expect(result.edges[0]).toEqual({ from: 'evt:A', to: 'evt:C', label: 'triggers' });
+    });
+
+    it('should preserve backLink property through reconnection', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'cmd:B', type: 'command', label: 'B' },
+          { id: 'evt:C', type: 'event', label: 'C' },
+        ],
+        edges: [
+          { from: 'evt:A', to: 'cmd:B', backLink: true },
+          { from: 'cmd:B', to: 'evt:C' },
+        ],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['command'], maintainEdges: true });
+
+      expect(result.edges[0]).toEqual({ from: 'evt:A', to: 'evt:C', backLink: true });
+    });
+  });
+
+  describe('P7: multiple excluded types', () => {
+    it('should filter multiple node types', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'cmd:B', type: 'command', label: 'B' },
+          { id: 'settled:C', type: 'settled', label: 'C' },
+        ],
+        edges: [
+          { from: 'evt:A', to: 'cmd:B' },
+          { from: 'cmd:B', to: 'settled:C' },
+        ],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['event', 'settled'], maintainEdges: true });
+
+      expect(result).toEqual({
+        nodes: [{ id: 'cmd:B', type: 'command', label: 'B' }],
+        edges: [],
+      });
+    });
+  });
+
+  describe('P8: edge deduplication', () => {
+    it('should deduplicate edges when multiple paths merge', () => {
+      const graph: GraphIR = {
+        nodes: [
+          { id: 'evt:A', type: 'event', label: 'A' },
+          { id: 'cmd:B1', type: 'command', label: 'B1' },
+          { id: 'cmd:B2', type: 'command', label: 'B2' },
+          { id: 'evt:C', type: 'event', label: 'C' },
+        ],
+        edges: [
+          { from: 'evt:A', to: 'cmd:B1' },
+          { from: 'evt:A', to: 'cmd:B2' },
+          { from: 'cmd:B1', to: 'evt:C' },
+          { from: 'cmd:B2', to: 'evt:C' },
+        ],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['command'], maintainEdges: true });
+
+      expect(result.edges).toHaveLength(1);
+      expect(result.edges[0]).toEqual({ from: 'evt:A', to: 'evt:C' });
+    });
+  });
+
+  describe('P9: edge cases', () => {
+    it('should return empty graph when all nodes filtered', () => {
+      const graph: GraphIR = {
+        nodes: [{ id: 'evt:A', type: 'event', label: 'A' }],
+        edges: [],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: ['event'], maintainEdges: true });
+
+      expect(result).toEqual({ nodes: [], edges: [] });
+    });
+
+    it('should return unchanged graph when no types excluded', () => {
+      const graph: GraphIR = {
+        nodes: [{ id: 'evt:A', type: 'event', label: 'A' }],
+        edges: [],
+      };
+
+      const result = filterGraph(graph, { excludeTypes: [], maintainEdges: false });
+
+      expect(result).toEqual(graph);
+    });
+
+    it('should handle empty graph input', () => {
+      const graph: GraphIR = { nodes: [], edges: [] };
+
+      const result = filterGraph(graph, { excludeTypes: ['event'], maintainEdges: true });
+
+      expect(result).toEqual({ nodes: [], edges: [] });
+    });
+  });
+});

package/src/graph/filter-graph.ts

@@ -0,0 +1,111 @@
+import type { FilterOptions, GraphEdge, GraphIR, NodeType } from './types';
+
+export function filterGraph(graph: GraphIR, options: FilterOptions): GraphIR {
+  const excludeSet = new Set<NodeType>(options.excludeTypes);
+  const remainingNodes = graph.nodes.filter((node) => !excludeSet.has(node.type));
+  const removedNodeIds = new Set(graph.nodes.filter((n) => excludeSet.has(n.type)).map((n) => n.id));
+
+  if (!options.maintainEdges) {
+    return filterEdgesSimple(graph, remainingNodes, removedNodeIds);
+  }
+
+  return reconnectEdges(graph, remainingNodes, removedNodeIds);
+}
+
+function filterEdgesSimple(graph: GraphIR, remainingNodes: GraphIR['nodes'], removedNodeIds: Set<string>): GraphIR {
+  const remainingEdges = graph.edges.filter((edge) => !removedNodeIds.has(edge.from) && !removedNodeIds.has(edge.to));
+  return { nodes: remainingNodes, edges: remainingEdges };
+}
+
+function reconnectEdges(graph: GraphIR, remainingNodes: GraphIR['nodes'], removedNodeIds: Set<string>): GraphIR {
+  const outgoingEdges = buildOutgoingEdgesMap(graph.edges);
+  const reconnectedEdges: GraphEdge[] = [];
+  const seenEdges = new Set<string>();
+
+  for (const edge of graph.edges) {
+    if (!removedNodeIds.has(edge.from)) {
+      processEdge(edge, removedNodeIds, outgoingEdges, reconnectedEdges, seenEdges);
+    }
+  }
+
+  return { nodes: remainingNodes, edges: reconnectedEdges };
+}
+
+function buildOutgoingEdgesMap(edges: GraphEdge[]): Map<string, GraphEdge[]> {
+  const outgoingEdges = new Map<string, GraphEdge[]>();
+  for (const edge of edges) {
+    const existing = outgoingEdges.get(edge.from) ?? [];
+    existing.push(edge);
+    outgoingEdges.set(edge.from, existing);
+  }
+  return outgoingEdges;
+}
+
+function processEdge(
+  edge: GraphEdge,
+  removedNodeIds: Set<string>,
+  outgoingEdges: Map<string, GraphEdge[]>,
+  reconnectedEdges: GraphEdge[],
+  seenEdges: Set<string>,
+): void {
+  if (!removedNodeIds.has(edge.to)) {
+    addEdgeIfNew(edge, reconnectedEdges, seenEdges);
+    return;
+  }
+
+  const targets = findFinalTargets(edge.to, removedNodeIds, outgoingEdges);
+  for (const target of targets) {
+    const isSelfLoop = target === edge.from;
+    addReconnectedEdge(edge, target, reconnectedEdges, seenEdges, isSelfLoop);
+  }
+}
+
+function addEdgeIfNew(edge: GraphEdge, reconnectedEdges: GraphEdge[], seenEdges: Set<string>): void {
+  const key = `${edge.from}->${edge.to}`;
+  if (!seenEdges.has(key)) {
+    seenEdges.add(key);
+    reconnectedEdges.push(edge);
+  }
+}
+
+function addReconnectedEdge(
+  sourceEdge: GraphEdge,
+  target: string,
+  reconnectedEdges: GraphEdge[],
+  seenEdges: Set<string>,
+  isSelfLoop: boolean,
+): void {
+  const key = `${sourceEdge.from}->${target}`;
+  if (seenEdges.has(key)) {
+    return;
+  }
+  seenEdges.add(key);
+
+  const newEdge: GraphEdge = { from: sourceEdge.from, to: target };
+  if (sourceEdge.label !== undefined) {
+    newEdge.label = sourceEdge.label;
+  }
+  if (isSelfLoop || sourceEdge.backLink === true) {
+    newEdge.backLink = true;
+  }
+  reconnectedEdges.push(newEdge);
+}
+
+function findFinalTargets(
+  nodeId: string,
+  removedNodeIds: Set<string>,
+  outgoingEdges: Map<string, GraphEdge[]>,
+): string[] {
+  const edges = outgoingEdges.get(nodeId) ?? [];
+  const targets: string[] = [];
+
+  for (const edge of edges) {
+    if (removedNodeIds.has(edge.to)) {
+      targets.push(...findFinalTargets(edge.to, removedNodeIds, outgoingEdges));
+    } else {
+      targets.push(edge.to);
+    }
+  }
+
+  return targets;
+}

package/src/graph/types.ts

@@ -1,9 +1,14 @@
 export type NodeType = 'event' | 'command' | 'settled';
 
+export type NodeStatus = 'idle' | 'running' | 'success' | 'error';
+
 export interface GraphNode {
   id: string;
   type: NodeType;
   label: string;
+  status?: NodeStatus;
+  pendingCount?: number;
+  endedCount?: number;
 }
 
 export interface GraphEdge {
@@ -17,3 +22,8 @@ export interface GraphIR {
   nodes: GraphNode[];
   edges: GraphEdge[];
 }
+
+export interface FilterOptions {
+  excludeTypes: NodeType[];
+  maintainEdges: boolean;
+}

package/src/index.ts

@@ -30,11 +30,10 @@ export type {
 } from './core/descriptors';
 export type { Command, CommandDispatch, Event } from './core/types';
 export { dispatch } from './core/types';
-
 export type { GraphEdge, GraphIR, GraphNode, NodeType } from './graph/types';
 export type { EventLoggerOptions, LogEntry } from './logging/event-logger';
 export { EventLogger } from './logging/event-logger';
-
+export type { AwaitEvent, AwaitTrackerDocument } from './projections/await-tracker-projection';
 export { AwaitTracker } from './runtime/await-tracker';
 export type { PipelineContext, RuntimeConfig } from './runtime/context';
 export { EventCommandMapper } from './runtime/event-command-map';

package/src/projections/await-tracker-projection.ts

@@ -0,0 +1,68 @@
+export interface AwaitTrackerDocument {
+  [key: string]: unknown;
+  correlationId: string;
+  pendingKeys: string[];
+  results: Record<string, unknown>;
+  status: 'pending' | 'completed';
+}
+
+export interface AwaitStartedEvent {
+  type: 'AwaitStarted';
+  data: {
+    correlationId: string;
+    keys: string[];
+  };
+}
+
+export interface AwaitItemCompletedEvent {
+  type: 'AwaitItemCompleted';
+  data: {
+    correlationId: string;
+    key: string;
+    result: unknown;
+  };
+}
+
+export interface AwaitCompletedEvent {
+  type: 'AwaitCompleted';
+  data: {
+    correlationId: string;
+  };
+}
+
+export type AwaitEvent = AwaitStartedEvent | AwaitItemCompletedEvent | AwaitCompletedEvent;
+
+export function evolve(document: AwaitTrackerDocument | null, event: AwaitEvent): AwaitTrackerDocument {
+  switch (event.type) {
+    case 'AwaitStarted': {
+      const { correlationId, keys } = event.data;
+      return {
+        correlationId,
+        pendingKeys: [...keys],
+        results: {},
+        status: 'pending',
+      };
+    }
+    case 'AwaitItemCompleted': {
+      if (document === null) {
+        throw new Error('Cannot apply AwaitItemCompleted to null document');
+      }
+      const { key, result } = event.data;
+      const newPendingKeys = document.pendingKeys.filter((k) => k !== key);
+      return {
+        ...document,
+        pendingKeys: newPendingKeys,
+        results: { ...document.results, [key]: result },
+      };
+    }
+    case 'AwaitCompleted': {
+      if (document === null) {
+        throw new Error('Cannot apply AwaitCompleted to null document');
+      }
+      return {
+        ...document,
+        status: 'completed',
+      };
+    }
+  }
+}

package/src/projections/index.ts

@@ -0,0 +1,11 @@
+export {
+  evolve as evolveItemStatus,
+  type ItemStatusChangedEvent,
+  type ItemStatusDocument,
+} from './item-status-projection';
+export { evolve as evolveLatestRun, type LatestRunDocument } from './latest-run-projection';
+export {
+  evolve as evolveNodeStatus,
+  type NodeStatusChangedEvent,
+  type NodeStatusDocument,
+} from './node-status-projection';