@auto-engineer/pipeline 0.0.1 → 0.15.0

package/src/server/sse-manager.specs.ts

@@ -49,14 +49,8 @@ describe('SSEManager', () => {
     manager = new SSEManager();
   });
 
-  describe('client management', () => {
-    it('should add client', () => {
-      const res = createMockResponse();
-      manager.addClient('c1', res);
-      expect(manager.clientCount).toBe(1);
-    });
-
-    it('should set correct SSE headers', () => {
+  describe('client connection', () => {
+    it('should set correct SSE headers when client connects', () => {
       const res = createMockResponse();
       manager.addClient('c1', res);
       expect(res.writeHeadMock).toHaveBeenCalledWith(200, {
@@ -73,23 +67,43 @@ describe('SSEManager', () => {
       expect(res.written[0]).toBe(':\n\n');
     });
 
-    it('should remove client', () => {
+    it('should receive broadcasts after connecting', () => {
+      const res = createMockResponse();
+      manager.addClient('c1', res);
+
+      const event: Event = { type: 'TestEvent', data: { foo: 'bar' } };
+      manager.broadcast(event);
+
+      expect(res.written).toContainEqual(`data: ${JSON.stringify(event)}\n\n`);
+    });
+  });
+
+  describe('client removal', () => {
+    it('should not receive broadcasts after explicit removal', () => {
       const res = createMockResponse();
       manager.addClient('c1', res);
       manager.removeClient('c1');
-      expect(manager.clientCount).toBe(0);
+
+      const event: Event = { type: 'TestEvent', data: { foo: 'bar' } };
+      manager.broadcast(event);
+
+      expect(res.written).toHaveLength(1);
     });
 
-    it('should cleanup client on response close', () => {
+    it('should not receive broadcasts after connection close', () => {
       const res = createMockResponse();
       manager.addClient('c1', res);
       res.triggerClose();
-      expect(manager.clientCount).toBe(0);
+
+      const event: Event = { type: 'TestEvent', data: { foo: 'bar' } };
+      manager.broadcast(event);
+
+      expect(res.written).toHaveLength(1);
     });
   });
 
   describe('broadcasting', () => {
-    it('should broadcast event to all clients', () => {
+    it('should broadcast event to all connected clients', () => {
       const res1 = createMockResponse();
       const res2 = createMockResponse();
 
@@ -103,21 +117,21 @@ describe('SSEManager', () => {
       expect(res2.written).toContainEqual(`data: ${JSON.stringify(event)}\n\n`);
     });
 
-    it('should filter by correlationId when set', () => {
-      const res1 = createMockResponse();
-      const res2 = createMockResponse();
+    it('should filter events by correlationId when client subscribes to specific correlation', () => {
+      const filteredRes = createMockResponse();
+      const unfilteredRes = createMockResponse();
 
-      manager.addClient('c1', res1, 'workflow-123');
-      manager.addClient('c2', res2);
+      manager.addClient('c1', filteredRes, 'workflow-123');
+      manager.addClient('c2', unfilteredRes);
 
       const event: Event = { type: 'TestEvent', correlationId: 'workflow-456', data: {} };
       manager.broadcast(event);
 
-      expect(res1.written.length).toBe(1);
-      expect(res2.written.length).toBe(2);
+      expect(filteredRes.written).toHaveLength(1);
+      expect(unfilteredRes.written).toHaveLength(2);
     });
 
-    it('should send to filtered client when correlationId matches', () => {
+    it('should send event to filtered client when correlationId matches', () => {
       const res = createMockResponse();
       manager.addClient('c1', res, 'workflow-123');
 
@@ -127,19 +141,19 @@ describe('SSEManager', () => {
       expect(res.written).toContainEqual(`data: ${JSON.stringify(event)}\n\n`);
     });
 
-    it('should not send to filtered client when correlationId does not match', () => {
+    it('should not send event to filtered client when correlationId does not match', () => {
       const res = createMockResponse();
       manager.addClient('c1', res, 'workflow-123');
 
       const event: Event = { type: 'TestEvent', correlationId: 'workflow-456', data: {} };
       manager.broadcast(event);
 
-      expect(res.written.length).toBe(1);
+      expect(res.written).toHaveLength(1);
     });
   });
 
   describe('broadcast error handling', () => {
-    it('should not throw when client.response.write fails', () => {
+    it('should not throw when client write fails', () => {
       const res = createMockResponse();
       manager.addClient('c1', res);
 
@@ -152,21 +166,28 @@ describe('SSEManager', () => {
       expect(() => manager.broadcast(event)).not.toThrow();
     });
 
-    it('should remove failed client from clients map', () => {
+    it('should stop sending to failed client on subsequent broadcasts', () => {
       const res = createMockResponse();
-      vi.mocked(res.write)
-        .mockImplementationOnce(() => true)
-        .mockImplementation(() => {
+      let writeCallCount = 0;
+      vi.mocked(res.write).mockImplementation((data: string) => {
+        writeCallCount++;
+        if (writeCallCount > 2) {
           throw new Error('Connection reset');
-        });
+        }
+        res.written.push(data);
+        return true;
+      });
 
       manager.addClient('c1', res);
-      expect(manager.clientCount).toBe(1);
 
-      const event: Event = { type: 'TestEvent', data: {} };
-      manager.broadcast(event);
+      const event1: Event = { type: 'TestEvent1', data: {} };
+      manager.broadcast(event1);
+
+      const event2: Event = { type: 'TestEvent2', data: {} };
+      manager.broadcast(event2);
 
-      expect(manager.clientCount).toBe(0);
+      const eventMessages = res.written.filter((w) => w.startsWith('data:'));
+      expect(eventMessages).toHaveLength(1);
     });
 
     it('should continue broadcasting to other clients after one fails', () => {
@@ -186,12 +207,11 @@ describe('SSEManager', () => {
       manager.broadcast(event);
 
       expect(workingRes.written).toContainEqual(`data: ${JSON.stringify(event)}\n\n`);
-      expect(manager.clientCount).toBe(1);
     });
   });
 
   describe('closeAll', () => {
-    it('should close all clients', () => {
+    it('should end all client connections', () => {
       const res1 = createMockResponse();
       const res2 = createMockResponse();
 
@@ -200,9 +220,20 @@ describe('SSEManager', () => {
 
       manager.closeAll();
 
-      expect(manager.clientCount).toBe(0);
       expect(res1.endMock).toHaveBeenCalled();
       expect(res2.endMock).toHaveBeenCalled();
     });
+
+    it('should not send broadcasts after closeAll', () => {
+      const res = createMockResponse();
+      manager.addClient('c1', res);
+
+      manager.closeAll();
+
+      const event: Event = { type: 'TestEvent', data: {} };
+      manager.broadcast(event);
+
+      expect(res.written).toHaveLength(1);
+    });
   });
 });

package/src/store/index.ts

@@ -0,0 +1,2 @@
+export { createPipelineEventStore, type PipelineEventStoreContext } from './pipeline-event-store';
+export { type CommandStats, PipelineReadModel } from './pipeline-read-model';

package/src/store/pipeline-event-store.specs.ts

@@ -0,0 +1,309 @@
+import { describe, expect, it } from 'vitest';
+import type { ItemStatusDocument } from '../projections/item-status-projection';
+import type { NodeStatusDocument } from '../projections/node-status-projection';
+import type { PhasedExecutionDocument } from '../projections/phased-execution-projection';
+import type { SettledInstanceDocument } from '../projections/settled-instance-projection';
+import { createPipelineEventStore } from './pipeline-event-store';
+
+describe('PipelineEventStore', () => {
+  describe('appendToStream', () => {
+    it('should append events and update projections', async () => {
+      const { eventStore, database, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('pipeline-c1', [
+          {
+            type: 'ItemStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandType: 'Cmd',
+              itemKey: 'a',
+              requestId: 'r1',
+              status: 'running',
+              attemptCount: 1,
+            },
+          },
+        ]);
+
+        const collection = database.collection<ItemStatusDocument & { _id: string }>('ItemStatus');
+        const items = await collection.find();
+
+        expect(items.length).toBe(1);
+        expect(items[0]?.status).toBe('running');
+      } finally {
+        await close();
+      }
+    });
+
+    it('should project NodeStatusChanged events', async () => {
+      const { eventStore, database, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('pipeline-c1', [
+          {
+            type: 'NodeStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandName: 'Cmd',
+              nodeId: 'node-1',
+              status: 'running',
+              previousStatus: 'idle',
+              pendingCount: 1,
+              endedCount: 0,
+            },
+          },
+        ]);
+
+        const collection = database.collection<NodeStatusDocument & { _id: string }>('NodeStatus');
+        const nodes = await collection.find();
+
+        expect(nodes.length).toBe(1);
+        expect(nodes[0]?.status).toBe('running');
+      } finally {
+        await close();
+      }
+    });
+
+    it('should update existing projection documents', async () => {
+      const { eventStore, database, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('pipeline-c1', [
+          {
+            type: 'ItemStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandType: 'Cmd',
+              itemKey: 'a',
+              requestId: 'r1',
+              status: 'running',
+              attemptCount: 1,
+            },
+          },
+        ]);
+
+        await eventStore.appendToStream('pipeline-c1', [
+          {
+            type: 'ItemStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandType: 'Cmd',
+              itemKey: 'a',
+              requestId: 'r1',
+              status: 'success',
+              attemptCount: 1,
+            },
+          },
+        ]);
+
+        const collection = database.collection<ItemStatusDocument & { _id: string }>('ItemStatus');
+        const items = await collection.find();
+
+        expect(items.length).toBe(1);
+        expect(items[0]?.status).toBe('success');
+      } finally {
+        await close();
+      }
+    });
+  });
+
+  describe('readModel integration', () => {
+    it('should provide working read model queries', async () => {
+      const { eventStore, readModel, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('pipeline-c1', [
+          {
+            type: 'ItemStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandType: 'Cmd',
+              itemKey: 'a',
+              requestId: 'r1',
+              status: 'running',
+              attemptCount: 1,
+            },
+          },
+          {
+            type: 'ItemStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandType: 'Cmd',
+              itemKey: 'b',
+              requestId: 'r2',
+              status: 'success',
+              attemptCount: 1,
+            },
+          },
+        ]);
+
+        const stats = await readModel.computeCommandStats('c1', 'Cmd');
+
+        expect(stats).toEqual({
+          pendingCount: 1,
+          endedCount: 1,
+          aggregateStatus: 'running',
+        });
+      } finally {
+        await close();
+      }
+    });
+
+    it('should detect correlation via read model', async () => {
+      const { eventStore, readModel, close } = createPipelineEventStore();
+      try {
+        expect(await readModel.hasCorrelation('c1')).toBe(false);
+
+        await eventStore.appendToStream('pipeline-c1', [
+          {
+            type: 'NodeStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandName: 'Cmd',
+              nodeId: 'node-1',
+              status: 'running',
+              previousStatus: 'idle',
+              pendingCount: 1,
+              endedCount: 0,
+            },
+          },
+        ]);
+
+        expect(await readModel.hasCorrelation('c1')).toBe(true);
+      } finally {
+        await close();
+      }
+    });
+  });
+
+  describe('settled instance projection', () => {
+    it('should project SettledInstanceCreated events', async () => {
+      const { eventStore, database, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('settled-c1', [
+          {
+            type: 'SettledInstanceCreated',
+            data: {
+              templateId: 'template-CmdA,CmdB',
+              correlationId: 'c1',
+              commandTypes: ['CmdA', 'CmdB'],
+            },
+          },
+        ]);
+
+        const collection = database.collection<SettledInstanceDocument & { _id: string }>('SettledInstance');
+        const instances = await collection.find();
+
+        expect(instances.length).toBe(1);
+        expect(instances[0]?.status).toBe('active');
+        expect(instances[0]?.commandTrackers).toHaveLength(2);
+      } finally {
+        await close();
+      }
+    });
+
+    it('should update settled instance through lifecycle events', async () => {
+      const { eventStore, database, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('settled-c1', [
+          {
+            type: 'SettledInstanceCreated',
+            data: {
+              templateId: 'template-CmdA',
+              correlationId: 'c1',
+              commandTypes: ['CmdA'],
+            },
+          },
+          {
+            type: 'SettledCommandStarted',
+            data: {
+              templateId: 'template-CmdA',
+              correlationId: 'c1',
+              commandType: 'CmdA',
+            },
+          },
+        ]);
+
+        const collection = database.collection<SettledInstanceDocument & { _id: string }>('SettledInstance');
+        const instances = await collection.find();
+
+        expect(instances.length).toBe(1);
+        expect(instances[0]?.commandTrackers[0]?.hasStarted).toBe(true);
+        expect(instances[0]?.commandTrackers[0]?.hasCompleted).toBe(false);
+      } finally {
+        await close();
+      }
+    });
+  });
+
+  describe('phased execution projection', () => {
+    it('should project PhasedExecutionStarted events', async () => {
+      const { eventStore, database, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('phased-c1', [
+          {
+            type: 'PhasedExecutionStarted',
+            data: {
+              executionId: 'exec-1',
+              correlationId: 'c1',
+              handlerId: 'handler-1',
+              triggerEvent: { type: 'TestEvent', correlationId: 'c1', data: {} },
+              items: [{ key: 'a', phase: 'prepare', dispatched: false, completed: false }],
+              phases: ['prepare', 'execute'],
+            },
+          },
+        ]);
+
+        const collection = database.collection<PhasedExecutionDocument & { _id: string }>('PhasedExecution');
+        const executions = await collection.find();
+
+        expect(executions.length).toBe(1);
+        expect(executions[0]?.status).toBe('active');
+        expect(executions[0]?.items).toHaveLength(1);
+      } finally {
+        await close();
+      }
+    });
+
+    it('should update phased execution through lifecycle events', async () => {
+      const { eventStore, database, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('phased-c1', [
+          {
+            type: 'PhasedExecutionStarted',
+            data: {
+              executionId: 'exec-1',
+              correlationId: 'c1',
+              handlerId: 'handler-1',
+              triggerEvent: { type: 'TestEvent', correlationId: 'c1', data: {} },
+              items: [{ key: 'a', phase: 'prepare', dispatched: false, completed: false }],
+              phases: ['prepare'],
+            },
+          },
+          {
+            type: 'PhasedItemDispatched',
+            data: { executionId: 'exec-1', itemKey: 'a', phase: 'prepare' },
+          },
+          {
+            type: 'PhasedItemCompleted',
+            data: {
+              executionId: 'exec-1',
+              itemKey: 'a',
+              resultEvent: { type: 'ItemDone', correlationId: 'c1', data: {} },
+            },
+          },
+          {
+            type: 'PhasedExecutionCompleted',
+            data: { executionId: 'exec-1', success: true, results: ['a'] },
+          },
+        ]);
+
+        const collection = database.collection<PhasedExecutionDocument & { _id: string }>('PhasedExecution');
+        const executions = await collection.find();
+
+        expect(executions.length).toBe(1);
+        expect(executions[0]?.status).toBe('completed');
+        expect(executions[0]?.items[0]?.dispatched).toBe(true);
+        expect(executions[0]?.items[0]?.completed).toBe(true);
+      } finally {
+        await close();
+      }
+    });
+  });
+});

package/src/store/pipeline-event-store.ts

@@ -0,0 +1,156 @@
+import {
+  getInMemoryDatabase,
+  getInMemoryEventStore,
+  type InMemoryDatabase,
+  type InMemoryEventStore,
+  type InMemoryReadEventMetadata,
+  inlineProjections,
+  inMemorySingleStreamProjection,
+} from '@event-driven-io/emmett';
+import type { AwaitEvent, AwaitTrackerDocument } from '../projections/await-tracker-projection';
+import { evolve as evolveAwaitTracker } from '../projections/await-tracker-projection';
+import type { ItemStatusChangedEvent, ItemStatusDocument } from '../projections/item-status-projection';
+import { evolve as evolveItemStatus } from '../projections/item-status-projection';
+import type { LatestRunDocument } from '../projections/latest-run-projection';
+import { evolve as evolveLatestRun } from '../projections/latest-run-projection';
+import type { MessageLogDocument, MessageLogEvent } from '../projections/message-log-projection';
+import { evolve as evolveMessageLog } from '../projections/message-log-projection';
+import type { NodeStatusChangedEvent, NodeStatusDocument } from '../projections/node-status-projection';
+import { evolve as evolveNodeStatus } from '../projections/node-status-projection';
+import type { PhasedExecutionDocument, PhasedExecutionEvent } from '../projections/phased-execution-projection';
+import { evolve as evolvePhasedExecution } from '../projections/phased-execution-projection';
+import type { SettledEvent, SettledInstanceDocument } from '../projections/settled-instance-projection';
+import { evolve as evolveSettledInstance } from '../projections/settled-instance-projection';
+import type { StatsDocument } from '../projections/stats-projection';
+import { evolve as evolveStats } from '../projections/stats-projection';
+import { PipelineReadModel } from './pipeline-read-model';
+
+interface PipelineRunStartedEvent {
+  type: 'PipelineRunStarted';
+  data: {
+    correlationId: string;
+    triggerCommand: string;
+  };
+}
+
+function createProjections() {
+  const itemStatusProjection = inMemorySingleStreamProjection<ItemStatusDocument, ItemStatusChangedEvent>({
+    collectionName: 'ItemStatus',
+    canHandle: ['ItemStatusChanged'],
+    getDocumentId: (event) => `${event.data.correlationId}-${event.data.commandType}-${event.data.itemKey}`,
+    evolve: (document: ItemStatusDocument | null, event: ItemStatusChangedEvent) => evolveItemStatus(document, event),
+  });
+
+  const nodeStatusProjection = inMemorySingleStreamProjection<NodeStatusDocument, NodeStatusChangedEvent>({
+    collectionName: 'NodeStatus',
+    canHandle: ['NodeStatusChanged'],
+    getDocumentId: (event) => `${event.data.correlationId}-${event.data.commandName}`,
+    evolve: (document: NodeStatusDocument | null, event: NodeStatusChangedEvent) => evolveNodeStatus(document, event),
+  });
+
+  const latestRunProjection = inMemorySingleStreamProjection<LatestRunDocument, PipelineRunStartedEvent>({
+    collectionName: 'LatestRun',
+    canHandle: ['PipelineRunStarted'],
+    getDocumentId: () => 'singleton',
+    evolve: (document: LatestRunDocument | null, event: PipelineRunStartedEvent) => evolveLatestRun(document, event),
+  });
+
+  const messageLogProjection = inMemorySingleStreamProjection<MessageLogDocument, MessageLogEvent>({
+    collectionName: 'MessageLog',
+    canHandle: ['CommandDispatched', 'DomainEventEmitted', 'PipelineRunStarted', 'NodeStatusChanged'],
+    getDocumentId: (event) => {
+      if (event.type === 'PipelineRunStarted') {
+        return `prs-${event.data.correlationId}`;
+      }
+      if (event.type === 'NodeStatusChanged') {
+        return `nsc-${event.data.correlationId}-${event.data.commandName}-${event.data.status}`;
+      }
+      if (event.type === 'DomainEventEmitted') {
+        return `dee-${event.data.requestId}-${event.data.eventType}`;
+      }
+      return `cmd-${event.data.requestId}`;
+    },
+    evolve: (document: MessageLogDocument | null, event: MessageLogEvent) => evolveMessageLog(document, event),
+  });
+
+  const statsProjection = inMemorySingleStreamProjection<StatsDocument, MessageLogEvent>({
+    collectionName: 'Stats',
+    canHandle: ['CommandDispatched', 'DomainEventEmitted'],
+    getDocumentId: () => 'global',
+    evolve: (document: StatsDocument | null, event: MessageLogEvent) => evolveStats(document, event),
+  });
+
+  const settledInstanceProjection = inMemorySingleStreamProjection<SettledInstanceDocument, SettledEvent>({
+    collectionName: 'SettledInstance',
+    canHandle: [
+      'SettledInstanceCreated',
+      'SettledCommandStarted',
+      'SettledEventReceived',
+      'SettledHandlerFired',
+      'SettledInstanceReset',
+      'SettledInstanceCleaned',
+    ],
+    getDocumentId: (event) => `${event.data.templateId}-${event.data.correlationId}`,
+    evolve: (document: SettledInstanceDocument | null, event: SettledEvent) => evolveSettledInstance(document, event),
+  });
+
+  const phasedExecutionProjection = inMemorySingleStreamProjection<PhasedExecutionDocument, PhasedExecutionEvent>({
+    collectionName: 'PhasedExecution',
+    canHandle: [
+      'PhasedExecutionStarted',
+      'PhasedItemDispatched',
+      'PhasedItemCompleted',
+      'PhasedItemFailed',
+      'PhasedPhaseAdvanced',
+      'PhasedExecutionCompleted',
+    ],
+    getDocumentId: (event) => event.data.executionId,
+    evolve: (document: PhasedExecutionDocument | null, event: PhasedExecutionEvent) =>
+      evolvePhasedExecution(document, event),
+  });
+
+  const awaitTrackerProjection = inMemorySingleStreamProjection<AwaitTrackerDocument, AwaitEvent>({
+    collectionName: 'AwaitTracker',
+    canHandle: ['AwaitStarted', 'AwaitItemCompleted', 'AwaitCompleted'],
+    getDocumentId: (event) => event.data.correlationId,
+    evolve: (document: AwaitTrackerDocument | null, event: AwaitEvent) => evolveAwaitTracker(document, event),
+  });
+
+  return inlineProjections<InMemoryReadEventMetadata>([
+    itemStatusProjection,
+    nodeStatusProjection,
+    latestRunProjection,
+    messageLogProjection,
+    statsProjection,
+    settledInstanceProjection,
+    phasedExecutionProjection,
+    awaitTrackerProjection,
+  ] as Parameters<typeof inlineProjections<InMemoryReadEventMetadata>>[0]);
+}
+
+export interface PipelineEventStoreContext {
+  eventStore: InMemoryEventStore;
+  database: InMemoryDatabase;
+  readModel: PipelineReadModel;
+  close: () => Promise<void>;
+}
+
+export function createPipelineEventStore(): PipelineEventStoreContext {
+  const database = getInMemoryDatabase();
+  const eventStore = getInMemoryEventStore({
+    database,
+    projections: createProjections() as Parameters<typeof getInMemoryEventStore>[0] extends { projections?: infer P }
+      ? P
+      : never,
+  });
+  const readModel = new PipelineReadModel(database);
+
+  return {
+    eventStore,
+    database,
+    readModel,
+    close: async () => {
+      // No-op for in-memory store
+    },
+  };
+}