@auto-engineer/pipeline 0.14.0 → 0.16.0

package/src/runtime/phased-executor.ts

@@ -1,119 +1,111 @@
 import type { Event } from '@auto-engineer/message-bus';
 import type { ForEachPhasedDescriptor } from '../core/descriptors';
-
-interface ItemTracker {
-  key: string;
-  phase: string;
-  dispatched: boolean;
-  completed: boolean;
-}
-
-interface PhasedSession {
-  correlationId: string;
-  handler: ForEachPhasedDescriptor;
-  triggerEvent: Event;
-  items: Map<string, ItemTracker>;
-  phases: readonly string[];
-  currentPhaseIndex: number;
-  pendingInPhase: Set<string>;
-  failedItems: Map<string, unknown>;
-}
+import type { PhasedExecutionDocument, PhasedExecutionEvent } from '../projections/phased-execution-projection';
+import type { PipelineReadModel } from '../store/pipeline-read-model';
 
 interface PhasedExecutorOptions {
+  readModel: PipelineReadModel;
   onDispatch: (commandType: string, data: unknown, correlationId: string) => void;
   onComplete: (event: Event, correlationId: string) => void;
+  onEventEmit?: (event: PhasedExecutionEvent) => void | Promise<void>;
 }
 
 export class PhasedExecutor {
-  private sessions = new Map<string, PhasedSession>();
-  private keyToSession = new Map<string, string>();
+  private handlerRegistry = new Map<string, ForEachPhasedDescriptor>();
+  private readonly readModel: PipelineReadModel;
   private readonly onDispatch: (commandType: string, data: unknown, correlationId: string) => void;
   private readonly onComplete: (event: Event, correlationId: string) => void;
+  private readonly onEventEmit?: (event: PhasedExecutionEvent) => void | Promise<void>;
 
   constructor(options: PhasedExecutorOptions) {
+    this.readModel = options.readModel;
     this.onDispatch = options.onDispatch;
     this.onComplete = options.onComplete;
+    this.onEventEmit = options.onEventEmit;
+  }
+
+  registerHandler(handler: ForEachPhasedDescriptor): void {
+    const handlerId = this.generateHandlerId(handler);
+    this.handlerRegistry.set(handlerId, handler);
   }
 
-  startPhased(handler: ForEachPhasedDescriptor, event: Event, correlationId: string): void {
+  async startPhased(handler: ForEachPhasedDescriptor, event: Event, correlationId: string): Promise<void> {
     const items = handler.itemsSelector(event);
-    const itemTrackers = new Map<string, ItemTracker>();
+    const itemData: Array<{ key: string; phase: string; dispatched: boolean; completed: boolean }> = [];
 
     for (const item of items) {
       const key = handler.completion.itemKey({ type: event.type, data: item as Record<string, unknown> });
       const phase = handler.classifier(item);
-      itemTrackers.set(key, {
-        key,
-        phase,
-        dispatched: false,
-        completed: false,
-      });
+      itemData.push({ key, phase, dispatched: false, completed: false });
     }
 
-    const session: PhasedSession = {
-      correlationId,
-      handler,
-      triggerEvent: event,
-      items: itemTrackers,
-      phases: handler.phases,
-      currentPhaseIndex: 0,
-      pendingInPhase: new Set(),
-      failedItems: new Map(),
-    };
-
-    const sessionId = this.generateSessionId(correlationId, handler);
-    this.sessions.set(sessionId, session);
-
-    for (const key of itemTrackers.keys()) {
-      this.keyToSession.set(this.keyWithCorrelation(key, correlationId), sessionId);
-    }
+    const executionId = this.generateSessionId(correlationId, handler);
+    const handlerId = this.generateHandlerId(handler);
+
+    await this.emitEvent({
+      type: 'PhasedExecutionStarted',
+      data: {
+        executionId,
+        correlationId,
+        handlerId,
+        triggerEvent: event,
+        items: itemData,
+        phases: handler.phases,
+      },
+    });
 
-    this.dispatchCurrentPhase(sessionId);
+    await this.dispatchCurrentPhase(executionId, handler);
   }
 
-  onEventReceived(event: Event, itemKey: string): void {
+  async onEventReceived(event: Event, itemKey: string): Promise<void> {
     const correlationId = event.correlationId;
     if (correlationId === undefined || correlationId === '') return;
 
-    const lookupKey = this.keyWithCorrelation(itemKey, correlationId);
-    const sessionId = this.keyToSession.get(lookupKey);
-    if (sessionId === undefined) return;
+    const executions = await this.readModel.getActivePhasedExecutions(correlationId);
+    const execution = executions.find((ex) => ex.items.some((item) => item.key === itemKey));
+
+    if (execution === undefined) return;
 
-    const session = this.sessions.get(sessionId)!;
-    const tracker = session.items.get(itemKey);
-    if (tracker === undefined || tracker.completed) return;
+    const itemDoc = execution.items.find((i) => i.key === itemKey);
+    if (itemDoc === undefined || itemDoc.completed) return;
 
-    tracker.completed = true;
-    session.pendingInPhase.delete(itemKey);
+    const handler = this.handlerRegistry.get(execution.handlerId)!;
 
-    if (session.handler.stopOnFailure && this.isFailureEvent(event, session.handler)) {
-      session.failedItems.set(itemKey, event);
-      this.handleFailure(sessionId, session);
+    if (handler.stopOnFailure && this.isFailureEvent(event, handler)) {
+      await this.emitEvent({
+        type: 'PhasedItemFailed',
+        data: { executionId: execution.executionId, itemKey, error: event },
+      });
+      await this.handleFailure(execution.executionId, handler);
       return;
     }
 
-    if (session.pendingInPhase.size === 0) {
-      this.advanceToNextPhase(sessionId, session);
-    }
-  }
+    await this.emitEvent({
+      type: 'PhasedItemCompleted',
+      data: { executionId: execution.executionId, itemKey, resultEvent: event },
+    });
 
-  getActiveSessionCount(): number {
-    return this.sessions.size;
-  }
+    const updatedExecution = (await this.readModel.getPhasedExecution(execution.executionId))!;
 
-  isPhaseComplete(correlationId: string, phase: string): boolean {
-    for (const session of this.sessions.values()) {
-      if (session.correlationId !== correlationId) continue;
+    const pendingCount = this.countPendingInPhase(updatedExecution);
+    if (pendingCount === 0) {
+      await this.advanceToNextPhase(execution.executionId, handler);
+    }
+  }
 
-      const phaseIndex = session.phases.indexOf(phase);
+  async isPhaseComplete(correlationId: string, phase: string): Promise<boolean> {
+    const executions = await this.readModel.getActivePhasedExecutions(correlationId);
+    for (const execution of executions) {
+      const phaseIndex = execution.phases.indexOf(phase);
       if (phaseIndex === -1) continue;
 
-      if (session.currentPhaseIndex > phaseIndex) {
+      if (execution.currentPhaseIndex > phaseIndex) {
         return true;
       }
 
-      if (session.currentPhaseIndex === phaseIndex) {
-        return session.pendingInPhase.size === 0;
+      if (execution.currentPhaseIndex === phaseIndex) {
+        const pendingCount = this.countPendingInPhase(execution);
+        return pendingCount === 0;
       }
 
       return false;
@@ -121,89 +113,99 @@ export class PhasedExecutor {
     return false;
   }
 
-  private dispatchCurrentPhase(sessionId: string): void {
-    const session = this.sessions.get(sessionId)!;
-
-    if (session.currentPhaseIndex >= session.phases.length) {
-      this.completeSession(sessionId, session, true);
-      return;
+  private countPendingInPhase(execution: PhasedExecutionDocument): number {
+    let pending = 0;
+    const currentPhase = execution.phases[execution.currentPhaseIndex];
+    for (const item of execution.items) {
+      if (item.phase === currentPhase && item.dispatched && !item.completed) {
+        pending++;
+      }
     }
+    return pending;
+  }
 
-    const currentPhase = session.phases[session.currentPhaseIndex];
-    const itemsToDispatch: ItemTracker[] = [];
+  private async dispatchCurrentPhase(executionId: string, handler: ForEachPhasedDescriptor): Promise<void> {
+    const execution = (await this.readModel.getPhasedExecution(executionId))!;
 
-    for (const tracker of session.items.values()) {
-      if (tracker.phase === currentPhase && !tracker.dispatched) {
-        itemsToDispatch.push(tracker);
-      }
+    if (execution.currentPhaseIndex >= execution.phases.length) {
+      await this.completeSession(executionId, handler, true);
+      return;
     }
 
+    const currentPhase = execution.phases[execution.currentPhaseIndex];
+    const itemsToDispatch = execution.items.filter((i) => i.phase === currentPhase && !i.dispatched);
+
     if (itemsToDispatch.length === 0) {
-      this.advanceToNextPhase(sessionId, session);
+      await this.advanceToNextPhase(executionId, handler);
       return;
     }
 
-    for (const tracker of itemsToDispatch) {
-      tracker.dispatched = true;
-      session.pendingInPhase.add(tracker.key);
+    for (const item of itemsToDispatch) {
+      await this.emitEvent({
+        type: 'PhasedItemDispatched',
+        data: { executionId, itemKey: item.key, phase: item.phase },
+      });
 
-      const item = this.findItemByKey(session, tracker.key);
-      if (item !== undefined) {
-        const command = session.handler.emitFactory(item, tracker.phase, session.triggerEvent);
-        this.onDispatch(command.commandType, command.data, session.correlationId);
+      const originalItem = this.findItemByKey(execution.triggerEvent, handler, item.key);
+      if (originalItem !== undefined) {
+        const command = handler.emitFactory(originalItem, item.phase, execution.triggerEvent);
+        this.onDispatch(command.commandType, command.data, execution.correlationId);
       }
     }
   }
 
-  private advanceToNextPhase(sessionId: string, session: PhasedSession): void {
-    session.currentPhaseIndex++;
-    this.dispatchCurrentPhase(sessionId);
+  private async advanceToNextPhase(executionId: string, handler: ForEachPhasedDescriptor): Promise<void> {
+    const execution = (await this.readModel.getPhasedExecution(executionId))!;
+
+    const fromPhase = execution.currentPhaseIndex;
+    const toPhase = fromPhase + 1;
+
+    await this.emitEvent({
+      type: 'PhasedPhaseAdvanced',
+      data: { executionId, fromPhase, toPhase },
+    });
+
+    await this.dispatchCurrentPhase(executionId, handler);
   }
 
-  private handleFailure(sessionId: string, session: PhasedSession): void {
-    this.completeSession(sessionId, session, false);
+  private async handleFailure(executionId: string, handler: ForEachPhasedDescriptor): Promise<void> {
+    await this.completeSession(executionId, handler, false);
   }
 
-  private completeSession(sessionId: string, session: PhasedSession, success: boolean): void {
-    const eventType = success ? session.handler.completion.successEvent : session.handler.completion.failureEvent;
+  private async completeSession(
+    executionId: string,
+    handler: ForEachPhasedDescriptor,
+    success: boolean,
+  ): Promise<void> {
+    const execution = (await this.readModel.getPhasedExecution(executionId))!;
 
-    const results = this.collectResults(session);
+    const eventDescriptor = success ? handler.completion.successEvent : handler.completion.failureEvent;
+    const eventType = eventDescriptor.name;
+
+    const results = execution.items.filter((i) => i.completed).map((i) => i.key);
     const eventData = success
-      ? { results, itemCount: session.items.size }
-      : { failures: Array.from(session.failedItems.entries()).map(([k, v]) => ({ key: k, error: v })) };
+      ? { results, itemCount: execution.items.length }
+      : { failures: execution.failedItems.map((f) => ({ key: f.key, error: f.error })) };
 
     const completionEvent: Event = {
       type: eventType,
-      correlationId: session.correlationId,
+      correlationId: execution.correlationId,
       data: eventData,
     };
 
-    this.onComplete(completionEvent, session.correlationId);
-    this.cleanupSession(sessionId, session);
-  }
-
-  private collectResults(session: PhasedSession): string[] {
-    const completed: string[] = [];
-    for (const tracker of session.items.values()) {
-      if (tracker.completed) {
-        completed.push(tracker.key);
-      }
-    }
-    return completed;
-  }
+    await this.emitEvent({
+      type: 'PhasedExecutionCompleted',
+      data: { executionId, success, results },
+    });
 
-  private cleanupSession(sessionId: string, session: PhasedSession): void {
-    for (const key of session.items.keys()) {
-      this.keyToSession.delete(this.keyWithCorrelation(key, session.correlationId));
-    }
-    this.sessions.delete(sessionId);
+    this.onComplete(completionEvent, execution.correlationId);
   }
 
-  private findItemByKey(session: PhasedSession, key: string): unknown {
-    const items = session.handler.itemsSelector(session.triggerEvent);
+  private findItemByKey(triggerEvent: Event, handler: ForEachPhasedDescriptor, key: string): unknown {
+    const items = handler.itemsSelector(triggerEvent);
     return items.find((item) => {
-      const itemKey = session.handler.completion.itemKey({
-        type: session.triggerEvent.type,
+      const itemKey = handler.completion.itemKey({
+        type: triggerEvent.type,
         data: item as Record<string, unknown>,
       });
       return itemKey === key;
@@ -211,14 +213,18 @@ export class PhasedExecutor {
   }
 
   private isFailureEvent(event: Event, handler: ForEachPhasedDescriptor): boolean {
-    return event.type === handler.completion.failureEvent;
+    return event.type === handler.completion.failureEvent.name;
   }
 
   private generateSessionId(correlationId: string, handler: ForEachPhasedDescriptor): string {
     return `phased-${correlationId}-${handler.eventType}`;
   }
 
-  private keyWithCorrelation(key: string, correlationId: string): string {
-    return `${correlationId}::${key}`;
+  private generateHandlerId(handler: ForEachPhasedDescriptor): string {
+    return `phased-handler-${handler.eventType}`;
+  }
+
+  private async emitEvent(event: PhasedExecutionEvent): Promise<void> {
+    await this.onEventEmit?.(event);
   }
 }

package/src/runtime/pipeline-runtime.specs.ts

@@ -211,4 +211,69 @@ describe('PipelineRuntime', () => {
     );
     expect(sent).toEqual(['2', '3', '1']);
   });
+
+  it('should use startPhased when provided for foreach-phased', async () => {
+    let phasedCalled = false;
+    const ctx = {
+      sendCommand: async () => {},
+      emit: async () => {},
+      correlationId: 'test',
+      startPhased: () => {
+        phasedCalled = true;
+      },
+    };
+    type Item = { id: string; p: 'high' | 'low' };
+    const pipeline = define('test')
+      .on('Items')
+      .forEach((e: { data: { items: Item[] } }) => e.data.items)
+      .groupInto(['high', 'low'], (i: Item) => i.p)
+      .process('Cmd', (i: Item) => ({ id: i.id }))
+      .onComplete({ success: 'Done', failure: 'Fail', itemKey: () => '' })
+      .build();
+    const runtime = new PipelineRuntime(pipeline.descriptor);
+    await runtime.handleEvent({ type: 'Items', data: { items: [{ id: '1', p: 'low' }] } }, ctx);
+    expect(phasedCalled).toBe(true);
+  });
+
+  it('should dispatch multiple emit commands in parallel, not sequentially', async () => {
+    const callOrder: string[] = [];
+    let resolveA: () => void;
+    let resolveB: () => void;
+    const promiseA = new Promise<void>((r) => {
+      resolveA = r;
+    });
+    const promiseB = new Promise<void>((r) => {
+      resolveB = r;
+    });
+
+    const ctx = {
+      sendCommand: async (type: string) => {
+        callOrder.push(`${type}:start`);
+        if (type === 'CmdA') {
+          await promiseA;
+        } else if (type === 'CmdB') {
+          await promiseB;
+        }
+        callOrder.push(`${type}:end`);
+      },
+      emit: async () => {},
+      correlationId: 'test',
+    };
+
+    const pipeline = define('test').on('Start').emit('CmdA', {}).emit('CmdB', {}).build();
+    const runtime = new PipelineRuntime(pipeline.descriptor);
+
+    const handlePromise = runtime.handleEvent({ type: 'Start', data: {} }, ctx);
+
+    await new Promise((r) => setTimeout(r, 10));
+
+    expect(callOrder).toContain('CmdA:start');
+    expect(callOrder).toContain('CmdB:start');
+
+    resolveB!();
+    resolveA!();
+    await handlePromise;
+
+    expect(callOrder).toEqual(['CmdA:start', 'CmdB:start', 'CmdB:end', 'CmdA:end']);
+  });
 });

package/src/runtime/pipeline-runtime.ts

@@ -53,10 +53,12 @@ export class PipelineRuntime {
   }
 
   private async executeEmitHandler(handler: EmitHandlerDescriptor, event: Event, ctx: PipelineContext): Promise<void> {
-    for (const command of handler.commands) {
-      const data = typeof command.data === 'function' ? command.data(event) : command.data;
-      await ctx.sendCommand(command.commandType, data);
-    }
+    await Promise.all(
+      handler.commands.map(async (command) => {
+        const data = typeof command.data === 'function' ? command.data(event) : command.data;
+        await ctx.sendCommand(command.commandType, data);
+      }),
+    );
   }
 
   private async executeCustomHandler(