@auto-engineer/pipeline 1.65.0 → 1.67.0

package/ketchup-plan.md

@@ -2,10 +2,23 @@
 
 ## TODO
 
+### Session-Based Status Tracking (Bursts S-1 to S-4) ✅
+
+- [x] Burst S-1: E2E test — sub-commands overwrite session status (b8246300)
+- [x] Burst S-2: E2E test — sub-command items counted under unified view (b8246300)
+- [x] Burst S-3: Add sessionId and RestartPipeline (b8246300)
+- [x] Burst S-4: Track status under sessionId (b8246300)
+
 ### Graph Rendering Fix (Burst 106)
 
 - [ ] Burst 106: Show source commands whose events are listened to by the pipeline [depends: none]
 
+### Parallel Scatter-Gather in Event Router (Bursts P-1 to P-3)
+
+- [x] Burst P-1: Test that emit mapping commands run in parallel [depends: none]
+- [x] Burst P-2: Fix event router to use Promise.all [depends: P-1]
+- [x] Burst P-3: Settled scatter-gather proves parallel [depends: P-2]
+
 ### Phase 11: 100% Test Coverage (Bursts 93-102)
 
 **Goal**: Achieve 100% test coverage by testing uncovered code or removing dead code.

package/package.json

@@ -13,8 +13,8 @@
     "get-port": "^7.1.0",
     "jose": "^5.9.6",
     "nanoid": "^5.0.0",
-    "@auto-engineer/file-store": "1.65.0",
-    "@auto-engineer/message-bus": "1.65.0"
+    "@auto-engineer/file-store": "1.67.0",
+    "@auto-engineer/message-bus": "1.67.0"
   },
   "devDependencies": {
     "@types/cors": "^2.8.17",
@@ -23,7 +23,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.65.0",
+  "version": "1.67.0",
   "scripts": {
     "build": "tsc && tsx ../../scripts/fix-esm-imports.ts",
     "test": "vitest run --reporter=dot",

package/src/builder/define-v2.specs.ts

@@ -0,0 +1,236 @@
+import { defineV2, toGraph } from './define-v2';
+
+describe('define-v2', () => {
+  it('exposes pipeline name', () => {
+    const pipeline = defineV2('my-pipeline').build();
+    expect(pipeline.name).toBe('my-pipeline');
+  });
+
+  describe('on().emit()', () => {
+    it('produces EmitRegistration for single emit', () => {
+      const pipeline = defineV2('test-pipeline').on('OrderCreated').emit('ProcessPayment', { amount: 100 }).build();
+
+      expect(pipeline.registrations).toEqual([
+        {
+          type: 'emit',
+          eventType: 'OrderCreated',
+          commands: [{ commandType: 'ProcessPayment', data: { amount: 100 } }],
+        },
+      ]);
+    });
+
+    it('produces EmitRegistration with multiple commands', () => {
+      const pipeline = defineV2('test-pipeline')
+        .on('OrderCreated')
+        .emit('ProcessPayment', { amount: 100 })
+        .emit('SendConfirmation', { template: 'order' })
+        .build();
+
+      expect(pipeline.registrations).toEqual([
+        {
+          type: 'emit',
+          eventType: 'OrderCreated',
+          commands: [
+            { commandType: 'ProcessPayment', data: { amount: 100 } },
+            { commandType: 'SendConfirmation', data: { template: 'order' } },
+          ],
+        },
+      ]);
+    });
+
+    it('supports factory function for data', () => {
+      const factory = (event: Record<string, unknown>) => ({ id: event.id });
+      const pipeline = defineV2('test-pipeline').on('OrderCreated').emit('ProcessPayment', factory).build();
+
+      expect(pipeline.registrations[0]).toEqual({
+        type: 'emit',
+        eventType: 'OrderCreated',
+        commands: [{ commandType: 'ProcessPayment', data: factory }],
+      });
+    });
+
+    it('chains multiple on().emit() blocks', () => {
+      const pipeline = defineV2('test-pipeline')
+        .on('OrderCreated')
+        .emit('ProcessPayment', { amount: 100 })
+        .on('PaymentProcessed')
+        .emit('ShipOrder', { warehouse: 'main' })
+        .build();
+
+      expect(pipeline.registrations).toEqual([
+        {
+          type: 'emit',
+          eventType: 'OrderCreated',
+          commands: [{ commandType: 'ProcessPayment', data: { amount: 100 } }],
+        },
+        {
+          type: 'emit',
+          eventType: 'PaymentProcessed',
+          commands: [{ commandType: 'ShipOrder', data: { warehouse: 'main' } }],
+        },
+      ]);
+    });
+  });
+
+  describe('on().handle()', () => {
+    it('produces CustomHandlerRegistration', () => {
+      const handler = () => [{ type: 'Result', data: {} }];
+      const pipeline = defineV2('test-pipeline').on('OrderCreated').handle(handler).build();
+
+      expect(pipeline.registrations).toEqual([
+        {
+          type: 'custom',
+          eventType: 'OrderCreated',
+          handler,
+        },
+      ]);
+    });
+  });
+
+  describe('settled()', () => {
+    it('produces SettledRegistration', () => {
+      const pipeline = defineV2('test').settled(['CheckTests', 'CheckTypes', 'CheckLint']).build();
+      expect(pipeline.registrations).toEqual([
+        { type: 'settled', commandTypes: ['CheckTests', 'CheckTypes', 'CheckLint'] },
+      ]);
+    });
+
+    it('supports maxRetries option', () => {
+      const pipeline = defineV2('test').settled(['A', 'B']).maxRetries(5).build();
+      expect(pipeline.registrations).toEqual([{ type: 'settled', commandTypes: ['A', 'B'], maxRetries: 5 }]);
+    });
+  });
+
+  describe('forEach().groupInto().process()', () => {
+    it('produces PhasedRegistration', () => {
+      const pipeline = defineV2('test')
+        .on('ComponentsGenerated')
+        .forEach()
+        .groupInto(['molecule', 'organism', 'page'])
+        .process()
+        .build();
+      expect(pipeline.registrations).toEqual([
+        {
+          type: 'phased',
+          eventType: 'ComponentsGenerated',
+          phases: ['molecule', 'organism', 'page'],
+          stopOnFailure: false,
+        },
+      ]);
+    });
+
+    it('supports stopOnFailure', () => {
+      const pipeline = defineV2('test')
+        .on('ComponentsGenerated')
+        .forEach()
+        .groupInto(['a', 'b'])
+        .process()
+        .stopOnFailure()
+        .build();
+      expect(pipeline.registrations).toEqual([
+        {
+          type: 'phased',
+          eventType: 'ComponentsGenerated',
+          phases: ['a', 'b'],
+          stopOnFailure: true,
+        },
+      ]);
+    });
+  });
+
+  describe('run().awaitAll()', () => {
+    it('produces AwaitRegistration', () => {
+      const pipeline = defineV2('test').on('BatchStarted').run(['fetchUsers', 'fetchRoles']).awaitAll().build();
+      expect(pipeline.registrations).toEqual([
+        {
+          type: 'await',
+          eventType: 'BatchStarted',
+          keys: ['fetchUsers', 'fetchRoles'],
+        },
+      ]);
+    });
+  });
+
+  describe('toGraph()', () => {
+    it('converts emit registration to graph nodes and edges', () => {
+      const pipeline = defineV2('test')
+        .on('OrderCreated')
+        .emit('ProcessPayment', { amount: 100 })
+        .emit('SendEmail', { template: 'order' })
+        .build();
+
+      const graph = toGraph(pipeline);
+
+      expect(graph.nodes).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ id: 'evt:OrderCreated', type: 'event' }),
+          expect.objectContaining({ id: 'cmd:ProcessPayment', type: 'command' }),
+          expect.objectContaining({ id: 'cmd:SendEmail', type: 'command' }),
+        ]),
+      );
+      expect(graph.edges).toEqual(
+        expect.arrayContaining([
+          { from: 'evt:OrderCreated', to: 'cmd:ProcessPayment' },
+          { from: 'evt:OrderCreated', to: 'cmd:SendEmail' },
+        ]),
+      );
+    });
+
+    it('converts settled registration to graph nodes', () => {
+      const pipeline = defineV2('test').settled(['CheckTests', 'CheckTypes']).build();
+
+      const graph = toGraph(pipeline);
+
+      expect(graph.nodes).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ id: 'cmd:CheckTests', type: 'command' }),
+          expect.objectContaining({ id: 'cmd:CheckTypes', type: 'command' }),
+          expect.objectContaining({ id: 'settled:CheckTests,CheckTypes', type: 'settled' }),
+        ]),
+      );
+      expect(graph.edges).toEqual(
+        expect.arrayContaining([
+          { from: 'cmd:CheckTests', to: 'settled:CheckTests,CheckTypes' },
+          { from: 'cmd:CheckTypes', to: 'settled:CheckTests,CheckTypes' },
+        ]),
+      );
+    });
+
+    it('converts phased registration to graph nodes', () => {
+      const pipeline = defineV2('test')
+        .on('ComponentsGenerated')
+        .forEach()
+        .groupInto(['molecule', 'organism'])
+        .process()
+        .build();
+
+      const graph = toGraph(pipeline);
+
+      expect(graph.nodes).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ id: 'evt:ComponentsGenerated', type: 'event' }),
+          expect.objectContaining({ type: 'phased' }),
+        ]),
+      );
+    });
+
+    it('converts await registration to graph nodes', () => {
+      const pipeline = defineV2('test').on('BatchStarted').run(['fetchUsers', 'fetchRoles']).awaitAll().build();
+
+      const graph = toGraph(pipeline);
+
+      expect(graph.nodes).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ id: 'evt:BatchStarted', type: 'event' }),
+          expect.objectContaining({ type: 'await' }),
+        ]),
+      );
+    });
+
+    it('returns empty graph for empty pipeline', () => {
+      const pipeline = defineV2('test').build();
+      const graph = toGraph(pipeline);
+      expect(graph).toEqual({ nodes: [], edges: [] });
+    });
+  });
+});

package/src/builder/define-v2.ts

@@ -0,0 +1,351 @@
+import type { GraphEdge, GraphIR, GraphNode, NodeType } from '../graph/types';
+
+type EmitRegistration = {
+  type: 'emit';
+  eventType: string;
+  commands: Array<{
+    commandType: string;
+    data: Record<string, unknown> | ((event: Record<string, unknown>) => Record<string, unknown>);
+  }>;
+};
+
+type CustomHandlerRegistration = {
+  type: 'custom';
+  eventType: string;
+  handler: (event: {
+    type: string;
+    data: Record<string, unknown>;
+  }) =>
+    | Array<{ type: string; data: Record<string, unknown> }>
+    | Promise<Array<{ type: string; data: Record<string, unknown> }>>;
+};
+
+export type SettledRegistration = {
+  type: 'settled';
+  commandTypes: string[];
+  maxRetries?: number;
+};
+
+export type PhasedRegistration = {
+  type: 'phased';
+  eventType: string;
+  phases: string[];
+  stopOnFailure: boolean;
+};
+
+export type AwaitRegistration = {
+  type: 'await';
+  eventType: string;
+  keys: string[];
+};
+
+type Registration =
+  | EmitRegistration
+  | CustomHandlerRegistration
+  | SettledRegistration
+  | PhasedRegistration
+  | AwaitRegistration;
+
+export type PipelineV2 = {
+  name: string;
+  registrations: Registration[];
+};
+
+type EmitChain = {
+  emit(
+    commandType: string,
+    data: Record<string, unknown> | ((event: Record<string, unknown>) => Record<string, unknown>),
+  ): EmitChain;
+  on(eventType: string): TriggerBuilder;
+  build(): PipelineV2;
+};
+
+type HandleChain = {
+  on(eventType: string): TriggerBuilder;
+  build(): PipelineV2;
+};
+
+type ProcessChain = {
+  stopOnFailure(): ProcessChain;
+  on(eventType: string): TriggerBuilder;
+  build(): PipelineV2;
+};
+
+type GroupIntoChain = {
+  process(): ProcessChain;
+};
+
+type ForEachChain = {
+  groupInto(phases: string[]): GroupIntoChain;
+};
+
+type AwaitAllChain = {
+  on(eventType: string): TriggerBuilder;
+  build(): PipelineV2;
+};
+
+type RunChain = {
+  awaitAll(): AwaitAllChain;
+};
+
+type TriggerBuilder = {
+  emit(
+    commandType: string,
+    data: Record<string, unknown> | ((event: Record<string, unknown>) => Record<string, unknown>),
+  ): EmitChain;
+  handle(
+    handler: (event: {
+      type: string;
+      data: Record<string, unknown>;
+    }) =>
+      | Array<{ type: string; data: Record<string, unknown> }>
+      | Promise<Array<{ type: string; data: Record<string, unknown> }>>,
+  ): HandleChain;
+  forEach(): ForEachChain;
+  run(keys: string[]): RunChain;
+};
+
+type SettledChain = {
+  maxRetries(n: number): SettledChain;
+  on(eventType: string): TriggerBuilder;
+  settled(commandTypes: string[]): SettledChain;
+  build(): PipelineV2;
+};
+
+type PipelineV2Builder = {
+  on(eventType: string): TriggerBuilder;
+  settled(commandTypes: string[]): SettledChain;
+  build(): PipelineV2;
+};
+
+export function defineV2(name: string): PipelineV2Builder {
+  const registrations: Registration[] = [];
+
+  function createEmitChain(registration: EmitRegistration): EmitChain {
+    return {
+      emit(commandType, data) {
+        registration.commands.push({ commandType, data });
+        return createEmitChain(registration);
+      },
+      on(eventType) {
+        return createTriggerBuilder(eventType);
+      },
+      build() {
+        return { name, registrations };
+      },
+    };
+  }
+
+  function createHandleChain(): HandleChain {
+    return {
+      on(eventType) {
+        return createTriggerBuilder(eventType);
+      },
+      build() {
+        return { name, registrations };
+      },
+    };
+  }
+
+  function createProcessChain(registration: PhasedRegistration): ProcessChain {
+    return {
+      stopOnFailure() {
+        registration.stopOnFailure = true;
+        return createProcessChain(registration);
+      },
+      on(eventType) {
+        return createTriggerBuilder(eventType);
+      },
+      build() {
+        return { name, registrations };
+      },
+    };
+  }
+
+  function createForEachChain(eventType: string): ForEachChain {
+    return {
+      groupInto(phases) {
+        return {
+          process() {
+            const registration: PhasedRegistration = {
+              type: 'phased',
+              eventType,
+              phases,
+              stopOnFailure: false,
+            };
+            registrations.push(registration);
+            return createProcessChain(registration);
+          },
+        };
+      },
+    };
+  }
+
+  function createAwaitAllChain(): AwaitAllChain {
+    return {
+      on(eventType) {
+        return createTriggerBuilder(eventType);
+      },
+      build() {
+        return { name, registrations };
+      },
+    };
+  }
+
+  function createRunChain(eventType: string, keys: string[]): RunChain {
+    return {
+      awaitAll() {
+        const registration: AwaitRegistration = {
+          type: 'await',
+          eventType,
+          keys,
+        };
+        registrations.push(registration);
+        return createAwaitAllChain();
+      },
+    };
+  }
+
+  function createTriggerBuilder(eventType: string): TriggerBuilder {
+    return {
+      emit(commandType, data) {
+        const registration: EmitRegistration = {
+          type: 'emit',
+          eventType,
+          commands: [{ commandType, data }],
+        };
+        registrations.push(registration);
+        return createEmitChain(registration);
+      },
+      handle(handler) {
+        const registration: CustomHandlerRegistration = {
+          type: 'custom',
+          eventType,
+          handler,
+        };
+        registrations.push(registration);
+        return createHandleChain();
+      },
+      forEach() {
+        return createForEachChain(eventType);
+      },
+      run(keys) {
+        return createRunChain(eventType, keys);
+      },
+    };
+  }
+
+  function createSettledChain(registration: SettledRegistration): SettledChain {
+    return {
+      maxRetries(n) {
+        registration.maxRetries = n;
+        return createSettledChain(registration);
+      },
+      on(eventType) {
+        return createTriggerBuilder(eventType);
+      },
+      settled(commandTypes) {
+        const reg: SettledRegistration = { type: 'settled', commandTypes };
+        registrations.push(reg);
+        return createSettledChain(reg);
+      },
+      build() {
+        return { name, registrations };
+      },
+    };
+  }
+
+  return {
+    on(eventType) {
+      return createTriggerBuilder(eventType);
+    },
+    settled(commandTypes) {
+      const registration: SettledRegistration = { type: 'settled', commandTypes };
+      registrations.push(registration);
+      return createSettledChain(registration);
+    },
+    build() {
+      return { name, registrations };
+    },
+  };
+}
+
+type GraphBuilderContext = {
+  nodeMap: Map<string, GraphNode>;
+  edges: GraphEdge[];
+};
+
+function addNode(ctx: GraphBuilderContext, id: string, type: NodeType, label: string): void {
+  if (!ctx.nodeMap.has(id)) {
+    ctx.nodeMap.set(id, { id, type, label });
+  }
+}
+
+function processEmitRegistration(ctx: GraphBuilderContext, reg: EmitRegistration): void {
+  addNode(ctx, `evt:${reg.eventType}`, 'event', reg.eventType);
+  for (const cmd of reg.commands) {
+    addNode(ctx, `cmd:${cmd.commandType}`, 'command', cmd.commandType);
+    ctx.edges.push({ from: `evt:${reg.eventType}`, to: `cmd:${cmd.commandType}` });
+  }
+}
+
+function processCustomRegistration(ctx: GraphBuilderContext, reg: CustomHandlerRegistration): void {
+  addNode(ctx, `evt:${reg.eventType}`, 'event', reg.eventType);
+  addNode(ctx, `handler:${reg.eventType}`, 'command', `${reg.eventType} handler`);
+  ctx.edges.push({ from: `evt:${reg.eventType}`, to: `handler:${reg.eventType}` });
+}
+
+function processSettledRegistration(ctx: GraphBuilderContext, reg: SettledRegistration): void {
+  const settledNodeId = `settled:${reg.commandTypes.join(',')}`;
+  addNode(ctx, settledNodeId, 'settled', 'Settled');
+  for (const commandType of reg.commandTypes) {
+    addNode(ctx, `cmd:${commandType}`, 'command', commandType);
+    ctx.edges.push({ from: `cmd:${commandType}`, to: settledNodeId });
+  }
+}
+
+function processPhasedRegistration(ctx: GraphBuilderContext, reg: PhasedRegistration): void {
+  addNode(ctx, `evt:${reg.eventType}`, 'event', reg.eventType);
+  const phasedNodeId = `phased:${reg.phases.join(',')}`;
+  addNode(ctx, phasedNodeId, 'phased', reg.phases.join(' → '));
+  ctx.edges.push({ from: `evt:${reg.eventType}`, to: phasedNodeId });
+}
+
+function processAwaitRegistration(ctx: GraphBuilderContext, reg: AwaitRegistration): void {
+  addNode(ctx, `evt:${reg.eventType}`, 'event', reg.eventType);
+  const awaitNodeId = `await:${reg.keys.join(',')}`;
+  addNode(ctx, awaitNodeId, 'await', reg.keys.join(', '));
+  ctx.edges.push({ from: `evt:${reg.eventType}`, to: awaitNodeId });
+}
+
+export function toGraph(pipeline: PipelineV2): GraphIR {
+  const ctx: GraphBuilderContext = {
+    nodeMap: new Map<string, GraphNode>(),
+    edges: [],
+  };
+
+  for (const reg of pipeline.registrations) {
+    switch (reg.type) {
+      case 'emit':
+        processEmitRegistration(ctx, reg);
+        break;
+      case 'custom':
+        processCustomRegistration(ctx, reg);
+        break;
+      case 'settled':
+        processSettledRegistration(ctx, reg);
+        break;
+      case 'phased':
+        processPhasedRegistration(ctx, reg);
+        break;
+      case 'await':
+        processAwaitRegistration(ctx, reg);
+        break;
+    }
+  }
+
+  return {
+    nodes: Array.from(ctx.nodeMap.values()),
+    edges: ctx.edges,
+  };
+}

package/src/engine/command-dispatcher.specs.ts

@@ -0,0 +1,62 @@
+import { createCommandDispatcher, dispatchAndStore } from './command-dispatcher.js';
+import { createPipelineStore } from './sqlite-store.js';
+
+describe('CommandDispatcher', () => {
+  it('calls registered handler for matching command type', async () => {
+    const dispatcher = createCommandDispatcher();
+    const calls: Array<{ type: string; data: Record<string, unknown> }> = [];
+
+    dispatcher.register('CheckTests', (cmd) => {
+      calls.push(cmd);
+      return [{ type: 'CheckTestsPassed', data: { duration: 100 } }];
+    });
+
+    const result = await dispatcher.dispatch({ type: 'CheckTests', data: { target: './src' } });
+
+    expect(calls).toEqual([{ type: 'CheckTests', data: { target: './src' } }]);
+    expect(result).toEqual([{ type: 'CheckTestsPassed', data: { duration: 100 } }]);
+  });
+
+  it('throws for unregistered command type', async () => {
+    const dispatcher = createCommandDispatcher();
+
+    await expect(dispatcher.dispatch({ type: 'Unknown', data: {} })).rejects.toThrow(
+      'No handler registered for command type: Unknown',
+    );
+  });
+
+  it('returns handler result events', async () => {
+    const dispatcher = createCommandDispatcher();
+    dispatcher.register('BuildProject', () => [
+      { type: 'BuildStarted', data: {} },
+      { type: 'BuildCompleted', data: { output: './dist' } },
+    ]);
+
+    const result = await dispatcher.dispatch({ type: 'BuildProject', data: {} });
+
+    expect(result).toEqual([
+      { type: 'BuildStarted', data: {} },
+      { type: 'BuildCompleted', data: { output: './dist' } },
+    ]);
+  });
+});
+
+describe('dispatchAndStore', () => {
+  it('dispatches command and stores result events', async () => {
+    const store = await createPipelineStore();
+    const dispatcher = createCommandDispatcher();
+    dispatcher.register('CheckTests', () => [{ type: 'CheckTestsPassed', data: { duration: 100 } }]);
+
+    const results = await dispatchAndStore(dispatcher, store.eventStore, 'pipeline-c1', {
+      type: 'CheckTests',
+      data: { target: './src' },
+    });
+
+    expect(results).toEqual([{ type: 'CheckTestsPassed', data: { duration: 100 } }]);
+
+    const stream = await store.eventStore.readStream('pipeline-c1');
+    expect(stream.events.map((e) => ({ type: e.type, data: e.data }))).toEqual([
+      { type: 'CheckTestsPassed', data: { duration: 100 } },
+    ]);
+  });
+});