@auto-engineer/pipeline 1.65.0 → 1.67.0

package/src/engine/pipeline-engine.integration.specs.ts

@@ -0,0 +1,126 @@
+import { defineV2, toGraph } from '../builder/define-v2.js';
+import { createPipelineEngine } from './pipeline-engine.js';
+import { createSettledWorkflow } from './workflows/settled-workflow.js';
+
+describe('PipelineEngine integration', () => {
+  it('runs emit chain: command A produces event, triggers command B', async () => {
+    const engine = await createPipelineEngine();
+    const events: string[] = [];
+
+    engine.registerCommandHandler('StartBuild', () => [{ type: 'BuildStarted', data: {} }]);
+    engine.registerCommandHandler('RunTests', () => [{ type: 'TestsPassed', data: {} }]);
+    engine.registerEmitMapping({
+      eventType: 'BuildStarted',
+      commands: [{ commandType: 'RunTests', data: {} }],
+    });
+
+    engine.onEvent((e) => events.push(e.type));
+    await engine.dispatch({ type: 'StartBuild', data: {} });
+
+    expect(events).toEqual(['BuildStarted', 'TestsPassed']);
+    await engine.close();
+  });
+
+  it('runs settled workflow through full lifecycle', async () => {
+    const engine = await createPipelineEngine();
+    const events: string[] = [];
+
+    engine.registerCommandHandler('CheckTests', () => [
+      { type: 'CommandCompleted', data: { commandType: 'CheckTests', result: 'success', event: {} } },
+    ]);
+    engine.registerCommandHandler('CheckTypes', () => [
+      { type: 'CommandCompleted', data: { commandType: 'CheckTypes', result: 'success', event: {} } },
+    ]);
+
+    engine.registerWorkflow({
+      id: 'settled-checks',
+      workflow: createSettledWorkflow({ commandTypes: ['CheckTests', 'CheckTypes'] }),
+      inputEvents: ['StartSettled', 'CommandCompleted'],
+    });
+
+    engine.onEvent((e) => events.push(e.type));
+
+    engine.processWorkflowEvent({
+      type: 'StartSettled',
+      data: { correlationId: 'c1', commandTypes: ['CheckTests', 'CheckTypes'] },
+    });
+
+    await engine.dispatch({ type: 'CheckTests', data: {} });
+    await engine.dispatch({ type: 'CheckTypes', data: {} });
+
+    expect(events).toContain('CommandCompleted');
+    expect(events).toContain('AllSettled');
+    await engine.close();
+  });
+
+  it('runs settled workflow with failure and retry', async () => {
+    const engine = await createPipelineEngine();
+    const events: string[] = [];
+    let callCount = 0;
+
+    engine.registerCommandHandler('CheckTests', () => {
+      callCount++;
+      if (callCount === 1) {
+        return [
+          {
+            type: 'CommandCompleted',
+            data: { commandType: 'CheckTests', result: 'failure', event: { error: 'failed' } },
+          },
+        ];
+      }
+      return [{ type: 'CommandCompleted', data: { commandType: 'CheckTests', result: 'success', event: {} } }];
+    });
+
+    engine.registerWorkflow({
+      id: 'retry-settled',
+      workflow: createSettledWorkflow({ commandTypes: ['CheckTests'], maxRetries: 3 }),
+      inputEvents: ['StartSettled', 'CommandCompleted'],
+    });
+
+    engine.onEvent((e) => events.push(e.type));
+
+    engine.processWorkflowEvent({
+      type: 'StartSettled',
+      data: { correlationId: 'c1', commandTypes: ['CheckTests'] },
+    });
+
+    await engine.dispatch({ type: 'CheckTests', data: {} });
+
+    expect(events).toContain('RetryCommands');
+    await engine.close();
+  });
+});
+
+describe('crash recovery', () => {
+  it('event store persists events across operations', async () => {
+    const engine = await createPipelineEngine();
+
+    engine.registerCommandHandler('StoreData', () => [{ type: 'DataStored', data: { key: 'test' } }]);
+
+    await engine.dispatch({ type: 'StoreData', data: {} });
+
+    const events: string[] = [];
+    engine.onEvent((e) => events.push(e.type));
+
+    engine.registerCommandHandler('MoreData', () => [{ type: 'MoreDataStored', data: {} }]);
+    await engine.dispatch({ type: 'MoreData', data: {} });
+
+    expect(events).toEqual(['MoreDataStored']);
+    await engine.close();
+  });
+});
+
+describe('A/B parity', () => {
+  it('v2 graph output has same structure as v1 format', () => {
+    const pipeline = defineV2('test').on('EventA').emit('CommandB', {}).build();
+
+    const graph = toGraph(pipeline);
+
+    expect(graph).toEqual({
+      nodes: expect.arrayContaining([
+        expect.objectContaining({ id: expect.any(String), type: expect.any(String), label: expect.any(String) }),
+      ]),
+      edges: expect.arrayContaining([expect.objectContaining({ from: expect.any(String), to: expect.any(String) })]),
+    });
+  });
+});

package/src/engine/pipeline-engine.specs.ts

@@ -0,0 +1,70 @@
+import { createPipelineEngine } from './pipeline-engine.js';
+import { createSettledWorkflow } from './workflows/settled-workflow.js';
+
+describe('PipelineEngine', () => {
+  it('dispatches command through registered handler', async () => {
+    const engine = await createPipelineEngine();
+    const calls: string[] = [];
+
+    engine.registerCommandHandler('CheckTests', () => {
+      calls.push('CheckTests');
+      return [{ type: 'CheckTestsPassed', data: {} }];
+    });
+
+    await engine.dispatch({ type: 'CheckTests', data: {} });
+
+    expect(calls).toEqual(['CheckTests']);
+    await engine.close();
+  });
+
+  it('routes events to emit mappings after command dispatch', async () => {
+    const engine = await createPipelineEngine();
+    const calls: string[] = [];
+
+    engine.registerCommandHandler('A', () => [{ type: 'ACompleted', data: {} }]);
+    engine.registerCommandHandler('B', () => {
+      calls.push('B');
+      return [{ type: 'BCompleted', data: {} }];
+    });
+    engine.registerEmitMapping({
+      eventType: 'ACompleted',
+      commands: [{ commandType: 'B', data: {} }],
+    });
+
+    await engine.dispatch({ type: 'A', data: {} });
+
+    expect(calls).toEqual(['B']);
+    await engine.close();
+  });
+
+  it('processes full settled workflow flow', async () => {
+    const engine = await createPipelineEngine();
+    const events: string[] = [];
+
+    engine.registerCommandHandler('CheckTests', () => [
+      { type: 'CommandCompleted', data: { commandType: 'CheckTests', result: 'success', event: {} } },
+    ]);
+    engine.registerCommandHandler('CheckTypes', () => [
+      { type: 'CommandCompleted', data: { commandType: 'CheckTypes', result: 'success', event: {} } },
+    ]);
+
+    engine.registerWorkflow({
+      id: 'settled-checks',
+      workflow: createSettledWorkflow({ commandTypes: ['CheckTests', 'CheckTypes'] }),
+      inputEvents: ['StartSettled', 'CommandCompleted'],
+    });
+
+    engine.onEvent((event) => events.push(event.type));
+
+    engine.processWorkflowEvent({
+      type: 'StartSettled',
+      data: { correlationId: 'c1', commandTypes: ['CheckTests', 'CheckTypes'] },
+    });
+
+    await engine.dispatch({ type: 'CheckTests', data: {} });
+    await engine.dispatch({ type: 'CheckTypes', data: {} });
+
+    expect(events).toContain('AllSettled');
+    await engine.close();
+  });
+});

package/src/engine/pipeline-engine.ts

@@ -0,0 +1,82 @@
+import { createCommandDispatcher, dispatchAndStore } from './command-dispatcher.js';
+import { createEventRouter } from './event-router.js';
+import { createPipelineStore } from './sqlite-store.js';
+import type { WorkflowRegistration } from './workflow-processor.js';
+import { createWorkflowProcessor } from './workflow-processor.js';
+
+type Event = { type: string; data: Record<string, unknown> };
+
+type CommandHandler = (command: {
+  type: string;
+  data: Record<string, unknown>;
+}) =>
+  | Array<{ type: string; data: Record<string, unknown> }>
+  | Promise<Array<{ type: string; data: Record<string, unknown> }>>;
+
+type EmitMapping = {
+  eventType: string;
+  commands: Array<{
+    commandType: string;
+    data: Record<string, unknown> | ((event: Event) => Record<string, unknown>);
+  }>;
+};
+
+export async function createPipelineEngine() {
+  const store = await createPipelineStore();
+  const dispatcher = createCommandDispatcher();
+  const router = createEventRouter(dispatcher);
+  const workflows = createWorkflowProcessor();
+  const eventListeners: Array<(event: Event) => void> = [];
+  let streamCounter = 0;
+
+  async function processEvents(events: Event[]): Promise<void> {
+    for (const event of events) {
+      for (const listener of eventListeners) {
+        listener(event);
+      }
+
+      const routedEvents = await router.route(event);
+      const workflowOutputs = workflows.process(event);
+
+      await processEvents(routedEvents);
+      await processEvents(workflowOutputs);
+    }
+  }
+
+  return {
+    registerCommandHandler(commandType: string, handler: CommandHandler): void {
+      dispatcher.register(commandType, handler);
+    },
+
+    registerEmitMapping(mapping: EmitMapping): void {
+      router.register(mapping);
+    },
+
+    registerWorkflow(registration: WorkflowRegistration): void {
+      workflows.register(registration);
+    },
+
+    registeredCommands(): string[] {
+      return dispatcher.registeredTypes();
+    },
+
+    onEvent(listener: (event: Event) => void): void {
+      eventListeners.push(listener);
+    },
+
+    processWorkflowEvent(event: Event): Event[] {
+      return workflows.process(event);
+    },
+
+    async dispatch(command: { type: string; data: Record<string, unknown> }): Promise<void> {
+      streamCounter++;
+      const streamName = `pipeline-${streamCounter}`;
+      const results = await dispatchAndStore(dispatcher, store.eventStore, streamName, command);
+      await processEvents(results);
+    },
+
+    async close(): Promise<void> {
+      await store.close();
+    },
+  };
+}

package/src/engine/projections/item-status.ts

@@ -0,0 +1,11 @@
+import { evolve, type ItemStatusChangedEvent, type ItemStatusDocument } from '../../projections/item-status-projection';
+
+export { evolve, type ItemStatusChangedEvent, type ItemStatusDocument };
+
+export const itemStatusProjection = {
+  name: 'item-status',
+  canHandle: ['ItemStatusChanged'] as const,
+  evolve,
+  getDocumentId: (event: ItemStatusChangedEvent) =>
+    `${event.data.correlationId}-${event.data.commandType}-${event.data.itemKey}`,
+};

package/src/engine/projections/latest-run.ts

@@ -0,0 +1,10 @@
+import { evolve, type LatestRunDocument } from '../../projections/latest-run-projection';
+
+export { evolve, type LatestRunDocument };
+
+export const latestRunProjection = {
+  name: 'latest-run',
+  canHandle: ['PipelineRunStarted'] as const,
+  evolve,
+  getDocumentId: () => 'latest',
+};

package/src/engine/projections/message-log.ts

@@ -0,0 +1,11 @@
+import { nanoid } from 'nanoid';
+import { evolve, type MessageLogDocument, type MessageLogEvent } from '../../projections/message-log-projection';
+
+export { evolve, type MessageLogDocument, type MessageLogEvent };
+
+export const messageLogProjection = {
+  name: 'message-log',
+  canHandle: ['CommandDispatched', 'DomainEventEmitted', 'PipelineRunStarted', 'NodeStatusChanged'] as const,
+  evolve,
+  getDocumentId: () => nanoid(),
+};

package/src/engine/projections/node-status.ts

@@ -0,0 +1,10 @@
+import { evolve, type NodeStatusChangedEvent, type NodeStatusDocument } from '../../projections/node-status-projection';
+
+export { evolve, type NodeStatusChangedEvent, type NodeStatusDocument };
+
+export const nodeStatusProjection = {
+  name: 'node-status',
+  canHandle: ['NodeStatusChanged'] as const,
+  evolve,
+  getDocumentId: (event: NodeStatusChangedEvent) => `${event.data.correlationId}-${event.data.commandName}`,
+};

package/src/engine/projections/projections.specs.ts

@@ -0,0 +1,176 @@
+import { describe, expect, it } from 'vitest';
+import { itemStatusProjection } from './item-status';
+import { latestRunProjection } from './latest-run';
+import { messageLogProjection } from './message-log';
+import { nodeStatusProjection } from './node-status';
+import { statsProjection } from './stats';
+
+describe('engine projections', () => {
+  describe('item-status', () => {
+    it('evolves ItemStatusChanged into document', () => {
+      const event = {
+        type: 'ItemStatusChanged' as const,
+        data: {
+          correlationId: 'c1',
+          commandType: 'CheckTests',
+          itemKey: 'k1',
+          requestId: 'r1',
+          status: 'running' as const,
+          attemptCount: 1,
+        },
+      };
+      const result = itemStatusProjection.evolve(null, event);
+      expect(result).toEqual({
+        correlationId: 'c1',
+        commandType: 'CheckTests',
+        itemKey: 'k1',
+        currentRequestId: 'r1',
+        status: 'running',
+        attemptCount: 1,
+      });
+    });
+
+    it('generates document id from correlation, command, and item key', () => {
+      const event = {
+        type: 'ItemStatusChanged' as const,
+        data: {
+          correlationId: 'c1',
+          commandType: 'CT',
+          itemKey: 'k1',
+          requestId: 'r1',
+          status: 'running' as const,
+          attemptCount: 1,
+        },
+      };
+      expect(itemStatusProjection.getDocumentId(event)).toBe('c1-CT-k1');
+    });
+  });
+
+  describe('node-status', () => {
+    it('evolves NodeStatusChanged into document', () => {
+      const event = {
+        type: 'NodeStatusChanged' as const,
+        data: {
+          correlationId: 'c1',
+          commandName: 'CheckTests',
+          nodeId: 'n1',
+          status: 'running' as const,
+          previousStatus: 'idle' as const,
+          pendingCount: 3,
+          endedCount: 0,
+        },
+      };
+      const result = nodeStatusProjection.evolve(null, event);
+      expect(result).toEqual({
+        correlationId: 'c1',
+        commandName: 'CheckTests',
+        status: 'running',
+        pendingCount: 3,
+        endedCount: 0,
+      });
+    });
+
+    it('generates document id from correlation and command name', () => {
+      const event = {
+        type: 'NodeStatusChanged' as const,
+        data: {
+          correlationId: 'c1',
+          commandName: 'CT',
+          nodeId: 'n1',
+          status: 'idle' as const,
+          previousStatus: 'idle' as const,
+          pendingCount: 0,
+          endedCount: 0,
+        },
+      };
+      expect(nodeStatusProjection.getDocumentId(event)).toBe('c1-CT');
+    });
+  });
+
+  describe('latest-run', () => {
+    it('evolves PipelineRunStarted into document', () => {
+      const event = {
+        type: 'PipelineRunStarted' as const,
+        data: { correlationId: 'c1', triggerCommand: 'StartPipeline' },
+      };
+      const result = latestRunProjection.evolve(null, event);
+      expect(result).toEqual({
+        latestCorrelationId: 'c1',
+        triggerCommand: 'StartPipeline',
+      });
+    });
+
+    it('uses constant document id', () => {
+      expect(latestRunProjection.getDocumentId()).toBe('latest');
+    });
+  });
+
+  describe('message-log', () => {
+    it('evolves CommandDispatched into document', () => {
+      const now = new Date();
+      const event = {
+        type: 'CommandDispatched' as const,
+        data: {
+          correlationId: 'c1',
+          requestId: 'r1',
+          commandType: 'CheckTests',
+          commandData: { x: 1 },
+          timestamp: now,
+        },
+      };
+      const result = messageLogProjection.evolve(null, event);
+      expect(result).toEqual({
+        correlationId: 'c1',
+        requestId: 'r1',
+        messageType: 'command',
+        messageName: 'CheckTests',
+        messageData: { x: 1 },
+        timestamp: now,
+      });
+    });
+
+    it('generates unique document ids', () => {
+      const id1 = messageLogProjection.getDocumentId();
+      const id2 = messageLogProjection.getDocumentId();
+      expect(id1).not.toBe(id2);
+      expect(typeof id1).toBe('string');
+    });
+  });
+
+  describe('stats', () => {
+    it('evolves CommandDispatched by incrementing command count', () => {
+      const event = {
+        type: 'CommandDispatched' as const,
+        data: {
+          correlationId: 'c1',
+          requestId: 'r1',
+          commandType: 'CT',
+          commandData: {},
+          timestamp: new Date(),
+        },
+      };
+      const result = statsProjection.evolve(null, event);
+      expect(result).toEqual({ totalMessages: 1, totalCommands: 1, totalEvents: 0 });
+    });
+
+    it('evolves DomainEventEmitted by incrementing event count', () => {
+      const existing = { totalMessages: 1, totalCommands: 1, totalEvents: 0 };
+      const event = {
+        type: 'DomainEventEmitted' as const,
+        data: {
+          correlationId: 'c1',
+          requestId: 'r1',
+          eventType: 'ET',
+          eventData: {},
+          timestamp: new Date(),
+        },
+      };
+      const result = statsProjection.evolve(existing, event);
+      expect(result).toEqual({ totalMessages: 2, totalCommands: 1, totalEvents: 1 });
+    });
+
+    it('uses constant document id', () => {
+      expect(statsProjection.getDocumentId()).toBe('global');
+    });
+  });
+});

package/src/engine/projections/stats.ts

@@ -0,0 +1,10 @@
+import { evolve, type StatsDocument } from '../../projections/stats-projection';
+
+export { evolve, type StatsDocument };
+
+export const statsProjection = {
+  name: 'stats',
+  canHandle: ['CommandDispatched', 'DomainEventEmitted'] as const,
+  evolve,
+  getDocumentId: () => 'global',
+};

package/src/engine/sqlite-consumer.specs.ts

@@ -0,0 +1,42 @@
+import { createConsumer } from './sqlite-consumer.js';
+import { createPipelineStore } from './sqlite-store.js';
+
+describe('SQLiteConsumer', () => {
+  it('delivers events to registered handler', async () => {
+    const store = await createPipelineStore();
+    const consumer = createConsumer(store);
+    const received: Array<{ type: string }> = [];
+
+    consumer.on('TestEvent', (event) => {
+      received.push(event);
+    });
+
+    await store.eventStore.appendToStream('test-stream', [{ type: 'TestEvent', data: { value: 1 } }]);
+
+    await consumer.poll();
+
+    expect(received).toEqual([expect.objectContaining({ type: 'TestEvent', data: { value: 1 } })]);
+  });
+
+  it('delivers events in order', async () => {
+    const store = await createPipelineStore();
+    const consumer = createConsumer(store);
+    const types: string[] = [];
+
+    consumer.on('A', (event) => {
+      types.push(event.type);
+    });
+    consumer.on('B', (event) => {
+      types.push(event.type);
+    });
+
+    await store.eventStore.appendToStream('s1', [
+      { type: 'A', data: {} },
+      { type: 'B', data: {} },
+    ]);
+
+    await consumer.poll();
+
+    expect(types).toEqual(['A', 'B']);
+  });
+});

package/src/engine/sqlite-consumer.ts

@@ -0,0 +1,34 @@
+import { readMessagesBatch, sqliteConnection } from '@event-driven-io/emmett-sqlite';
+import type { PipelineStore } from './sqlite-store.js';
+
+type EventHandler = (event: { type: string; data: Record<string, unknown> }) => void;
+
+export function createConsumer(store: PipelineStore) {
+  const handlers = new Map<string, EventHandler>();
+  let lastPosition = 0n;
+  const connection = sqliteConnection({ fileName: store.fileName });
+
+  return {
+    on(eventType: string, handler: EventHandler): void {
+      handlers.set(eventType, handler);
+    },
+
+    async poll(): Promise<void> {
+      const { messages, currentGlobalPosition } = await readMessagesBatch(connection, {
+        after: lastPosition,
+        batchSize: 1000,
+      });
+
+      for (const message of messages) {
+        const handler = handlers.get(message.type);
+        if (handler) {
+          handler({ type: message.type, data: message.data as Record<string, unknown> });
+        }
+      }
+
+      if (messages.length > 0) {
+        lastPosition = currentGlobalPosition;
+      }
+    },
+  };
+}

package/src/engine/sqlite-store.specs.ts

@@ -0,0 +1,46 @@
+import { createPipelineStore } from './sqlite-store.js';
+
+describe('createPipelineStore', () => {
+  it('creates a pipeline store with in-memory SQLite', async () => {
+    const store = await createPipelineStore();
+    expect(store.eventStore).toBeDefined();
+    expect(store.close).toBeInstanceOf(Function);
+    await store.close();
+  });
+
+  it('appends events to a stream and reads them back', async () => {
+    const store = await createPipelineStore();
+
+    await store.eventStore.appendToStream('test-stream-1', [
+      { type: 'TaskStarted', data: { taskId: 'a' } },
+      { type: 'TaskCompleted', data: { taskId: 'a' } },
+    ]);
+
+    const result = await store.eventStore.readStream('test-stream-1');
+
+    expect(result.events).toHaveLength(2);
+    expect(result.events[0]!.type).toEqual('TaskStarted');
+    expect(result.events[0]!.data).toEqual({ taskId: 'a' });
+    expect(result.events[1]!.type).toEqual('TaskCompleted');
+    expect(result.events[1]!.data).toEqual({ taskId: 'a' });
+
+    await store.close();
+  });
+
+  it('returns events in append order', async () => {
+    const store = await createPipelineStore();
+
+    await store.eventStore.appendToStream('test-stream-2', [
+      { type: 'Step1', data: { order: 1 } },
+      { type: 'Step2', data: { order: 2 } },
+      { type: 'Step3', data: { order: 3 } },
+    ]);
+
+    const result = await store.eventStore.readStream('test-stream-2');
+
+    expect(result.events.map((e) => e.type)).toEqual(['Step1', 'Step2', 'Step3']);
+    expect(result.events.map((e) => e.data)).toEqual([{ order: 1 }, { order: 2 }, { order: 3 }]);
+
+    await store.close();
+  });
+});

package/src/engine/sqlite-store.ts

@@ -0,0 +1,21 @@
+import { getSQLiteEventStore, type SQLiteEventStore } from '@event-driven-io/emmett-sqlite';
+
+export interface PipelineStore {
+  eventStore: SQLiteEventStore;
+  fileName: string;
+  close: () => Promise<void>;
+}
+
+export async function createPipelineStore(options?: { fileName?: string }): Promise<PipelineStore> {
+  const fileName = options?.fileName ?? 'file::memory:?cache=shared';
+  const eventStore = getSQLiteEventStore({
+    fileName,
+    schema: { autoMigration: 'CreateOrUpdate' },
+  });
+
+  return {
+    eventStore,
+    fileName,
+    close: async () => {},
+  };
+}