@auto-engineer/pipeline 1.65.0 → 1.67.0

package/src/server/pipeline-server.ts

@@ -6,6 +6,8 @@ import {
   type Event,
   type MessageBus,
 } from '@auto-engineer/message-bus';
+import type { EventStore } from '@event-driven-io/emmett';
+import { getSQLiteEventStore, readMessagesBatch, sqliteConnection } from '@event-driven-io/emmett-sqlite';
 import cors from 'cors';
 import express from 'express';
 import getPort from 'get-port';
@@ -36,6 +38,7 @@ export interface CommandHandlerWithMetadata extends CommandHandler {
 
 export interface PipelineServerConfig {
   port: number;
+  storeFileName?: string;
 }
 
 interface EventWithCorrelation extends Event {
@@ -58,9 +61,12 @@ export class PipelineServer {
   private readonly eventStoreContext: PipelineEventStoreContext;
   private readonly itemKeyExtractors = new Map<string, (data: unknown) => string | undefined>();
   private readonly middleware: express.RequestHandler[] = [];
-  private readonly seenCorrelations = new Set<string>();
+  private readonly storeFileName?: string;
+  private sqliteEventStore?: EventStore;
+  private currentSessionId = '';
 
   constructor(config: PipelineServerConfig) {
+    this.storeFileName = config.storeFileName;
     this.requestedPort = config.port;
     this.actualPort = config.port;
     this.app = express();
@@ -173,9 +179,61 @@ export class PipelineServer {
       });
     });
 
+    if (this.storeFileName) {
+      this.sqliteEventStore = getSQLiteEventStore({
+        fileName: this.storeFileName,
+        schema: { autoMigration: 'CreateOrUpdate' },
+      });
+
+      await this.replayEventsFromSQLite();
+
+      const originalAppend = this.eventStoreContext.eventStore.appendToStream.bind(this.eventStoreContext.eventStore);
+      this.eventStoreContext.eventStore.appendToStream = async (streamName, events, options) => {
+        const result = await originalAppend(streamName, events, options);
+        await this.sqliteEventStore!.appendToStream(streamName, events);
+        return result;
+      };
+    }
+
+    const restoredSessionId = await this.eventStoreContext.readModel.getLatestCorrelationId();
+    if (restoredSessionId) {
+      this.currentSessionId = restoredSessionId;
+    } else {
+      this.currentSessionId = `session-${nanoid()}`;
+      await this.broadcastPipelineRunStarted(this.currentSessionId, 'PipelineStarted');
+    }
     await this.emitPipelineStartedEvent();
   }
 
+  private async replayEventsFromSQLite(): Promise<void> {
+    const connection = sqliteConnection({ fileName: this.storeFileName! });
+    let lastPosition = 0n;
+
+    try {
+      while (true) {
+        const { messages, currentGlobalPosition } = await readMessagesBatch(connection, {
+          after: lastPosition,
+          batchSize: 1000,
+        });
+        if (messages.length === 0) break;
+
+        for (const message of messages) {
+          const metadata: { streamName?: string } = message.metadata ?? {};
+          const streamName = metadata.streamName ?? 'pipeline-replay';
+          await this.eventStoreContext.eventStore.appendToStream(streamName, [
+            { type: message.type, data: message.data },
+          ]);
+        }
+        lastPosition = currentGlobalPosition;
+      }
+    } catch (error: unknown) {
+      const isMissingTable = error instanceof Error && error.message.includes('no such table');
+      if (!isMissingTable) throw error;
+    } finally {
+      connection.close();
+    }
+  }
+
   async stop(): Promise<void> {
     this.sseManager.closeAll();
     await new Promise<void>((resolve) => {
@@ -241,7 +299,7 @@ export class PipelineServer {
         const completeGraph = this.markBackLinks(graphWithEnrichedEvents);
         const filterOptions = this.parseFilterOptions(req.query);
         const filteredGraph = filterGraph(completeGraph, filterOptions);
-        const correlationId = req.query.correlationId as string | undefined;
+        const correlationId = (req.query.correlationId as string | undefined) ?? this.currentSessionId;
         const graphWithStatus = await this.addStatusToCommandNodes(filteredGraph, correlationId);
 
         const latestRun = await this.eventStoreContext.readModel.getLatestCorrelationId();
@@ -270,6 +328,19 @@ export class PipelineServer {
       void (async () => {
         const command = req.body as Command;
 
+        if (command.type === 'RestartPipeline') {
+          const previousSessionId = this.currentSessionId;
+          this.currentSessionId = `session-${nanoid()}`;
+          const requestId = `req-${nanoid()}`;
+          await this.broadcastPipelineRunStarted(this.currentSessionId, 'RestartPipeline');
+          await this.emitDomainEventEmitted(this.currentSessionId, requestId, 'PipelineRestarted', {
+            correlationId: this.currentSessionId,
+            previousSessionId,
+          });
+          res.json({ status: 'ack', sessionId: this.currentSessionId });
+          return;
+        }
+
         if (!this.commandHandlers.has(command.type)) {
           res.status(404).json({
             status: 'nack',
@@ -922,15 +993,10 @@ export class PipelineServer {
     const handler = this.commandHandlers.get(command.type);
     if (!handler) return;
 
-    if (!this.seenCorrelations.has(command.correlationId)) {
-      this.seenCorrelations.add(command.correlationId);
-      await this.broadcastPipelineRunStarted(command.correlationId, command.type);
-    }
-
     const itemKey = this.extractItemKey(command.type, command.data, command.requestId);
-    await this.getOrCreateItemStatus(command.correlationId, command.type, itemKey, command.requestId);
+    await this.getOrCreateItemStatus(this.currentSessionId, command.type, itemKey, command.requestId);
 
-    await this.updateNodeStatus(command.correlationId, command.type, 'running');
+    await this.updateNodeStatus(this.currentSessionId, command.type, 'running');
     await this.settledTracker.onCommandStarted(command);
 
     const ctx = this.createContext(command.correlationId);
@@ -956,8 +1022,8 @@ export class PipelineServer {
     }
 
     const finalStatus = this.getStatusFromEvents(events);
-    await this.updateItemStatus(command.correlationId, command.type, itemKey, finalStatus);
-    await this.updateNodeStatus(command.correlationId, command.type, finalStatus);
+    await this.updateItemStatus(this.currentSessionId, command.type, itemKey, finalStatus);
+    await this.updateNodeStatus(this.currentSessionId, command.type, finalStatus);
 
     const eventsWithIds: EventWithCorrelation[] = events.map((event) => ({
       ...event,