@auto-engineer/pipeline 1.97.0 → 1.97.2

package/package.json

@@ -4,8 +4,9 @@
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
   "dependencies": {
-    "@event-driven-io/emmett": "^0.38.2",
-    "@event-driven-io/emmett-sqlite": "^0.38.5",
+    "@event-driven-io/dumbo": "0.13.0-beta.31",
+    "@event-driven-io/emmett": "0.43.0-beta.5",
+    "@event-driven-io/emmett-sqlite": "0.43.0-beta.5",
     "chokidar": "^3.6.0",
     "cors": "^2.8.5",
     "dotenv": "^16.4.5",
@@ -13,8 +14,8 @@
     "get-port": "^7.1.0",
     "jose": "^5.9.6",
     "nanoid": "^5.0.0",
-    "@auto-engineer/file-store": "1.97.0",
-    "@auto-engineer/message-bus": "1.97.0"
+    "@auto-engineer/message-bus": "1.97.2",
+    "@auto-engineer/file-store": "1.97.2"
   },
   "devDependencies": {
     "@types/cors": "^2.8.17",
@@ -23,7 +24,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.97.0",
+  "version": "1.97.2",
   "scripts": {
     "build": "tsc && tsx ../../scripts/fix-esm-imports.ts",
     "test": "vitest run --reporter=dot",

package/src/engine/sqlite-consumer.ts

@@ -1,4 +1,6 @@
-import { readMessagesBatch, sqliteConnection } from '@event-driven-io/emmett-sqlite';
+import { dumbo } from '@event-driven-io/dumbo';
+import { sqlite3DumboDriver } from '@event-driven-io/dumbo/sqlite3';
+import { readMessagesBatch } from '@event-driven-io/emmett-sqlite';
 import type { PipelineStore } from './sqlite-store.js';
 
 type EventHandler = (event: { type: string; data: Record<string, unknown> }) => void;
@@ -6,7 +8,7 @@ type EventHandler = (event: { type: string; data: Record<string, unknown> }) =>
 export function createConsumer(store: PipelineStore) {
   const handlers = new Map<string, EventHandler>();
   let lastPosition = 0n;
-  const connection = sqliteConnection({ fileName: store.fileName });
+  const pool = dumbo({ driver: sqlite3DumboDriver, fileName: store.fileName });
 
   return {
     on(eventType: string, handler: EventHandler): void {
@@ -14,7 +16,7 @@ export function createConsumer(store: PipelineStore) {
     },
 
     async poll(): Promise<void> {
-      const { messages, currentGlobalPosition } = await readMessagesBatch(connection, {
+      const { messages, currentGlobalPosition } = await readMessagesBatch(pool.execute, {
         after: lastPosition,
         batchSize: 1000,
       });

package/src/engine/sqlite-store.ts

@@ -1,4 +1,5 @@
 import { getSQLiteEventStore, type SQLiteEventStore } from '@event-driven-io/emmett-sqlite';
+import { sqlite3EventStoreDriver } from '@event-driven-io/emmett-sqlite/sqlite3';
 
 export interface PipelineStore {
   eventStore: SQLiteEventStore;
@@ -9,6 +10,7 @@ export interface PipelineStore {
 export async function createPipelineStore(options?: { fileName?: string }): Promise<PipelineStore> {
   const fileName = options?.fileName ?? 'file::memory:?cache=shared';
   const eventStore = getSQLiteEventStore({
+    driver: sqlite3EventStoreDriver,
     fileName,
     schema: { autoMigration: 'CreateOrUpdate' },
   });

package/src/server/pipeline-server.specs.ts

@@ -1,7 +1,9 @@
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
-import { readMessagesBatch, sqliteConnection } from '@event-driven-io/emmett-sqlite';
+import { dumbo } from '@event-driven-io/dumbo';
+import { sqlite3DumboDriver } from '@event-driven-io/dumbo/sqlite3';
+import { readMessagesBatch } from '@event-driven-io/emmett-sqlite';
 import { nanoid } from 'nanoid';
 import { define } from '../builder/define';
 import { PipelineServer } from './pipeline-server';
@@ -2376,16 +2378,15 @@ describe('PipelineServer', () => {
       await new Promise((r) => setTimeout(r, 200));
       await server.stop();
 
-      const connection = sqliteConnection({ fileName: tmpFile });
-      const { messages } = await readMessagesBatch(connection, {
+      const pool = dumbo({ driver: sqlite3DumboDriver, fileName: tmpFile });
+      const { messages } = await readMessagesBatch(pool.execute, {
         after: 0n,
         batchSize: 1000,
       });
-      connection.close();
+      await pool.close();
 
-      expect(messages.length).toBeGreaterThan(0);
-      expect(messages.some((m) => m.type === 'CommandDispatched')).toBe(true);
-      expect(messages.some((m) => m.type === 'PipelineRunStarted')).toBe(true);
+      const types = messages.map((m) => m.type);
+      expect(types).toEqual(expect.arrayContaining(['CommandDispatched', 'PipelineRunStarted']));
 
       fs.unlinkSync(tmpFile);
     });

package/src/server/pipeline-server.ts

@@ -6,8 +6,11 @@ import {
   type Event,
   type MessageBus,
 } from '@auto-engineer/message-bus';
+import { dumbo } from '@event-driven-io/dumbo';
+import { sqlite3DumboDriver } from '@event-driven-io/dumbo/sqlite3';
 import type { EventStore } from '@event-driven-io/emmett';
-import { getSQLiteEventStore, readMessagesBatch, sqliteConnection } from '@event-driven-io/emmett-sqlite';
+import { getSQLiteEventStore, readMessagesBatch } from '@event-driven-io/emmett-sqlite';
+import { sqlite3EventStoreDriver } from '@event-driven-io/emmett-sqlite/sqlite3';
 import cors from 'cors';
 import express from 'express';
 import getPort from 'get-port';
@@ -180,6 +183,7 @@ export class PipelineServer {
 
     if (this.storeFileName) {
       this.sqliteEventStore = getSQLiteEventStore({
+        driver: sqlite3EventStoreDriver,
         fileName: this.storeFileName,
         schema: { autoMigration: 'CreateOrUpdate' },
       });
@@ -205,12 +209,12 @@ export class PipelineServer {
   }
 
   private async replayEventsFromSQLite(): Promise<void> {
-    const connection = sqliteConnection({ fileName: this.storeFileName! });
+    const pool = dumbo({ driver: sqlite3DumboDriver, fileName: this.storeFileName! });
     let lastPosition = 0n;
 
     try {
       while (true) {
-        const { messages, currentGlobalPosition } = await readMessagesBatch(connection, {
+        const { messages, currentGlobalPosition } = await readMessagesBatch(pool.execute, {
           after: lastPosition,
           batchSize: 1000,
         });
@@ -229,7 +233,7 @@ export class PipelineServer {
       const isMissingTable = error instanceof Error && error.message.includes('no such table');
       if (!isMissingTable) throw error;
     } finally {
-      connection.close();
+      await pool.close();
     }
   }
 

package/src/store/pipeline-event-store.specs.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from 'vitest';
 import type { ItemStatusDocument } from '../projections/item-status-projection';
+import type { MessageLogDocument } from '../projections/message-log-projection';
 import type { NodeStatusDocument } from '../projections/node-status-projection';
 import { createPipelineEventStore } from './pipeline-event-store';
 
@@ -169,6 +170,72 @@ describe('PipelineEventStore', () => {
       }
     });
 
+    it('should append a new message log entry for each NodeStatusChanged even on retry', async () => {
+      const { eventStore, readModel, close } = createPipelineEventStore();
+      try {
+        await eventStore.appendToStream('pipeline-c1', [
+          {
+            type: 'NodeStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandName: 'RebuildComponentDB',
+              nodeId: 'node-1',
+              status: 'running',
+              previousStatus: 'idle',
+              pendingCount: 1,
+              endedCount: 0,
+            },
+          },
+          {
+            type: 'NodeStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandName: 'RebuildComponentDB',
+              nodeId: 'node-1',
+              status: 'error',
+              previousStatus: 'running',
+              pendingCount: 0,
+              endedCount: 1,
+            },
+          },
+        ]);
+
+        await eventStore.appendToStream('pipeline-c1', [
+          {
+            type: 'NodeStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandName: 'RebuildComponentDB',
+              nodeId: 'node-1',
+              status: 'running',
+              previousStatus: 'error',
+              pendingCount: 1,
+              endedCount: 0,
+            },
+          },
+          {
+            type: 'NodeStatusChanged',
+            data: {
+              correlationId: 'c1',
+              commandName: 'RebuildComponentDB',
+              nodeId: 'node-1',
+              status: 'success',
+              previousStatus: 'running',
+              pendingCount: 0,
+              endedCount: 1,
+            },
+          },
+        ]);
+
+        const messages = await readModel.getMessages('c1');
+        const statuses = messages.map((m: MessageLogDocument) => (m.messageData as { status: string }).status);
+
+        expect(statuses).toEqual(['running', 'error', 'running', 'success']);
+      } finally {
+        await close();
+      }
+    });
+
     it('should track message stats through CommandDispatched and DomainEventEmitted', async () => {
       const { eventStore, readModel, close } = createPipelineEventStore();
       try {

package/src/store/pipeline-event-store.ts

@@ -7,6 +7,7 @@ import {
   inlineProjections,
   inMemorySingleStreamProjection,
 } from '@event-driven-io/emmett';
+import { nanoid } from 'nanoid';
 import type { AwaitEvent, AwaitTrackerDocument } from '../projections/await-tracker-projection';
 import { evolve as evolveAwaitTracker } from '../projections/await-tracker-projection';
 import type { ItemStatusChangedEvent, ItemStatusDocument } from '../projections/item-status-projection';
@@ -59,7 +60,7 @@ function createProjections() {
         return `prs-${event.data.correlationId}`;
       }
       if (event.type === 'NodeStatusChanged') {
-        return `nsc-${event.data.correlationId}-${event.data.commandName}-${event.data.status}`;
+        return `nsc-${nanoid()}`;
       }
       if (event.type === 'DomainEventEmitted') {
         return `dee-${event.data.requestId}-${event.data.eventType}`;

package/src/store/sqlite-pipeline-event-store.ts

@@ -1,5 +1,6 @@
 import { getInMemoryDatabase, type InMemoryDatabase } from '@event-driven-io/emmett';
 import { getSQLiteEventStore, type SQLiteEventStore } from '@event-driven-io/emmett-sqlite';
+import { sqlite3EventStoreDriver } from '@event-driven-io/emmett-sqlite/sqlite3';
 import { PipelineReadModel } from './pipeline-read-model';
 
 export interface SQLitePipelineEventStoreContext {
@@ -19,6 +20,7 @@ export async function createSQLitePipelineEventStore(
   const database = getInMemoryDatabase();
 
   const eventStore = getSQLiteEventStore({
+    driver: sqlite3EventStoreDriver,
     fileName: config.fileName,
     schema: { autoMigration: 'CreateOrUpdate' },
   });