@auto-engineer/pipeline 1.135.0 → 1.136.0

package/ketchup-plan.md

@@ -1,5 +1,7 @@
 # Ketchup Plan: Fix Pipeline Diagram Wrong Status for Retried Commands
 
+### Bottle: RetryStatusFix
+
 ## TODO
 
 ## DONE
@@ -8,3 +10,6 @@
 - [x] Burst 2: Add `autoRegisterItemKeyExtractor` and call from `registerCommandHandlers` [depends: Burst 1] (0bd4e2a7)
 - [x] Burst 3: Covered by Bursts 1+2 tests [depends: Burst 2]
 - [x] Burst 4: Test manual extractor takes precedence over auto-derived [depends: Burst 2]
+- [x] Burst 5: Add `batchId` field to `ItemStatusDocument` and `ItemStatusChangedEvent`, preserve via `evolve` fallback [depends: none] (752e1329)
+- [x] Burst 6: Scope `computeCommandStats` to latest batch — filter items by highest `batchId` with backward compat fallback [depends: Burst 5] (a34a7e6a)
+- [x] Burst 7+8: Wire sync `resolveBatchId` with pending counter, update test expectations for batch-scoped counts [depends: Burst 6] (7e33cf98)

package/package.json

@@ -15,8 +15,8 @@
     "jose": "^5.9.6",
     "nanoid": "^5.0.0",
     "uuid": "^11.0.0",
-    "@auto-engineer/file-store": "1.135.0",
-    "@auto-engineer/message-bus": "1.135.0"
+    "@auto-engineer/file-store": "1.136.0",
+    "@auto-engineer/message-bus": "1.136.0"
   },
   "devDependencies": {
     "@types/cors": "^2.8.17",
@@ -25,7 +25,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.135.0",
+  "version": "1.136.0",
   "scripts": {
     "build": "tsc && tsx ../../scripts/fix-esm-imports.ts",
     "test": "vitest run --reporter=dot",

package/src/projections/item-status-projection.specs.ts

@@ -179,6 +179,68 @@ describe('ItemStatusProjection', () => {
       expect(result.endedAt).toEqual(endTime.toISOString());
     });
 
+    it('preserves batchId from creation event on subsequent updates', () => {
+      const existing: ItemStatusDocument = {
+        correlationId: 'c1',
+        commandType: 'ProcessItem',
+        itemKey: 'item-1',
+        currentRequestId: 'req-1',
+        status: 'running',
+        attemptCount: 1,
+        batchId: '2025-01-01T00:00:00.000Z',
+      };
+      const event: ItemStatusChangedEvent = {
+        type: 'ItemStatusChanged',
+        data: {
+          correlationId: 'c1',
+          commandType: 'ProcessItem',
+          itemKey: 'item-1',
+          requestId: 'req-1',
+          status: 'success',
+          attemptCount: 1,
+        },
+      };
+
+      const result = evolve(existing, event);
+
+      expect(result).toEqual({
+        correlationId: 'c1',
+        commandType: 'ProcessItem',
+        itemKey: 'item-1',
+        currentRequestId: 'req-1',
+        status: 'success',
+        attemptCount: 1,
+        batchId: '2025-01-01T00:00:00.000Z',
+      });
+    });
+
+    it('sets batchId from event data on creation', () => {
+      const event: ItemStatusChangedEvent = {
+        type: 'ItemStatusChanged',
+        data: {
+          correlationId: 'c1',
+          commandType: 'ProcessItem',
+          itemKey: 'item-1',
+          requestId: 'req-1',
+          status: 'running',
+          attemptCount: 1,
+          batchId: '2025-01-01T00:00:00.000Z',
+        },
+      };
+
+      const result = evolve(null, event);
+
+      expect(result).toEqual({
+        correlationId: 'c1',
+        commandType: 'ProcessItem',
+        itemKey: 'item-1',
+        currentRequestId: 'req-1',
+        status: 'running',
+        attemptCount: 1,
+        batchId: '2025-01-01T00:00:00.000Z',
+      });
+    });
+
     it('sets endedAt when status becomes error', () => {
       const startTime = new Date('2025-01-01T00:00:00Z');
       const endTime = new Date('2025-01-01T00:00:03Z');

package/src/projections/item-status-projection.ts

@@ -8,6 +8,7 @@ export interface ItemStatusDocument {
   attemptCount: number;
   startedAt?: string;
   endedAt?: string;
+  batchId?: string;
 }
 
 export interface ItemStatusChangedEvent {
@@ -20,6 +21,7 @@ export interface ItemStatusChangedEvent {
     status: 'running' | 'success' | 'error';
     attemptCount: number;
     timestamp?: string;
+    batchId?: string;
   };
 }
 
@@ -31,6 +33,7 @@ export function evolve(document: ItemStatusDocument | null, event: ItemStatusCha
     currentRequestId: event.data.requestId,
     status: event.data.status,
     attemptCount: event.data.attemptCount,
+    batchId: event.data.batchId ?? document?.batchId,
   };
 
   if (event.data.status === 'running') {

package/src/server/pipeline-server.specs.ts

@@ -832,7 +832,7 @@ describe('PipelineServer', () => {
       const data = await fetchAs<PipelineResponse>(`http://localhost:${server.port}/pipeline`);
       const cmdNode = data.nodes.find((n) => n.id === 'cmd:IndependentCmd');
       expect(cmdNode?.status).toBe('success');
-      expect(cmdNode?.endedCount).toBe(2);
+      expect(cmdNode?.endedCount).toBe(1);
 
       await server.stop();
     });
@@ -1707,10 +1707,16 @@ describe('PipelineServer', () => {
     });
 
     it('should count multiple parallel items correctly', async () => {
+      const resolvers: Array<() => void> = [];
       const handler = {
         name: 'ImplementSlice',
         events: ['SliceImplemented'],
-        handle: async () => ({ type: 'SliceImplemented', data: {} }),
+        handle: async () => {
+          await new Promise<void>((resolve) => {
+            resolvers.push(resolve);
+          });
+          return { type: 'SliceImplemented', data: {} };
+        },
       };
       const pipeline = define('test').on('Trigger').emit('ImplementSlice', {}).build();
       const server = new PipelineServer({ port: 0 });
@@ -1719,24 +1725,26 @@ describe('PipelineServer', () => {
       server.registerItemKeyExtractor('ImplementSlice', (d) => (d as { slicePath?: string }).slicePath);
       await server.start();
 
-      await Promise.all([
-        fetch(`http://localhost:${server.port}/command`, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ type: 'ImplementSlice', data: { slicePath: '/server/slice-1' } }),
-        }),
-        fetch(`http://localhost:${server.port}/command`, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ type: 'ImplementSlice', data: { slicePath: '/server/slice-2' } }),
-        }),
-        fetch(`http://localhost:${server.port}/command`, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ type: 'ImplementSlice', data: { slicePath: '/server/slice-3' } }),
-        }),
-      ]);
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'ImplementSlice', data: { slicePath: '/server/slice-1' } }),
+      });
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'ImplementSlice', data: { slicePath: '/server/slice-2' } }),
+      });
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'ImplementSlice', data: { slicePath: '/server/slice-3' } }),
+      });
 
+      while (resolvers.length < 3) {
+        await new Promise((r) => setTimeout(r, 10));
+      }
+      resolvers.forEach((r) => r());
       await new Promise((r) => setTimeout(r, 100));
 
       const data = await fetchAs<PipelineResponse>(`http://localhost:${server.port}/pipeline`);
@@ -1805,10 +1813,14 @@ describe('PipelineServer', () => {
     });
 
     it('should show error status when any item fails', async () => {
+      const resolvers: Array<() => void> = [];
       const handler = {
         name: 'MixedSlice',
         events: ['MixedSliceDone', 'MixedSliceFailed'],
         handle: async (cmd: { data: { shouldFail?: boolean } }) => {
+          await new Promise<void>((resolve) => {
+            resolvers.push(resolve);
+          });
           if (cmd.data.shouldFail === true) {
             return { type: 'MixedSliceFailed', data: {} };
           }
@@ -1822,24 +1834,26 @@ describe('PipelineServer', () => {
       server.registerItemKeyExtractor('MixedSlice', (d) => (d as { id?: string }).id);
       await server.start();
 
-      await Promise.all([
-        fetch(`http://localhost:${server.port}/command`, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ type: 'MixedSlice', data: { id: 'pass-1' } }),
-        }),
-        fetch(`http://localhost:${server.port}/command`, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ type: 'MixedSlice', data: { id: 'fail-1', shouldFail: true } }),
-        }),
-        fetch(`http://localhost:${server.port}/command`, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ type: 'MixedSlice', data: { id: 'pass-2' } }),
-        }),
-      ]);
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'MixedSlice', data: { id: 'pass-1' } }),
+      });
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'MixedSlice', data: { id: 'fail-1', shouldFail: true } }),
+      });
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'MixedSlice', data: { id: 'pass-2' } }),
+      });
 
+      while (resolvers.length < 3) {
+        await new Promise((r) => setTimeout(r, 10));
+      }
+      resolvers.forEach((r) => r());
       await new Promise((r) => setTimeout(r, 100));
 
       const data = await fetchAs<PipelineResponse>(`http://localhost:${server.port}/pipeline`);
@@ -2120,7 +2134,7 @@ describe('PipelineServer', () => {
         label: 'RetryNoExtractor',
         status: 'success',
         pendingCount: 0,
-        endedCount: 2,
+        endedCount: 1,
         lastDurationMs: expect.any(Number),
       });
 
@@ -2168,6 +2182,78 @@ describe('PipelineServer', () => {
       resolveHandler();
       await server.stop();
     });
+
+    it('should not let stale error items from a previous batch contaminate status', async () => {
+      let callCount = 0;
+      const resolvers: Array<() => void> = [];
+      const handler = {
+        name: 'BatchCmd',
+        events: ['BatchDone', 'BatchFailed'],
+        handle: async () => {
+          callCount++;
+          const thisCall = callCount;
+          await new Promise<void>((resolve) => {
+            resolvers.push(resolve);
+          });
+          if (thisCall <= 2) {
+            return { type: 'BatchFailed', data: {} };
+          }
+          return { type: 'BatchDone', data: {} };
+        },
+      };
+      const pipeline = define('test').on('Trigger').emit('BatchCmd', {}).build();
+      const server = new PipelineServer({ port: 0 });
+      server.registerCommandHandlers([handler]);
+      server.registerPipeline(pipeline);
+      server.registerItemKeyExtractor('BatchCmd', (d) => (d as { id?: string }).id);
+      await server.start();
+
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'BatchCmd', data: { id: 'item-a' } }),
+      });
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'BatchCmd', data: { id: 'item-b' } }),
+      });
+
+      while (resolvers.length < 2) {
+        await new Promise((r) => setTimeout(r, 10));
+      }
+      resolvers.forEach((r) => r());
+      await new Promise((r) => setTimeout(r, 100));
+
+      const firstBatch = await fetchAs<PipelineResponse>(`http://localhost:${server.port}/pipeline`);
+      const firstNode = firstBatch.nodes.find((n) => n.id === 'cmd:BatchCmd');
+      expect(firstNode?.status).toBe('error');
+      expect(firstNode?.endedCount).toBe(2);
+
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'BatchCmd', data: { id: 'item-c' } }),
+      });
+      void fetch(`http://localhost:${server.port}/command`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ type: 'BatchCmd', data: { id: 'item-d' } }),
+      });
+
+      while (resolvers.length < 4) {
+        await new Promise((r) => setTimeout(r, 10));
+      }
+      resolvers.slice(2).forEach((r) => r());
+      await new Promise((r) => setTimeout(r, 100));
+
+      const secondBatch = await fetchAs<PipelineResponse>(`http://localhost:${server.port}/pipeline`);
+      const secondNode = secondBatch.nodes.find((n) => n.id === 'cmd:BatchCmd');
+      expect(secondNode?.status).toBe('success');
+      expect(secondNode?.endedCount).toBe(2);
+
+      await server.stop();
+    });
   });
 
   describe('auto-derived item key extractors', () => {

package/src/server/pipeline-server.ts

@@ -76,6 +76,9 @@ export class PipelineServer {
   private currentSessionId = '';
   private readonly quiescenceTracker: QuiescenceTracker;
   private readonly requestIdToSourceEvent = new Map<string, string>();
+  private readonly settledRequestIds = new Set<string>();
+  private readonly currentBatchIds = new Map<string, string>();
+  private readonly batchPendingCounts = new Map<string, number>();
 
   constructor(config: PipelineServerConfig) {
     this.storeFileName = config.storeFileName;
@@ -666,6 +669,7 @@ export class PipelineServer {
     requestId: string,
     status: 'running' | 'success' | 'error',
     attemptCount: number,
+    batchId?: string,
   ): Promise<void> {
     await this.eventStoreContext.eventStore.appendToStream(`pipeline-${correlationId}`, [
       {
@@ -678,6 +682,7 @@ export class PipelineServer {
           status,
           attemptCount,
           timestamp: new Date().toISOString(),
+          ...(batchId !== undefined && { batchId }),
         },
       },
     ]);
@@ -828,6 +833,30 @@ export class PipelineServer {
     this.sseManager.broadcast(event);
   }
 
+  private resolveBatchId(correlationId: string, commandType: string, requestId: string): string {
+    const key = `${correlationId}:${commandType}`;
+    const isFromSettled = this.settledRequestIds.delete(requestId);
+
+    if (isFromSettled) {
+      const existing = this.currentBatchIds.get(key);
+      if (existing) return existing;
+    } else {
+      const pending = this.batchPendingCounts.get(key) ?? 0;
+      if (pending === 0) {
+        const batchId = new Date().toISOString();
+        this.currentBatchIds.set(key, batchId);
+        return batchId;
+      }
+    }
+
+    const existing = this.currentBatchIds.get(key);
+    if (existing) return existing;
+
+    const batchId = new Date().toISOString();
+    this.currentBatchIds.set(key, batchId);
+    return batchId;
+  }
+
   private extractItemKey(commandType: string, data: unknown, requestId: string): string {
     const extractor = this.itemKeyExtractors.get(commandType);
     if (extractor !== undefined) {
@@ -842,11 +871,12 @@ export class PipelineServer {
     commandType: string,
     itemKey: string,
     requestId: string,
+    batchId?: string,
   ): Promise<{ attemptCount: number }> {
     const existing = await this.eventStoreContext.readModel.getItemStatus(correlationId, commandType, itemKey);
     const attemptCount = (existing?.attemptCount ?? 0) + 1;
 
-    await this.emitItemStatusChanged(correlationId, commandType, itemKey, requestId, 'running', attemptCount);
+    await this.emitItemStatusChanged(correlationId, commandType, itemKey, requestId, 'running', attemptCount, batchId);
 
     return { attemptCount };
   }
@@ -1171,8 +1201,12 @@ export class PipelineServer {
     handler: CommandHandlerWithMetadata,
     signal: AbortSignal,
   ): Promise<void> {
+    const batchKey = `${this.currentSessionId}:${command.type}`;
+    const batchId = this.resolveBatchId(this.currentSessionId, command.type, command.requestId);
+    this.batchPendingCounts.set(batchKey, (this.batchPendingCounts.get(batchKey) ?? 0) + 1);
+
     const itemKey = this.extractItemKey(command.type, command.data, command.requestId);
-    await this.getOrCreateItemStatus(this.currentSessionId, command.type, itemKey, command.requestId);
+    await this.getOrCreateItemStatus(this.currentSessionId, command.type, itemKey, command.requestId, batchId);
 
     await this.updateNodeStatus(this.currentSessionId, command.type, 'running');
     const sourceEventType = this.requestIdToSourceEvent.get(command.requestId);
@@ -1202,10 +1236,14 @@ export class PipelineServer {
       ];
     }
 
-    if (signal.aborted) return;
+    if (signal.aborted) {
+      this.batchPendingCounts.set(batchKey, (this.batchPendingCounts.get(batchKey) ?? 1) - 1);
+      return;
+    }
 
     const finalStatus = this.getStatusFromEvents(events);
     await this.updateItemStatus(this.currentSessionId, command.type, itemKey, finalStatus);
+    this.batchPendingCounts.set(batchKey, (this.batchPendingCounts.get(batchKey) ?? 1) - 1);
     const completedItem = await this.eventStoreContext.readModel.getItemStatus(
       this.currentSessionId,
       command.type,
@@ -1255,9 +1293,10 @@ export class PipelineServer {
     return 'success';
   }
 
-  /* v8 ignore next 11 - integration callback tested via v2-runtime-bridge.specs.ts */
+  /* v8 ignore next 12 - integration callback tested via v2-runtime-bridge.specs.ts */
   private async dispatchFromSettled(commandType: string, data: unknown, correlationId: string): Promise<void> {
     const requestId = `req-${nanoid()}`;
+    this.settledRequestIds.add(requestId);
     const command: Command & { correlationId: string; requestId: string } = {
       type: commandType,
       data: data as Record<string, unknown>,

package/src/store/pipeline-read-model.specs.ts

@@ -277,6 +277,88 @@ describe('PipelineReadModel', () => {
       });
     });
 
+    it('should scope to latest batch and ignore stale errors from previous batches', async () => {
+      const collection = database.collection<WithId<ItemStatusDocument>>('ItemStatus');
+      await collection.insertOne({
+        _id: 'c1-Cmd-old-a',
+        correlationId: 'c1',
+        commandType: 'Cmd',
+        itemKey: 'old-a',
+        currentRequestId: 'r1',
+        status: 'error',
+        attemptCount: 1,
+        batchId: '2025-01-01T00:00:00.000Z',
+      });
+      await collection.insertOne({
+        _id: 'c1-Cmd-old-b',
+        correlationId: 'c1',
+        commandType: 'Cmd',
+        itemKey: 'old-b',
+        currentRequestId: 'r2',
+        status: 'error',
+        attemptCount: 1,
+        batchId: '2025-01-01T00:00:00.000Z',
+      });
+      await collection.insertOne({
+        _id: 'c1-Cmd-new-a',
+        correlationId: 'c1',
+        commandType: 'Cmd',
+        itemKey: 'new-a',
+        currentRequestId: 'r3',
+        status: 'success',
+        attemptCount: 1,
+        batchId: '2025-01-01T00:01:00.000Z',
+      });
+      await collection.insertOne({
+        _id: 'c1-Cmd-new-b',
+        correlationId: 'c1',
+        commandType: 'Cmd',
+        itemKey: 'new-b',
+        currentRequestId: 'r4',
+        status: 'success',
+        attemptCount: 1,
+        batchId: '2025-01-01T00:01:00.000Z',
+      });
+
+      const result = await readModel.computeCommandStats('c1', 'Cmd');
+
+      expect(result).toEqual({
+        pendingCount: 0,
+        endedCount: 2,
+        aggregateStatus: 'success',
+      });
+    });
+
+    it('should use all items when no batchId exists for backward compatibility', async () => {
+      const collection = database.collection<WithId<ItemStatusDocument>>('ItemStatus');
+      await collection.insertOne({
+        _id: 'c1-Cmd-a',
+        correlationId: 'c1',
+        commandType: 'Cmd',
+        itemKey: 'a',
+        currentRequestId: 'r1',
+        status: 'success',
+        attemptCount: 1,
+      });
+      await collection.insertOne({
+        _id: 'c1-Cmd-b',
+        correlationId: 'c1',
+        commandType: 'Cmd',
+        itemKey: 'b',
+        currentRequestId: 'r2',
+        status: 'error',
+        attemptCount: 1,
+      });
+
+      const result = await readModel.computeCommandStats('c1', 'Cmd');
+
+      expect(result).toEqual({
+        pendingCount: 0,
+        endedCount: 2,
+        aggregateStatus: 'error',
+      });
+    });
+
     it('documents behavior: returns error when stale error items exist (BUG: requestId-based itemKey needs proper extractor)', async () => {
       const collection = database.collection<WithId<ItemStatusDocument>>('ItemStatus');
       await collection.insertOne({

package/src/store/pipeline-read-model.ts

@@ -50,11 +50,18 @@ export class PipelineReadModel {
       return { pendingCount: 0, endedCount: 0, aggregateStatus: 'idle' };
     }
 
+    const latestBatchId = items.reduce<string | undefined>((latest, item) => {
+      if (item.batchId === undefined) return latest;
+      if (latest === undefined || item.batchId > latest) return item.batchId;
+      return latest;
+    }, undefined);
+    const currentItems = latestBatchId !== undefined ? items.filter((item) => item.batchId === latestBatchId) : items;
+
     let pendingCount = 0;
     let endedCount = 0;
     let hasError = false;
 
-    for (const item of items) {
+    for (const item of currentItems) {
       if (item.status === 'running') {
         pendingCount++;
       } else {
@@ -117,6 +124,7 @@ export class PipelineReadModel {
       attemptCount: item.attemptCount,
       startedAt: item.startedAt,
       endedAt: item.endedAt,
+      batchId: item.batchId,
     };
   }
 
@@ -131,6 +139,7 @@ export class PipelineReadModel {
       attemptCount: item.attemptCount,
       startedAt: item.startedAt,
       endedAt: item.endedAt,
+      batchId: item.batchId,
     }));
   }