@flowcore/data-pump 0.14.0 → 0.15.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [0.15.0](https://github.com/flowcore-io/data-pump/compare/v0.14.0...v0.15.0) (2025-06-03)
+
+
+### Features
+
+* **data-pump:** :sparkles: add acknowledgment timeout scaling and timeout management ([17ea18d](https://github.com/flowcore-io/data-pump/commit/17ea18d6cef86ad73a77158a1008ce353e3471bb))
+
+
+### Bug Fixes
+
+* **data-pump:** :art: reverted concurrency changes ([0f0b29c](https://github.com/flowcore-io/data-pump/commit/0f0b29ce5a807cd5bf0e6a44798310ee409adab1))
+* **data-pump:** :art: update timeoutId type to use ReturnType of setTimeout ([132aba2](https://github.com/flowcore-io/data-pump/commit/132aba2153e70b8532fab3cfc53816e2d9450c21))
+* **deno:** :art: update deno.lock version and format deno.json ([9bf22c7](https://github.com/flowcore-io/data-pump/commit/9bf22c767f3e2c3c413386d38e4f2d60cdd586f8))
+
 ## [0.14.0](https://github.com/flowcore-io/data-pump/compare/v0.13.0...v0.14.0) (2025-06-03)
 
 

package/README.md

@@ -39,28 +39,3 @@ await dataPump.start((error?: Error) => {
   console.log("Datapump ended with: ", error)
 })
 ```
-
-## Configuration
-
-### Concurrency and Acknowledgment Timeout
-
-When using higher concurrency values, the data pump automatically scales the acknowledgment timeout to prevent events
-from being prematurely failed due to processing time. The default acknowledgment timeout is calculated as:
-
-```ts
-const defaultAckTimeout = Math.max(10_000, concurrency * 2_000) // At least 10s, scale with concurrency
-```
-
-For example:
-
-- `concurrency: 1` → `achknowledgeTimeoutMs: 10_000` (10 seconds)
-- `concurrency: 10` → `achknowledgeTimeoutMs: 20_000` (20 seconds)
-- `concurrency: 50` → `achknowledgeTimeoutMs: 100_000` (100 seconds)
-
-You can override this by explicitly setting `achknowledgeTimeoutMs` in the options.
-
-### Important Notes
-
-- When using high concurrency, ensure your `achknowledgeTimeoutMs` is sufficient for your processing time
-- Events that exceed `maxRedeliveryCount` (default: 3) will be permanently removed from the buffer
-- The buffer management has been improved to prevent timeout race conditions with high concurrency

package/esm/data-pump/data-pump.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAGrD,OAAO,KAAK,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,cAAc,EACf,MAAM,YAAY,CAAA;AAEnB,UAAU,mCAAmC;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED,UAAU,wCAAwC;IAChD,IAAI,EAAE,WAAW,CAAA;CAClB;AAED,UAAU,qCAAqC;IAC7C,IAAI,EAAE,QAAQ,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,KAAK,+BAA+B,GAChC,mCAAmC,GACnC,wCAAwC,GACxC,qCAAqC,CAAA;AAEzC,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;IACtC,YAAY,EAAE,4BAA4B,CAAA;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAC9B,SAAS,CAAC,EAAE,yBAAyB,CAAA;IACrC,QAAQ,CAAC,EAAE,+BAA+B,CAAA;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAoBD,qBAAa,gBAAgB;IAWzB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAd1B,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAC,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAQ;IAEtB,OAAO;IAaP,IAAW,SAAS,IAAI,OAAO,CAE9B;WAEa,MAAM,CAAC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,gBAAgB;IAuCpG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA2C9D,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAQjE,IAAI,CAAC,SAAS,UAAQ,GAAG,IAAI;IAWpC,OAAO,CAAC,WAAW;YAaL,IAAI;IAmFL,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA4CjD,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE;IAqC9B,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;YAwCtB,MAAM;YAkDN,WAAW;IAiBzB,OAAO,CAAC,mBAAmB;IA4D3B,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,YAAY,CAAC,CAAY;YACnB,aAAa;IAO3B,OAAO,CAAC,qBAAqB,CAAC,CAAY;YAC5B,sBAAsB;IAOpC,OAAO,CAAC,iBAAiB,CAAC,CAAY;YACxB,kBAAkB;CAWjC"}
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAGrD,OAAO,KAAK,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,cAAc,EACf,MAAM,YAAY,CAAA;AAEnB,UAAU,mCAAmC;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED,UAAU,wCAAwC;IAChD,IAAI,EAAE,WAAW,CAAA;CAClB;AAED,UAAU,qCAAqC;IAC7C,IAAI,EAAE,QAAQ,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,KAAK,+BAA+B,GAChC,mCAAmC,GACnC,wCAAwC,GACxC,qCAAqC,CAAA;AAEzC,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;IACtC,YAAY,EAAE,4BAA4B,CAAA;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAC9B,SAAS,CAAC,EAAE,yBAAyB,CAAA;IACrC,QAAQ,CAAC,EAAE,+BAA+B,CAAA;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAmBD,qBAAa,gBAAgB;IAWzB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAd1B,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAC,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAQ;IAEtB,OAAO;IAaP,IAAW,SAAS,IAAI,OAAO,CAE9B;WAEa,MAAM,CAAC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,gBAAgB;IAkCpG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA2C9D,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAQjE,IAAI,CAAC,SAAS,UAAQ,GAAG,IAAI;IAWpC,OAAO,CAAC,WAAW;YAaL,IAAI;IAmFL,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAqCjD,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE;IA0B9B,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;YA4BtB,MAAM;YAiDN,WAAW;IAiBzB,OAAO,CAAC,mBAAmB;IA4D3B,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,YAAY,CAAC,CAAY;YACnB,aAAa;IAO3B,OAAO,CAAC,qBAAqB,CAAC,CAAY;YAC5B,sBAAsB;IAOpC,OAAO,CAAC,iBAAiB,CAAC,CAAY;YACxB,kBAAkB;CAWjC"}

package/esm/data-pump/data-pump.js

@@ -129,14 +129,11 @@ export class FlowcoreDataPump {
             directMode: options.directMode,
             noTranslation: options.noTranslation,
         });
-        // Calculate acknowledgment timeout based on concurrency if not explicitly set
-        const concurrency = options.processor?.concurrency ?? 1;
-        const defaultAckTimeout = Math.max(10_000, concurrency * 2_000); // At least 10s, scale with concurrency
         return new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
             bufferSize: options.bufferSize ?? 1000,
             bufferThreshold: Math.ceil((options.bufferSize ?? 1000) * 0.1),
             maxRedeliveryCount: options.maxRedeliveryCount ?? 3,
-            achknowledgeTimeoutMs: options.achknowledgeTimeoutMs ?? defaultAckTimeout,
+            achknowledgeTimeoutMs: options.achknowledgeTimeoutMs ?? 5_000,
             includeSensitiveData: options.includeSensitiveData ?? false,
             processor: options.processor,
             stopAt: options.stopAt,
@@ -273,14 +270,12 @@ export class FlowcoreDataPump {
         }
         const events = [];
         const deliveryId = crypto.randomUUID();
-        const reservedBufferItems = [];
         for (const event of this.buffer) {
             if (event.status === "open") {
                 event.status = "reserved";
                 event.deliveryId = deliveryId;
                 event.deliveryCount++;
                 events.push(event.event);
-                reservedBufferItems.push(event);
                 this.incMetricsCounter("pulled", event.event.eventType, JSON.stringify(event.event).length);
                 if (events.length === amount) {
                     break;
@@ -292,13 +287,9 @@ export class FlowcoreDataPump {
             return this.reserve(amount);
         }
         this.updateMetricsGauges();
-        const timeoutId = setTimeout(() => {
+        setTimeout(() => {
             this.reOpen(events.map((event) => event.eventId), deliveryId);
         }, this.options.achknowledgeTimeoutMs);
-        // Store timeout ID in all reserved items so we can clear it on acknowledgment
-        for (const item of reservedBufferItems) {
-            item.timeoutId = timeoutId;
-        }
         return events;
     }
     async acknowledge(eventIds) {
@@ -306,22 +297,13 @@ export class FlowcoreDataPump {
             return;
         }
         const lastEventInBuffer = this.buffer[this.buffer.length - 1];
-        const acknowledgedTimeoutIds = new Set();
         this.buffer = this.buffer.filter((event) => {
             if (eventIds.includes(event.event.eventId)) {
                 this.incMetricsCounter("acknowledged", event.event.eventType, 1);
-                // Collect timeout IDs to clear them
-                if (event.timeoutId !== undefined) {
-                    acknowledgedTimeoutIds.add(event.timeoutId);
-                }
                 return false;
             }
             return true;
         });
-        // Clear timeouts for acknowledged events to prevent unnecessary reOpen calls
-        for (const timeoutId of acknowledgedTimeoutIds) {
-            clearTimeout(timeoutId);
-        }
         if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
             this.waiterBufferThreshold?.();
         }
@@ -337,23 +319,14 @@ export class FlowcoreDataPump {
         }
         const lastEventInBuffer = this.buffer[this.buffer.length - 1];
         const failedEvents = [];
-        const failedTimeoutIds = new Set();
         this.buffer = this.buffer.filter((event) => {
             if (eventIds.includes(event.event.eventId)) {
                 this.incMetricsCounter("failed", event.event.eventType, 1);
                 failedEvents.push(event.event);
-                // Collect timeout IDs to clear them
-                if (event.timeoutId !== undefined) {
-                    failedTimeoutIds.add(event.timeoutId);
-                }
                 return false;
             }
             return true;
         });
-        // Clear timeouts for failed events
-        for (const timeoutId of failedTimeoutIds) {
-            clearTimeout(timeoutId);
-        }
         this.logger?.info(`Failed ${failedEvents.length} events`);
         void this.options.processor?.failedHandler?.(failedEvents);
         if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
@@ -380,7 +353,6 @@ export class FlowcoreDataPump {
             }
             event.status = "open";
             event.deliveryId = undefined;
-            event.timeoutId = undefined;
             reopenedEvents.push(event.event);
             return true;
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowcore/data-pump",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "Flowcore Data Pump",
   "homepage": "https://github.com/flowcore-io/flowcore-data-pump#readme",
   "repository": {

package/script/data-pump/data-pump.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAGrD,OAAO,KAAK,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,cAAc,EACf,MAAM,YAAY,CAAA;AAEnB,UAAU,mCAAmC;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED,UAAU,wCAAwC;IAChD,IAAI,EAAE,WAAW,CAAA;CAClB;AAED,UAAU,qCAAqC;IAC7C,IAAI,EAAE,QAAQ,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,KAAK,+BAA+B,GAChC,mCAAmC,GACnC,wCAAwC,GACxC,qCAAqC,CAAA;AAEzC,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;IACtC,YAAY,EAAE,4BAA4B,CAAA;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAC9B,SAAS,CAAC,EAAE,yBAAyB,CAAA;IACrC,QAAQ,CAAC,EAAE,+BAA+B,CAAA;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAoBD,qBAAa,gBAAgB;IAWzB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAd1B,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAC,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAQ;IAEtB,OAAO;IAaP,IAAW,SAAS,IAAI,OAAO,CAE9B;WAEa,MAAM,CAAC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,gBAAgB;IAuCpG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA2C9D,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAQjE,IAAI,CAAC,SAAS,UAAQ,GAAG,IAAI;IAWpC,OAAO,CAAC,WAAW;YAaL,IAAI;IAmFL,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA4CjD,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE;IAqC9B,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;YAwCtB,MAAM;YAkDN,WAAW;IAiBzB,OAAO,CAAC,mBAAmB;IA4D3B,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,YAAY,CAAC,CAAY;YACnB,aAAa;IAO3B,OAAO,CAAC,qBAAqB,CAAC,CAAY;YAC5B,sBAAsB;IAOpC,OAAO,CAAC,iBAAiB,CAAC,CAAY;YACxB,kBAAkB;CAWjC"}
+{"version":3,"file":"data-pump.d.ts","sourceRoot":"","sources":["../../src/data-pump/data-pump.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAGrD,OAAO,KAAK,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,cAAc,EACf,MAAM,YAAY,CAAA;AAEnB,UAAU,mCAAmC;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED,UAAU,wCAAwC;IAChD,IAAI,EAAE,WAAW,CAAA;CAClB;AAED,UAAU,qCAAqC;IAC7C,IAAI,EAAE,QAAQ,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,KAAK,+BAA+B,GAChC,mCAAmC,GACnC,wCAAwC,GACxC,qCAAqC,CAAA;AAEzC,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,oBAAoB,CAAA;IAC1B,UAAU,EAAE,0BAA0B,CAAA;IACtC,YAAY,EAAE,4BAA4B,CAAA;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAC9B,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAC9B,SAAS,CAAC,EAAE,yBAAyB,CAAA;IACrC,QAAQ,CAAC,EAAE,+BAA+B,CAAA;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAmBD,qBAAa,gBAAgB;IAWzB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IAd1B,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAiB;IACzC,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAC,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAQ;IAEtB,OAAO;IAaP,IAAW,SAAS,IAAI,OAAO,CAE9B;WAEa,MAAM,CAAC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,gBAAgB;IAkCpG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA2C9D,OAAO,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAQjE,IAAI,CAAC,SAAS,UAAQ,GAAG,IAAI;IAWpC,OAAO,CAAC,WAAW;YAaL,IAAI;IAmFL,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAqCjD,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE;IA0B9B,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;YA4BtB,MAAM;YAiDN,WAAW;IAiBzB,OAAO,CAAC,mBAAmB;IA4D3B,OAAO,CAAC,iBAAiB;IA0CzB,OAAO,CAAC,YAAY,CAAC,CAAY;YACnB,aAAa;IAO3B,OAAO,CAAC,qBAAqB,CAAC,CAAY;YAC5B,sBAAsB;IAOpC,OAAO,CAAC,iBAAiB,CAAC,CAAY;YACxB,kBAAkB;CAWjC"}

package/script/data-pump/data-pump.js

@@ -132,14 +132,11 @@ class FlowcoreDataPump {
             directMode: options.directMode,
             noTranslation: options.noTranslation,
         });
-        // Calculate acknowledgment timeout based on concurrency if not explicitly set
-        const concurrency = options.processor?.concurrency ?? 1;
-        const defaultAckTimeout = Math.max(10_000, concurrency * 2_000); // At least 10s, scale with concurrency
         return new FlowcoreDataPump(dataSource, notifier, options.stateManager, {
             bufferSize: options.bufferSize ?? 1000,
             bufferThreshold: Math.ceil((options.bufferSize ?? 1000) * 0.1),
             maxRedeliveryCount: options.maxRedeliveryCount ?? 3,
-            achknowledgeTimeoutMs: options.achknowledgeTimeoutMs ?? defaultAckTimeout,
+            achknowledgeTimeoutMs: options.achknowledgeTimeoutMs ?? 5_000,
             includeSensitiveData: options.includeSensitiveData ?? false,
             processor: options.processor,
             stopAt: options.stopAt,
@@ -276,14 +273,12 @@ class FlowcoreDataPump {
         }
         const events = [];
         const deliveryId = crypto.randomUUID();
-        const reservedBufferItems = [];
         for (const event of this.buffer) {
             if (event.status === "open") {
                 event.status = "reserved";
                 event.deliveryId = deliveryId;
                 event.deliveryCount++;
                 events.push(event.event);
-                reservedBufferItems.push(event);
                 this.incMetricsCounter("pulled", event.event.eventType, JSON.stringify(event.event).length);
                 if (events.length === amount) {
                     break;
@@ -295,13 +290,9 @@ class FlowcoreDataPump {
             return this.reserve(amount);
         }
         this.updateMetricsGauges();
-        const timeoutId = setTimeout(() => {
+        setTimeout(() => {
             this.reOpen(events.map((event) => event.eventId), deliveryId);
         }, this.options.achknowledgeTimeoutMs);
-        // Store timeout ID in all reserved items so we can clear it on acknowledgment
-        for (const item of reservedBufferItems) {
-            item.timeoutId = timeoutId;
-        }
         return events;
     }
     async acknowledge(eventIds) {
@@ -309,22 +300,13 @@ class FlowcoreDataPump {
             return;
         }
         const lastEventInBuffer = this.buffer[this.buffer.length - 1];
-        const acknowledgedTimeoutIds = new Set();
         this.buffer = this.buffer.filter((event) => {
             if (eventIds.includes(event.event.eventId)) {
                 this.incMetricsCounter("acknowledged", event.event.eventType, 1);
-                // Collect timeout IDs to clear them
-                if (event.timeoutId !== undefined) {
-                    acknowledgedTimeoutIds.add(event.timeoutId);
-                }
                 return false;
             }
             return true;
         });
-        // Clear timeouts for acknowledged events to prevent unnecessary reOpen calls
-        for (const timeoutId of acknowledgedTimeoutIds) {
-            clearTimeout(timeoutId);
-        }
         if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
             this.waiterBufferThreshold?.();
         }
@@ -340,23 +322,14 @@ class FlowcoreDataPump {
         }
         const lastEventInBuffer = this.buffer[this.buffer.length - 1];
         const failedEvents = [];
-        const failedTimeoutIds = new Set();
         this.buffer = this.buffer.filter((event) => {
             if (eventIds.includes(event.event.eventId)) {
                 this.incMetricsCounter("failed", event.event.eventType, 1);
                 failedEvents.push(event.event);
-                // Collect timeout IDs to clear them
-                if (event.timeoutId !== undefined) {
-                    failedTimeoutIds.add(event.timeoutId);
-                }
                 return false;
             }
             return true;
         });
-        // Clear timeouts for failed events
-        for (const timeoutId of failedTimeoutIds) {
-            clearTimeout(timeoutId);
-        }
         this.logger?.info(`Failed ${failedEvents.length} events`);
         void this.options.processor?.failedHandler?.(failedEvents);
         if (this.buffer.length <= this.options.bufferSize - this.options.bufferThreshold) {
@@ -383,7 +356,6 @@ class FlowcoreDataPump {
             }
             event.status = "open";
             event.deliveryId = undefined;
-            event.timeoutId = undefined;
             reopenedEvents.push(event.event);
             return true;
         });