@rudderstack/integrations-lib 0.2.33 → 0.2.35

package/build/utils/batch-processing.d.ts

@@ -1,15 +1,23 @@
 /**
- * Configuration options for batch processing operations
+ * Options for batch processing operations
  */
-export interface BatchProcessingDefaults {
+export interface BatchProcessingOptions {
     /**
-     * Default number of items to process in each batch
+     * Number of items to process in each batch (default: 10)
+     * Must be a positive integer.
      */
     batchSize?: number;
     /**
-     * Default time threshold in milliseconds before yielding control back to the event loop
+     * Time threshold in milliseconds (default: 10) before yielding control back to the event loop.
+     * Set to 0 to yield after every batch. Must be a non-negative integer.
      */
     yieldThreshold?: number;
+    /**
+     * Whether to process items sequentially within each batch. When true (default), each item in a batch
+     * will be processed one at a time. When false, all items in a batch will be processed concurrently.
+     * Consider the implications of concurrency on your processing logic before setting this to false, e.g. race conditions, rate limits, memory pressure, etc.
+     */
+    sequentialProcessing?: boolean;
 }
 /**
  * Configure global defaults for batch processing operations
@@ -25,11 +33,14 @@ export interface BatchProcessingDefaults {
  * // Set only batch size
  * configureBatchProcessingDefaults({ batchSize: 50 });
  *
+ * // Enable concurrent processing within batches
+ * configureBatchProcessingDefaults({ sequentialProcessing: false });
+ *
  * // Get current configuration
  * const currentConfig = configureBatchProcessingDefaults();
  * ```
  */
-export declare function configureBatchProcessingDefaults(config?: BatchProcessingDefaults): BatchProcessingDefaults;
+export declare function configureBatchProcessingDefaults(config?: BatchProcessingOptions): BatchProcessingOptions;
 /**
  * Maps over an array in batches to avoid blocking the event loop.
  * Processes items in chunks and yields control back to the event loop between batches
@@ -39,26 +50,20 @@ export declare function configureBatchProcessingDefaults(config?: BatchProcessin
  * @template R - The type of items in the result array
  * @param items - The array to map over
  * @param mapFn - The mapping function to apply to each item. Receives the item and its index.
- * @param batchSize - The number of items to process in each batch (default: @see defaultBatchSize)
- * @param yieldThreshold - Time threshold in milliseconds before yielding control (default: @see defaultYieldThreshold)
+ * @param options - Batch processing options
  * @returns A promise that resolves to the mapped array
  * @throws {Error} When batchSize is not a positive integer
  *
  * @example
  * ```typescript
- * // Synchronous mapping
- * const doubled = await mapInBatches([1, 2, 3, 4], (x) => x * 2, 2);
- * // Result: [2, 4, 6, 8]
- *
- * // Asynchronous mapping
- * const fetched = await mapInBatches(urls, async (url) => fetch(url), 3);
+ * // Synchronous mapping with default options (sequential processing)
+ * const doubled = await mapInBatches([1, 2, 3, 4], (x) => x * 2);
  *
- * // With index
- * const indexed = await mapInBatches(['a', 'b'], (item, index) => `${item}-${index}`);
- * // Result: ['a-0', 'b-1']
+ * // With concurrent processing within batches
+ * const doubled = await mapInBatches([1, 2, 3, 4], (x) => x * 2, { sequentialProcessing: false });
  * ```
  */
-export declare function mapInBatches<T, R>(items: T[], mapFn: (item: T, index: number) => R | Promise<R>, batchSize?: number, yieldThreshold?: number): Promise<R[]>;
+export declare function mapInBatches<T, R>(items: T[], mapFn: (item: T, index: number) => R | Promise<R>, options?: BatchProcessingOptions): Promise<R[]>;
 /**
  * Filters an array in batches to avoid blocking the event loop.
  * Processes items in chunks and yields control back to the event loop between batches
@@ -67,29 +72,21 @@ export declare function mapInBatches<T, R>(items: T[], mapFn: (item: T, index: n
  * @template T - The type of items in the array
  * @param items - The array to filter
  * @param predicate - The predicate function to test each item. Receives the item and its index.
- * @param batchSize - The number of items to process in each batch (default: @see defaultBatchSize)
- * @param yieldThreshold - Time threshold in milliseconds before yielding control (default: @see defaultYieldThreshold)
+ * @param options - Batch processing options
  * @returns A promise that resolves to the filtered array
  * @throws {Error} When batchSize is not a positive integer
  *
  * @example
  * ```typescript
- * // Synchronous filtering
- * const evens = await filterInBatches([1, 2, 3, 4, 5], (x) => x % 2 === 0, 2);
+ * // Synchronous filtering with default options
+ * const evens = await filterInBatches([1, 2, 3, 4, 5], (x) => x % 2 === 0);
  * // Result: [2, 4]
  *
- * // Asynchronous filtering
- * const valid = await filterInBatches(urls, async (url) => {
- *   const response = await fetch(url);
- *   return response.ok;
- * }, 3);
- *
- * // With index
- * const evenIndices = await filterInBatches(['a', 'b', 'c'], (_, index) => index % 2 === 0);
- * // Result: ['a', 'c']
+ * // With custom batch size
+ * const evens = await filterInBatches([1, 2, 3, 4, 5], (x) => x % 2 === 0, { batchSize: 2 });
  * ```
  */
-export declare function filterInBatches<T>(items: T[], predicate: (item: T, index: number) => boolean | Promise<boolean>, batchSize?: number, yieldThreshold?: number): Promise<T[]>;
+export declare function filterInBatches<T>(items: T[], predicate: (item: T, index: number) => boolean | Promise<boolean>, options?: BatchProcessingOptions): Promise<T[]>;
 /**
  * Groups an array by a key function in batches to avoid blocking the event loop.
  * Processes items in chunks and yields control back to the event loop between batches
@@ -99,73 +96,54 @@ export declare function filterInBatches<T>(items: T[], predicate: (item: T, inde
  * @template K - The type of the grouping key (must extend PropertyKey)
  * @param items - The array to group
  * @param keyFn - The function to extract the grouping key from each item. Receives the item and its index.
- * @param batchSize - The number of items to process in each batch (default: @see defaultBatchSize)
- * @param yieldThreshold - Time threshold in milliseconds before yielding control (default: @see defaultYieldThreshold)
+ * @param options - Batch processing options
  * @returns A promise that resolves to an object with grouped items
  * @throws {Error} When batchSize is not a positive integer
  *
  * @example
  * ```typescript
- * // Group by property
+ * // Group by property with default options
  * const byType = await groupByInBatches(
  *   [{type: 'A', value: 1}, {type: 'B', value: 2}, {type: 'A', value: 3}],
- *   (item) => item.type,
- *   2
+ *   (item) => item.type
  * );
  * // Result: {A: [{type: 'A', value: 1}, {type: 'A', value: 3}], B: [{type: 'B', value: 2}]}
  *
- * // Group by computed value
- * const byParity = await groupByInBatches([1, 2, 3, 4], (x) => x % 2 === 0 ? 'even' : 'odd');
- * // Result: {odd: [1, 3], even: [2, 4]}
- *
- * // Asynchronous key function
- * const byCategory = await groupByInBatches(items, async (item) => {
- *   return await getCategoryForItem(item);
- * });
+ * // With custom batch size
+ * const byType = await groupByInBatches(
+ *   [{type: 'A', value: 1}, {type: 'B', value: 2}, {type: 'A', value: 3}],
+ *   (item) => item.type,
+ *   { batchSize: 2 }
+ * );
  * ```
  */
-export declare function groupByInBatches<T, K extends PropertyKey>(items: T[], keyFn: (item: T, index: number) => K | Promise<K>, batchSize?: number, yieldThreshold?: number): Promise<Record<K, T[]>>;
+export declare function groupByInBatches<T, K extends PropertyKey>(items: T[], keyFn: (item: T, index: number) => K | Promise<K>, options?: BatchProcessingOptions): Promise<Record<K, T[]>>;
 /**
  * Reduces an array in batches to avoid blocking the event loop.
  * Processes items in chunks and yields control back to the event loop between batches
- * when the processing time exceeds the threshold.
+ * when the processing time exceeds the threshold. Sequential processing is always used for the reducer function,
+ * irrespective of the `sequentialProcessing` option.
  *
  * @template T - The type of items in the array
  * @template R - The type of the accumulator/result
  * @param items - The array to reduce
  * @param reducer - The reducer function. Receives the accumulator, current item, and index.
  * @param initialValue - The initial value for the accumulator
- * @param batchSize - The number of items to process in each batch (default: @see defaultBatchSize)
- * @param yieldThreshold - Time threshold in milliseconds before yielding control (default: @see defaultYieldThreshold)
+ * @param options - Batch processing options
  * @returns A promise that resolves to the reduced value
  * @throws {Error} When batchSize is not a positive integer
  *
  * @example
  * ```typescript
- * // Sum numbers
- * const sum = await reduceInBatches([1, 2, 3, 4], (acc, x) => acc + x, 0, 2);
+ * // Sum numbers with default options
+ * const sum = await reduceInBatches([1, 2, 3, 4], (acc, x) => acc + x, 0);
  * // Result: 10
  *
- * // Concatenate strings
- * const joined = await reduceInBatches(['a', 'b', 'c'], (acc, x) => acc + x, '');
- * // Result: 'abc'
- *
- * // Build object with index
- * const indexed = await reduceInBatches(
- *   ['x', 'y'],
- *   (acc, item, index) => ({ ...acc, [index]: item }),
- *   {}
- * );
- * // Result: {0: 'x', 1: 'y'}
- *
- * // Asynchronous reducer
- * const processed = await reduceInBatches(urls, async (acc, url) => {
- *   const data = await fetch(url).then(r => r.json());
- *   return [...acc, data];
- * }, []);
+ * // With custom batch size
+ * const sum = await reduceInBatches([1, 2, 3, 4], (acc, x) => acc + x, 0, { batchSize: 2 });
  * ```
  */
-export declare function reduceInBatches<T, R>(items: T[], reducer: (acc: R, item: T, index: number) => R | Promise<R>, initialValue: R, batchSize?: number, yieldThreshold?: number): Promise<R>;
+export declare function reduceInBatches<T, R>(items: T[], reducer: (acc: R, item: T, index: number) => R | Promise<R>, initialValue: R, options?: BatchProcessingOptions): Promise<R>;
 /**
  * FlatMaps an array in batches to avoid blocking the event loop.
  * Processes items in chunks, flattens the results, and yields control back to the event loop between batches
@@ -175,31 +153,45 @@ export declare function reduceInBatches<T, R>(items: T[], reducer: (acc: R, item
  * @template R - The type of items in the flattened result array
  * @param items - The array to flatMap over
  * @param mapFn - The mapping function that returns an array for each item. Receives the item and its index.
- * @param batchSize - The number of items to process in each batch (default: @see defaultBatchSize)
- * @param yieldThreshold - Time threshold in milliseconds before yielding control (default: @see defaultYieldThreshold)
+ * @param options - Batch processing options
  * @returns A promise that resolves to the flattened mapped array
  * @throws {Error} When batchSize is not a positive integer
  *
  * @example
  * ```typescript
- * // Duplicate each item
- * const duplicated = await flatMapInBatches([1, 2, 3], (x) => [x, x], 2);
+ * // Duplicate each item with default options
+ * const duplicated = await flatMapInBatches([1, 2, 3], (x) => [x, x]);
  * // Result: [1, 1, 2, 2, 3, 3]
  *
- * // Variable length results
- * const repeated = await flatMapInBatches([1, 2, 3], (x) => Array(x).fill(x));
- * // Result: [1, 2, 2, 3, 3, 3]
+ * // With custom batch size
+ * const duplicated = await flatMapInBatches([1, 2, 3], (x) => [x, x], { batchSize: 2 });
+ * ```
+ */
+export declare function flatMapInBatches<T, R>(items: T[], mapFn: (item: T, index: number) => R[] | Promise<R[]>, options?: BatchProcessingOptions): Promise<R[]>;
+/**
+ * forEach over an array in batches to avoid blocking the event loop.
+ * Processes items in chunks and yields control back to the event loop between batches
+ * when the processing time exceeds the threshold.
  *
- * // Split strings into characters
- * const chars = await flatMapInBatches(['ab', 'cd'], (str) => str.split(''));
- * // Result: ['a', 'b', 'c', 'd']
+ * @template T - The type of items in the input array
+ * @param items - The array to iterate over
+ * @param fn - The function to apply to each item. Receives the item and its index. Can be async.
+ * @param options - Batch processing options
+ * @returns A promise that resolves when all items have been processed
+ * @throws {Error} When batchSize is not a positive integer
  *
- * // Asynchronous mapping
- * const expanded = await flatMapInBatches(categories, async (category) => {
- *   const items = await fetchItemsForCategory(category);
- *   return items;
+ * @example
+ * ```typescript
+ * // Process items in batches with default options
+ * await forEachInBatches([1, 2, 3, 4], async (x) => {
+ *   await doSomething(x);
  * });
+ *
+ * // With custom batch size
+ * await forEachInBatches([1, 2, 3, 4], async (x) => {
+ *   await doSomething(x);
+ * }, { batchSize: 2 });
  * ```
  */
-export declare function flatMapInBatches<T, R>(items: T[], mapFn: (item: T, index: number) => R[] | Promise<R[]>, batchSize?: number, yieldThreshold?: number): Promise<R[]>;
+export declare function forEachInBatches<T>(items: T[], fn: (item: T, index: number) => void | Promise<void>, options?: BatchProcessingOptions): Promise<void>;
 //# sourceMappingURL=batch-processing.d.ts.map

package/build/utils/batch-processing.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"batch-processing.d.ts","sourceRoot":"","sources":["../../src/utils/batch-processing.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAMD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,gCAAgC,CAC9C,MAAM,CAAC,EAAE,uBAAuB,GAC/B,uBAAuB,CAqBzB;AA0BD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,CAAC,EAC/B,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,SAAmB,EAC5B,cAAc,SAAwB,GACrC,OAAO,CAAC,CAAC,EAAE,CAAC,CAmBd;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAC/B,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,EACjE,SAAS,SAAmB,EAC5B,cAAc,SAAwB,GACrC,OAAO,CAAC,CAAC,EAAE,CAAC,CAqBd;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,CAAC,SAAS,WAAW,EACvD,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EACjD,SAAS,SAAmB,EAC5B,cAAc,SAAwB,GACrC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAiCzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,CAAC,EAClC,KAAK,EAAE,CAAC,EAAE,EACV,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EAC3D,YAAY,EAAE,CAAC,EACf,SAAS,SAAmB,EAC5B,cAAc,SAAwB,GACrC,OAAO,CAAC,CAAC,CAAC,CAsBZ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,CAAC,EACnC,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,EACrD,SAAS,SAAmB,EAC5B,cAAc,SAAwB,GACrC,OAAO,CAAC,CAAC,EAAE,CAAC,CAoBd"}
+{"version":3,"file":"batch-processing.d.ts","sourceRoot":"","sources":["../../src/utils/batch-processing.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAOD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,gCAAgC,CAC9C,MAAM,CAAC,EAAE,sBAAsB,GAC9B,sBAAsB,CA0BxB;AAoFD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,CAAC,EAC/B,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EACjD,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,CAAC,EAAE,CAAC,CAqBd;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAC/B,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,EACjE,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,CAAC,EAAE,CAAC,CAuBd;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,CAAC,SAAS,WAAW,EACvD,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EACjD,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CA0BzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,CAAC,EAClC,KAAK,EAAE,CAAC,EAAE,EACV,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EAC3D,YAAY,EAAE,CAAC,EACf,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,CAAC,CAAC,CAcZ;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,CAAC,EACnC,KAAK,EAAE,CAAC,EAAE,EACV,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,EACrD,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,CAAC,EAAE,CAAC,CAuBd;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EACV,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EACpD,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAef"}