@devrev/ts-adaas 1.7.2-beta.0 → 1.8.0-beta.0

package/dist/attachments-streaming/attachments-streaming-pool.d.ts

@@ -0,0 +1,17 @@
+import { NormalizedAttachment, ExternalSystemAttachmentStreamingFunction, ProcessAttachmentReturnType } from '../types';
+import { WorkerAdapter } from '../workers/worker-adapter';
+export declare class AttachmentsStreamingPool<ConnectorState> {
+    private adapter;
+    private attachments;
+    private batchSize;
+    private delay;
+    private stream;
+    constructor({ adapter, attachments, batchSize, stream, }: {
+        adapter: WorkerAdapter<ConnectorState>;
+        attachments: NormalizedAttachment[];
+        batchSize?: number;
+        stream: ExternalSystemAttachmentStreamingFunction;
+    });
+    streamAll(): Promise<ProcessAttachmentReturnType>;
+    startPoolWorker(): Promise<void>;
+}

package/dist/attachments-streaming/attachments-streaming-pool.js

@@ -0,0 +1,75 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AttachmentsStreamingPool = void 0;
+class AttachmentsStreamingPool {
+    constructor({ adapter, attachments, batchSize = 10, stream, }) {
+        this.adapter = adapter;
+        this.attachments = [...attachments]; // Create a copy we can mutate
+        this.batchSize = batchSize;
+        this.delay = undefined;
+        this.stream = stream;
+    }
+    async streamAll() {
+        console.log(`Starting download of ${this.attachments.length} attachments with ${this.batchSize} workers.`);
+        if (!this.adapter.state.toDevRev) {
+            const error = new Error('toDevRev state is not initialized');
+            console.error(error);
+            return { error };
+        }
+        // Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction.
+        // If no such list exists, create an empty one.
+        if (!this.adapter.state.toDevRev.attachmentsMetadata
+            .lastProcessedAttachmentsIdsList) {
+            this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
+                [];
+        }
+        // Start initial batch of workers up to batchSize limit
+        const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
+        const initialPromises = [];
+        for (let i = 0; i < initialBatchSize; i++) {
+            initialPromises.push(this.startPoolWorker());
+        }
+        // Wait for all workers to complete
+        await Promise.all(initialPromises);
+        if (this.delay) {
+            return { delay: this.delay };
+        }
+        return {};
+    }
+    async startPoolWorker() {
+        var _a, _b, _c, _d;
+        // Worker keeps taking jobs until the attachments array is empty
+        while (this.attachments.length > 0) {
+            if (this.delay) {
+                break; // Exit if we have a delay
+            }
+            // Check if we can start a new worker
+            const attachment = this.attachments.shift();
+            if (!attachment) {
+                break; // Exit if no more attachments
+            }
+            if (this.adapter.state.toDevRev &&
+                ((_a = this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.includes(attachment.id))) {
+                console.log(`Attachment with ID ${attachment.id} has already been processed. Skipping.`);
+                continue; // Skip if the attachment ID is already processed
+            }
+            try {
+                const response = await this.adapter.processAttachment(attachment, this.stream);
+                // Check if rate limit was hit
+                if (response === null || response === void 0 ? void 0 : response.delay) {
+                    this.delay = response.delay; // Set the delay for rate limiting
+                    return;
+                }
+                // No rate limiting, process normally
+                if ((_c = (_b = this.adapter.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata) === null || _c === void 0 ? void 0 : _c.lastProcessedAttachmentsIdsList) {
+                    (_d = this.adapter.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata.lastProcessedAttachmentsIdsList.push(attachment.id);
+                    console.log(`Successfully processed attachment: ${attachment.id}`);
+                }
+            }
+            catch (error) {
+                console.warn(`Skipping attachment with ID ${attachment.id} due to error: ${error}`);
+            }
+        }
+    }
+}
+exports.AttachmentsStreamingPool = AttachmentsStreamingPool;

package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/attachments-streaming/attachments-streaming-pool.test.js

@@ -0,0 +1,273 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const attachments_streaming_pool_1 = require("./attachments-streaming-pool");
+describe('AttachmentsStreamingPool', () => {
+    let mockAdapter;
+    let mockStream;
+    let mockAttachments;
+    beforeEach(() => {
+        // Create mock adapter
+        mockAdapter = {
+            state: {
+                attachments: { completed: false },
+                toDevRev: {
+                    attachmentsMetadata: {
+                        lastProcessedAttachmentsIdsList: []
+                    }
+                }
+            },
+            processAttachment: jest.fn().mockResolvedValue({})
+        };
+        // Create mock stream function
+        mockStream = jest.fn().mockResolvedValue({ success: true });
+        // Create mock attachments
+        mockAttachments = [
+            {
+                id: 'attachment-1',
+                url: 'https://example.com/file1.pdf',
+                file_name: 'file1.pdf',
+                parent_id: 'parent-1'
+            },
+            {
+                id: 'attachment-2',
+                url: 'https://example.com/file2.jpg',
+                file_name: 'file2.jpg',
+                parent_id: 'parent-2'
+            },
+            {
+                id: 'attachment-3',
+                url: 'https://example.com/file3.doc',
+                file_name: 'file3.doc',
+                parent_id: 'parent-3'
+            }
+        ];
+        // Mock console methods
+        jest.spyOn(console, 'log').mockImplementation();
+        jest.spyOn(console, 'error').mockImplementation();
+        jest.spyOn(console, 'warn').mockImplementation();
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+        jest.restoreAllMocks();
+    });
+    describe('constructor', () => {
+        it('should initialize with default values', () => {
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            expect(pool).toBeDefined();
+            expect(pool['adapter']).toBe(mockAdapter);
+            expect(pool['attachments']).toEqual(mockAttachments);
+            expect(pool['batchSize']).toBe(10);
+            expect(pool['stream']).toBe(mockStream);
+        });
+        it('should initialize with custom batch size', () => {
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                batchSize: 5,
+                stream: mockStream
+            });
+            expect(pool['batchSize']).toBe(5);
+        });
+        it('should create a copy of attachments array', () => {
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            expect(pool['attachments']).toEqual(mockAttachments);
+            expect(pool['attachments']).not.toBe(mockAttachments); // Different reference
+        });
+    });
+    describe('streamAll', () => {
+        it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => {
+            mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = undefined;
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            // Mock startPoolWorker to avoid actual processing
+            jest.spyOn(pool, 'startPoolWorker').mockResolvedValue(undefined);
+            await pool.streamAll();
+            expect(mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([]);
+        });
+        it('should process all attachments successfully', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            const result = await pool.streamAll();
+            expect(result).toEqual({});
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+        });
+        it('should handle empty attachments array', async () => {
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: [],
+                stream: mockStream
+            });
+            const result = await pool.streamAll();
+            expect(result).toEqual({});
+            expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
+            expect(console.log).toHaveBeenCalledWith('Starting download of 0 attachments with 10 workers.');
+        });
+        it('should return delay when rate limit is hit', async () => {
+            const delayResponse = { delay: 5000 };
+            mockAdapter.processAttachment.mockResolvedValue(delayResponse);
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            const result = await pool.streamAll();
+            expect(result).toEqual({ delay: 5000 });
+        });
+    });
+    describe('startPoolWorker', () => {
+        it('should skip already processed attachments', async () => {
+            mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = ['attachment-1'];
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            await pool.streamAll();
+            expect(console.log).toHaveBeenCalledWith('Attachment with ID attachment-1 has already been processed. Skipping.');
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3
+        });
+        it('should add successfully processed attachment IDs to the list', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            await pool.streamAll();
+            expect(mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
+                'attachment-1',
+                'attachment-2',
+                'attachment-3'
+            ]);
+            expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-1');
+            expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-2');
+            expect(console.log).toHaveBeenCalledWith('Successfully processed attachment: attachment-3');
+        });
+        it('should handle processing errors gracefully', async () => {
+            const error = new Error('Processing failed');
+            mockAdapter.processAttachment
+                .mockResolvedValueOnce({}) // First attachment succeeds
+                .mockRejectedValueOnce(error) // Second attachment fails
+                .mockResolvedValueOnce({}); // Third attachment succeeds
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            await pool.streamAll();
+            expect(console.warn).toHaveBeenCalledWith('Skipping attachment with ID attachment-2 due to error: Error: Processing failed');
+            expect(mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
+                'attachment-1',
+                'attachment-3'
+            ]);
+        });
+        it('should stop processing when rate limit delay is encountered', async () => {
+            mockAdapter.processAttachment
+                .mockResolvedValueOnce({}) // First attachment succeeds
+                .mockResolvedValueOnce({ delay: 5000 }) // Second attachment triggers rate limit
+                .mockResolvedValueOnce({}); // Third attachment succeeds
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream
+            });
+            await pool.streamAll();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+            expect(mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([
+                'attachment-1',
+                'attachment-3'
+            ]);
+        });
+        it('should pass correct parameters to processAttachment', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: [mockAttachments[0]],
+                stream: mockStream
+            });
+            await pool.streamAll();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0], mockStream);
+        });
+    });
+    describe('edge cases', () => {
+        it('should handle single attachment', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: [mockAttachments[0]],
+                stream: mockStream
+            });
+            const result = await pool.streamAll();
+            expect(result).toEqual({});
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
+        });
+        it('should handle batch size larger than attachments array', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                batchSize: 100,
+                stream: mockStream
+            });
+            await pool.streamAll();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+        });
+        it('should handle batch size of 1', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                batchSize: 1,
+                stream: mockStream
+            });
+            await pool.streamAll();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+        });
+    });
+    describe('concurrency behavior', () => {
+        it('should process attachments concurrently within batch size', async () => {
+            let processCallCount = 0;
+            const processPromises = [];
+            mockAdapter.processAttachment.mockImplementation(() => {
+                const promise = new Promise((resolve) => {
+                    setTimeout(() => {
+                        processCallCount++;
+                        resolve({});
+                    }, 100);
+                });
+                processPromises.push(promise);
+                return promise;
+            });
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                batchSize: 2,
+                stream: mockStream
+            });
+            const startTime = Date.now();
+            await pool.streamAll();
+            const endTime = Date.now();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+            expect(processCallCount).toBe(3);
+            // Should complete in roughly 200ms (2 batches of 100ms each) rather than 300ms (sequential)
+            expect(endTime - startTime).toBeLessThan(250);
+        });
+    });
+});

package/dist/types/index.d.ts

@@ -1,5 +1,5 @@
 export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
-export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, ExternalProcessAttachmentFunction, ExternalSystemAttachmentReducerFunction, ExternalSystemAttachmentIteratorFunction, } from './extraction';
+export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, SyncMode, ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingFunction, ExternalProcessAttachmentFunction, ExternalSystemAttachmentReducerFunction, ExternalSystemAttachmentIteratorFunction, ProcessAttachmentReturnType, } from './extraction';
 export { LoaderEventType, ExternalSystemItem, ExternalSystemItemLoadingResponse, ExternalSystemItemLoadingParams, ExternalSystemAttachment, } from './loading';
 export { NormalizedItem, NormalizedAttachment, RepoInterface, } from '../repo/repo.interfaces';
 export { AdapterState } from '../state/state.interfaces';

package/dist/workers/worker-adapter.d.ts

@@ -68,27 +68,6 @@ export declare class WorkerAdapter<ConnectorState> {
         item: ExternalSystemAttachment;
         create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
     }): Promise<LoadItemResponse>;
-    /**
-     * Transforms an array of attachments into array of batches of the specified size.
-     *
-     * @param {Object} parameters - The parameters object
-     * @param {NormalizedAttachment[]} parameters.attachments - Array of attachments to be processed
-     * @param {number} [parameters.batchSize=1] - The size of each batch (defaults to 1)
-     * @param {ConnectorState} parameters.adapter - The adapter instance
-     * @returns {NormalizedAttachment[][]} An array of attachment batches
-     */
-    private defaultAttachmentsReducer;
-    /**
-     * This iterator function processes attachments batch by batch, saves progress to state, and handles rate limiting.
-     *
-     * @param {Object} parameters - The parameters object
-     * @param {NormalizedAttachment[][]} parameters.reducedAttachments - Array of attachment batches to process
-     * @param {Object} parameters.adapter - The connector adapter that contains state and processing methods
-     * @param {Object} parameters.stream - Stream object for logging or progress reporting
-     * @returns {Promise<{delay?: number} | void>} Returns an object with delay information if rate-limited, otherwise void
-     * @throws Will not throw exceptions but will log warnings for processing failures
-     */
-    private defaultAttachmentsIterator;
     /**
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.

package/dist/workers/worker-adapter.js

@@ -15,7 +15,7 @@ const mappers_1 = require("../mappers/mappers");
 const uploader_1 = require("../uploader/uploader");
 const logger_1 = require("../logger/logger");
 const mappers_interface_1 = require("../mappers/mappers.interface");
-const helpers_2 = require("../common/helpers");
+const attachments_streaming_pool_1 = require("../attachments-streaming/attachments-streaming-pool");
 function createWorkerAdapter({ event, adapterState, options, }) {
     return new WorkerAdapter({
         event,
@@ -40,108 +40,6 @@ function createWorkerAdapter({ event, adapterState, options, }) {
 class WorkerAdapter {
     constructor({ event, adapterState, options, }) {
         this.repos = [];
-        /**
-         * Transforms an array of attachments into array of batches of the specified size.
-         *
-         * @param {Object} parameters - The parameters object
-         * @param {NormalizedAttachment[]} parameters.attachments - Array of attachments to be processed
-         * @param {number} [parameters.batchSize=1] - The size of each batch (defaults to 1)
-         * @param {ConnectorState} parameters.adapter - The adapter instance
-         * @returns {NormalizedAttachment[][]} An array of attachment batches
-         */
-        this.defaultAttachmentsReducer = ({ attachments, batchSize = 1 }) => {
-            // Transform the attachments array into smaller batches
-            const batches = attachments.reduce((result, item, index) => {
-                // Determine the index of the current batch
-                const batchIndex = Math.floor(index / batchSize);
-                // Initialize a new batch if it doesn't already exist
-                if (!result[batchIndex]) {
-                    result[batchIndex] = [];
-                }
-                // Append the current item to the current batch
-                result[batchIndex].push(item);
-                return result;
-            }, []);
-            // Return the array of batches
-            return batches;
-        };
-        /**
-         * This iterator function processes attachments batch by batch, saves progress to state, and handles rate limiting.
-         *
-         * @param {Object} parameters - The parameters object
-         * @param {NormalizedAttachment[][]} parameters.reducedAttachments - Array of attachment batches to process
-         * @param {Object} parameters.adapter - The connector adapter that contains state and processing methods
-         * @param {Object} parameters.stream - Stream object for logging or progress reporting
-         * @returns {Promise<{delay?: number} | void>} Returns an object with delay information if rate-limited, otherwise void
-         * @throws Will not throw exceptions but will log warnings for processing failures
-         */
-        this.defaultAttachmentsIterator = async ({ reducedAttachments, adapter, stream }) => {
-            if (!adapter.state.toDevRev) {
-                const error = new Error(`toDevRev state is not defined.`);
-                console.error(error.message);
-                return { error };
-            }
-            // Get index of the last processed batch of this artifact
-            const lastProcessedBatchIndex = adapter.state.toDevRev.attachmentsMetadata.lastProcessed || 0;
-            // Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction.
-            // If no such list exists, create an empty one.
-            if (!adapter.state.toDevRev.attachmentsMetadata
-                .lastProcessedAttachmentsIdsList) {
-                adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
-                    [];
-            }
-            // Loop through the batches of attachments
-            for (let i = lastProcessedBatchIndex; i < reducedAttachments.length; i++) {
-                // Check if we hit timeout
-                if (adapter.isTimeout) {
-                    await (0, helpers_2.sleep)(constants_1.DEFAULT_SLEEP_DELAY_MS);
-                }
-                const attachmentsBatch = reducedAttachments[i];
-                // Create a list of promises for parallel processing
-                const promises = [];
-                for (const attachment of attachmentsBatch) {
-                    if (adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList.includes(attachment.id)) {
-                        console.log(`Attachment with ID ${attachment.id} has already been processed. Skipping.`);
-                        continue; // Skip if the attachment ID is already processed
-                    }
-                    const promise = adapter
-                        .processAttachment(attachment, stream)
-                        .then((response) => {
-                        var _a, _b, _c;
-                        // Check if rate limit was hit
-                        if (response === null || response === void 0 ? void 0 : response.delay) {
-                            // Store this promise result to be checked later
-                            return { delay: response.delay };
-                        }
-                        // No rate limiting, process normally
-                        if ((_b = (_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata) === null || _b === void 0 ? void 0 : _b.lastProcessedAttachmentsIdsList) {
-                            (_c = adapter.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata.lastProcessedAttachmentsIdsList.push(attachment.id);
-                        }
-                        return null; // Return null for successful processing
-                    })
-                        .catch((error) => {
-                        console.warn(`Skipping attachment with ID ${attachment.id} due to error: ${error}`);
-                        return null; // Return null for errors too
-                    });
-                    promises.push(promise);
-                }
-                // Wait for all promises to settle and check for rate limiting
-                const results = await Promise.all(promises);
-                // Check if any of the results indicate rate limiting
-                const rateLimit = results.find((result) => result === null || result === void 0 ? void 0 : result.delay);
-                if (rateLimit) {
-                    // Return the delay information to the caller
-                    return { delay: rateLimit.delay };
-                }
-                if (adapter.state.toDevRev) {
-                    // Update the last processed batch index
-                    adapter.state.toDevRev.attachmentsMetadata.lastProcessed = i + 1;
-                    // Reset successfullyProcessedAttachments list
-                    adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
-                }
-            }
-            return {};
-        };
         this.event = event;
         this.options = options;
         this.adapterState = adapterState;
@@ -728,38 +626,41 @@ class WorkerAdapter {
                 continue;
             }
             console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
-            // Use the reducer to split into batches.
-            let reducer;
-            // Use the iterator to process each batch, streaming all attachments inside one batch in parallel.
-            let iterator;
+            let response;
             if (processors) {
                 console.log(`Using custom processors for attachments.`);
-                reducer = processors.reducer;
-                iterator = processors.iterator;
+                const reducer = processors.reducer;
+                const iterator = processors.iterator;
+                const reducedAttachments = reducer({
+                    attachments,
+                    adapter: this,
+                    batchSize,
+                });
+                response = await iterator({
+                    reducedAttachments,
+                    adapter: this,
+                    stream,
+                });
             }
             else {
-                console.log(`Using default processors for attachments.`);
-                reducer = this
-                    .defaultAttachmentsReducer;
-                iterator = this
-                    .defaultAttachmentsIterator;
+                console.log(`Using attachments streaming pool for attachments streaming.`);
+                const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                    adapter: this,
+                    attachments,
+                    batchSize,
+                    stream,
+                });
+                response = await attachmentsPool.streamAll();
             }
-            const reducedAttachments = reducer({
-                attachments,
-                adapter: this,
-                batchSize,
-            });
-            const response = await iterator({
-                reducedAttachments,
-                adapter: this,
-                stream,
-            });
             if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
                 return response;
             }
             console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
             this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
             this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
+            if (this.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) {
+                this.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
+            }
         }
         return;
     }

package/dist/workers/worker-adapter.test.js

@@ -4,6 +4,7 @@ const worker_adapter_1 = require("./worker-adapter");
 const state_1 = require("../state/state");
 const test_helpers_1 = require("../tests/test-helpers");
 const types_1 = require("../types");
+const attachments_streaming_pool_1 = require("../attachments-streaming/attachments-streaming-pool");
 // Mock dependencies
 jest.mock('../common/control-protocol', () => ({
     emit: jest.fn().mockResolvedValue({}),
@@ -19,6 +20,15 @@ jest.mock('node:worker_threads', () => ({
         postMessage: jest.fn(),
     },
 }));
+jest.mock('../attachments-streaming/attachments-streaming-pool', () => {
+    return {
+        AttachmentsStreamingPool: jest.fn().mockImplementation(() => {
+            return {
+                streamAll: jest.fn().mockResolvedValue(undefined),
+            };
+        }),
+    };
+});
 describe('WorkerAdapter', () => {
     let adapter;
     let mockEvent;
@@ -51,269 +61,6 @@ describe('WorkerAdapter', () => {
             adapterState: mockAdapterState,
         });
     });
-    describe('defaultAttachmentsReducer', () => {
-        it('should correctly batch attachments based on the batchSize', () => {
-            // Arrange
-            const attachments = [
-                { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-                { url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
-                { url: 'http://example.com/file4.pdf', id: 'attachment4', file_name: 'file4.pdf', parent_id: 'parent4' },
-                { url: 'http://example.com/file5.pdf', id: 'attachment5', file_name: 'file5.pdf', parent_id: 'parent5' },
-            ];
-            // Act - call the private method using function call notation
-            const result = adapter['defaultAttachmentsReducer']({
-                attachments,
-                adapter,
-                batchSize: 2,
-            });
-            // Assert
-            expect(result).toHaveLength(3);
-            expect(result[0]).toHaveLength(2);
-            expect(result[1]).toHaveLength(2);
-            expect(result[2]).toHaveLength(1);
-            expect(result[0][0].id).toBe('attachment1');
-            expect(result[0][1].id).toBe('attachment2');
-            expect(result[1][0].id).toBe('attachment3');
-            expect(result[1][1].id).toBe('attachment4');
-            expect(result[2][0].id).toBe('attachment5');
-        });
-        it('should return a single batch when batchSize equals the number of attachments', () => {
-            // Arrange
-            const attachments = [
-                { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-            ];
-            // Act
-            const result = adapter['defaultAttachmentsReducer']({
-                attachments,
-                adapter,
-                batchSize: 2,
-            });
-            // Assert
-            expect(result).toHaveLength(1);
-            expect(result[0]).toHaveLength(2);
-            expect(result[0][0].id).toBe('attachment1');
-            expect(result[0][1].id).toBe('attachment2');
-        });
-        it('should return a single batch when batchSize is bigger than the number of attachments', () => {
-            // Arrange
-            const attachments = [
-                { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-            ];
-            // Act
-            const result = adapter['defaultAttachmentsReducer']({
-                attachments,
-                adapter,
-                batchSize: 10,
-            });
-            // Assert
-            expect(result).toHaveLength(1);
-            expect(result[0]).toHaveLength(2);
-            expect(result[0][0].id).toBe('attachment1');
-            expect(result[0][1].id).toBe('attachment2');
-        });
-        it('should handle empty attachments array', () => {
-            // Arrange
-            const attachments = [];
-            // Act
-            const result = adapter['defaultAttachmentsReducer']({
-                attachments,
-                adapter,
-                batchSize: 2,
-            });
-            // Assert
-            expect(result).toHaveLength(0);
-        });
-        it('should default to batchSize of 1 if not provided', () => {
-            // Arrange
-            const attachments = [
-                { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-            ];
-            // Act
-            const result = adapter['defaultAttachmentsReducer']({
-                attachments,
-                adapter
-            });
-            // Assert
-            expect(result).toHaveLength(2);
-            expect(result[0]).toHaveLength(1);
-            expect(result[1]).toHaveLength(1);
-        });
-    });
-    describe('defaultAttachmentsIterator', () => {
-        it('should process all batches of attachments', async () => {
-            var _a, _b;
-            // Arrange
-            const mockAttachments = [
-                [
-                    { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                    { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-                ],
-                [
-                    { url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
-                ],
-            ];
-            const mockStream = jest.fn();
-            // Mock the processAttachment method 
-            adapter.processAttachment = jest.fn().mockResolvedValue(null);
-            // Act
-            const result = await adapter['defaultAttachmentsIterator']({
-                reducedAttachments: mockAttachments,
-                adapter: adapter,
-                stream: mockStream,
-            });
-            // Assert
-            expect(adapter.processAttachment).toHaveBeenCalledTimes(3);
-            expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][0], mockStream);
-            expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][1], mockStream);
-            expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[1][0], mockStream);
-            // Verify the state was updated correctly
-            expect((_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.lastProcessed).toBe(2);
-            expect((_b = adapter.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([]);
-            expect(result).toEqual({});
-        });
-        it('should handle rate limiting during processing', async () => {
-            var _a;
-            // Arrange
-            const mockAttachments = [
-                [
-                    { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                    { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-                ],
-                [
-                    { url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
-                ],
-            ];
-            const mockStream = jest.fn();
-            // Mock the processAttachment method to simulate rate limiting on the second attachment
-            adapter.processAttachment = jest.fn()
-                .mockResolvedValueOnce(null) // First attachment processes successfully
-                .mockResolvedValueOnce({ delay: 30 }); // Second attachment hits rate limit
-            // Set up adapter state
-            adapter.state.toDevRev = {
-                attachmentsMetadata: {
-                    lastProcessed: 0,
-                    artifactIds: [],
-                    lastProcessedAttachmentsIdsList: [],
-                },
-            };
-            // Act
-            const result = await adapter['defaultAttachmentsIterator']({
-                reducedAttachments: mockAttachments,
-                adapter: adapter,
-                stream: mockStream,
-            });
-            // Assert
-            expect(adapter.processAttachment).toHaveBeenCalledTimes(2);
-            expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][0], mockStream);
-            expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][1], mockStream);
-            // Verify the delay was returned
-            expect(result).toEqual({ delay: 30 });
-            // And lastProcessed wasn't updated yet
-            expect((_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.lastProcessed).toBe(0);
-        });
-        it('should skip already processed attachments', async () => {
-            // Arrange
-            const mockAttachments = [
-                [
-                    { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                    { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-                ],
-            ];
-            const mockStream = jest.fn();
-            // Mock the processAttachment method
-            adapter.processAttachment = jest.fn().mockResolvedValue(null);
-            // Set up adapter state to indicate attachment1 was already processed
-            adapter.state.toDevRev = {
-                attachmentsMetadata: {
-                    lastProcessed: 0,
-                    artifactIds: [],
-                    lastProcessedAttachmentsIdsList: ['attachment1'],
-                },
-            };
-            // Act
-            await adapter['defaultAttachmentsIterator']({
-                reducedAttachments: mockAttachments,
-                adapter: adapter,
-                stream: mockStream,
-            });
-            // Assert
-            expect(adapter.processAttachment).toHaveBeenCalledTimes(1);
-            expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0][1], mockStream);
-            expect(adapter.processAttachment).not.toHaveBeenCalledWith(mockAttachments[0][0], mockStream);
-        });
-        it('should continue from last processed batch', async () => {
-            // Arrange
-            const mockAttachments = [
-                [
-                    { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                ],
-                [
-                    { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-                ],
-            ];
-            const mockStream = jest.fn();
-            // Mock the processAttachment method
-            adapter.processAttachment = jest.fn().mockResolvedValue(null);
-            // Set up adapter state to indicate we already processed the first batch
-            adapter.state.toDevRev = {
-                attachmentsMetadata: {
-                    lastProcessed: 1, // Skip first batch (index 0)
-                    artifactIds: [],
-                    lastProcessedAttachmentsIdsList: [],
-                },
-            };
-            // Act
-            await adapter['defaultAttachmentsIterator']({
-                reducedAttachments: mockAttachments,
-                adapter: adapter,
-                stream: mockStream,
-            });
-            // Assert
-            expect(adapter.processAttachment).toHaveBeenCalledTimes(1);
-            expect(adapter.processAttachment).toHaveBeenCalledWith(mockAttachments[1][0], mockStream);
-            expect(adapter.processAttachment).not.toHaveBeenCalledWith(mockAttachments[0][0], mockStream);
-        });
-        it('should handle errors during processing and continue', async () => {
-            // Arrange
-            const mockAttachments = [
-                [
-                    { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                    { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-                ],
-            ];
-            const mockStream = jest.fn();
-            // Mock processAttachment to throw an error for the first attachment
-            adapter.processAttachment = jest.fn()
-                .mockRejectedValueOnce(new Error('Processing error'))
-                .mockResolvedValueOnce(null);
-            // Mock console.warn to avoid test output noise
-            const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
-            // Set up adapter state
-            adapter.state.toDevRev = {
-                attachmentsMetadata: {
-                    lastProcessed: 0,
-                    artifactIds: [],
-                    lastProcessedAttachmentsIdsList: [],
-                },
-            };
-            // Act
-            const result = await adapter['defaultAttachmentsIterator']({
-                reducedAttachments: mockAttachments,
-                adapter: adapter,
-                stream: mockStream,
-            });
-            // Assert - both attachments should have been processed
-            expect(adapter.processAttachment).toHaveBeenCalledTimes(2);
-            expect(consoleWarnSpy).toHaveBeenCalled();
-            expect(result).toEqual({});
-            // Restore console.warn
-            consoleWarnSpy.mockRestore();
-        });
-    });
     describe('streamAttachments', () => {
         it('should process all artifact batches successfully', async () => {
             // Arrange
@@ -341,11 +88,6 @@ describe('WorkerAdapter', () => {
             });
             // Mock the initializeRepos method
             adapter.initializeRepos = jest.fn();
-            // Mock the defaultAttachmentsReducer and defaultAttachmentsIterator
-            const mockReducedAttachments = [['batch1']];
-            adapter['defaultAttachmentsReducer'] = jest.fn().mockReturnValue(mockReducedAttachments);
-            adapter['defaultAttachmentsIterator'] = jest.fn().mockResolvedValue({});
-            // Act
             const result = await adapter.streamAttachments({
                 stream: mockStream,
             });
@@ -353,9 +95,8 @@ describe('WorkerAdapter', () => {
             expect(adapter.initializeRepos).toHaveBeenCalledWith([
                 { itemType: 'ssor_attachment' },
             ]);
+            expect(adapter.initializeRepos).toHaveBeenCalledTimes(1);
             expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
-            expect(adapter['defaultAttachmentsReducer']).toHaveBeenCalledTimes(2);
-            expect(adapter['defaultAttachmentsIterator']).toHaveBeenCalledTimes(2);
             // Verify state was updated correctly
             expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
             expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
@@ -379,11 +120,7 @@ describe('WorkerAdapter', () => {
                     { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
                 ],
             });
-            // Mock the required methods
             adapter.initializeRepos = jest.fn();
-            const mockReducedAttachments = [['batch1']];
-            adapter['defaultAttachmentsReducer'] = jest.fn().mockReturnValue(mockReducedAttachments);
-            adapter['defaultAttachmentsIterator'] = jest.fn().mockResolvedValue({});
             // Act
             const result = await adapter.streamAttachments({
                 stream: mockStream,
@@ -391,12 +128,6 @@ describe('WorkerAdapter', () => {
             });
             // Assert
             expect(consoleWarnSpy).toHaveBeenCalledWith('The specified batch size (0) is invalid. Using 1 instead.');
-            // Verify that the reducer was called with batchSize 50 (not 100)
-            expect(adapter['defaultAttachmentsReducer']).toHaveBeenCalledWith({
-                attachments: expect.any(Array),
-                adapter: adapter,
-                batchSize: 1,
-            });
             expect(result).toBeUndefined();
             // Restore console.warn
             consoleWarnSpy.mockRestore();
@@ -421,9 +152,6 @@ describe('WorkerAdapter', () => {
             });
             // Mock the required methods
             adapter.initializeRepos = jest.fn();
-            const mockReducedAttachments = [['batch1']];
-            adapter['defaultAttachmentsReducer'] = jest.fn().mockReturnValue(mockReducedAttachments);
-            adapter['defaultAttachmentsIterator'] = jest.fn().mockResolvedValue({});
             // Act
             const result = await adapter.streamAttachments({
                 stream: mockStream,
@@ -431,12 +159,6 @@ describe('WorkerAdapter', () => {
             });
             // Assert
             expect(consoleWarnSpy).toHaveBeenCalledWith('The specified batch size (100) is too large. Using 50 instead.');
-            // Verify that the reducer was called with batchSize 50 (not 100)
-            expect(adapter['defaultAttachmentsReducer']).toHaveBeenCalledWith({
-                attachments: expect.any(Array),
-                adapter: adapter,
-                batchSize: 50, // Should be capped at 50
-            });
             expect(result).toBeUndefined();
             // Restore console.warn
             consoleWarnSpy.mockRestore();
@@ -568,6 +290,12 @@ describe('WorkerAdapter', () => {
         it('should handle rate limiting from iterator', async () => {
             // Arrange
             const mockStream = jest.fn();
+            attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
+                return {
+                    // Return an object with a `streamAll` method that resolves to your desired value.
+                    streamAll: jest.fn().mockResolvedValue({ delay: 30 }),
+                };
+            });
             // Set up adapter state with artifact IDs
             adapter.state.toDevRev = {
                 attachmentsMetadata: {
@@ -582,10 +310,6 @@ describe('WorkerAdapter', () => {
             });
             // Mock methods
             adapter.initializeRepos = jest.fn();
-            adapter['defaultAttachmentsReducer'] = jest.fn().mockReturnValue([]);
-            adapter['defaultAttachmentsIterator'] = jest.fn().mockResolvedValue({
-                delay: 30,
-            });
             // Act
             const result = await adapter.streamAttachments({
                 stream: mockStream,
@@ -600,6 +324,14 @@ describe('WorkerAdapter', () => {
         it('should handle error from iterator', async () => {
             // Arrange
             const mockStream = jest.fn();
+            attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
+                return {
+                    // Return an object with a `streamAll` method that resolves to your desired value.
+                    streamAll: jest.fn().mockResolvedValue({
+                        error: "Mock error",
+                    }),
+                };
+            });
             // Set up adapter state with artifact IDs
             adapter.state.toDevRev = {
                 attachmentsMetadata: {
@@ -614,51 +346,17 @@ describe('WorkerAdapter', () => {
             });
             // Mock methods
             adapter.initializeRepos = jest.fn();
-            adapter['defaultAttachmentsReducer'] = jest.fn().mockReturnValue([]);
-            const mockError = new Error('Iterator error');
-            adapter['defaultAttachmentsIterator'] = jest.fn().mockResolvedValue({
-                error: mockError,
-            });
             // Act
             const result = await adapter.streamAttachments({
                 stream: mockStream,
             });
             // Assert
             expect(result).toEqual({
-                error: mockError,
+                error: "Mock error",
             });
             // The artifactIds array should remain unchanged
             expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1']);
         });
-        it('should continue processing from last processed attachment for the current artifact', async () => {
-            const mockStream = jest.fn();
-            adapter.state.toDevRev = {
-                attachmentsMetadata: {
-                    artifactIds: ['artifact1'],
-                    lastProcessed: 0,
-                    lastProcessedAttachmentsIdsList: ['attachment1', 'attachment2'],
-                },
-            };
-            adapter['uploader'].getAttachmentsFromArtifactId = jest.fn().mockResolvedValue({
-                attachments: [
-                    { url: 'http://example.com/file1.pdf', id: 'attachment1', file_name: 'file1.pdf', parent_id: 'parent1' },
-                    { url: 'http://example.com/file2.pdf', id: 'attachment2', file_name: 'file2.pdf', parent_id: 'parent2' },
-                    { url: 'http://example.com/file3.pdf', id: 'attachment3', file_name: 'file3.pdf', parent_id: 'parent3' },
-                ],
-            });
-            adapter.processAttachment = jest.fn().mockResolvedValue(null);
-            await adapter.streamAttachments({
-                stream: mockStream,
-                batchSize: 3,
-            });
-            expect(adapter.processAttachment).toHaveBeenCalledTimes(1);
-            expect(adapter.processAttachment).toHaveBeenCalledWith({
-                url: 'http://example.com/file3.pdf',
-                id: 'attachment3',
-                file_name: 'file3.pdf',
-                parent_id: 'parent3'
-            }, mockStream);
-        });
         it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => {
             const mockStream = jest.fn();
             adapter.state.toDevRev = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.7.2-beta.0",
+  "version": "1.8.0-beta.0",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",