@devrev/ts-adaas 1.2.6 → 1.4.0

package/dist/workers/worker-adapter.js

@@ -39,6 +39,104 @@ function createWorkerAdapter({ event, adapterState, options, }) {
 class WorkerAdapter {
     constructor({ event, adapterState, options, }) {
         this.repos = [];
+        /**
+         * Transforms an array of attachments into array of batches of the specified size.
+         *
+         * @param {Object} parameters - The parameters object
+         * @param {NormalizedAttachment[]} parameters.attachments - Array of attachments to be processed
+         * @param {number} [parameters.batchSize=1] - The size of each batch (defaults to 1)
+         * @param {ConnectorState} parameters.adapter - The adapter instance
+         * @returns {NormalizedAttachment[][]} An array of attachment batches
+         */
+        this.defaultAttachmentsReducer = ({ attachments, batchSize = 1 }) => {
+            // Transform the attachments array into smaller batches
+            const batches = attachments.reduce((result, item, index) => {
+                // Determine the index of the current batch
+                const batchIndex = Math.floor(index / batchSize);
+                // Initialize a new batch if it doesn't already exist
+                if (!result[batchIndex]) {
+                    result[batchIndex] = [];
+                }
+                // Append the current item to the current batch
+                result[batchIndex].push(item);
+                return result;
+            }, []);
+            // Return the array of batches
+            return batches;
+        };
+        /**
+         * This iterator function processes attachments batch by batch, saves progress to state, and handles rate limiting.
+         *
+         * @param {Object} parameters - The parameters object
+         * @param {NormalizedAttachment[][]} parameters.reducedAttachments - Array of attachment batches to process
+         * @param {Object} parameters.adapter - The connector adapter that contains state and processing methods
+         * @param {Object} parameters.stream - Stream object for logging or progress reporting
+         * @returns {Promise<{delay?: number} | void>} Returns an object with delay information if rate-limited, otherwise void
+         * @throws Will not throw exceptions but will log warnings for processing failures
+         */
+        this.defaultAttachmentsIterator = async ({ reducedAttachments, adapter, stream }) => {
+            if (!adapter.state.toDevRev) {
+                const error = new Error(`toDevRev state is not defined.`);
+                console.error(error.message);
+                return { error };
+            }
+            // Get index of the last processed batch of this artifact
+            const lastProcessedBatchIndex = adapter.state.toDevRev.attachmentsMetadata.lastProcessed || 0;
+            // Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction.
+            // If no such list exists, create an empty one.
+            if (!adapter.state.toDevRev.attachmentsMetadata
+                .lastProcessedAttachmentsIdsList) {
+                adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
+                    [];
+            }
+            // Loop through the batches of attachments
+            for (let i = lastProcessedBatchIndex; i < reducedAttachments.length; i++) {
+                const attachmentsBatch = reducedAttachments[i];
+                // Create a list of promises for parallel processing
+                const promises = [];
+                for (const attachment of attachmentsBatch) {
+                    if (adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList.includes(attachment.id)) {
+                        console.log(`Attachment with ID ${attachment.id} has already been processed. Skipping.`);
+                        continue; // Skip if the attachment ID is already processed
+                    }
+                    const promise = adapter
+                        .processAttachment(attachment, stream)
+                        .then((response) => {
+                        var _a, _b, _c;
+                        // Check if rate limit was hit
+                        if (response === null || response === void 0 ? void 0 : response.delay) {
+                            // Store this promise result to be checked later
+                            return { delay: response.delay };
+                        }
+                        // No rate limiting, process normally
+                        if ((_b = (_a = adapter.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata) === null || _b === void 0 ? void 0 : _b.lastProcessedAttachmentsIdsList) {
+                            (_c = adapter.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata.lastProcessedAttachmentsIdsList.push(attachment.id);
+                        }
+                        return null; // Return null for successful processing
+                    })
+                        .catch((error) => {
+                        console.warn(`Skipping attachment with ID ${attachment.id} due to error: ${error}`);
+                        return null; // Return null for errors too
+                    });
+                    promises.push(promise);
+                }
+                // Wait for all promises to settle and check for rate limiting
+                const results = await Promise.all(promises);
+                // Check if any of the results indicate rate limiting
+                const rateLimit = results.find(result => result === null || result === void 0 ? void 0 : result.delay);
+                if (rateLimit) {
+                    // Return the delay information to the caller
+                    return { delay: rateLimit.delay };
+                }
+                if (adapter.state.toDevRev) {
+                    // Update the last processed batch index
+                    adapter.state.toDevRev.attachmentsMetadata.lastProcessed = i + 1;
+                    // Reset successfullyProcessedAttachments list
+                    adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
+                }
+            }
+            return {};
+        };
         this.event = event;
         this.options = options;
         this.adapterState = adapterState;
@@ -88,7 +186,7 @@ class WorkerAdapter {
     getRepo(itemType) {
         const repo = this.repos.find((repo) => repo.itemType === itemType);
         if (!repo) {
-            console.error(`Repo not found for item type: ${itemType}.`);
+            console.error(`Repo for item type ${itemType} not found.`);
             return;
         }
         return repo;
@@ -117,7 +215,14 @@ class WorkerAdapter {
         }
         // We want to upload all the repos before emitting the event, except for the external sync units done event
         if (newEventType !== extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone) {
-            await this.uploadAllRepos();
+            try {
+                await this.uploadAllRepos();
+            }
+            catch (error) {
+                console.error('Error while uploading repos', error);
+                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+                return;
+            }
         }
         // If the extraction is done, we want to save the timestamp of the last successful sync
         if (newEventType === extraction_1.ExtractorEventType.ExtractionAttachmentsDone) {
@@ -164,7 +269,10 @@ class WorkerAdapter {
     }
     async uploadAllRepos() {
         for (const repo of this.repos) {
-            await repo.upload();
+            const error = await repo.upload();
+            if (error) {
+                throw error;
+            }
         }
     }
     handleTimeout() {
@@ -501,7 +609,7 @@ class WorkerAdapter {
             event: this.event,
         });
         if (error) {
-            console.warn('Error while streaming attachment', error === null || error === void 0 ? void 0 : error.message);
+            console.warn('Error while streaming attachment', error);
             return { error };
         }
         else if (delay) {
@@ -573,27 +681,35 @@ class WorkerAdapter {
      * Streams the attachments to the DevRev platform.
      * The attachments are streamed to the platform and the artifact information is returned.
      * @param {{ stream, processors }: { stream: ExternalSystemAttachmentStreamingFunction, processors?: ExternalSystemAttachmentProcessors  }} Params - The parameters to stream the attachments
-     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssoAttachment artifact information
+     * @returns {Promise<StreamAttachmentsReturnType>} - The response object containing the ssorAttachment artifact information
      * or error information if there was an error
      */
-    async streamAttachments({ stream, processors, }) {
-        var _a, _b, _c, _d;
+    async streamAttachments({ stream, processors, batchSize = 1, // By default, we want to stream one attachment at a time
+     }) {
+        var _a, _b;
+        if (batchSize <= 0) {
+            const error = new Error(`Invalid attachments batch size: ${batchSize}. Batch size must be greater than 0.`);
+            console.error(error.message);
+            return { error };
+        }
         const repos = [
             {
                 itemType: 'ssor_attachment',
             },
         ];
         this.initializeRepos(repos);
-        const attachmentsMetadataArtifactIds = (_b = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata) === null || _b === void 0 ? void 0 : _b.artifactIds;
-        if (!attachmentsMetadataArtifactIds ||
-            attachmentsMetadataArtifactIds.length === 0) {
+        // If there are no attachments metadata artifact IDs in state, finish here
+        if (!((_b = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata) === null || _b === void 0 ? void 0 : _b.artifactIds) ||
+            this.state.toDevRev.attachmentsMetadata.artifactIds.length === 0) {
             console.log(`No attachments metadata artifact IDs found in state.`);
             return;
         }
         else {
-            console.log(`Found ${attachmentsMetadataArtifactIds.length} attachments metadata artifact IDs in state.`);
+            console.log(`Found ${this.state.toDevRev.attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
         }
-        for (const attachmentsMetadataArtifactId of attachmentsMetadataArtifactIds) {
+        // Loop through the attachments metadata artifact IDs
+        while (this.state.toDevRev.attachmentsMetadata.artifactIds.length > 0) {
+            const attachmentsMetadataArtifactId = this.state.toDevRev.attachmentsMetadata.artifactIds[0];
             console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
             const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
                 artifact: attachmentsMetadataArtifactId,
@@ -604,43 +720,44 @@ class WorkerAdapter {
             }
             if (!attachments || attachments.length === 0) {
                 console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                // Remove empty artifact and reset lastProcessed
+                this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
+                this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
                 continue;
             }
             console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+            // Use the reducer to split into batches.
+            let reducer;
+            // Use the iterator to process each batch, streaming all attachments inside one batch in parallel.
+            let iterator;
             if (processors) {
                 console.log(`Using custom processors for attachments.`);
-                const { reducer, iterator } = processors;
-                const reducedAttachments = reducer({ attachments, adapter: this });
-                const response = await iterator({
-                    reducedAttachments,
-                    adapter: this,
-                    stream,
-                });
-                if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
-                    return response;
-                }
+                reducer = processors.reducer;
+                iterator = processors.iterator;
             }
             else {
                 console.log(`Using default processors for attachments.`);
-                const attachmentsToProcess = attachments.slice((_d = (_c = this.state.toDevRev) === null || _c === void 0 ? void 0 : _c.attachmentsMetadata) === null || _d === void 0 ? void 0 : _d.lastProcessed, attachments.length);
-                for (const attachment of attachmentsToProcess) {
-                    const response = await this.processAttachment(attachment, stream);
-                    if (response === null || response === void 0 ? void 0 : response.delay) {
-                        return response;
-                    }
-                    else if (response === null || response === void 0 ? void 0 : response.error) {
-                        console.warn(`Skipping attachment with ID ${attachment.id} due to error.`);
-                    }
-                    if (this.state.toDevRev) {
-                        this.state.toDevRev.attachmentsMetadata.lastProcessed += 1;
-                    }
-                }
+                reducer = this
+                    .defaultAttachmentsReducer;
+                iterator = this
+                    .defaultAttachmentsIterator;
             }
-            if (this.state.toDevRev) {
-                console.log(`Finished processing attachments for artifact ID. Setting last processed to 0 and removing artifact ID from state.`);
-                this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
-                this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
+            const reducedAttachments = reducer({
+                attachments,
+                adapter: this,
+                batchSize,
+            });
+            const response = await iterator({
+                reducedAttachments,
+                adapter: this,
+                stream,
+            });
+            if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
+                return response;
             }
+            console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+            this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
+            this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
         }
         return;
     }

package/dist/workers/worker-adapter.test.d.ts

@@ -0,0 +1 @@
+export {};