@devrev/ts-adaas 1.8.0-beta.0 → 1.9.0-beta.0

package/dist/attachments-streaming/attachments-streaming-pool.d.ts

@@ -1,17 +1,12 @@
-import { NormalizedAttachment, ExternalSystemAttachmentStreamingFunction, ProcessAttachmentReturnType } from '../types';
-import { WorkerAdapter } from '../workers/worker-adapter';
+import { ProcessAttachmentReturnType } from '../types';
+import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces';
 export declare class AttachmentsStreamingPool<ConnectorState> {
     private adapter;
     private attachments;
     private batchSize;
     private delay;
     private stream;
-    constructor({ adapter, attachments, batchSize, stream, }: {
-        adapter: WorkerAdapter<ConnectorState>;
-        attachments: NormalizedAttachment[];
-        batchSize?: number;
-        stream: ExternalSystemAttachmentStreamingFunction;
-    });
+    constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
     streamAll(): Promise<ProcessAttachmentReturnType>;
-    startPoolWorker(): Promise<void>;
+    startPoolStreaming(): Promise<void>;
 }

package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts

@@ -0,0 +1,8 @@
+import { NormalizedAttachment, ExternalSystemAttachmentStreamingFunction } from '../types';
+import { WorkerAdapter } from '../workers/worker-adapter';
+export interface AttachmentsStreamingPoolParams<ConnectorState> {
+    adapter: WorkerAdapter<ConnectorState>;
+    attachments: NormalizedAttachment[];
+    batchSize?: number;
+    stream: ExternalSystemAttachmentStreamingFunction;
+}

package/dist/attachments-streaming/attachments-streaming-pool.interfaces.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/attachments-streaming/attachments-streaming-pool.js

@@ -10,7 +10,7 @@ class AttachmentsStreamingPool {
         this.stream = stream;
     }
     async streamAll() {
-        console.log(`Starting download of ${this.attachments.length} attachments with ${this.batchSize} workers.`);
+        console.log(`Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.`);
         if (!this.adapter.state.toDevRev) {
             const error = new Error('toDevRev state is not initialized');
             console.error(error);
@@ -23,27 +23,27 @@ class AttachmentsStreamingPool {
             this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
                 [];
         }
-        // Start initial batch of workers up to batchSize limit
+        // Start initial batch of promises up to batchSize limit
         const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
         const initialPromises = [];
         for (let i = 0; i < initialBatchSize; i++) {
-            initialPromises.push(this.startPoolWorker());
+            initialPromises.push(this.startPoolStreaming());
         }
-        // Wait for all workers to complete
+        // Wait for all promises to complete
         await Promise.all(initialPromises);
         if (this.delay) {
             return { delay: this.delay };
         }
         return {};
     }
-    async startPoolWorker() {
+    async startPoolStreaming() {
         var _a, _b, _c, _d;
-        // Worker keeps taking jobs until the attachments array is empty
+        // Process attachments until the attachments array is empty
         while (this.attachments.length > 0) {
             if (this.delay) {
                 break; // Exit if we have a delay
             }
-            // Check if we can start a new worker
+            // Check if we can process next attachment
             const attachment = this.attachments.shift();
             if (!attachment) {
                 break; // Exit if no more attachments

package/dist/attachments-streaming/attachments-streaming-pool.test.js

@@ -91,8 +91,8 @@ describe('AttachmentsStreamingPool', () => {
                 attachments: mockAttachments,
                 stream: mockStream
             });
-            // Mock startPoolWorker to avoid actual processing
-            jest.spyOn(pool, 'startPoolWorker').mockResolvedValue(undefined);
+            // Mock startPoolStreaming to avoid actual processing
+            jest.spyOn(pool, 'startPoolStreaming').mockResolvedValue(undefined);
             await pool.streamAll();
             expect(mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList).toEqual([]);
         });
@@ -116,7 +116,7 @@ describe('AttachmentsStreamingPool', () => {
             const result = await pool.streamAll();
             expect(result).toEqual({});
             expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
-            expect(console.log).toHaveBeenCalledWith('Starting download of 0 attachments with 10 workers.');
+            expect(console.log).toHaveBeenCalledWith('Starting download of 0 attachments, streaming 10 at once.');
         });
         it('should return delay when rate limit is hit', async () => {
             const delayResponse = { delay: 5000 };
@@ -130,7 +130,7 @@ describe('AttachmentsStreamingPool', () => {
             expect(result).toEqual({ delay: 5000 });
         });
     });
-    describe('startPoolWorker', () => {
+    describe('startPoolStreaming', () => {
         it('should skip already processed attachments', async () => {
             mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = ['attachment-1'];
             mockAdapter.processAttachment.mockResolvedValue({});

package/dist/workers/worker-adapter.js

@@ -598,18 +598,18 @@ class WorkerAdapter {
             },
         ];
         this.initializeRepos(repos);
+        const attachmentsMetadata = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
         // If there are no attachments metadata artifact IDs in state, finish here
-        if (!((_b = (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata) === null || _b === void 0 ? void 0 : _b.artifactIds) ||
-            this.state.toDevRev.attachmentsMetadata.artifactIds.length === 0) {
+        if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
             console.log(`No attachments metadata artifact IDs found in state.`);
             return;
         }
         else {
-            console.log(`Found ${this.state.toDevRev.attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
+            console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
         }
         // Loop through the attachments metadata artifact IDs
-        while (this.state.toDevRev.attachmentsMetadata.artifactIds.length > 0) {
-            const attachmentsMetadataArtifactId = this.state.toDevRev.attachmentsMetadata.artifactIds[0];
+        while (attachmentsMetadata.artifactIds.length > 0) {
+            const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
             console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
             const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
                 artifact: attachmentsMetadataArtifactId,
@@ -621,8 +621,8 @@ class WorkerAdapter {
             if (!attachments || attachments.length === 0) {
                 console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
                 // Remove empty artifact and reset lastProcessed
-                this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
-                this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
+                attachmentsMetadata.artifactIds.shift();
+                attachmentsMetadata.lastProcessed = 0;
                 continue;
             }
             console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
@@ -656,10 +656,10 @@ class WorkerAdapter {
                 return response;
             }
             console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
-            this.state.toDevRev.attachmentsMetadata.artifactIds.shift();
-            this.state.toDevRev.attachmentsMetadata.lastProcessed = 0;
-            if (this.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) {
-                this.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
+            attachmentsMetadata.artifactIds.shift();
+            attachmentsMetadata.lastProcessed = 0;
+            if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
+                attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
             }
         }
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.8.0-beta.0",
+  "version": "1.9.0-beta.0",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",