@devrev/ts-adaas 1.10.0 → 1.10.1-beta.2

package/dist/repo/repo.d.ts

@@ -1,5 +1,6 @@
 import { ErrorRecord } from '../types/common';
 import { Item } from '../repo/repo.interfaces';
+import { Artifact } from '../uploader/uploader.interfaces';
 import { RepoFactoryInterface, NormalizedItem, NormalizedAttachment } from './repo.interfaces';
 export declare class Repo {
     readonly itemType: string;
@@ -8,6 +9,7 @@ export declare class Repo {
     private uploader;
     private onUpload;
     private options?;
+    uploadedArtifacts: Artifact[];
     constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
     getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
     upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;

package/dist/repo/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAI/C,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAG3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;gBAE3B,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IASvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAmCxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAuC5C"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAG3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;gBAEzB,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAqCxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAuC5C"}

package/dist/repo/repo.js

@@ -11,6 +11,7 @@ class Repo {
         this.onUpload = onUpload;
         this.uploader = new uploader_1.Uploader({ event, options });
         this.options = options;
+        this.uploadedArtifacts = [];
     }
     getItems() {
         return this.items;
@@ -25,6 +26,7 @@ class Repo {
                 return error;
             }
             this.onUpload(artifact);
+            this.uploadedArtifacts.push(artifact);
             // Clear the uploaded items from the main items array if no batch was specified
             if (!batch) {
                 this.items = [];

package/dist/uploader/uploader.d.ts

@@ -23,6 +23,11 @@ export declare class Uploader {
     uploadArtifact(artifact: ArtifactToUpload, file: Buffer): Promise<AxiosResponse | void>;
     streamArtifact(artifact: ArtifactToUpload, fileStream: any): Promise<AxiosResponse | void>;
     confirmArtifactUpload(artifactId: string): Promise<AxiosResponse | void>;
+    /**
+     * Destroys a stream to prevent resource leaks.
+     * @param {any} fileStream - The axios response stream to destroy
+     */
+    private destroyStream;
     getAttachmentsFromArtifactId({ artifact, }: {
         artifact: string;
     }): Promise<{

package/dist/uploader/uploader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAEL,cAAc,EACd,wBAAwB,EACxB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;;;;OAQG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwDpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAuB7B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmB1B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,GAAG,GACd,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAkC1B,qBAAqB,CACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAwB1B,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAqCY,sBAAsB;YAyBtB,gBAAgB;IAe9B,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,UAAU;IASZ,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC;YAyBvB,eAAe;CA8B9B"}
+{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAEL,cAAc,EACd,wBAAwB,EACxB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,kBAAkB,CAAC,CAAU;IACrC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAyB;gBAEnC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,wBAAwB;IAWxD;;;;;;;;OAQG;IACG,MAAM,CACV,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,GAChC,OAAO,CAAC,cAAc,CAAC;IAwDpB,oBAAoB,CACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAuB7B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAmB1B,cAAc,CAClB,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,GAAG,GACd,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAoC1B,qBAAqB,CACzB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAwBhC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAef,4BAA4B,CAAC,EACjC,QAAQ,GACT,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC;QACV,WAAW,CAAC,EAAE,oBAAoB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7B,CAAC;YAqCY,sBAAsB;YAyBtB,gBAAgB;IAe9B,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,UAAU;IASZ,yBAAyB,CAAC,EAC9B,UAAU,EACV,SAAiB,GAClB,EAAE;QACD,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC;YAyBvB,eAAe;CA8B9B"}

package/dist/uploader/uploader.js

@@ -148,6 +148,7 @@ class Uploader {
         formData.append('file', fileStream.data);
         if (fileStream.headers['content-length'] > constants_1.MAX_DEVREV_ARTIFACT_SIZE) {
             console.warn(`File size exceeds the maximum limit of ${constants_1.MAX_DEVREV_ARTIFACT_SIZE} bytes.`);
+            this.destroyStream(fileStream);
             return;
         }
         try {
@@ -164,6 +165,7 @@ class Uploader {
         }
         catch (error) {
             console.error('Error while streaming artifact.', (0, logger_1.serializeError)(error));
+            this.destroyStream(fileStream);
             return;
         }
     }
@@ -182,6 +184,26 @@ class Uploader {
             console.error('Error while confirming artifact upload.', (0, logger_1.serializeError)(error));
         }
     }
+    /**
+     * Destroys a stream to prevent resource leaks.
+     * @param {any} fileStream - The axios response stream to destroy
+     */
+    destroyStream(fileStream) {
+        try {
+            if (fileStream && fileStream.data) {
+                // For axios response streams, the data property contains the actual stream
+                if (typeof fileStream.data.destroy === 'function') {
+                    fileStream.data.destroy();
+                }
+                else if (typeof fileStream.data.close === 'function') {
+                    fileStream.data.close();
+                }
+            }
+        }
+        catch (error) {
+            console.warn('Error while destroying stream:', (0, logger_1.serializeError)(error));
+        }
+    }
     async getAttachmentsFromArtifactId({ artifact, }) {
         // Get the URL of the attachments metadata artifact
         const artifactUrl = await this.getArtifactDownloadUrl(artifact);

package/dist/workers/default-workers/attachments-extraction.js

@@ -6,10 +6,28 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const axios_1 = __importDefault(require("axios"));
 const index_1 = require("../../index");
 const axios_client_internal_1 = require("../../http/axios-client-internal");
+const constants_1 = require("../../common/constants");
 const getAttachmentStream = async ({ item, }) => {
     const { id, url } = item;
+    let fileStreamResponse = null;
     try {
-        const fileStreamResponse = await axios_client_internal_1.axiosClient.get(url, {
+        // First, check file size with HEAD request to avoid downloading large files
+        const headResponse = await axios_client_internal_1.axiosClient.head(url, {
+            headers: {
+                'Accept-Encoding': 'identity',
+            },
+        });
+        const contentLength = headResponse.headers['content-length'];
+        if (contentLength && parseInt(contentLength) > constants_1.MAX_DEVREV_ARTIFACT_SIZE) {
+            console.warn(`Attachment ${id} size (${contentLength} bytes) exceeds maximum limit of ${constants_1.MAX_DEVREV_ARTIFACT_SIZE} bytes. Skipping download.`);
+            return {
+                error: {
+                    message: `File size exceeds maximum limit of ${constants_1.MAX_DEVREV_ARTIFACT_SIZE} bytes.`,
+                },
+            };
+        }
+        // If size is acceptable, proceed with streaming
+        fileStreamResponse = await axios_client_internal_1.axiosClient.get(url, {
             responseType: 'stream',
             headers: {
                 'Accept-Encoding': 'identity',
@@ -18,6 +36,10 @@ const getAttachmentStream = async ({ item, }) => {
         return { httpStream: fileStreamResponse };
     }
     catch (error) {
+        // If we created a stream but failed afterwards, destroy it
+        if (fileStreamResponse) {
+            destroyHttpStream(fileStreamResponse);
+        }
         if (axios_1.default.isAxiosError(error)) {
             console.warn(`Error while fetching attachment ${id} from URL.`, (0, index_1.serializeAxiosError)(error));
             console.warn('Failed attachment metadata', item);
@@ -33,6 +55,25 @@ const getAttachmentStream = async ({ item, }) => {
         };
     }
 };
+/**
+ * Destroys a stream to prevent memory leaks.
+ * @param {any} httpStream - The axios response stream to destroy
+ */
+const destroyHttpStream = (httpStream) => {
+    try {
+        if (httpStream && httpStream.data) {
+            if (typeof httpStream.data.destroy === 'function') {
+                httpStream.data.destroy();
+            }
+            else if (typeof httpStream.data.close === 'function') {
+                httpStream.data.close();
+            }
+        }
+    }
+    catch (error) {
+        console.warn('Error while destroying HTTP stream:', error);
+    }
+};
 (0, index_1.processTask)({
     task: async ({ adapter }) => {
         try {

package/dist/workers/worker-adapter.artifacts.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=worker-adapter.artifacts.test.d.ts.map

package/dist/workers/worker-adapter.artifacts.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-adapter.artifacts.test.d.ts","sourceRoot":"","sources":["../../src/workers/worker-adapter.artifacts.test.ts"],"names":[],"mappings":""}

package/dist/workers/worker-adapter.artifacts.test.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const test_helpers_1 = require("../tests/test-helpers");
+const worker_adapter_1 = require("./worker-adapter");
+const state_1 = require("../state/state");
+const types_1 = require("../types");
+// 1. Create a mock function for the method you want to override.
+const mockUpload = (itemType, objects) => {
+    return {
+        error: null,
+        artifact: {
+            id: `artifact-${itemType}-${Math.random().toString(36).substring(2, 15)}`,
+            item_type: itemType,
+            item_count: objects.length,
+        }
+    };
+};
+// 2. Mock the entire 'uploader' module.
+// The factory function () => { ... } returns the mock implementation.
+jest.mock('../uploader/uploader', () => {
+    return {
+        // The mocked Uploader class
+        Uploader: jest.fn().mockImplementation(() => {
+            // The constructor of the mocked Uploader returns an object
+            // with the methods you want to control.
+            return {
+                upload: mockUpload,
+            };
+        }),
+    };
+});
+function checkArtifactOrder(artifacts, expectedOrder) {
+    let outerIndex = 0;
+    for (const artifact of artifacts) {
+        try {
+            // Always increase outer index. If items are out of order, the array will overflow and exception will be thrown
+            while (artifact.item_type != expectedOrder[outerIndex].itemType) {
+                outerIndex++;
+            }
+        }
+        catch (e) {
+            console.error("Error finding artifact type in repos:", e);
+            return false;
+        }
+    }
+    return true;
+}
+describe("Artifact ordering when artifacts overflow batch sizes in repositories", () => {
+    let testAdapter;
+    beforeEach(() => {
+        // Create a fresh adapter instance for this test to avoid mocking conflicts
+        let mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart });
+        let mockAdapterState = new state_1.State({
+            event: mockEvent,
+            initialState: { attachments: { completed: false } },
+        });
+        testAdapter = new worker_adapter_1.WorkerAdapter({
+            event: mockEvent,
+            adapterState: mockAdapterState,
+            options: {
+                batchSize: 50
+            }
+        });
+    });
+    it('should maintain artifact ordering when repo ItemTypeA has items below batch size and repo ItemTypeB has items above batch size', async () => {
+        var _a, _b;
+        const repos = [
+            { itemType: 'ItemTypeA' },
+            { itemType: 'ItemTypeB' }
+        ];
+        // Initialize repos
+        testAdapter.initializeRepos(repos);
+        await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
+        await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(105)));
+        await testAdapter.uploadAllRepos();
+        const artifacts = testAdapter.artifacts;
+        expect(artifacts.length).toBe(4);
+        expect(checkArtifactOrder(artifacts, repos)).toBe(true);
+    });
+    it('should work with more than 2 repos', async () => {
+        var _a, _b, _c, _d;
+        const repos = [
+            { itemType: 'ItemTypeA' },
+            { itemType: 'ItemTypeB' },
+            { itemType: 'ItemTypeC' },
+            { itemType: 'ItemTypeD' },
+        ];
+        // Initialize repos
+        testAdapter.initializeRepos(repos);
+        await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
+        await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
+        await ((_c = testAdapter.getRepo('ItemTypeC')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(103)));
+        await ((_d = testAdapter.getRepo('ItemTypeD')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(104)));
+        await testAdapter.uploadAllRepos();
+        const artifacts = testAdapter.artifacts;
+        expect(artifacts.length).toBe(12);
+        expect(checkArtifactOrder(artifacts, repos)).toBe(true);
+    });
+    it('should maintain order with multiple pushes and uploads', async () => {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
+        const repos = [
+            { itemType: 'ItemTypeA' },
+            { itemType: 'ItemTypeB' },
+        ];
+        // Initialize repos
+        testAdapter.initializeRepos(repos);
+        await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
+        await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
+        await ((_c = testAdapter.getRepo('ItemTypeA')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(101)));
+        await ((_d = testAdapter.getRepo('ItemTypeB')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(102)));
+        await ((_e = testAdapter.getRepo('ItemTypeA')) === null || _e === void 0 ? void 0 : _e.upload());
+        await ((_f = testAdapter.getRepo('ItemTypeB')) === null || _f === void 0 ? void 0 : _f.upload());
+        await ((_g = testAdapter.getRepo('ItemTypeA')) === null || _g === void 0 ? void 0 : _g.push((0, test_helpers_1.createItems)(101)));
+        await ((_h = testAdapter.getRepo('ItemTypeB')) === null || _h === void 0 ? void 0 : _h.push((0, test_helpers_1.createItems)(102)));
+        await ((_j = testAdapter.getRepo('ItemTypeA')) === null || _j === void 0 ? void 0 : _j.push((0, test_helpers_1.createItems)(101)));
+        await ((_k = testAdapter.getRepo('ItemTypeB')) === null || _k === void 0 ? void 0 : _k.push((0, test_helpers_1.createItems)(102)));
+        await testAdapter.uploadAllRepos();
+        const artifacts = testAdapter.artifacts;
+        expect(artifacts.length).toBe(20);
+        expect(checkArtifactOrder(artifacts, repos)).toBe(true);
+    });
+    it('should not count artifacts if 0 items are pushed to the repo', async () => {
+        var _a;
+        const repos = [
+            { itemType: 'ItemTypeA' }
+        ];
+        // Initialize repos
+        testAdapter.initializeRepos(repos);
+        await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push([]));
+        await testAdapter.uploadAllRepos();
+        const artifacts = testAdapter.artifacts;
+        expect(artifacts.length).toBe(0);
+        expect(checkArtifactOrder(artifacts, repos)).toBe(true);
+    });
+});

package/dist/workers/worker-adapter.d.ts

@@ -67,6 +67,11 @@ export declare class WorkerAdapter<ConnectorState> {
         itemTypeToLoad: ItemTypeToLoad;
     }): Promise<LoadItemResponse>;
     processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
+    /**
+     * Destroys a stream to prevent memory leaks.
+     * @param {any} httpStream - The axios response stream to destroy
+     */
+    private destroyHttpStream;
     loadAttachment({ item, create, }: {
         item: ExternalSystemAttachment;
         create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;

package/dist/workers/worker-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../src/workers/worker-adapter.ts"],"names":[],"mappings":"AACA,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,SAAS,EAET,yCAAyC,EACzC,kCAAkC,EAClC,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAEL,wBAAwB,EACxB,6BAA6B,EAC7B,UAAU,EACV,eAAe,EAEhB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAkB,MAAM,iCAAiC,CAAC;AAO3E,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAIhF,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EACtB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAM7C,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAyBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAWrC,SAAS;IAIf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAiFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IASrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAgHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IAuBK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAuE5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwKvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAsFjC,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAgC7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAsHzC"}
+{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../src/workers/worker-adapter.ts"],"names":[],"mappings":"AACA,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,SAAS,EAET,yCAAyC,EACzC,kCAAkC,EAClC,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAEL,wBAAwB,EACxB,6BAA6B,EAC7B,UAAU,EACV,eAAe,EAEhB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAkB,MAAM,iCAAiC,CAAC;AAO3E,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAIhF,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EACtB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAM7C,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAWrC,SAAS;IAIf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAiFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAgHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IAuBK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAuE5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwKvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAwFvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAgC7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAsHzC"}

package/dist/workers/worker-adapter.js

@@ -85,7 +85,6 @@ class WorkerAdapter {
                 repo.itemType !== constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT;
             return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
                     var _a;
-                    this.artifacts.push(artifact);
                     // We need to store artifacts ids in state for later use when streaming attachments
                     if (repo.itemType === constants_1.AIRDROP_DEFAULT_ITEM_TYPES.ATTACHMENTS) {
                         (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
@@ -180,6 +179,7 @@ class WorkerAdapter {
     async uploadAllRepos() {
         for (const repo of this.repos) {
             const error = await repo.upload();
+            this.artifacts.push(...repo.uploadedArtifacts);
             if (error) {
                 throw error;
             }
@@ -514,9 +514,11 @@ class WorkerAdapter {
             const preparedArtifact = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType);
             if (!preparedArtifact) {
                 console.warn(`Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment.`);
+                this.destroyHttpStream(httpStream);
                 return;
             }
             if (this.isTimeout) {
+                this.destroyHttpStream(httpStream);
                 return;
             }
             // Stream attachment
@@ -557,6 +559,25 @@ class WorkerAdapter {
         }
         return;
     }
+    /**
+     * Destroys a stream to prevent memory leaks.
+     * @param {any} httpStream - The axios response stream to destroy
+     */
+    destroyHttpStream(httpStream) {
+        try {
+            if (httpStream && httpStream.data) {
+                if (typeof httpStream.data.destroy === 'function') {
+                    httpStream.data.destroy();
+                }
+                else if (typeof httpStream.data.close === 'function') {
+                    httpStream.data.close();
+                }
+            }
+        }
+        catch (error) {
+            console.warn('Error while destroying HTTP stream:', error);
+        }
+    }
     async loadAttachment({ item, create, }) {
         // Create item
         const { id, delay, error } = await create({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.10.0",
+  "version": "1.10.1-beta.2",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -38,17 +38,16 @@
     "eslint-config-prettier": "^10.1.5",
     "eslint-plugin-prettier": "^5.1.3",
     "jest": "^29.7.0",
-    "ts-jest": "^29.3.3"
+    "ts-jest": "^29.4.4",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
   },
   "dependencies": {
     "@devrev/typescript-sdk": "^1.1.59",
     "axios": "^1.12.1",
-    "@microsoft/api-extractor": "^7.52.11",
     "axios-retry": "^4.5.0",
     "form-data": "^4.0.4",
     "js-jsonl": "^1.1.1",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.3.3",
     "yargs": "^17.7.2"
   },
   "files": [