@devrev/ts-adaas 1.10.1-beta.4 → 1.10.1-beta.5

package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,2BAA2B,EAC5B,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAGzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD,OAAO,CAAC,cAAc;IAOhB,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IAuCjD,kBAAkB;CAiEzB"}
+{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,2BAA2B,EAC5B,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAGzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;IAQjD,OAAO,CAAC,cAAc;IAOhB,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IAuCjD,kBAAkB;CAgEzB"}

package/dist/attachments-streaming/attachments-streaming-pool.js

@@ -64,7 +64,6 @@ class AttachmentsStreamingPool {
             }
             if (this.adapter.state.toDevRev &&
                 ((_a = this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.includes(attachment.id))) {
-                this.updateProgress();
                 continue; // Skip if the attachment ID is already processed
             }
             try {

package/dist/workers/default-workers/attachments-extraction.js

@@ -5,34 +5,30 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const axios_1 = __importDefault(require("axios"));
 const index_1 = require("../../index");
-const axios_client_internal_1 = require("../../http/axios-client-internal");
 const constants_1 = require("../../common/constants");
 const getAttachmentStream = async ({ item, }) => {
     const { id, url } = item;
     let fileStreamResponse = null;
     try {
-        // First, check file size with HEAD request to avoid downloading large files
-        const headResponse = await axios_client_internal_1.axiosClient.head(url, {
+        // Get the stream response directly
+        fileStreamResponse = await axios_1.default.get(url, {
+            responseType: 'stream',
             headers: {
                 'Accept-Encoding': 'identity',
             },
         });
-        const contentLength = headResponse.headers['content-length'];
+        // Check content-length from the stream response headers
+        const contentLength = fileStreamResponse.headers['content-length'];
         if (contentLength && parseInt(contentLength) > constants_1.MAX_DEVREV_ARTIFACT_SIZE) {
             console.warn(`Attachment ${id} size (${contentLength} bytes) exceeds maximum limit of ${constants_1.MAX_DEVREV_ARTIFACT_SIZE} bytes. Skipping download.`);
+            // Destroy the stream since we won't use it
+            destroyHttpStream(fileStreamResponse);
             return {
                 error: {
                     message: `File size exceeds maximum limit of ${constants_1.MAX_DEVREV_ARTIFACT_SIZE} bytes.`,
                 },
             };
         }
-        // If size is acceptable, proceed with streaming
-        fileStreamResponse = await axios_client_internal_1.axiosClient.get(url, {
-            responseType: 'stream',
-            headers: {
-                'Accept-Encoding': 'identity',
-            },
-        });
         return { httpStream: fileStreamResponse };
     }
     catch (error) {
@@ -40,17 +36,9 @@ const getAttachmentStream = async ({ item, }) => {
         if (fileStreamResponse) {
             destroyHttpStream(fileStreamResponse);
         }
-        if (axios_1.default.isAxiosError(error)) {
-            console.warn(`Error while fetching attachment ${id} from URL.`, (0, index_1.serializeAxiosError)(error));
-            console.warn('Failed attachment metadata', item);
-        }
-        else {
-            console.warn(`Error while fetching attachment ${id} from URL.`, error);
-            console.warn('Failed attachment metadata', item);
-        }
         return {
             error: {
-                message: 'Error while fetching attachment ' + id + ' from URL.',
+                message: `Error while getting attachment stream for attachment with id ${id}.`,
             },
         };
     }
@@ -79,6 +67,7 @@ const destroyHttpStream = (httpStream) => {
         try {
             const response = await adapter.streamAttachments({
                 stream: getAttachmentStream,
+                batchSize: 10,
             });
             if (response === null || response === void 0 ? void 0 : response.delay) {
                 await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsDelay, {
@@ -99,8 +88,6 @@ const destroyHttpStream = (httpStream) => {
         }
     },
     onTimeout: async ({ adapter }) => {
-        await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsProgress, {
-            progress: 50,
-        });
+        await adapter.emit(index_1.ExtractorEventType.ExtractionAttachmentsProgress);
     },
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.10.1-beta.4",
+  "version": "1.10.1-beta.5",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",