@devrev/ts-adaas 1.0.4 → 1.1.0

package/dist/workers/worker-adapter.js

@@ -1,12 +1,22 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkerAdapter = void 0;
 exports.createWorkerAdapter = createWorkerAdapter;
+const axios_1 = __importDefault(require("axios"));
 const extraction_1 = require("../types/extraction");
+const loading_1 = require("../types/loading");
 const constants_1 = require("../common/constants");
 const control_protocol_1 = require("../common/control-protocol");
 const workers_1 = require("../types/workers");
 const repo_1 = require("../repo/repo");
+const helpers_1 = require("../common/helpers");
+const mappers_1 = require("../mappers/mappers");
+const uploader_1 = require("../uploader/uploader");
+const logger_1 = require("../logger/logger");
+const mappers_interface_1 = require("../mappers/mappers.interface");
 function createWorkerAdapter({ event, adapterState, parentPort, options, }) {
     return new WorkerAdapter({
         event,
@@ -39,6 +49,17 @@ class WorkerAdapter {
         this.parentPort = parentPort;
         this.hasWorkerEmitted = false;
         this.isTimeout = false;
+        // Loader
+        this.loaderReports = [];
+        this._processedFiles = [];
+        this.mappers = new mappers_1.Mappers({
+            event,
+            options,
+        });
+        this.uploader = new uploader_1.Uploader({
+            event,
+            options,
+        });
     }
     get state() {
         return this.adapterState.state;
@@ -48,6 +69,12 @@ class WorkerAdapter {
             this.adapterState.state = value;
         }
     }
+    get reports() {
+        return this.loaderReports;
+    }
+    get processedFiles() {
+        return this._processedFiles;
+    }
     initializeRepos(repos) {
         this.repos = repos.map((repo) => {
             const shouldNormalize = !Object.values(constants_1.AIRDROP_DEFAULT_ITEM_TYPES).includes(repo.itemType);
@@ -112,7 +139,9 @@ class WorkerAdapter {
             await (0, control_protocol_1.emit)({
                 eventType: newEventType,
                 event: this.event,
-                data: Object.assign(Object.assign({}, data), { artifacts: this.artifacts }),
+                data: Object.assign(Object.assign({}, data), (constants_1.ALLOWED_EXTRACTION_EVENT_TYPES.includes(this.event.payload.event_type)
+                    ? { artifacts: this.artifacts }
+                    : {})),
             });
             const message = {
                 subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
@@ -134,5 +163,305 @@ class WorkerAdapter {
     handleTimeout() {
         this.isTimeout = true;
     }
+    async loadItemTypes({ itemTypesToLoad, }) {
+        var _a, _b, _c;
+        const statsFileArtifactId = (_a = this.event.payload.event_data) === null || _a === void 0 ? void 0 : _a.stats_file;
+        if (!statsFileArtifactId) {
+            console.error('Stats file artifact id not found in event data.');
+            await (0, control_protocol_1.emit)({
+                event: this.event,
+                eventType: loading_1.LoaderEventType.DataLoadingError,
+                data: {
+                    error: {
+                        message: 'Stats file artifact id not found in event data.',
+                    },
+                },
+            });
+            return {
+                reports: this.reports,
+                processed_files: this.processedFiles,
+            };
+        }
+        const statsFile = (await this.uploader.getJsonObjectByArtifactId({
+            artifactId: statsFileArtifactId,
+        }));
+        console.log('Stats file', statsFile);
+        if (!statsFile || statsFile.length === 0) {
+            console.warn('Stats file not found or empty.');
+            return {
+                reports: this.reports,
+                processed_files: this.processedFiles,
+            };
+        }
+        if (this.event.payload.event_type === extraction_1.EventType.StartLoadingData) {
+            console.log('Recieved event type ' +
+                extraction_1.EventType.StartLoadingData +
+                '. Preparing files to load.');
+            const filesToLoad = (0, helpers_1.getFilesToLoad)({
+                itemTypesToLoad,
+                statsFile,
+            });
+            if (filesToLoad.length === 0) {
+                console.warn('No files to load, returning.');
+                return {
+                    reports: this.reports,
+                    processed_files: this.processedFiles,
+                };
+            }
+            this.adapterState.state.fromDevRev = { filesToLoad };
+        }
+        console.log('Files to load in state', (_b = this.adapterState.state.fromDevRev) === null || _b === void 0 ? void 0 : _b.filesToLoad);
+        if (!this.adapterState.state.fromDevRev) {
+            await (0, control_protocol_1.emit)({
+                event: this.event,
+                eventType: loading_1.LoaderEventType.DataLoadingError,
+                data: {
+                    error: {
+                        message: 'Unexpected state set in LOAD_DATA.',
+                    },
+                },
+            });
+            return {
+                reports: this.reports,
+                processed_files: this.processedFiles,
+            };
+        }
+        outerloop: for (const fileToLoad of (_c = this.adapterState.state.fromDevRev) === null || _c === void 0 ? void 0 : _c.filesToLoad) {
+            const itemTypeToLoad = itemTypesToLoad.find((itemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType);
+            if (!itemTypeToLoad) {
+                console.error(`Item type to load not found for item type: ${fileToLoad.itemType}.`);
+                await (0, control_protocol_1.emit)({
+                    event: this.event,
+                    eventType: loading_1.LoaderEventType.DataLoadingError,
+                    data: {
+                        error: {
+                            message: `Item type to load not found for item type: ${fileToLoad.itemType}.`,
+                        },
+                    },
+                });
+                break;
+            }
+            if (!fileToLoad.completed) {
+                const transformerFile = (await this.uploader.getJsonObjectByArtifactId({
+                    artifactId: fileToLoad.id,
+                    isGzipped: true,
+                }));
+                if (!transformerFile) {
+                    console.error('Transformer file not found.');
+                    break outerloop;
+                }
+                for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) {
+                    const { report, rateLimit, error } = await this.loadItem({
+                        item: transformerFile[i],
+                        itemTypeToLoad,
+                    });
+                    if (error) {
+                        await (0, control_protocol_1.emit)({
+                            event: this.event,
+                            eventType: loading_1.LoaderEventType.DataLoadingError,
+                            data: {
+                                error,
+                            },
+                        });
+                        break outerloop;
+                    }
+                    if (rateLimit === null || rateLimit === void 0 ? void 0 : rateLimit.delay) {
+                        await (0, control_protocol_1.emit)({
+                            event: this.event,
+                            eventType: loading_1.LoaderEventType.DataLoadingDelay,
+                            data: {
+                                delay: rateLimit.delay,
+                                reports: this.reports,
+                                processed_files: this.processedFiles,
+                            },
+                        });
+                        break outerloop;
+                    }
+                    if (report) {
+                        (0, helpers_1.addReportToLoaderReport)({
+                            loaderReports: this.loaderReports,
+                            report,
+                        });
+                        fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1;
+                    }
+                }
+                fileToLoad.completed = true;
+                this._processedFiles.push(fileToLoad.id);
+            }
+        }
+        return {
+            reports: this.reports,
+            processed_files: this.processedFiles,
+        };
+    }
+    async loadItem({ item, itemTypeToLoad, }) {
+        var _a;
+        const devrevId = item.id.devrev;
+        try {
+            const syncMapperRecordResponse = await this.mappers.getByTargetId({
+                sync_unit: this.event.payload.event_context.sync_unit,
+                target: devrevId,
+            });
+            const syncMapperRecord = syncMapperRecordResponse.data;
+            if (!syncMapperRecord) {
+                console.error('Failed to get sync mapper record from response.');
+                return {
+                    error: {
+                        message: 'Failed to get sync mapper record from response.',
+                    },
+                };
+            }
+            // Update item
+            const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({
+                item,
+                mappers: this.mappers,
+                event: this.event,
+            });
+            if (id) {
+                if (modifiedDate) {
+                    try {
+                        const updateSyncMapperRecordResponse = await this.mappers.update({
+                            id: syncMapperRecord.sync_mapper_record.id,
+                            sync_unit: this.event.payload.event_context.sync_unit,
+                            status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
+                            external_versions: {
+                                add: [
+                                    {
+                                        modified_date: modifiedDate,
+                                        recipe_version: 0,
+                                    },
+                                ],
+                            },
+                            external_ids: {
+                                add: [id],
+                            },
+                            targets: {
+                                add: [devrevId],
+                            },
+                        });
+                        console.log('Updated sync mapper record', JSON.stringify(updateSyncMapperRecordResponse.data));
+                    }
+                    catch (error) {
+                        if (axios_1.default.isAxiosError(error)) {
+                            console.error('Failed to update sync mapper record', (0, logger_1.formatAxiosError)(error));
+                            return {
+                                error: {
+                                    message: error.message,
+                                },
+                            };
+                        }
+                        else {
+                            console.error('Failed to update sync mapper record', error);
+                            return {
+                                error: {
+                                    message: 'Failed to update sync mapper record' + error,
+                                },
+                            };
+                        }
+                    }
+                }
+                return {
+                    report: {
+                        item_type: itemTypeToLoad.itemType,
+                        [loading_1.ActionType.UPDATED]: 1,
+                    },
+                };
+            }
+            else if (delay) {
+                console.log('Rate limited, delaying for', delay);
+                return {
+                    rateLimit: {
+                        delay,
+                    },
+                };
+            }
+            else {
+                console.error('Failed to update item', error);
+                return {
+                    report: {
+                        item_type: itemTypeToLoad.itemType,
+                        [loading_1.ActionType.FAILED]: 1,
+                    },
+                };
+            }
+            // Update mapper (optional)
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error)) {
+                if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
+                    // Create item
+                    const { id, delay, error } = await itemTypeToLoad.create({
+                        item,
+                        mappers: this.mappers,
+                        event: this.event,
+                    });
+                    if (id) {
+                        // Create mapper
+                        try {
+                            const createdSyncMapperRecordResponse = await this.mappers.create({
+                                sync_unit: this.event.payload.event_context.sync_unit,
+                                status: mappers_interface_1.SyncMapperRecordStatus.OPERATIONAL,
+                                external_ids: [id],
+                                targets: [devrevId],
+                            });
+                            console.log('Created sync mapper record', createdSyncMapperRecordResponse.data.sync_mapper_record.id);
+                            return {
+                                report: {
+                                    item_type: itemTypeToLoad.itemType,
+                                    [loading_1.ActionType.CREATED]: 1,
+                                },
+                            };
+                        }
+                        catch (error) {
+                            if (axios_1.default.isAxiosError(error)) {
+                                console.error('Failed to create sync mapper record', (0, logger_1.formatAxiosError)(error));
+                                return {
+                                    error: {
+                                        message: error.message,
+                                    },
+                                };
+                            }
+                            console.error('Failed to create sync mapper record', error);
+                            return {
+                                error: {
+                                    message: 'Failed to create sync mapper record' + error,
+                                },
+                            };
+                        }
+                    }
+                    else if (delay) {
+                        return {
+                            rateLimit: {
+                                delay,
+                            },
+                        };
+                    }
+                    else {
+                        console.error('Failed to create item', error);
+                        return {
+                            report: {
+                                item_type: itemTypeToLoad.itemType,
+                                [loading_1.ActionType.FAILED]: 1,
+                            },
+                        };
+                    }
+                }
+                else {
+                    console.error('Failed to get sync mapper record', (0, logger_1.formatAxiosError)(error));
+                    return {
+                        error: {
+                            message: error.message,
+                        },
+                    };
+                }
+            }
+            console.error('Failed to get sync mapper record', error);
+            return {
+                error: {
+                    message: 'Failed to get sync mapper record' + error,
+                },
+            };
+        }
+    }
 }
 exports.WorkerAdapter = WorkerAdapter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.0.4",
+  "version": "1.1.0",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",