@devrev/airsync-sdk 2.0.0-beta.0

package/dist/multithreading/spawn/spawn.js

@@ -0,0 +1,249 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Spawn = void 0;
+exports.spawn = spawn;
+const yargs_1 = __importDefault(require("yargs"));
+const helpers_1 = require("yargs/helpers");
+const control_protocol_1 = require("../../common/control-protocol");
+const event_type_translation_1 = require("../../common/event-type-translation");
+const helpers_2 = require("../../common/helpers");
+const logger_1 = require("../../logger/logger");
+const extraction_1 = require("../../types/extraction");
+const workers_1 = require("../../types/workers");
+const constants_1 = require("../../common/constants");
+const create_worker_1 = require("../create-worker");
+const spawn_helpers_1 = require("./spawn.helpers");
+function getWorkerPath({ event, workerBasePath, }) {
+    let path = null;
+    switch (event.payload.event_type) {
+        case extraction_1.EventType.StartExtractingExternalSyncUnits:
+            path = '/workers/external-sync-units-extraction';
+            break;
+        case extraction_1.EventType.StartExtractingMetadata:
+            path = '/workers/metadata-extraction';
+            break;
+        case extraction_1.EventType.StartExtractingData:
+        case extraction_1.EventType.ContinueExtractingData:
+            path = '/workers/data-extraction';
+            break;
+        case extraction_1.EventType.StartExtractingAttachments:
+        case extraction_1.EventType.ContinueExtractingAttachments:
+            path = '/workers/attachments-extraction';
+            break;
+        case extraction_1.EventType.StartLoadingData:
+        case extraction_1.EventType.ContinueLoadingData:
+            path = '/workers/load-data';
+            break;
+        case extraction_1.EventType.StartLoadingAttachments:
+        case extraction_1.EventType.ContinueLoadingAttachments:
+            path = '/workers/load-attachments';
+            break;
+    }
+    return path ? workerBasePath + path : null;
+}
+/**
+ * Creates a new instance of Spawn class.
+ * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
+ * The class provides utilities to emit control events to the platform and exit the worker gracefully.
+ * In case of lambda timeout, the class emits a lambda timeout event to the platform.
+ * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class
+ * @param {AirdropEvent} options.event - The event object received from the platform
+ * @param {object} options.initialState - The initial state of the adapter
+ * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath
+ * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname`
+ * @returns {Promise<Spawn>} - A new instance of Spawn class
+ */
+async function spawn({ event, initialState, workerPath, initialDomainMapping, options, baseWorkerPath, }) {
+    // Translate incoming event type for backwards compatibility. This allows the
+    // SDK to accept both old and new event type formats. Then update the event with the translated event type.
+    const originalEventType = event.payload.event_type;
+    const translatedEventType = (0, event_type_translation_1.translateIncomingEventType)(event.payload.event_type);
+    event.payload.event_type = translatedEventType;
+    // Read the command line arguments to check if the local flag is passed.
+    const argv = await (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv)).argv;
+    if (argv._.includes('local')) {
+        options = Object.assign(Object.assign({}, (options || {})), { isLocalDevelopment: true });
+    }
+    const originalConsole = console;
+    // eslint-disable-next-line no-global-assign
+    console = new logger_1.Logger({ event, options });
+    if (translatedEventType !== originalEventType) {
+        console.log(`Event type translated from ${originalEventType} to ${translatedEventType}.`);
+    }
+    if (options === null || options === void 0 ? void 0 : options.isLocalDevelopment) {
+        console.log('Snap-in is running in local development mode.');
+    }
+    let script = null;
+    if (workerPath != null) {
+        script = workerPath;
+    }
+    else if (baseWorkerPath != null &&
+        (options === null || options === void 0 ? void 0 : options.workerPathOverrides) != null &&
+        options.workerPathOverrides[translatedEventType] != null) {
+        script =
+            baseWorkerPath +
+                options.workerPathOverrides[translatedEventType];
+    }
+    else {
+        script = getWorkerPath({
+            event,
+            workerBasePath: baseWorkerPath !== null && baseWorkerPath !== void 0 ? baseWorkerPath : __dirname,
+        });
+    }
+    // If a script is found for the event type, spawn a new worker.
+    if (script) {
+        try {
+            const worker = await (0, create_worker_1.createWorker)({
+                event,
+                initialState,
+                workerPath: script,
+                initialDomainMapping,
+                options,
+            });
+            return new Promise((resolve) => {
+                new Spawn({
+                    event,
+                    worker,
+                    options,
+                    resolve,
+                    originalConsole,
+                });
+            });
+        }
+        catch (error) {
+            console.error('Worker error while processing task', error);
+            // eslint-disable-next-line no-global-assign
+            console = originalConsole;
+            return Promise.reject(error);
+        }
+    }
+    else {
+        const { eventType } = (0, spawn_helpers_1.getNoScriptEventType)(event.payload.event_type);
+        await (0, control_protocol_1.emit)({
+            event,
+            eventType,
+        });
+        // eslint-disable-next-line no-global-assign
+        console = originalConsole;
+        return Promise.resolve();
+    }
+}
+class Spawn {
+    constructor({ event, worker, options, resolve, originalConsole, }) {
+        this.defaultLambdaTimeout = constants_1.DEFAULT_LAMBDA_TIMEOUT;
+        this.originalConsole = originalConsole || console;
+        this.logger = console;
+        this.alreadyEmitted = false;
+        this.event = event;
+        this.lambdaTimeout = (options === null || options === void 0 ? void 0 : options.timeout)
+            ? Math.min(options.timeout, this.defaultLambdaTimeout)
+            : this.defaultLambdaTimeout;
+        this.resolve = resolve;
+        // If soft timeout is reached, send a message to the worker to gracefully exit.
+        this.softTimeoutTimer = setTimeout(() => void (async () => {
+            console.log('SOFT TIMEOUT: Sending a message to the worker to gracefully exit.');
+            if (worker) {
+                worker.postMessage({
+                    subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
+                });
+            }
+            else {
+                console.log('Worker does not exist. Exiting from main thread.');
+                await this.exitFromMainThread();
+            }
+        })(), this.lambdaTimeout);
+        // If hard timeout is reached, that means the worker did not exit in time. Terminate the worker.
+        this.hardTimeoutTimer = setTimeout(() => void (async () => {
+            console.error('HARD TIMEOUT: Worker did not exit in time. Terminating the worker.');
+            if (worker) {
+                await worker.terminate();
+            }
+            else {
+                console.log('Worker does not exist. Exiting from main thread.');
+                await this.exitFromMainThread();
+            }
+        })(), this.lambdaTimeout * constants_1.HARD_TIMEOUT_MULTIPLIER);
+        // If worker exits with process.exit(code), clear the timeouts and exit from
+        // main thread.
+        worker.on(workers_1.WorkerEvent.WorkerExit, (code) => void (async () => {
+            console.info('Worker exited with exit code: ' + code + '.');
+            this.clearTimeouts();
+            await this.exitFromMainThread();
+        })());
+        worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
+            var _a, _b, _c, _d;
+            // Since logs from the worker thread are handled differently in snap-in
+            // platform,  we need to catch the log messages from worker thread and log
+            // them in main thread.
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageLog) {
+                const stringifiedArgs = (_a = message.payload) === null || _a === void 0 ? void 0 : _a.stringifiedArgs;
+                const level = (_b = message.payload) === null || _b === void 0 ? void 0 : _b.level;
+                const isSdkLog = (_d = (_c = message.payload) === null || _c === void 0 ? void 0 : _c.isSdkLog) !== null && _d !== void 0 ? _d : false;
+                this.logger.logFn(stringifiedArgs, level, isSdkLog);
+            }
+            // If worker sends a message that it has emitted an event, then set alreadyEmitted to true.
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
+                console.info('Worker has emitted message to ADaaS.');
+                this.alreadyEmitted = true;
+            }
+        });
+        // Log memory usage every 30 seconds
+        this.memoryMonitoringInterval = setInterval(() => {
+            try {
+                const memoryInfo = (0, helpers_2.getMemoryUsage)();
+                if (memoryInfo) {
+                    console.info(memoryInfo.formattedMessage);
+                }
+            }
+            catch (error) {
+                // If memory monitoring fails, log the warning and clear the interval to prevent further issues
+                console.warn('Memory monitoring failed, stopping logging of memory usage interval', error);
+                if (this.memoryMonitoringInterval) {
+                    clearInterval(this.memoryMonitoringInterval);
+                    this.memoryMonitoringInterval = undefined;
+                }
+            }
+        }, constants_1.MEMORY_LOG_INTERVAL);
+    }
+    clearTimeouts() {
+        if (this.softTimeoutTimer) {
+            clearTimeout(this.softTimeoutTimer);
+        }
+        if (this.hardTimeoutTimer) {
+            clearTimeout(this.hardTimeoutTimer);
+        }
+        if (this.memoryMonitoringInterval) {
+            clearInterval(this.memoryMonitoringInterval);
+        }
+    }
+    async exitFromMainThread() {
+        this.clearTimeouts();
+        // eslint-disable-next-line no-global-assign
+        console = this.originalConsole;
+        if (this.alreadyEmitted) {
+            this.resolve();
+            return;
+        }
+        this.alreadyEmitted = true;
+        const { eventType } = (0, spawn_helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
+        try {
+            await (0, control_protocol_1.emit)({
+                eventType,
+                event: this.event,
+                data: {
+                    error: {
+                        message: 'Worker exited the process without emitting an event. Check other logs for more information.',
+                    },
+                },
+            });
+            this.resolve();
+        }
+        catch (error) {
+            console.error('Error while emitting event.', (0, logger_1.serializeError)(error));
+        }
+    }
+}
+exports.Spawn = Spawn;

package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=worker-adapter.artifacts.test.d.ts.map

package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-adapter.artifacts.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts"],"names":[],"mappings":""}

package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js

@@ -0,0 +1,127 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const state_1 = require("../../state/state");
+const test_helpers_1 = require("../../tests/test-helpers");
+const types_1 = require("../../types");
+const worker_adapter_1 = require("./worker-adapter");
+// 1. Create a mock function for the method you want to override.
+const mockUpload = (itemType, objects) => {
+    return {
+        error: null,
+        artifact: {
+            id: `artifact-${itemType}-${Math.random().toString(36).substring(2, 15)}`,
+            item_type: itemType,
+            item_count: objects.length,
+        },
+    };
+};
+// 2. Mock the entire 'uploader' module.
+// The factory function () => { ... } returns the mock implementation.
+jest.mock('../../uploader/uploader', () => {
+    return {
+        // The mocked Uploader class
+        Uploader: jest.fn().mockImplementation(() => {
+            // The constructor of the mocked Uploader returns an object
+            // with the methods you want to control.
+            return {
+                upload: mockUpload,
+            };
+        }),
+    };
+});
+function checkArtifactOrder(artifacts, expectedOrder) {
+    let outerIndex = 0;
+    for (const artifact of artifacts) {
+        try {
+            // Always increase outer index. If items are out of order, the array will overflow and exception will be thrown
+            while (artifact.item_type != expectedOrder[outerIndex].itemType) {
+                outerIndex++;
+            }
+        }
+        catch (e) {
+            console.error('Error finding artifact type in repos:', e);
+            return false;
+        }
+    }
+    return true;
+}
+describe('Artifact ordering when artifacts overflow batch sizes in repositories', () => {
+    let testAdapter;
+    beforeEach(() => {
+        // Create a fresh adapter instance for this test to avoid mocking conflicts
+        const mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart });
+        const mockAdapterState = new state_1.State({
+            event: mockEvent,
+            initialState: { attachments: { completed: false } },
+        });
+        testAdapter = new worker_adapter_1.WorkerAdapter({
+            event: mockEvent,
+            adapterState: mockAdapterState,
+            options: {
+                batchSize: 50,
+            },
+        });
+    });
+    it('should maintain artifact ordering when repo ItemTypeA has items below batch size and repo ItemTypeB has items above batch size', async () => {
+        var _a, _b;
+        const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }];
+        // Initialize repos
+        testAdapter.initializeRepos(repos);
+        await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
+        await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(105)));
+        await testAdapter.uploadAllRepos();
+        const artifacts = testAdapter.artifacts;
+        expect(artifacts.length).toBe(4);
+        expect(checkArtifactOrder(artifacts, repos)).toBe(true);
+    });
+    it('should work with more than 2 repos', async () => {
+        var _a, _b, _c, _d;
+        const repos = [
+            { itemType: 'ItemTypeA' },
+            { itemType: 'ItemTypeB' },
+            { itemType: 'ItemTypeC' },
+            { itemType: 'ItemTypeD' },
+        ];
+        // Initialize repos
+        testAdapter.initializeRepos(repos);
+        await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
+        await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
+        await ((_c = testAdapter.getRepo('ItemTypeC')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(103)));
+        await ((_d = testAdapter.getRepo('ItemTypeD')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(104)));
+        await testAdapter.uploadAllRepos();
+        const artifacts = testAdapter.artifacts;
+        expect(artifacts.length).toBe(12);
+        expect(checkArtifactOrder(artifacts, repos)).toBe(true);
+    });
+    it('should maintain order with multiple pushes and uploads', async () => {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
+        const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }];
+        // Initialize repos
+        testAdapter.initializeRepos(repos);
+        await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(101)));
+        await ((_b = testAdapter.getRepo('ItemTypeB')) === null || _b === void 0 ? void 0 : _b.push((0, test_helpers_1.createItems)(102)));
+        await ((_c = testAdapter.getRepo('ItemTypeA')) === null || _c === void 0 ? void 0 : _c.push((0, test_helpers_1.createItems)(101)));
+        await ((_d = testAdapter.getRepo('ItemTypeB')) === null || _d === void 0 ? void 0 : _d.push((0, test_helpers_1.createItems)(102)));
+        await ((_e = testAdapter.getRepo('ItemTypeA')) === null || _e === void 0 ? void 0 : _e.upload());
+        await ((_f = testAdapter.getRepo('ItemTypeB')) === null || _f === void 0 ? void 0 : _f.upload());
+        await ((_g = testAdapter.getRepo('ItemTypeA')) === null || _g === void 0 ? void 0 : _g.push((0, test_helpers_1.createItems)(101)));
+        await ((_h = testAdapter.getRepo('ItemTypeB')) === null || _h === void 0 ? void 0 : _h.push((0, test_helpers_1.createItems)(102)));
+        await ((_j = testAdapter.getRepo('ItemTypeA')) === null || _j === void 0 ? void 0 : _j.push((0, test_helpers_1.createItems)(101)));
+        await ((_k = testAdapter.getRepo('ItemTypeB')) === null || _k === void 0 ? void 0 : _k.push((0, test_helpers_1.createItems)(102)));
+        await testAdapter.uploadAllRepos();
+        const artifacts = testAdapter.artifacts;
+        expect(artifacts.length).toBe(20);
+        expect(checkArtifactOrder(artifacts, repos)).toBe(true);
+    });
+    it('should not count artifacts if 0 items are pushed to the repo', async () => {
+        var _a;
+        const repos = [{ itemType: 'ItemTypeA' }];
+        // Initialize repos
+        testAdapter.initializeRepos(repos);
+        await ((_a = testAdapter.getRepo('ItemTypeA')) === null || _a === void 0 ? void 0 : _a.push([]));
+        await testAdapter.uploadAllRepos();
+        const artifacts = testAdapter.artifacts;
+        expect(artifacts.length).toBe(0);
+        expect(checkArtifactOrder(artifacts, repos)).toBe(true);
+    });
+});

package/dist/multithreading/worker-adapter/worker-adapter.d.ts

@@ -0,0 +1,91 @@
+import { Mappers } from '../../mappers/mappers';
+import { Repo } from '../../repo/repo';
+import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
+import { AdapterState } from '../../state/state.interfaces';
+import { AirdropEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, StreamAttachmentsReturnType } from '../../types/extraction';
+import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../../types/loading';
+import { WorkerAdapterInterface, WorkerAdapterOptions } from '../../types/workers';
+import { Artifact } from '../../uploader/uploader.interfaces';
+export declare function createWorkerAdapter<ConnectorState>({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>): WorkerAdapter<ConnectorState>;
+/**
+ * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in
+ * as parameter in processTask and onTimeout functions. The class provides
+ * utilities to emit control events to the platform, update the state of the connector,
+ * and upload artifacts to the platform.
+ * @class WorkerAdapter
+ * @constructor
+ * @param options - The options to create a new instance of WorkerAdapter class
+ * @param event - The event object received from the platform
+ * @param initialState - The initial state of the adapter
+ * @param isLocalDevelopment - A flag to indicate if the adapter is being used in local development
+ * @param workerPath - The path to the worker file
+ *
+ * @public
+ */
+export declare class WorkerAdapter<ConnectorState> {
+    readonly event: AirdropEvent;
+    readonly options?: WorkerAdapterOptions;
+    isTimeout: boolean;
+    private adapterState;
+    private _artifacts;
+    private hasWorkerEmitted;
+    private repos;
+    private loaderReports;
+    private _processedFiles;
+    private _mappers;
+    private uploader;
+    constructor({ event, adapterState, options, }: WorkerAdapterInterface<ConnectorState>);
+    get state(): AdapterState<ConnectorState>;
+    set state(value: AdapterState<ConnectorState>);
+    get reports(): LoaderReport[];
+    get processedFiles(): string[];
+    get mappers(): Mappers;
+    initializeRepos(repos: RepoInterface[]): void;
+    getRepo(itemType: string): Repo | undefined;
+    postState(): Promise<void>;
+    get artifacts(): Artifact[];
+    set artifacts(artifacts: Artifact[]);
+    /**
+     *  Emits an event to the platform.
+     *
+     * @param newEventType - The event type to be emitted
+     * @param data - The data to be sent with the event
+     */
+    emit(newEventType: ExtractorEventType | LoaderEventType, data?: EventData): Promise<void>;
+    uploadAllRepos(): Promise<void>;
+    handleTimeout(): void;
+    loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
+    getLoaderBatches({ supportedItemTypes, }: {
+        supportedItemTypes: string[];
+    }): Promise<FileToLoad[]>;
+    loadAttachments({ create, }: {
+        create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
+    }): Promise<LoadItemTypesResponse>;
+    loadItem({ item, itemTypeToLoad, }: {
+        item: ExternalSystemItem;
+        itemTypeToLoad: ItemTypeToLoad;
+    }): Promise<LoadItemResponse>;
+    processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
+    /**
+     * Destroys a stream to prevent memory leaks.
+     * @param httpStream - The axios response stream to destroy
+     */
+    private destroyHttpStream;
+    loadAttachment({ item, create, }: {
+        item: ExternalSystemAttachment;
+        create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
+    }): Promise<LoadItemResponse>;
+    /**
+     * Streams the attachments to the DevRev platform.
+     * The attachments are streamed to the platform and the artifact information is returned.
+     * @param params - The parameters to stream the attachments
+     * @returns The response object containing the ssorAttachment artifact information
+     * or error information if there was an error
+     */
+    streamAttachments<NewBatch>({ stream, processors, batchSize, }: {
+        stream: ExternalSystemAttachmentStreamingFunction;
+        processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
+        batchSize?: number;
+    }): Promise<StreamAttachmentsReturnType>;
+}
+//# sourceMappingURL=worker-adapter.d.ts.map

package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAClB,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAG9E,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,EAClD,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC,GAAG,aAAa,CAAC,cAAc,CAAC,CAMxE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAa,CAAC,cAAc;IACvC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,KAAK,CAAc;IAG3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,QAAQ,CAAW;gBAEf,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE,sBAAsB,CAAC,cAAc,CAAC;IAqBzC,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAI5C;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAuBtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAarC,SAAS;IAMf,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;IAED;;;;;OAKG;IACG,IAAI,CACR,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;IAqFV,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC,aAAa;IAIP,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAqHnD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA6E5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkMvB,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAoGvC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAcnB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAqD7B;;;;;;OAMG;IACG,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAyHzC"}

package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts

@@ -0,0 +1,22 @@
+import { FileToLoad, LoaderReport, StatsFileObject } from '../../types/loading';
+/**
+ * Gets the files to load for the loader.
+ * @param {string[]} supportedItemTypes - The supported item types
+ * @param {StatsFileObject[]} statsFile - The stats file
+ * @returns {FileToLoad[]} The files to load
+ */
+export declare function getFilesToLoad({ supportedItemTypes, statsFile, }: {
+    supportedItemTypes: string[];
+    statsFile: StatsFileObject[];
+}): FileToLoad[];
+/**
+ * Adds a report to the loader report.
+ * @param {LoaderReport[]} loaderReports - The loader reports
+ * @param {LoaderReport} report - The report to add
+ * @returns {LoaderReport[]} The updated loader reports
+ */
+export declare function addReportToLoaderReport({ loaderReports, report, }: {
+    loaderReports: LoaderReport[];
+    report: LoaderReport;
+}): LoaderReport[];
+//# sourceMappingURL=worker-adapter.helpers.d.ts.map

package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}

package/dist/multithreading/worker-adapter/worker-adapter.helpers.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFilesToLoad = getFilesToLoad;
+exports.addReportToLoaderReport = addReportToLoaderReport;
+const loading_1 = require("../../types/loading");
+/**
+ * Gets the files to load for the loader.
+ * @param {string[]} supportedItemTypes - The supported item types
+ * @param {StatsFileObject[]} statsFile - The stats file
+ * @returns {FileToLoad[]} The files to load
+ */
+function getFilesToLoad({ supportedItemTypes, statsFile, }) {
+    const filesToLoad = [];
+    if (supportedItemTypes.length === 0 || statsFile.length === 0) {
+        return [];
+    }
+    const filteredStatsFile = statsFile.filter((file) => supportedItemTypes.includes(file.item_type));
+    const orderedFiles = filteredStatsFile.sort((a, b) => {
+        const aIndex = supportedItemTypes.indexOf(a.item_type);
+        const bIndex = supportedItemTypes.indexOf(b.item_type);
+        return aIndex - bIndex;
+    });
+    for (const file of orderedFiles) {
+        filesToLoad.push({
+            id: file.id,
+            file_name: file.file_name,
+            itemType: file.item_type,
+            count: parseInt(file.count),
+            completed: false,
+            lineToProcess: 0,
+        });
+    }
+    return filesToLoad;
+}
+/**
+ * Adds a report to the loader report.
+ * @param {LoaderReport[]} loaderReports - The loader reports
+ * @param {LoaderReport} report - The report to add
+ * @returns {LoaderReport[]} The updated loader reports
+ */
+function addReportToLoaderReport({ loaderReports, report, }) {
+    const existingReport = loaderReports.find((loaderReport) => loaderReport.item_type === report.item_type);
+    if (existingReport) {
+        existingReport[loading_1.ActionType.CREATED] = existingReport[loading_1.ActionType.CREATED]
+            ? report[loading_1.ActionType.CREATED]
+                ? existingReport[loading_1.ActionType.CREATED] + report[loading_1.ActionType.CREATED]
+                : existingReport[loading_1.ActionType.CREATED]
+            : report[loading_1.ActionType.CREATED];
+        existingReport[loading_1.ActionType.UPDATED] = existingReport[loading_1.ActionType.UPDATED]
+            ? report[loading_1.ActionType.UPDATED]
+                ? existingReport[loading_1.ActionType.UPDATED] + report[loading_1.ActionType.UPDATED]
+                : existingReport[loading_1.ActionType.UPDATED]
+            : report[loading_1.ActionType.UPDATED];
+        existingReport[loading_1.ActionType.FAILED] = existingReport[loading_1.ActionType.FAILED]
+            ? report[loading_1.ActionType.FAILED]
+                ? existingReport[loading_1.ActionType.FAILED] + report[loading_1.ActionType.FAILED]
+                : existingReport[loading_1.ActionType.FAILED]
+            : report[loading_1.ActionType.FAILED];
+    }
+    else {
+        loaderReports.push(report);
+    }
+    return loaderReports;
+}

package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=worker-adapter.helpers.test.d.ts.map

package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-adapter.helpers.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.helpers.test.ts"],"names":[],"mappings":""}