@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.3

package/dist/multithreading/spawn/spawn.js

@@ -8,7 +8,6 @@ exports.spawn = spawn;
 const yargs_1 = __importDefault(require("yargs"));
 const helpers_1 = require("yargs/helpers");
 const control_protocol_1 = require("../../common/control-protocol");
-const event_type_translation_1 = require("../../common/event-type-translation");
 const helpers_2 = require("../../common/helpers");
 const logger_1 = require("../../logger/logger");
 const extraction_1 = require("../../types/extraction");
@@ -50,42 +49,30 @@ function getWorkerPath({ event, workerBasePath, }) {
  * The class provides utilities to emit control events to the platform and exit the worker gracefully.
  * In case of lambda timeout, the class emits a lambda timeout event to the platform.
  * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class
- * @param {AirdropEvent} options.event - The event object received from the platform
+ * @param {AirSyncEvent} options.event - The event object received from the platform
  * @param {object} options.initialState - The initial state of the adapter
- * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath
  * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname`
  * @returns {Promise<Spawn>} - A new instance of Spawn class
  */
-async function spawn({ event, initialState, workerPath, initialDomainMapping, options, baseWorkerPath, }) {
-    // Translate incoming event type for backwards compatibility. This allows the
-    // SDK to accept both old and new event type formats. Then update the event with the translated event type.
-    const originalEventType = event.payload.event_type;
-    const translatedEventType = (0, event_type_translation_1.translateIncomingEventType)(event.payload.event_type);
-    event.payload.event_type = translatedEventType;
+async function spawn({ event, initialState, initialDomainMapping, options, baseWorkerPath, }) {
     // Read the command line arguments to check if the local flag is passed.
     const argv = await (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv)).argv;
-    if (argv._.includes('local')) {
+    if (argv._.includes('local') || argv.local) {
         options = Object.assign(Object.assign({}, (options || {})), { isLocalDevelopment: true });
     }
     const originalConsole = console;
     // eslint-disable-next-line no-global-assign
     console = new logger_1.Logger({ event, options });
-    if (translatedEventType !== originalEventType) {
-        console.log(`Event type translated from ${originalEventType} to ${translatedEventType}.`);
-    }
     if (options === null || options === void 0 ? void 0 : options.isLocalDevelopment) {
         console.log('Snap-in is running in local development mode.');
     }
+    const eventType = event.payload.event_type;
     let script = null;
-    if (workerPath != null) {
-        script = workerPath;
-    }
-    else if (baseWorkerPath != null &&
+    if (baseWorkerPath != null &&
         (options === null || options === void 0 ? void 0 : options.workerPathOverrides) != null &&
-        options.workerPathOverrides[translatedEventType] != null) {
+        options.workerPathOverrides[eventType] != null) {
         script =
-            baseWorkerPath +
-                options.workerPathOverrides[translatedEventType];
+            baseWorkerPath + options.workerPathOverrides[eventType];
     }
     else {
         script = getWorkerPath({
@@ -189,21 +176,25 @@ class Spawn {
             }
         });
         worker.on(workers_1.WorkerEvent.WorkerMessage, (message) => {
-            var _a, _b, _c, _d;
+            var _a, _b, _c, _d, _e;
             // Since logs from the worker thread are handled differently in snap-in
             // platform,  we need to catch the log messages from worker thread and log
             // them in main thread.
             if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageLog) {
                 const stringifiedArgs = (_a = message.payload) === null || _a === void 0 ? void 0 : _a.stringifiedArgs;
                 const level = (_b = message.payload) === null || _b === void 0 ? void 0 : _b.level;
-                const isSdkLog = (_d = (_c = message.payload) === null || _c === void 0 ? void 0 : _c.isSdkLog) !== null && _d !== void 0 ? _d : false;
+                const isSdkLog = (_d = (_c = message.payload) === null || _c === void 0 ? void 0 : _c.isSdkLog) !== null && _d !== void 0 ? _d : true;
                 this.logger.logFn(stringifiedArgs, level, isSdkLog);
             }
             // If worker sends a message that it has emitted an event, then set alreadyEmitted to true.
             if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageEmitted) {
-                console.info('Worker has emitted message to ADaaS.');
+                console.info('Worker has emitted message to the AirSync platform.');
                 this.alreadyEmitted = true;
             }
+            // If worker sends a failure message before exiting, capture it for use in the error event.
+            if ((message === null || message === void 0 ? void 0 : message.subject) === workers_1.WorkerMessageSubject.WorkerMessageFailed) {
+                this.workerFailedMessage = (_e = message.payload) === null || _e === void 0 ? void 0 : _e.message;
+            }
         });
         // Log memory usage every 30 seconds
         this.memoryMonitoringInterval = setInterval(() => {
@@ -250,15 +241,19 @@ class Spawn {
                 event: this.event,
                 data: {
                     error: {
-                        message: 'Worker exited the process without emitting an event. Check other logs for more information.',
+                        message: `Worker exited without emitting event. ${this.workerFailedMessage
+                            ? `Error: ${this.workerFailedMessage}`
+                            : 'Check the logs for more information.'}`,
                     },
                 },
             });
-            this.resolve();
         }
         catch (error) {
             console.error('Error while emitting event.', (0, logger_1.serializeError)(error));
         }
+        finally {
+            this.resolve();
+        }
     }
 }
 exports.Spawn = Spawn;

package/dist/multithreading/spawn/spawn.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=spawn.test.d.ts.map

package/dist/multithreading/spawn/spawn.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spawn.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/spawn/spawn.test.ts"],"names":[],"mappings":""}

package/dist/multithreading/spawn/spawn.test.js

@@ -0,0 +1,232 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const events_1 = require("events");
+const constants_1 = require("../../common/constants");
+const extraction_1 = require("../../types/extraction");
+const workers_1 = require("../../types/workers");
+const test_utils_1 = require("../../common/test-utils");
+// ---------------------------------------------------------------------------
+// Mocks
+// ---------------------------------------------------------------------------
+jest.mock('../create-worker', () => ({
+    createWorker: jest.fn(),
+}));
+jest.mock('../../common/control-protocol', () => ({
+    emit: jest.fn().mockResolvedValue({}),
+}));
+jest.mock('../../logger/logger', () => ({
+    Logger: jest.fn().mockImplementation(() => ({
+        log: jest.fn(),
+        warn: jest.fn(),
+        error: jest.fn(),
+        info: jest.fn(),
+        logFn: jest.fn(),
+    })),
+    serializeError: jest.fn((e) => String(e)),
+}));
+jest.mock('../../common/helpers', () => ({
+    getLibraryVersion: jest.fn().mockReturnValue('1.0.0-test'),
+    getMemoryUsage: jest.fn().mockReturnValue({
+        formattedMessage: 'Memory: RSS 100/512MB (19.53%) [...]',
+        rssUsedMB: '100.00',
+        rssUsedPercent: '19.53%',
+        heapUsedPercent: '30.00%',
+        externalMB: '10.00',
+        arrayBuffersMB: '5.00',
+    }),
+    sleep: jest.fn(),
+    truncateFilename: jest.fn((f) => f),
+    truncateMessage: jest.fn((m) => m),
+}));
+// ---------------------------------------------------------------------------
+// Imports after mocks
+// ---------------------------------------------------------------------------
+const spawn_1 = require("./spawn");
+const create_worker_1 = require("../create-worker");
+const control_protocol_1 = require("../../common/control-protocol");
+const helpers_1 = require("../../common/helpers");
+// ---------------------------------------------------------------------------
+// Factory for a fake worker (EventEmitter with postMessage + terminate)
+// ---------------------------------------------------------------------------
+function makeWorker() {
+    const w = new events_1.EventEmitter();
+    w.postMessage = jest.fn();
+    w.terminate = jest.fn().mockResolvedValue(0);
+    return w;
+}
+// ---------------------------------------------------------------------------
+// Helper: instantiate Spawn directly, injecting a mock logger via console swap
+// ---------------------------------------------------------------------------
+function buildSpawn(overrides) {
+    var _a;
+    const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
+        payload: { event_type: extraction_1.EventType.StartExtractingData },
+    });
+    const mockLogger = {
+        log: jest.fn(),
+        warn: jest.fn(),
+        error: jest.fn(),
+        info: jest.fn(),
+        logFn: jest.fn(),
+    };
+    const originalConsole = console;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    global.console = mockLogger;
+    const s = new spawn_1.Spawn({
+        event,
+        worker: overrides.worker,
+        options: overrides.options,
+        resolve: (_a = overrides.resolve) !== null && _a !== void 0 ? _a : jest.fn(),
+        originalConsole: originalConsole,
+    });
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    global.console = originalConsole;
+    return s;
+}
+// ---------------------------------------------------------------------------
+// spawn() factory tests
+// ---------------------------------------------------------------------------
+describe('spawn() factory', () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+        jest.useFakeTimers({ legacyFakeTimers: true });
+    });
+    afterEach(() => {
+        jest.useRealTimers();
+    });
+    it('should emit a no-script event and NOT spawn a worker for an unknown event type', async () => {
+        // Arrange
+        const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
+            payload: { event_type: extraction_1.EventType.UnknownEventType },
+        });
+        // Act
+        await (0, spawn_1.spawn)({ event, initialState: {} });
+        // Assert: no worker process should be started, but the platform should
+        // still receive a terminal event (so the run doesn't hang).
+        expect(create_worker_1.createWorker).not.toHaveBeenCalled();
+        expect(control_protocol_1.emit).toHaveBeenCalledWith(expect.objectContaining({ event, eventType: expect.any(String) }));
+    });
+    it('should reject the returned promise when createWorker throws', async () => {
+        // Arrange
+        create_worker_1.createWorker.mockRejectedValue(new Error('worker boom'));
+        const event = (0, test_utils_1.createMockEvent)('http://localhost:0', {
+            payload: { event_type: extraction_1.EventType.StartExtractingData },
+        });
+        // Act & Assert
+        await expect((0, spawn_1.spawn)({
+            event,
+            initialState: {},
+            baseWorkerPath: '/fake',
+            options: {
+                workerPathOverrides: {
+                    [extraction_1.EventType.StartExtractingData]: '/path.js',
+                },
+            },
+        })).rejects.toThrow('worker boom');
+    });
+});
+// ---------------------------------------------------------------------------
+// Spawn class — lifecycle tests
+// ---------------------------------------------------------------------------
+describe('Spawn class', () => {
+    let worker;
+    let resolveMock;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        jest.useFakeTimers({ legacyFakeTimers: true });
+        worker = makeWorker();
+        resolveMock = jest.fn();
+    });
+    afterEach(() => {
+        jest.useRealTimers();
+    });
+    // -------------------------------------------------------------------------
+    // WorkerMessageFailed captured and propagated in error emit
+    // -------------------------------------------------------------------------
+    it('should include the WorkerMessageFailed reason in the error event emitted to the platform', async () => {
+        // Arrange
+        buildSpawn({ worker, resolve: resolveMock });
+        // Act
+        worker.emit(workers_1.WorkerEvent.WorkerMessage, {
+            subject: workers_1.WorkerMessageSubject.WorkerMessageFailed,
+            payload: { message: 'connector exploded' },
+        });
+        worker.emit(workers_1.WorkerEvent.WorkerExit, 1);
+        await Promise.resolve();
+        await Promise.resolve();
+        // Assert: the platform receives an error event whose message contains
+        // the reason sent by the worker — this is what operators see in the run log.
+        expect(control_protocol_1.emit).toHaveBeenCalledWith(expect.objectContaining({
+            data: expect.objectContaining({
+                error: expect.objectContaining({
+                    message: expect.stringContaining('connector exploded'),
+                }),
+            }),
+        }));
+        expect(resolveMock).toHaveBeenCalled();
+    });
+    it('should emit a data extraction error when the worker exits without ever emitting', async () => {
+        // Arrange
+        buildSpawn({ worker, resolve: resolveMock });
+        // Act
+        worker.emit(workers_1.WorkerEvent.WorkerExit, 1);
+        await Promise.resolve();
+        await Promise.resolve();
+        // Assert: a silent worker exit must surface a DataExtractionError to the
+        // platform with a non-empty message. The event type is the upstream
+        // contract; the exact message wording is implementation detail.
+        expect(control_protocol_1.emit).toHaveBeenCalledTimes(1);
+        const [emitted] = control_protocol_1.emit.mock.calls[0];
+        expect(emitted.eventType).toBe(extraction_1.ExtractorEventType.DataExtractionError);
+        expect(emitted.data.error.message).toEqual(expect.any(String));
+        expect(emitted.data.error.message.length).toBeGreaterThan(0);
+        expect(resolveMock).toHaveBeenCalled();
+    });
+    // Soft-timeout and hard-timeout timer behavior is covered end-to-end by
+    // src/tests/timeout-handling/ (real workers, real timers). Unit tests with
+    // fake timers only re-asserted the mocked setTimeout call and gave no signal.
+    // -------------------------------------------------------------------------
+    // Memory monitoring — error clears the interval
+    // -------------------------------------------------------------------------
+    it('should clear the memory monitoring interval when getMemoryUsage throws to prevent repeated crashes', async () => {
+        // Arrange
+        helpers_1.getMemoryUsage.mockImplementation(() => {
+            throw new Error('OOM');
+        });
+        const clearIntervalSpy = jest.spyOn(global, 'clearInterval');
+        buildSpawn({ worker, resolve: resolveMock });
+        // Act
+        jest.advanceTimersByTime(30001); // MEMORY_LOG_INTERVAL = 30 s
+        await Promise.resolve();
+        // Assert
+        expect(clearIntervalSpy).toHaveBeenCalled();
+    });
+    // -------------------------------------------------------------------------
+    // Soft-timeout race: worker emits AFTER softTimeoutSent — no double-emit
+    // -------------------------------------------------------------------------
+    it('should NOT emit an error when the worker successfully emits just after receiving the soft-timeout signal', async () => {
+        // Arrange
+        buildSpawn({ worker, resolve: resolveMock });
+        // Act: trigger soft timeout — sends WorkerMessageExit to the worker
+        jest.advanceTimersByTime(constants_1.DEFAULT_LAMBDA_TIMEOUT + 1);
+        await Promise.resolve();
+        // Confirm the soft-timeout path actually fired — guards against a silent
+        // pass if the timeout constant changes and the timer never runs.
+        expect(worker.postMessage).toHaveBeenCalledWith(expect.objectContaining({
+            subject: workers_1.WorkerMessageSubject.WorkerMessageExit,
+        }));
+        // Worker responds: emits its event successfully, then exits normally
+        worker.emit(workers_1.WorkerEvent.WorkerMessage, {
+            subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
+        });
+        worker.emit(workers_1.WorkerEvent.WorkerExit, 0);
+        // The exit handler defers via setImmediate when softTimeoutSent=true
+        jest.runAllImmediates();
+        await Promise.resolve();
+        await Promise.resolve();
+        // Assert: no error should reach the platform — the worker completed its job
+        const errorEmits = control_protocol_1.emit.mock.calls.filter((call) => { var _a, _b; return (_b = (_a = call[0]) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.error; });
+        expect(errorEmits).toHaveLength(0);
+        expect(resolveMock).toHaveBeenCalled();
+    });
+});

package/dist/repo/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAG3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;gBAEzB,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAqCxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAuC5C"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;gBAEzB,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAqCxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAyC5C"}

package/dist/repo/repo.interfaces.d.ts

@@ -1,5 +1,5 @@
 import { Artifact } from '../uploader/uploader.interfaces';
-import { AirdropEvent } from '../types/extraction';
+import { AirSyncEvent } from '../types/extraction';
 import { WorkerAdapterOptions } from '../types/workers';
 /**
  * RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data.
@@ -7,12 +7,13 @@ import { WorkerAdapterOptions } from '../types/workers';
 export interface RepoInterface {
     itemType: string;
     normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
+    overridenOptions?: WorkerAdapterOptions;
 }
 /**
  * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo.
  */
 export interface RepoFactoryInterface {
-    event: AirdropEvent;
+    event: AirSyncEvent;
     itemType: string;
     normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
     onUpload: (artifact: Artifact) => void;
@@ -37,6 +38,7 @@ export interface NormalizedAttachment {
     parent_id: string;
     author_id?: string;
     inline?: boolean;
+    content_type?: string;
     grand_parent_id?: number | string;
 }
 /**

package/dist/repo/repo.interfaces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;CACvE;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IAIjB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
+{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IAItB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}

package/dist/repo/repo.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Repo = void 0;
 const constants_1 = require("../common/constants");
 const uploader_1 = require("../uploader/uploader");
+const logger_context_1 = require("../logger/logger.context");
 class Repo {
     constructor({ event, itemType, normalize, onUpload, options, }) {
         this.items = [];
@@ -46,9 +47,9 @@ class Repo {
         }
         // Normalize items if needed
         if (this.normalize &&
-            this.itemType != constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA &&
-            this.itemType != constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT) {
-            recordsToPush = items.map((item) => this.normalize(item));
+            this.itemType != constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
+            this.itemType != constants_1.SSOR_ATTACHMENT) {
+            recordsToPush = (0, logger_context_1.runWithUserLogContext)(() => items.map((item) => this.normalize(item)));
         }
         else {
             recordsToPush = items;

package/dist/repo/repo.test.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const constants_1 = require("../common/constants");
 const test_helpers_1 = require("../tests/test-helpers");
+const jest_setup_1 = require("../tests/jest.setup");
+const test_utils_1 = require("../common/test-utils");
 const types_1 = require("../types");
 const repo_1 = require("./repo");
 jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
@@ -11,7 +13,9 @@ describe(repo_1.Repo.name, () => {
     beforeEach(() => {
         normalize = jest.fn();
         repo = new repo_1.Repo({
-            event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
+            event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                payload: { event_type: types_1.EventType.StartExtractingData },
+            }),
             itemType: 'test_item_type',
             normalize,
             onUpload: jest.fn(),
@@ -21,77 +25,111 @@ describe(repo_1.Repo.name, () => {
         jest.clearAllMocks();
     });
     it('should normalize and push items when array contains items', async () => {
+        // Arrange
         const items = (0, test_helpers_1.createItems)(10);
+        // Act
         await repo.push(items);
+        // Assert
         expect(normalize).toHaveBeenCalledTimes(10);
         const normalizedItems = items.map((item) => (0, test_helpers_1.normalizeItem)(item));
         expect(repo.getItems()).toEqual(normalizedItems);
     });
     it('should not normalize items when normalize function is not provided', async () => {
+        // Arrange
         repo = new repo_1.Repo({
-            event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
+            event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                payload: { event_type: types_1.EventType.StartExtractingData },
+            }),
             itemType: 'test_item_type',
             onUpload: jest.fn(),
             options: {},
         });
         const items = (0, test_helpers_1.createItems)(10);
+        // Act
         await repo.push(items);
+        // Assert
         expect(normalize).not.toHaveBeenCalled();
     });
     it('[edge] should not push items when items array is empty', async () => {
+        // Act
         await repo.push([]);
+        // Assert
         expect(repo.getItems()).toEqual([]);
     });
     it('should not normalize items when item type is external_domain_metadata', async () => {
+        // Arrange
         repo = new repo_1.Repo({
-            event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
-            itemType: constants_1.AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA,
+            event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                payload: { event_type: types_1.EventType.StartExtractingData },
+            }),
+            itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA,
             normalize,
             onUpload: jest.fn(),
             options: {},
         });
         const items = (0, test_helpers_1.createItems)(10);
+        // Act
         await repo.push(items);
+        // Assert
         expect(normalize).not.toHaveBeenCalled();
     });
     it('should not normalize items when item type is ssor_attachment', async () => {
+        // Arrange
         repo = new repo_1.Repo({
-            event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
-            itemType: constants_1.AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT,
+            event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                payload: { event_type: types_1.EventType.StartExtractingData },
+            }),
+            itemType: constants_1.SSOR_ATTACHMENT,
             normalize,
             onUpload: jest.fn(),
             options: {},
         });
         const items = (0, test_helpers_1.createItems)(10);
+        // Act
         await repo.push(items);
+        // Assert
         expect(normalize).not.toHaveBeenCalled();
     });
     it('should leave 5 items in the items array after pushing 2005 items with batch size of 2000', async () => {
+        // Arrange
         const items = (0, test_helpers_1.createItems)(2005);
+        // Act
         await repo.push(items);
+        // Assert
         expect(repo.getItems().length).toBe(5);
     });
     it('should normalize all items when pushing 4005 items with batch size of 2000', async () => {
+        // Arrange
         const items = (0, test_helpers_1.createItems)(4005);
+        // Act
         await repo.push(items);
+        // Assert
         expect(normalize).toHaveBeenCalledTimes(4005);
     });
     it('should upload 2 batches when pushing 4005 items with batch size of 2000', async () => {
+        // Arrange
         const uploadSpy = jest.spyOn(repo, 'upload');
         const items = (0, test_helpers_1.createItems)(4005);
+        // Act
         await repo.push(items);
+        // Assert
         expect(uploadSpy).toHaveBeenCalledTimes(2);
         uploadSpy.mockRestore();
     });
     it('should leave 5 items in array after pushing 4005 items with batch size of 2000', async () => {
+        // Arrange
         const items = (0, test_helpers_1.createItems)(4005);
+        // Act
         await repo.push(items);
+        // Assert
         expect(repo.getItems().length).toBe(5);
     });
     describe('should take batch size into account', () => {
         beforeEach(() => {
             repo = new repo_1.Repo({
-                event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
+                event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                    payload: { event_type: types_1.EventType.StartExtractingData },
+                }),
                 itemType: 'test_item_type',
                 normalize,
                 onUpload: jest.fn(),
@@ -101,30 +139,45 @@ describe(repo_1.Repo.name, () => {
             });
         });
         it('should empty the items array after pushing 50 items with batch size of 50', async () => {
+            // Arrange
             const items = (0, test_helpers_1.createItems)(50);
+            // Act
             await repo.push(items);
+            // Assert
             expect(repo.getItems()).toEqual([]);
         });
         it('should leave 5 items in the items array after pushing 205 items with batch size of 50', async () => {
+            // Arrange
             const items = (0, test_helpers_1.createItems)(205);
+            // Act
             await repo.push(items);
+            // Assert
             expect(repo.getItems().length).toBe(5);
         });
         it('should normalize all items when pushing 205 items with batch size of 50', async () => {
+            // Arrange
             const items = (0, test_helpers_1.createItems)(205);
+            // Act
             await repo.push(items);
+            // Assert
             expect(normalize).toHaveBeenCalledTimes(205);
         });
         it('should upload 4 batches when pushing 205 items with batch size of 50', async () => {
+            // Arrange
             const uploadSpy = jest.spyOn(repo, 'upload');
             const items = (0, test_helpers_1.createItems)(205);
+            // Act
             await repo.push(items);
+            // Assert
             expect(uploadSpy).toHaveBeenCalledTimes(4);
             uploadSpy.mockRestore();
         });
         it('should leave 5 items in array after pushing 205 items with batch size of 50', async () => {
+            // Arrange
             const items = (0, test_helpers_1.createItems)(205);
+            // Act
             await repo.push(items);
+            // Assert
             expect(repo.getItems().length).toBe(5);
         });
     });