@devrev/ts-adaas 1.20.2-beta.4 → 1.20.2-beta.6

package/dist/multithreading/worker-adapter/worker-adapter.js

@@ -22,6 +22,7 @@ const workers_1 = require("../../types/workers");
 const uploader_1 = require("../../uploader/uploader");
 const event_type_translation_1 = require("../../common/event-type-translation");
 const helpers_1 = require("../../common/helpers");
+const spawn_helpers_1 = require("../spawn/spawn.helpers");
 function createWorkerAdapter({ event, adapterState, options, }) {
     return new WorkerAdapter({
         event,
@@ -159,93 +160,49 @@ class WorkerAdapter {
      */
     async emit(newEventType, data) {
         return (0, logger_context_1.runWithSdkLogContext)(async () => {
-            var _a, _b;
+            var _a;
             newEventType = (0, event_type_translation_1.translateOutgoingEventType)(newEventType);
             if (this.hasWorkerEmitted) {
                 console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
                 return;
             }
-            // If the event is ExternalSyncUnitExtractionDone, upload external sync units via a Repo before emitting
-            // TODO: Remove in v2.0.0
-            if (newEventType === extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone &&
-                (data === null || data === void 0 ? void 0 : data.external_sync_units) &&
-                data.external_sync_units.length > 0) {
-                console.log(`Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.`);
-                this.initializeRepos([
-                    {
-                        itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
-                        overridenOptions: {
-                            batchSize: 25000,
-                            skipConfirmation: true,
-                        },
-                    },
-                ]);
-                await ((_a = this.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)) === null || _a === void 0 ? void 0 : _a.push(data.external_sync_units));
-                // Remove inline external_sync_units from data to avoid SQS size issues
-                delete data.external_sync_units;
-            }
-            // Upload all repos before emitting the event
-            console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
+            let eventPayload;
             try {
-                await this.uploadAllRepos();
+                // If the event is ExternalSyncUnitExtractionDone, upload external sync units via a Repo before emitting.
+                // TODO: Remove in v2.0.0
+                if (newEventType === extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone &&
+                    (data === null || data === void 0 ? void 0 : data.external_sync_units) &&
+                    data.external_sync_units.length > 0) {
+                    console.log(`Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.`);
+                    this.initializeRepos([
+                        {
+                            itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
+                            overridenOptions: {
+                                batchSize: 25000,
+                                skipConfirmation: true,
+                            },
+                        },
+                    ]);
+                    await ((_a = this.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)) === null || _a === void 0 ? void 0 : _a.push(data.external_sync_units));
+                    // Remove inline external_sync_units from data to avoid SQS size issues
+                    delete data.external_sync_units;
+                }
+                await this.beforeEmit(newEventType);
+                eventPayload = this.buildEmitPayload(newEventType, data);
             }
             catch (error) {
-                console.error('Error while uploading repos', error);
-                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
-                this.hasWorkerEmitted = true;
+                console.error('Error while preparing event for emission.', (0, logger_1.serializeError)(error));
+                await this.emitError(error);
                 return;
             }
-            // If the extraction is done, we want to save the timestamp of the last successful sync
-            if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
-                console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
-                this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
-                this.state.lastSyncStarted = '';
-                // Clear pending extraction boundaries now that the cycle is complete
-                this.state.pendingWorkersOldest = '';
-                this.state.pendingWorkersNewest = '';
-                // Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
-                // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
-                const extractionStart = this.event.payload.event_context.extract_from;
-                const extractionEnd = this.event.payload.event_context.extract_to;
-                if (extractionStart &&
-                    (!this.state.workersOldest ||
-                        extractionStart < this.state.workersOldest)) {
-                    console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
-                    this.state.workersOldest = extractionStart;
-                }
-                if (extractionEnd &&
-                    (!this.state.workersNewest ||
-                        extractionEnd > this.state.workersNewest)) {
-                    console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
-                    this.state.workersNewest = extractionEnd;
-                }
-            }
-            // We want to save the state every time we emit an event, except for the start and delete events
-            if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
-                console.log(`Saving state before emitting event with event type: ${newEventType}.`);
-                try {
-                    await this.adapterState.postState(this.state);
-                }
-                catch (error) {
-                    console.error('Error while posting state', error);
-                    node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
-                    this.hasWorkerEmitted = true;
-                    return;
-                }
-            }
             try {
-                // Always prune error messages to make them shorter before emit
-                if ((_b = data === null || data === void 0 ? void 0 : data.error) === null || _b === void 0 ? void 0 : _b.message) {
-                    data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
-                }
                 const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
-                const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
                 const progressData = {};
                 if (isExtractionEvent &&
-                    (newEventType == extraction_1.ExtractorEventType.DataExtractionDone ||
-                        newEventType == extraction_1.ExtractorEventType.DataExtractionProgress ||
-                        newEventType == extraction_1.ExtractorEventType.AttachmentExtractionDone ||
-                        newEventType == extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
+                    (newEventType === extraction_1.ExtractorEventType.DataExtractionDone ||
+                        newEventType === extraction_1.ExtractorEventType.DataExtractionProgress ||
+                        newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone ||
+                        newEventType === extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
                     const repo = this.lastExtractedItemType
                         ? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
                         : undefined;
@@ -260,9 +217,7 @@ class WorkerAdapter {
                 await (0, control_protocol_1.emit)({
                     eventType: newEventType,
                     event: this.event,
-                    data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
-                        ? { reports: this.reports, processed_files: this.processedFiles }
-                        : {})),
+                    data: eventPayload,
                     worker_metadata: Object.assign({}, progressData),
                 });
                 const message = {
@@ -280,14 +235,111 @@ class WorkerAdapter {
             }
         });
     }
+    /**
+     * Emits a phase-specific error after task or pre-emit execution fails. This
+     * path avoids flushing repos again, so artifacts uploaded before the failure
+     * are still reported without retrying the failed batch.
+     */
+    async emitError(error) {
+        return (0, logger_context_1.runWithSdkLogContext)(async () => {
+            if (this.hasWorkerEmitted) {
+                return;
+            }
+            // Include artifacts uploaded before a failure so they are not orphaned
+            // and can be associated with the emitted phase error.
+            for (const repo of this.repos) {
+                this.artifacts = [...this.artifacts, ...repo.uploadedArtifacts];
+            }
+            const { eventType } = (0, spawn_helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
+            try {
+                await (0, control_protocol_1.emit)({
+                    eventType,
+                    event: this.event,
+                    data: this.buildEmitPayload(eventType, {
+                        error: { message: (0, logger_1.serializeError)(error) },
+                    }),
+                });
+                const message = {
+                    subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
+                    payload: { eventType },
+                };
+                this.artifacts = [];
+                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
+                this.hasWorkerEmitted = true;
+            }
+            catch (emitError) {
+                console.error(`Error while emitting error event with event type: ${eventType}.`, (0, logger_1.serializeError)(emitError));
+            }
+        });
+    }
     async uploadAllRepos() {
         for (const repo of this.repos) {
-            const error = await repo.upload();
+            await repo.upload();
             this.artifacts.push(...repo.uploadedArtifacts);
-            if (error) {
-                throw error;
+        }
+    }
+    /**
+     * Pre-emit hook: uploads all repos and updates extraction boundaries/state.
+     * Throws if uploading repos or persisting state fails. The caller emits a
+     * compensating error event with whatever artifacts did upload.
+     */
+    async beforeEmit(eventType) {
+        // Upload all repos before emitting the event.
+        console.log(`Uploading all repos before emitting event with event type: ${eventType}.`);
+        await this.uploadAllRepos();
+        // If the extraction is done, save the timestamp of the last successful sync.
+        if (eventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
+            console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
+            this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
+            this.state.lastSyncStarted = '';
+            // Clear pending extraction boundaries now that the cycle is complete.
+            this.state.pendingWorkersOldest = '';
+            this.state.pendingWorkersNewest = '';
+            // Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
+            const extractionStart = this.event.payload.event_context.extract_from;
+            const extractionEnd = this.event.payload.event_context.extract_to;
+            if (extractionStart &&
+                (!this.state.workersOldest ||
+                    extractionStart < this.state.workersOldest)) {
+                console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
+                this.state.workersOldest = extractionStart;
+            }
+            if (extractionEnd &&
+                (!this.state.workersNewest || extractionEnd > this.state.workersNewest)) {
+                console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
+                this.state.workersNewest = extractionEnd;
             }
         }
+        if (constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
+            return;
+        }
+        let stateSizeKb = 0;
+        try {
+            stateSizeKb =
+                Buffer.byteLength(JSON.stringify(this.state), 'utf8') / 1024;
+        }
+        catch (_a) {
+            stateSizeKb = 0;
+        }
+        console.log(`Saving ${stateSizeKb.toFixed(2)} KB state before emitting event with event type: ${eventType}. Current state`, (0, logger_1.getPrintableState)(this.state));
+        try {
+            await this.adapterState.postState(this.state);
+        }
+        catch (error) {
+            console.error('Error while posting state.', (0, logger_1.serializeError)(error));
+            throw error;
+        }
+    }
+    buildEmitPayload(eventType, data) {
+        var _a;
+        if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
+            data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
+        }
+        const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(eventType);
+        const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(eventType);
+        return Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
+            ? { reports: this.reports, processed_files: this.processedFiles }
+            : {}));
     }
     async loadItemTypes({ itemTypesToLoad, }) {
         return (0, logger_context_1.runWithSdkLogContext)(async () => {

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=worker-adapter.upload-failure.test.d.ts.map

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-adapter.upload-failure.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.upload-failure.test.ts"],"names":[],"mappings":""}

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const state_1 = require("../../state/state");
+const jest_setup_1 = require("../../tests/jest.setup");
+const test_helpers_1 = require("../../tests/test-helpers");
+const test_utils_1 = require("../../common/test-utils");
+const types_1 = require("../../types");
+const worker_adapter_1 = require("./worker-adapter");
+/* eslint-disable @typescript-eslint/no-require-imports */
+jest.mock('../../common/control-protocol', () => ({
+    emit: jest.fn().mockResolvedValue({}),
+}));
+jest.mock('../../mappers/mappers');
+jest.mock('node:worker_threads', () => ({
+    parentPort: { postMessage: jest.fn() },
+}));
+jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({
+    AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({
+        streamAll: jest.fn().mockResolvedValue(undefined),
+    })),
+}));
+const UPLOAD_URL_PATH = '/internal/airdrop.artifacts.upload-url';
+function makeAdapter() {
+    const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+        payload: { event_type: types_1.EventType.StartExtractingData },
+    });
+    const initialState = {
+        attachments: { completed: false },
+        lastSyncStarted: '',
+        lastSuccessfulSyncStarted: '',
+        snapInVersionId: '',
+        toDevRev: {
+            attachmentsMetadata: {
+                artifactIds: [],
+                lastProcessed: 0,
+                lastProcessedAttachmentsIdsList: [],
+            },
+        },
+    };
+    const adapterState = new state_1.State({ event, initialState });
+    const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
+    const workerThreads = require('node:worker_threads');
+    const mockPostMessage = jest.fn();
+    if (workerThreads.parentPort) {
+        jest
+            .spyOn(workerThreads.parentPort, 'postMessage')
+            .mockImplementation(mockPostMessage);
+    }
+    else {
+        workerThreads.parentPort = { postMessage: mockPostMessage };
+    }
+    return { adapter, mockPostMessage };
+}
+describe(`${worker_adapter_1.WorkerAdapter.name} upload failure (near-integration)`, () => {
+    let adapter;
+    let mockPostMessage;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        jest_setup_1.mockServer.resetRoutes();
+        ({ adapter, mockPostMessage } = makeAdapter());
+        adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
+    });
+    afterEach(() => {
+        jest.restoreAllMocks();
+    });
+    it('should throw from repo.push when upload-url returns 400 during batch upload', async () => {
+        var _a, _b;
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 400,
+        });
+        adapter.initializeRepos([
+            {
+                itemType: 'tasks',
+                overridenOptions: { batchSize: 10 },
+            },
+        ]);
+        await expect((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(20))).rejects.toThrow('artifact upload URL');
+        expect((_b = adapter.getRepo('tasks')) === null || _b === void 0 ? void 0 : _b.getItems().length).toBe(20);
+    });
+    it('should emit DataExtractionError when uploadAllRepos fails', async () => {
+        var _a;
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 400,
+        });
+        adapter.initializeRepos([{ itemType: 'tasks' }]);
+        await ((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
+        expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
+            eventType: types_1.ExtractorEventType.DataExtractionError,
+            data: expect.objectContaining({
+                error: expect.objectContaining({
+                    message: expect.stringContaining('artifact upload URL'),
+                }),
+            }),
+        }));
+        expect(mockPostMessage).toHaveBeenCalledWith(expect.objectContaining({
+            subject: 'emit',
+            payload: { eventType: types_1.ExtractorEventType.DataExtractionError },
+        }));
+    });
+    it('should include partial artifacts when push batch succeeded before uploadAllRepos fails', async () => {
+        var _a;
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 200,
+            succeedThenFail: {
+                successCount: 1,
+                errorStatus: 400,
+            },
+        });
+        adapter.initializeRepos([
+            {
+                itemType: 'tasks',
+                overridenOptions: { batchSize: 10 },
+            },
+        ]);
+        await ((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(15)));
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
+        expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
+            eventType: types_1.ExtractorEventType.DataExtractionError,
+            data: expect.objectContaining({
+                artifacts: expect.arrayContaining([
+                    expect.objectContaining({ item_count: 10, item_type: 'tasks' }),
+                ]),
+            }),
+        }));
+    });
+});

package/dist/repo/repo.d.ts

@@ -1,5 +1,4 @@
 import { Item } from '../repo/repo.interfaces';
-import { ErrorRecord } from '../types/common';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { NormalizedAttachment, NormalizedItem, RepoFactoryInterface } from './repo.interfaces';
 export declare class Repo {
@@ -22,7 +21,7 @@ export declare class Repo {
     };
     constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
     getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
-    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
+    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
     push(items: Item[]): Promise<boolean>;
 }
 //# sourceMappingURL=repo.d.ts.map

package/dist/repo/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAG3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAsDxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAyC5C"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAG3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IA4DV,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAkC5C"}

package/dist/repo/repo.js

@@ -23,6 +23,7 @@ class Repo {
         return this.items;
     }
     async upload(batch) {
+        var _a;
         const itemsToUpload = batch || this.items;
         if (itemsToUpload.length > 0) {
             for (const item of itemsToUpload) {
@@ -44,8 +45,8 @@ class Repo {
             console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
             const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
             if (error || !artifact) {
-                console.error('Error while uploading batch', error);
-                return error;
+                console.error(`Error while uploading batch of ${itemsToUpload.length} items of type ${this.itemType}.`, error);
+                throw new Error((_a = error === null || error === void 0 ? void 0 : error.message) !== null && _a !== void 0 ? _a : `Upload failed for item type "${this.itemType}" without artifact.`);
             }
             this.onUpload(artifact);
             this.uploadedArtifacts.push(artifact);
@@ -80,16 +81,9 @@ class Repo {
         // Upload in batches while the number of items exceeds the batch size
         const batchSize = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.batchSize) || constants_1.ARTIFACT_BATCH_SIZE;
         while (this.items.length >= batchSize) {
-            // Slice out a batch of batchSize items to upload
-            const batch = this.items.splice(0, batchSize);
-            try {
-                // Upload the batch
-                await this.upload(batch);
-            }
-            catch (error) {
-                console.error('Error while uploading batch', error);
-                return false;
-            }
+            const batch = this.items.slice(0, batchSize);
+            await this.upload(batch);
+            this.items.splice(0, batchSize);
         }
         return true;
     }

package/dist/repo/repo.test.js

@@ -313,7 +313,7 @@ describe(repo_1.Repo.name, () => {
                 error: new Error('fail'),
                 artifact: null,
             });
-            await repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')]);
+            await expect(repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')])).rejects.toThrow('fail');
             expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
             expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
         });
@@ -343,4 +343,35 @@ describe(repo_1.Repo.name, () => {
             expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
         });
     });
+    it('should throw when upload fails', async () => {
+        mockUploadFn.mockResolvedValueOnce({
+            error: { message: 'upload failed' },
+        });
+        await expect(repo.upload((0, test_helpers_1.createItems)(1))).rejects.toThrow('upload failed');
+    });
+    it('should retain items in repo when batch upload fails during push', async () => {
+        repo = new repo_1.Repo({
+            event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                payload: { event_type: types_1.EventType.ExtractionDataStart },
+            }),
+            itemType: 'test_item_type',
+            normalize,
+            onUpload: jest.fn(),
+            options: { batchSize: 10 },
+        });
+        const items = (0, test_helpers_1.createItems)(20);
+        mockUploadFn
+            .mockResolvedValueOnce({
+            artifact: {
+                id: 'artifact-1',
+                item_type: 'test_item_type',
+                item_count: 10,
+            },
+        })
+            .mockResolvedValueOnce({
+            error: { message: 'second batch failed' },
+        });
+        await expect(repo.push(items)).rejects.toThrow('second batch failed');
+        expect(repo.getItems().length).toBe(10);
+    });
 });

package/dist/state/state.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/state/state.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,YAAY,EAIZ,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,wBAAsB,kBAAkB,CAAC,cAAc,EAAE,EACvD,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CA8KjE;AAED,qBAAa,KAAK,CAAC,cAAc;IAC/B,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAEd,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC;IAenE,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDvD;;;OAGG;IACG,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC;IAqDpD;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}
+{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/state/state.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,YAAY,EAIZ,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,wBAAsB,kBAAkB,CAAC,cAAc,EAAE,EACvD,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAqLjE;AAED,qBAAa,KAAK,CAAC,cAAc;IAC/B,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAEd,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC;IAenE,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDvD;;;OAGG;IACG,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC;IAqDpD;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}

package/dist/state/state.js

@@ -67,7 +67,7 @@ async function createAdapterState({ event, initialState, initialDomainMapping, o
         const eventContext = event.payload.event_context;
         const overrideExtractFrom = (value) => {
             if (eventContext.extract_from !== undefined) {
-                eventContext.platform_extract_from = eventContext.extract_from;
+                eventContext.original_extract_from = eventContext.extract_from;
             }
             eventContext.extract_from = value;
         };
@@ -126,10 +126,14 @@ async function createAdapterState({ event, initialState, initialDomainMapping, o
         }
         else {
             // Non-StartExtractingMetadata events: reuse pending values from state
-            if (eventContext.extract_from === undefined &&
-                as.state.pendingWorkersOldest) {
-                overrideExtractFrom(as.state.pendingWorkersOldest);
-                console.log(`Reusing pendingWorkersOldest as extract_from: ${as.state.pendingWorkersOldest}.`);
+            const pendingExtractFrom = as.state.pendingWorkersOldest || as.state.lastSuccessfulSyncStarted;
+            if (pendingExtractFrom &&
+                (eventContext.extract_from === undefined ||
+                    eventContext.extract_from !== pendingExtractFrom)) {
+                overrideExtractFrom(pendingExtractFrom);
+                console.log(`Using ${as.state.pendingWorkersOldest
+                    ? 'pendingWorkersOldest'
+                    : 'lastSuccessfulSyncStarted'} as extract_from: ${pendingExtractFrom}.`);
             }
             else {
                 console.log('pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.');

package/dist/state/state.time-value-resolution.test.js

@@ -102,7 +102,7 @@ describe('State — TimeValue resolution', () => {
             // Assert: the successful sync boundary takes precedence and is retained
             // for later phases.
             expect(event.payload.event_context.extract_from).toBe('2025-05-01T00:00:00.000Z');
-            expect(event.payload.event_context.platform_extract_from).toBe(platformExtractFrom);
+            expect(event.payload.event_context.original_extract_from).toBe(platformExtractFrom);
             expect(state.state.pendingWorkersOldest).toBe('2025-05-01T00:00:00.000Z');
         });
         it('should preserve the platform extract_from when resolving a CPv2 start time', async () => {
@@ -134,7 +134,7 @@ describe('State — TimeValue resolution', () => {
                 initialDomainMapping: {},
             });
             expect(event.payload.event_context.extract_from).toBe(resolvedExtractFrom);
-            expect(event.payload.event_context.platform_extract_from).toBe(platformExtractFrom);
+            expect(event.payload.event_context.original_extract_from).toBe(platformExtractFrom);
         });
         it('should use the platform extract_from when lastSuccessfulSyncStarted is not set', async () => {
             // Arrange: CPv1.1 sends a concrete extract_from instead of a TimeValue.
@@ -228,6 +228,35 @@ describe('State — TimeValue resolution', () => {
             expect(event.payload.event_context.extract_from).toBe(lastSuccessfulSyncStarted);
             expect(state.state.pendingWorkersOldest).toBe(lastSuccessfulSyncStarted);
         });
+        it('should override platform extract_from on StartExtractingData and preserve its original value', async () => {
+            const platformExtractFrom = '2026-08-24T13:26:15.388Z';
+            const lastSuccessfulSyncStarted = '2026-08-25T13:26:15.388Z';
+            const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                context: {
+                    snap_in_version_id: 'test_snap_in_version_id',
+                },
+                payload: {
+                    event_type: extraction_1.EventType.StartExtractingData,
+                    event_context: {
+                        extract_from: platformExtractFrom,
+                        extraction_start_time: {},
+                    },
+                },
+            });
+            fetchStateSpy.mockResolvedValue({
+                state: JSON.stringify({
+                    snapInVersionId: 'test_snap_in_version_id',
+                    lastSuccessfulSyncStarted,
+                }),
+            });
+            await (0, state_1.createAdapterState)({
+                event,
+                initialState: {},
+                initialDomainMapping: {},
+            });
+            expect(event.payload.event_context.extract_from).toBe(lastSuccessfulSyncStarted);
+            expect(event.payload.event_context.original_extract_from).toBe(platformExtractFrom);
+        });
         it('should skip resolution when extraction_end_time has no type', async () => {
             // Arrange: platform sends extraction_end_time without a type field
             const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {

package/dist/types/extraction.d.ts

@@ -56,7 +56,8 @@ export declare enum EventType {
     StartDeletingExtractorState = "START_DELETING_EXTRACTOR_STATE",
     StartExtractingAttachments = "START_EXTRACTING_ATTACHMENTS",
     ContinueExtractingAttachments = "CONTINUE_EXTRACTING_ATTACHMENTS",
-    StartDeletingExtractorAttachmentsState = "START_DELETING_EXTRACTOR_ATTACHMENTS_STATE"
+    StartDeletingExtractorAttachmentsState = "START_DELETING_EXTRACTOR_ATTACHMENTS_STATE",
+    ContinueExtractingMetadata = "CONTINUE_EXTRACTING_METADATA"
 }
 /**
  * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
@@ -143,7 +144,9 @@ export declare enum ExtractorEventType {
     AttachmentExtractionDone = "ATTACHMENT_EXTRACTION_DONE",
     AttachmentExtractionError = "ATTACHMENT_EXTRACTION_ERROR",
     ExtractorAttachmentsStateDeletionDone = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE",
-    ExtractorAttachmentsStateDeletionError = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR"
+    ExtractorAttachmentsStateDeletionError = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR",
+    MetadataExtractionProgress = "METADATA_EXTRACTION_PROGRESS",
+    MetadataExtractionDelayed = "METADATA_EXTRACTION_DELAYED"
 }
 /**
  * @deprecated
@@ -303,7 +306,7 @@ export interface EventContext {
      * Original extraction start timestamp supplied by the platform before the SDK
      * overrides extract_from.
      */
-    platform_extract_from?: string;
+    original_extract_from?: string;
     import_slug: string;
     initial_sync_scope?: InitialSyncScope;
     mode: string;