@devrev/ts-adaas 1.20.2-beta.3 → 1.20.2-beta.5

package/dist/multithreading/worker-adapter/worker-adapter.js

@@ -22,7 +22,6 @@ const workers_1 = require("../../types/workers");
 const uploader_1 = require("../../uploader/uploader");
 const event_type_translation_1 = require("../../common/event-type-translation");
 const helpers_1 = require("../../common/helpers");
-const spawn_helpers_1 = require("../spawn/spawn.helpers");
 function createWorkerAdapter({ event, adapterState, options, }) {
     return new WorkerAdapter({
         event,
@@ -160,49 +159,93 @@ class WorkerAdapter {
      */
     async emit(newEventType, data) {
         return (0, logger_context_1.runWithSdkLogContext)(async () => {
-            var _a;
+            var _a, _b;
             newEventType = (0, event_type_translation_1.translateOutgoingEventType)(newEventType);
             if (this.hasWorkerEmitted) {
                 console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
                 return;
             }
-            let eventPayload;
-            try {
-                // If the event is ExternalSyncUnitExtractionDone, upload external sync units via a Repo before emitting.
-                // TODO: Remove in v2.0.0
-                if (newEventType === extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone &&
-                    (data === null || data === void 0 ? void 0 : data.external_sync_units) &&
-                    data.external_sync_units.length > 0) {
-                    console.log(`Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.`);
-                    this.initializeRepos([
-                        {
-                            itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
-                            overridenOptions: {
-                                batchSize: 25000,
-                                skipConfirmation: true,
-                            },
+            // If the event is ExternalSyncUnitExtractionDone, upload external sync units via a Repo before emitting
+            // TODO: Remove in v2.0.0
+            if (newEventType === extraction_1.ExtractorEventType.ExternalSyncUnitExtractionDone &&
+                (data === null || data === void 0 ? void 0 : data.external_sync_units) &&
+                data.external_sync_units.length > 0) {
+                console.log(`Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.`);
+                this.initializeRepos([
+                    {
+                        itemType: constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS,
+                        overridenOptions: {
+                            batchSize: 25000,
+                            skipConfirmation: true,
                         },
-                    ]);
-                    await ((_a = this.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)) === null || _a === void 0 ? void 0 : _a.push(data.external_sync_units));
-                    // Remove inline external_sync_units from data to avoid SQS size issues
-                    delete data.external_sync_units;
-                }
-                await this.beforeEmit(newEventType);
-                eventPayload = this.buildEmitPayload(newEventType, data);
+                    },
+                ]);
+                await ((_a = this.getRepo(constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)) === null || _a === void 0 ? void 0 : _a.push(data.external_sync_units));
+                // Remove inline external_sync_units from data to avoid SQS size issues
+                delete data.external_sync_units;
+            }
+            // Upload all repos before emitting the event
+            console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
+            try {
+                await this.uploadAllRepos();
             }
             catch (error) {
-                console.error('Error while preparing event for emission.', (0, logger_1.serializeError)(error));
-                await this.emitError(error);
+                console.error('Error while uploading repos', error);
+                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+                this.hasWorkerEmitted = true;
                 return;
             }
+            // If the extraction is done, we want to save the timestamp of the last successful sync
+            if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
+                console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
+                this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
+                this.state.lastSyncStarted = '';
+                // Clear pending extraction boundaries now that the cycle is complete
+                this.state.pendingWorkersOldest = '';
+                this.state.pendingWorkersNewest = '';
+                // Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
+                // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
+                const extractionStart = this.event.payload.event_context.extract_from;
+                const extractionEnd = this.event.payload.event_context.extract_to;
+                if (extractionStart &&
+                    (!this.state.workersOldest ||
+                        extractionStart < this.state.workersOldest)) {
+                    console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
+                    this.state.workersOldest = extractionStart;
+                }
+                if (extractionEnd &&
+                    (!this.state.workersNewest ||
+                        extractionEnd > this.state.workersNewest)) {
+                    console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
+                    this.state.workersNewest = extractionEnd;
+                }
+            }
+            // We want to save the state every time we emit an event, except for the start and delete events
+            if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
+                console.log(`Saving state before emitting event with event type: ${newEventType}.`);
+                try {
+                    await this.adapterState.postState(this.state);
+                }
+                catch (error) {
+                    console.error('Error while posting state', error);
+                    node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+                    this.hasWorkerEmitted = true;
+                    return;
+                }
+            }
             try {
+                // Always prune error messages to make them shorter before emit
+                if ((_b = data === null || data === void 0 ? void 0 : data.error) === null || _b === void 0 ? void 0 : _b.message) {
+                    data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
+                }
                 const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
+                const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
                 const progressData = {};
                 if (isExtractionEvent &&
-                    (newEventType === extraction_1.ExtractorEventType.DataExtractionDone ||
-                        newEventType === extraction_1.ExtractorEventType.DataExtractionProgress ||
-                        newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone ||
-                        newEventType === extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
+                    (newEventType == extraction_1.ExtractorEventType.DataExtractionDone ||
+                        newEventType == extraction_1.ExtractorEventType.DataExtractionProgress ||
+                        newEventType == extraction_1.ExtractorEventType.AttachmentExtractionDone ||
+                        newEventType == extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
                     const repo = this.lastExtractedItemType
                         ? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
                         : undefined;
@@ -217,7 +260,9 @@ class WorkerAdapter {
                 await (0, control_protocol_1.emit)({
                     eventType: newEventType,
                     event: this.event,
-                    data: eventPayload,
+                    data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
+                        ? { reports: this.reports, processed_files: this.processedFiles }
+                        : {})),
                     worker_metadata: Object.assign({}, progressData),
                 });
                 const message = {
@@ -235,111 +280,14 @@ class WorkerAdapter {
             }
         });
     }
-    /**
-     * Emits a phase-specific error after task or pre-emit execution fails. This
-     * path avoids flushing repos again, so artifacts uploaded before the failure
-     * are still reported without retrying the failed batch.
-     */
-    async emitError(error) {
-        return (0, logger_context_1.runWithSdkLogContext)(async () => {
-            if (this.hasWorkerEmitted) {
-                return;
-            }
-            // Include artifacts uploaded before a failure so they are not orphaned
-            // and can be associated with the emitted phase error.
-            for (const repo of this.repos) {
-                this.artifacts = [...this.artifacts, ...repo.uploadedArtifacts];
-            }
-            const { eventType } = (0, spawn_helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
-            try {
-                await (0, control_protocol_1.emit)({
-                    eventType,
-                    event: this.event,
-                    data: this.buildEmitPayload(eventType, {
-                        error: { message: (0, logger_1.serializeError)(error) },
-                    }),
-                });
-                const message = {
-                    subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
-                    payload: { eventType },
-                };
-                this.artifacts = [];
-                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
-                this.hasWorkerEmitted = true;
-            }
-            catch (emitError) {
-                console.error(`Error while emitting error event with event type: ${eventType}.`, (0, logger_1.serializeError)(emitError));
-            }
-        });
-    }
     async uploadAllRepos() {
         for (const repo of this.repos) {
-            await repo.upload();
+            const error = await repo.upload();
             this.artifacts.push(...repo.uploadedArtifacts);
-        }
-    }
-    /**
-     * Pre-emit hook: uploads all repos and updates extraction boundaries/state.
-     * Throws if uploading repos or persisting state fails. The caller emits a
-     * compensating error event with whatever artifacts did upload.
-     */
-    async beforeEmit(eventType) {
-        // Upload all repos before emitting the event.
-        console.log(`Uploading all repos before emitting event with event type: ${eventType}.`);
-        await this.uploadAllRepos();
-        // If the extraction is done, save the timestamp of the last successful sync.
-        if (eventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
-            console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).`);
-            this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted;
-            this.state.lastSyncStarted = '';
-            // Clear pending extraction boundaries now that the cycle is complete.
-            this.state.pendingWorkersOldest = '';
-            this.state.pendingWorkersNewest = '';
-            // Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
-            const extractionStart = this.event.payload.event_context.extract_from;
-            const extractionEnd = this.event.payload.event_context.extract_to;
-            if (extractionStart &&
-                (!this.state.workersOldest ||
-                    extractionStart < this.state.workersOldest)) {
-                console.log(`Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.`);
-                this.state.workersOldest = extractionStart;
-            }
-            if (extractionEnd &&
-                (!this.state.workersNewest || extractionEnd > this.state.workersNewest)) {
-                console.log(`Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.`);
-                this.state.workersNewest = extractionEnd;
+            if (error) {
+                throw error;
             }
         }
-        if (constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
-            return;
-        }
-        let stateSizeKb = 0;
-        try {
-            stateSizeKb =
-                Buffer.byteLength(JSON.stringify(this.state), 'utf8') / 1024;
-        }
-        catch (_a) {
-            stateSizeKb = 0;
-        }
-        console.log(`Saving ${stateSizeKb.toFixed(2)} KB state before emitting event with event type: ${eventType}. Current state`, (0, logger_1.getPrintableState)(this.state));
-        try {
-            await this.adapterState.postState(this.state);
-        }
-        catch (error) {
-            console.error('Error while posting state.', (0, logger_1.serializeError)(error));
-            throw error;
-        }
-    }
-    buildEmitPayload(eventType, data) {
-        var _a;
-        if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
-            data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
-        }
-        const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(eventType);
-        const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(eventType);
-        return Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
-            ? { reports: this.reports, processed_files: this.processedFiles }
-            : {}));
     }
     async loadItemTypes({ itemTypesToLoad, }) {
         return (0, logger_context_1.runWithSdkLogContext)(async () => {

package/dist/repo/repo.d.ts

@@ -1,4 +1,5 @@
 import { Item } from '../repo/repo.interfaces';
+import { ErrorRecord } from '../types/common';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { NormalizedAttachment, NormalizedItem, RepoFactoryInterface } from './repo.interfaces';
 export declare class Repo {
@@ -21,7 +22,7 @@ export declare class Repo {
     };
     constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
     getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
-    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
+    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
     push(items: Item[]): Promise<boolean>;
 }
 //# sourceMappingURL=repo.d.ts.map

package/dist/repo/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAG3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IA4DV,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAkC5C"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAG3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU,EAAE;QACjB,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,YAAY,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACpD,CAGC;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAsDxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAyC5C"}

package/dist/repo/repo.js

@@ -23,7 +23,6 @@ class Repo {
         return this.items;
     }
     async upload(batch) {
-        var _a;
         const itemsToUpload = batch || this.items;
         if (itemsToUpload.length > 0) {
             for (const item of itemsToUpload) {
@@ -45,8 +44,8 @@ class Repo {
             console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
             const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
             if (error || !artifact) {
-                console.error(`Error while uploading batch of ${itemsToUpload.length} items of type ${this.itemType}.`, error);
-                throw new Error((_a = error === null || error === void 0 ? void 0 : error.message) !== null && _a !== void 0 ? _a : `Upload failed for item type "${this.itemType}" without artifact.`);
+                console.error('Error while uploading batch', error);
+                return error;
             }
             this.onUpload(artifact);
             this.uploadedArtifacts.push(artifact);
@@ -81,9 +80,16 @@ class Repo {
         // Upload in batches while the number of items exceeds the batch size
         const batchSize = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.batchSize) || constants_1.ARTIFACT_BATCH_SIZE;
         while (this.items.length >= batchSize) {
-            const batch = this.items.slice(0, batchSize);
-            await this.upload(batch);
-            this.items.splice(0, batchSize);
+            // Slice out a batch of batchSize items to upload
+            const batch = this.items.splice(0, batchSize);
+            try {
+                // Upload the batch
+                await this.upload(batch);
+            }
+            catch (error) {
+                console.error('Error while uploading batch', error);
+                return false;
+            }
         }
         return true;
     }

package/dist/repo/repo.test.js

@@ -313,7 +313,7 @@ describe(repo_1.Repo.name, () => {
                 error: new Error('fail'),
                 artifact: null,
             });
-            await expect(repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')])).rejects.toThrow('fail');
+            await repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')]);
             expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
             expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
         });
@@ -343,35 +343,4 @@ describe(repo_1.Repo.name, () => {
             expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
         });
     });
-    it('should throw when upload fails', async () => {
-        mockUploadFn.mockResolvedValueOnce({
-            error: { message: 'upload failed' },
-        });
-        await expect(repo.upload((0, test_helpers_1.createItems)(1))).rejects.toThrow('upload failed');
-    });
-    it('should retain items in repo when batch upload fails during push', async () => {
-        repo = new repo_1.Repo({
-            event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
-                payload: { event_type: types_1.EventType.ExtractionDataStart },
-            }),
-            itemType: 'test_item_type',
-            normalize,
-            onUpload: jest.fn(),
-            options: { batchSize: 10 },
-        });
-        const items = (0, test_helpers_1.createItems)(20);
-        mockUploadFn
-            .mockResolvedValueOnce({
-            artifact: {
-                id: 'artifact-1',
-                item_type: 'test_item_type',
-                item_count: 10,
-            },
-        })
-            .mockResolvedValueOnce({
-            error: { message: 'second batch failed' },
-        });
-        await expect(repo.push(items)).rejects.toThrow('second batch failed');
-        expect(repo.getItems().length).toBe(10);
-    });
 });

package/dist/state/state.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/state/state.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,YAAY,EAIZ,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,wBAAsB,kBAAkB,CAAC,cAAc,EAAE,EACvD,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CA6IjE;AAED,qBAAa,KAAK,CAAC,cAAc;IAC/B,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAEd,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC;IAenE,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDvD;;;OAGG;IACG,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC;IAqDpD;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}
+{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/state/state.ts"],"names":[],"mappings":"AAYA,OAAO,EACL,YAAY,EAIZ,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,wBAAsB,kBAAkB,CAAC,cAAc,EAAE,EACvD,KAAK,EACL,YAAY,EACZ,oBAAoB,EACpB,OAAO,GACR,EAAE,cAAc,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAqLjE;AAED,qBAAa,KAAK,CAAC,cAAc;IAC/B,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAS;gBAEd,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,cAAc,CAAC;IAenE,IAAI,KAAK,IAAI,YAAY,CAAC,cAAc,CAAC,CAExC;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,cAAc,CAAC,EAE5C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED;;;;OAIG;IACG,IAAI,CAAC,YAAY,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDvD;;;OAGG;IACG,SAAS,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC;IAqDpD;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAqBjE"}

package/dist/state/state.js

@@ -65,6 +65,12 @@ async function createAdapterState({ event, initialState, initialDomainMapping, o
         // On StartExtractingMetadata: resolve fresh from TimeValue objects and store in pending state (always overwrite).
         // On all other events: reuse the pending values cached during StartExtractingMetadata.
         const eventContext = event.payload.event_context;
+        const overrideExtractFrom = (value) => {
+            if (eventContext.extract_from !== undefined) {
+                eventContext.platform_extract_from = eventContext.extract_from;
+            }
+            eventContext.extract_from = value;
+        };
         if (event.payload.event_type === extraction_1.EventType.StartExtractingMetadata) {
             const timeFields = [
                 {
@@ -83,7 +89,12 @@ async function createAdapterState({ event, initialState, initialDomainMapping, o
                 if (timeValue && timeValue.type) {
                     try {
                         const resolved = (0, time_value_resolver_1.resolveTimeValue)(timeValue, as.state);
-                        eventContext[target] = resolved;
+                        if (target === 'extract_from') {
+                            overrideExtractFrom(resolved);
+                        }
+                        else {
+                            eventContext[target] = resolved;
+                        }
                         as.state[pending] = resolved;
                         console.log(`Resolved ${target} to ${resolved}. Stored in ${pending}.`);
                     }
@@ -97,13 +108,32 @@ async function createAdapterState({ event, initialState, initialDomainMapping, o
                         process.exit(1);
                     }
                 }
+                else if (target === 'extract_from' &&
+                    as.state.lastSuccessfulSyncStarted) {
+                    overrideExtractFrom(as.state.lastSuccessfulSyncStarted);
+                    as.state.pendingWorkersOldest = as.state.lastSuccessfulSyncStarted;
+                    console.log(`Using lastSuccessfulSyncStarted as extract_from: ${as.state.lastSuccessfulSyncStarted}.`);
+                }
+                else if (target === 'extract_from' &&
+                    !as.state.lastSuccessfulSyncStarted &&
+                    eventContext.extract_from !== undefined) {
+                    // CPv1.1 supplies the resolved boundary directly. Keep it pending so
+                    // later extraction phases expose the same event_context value.
+                    as.state.pendingWorkersOldest = eventContext.extract_from;
+                    console.log(`Using platform-provided extract_from: ${eventContext.extract_from}. Stored in pendingWorkersOldest.`);
+                }
             }
         }
         else {
             // Non-StartExtractingMetadata events: reuse pending values from state
-            if (as.state.pendingWorkersOldest) {
-                eventContext.extract_from = as.state.pendingWorkersOldest;
-                console.log(`Reusing pendingWorkersOldest as extract_from: ${as.state.pendingWorkersOldest}.`);
+            const pendingExtractFrom = as.state.pendingWorkersOldest || as.state.lastSuccessfulSyncStarted;
+            if (pendingExtractFrom &&
+                (eventContext.extract_from === undefined ||
+                    eventContext.extract_from !== pendingExtractFrom)) {
+                overrideExtractFrom(pendingExtractFrom);
+                console.log(`Using ${as.state.pendingWorkersOldest
+                    ? 'pendingWorkersOldest'
+                    : 'lastSuccessfulSyncStarted'} as extract_from: ${pendingExtractFrom}.`);
             }
             else {
                 console.log('pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.');

package/dist/state/state.time-value-resolution.test.js

@@ -73,6 +73,98 @@ describe('State — TimeValue resolution', () => {
         });
     });
     describe('Backwards compatibility - missing TimeValue type', () => {
+        it('should prefer lastSuccessfulSyncStarted over the platform extract_from when no CPv2 start time is provided', async () => {
+            // Arrange: CPv1.1 sends a concrete extract_from instead of a TimeValue.
+            const platformExtractFrom = '2025-04-01T00:00:00.000Z';
+            const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                context: {
+                    snap_in_version_id: 'test_snap_in_version_id',
+                },
+                payload: {
+                    event_type: extraction_1.EventType.StartExtractingMetadata,
+                    event_context: {
+                        extract_from: platformExtractFrom,
+                    },
+                },
+            });
+            fetchStateSpy.mockResolvedValue({
+                state: JSON.stringify({
+                    snapInVersionId: 'test_snap_in_version_id',
+                    lastSuccessfulSyncStarted: '2025-05-01T00:00:00.000Z',
+                }),
+            });
+            // Act
+            const state = await (0, state_1.createAdapterState)({
+                event,
+                initialState: {},
+                initialDomainMapping: {},
+            });
+            // Assert: the successful sync boundary takes precedence and is retained
+            // for later phases.
+            expect(event.payload.event_context.extract_from).toBe('2025-05-01T00:00:00.000Z');
+            expect(event.payload.event_context.platform_extract_from).toBe(platformExtractFrom);
+            expect(state.state.pendingWorkersOldest).toBe('2025-05-01T00:00:00.000Z');
+        });
+        it('should preserve the platform extract_from when resolving a CPv2 start time', async () => {
+            const platformExtractFrom = '2025-04-01T00:00:00.000Z';
+            const resolvedExtractFrom = '2025-06-01T00:00:00.000Z';
+            const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                context: {
+                    snap_in_version_id: 'test_snap_in_version_id',
+                },
+                payload: {
+                    event_type: extraction_1.EventType.StartExtractingMetadata,
+                    event_context: {
+                        extraction_start_time: {
+                            type: extraction_1.TimeValueType.ABSOLUTE_TIME,
+                            value: resolvedExtractFrom,
+                        },
+                        extract_from: platformExtractFrom,
+                    },
+                },
+            });
+            fetchStateSpy.mockResolvedValue({
+                state: JSON.stringify({
+                    snapInVersionId: 'test_snap_in_version_id',
+                }),
+            });
+            await (0, state_1.createAdapterState)({
+                event,
+                initialState: {},
+                initialDomainMapping: {},
+            });
+            expect(event.payload.event_context.extract_from).toBe(resolvedExtractFrom);
+            expect(event.payload.event_context.platform_extract_from).toBe(platformExtractFrom);
+        });
+        it('should use the platform extract_from when lastSuccessfulSyncStarted is not set', async () => {
+            // Arrange: CPv1.1 sends a concrete extract_from instead of a TimeValue.
+            const platformExtractFrom = '2025-04-01T00:00:00.000Z';
+            const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                context: {
+                    snap_in_version_id: 'test_snap_in_version_id',
+                },
+                payload: {
+                    event_type: extraction_1.EventType.StartExtractingMetadata,
+                    event_context: {
+                        extract_from: platformExtractFrom,
+                    },
+                },
+            });
+            fetchStateSpy.mockResolvedValue({
+                state: JSON.stringify({
+                    snapInVersionId: 'test_snap_in_version_id',
+                }),
+            });
+            // Act
+            const state = await (0, state_1.createAdapterState)({
+                event,
+                initialState: {},
+                initialDomainMapping: {},
+            });
+            // Assert: the platform value is used and retained for later phases.
+            expect(event.payload.event_context.extract_from).toBe(platformExtractFrom);
+            expect(state.state.pendingWorkersOldest).toBe(platformExtractFrom);
+        });
         it('should skip resolution when extraction_start_time has no type', async () => {
             // Arrange: platform sends extraction_start_time without a type field (old platform version)
             const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
@@ -106,6 +198,65 @@ describe('State — TimeValue resolution', () => {
             expect(event.payload.event_context.extract_to).toBe('2025-06-01T00:00:00.000Z');
             expect(state.state.pendingWorkersNewest).toBe('2025-06-01T00:00:00.000Z');
         });
+        it('should use lastSuccessfulSyncStarted for extract_from when extraction_start_time has no type', async () => {
+            // Arrange: platform sends an empty extraction_start_time on a subsequent sync
+            const lastSuccessfulSyncStarted = '2025-05-01T00:00:00.000Z';
+            const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                context: {
+                    snap_in_version_id: 'test_snap_in_version_id',
+                },
+                payload: {
+                    event_type: extraction_1.EventType.StartExtractingMetadata,
+                    event_context: {
+                        extraction_start_time: {},
+                    },
+                },
+            });
+            fetchStateSpy.mockResolvedValue({
+                state: JSON.stringify({
+                    snapInVersionId: 'test_snap_in_version_id',
+                    lastSuccessfulSyncStarted,
+                }),
+            });
+            // Act
+            const state = await (0, state_1.createAdapterState)({
+                event,
+                initialState: {},
+                initialDomainMapping: {},
+            });
+            // Assert: the fallback is also stored for the following data phase
+            expect(event.payload.event_context.extract_from).toBe(lastSuccessfulSyncStarted);
+            expect(state.state.pendingWorkersOldest).toBe(lastSuccessfulSyncStarted);
+        });
+        it('should override platform extract_from on StartExtractingData and preserve its original value', async () => {
+            const platformExtractFrom = '2026-08-24T13:26:15.388Z';
+            const lastSuccessfulSyncStarted = '2026-08-25T13:26:15.388Z';
+            const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                context: {
+                    snap_in_version_id: 'test_snap_in_version_id',
+                },
+                payload: {
+                    event_type: extraction_1.EventType.StartExtractingData,
+                    event_context: {
+                        extract_from: platformExtractFrom,
+                        extraction_start_time: {},
+                    },
+                },
+            });
+            fetchStateSpy.mockResolvedValue({
+                state: JSON.stringify({
+                    snapInVersionId: 'test_snap_in_version_id',
+                    lastSuccessfulSyncStarted,
+                }),
+            });
+            await (0, state_1.createAdapterState)({
+                event,
+                initialState: {},
+                initialDomainMapping: {},
+            });
+            expect(event.payload.event_context.extract_from).toBe(lastSuccessfulSyncStarted);
+            expect(event.payload.event_context.platform_extract_from).toBe(platformExtractFrom);
+        });
         it('should skip resolution when extraction_end_time has no type', async () => {
             // Arrange: platform sends extraction_end_time without a type field
             const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {

package/dist/types/extraction.d.ts

@@ -56,8 +56,7 @@ export declare enum EventType {
     StartDeletingExtractorState = "START_DELETING_EXTRACTOR_STATE",
     StartExtractingAttachments = "START_EXTRACTING_ATTACHMENTS",
     ContinueExtractingAttachments = "CONTINUE_EXTRACTING_ATTACHMENTS",
-    StartDeletingExtractorAttachmentsState = "START_DELETING_EXTRACTOR_ATTACHMENTS_STATE",
-    ContinueExtractingMetadata = "CONTINUE_EXTRACTING_METADATA"
+    StartDeletingExtractorAttachmentsState = "START_DELETING_EXTRACTOR_ATTACHMENTS_STATE"
 }
 /**
  * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
@@ -144,9 +143,7 @@ export declare enum ExtractorEventType {
     AttachmentExtractionDone = "ATTACHMENT_EXTRACTION_DONE",
     AttachmentExtractionError = "ATTACHMENT_EXTRACTION_ERROR",
     ExtractorAttachmentsStateDeletionDone = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE",
-    ExtractorAttachmentsStateDeletionError = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR",
-    MetadataExtractionProgress = "METADATA_EXTRACTION_PROGRESS",
-    MetadataExtractionDelayed = "METADATA_EXTRACTION_DELAYED"
+    ExtractorAttachmentsStateDeletionError = "EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR"
 }
 /**
  * @deprecated
@@ -297,11 +294,16 @@ export interface EventContext {
     external_system_name: string;
     external_system_type: string;
     /**
-     * Resolved start timestamp of extraction (ISO 8601 format).
-     * Automatically computed by the SDK from extraction_start_time and worker state.
+     * Extraction start timestamp (ISO 8601 format), supplied directly by CPv1.1
+     * or resolved by the SDK from extraction_start_time and worker state for CPv2.
      * This is the field developers should read to know when to start extracting from.
      */
     extract_from?: string;
+    /**
+     * Original extraction start timestamp supplied by the platform before the SDK
+     * overrides extract_from.
+     */
+    platform_extract_from?: string;
     import_slug: string;
     initial_sync_scope?: InitialSyncScope;
     mode: string;