@devrev/ts-adaas 1.19.7 → 1.19.9-beta.0

package/dist/multithreading/worker-adapter/worker-adapter.js

@@ -22,6 +22,13 @@ const workers_1 = require("../../types/workers");
 const uploader_1 = require("../../uploader/uploader");
 const event_type_translation_1 = require("../../common/event-type-translation");
 const helpers_1 = require("../../common/helpers");
+const spawn_helpers_1 = require("../spawn/spawn.helpers");
+function toRfc3339Timestamp(ms) {
+    if (!Number.isFinite(ms) || ms === 0) {
+        return undefined;
+    }
+    return new Date(ms).toISOString();
+}
 function createWorkerAdapter({ event, adapterState, options, }) {
     return new WorkerAdapter({
         event,
@@ -102,6 +109,7 @@ class WorkerAdapter {
                 repo.itemType !== constants_1.SSOR_ATTACHMENT;
             return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
                     var _a;
+                    this.lastExtractedItemType = repo.itemType;
                     // We need to store artifacts ids in state for later use when streaming attachments
                     if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
                         (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
@@ -146,7 +154,7 @@ class WorkerAdapter {
      */
     async emit(newEventType, data) {
         return (0, logger_context_1.runWithSdkLogContext)(async () => {
-            var _a, _b;
+            var _a;
             newEventType = (0, event_type_translation_1.translateOutgoingEventType)(newEventType);
             if (this.hasWorkerEmitted) {
                 console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
@@ -178,8 +186,13 @@ class WorkerAdapter {
             }
             catch (error) {
                 console.error('Error while uploading repos', error);
-                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
-                this.hasWorkerEmitted = true;
+                for (const repo of this.repos) {
+                    this.artifacts.push(...repo.uploadedArtifacts);
+                }
+                const { eventType } = (0, spawn_helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
+                await this.emitToPlatform(eventType, {
+                    error: { message: (0, logger_1.serializeError)(error) },
+                });
                 return;
             }
             // If the extraction is done, we want to save the timestamp of the last successful sync
@@ -207,55 +220,87 @@ class WorkerAdapter {
                     this.state.workersNewest = extractionEnd;
                 }
             }
-            // We want to save the state every time we emit an event, except for the start and delete events
-            if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
-                console.log(`Saving state before emitting event with event type: ${newEventType}.`);
-                try {
-                    await this.adapterState.postState(this.state);
-                }
-                catch (error) {
-                    console.error('Error while posting state', error);
-                    node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
-                    this.hasWorkerEmitted = true;
-                    return;
-                }
-            }
-            try {
-                // Always prune error messages to make them shorter before emit
-                if ((_b = data === null || data === void 0 ? void 0 : data.error) === null || _b === void 0 ? void 0 : _b.message) {
-                    data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
-                }
-                const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
-                const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
-                await (0, control_protocol_1.emit)({
-                    eventType: newEventType,
-                    event: this.event,
-                    data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
-                        ? { reports: this.reports, processed_files: this.processedFiles }
-                        : {})),
-                });
-                const message = {
-                    subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
-                    payload: { eventType: newEventType },
-                };
-                this.artifacts = [];
-                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
-                this.hasWorkerEmitted = true;
-            }
-            catch (error) {
-                console.error(`Error while emitting event with event type: ${newEventType}.`, (0, logger_1.serializeError)(error));
-                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
-                this.hasWorkerEmitted = true;
-            }
+            await this.emitToPlatform(newEventType, data);
         });
     }
     async uploadAllRepos() {
         for (const repo of this.repos) {
-            const error = await repo.upload();
+            await repo.upload();
             this.artifacts.push(...repo.uploadedArtifacts);
-            if (error) {
-                throw error;
+        }
+    }
+    /**
+     * Persists adapter state before emitting, except for stateless worker phases.
+     */
+    async postStateBeforeEmit(eventType) {
+        if (constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
+            return true;
+        }
+        console.log(`Saving state before emitting event with event type: ${eventType}.`);
+        try {
+            await this.adapterState.postState(this.state);
+            return true;
+        }
+        catch (error) {
+            console.error('Error while posting state', error);
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+            this.hasWorkerEmitted = true;
+            return false;
+        }
+    }
+    /**
+     * Posts state (when required), emits to the platform, and notifies the parent
+     * worker that an event was emitted.
+     */
+    async emitToPlatform(eventType, data) {
+        var _a;
+        const posted = await this.postStateBeforeEmit(eventType);
+        if (!posted) {
+            return;
+        }
+        try {
+            if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
+                data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
+            }
+            const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(eventType);
+            const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(eventType);
+            const progress_data = {};
+            if (isExtractionEvent &&
+                (eventType == extraction_1.ExtractorEventType.DataExtractionDone ||
+                    eventType == extraction_1.ExtractorEventType.DataExtractionProgress ||
+                    eventType == extraction_1.ExtractorEventType.AttachmentExtractionDone ||
+                    eventType == extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
+                const repo = this.lastExtractedItemType
+                    ? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
+                    : undefined;
+                if (repo) {
+                    progress_data.item_type = repo.itemType;
+                    progress_data.newest_created_date = toRfc3339Timestamp(repo.dateRanges.creationDate.newest);
+                    progress_data.oldest_created_date = toRfc3339Timestamp(repo.dateRanges.creationDate.oldest);
+                    progress_data.newest_modified_date = toRfc3339Timestamp(repo.dateRanges.modifiedDate.newest);
+                    progress_data.oldest_modified_date = toRfc3339Timestamp(repo.dateRanges.modifiedDate.oldest);
+                }
             }
+            await (0, control_protocol_1.emit)({
+                eventType,
+                event: this.event,
+                data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
+                    ? { reports: this.reports, processed_files: this.processedFiles }
+                    : {})),
+                worker_metadata: Object.assign({}, progress_data),
+            });
+            const message = {
+                subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
+                payload: { eventType },
+            };
+            this.artifacts = [];
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
+            this.hasWorkerEmitted = true;
+        }
+        catch (error) {
+            console.error(`Error while emitting event with event type: ${eventType}.`, (0, logger_1.serializeError)(error));
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+            this.hasWorkerEmitted = true;
         }
     }
     async loadItemTypes({ itemTypesToLoad, }) {

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=worker-adapter.upload-failure.test.d.ts.map

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-adapter.upload-failure.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.upload-failure.test.ts"],"names":[],"mappings":""}

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const state_1 = require("../../state/state");
+const jest_setup_1 = require("../../tests/jest.setup");
+const test_helpers_1 = require("../../tests/test-helpers");
+const test_utils_1 = require("../../common/test-utils");
+const types_1 = require("../../types");
+const worker_adapter_1 = require("./worker-adapter");
+/* eslint-disable @typescript-eslint/no-require-imports */
+jest.mock('../../common/control-protocol', () => ({
+    emit: jest.fn().mockResolvedValue({}),
+}));
+jest.mock('../../mappers/mappers');
+jest.mock('node:worker_threads', () => ({
+    parentPort: { postMessage: jest.fn() },
+}));
+jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({
+    AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({
+        streamAll: jest.fn().mockResolvedValue(undefined),
+    })),
+}));
+const UPLOAD_URL_PATH = '/internal/airdrop.artifacts.upload-url';
+function makeAdapter() {
+    const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+        payload: { event_type: types_1.EventType.StartExtractingData },
+    });
+    const initialState = {
+        attachments: { completed: false },
+        lastSyncStarted: '',
+        lastSuccessfulSyncStarted: '',
+        snapInVersionId: '',
+        toDevRev: {
+            attachmentsMetadata: {
+                artifactIds: [],
+                lastProcessed: 0,
+                lastProcessedAttachmentsIdsList: [],
+            },
+        },
+    };
+    const adapterState = new state_1.State({ event, initialState });
+    const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
+    const workerThreads = require('node:worker_threads');
+    const mockPostMessage = jest.fn();
+    if (workerThreads.parentPort) {
+        jest
+            .spyOn(workerThreads.parentPort, 'postMessage')
+            .mockImplementation(mockPostMessage);
+    }
+    else {
+        workerThreads.parentPort = { postMessage: mockPostMessage };
+    }
+    return { adapter, mockPostMessage };
+}
+describe(`${worker_adapter_1.WorkerAdapter.name} upload failure (near-integration)`, () => {
+    let adapter;
+    let mockPostMessage;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        jest_setup_1.mockServer.resetRoutes();
+        ({ adapter, mockPostMessage } = makeAdapter());
+        adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
+    });
+    afterEach(() => {
+        jest.restoreAllMocks();
+    });
+    it('should throw from repo.push when upload-url returns 400 during batch upload', async () => {
+        var _a, _b;
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 400,
+        });
+        adapter.initializeRepos([
+            {
+                itemType: 'tasks',
+                overridenOptions: { batchSize: 10 },
+            },
+        ]);
+        await expect((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(20))).rejects.toThrow('artifact upload URL');
+        expect((_b = adapter.getRepo('tasks')) === null || _b === void 0 ? void 0 : _b.getItems().length).toBe(20);
+    });
+    it('should emit DataExtractionError via emitToPlatform when uploadAllRepos fails', async () => {
+        var _a;
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 400,
+        });
+        adapter.initializeRepos([{ itemType: 'tasks' }]);
+        await ((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
+        expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
+            eventType: types_1.ExtractorEventType.DataExtractionError,
+            data: expect.objectContaining({
+                error: expect.objectContaining({
+                    message: expect.stringContaining('artifact upload URL'),
+                }),
+            }),
+        }));
+        expect(mockPostMessage).toHaveBeenCalledWith(expect.objectContaining({
+            subject: 'emit',
+            payload: { eventType: types_1.ExtractorEventType.DataExtractionError },
+        }));
+    });
+    it('should include partial artifacts when push batch succeeded before uploadAllRepos fails', async () => {
+        var _a;
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 200,
+            succeedThenFail: {
+                successCount: 1,
+                errorStatus: 400,
+            },
+        });
+        adapter.initializeRepos([
+            {
+                itemType: 'tasks',
+                overridenOptions: { batchSize: 10 },
+            },
+        ]);
+        await ((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(15)));
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
+        expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
+            eventType: types_1.ExtractorEventType.DataExtractionError,
+            data: expect.objectContaining({
+                artifacts: expect.arrayContaining([
+                    expect.objectContaining({ item_count: 10, item_type: 'tasks' }),
+                ]),
+            }),
+        }));
+    });
+});

package/dist/repo/repo.d.ts

@@ -1,5 +1,4 @@
 import { Item } from '../repo/repo.interfaces';
-import { ErrorRecord } from '../types/common';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { NormalizedAttachment, NormalizedItem, RepoFactoryInterface } from './repo.interfaces';
 export declare class Repo {
@@ -10,9 +9,19 @@ export declare class Repo {
     private onUpload;
     private options?;
     uploadedArtifacts: Artifact[];
+    dateRanges: {
+        creationDate: {
+            oldest: number;
+            newest: number;
+        };
+        modifiedDate: {
+            oldest: number;
+            newest: number;
+        };
+    };
     constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
     getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
-    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
+    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
     push(items: Item[]): Promise<boolean>;
 }
 //# sourceMappingURL=repo.d.ts.map

package/dist/repo/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;gBAEzB,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAqCxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAyC5C"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAmB3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU;;;;;;;;;MAGf;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IAyDV,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAkC5C"}

package/dist/repo/repo.interfaces.d.ts

@@ -39,6 +39,8 @@ export interface NormalizedAttachment {
     author_id?: string;
     inline?: boolean;
     content_type?: string;
+    created_date?: string;
+    modified_date?: string;
     grand_parent_id?: number | string;
 }
 /**

package/dist/repo/repo.interfaces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IAItB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
+{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IAIvB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}

package/dist/repo/repo.js

@@ -4,8 +4,24 @@ exports.Repo = void 0;
 const constants_1 = require("../common/constants");
 const uploader_1 = require("../uploader/uploader");
 const logger_context_1 = require("../logger/logger.context");
+function updateRange(range, ms) {
+    if (range.oldest === 0 || ms < range.oldest) {
+        range.oldest = ms;
+    }
+    if (range.newest === 0 || ms > range.newest) {
+        range.newest = ms;
+    }
+}
+function toValidTimestamp(value) {
+    const ms = new Date(value).getTime();
+    return Number.isFinite(ms) ? ms : undefined;
+}
 class Repo {
     constructor({ event, itemType, normalize, onUpload, options, }) {
+        this.dateRanges = {
+            creationDate: { oldest: 0, newest: 0 },
+            modifiedDate: { oldest: 0, newest: 0 },
+        };
         this.items = [];
         this.itemType = itemType;
         this.normalize = normalize;
@@ -18,13 +34,30 @@ class Repo {
         return this.items;
     }
     async upload(batch) {
+        var _a;
         const itemsToUpload = batch || this.items;
         if (itemsToUpload.length > 0) {
+            for (const item of itemsToUpload) {
+                const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
+                if (createdDate != null) {
+                    const createdMs = toValidTimestamp(createdDate);
+                    if (createdMs !== undefined) {
+                        updateRange(this.dateRanges.creationDate, createdMs);
+                    }
+                }
+                const modifiedDate = item === null || item === void 0 ? void 0 : item.modified_date;
+                if (modifiedDate != null && modifiedDate !== '') {
+                    const modifiedMs = toValidTimestamp(modifiedDate);
+                    if (modifiedMs !== undefined) {
+                        updateRange(this.dateRanges.modifiedDate, modifiedMs);
+                    }
+                }
+            }
             console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
             const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
             if (error || !artifact) {
                 console.error('Error while uploading batch', error);
-                return error;
+                throw new Error((_a = error === null || error === void 0 ? void 0 : error.message) !== null && _a !== void 0 ? _a : `Upload failed for item type "${this.itemType}" without artifact.`);
             }
             this.onUpload(artifact);
             this.uploadedArtifacts.push(artifact);
@@ -59,16 +92,9 @@ class Repo {
         // Upload in batches while the number of items exceeds the batch size
         const batchSize = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.batchSize) || constants_1.ARTIFACT_BATCH_SIZE;
         while (this.items.length >= batchSize) {
-            // Slice out a batch of batchSize items to upload
-            const batch = this.items.splice(0, batchSize);
-            try {
-                // Upload the batch
-                await this.upload(batch);
-            }
-            catch (error) {
-                console.error('Error while uploading batch', error);
-                return false;
-            }
+            const batch = this.items.slice(0, batchSize);
+            await this.upload(batch);
+            this.items.splice(0, batchSize);
         }
         return true;
     }

package/dist/repo/repo.test.js

@@ -7,6 +7,22 @@ const test_utils_1 = require("../common/test-utils");
 const types_1 = require("../types");
 const repo_1 = require("./repo");
 jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
+const mockUploadFn = jest.fn().mockResolvedValue({
+    error: null,
+    artifact: { id: 'art-1', item_type: 'test', item_count: 0 },
+});
+jest.mock('../uploader/uploader', () => ({
+    Uploader: jest.fn().mockImplementation(() => ({
+        upload: mockUploadFn,
+    })),
+}));
+function itemWithDate(id, created_date) {
+    return { id, created_date, modified_date: created_date, data: {} };
+}
+function itemWithDates(id, created_date, modified_date) {
+    return { id, created_date, modified_date, data: {} };
+}
+const ts = (iso) => new Date(iso).getTime();
 describe(repo_1.Repo.name, () => {
     let repo;
     let normalize;
@@ -181,4 +197,172 @@ describe(repo_1.Repo.name, () => {
             expect(repo.getItems().length).toBe(5);
         });
     });
+    describe('dateRanges', () => {
+        beforeEach(() => {
+            mockUploadFn.mockResolvedValue({
+                error: null,
+                artifact: { id: 'art-1', item_type: 'test', item_count: 0 },
+            });
+        });
+        it('should track min and max created_date from a single upload batch', async () => {
+            await repo.upload([
+                itemWithDate('1', '2023-06-15T12:00:00.000Z'),
+                itemWithDate('2', '2020-01-01T00:00:00.000Z'),
+                itemWithDate('3', '2021-03-01T00:00:00.000Z'),
+            ]);
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2023-06-15T12:00:00.000Z'));
+            expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-06-15T12:00:00.000Z'));
+        });
+        it('should skip items without created_date', async () => {
+            const attachmentWithoutDate = {
+                id: 'att-1',
+                url: 'https://example.com/file',
+                file_name: 'file.txt',
+                parent_id: 'parent-1',
+            };
+            await repo.upload([
+                itemWithDate('1', '2022-06-01T00:00:00.000Z'),
+                { id: '2', created_date: null, modified_date: '', data: {} },
+                { id: '3', modified_date: '', data: {} },
+                attachmentWithoutDate,
+            ]);
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-06-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-06-01T00:00:00.000Z'));
+        });
+        it('should leave timestamps at zero when no items have created_date', async () => {
+            await repo.upload([
+                { id: '1', modified_date: '', data: {} },
+                {
+                    id: 'att-1',
+                    url: 'https://example.com/file',
+                    file_name: 'file.txt',
+                    parent_id: 'parent-1',
+                },
+            ]);
+            expect(repo.dateRanges).toEqual({
+                creationDate: { oldest: 0, newest: 0 },
+                modifiedDate: { oldest: 0, newest: 0 },
+            });
+        });
+        it('should not update timestamps or call uploader on empty upload', async () => {
+            await repo.upload([]);
+            expect(repo.dateRanges).toEqual({
+                creationDate: { oldest: 0, newest: 0 },
+                modifiedDate: { oldest: 0, newest: 0 },
+            });
+            expect(mockUploadFn).not.toHaveBeenCalled();
+        });
+        it('should accumulate min and max across multiple upload batches via push', async () => {
+            repo = new repo_1.Repo({
+                event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                    payload: { event_type: types_1.EventType.ExtractionDataStart },
+                }),
+                itemType: 'test_item_type',
+                onUpload: jest.fn(),
+                options: { batchSize: 3 },
+            });
+            const dates = [
+                '2020-01-01T00:00:00.000Z',
+                '2021-01-01T00:00:00.000Z',
+                '2022-01-01T00:00:00.000Z',
+                '2023-01-01T00:00:00.000Z',
+                '2024-06-01T00:00:00.000Z',
+                '2024-12-01T00:00:00.000Z',
+                '2024-12-31T00:00:00.000Z',
+            ];
+            const items = dates.map((created_date, index) => itemWithDate(String(index), created_date));
+            await repo.push(items);
+            expect(repo.getItems()).toHaveLength(1);
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-12-01T00:00:00.000Z'));
+            await repo.push([
+                itemWithDate('7', '2019-01-01T00:00:00.000Z'),
+                itemWithDate('8', '2025-01-01T00:00:00.000Z'),
+            ]);
+            expect(repo.getItems()).toHaveLength(0);
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2019-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2025-01-01T00:00:00.000Z'));
+        });
+        it('should extend min and max when subsequent batches have wider date range', async () => {
+            await repo.upload([
+                itemWithDate('1', '2022-06-01T00:00:00.000Z'),
+                itemWithDate('2', '2023-06-01T00:00:00.000Z'),
+            ]);
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-06-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2023-06-01T00:00:00.000Z'));
+            await repo.upload([
+                itemWithDate('3', '2020-01-01T00:00:00.000Z'),
+                itemWithDate('4', '2024-01-01T00:00:00.000Z'),
+            ]);
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-01-01T00:00:00.000Z'));
+        });
+        it('should update timestamps even when upload fails', async () => {
+            mockUploadFn.mockResolvedValueOnce({
+                error: new Error('fail'),
+                artifact: null,
+            });
+            await expect(repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')])).rejects.toThrow('fail');
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
+        });
+        it('should ignore invalid created_date and modified_date values', async () => {
+            await repo.upload([
+                {
+                    id: '1',
+                    created_date: 'not-a-date',
+                    modified_date: 'still-not-a-date',
+                    data: {},
+                },
+                itemWithDates('2', '2022-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'),
+            ]);
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2022-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2022-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2023-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
+        });
+        it('should track modified_date independently from created_date', async () => {
+            await repo.upload([
+                itemWithDates('1', '2020-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'),
+                itemWithDates('2', '2024-01-01T00:00:00.000Z', '2021-06-01T00:00:00.000Z'),
+            ]);
+            expect(repo.dateRanges.creationDate.oldest).toBe(ts('2020-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.creationDate.newest).toBe(ts('2024-01-01T00:00:00.000Z'));
+            expect(repo.dateRanges.modifiedDate.oldest).toBe(ts('2021-06-01T00:00:00.000Z'));
+            expect(repo.dateRanges.modifiedDate.newest).toBe(ts('2023-01-01T00:00:00.000Z'));
+        });
+    });
+    it('should throw when upload fails', async () => {
+        mockUploadFn.mockResolvedValueOnce({
+            error: { message: 'upload failed' },
+        });
+        await expect(repo.upload((0, test_helpers_1.createItems)(1))).rejects.toThrow('upload failed');
+    });
+    it('should retain items in repo when batch upload fails during push', async () => {
+        repo = new repo_1.Repo({
+            event: (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+                payload: { event_type: types_1.EventType.ExtractionDataStart },
+            }),
+            itemType: 'test_item_type',
+            normalize,
+            onUpload: jest.fn(),
+            options: { batchSize: 10 },
+        });
+        const items = (0, test_helpers_1.createItems)(20);
+        mockUploadFn
+            .mockResolvedValueOnce({
+            artifact: {
+                id: 'artifact-1',
+                item_type: 'test_item_type',
+                item_count: 10,
+            },
+        })
+            .mockResolvedValueOnce({
+            error: { message: 'second batch failed' },
+        });
+        await expect(repo.push(items)).rejects.toThrow('second batch failed');
+        expect(repo.getItems().length).toBe(10);
+    });
 });

package/dist/types/extraction.d.ts

@@ -388,7 +388,14 @@ export interface EventData {
  * WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS.
  */
 export interface WorkerMetadata {
-    adaas_library_version: string;
+    adaas_library_version?: string;
+    item_type?: string;
+    oldest_created_date?: string;
+    newest_created_date?: string;
+    oldest_modified_date?: string;
+    newest_modified_date?: string;
+    oldest_state_date?: string;
+    newest_state_date?: string;
 }
 /**
  * DomainObject is an interface that defines the structure of a domain object that can be extracted.