@devrev/ts-adaas 1.19.8 → 1.19.9-beta.0

package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js

@@ -43,6 +43,7 @@ function makeAdapter(eventType = types_1.EventType.StartExtractingData) {
     const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
     return { adapter, event, adapterState };
 }
+const iso = (ms) => new Date(ms).toISOString();
 describe(`${worker_adapter_1.WorkerAdapter.name}.emit`, () => {
     let adapter;
     let mockPostMessage;
@@ -113,19 +114,25 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.emit`, () => {
         // Assert
         expect(mockPostMessage).toHaveBeenCalledTimes(1);
     });
-    it('should correctly emit one event even if uploadAllRepos errors', async () => {
-        // Arrange
+    it('should emit phase extraction error when uploadAllRepos fails', async () => {
+        const { emit: mockEmit } = require('../../common/control-protocol');
         adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
         adapter.uploadAllRepos = jest
             .fn()
             .mockRejectedValue(new Error('uploadAllRepos error'));
-        // Act
-        await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
-            reports: [],
-            processed_files: [],
-        });
-        // Assert
-        expect(mockPostMessage).toHaveBeenCalledTimes(1);
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
+        expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
+            eventType: types_1.ExtractorEventType.DataExtractionError,
+            data: expect.objectContaining({
+                error: expect.objectContaining({
+                    message: expect.stringContaining('uploadAllRepos error'),
+                }),
+            }),
+        }));
+        expect(mockPostMessage).toHaveBeenCalledWith(expect.objectContaining({
+            subject: 'emit',
+            payload: { eventType: types_1.ExtractorEventType.DataExtractionError },
+        }));
     });
     it('should include artifacts in data for extraction events', async () => {
         // Arrange
@@ -262,6 +269,80 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.emit`, () => {
         expect(emittedMessage.startsWith('E'.repeat(100))).toBe(true);
     });
 });
+describe(`${worker_adapter_1.WorkerAdapter.name}.emit — worker_metadata`, () => {
+    let adapter;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        ({ adapter } = makeAdapter());
+        adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
+        adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
+    });
+    function setupRepos(adapterInstance, lastExtractedItemType) {
+        adapterInstance['repos'] = [
+            {
+                itemType: 'issues',
+                dateRanges: {
+                    creationDate: { oldest: 100, newest: 200 },
+                    modifiedDate: { oldest: 150, newest: 250 },
+                },
+            },
+            {
+                itemType: 'tasks',
+                dateRanges: {
+                    creationDate: { oldest: 300, newest: 400 },
+                    modifiedDate: { oldest: 350, newest: 450 },
+                },
+            },
+        ];
+        adapterInstance['lastExtractedItemType'] = lastExtractedItemType;
+    }
+    it('should emit flat worker_metadata for the latest extracted item type only', async () => {
+        setupRepos(adapter, 'tasks');
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({
+            item_type: 'tasks',
+            oldest_created_date: iso(300),
+            newest_created_date: iso(400),
+            oldest_modified_date: iso(350),
+            newest_modified_date: iso(450),
+        });
+    });
+    it('should omit unset RFC3339 bounds from worker_metadata', async () => {
+        adapter['repos'] = [
+            {
+                itemType: 'tasks',
+                dateRanges: {
+                    creationDate: { oldest: 0, newest: 0 },
+                    modifiedDate: { oldest: 0, newest: 0 },
+                },
+            },
+        ];
+        adapter['lastExtractedItemType'] = 'tasks';
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({
+            item_type: 'tasks',
+        });
+    });
+    it('should send empty worker_metadata when no item type has been extracted', async () => {
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionProgress);
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
+    });
+    it('should send empty worker_metadata for loader events', async () => {
+        setupRepos(adapter, 'tasks');
+        await adapter.emit(types_1.LoaderEventType.DataLoadingProgress);
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
+    });
+    it('should send empty worker_metadata for non-progress extraction events', async () => {
+        setupRepos(adapter, 'tasks');
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionError);
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({});
+    });
+});
 describe(`${worker_adapter_1.WorkerAdapter.name}.emit — ExternalSyncUnitExtractionDone legacy path`, () => {
     it('should upload ESUs via a repo and strip external_sync_units from the emitted payload', async () => {
         // Arrange

package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
+const test_utils_1 = require("../../common/test-utils");
 const state_1 = require("../../state/state");
 const jest_setup_1 = require("../../tests/jest.setup");
-const test_utils_1 = require("../../common/test-utils");
 const types_1 = require("../../types");
 const worker_adapter_1 = require("./worker-adapter");
 /* eslint-disable @typescript-eslint/no-require-imports */

package/dist/multithreading/worker-adapter/worker-adapter.js

@@ -22,6 +22,13 @@ const workers_1 = require("../../types/workers");
 const uploader_1 = require("../../uploader/uploader");
 const event_type_translation_1 = require("../../common/event-type-translation");
 const helpers_1 = require("../../common/helpers");
+const spawn_helpers_1 = require("../spawn/spawn.helpers");
+function toRfc3339Timestamp(ms) {
+    if (!Number.isFinite(ms) || ms === 0) {
+        return undefined;
+    }
+    return new Date(ms).toISOString();
+}
 function createWorkerAdapter({ event, adapterState, options, }) {
     return new WorkerAdapter({
         event,
@@ -46,10 +53,6 @@ function createWorkerAdapter({ event, adapterState, options, }) {
  */
 class WorkerAdapter {
     constructor({ event, adapterState, options, }) {
-        this._isTimeout = false;
-        this.timeoutSignal = new Promise((resolve) => {
-            this.resolveTimeoutSignal = resolve;
-        });
         this.repos = [];
         this.currentEventDataLength = 0;
         this.event = event;
@@ -57,6 +60,7 @@ class WorkerAdapter {
         this.adapterState = adapterState;
         this._artifacts = [];
         this.hasWorkerEmitted = false;
+        this.isTimeout = false;
         // Loader
         this.loaderReports = [];
         this._processedFiles = [];
@@ -69,15 +73,6 @@ class WorkerAdapter {
             options,
         });
     }
-    get isTimeout() {
-        return this._isTimeout;
-    }
-    set isTimeout(value) {
-        this._isTimeout = value;
-        if (value) {
-            this.resolveTimeoutSignal();
-        }
-    }
     get state() {
         return this.adapterState.state;
     }
@@ -114,6 +109,7 @@ class WorkerAdapter {
                 repo.itemType !== constants_1.SSOR_ATTACHMENT;
             return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
                     var _a;
+                    this.lastExtractedItemType = repo.itemType;
                     // We need to store artifacts ids in state for later use when streaming attachments
                     if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
                         (_a = this.state.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
@@ -158,7 +154,7 @@ class WorkerAdapter {
      */
     async emit(newEventType, data) {
         return (0, logger_context_1.runWithSdkLogContext)(async () => {
-            var _a, _b;
+            var _a;
             newEventType = (0, event_type_translation_1.translateOutgoingEventType)(newEventType);
             if (this.hasWorkerEmitted) {
                 console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
@@ -190,8 +186,13 @@ class WorkerAdapter {
             }
             catch (error) {
                 console.error('Error while uploading repos', error);
-                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
-                this.hasWorkerEmitted = true;
+                for (const repo of this.repos) {
+                    this.artifacts.push(...repo.uploadedArtifacts);
+                }
+                const { eventType } = (0, spawn_helpers_1.getTimeoutErrorEventType)(this.event.payload.event_type);
+                await this.emitToPlatform(eventType, {
+                    error: { message: (0, logger_1.serializeError)(error) },
+                });
                 return;
             }
             // If the extraction is done, we want to save the timestamp of the last successful sync
@@ -219,55 +220,87 @@ class WorkerAdapter {
                     this.state.workersNewest = extractionEnd;
                 }
             }
-            // We want to save the state every time we emit an event, except for the start and delete events
-            if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
-                console.log(`Saving state before emitting event with event type: ${newEventType}.`);
-                try {
-                    await this.adapterState.postState(this.state);
-                }
-                catch (error) {
-                    console.error('Error while posting state', error);
-                    node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
-                    this.hasWorkerEmitted = true;
-                    return;
-                }
-            }
-            try {
-                // Always prune error messages to make them shorter before emit
-                if ((_b = data === null || data === void 0 ? void 0 : data.error) === null || _b === void 0 ? void 0 : _b.message) {
-                    data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
-                }
-                const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
-                const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(newEventType);
-                await (0, control_protocol_1.emit)({
-                    eventType: newEventType,
-                    event: this.event,
-                    data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
-                        ? { reports: this.reports, processed_files: this.processedFiles }
-                        : {})),
-                });
-                const message = {
-                    subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
-                    payload: { eventType: newEventType },
-                };
-                this.artifacts = [];
-                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
-                this.hasWorkerEmitted = true;
-            }
-            catch (error) {
-                console.error(`Error while emitting event with event type: ${newEventType}.`, (0, logger_1.serializeError)(error));
-                node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
-                this.hasWorkerEmitted = true;
-            }
+            await this.emitToPlatform(newEventType, data);
         });
     }
     async uploadAllRepos() {
         for (const repo of this.repos) {
-            const error = await repo.upload();
+            await repo.upload();
             this.artifacts.push(...repo.uploadedArtifacts);
-            if (error) {
-                throw error;
-            }
+        }
+    }
+    /**
+     * Persists adapter state before emitting, except for stateless worker phases.
+     */
+    async postStateBeforeEmit(eventType) {
+        if (constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
+            return true;
+        }
+        console.log(`Saving state before emitting event with event type: ${eventType}.`);
+        try {
+            await this.adapterState.postState(this.state);
+            return true;
+        }
+        catch (error) {
+            console.error('Error while posting state', error);
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+            this.hasWorkerEmitted = true;
+            return false;
+        }
+    }
+    /**
+     * Posts state (when required), emits to the platform, and notifies the parent
+     * worker that an event was emitted.
+     */
+    async emitToPlatform(eventType, data) {
+        var _a;
+        const posted = await this.postStateBeforeEmit(eventType);
+        if (!posted) {
+            return;
+        }
+        try {
+            if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
+                data.error.message = (0, helpers_1.truncateMessage)(data.error.message);
+            }
+            const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(eventType);
+            const isLoaderEvent = Object.values(loading_1.LoaderEventType).includes(eventType);
+            const progress_data = {};
+            if (isExtractionEvent &&
+                (eventType == extraction_1.ExtractorEventType.DataExtractionDone ||
+                    eventType == extraction_1.ExtractorEventType.DataExtractionProgress ||
+                    eventType == extraction_1.ExtractorEventType.AttachmentExtractionDone ||
+                    eventType == extraction_1.ExtractorEventType.AttachmentExtractionProgress)) {
+                const repo = this.lastExtractedItemType
+                    ? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
+                    : undefined;
+                if (repo) {
+                    progress_data.item_type = repo.itemType;
+                    progress_data.newest_created_date = toRfc3339Timestamp(repo.dateRanges.creationDate.newest);
+                    progress_data.oldest_created_date = toRfc3339Timestamp(repo.dateRanges.creationDate.oldest);
+                    progress_data.newest_modified_date = toRfc3339Timestamp(repo.dateRanges.modifiedDate.newest);
+                    progress_data.oldest_modified_date = toRfc3339Timestamp(repo.dateRanges.modifiedDate.oldest);
+                }
+            }
+            await (0, control_protocol_1.emit)({
+                eventType,
+                event: this.event,
+                data: Object.assign(Object.assign(Object.assign({}, data), (isExtractionEvent ? { artifacts: this.artifacts } : {})), (isLoaderEvent
+                    ? { reports: this.reports, processed_files: this.processedFiles }
+                    : {})),
+                worker_metadata: Object.assign({}, progress_data),
+            });
+            const message = {
+                subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
+                payload: { eventType },
+            };
+            this.artifacts = [];
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
+            this.hasWorkerEmitted = true;
+        }
+        catch (error) {
+            console.error(`Error while emitting event with event type: ${eventType}.`, (0, logger_1.serializeError)(error));
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+            this.hasWorkerEmitted = true;
         }
     }
     async loadItemTypes({ itemTypesToLoad, }) {
@@ -703,10 +736,6 @@ class WorkerAdapter {
                 else if (attachment.inline === false) {
                     ssorAttachment.inline = false;
                 }
-                if (this.isTimeout) {
-                    this.destroyHttpStream(httpStream);
-                    return;
-                }
                 await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
                 return;
             }

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=worker-adapter.upload-failure.test.d.ts.map

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-adapter.upload-failure.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.upload-failure.test.ts"],"names":[],"mappings":""}

package/dist/multithreading/worker-adapter/worker-adapter.upload-failure.test.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const state_1 = require("../../state/state");
+const jest_setup_1 = require("../../tests/jest.setup");
+const test_helpers_1 = require("../../tests/test-helpers");
+const test_utils_1 = require("../../common/test-utils");
+const types_1 = require("../../types");
+const worker_adapter_1 = require("./worker-adapter");
+/* eslint-disable @typescript-eslint/no-require-imports */
+jest.mock('../../common/control-protocol', () => ({
+    emit: jest.fn().mockResolvedValue({}),
+}));
+jest.mock('../../mappers/mappers');
+jest.mock('node:worker_threads', () => ({
+    parentPort: { postMessage: jest.fn() },
+}));
+jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({
+    AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({
+        streamAll: jest.fn().mockResolvedValue(undefined),
+    })),
+}));
+const UPLOAD_URL_PATH = '/internal/airdrop.artifacts.upload-url';
+function makeAdapter() {
+    const event = (0, test_utils_1.createMockEvent)(jest_setup_1.mockServer.baseUrl, {
+        payload: { event_type: types_1.EventType.StartExtractingData },
+    });
+    const initialState = {
+        attachments: { completed: false },
+        lastSyncStarted: '',
+        lastSuccessfulSyncStarted: '',
+        snapInVersionId: '',
+        toDevRev: {
+            attachmentsMetadata: {
+                artifactIds: [],
+                lastProcessed: 0,
+                lastProcessedAttachmentsIdsList: [],
+            },
+        },
+    };
+    const adapterState = new state_1.State({ event, initialState });
+    const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
+    const workerThreads = require('node:worker_threads');
+    const mockPostMessage = jest.fn();
+    if (workerThreads.parentPort) {
+        jest
+            .spyOn(workerThreads.parentPort, 'postMessage')
+            .mockImplementation(mockPostMessage);
+    }
+    else {
+        workerThreads.parentPort = { postMessage: mockPostMessage };
+    }
+    return { adapter, mockPostMessage };
+}
+describe(`${worker_adapter_1.WorkerAdapter.name} upload failure (near-integration)`, () => {
+    let adapter;
+    let mockPostMessage;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        jest_setup_1.mockServer.resetRoutes();
+        ({ adapter, mockPostMessage } = makeAdapter());
+        adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined);
+    });
+    afterEach(() => {
+        jest.restoreAllMocks();
+    });
+    it('should throw from repo.push when upload-url returns 400 during batch upload', async () => {
+        var _a, _b;
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 400,
+        });
+        adapter.initializeRepos([
+            {
+                itemType: 'tasks',
+                overridenOptions: { batchSize: 10 },
+            },
+        ]);
+        await expect((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(20))).rejects.toThrow('artifact upload URL');
+        expect((_b = adapter.getRepo('tasks')) === null || _b === void 0 ? void 0 : _b.getItems().length).toBe(20);
+    });
+    it('should emit DataExtractionError via emitToPlatform when uploadAllRepos fails', async () => {
+        var _a;
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 400,
+        });
+        adapter.initializeRepos([{ itemType: 'tasks' }]);
+        await ((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(5)));
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
+        expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
+            eventType: types_1.ExtractorEventType.DataExtractionError,
+            data: expect.objectContaining({
+                error: expect.objectContaining({
+                    message: expect.stringContaining('artifact upload URL'),
+                }),
+            }),
+        }));
+        expect(mockPostMessage).toHaveBeenCalledWith(expect.objectContaining({
+            subject: 'emit',
+            payload: { eventType: types_1.ExtractorEventType.DataExtractionError },
+        }));
+    });
+    it('should include partial artifacts when push batch succeeded before uploadAllRepos fails', async () => {
+        var _a;
+        const { emit: mockEmit } = require('../../common/control-protocol');
+        jest_setup_1.mockServer.setRoute({
+            path: UPLOAD_URL_PATH,
+            method: 'GET',
+            status: 200,
+            succeedThenFail: {
+                successCount: 1,
+                errorStatus: 400,
+            },
+        });
+        adapter.initializeRepos([
+            {
+                itemType: 'tasks',
+                overridenOptions: { batchSize: 10 },
+            },
+        ]);
+        await ((_a = adapter.getRepo('tasks')) === null || _a === void 0 ? void 0 : _a.push((0, test_helpers_1.createItems)(15)));
+        await adapter.emit(types_1.ExtractorEventType.DataExtractionDone);
+        expect(mockEmit).toHaveBeenCalledWith(expect.objectContaining({
+            eventType: types_1.ExtractorEventType.DataExtractionError,
+            data: expect.objectContaining({
+                artifacts: expect.arrayContaining([
+                    expect.objectContaining({ item_count: 10, item_type: 'tasks' }),
+                ]),
+            }),
+        }));
+    });
+});

package/dist/repo/repo.d.ts

@@ -1,5 +1,4 @@
 import { Item } from '../repo/repo.interfaces';
-import { ErrorRecord } from '../types/common';
 import { Artifact } from '../uploader/uploader.interfaces';
 import { NormalizedAttachment, NormalizedItem, RepoFactoryInterface } from './repo.interfaces';
 export declare class Repo {
@@ -10,9 +9,19 @@ export declare class Repo {
     private onUpload;
     private options?;
     uploadedArtifacts: Artifact[];
+    dateRanges: {
+        creationDate: {
+            oldest: number;
+            newest: number;
+        };
+        modifiedDate: {
+            oldest: number;
+            newest: number;
+        };
+    };
     constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
     getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
-    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
+    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void>;
     push(items: Item[]): Promise<boolean>;
 }
 //# sourceMappingURL=repo.d.ts.map

package/dist/repo/repo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;gBAEzB,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,GAAG,WAAW,CAAC;IAqCxB,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAyC5C"}
+{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../src/repo/repo.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAI3D,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAmB3B,qBAAa,IAAI;IACf,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAmD;IAChE,OAAO,CAAC,SAAS,CAAC,CAAwD;IAC1E,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,OAAO,CAAC,CAAuB;IAChC,iBAAiB,EAAE,QAAQ,EAAE,CAAC;IAC9B,UAAU;;;;;;;;;MAGf;gBAEU,EACV,KAAK,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,OAAO,GACR,EAAE,oBAAoB;IAUvB,QAAQ,IAAI,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE;IAItD,MAAM,CACV,KAAK,CAAC,EAAE,CAAC,cAAc,GAAG,oBAAoB,GAAG,IAAI,CAAC,EAAE,GACvD,OAAO,CAAC,IAAI,CAAC;IAyDV,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CAkC5C"}

package/dist/repo/repo.interfaces.d.ts

@@ -39,6 +39,8 @@ export interface NormalizedAttachment {
     author_id?: string;
     inline?: boolean;
     content_type?: string;
+    created_date?: string;
+    modified_date?: string;
     grand_parent_id?: number | string;
 }
 /**

package/dist/repo/repo.interfaces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IAItB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
+{"version":3,"file":"repo.interfaces.d.ts","sourceRoot":"","sources":["../../src/repo/repo.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,cAAc,GAAG,oBAAoB,CAAC;IACtE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IAIvB,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED;;GAEG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}

package/dist/repo/repo.js

@@ -4,8 +4,24 @@ exports.Repo = void 0;
 const constants_1 = require("../common/constants");
 const uploader_1 = require("../uploader/uploader");
 const logger_context_1 = require("../logger/logger.context");
+function updateRange(range, ms) {
+    if (range.oldest === 0 || ms < range.oldest) {
+        range.oldest = ms;
+    }
+    if (range.newest === 0 || ms > range.newest) {
+        range.newest = ms;
+    }
+}
+function toValidTimestamp(value) {
+    const ms = new Date(value).getTime();
+    return Number.isFinite(ms) ? ms : undefined;
+}
 class Repo {
     constructor({ event, itemType, normalize, onUpload, options, }) {
+        this.dateRanges = {
+            creationDate: { oldest: 0, newest: 0 },
+            modifiedDate: { oldest: 0, newest: 0 },
+        };
         this.items = [];
         this.itemType = itemType;
         this.normalize = normalize;
@@ -18,13 +34,30 @@ class Repo {
         return this.items;
     }
     async upload(batch) {
+        var _a;
         const itemsToUpload = batch || this.items;
         if (itemsToUpload.length > 0) {
+            for (const item of itemsToUpload) {
+                const createdDate = item === null || item === void 0 ? void 0 : item.created_date;
+                if (createdDate != null) {
+                    const createdMs = toValidTimestamp(createdDate);
+                    if (createdMs !== undefined) {
+                        updateRange(this.dateRanges.creationDate, createdMs);
+                    }
+                }
+                const modifiedDate = item === null || item === void 0 ? void 0 : item.modified_date;
+                if (modifiedDate != null && modifiedDate !== '') {
+                    const modifiedMs = toValidTimestamp(modifiedDate);
+                    if (modifiedMs !== undefined) {
+                        updateRange(this.dateRanges.modifiedDate, modifiedMs);
+                    }
+                }
+            }
             console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}. `);
             const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
             if (error || !artifact) {
                 console.error('Error while uploading batch', error);
-                return error;
+                throw new Error((_a = error === null || error === void 0 ? void 0 : error.message) !== null && _a !== void 0 ? _a : `Upload failed for item type "${this.itemType}" without artifact.`);
             }
             this.onUpload(artifact);
             this.uploadedArtifacts.push(artifact);
@@ -59,16 +92,9 @@ class Repo {
         // Upload in batches while the number of items exceeds the batch size
         const batchSize = ((_a = this.options) === null || _a === void 0 ? void 0 : _a.batchSize) || constants_1.ARTIFACT_BATCH_SIZE;
         while (this.items.length >= batchSize) {
-            // Slice out a batch of batchSize items to upload
-            const batch = this.items.splice(0, batchSize);
-            try {
-                // Upload the batch
-                await this.upload(batch);
-            }
-            catch (error) {
-                console.error('Error while uploading batch', error);
-                return false;
-            }
+            const batch = this.items.slice(0, batchSize);
+            await this.upload(batch);
+            this.items.splice(0, batchSize);
         }
         return true;
     }