@devrev/airsync-sdk 2.0.0-beta.2 → 2.0.0-beta.3

package/dist/multithreading/adapters/extraction-adapter.js

@@ -0,0 +1,325 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExtractionAdapter = void 0;
+const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
+const constants_1 = require("../../common/constants");
+const logger_1 = require("../../logger/logger");
+const logger_context_1 = require("../../logger/logger.context");
+const repo_1 = require("../../repo/repo");
+const extraction_1 = require("../../types/extraction");
+const base_adapter_1 = require("./base-adapter");
+/**
+ * ExtractionAdapter is the adapter passed to extraction tasks. It exposes the
+ * extraction surface (repos, artifacts, attachment streaming) and updates the
+ * extraction boundaries before emitting.
+ *
+ * @public
+ */
+class ExtractionAdapter extends base_adapter_1.BaseAdapter {
+    constructor(params) {
+        super(params);
+        this.repos = [];
+        this.currentEventDataLength = 0;
+        this._artifacts = [];
+    }
+    get extractionScope() {
+        return this.adapterState.extractionScope;
+    }
+    /**
+     * Returns whether the given item type should be extracted.
+     * Defaults to true if the scope is empty or the item type is not listed.
+     */
+    shouldExtract(itemType) {
+        const scope = this.extractionScope;
+        if (Object.keys(scope).length === 0)
+            return true;
+        if (!(itemType in scope))
+            return true;
+        return scope[itemType].extract;
+    }
+    initializeRepos(repos) {
+        this.repos = repos.map((repo) => {
+            const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
+                repo.itemType !== constants_1.SSOR_ATTACHMENT;
+            return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { onUpload: (artifact) => {
+                    var _a;
+                    // We need to store artifacts ids in state for later use when streaming attachments
+                    if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
+                        (_a = this.adapterState.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata.artifactIds.push(artifact.id);
+                    }
+                    // Calculate size of the entire artifact object that goes in the SQS message
+                    this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
+                    if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
+                        !this.isTimeout) {
+                        this.isTimeout = true;
+                    }
+                }, options: Object.assign(Object.assign({}, this.options), repo.overridenOptions) }));
+        });
+    }
+    getRepo(itemType) {
+        return (0, logger_context_1.runWithSdkLogContext)(() => {
+            const repo = this.repos.find((repo) => repo.itemType === itemType);
+            if (!repo) {
+                console.error(`Repo for item type ${itemType} not found.`);
+                return;
+            }
+            return repo;
+        });
+    }
+    get artifacts() {
+        return this._artifacts;
+    }
+    set artifacts(artifacts) {
+        this._artifacts = this._artifacts
+            .concat(artifacts)
+            .filter((value, index, self) => self.indexOf(value) === index);
+    }
+    async beforeEmit(newEventType) {
+        // Upload all repos before emitting the event
+        console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
+        await this.uploadAllRepos();
+        if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
+            const sdkState = this.adapterState.sdkState;
+            // Clear pending extraction boundaries now that the cycle is complete
+            sdkState.pendingWorkersOldest = '';
+            sdkState.pendingWorkersNewest = '';
+            // Update workersOldest and workersNewest boundaries from resolved extraction timestamps.
+            // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest.
+            const extractionStart = this.event.payload.event_context.extract_from;
+            const extractionEnd = this.event.payload.event_context.extract_to;
+            if (extractionStart &&
+                (!sdkState.workersOldest || extractionStart < sdkState.workersOldest)) {
+                console.log(`Updating workersOldest from '${sdkState.workersOldest}' to '${extractionStart}'.`);
+                sdkState.workersOldest = extractionStart;
+            }
+            if (extractionEnd &&
+                (!sdkState.workersNewest || extractionEnd > sdkState.workersNewest)) {
+                console.log(`Updating workersNewest from '${sdkState.workersNewest}' to '${extractionEnd}'.`);
+                sdkState.workersNewest = extractionEnd;
+            }
+        }
+    }
+    buildEmitPayload(newEventType) {
+        const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
+        return isExtractionEvent ? { artifacts: this.artifacts } : {};
+    }
+    afterEmit() {
+        this.artifacts = [];
+    }
+    async uploadAllRepos() {
+        for (const repo of this.repos) {
+            const error = await repo.upload();
+            this.artifacts.push(...repo.uploadedArtifacts);
+            if (error) {
+                throw error;
+            }
+        }
+    }
+    async processAttachment(attachment, stream) {
+        return (0, logger_context_1.runWithSdkLogContext)(async () => {
+            var _a, _b, _c;
+            const { httpStream, delay, error } = await (0, logger_context_1.runWithUserLogContext)(async () => stream({
+                item: attachment,
+                event: this.event,
+            }));
+            if (error) {
+                return { error };
+            }
+            else if (delay) {
+                return { delay };
+            }
+            if (httpStream) {
+                const fileType = attachment.content_type ||
+                    ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
+                    'application/octet-stream';
+                const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
+                const fileSize = contentLength ? parseInt(contentLength) : undefined;
+                // Get upload URL
+                const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
+                if (artifactUrlError) {
+                    this.destroyHttpStream(httpStream);
+                    return {
+                        error: {
+                            message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
+                            fileSize: fileSize,
+                        },
+                    };
+                }
+                // Stream attachment
+                const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
+                if (uploadedArtifactError) {
+                    this.destroyHttpStream(httpStream);
+                    return {
+                        error: {
+                            message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
+                                (0, logger_1.serializeError)(uploadedArtifactError),
+                            fileSize: fileSize,
+                        },
+                    };
+                }
+                // Confirm attachment upload
+                const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
+                if (confirmArtifactUploadError) {
+                    return {
+                        error: {
+                            message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
+                                (0, logger_1.serializeError)(confirmArtifactUploadError),
+                            fileSize: fileSize,
+                        },
+                    };
+                }
+                const ssorAttachment = {
+                    id: {
+                        devrev: artifactUrlResponse.artifact_id,
+                        external: attachment.id,
+                    },
+                    parent_id: {
+                        external: attachment.parent_id,
+                    },
+                };
+                if (attachment.author_id) {
+                    ssorAttachment.actor_id = {
+                        external: attachment.author_id,
+                    };
+                }
+                // This will set inline flag in ssor_attachment only if it is explicity
+                // set in the attachment object.
+                if (attachment.inline === true) {
+                    ssorAttachment.inline = true;
+                }
+                else if (attachment.inline === false) {
+                    ssorAttachment.inline = false;
+                }
+                await ((_c = this.getRepo('ssor_attachment')) === null || _c === void 0 ? void 0 : _c.push([ssorAttachment]));
+                return;
+            }
+            return {
+                error: {
+                    message: `Error while opening attachment stream. Skipping attachment.`,
+                },
+            };
+        });
+    }
+    /**
+     * Destroys a stream to prevent memory leaks.
+     * @param httpStream - The axios response stream to destroy
+     */
+    destroyHttpStream(httpStream) {
+        try {
+            if (httpStream && httpStream.data) {
+                if (typeof httpStream.data.destroy === 'function') {
+                    httpStream.data.destroy();
+                }
+                else if (typeof httpStream.data.close === 'function') {
+                    httpStream.data.close();
+                }
+            }
+        }
+        catch (error) {
+            console.warn('Error while destroying HTTP stream:', error);
+        }
+    }
+    /**
+     * Streams the attachments to the DevRev platform.
+     * The attachments are streamed to the platform and the artifact information is returned.
+     * @param params - The parameters to stream the attachments
+     * @returns The response object containing the ssorAttachment artifact information
+     * or error information if there was an error
+     */
+    async streamAttachments({ stream, processors, batchSize = 1, // By default, we want to stream one attachment at a time
+     }) {
+        return (0, logger_context_1.runWithSdkLogContext)(async () => {
+            var _a, _b;
+            if (batchSize <= 0) {
+                console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
+                batchSize = 1;
+            }
+            if (batchSize > 50) {
+                console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
+                batchSize = 50;
+            }
+            const repos = [
+                {
+                    itemType: 'ssor_attachment',
+                },
+            ];
+            this.initializeRepos(repos);
+            const attachmentsMetadata = (_a = this.adapterState.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
+            // If there are no attachments metadata artifact IDs in state, finish here
+            if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
+                console.log(`No attachments metadata artifact IDs found in state.`);
+                return;
+            }
+            else {
+                console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
+            }
+            // Loop through the attachments metadata artifact IDs
+            while (attachmentsMetadata.artifactIds.length > 0) {
+                const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
+                console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
+                const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
+                    artifact: attachmentsMetadataArtifactId,
+                });
+                if (error) {
+                    console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                    return { error };
+                }
+                if (!attachments || attachments.length === 0) {
+                    console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                    // Remove empty artifact and reset lastProcessed
+                    attachmentsMetadata.artifactIds.shift();
+                    attachmentsMetadata.lastProcessed = 0;
+                    continue;
+                }
+                console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                let response;
+                if (processors) {
+                    console.log(`Using custom processors for attachments.`);
+                    const reducer = processors.reducer;
+                    const iterator = processors.iterator;
+                    const reducedAttachments = (0, logger_context_1.runWithUserLogContext)(() => reducer({
+                        attachments,
+                        adapter: this,
+                        batchSize,
+                    }));
+                    response = await (0, logger_context_1.runWithUserLogContext)(async () => {
+                        return await iterator({
+                            reducedAttachments,
+                            adapter: this,
+                            stream,
+                        });
+                    });
+                }
+                else {
+                    console.log(`Using attachments streaming pool for attachments streaming.`);
+                    const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                        adapter: this,
+                        attachments,
+                        attachmentsMetadata,
+                        batchSize,
+                        stream,
+                    });
+                    response = await attachmentsPool.streamAll();
+                }
+                if ((response === null || response === void 0 ? void 0 : response.delay) || (response === null || response === void 0 ? void 0 : response.error)) {
+                    return response;
+                }
+                // On timeout, emit progress and exit to allow continuation.
+                if (this.isTimeout) {
+                    console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.`);
+                    await this.emit(extraction_1.ExtractorEventType.AttachmentExtractionProgress);
+                    process.exit(0);
+                    return;
+                }
+                console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                attachmentsMetadata.artifactIds.shift();
+                attachmentsMetadata.lastProcessed = 0;
+                if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
+                    attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
+                }
+            }
+            return;
+        });
+    }
+}
+exports.ExtractionAdapter = ExtractionAdapter;

package/dist/multithreading/adapters/extraction-adapter.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=extraction-adapter.test.d.ts.map

package/dist/multithreading/adapters/extraction-adapter.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction-adapter.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.test.ts"],"names":[],"mappings":""}

package/dist/multithreading/{worker-adapter/worker-adapter.extraction.test.js → adapters/extraction-adapter.test.js}

@@ -1,11 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
-const state_1 = require("../../state/state");
+const extraction_state_1 = require("../../state/extraction-state");
 const jest_setup_1 = require("../../tests/jest.setup");
 const test_utils_1 = require("../../common/test-utils");
 const types_1 = require("../../types");
-const worker_adapter_1 = require("./worker-adapter");
+const extraction_adapter_1 = require("./extraction-adapter");
 /* eslint-disable @typescript-eslint/no-require-imports */
 jest.mock('../../common/control-protocol', () => ({
     emit: jest.fn().mockResolvedValue({}),
@@ -27,29 +27,30 @@ function makeAdapter(eventType = types_1.EventType.StartExtractingData) {
     });
     const initialState = {
         attachments: { completed: false },
-        snapInVersionId: '',
-        toDevRev: {
-            attachmentsMetadata: {
-                artifactIds: [],
-                lastProcessed: 0,
-                lastProcessedAttachmentsIdsList: [],
-            },
+    };
+    const adapterState = new extraction_state_1.ExtractionState({ event, initialState });
+    adapterState.sdkState.snapInVersionId = '';
+    adapterState.sdkState.toDevRev = {
+        attachmentsMetadata: {
+            artifactIds: [],
+            lastProcessed: 0,
+            lastProcessedAttachmentsIdsList: [],
         },
     };
-    const adapterState = new state_1.State({ event, initialState });
-    const adapter = new worker_adapter_1.WorkerAdapter({ event, adapterState });
+    const adapter = new extraction_adapter_1.ExtractionAdapter({ event, adapterState });
     return { adapter, event, adapterState };
 }
-describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
+describe(`${extraction_adapter_1.ExtractionAdapter.name}.streamAttachments`, () => {
     let adapter;
+    let adapterState;
     beforeEach(() => {
         jest.clearAllMocks();
-        ({ adapter } = makeAdapter());
+        ({ adapter, adapterState } = makeAdapter());
     });
     it('should process all artifact batches successfully', async () => {
         // Arrange
         const mockStream = jest.fn();
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1', 'artifact2'],
                 lastProcessed: 0,
@@ -95,14 +96,14 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         ]);
         expect(adapter.initializeRepos).toHaveBeenCalledTimes(1);
         expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
-        expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
-        expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
         expect(result).toBeUndefined();
     });
     it('[edge] should handle invalid batch size by using 1 instead', async () => {
         // Arrange
         const mockStream = jest.fn();
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1'],
                 lastProcessed: 0,
@@ -133,7 +134,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
     it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => {
         // Arrange
         const mockStream = jest.fn();
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1'],
                 lastProcessed: 0,
@@ -164,7 +165,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
     it('[edge] should handle empty attachments metadata artifact IDs', async () => {
         // Arrange
         const mockStream = jest.fn();
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: [],
                 lastProcessed: 0,
@@ -180,7 +181,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
     it('[edge] should handle errors when getting attachments', async () => {
         // Arrange
         const mockStream = jest.fn();
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1'],
                 lastProcessed: 0,
@@ -206,7 +207,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
     it('[edge] should handle empty attachments array from artifact', async () => {
         // Arrange
         const mockStream = jest.fn();
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1'],
                 lastProcessed: 0,
@@ -224,7 +225,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
             stream: mockStream,
         });
         // Assert
-        expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
         expect(result).toBeUndefined();
     });
     it('should use custom processors when provided', async () => {
@@ -232,7 +233,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         const mockStream = jest.fn();
         const mockReducer = jest.fn().mockReturnValue(['custom-reduced']);
         const mockIterator = jest.fn().mockResolvedValue({});
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1'],
                 lastProcessed: 0,
@@ -272,7 +273,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => ({
             streamAll: jest.fn().mockResolvedValue({ delay: 30 }),
         }));
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1'],
                 lastProcessed: 0,
@@ -291,9 +292,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         });
         // Assert
         expect(result).toEqual({ delay: 30 });
-        expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
-            'artifact1',
-        ]);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1']);
     });
     it('should handle error from iterator', async () => {
         // Arrange
@@ -303,7 +302,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
                 error: 'Mock error',
             }),
         }));
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1'],
                 lastProcessed: 0,
@@ -322,9 +321,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         });
         // Assert
         expect(result).toEqual({ error: 'Mock error' });
-        expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
-            'artifact1',
-        ]);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1']);
     });
     it('should emit progress event and exit process on timeout, preserving state for resumption', async () => {
         // Arrange
@@ -332,7 +329,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         const exitSpy = jest
             .spyOn(process, 'exit')
             .mockImplementation(() => undefined);
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1', 'artifact2', 'artifact3'],
                 lastProcessed: 0,
@@ -366,11 +363,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         // Assert
         expect(emitSpy).toHaveBeenCalledWith(types_1.ExtractorEventType.AttachmentExtractionProgress);
         expect(exitSpy).toHaveBeenCalledWith(0);
-        expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
-            'artifact1',
-            'artifact2',
-            'artifact3',
-        ]);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact1', 'artifact2', 'artifact3']);
         expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(1);
         exitSpy.mockRestore();
     });
@@ -382,7 +375,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         const exitSpy = jest
             .spyOn(process, 'exit')
             .mockImplementation(() => undefined);
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1', 'artifact2', 'artifact3'],
                 lastProcessed: 0,
@@ -424,16 +417,13 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
         expect(exitSpy).toHaveBeenCalledWith(0);
         // - Artifact 1 was shifted out cleanly; artifact 2 remains (timeout caught
         //   before its shift) along with the untouched artifact 3
-        expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
-            'artifact2',
-            'artifact3',
-        ]);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toEqual(['artifact2', 'artifact3']);
         exitSpy.mockRestore();
     });
     it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => {
         // Arrange
         const mockStream = jest.fn();
-        adapter.state.toDevRev = {
+        adapterState.sdkState.toDevRev = {
             attachmentsMetadata: {
                 artifactIds: ['artifact1'],
                 lastProcessed: 0,
@@ -470,11 +460,11 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.streamAttachments`, () => {
             stream: mockStream,
         });
         // Assert
-        expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength(0);
-        expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.artifactIds).toHaveLength(0);
+        expect(adapterState.sdkState.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
     });
 });
-describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
+describe(`${extraction_adapter_1.ExtractionAdapter.name}.processAttachment`, () => {
     let adapter;
     beforeEach(() => {
         jest.clearAllMocks();
@@ -662,7 +652,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.processAttachment`, () => {
         expect((_a = result === null || result === void 0 ? void 0 : result.error) === null || _a === void 0 ? void 0 : _a.message).toContain('Error while opening attachment stream');
     });
 });
-describe(`${worker_adapter_1.WorkerAdapter.name}.initializeRepos — event size threshold`, () => {
+describe(`${extraction_adapter_1.ExtractionAdapter.name}.initializeRepos — event size threshold`, () => {
     it('should set isTimeout=true once the cumulative artifact payload exceeds EVENT_SIZE_THRESHOLD_BYTES', () => {
         // Arrange
         const { adapter } = makeAdapter();
@@ -684,7 +674,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.initializeRepos — event size
         expect(adapter.isTimeout).toBe(true);
     });
 });
-describe(`${worker_adapter_1.WorkerAdapter.name}.getRepo`, () => {
+describe(`${extraction_adapter_1.ExtractionAdapter.name}.getRepo`, () => {
     it('should return undefined when the requested repo was never initialised', () => {
         // Arrange
         const { adapter } = makeAdapter();
@@ -694,7 +684,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.getRepo`, () => {
         expect(result).toBeUndefined();
     });
 });
-describe(`${worker_adapter_1.WorkerAdapter.name}.destroyHttpStream`, () => {
+describe(`${extraction_adapter_1.ExtractionAdapter.name}.destroyHttpStream`, () => {
     let adapter;
     beforeEach(() => {
         ({ adapter } = makeAdapter());
@@ -752,7 +742,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name}.destroyHttpStream`, () => {
         expect(() => adapter['destroyHttpStream'](httpStream)).not.toThrow();
     });
 });
-describe(`${worker_adapter_1.WorkerAdapter.name} — extractionScope`, () => {
+describe(`${extraction_adapter_1.ExtractionAdapter.name} — extractionScope`, () => {
     it('should return empty object by default', () => {
         const { adapter } = makeAdapter();
         expect(adapter.extractionScope).toEqual({});
@@ -767,7 +757,7 @@ describe(`${worker_adapter_1.WorkerAdapter.name} — extractionScope`, () => {
         expect(adapter.extractionScope).toEqual(extractionScope);
     });
 });
-describe(`${worker_adapter_1.WorkerAdapter.name} — shouldExtract`, () => {
+describe(`${extraction_adapter_1.ExtractionAdapter.name} — shouldExtract`, () => {
     it('should return true when extraction scope is empty', () => {
         const { adapter } = makeAdapter();
         expect(adapter.shouldExtract('tasks')).toBe(true);

package/dist/multithreading/adapters/loading-adapter.d.ts

@@ -0,0 +1,44 @@
+import { Mappers } from '../../mappers/mappers';
+import { LoadingState } from '../../state/loading-state';
+import { AirSyncEvent, EventData, ExtractorEventType } from '../../types/extraction';
+import { ExternalSystemAttachment, ExternalSystemItem, ExternalSystemLoadingFunction, FileToLoad, ItemTypesToLoadParams, ItemTypeToLoad, LoaderEventType, LoaderReport, LoadItemResponse, LoadItemTypesResponse } from '../../types/loading';
+import { WorkerAdapterOptions } from '../../types/workers';
+import { BaseAdapter } from './base-adapter';
+/**
+ * LoadingAdapter is the adapter passed to loading tasks. It exposes the loading
+ * surface (item/attachment loading, mappers, loader reports).
+ *
+ * @public
+ */
+export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState, LoadingState<ConnectorState>> {
+    private loaderReports;
+    private _processedFiles;
+    private _mappers;
+    constructor(params: {
+        event: AirSyncEvent;
+        adapterState: LoadingState<ConnectorState>;
+        options?: WorkerAdapterOptions;
+    });
+    get reports(): LoaderReport[];
+    get processedFiles(): string[];
+    get mappers(): Mappers;
+    protected beforeEmit(): Promise<void>;
+    protected buildEmitPayload(newEventType: ExtractorEventType | LoaderEventType): EventData;
+    protected afterEmit(): void;
+    loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<LoadItemTypesResponse>;
+    getLoaderBatches({ supportedItemTypes, }: {
+        supportedItemTypes: string[];
+    }): Promise<FileToLoad[]>;
+    loadAttachments({ create, }: {
+        create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
+    }): Promise<LoadItemTypesResponse>;
+    loadItem({ item, itemTypeToLoad, }: {
+        item: ExternalSystemItem;
+        itemTypeToLoad: ItemTypeToLoad;
+    }): Promise<LoadItemResponse>;
+    loadAttachment({ item, create, }: {
+        item: ExternalSystemAttachment;
+        create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
+    }): Promise<LoadItemResponse>;
+}
+//# sourceMappingURL=loading-adapter.d.ts.map

package/dist/multithreading/adapters/loading-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EACnB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,wBAAwB,EACxB,kBAAkB,EAClB,6BAA6B,EAC7B,UAAU,EACV,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C;;;;;GAKG;AACH,qBAAa,cAAc,CAAC,cAAc,CAAE,SAAQ,WAAW,CAC7D,cAAc,EACd,YAAY,CAAC,cAAc,CAAC,CAC7B;IACC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAU;gBAEd,MAAM,EAAE;QAClB,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,YAAY,CAAC,cAAc,CAAC,CAAC;QAC3C,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAUD,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;cAEe,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3C,SAAS,CAAC,gBAAgB,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAYZ,SAAS,CAAC,SAAS,IAAI,IAAI;IAIrB,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwInD,gBAAgB,CAAC,EACrB,kBAAkB,GACnB,EAAE;QACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B;IA4BK,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyG5B,QAAQ,CAAC,EACb,IAAI,EACJ,cAAc,GACf,EAAE;QACD,IAAI,EAAE,kBAAkB,CAAC;QACzB,cAAc,EAAE,cAAc,CAAC;KAChC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwMvB,cAAc,CAAC,EACnB,IAAI,EACJ,MAAM,GACP,EAAE;QACD,IAAI,EAAE,wBAAwB,CAAC;QAC/B,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsD9B"}

package/dist/multithreading/{worker-adapter/worker-adapter.helpers.d.ts → adapters/loading-adapter.helpers.d.ts}

@@ -19,4 +19,4 @@ export declare function addReportToLoaderReport({ loaderReports, report, }: {
     loaderReports: LoaderReport[];
     report: LoaderReport;
 }): LoaderReport[];
-//# sourceMappingURL=worker-adapter.helpers.d.ts.map
+//# sourceMappingURL=loading-adapter.helpers.d.ts.map

package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loading-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}