@devrev/airsync-sdk 2.0.0-beta.0

package/README.md

@@ -0,0 +1,31 @@
+# AirSync SDK
+
+[![Coverage Status](https://coveralls.io/repos/github/devrev/adaas-sdk/badge.svg?branch=v2&t=s4Otlm)](https://coveralls.io/github/devrev/adaas-sdk?branch=v2)
+
+> **Note:** This is the v2 beta of the AirSync SDK (formerly Airdrop SDK). The package has been renamed from `@devrev/ts-adaas` to `@devrev/airsync-sdk`.
+
+## Overview
+
+The AirSync SDK for TypeScript helps developers build snap-ins that integrate with DevRev's AirSync platform. This SDK simplifies the workflow for handling data extraction and loading, event-driven actions, state management, and artifact handling.
+
+It provides features such as:
+
+- Type Definitions: Structured types for AirSync control protocol
+- Event Management: Easily emit events for different extraction or loading phases
+- State Handling: Update and access state in real-time within tasks
+- Artifact Management: Supports batched storage of artifacts
+- Error & Timeout Support: Error handling and timeout management for long-running tasks
+
+## Installation
+
+```bash
+npm install @devrev/airsync-sdk@beta
+```
+
+## Changelog
+
+See [CHANGELOG.md](./CHANGELOG.md) for release notes and migration guides.
+
+## Reference
+
+Please refer to the [REFERENCE.md](./REFERENCE.md) file for more information on the types, interfaces and functions used in the library.

package/dist/attachments-streaming/attachments-streaming-pool.d.ts

@@ -0,0 +1,16 @@
+import { ProcessAttachmentReturnType } from '../types';
+import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces';
+export declare class AttachmentsStreamingPool<ConnectorState> {
+    private adapter;
+    private attachments;
+    private batchSize;
+    private delay;
+    private stream;
+    private totalProcessedCount;
+    private readonly PROGRESS_REPORT_INTERVAL;
+    constructor({ adapter, attachments, batchSize, stream, }: AttachmentsStreamingPoolParams<ConnectorState>);
+    private updateProgress;
+    streamAll(): Promise<ProcessAttachmentReturnType>;
+    startPoolStreaming(): Promise<void>;
+}
+//# sourceMappingURL=attachments-streaming-pool.d.ts.map

package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"attachments-streaming-pool.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,2BAA2B,EAC5B,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,8BAA8B,EAAE,MAAM,yCAAyC,CAAC;AAEzF,qBAAa,wBAAwB,CAAC,cAAc;IAClD,OAAO,CAAC,OAAO,CAAgC;IAC/C,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAM;gBAEnC,EACV,OAAO,EACP,WAAW,EACX,SAAc,EACd,MAAM,GACP,EAAE,8BAA8B,CAAC,cAAc,CAAC;YAQnC,cAAc;IAStB,SAAS,IAAI,OAAO,CAAC,2BAA2B,CAAC;IAuCjD,kBAAkB;CA0EzB"}

package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts

@@ -0,0 +1,9 @@
+import { ExternalSystemAttachmentStreamingFunction, NormalizedAttachment } from '../types';
+import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter';
+export interface AttachmentsStreamingPoolParams<ConnectorState> {
+    adapter: WorkerAdapter<ConnectorState>;
+    attachments: NormalizedAttachment[];
+    batchSize?: number;
+    stream: ExternalSystemAttachmentStreamingFunction;
+}
+//# sourceMappingURL=attachments-streaming-pool.interfaces.d.ts.map

package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yCAAyC,EACzC,oBAAoB,EACrB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAEhF,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}

package/dist/attachments-streaming/attachments-streaming-pool.interfaces.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/attachments-streaming/attachments-streaming-pool.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AttachmentsStreamingPool = void 0;
+const helpers_1 = require("../common/helpers");
+class AttachmentsStreamingPool {
+    constructor({ adapter, attachments, batchSize = 10, stream, }) {
+        this.totalProcessedCount = 0;
+        this.PROGRESS_REPORT_INTERVAL = 50;
+        this.adapter = adapter;
+        this.attachments = [...attachments]; // Create a copy we can mutate
+        this.batchSize = batchSize;
+        this.delay = undefined;
+        this.stream = stream;
+    }
+    async updateProgress() {
+        this.totalProcessedCount++;
+        if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) {
+            console.info(`Processed ${this.totalProcessedCount} attachments so far.`);
+            // Sleep for 100ms to avoid blocking the event loop
+            await (0, helpers_1.sleep)(100);
+        }
+    }
+    async streamAll() {
+        console.log(`Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.`);
+        if (!this.adapter.state.toDevRev) {
+            const error = new Error('toDevRev state is not initialized');
+            console.error(error);
+            return { error };
+        }
+        // Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction.
+        // If no such list exists, create an empty one.
+        if (!this.adapter.state.toDevRev.attachmentsMetadata
+            .lastProcessedAttachmentsIdsList) {
+            this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
+                [];
+        }
+        // Start initial batch of promises up to batchSize limit
+        const initialBatchSize = Math.min(this.batchSize, this.attachments.length);
+        const initialPromises = [];
+        for (let i = 0; i < initialBatchSize; i++) {
+            initialPromises.push(this.startPoolStreaming());
+        }
+        // Wait for all promises to complete
+        await Promise.all(initialPromises);
+        if (this.delay) {
+            return { delay: this.delay };
+        }
+        return {};
+    }
+    async startPoolStreaming() {
+        var _a, _b, _c, _d;
+        // Process attachments until the attachments array is empty
+        while (this.attachments.length > 0) {
+            // If delay is set, stop streaming
+            if (this.delay) {
+                break;
+            }
+            // If timeout is set, stop streaming
+            if (this.adapter.isTimeout) {
+                console.log('Timeout detected while streaming attachments. Stopping streaming.');
+                break;
+            }
+            // Check if we can process next attachment
+            const attachment = this.attachments.shift();
+            if (!attachment) {
+                break; // Exit if no more attachments
+            }
+            if (this.adapter.state.toDevRev &&
+                ((_a = this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList) === null || _a === void 0 ? void 0 : _a.includes(attachment.id))) {
+                continue; // Skip if the attachment ID is already processed
+            }
+            try {
+                const response = await this.adapter.processAttachment(attachment, this.stream);
+                // Check if rate limit was hit
+                if (response === null || response === void 0 ? void 0 : response.delay) {
+                    this.delay = response.delay; // Set the delay for rate limiting
+                    return;
+                }
+                if (response === null || response === void 0 ? void 0 : response.error) {
+                    console.warn(`Skipping attachment with ID ${attachment.id} due to error returned by the stream function`, response.error);
+                    await this.updateProgress();
+                    continue;
+                }
+                // No rate limiting, process normally
+                if ((_c = (_b = this.adapter.state.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata) === null || _c === void 0 ? void 0 : _c.lastProcessedAttachmentsIdsList) {
+                    (_d = this.adapter.state.toDevRev) === null || _d === void 0 ? void 0 : _d.attachmentsMetadata.lastProcessedAttachmentsIdsList.push(attachment.id);
+                }
+                await this.updateProgress();
+            }
+            catch (error) {
+                console.warn(`Skipping attachment with ID ${attachment.id} due to error in processAttachment function`, error);
+                await this.updateProgress();
+            }
+        }
+    }
+}
+exports.AttachmentsStreamingPool = AttachmentsStreamingPool;

package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=attachments-streaming-pool.test.d.ts.map

package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"attachments-streaming-pool.test.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.test.ts"],"names":[],"mappings":""}

package/dist/attachments-streaming/attachments-streaming-pool.test.js

@@ -0,0 +1,267 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const attachments_streaming_pool_1 = require("./attachments-streaming-pool");
+/* eslint-disable @typescript-eslint/no-explicit-any */
+describe(attachments_streaming_pool_1.AttachmentsStreamingPool.name, () => {
+    let mockAdapter;
+    let mockStream;
+    let mockAttachments;
+    beforeEach(() => {
+        // Create mock adapter
+        mockAdapter = {
+            state: {
+                attachments: { completed: false },
+                toDevRev: {
+                    attachmentsMetadata: {
+                        lastProcessedAttachmentsIdsList: [],
+                    },
+                },
+            },
+            processAttachment: jest
+                .fn()
+                .mockResolvedValue({}),
+        };
+        // Create mock stream function
+        mockStream = jest.fn().mockResolvedValue({ success: true });
+        // Create mock attachments
+        mockAttachments = [
+            {
+                id: 'attachment-1',
+                url: 'https://example.com/file1.pdf',
+                file_name: 'file1.pdf',
+                parent_id: 'parent-1',
+            },
+            {
+                id: 'attachment-2',
+                url: 'https://example.com/file2.jpg',
+                file_name: 'file2.jpg',
+                parent_id: 'parent-2',
+            },
+            {
+                id: 'attachment-3',
+                url: 'https://example.com/file3.doc',
+                file_name: 'file3.doc',
+                parent_id: 'parent-3',
+            },
+        ];
+        // Mock console methods
+        jest.spyOn(console, 'log').mockImplementation();
+        jest.spyOn(console, 'error').mockImplementation();
+        jest.spyOn(console, 'warn').mockImplementation();
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+        jest.restoreAllMocks();
+    });
+    describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.constructor.name, () => {
+        it('should initialize with default values', () => {
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            expect(pool).toBeDefined();
+            expect(pool['adapter']).toBe(mockAdapter);
+            expect(pool['attachments']).toEqual(mockAttachments);
+            expect(pool['batchSize']).toBe(10);
+            expect(pool['stream']).toBe(mockStream);
+        });
+        it('should initialize with custom batch size', () => {
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                batchSize: 5,
+                stream: mockStream,
+            });
+            expect(pool['batchSize']).toBe(5);
+        });
+        it('should create a copy of attachments array', () => {
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            expect(pool['attachments']).toEqual(mockAttachments);
+            expect(pool['attachments']).not.toBe(mockAttachments); // Different reference
+        });
+    });
+    describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.streamAll.name, () => {
+        it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => {
+            mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
+                undefined;
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            // Mock startPoolStreaming to avoid actual processing
+            jest
+                .spyOn(pool, 'startPoolStreaming')
+                .mockResolvedValue(undefined);
+            await pool.streamAll();
+            expect(mockAdapter.state.toDevRev.attachmentsMetadata
+                .lastProcessedAttachmentsIdsList).toEqual([]);
+        });
+        it('should process all attachments successfully', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            const result = await pool.streamAll();
+            expect(result).toEqual({});
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+        });
+        it('should handle empty attachments array', async () => {
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: [],
+                stream: mockStream,
+            });
+            const result = await pool.streamAll();
+            expect(result).toEqual({});
+            expect(mockAdapter.processAttachment).not.toHaveBeenCalled();
+        });
+        it('should return delay when rate limit is hit', async () => {
+            const delayResponse = { delay: 5000 };
+            mockAdapter.processAttachment.mockResolvedValue(delayResponse);
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            const result = await pool.streamAll();
+            expect(result).toEqual({ delay: 5000 });
+        });
+    });
+    describe(attachments_streaming_pool_1.AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => {
+        it('should skip already processed attachments', async () => {
+            mockAdapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList =
+                ['attachment-1'];
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            await pool.streamAll();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3
+        });
+        it('should add successfully processed attachment IDs to the list', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            await pool.streamAll();
+            expect(mockAdapter.state.toDevRev.attachmentsMetadata
+                .lastProcessedAttachmentsIdsList).toEqual(['attachment-1', 'attachment-2', 'attachment-3']);
+        });
+        it('should handle processing errors gracefully', async () => {
+            const error = new Error('Processing failed');
+            mockAdapter.processAttachment
+                .mockResolvedValueOnce({}) // First attachment succeeds
+                .mockRejectedValueOnce(error) // Second attachment fails
+                .mockResolvedValueOnce({}); // Third attachment succeeds
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            await pool.streamAll();
+            expect(console.warn).toHaveBeenCalledWith('Skipping attachment with ID attachment-2 due to error in processAttachment function', error);
+            expect(mockAdapter.state.toDevRev.attachmentsMetadata
+                .lastProcessedAttachmentsIdsList).toEqual(['attachment-1', 'attachment-3']);
+        });
+        it('should stop processing when rate limit delay is encountered', async () => {
+            mockAdapter.processAttachment
+                .mockResolvedValueOnce({}) // First attachment succeeds
+                .mockResolvedValueOnce({ delay: 5000 }) // Second attachment triggers rate limit
+                .mockResolvedValueOnce({}); // Third attachment succeeds
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                stream: mockStream,
+            });
+            await pool.streamAll();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+            expect(mockAdapter.state.toDevRev.attachmentsMetadata
+                .lastProcessedAttachmentsIdsList).toEqual(['attachment-1', 'attachment-3']);
+        });
+        it('should pass correct parameters to processAttachment', async () => {
+            mockAdapter.processAttachment.mockResolvedValue({});
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: [mockAttachments[0]],
+                stream: mockStream,
+            });
+            await pool.streamAll();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledWith(mockAttachments[0], mockStream);
+        });
+    });
+    it('[edge] should handle single attachment', async () => {
+        mockAdapter.processAttachment.mockResolvedValue({});
+        const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+            adapter: mockAdapter,
+            attachments: [mockAttachments[0]],
+            stream: mockStream,
+        });
+        const result = await pool.streamAll();
+        expect(result).toEqual({});
+        expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1);
+    });
+    it('[edge] should handle batch size larger than attachments array', async () => {
+        mockAdapter.processAttachment.mockResolvedValue({});
+        const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+            adapter: mockAdapter,
+            attachments: mockAttachments,
+            batchSize: 100,
+            stream: mockStream,
+        });
+        await pool.streamAll();
+        expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+    });
+    it('[edge] should handle batch size of 1', async () => {
+        mockAdapter.processAttachment.mockResolvedValue({});
+        const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+            adapter: mockAdapter,
+            attachments: mockAttachments,
+            batchSize: 1,
+            stream: mockStream,
+        });
+        await pool.streamAll();
+        expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+    });
+    describe('concurrency behavior', () => {
+        it('should process attachments concurrently within batch size', async () => {
+            let processCallCount = 0;
+            const processPromises = [];
+            mockAdapter.processAttachment.mockImplementation(async () => {
+                const promise = new Promise((resolve) => {
+                    setTimeout(() => {
+                        processCallCount++;
+                        resolve({});
+                    }, 100);
+                });
+                processPromises.push(promise);
+                return promise;
+            });
+            const pool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                adapter: mockAdapter,
+                attachments: mockAttachments,
+                batchSize: 2,
+                stream: mockStream,
+            });
+            const startTime = Date.now();
+            await pool.streamAll();
+            const endTime = Date.now();
+            expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3);
+            expect(processCallCount).toBe(3);
+            // Should complete in roughly 200ms (2 batches of 100ms each) rather than 300ms (sequential)
+            expect(endTime - startTime).toBeLessThan(250);
+        });
+    });
+});

package/dist/common/constants.d.ts

@@ -0,0 +1,25 @@
+import { EventType } from '../types/extraction';
+export declare const ALLOWED_EXTRACTION_EVENT_TYPES: EventType[];
+export declare const ALLOWED_LOADING_EVENT_TYPES: EventType[];
+export declare const ALLOWED_EVENT_TYPES: EventType[];
+export declare const STATELESS_EXTRACTION_EVENT_TYPES: EventType[];
+export declare const STATELESS_LOADING_EVENT_TYPES: EventType[];
+export declare const STATELESS_EVENT_TYPES: EventType[];
+export declare const STATEFUL_EXTRACTION_EVENT_TYPES: EventType[];
+export declare const STATEFUL_LOADING_EVENT_TYPES: EventType[];
+export declare const STATEFUL_EVENT_TYPES: EventType[];
+export declare const ARTIFACT_BATCH_SIZE = 2000;
+export declare const MAX_DEVREV_ARTIFACT_SIZE: number;
+export declare const MAX_DEVREV_FILENAME_LENGTH = 256;
+export declare const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20;
+export declare const AIRDROP_DEFAULT_ITEM_TYPES: {
+    EXTERNAL_DOMAIN_METADATA: string;
+    ATTACHMENTS: string;
+    SSOR_ATTACHMENT: string;
+};
+export declare const LIBRARY_VERSION: any;
+export declare const DEFAULT_LAMBDA_TIMEOUT: number;
+export declare const HARD_TIMEOUT_MULTIPLIER = 1.3;
+export declare const MEMORY_LOG_INTERVAL: number;
+export declare const DEFAULT_SLEEP_DELAY_MS: number;
+//# sourceMappingURL=constants.d.ts.map

package/dist/common/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/common/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGhD,eAAO,MAAM,8BAA8B,aAS1C,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAKvC,CAAC;AAEF,eAAO,MAAM,mBAAmB,aAG/B,CAAC;AAEF,eAAO,MAAM,gCAAgC,aAI5C,CAAC;AAEF,eAAO,MAAM,6BAA6B,aAGzC,CAAC;AAEF,eAAO,MAAM,qBAAqB,aAGjC,CAAC;AAEF,eAAO,MAAM,+BAA+B,aAGzC,CAAC;AAEJ,eAAO,MAAM,4BAA4B,aAExC,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAGhC,CAAC;AAEF,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,eAAO,MAAM,wBAAwB,QAAyB,CAAC;AAC/D,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAC9C,eAAO,MAAM,oCAAoC,KAAK,CAAC;AAEvD,eAAO,MAAM,0BAA0B;;;;CAItC,CAAC;AAEF,eAAO,MAAM,eAAe,KAAsB,CAAC;AAEnD,eAAO,MAAM,sBAAsB,QAAiB,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C,eAAO,MAAM,sBAAsB,QAAgB,CAAC"}

package/dist/common/constants.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_SLEEP_DELAY_MS = exports.MEMORY_LOG_INTERVAL = exports.HARD_TIMEOUT_MULTIPLIER = exports.DEFAULT_LAMBDA_TIMEOUT = exports.LIBRARY_VERSION = exports.AIRDROP_DEFAULT_ITEM_TYPES = exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = exports.MAX_DEVREV_FILENAME_LENGTH = exports.MAX_DEVREV_ARTIFACT_SIZE = exports.ARTIFACT_BATCH_SIZE = exports.STATEFUL_EVENT_TYPES = exports.STATEFUL_LOADING_EVENT_TYPES = exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.STATELESS_EVENT_TYPES = exports.STATELESS_LOADING_EVENT_TYPES = exports.STATELESS_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES = void 0;
+const extraction_1 = require("../types/extraction");
+const helpers_1 = require("./helpers");
+exports.ALLOWED_EXTRACTION_EVENT_TYPES = [
+    extraction_1.EventType.StartExtractingExternalSyncUnits,
+    extraction_1.EventType.StartExtractingMetadata,
+    extraction_1.EventType.StartExtractingData,
+    extraction_1.EventType.ContinueExtractingData,
+    extraction_1.EventType.StartDeletingExtractorState,
+    extraction_1.EventType.StartExtractingAttachments,
+    extraction_1.EventType.ContinueExtractingAttachments,
+    extraction_1.EventType.StartDeletingExtractorAttachmentsState,
+];
+exports.ALLOWED_LOADING_EVENT_TYPES = [
+    extraction_1.EventType.StartLoadingData,
+    extraction_1.EventType.ContinueLoadingData,
+    extraction_1.EventType.StartDeletingLoaderState,
+    extraction_1.EventType.StartDeletingLoaderAttachmentState,
+];
+exports.ALLOWED_EVENT_TYPES = [
+    ...exports.ALLOWED_EXTRACTION_EVENT_TYPES,
+    ...exports.ALLOWED_LOADING_EVENT_TYPES,
+];
+exports.STATELESS_EXTRACTION_EVENT_TYPES = [
+    extraction_1.EventType.StartExtractingExternalSyncUnits,
+    extraction_1.EventType.StartDeletingExtractorState,
+    extraction_1.EventType.StartDeletingExtractorAttachmentsState,
+];
+exports.STATELESS_LOADING_EVENT_TYPES = [
+    extraction_1.EventType.StartDeletingLoaderState,
+    extraction_1.EventType.StartDeletingLoaderAttachmentState,
+];
+exports.STATELESS_EVENT_TYPES = [
+    ...exports.STATELESS_EXTRACTION_EVENT_TYPES,
+    ...exports.STATELESS_LOADING_EVENT_TYPES,
+];
+exports.STATEFUL_EXTRACTION_EVENT_TYPES = exports.ALLOWED_EXTRACTION_EVENT_TYPES.filter((eventType) => !exports.STATELESS_EXTRACTION_EVENT_TYPES.includes(eventType));
+exports.STATEFUL_LOADING_EVENT_TYPES = exports.ALLOWED_LOADING_EVENT_TYPES.filter((eventType) => !exports.STATELESS_LOADING_EVENT_TYPES.includes(eventType));
+exports.STATEFUL_EVENT_TYPES = [
+    ...exports.STATEFUL_EXTRACTION_EVENT_TYPES,
+    ...exports.STATEFUL_LOADING_EVENT_TYPES,
+];
+exports.ARTIFACT_BATCH_SIZE = 2000;
+exports.MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
+exports.MAX_DEVREV_FILENAME_LENGTH = 256;
+exports.MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20; // 20 characters for the file extension
+exports.AIRDROP_DEFAULT_ITEM_TYPES = {
+    EXTERNAL_DOMAIN_METADATA: 'external_domain_metadata',
+    ATTACHMENTS: 'attachments',
+    SSOR_ATTACHMENT: 'ssor_attachment',
+};
+exports.LIBRARY_VERSION = (0, helpers_1.getLibraryVersion)();
+exports.DEFAULT_LAMBDA_TIMEOUT = 10 * 60 * 1000; // 10 minutes
+exports.HARD_TIMEOUT_MULTIPLIER = 1.3;
+exports.MEMORY_LOG_INTERVAL = 30 * 1000; // 30 seconds
+exports.DEFAULT_SLEEP_DELAY_MS = 3 * 60 * 1000; // 3 minutes

package/dist/common/control-protocol.d.ts

@@ -0,0 +1,10 @@
+import { AxiosResponse } from 'axios';
+import { AirdropEvent, EventData, ExtractorEventType } from '../types/extraction';
+import { LoaderEventType } from '../types/loading';
+export interface EmitInterface {
+    event: AirdropEvent;
+    eventType: ExtractorEventType | LoaderEventType;
+    data?: EventData;
+}
+export declare const emit: ({ event, eventType, data, }: EmitInterface) => Promise<AxiosResponse>;
+//# sourceMappingURL=control-protocol.d.ts.map

package/dist/common/control-protocol.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"control-protocol.d.ts","sourceRoot":"","sources":["../../src/common/control-protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAEtC,OAAO,EACL,YAAY,EACZ,SAAS,EAET,kBAAkB,EAEnB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,YAAY,CAAC;IACpB,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;IAChD,IAAI,CAAC,EAAE,SAAS,CAAC;CAClB;AAED,eAAO,MAAM,IAAI,GAAU,6BAIxB,aAAa,KAAG,OAAO,CAAC,aAAa,CAgCvC,CAAC"}

package/dist/common/control-protocol.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.emit = void 0;
+const axios_client_internal_1 = require("../http/axios-client-internal");
+const constants_1 = require("./constants");
+const event_type_translation_1 = require("./event-type-translation");
+const emit = async ({ event, eventType, data, }) => {
+    // Translate outgoing event type to ensure we always send new event types
+    // TODO: Remove when the old types are completely phased out
+    const translatedEventType = (0, event_type_translation_1.translateOutgoingEventType)(eventType);
+    const newEvent = {
+        event_type: translatedEventType,
+        event_context: event.payload.event_context,
+        event_data: Object.assign({}, data),
+        worker_metadata: {
+            adaas_library_version: constants_1.LIBRARY_VERSION,
+        },
+    };
+    console.info('Emitting event', newEvent);
+    return axios_client_internal_1.axiosClient.post(event.payload.event_context.callback_url, Object.assign({}, newEvent), {
+        headers: {
+            Accept: 'application/json, text/plain, */*',
+            Authorization: event.context.secrets.service_account_token,
+            'Content-Type': 'application/json',
+            'X-DevRev-Client-Platform': event.payload.event_context.snap_in_slug,
+            'X-DevRev-Client-Id': event.payload.event_context.snap_in_version_id,
+            'X-DevRev-Client-Version': constants_1.LIBRARY_VERSION,
+        },
+    });
+};
+exports.emit = emit;

package/dist/common/errors.d.ts

@@ -0,0 +1,6 @@
+export declare const enum ExtractionCommonError {
+    EXTERNAL_SYNC_UNIT_DELETED = "ERROR_CODE=EXTERNAL_SYNC_UNIT_DELETED",
+    EXTERNAL_SYNC_UNIT_DEACTIVATED = "ERROR_CODE=EXTERNAL_SYNC_UNIT_DEACTIVATED",
+    USER_DELETED = "ERROR_CODE=USER_DELETED"
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/common/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/common/errors.ts"],"names":[],"mappings":"AAGA,0BAAkB,qBAAqB;IACrC,0BAA0B,0CAAgE;IAC1F,8BAA8B,8CAAoE;IAClG,YAAY,4BAAkD;CAC/D"}

package/dist/common/errors.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const ERROR_PREFIX = 'ERROR_CODE';
+const ERROR_DELIMITER = '=';

package/dist/common/event-type-translation.d.ts

@@ -0,0 +1,24 @@
+import { EventType, ExtractorEventType } from '../types/extraction';
+import { LoaderEventType } from '../types/loading';
+/**
+ * Maps old incoming event type strings to new EventType enum values.
+ * This ensures backwards compatibility when the platform sends old event types.
+ * @param eventTypeString The raw event type string from the platform
+ * @returns The translated EventType enum value
+ */
+export declare function translateIncomingEventType(eventTypeString: string): EventType;
+/**
+ * Translates ExtractorEventType enum values by converting old enum members to new ones.
+ * Old enum members are deprecated and should be replaced with new ones.
+ */
+export declare function translateExtractorEventType(eventType: ExtractorEventType): ExtractorEventType;
+/**
+ * Translates LoaderEventType enum values by converting old enum members to new ones.
+ * Old enum members are deprecated and should be replaced with new ones.
+ */
+export declare function translateLoaderEventType(eventType: LoaderEventType): LoaderEventType;
+/**
+ * Translates any outgoing event type (Extractor or Loader) to ensure new event types are used.
+ */
+export declare function translateOutgoingEventType(eventType: ExtractorEventType | LoaderEventType): ExtractorEventType | LoaderEventType;
+//# sourceMappingURL=event-type-translation.d.ts.map

package/dist/common/event-type-translation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"event-type-translation.d.ts","sourceRoot":"","sources":["../../src/common/event-type-translation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,eAAe,EAAE,MAAM,GAAG,SAAS,CAwD7E;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,SAAS,EAAE,kBAAkB,GAC5B,kBAAkB,CA0CpB;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,eAAe,GACzB,eAAe,CAmBjB;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,kBAAkB,GAAG,eAAe,GAC9C,kBAAkB,GAAG,eAAe,CAatC"}