@devrev/ts-adaas 0.0.2 → 1.0.0

package/dist/repo/repo.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Repo = void 0;
+const constants_1 = require("../common/constants");
+const uploader_1 = require("../uploader/uploader");
+class Repo {
+    constructor({ event, itemType, normalize, onUpload, options, }) {
+        this.items = [];
+        this.itemType = itemType;
+        this.normalize = normalize;
+        this.onUpload = onUpload;
+        this.uploader = new uploader_1.Uploader({ event, options });
+    }
+    getItems() {
+        return this.items;
+    }
+    async upload(batch) {
+        const itemsToUpload = batch || this.items;
+        if (itemsToUpload.length > 0) {
+            console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}.`);
+            const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
+            if (error || !artifact) {
+                console.error('Error while uploading batch', error);
+                return error;
+            }
+            this.onUpload(artifact);
+            // Clear the uploaded items from the main items array if no batch was specified
+            if (!batch) {
+                this.items = [];
+            }
+        }
+        else {
+            console.log(`No items to upload for type ${this.itemType}. Skipping upload.`);
+        }
+    }
+    async push(items) {
+        return new Promise(async (resolve, reject) => {
+            let recordsToPush;
+            // Normalize items if needed
+            if (this.normalize &&
+                !Object.values(constants_1.AIRDROP_DEFAULT_ITEM_TYPES).includes(this.itemType)) {
+                recordsToPush = items.map((item) => this.normalize(item));
+            }
+            else {
+                recordsToPush = items;
+            }
+            // Add the new records to the items array
+            this.items.push(...recordsToPush);
+            console.log(`Extracted ${this.items.length} items of type ${this.itemType}.`);
+            // Upload in batches while the number of items exceeds the batch size
+            while (this.items.length >= constants_1.ARTIFACT_BATCH_SIZE) {
+                // Slice out a batch of ARTIFACT_BATCH_SIZE items to upload
+                const batch = this.items.splice(0, constants_1.ARTIFACT_BATCH_SIZE);
+                try {
+                    // Upload the batch
+                    await this.upload(batch);
+                }
+                catch (error) {
+                    console.error('Error while uploading batch', error);
+                    reject(error);
+                    return;
+                }
+            }
+            resolve();
+        });
+    }
+}
+exports.Repo = Repo;

package/dist/repo/repo.test.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const constants_1 = require("../common/constants");
+const test_helpers_1 = require("../tests/test-helpers");
+const types_1 = require("../types");
+const repo_1 = require("./repo");
+jest.mock('../tests/test-helpers', () => (Object.assign(Object.assign({}, jest.requireActual('../tests/test-helpers')), { normalizeItem: jest.fn() })));
+describe('Repo class push method', () => {
+    let repo;
+    let normalize;
+    beforeEach(() => {
+        normalize = jest.fn();
+        repo = new repo_1.Repo({
+            event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
+            itemType: 'test_item_type',
+            normalize,
+            onUpload: jest.fn(),
+            options: {},
+        });
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+    it('should not push items if items array is empty', async () => {
+        await repo.push([]);
+        expect(repo.getItems()).toEqual([]);
+    });
+    it('should normalize and push 10 items if array is not empty', async () => {
+        const items = (0, test_helpers_1.createItems)(10);
+        await repo.push(items);
+        expect(normalize).toHaveBeenCalledTimes(10);
+        const normalizedItems = items.map((item) => (0, test_helpers_1.normalizeItem)(item));
+        expect(repo.getItems()).toEqual(normalizedItems);
+    });
+    it('should not normalize items if normalize function is not provided', async () => {
+        repo = new repo_1.Repo({
+            event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
+            itemType: 'test_item_type',
+            onUpload: jest.fn(),
+            options: {},
+        });
+        const items = (0, test_helpers_1.createItems)(10);
+        await repo.push(items);
+        expect(normalize).not.toHaveBeenCalled();
+    });
+    describe('should not normalize items if they are airdrop default item types', () => {
+        it.each(Object.values(constants_1.AIRDROP_DEFAULT_ITEM_TYPES))('item type: %s', async (itemType) => {
+            repo = new repo_1.Repo({
+                event: (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.ExtractionDataStart }),
+                itemType,
+                normalize,
+                onUpload: jest.fn(),
+                options: {},
+            });
+            const items = (0, test_helpers_1.createItems)(10);
+            await repo.push(items);
+            expect(normalize).not.toHaveBeenCalled();
+        });
+    });
+    it('should leave 5 items in the items array after pushing 2005 items with batch size of 2000', async () => {
+        const items = (0, test_helpers_1.createItems)(2005);
+        await repo.push(items);
+        expect(repo.getItems().length).toBe(5);
+    });
+    it('should upload 2 batches of 2000 and leave 5 items in the items array after pushing 4005 items with batch size of 2000', async () => {
+        const uploadSpy = jest.spyOn(repo, 'upload');
+        const items = (0, test_helpers_1.createItems)(4005);
+        await repo.push(items);
+        expect(normalize).toHaveBeenCalledTimes(4005);
+        expect(repo.getItems().length).toBe(5);
+        expect(uploadSpy).toHaveBeenCalledTimes(2); // Check that upload was called twice
+        uploadSpy.mockRestore();
+    });
+});

package/dist/state/state.d.ts

@@ -0,0 +1,24 @@
+import { ErrorRecord } from '../types/common';
+import { AdapterState, StateInterface } from './state.interfaces';
+export declare function createAdapterState<ConnectorState>({ event, initialState, options, }: StateInterface<ConnectorState>): Promise<State<ConnectorState>>;
+export declare class State<ConnectorState> {
+    private _state;
+    private event;
+    private workerUrl;
+    private devrevToken;
+    constructor({ event, initialState }: StateInterface<ConnectorState>);
+    get state(): AdapterState<ConnectorState>;
+    set state(value: AdapterState<ConnectorState>);
+    /**
+     *  Updates the state of the adapter.
+     *
+     * @param {object} state - The state to be updated
+     */
+    postState(state?: AdapterState<ConnectorState>): Promise<void>;
+    /**
+     *  Fetches the state of the adapter.
+     *
+     * @return  The state of the adapter
+     */
+    fetchState(initialState: ConnectorState): Promise<AdapterState<ConnectorState> | ErrorRecord>;
+}

package/dist/state/state.interfaces.d.ts

@@ -0,0 +1,24 @@
+import { AirdropEvent } from '../types/extraction';
+import { WorkerAdapterOptions } from '../types/workers';
+/**
+ * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, and attachmentsMetadata.
+ */
+export type AdapterState<ConnectorState> = ConnectorState & {
+    lastSyncStarted?: string;
+    lastSuccessfulSyncStarted?: string;
+    toDevRev?: ToDevRev;
+    fromDevRev?: FromDevRev;
+};
+export interface ToDevRev {
+    attachmentsMetadata: {
+        artifactIds: string[];
+        lastProcessed: number;
+    };
+}
+export interface FromDevRev {
+}
+export interface StateInterface<ConnectorState> {
+    event: AirdropEvent;
+    initialState: ConnectorState;
+    options?: WorkerAdapterOptions;
+}

package/dist/state/state.interfaces.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/state/state.js

@@ -0,0 +1,115 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.State = void 0;
+exports.createAdapterState = createAdapterState;
+const axios_1 = __importDefault(require("axios"));
+const constants_1 = require("../common/constants");
+const logger_1 = require("../logger/logger");
+async function createAdapterState({ event, initialState, options, }) {
+    const newInitialState = structuredClone(initialState);
+    const as = new State({
+        event,
+        initialState: newInitialState,
+        options,
+    });
+    if (!constants_1.STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
+        await as.fetchState(newInitialState);
+    }
+    return as;
+}
+class State {
+    constructor({ event, initialState }) {
+        this._state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: new Date().toISOString(), lastSuccessfulSyncStarted: '', fromDevRev: {
+                attachmentsMetadata: {
+                    artifactIds: [],
+                    lastProcessed: 0,
+                },
+            }, toDevRev: {} });
+        this.event = event;
+        this.workerUrl = event.payload.event_context.worker_data_url;
+        this.devrevToken = event.context.secrets.service_account_token;
+    }
+    get state() {
+        return this._state;
+    }
+    set state(value) {
+        this._state = value;
+    }
+    /**
+     *  Updates the state of the adapter.
+     *
+     * @param {object} state - The state to be updated
+     */
+    async postState(state) {
+        try {
+            await axios_1.default.post(this.workerUrl + '.update', {
+                state: JSON.stringify(state || this.state),
+            }, {
+                headers: {
+                    Authorization: this.devrevToken,
+                },
+                params: {
+                    sync_unit: this.event.payload.event_context.sync_unit_id,
+                    request_id: this.event.payload.event_context.uuid,
+                },
+            });
+            this.state = state || this.state;
+            console.log('State updated successfully to:', (0, logger_1.getPrintableState)(this.state));
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error)) {
+                console.error('Failed to update state.', (0, logger_1.formatAxiosError)(error));
+            }
+            else {
+                console.error('Failed to update state.', error);
+            }
+        }
+    }
+    /**
+     *  Fetches the state of the adapter.
+     *
+     * @return  The state of the adapter
+     */
+    async fetchState(initialState) {
+        var _a;
+        console.log('Fetching state with sync unit id: ' +
+            this.event.payload.event_context.sync_unit_id +
+            '.');
+        try {
+            const response = await axios_1.default.post(this.workerUrl + '.get', {}, {
+                headers: {
+                    Authorization: this.devrevToken,
+                },
+                params: {
+                    sync_unit: this.event.payload.event_context.sync_unit_id,
+                    request_id: this.event.payload.event_context.uuid,
+                },
+            });
+            this.state = JSON.parse(response.data.state);
+            console.log('State fetched successfully. Current state:', (0, logger_1.getPrintableState)(this.state));
+            return this.state;
+        }
+        catch (error) {
+            if (axios_1.default.isAxiosError(error) && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
+                const state = Object.assign(Object.assign({}, initialState), { lastSyncStarted: new Date().toISOString(), lastSuccessfulSyncStarted: '', toDevRev: {
+                        attachmentsMetadata: {
+                            artifactIds: [],
+                            lastProcessed: 0,
+                        },
+                    }, fromDevRev: {} });
+                this.state = state;
+                console.log('State not found, returning initial state. Current state:', (0, logger_1.getPrintableState)(this.state));
+                await this.postState(this.state);
+                return this.state;
+            }
+            else {
+                console.error('Failed to fetch state.', error);
+                return { message: 'Failed to fetch state.' };
+            }
+        }
+    }
+}
+exports.State = State;

package/dist/tests/test-helpers.d.ts

@@ -0,0 +1,9 @@
+import { Item, NormalizedAttachment, NormalizedItem } from '../repo/repo.interfaces';
+import { AirdropEvent } from '../types/extraction';
+import { CreateEventInterface } from './test-helpers.interfaces';
+export declare function createEvent({ eventType, externalSyncUnits, progress, error, delay, contextOverrides, payloadOverrides, }: CreateEventInterface): AirdropEvent;
+export declare function createItem(id: number): Item;
+export declare function createItems(count: number): Item[];
+export declare function normalizeItem(item: Item): NormalizedItem;
+export declare function createAttachment(id: number): NormalizedAttachment;
+export declare function createAttachments(count: number): NormalizedAttachment[];

package/dist/tests/test-helpers.interfaces.d.ts

@@ -0,0 +1,11 @@
+import { ErrorRecord } from '../types/common';
+import { AirdropEvent, EventType, ExternalSyncUnit } from '../types/extraction';
+export interface CreateEventInterface {
+    eventType: EventType;
+    externalSyncUnits?: ExternalSyncUnit[];
+    progress?: number;
+    error?: ErrorRecord;
+    delay?: number;
+    contextOverrides?: Partial<AirdropEvent['context']>;
+    payloadOverrides?: Partial<AirdropEvent['payload']>;
+}

package/dist/tests/test-helpers.interfaces.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/tests/test-helpers.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEvent = createEvent;
+exports.createItem = createItem;
+exports.createItems = createItems;
+exports.normalizeItem = normalizeItem;
+exports.createAttachment = createAttachment;
+exports.createAttachments = createAttachments;
+function createEvent({ eventType, externalSyncUnits = [], progress, error, delay, contextOverrides = {}, payloadOverrides = {}, }) {
+    return {
+        context: Object.assign({ secrets: {
+                service_account_token: 'test_token',
+            }, snap_in_version_id: 'test_snap_in_version_id' }, contextOverrides),
+        payload: Object.assign({ connection_data: {
+                org_id: 'test_org_id',
+                org_name: 'test_org_name',
+                key: 'test_key',
+                key_type: 'test_key_type',
+            }, event_context: {
+                mode: 'test_mode',
+                callback_url: 'test_callback_url',
+                dev_org_id: 'test_dev_org_id',
+                dev_user_id: 'test_dev_user_id',
+                external_sync_unit_id: 'test_external_sync_unit_id',
+                sync_unit_id: 'test_sync_unit_id',
+                sync_run_id: 'test_sync_run_id',
+                external_system_id: 'test_external_system_id',
+                uuid: 'test_uuid',
+                worker_data_url: 'test_worker_data_url',
+                external_system: 'test_external_system',
+                external_system_type: 'test_external_system_type',
+                import_slug: 'test_import_slug',
+                snap_in_slug: 'test_snap_in_slug',
+                sync_tier: 'test_sync_tier',
+            }, event_type: eventType, event_data: {
+                external_sync_units: externalSyncUnits,
+                progress,
+                error,
+                delay,
+            }, extractor_state: {
+                test_extractor_state_key: 'test_extractor_state_value',
+            } }, payloadOverrides),
+        execution_metadata: {
+            devrev_endpoint: 'test_devrev_endpoint',
+        },
+        input_data: {
+            global_values: {
+                test_global_key: 'test_global_value',
+            },
+            event_sources: {
+                test_event_source_key: 'test_event_source_id',
+            },
+        },
+    };
+}
+function createItem(id) {
+    return {
+        id,
+        created_at: '2021-01-01',
+        updated_at: '2021-01-01',
+        name: 'item' + id,
+    };
+}
+function createItems(count) {
+    return Array.from({ length: count }, (_, index) => createItem(index));
+}
+function normalizeItem(item) {
+    return {
+        id: item.id,
+        created_date: item.created_at,
+        modified_date: item.updated_at,
+        data: {
+            name: item.name,
+        },
+    };
+}
+function createAttachment(id) {
+    return {
+        id: id.toString(),
+        url: 'https://test.com/' + id,
+        author_id: 'author' + id,
+        file_name: 'file' + id,
+        parent_id: 'parent' + id,
+    };
+}
+function createAttachments(count) {
+    return Array.from({ length: count }, (_, index) => createAttachment(index));
+}

package/dist/tests/test-worker.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const process_task_1 = require("../workers/process-task");
+const extraction_1 = require("../types/extraction");
+(0, process_task_1.processTask)({
+    task: async ({ adapter }) => {
+        await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {});
+        return 0;
+    },
+    onTimeout: async ({ adapter }) => {
+        await adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError, {
+            error: { message: 'External sync unit failed.' },
+        });
+        return 0;
+    },
+});

package/dist/types/common.d.ts

@@ -0,0 +1,39 @@
+import { Artifact } from '../uploader/uploader.interfaces';
+/**
+ * ErrorLevel is an enum that represents the level of an error.
+ * @deprecated
+ */
+export declare enum ErrorLevel {
+    Warning = "WARNING",
+    Error = "ERROR",
+    Info = "INFO"
+}
+/**
+ * ErrorRecord is an interface that defines the structure of an error record.
+ */
+export interface ErrorRecord {
+    message: string;
+}
+/**
+ * LogRecord is an interface that defines the structure of a log record.
+ * @deprecated
+ */
+export interface LogRecord {
+    level: ErrorLevel;
+    message: string;
+}
+/**
+ * AdapterUpdateParams is an interface that defines the structure of the parameters that can be passed to the update adapter.
+ * @deprecated
+ */
+export interface AdapterUpdateParams {
+    artifact?: Artifact;
+    extractor_state?: object;
+}
+/**
+ * InitialDomainMapping is an interface that defines the structure of the initial domain mapping.
+ */
+export interface InitialDomainMapping {
+    starting_recipe_blueprint?: object;
+    additional_mappings?: object;
+}

package/dist/{src/types → types}/common.js

@@ -1,6 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ErrorLevel = void 0;
+/**
+ * ErrorLevel is an enum that represents the level of an error.
+ * @deprecated
+ */
 var ErrorLevel;
 (function (ErrorLevel) {
     ErrorLevel["Warning"] = "WARNING";

package/dist/{src/types → types}/extraction.d.ts

@@ -1,5 +1,10 @@
 import { InputData } from '@devrev/typescript-sdk/dist/snap-ins';
-import { Artifact, ErrorRecord } from './common';
+import { Artifact } from '../uploader/uploader.interfaces';
+import { ErrorRecord } from './common';
+/**
+ * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
+ * The external extractor can use these events to know what to do next in the extraction process.
+ */
 export declare enum EventType {
     ExtractionExternalSyncUnitsStart = "EXTRACTION_EXTERNAL_SYNC_UNITS_START",
     ExtractionMetadataStart = "EXTRACTION_METADATA_START",
@@ -10,6 +15,10 @@ export declare enum EventType {
     ExtractionAttachmentsContinue = "EXTRACTION_ATTACHMENTS_CONTINUE",
     ExtractionAttachmentsDelete = "EXTRACTION_ATTACHMENTS_DELETE"
 }
+/**
+ * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
+ * The external extractor can use these events to inform ADaaS about the progress of the extraction process.
+ */
 export declare enum ExtractorEventType {
     ExtractionExternalSyncUnitsDone = "EXTRACTION_EXTERNAL_SYNC_UNITS_DONE",
     ExtractionExternalSyncUnitsError = "EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR",
@@ -28,23 +37,28 @@ export declare enum ExtractorEventType {
     ExtractionAttachmentsDeleteDone = "EXTRACTION_ATTACHMENTS_DELETE_DONE",
     ExtractionAttachmentsDeleteError = "EXTRACTION_ATTACHMENTS_DELETE_ERROR"
 }
-export interface EventData {
-    external_sync_units?: ExternalSyncUnit[];
-    progress?: number;
-    error?: ErrorRecord;
-    delay?: number;
-    artifacts?: Artifact[];
-}
+/**
+ * ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor.
+ * It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs.
+ */
 export declare enum ExtractionMode {
     INITIAL = "INITIAL",
     INCREMENTAL = "INCREMENTAL"
 }
+/**
+ * ExternalSyncUnit is an interface that defines the structure of an external sync unit (repos, projects, ...) that can be extracted.
+ * It must contain an ID, a name, and a description. It can also contain the number of items in the external sync unit.
+ */
 export interface ExternalSyncUnit {
     id: string;
     name: string;
     description: string;
     item_count?: number;
+    item_type?: string;
 }
+/**
+ * EventContextIn is an interface that defines the structure of the input event context that is sent to the external extractor from ADaaS.
+ */
 export interface EventContextIn {
     mode: string;
     callback_url: string;
@@ -56,25 +70,48 @@ export interface EventContextIn {
     external_system_id: string;
     uuid: string;
     worker_data_url: string;
+    external_system: string;
+    external_system_type: string;
+    import_slug: string;
+    snap_in_slug: string;
+    sync_tier: string;
 }
+/**
+ * EventContextOut is an interface that defines the structure of the output event context that is sent from the external extractor to ADaaS.
+ */
 export interface EventContextOut {
     uuid: string;
     sync_run: string;
     sync_unit?: string;
 }
+/**
+ * ConnectionData is an interface that defines the structure of the connection data that is sent to the external extractor from ADaaS.
+ * It contains the organization ID, organization name, key, and key type.
+ */
 export interface ConnectionData {
     org_id: string;
     org_name: string;
     key: string;
     key_type: string;
 }
+/**
+ * EventData is an interface that defines the structure of the event data that is sent from the external extractor to ADaaS.
+ */
 export interface EventData {
     external_sync_units?: ExternalSyncUnit[];
     progress?: number;
     error?: ErrorRecord;
     delay?: number;
+    /**
+     * @deprecated This field is deprecated and should not be used.
+     */
     artifacts?: Artifact[];
 }
+/**
+ * DomainObject is an interface that defines the structure of a domain object that can be extracted.
+ * It must contain a name, a next chunk ID, the pages, the last modified date, whether it is done, and the count.
+ * @deprecated
+ */
 export interface DomainObjectState {
     name: string;
     nextChunkId: number;
@@ -85,11 +122,16 @@ export interface DomainObjectState {
     isDone: boolean;
     count: number;
 }
+/**
+ * AirdropEvent is an interface that defines the structure of the event that is sent to the external extractor from ADaaS.
+ * It contains the context, payload, execution metadata, and input data as common snap-ins.
+ */
 export interface AirdropEvent {
     context: {
         secrets: {
             service_account_token: string;
         };
+        snap_in_version_id: string;
     };
     payload: AirdropMessage;
     execution_metadata: {
@@ -97,6 +139,9 @@ export interface AirdropEvent {
     };
     input_data: InputData;
 }
+/**
+ * AirdropMessage is an interface that defines the structure of the payload/message that is sent to the external extractor from ADaaS.
+ */
 export interface AirdropMessage {
     connection_data: ConnectionData;
     event_context: EventContextIn;
@@ -104,13 +149,13 @@ export interface AirdropMessage {
     event_data?: EventData;
     extractor_state?: any;
 }
+/**
+ * ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to ADaaS.
+ * It contains the event type, event context, extractor state, and event data.
+ */
 export interface ExtractorEvent {
     event_type: string;
     event_context: EventContextOut;
     extractor_state?: string;
     event_data?: EventData;
 }
-export type AdapterState<ExtractorState> = ExtractorState & {
-    lastSyncStarted?: string;
-    lastSuccessfulSyncStarted?: string;
-};