@devrev/ts-adaas 0.0.3 → 1.0.0

package/dist/{src → deprecated}/adapter/index.js

@@ -3,12 +3,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Adapter = exports.createAdapter = void 0;
+exports.Adapter = void 0;
+exports.createAdapter = createAdapter;
 const axios_1 = __importDefault(require("axios"));
-const constants_1 = require("../common/constants");
+const constants_1 = require("../../common/constants");
 const helpers_1 = require("../common/helpers");
-const logging_1 = require("../logging");
-const state_1 = require("../state");
+// import { Logger } from '../../logger/logger';
+const state_1 = require("../../state/state");
 /**
  * Adapter class is used to interact with Airdrop platform. The class provides
  * utilities to
@@ -18,6 +19,7 @@ const state_1 = require("../state");
  *
  * @class Adapter
  * @constructor
+ * @deprecated
  * @param {AirdropEvent} event - The event object received from the platform
  * @param {object=} initialState - The initial state of the adapter
  * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
@@ -32,19 +34,21 @@ const state_1 = require("../state");
  */
 async function createAdapter(event, initialState, isLocalDevelopment = false) {
     const newInitialState = structuredClone(initialState);
-    const adapterState = await (0, state_1.createAdapterState)(event, newInitialState);
+    const adapterState = await (0, state_1.createAdapterState)({
+        event,
+        initialState: newInitialState,
+    });
     const a = new Adapter(event, adapterState, isLocalDevelopment);
     return a;
 }
-exports.createAdapter = createAdapter;
 class Adapter {
     constructor(event, adapterState, isLocalDevelopment = false) {
+        // if (!isLocalDevelopment) {
+        //   Logger.init(event);
+        // }
         this.exit = false;
         this.lambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
         this.heartBeatInterval = 30 * 1000; // 30 seconds in milliseconds
-        if (!isLocalDevelopment) {
-            logging_1.Logger.init(event);
-        }
         this.adapterState = adapterState;
         this._artifacts = [];
         this.event = event;
@@ -122,7 +126,6 @@ class Adapter {
      */
     exitAdapter() {
         this.exit = true;
-        clearInterval(this.heartBeatFn);
     }
     /**
      * Heartbeat function to check if the lambda is about to timeout.

package/dist/{src → deprecated}/common/helpers.d.ts

@@ -1,6 +1,5 @@
-import { Artifact, EventType, ExtractorEventType } from '../types';
+import { EventType, ExtractorEventType } from '../../types/extraction';
 export declare function createFormData(preparedArtifact: any, fetchedObjects: object[] | object): FormData;
-export declare function createArtifact(preparedArtifact: any, fetchedObjects: object[] | object, entity: string): Artifact;
 export declare function getTimeoutExtractorEventType(eventType: EventType): {
     eventType: ExtractorEventType;
     isError: boolean;

package/dist/deprecated/common/helpers.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createFormData = createFormData;
+exports.getTimeoutExtractorEventType = getTimeoutExtractorEventType;
+const js_jsonl_1 = require("js-jsonl");
+const extraction_1 = require("../../types/extraction");
+function createFormData(
+//eslint-disable-next-line @typescript-eslint/no-explicit-any
+preparedArtifact, fetchedObjects) {
+    const formData = new FormData();
+    for (const item of preparedArtifact.form_data) {
+        formData.append(item.key, item.value);
+    }
+    const output = js_jsonl_1.jsonl.stringify(fetchedObjects);
+    formData.append('file', output);
+    return formData;
+}
+function getTimeoutExtractorEventType(eventType) {
+    switch (eventType) {
+        case extraction_1.EventType.ExtractionMetadataStart:
+            return {
+                eventType: extraction_1.ExtractorEventType.ExtractionMetadataError,
+                isError: true,
+            };
+        case extraction_1.EventType.ExtractionDataStart:
+        case extraction_1.EventType.ExtractionDataContinue:
+            return {
+                eventType: extraction_1.ExtractorEventType.ExtractionDataProgress,
+                isError: false,
+            };
+        case extraction_1.EventType.ExtractionAttachmentsStart:
+        case extraction_1.EventType.ExtractionAttachmentsContinue:
+            return {
+                eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsProgress,
+                isError: false,
+            };
+        case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
+            return {
+                eventType: extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
+                isError: true,
+            };
+        default:
+            console.log('Event type not recognized in getTimeoutExtractorEventType function: ' +
+                eventType);
+            return null;
+    }
+}

package/dist/deprecated/demo-extractor/index.d.ts

@@ -0,0 +1,17 @@
+import { AirdropEvent } from '../../types/extraction';
+import { Adapter } from '../adapter';
+type ConnectorState = object;
+/**
+ * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
+ *
+ * @class DemoExtractor
+ * @deprecated
+ **/
+export declare class DemoExtractor {
+    private event;
+    private adapter;
+    private uploader;
+    constructor(event: AirdropEvent, adapter: Adapter<ConnectorState>);
+    run(): Promise<void>;
+}
+export {};

package/dist/{src → deprecated}/demo-extractor/index.js

@@ -4,9 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DemoExtractor = void 0;
-const types_1 = require("../types");
+const extraction_1 = require("../../types/extraction");
 const uploader_1 = require("../uploader");
 const external_domain_metadata_json_1 = __importDefault(require("./external_domain_metadata.json"));
+/**
+ * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
+ *
+ * @class DemoExtractor
+ * @deprecated
+ **/
 class DemoExtractor {
     constructor(event, adapter) {
         this.event = event;
@@ -15,7 +21,7 @@ class DemoExtractor {
     }
     async run() {
         switch (this.event.payload.event_type) {
-            case types_1.EventType.ExtractionExternalSyncUnitsStart: {
+            case extraction_1.EventType.ExtractionExternalSyncUnitsStart: {
                 const externalSyncUnits = [
                     {
                         id: 'devrev',
@@ -23,25 +29,25 @@ class DemoExtractor {
                         description: 'Demo external sync unit',
                     },
                 ];
-                await this.adapter.emit(types_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
+                await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
                     external_sync_units: externalSyncUnits,
                 });
                 break;
             }
-            case types_1.EventType.ExtractionMetadataStart: {
+            case extraction_1.EventType.ExtractionMetadataStart: {
                 const { artifact, error } = await this.uploader.upload('metadata_1.jsonl', 'external_domain_metadata', external_domain_metadata_json_1.default);
                 if (error || !artifact) {
-                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionMetadataError, {
+                    await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataError, {
                         error,
                     });
                     return;
                 }
-                await this.adapter.emit(types_1.ExtractorEventType.ExtractionMetadataDone, {
+                await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataDone, {
                     artifacts: [artifact],
                 });
                 break;
             }
-            case types_1.EventType.ExtractionDataStart: {
+            case extraction_1.EventType.ExtractionDataStart: {
                 const contacts = [
                     {
                         id: 'contact-1',
@@ -64,18 +70,18 @@ class DemoExtractor {
                 ];
                 const { artifact, error } = await this.uploader.upload('contacts_1.json', 'contacts', contacts);
                 if (error || !artifact) {
-                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
+                    await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
                         error,
                     });
                     return;
                 }
-                await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataProgress, {
+                await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataProgress, {
                     progress: 50,
                     artifacts: [artifact],
                 });
                 break;
             }
-            case types_1.EventType.ExtractionDataContinue: {
+            case extraction_1.EventType.ExtractionDataContinue: {
                 const users = [
                     {
                         id: 'user-1',
@@ -98,51 +104,51 @@ class DemoExtractor {
                 ];
                 const { artifact, error } = await this.uploader.upload('users_1.json', 'users', users);
                 if (error || !artifact) {
-                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataError, {
+                    await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
                         error,
                     });
                     return;
                 }
-                await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataDone, {
+                await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDone, {
                     progress: 100,
                     artifacts: [artifact],
                 });
                 break;
             }
-            case types_1.EventType.ExtractionDataDelete: {
-                await this.adapter.emit(types_1.ExtractorEventType.ExtractionDataDeleteDone);
+            case extraction_1.EventType.ExtractionDataDelete: {
+                await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDeleteDone);
                 break;
             }
-            case types_1.EventType.ExtractionAttachmentsStart: {
+            case extraction_1.EventType.ExtractionAttachmentsStart: {
                 const attachment1 = ['This is attachment1.txt content'];
                 const { artifact, error } = await this.uploader.upload('attachment1.txt', 'attachment', attachment1);
                 if (error || !artifact) {
-                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
+                    await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
                         error,
                     });
                     return;
                 }
-                await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsProgress, {
+                await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsProgress, {
                     artifacts: [artifact],
                 });
                 break;
             }
-            case types_1.EventType.ExtractionAttachmentsContinue: {
+            case extraction_1.EventType.ExtractionAttachmentsContinue: {
                 const attachment2 = ['This is attachment2.txt content'];
                 const { artifact, error } = await this.uploader.upload('attachment2.txt', 'attachment', attachment2);
                 if (error || !artifact) {
-                    await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsError, {
+                    await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
                         error,
                     });
                     return;
                 }
-                await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDone, {
+                await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDone, {
                     artifacts: [artifact],
                 });
                 break;
             }
-            case types_1.EventType.ExtractionAttachmentsDelete: {
-                await this.adapter.emit(types_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
+            case extraction_1.EventType.ExtractionAttachmentsDelete: {
+                await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
                 break;
             }
             default: {

package/dist/{src → deprecated}/uploader/index.d.ts

@@ -1,4 +1,4 @@
-import { UploadResponse } from '../types/common';
+import { UploadResponse } from '../../uploader/uploader.interfaces';
 /**
  * Uploader class is used to upload files to the DevRev platform.
  * The class provides utilities to

package/dist/{src → deprecated}/uploader/index.js

@@ -106,7 +106,9 @@ class Uploader {
     }
     async uploadToArtifact(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    preparedArtifact, fetchedObjects) {
+    preparedArtifact, fetchedObjects
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ) {
         const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
         try {
             const response = await axios_1.default.post(preparedArtifact.url, formData, {

package/dist/{src/http → http}/client.d.ts

@@ -1,3 +1,4 @@
+import { RawAxiosRequestHeaders } from 'axios';
 import { HTTPResponse } from './types';
 export declare const defaultResponse: HTTPResponse;
 export declare class HTTPClient {
@@ -12,5 +13,5 @@ export declare class HTTPClient {
      * In case of rate limit exceeded, the function returns success as true and the delay time in seconds
      * In case of any other error, the function returns success as false and the error message
      */
-    getCall(endpoint: string, headers: Record<string, string>, params?: any): Promise<HTTPResponse>;
+    getCall(endpoint: string, headers: RawAxiosRequestHeaders, params?: any): Promise<HTTPResponse>;
 }

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+export * from './deprecated/adapter';
+export * from './deprecated/demo-extractor';
+export * from './deprecated/uploader';
+export * from './types';
+export * from './http';
+export * from './common/install-initial-domain-mapping';
+export { WorkerAdapter } from './workers/worker-adapter';
+export { processTask } from './workers/process-task';
+export { spawn } from './workers/spawn';
+export * from './types/workers';

package/dist/{src/index.js → index.js}

@@ -14,9 +14,17 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./adapter"), exports);
-__exportStar(require("./demo-extractor"), exports);
-__exportStar(require("./uploader"), exports);
+exports.spawn = exports.processTask = exports.WorkerAdapter = void 0;
+__exportStar(require("./deprecated/adapter"), exports);
+__exportStar(require("./deprecated/demo-extractor"), exports);
+__exportStar(require("./deprecated/uploader"), exports);
 __exportStar(require("./types"), exports);
 __exportStar(require("./http"), exports);
 __exportStar(require("./common/install-initial-domain-mapping"), exports);
+var worker_adapter_1 = require("./workers/worker-adapter");
+Object.defineProperty(exports, "WorkerAdapter", { enumerable: true, get: function () { return worker_adapter_1.WorkerAdapter; } });
+var process_task_1 = require("./workers/process-task");
+Object.defineProperty(exports, "processTask", { enumerable: true, get: function () { return process_task_1.processTask; } });
+var spawn_1 = require("./workers/spawn");
+Object.defineProperty(exports, "spawn", { enumerable: true, get: function () { return spawn_1.spawn; } });
+__exportStar(require("./types/workers"), exports);

package/dist/logger/logger.d.ts

@@ -0,0 +1,14 @@
+import { Console } from 'node:console';
+import { LogLevel, PrintableState } from './logger.interfaces';
+import { AirdropEvent } from '../types';
+import { AxiosError } from 'axios';
+export declare class Logger extends Console {
+    constructor(event: AirdropEvent);
+    logFn(args: unknown[], level: LogLevel): void;
+    log(...args: unknown[]): void;
+    info(...args: unknown[]): void;
+    warn(...args: unknown[]): void;
+    error(...args: unknown[]): void;
+}
+export declare function getPrintableState(state: Record<string, any>): PrintableState;
+export declare function formatAxiosError(error: AxiosError): object;

package/dist/logger/logger.interfaces.d.ts

@@ -0,0 +1,14 @@
+export declare enum LogLevel {
+    INFO = "info",
+    WARN = "warn",
+    ERROR = "error"
+}
+export interface PrintableArray {
+    type: 'array';
+    length: number;
+    firstItem?: any;
+    lastItem?: any;
+}
+export interface PrintableState {
+    [key: string]: any | PrintableArray | PrintableState;
+}

package/dist/logger/logger.interfaces.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LogLevel = void 0;
+var LogLevel;
+(function (LogLevel) {
+    LogLevel["INFO"] = "info";
+    LogLevel["WARN"] = "warn";
+    LogLevel["ERROR"] = "error";
+})(LogLevel || (exports.LogLevel = LogLevel = {}));

package/dist/logger/logger.js

@@ -0,0 +1,91 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Logger = void 0;
+exports.getPrintableState = getPrintableState;
+exports.formatAxiosError = formatAxiosError;
+/* eslint-disable @typescript-eslint/no-explicit-any */
+const lambda_log_1 = __importDefault(require("lambda-log"));
+const node_console_1 = require("node:console");
+const logger_interfaces_1 = require("./logger.interfaces");
+const node_worker_threads_1 = require("node:worker_threads");
+const workers_1 = require("../types/workers");
+class Logger extends node_console_1.Console {
+    constructor(event) {
+        super(process.stdout, process.stderr);
+        lambda_log_1.default.options.levelKey = null;
+        lambda_log_1.default.options.tagsKey = null;
+        lambda_log_1.default.options.messageKey = 'message';
+        lambda_log_1.default.options.meta = Object.assign({}, event.payload.event_context);
+    }
+    logFn(args, level) {
+        if (node_worker_threads_1.isMainThread) {
+            lambda_log_1.default.log(level, JSON.stringify(args));
+        }
+        else {
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
+                subject: workers_1.WorkerMessageSubject.WorkerMessageLog,
+                payload: {
+                    args,
+                    level,
+                },
+            });
+        }
+    }
+    log(...args) {
+        this.logFn(args, logger_interfaces_1.LogLevel.INFO);
+    }
+    info(...args) {
+        this.logFn(args, logger_interfaces_1.LogLevel.INFO);
+    }
+    warn(...args) {
+        this.logFn(args, logger_interfaces_1.LogLevel.WARN);
+    }
+    error(...args) {
+        this.logFn(args, logger_interfaces_1.LogLevel.ERROR);
+    }
+}
+exports.Logger = Logger;
+// Helper function to process each value in the state
+function getPrintableState(state) {
+    function processValue(value) {
+        if (Array.isArray(value)) {
+            // If the value is an array, summarize it
+            return {
+                type: 'array',
+                length: value.length,
+                firstItem: value.length > 0 ? value[0] : undefined,
+                lastItem: value.length > 1 ? value[value.length - 1] : undefined,
+            };
+        }
+        else if (typeof value === 'object' && value !== null) {
+            // If the value is an object, recursively process its properties
+            const processedObject = {};
+            for (const key in value) {
+                if (value.hasOwnProperty(key)) {
+                    processedObject[key] = processValue(value[key]);
+                }
+            }
+            return processedObject;
+        }
+        // For primitive types, return the value as is
+        return value;
+    }
+    // Process the state object directly since it's guaranteed to be an object
+    return processValue(state);
+}
+function formatAxiosError(error) {
+    var _a, _b, _c;
+    if (error.response) {
+        return {
+            status: error.response.status,
+            data: error.response.data,
+            method: (_a = error.config) === null || _a === void 0 ? void 0 : _a.method,
+            baseURL: (_b = error.config) === null || _b === void 0 ? void 0 : _b.baseURL,
+            url: (_c = error.config) === null || _c === void 0 ? void 0 : _c.url,
+        };
+    }
+    return error;
+}

package/dist/logger/logger.test.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const logger_1 = require("./logger");
+it('getPrintableState should return printable state', () => {
+    const state = {
+        test_key: 'test_value',
+        big_array: Array.from({ length: 1000 }, (_, index) => index),
+        nested_object: {
+            nested_key: 'nested_value',
+            nested_array: Array.from({ length: 1000 }, (_, index) => index),
+        },
+    };
+    const printableState = (0, logger_1.getPrintableState)(state);
+    expect(printableState).toEqual({
+        test_key: 'test_value',
+        big_array: {
+            type: 'array',
+            length: 1000,
+            firstItem: 0,
+            lastItem: 999,
+        },
+        nested_object: {
+            nested_key: 'nested_value',
+            nested_array: {
+                type: 'array',
+                length: 1000,
+                firstItem: 0,
+                lastItem: 999,
+            },
+        },
+    });
+});
+it('formatAxiosError should return formatted error', () => {
+    const error = {
+        response: {
+            status: 500,
+            data: 'Internal server error',
+        },
+        config: {
+            method: 'GET',
+        },
+    };
+    const formattedError = (0, logger_1.formatAxiosError)(error);
+    expect(formattedError).toEqual({
+        status: 500,
+        data: 'Internal server error',
+        method: 'GET',
+    });
+});

package/dist/repo/repo.d.ts

@@ -0,0 +1,14 @@
+import { ErrorRecord } from '../types/common';
+import { Item } from '../repo/repo.interfaces';
+import { RepoFactoryInterface, NormalizedItem, NormalizedAttachment } from './repo.interfaces';
+export declare class Repo {
+    readonly itemType: string;
+    private items;
+    private normalize?;
+    private uploader;
+    private onUpload;
+    constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
+    getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
+    upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
+    push(items: Item[]): Promise<void | ErrorRecord>;
+}

package/dist/repo/repo.interfaces.d.ts

@@ -0,0 +1,43 @@
+import { Artifact } from '../uploader/uploader.interfaces';
+import { AirdropEvent } from '../types/extraction';
+import { WorkerAdapterOptions } from '../types/workers';
+/**
+ * RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data.
+ */
+export interface RepoInterface {
+    itemType: string;
+    normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
+}
+/**
+ * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo.
+ */
+export interface RepoFactoryInterface {
+    event: AirdropEvent;
+    itemType: string;
+    normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
+    onUpload: (artifact: Artifact) => void;
+    options?: WorkerAdapterOptions;
+}
+/**
+ * NormalizedItem is an interface of item after normalization.
+ */
+export interface NormalizedItem {
+    id: string;
+    created_date: string;
+    modified_date: string;
+    data: object;
+}
+/**
+ * NormalizedAttachment is an interface of attachment after normalization.
+ */
+export interface NormalizedAttachment {
+    url: string;
+    id: string;
+    file_name: string;
+    author_id: string;
+    parent_id: string;
+}
+/**
+ * Item is an interface that defines the structure of an item.
+ */
+export type Item = Record<string, any>;

package/dist/repo/repo.interfaces.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/repo/repo.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Repo = void 0;
+const constants_1 = require("../common/constants");
+const uploader_1 = require("../uploader/uploader");
+class Repo {
+    constructor({ event, itemType, normalize, onUpload, options, }) {
+        this.items = [];
+        this.itemType = itemType;
+        this.normalize = normalize;
+        this.onUpload = onUpload;
+        this.uploader = new uploader_1.Uploader({ event, options });
+    }
+    getItems() {
+        return this.items;
+    }
+    async upload(batch) {
+        const itemsToUpload = batch || this.items;
+        if (itemsToUpload.length > 0) {
+            console.log(`Uploading ${itemsToUpload.length} items of type ${this.itemType}.`);
+            const { artifact, error } = await this.uploader.upload(this.itemType, itemsToUpload);
+            if (error || !artifact) {
+                console.error('Error while uploading batch', error);
+                return error;
+            }
+            this.onUpload(artifact);
+            // Clear the uploaded items from the main items array if no batch was specified
+            if (!batch) {
+                this.items = [];
+            }
+        }
+        else {
+            console.log(`No items to upload for type ${this.itemType}. Skipping upload.`);
+        }
+    }
+    async push(items) {
+        return new Promise(async (resolve, reject) => {
+            let recordsToPush;
+            // Normalize items if needed
+            if (this.normalize &&
+                !Object.values(constants_1.AIRDROP_DEFAULT_ITEM_TYPES).includes(this.itemType)) {
+                recordsToPush = items.map((item) => this.normalize(item));
+            }
+            else {
+                recordsToPush = items;
+            }
+            // Add the new records to the items array
+            this.items.push(...recordsToPush);
+            console.log(`Extracted ${this.items.length} items of type ${this.itemType}.`);
+            // Upload in batches while the number of items exceeds the batch size
+            while (this.items.length >= constants_1.ARTIFACT_BATCH_SIZE) {
+                // Slice out a batch of ARTIFACT_BATCH_SIZE items to upload
+                const batch = this.items.splice(0, constants_1.ARTIFACT_BATCH_SIZE);
+                try {
+                    // Upload the batch
+                    await this.upload(batch);
+                }
+                catch (error) {
+                    console.error('Error while uploading batch', error);
+                    reject(error);
+                    return;
+                }
+            }
+            resolve();
+        });
+    }
+}
+exports.Repo = Repo;