@devrev/ts-adaas 0.0.2 → 1.0.0

package/dist/{src/types → types}/extraction.js

@@ -1,65 +1,48 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ExtractionMode = exports.ExtractorEventType = exports.EventType = void 0;
+/**
+ * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
+ * The external extractor can use these events to know what to do next in the extraction process.
+ */
 var EventType;
 (function (EventType) {
-    // Get the list of sync units (repos, projects, ...) that can be extracted
     EventType["ExtractionExternalSyncUnitsStart"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_START";
-    // Start the extraction of external sync unit's metadata (repos, projects, ...)
     EventType["ExtractionMetadataStart"] = "EXTRACTION_METADATA_START";
-    // Start the extraction of a specific external sync unit
     EventType["ExtractionDataStart"] = "EXTRACTION_DATA_START";
-    // Continue the extraction of a specific external sync unit
     EventType["ExtractionDataContinue"] = "EXTRACTION_DATA_CONTINUE";
-    // Give the external extractor an opportunity to clean up after itself
     EventType["ExtractionDataDelete"] = "EXTRACTION_DATA_DELETE";
-    // Extract all the attachments for a specific external sync unit
     EventType["ExtractionAttachmentsStart"] = "EXTRACTION_ATTACHMENTS_START";
-    // Continue the extraction of attachments for a specific external sync unit
     EventType["ExtractionAttachmentsContinue"] = "EXTRACTION_ATTACHMENTS_CONTINUE";
-    // Give the external extractor an opportunity to clean up after extractions of external sync units
     EventType["ExtractionAttachmentsDelete"] = "EXTRACTION_ATTACHMENTS_DELETE";
 })(EventType || (exports.EventType = EventType = {}));
+/**
+ * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS.
+ * The external extractor can use these events to inform ADaaS about the progress of the extraction process.
+ */
 var ExtractorEventType;
 (function (ExtractorEventType) {
-    /* Sync Units */
-    // Sent when the extraction of external sync units finished
     ExtractorEventType["ExtractionExternalSyncUnitsDone"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_DONE";
-    // Sent when there was an unrecoverable error for extraction of external sync units.
-    // Must contain a list of error records.
     ExtractorEventType["ExtractionExternalSyncUnitsError"] = "EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR";
-    /* Metadata */
-    // Sent when the extraction of metadata finished
     ExtractorEventType["ExtractionMetadataDone"] = "EXTRACTION_METADATA_DONE";
-    // Sent when there was an unrecoverable error for extraction of metadata.
     ExtractorEventType["ExtractionMetadataError"] = "EXTRACTION_METADATA_ERROR";
-    /* Data */
-    // Sent after a batch was extracted, contains artifact IDs of the uploaded files
     ExtractorEventType["ExtractionDataProgress"] = "EXTRACTION_DATA_PROGRESS";
-    // Sent when there is a rate limit of more than ~1m, adapter will restart the extraction after the delay
     ExtractorEventType["ExtractionDataDelay"] = "EXTRACTION_DATA_DELAY";
-    // Sent when the extraction of data finished
     ExtractorEventType["ExtractionDataDone"] = "EXTRACTION_DATA_DONE";
-    // Sent when there was an unrecoverable error for extraction of data
     ExtractorEventType["ExtractionDataError"] = "EXTRACTION_DATA_ERROR";
-    // Sent when the external extractor has finished cleaning up after itself
     ExtractorEventType["ExtractionDataDeleteDone"] = "EXTRACTION_DATA_DELETE_DONE";
-    // Sent when there was an unrecoverable error for extraction of data
     ExtractorEventType["ExtractionDataDeleteError"] = "EXTRACTION_DATA_DELETE_ERROR";
-    /* Attachments */
-    // Sent after a batch was extracted, contains artifact IDs of the uploaded files
     ExtractorEventType["ExtractionAttachmentsProgress"] = "EXTRACTION_ATTACHMENTS_PROGRESS";
-    // Sent when there is a rate limit of more than ~30s, adapter will restart the extraction after the delay
     ExtractorEventType["ExtractionAttachmentsDelay"] = "EXTRACTION_ATTACHMENTS_DELAY";
-    // Sent when the extraction of attachements is finished
     ExtractorEventType["ExtractionAttachmentsDone"] = "EXTRACTION_ATTACHMENTS_DONE";
-    // Sent when there was an unrecoverable error for extraction of attachments
     ExtractorEventType["ExtractionAttachmentsError"] = "EXTRACTION_ATTACHMENTS_ERROR";
-    // Sent when the external extractor has finished cleaning up after itself
     ExtractorEventType["ExtractionAttachmentsDeleteDone"] = "EXTRACTION_ATTACHMENTS_DELETE_DONE";
-    // Sent when there was an unrecoverable error for extraction of attachments
     ExtractorEventType["ExtractionAttachmentsDeleteError"] = "EXTRACTION_ATTACHMENTS_DELETE_ERROR";
 })(ExtractorEventType || (exports.ExtractorEventType = ExtractorEventType = {}));
+/**
+ * ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor.
+ * It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs.
+ */
 var ExtractionMode;
 (function (ExtractionMode) {
     ExtractionMode["INITIAL"] = "INITIAL";

package/dist/types/index.d.ts

@@ -0,0 +1,5 @@
+export { ErrorLevel, ErrorRecord, LogRecord, AdapterUpdateParams, InitialDomainMapping, } from './common';
+export { EventType, ExtractorEventType, ExtractionMode, ExternalSyncUnit, EventContextIn, EventContextOut, ConnectionData, EventData, DomainObjectState, AirdropEvent, AirdropMessage, ExtractorEvent, } from './extraction';
+export { NormalizedItem, NormalizedAttachment } from '../repo/repo.interfaces';
+export { AdapterState } from '../state/state.interfaces';
+export { Artifact, ArtifactsPrepareResponse, UploadResponse, StreamResponse, StreamAttachmentsResponse, SsorAttachment, } from '../uploader/uploader.interfaces';

package/dist/types/index.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExtractionMode = exports.ExtractorEventType = exports.EventType = exports.ErrorLevel = void 0;
+// Common
+var common_1 = require("./common");
+Object.defineProperty(exports, "ErrorLevel", { enumerable: true, get: function () { return common_1.ErrorLevel; } });
+// Extraction
+var extraction_1 = require("./extraction");
+Object.defineProperty(exports, "EventType", { enumerable: true, get: function () { return extraction_1.EventType; } });
+Object.defineProperty(exports, "ExtractorEventType", { enumerable: true, get: function () { return extraction_1.ExtractorEventType; } });
+Object.defineProperty(exports, "ExtractionMode", { enumerable: true, get: function () { return extraction_1.ExtractionMode; } });

package/dist/types/workers.d.ts

@@ -0,0 +1,142 @@
+import { Worker } from 'worker_threads';
+import { MessagePort } from 'node:worker_threads';
+import { State } from '../state/state';
+import { WorkerAdapter } from '../workers/worker-adapter';
+import { ExtractorEventType, AirdropEvent } from './extraction';
+/**
+ * WorkerAdapterInterface is an interface for WorkerAdapter class.
+ * @interface WorkerAdapterInterface
+ * @constructor
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object=} initialState - The initial state of the adapter
+ * @param {MessagePort} parentPort - The parent port of the worker thread
+ * @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class
+ */
+export interface WorkerAdapterInterface<ConnectorState> {
+    event: AirdropEvent;
+    adapterState: State<ConnectorState>;
+    parentPort: MessagePort;
+    options: WorkerAdapterOptions;
+}
+/**
+ * WorkerAdapterOptions represents the options for WorkerAdapter class.
+ * @interface WorkerAdapterOptions
+ * @constructor
+ * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
+ * @param {number=} timeout - The timeout for the worker thread
+ */
+export interface WorkerAdapterOptions {
+    isLocalDevelopment?: boolean;
+    timeout?: number;
+}
+export type SpawnResolve = (value: boolean | PromiseLike<boolean>) => void;
+/**
+ * SpawnInterface is an interface for Spawn class.
+ * @interface SpawnInterface
+ * @constructor
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {Worker} worker - The worker thread
+ */
+export interface SpawnInterface {
+    event: AirdropEvent;
+    worker: Worker;
+    options: WorkerAdapterOptions | null;
+    resolve: SpawnResolve;
+}
+/**
+ * SpawnFactoryInterface is an interface for Spawn class factory.
+ * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
+ * The class provides utilities to emit control events to the platform and exit the worker gracefully.
+ * In case of lambda timeout, the class emits a lambda timeout event to the platform.
+ * @interface SpawnFactoryInterface
+ * @constructor
+ * @param {AirdropEvent} event - The event object received from the platform
+ * @param {object=} initialState - The initial state of the adapter
+ * @param {string} workerPath - The path to the worker file
+ * @param {WorkerAdapterOptions} options - The options to create a new instance of Spawn class
+ */
+export interface SpawnFactoryInterface<ConnectorState> {
+    event: AirdropEvent;
+    initialState: ConnectorState;
+    workerPath?: string;
+    options?: WorkerAdapterOptions;
+}
+/**
+ * TaskAdapterInterface is an interface for TaskAdapter class.
+ * @interface TaskAdapterInterface
+ * @constructor
+ * @param {WorkerAdapter} adapter - The adapter object
+ */
+export interface TaskAdapterInterface<ConnectorState> {
+    adapter: WorkerAdapter<ConnectorState>;
+}
+/**
+ * ProcessTaskInterface is an interface for ProcessTask class.
+ * @interface ProcessTaskInterface
+ * @constructor
+ * @param {function} task - The task to be executed, returns exit code
+ * @param {function} onTimeout - The task to be executed on timeout, returns exit code
+ */
+export interface ProcessTaskInterface<ConnectorState> {
+    task: (params: TaskAdapterInterface<ConnectorState>) => void;
+    onTimeout: (params: TaskAdapterInterface<ConnectorState>) => void;
+}
+/**
+ * WorkerEvent represents the standard worker events.
+ */
+export declare enum WorkerEvent {
+    WorkerMessage = "message",
+    WorkerOnline = "online",
+    WorkerError = "error",
+    WorkerExit = "exit"
+}
+/**
+ * WorkerMessageSubject represents the handled worker message subjects.
+ */
+export declare enum WorkerMessageSubject {
+    WorkerMessageEmitted = "emit",
+    WorkerMessageDone = "done",
+    WorkerMessageExit = "exit",
+    WorkerMessageLog = "log"
+}
+/**
+ * WorkerMessageEmitted interface represents the structure of the emitted worker message.
+ */
+export interface WorkerMessageEmitted {
+    subject: WorkerMessageSubject.WorkerMessageEmitted;
+    payload: {
+        eventType: ExtractorEventType;
+    };
+}
+/**
+ * WorkerMessageDone interface represents the structure of the done worker message.
+ */
+export interface WorkerMessageDone {
+    subject: WorkerMessageSubject.WorkerMessageDone;
+}
+/**
+ * WorkerMessageExit interface represents the structure of the exit worker message.
+ */
+export interface WorkerMessageExit {
+    subject: WorkerMessageSubject.WorkerMessageExit;
+}
+/**
+ * WorkerMessage represents the structure of the worker message.
+ */
+export type WorkerMessage = WorkerMessageDone | WorkerMessageEmitted | WorkerMessageExit;
+/**
+ * WorkerData represents the structure of the worker data object.
+ */
+export interface WorkerData<ConnectorState> {
+    event: AirdropEvent;
+    initialState: ConnectorState;
+    workerPath: string;
+    options?: WorkerAdapterOptions;
+}
+/**
+ * GetWorkerPathInterface is an interface for getting the worker path.
+ */
+export interface GetWorkerPathInterface {
+    event: AirdropEvent;
+    connectorWorkerPath?: string | null;
+}

package/dist/types/workers.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerMessageSubject = exports.WorkerEvent = void 0;
+/**
+ * WorkerEvent represents the standard worker events.
+ */
+var WorkerEvent;
+(function (WorkerEvent) {
+    WorkerEvent["WorkerMessage"] = "message";
+    WorkerEvent["WorkerOnline"] = "online";
+    WorkerEvent["WorkerError"] = "error";
+    WorkerEvent["WorkerExit"] = "exit";
+})(WorkerEvent || (exports.WorkerEvent = WorkerEvent = {}));
+/**
+ * WorkerMessageSubject represents the handled worker message subjects.
+ */
+var WorkerMessageSubject;
+(function (WorkerMessageSubject) {
+    WorkerMessageSubject["WorkerMessageEmitted"] = "emit";
+    WorkerMessageSubject["WorkerMessageDone"] = "done";
+    WorkerMessageSubject["WorkerMessageExit"] = "exit";
+    WorkerMessageSubject["WorkerMessageLog"] = "log";
+})(WorkerMessageSubject || (exports.WorkerMessageSubject = WorkerMessageSubject = {}));

package/dist/uploader/uploader.d.ts

@@ -0,0 +1,38 @@
+import { UploadResponse, StreamAttachmentsResponse, UploaderFactoryInterface } from './uploader.interfaces';
+export declare class Uploader {
+    private event;
+    private betaDevrevSdk;
+    private isLocalDevelopment?;
+    constructor({ event, options }: UploaderFactoryInterface);
+    /**
+     * Uploads the fetched objects to the DevRev platform.
+     * The fetched objects are uploaded to the platform and the artifact information is returned.
+     * @param {string} filename - The name of the file to be uploaded
+     * @param {string} itemType - The type of the item to be uploaded
+     * @param {object[] | object} fetchedObjects - The fetched objects to be uploaded
+     * @returns {Promise<UploadResponse>} - The response object containing the artifact information
+     * or error information if there was an error
+     */
+    upload(itemType: string, fetchedObjects: object[] | object): Promise<UploadResponse>;
+    private prepareArtifact;
+    private uploadToArtifact;
+    private streamToArtifact;
+    /**
+     * Streams the attachments to the DevRev platform.
+     * The attachments are streamed to the platform and the artifact information is returned.
+     * @param {string} attachmentsMetadataArtifactId - The artifact ID of the attachments metadata
+     * @returns {Promise<UploadResponse>} - The response object containing the ssoAttachment artifact information
+     * or error information if there was an error
+     */
+    streamAttachments({ attachmentsMetadataArtifactId, }: {
+        attachmentsMetadataArtifactId: string;
+    }): Promise<StreamAttachmentsResponse>;
+    private getArtifactDownloadUrl;
+    private downloadArtifact;
+    private compressGzip;
+    private decompressGzip;
+    private parseJsonl;
+    private stream;
+    private getFileStreamResponse;
+    private downloadToLocal;
+}

package/dist/uploader/uploader.interfaces.d.ts

@@ -0,0 +1,63 @@
+import { AirdropEvent } from '../types/extraction';
+import { WorkerAdapterOptions } from '../types/workers';
+import { ErrorRecord } from '../types/common';
+export interface UploaderFactoryInterface {
+    event: AirdropEvent;
+    options?: WorkerAdapterOptions;
+}
+/**
+ * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
+ */
+export interface Artifact {
+    id: string;
+    item_type: string;
+    item_count: number;
+}
+/**
+ * ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
+ * @deprecated
+ */
+export interface ArtifactsPrepareResponse {
+    url: string;
+    id: string;
+    form_data: {
+        key: string;
+        value: string;
+    }[];
+}
+/**
+ * UploadResponse is an interface that defines the structure of the response from upload through Uploader.
+ */
+export interface UploadResponse {
+    artifact?: Artifact;
+    error?: ErrorRecord;
+}
+/**
+ * StreamAttachmentsResponse is an interface that defines the structure of the response from the stream attachments through Uploader.
+ */
+export interface StreamAttachmentsResponse {
+    ssorAttachments?: SsorAttachment[];
+    error?: ErrorRecord;
+}
+/**
+ * StreamResponse is an interface that defines the structure of the response from the stream of single attachment through Uploader.
+ */
+export interface StreamResponse {
+    ssorAttachment?: SsorAttachment;
+    error?: ErrorRecord;
+}
+/**
+ * SsorAttachment is an interface that defines the structure of the SSOR attachment.
+ */
+export interface SsorAttachment {
+    id: {
+        devrev: string;
+        external: string;
+    };
+    parent_id: {
+        external: string;
+    };
+    actor_id: {
+        external: string;
+    };
+}

package/dist/uploader/uploader.interfaces.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });