@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.10

package/dist/multithreading/adapters/extraction-adapter.js

@@ -0,0 +1,375 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExtractionAdapter = void 0;
+const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
+const constants_1 = require("../../common/constants");
+const logger_1 = require("../../logger/logger");
+const repo_1 = require("../../repo/repo");
+const extraction_1 = require("../../types/extraction");
+const uploader_helpers_1 = require("../../uploader/uploader.helpers");
+const base_adapter_1 = require("./base-adapter");
+/** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
+class ExtractionAdapter extends base_adapter_1.BaseAdapter {
+    /** @internal */
+    constructor(params) {
+        super(params);
+        this.repos = [];
+        this.currentEventDataLength = 0;
+        this._artifacts = [];
+    }
+    /** Defaults to true if the scope is empty or the item type is not listed. */
+    shouldExtract(itemType) {
+        const scope = this.extractionScope;
+        if (Object.keys(scope).length === 0)
+            return true;
+        if (!(itemType in scope))
+            return true;
+        return scope[itemType].extract;
+    }
+    initializeRepos(repos) {
+        const existingSsorRepo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
+        this.repos = repos.map((repo) => this.createRepo(repo));
+        if (existingSsorRepo &&
+            !this.repos.some((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT)) {
+            this.repos.push(existingSsorRepo);
+        }
+    }
+    createRepo(repo) {
+        const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
+            repo.itemType !== constants_1.SSOR_ATTACHMENT;
+        const isDeferred = this.isDeferredRepo(repo.itemType);
+        return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact) => {
+                var _a, _b;
+                if (isDeferred) {
+                    const toDevRev = this.adapterState.sdkState.toDevRev;
+                    if (toDevRev) {
+                        (_a = toDevRev.deferredSsorArtifacts) !== null && _a !== void 0 ? _a : (toDevRev.deferredSsorArtifacts = []);
+                        toDevRev.deferredSsorArtifacts.push(artifact);
+                    }
+                    return;
+                }
+                this.lastExtractedItemType = repo.itemType;
+                // Artifact ids are kept in state for later attachment streaming
+                if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
+                    (_b = this.adapterState.sdkState.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.push(artifact.id);
+                }
+                // Track size of artifact objects that go in the SQS message; flip
+                // isTimeout once the threshold is exceeded to force an early emit.
+                this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
+                if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
+                    !this.isTimeout) {
+                    this.isTimeout = true;
+                }
+            }, options: Object.assign(Object.assign({}, this.options), repo.overriddenOptions) }));
+    }
+    /**
+     * SSOR attachments created outside the attachments phase are held in state
+     * and reported with ATTACHMENT_EXTRACTION_DONE instead of the current event.
+     */
+    isDeferredRepo(itemType) {
+        return itemType === constants_1.SSOR_ATTACHMENT && !this.isAttachmentsPhase();
+    }
+    isAttachmentsPhase() {
+        const eventType = this.event.payload.event_type;
+        return (eventType === extraction_1.EventType.StartExtractingAttachments ||
+            eventType === extraction_1.EventType.ContinueExtractingAttachments);
+    }
+    getOrCreateSsorRepo() {
+        let repo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
+        if (!repo) {
+            repo = this.createRepo({ itemType: constants_1.SSOR_ATTACHMENT });
+            this.repos.push(repo);
+        }
+        return repo;
+    }
+    getRepo(itemType) {
+        const repo = this.repos.find((repo) => repo.itemType === itemType);
+        if (!repo) {
+            console.error(`Repo for item type ${itemType} not found.`);
+            return;
+        }
+        return repo;
+    }
+    /** Artifacts uploaded so far that the next emit will report. */
+    get artifacts() {
+        return this._artifacts;
+    }
+    addArtifacts(artifacts) {
+        this._artifacts = this._artifacts
+            .concat(artifacts)
+            .filter((value, index, self) => self.indexOf(value) === index);
+    }
+    /**
+     * Per-record-type counts for sync-duration estimation. Set during the
+     * metadata phase; the SDK attaches them to the metadata-done event.
+     */
+    get preExtractionItemCounts() {
+        return this._preExtractionItemCounts;
+    }
+    set preExtractionItemCounts(counts) {
+        this._preExtractionItemCounts = counts;
+    }
+    /** @internal */
+    async beforeEmit(newEventType) {
+        var _a;
+        console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
+        await this.uploadAllRepos();
+        // When the full extraction cycle completes, commit the extraction window
+        // boundaries so subsequent incremental syncs can resume from them.
+        if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
+            const sdkState = this.adapterState.sdkState;
+            // Released before the state save so they're never reported twice.
+            const toDevRev = sdkState.toDevRev;
+            if ((_a = toDevRev === null || toDevRev === void 0 ? void 0 : toDevRev.deferredSsorArtifacts) === null || _a === void 0 ? void 0 : _a.length) {
+                this.addArtifacts(toDevRev.deferredSsorArtifacts);
+                toDevRev.deferredSsorArtifacts = [];
+            }
+            console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${sdkState.lastSyncStarted}).`);
+            sdkState.lastSuccessfulSyncStarted = sdkState.lastSyncStarted;
+            sdkState.lastSyncStarted = '';
+            sdkState.pendingWorkersOldest = '';
+            sdkState.pendingWorkersNewest = '';
+            // Expand boundaries: workersOldest keeps the earliest timestamp seen,
+            // workersNewest the latest.
+            const extractionStart = this.event.payload.event_context.extract_from;
+            const extractionEnd = this.event.payload.event_context.extract_to;
+            if (extractionStart &&
+                (!sdkState.workersOldest || extractionStart < sdkState.workersOldest)) {
+                console.log(`Updating workersOldest from '${sdkState.workersOldest}' to '${extractionStart}'.`);
+                sdkState.workersOldest = extractionStart;
+            }
+            if (extractionEnd &&
+                (!sdkState.workersNewest || extractionEnd > sdkState.workersNewest)) {
+                console.log(`Updating workersNewest from '${sdkState.workersNewest}' to '${extractionEnd}'.`);
+                sdkState.workersNewest = extractionEnd;
+            }
+        }
+    }
+    /** @internal */
+    buildEmitPayload(newEventType) {
+        const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
+        if (!isExtractionEvent) {
+            return {};
+        }
+        const payload = { artifacts: [...this._artifacts] };
+        if (newEventType === extraction_1.ExtractorEventType.MetadataExtractionDone &&
+            this._preExtractionItemCounts) {
+            payload.pre_extraction_item_counts = this._preExtractionItemCounts;
+        }
+        return payload;
+    }
+    /**
+     * @internal
+     * Attaches the last-extracted item type and its created/modified date ranges
+     * to data/attachment done/progress events, mirroring the metadata v1 sent.
+     */
+    buildWorkerMetadata(newEventType) {
+        if (newEventType !== extraction_1.ExtractorEventType.DataExtractionDone &&
+            newEventType !== extraction_1.ExtractorEventType.DataExtractionProgress &&
+            newEventType !== extraction_1.ExtractorEventType.AttachmentExtractionDone &&
+            newEventType !== extraction_1.ExtractorEventType.AttachmentExtractionProgress) {
+            return undefined;
+        }
+        const repo = this.lastExtractedItemType
+            ? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
+            : undefined;
+        if (!repo) {
+            return undefined;
+        }
+        return {
+            item_type: repo.itemType,
+            newest_created_date: this.toRfc3339Timestamp(repo.dateRanges.creationDate.newest),
+            oldest_created_date: this.toRfc3339Timestamp(repo.dateRanges.creationDate.oldest),
+            newest_modified_date: this.toRfc3339Timestamp(repo.dateRanges.modifiedDate.newest),
+            oldest_modified_date: this.toRfc3339Timestamp(repo.dateRanges.modifiedDate.oldest),
+        };
+    }
+    toRfc3339Timestamp(ms) {
+        if (ms === undefined || !Number.isFinite(ms)) {
+            return undefined;
+        }
+        return new Date(ms).toISOString();
+    }
+    /** @internal */
+    afterEmit() {
+        this._artifacts = [];
+    }
+    async uploadAllRepos() {
+        for (const repo of this.repos) {
+            await repo.upload();
+            if (!this.isDeferredRepo(repo.itemType)) {
+                this.addArtifacts(repo.uploadedArtifacts);
+            }
+        }
+    }
+    /** @internal */
+    beforeErrorEmit() {
+        // Report artifacts that uploaded before the failure so they aren't orphaned.
+        for (const repo of this.repos) {
+            if (!this.isDeferredRepo(repo.itemType)) {
+                this.addArtifacts(repo.uploadedArtifacts);
+            }
+        }
+    }
+    /**
+     * Streams one attachment into DevRev. Usable in any extraction phase; SSOR
+     * records created outside the attachments phase are reported with
+     * ATTACHMENT_EXTRACTION_DONE.
+     */
+    async processAttachment(attachment, stream) {
+        var _a, _b;
+        const response = await stream({
+            item: attachment,
+            event: this.event,
+        });
+        if (response.status !== 'success') {
+            return response;
+        }
+        const { httpStream } = response;
+        const fileType = attachment.content_type ||
+            ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
+            'application/octet-stream';
+        const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
+        const fileSize = contentLength ? parseInt(contentLength) : undefined;
+        const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
+        if (artifactUrlError) {
+            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+            return {
+                status: 'error',
+                error: {
+                    message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
+                    fileSize,
+                },
+            };
+        }
+        const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
+        if (uploadedArtifactError) {
+            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+            return {
+                status: 'error',
+                error: {
+                    message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
+                        (0, logger_1.serializeError)(uploadedArtifactError),
+                    fileSize,
+                },
+            };
+        }
+        const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
+        if (confirmArtifactUploadError) {
+            return {
+                status: 'error',
+                error: {
+                    message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
+                        (0, logger_1.serializeError)(confirmArtifactUploadError),
+                    fileSize,
+                },
+            };
+        }
+        const ssorAttachment = {
+            id: {
+                devrev: artifactUrlResponse.artifact_id,
+                external: attachment.id,
+            },
+            parent_id: {
+                external: attachment.parent_id,
+            },
+        };
+        if (attachment.author_id) {
+            ssorAttachment.actor_id = {
+                external: attachment.author_id,
+            };
+        }
+        // Set inline flag only if it is explicitly set on the attachment.
+        if (attachment.inline === true) {
+            ssorAttachment.inline = true;
+        }
+        else if (attachment.inline === false) {
+            ssorAttachment.inline = false;
+        }
+        if (this.isTimeout) {
+            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+            return { status: 'success' };
+        }
+        await this.getOrCreateSsorRepo().push([ssorAttachment]);
+        return { status: 'success' };
+    }
+    async streamAttachments({ stream, processors, batchSize = 1, }) {
+        var _a, _b;
+        if (batchSize <= 0) {
+            console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
+            batchSize = 1;
+        }
+        if (batchSize > 50) {
+            console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
+            batchSize = 50;
+        }
+        this.initializeRepos([{ itemType: constants_1.SSOR_ATTACHMENT }]);
+        const attachmentsMetadata = (_a = this.adapterState.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
+        if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
+            console.log(`No attachments metadata artifact IDs found in state.`);
+            return { status: 'success' };
+        }
+        else {
+            console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
+        }
+        while (attachmentsMetadata.artifactIds.length > 0) {
+            const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
+            console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
+            const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
+                artifact: attachmentsMetadataArtifactId,
+            });
+            if (error) {
+                console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                return { status: 'error', error };
+            }
+            if (!attachments || attachments.length === 0) {
+                console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                attachmentsMetadata.artifactIds.shift();
+                attachmentsMetadata.lastProcessed = 0;
+                continue;
+            }
+            console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+            let response;
+            if (processors) {
+                console.log(`Using custom processors for attachments.`);
+                const reducer = processors.reducer;
+                const iterator = processors.iterator;
+                const reducedAttachments = reducer({
+                    attachments,
+                    adapter: this,
+                    batchSize,
+                });
+                response = await iterator({
+                    reducedAttachments,
+                    adapter: this,
+                    stream,
+                });
+            }
+            else {
+                console.log(`Using attachments streaming pool for attachments streaming.`);
+                const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                    adapter: this,
+                    attachments,
+                    batchSize,
+                    stream,
+                });
+                response = await attachmentsPool.streamAll();
+            }
+            if (response.status !== 'success') {
+                return response;
+            }
+            if (this.isTimeout) {
+                console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
+                return { status: 'progress' };
+            }
+            console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+            attachmentsMetadata.artifactIds.shift();
+            attachmentsMetadata.lastProcessed = 0;
+            if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
+                attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
+            }
+        }
+        return { status: 'success' };
+    }
+}
+exports.ExtractionAdapter = ExtractionAdapter;

package/dist/multithreading/adapters/loading-adapter.d.ts

@@ -0,0 +1,26 @@
+import { ExternalSystemAttachment, ExternalSystemLoadingFunction, ItemTypesToLoadParams, LoaderReport } from '../../types/loading';
+import { TaskResult } from '../../types/workers';
+import { BaseAdapter } from './base-adapter';
+/** Adapter passed to loading tasks (item/attachment loading, loader reports). @public */
+export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
+    private loaderReports;
+    private _processedFiles;
+    get reports(): LoaderReport[];
+    get processedFiles(): string[];
+    loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<TaskResult>;
+    loadAttachments({ create, }: {
+        create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
+    }): Promise<TaskResult>;
+    /**
+     * Walks the transformer files in `fromDevRev.filesToLoad`, resuming from
+     * each file's `lineToProcess`. `getLoader` returns undefined when a file has
+     * an item type the connector can't load, which fails the phase.
+     */
+    private loadFiles;
+    private getLoaderBatches;
+    private loadItem;
+    /** Maps a non-success connector response to a delay or a failed report. */
+    private toFailedLoad;
+    private loadAttachment;
+}
+//# sourceMappingURL=loading-adapter.d.ts.map

package/dist/multithreading/adapters/loading-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,wBAAwB,EAGxB,6BAA6B,EAE7B,qBAAqB,EAGrB,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,yFAAyF;AACzF,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAalC,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IA2BK,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IA8BxC,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAavB;;;;OAIG;YACW,SAAS;YAiFT,gBAAgB;YAuBhB,QAAQ;IA+HtB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;YAwBN,cAAc;CA2C7B"}

package/dist/multithreading/adapters/loading-adapter.helpers.d.ts

@@ -0,0 +1,12 @@
+import { FileToLoad, LoaderReport, StatsFileObject } from '../../types/loading';
+/** Filters the stats file to supported item types, ordered by their position in supportedItemTypes. */
+export declare function getFilesToLoad({ supportedItemTypes, statsFile, }: {
+    supportedItemTypes: string[];
+    statsFile: StatsFileObject[];
+}): FileToLoad[];
+/** Merges a report into the loader reports, summing counts per item type. */
+export declare function addReportToLoaderReport({ loaderReports, report, }: {
+    loaderReports: LoaderReport[];
+    report: LoaderReport;
+}): LoaderReport[];
+//# sourceMappingURL=loading-adapter.helpers.d.ts.map

package/dist/multithreading/adapters/loading-adapter.helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loading-adapter.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EACV,YAAY,EACZ,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,uGAAuG;AACvG,wBAAgB,cAAc,CAAC,EAC7B,kBAAkB,EAClB,SAAS,GACV,EAAE;IACD,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,GAAG,UAAU,EAAE,CA8Bf;AAED,6EAA6E;AAC7E,wBAAgB,uBAAuB,CAAC,EACtC,aAAa,EACb,MAAM,GACP,EAAE;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,MAAM,EAAE,YAAY,CAAC;CACtB,GAAG,YAAY,EAAE,CA4BjB"}

package/dist/multithreading/{worker-adapter/worker-adapter.helpers.js → adapters/loading-adapter.helpers.js}

@@ -3,12 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getFilesToLoad = getFilesToLoad;
 exports.addReportToLoaderReport = addReportToLoaderReport;
 const loading_1 = require("../../types/loading");
-/**
- * Gets the files to load for the loader.
- * @param {string[]} supportedItemTypes - The supported item types
- * @param {StatsFileObject[]} statsFile - The stats file
- * @returns {FileToLoad[]} The files to load
- */
+/** Filters the stats file to supported item types, ordered by their position in supportedItemTypes. */
 function getFilesToLoad({ supportedItemTypes, statsFile, }) {
     const filesToLoad = [];
     if (supportedItemTypes.length === 0 || statsFile.length === 0) {
@@ -32,12 +27,7 @@ function getFilesToLoad({ supportedItemTypes, statsFile, }) {
     }
     return filesToLoad;
 }
-/**
- * Adds a report to the loader report.
- * @param {LoaderReport[]} loaderReports - The loader reports
- * @param {LoaderReport} report - The report to add
- * @returns {LoaderReport[]} The updated loader reports
- */
+/** Merges a report into the loader reports, summing counts per item type. */
 function addReportToLoaderReport({ loaderReports, report, }) {
     const existingReport = loaderReports.find((loaderReport) => loaderReport.item_type === report.item_type);
     if (existingReport) {