@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.11

package/dist/multithreading/adapters/base-adapter.js

@@ -0,0 +1,212 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseAdapter = void 0;
+const node_worker_threads_1 = require("node:worker_threads");
+const constants_1 = require("../../common/constants");
+const errors_1 = require("../../http/errors");
+const logger_1 = require("../../logger/logger");
+const mappers_1 = require("../../mappers/mappers");
+const errors_2 = require("../../types/errors");
+const workers_1 = require("../../types/workers");
+const uploader_1 = require("../../uploader/uploader");
+const emit_1 = require("../emit");
+const spawn_helpers_1 = require("../spawn/spawn.helpers");
+/**
+ * Shared state/behavior for both sync modes; owns the `emit` control-protocol
+ * flow as a template method with mode-specific hooks.
+ */
+class BaseAdapter {
+    /** @internal */
+    constructor({ event, adapterState, options, }) {
+        this._hasWorkerEmitted = false;
+        this._isTimeout = false;
+        /** @internal */
+        this.timeoutSignal = new Promise((resolve) => {
+            this.resolveTimeoutSignal = resolve;
+        });
+        this.event = event;
+        this.options = options;
+        this.adapterState = adapterState;
+        this.uploader = new uploader_1.Uploader({
+            event,
+            options,
+        });
+        this._mappers = new mappers_1.Mappers({ event });
+    }
+    /** Sync mapper records for this event's sync unit, in both extraction and loading. */
+    get mappers() {
+        return this._mappers;
+    }
+    /** @internal */
+    get hasWorkerEmitted() {
+        return this._hasWorkerEmitted;
+    }
+    /** Set once the soft timeout fires; the task should wrap up and return. */
+    get isTimeout() {
+        return this._isTimeout;
+    }
+    /** @internal */
+    set isTimeout(value) {
+        this._isTimeout = value;
+        if (value) {
+            this.resolveTimeoutSignal();
+        }
+    }
+    /** Connector-owned state exposed to snap-in code. */
+    get state() {
+        return this.adapterState.state;
+    }
+    set state(value) {
+        this.adapterState.state = value;
+    }
+    /** @internal */
+    get sdkState() {
+        return this.adapterState.sdkState;
+    }
+    get extractionScope() {
+        return this.adapterState.extractionScope;
+    }
+    /**
+     * Persists connector and SDK state right now. State is saved on every emit
+     * anyway; call this only to record intent before an external write that
+     * must not be repeated. Throws if the save fails.
+     */
+    async postState() {
+        await this.adapterState.postState();
+    }
+    /**
+     * Uploads connector-owned data (e.g. state too large to keep inline) as a
+     * gzipped JSONL artifact. Unlike repo uploads it is not reported in the
+     * emitted event. Throws on failure.
+     */
+    async uploadArtifact(itemType, data) {
+        var _a;
+        const { artifact, error } = await this.uploader.upload(itemType, data);
+        if (error || !artifact) {
+            throw new errors_1.AirSyncHttpError(`Failed to upload artifact of type ${itemType}. ${(_a = error === null || error === void 0 ? void 0 : error.message) !== null && _a !== void 0 ? _a : 'No artifact returned.'}`);
+        }
+        return { id: artifact.id };
+    }
+    /** Downloads an artifact written by {@link uploadArtifact}. Throws on failure. */
+    async downloadArtifact(artifactId) {
+        const { response, error } = await this.uploader.getJsonObjectByArtifactId({
+            artifactId,
+            isGzipped: true,
+        });
+        if (error !== undefined || response === undefined) {
+            throw error instanceof errors_1.AirSyncHttpError
+                ? error
+                : new errors_1.AirSyncHttpError(`Failed to download artifact ${artifactId}. ${(0, logger_1.serializeError)(error)}`);
+        }
+        return (Array.isArray(response) ? response : [response]);
+    }
+    /**
+     * @internal
+     * Mode-specific worker metadata merged into the emitted event. The library
+     * version and state-date range are added centrally in `emit`, so subclasses
+     * only contribute their own statistics.
+     */
+    buildWorkerMetadata(
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    newEventType) {
+        return undefined;
+    }
+    /** @internal Runs before a failure is reported via `emitError`. */
+    beforeErrorEmit() { }
+    /**
+     * Maps a {@link TaskResult} to the phase-appropriate platform event and emits
+     * it exactly once. Invoked by the worker driver, not by connectors —
+     * connectors signal outcomes by returning a `TaskResult`, never by emitting.
+     *
+     * @internal
+     */
+    async emitFromResult(result) {
+        const { eventType, illegal } = (0, spawn_helpers_1.getEventTypeForResult)(this.event.payload.event_type, result.status);
+        const data = {};
+        if (result.status === 'delay') {
+            data.delay = result.delaySeconds;
+        }
+        else if (result.status === 'error') {
+            data.error = result.error;
+        }
+        else if (illegal) {
+            data.error = {
+                message: `Worker returned status '${result.status}' for a non-resumable phase (${this.event.payload.event_type}), which is not allowed. Emitting an error event instead.`,
+            };
+        }
+        await this.emit(eventType, data);
+    }
+    /** @internal */
+    async emit(newEventType, data) {
+        if (this._hasWorkerEmitted) {
+            console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
+            return;
+        }
+        try {
+            await this.beforeEmit(newEventType);
+            // Save state on every emit, except for start and delete events
+            if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
+                console.log(`Saving state before emitting event with event type: ${newEventType}.`);
+                await this.adapterState.postState();
+            }
+        }
+        catch (error) {
+            console.error('Error while preparing event for emission.', (0, logger_1.serializeError)(error));
+            await this.emitError(error);
+            return;
+        }
+        await this.send(newEventType, data);
+    }
+    /**
+     * Emits the phase error for a failed task or pre-emit step. Skips the
+     * pre-emit flush and state save, so a failed batch isn't retried; artifacts
+     * uploaded before the failure are still reported.
+     *
+     * @internal
+     */
+    async emitError(error) {
+        if (this._hasWorkerEmitted) {
+            return;
+        }
+        this.beforeErrorEmit();
+        const { eventType } = (0, spawn_helpers_1.getEventTypeForResult)(this.event.payload.event_type, 'error');
+        await this.send(eventType, { error: { message: (0, logger_1.serializeError)(error) } });
+    }
+    async send(newEventType, data) {
+        try {
+            if (data === null || data === void 0 ? void 0 : data.error) {
+                data.error = toWireError(data.error);
+            }
+            await (0, emit_1.emit)({
+                eventType: newEventType,
+                event: this.event,
+                data: Object.assign(Object.assign({}, data), this.buildEmitPayload(newEventType)),
+                worker_metadata: this.buildWorkerMetadata(newEventType),
+            });
+            const message = {
+                subject: workers_1.WorkerMessageSubject.WorkerMessageEmitted,
+                payload: { eventType: newEventType },
+            };
+            this.afterEmit(newEventType);
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
+            this._hasWorkerEmitted = true;
+        }
+        catch (error) {
+            console.error(`Error while emitting event with event type: ${newEventType}.`, (0, logger_1.serializeError)(error));
+            node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
+            this._hasWorkerEmitted = true;
+        }
+    }
+}
+exports.BaseAdapter = BaseAdapter;
+/**
+ * The platform matches error codes against the whole message, so a coded
+ * error is sent as just the code and its details only go to the logs.
+ */
+function toWireError({ message, code }) {
+    if (!code) {
+        return { message: (0, logger_1.truncateMessage)(message) };
+    }
+    console.error(`Reporting ${code}: ${message}`);
+    return { message: (0, errors_2.formatErrorCode)(code) };
+}

package/dist/multithreading/adapters/extraction-adapter.d.ts

@@ -0,0 +1,49 @@
+import { Repo } from '../../repo/repo';
+import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
+import { ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ItemTypeCount, ProcessAttachmentReturnType } from '../../types/extraction';
+import { TaskResult } from '../../types/workers';
+import { Artifact } from '../../uploader/uploader.interfaces';
+import { BaseAdapter } from './base-adapter';
+/** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
+export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
+    private _artifacts;
+    private _preExtractionItemCounts?;
+    private repos;
+    private lastExtractedItemType?;
+    private currentEventDataLength;
+    /** Defaults to true if the scope is empty or the item type is not listed. */
+    shouldExtract(itemType: string): boolean;
+    initializeRepos(repos: RepoInterface[]): void;
+    private createRepo;
+    /**
+     * SSOR attachments created outside the attachments phase are held in state
+     * and reported with ATTACHMENT_EXTRACTION_DONE instead of the current event.
+     */
+    private isDeferredRepo;
+    private isAttachmentsPhase;
+    private getOrCreateSsorRepo;
+    getRepo(itemType: string): Repo | undefined;
+    /** Artifacts uploaded so far that the next emit will report. */
+    get artifacts(): readonly Artifact[];
+    private addArtifacts;
+    /**
+     * Per-record-type counts for sync-duration estimation. Set during the
+     * metadata phase; the SDK attaches them to the metadata-done event.
+     */
+    get preExtractionItemCounts(): ItemTypeCount[] | undefined;
+    set preExtractionItemCounts(counts: ItemTypeCount[] | undefined);
+    private toRfc3339Timestamp;
+    private uploadAllRepos;
+    /**
+     * Streams one attachment into DevRev. Usable in any extraction phase; SSOR
+     * records created outside the attachments phase are reported with
+     * ATTACHMENT_EXTRACTION_DONE.
+     */
+    processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
+    streamAttachments<NewBatch>({ stream, processors, batchSize, }: {
+        stream: ExternalSystemAttachmentStreamingFunction;
+        processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
+        batchSize?: number;
+    }): Promise<TaskResult>;
+}
+//# sourceMappingURL=extraction-adapter.d.ts.map

package/dist/multithreading/adapters/extraction-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAoDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAwGjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}

package/dist/multithreading/adapters/extraction-adapter.js

@@ -0,0 +1,375 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExtractionAdapter = void 0;
+const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
+const constants_1 = require("../../common/constants");
+const logger_1 = require("../../logger/logger");
+const repo_1 = require("../../repo/repo");
+const extraction_1 = require("../../types/extraction");
+const uploader_helpers_1 = require("../../uploader/uploader.helpers");
+const base_adapter_1 = require("./base-adapter");
+/** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
+class ExtractionAdapter extends base_adapter_1.BaseAdapter {
+    /** @internal */
+    constructor(params) {
+        super(params);
+        this.repos = [];
+        this.currentEventDataLength = 0;
+        this._artifacts = [];
+    }
+    /** Defaults to true if the scope is empty or the item type is not listed. */
+    shouldExtract(itemType) {
+        const scope = this.extractionScope;
+        if (Object.keys(scope).length === 0)
+            return true;
+        if (!(itemType in scope))
+            return true;
+        return scope[itemType].extract;
+    }
+    initializeRepos(repos) {
+        const existingSsorRepo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
+        this.repos = repos.map((repo) => this.createRepo(repo));
+        if (existingSsorRepo &&
+            !this.repos.some((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT)) {
+            this.repos.push(existingSsorRepo);
+        }
+    }
+    createRepo(repo) {
+        const shouldNormalize = repo.itemType !== constants_1.AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA &&
+            repo.itemType !== constants_1.SSOR_ATTACHMENT;
+        const isDeferred = this.isDeferredRepo(repo.itemType);
+        return new repo_1.Repo(Object.assign(Object.assign({ event: this.event, itemType: repo.itemType }, (shouldNormalize && { normalize: repo.normalize })), { skipConfirmation: repo.itemType === constants_1.AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, onUpload: (artifact) => {
+                var _a, _b;
+                if (isDeferred) {
+                    const toDevRev = this.adapterState.sdkState.toDevRev;
+                    if (toDevRev) {
+                        (_a = toDevRev.deferredSsorArtifacts) !== null && _a !== void 0 ? _a : (toDevRev.deferredSsorArtifacts = []);
+                        toDevRev.deferredSsorArtifacts.push(artifact);
+                    }
+                    return;
+                }
+                this.lastExtractedItemType = repo.itemType;
+                // Artifact ids are kept in state for later attachment streaming
+                if (repo.itemType === constants_1.AirSyncDefaultItemTypes.ATTACHMENTS) {
+                    (_b = this.adapterState.sdkState.toDevRev) === null || _b === void 0 ? void 0 : _b.attachmentsMetadata.artifactIds.push(artifact.id);
+                }
+                // Track size of artifact objects that go in the SQS message; flip
+                // isTimeout once the threshold is exceeded to force an early emit.
+                this.currentEventDataLength += Buffer.byteLength(JSON.stringify(artifact), 'utf8');
+                if (this.currentEventDataLength > constants_1.EVENT_SIZE_THRESHOLD_BYTES &&
+                    !this.isTimeout) {
+                    this.isTimeout = true;
+                }
+            }, options: Object.assign(Object.assign({}, this.options), repo.overriddenOptions) }));
+    }
+    /**
+     * SSOR attachments created outside the attachments phase are held in state
+     * and reported with ATTACHMENT_EXTRACTION_DONE instead of the current event.
+     */
+    isDeferredRepo(itemType) {
+        return itemType === constants_1.SSOR_ATTACHMENT && !this.isAttachmentsPhase();
+    }
+    isAttachmentsPhase() {
+        const eventType = this.event.payload.event_type;
+        return (eventType === extraction_1.EventType.StartExtractingAttachments ||
+            eventType === extraction_1.EventType.ContinueExtractingAttachments);
+    }
+    getOrCreateSsorRepo() {
+        let repo = this.repos.find((repo) => repo.itemType === constants_1.SSOR_ATTACHMENT);
+        if (!repo) {
+            repo = this.createRepo({ itemType: constants_1.SSOR_ATTACHMENT });
+            this.repos.push(repo);
+        }
+        return repo;
+    }
+    getRepo(itemType) {
+        const repo = this.repos.find((repo) => repo.itemType === itemType);
+        if (!repo) {
+            console.error(`Repo for item type ${itemType} not found.`);
+            return;
+        }
+        return repo;
+    }
+    /** Artifacts uploaded so far that the next emit will report. */
+    get artifacts() {
+        return this._artifacts;
+    }
+    addArtifacts(artifacts) {
+        this._artifacts = this._artifacts
+            .concat(artifacts)
+            .filter((value, index, self) => self.indexOf(value) === index);
+    }
+    /**
+     * Per-record-type counts for sync-duration estimation. Set during the
+     * metadata phase; the SDK attaches them to the metadata-done event.
+     */
+    get preExtractionItemCounts() {
+        return this._preExtractionItemCounts;
+    }
+    set preExtractionItemCounts(counts) {
+        this._preExtractionItemCounts = counts;
+    }
+    /** @internal */
+    async beforeEmit(newEventType) {
+        var _a;
+        console.log(`Uploading all repos before emitting event with event type: ${newEventType}.`);
+        await this.uploadAllRepos();
+        // When the full extraction cycle completes, commit the extraction window
+        // boundaries so subsequent incremental syncs can resume from them.
+        if (newEventType === extraction_1.ExtractorEventType.AttachmentExtractionDone) {
+            const sdkState = this.adapterState.sdkState;
+            // Released before the state save so they're never reported twice.
+            const toDevRev = sdkState.toDevRev;
+            if ((_a = toDevRev === null || toDevRev === void 0 ? void 0 : toDevRev.deferredSsorArtifacts) === null || _a === void 0 ? void 0 : _a.length) {
+                this.addArtifacts(toDevRev.deferredSsorArtifacts);
+                toDevRev.deferredSsorArtifacts = [];
+            }
+            console.log(`Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${sdkState.lastSyncStarted}).`);
+            sdkState.lastSuccessfulSyncStarted = sdkState.lastSyncStarted;
+            sdkState.lastSyncStarted = '';
+            sdkState.pendingWorkersOldest = '';
+            sdkState.pendingWorkersNewest = '';
+            // Expand boundaries: workersOldest keeps the earliest timestamp seen,
+            // workersNewest the latest.
+            const extractionStart = this.event.payload.event_context.extract_from;
+            const extractionEnd = this.event.payload.event_context.extract_to;
+            if (extractionStart &&
+                (!sdkState.workersOldest || extractionStart < sdkState.workersOldest)) {
+                console.log(`Updating workersOldest from '${sdkState.workersOldest}' to '${extractionStart}'.`);
+                sdkState.workersOldest = extractionStart;
+            }
+            if (extractionEnd &&
+                (!sdkState.workersNewest || extractionEnd > sdkState.workersNewest)) {
+                console.log(`Updating workersNewest from '${sdkState.workersNewest}' to '${extractionEnd}'.`);
+                sdkState.workersNewest = extractionEnd;
+            }
+        }
+    }
+    /** @internal */
+    buildEmitPayload(newEventType) {
+        const isExtractionEvent = Object.values(extraction_1.ExtractorEventType).includes(newEventType);
+        if (!isExtractionEvent) {
+            return {};
+        }
+        const payload = { artifacts: [...this._artifacts] };
+        if (newEventType === extraction_1.ExtractorEventType.MetadataExtractionDone &&
+            this._preExtractionItemCounts) {
+            payload.pre_extraction_item_counts = this._preExtractionItemCounts;
+        }
+        return payload;
+    }
+    /**
+     * @internal
+     * Attaches the last-extracted item type and its created/modified date ranges
+     * to data/attachment done/progress events, mirroring the metadata v1 sent.
+     */
+    buildWorkerMetadata(newEventType) {
+        if (newEventType !== extraction_1.ExtractorEventType.DataExtractionDone &&
+            newEventType !== extraction_1.ExtractorEventType.DataExtractionProgress &&
+            newEventType !== extraction_1.ExtractorEventType.AttachmentExtractionDone &&
+            newEventType !== extraction_1.ExtractorEventType.AttachmentExtractionProgress) {
+            return undefined;
+        }
+        const repo = this.lastExtractedItemType
+            ? this.repos.find((r) => r.itemType === this.lastExtractedItemType)
+            : undefined;
+        if (!repo) {
+            return undefined;
+        }
+        return {
+            item_type: repo.itemType,
+            newest_created_date: this.toRfc3339Timestamp(repo.dateRanges.creationDate.newest),
+            oldest_created_date: this.toRfc3339Timestamp(repo.dateRanges.creationDate.oldest),
+            newest_modified_date: this.toRfc3339Timestamp(repo.dateRanges.modifiedDate.newest),
+            oldest_modified_date: this.toRfc3339Timestamp(repo.dateRanges.modifiedDate.oldest),
+        };
+    }
+    toRfc3339Timestamp(ms) {
+        if (ms === undefined || !Number.isFinite(ms)) {
+            return undefined;
+        }
+        return new Date(ms).toISOString();
+    }
+    /** @internal */
+    afterEmit() {
+        this._artifacts = [];
+    }
+    async uploadAllRepos() {
+        for (const repo of this.repos) {
+            await repo.upload();
+            if (!this.isDeferredRepo(repo.itemType)) {
+                this.addArtifacts(repo.uploadedArtifacts);
+            }
+        }
+    }
+    /** @internal */
+    beforeErrorEmit() {
+        // Report artifacts that uploaded before the failure so they aren't orphaned.
+        for (const repo of this.repos) {
+            if (!this.isDeferredRepo(repo.itemType)) {
+                this.addArtifacts(repo.uploadedArtifacts);
+            }
+        }
+    }
+    /**
+     * Streams one attachment into DevRev. Usable in any extraction phase; SSOR
+     * records created outside the attachments phase are reported with
+     * ATTACHMENT_EXTRACTION_DONE.
+     */
+    async processAttachment(attachment, stream) {
+        var _a, _b;
+        const response = await stream({
+            item: attachment,
+            event: this.event,
+        });
+        if (response.status !== 'success') {
+            return response;
+        }
+        const { httpStream } = response;
+        const fileType = attachment.content_type ||
+            ((_a = httpStream.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toString()) ||
+            'application/octet-stream';
+        const contentLength = (_b = httpStream.headers['content-length']) === null || _b === void 0 ? void 0 : _b.toString();
+        const fileSize = contentLength ? parseInt(contentLength) : undefined;
+        const { error: artifactUrlError, response: artifactUrlResponse } = await this.uploader.getArtifactUploadUrl(attachment.file_name, fileType, fileSize);
+        if (artifactUrlError) {
+            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+            return {
+                status: 'error',
+                error: {
+                    message: `Error while preparing artifact for attachment ID ${attachment.id}. Skipping attachment. ${(0, logger_1.serializeError)(artifactUrlError)}`,
+                    fileSize,
+                },
+            };
+        }
+        const { error: uploadedArtifactError } = await this.uploader.streamArtifact(artifactUrlResponse, httpStream);
+        if (uploadedArtifactError) {
+            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+            return {
+                status: 'error',
+                error: {
+                    message: `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` +
+                        (0, logger_1.serializeError)(uploadedArtifactError),
+                    fileSize,
+                },
+            };
+        }
+        const { error: confirmArtifactUploadError } = await this.uploader.confirmArtifactUpload(artifactUrlResponse.artifact_id);
+        if (confirmArtifactUploadError) {
+            return {
+                status: 'error',
+                error: {
+                    message: `Error while confirming upload for attachment ID ${attachment.id}. ` +
+                        (0, logger_1.serializeError)(confirmArtifactUploadError),
+                    fileSize,
+                },
+            };
+        }
+        const ssorAttachment = {
+            id: {
+                devrev: artifactUrlResponse.artifact_id,
+                external: attachment.id,
+            },
+            parent_id: {
+                external: attachment.parent_id,
+            },
+        };
+        if (attachment.author_id) {
+            ssorAttachment.actor_id = {
+                external: attachment.author_id,
+            };
+        }
+        // Set inline flag only if it is explicitly set on the attachment.
+        if (attachment.inline === true) {
+            ssorAttachment.inline = true;
+        }
+        else if (attachment.inline === false) {
+            ssorAttachment.inline = false;
+        }
+        if (this.isTimeout) {
+            (0, uploader_helpers_1.destroyHttpStream)(httpStream);
+            return { status: 'success' };
+        }
+        await this.getOrCreateSsorRepo().push([ssorAttachment]);
+        return { status: 'success' };
+    }
+    async streamAttachments({ stream, processors, batchSize = 1, }) {
+        var _a, _b;
+        if (batchSize <= 0) {
+            console.warn(`The specified batch size (${batchSize}) is invalid. Using 1 instead.`);
+            batchSize = 1;
+        }
+        if (batchSize > 50) {
+            console.warn(`The specified batch size (${batchSize}) is too large. Using 50 instead.`);
+            batchSize = 50;
+        }
+        this.initializeRepos([{ itemType: constants_1.SSOR_ATTACHMENT }]);
+        const attachmentsMetadata = (_a = this.adapterState.sdkState.toDevRev) === null || _a === void 0 ? void 0 : _a.attachmentsMetadata;
+        if (!((_b = attachmentsMetadata === null || attachmentsMetadata === void 0 ? void 0 : attachmentsMetadata.artifactIds) === null || _b === void 0 ? void 0 : _b.length)) {
+            console.log(`No attachments metadata artifact IDs found in state.`);
+            return { status: 'success' };
+        }
+        else {
+            console.log(`Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.`);
+        }
+        while (attachmentsMetadata.artifactIds.length > 0) {
+            const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0];
+            console.log(`Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.`);
+            const { attachments, error } = await this.uploader.getAttachmentsFromArtifactId({
+                artifact: attachmentsMetadataArtifactId,
+            });
+            if (error) {
+                console.error(`Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                return { status: 'error', error };
+            }
+            if (!attachments || attachments.length === 0) {
+                console.warn(`No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.`);
+                attachmentsMetadata.artifactIds.shift();
+                attachmentsMetadata.lastProcessed = 0;
+                continue;
+            }
+            console.log(`Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+            let response;
+            if (processors) {
+                console.log(`Using custom processors for attachments.`);
+                const reducer = processors.reducer;
+                const iterator = processors.iterator;
+                const reducedAttachments = reducer({
+                    attachments,
+                    adapter: this,
+                    batchSize,
+                });
+                response = await iterator({
+                    reducedAttachments,
+                    adapter: this,
+                    stream,
+                });
+            }
+            else {
+                console.log(`Using attachments streaming pool for attachments streaming.`);
+                const attachmentsPool = new attachments_streaming_pool_1.AttachmentsStreamingPool({
+                    adapter: this,
+                    attachments,
+                    batchSize,
+                    stream,
+                });
+                response = await attachmentsPool.streamAll();
+            }
+            if (response.status !== 'success') {
+                return response;
+            }
+            if (this.isTimeout) {
+                console.log(`Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.`);
+                return { status: 'progress' };
+            }
+            console.log(`Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.`);
+            attachmentsMetadata.artifactIds.shift();
+            attachmentsMetadata.lastProcessed = 0;
+            if (attachmentsMetadata.lastProcessedAttachmentsIdsList) {
+                attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0;
+            }
+        }
+        return { status: 'success' };
+    }
+}
+exports.ExtractionAdapter = ExtractionAdapter;

package/dist/multithreading/adapters/loading-adapter.d.ts

@@ -0,0 +1,26 @@
+import { ExternalSystemAttachment, ExternalSystemLoadingFunction, ItemTypesToLoadParams, LoaderReport } from '../../types/loading';
+import { TaskResult } from '../../types/workers';
+import { BaseAdapter } from './base-adapter';
+/** Adapter passed to loading tasks (item/attachment loading, loader reports). @public */
+export declare class LoadingAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
+    private loaderReports;
+    private _processedFiles;
+    get reports(): LoaderReport[];
+    get processedFiles(): string[];
+    loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise<TaskResult>;
+    loadAttachments({ create, }: {
+        create: ExternalSystemLoadingFunction<ExternalSystemAttachment>;
+    }): Promise<TaskResult>;
+    /**
+     * Walks the transformer files in `fromDevRev.filesToLoad`, resuming from
+     * each file's `lineToProcess`. `getLoader` returns undefined when a file has
+     * an item type the connector can't load, which fails the phase.
+     */
+    private loadFiles;
+    private getLoaderBatches;
+    private loadItem;
+    /** Maps a non-success connector response to a delay or a failed report. */
+    private toFailedLoad;
+    private loadAttachment;
+}
+//# sourceMappingURL=loading-adapter.d.ts.map

package/dist/multithreading/adapters/loading-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loading-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/loading-adapter.ts"],"names":[],"mappings":"AASA,OAAO,EAEL,wBAAwB,EAGxB,6BAA6B,EAE7B,qBAAqB,EAGrB,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAM7C,yFAAyF;AACzF,qBAAa,cAAc,CACzB,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAW;IAalC,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,IAAI,cAAc,IAAI,MAAM,EAAE,CAE7B;IA2BK,aAAa,CAAC,EAClB,eAAe,GAChB,EAAE,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC;IA8BxC,eAAe,CAAC,EACpB,MAAM,GACP,EAAE;QACD,MAAM,EAAE,6BAA6B,CAAC,wBAAwB,CAAC,CAAC;KACjE,GAAG,OAAO,CAAC,UAAU,CAAC;IAavB;;;;OAIG;YACW,SAAS;YAiFT,gBAAgB;YAuBhB,QAAQ;IA+HtB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;YAwBN,cAAc;CA2C7B"}