@strapi/data-transfer 5.10.3 → 5.11.0

package/dist/index.mjs

@@ -1,4 +1,5 @@
-import { Transform, PassThrough, Writable, Readable, Duplex, pipeline } from 'stream';
+import { Transform, PassThrough, Writable, Readable, Duplex, pipeline as pipeline$1 } from 'stream';
+import { pipeline } from 'stream/promises';
 import path, { extname, join, posix } from 'path';
 import { EOL } from 'os';
 import { chain } from 'stream-chain';
@@ -633,8 +634,7 @@ const TRANSFER_STAGES = Object.freeze([
 };
 const DEFAULT_VERSION_STRATEGY = 'ignore';
 const DEFAULT_SCHEMA_STRATEGY = 'strict';
-var _metadata$1 = /*#__PURE__*/ _class_private_field_loose_key$6("_metadata"), _schema = /*#__PURE__*/ _class_private_field_loose_key$6("_schema"), _handlers = /*#__PURE__*/ _class_private_field_loose_key$6("_handlers"), // Save the currently open stream so that we can access it at any time
-_currentStream = /*#__PURE__*/ _class_private_field_loose_key$6("_currentStream"), /**
+var _metadata$1 = /*#__PURE__*/ _class_private_field_loose_key$6("_metadata"), _schema = /*#__PURE__*/ _class_private_field_loose_key$6("_schema"), _handlers = /*#__PURE__*/ _class_private_field_loose_key$6("_handlers"), _currentStreamController = /*#__PURE__*/ _class_private_field_loose_key$6("_currentStreamController"), _aborted = /*#__PURE__*/ _class_private_field_loose_key$6("_aborted"), /**
    * Create and return a transform stream based on the given stage and options.
    *
    * Allowed transformations includes 'filter' and 'map'.
@@ -749,11 +749,9 @@ class TransferEngine {
     }
     // Cause an ongoing transfer to abort gracefully
     async abortTransfer() {
-        const err = new TransferEngineError('fatal', 'Transfer aborted.');
-        if (!_class_private_field_loose_base$6(this, _currentStream)[_currentStream]) {
-            throw err;
-        }
-        _class_private_field_loose_base$6(this, _currentStream)[_currentStream].destroy(err);
+        _class_private_field_loose_base$6(this, _aborted)[_aborted] = true;
+        _class_private_field_loose_base$6(this, _currentStreamController)[_currentStreamController]?.abort();
+        throw new TransferEngineError('fatal', 'Transfer aborted.');
     }
     async init() {
         // Resolve providers' resource and store
@@ -1063,7 +1061,11 @@ class TransferEngine {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, _currentStream, {
+        Object.defineProperty(this, _currentStreamController, {
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, _aborted, {
             writable: true,
             value: void 0
         });
@@ -1073,6 +1075,7 @@ class TransferEngine {
             schemaDiff: [],
             errors: {}
         };
+        _class_private_field_loose_base$6(this, _aborted)[_aborted] = false;
         this.diagnostics = createDiagnosticReporter();
         validateProvider('source', sourceProvider);
         validateProvider('destination', destinationProvider);
@@ -1271,6 +1274,9 @@ function assertSchemasMatching(sourceSchemas, destinationSchemas) {
     }
 }
 async function transferStage(options) {
+    if (_class_private_field_loose_base$6(this, _aborted)[_aborted]) {
+        throw new TransferEngineError('fatal', 'Transfer aborted.');
+    }
     const { stage, source, destination, transform, tracker } = options;
     const updateEndTime = ()=>{
         const stageData = this.progress.data[stage];
@@ -1302,27 +1308,42 @@ async function transferStage(options) {
         return;
     }
     _class_private_field_loose_base$6(this, _emitStageUpdate)[_emitStageUpdate]('start', stage);
-    await new Promise((resolve, reject)=>{
-        let stream = source;
+    try {
+        const streams = [
+            source
+        ];
         if (transform) {
-            stream = stream.pipe(transform);
+            streams.push(transform);
         }
         if (tracker) {
-            stream = stream.pipe(tracker);
-        }
-        _class_private_field_loose_base$6(this, _currentStream)[_currentStream] = stream.pipe(destination).on('error', (e)=>{
-            updateEndTime();
-            _class_private_field_loose_base$6(this, _emitStageUpdate)[_emitStageUpdate]('error', stage);
-            this.reportError(e, 'error');
-            destination.destroy(e);
-            reject(e);
-        }).on('close', ()=>{
-            _class_private_field_loose_base$6(this, _currentStream)[_currentStream] = undefined;
-            updateEndTime();
-            resolve();
+            streams.push(tracker);
+        }
+        streams.push(destination);
+        // NOTE: to debug/confirm backpressure issues from misbehaving stream, uncomment the following lines
+        // source.on('pause', () => console.log(`[${stage}] Source paused due to backpressure`));
+        // source.on('resume', () => console.log(`[${stage}] Source resumed`));
+        // destination.on('drain', () =>
+        //   console.log(`[${stage}] Destination drained, resuming data flow`)
+        // );
+        // destination.on('error', (err) => console.error(`[${stage}] Destination error:`, err));
+        const controller = new AbortController();
+        const { signal } = controller;
+        // Store the controller so you can cancel later
+        _class_private_field_loose_base$6(this, _currentStreamController)[_currentStreamController] = controller;
+        await pipeline(streams, {
+            signal
         });
-    });
-    _class_private_field_loose_base$6(this, _emitStageUpdate)[_emitStageUpdate]('finish', stage);
+        _class_private_field_loose_base$6(this, _emitStageUpdate)[_emitStageUpdate]('finish', stage);
+    } catch (e) {
+        updateEndTime();
+        _class_private_field_loose_base$6(this, _emitStageUpdate)[_emitStageUpdate]('error', stage);
+        this.reportError(e, 'error');
+        if (!destination.destroyed) {
+            destination.destroy(e);
+        }
+    } finally{
+        updateEndTime();
+    }
 }
 async function resolveProviderResource() {
     const sourceMetadata = await this.sourceProvider.getMetadata();
@@ -1894,9 +1915,7 @@ const createLinkQuery = (strapi, trx)=>{
                 assignOrderColumns();
                 const qb = connection.insert(payload).into(addSchema(joinTable.name));
                 if (trx) {
-                    await trx.transaction(async (nestedTrx)=>{
-                        await qb.transacting(nestedTrx);
-                    });
+                    await qb.transacting(trx);
                 }
             }
             if ('morphColumn' in attribute && attribute.morphColumn) {
@@ -2349,7 +2368,7 @@ class LocalStrapiDestinationProvider {
                     const provider = strapi.config.get('plugin::upload').provider;
                     const fileId = fileEntitiesMapper?.[uploadData.id];
                     if (!fileId) {
-                        callback(new Error(`File ID not found for ID: ${uploadData.id}`));
+                        return callback(new Error(`File ID not found for ID: ${uploadData.id}`));
                     }
                     try {
                         await strapi.plugin('upload').provider.uploadStream(uploadData);
@@ -2400,9 +2419,9 @@ class LocalStrapiDestinationProvider {
                                 provider
                             }
                         });
-                        callback();
+                        return callback();
                     } catch (error) {
-                        callback(new Error(`Error while uploading asset ${chunk.filename} ${error}`));
+                        return callback(new Error(`Error while uploading asset ${chunk.filename} ${error}`));
                     }
                 });
             }
@@ -2824,7 +2843,11 @@ function _class_private_field_loose_key$4(name) {
 const createLocalStrapiSourceProvider = (options)=>{
     return new LocalStrapiSourceProvider(options);
 };
-var _diagnostics$4 = /*#__PURE__*/ _class_private_field_loose_key$4("_diagnostics"), _reportInfo$4 = /*#__PURE__*/ _class_private_field_loose_key$4("_reportInfo");
+var _diagnostics$4 = /*#__PURE__*/ _class_private_field_loose_key$4("_diagnostics"), _reportInfo$4 = /*#__PURE__*/ _class_private_field_loose_key$4("_reportInfo"), /**
+   * Reports an error to the diagnostic reporter.
+   */ _reportError = /*#__PURE__*/ _class_private_field_loose_key$4("_reportError"), /**
+   * Handles errors that occur in read streams.
+   */ _handleStreamError = /*#__PURE__*/ _class_private_field_loose_key$4("_handleStreamError");
 class LocalStrapiSourceProvider {
     async bootstrap(diagnostics) {
         _class_private_field_loose_base$4(this, _diagnostics$4)[_diagnostics$4] = diagnostics;
@@ -2886,12 +2909,22 @@ class LocalStrapiSourceProvider {
     createAssetsReadStream() {
         assertValidStrapi(this.strapi, 'Not able to stream assets');
         _class_private_field_loose_base$4(this, _reportInfo$4)[_reportInfo$4]('creating assets read stream');
-        return createAssetsStream(this.strapi);
+        const stream = createAssetsStream(this.strapi);
+        stream.on('error', (err)=>{
+            _class_private_field_loose_base$4(this, _handleStreamError)[_handleStreamError]('assets', err);
+        });
+        return stream;
     }
     constructor(options){
         Object.defineProperty(this, _reportInfo$4, {
             value: reportInfo$4
         });
+        Object.defineProperty(this, _reportError, {
+            value: reportError
+        });
+        Object.defineProperty(this, _handleStreamError, {
+            value: handleStreamError
+        });
         Object.defineProperty(this, _diagnostics$4, {
             writable: true,
             value: void 0
@@ -2911,6 +2944,29 @@ function reportInfo$4(message) {
         kind: 'info'
     });
 }
+function reportError(message, error) {
+    _class_private_field_loose_base$4(this, _diagnostics$4)[_diagnostics$4]?.report({
+        details: {
+            createdAt: new Date(),
+            message,
+            error,
+            severity: 'fatal',
+            name: error.name
+        },
+        kind: 'error'
+    });
+}
+function handleStreamError(streamType, err) {
+    const { message, stack } = err;
+    const errorMessage = `[Data transfer] Error in ${streamType} read stream: ${message}`;
+    const formattedError = {
+        message: errorMessage,
+        stack,
+        timestamp: new Date().toISOString()
+    };
+    this.strapi?.log.error(formattedError);
+    _class_private_field_loose_base$4(this, _reportError)[_reportError](formattedError.message, err);
+}
 
 const createDispatcher = (ws, retryMessageOptions = {
     retryMessageMaxRetries: 5,
@@ -3074,17 +3130,6 @@ const connectToWebsocket = (address, options, diagnostics)=>{
 const trimTrailingSlash = (input)=>{
     return input.replace(/\/$/, '');
 };
-const wait = (ms)=>{
-    return new Promise((resolve)=>{
-        setTimeout(resolve, ms);
-    });
-};
-const waitUntil = async (test, interval)=>{
-    while(!test()){
-        await wait(interval);
-    }
-    return Promise.resolve();
-};
 
 const TRANSFER_PATH = '/transfer/runner';
 const TRANSFER_METHODS = [
@@ -3505,6 +3550,17 @@ class RemoteStrapiSourceProvider {
         });
         // Init the asset map
         const assets = {};
+        // Watch for stalled assets; if we don't receive a chunk within timeout, abort transfer
+        const resetTimeout = (assetID)=>{
+            if (assets[assetID].timeout) {
+                clearTimeout(assets[assetID].timeout);
+            }
+            assets[assetID].timeout = setTimeout(()=>{
+                _class_private_field_loose_base$2(this, _reportInfo$2)[_reportInfo$2](`Asset ${assetID} transfer stalled, aborting.`);
+                assets[assetID].status = 'errored';
+                assets[assetID].stream.destroy(new Error(`Asset ${assetID} transfer timed out`));
+            }, this.options.streamTimeout);
+        };
         stream/**
        * Process a payload of many transfer assets and performs the following tasks:
        * - Start: creates a stream for new assets.
@@ -3515,56 +3571,46 @@ class RemoteStrapiSourceProvider {
                 const { action, assetID } = item;
                 // Creates the stream to send the incoming asset through
                 if (action === 'start') {
-                    // Ignore the item if a transfer has already been started for the same asset ID
+                    // if a transfer has already been started for the same asset ID, something is wrong
                     if (assets[assetID]) {
-                        continue;
+                        throw new Error(`Asset ${assetID} already started`);
                     }
+                    _class_private_field_loose_base$2(this, _reportInfo$2)[_reportInfo$2](`Asset ${assetID} starting`);
                     // Register the asset
                     assets[assetID] = {
                         ...item.data,
                         stream: new PassThrough(),
-                        status: 'idle',
+                        status: 'ok',
                         queue: []
                     };
+                    resetTimeout(assetID);
                     // Connect the individual asset stream to the main asset stage stream
                     // Note: nothing is transferred until data chunks are fed to the asset stream
                     await this.writeAsync(pass, assets[assetID]);
-                } else if (action === 'stream') {
-                    // If the asset hasn't been registered, or if it's been closed already, then ignore the message
+                } else if (action === 'stream' || action === 'end') {
+                    // If the asset hasn't been registered, or if it's been closed already, something is wrong
                     if (!assets[assetID]) {
-                        continue;
+                        throw new Error(`No id matching ${assetID} for stream action`);
                     }
-                    switch(assets[assetID].status){
-                        // The asset is ready to accept a new chunk, write it now
-                        case 'idle':
-                            await writeAssetChunk(assetID, item.data);
-                            break;
-                        // The resource is busy, queue the current chunk so that it gets transferred as soon as possible
-                        case 'busy':
-                            assets[assetID].queue.push(item);
-                            break;
+                    // On every action, reset the timeout timer
+                    if (action === 'stream') {
+                        resetTimeout(assetID);
+                    } else {
+                        clearTimeout(assets[assetID].timeout);
                     }
-                } else if (action === 'end') {
-                    // If the asset has already been closed, or if it was never registered, ignore the command
-                    if (!assets[assetID]) {
-                        continue;
+                    if (assets[assetID].status === 'closed') {
+                        throw new Error(`Asset ${assetID} is closed`);
                     }
-                    switch(assets[assetID].status){
-                        // There's no ongoing activity, the asset is ready to be closed
-                        case 'idle':
-                        case 'errored':
-                            await closeAssetStream(assetID);
-                            break;
-                        // The resource is busy, wait for a different state and close the stream.
-                        case 'busy':
-                            await Promise.race([
-                                // Either: wait for the asset to be ready to be closed
-                                waitUntil(()=>assets[assetID].status !== 'busy', 100),
-                                // Or: if the last chunks are still not processed after ten seconds
-                                wait(10000)
-                            ]);
-                            await closeAssetStream(assetID);
-                            break;
+                    assets[assetID].queue.push(item);
+                }
+            }
+            // each new payload will start new processQueue calls, which may cause some extra calls
+            // it's essentially saying "start processing this asset again, I added more data to the queue"
+            for(const assetID in assets){
+                if (Object.prototype.hasOwnProperty.call(assets, assetID)) {
+                    const asset = assets[assetID];
+                    if (asset.queue?.length > 0) {
+                        await processQueue(assetID);
                     }
                 }
             }
@@ -3572,38 +3618,48 @@ class RemoteStrapiSourceProvider {
             pass.end();
         });
         /**
-     * Writes a chunk of data for the specified asset with the given id.
-     */ const writeAssetChunk = async (id, data)=>{
+     * Start processing the queue for a given assetID
+     *
+     * Even though this is a loop that attempts to process the entire queue, it is safe to call this more than once
+     * for the same asset id because the queue is shared globally, the items are shifted off, and immediately written
+     */ const processQueue = async (id)=>{
             if (!assets[id]) {
                 throw new Error(`Failed to write asset chunk for "${id}". Asset not found.`);
             }
-            const { status: currentStatus } = assets[id];
-            if (currentStatus !== 'idle') {
+            const asset = assets[id];
+            const { status: currentStatus } = asset;
+            if ([
+                'closed',
+                'errored'
+            ].includes(currentStatus)) {
                 throw new Error(`Failed to write asset chunk for "${id}". The asset is currently "${currentStatus}"`);
             }
-            const nextItemInQueue = ()=>assets[id].queue.shift();
-            try {
-                // Lock the asset
-                assets[id].status = 'busy';
-                // Save the current chunk
-                await unsafe_writeAssetChunk(id, data);
-                // Empty the queue if needed
-                let item = nextItemInQueue();
-                while(item){
-                    await unsafe_writeAssetChunk(id, item.data);
-                    item = nextItemInQueue();
+            while(asset.queue.length > 0){
+                const data = asset.queue.shift();
+                if (!data) {
+                    throw new Error(`Invalid chunk found for ${id}`);
+                }
+                try {
+                    // if this is an end chunk, close the asset stream
+                    if (data.action === 'end') {
+                        _class_private_field_loose_base$2(this, _reportInfo$2)[_reportInfo$2](`Ending asset stream for ${id}`);
+                        await closeAssetStream(id);
+                        break; // Exit the loop after closing the stream
+                    }
+                    // Save the current chunk
+                    await writeChunkToStream(id, data);
+                } catch  {
+                    if (!assets[id]) {
+                        throw new Error(`No id matching ${id} for writeAssetChunk`);
+                    }
                 }
-                // Unlock the asset
-                assets[id].status = 'idle';
-            } catch  {
-                assets[id].status = 'errored';
             }
         };
         /**
      * Writes a chunk of data to the asset's stream.
      *
      * Only check if the targeted asset exists, no other validation is done.
-     */ const unsafe_writeAssetChunk = async (id, data)=>{
+     */ const writeChunkToStream = async (id, data)=>{
             const asset = assets[id];
             if (!asset) {
                 throw new Error(`Failed to write asset chunk for "${id}". Asset not found.`);
@@ -3624,9 +3680,11 @@ class RemoteStrapiSourceProvider {
             await new Promise((resolve, reject)=>{
                 const { stream } = assets[id];
                 stream.on('close', ()=>{
-                    delete assets[id];
                     resolve();
-                }).on('error', reject).end();
+                }).on('error', (e)=>{
+                    assets[id].status = 'errored';
+                    reject(new Error(`Failed to close asset "${id}". Asset stream error: ${e.toString()}`));
+                }).end();
             });
         };
         return pass;
@@ -3741,6 +3799,9 @@ class RemoteStrapiSourceProvider {
         });
         this.name = 'source::remote-strapi';
         this.type = 'source';
+        this.defaultOptions = {
+            streamTimeout: 15000
+        };
         this.writeAsync = (stream, data)=>{
             return new Promise((resolve, reject)=>{
                 stream.write(data, (error)=>{
@@ -3751,7 +3812,10 @@ class RemoteStrapiSourceProvider {
                 });
             });
         };
-        this.options = options;
+        this.options = {
+            ...this.defaultOptions,
+            ...options
+        };
         this.ws = null;
         this.dispatcher = null;
     }
@@ -4684,6 +4748,18 @@ const createPullController = handlerControllerFactory((proto)=>({
                 kind: 'warning'
             });
         },
+        onError (error) {
+            this.diagnostics?.report({
+                details: {
+                    message: error.message,
+                    error,
+                    createdAt: new Date(),
+                    name: error.name,
+                    severity: 'fatal'
+                },
+                kind: 'error'
+            });
+        },
         assertValidTransferAction (action) {
             // Abstract the constant to string[] to allow looser check on the given action
             const validActions = VALID_TRANSFER_ACTIONS;
@@ -4757,6 +4833,15 @@ const createPullController = handlerControllerFactory((proto)=>({
             let batch = [];
             const stream = this.streams?.[stage];
             const batchLength = ()=>Buffer.byteLength(JSON.stringify(batch));
+            const maybeConfirm = async (data)=>{
+                try {
+                    await this.confirm(data);
+                } catch (error) {
+                    // Handle the error, log it, or take other appropriate actions
+                    strapi?.log.error(`[Data transfer] Message confirmation failed: ${error?.message}`);
+                    this.onError(error);
+                }
+            };
             const sendBatch = async ()=>{
                 await this.confirm({
                     type: 'transfer',
@@ -4765,6 +4850,7 @@ const createPullController = handlerControllerFactory((proto)=>({
                     error: null,
                     id
                 });
+                batch = [];
             };
             if (!stream) {
                 throw new ProviderTransferError(`No available stream found for ${stage}`);
@@ -4775,7 +4861,6 @@ const createPullController = handlerControllerFactory((proto)=>({
                         batch.push(chunk);
                         if (batchLength() >= batchSize) {
                             await sendBatch();
-                            batch = [];
                         }
                     } else {
                         await this.confirm({
@@ -4791,7 +4876,6 @@ const createPullController = handlerControllerFactory((proto)=>({
                 }
                 if (batch.length > 0 && stage !== 'assets') {
                     await sendBatch();
-                    batch = [];
                 }
                 await this.confirm({
                     type: 'transfer',
@@ -4801,7 +4885,8 @@ const createPullController = handlerControllerFactory((proto)=>({
                     id
                 });
             } catch (e) {
-                await this.confirm({
+                // TODO: if this confirm fails, can we abort the whole transfer?
+                await maybeConfirm({
                     type: 'transfer',
                     data: null,
                     ended: true,
@@ -4850,7 +4935,7 @@ const createPullController = handlerControllerFactory((proto)=>({
                     };
                     const BATCH_MAX_SIZE = 1024 * 1024; // 1MB
                     if (!assets) {
-                        throw new Error('bad');
+                        throw new Error('Assets read stream could not be created');
                     }
                     /**
          * Generates batches of 1MB of data from the assets stream to avoid
@@ -5101,7 +5186,7 @@ class LocalFileSourceProvider {
         });
         const loadAssetMetadata = _class_private_field_loose_base$1(this, _loadAssetMetadata)[_loadAssetMetadata].bind(this);
         _class_private_field_loose_base$1(this, _reportInfo$1)[_reportInfo$1]('creating assets read stream');
-        pipeline([
+        pipeline$1([
             inStream,
             new tar.Parse({
                 // find only files in the assets/uploads folder
@@ -5212,7 +5297,7 @@ function streamJsonlDirectory(directory) {
     const outStream = new PassThrough({
         objectMode: true
     });
-    pipeline([
+    pipeline$1([
         inStream,
         new tar.Parse({
             filter (filePath, entry) {
@@ -5253,7 +5338,7 @@ function streamJsonlDirectory(directory) {
 }
 async function parseJSONFile(fileStream, filePath) {
     return new Promise((resolve, reject)=>{
-        pipeline([
+        pipeline$1([
             fileStream,
             // Custom backup archive parsing
             new tar.Parse({