@strapi/data-transfer 5.12.0 → 5.12.2

package/dist/file/providers/destination/index.js

@@ -0,0 +1,248 @@
+'use strict';
+
+var path = require('path');
+var zip = require('zlib');
+var stream = require('stream');
+var fse = require('fs-extra');
+var tar = require('tar-stream');
+var Stringer = require('stream-json/jsonl/Stringer');
+var streamChain = require('stream-chain');
+var encrypt = require('../../../utils/encryption/encrypt.js');
+require('crypto');
+var utils = require('./utils.js');
+var providers = require('../../../errors/providers.js');
+
+function _class_private_field_loose_base(receiver, privateKey) {
+    if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
+        throw new TypeError("attempted to use private field on non-instance");
+    }
+    return receiver;
+}
+var id = 0;
+function _class_private_field_loose_key(name) {
+    return "__private_" + id++ + "_" + name;
+}
+const createLocalFileDestinationProvider = (options)=>{
+    return new LocalFileDestinationProvider(options);
+};
+var _providersMetadata = /*#__PURE__*/ _class_private_field_loose_key("_providersMetadata"), _archive = /*#__PURE__*/ _class_private_field_loose_key("_archive"), _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), _archivePath = /*#__PURE__*/ _class_private_field_loose_key("_archivePath"), _writeMetadata = /*#__PURE__*/ _class_private_field_loose_key("_writeMetadata"), _getMetadataStream = /*#__PURE__*/ _class_private_field_loose_key("_getMetadataStream");
+class LocalFileDestinationProvider {
+    setMetadata(target, metadata) {
+        _class_private_field_loose_base(this, _providersMetadata)[_providersMetadata][target] = metadata;
+        return this;
+    }
+    createGzip() {
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating gzip');
+        return zip.createGzip();
+    }
+    bootstrap(diagnostics) {
+        _class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
+        const { compression, encryption } = this.options;
+        if (encryption.enabled && !encryption.key) {
+            throw new Error("Can't encrypt without a key");
+        }
+        _class_private_field_loose_base(this, _archive)[_archive].stream = tar.pack();
+        const outStream = fse.createWriteStream(_class_private_field_loose_base(this, _archivePath)[_archivePath]);
+        outStream.on('error', (err)=>{
+            if (err.code === 'ENOSPC') {
+                throw new providers.ProviderTransferError("Your server doesn't have space to proceed with the import.");
+            }
+            throw err;
+        });
+        const archiveTransforms = [];
+        if (compression.enabled) {
+            archiveTransforms.push(this.createGzip());
+        }
+        if (encryption.enabled && encryption.key) {
+            archiveTransforms.push(encrypt.createEncryptionCipher(encryption.key));
+        }
+        _class_private_field_loose_base(this, _archive)[_archive].pipeline = streamChain.chain([
+            _class_private_field_loose_base(this, _archive)[_archive].stream,
+            ...archiveTransforms,
+            outStream
+        ]);
+        this.results.file = {
+            path: _class_private_field_loose_base(this, _archivePath)[_archivePath]
+        };
+    }
+    async close() {
+        const { stream, pipeline } = _class_private_field_loose_base(this, _archive)[_archive];
+        if (!stream) {
+            return;
+        }
+        await _class_private_field_loose_base(this, _writeMetadata)[_writeMetadata]();
+        stream.finalize();
+        if (pipeline && !pipeline.closed) {
+            await new Promise((resolve, reject)=>{
+                pipeline.on('close', resolve).on('error', reject);
+            });
+        }
+    }
+    async rollback() {
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('rolling back');
+        await this.close();
+        await fse.rm(_class_private_field_loose_base(this, _archivePath)[_archivePath], {
+            force: true
+        });
+    }
+    getMetadata() {
+        return null;
+    }
+    createSchemasWriteStream() {
+        if (!_class_private_field_loose_base(this, _archive)[_archive].stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating schemas write stream');
+        const filePathFactory = utils.createFilePathFactory('schemas');
+        const entryStream = utils.createTarEntryStream(_class_private_field_loose_base(this, _archive)[_archive].stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return streamChain.chain([
+            Stringer.stringer(),
+            entryStream
+        ]);
+    }
+    createEntitiesWriteStream() {
+        if (!_class_private_field_loose_base(this, _archive)[_archive].stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating entities write stream');
+        const filePathFactory = utils.createFilePathFactory('entities');
+        const entryStream = utils.createTarEntryStream(_class_private_field_loose_base(this, _archive)[_archive].stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return streamChain.chain([
+            Stringer.stringer(),
+            entryStream
+        ]);
+    }
+    createLinksWriteStream() {
+        if (!_class_private_field_loose_base(this, _archive)[_archive].stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating links write stream');
+        const filePathFactory = utils.createFilePathFactory('links');
+        const entryStream = utils.createTarEntryStream(_class_private_field_loose_base(this, _archive)[_archive].stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return streamChain.chain([
+            Stringer.stringer(),
+            entryStream
+        ]);
+    }
+    createConfigurationWriteStream() {
+        if (!_class_private_field_loose_base(this, _archive)[_archive].stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating configuration write stream');
+        const filePathFactory = utils.createFilePathFactory('configuration');
+        const entryStream = utils.createTarEntryStream(_class_private_field_loose_base(this, _archive)[_archive].stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return streamChain.chain([
+            Stringer.stringer(),
+            entryStream
+        ]);
+    }
+    createAssetsWriteStream() {
+        const { stream: archiveStream } = _class_private_field_loose_base(this, _archive)[_archive];
+        if (!archiveStream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets write stream');
+        return new stream.Writable({
+            objectMode: true,
+            write (data, _encoding, callback) {
+                // always write tar files with posix paths so we have a standard format for paths regardless of system
+                const entryPath = path.posix.join('assets', 'uploads', data.filename);
+                const entryMetadataPath = path.posix.join('assets', 'metadata', `${data.filename}.json`);
+                const stringifiedMetadata = JSON.stringify(data.metadata);
+                archiveStream.entry({
+                    name: entryMetadataPath,
+                    size: stringifiedMetadata.length
+                }, stringifiedMetadata);
+                const entry = archiveStream.entry({
+                    name: entryPath,
+                    size: data.stats.size
+                });
+                if (!entry) {
+                    callback(new Error(`Failed to created an asset tar entry for ${entryPath}`));
+                    return;
+                }
+                data.stream.pipe(entry);
+                entry.on('finish', ()=>{
+                    callback(null);
+                }).on('error', (error)=>{
+                    callback(error);
+                });
+            }
+        });
+    }
+    constructor(options){
+        Object.defineProperty(this, _reportInfo, {
+            value: reportInfo
+        });
+        Object.defineProperty(this, _archivePath, {
+            get: get_archivePath,
+            set: void 0
+        });
+        Object.defineProperty(this, _writeMetadata, {
+            value: writeMetadata
+        });
+        Object.defineProperty(this, _getMetadataStream, {
+            value: getMetadataStream
+        });
+        Object.defineProperty(this, _providersMetadata, {
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, _archive, {
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, _diagnostics, {
+            writable: true,
+            value: void 0
+        });
+        this.name = 'destination::local-file';
+        this.type = 'destination';
+        this.results = {};
+        _class_private_field_loose_base(this, _providersMetadata)[_providersMetadata] = {};
+        _class_private_field_loose_base(this, _archive)[_archive] = {};
+        this.options = options;
+    }
+}
+function reportInfo(message) {
+    _class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
+        details: {
+            createdAt: new Date(),
+            message,
+            origin: 'file-destination-provider'
+        },
+        kind: 'info'
+    });
+}
+function get_archivePath() {
+    const { encryption, compression, file } = this.options;
+    let filePath = `${file.path}.tar`;
+    if (compression.enabled) {
+        filePath += '.gz';
+    }
+    if (encryption.enabled) {
+        filePath += '.enc';
+    }
+    return filePath;
+}
+async function writeMetadata() {
+    _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('writing metadata');
+    const metadata = _class_private_field_loose_base(this, _providersMetadata)[_providersMetadata].source;
+    if (metadata) {
+        await new Promise((resolve)=>{
+            const outStream = _class_private_field_loose_base(this, _getMetadataStream)[_getMetadataStream]();
+            const data = JSON.stringify(metadata, null, 2);
+            stream.Readable.from(data).pipe(outStream).on('close', resolve);
+        });
+    }
+}
+function getMetadataStream() {
+    const { stream } = _class_private_field_loose_base(this, _archive)[_archive];
+    if (!stream) {
+        throw new Error('Archive stream is unavailable');
+    }
+    return utils.createTarEntryStream(stream, ()=>'metadata.json');
+}
+
+exports.createLocalFileDestinationProvider = createLocalFileDestinationProvider;
+//# sourceMappingURL=index.js.map

package/dist/file/providers/destination/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sources":["../../../../src/file/providers/destination/index.ts"],"sourcesContent":["import path from 'path';\nimport zlib from 'zlib';\nimport { Readable, Writable } from 'stream';\nimport { rm, createWriteStream } from 'fs-extra';\nimport tar from 'tar-stream';\nimport { stringer } from 'stream-json/jsonl/Stringer';\nimport { chain } from 'stream-chain';\n\nimport { createEncryptionCipher } from '../../../utils/encryption';\nimport type {\n  IAsset,\n  IDestinationProvider,\n  IDestinationProviderTransferResults,\n  IMetadata,\n  ProviderType,\n  Stream,\n} from '../../../../types';\nimport type { IDiagnosticReporter } from '../../../utils/diagnostic';\nimport { createFilePathFactory, createTarEntryStream } from './utils';\nimport { ProviderTransferError } from '../../../errors/providers';\n\nexport interface ILocalFileDestinationProviderOptions {\n  encryption: {\n    enabled: boolean; // if the file should be encrypted\n    key?: string; // the key to use when encryption.enabled is true\n  };\n\n  compression: {\n    enabled: boolean; // if the file should be compressed with gzip\n  };\n\n  file: {\n    path: string; // the filename to create\n    maxSize?: number; // the max size of a single backup file\n    maxSizeJsonl?: number; // the max lines of each jsonl file before creating the next file\n  };\n}\n\nexport interface ILocalFileDestinationProviderTransferResults\n  extends IDestinationProviderTransferResults {\n  file?: {\n    path?: string;\n  };\n}\n\nexport const createLocalFileDestinationProvider = (\n  options: ILocalFileDestinationProviderOptions\n) => {\n  return new LocalFileDestinationProvider(options);\n};\n\nclass LocalFileDestinationProvider implements IDestinationProvider {\n  name = 'destination::local-file';\n\n  type: ProviderType = 'destination';\n\n  options: ILocalFileDestinationProviderOptions;\n\n  results: ILocalFileDestinationProviderTransferResults = {};\n\n  #providersMetadata: { source?: IMetadata; destination?: IMetadata } = {};\n\n  #archive: { stream?: tar.Pack; pipeline?: Stream } = {};\n\n  #diagnostics?: IDiagnosticReporter;\n\n  constructor(options: ILocalFileDestinationProviderOptions) {\n    this.options = options;\n  }\n\n  #reportInfo(message: string) {\n    this.#diagnostics?.report({\n      details: {\n        createdAt: new Date(),\n        message,\n        origin: 'file-destination-provider',\n      },\n      kind: 'info',\n    });\n  }\n\n  get #archivePath() {\n    const { encryption, compression, file } = this.options;\n\n    let filePath = `${file.path}.tar`;\n\n    if (compression.enabled) {\n      filePath += '.gz';\n    }\n\n    if (encryption.enabled) {\n      filePath += '.enc';\n    }\n\n    return filePath;\n  }\n\n  setMetadata(target: ProviderType, metadata: IMetadata): IDestinationProvider {\n    this.#providersMetadata[target] = metadata;\n\n    return this;\n  }\n\n  createGzip(): zlib.Gzip {\n    this.#reportInfo('creating gzip');\n    return zlib.createGzip();\n  }\n\n  bootstrap(diagnostics: IDiagnosticReporter): void | Promise<void> {\n    this.#diagnostics = diagnostics;\n    const { compression, encryption } = this.options;\n\n    if (encryption.enabled && !encryption.key) {\n      throw new Error(\"Can't encrypt without a key\");\n    }\n\n    this.#archive.stream = tar.pack();\n\n    const outStream = createWriteStream(this.#archivePath);\n\n    outStream.on('error', (err: NodeJS.ErrnoException) => {\n      if (err.code === 'ENOSPC') {\n        throw new ProviderTransferError(\n          \"Your server doesn't have space to proceed with the import.\"\n        );\n      }\n      throw err;\n    });\n\n    const archiveTransforms: Stream[] = [];\n\n    if (compression.enabled) {\n      archiveTransforms.push(this.createGzip());\n    }\n\n    if (encryption.enabled && encryption.key) {\n      archiveTransforms.push(createEncryptionCipher(encryption.key));\n    }\n\n    this.#archive.pipeline = chain([this.#archive.stream, ...archiveTransforms, outStream]);\n\n    this.results.file = { path: this.#archivePath };\n  }\n\n  async close() {\n    const { stream, pipeline } = this.#archive;\n\n    if (!stream) {\n      return;\n    }\n\n    await this.#writeMetadata();\n    stream.finalize();\n\n    if (pipeline && !pipeline.closed) {\n      await new Promise<void>((resolve, reject) => {\n        pipeline.on('close', resolve).on('error', reject);\n      });\n    }\n  }\n\n  async rollback(): Promise<void> {\n    this.#reportInfo('rolling back');\n    await this.close();\n    await rm(this.#archivePath, { force: true });\n  }\n\n  getMetadata() {\n    return null;\n  }\n\n  async #writeMetadata(): Promise<void> {\n    this.#reportInfo('writing metadata');\n    const metadata = this.#providersMetadata.source;\n\n    if (metadata) {\n      await new Promise((resolve) => {\n        const outStream = this.#getMetadataStream();\n        const data = JSON.stringify(metadata, null, 2);\n\n        Readable.from(data).pipe(outStream).on('close', resolve);\n      });\n    }\n  }\n\n  #getMetadataStream() {\n    const { stream } = this.#archive;\n\n    if (!stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n\n    return createTarEntryStream(stream, () => 'metadata.json');\n  }\n\n  createSchemasWriteStream() {\n    if (!this.#archive.stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n    this.#reportInfo('creating schemas write stream');\n    const filePathFactory = createFilePathFactory('schemas');\n\n    const entryStream = createTarEntryStream(\n      this.#archive.stream,\n      filePathFactory,\n      this.options.file.maxSizeJsonl\n    );\n\n    return chain([stringer(), entryStream]);\n  }\n\n  createEntitiesWriteStream(): Writable {\n    if (!this.#archive.stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n    this.#reportInfo('creating entities write stream');\n    const filePathFactory = createFilePathFactory('entities');\n\n    const entryStream = createTarEntryStream(\n      this.#archive.stream,\n      filePathFactory,\n      this.options.file.maxSizeJsonl\n    );\n\n    return chain([stringer(), entryStream]);\n  }\n\n  createLinksWriteStream(): Writable {\n    if (!this.#archive.stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n    this.#reportInfo('creating links write stream');\n    const filePathFactory = createFilePathFactory('links');\n\n    const entryStream = createTarEntryStream(\n      this.#archive.stream,\n      filePathFactory,\n      this.options.file.maxSizeJsonl\n    );\n\n    return chain([stringer(), entryStream]);\n  }\n\n  createConfigurationWriteStream(): Writable {\n    if (!this.#archive.stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n    this.#reportInfo('creating configuration write stream');\n    const filePathFactory = createFilePathFactory('configuration');\n\n    const entryStream = createTarEntryStream(\n      this.#archive.stream,\n      filePathFactory,\n      this.options.file.maxSizeJsonl\n    );\n\n    return chain([stringer(), entryStream]);\n  }\n\n  createAssetsWriteStream(): Writable {\n    const { stream: archiveStream } = this.#archive;\n\n    if (!archiveStream) {\n      throw new Error('Archive stream is unavailable');\n    }\n\n    this.#reportInfo('creating assets write stream');\n    return new Writable({\n      objectMode: true,\n      write(data: IAsset, _encoding, callback) {\n        // always write tar files with posix paths so we have a standard format for paths regardless of system\n        const entryPath = path.posix.join('assets', 'uploads', data.filename);\n\n        const entryMetadataPath = path.posix.join('assets', 'metadata', `${data.filename}.json`);\n        const stringifiedMetadata = JSON.stringify(data.metadata);\n        archiveStream.entry(\n          {\n            name: entryMetadataPath,\n            size: stringifiedMetadata.length,\n          },\n          stringifiedMetadata\n        );\n\n        const entry = archiveStream.entry({\n          name: entryPath,\n          size: data.stats.size,\n        });\n\n        if (!entry) {\n          callback(new Error(`Failed to created an asset tar entry for ${entryPath}`));\n          return;\n        }\n\n        data.stream.pipe(entry);\n\n        entry\n          .on('finish', () => {\n            callback(null);\n          })\n          .on('error', (error) => {\n            callback(error);\n          });\n      },\n    });\n  }\n}\n"],"names":["createLocalFileDestinationProvider","options","LocalFileDestinationProvider","setMetadata","target","metadata","providersMetadata","createGzip","reportInfo","zlib","bootstrap","diagnostics","compression","encryption","enabled","key","Error","archive","stream","tar","pack","outStream","createWriteStream","archivePath","on","err","code","ProviderTransferError","archiveTransforms","push","createEncryptionCipher","pipeline","chain","results","file","path","close","writeMetadata","finalize","closed","Promise","resolve","reject","rollback","rm","force","getMetadata","createSchemasWriteStream","filePathFactory","createFilePathFactory","entryStream","createTarEntryStream","maxSizeJsonl","stringer","createEntitiesWriteStream","createLinksWriteStream","createConfigurationWriteStream","createAssetsWriteStream","archiveStream","Writable","objectMode","write","data","_encoding","callback","entryPath","posix","join","filename","entryMetadataPath","stringifiedMetadata","JSON","stringify","entry","name","size","length","stats","pipe","error","constructor","type","message","report","details","createdAt","Date","origin","kind","filePath","source","getMetadataStream","Readable","from"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA6CO,MAAMA,qCAAqC,CAChDC,OAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,4BAA6BD,CAAAA,OAAAA,CAAAA;AAC1C;AAWE,IAAA,kBAAA,iBAAA,8BAAA,CAAA,oBAAA,CAAA,EAEA,QAEA,iBAAA,8BAAA,CAAA,UAAA,CAAA,EAAA,YAAA,iBAAA,8BAAA,CAAA,cAAA,CAAA,EAMA,WAWI,iBAAA,8BAAA,CAAA,aAAA,CAAA,EAAA,YAAA,iBAAA,8BAAA,CAAA,cAAA,CAAA,EA0FE,cAcN,iBAAA,8BAAA,CAAA,gBAAA,CAAA,EAAA,kBAAA,iBAAA,8BAAA,CAAA,oBAAA,CAAA;AAtIF,MAAMC,4BAAAA,CAAAA;IA8CJC,WAAYC,CAAAA,MAAoB,EAAEC,QAAmB,EAAwB;AAC3E,QAAA,+BAAA,CAAA,IAAI,EAAEC,kBAAAA,CAAAA,CAAAA,kBAAiB,CAAA,CAACF,OAAO,GAAGC,QAAAA;AAElC,QAAA,OAAO,IAAI;AACb;IAEAE,UAAwB,GAAA;QACtB,+BAAA,CAAA,IAAI,EAAEC,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,eAAA,CAAA;AACjB,QAAA,OAAOC,IAAKF,UAAU,EAAA;AACxB;AAEAG,IAAAA,SAAAA,CAAUC,WAAgC,EAAwB;QAChE,+BAAA,CAAA,IAAI,EAAEA,YAAAA,CAAAA,CAAAA,YAAcA,CAAAA,GAAAA,WAAAA;QACpB,MAAM,EAAEC,WAAW,EAAEC,UAAU,EAAE,GAAG,IAAI,CAACZ,OAAO;AAEhD,QAAA,IAAIY,WAAWC,OAAO,IAAI,CAACD,UAAAA,CAAWE,GAAG,EAAE;AACzC,YAAA,MAAM,IAAIC,KAAM,CAAA,6BAAA,CAAA;AAClB;AAEA,QAAA,+BAAA,CAAA,IAAI,EAAEC,QAAAA,CAAAA,CAAAA,UAAQC,MAAM,GAAGC,IAAIC,IAAI,EAAA;AAE/B,QAAA,MAAMC,SAAYC,GAAAA,qBAAAA,CAAkB,+BAAA,CAAA,IAAI,EAAEC,YAAAA,CAAAA,CAAAA,YAAAA,CAAAA,CAAAA;QAE1CF,SAAUG,CAAAA,EAAE,CAAC,OAAA,EAAS,CAACC,GAAAA,GAAAA;YACrB,IAAIA,GAAAA,CAAIC,IAAI,KAAK,QAAU,EAAA;AACzB,gBAAA,MAAM,IAAIC,+BACR,CAAA,4DAAA,CAAA;AAEJ;YACA,MAAMF,GAAAA;AACR,SAAA,CAAA;AAEA,QAAA,MAAMG,oBAA8B,EAAE;QAEtC,IAAIhB,WAAAA,CAAYE,OAAO,EAAE;AACvBc,YAAAA,iBAAAA,CAAkBC,IAAI,CAAC,IAAI,CAACtB,UAAU,EAAA,CAAA;AACxC;AAEA,QAAA,IAAIM,UAAWC,CAAAA,OAAO,IAAID,UAAAA,CAAWE,GAAG,EAAE;AACxCa,YAAAA,iBAAAA,CAAkBC,IAAI,CAACC,8BAAuBjB,CAAAA,UAAAA,CAAWE,GAAG,CAAA,CAAA;AAC9D;AAEA,QAAA,+BAAA,CAAA,IAAI,EAAEE,QAAAA,CAAAA,CAAAA,QAAQc,CAAAA,CAAAA,QAAQ,GAAGC,iBAAM,CAAA;AAAC,YAAA,+BAAA,CAAA,IAAI,EAAEf,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM;AAAKU,YAAAA,GAAAA,iBAAAA;AAAmBP,YAAAA;AAAU,SAAA,CAAA;AAEtF,QAAA,IAAI,CAACY,OAAO,CAACC,IAAI,GAAG;AAAEC,YAAAA,IAAI,EAAE,+BAAA,CAAA,IAAI,EAAEZ,YAAAA,CAAAA,CAAAA,YAAAA;AAAY,SAAA;AAChD;AAEA,IAAA,MAAMa,KAAQ,GAAA;QACZ,MAAM,EAAElB,MAAM,EAAEa,QAAQ,EAAE,GAAG,+BAAA,CAAA,IAAI,EAAEd,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA;AAEnC,QAAA,IAAI,CAACC,MAAQ,EAAA;AACX,YAAA;AACF;QAEA,MAAM,+BAAA,CAAA,IAAI,EAAEmB,cAAAA,CAAAA,CAAAA,cAAAA,CAAAA,EAAAA;AACZnB,QAAAA,MAAAA,CAAOoB,QAAQ,EAAA;AAEf,QAAA,IAAIP,QAAY,IAAA,CAACA,QAASQ,CAAAA,MAAM,EAAE;YAChC,MAAM,IAAIC,OAAc,CAAA,CAACC,OAASC,EAAAA,MAAAA,GAAAA;AAChCX,gBAAAA,QAAAA,CAASP,EAAE,CAAC,OAAA,EAASiB,OAASjB,CAAAA,CAAAA,EAAE,CAAC,OAASkB,EAAAA,MAAAA,CAAAA;AAC5C,aAAA,CAAA;AACF;AACF;AAEA,IAAA,MAAMC,QAA0B,GAAA;QAC9B,+BAAA,CAAA,IAAI,EAAEnC,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,cAAA,CAAA;QACjB,MAAM,IAAI,CAAC4B,KAAK,EAAA;AAChB,QAAA,MAAMQ,MAAG,CAAA,+BAAA,CAAA,IAAI,EAAErB,cAAAA,YAAa,CAAA,EAAA;YAAEsB,KAAO,EAAA;AAAK,SAAA,CAAA;AAC5C;IAEAC,WAAc,GAAA;QACZ,OAAO,IAAA;AACT;IA0BAC,wBAA2B,GAAA;AACzB,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAE9B,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM,EAAE;AACzB,YAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;QACA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,+BAAA,CAAA;AACjB,QAAA,MAAMwC,kBAAkBC,2BAAsB,CAAA,SAAA,CAAA;AAE9C,QAAA,MAAMC,cAAcC,0BAClB,CAAA,+BAAA,CAAA,IAAI,EAAElC,UAAAA,QAAQC,CAAAA,CAAAA,MAAM,EACpB8B,eAAAA,EACA,IAAI,CAAC/C,OAAO,CAACiC,IAAI,CAACkB,YAAY,CAAA;AAGhC,QAAA,OAAOpB,iBAAM,CAAA;AAACqB,YAAAA,iBAAAA,EAAAA;AAAYH,YAAAA;AAAY,SAAA,CAAA;AACxC;IAEAI,yBAAsC,GAAA;AACpC,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAErC,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM,EAAE;AACzB,YAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;QACA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,gCAAA,CAAA;AACjB,QAAA,MAAMwC,kBAAkBC,2BAAsB,CAAA,UAAA,CAAA;AAE9C,QAAA,MAAMC,cAAcC,0BAClB,CAAA,+BAAA,CAAA,IAAI,EAAElC,UAAAA,QAAQC,CAAAA,CAAAA,MAAM,EACpB8B,eAAAA,EACA,IAAI,CAAC/C,OAAO,CAACiC,IAAI,CAACkB,YAAY,CAAA;AAGhC,QAAA,OAAOpB,iBAAM,CAAA;AAACqB,YAAAA,iBAAAA,EAAAA;AAAYH,YAAAA;AAAY,SAAA,CAAA;AACxC;IAEAK,sBAAmC,GAAA;AACjC,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAEtC,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM,EAAE;AACzB,YAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;QACA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,6BAAA,CAAA;AACjB,QAAA,MAAMwC,kBAAkBC,2BAAsB,CAAA,OAAA,CAAA;AAE9C,QAAA,MAAMC,cAAcC,0BAClB,CAAA,+BAAA,CAAA,IAAI,EAAElC,UAAAA,QAAQC,CAAAA,CAAAA,MAAM,EACpB8B,eAAAA,EACA,IAAI,CAAC/C,OAAO,CAACiC,IAAI,CAACkB,YAAY,CAAA;AAGhC,QAAA,OAAOpB,iBAAM,CAAA;AAACqB,YAAAA,iBAAAA,EAAAA;AAAYH,YAAAA;AAAY,SAAA,CAAA;AACxC;IAEAM,8BAA2C,GAAA;AACzC,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAEvC,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM,EAAE;AACzB,YAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;QACA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,qCAAA,CAAA;AACjB,QAAA,MAAMwC,kBAAkBC,2BAAsB,CAAA,eAAA,CAAA;AAE9C,QAAA,MAAMC,cAAcC,0BAClB,CAAA,+BAAA,CAAA,IAAI,EAAElC,UAAAA,QAAQC,CAAAA,CAAAA,MAAM,EACpB8B,eAAAA,EACA,IAAI,CAAC/C,OAAO,CAACiC,IAAI,CAACkB,YAAY,CAAA;AAGhC,QAAA,OAAOpB,iBAAM,CAAA;AAACqB,YAAAA,iBAAAA,EAAAA;AAAYH,YAAAA;AAAY,SAAA,CAAA;AACxC;IAEAO,uBAAoC,GAAA;QAClC,MAAM,EAAEvC,QAAQwC,aAAa,EAAE,GAAG,+BAAA,CAAA,IAAI,EAAEzC,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA;AAExC,QAAA,IAAI,CAACyC,aAAe,EAAA;AAClB,YAAA,MAAM,IAAI1C,KAAM,CAAA,+BAAA,CAAA;AAClB;QAEA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,8BAAA,CAAA;AACjB,QAAA,OAAO,IAAImD,eAAS,CAAA;YAClBC,UAAY,EAAA,IAAA;AACZC,YAAAA,KAAAA,CAAAA,CAAMC,IAAY,EAAEC,SAAS,EAAEC,QAAQ,EAAA;;gBAErC,MAAMC,SAAAA,GAAY9B,KAAK+B,KAAK,CAACC,IAAI,CAAC,QAAA,EAAU,SAAWL,EAAAA,IAAAA,CAAKM,QAAQ,CAAA;AAEpE,gBAAA,MAAMC,iBAAoBlC,GAAAA,IAAAA,CAAK+B,KAAK,CAACC,IAAI,CAAC,QAAA,EAAU,UAAY,EAAA,CAAC,EAAEL,IAAAA,CAAKM,QAAQ,CAAC,KAAK,CAAC,CAAA;AACvF,gBAAA,MAAME,mBAAsBC,GAAAA,IAAAA,CAAKC,SAAS,CAACV,KAAKzD,QAAQ,CAAA;AACxDqD,gBAAAA,aAAAA,CAAce,KAAK,CACjB;oBACEC,IAAML,EAAAA,iBAAAA;AACNM,oBAAAA,IAAAA,EAAML,oBAAoBM;iBAE5BN,EAAAA,mBAAAA,CAAAA;gBAGF,MAAMG,KAAAA,GAAQf,aAAce,CAAAA,KAAK,CAAC;oBAChCC,IAAMT,EAAAA,SAAAA;oBACNU,IAAMb,EAAAA,IAAAA,CAAKe,KAAK,CAACF;AACnB,iBAAA,CAAA;AAEA,gBAAA,IAAI,CAACF,KAAO,EAAA;AACVT,oBAAAA,QAAAA,CAAS,IAAIhD,KAAM,CAAA,CAAC,yCAAyC,EAAEiD,UAAU,CAAC,CAAA,CAAA;AAC1E,oBAAA;AACF;gBAEAH,IAAK5C,CAAAA,MAAM,CAAC4D,IAAI,CAACL,KAAAA,CAAAA;gBAEjBA,KACGjD,CAAAA,EAAE,CAAC,QAAU,EAAA,IAAA;oBACZwC,QAAS,CAAA,IAAA,CAAA;iBAEVxC,CAAAA,CAAAA,EAAE,CAAC,OAAA,EAAS,CAACuD,KAAAA,GAAAA;oBACZf,QAASe,CAAAA,KAAAA,CAAAA;AACX,iBAAA,CAAA;AACJ;AACF,SAAA,CAAA;AACF;AA9OAC,IAAAA,WAAAA,CAAY/E,OAA6C,CAAE;QAI3D,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,WAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAWA,MAAI,CAAA,cAAA,CAAA,IAAA,EAAA,YAAA,EAAA;AAAJ,YAAA,GAAA,EAAA,eAAA;;;QA0FA,MAAM,CAAA,cAAA,CAAA,IAAA,EAAA,cAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAcA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,kBAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QA7HA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,kBAAA,EAAA;;mBAAA,KAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,QAAA,EAAA;;mBAAA,KAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,YAAA,EAAA;;mBAAA,KAAA;;aAZAyE,IAAO,GAAA,yBAAA;aAEPO,IAAqB,GAAA,aAAA;AAIrBhD,QAAAA,IAAAA,CAAAA,OAAAA,GAAwD,EAAC;AAExD3B,QAAAA,+BAAAA,CAAAA,IAAAA,EAAAA,kBAAAA,CAAAA,CAAAA,sBAAqE,EAAC;AAEtEW,QAAAA,+BAAAA,CAAAA,IAAAA,EAAAA,QAAAA,CAAAA,CAAAA,YAAoD,EAAC;QAKpD,IAAI,CAAChB,OAAO,GAAGA,OAAAA;AACjB;AA6OF;AA3OE,SAAA,WAAYiF,OAAe,EAAA;AACzB,IAAA,+BAAA,CAAA,IAAI,EAAEvE,YAAAA,CAAAA,CAAAA,YAAAA,CAAAA,EAAawE,MAAO,CAAA;QACxBC,OAAS,EAAA;AACPC,YAAAA,SAAAA,EAAW,IAAIC,IAAAA,EAAAA;AACfJ,YAAAA,OAAAA;YACAK,MAAQ,EAAA;AACV,SAAA;QACAC,IAAM,EAAA;AACR,KAAA,CAAA;AACF;AAEA,SAAA,eAAA,GAAA;IACE,MAAM,EAAE3E,UAAU,EAAED,WAAW,EAAEsB,IAAI,EAAE,GAAG,IAAI,CAACjC,OAAO;AAEtD,IAAA,IAAIwF,WAAW,CAAC,EAAEvD,KAAKC,IAAI,CAAC,IAAI,CAAC;IAEjC,IAAIvB,WAAAA,CAAYE,OAAO,EAAE;QACvB2E,QAAY,IAAA,KAAA;AACd;IAEA,IAAI5E,UAAAA,CAAWC,OAAO,EAAE;QACtB2E,QAAY,IAAA,MAAA;AACd;IAEA,OAAOA,QAAAA;AACT;AA4EA,eAAA,aAAA,GAAA;IACE,+BAAA,CAAA,IAAI,EAAEjF,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,kBAAA,CAAA;AACjB,IAAA,MAAMH,WAAW,+BAAA,CAAA,IAAI,EAAEC,kBAAAA,CAAAA,CAAAA,oBAAkBoF,MAAM;AAE/C,IAAA,IAAIrF,QAAU,EAAA;QACZ,MAAM,IAAImC,QAAQ,CAACC,OAAAA,GAAAA;AACjB,YAAA,MAAMpB,SAAY,GAAA,+BAAA,CAAA,IAAI,EAAEsE,kBAAAA,CAAAA,CAAAA,kBAAAA,CAAAA,EAAAA;AACxB,YAAA,MAAM7B,IAAOS,GAAAA,IAAAA,CAAKC,SAAS,CAACnE,UAAU,IAAM,EAAA,CAAA,CAAA;YAE5CuF,eAASC,CAAAA,IAAI,CAAC/B,IAAMgB,CAAAA,CAAAA,IAAI,CAACzD,SAAWG,CAAAA,CAAAA,EAAE,CAAC,OAASiB,EAAAA,OAAAA,CAAAA;AAClD,SAAA,CAAA;AACF;AACF;AAEA,SAAA,iBAAA,GAAA;AACE,IAAA,MAAM,EAAEvB,MAAM,EAAE,GAAG,+BAAA,CAAA,IAAI,EAAED,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA;AAEzB,IAAA,IAAI,CAACC,MAAQ,EAAA;AACX,QAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;IAEA,OAAOmC,0BAAAA,CAAqBjC,QAAQ,IAAM,eAAA,CAAA;AAC5C;;;;"}

package/dist/file/providers/destination/index.mjs

@@ -0,0 +1,246 @@
+import path from 'path';
+import zip from 'zlib';
+import { Writable, Readable } from 'stream';
+import { createWriteStream, rm } from 'fs-extra';
+import tar from 'tar-stream';
+import { stringer } from 'stream-json/jsonl/Stringer';
+import { chain } from 'stream-chain';
+import { createEncryptionCipher } from '../../../utils/encryption/encrypt.mjs';
+import 'crypto';
+import { createTarEntryStream, createFilePathFactory } from './utils.mjs';
+import { ProviderTransferError } from '../../../errors/providers.mjs';
+
+function _class_private_field_loose_base(receiver, privateKey) {
+    if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
+        throw new TypeError("attempted to use private field on non-instance");
+    }
+    return receiver;
+}
+var id = 0;
+function _class_private_field_loose_key(name) {
+    return "__private_" + id++ + "_" + name;
+}
+const createLocalFileDestinationProvider = (options)=>{
+    return new LocalFileDestinationProvider(options);
+};
+var _providersMetadata = /*#__PURE__*/ _class_private_field_loose_key("_providersMetadata"), _archive = /*#__PURE__*/ _class_private_field_loose_key("_archive"), _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), _archivePath = /*#__PURE__*/ _class_private_field_loose_key("_archivePath"), _writeMetadata = /*#__PURE__*/ _class_private_field_loose_key("_writeMetadata"), _getMetadataStream = /*#__PURE__*/ _class_private_field_loose_key("_getMetadataStream");
+class LocalFileDestinationProvider {
+    setMetadata(target, metadata) {
+        _class_private_field_loose_base(this, _providersMetadata)[_providersMetadata][target] = metadata;
+        return this;
+    }
+    createGzip() {
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating gzip');
+        return zip.createGzip();
+    }
+    bootstrap(diagnostics) {
+        _class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
+        const { compression, encryption } = this.options;
+        if (encryption.enabled && !encryption.key) {
+            throw new Error("Can't encrypt without a key");
+        }
+        _class_private_field_loose_base(this, _archive)[_archive].stream = tar.pack();
+        const outStream = createWriteStream(_class_private_field_loose_base(this, _archivePath)[_archivePath]);
+        outStream.on('error', (err)=>{
+            if (err.code === 'ENOSPC') {
+                throw new ProviderTransferError("Your server doesn't have space to proceed with the import.");
+            }
+            throw err;
+        });
+        const archiveTransforms = [];
+        if (compression.enabled) {
+            archiveTransforms.push(this.createGzip());
+        }
+        if (encryption.enabled && encryption.key) {
+            archiveTransforms.push(createEncryptionCipher(encryption.key));
+        }
+        _class_private_field_loose_base(this, _archive)[_archive].pipeline = chain([
+            _class_private_field_loose_base(this, _archive)[_archive].stream,
+            ...archiveTransforms,
+            outStream
+        ]);
+        this.results.file = {
+            path: _class_private_field_loose_base(this, _archivePath)[_archivePath]
+        };
+    }
+    async close() {
+        const { stream, pipeline } = _class_private_field_loose_base(this, _archive)[_archive];
+        if (!stream) {
+            return;
+        }
+        await _class_private_field_loose_base(this, _writeMetadata)[_writeMetadata]();
+        stream.finalize();
+        if (pipeline && !pipeline.closed) {
+            await new Promise((resolve, reject)=>{
+                pipeline.on('close', resolve).on('error', reject);
+            });
+        }
+    }
+    async rollback() {
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('rolling back');
+        await this.close();
+        await rm(_class_private_field_loose_base(this, _archivePath)[_archivePath], {
+            force: true
+        });
+    }
+    getMetadata() {
+        return null;
+    }
+    createSchemasWriteStream() {
+        if (!_class_private_field_loose_base(this, _archive)[_archive].stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating schemas write stream');
+        const filePathFactory = createFilePathFactory('schemas');
+        const entryStream = createTarEntryStream(_class_private_field_loose_base(this, _archive)[_archive].stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return chain([
+            stringer(),
+            entryStream
+        ]);
+    }
+    createEntitiesWriteStream() {
+        if (!_class_private_field_loose_base(this, _archive)[_archive].stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating entities write stream');
+        const filePathFactory = createFilePathFactory('entities');
+        const entryStream = createTarEntryStream(_class_private_field_loose_base(this, _archive)[_archive].stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return chain([
+            stringer(),
+            entryStream
+        ]);
+    }
+    createLinksWriteStream() {
+        if (!_class_private_field_loose_base(this, _archive)[_archive].stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating links write stream');
+        const filePathFactory = createFilePathFactory('links');
+        const entryStream = createTarEntryStream(_class_private_field_loose_base(this, _archive)[_archive].stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return chain([
+            stringer(),
+            entryStream
+        ]);
+    }
+    createConfigurationWriteStream() {
+        if (!_class_private_field_loose_base(this, _archive)[_archive].stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating configuration write stream');
+        const filePathFactory = createFilePathFactory('configuration');
+        const entryStream = createTarEntryStream(_class_private_field_loose_base(this, _archive)[_archive].stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return chain([
+            stringer(),
+            entryStream
+        ]);
+    }
+    createAssetsWriteStream() {
+        const { stream: archiveStream } = _class_private_field_loose_base(this, _archive)[_archive];
+        if (!archiveStream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets write stream');
+        return new Writable({
+            objectMode: true,
+            write (data, _encoding, callback) {
+                // always write tar files with posix paths so we have a standard format for paths regardless of system
+                const entryPath = path.posix.join('assets', 'uploads', data.filename);
+                const entryMetadataPath = path.posix.join('assets', 'metadata', `${data.filename}.json`);
+                const stringifiedMetadata = JSON.stringify(data.metadata);
+                archiveStream.entry({
+                    name: entryMetadataPath,
+                    size: stringifiedMetadata.length
+                }, stringifiedMetadata);
+                const entry = archiveStream.entry({
+                    name: entryPath,
+                    size: data.stats.size
+                });
+                if (!entry) {
+                    callback(new Error(`Failed to created an asset tar entry for ${entryPath}`));
+                    return;
+                }
+                data.stream.pipe(entry);
+                entry.on('finish', ()=>{
+                    callback(null);
+                }).on('error', (error)=>{
+                    callback(error);
+                });
+            }
+        });
+    }
+    constructor(options){
+        Object.defineProperty(this, _reportInfo, {
+            value: reportInfo
+        });
+        Object.defineProperty(this, _archivePath, {
+            get: get_archivePath,
+            set: void 0
+        });
+        Object.defineProperty(this, _writeMetadata, {
+            value: writeMetadata
+        });
+        Object.defineProperty(this, _getMetadataStream, {
+            value: getMetadataStream
+        });
+        Object.defineProperty(this, _providersMetadata, {
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, _archive, {
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, _diagnostics, {
+            writable: true,
+            value: void 0
+        });
+        this.name = 'destination::local-file';
+        this.type = 'destination';
+        this.results = {};
+        _class_private_field_loose_base(this, _providersMetadata)[_providersMetadata] = {};
+        _class_private_field_loose_base(this, _archive)[_archive] = {};
+        this.options = options;
+    }
+}
+function reportInfo(message) {
+    _class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
+        details: {
+            createdAt: new Date(),
+            message,
+            origin: 'file-destination-provider'
+        },
+        kind: 'info'
+    });
+}
+function get_archivePath() {
+    const { encryption, compression, file } = this.options;
+    let filePath = `${file.path}.tar`;
+    if (compression.enabled) {
+        filePath += '.gz';
+    }
+    if (encryption.enabled) {
+        filePath += '.enc';
+    }
+    return filePath;
+}
+async function writeMetadata() {
+    _class_private_field_loose_base(this, _reportInfo)[_reportInfo]('writing metadata');
+    const metadata = _class_private_field_loose_base(this, _providersMetadata)[_providersMetadata].source;
+    if (metadata) {
+        await new Promise((resolve)=>{
+            const outStream = _class_private_field_loose_base(this, _getMetadataStream)[_getMetadataStream]();
+            const data = JSON.stringify(metadata, null, 2);
+            Readable.from(data).pipe(outStream).on('close', resolve);
+        });
+    }
+}
+function getMetadataStream() {
+    const { stream } = _class_private_field_loose_base(this, _archive)[_archive];
+    if (!stream) {
+        throw new Error('Archive stream is unavailable');
+    }
+    return createTarEntryStream(stream, ()=>'metadata.json');
+}
+
+export { createLocalFileDestinationProvider };
+//# sourceMappingURL=index.mjs.map

package/dist/file/providers/destination/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","sources":["../../../../src/file/providers/destination/index.ts"],"sourcesContent":["import path from 'path';\nimport zlib from 'zlib';\nimport { Readable, Writable } from 'stream';\nimport { rm, createWriteStream } from 'fs-extra';\nimport tar from 'tar-stream';\nimport { stringer } from 'stream-json/jsonl/Stringer';\nimport { chain } from 'stream-chain';\n\nimport { createEncryptionCipher } from '../../../utils/encryption';\nimport type {\n  IAsset,\n  IDestinationProvider,\n  IDestinationProviderTransferResults,\n  IMetadata,\n  ProviderType,\n  Stream,\n} from '../../../../types';\nimport type { IDiagnosticReporter } from '../../../utils/diagnostic';\nimport { createFilePathFactory, createTarEntryStream } from './utils';\nimport { ProviderTransferError } from '../../../errors/providers';\n\nexport interface ILocalFileDestinationProviderOptions {\n  encryption: {\n    enabled: boolean; // if the file should be encrypted\n    key?: string; // the key to use when encryption.enabled is true\n  };\n\n  compression: {\n    enabled: boolean; // if the file should be compressed with gzip\n  };\n\n  file: {\n    path: string; // the filename to create\n    maxSize?: number; // the max size of a single backup file\n    maxSizeJsonl?: number; // the max lines of each jsonl file before creating the next file\n  };\n}\n\nexport interface ILocalFileDestinationProviderTransferResults\n  extends IDestinationProviderTransferResults {\n  file?: {\n    path?: string;\n  };\n}\n\nexport const createLocalFileDestinationProvider = (\n  options: ILocalFileDestinationProviderOptions\n) => {\n  return new LocalFileDestinationProvider(options);\n};\n\nclass LocalFileDestinationProvider implements IDestinationProvider {\n  name = 'destination::local-file';\n\n  type: ProviderType = 'destination';\n\n  options: ILocalFileDestinationProviderOptions;\n\n  results: ILocalFileDestinationProviderTransferResults = {};\n\n  #providersMetadata: { source?: IMetadata; destination?: IMetadata } = {};\n\n  #archive: { stream?: tar.Pack; pipeline?: Stream } = {};\n\n  #diagnostics?: IDiagnosticReporter;\n\n  constructor(options: ILocalFileDestinationProviderOptions) {\n    this.options = options;\n  }\n\n  #reportInfo(message: string) {\n    this.#diagnostics?.report({\n      details: {\n        createdAt: new Date(),\n        message,\n        origin: 'file-destination-provider',\n      },\n      kind: 'info',\n    });\n  }\n\n  get #archivePath() {\n    const { encryption, compression, file } = this.options;\n\n    let filePath = `${file.path}.tar`;\n\n    if (compression.enabled) {\n      filePath += '.gz';\n    }\n\n    if (encryption.enabled) {\n      filePath += '.enc';\n    }\n\n    return filePath;\n  }\n\n  setMetadata(target: ProviderType, metadata: IMetadata): IDestinationProvider {\n    this.#providersMetadata[target] = metadata;\n\n    return this;\n  }\n\n  createGzip(): zlib.Gzip {\n    this.#reportInfo('creating gzip');\n    return zlib.createGzip();\n  }\n\n  bootstrap(diagnostics: IDiagnosticReporter): void | Promise<void> {\n    this.#diagnostics = diagnostics;\n    const { compression, encryption } = this.options;\n\n    if (encryption.enabled && !encryption.key) {\n      throw new Error(\"Can't encrypt without a key\");\n    }\n\n    this.#archive.stream = tar.pack();\n\n    const outStream = createWriteStream(this.#archivePath);\n\n    outStream.on('error', (err: NodeJS.ErrnoException) => {\n      if (err.code === 'ENOSPC') {\n        throw new ProviderTransferError(\n          \"Your server doesn't have space to proceed with the import.\"\n        );\n      }\n      throw err;\n    });\n\n    const archiveTransforms: Stream[] = [];\n\n    if (compression.enabled) {\n      archiveTransforms.push(this.createGzip());\n    }\n\n    if (encryption.enabled && encryption.key) {\n      archiveTransforms.push(createEncryptionCipher(encryption.key));\n    }\n\n    this.#archive.pipeline = chain([this.#archive.stream, ...archiveTransforms, outStream]);\n\n    this.results.file = { path: this.#archivePath };\n  }\n\n  async close() {\n    const { stream, pipeline } = this.#archive;\n\n    if (!stream) {\n      return;\n    }\n\n    await this.#writeMetadata();\n    stream.finalize();\n\n    if (pipeline && !pipeline.closed) {\n      await new Promise<void>((resolve, reject) => {\n        pipeline.on('close', resolve).on('error', reject);\n      });\n    }\n  }\n\n  async rollback(): Promise<void> {\n    this.#reportInfo('rolling back');\n    await this.close();\n    await rm(this.#archivePath, { force: true });\n  }\n\n  getMetadata() {\n    return null;\n  }\n\n  async #writeMetadata(): Promise<void> {\n    this.#reportInfo('writing metadata');\n    const metadata = this.#providersMetadata.source;\n\n    if (metadata) {\n      await new Promise((resolve) => {\n        const outStream = this.#getMetadataStream();\n        const data = JSON.stringify(metadata, null, 2);\n\n        Readable.from(data).pipe(outStream).on('close', resolve);\n      });\n    }\n  }\n\n  #getMetadataStream() {\n    const { stream } = this.#archive;\n\n    if (!stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n\n    return createTarEntryStream(stream, () => 'metadata.json');\n  }\n\n  createSchemasWriteStream() {\n    if (!this.#archive.stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n    this.#reportInfo('creating schemas write stream');\n    const filePathFactory = createFilePathFactory('schemas');\n\n    const entryStream = createTarEntryStream(\n      this.#archive.stream,\n      filePathFactory,\n      this.options.file.maxSizeJsonl\n    );\n\n    return chain([stringer(), entryStream]);\n  }\n\n  createEntitiesWriteStream(): Writable {\n    if (!this.#archive.stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n    this.#reportInfo('creating entities write stream');\n    const filePathFactory = createFilePathFactory('entities');\n\n    const entryStream = createTarEntryStream(\n      this.#archive.stream,\n      filePathFactory,\n      this.options.file.maxSizeJsonl\n    );\n\n    return chain([stringer(), entryStream]);\n  }\n\n  createLinksWriteStream(): Writable {\n    if (!this.#archive.stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n    this.#reportInfo('creating links write stream');\n    const filePathFactory = createFilePathFactory('links');\n\n    const entryStream = createTarEntryStream(\n      this.#archive.stream,\n      filePathFactory,\n      this.options.file.maxSizeJsonl\n    );\n\n    return chain([stringer(), entryStream]);\n  }\n\n  createConfigurationWriteStream(): Writable {\n    if (!this.#archive.stream) {\n      throw new Error('Archive stream is unavailable');\n    }\n    this.#reportInfo('creating configuration write stream');\n    const filePathFactory = createFilePathFactory('configuration');\n\n    const entryStream = createTarEntryStream(\n      this.#archive.stream,\n      filePathFactory,\n      this.options.file.maxSizeJsonl\n    );\n\n    return chain([stringer(), entryStream]);\n  }\n\n  createAssetsWriteStream(): Writable {\n    const { stream: archiveStream } = this.#archive;\n\n    if (!archiveStream) {\n      throw new Error('Archive stream is unavailable');\n    }\n\n    this.#reportInfo('creating assets write stream');\n    return new Writable({\n      objectMode: true,\n      write(data: IAsset, _encoding, callback) {\n        // always write tar files with posix paths so we have a standard format for paths regardless of system\n        const entryPath = path.posix.join('assets', 'uploads', data.filename);\n\n        const entryMetadataPath = path.posix.join('assets', 'metadata', `${data.filename}.json`);\n        const stringifiedMetadata = JSON.stringify(data.metadata);\n        archiveStream.entry(\n          {\n            name: entryMetadataPath,\n            size: stringifiedMetadata.length,\n          },\n          stringifiedMetadata\n        );\n\n        const entry = archiveStream.entry({\n          name: entryPath,\n          size: data.stats.size,\n        });\n\n        if (!entry) {\n          callback(new Error(`Failed to created an asset tar entry for ${entryPath}`));\n          return;\n        }\n\n        data.stream.pipe(entry);\n\n        entry\n          .on('finish', () => {\n            callback(null);\n          })\n          .on('error', (error) => {\n            callback(error);\n          });\n      },\n    });\n  }\n}\n"],"names":["createLocalFileDestinationProvider","options","LocalFileDestinationProvider","setMetadata","target","metadata","providersMetadata","createGzip","reportInfo","zlib","bootstrap","diagnostics","compression","encryption","enabled","key","Error","archive","stream","tar","pack","outStream","createWriteStream","archivePath","on","err","code","ProviderTransferError","archiveTransforms","push","createEncryptionCipher","pipeline","chain","results","file","path","close","writeMetadata","finalize","closed","Promise","resolve","reject","rollback","rm","force","getMetadata","createSchemasWriteStream","filePathFactory","createFilePathFactory","entryStream","createTarEntryStream","maxSizeJsonl","stringer","createEntitiesWriteStream","createLinksWriteStream","createConfigurationWriteStream","createAssetsWriteStream","archiveStream","Writable","objectMode","write","data","_encoding","callback","entryPath","posix","join","filename","entryMetadataPath","stringifiedMetadata","JSON","stringify","entry","name","size","length","stats","pipe","error","constructor","type","message","report","details","createdAt","Date","origin","kind","filePath","source","getMetadataStream","Readable","from"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA6CO,MAAMA,qCAAqC,CAChDC,OAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,4BAA6BD,CAAAA,OAAAA,CAAAA;AAC1C;AAWE,IAAA,kBAAA,iBAAA,8BAAA,CAAA,oBAAA,CAAA,EAEA,QAEA,iBAAA,8BAAA,CAAA,UAAA,CAAA,EAAA,YAAA,iBAAA,8BAAA,CAAA,cAAA,CAAA,EAMA,WAWI,iBAAA,8BAAA,CAAA,aAAA,CAAA,EAAA,YAAA,iBAAA,8BAAA,CAAA,cAAA,CAAA,EA0FE,cAcN,iBAAA,8BAAA,CAAA,gBAAA,CAAA,EAAA,kBAAA,iBAAA,8BAAA,CAAA,oBAAA,CAAA;AAtIF,MAAMC,4BAAAA,CAAAA;IA8CJC,WAAYC,CAAAA,MAAoB,EAAEC,QAAmB,EAAwB;AAC3E,QAAA,+BAAA,CAAA,IAAI,EAAEC,kBAAAA,CAAAA,CAAAA,kBAAiB,CAAA,CAACF,OAAO,GAAGC,QAAAA;AAElC,QAAA,OAAO,IAAI;AACb;IAEAE,UAAwB,GAAA;QACtB,+BAAA,CAAA,IAAI,EAAEC,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,eAAA,CAAA;AACjB,QAAA,OAAOC,IAAKF,UAAU,EAAA;AACxB;AAEAG,IAAAA,SAAAA,CAAUC,WAAgC,EAAwB;QAChE,+BAAA,CAAA,IAAI,EAAEA,YAAAA,CAAAA,CAAAA,YAAcA,CAAAA,GAAAA,WAAAA;QACpB,MAAM,EAAEC,WAAW,EAAEC,UAAU,EAAE,GAAG,IAAI,CAACZ,OAAO;AAEhD,QAAA,IAAIY,WAAWC,OAAO,IAAI,CAACD,UAAAA,CAAWE,GAAG,EAAE;AACzC,YAAA,MAAM,IAAIC,KAAM,CAAA,6BAAA,CAAA;AAClB;AAEA,QAAA,+BAAA,CAAA,IAAI,EAAEC,QAAAA,CAAAA,CAAAA,UAAQC,MAAM,GAAGC,IAAIC,IAAI,EAAA;AAE/B,QAAA,MAAMC,SAAYC,GAAAA,iBAAAA,CAAkB,+BAAA,CAAA,IAAI,EAAEC,YAAAA,CAAAA,CAAAA,YAAAA,CAAAA,CAAAA;QAE1CF,SAAUG,CAAAA,EAAE,CAAC,OAAA,EAAS,CAACC,GAAAA,GAAAA;YACrB,IAAIA,GAAAA,CAAIC,IAAI,KAAK,QAAU,EAAA;AACzB,gBAAA,MAAM,IAAIC,qBACR,CAAA,4DAAA,CAAA;AAEJ;YACA,MAAMF,GAAAA;AACR,SAAA,CAAA;AAEA,QAAA,MAAMG,oBAA8B,EAAE;QAEtC,IAAIhB,WAAAA,CAAYE,OAAO,EAAE;AACvBc,YAAAA,iBAAAA,CAAkBC,IAAI,CAAC,IAAI,CAACtB,UAAU,EAAA,CAAA;AACxC;AAEA,QAAA,IAAIM,UAAWC,CAAAA,OAAO,IAAID,UAAAA,CAAWE,GAAG,EAAE;AACxCa,YAAAA,iBAAAA,CAAkBC,IAAI,CAACC,sBAAuBjB,CAAAA,UAAAA,CAAWE,GAAG,CAAA,CAAA;AAC9D;AAEA,QAAA,+BAAA,CAAA,IAAI,EAAEE,QAAAA,CAAAA,CAAAA,QAAQc,CAAAA,CAAAA,QAAQ,GAAGC,KAAM,CAAA;AAAC,YAAA,+BAAA,CAAA,IAAI,EAAEf,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM;AAAKU,YAAAA,GAAAA,iBAAAA;AAAmBP,YAAAA;AAAU,SAAA,CAAA;AAEtF,QAAA,IAAI,CAACY,OAAO,CAACC,IAAI,GAAG;AAAEC,YAAAA,IAAI,EAAE,+BAAA,CAAA,IAAI,EAAEZ,YAAAA,CAAAA,CAAAA,YAAAA;AAAY,SAAA;AAChD;AAEA,IAAA,MAAMa,KAAQ,GAAA;QACZ,MAAM,EAAElB,MAAM,EAAEa,QAAQ,EAAE,GAAG,+BAAA,CAAA,IAAI,EAAEd,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA;AAEnC,QAAA,IAAI,CAACC,MAAQ,EAAA;AACX,YAAA;AACF;QAEA,MAAM,+BAAA,CAAA,IAAI,EAAEmB,cAAAA,CAAAA,CAAAA,cAAAA,CAAAA,EAAAA;AACZnB,QAAAA,MAAAA,CAAOoB,QAAQ,EAAA;AAEf,QAAA,IAAIP,QAAY,IAAA,CAACA,QAASQ,CAAAA,MAAM,EAAE;YAChC,MAAM,IAAIC,OAAc,CAAA,CAACC,OAASC,EAAAA,MAAAA,GAAAA;AAChCX,gBAAAA,QAAAA,CAASP,EAAE,CAAC,OAAA,EAASiB,OAASjB,CAAAA,CAAAA,EAAE,CAAC,OAASkB,EAAAA,MAAAA,CAAAA;AAC5C,aAAA,CAAA;AACF;AACF;AAEA,IAAA,MAAMC,QAA0B,GAAA;QAC9B,+BAAA,CAAA,IAAI,EAAEnC,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,cAAA,CAAA;QACjB,MAAM,IAAI,CAAC4B,KAAK,EAAA;AAChB,QAAA,MAAMQ,EAAG,CAAA,+BAAA,CAAA,IAAI,EAAErB,cAAAA,YAAa,CAAA,EAAA;YAAEsB,KAAO,EAAA;AAAK,SAAA,CAAA;AAC5C;IAEAC,WAAc,GAAA;QACZ,OAAO,IAAA;AACT;IA0BAC,wBAA2B,GAAA;AACzB,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAE9B,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM,EAAE;AACzB,YAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;QACA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,+BAAA,CAAA;AACjB,QAAA,MAAMwC,kBAAkBC,qBAAsB,CAAA,SAAA,CAAA;AAE9C,QAAA,MAAMC,cAAcC,oBAClB,CAAA,+BAAA,CAAA,IAAI,EAAElC,UAAAA,QAAQC,CAAAA,CAAAA,MAAM,EACpB8B,eAAAA,EACA,IAAI,CAAC/C,OAAO,CAACiC,IAAI,CAACkB,YAAY,CAAA;AAGhC,QAAA,OAAOpB,KAAM,CAAA;AAACqB,YAAAA,QAAAA,EAAAA;AAAYH,YAAAA;AAAY,SAAA,CAAA;AACxC;IAEAI,yBAAsC,GAAA;AACpC,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAErC,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM,EAAE;AACzB,YAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;QACA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,gCAAA,CAAA;AACjB,QAAA,MAAMwC,kBAAkBC,qBAAsB,CAAA,UAAA,CAAA;AAE9C,QAAA,MAAMC,cAAcC,oBAClB,CAAA,+BAAA,CAAA,IAAI,EAAElC,UAAAA,QAAQC,CAAAA,CAAAA,MAAM,EACpB8B,eAAAA,EACA,IAAI,CAAC/C,OAAO,CAACiC,IAAI,CAACkB,YAAY,CAAA;AAGhC,QAAA,OAAOpB,KAAM,CAAA;AAACqB,YAAAA,QAAAA,EAAAA;AAAYH,YAAAA;AAAY,SAAA,CAAA;AACxC;IAEAK,sBAAmC,GAAA;AACjC,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAEtC,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM,EAAE;AACzB,YAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;QACA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,6BAAA,CAAA;AACjB,QAAA,MAAMwC,kBAAkBC,qBAAsB,CAAA,OAAA,CAAA;AAE9C,QAAA,MAAMC,cAAcC,oBAClB,CAAA,+BAAA,CAAA,IAAI,EAAElC,UAAAA,QAAQC,CAAAA,CAAAA,MAAM,EACpB8B,eAAAA,EACA,IAAI,CAAC/C,OAAO,CAACiC,IAAI,CAACkB,YAAY,CAAA;AAGhC,QAAA,OAAOpB,KAAM,CAAA;AAACqB,YAAAA,QAAAA,EAAAA;AAAYH,YAAAA;AAAY,SAAA,CAAA;AACxC;IAEAM,8BAA2C,GAAA;AACzC,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAEvC,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA,CAAQC,MAAM,EAAE;AACzB,YAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;QACA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,qCAAA,CAAA;AACjB,QAAA,MAAMwC,kBAAkBC,qBAAsB,CAAA,eAAA,CAAA;AAE9C,QAAA,MAAMC,cAAcC,oBAClB,CAAA,+BAAA,CAAA,IAAI,EAAElC,UAAAA,QAAQC,CAAAA,CAAAA,MAAM,EACpB8B,eAAAA,EACA,IAAI,CAAC/C,OAAO,CAACiC,IAAI,CAACkB,YAAY,CAAA;AAGhC,QAAA,OAAOpB,KAAM,CAAA;AAACqB,YAAAA,QAAAA,EAAAA;AAAYH,YAAAA;AAAY,SAAA,CAAA;AACxC;IAEAO,uBAAoC,GAAA;QAClC,MAAM,EAAEvC,QAAQwC,aAAa,EAAE,GAAG,+BAAA,CAAA,IAAI,EAAEzC,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA;AAExC,QAAA,IAAI,CAACyC,aAAe,EAAA;AAClB,YAAA,MAAM,IAAI1C,KAAM,CAAA,+BAAA,CAAA;AAClB;QAEA,+BAAA,CAAA,IAAI,EAAER,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,8BAAA,CAAA;AACjB,QAAA,OAAO,IAAImD,QAAS,CAAA;YAClBC,UAAY,EAAA,IAAA;AACZC,YAAAA,KAAAA,CAAAA,CAAMC,IAAY,EAAEC,SAAS,EAAEC,QAAQ,EAAA;;gBAErC,MAAMC,SAAAA,GAAY9B,KAAK+B,KAAK,CAACC,IAAI,CAAC,QAAA,EAAU,SAAWL,EAAAA,IAAAA,CAAKM,QAAQ,CAAA;AAEpE,gBAAA,MAAMC,iBAAoBlC,GAAAA,IAAAA,CAAK+B,KAAK,CAACC,IAAI,CAAC,QAAA,EAAU,UAAY,EAAA,CAAC,EAAEL,IAAAA,CAAKM,QAAQ,CAAC,KAAK,CAAC,CAAA;AACvF,gBAAA,MAAME,mBAAsBC,GAAAA,IAAAA,CAAKC,SAAS,CAACV,KAAKzD,QAAQ,CAAA;AACxDqD,gBAAAA,aAAAA,CAAce,KAAK,CACjB;oBACEC,IAAML,EAAAA,iBAAAA;AACNM,oBAAAA,IAAAA,EAAML,oBAAoBM;iBAE5BN,EAAAA,mBAAAA,CAAAA;gBAGF,MAAMG,KAAAA,GAAQf,aAAce,CAAAA,KAAK,CAAC;oBAChCC,IAAMT,EAAAA,SAAAA;oBACNU,IAAMb,EAAAA,IAAAA,CAAKe,KAAK,CAACF;AACnB,iBAAA,CAAA;AAEA,gBAAA,IAAI,CAACF,KAAO,EAAA;AACVT,oBAAAA,QAAAA,CAAS,IAAIhD,KAAM,CAAA,CAAC,yCAAyC,EAAEiD,UAAU,CAAC,CAAA,CAAA;AAC1E,oBAAA;AACF;gBAEAH,IAAK5C,CAAAA,MAAM,CAAC4D,IAAI,CAACL,KAAAA,CAAAA;gBAEjBA,KACGjD,CAAAA,EAAE,CAAC,QAAU,EAAA,IAAA;oBACZwC,QAAS,CAAA,IAAA,CAAA;iBAEVxC,CAAAA,CAAAA,EAAE,CAAC,OAAA,EAAS,CAACuD,KAAAA,GAAAA;oBACZf,QAASe,CAAAA,KAAAA,CAAAA;AACX,iBAAA,CAAA;AACJ;AACF,SAAA,CAAA;AACF;AA9OAC,IAAAA,WAAAA,CAAY/E,OAA6C,CAAE;QAI3D,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,WAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAWA,MAAI,CAAA,cAAA,CAAA,IAAA,EAAA,YAAA,EAAA;AAAJ,YAAA,GAAA,EAAA,eAAA;;;QA0FA,MAAM,CAAA,cAAA,CAAA,IAAA,EAAA,cAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAcA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,kBAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QA7HA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,kBAAA,EAAA;;mBAAA,KAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,QAAA,EAAA;;mBAAA,KAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,YAAA,EAAA;;mBAAA,KAAA;;aAZAyE,IAAO,GAAA,yBAAA;aAEPO,IAAqB,GAAA,aAAA;AAIrBhD,QAAAA,IAAAA,CAAAA,OAAAA,GAAwD,EAAC;AAExD3B,QAAAA,+BAAAA,CAAAA,IAAAA,EAAAA,kBAAAA,CAAAA,CAAAA,sBAAqE,EAAC;AAEtEW,QAAAA,+BAAAA,CAAAA,IAAAA,EAAAA,QAAAA,CAAAA,CAAAA,YAAoD,EAAC;QAKpD,IAAI,CAAChB,OAAO,GAAGA,OAAAA;AACjB;AA6OF;AA3OE,SAAA,WAAYiF,OAAe,EAAA;AACzB,IAAA,+BAAA,CAAA,IAAI,EAAEvE,YAAAA,CAAAA,CAAAA,YAAAA,CAAAA,EAAawE,MAAO,CAAA;QACxBC,OAAS,EAAA;AACPC,YAAAA,SAAAA,EAAW,IAAIC,IAAAA,EAAAA;AACfJ,YAAAA,OAAAA;YACAK,MAAQ,EAAA;AACV,SAAA;QACAC,IAAM,EAAA;AACR,KAAA,CAAA;AACF;AAEA,SAAA,eAAA,GAAA;IACE,MAAM,EAAE3E,UAAU,EAAED,WAAW,EAAEsB,IAAI,EAAE,GAAG,IAAI,CAACjC,OAAO;AAEtD,IAAA,IAAIwF,WAAW,CAAC,EAAEvD,KAAKC,IAAI,CAAC,IAAI,CAAC;IAEjC,IAAIvB,WAAAA,CAAYE,OAAO,EAAE;QACvB2E,QAAY,IAAA,KAAA;AACd;IAEA,IAAI5E,UAAAA,CAAWC,OAAO,EAAE;QACtB2E,QAAY,IAAA,MAAA;AACd;IAEA,OAAOA,QAAAA;AACT;AA4EA,eAAA,aAAA,GAAA;IACE,+BAAA,CAAA,IAAI,EAAEjF,WAAAA,CAAAA,CAAAA,WAAW,CAAA,CAAA,kBAAA,CAAA;AACjB,IAAA,MAAMH,WAAW,+BAAA,CAAA,IAAI,EAAEC,kBAAAA,CAAAA,CAAAA,oBAAkBoF,MAAM;AAE/C,IAAA,IAAIrF,QAAU,EAAA;QACZ,MAAM,IAAImC,QAAQ,CAACC,OAAAA,GAAAA;AACjB,YAAA,MAAMpB,SAAY,GAAA,+BAAA,CAAA,IAAI,EAAEsE,kBAAAA,CAAAA,CAAAA,kBAAAA,CAAAA,EAAAA;AACxB,YAAA,MAAM7B,IAAOS,GAAAA,IAAAA,CAAKC,SAAS,CAACnE,UAAU,IAAM,EAAA,CAAA,CAAA;YAE5CuF,QAASC,CAAAA,IAAI,CAAC/B,IAAMgB,CAAAA,CAAAA,IAAI,CAACzD,SAAWG,CAAAA,CAAAA,EAAE,CAAC,OAASiB,EAAAA,OAAAA,CAAAA;AAClD,SAAA,CAAA;AACF;AACF;AAEA,SAAA,iBAAA,GAAA;AACE,IAAA,MAAM,EAAEvB,MAAM,EAAE,GAAG,+BAAA,CAAA,IAAI,EAAED,QAAAA,CAAAA,CAAAA,QAAAA,CAAAA;AAEzB,IAAA,IAAI,CAACC,MAAQ,EAAA;AACX,QAAA,MAAM,IAAIF,KAAM,CAAA,+BAAA,CAAA;AAClB;IAEA,OAAOmC,oBAAAA,CAAqBjC,QAAQ,IAAM,eAAA,CAAA;AAC5C;;;;"}

package/dist/file/providers/destination/utils.js

@@ -0,0 +1,63 @@
+'use strict';
+
+var stream = require('stream');
+var path = require('path');
+
+/**
+ * Create a file path factory for a given path & prefix.
+ * Upon being called, the factory will return a file path for a given index
+ */ const createFilePathFactory = (type)=>(fileIndex = 0)=>{
+        // always write tar files with posix paths so we have a standard format for paths regardless of system
+        return path.posix.join(// "{type}" directory
+        type, // "${type}_XXXXX.jsonl" file
+        `${type}_${String(fileIndex).padStart(5, '0')}.jsonl`);
+    };
+const createTarEntryStream = (archive, pathFactory, maxSize = 2.56e8)=>{
+    let fileIndex = 0;
+    let buffer = '';
+    const flush = async ()=>{
+        if (!buffer) {
+            return;
+        }
+        fileIndex += 1;
+        const name = pathFactory(fileIndex);
+        const size = buffer.length;
+        await new Promise((resolve, reject)=>{
+            archive.entry({
+                name,
+                size
+            }, buffer, (err)=>{
+                if (err) {
+                    reject(err);
+                }
+                resolve();
+            });
+        });
+        buffer = '';
+    };
+    const push = (chunk)=>{
+        buffer += chunk;
+    };
+    return new stream.Writable({
+        async destroy (err, callback) {
+            await flush();
+            callback(err);
+        },
+        async write (chunk, _encoding, callback) {
+            const size = chunk.length;
+            if (chunk.length > maxSize) {
+                callback(new Error(`payload too large: ${chunk.length}>${maxSize}`));
+                return;
+            }
+            if (buffer.length + size > maxSize) {
+                await flush();
+            }
+            push(chunk);
+            callback(null);
+        }
+    });
+};
+
+exports.createFilePathFactory = createFilePathFactory;
+exports.createTarEntryStream = createTarEntryStream;
+//# sourceMappingURL=utils.js.map

package/dist/file/providers/destination/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sources":["../../../../src/file/providers/destination/utils.ts"],"sourcesContent":["import { Writable } from 'stream';\nimport { posix } from 'path';\nimport tar from 'tar-stream';\n\n/**\n * Create a file path factory for a given path & prefix.\n * Upon being called, the factory will return a file path for a given index\n */\nexport const createFilePathFactory =\n  (type: string) =>\n  (fileIndex = 0): string => {\n    // always write tar files with posix paths so we have a standard format for paths regardless of system\n    return posix.join(\n      // \"{type}\" directory\n      type,\n      // \"${type}_XXXXX.jsonl\" file\n      `${type}_${String(fileIndex).padStart(5, '0')}.jsonl`\n    );\n  };\n\nexport const createTarEntryStream = (\n  archive: tar.Pack,\n  pathFactory: (index?: number) => string,\n  maxSize = 2.56e8\n) => {\n  let fileIndex = 0;\n  let buffer = '';\n\n  const flush = async () => {\n    if (!buffer) {\n      return;\n    }\n\n    fileIndex += 1;\n    const name = pathFactory(fileIndex);\n    const size = buffer.length;\n\n    await new Promise<void>((resolve, reject) => {\n      archive.entry({ name, size }, buffer, (err) => {\n        if (err) {\n          reject(err);\n        }\n\n        resolve();\n      });\n    });\n\n    buffer = '';\n  };\n\n  const push = (chunk: string | Buffer) => {\n    buffer += chunk;\n  };\n\n  return new Writable({\n    async destroy(err, callback) {\n      await flush();\n      callback(err);\n    },\n\n    async write(chunk, _encoding, callback) {\n      const size = chunk.length;\n\n      if (chunk.length > maxSize) {\n        callback(new Error(`payload too large: ${chunk.length}>${maxSize}`));\n        return;\n      }\n\n      if (buffer.length + size > maxSize) {\n        await flush();\n      }\n\n      push(chunk);\n\n      callback(null);\n    },\n  });\n};\n"],"names":["createFilePathFactory","type","fileIndex","posix","join","String","padStart","createTarEntryStream","archive","pathFactory","maxSize","buffer","flush","name","size","length","Promise","resolve","reject","entry","err","push","chunk","Writable","destroy","callback","write","_encoding","Error"],"mappings":";;;;;AAIA;;;AAGC,IACYA,MAAAA,qBAAAA,GACX,CAACC,IACD,GAAA,CAACC,YAAY,CAAC,GAAA;;QAEZ,OAAOC,UAAAA,CAAMC,IAAI;AAEfH,QAAAA,IAAAA;QAEA,CAAC,EAAEA,IAAK,CAAA,CAAC,EAAEI,MAAAA,CAAOH,SAAWI,CAAAA,CAAAA,QAAQ,CAAC,CAAA,EAAG,GAAK,CAAA,CAAA,MAAM,CAAC,CAAA;;MAI9CC,oBAAuB,GAAA,CAClCC,OACAC,EAAAA,WAAAA,EACAC,UAAU,MAAM,GAAA;AAEhB,IAAA,IAAIR,SAAY,GAAA,CAAA;AAChB,IAAA,IAAIS,MAAS,GAAA,EAAA;AAEb,IAAA,MAAMC,KAAQ,GAAA,UAAA;AACZ,QAAA,IAAI,CAACD,MAAQ,EAAA;AACX,YAAA;AACF;QAEAT,SAAa,IAAA,CAAA;AACb,QAAA,MAAMW,OAAOJ,WAAYP,CAAAA,SAAAA,CAAAA;QACzB,MAAMY,IAAAA,GAAOH,OAAOI,MAAM;QAE1B,MAAM,IAAIC,OAAc,CAAA,CAACC,OAASC,EAAAA,MAAAA,GAAAA;AAChCV,YAAAA,OAAAA,CAAQW,KAAK,CAAC;AAAEN,gBAAAA,IAAAA;AAAMC,gBAAAA;AAAK,aAAA,EAAGH,QAAQ,CAACS,GAAAA,GAAAA;AACrC,gBAAA,IAAIA,GAAK,EAAA;oBACPF,MAAOE,CAAAA,GAAAA,CAAAA;AACT;AAEAH,gBAAAA,OAAAA,EAAAA;AACF,aAAA,CAAA;AACF,SAAA,CAAA;QAEAN,MAAS,GAAA,EAAA;AACX,KAAA;AAEA,IAAA,MAAMU,OAAO,CAACC,KAAAA,GAAAA;QACZX,MAAUW,IAAAA,KAAAA;AACZ,KAAA;AAEA,IAAA,OAAO,IAAIC,eAAS,CAAA;QAClB,MAAMC,OAAAA,CAAAA,CAAQJ,GAAG,EAAEK,QAAQ,EAAA;YACzB,MAAMb,KAAAA,EAAAA;YACNa,QAASL,CAAAA,GAAAA,CAAAA;AACX,SAAA;AAEA,QAAA,MAAMM,KAAMJ,CAAAA,CAAAA,KAAK,EAAEK,SAAS,EAAEF,QAAQ,EAAA;YACpC,MAAMX,IAAAA,GAAOQ,MAAMP,MAAM;YAEzB,IAAIO,KAAAA,CAAMP,MAAM,GAAGL,OAAS,EAAA;gBAC1Be,QAAS,CAAA,IAAIG,KAAM,CAAA,CAAC,mBAAmB,EAAEN,KAAMP,CAAAA,MAAM,CAAC,CAAC,EAAEL,OAAAA,CAAQ,CAAC,CAAA,CAAA;AAClE,gBAAA;AACF;AAEA,YAAA,IAAIC,MAAOI,CAAAA,MAAM,GAAGD,IAAAA,GAAOJ,OAAS,EAAA;gBAClC,MAAME,KAAAA,EAAAA;AACR;YAEAS,IAAKC,CAAAA,KAAAA,CAAAA;YAELG,QAAS,CAAA,IAAA,CAAA;AACX;AACF,KAAA,CAAA;AACF;;;;;"}