@strapi/data-transfer 5.56.0 → 5.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/directory/providers/source/index.js +3 -3
- package/dist/directory/providers/source/index.js.map +1 -1
- package/dist/directory/providers/source/index.mjs +2 -2
- package/dist/directory/providers/source/index.mjs.map +1 -1
- package/dist/engine/index.js +13 -8
- package/dist/engine/index.js.map +1 -1
- package/dist/engine/index.mjs +9 -4
- package/dist/engine/index.mjs.map +1 -1
- package/dist/engine/validation/provider.js +2 -2
- package/dist/engine/validation/provider.js.map +1 -1
- package/dist/engine/validation/provider.mjs +1 -1
- package/dist/engine/validation/provider.mjs.map +1 -1
- package/dist/engine/validation/schemas/index.js +4 -4
- package/dist/engine/validation/schemas/index.js.map +1 -1
- package/dist/engine/validation/schemas/index.mjs +3 -3
- package/dist/engine/validation/schemas/index.mjs.map +1 -1
- package/dist/file/providers/source/index.js +3 -3
- package/dist/file/providers/source/index.js.map +1 -1
- package/dist/file/providers/source/index.mjs +2 -2
- package/dist/file/providers/source/index.mjs.map +1 -1
- package/dist/strapi/providers/local-destination/index.js +1 -1
- package/dist/strapi/providers/local-destination/index.mjs +1 -1
- package/dist/strapi/providers/local-destination/strategies/restore/configuration.js +4 -4
- package/dist/strapi/providers/local-destination/strategies/restore/configuration.js.map +1 -1
- package/dist/strapi/providers/local-destination/strategies/restore/configuration.mjs +4 -4
- package/dist/strapi/providers/local-destination/strategies/restore/configuration.mjs.map +1 -1
- package/dist/strapi/providers/local-destination/strategies/restore/entities.js +1 -1
- package/dist/strapi/providers/local-destination/strategies/restore/entities.mjs +1 -1
- package/dist/strapi/providers/local-destination/strategies/restore/index.js +5 -5
- package/dist/strapi/providers/local-destination/strategies/restore/index.js.map +1 -1
- package/dist/strapi/providers/local-destination/strategies/restore/index.mjs +5 -3
- package/dist/strapi/providers/local-destination/strategies/restore/index.mjs.map +1 -1
- package/dist/strapi/providers/local-source/configuration.d.ts.map +1 -1
- package/dist/strapi/providers/local-source/configuration.js +4 -2
- package/dist/strapi/providers/local-source/configuration.js.map +1 -1
- package/dist/strapi/providers/local-source/configuration.mjs +4 -2
- package/dist/strapi/providers/local-source/configuration.mjs.map +1 -1
- package/dist/strapi/providers/local-source/entities.js +1 -1
- package/dist/strapi/providers/local-source/entities.mjs +1 -1
- package/dist/strapi/providers/local-source/index.js +1 -1
- package/dist/strapi/providers/local-source/index.mjs +1 -1
- package/dist/strapi/providers/remote-destination/index.js +2 -2
- package/dist/strapi/providers/remote-destination/index.js.map +1 -1
- package/dist/strapi/providers/remote-destination/index.mjs +1 -1
- package/dist/strapi/providers/remote-destination/index.mjs.map +1 -1
- package/dist/strapi/providers/remote-source/index.d.ts.map +1 -1
- package/dist/strapi/providers/remote-source/index.js +3 -2
- package/dist/strapi/providers/remote-source/index.js.map +1 -1
- package/dist/strapi/providers/remote-source/index.mjs +3 -2
- package/dist/strapi/providers/remote-source/index.mjs.map +1 -1
- package/dist/strapi/queries/entity.js +13 -13
- package/dist/strapi/queries/entity.js.map +1 -1
- package/dist/strapi/queries/entity.mjs +13 -13
- package/dist/strapi/queries/entity.mjs.map +1 -1
- package/dist/strapi/queries/link.js +7 -7
- package/dist/strapi/queries/link.js.map +1 -1
- package/dist/strapi/queries/link.mjs +3 -3
- package/dist/strapi/queries/link.mjs.map +1 -1
- package/dist/strapi/remote/handlers/pull.js +2 -3
- package/dist/strapi/remote/handlers/pull.js.map +1 -1
- package/dist/strapi/remote/handlers/pull.mjs +2 -3
- package/dist/strapi/remote/handlers/pull.mjs.map +1 -1
- package/dist/strapi/remote/handlers/push.js +2 -2
- package/dist/strapi/remote/handlers/push.mjs +2 -2
- package/dist/strapi/utils/project-settings-logos.js +2 -2
- package/dist/strapi/utils/project-settings-logos.js.map +1 -1
- package/dist/strapi/utils/project-settings-logos.mjs +2 -2
- package/dist/strapi/utils/project-settings-logos.mjs.map +1 -1
- package/dist/utils/components.d.ts.map +1 -1
- package/dist/utils/components.js +8 -8
- package/dist/utils/components.js.map +1 -1
- package/dist/utils/components.mjs +9 -9
- package/dist/utils/components.mjs.map +1 -1
- package/dist/utils/json.js +8 -6
- package/dist/utils/json.js.map +1 -1
- package/dist/utils/json.mjs +5 -3
- package/dist/utils/json.mjs.map +1 -1
- package/dist/utils/schema.d.ts.map +1 -1
- package/dist/utils/schema.js +2 -2
- package/dist/utils/schema.js.map +1 -1
- package/dist/utils/schema.mjs +2 -2
- package/dist/utils/schema.mjs.map +1 -1
- package/package.json +6 -6
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
var path = require('path');
|
|
4
4
|
var stream$1 = require('stream');
|
|
5
5
|
var fs = require('fs-extra');
|
|
6
|
-
var
|
|
6
|
+
var _ = require('lodash');
|
|
7
7
|
var streamChain = require('stream-chain');
|
|
8
8
|
var Parser = require('stream-json/jsonl/Parser');
|
|
9
9
|
require('crypto');
|
|
@@ -68,10 +68,10 @@ class LocalDirectorySourceProvider {
|
|
|
68
68
|
async getSchemas() {
|
|
69
69
|
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting schemas');
|
|
70
70
|
const schemaCollection = await stream.collect(this.createSchemasReadStream());
|
|
71
|
-
if (
|
|
71
|
+
if (_.isEmpty(schemaCollection)) {
|
|
72
72
|
throw new providers.ProviderInitializationError('Could not load schemas from Strapi data export.');
|
|
73
73
|
}
|
|
74
|
-
const schemas =
|
|
74
|
+
const schemas = _.keyBy(schemaCollection, 'uid');
|
|
75
75
|
return schema.schemasToValidJSON(schemas);
|
|
76
76
|
}
|
|
77
77
|
async validateStage(stage) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sources":["../../../../src/directory/providers/source/index.ts"],"sourcesContent":["import path from 'path';\nimport type { Readable } from 'stream';\nimport { PassThrough } from 'stream';\n\nimport fs from 'fs-extra';\nimport { isEmpty, keyBy } from 'lodash/fp';\nimport { chain } from 'stream-chain';\nimport { parser } from 'stream-json/jsonl/Parser';\nimport type { Struct } from '@strapi/types';\n\nimport type {\n IAsset,\n IMetadata,\n ISourceProvider,\n ProviderType,\n TransferStage,\n} from '../../../types';\nimport type { IDiagnosticReporter } from '../../../utils/diagnostic';\n\nimport * as utils from '../../../utils';\nimport { write } from '../../../utils/writable-async-write';\nimport {\n ProviderInitializationError,\n ProviderTransferError,\n ProviderValidationError,\n} from '../../../errors/providers';\nimport { unknownPathToPosix, validateAssetMetadata } from '../../../file/providers/source/utils';\n\nconst METADATA_FILE_PATH = 'metadata.json';\n\nexport interface ILocalDirectorySourceProviderOptions {\n directory: {\n /** Absolute or relative path to the root of an unpacked Strapi data-transfer export */\n path: string;\n };\n}\n\nexport const createLocalDirectorySourceProvider = (\n options: ILocalDirectorySourceProviderOptions\n) => {\n return new LocalDirectorySourceProvider(options);\n};\n\nconst isPathInsideRoot = (root: string, candidate: string): boolean => {\n const relative = path.relative(root, candidate);\n return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));\n};\n\nclass LocalDirectorySourceProvider implements ISourceProvider {\n type: ProviderType = 'source';\n\n name = 'source::local-directory';\n\n options: ILocalDirectorySourceProviderOptions;\n\n #rootResolved: string;\n\n #metadata?: IMetadata;\n\n #assetMetadata?: Map<string, IAsset['metadata']>;\n\n #diagnostics?: IDiagnosticReporter;\n\n constructor(options: ILocalDirectorySourceProviderOptions) {\n this.options = options;\n this.#rootResolved = path.resolve(options.directory.path);\n }\n\n #reportInfo(message: string) {\n this.#diagnostics?.report({\n details: {\n createdAt: new Date(),\n message,\n origin: 'directory-source-provider',\n },\n kind: 'info',\n });\n }\n\n /** Resolve a posix-style relative path under the export root; rejects escapes. */\n #safePath(...posixSegments: string[]): string {\n const joined = path.posix.join(...posixSegments);\n const segments = joined.split(path.posix.sep).filter(Boolean);\n const resolved = path.resolve(this.#rootResolved, ...segments);\n if (!isPathInsideRoot(this.#rootResolved, resolved)) {\n throw new ProviderInitializationError(`Invalid path \"${joined}\" — escapes backup directory`);\n }\n return resolved;\n }\n\n async bootstrap(diagnostics: IDiagnosticReporter) {\n this.#diagnostics = diagnostics;\n const root = this.#rootResolved;\n\n try {\n const stat = await fs.stat(root);\n if (!stat.isDirectory()) {\n throw new ProviderInitializationError(`Path '${root}' is not a directory.`);\n }\n await this.#loadMetadata();\n } catch (e) {\n if (e instanceof ProviderInitializationError) {\n throw e;\n }\n throw new ProviderInitializationError(\n `Directory '${root}' is not a valid Strapi data export.`\n );\n }\n\n if (!this.#metadata) {\n throw new ProviderInitializationError('Could not load metadata from Strapi data export.');\n }\n }\n\n async #loadMetadata() {\n const metadataPath = this.#safePath(METADATA_FILE_PATH);\n if (!(await fs.pathExists(metadataPath))) {\n throw new ProviderInitializationError(\n `Missing ${METADATA_FILE_PATH} in export directory '${this.#rootResolved}'.`\n );\n }\n this.#metadata = await fs.readJson(metadataPath);\n }\n\n async getMetadata() {\n this.#reportInfo('getting metadata');\n if (!this.#metadata) {\n await this.#loadMetadata();\n }\n return this.#metadata ?? null;\n }\n\n async getSchemas() {\n this.#reportInfo('getting schemas');\n const schemaCollection = await utils.stream.collect<Struct.Schema>(\n this.createSchemasReadStream()\n );\n\n if (isEmpty(schemaCollection)) {\n throw new ProviderInitializationError('Could not load schemas from Strapi data export.');\n }\n\n const schemas = keyBy('uid', schemaCollection);\n return utils.schema.schemasToValidJSON(schemas);\n }\n\n async validateStage(stage: TransferStage) {\n if (stage !== 'assets') {\n return;\n }\n\n const uploadsDir = this.#safePath('assets', 'uploads');\n if (!(await fs.pathExists(uploadsDir))) {\n this.#assetMetadata = new Map();\n return;\n }\n\n const metadata = new Map<string, IAsset['metadata']>();\n const names = (await fs.readdir(uploadsDir)).sort();\n for (const name of names) {\n const uploadPath = path.join(uploadsDir, name);\n if (!(await fs.stat(uploadPath)).isFile()) {\n continue;\n }\n\n try {\n metadata.set(name, await this.#readAssetMetadata(name));\n } catch (error) {\n const reason = error instanceof Error ? error.message : String(error);\n throw new ProviderValidationError(\n `Asset metadata preflight failed for \"${name}\": ${reason}. The destination was not modified; re-export and try again.`,\n { error }\n );\n }\n }\n\n this.#assetMetadata = metadata;\n }\n\n createEntitiesReadStream(): Readable {\n this.#reportInfo('creating entities read stream');\n return this.#streamJsonlDirectory('entities');\n }\n\n createSchemasReadStream(): Readable {\n this.#reportInfo('creating schemas read stream');\n return this.#streamJsonlDirectory('schemas');\n }\n\n createLinksReadStream(): Readable {\n this.#reportInfo('creating links read stream');\n return this.#streamJsonlDirectory('links');\n }\n\n createConfigurationReadStream(): Readable {\n this.#reportInfo('creating configuration read stream');\n return this.#streamJsonlDirectory('configuration');\n }\n\n createAssetsReadStream(): Readable {\n const outStream = new PassThrough({ objectMode: true });\n const uploadsDir = this.#safePath('assets', 'uploads');\n this.#reportInfo('creating assets read stream');\n\n this.#pipeAssetsToStream(outStream, uploadsDir).catch((e: unknown) => {\n outStream.destroy(\n e instanceof Error ? e : new ProviderTransferError(String(e), { details: { error: e } })\n );\n });\n\n return outStream;\n }\n\n async #pipeAssetsToStream(outStream: PassThrough, uploadsDir: string): Promise<void> {\n if (!(await fs.pathExists(uploadsDir))) {\n outStream.end();\n return;\n }\n\n const names = (await fs.readdir(uploadsDir)).sort();\n for (const name of names) {\n const absUpload = path.join(uploadsDir, name);\n const stat = await fs.stat(absUpload);\n if (stat.isFile()) {\n let metadata = this.#assetMetadata?.get(name);\n if (!metadata) {\n try {\n metadata = await this.#readAssetMetadata(name);\n } catch (error) {\n outStream.destroy(\n new ProviderTransferError(`Failed to read metadata for ${name}`, {\n details: { error },\n })\n );\n return;\n }\n }\n\n const normalizedPath = unknownPathToPosix(path.posix.join('assets', 'uploads', name));\n const asset: IAsset = {\n metadata,\n filename: name,\n filepath: normalizedPath,\n stats: { size: stat.size },\n stream: fs.createReadStream(absUpload),\n };\n await write(outStream, asset);\n }\n }\n outStream.end();\n }\n\n async #readAssetMetadata(filename: string): Promise<IAsset['metadata']> {\n const metadataPath = this.#safePath('assets', 'metadata', `${filename}.json`);\n return validateAssetMetadata(await fs.readJson(metadataPath), filename);\n }\n\n async #listJsonlFiles(posixSubdir: string): Promise<string[]> {\n const dirAbs = this.#safePath(...posixSubdir.split('/').filter(Boolean));\n if (!(await fs.pathExists(dirAbs))) {\n return [];\n }\n const names = await fs.readdir(dirAbs);\n return names\n .filter((n) => n.endsWith('.jsonl'))\n .sort()\n .map((n) => path.join(dirAbs, n));\n }\n\n #streamJsonlDirectory(posixSubdir: string): Readable {\n const outStream = new PassThrough({ objectMode: true });\n this.#reportInfo(`streaming jsonl from ${posixSubdir}`);\n\n this.#pipeJsonlDirectoryToStream(outStream, posixSubdir).catch((e: unknown) => {\n outStream.destroy(\n e instanceof Error ? e : new ProviderTransferError(String(e), { details: { error: e } })\n );\n });\n\n return outStream;\n }\n\n async #pipeJsonlDirectoryToStream(outStream: PassThrough, posixSubdir: string): Promise<void> {\n const files = await this.#listJsonlFiles(posixSubdir);\n for (const absPath of files) {\n const transforms = [\n parser({\n checkErrors: true,\n }),\n (line: { key: string; value: object }) => line.value,\n ];\n\n const stream = fs.createReadStream(absPath).pipe(chain(transforms));\n\n try {\n for await (const chunk of stream) {\n await write(outStream, chunk);\n }\n } catch (e: unknown) {\n outStream.destroy(\n new ProviderTransferError(`Error parsing JSONL in ${absPath}: ${(e as Error).message}`, {\n details: {\n error: e,\n },\n })\n );\n return;\n }\n }\n outStream.end();\n }\n}\n"],"names":["METADATA_FILE_PATH","createLocalDirectorySourceProvider","options","LocalDirectorySourceProvider","isPathInsideRoot","root","candidate","relative","path","startsWith","sep","isAbsolute","bootstrap","diagnostics","stat","fs","isDirectory","ProviderInitializationError","e","getMetadata","getSchemas","schemaCollection","utils","createSchemasReadStream","isEmpty","schemas","keyBy","validateStage","stage","uploadsDir","pathExists","Map","metadata","names","readdir","sort","name","uploadPath","join","isFile","set","error","reason","Error","message","String","ProviderValidationError","createEntitiesReadStream","createLinksReadStream","createConfigurationReadStream","createAssetsReadStream","outStream","PassThrough","objectMode","catch","destroy","ProviderTransferError","details","type","resolve","directory","report","createdAt","Date","origin","kind","posixSegments","joined","posix","segments","split","filter","Boolean","resolved","metadataPath","readJson","end","absUpload","get","normalizedPath","unknownPathToPosix","asset","filename","filepath","stats","size","stream","createReadStream","write","validateAssetMetadata","posixSubdir","dirAbs","n","endsWith","map","files","absPath","transforms","parser","checkErrors","line","value","pipe","chain","chunk"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAMA,kBAAAA,GAAqB,eAAA;AASpB,MAAMC,qCAAqC,CAChDC,OAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,4BAAAA,CAA6BD,OAAAA,CAAAA;AAC1C;AAEA,MAAME,gBAAAA,GAAmB,CAACC,IAAAA,EAAcC,SAAAA,GAAAA;AACtC,IAAA,MAAMC,QAAAA,GAAWC,qBAAAA,CAAKD,QAAQ,CAACF,IAAAA,EAAMC,SAAAA,CAAAA;AACrC,IAAA,OAAOC,aAAa,EAAA,IAAO,CAACA,QAAAA,CAASE,UAAU,CAAC,CAAC,EAAE,EAAED,qBAAAA,CAAKE,GAAG,CAAA,CAAE,CAAA,IAAK,CAACF,qBAAAA,CAAKG,UAAU,CAACJ,QAAAA,CAAAA;AACvF,CAAA;IASE,aAAA,iBAAA,8BAAA,CAAA,eAAA,CAAA,EAEA,SAAA,iBAAA,8BAAA,CAAA,WAAA,CAAA,EAEA,cAAA,iBAAA,8BAAA,CAAA,gBAAA,CAAA,EAEA,YAAA,iBAAA,8BAAA,CAAA,cAAA,CAAA,EAOA,WAAA,iBAAA,8BAAA,CAAA,aAAA,CAAA,qFAYA,SAAA,iBAAA,8BAAA,CAAA,WAAA,CAAA,EAkCM,aAAA,iBAAA,8BAAA,CAAA,eAAA,CAAA,EAmGA,mBAAA,iBAAA,8BAAA,CAAA,qBAAA,CAAA,EAuCA,kBAAA,iBAAA,8BAAA,CAAA,oBAAA,CAAA,EAKA,eAAA,iBAAA,8BAAA,CAAA,iBAAA,CAAA,EAYN,qBAAA,iBAAA,8BAAA,CAAA,uBAAA,CAAA,EAaM,2BAAA,iBAAA,8BAAA,CAAA,6BAAA,CAAA;AA1OR,MAAMJ,4BAAAA,CAAAA;IA0CJ,MAAMS,SAAAA,CAAUC,WAAgC,EAAE;QAChD,+BAAA,CAAA,IAAI,EAAC,YAAA,CAAA,CAAA,YAAA,CAAA,GAAeA,WAAAA;AACpB,QAAA,MAAMR,IAAAA,GAAO,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA;QAElB,IAAI;AACF,YAAA,MAAMS,IAAAA,GAAO,MAAMC,mBAAAA,CAAGD,IAAI,CAACT,IAAAA,CAAAA;YAC3B,IAAI,CAACS,IAAAA,CAAKE,WAAW,EAAA,EAAI;AACvB,gBAAA,MAAM,IAAIC,qCAAAA,CAA4B,CAAC,MAAM,EAAEZ,IAAAA,CAAK,qBAAqB,CAAC,CAAA;AAC5E,YAAA;YACA,MAAM,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,EAAA;AACb,QAAA,CAAA,CAAE,OAAOa,CAAAA,EAAG;AACV,YAAA,IAAIA,aAAaD,qCAAAA,EAA6B;gBAC5C,MAAMC,CAAAA;AACR,YAAA;AACA,YAAA,MAAM,IAAID,qCAAAA,CACR,CAAC,WAAW,EAAEZ,IAAAA,CAAK,oCAAoC,CAAC,CAAA;AAE5D,QAAA;AAEA,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,EAAW;AACnB,YAAA,MAAM,IAAIY,qCAAAA,CAA4B,kDAAA,CAAA;AACxC,QAAA;AACF,IAAA;AAYA,IAAA,MAAME,WAAAA,GAAc;QAClB,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,kBAAA,CAAA;AACjB,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,EAAW;YACnB,MAAM,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,EAAA;AACb,QAAA;AACA,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,IAAa,IAAA;AAC3B,IAAA;AAEA,IAAA,MAAMC,UAAAA,GAAa;QACjB,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,iBAAA,CAAA;QACjB,MAAMC,gBAAAA,GAAmB,MAAMC,cAAoB,CACjD,IAAI,CAACC,uBAAuB,EAAA,CAAA;AAG9B,QAAA,IAAIC,WAAQH,gBAAAA,CAAAA,EAAmB;AAC7B,YAAA,MAAM,IAAIJ,qCAAAA,CAA4B,iDAAA,CAAA;AACxC,QAAA;QAEA,MAAMQ,OAAAA,GAAUC,SAAM,KAAA,EAAOL,gBAAAA,CAAAA;AAC7B,QAAA,OAAOC,yBAA+B,CAACG,OAAAA,CAAAA;AACzC,IAAA;IAEA,MAAME,aAAAA,CAAcC,KAAoB,EAAE;AACxC,QAAA,IAAIA,UAAU,QAAA,EAAU;AACtB,YAAA;AACF,QAAA;AAEA,QAAA,MAAMC,aAAa,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,WAAU,QAAA,EAAU,SAAA,CAAA;AAC5C,QAAA,IAAI,CAAE,MAAMd,mBAAAA,CAAGe,UAAU,CAACD,UAAAA,CAAAA,EAAc;AACtC,YAAA,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,cAAA,CAAA,GAAiB,IAAIE,GAAAA,EAAAA;AAC1B,YAAA;AACF,QAAA;AAEA,QAAA,MAAMC,WAAW,IAAID,GAAAA,EAAAA;QACrB,MAAME,KAAAA,GAAQ,CAAC,MAAMlB,oBAAGmB,OAAO,CAACL,UAAAA,CAAU,EAAGM,IAAI,EAAA;QACjD,KAAK,MAAMC,QAAQH,KAAAA,CAAO;AACxB,YAAA,MAAMI,UAAAA,GAAa7B,qBAAAA,CAAK8B,IAAI,CAACT,UAAAA,EAAYO,IAAAA,CAAAA;YACzC,IAAI,CAAC,CAAC,MAAMrB,mBAAAA,CAAGD,IAAI,CAACuB,UAAAA,CAAU,EAAGE,MAAM,EAAA,EAAI;AACzC,gBAAA;AACF,YAAA;YAEA,IAAI;gBACFP,QAAAA,CAASQ,GAAG,CAACJ,IAAAA,EAAM,MAAM,gCAAA,IAAI,EAAC,oBAAA,kBAAA,CAAA,CAAmBA,IAAAA,CAAAA,CAAAA;AACnD,YAAA,CAAA,CAAE,OAAOK,KAAAA,EAAO;AACd,gBAAA,MAAMC,SAASD,KAAAA,YAAiBE,KAAAA,GAAQF,KAAAA,CAAMG,OAAO,GAAGC,MAAAA,CAAOJ,KAAAA,CAAAA;gBAC/D,MAAM,IAAIK,iCAAAA,CACR,CAAC,qCAAqC,EAAEV,IAAAA,CAAK,GAAG,EAAEM,MAAAA,CAAO,4DAA4D,CAAC,EACtH;AAAED,oBAAAA;AAAM,iBAAA,CAAA;AAEZ,YAAA;AACF,QAAA;QAEA,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,cAAA,CAAA,GAAiBT,QAAAA;AACxB,IAAA;IAEAe,wBAAAA,GAAqC;QACnC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,+BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,UAAA,CAAA;AACpC,IAAA;IAEAxB,uBAAAA,GAAoC;QAClC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,8BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,SAAA,CAAA;AACpC,IAAA;IAEAyB,qBAAAA,GAAkC;QAChC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,4BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,OAAA,CAAA;AACpC,IAAA;IAEAC,6BAAAA,GAA0C;QACxC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,oCAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,eAAA,CAAA;AACpC,IAAA;IAEAC,sBAAAA,GAAmC;QACjC,MAAMC,SAAAA,GAAY,IAAIC,oBAAAA,CAAY;YAAEC,UAAAA,EAAY;AAAK,SAAA,CAAA;AACrD,QAAA,MAAMxB,aAAa,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,WAAU,QAAA,EAAU,SAAA,CAAA;QAC5C,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,6BAAA,CAAA;QAEjB,+BAAA,CAAA,IAAI,EAAC,mBAAA,CAAA,CAAA,mBAAA,CAAA,CAAoBsB,WAAWtB,UAAAA,CAAAA,CAAYyB,KAAK,CAAC,CAACpC,CAAAA,GAAAA;YACrDiC,SAAAA,CAAUI,OAAO,CACfrC,CAAAA,YAAayB,KAAAA,GAAQzB,IAAI,IAAIsC,+BAAAA,CAAsBX,OAAO3B,CAAAA,CAAAA,EAAI;gBAAEuC,OAAAA,EAAS;oBAAEhB,KAAAA,EAAOvB;AAAE;AAAE,aAAA,CAAA,CAAA;AAE1F,QAAA,CAAA,CAAA;QAEA,OAAOiC,SAAAA;AACT,IAAA;AApJA,IAAA,WAAA,CAAYjD,OAA6C,CAAE;QAK3D,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,WAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAYA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,SAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAkCA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,aAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAmGA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,mBAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAuCA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,kBAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAKA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,eAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAYA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,qBAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAaA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,2BAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAnOA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,aAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,SAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,cAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,YAAA,EAAA;;mBAAA;;aAZAwD,IAAAA,GAAqB,QAAA;aAErBtB,IAAAA,GAAO,yBAAA;QAaL,IAAI,CAAClC,OAAO,GAAGA,OAAAA;QACf,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,GAAgBM,qBAAAA,CAAKmD,OAAO,CAACzD,OAAAA,CAAQ0D,SAAS,CAACpD,IAAI,CAAA;AAC1D,IAAA;AAqPF;AAnPE,SAAA,WAAYoC,OAAe,EAAA;AACzB,IAAA,+BAAA,CAAA,IAAI,EAAC,YAAA,CAAA,CAAA,YAAA,CAAA,EAAciB,MAAAA,CAAO;QACxBJ,OAAAA,EAAS;AACPK,YAAAA,SAAAA,EAAW,IAAIC,IAAAA,EAAAA;AACfnB,YAAAA,OAAAA;YACAoB,MAAAA,EAAQ;AACV,SAAA;QACAC,IAAAA,EAAM;AACR,KAAA,CAAA;AACF;AAGA,SAAA,QAAA,CAAU,GAAGC,aAAuB,EAAA;AAClC,IAAA,MAAMC,MAAAA,GAAS3D,qBAAAA,CAAK4D,KAAK,CAAC9B,IAAI,CAAA,GAAI4B,aAAAA,CAAAA;IAClC,MAAMG,QAAAA,GAAWF,MAAAA,CAAOG,KAAK,CAAC9D,qBAAAA,CAAK4D,KAAK,CAAC1D,GAAG,CAAA,CAAE6D,MAAM,CAACC,OAAAA,CAAAA;IACrD,MAAMC,QAAAA,GAAWjE,sBAAKmD,OAAO,CAAC,gCAAA,IAAI,EAAC,eAAA,aAAA,CAAA,EAAA,GAAkBU,QAAAA,CAAAA;AACrD,IAAA,IAAI,CAACjE,gBAAAA,CAAiB,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,gBAAeqE,QAAAA,CAAAA,EAAW;AACnD,QAAA,MAAM,IAAIxD,qCAAAA,CAA4B,CAAC,cAAc,EAAEkD,MAAAA,CAAO,4BAA4B,CAAC,CAAA;AAC7F,IAAA;IACA,OAAOM,QAAAA;AACT;AA0BA,eAAA,YAAA,GAAA;AACE,IAAA,MAAMC,YAAAA,GAAe,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,CAAU1E,kBAAAA,CAAAA;AACpC,IAAA,IAAI,CAAE,MAAMe,mBAAAA,CAAGe,UAAU,CAAC4C,YAAAA,CAAAA,EAAgB;AACxC,QAAA,MAAM,IAAIzD,qCAAAA,CACR,CAAC,QAAQ,EAAEjB,kBAAAA,CAAmB,sBAAsB,EAAE,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,CAAc,EAAE,CAAC,CAAA;AAEhF,IAAA;AACA,IAAA,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,aAAY,MAAMe,mBAAAA,CAAG4D,QAAQ,CAACD,YAAAA,CAAAA;AACrC;AA2FA,eAAA,kBAAA,CAA0BvB,SAAsB,EAAEtB,UAAkB,EAAA;AAClE,IAAA,IAAI,CAAE,MAAMd,mBAAAA,CAAGe,UAAU,CAACD,UAAAA,CAAAA,EAAc;AACtCsB,QAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACb,QAAA;AACF,IAAA;IAEA,MAAM3C,KAAAA,GAAQ,CAAC,MAAMlB,oBAAGmB,OAAO,CAACL,UAAAA,CAAU,EAAGM,IAAI,EAAA;IACjD,KAAK,MAAMC,QAAQH,KAAAA,CAAO;AACxB,QAAA,MAAM4C,SAAAA,GAAYrE,qBAAAA,CAAK8B,IAAI,CAACT,UAAAA,EAAYO,IAAAA,CAAAA;AACxC,QAAA,MAAMtB,IAAAA,GAAO,MAAMC,mBAAAA,CAAGD,IAAI,CAAC+D,SAAAA,CAAAA;QAC3B,IAAI/D,IAAAA,CAAKyB,MAAM,EAAA,EAAI;AACjB,YAAA,IAAIP,WAAW,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,iBAAgB8C,GAAAA,CAAI1C,IAAAA,CAAAA;AACxC,YAAA,IAAI,CAACJ,QAAAA,EAAU;gBACb,IAAI;AACFA,oBAAAA,QAAAA,GAAW,MAAM,+BAAA,CAAA,IAAI,EAAC,oBAAA,kBAAA,CAAA,CAAmBI,IAAAA,CAAAA;AAC3C,gBAAA,CAAA,CAAE,OAAOK,KAAAA,EAAO;oBACdU,SAAAA,CAAUI,OAAO,CACf,IAAIC,+BAAAA,CAAsB,CAAC,4BAA4B,EAAEpB,MAAM,EAAE;wBAC/DqB,OAAAA,EAAS;AAAEhB,4BAAAA;AAAM;AACnB,qBAAA,CAAA,CAAA;AAEF,oBAAA;AACF,gBAAA;AACF,YAAA;YAEA,MAAMsC,cAAAA,GAAiBC,yBAAmBxE,qBAAAA,CAAK4D,KAAK,CAAC9B,IAAI,CAAC,UAAU,SAAA,EAAWF,IAAAA,CAAAA,CAAAA;AAC/E,YAAA,MAAM6C,KAAAA,GAAgB;AACpBjD,gBAAAA,QAAAA;gBACAkD,QAAAA,EAAU9C,IAAAA;gBACV+C,QAAAA,EAAUJ,cAAAA;gBACVK,KAAAA,EAAO;AAAEC,oBAAAA,IAAAA,EAAMvE,KAAKuE;AAAK,iBAAA;gBACzBC,MAAAA,EAAQvE,mBAAAA,CAAGwE,gBAAgB,CAACV,SAAAA;AAC9B,aAAA;AACA,YAAA,MAAMW,yBAAMrC,SAAAA,EAAW8B,KAAAA,CAAAA;AACzB,QAAA;AACF,IAAA;AACA9B,IAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACf;AAEA,eAAA,kBAAyBM,QAAgB,EAAA;IACvC,MAAMR,YAAAA,GAAe,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,CAAU,QAAA,EAAU,UAAA,EAAY,CAAA,EAAGQ,QAAAA,CAAS,KAAK,CAAC,CAAA;AAC5E,IAAA,OAAOO,2BAAAA,CAAsB,MAAM1E,mBAAAA,CAAG4D,QAAQ,CAACD,YAAAA,CAAAA,EAAeQ,QAAAA,CAAAA;AAChE;AAEA,eAAA,eAAsBQ,WAAmB,EAAA;IACvC,MAAMC,MAAAA,GAAS,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,CAAA,GAAaD,WAAAA,CAAYpB,KAAK,CAAC,GAAA,CAAA,CAAKC,MAAM,CAACC,OAAAA,CAAAA,CAAAA;AAC/D,IAAA,IAAI,CAAE,MAAMzD,mBAAAA,CAAGe,UAAU,CAAC6D,MAAAA,CAAAA,EAAU;AAClC,QAAA,OAAO,EAAE;AACX,IAAA;AACA,IAAA,MAAM1D,KAAAA,GAAQ,MAAMlB,mBAAAA,CAAGmB,OAAO,CAACyD,MAAAA,CAAAA;AAC/B,IAAA,OAAO1D,MACJsC,MAAM,CAAC,CAACqB,CAAAA,GAAMA,CAAAA,CAAEC,QAAQ,CAAC,QAAA,CAAA,CAAA,CACzB1D,IAAI,EAAA,CACJ2D,GAAG,CAAC,CAACF,IAAMpF,qBAAAA,CAAK8B,IAAI,CAACqD,MAAAA,EAAQC,CAAAA,CAAAA,CAAAA;AAClC;AAEA,SAAA,qBAAsBF,WAAmB,EAAA;IACvC,MAAMvC,SAAAA,GAAY,IAAIC,oBAAAA,CAAY;QAAEC,UAAAA,EAAY;AAAK,KAAA,CAAA;AACrD,IAAA,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,aAAY,CAAC,qBAAqB,EAAEqC,WAAAA,CAAAA,CAAa,CAAA;IAEtD,+BAAA,CAAA,IAAI,EAAC,2BAAA,CAAA,CAAA,2BAAA,CAAA,CAA4BvC,WAAWuC,WAAAA,CAAAA,CAAapC,KAAK,CAAC,CAACpC,CAAAA,GAAAA;QAC9DiC,SAAAA,CAAUI,OAAO,CACfrC,CAAAA,YAAayB,KAAAA,GAAQzB,IAAI,IAAIsC,+BAAAA,CAAsBX,OAAO3B,CAAAA,CAAAA,EAAI;YAAEuC,OAAAA,EAAS;gBAAEhB,KAAAA,EAAOvB;AAAE;AAAE,SAAA,CAAA,CAAA;AAE1F,IAAA,CAAA,CAAA;IAEA,OAAOiC,SAAAA;AACT;AAEA,eAAA,0BAAA,CAAkCA,SAAsB,EAAEuC,WAAmB,EAAA;AAC3E,IAAA,MAAMK,QAAQ,MAAM,+BAAA,CAAA,IAAI,EAAC,iBAAA,eAAA,CAAA,CAAgBL,WAAAA,CAAAA;IACzC,KAAK,MAAMM,WAAWD,KAAAA,CAAO;AAC3B,QAAA,MAAME,UAAAA,GAAa;YACjBC,aAAAA,CAAO;gBACLC,WAAAA,EAAa;AACf,aAAA,CAAA;YACA,CAACC,IAAAA,GAAyCA,KAAKC;AAChD,SAAA;AAED,QAAA,MAAMf,SAASvE,mBAAAA,CAAGwE,gBAAgB,CAACS,OAAAA,CAAAA,CAASM,IAAI,CAACC,iBAAAA,CAAMN,UAAAA,CAAAA,CAAAA;QAEvD,IAAI;YACF,WAAW,MAAMO,SAASlB,MAAAA,CAAQ;AAChC,gBAAA,MAAME,yBAAMrC,SAAAA,EAAWqD,KAAAA,CAAAA;AACzB,YAAA;AACF,QAAA,CAAA,CAAE,OAAOtF,CAAAA,EAAY;AACnBiC,YAAAA,SAAAA,CAAUI,OAAO,CACf,IAAIC,+BAAAA,CAAsB,CAAC,uBAAuB,EAAEwC,OAAAA,CAAQ,EAAE,EAAG9E,CAAAA,CAAY0B,OAAO,EAAE,EAAE;gBACtFa,OAAAA,EAAS;oBACPhB,KAAAA,EAAOvB;AACT;AACF,aAAA,CAAA,CAAA;AAEF,YAAA;AACF,QAAA;AACF,IAAA;AACAiC,IAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACf;;;;"}
|
|
1
|
+
{"version":3,"file":"index.js","sources":["../../../../src/directory/providers/source/index.ts"],"sourcesContent":["import path from 'path';\nimport type { Readable } from 'stream';\nimport { PassThrough } from 'stream';\n\nimport fs from 'fs-extra';\nimport { isEmpty, keyBy } from 'lodash';\nimport { chain } from 'stream-chain';\nimport { parser } from 'stream-json/jsonl/Parser';\nimport type { Struct } from '@strapi/types';\n\nimport type {\n IAsset,\n IMetadata,\n ISourceProvider,\n ProviderType,\n TransferStage,\n} from '../../../types';\nimport type { IDiagnosticReporter } from '../../../utils/diagnostic';\n\nimport * as utils from '../../../utils';\nimport { write } from '../../../utils/writable-async-write';\nimport {\n ProviderInitializationError,\n ProviderTransferError,\n ProviderValidationError,\n} from '../../../errors/providers';\nimport { unknownPathToPosix, validateAssetMetadata } from '../../../file/providers/source/utils';\n\nconst METADATA_FILE_PATH = 'metadata.json';\n\nexport interface ILocalDirectorySourceProviderOptions {\n directory: {\n /** Absolute or relative path to the root of an unpacked Strapi data-transfer export */\n path: string;\n };\n}\n\nexport const createLocalDirectorySourceProvider = (\n options: ILocalDirectorySourceProviderOptions\n) => {\n return new LocalDirectorySourceProvider(options);\n};\n\nconst isPathInsideRoot = (root: string, candidate: string): boolean => {\n const relative = path.relative(root, candidate);\n return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));\n};\n\nclass LocalDirectorySourceProvider implements ISourceProvider {\n type: ProviderType = 'source';\n\n name = 'source::local-directory';\n\n options: ILocalDirectorySourceProviderOptions;\n\n #rootResolved: string;\n\n #metadata?: IMetadata;\n\n #assetMetadata?: Map<string, IAsset['metadata']>;\n\n #diagnostics?: IDiagnosticReporter;\n\n constructor(options: ILocalDirectorySourceProviderOptions) {\n this.options = options;\n this.#rootResolved = path.resolve(options.directory.path);\n }\n\n #reportInfo(message: string) {\n this.#diagnostics?.report({\n details: {\n createdAt: new Date(),\n message,\n origin: 'directory-source-provider',\n },\n kind: 'info',\n });\n }\n\n /** Resolve a posix-style relative path under the export root; rejects escapes. */\n #safePath(...posixSegments: string[]): string {\n const joined = path.posix.join(...posixSegments);\n const segments = joined.split(path.posix.sep).filter(Boolean);\n const resolved = path.resolve(this.#rootResolved, ...segments);\n if (!isPathInsideRoot(this.#rootResolved, resolved)) {\n throw new ProviderInitializationError(`Invalid path \"${joined}\" — escapes backup directory`);\n }\n return resolved;\n }\n\n async bootstrap(diagnostics: IDiagnosticReporter) {\n this.#diagnostics = diagnostics;\n const root = this.#rootResolved;\n\n try {\n const stat = await fs.stat(root);\n if (!stat.isDirectory()) {\n throw new ProviderInitializationError(`Path '${root}' is not a directory.`);\n }\n await this.#loadMetadata();\n } catch (e) {\n if (e instanceof ProviderInitializationError) {\n throw e;\n }\n throw new ProviderInitializationError(\n `Directory '${root}' is not a valid Strapi data export.`\n );\n }\n\n if (!this.#metadata) {\n throw new ProviderInitializationError('Could not load metadata from Strapi data export.');\n }\n }\n\n async #loadMetadata() {\n const metadataPath = this.#safePath(METADATA_FILE_PATH);\n if (!(await fs.pathExists(metadataPath))) {\n throw new ProviderInitializationError(\n `Missing ${METADATA_FILE_PATH} in export directory '${this.#rootResolved}'.`\n );\n }\n this.#metadata = await fs.readJson(metadataPath);\n }\n\n async getMetadata() {\n this.#reportInfo('getting metadata');\n if (!this.#metadata) {\n await this.#loadMetadata();\n }\n return this.#metadata ?? null;\n }\n\n async getSchemas() {\n this.#reportInfo('getting schemas');\n const schemaCollection = await utils.stream.collect<Struct.Schema>(\n this.createSchemasReadStream()\n );\n\n if (isEmpty(schemaCollection)) {\n throw new ProviderInitializationError('Could not load schemas from Strapi data export.');\n }\n\n const schemas = keyBy(schemaCollection, 'uid');\n return utils.schema.schemasToValidJSON(schemas);\n }\n\n async validateStage(stage: TransferStage) {\n if (stage !== 'assets') {\n return;\n }\n\n const uploadsDir = this.#safePath('assets', 'uploads');\n if (!(await fs.pathExists(uploadsDir))) {\n this.#assetMetadata = new Map();\n return;\n }\n\n const metadata = new Map<string, IAsset['metadata']>();\n const names = (await fs.readdir(uploadsDir)).sort();\n for (const name of names) {\n const uploadPath = path.join(uploadsDir, name);\n if (!(await fs.stat(uploadPath)).isFile()) {\n continue;\n }\n\n try {\n metadata.set(name, await this.#readAssetMetadata(name));\n } catch (error) {\n const reason = error instanceof Error ? error.message : String(error);\n throw new ProviderValidationError(\n `Asset metadata preflight failed for \"${name}\": ${reason}. The destination was not modified; re-export and try again.`,\n { error }\n );\n }\n }\n\n this.#assetMetadata = metadata;\n }\n\n createEntitiesReadStream(): Readable {\n this.#reportInfo('creating entities read stream');\n return this.#streamJsonlDirectory('entities');\n }\n\n createSchemasReadStream(): Readable {\n this.#reportInfo('creating schemas read stream');\n return this.#streamJsonlDirectory('schemas');\n }\n\n createLinksReadStream(): Readable {\n this.#reportInfo('creating links read stream');\n return this.#streamJsonlDirectory('links');\n }\n\n createConfigurationReadStream(): Readable {\n this.#reportInfo('creating configuration read stream');\n return this.#streamJsonlDirectory('configuration');\n }\n\n createAssetsReadStream(): Readable {\n const outStream = new PassThrough({ objectMode: true });\n const uploadsDir = this.#safePath('assets', 'uploads');\n this.#reportInfo('creating assets read stream');\n\n this.#pipeAssetsToStream(outStream, uploadsDir).catch((e: unknown) => {\n outStream.destroy(\n e instanceof Error ? e : new ProviderTransferError(String(e), { details: { error: e } })\n );\n });\n\n return outStream;\n }\n\n async #pipeAssetsToStream(outStream: PassThrough, uploadsDir: string): Promise<void> {\n if (!(await fs.pathExists(uploadsDir))) {\n outStream.end();\n return;\n }\n\n const names = (await fs.readdir(uploadsDir)).sort();\n for (const name of names) {\n const absUpload = path.join(uploadsDir, name);\n const stat = await fs.stat(absUpload);\n if (stat.isFile()) {\n let metadata = this.#assetMetadata?.get(name);\n if (!metadata) {\n try {\n metadata = await this.#readAssetMetadata(name);\n } catch (error) {\n outStream.destroy(\n new ProviderTransferError(`Failed to read metadata for ${name}`, {\n details: { error },\n })\n );\n return;\n }\n }\n\n const normalizedPath = unknownPathToPosix(path.posix.join('assets', 'uploads', name));\n const asset: IAsset = {\n metadata,\n filename: name,\n filepath: normalizedPath,\n stats: { size: stat.size },\n stream: fs.createReadStream(absUpload),\n };\n await write(outStream, asset);\n }\n }\n outStream.end();\n }\n\n async #readAssetMetadata(filename: string): Promise<IAsset['metadata']> {\n const metadataPath = this.#safePath('assets', 'metadata', `${filename}.json`);\n return validateAssetMetadata(await fs.readJson(metadataPath), filename);\n }\n\n async #listJsonlFiles(posixSubdir: string): Promise<string[]> {\n const dirAbs = this.#safePath(...posixSubdir.split('/').filter(Boolean));\n if (!(await fs.pathExists(dirAbs))) {\n return [];\n }\n const names = await fs.readdir(dirAbs);\n return names\n .filter((n) => n.endsWith('.jsonl'))\n .sort()\n .map((n) => path.join(dirAbs, n));\n }\n\n #streamJsonlDirectory(posixSubdir: string): Readable {\n const outStream = new PassThrough({ objectMode: true });\n this.#reportInfo(`streaming jsonl from ${posixSubdir}`);\n\n this.#pipeJsonlDirectoryToStream(outStream, posixSubdir).catch((e: unknown) => {\n outStream.destroy(\n e instanceof Error ? e : new ProviderTransferError(String(e), { details: { error: e } })\n );\n });\n\n return outStream;\n }\n\n async #pipeJsonlDirectoryToStream(outStream: PassThrough, posixSubdir: string): Promise<void> {\n const files = await this.#listJsonlFiles(posixSubdir);\n for (const absPath of files) {\n const transforms = [\n parser({\n checkErrors: true,\n }),\n (line: { key: string; value: object }) => line.value,\n ];\n\n const stream = fs.createReadStream(absPath).pipe(chain(transforms));\n\n try {\n for await (const chunk of stream) {\n await write(outStream, chunk);\n }\n } catch (e: unknown) {\n outStream.destroy(\n new ProviderTransferError(`Error parsing JSONL in ${absPath}: ${(e as Error).message}`, {\n details: {\n error: e,\n },\n })\n );\n return;\n }\n }\n outStream.end();\n }\n}\n"],"names":["METADATA_FILE_PATH","createLocalDirectorySourceProvider","options","LocalDirectorySourceProvider","isPathInsideRoot","root","candidate","relative","path","startsWith","sep","isAbsolute","bootstrap","diagnostics","stat","fs","isDirectory","ProviderInitializationError","e","getMetadata","getSchemas","schemaCollection","utils","createSchemasReadStream","isEmpty","schemas","keyBy","validateStage","stage","uploadsDir","pathExists","Map","metadata","names","readdir","sort","name","uploadPath","join","isFile","set","error","reason","Error","message","String","ProviderValidationError","createEntitiesReadStream","createLinksReadStream","createConfigurationReadStream","createAssetsReadStream","outStream","PassThrough","objectMode","catch","destroy","ProviderTransferError","details","type","resolve","directory","report","createdAt","Date","origin","kind","posixSegments","joined","posix","segments","split","filter","Boolean","resolved","metadataPath","readJson","end","absUpload","get","normalizedPath","unknownPathToPosix","asset","filename","filepath","stats","size","stream","createReadStream","write","validateAssetMetadata","posixSubdir","dirAbs","n","endsWith","map","files","absPath","transforms","parser","checkErrors","line","value","pipe","chain","chunk"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAMA,kBAAAA,GAAqB,eAAA;AASpB,MAAMC,qCAAqC,CAChDC,OAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,4BAAAA,CAA6BD,OAAAA,CAAAA;AAC1C;AAEA,MAAME,gBAAAA,GAAmB,CAACC,IAAAA,EAAcC,SAAAA,GAAAA;AACtC,IAAA,MAAMC,QAAAA,GAAWC,qBAAAA,CAAKD,QAAQ,CAACF,IAAAA,EAAMC,SAAAA,CAAAA;AACrC,IAAA,OAAOC,aAAa,EAAA,IAAO,CAACA,QAAAA,CAASE,UAAU,CAAC,CAAC,EAAE,EAAED,qBAAAA,CAAKE,GAAG,CAAA,CAAE,CAAA,IAAK,CAACF,qBAAAA,CAAKG,UAAU,CAACJ,QAAAA,CAAAA;AACvF,CAAA;IASE,aAAA,iBAAA,8BAAA,CAAA,eAAA,CAAA,EAEA,SAAA,iBAAA,8BAAA,CAAA,WAAA,CAAA,EAEA,cAAA,iBAAA,8BAAA,CAAA,gBAAA,CAAA,EAEA,YAAA,iBAAA,8BAAA,CAAA,cAAA,CAAA,EAOA,WAAA,iBAAA,8BAAA,CAAA,aAAA,CAAA,qFAYA,SAAA,iBAAA,8BAAA,CAAA,WAAA,CAAA,EAkCM,aAAA,iBAAA,8BAAA,CAAA,eAAA,CAAA,EAmGA,mBAAA,iBAAA,8BAAA,CAAA,qBAAA,CAAA,EAuCA,kBAAA,iBAAA,8BAAA,CAAA,oBAAA,CAAA,EAKA,eAAA,iBAAA,8BAAA,CAAA,iBAAA,CAAA,EAYN,qBAAA,iBAAA,8BAAA,CAAA,uBAAA,CAAA,EAaM,2BAAA,iBAAA,8BAAA,CAAA,6BAAA,CAAA;AA1OR,MAAMJ,4BAAAA,CAAAA;IA0CJ,MAAMS,SAAAA,CAAUC,WAAgC,EAAE;QAChD,+BAAA,CAAA,IAAI,EAAC,YAAA,CAAA,CAAA,YAAA,CAAA,GAAeA,WAAAA;AACpB,QAAA,MAAMR,IAAAA,GAAO,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA;QAElB,IAAI;AACF,YAAA,MAAMS,IAAAA,GAAO,MAAMC,mBAAAA,CAAGD,IAAI,CAACT,IAAAA,CAAAA;YAC3B,IAAI,CAACS,IAAAA,CAAKE,WAAW,EAAA,EAAI;AACvB,gBAAA,MAAM,IAAIC,qCAAAA,CAA4B,CAAC,MAAM,EAAEZ,IAAAA,CAAK,qBAAqB,CAAC,CAAA;AAC5E,YAAA;YACA,MAAM,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,EAAA;AACb,QAAA,CAAA,CAAE,OAAOa,CAAAA,EAAG;AACV,YAAA,IAAIA,aAAaD,qCAAAA,EAA6B;gBAC5C,MAAMC,CAAAA;AACR,YAAA;AACA,YAAA,MAAM,IAAID,qCAAAA,CACR,CAAC,WAAW,EAAEZ,IAAAA,CAAK,oCAAoC,CAAC,CAAA;AAE5D,QAAA;AAEA,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,EAAW;AACnB,YAAA,MAAM,IAAIY,qCAAAA,CAA4B,kDAAA,CAAA;AACxC,QAAA;AACF,IAAA;AAYA,IAAA,MAAME,WAAAA,GAAc;QAClB,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,kBAAA,CAAA;AACjB,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,EAAW;YACnB,MAAM,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,EAAA;AACb,QAAA;AACA,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,IAAa,IAAA;AAC3B,IAAA;AAEA,IAAA,MAAMC,UAAAA,GAAa;QACjB,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,iBAAA,CAAA;QACjB,MAAMC,gBAAAA,GAAmB,MAAMC,cAAoB,CACjD,IAAI,CAACC,uBAAuB,EAAA,CAAA;AAG9B,QAAA,IAAIC,UAAQH,gBAAAA,CAAAA,EAAmB;AAC7B,YAAA,MAAM,IAAIJ,qCAAAA,CAA4B,iDAAA,CAAA;AACxC,QAAA;QAEA,MAAMQ,OAAAA,GAAUC,QAAML,gBAAAA,EAAkB,KAAA,CAAA;AACxC,QAAA,OAAOC,yBAA+B,CAACG,OAAAA,CAAAA;AACzC,IAAA;IAEA,MAAME,aAAAA,CAAcC,KAAoB,EAAE;AACxC,QAAA,IAAIA,UAAU,QAAA,EAAU;AACtB,YAAA;AACF,QAAA;AAEA,QAAA,MAAMC,aAAa,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,WAAU,QAAA,EAAU,SAAA,CAAA;AAC5C,QAAA,IAAI,CAAE,MAAMd,mBAAAA,CAAGe,UAAU,CAACD,UAAAA,CAAAA,EAAc;AACtC,YAAA,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,cAAA,CAAA,GAAiB,IAAIE,GAAAA,EAAAA;AAC1B,YAAA;AACF,QAAA;AAEA,QAAA,MAAMC,WAAW,IAAID,GAAAA,EAAAA;QACrB,MAAME,KAAAA,GAAQ,CAAC,MAAMlB,oBAAGmB,OAAO,CAACL,UAAAA,CAAU,EAAGM,IAAI,EAAA;QACjD,KAAK,MAAMC,QAAQH,KAAAA,CAAO;AACxB,YAAA,MAAMI,UAAAA,GAAa7B,qBAAAA,CAAK8B,IAAI,CAACT,UAAAA,EAAYO,IAAAA,CAAAA;YACzC,IAAI,CAAC,CAAC,MAAMrB,mBAAAA,CAAGD,IAAI,CAACuB,UAAAA,CAAU,EAAGE,MAAM,EAAA,EAAI;AACzC,gBAAA;AACF,YAAA;YAEA,IAAI;gBACFP,QAAAA,CAASQ,GAAG,CAACJ,IAAAA,EAAM,MAAM,gCAAA,IAAI,EAAC,oBAAA,kBAAA,CAAA,CAAmBA,IAAAA,CAAAA,CAAAA;AACnD,YAAA,CAAA,CAAE,OAAOK,KAAAA,EAAO;AACd,gBAAA,MAAMC,SAASD,KAAAA,YAAiBE,KAAAA,GAAQF,KAAAA,CAAMG,OAAO,GAAGC,MAAAA,CAAOJ,KAAAA,CAAAA;gBAC/D,MAAM,IAAIK,iCAAAA,CACR,CAAC,qCAAqC,EAAEV,IAAAA,CAAK,GAAG,EAAEM,MAAAA,CAAO,4DAA4D,CAAC,EACtH;AAAED,oBAAAA;AAAM,iBAAA,CAAA;AAEZ,YAAA;AACF,QAAA;QAEA,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,cAAA,CAAA,GAAiBT,QAAAA;AACxB,IAAA;IAEAe,wBAAAA,GAAqC;QACnC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,+BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,UAAA,CAAA;AACpC,IAAA;IAEAxB,uBAAAA,GAAoC;QAClC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,8BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,SAAA,CAAA;AACpC,IAAA;IAEAyB,qBAAAA,GAAkC;QAChC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,4BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,OAAA,CAAA;AACpC,IAAA;IAEAC,6BAAAA,GAA0C;QACxC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,oCAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,eAAA,CAAA;AACpC,IAAA;IAEAC,sBAAAA,GAAmC;QACjC,MAAMC,SAAAA,GAAY,IAAIC,oBAAAA,CAAY;YAAEC,UAAAA,EAAY;AAAK,SAAA,CAAA;AACrD,QAAA,MAAMxB,aAAa,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,WAAU,QAAA,EAAU,SAAA,CAAA;QAC5C,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,6BAAA,CAAA;QAEjB,+BAAA,CAAA,IAAI,EAAC,mBAAA,CAAA,CAAA,mBAAA,CAAA,CAAoBsB,WAAWtB,UAAAA,CAAAA,CAAYyB,KAAK,CAAC,CAACpC,CAAAA,GAAAA;YACrDiC,SAAAA,CAAUI,OAAO,CACfrC,CAAAA,YAAayB,KAAAA,GAAQzB,IAAI,IAAIsC,+BAAAA,CAAsBX,OAAO3B,CAAAA,CAAAA,EAAI;gBAAEuC,OAAAA,EAAS;oBAAEhB,KAAAA,EAAOvB;AAAE;AAAE,aAAA,CAAA,CAAA;AAE1F,QAAA,CAAA,CAAA;QAEA,OAAOiC,SAAAA;AACT,IAAA;AApJA,IAAA,WAAA,CAAYjD,OAA6C,CAAE;QAK3D,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,WAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAYA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,SAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAkCA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,aAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAmGA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,mBAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAuCA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,kBAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAKA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,eAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAYA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,qBAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAaA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,2BAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAnOA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,aAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,SAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,cAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,YAAA,EAAA;;mBAAA;;aAZAwD,IAAAA,GAAqB,QAAA;aAErBtB,IAAAA,GAAO,yBAAA;QAaL,IAAI,CAAClC,OAAO,GAAGA,OAAAA;QACf,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,GAAgBM,qBAAAA,CAAKmD,OAAO,CAACzD,OAAAA,CAAQ0D,SAAS,CAACpD,IAAI,CAAA;AAC1D,IAAA;AAqPF;AAnPE,SAAA,WAAYoC,OAAe,EAAA;AACzB,IAAA,+BAAA,CAAA,IAAI,EAAC,YAAA,CAAA,CAAA,YAAA,CAAA,EAAciB,MAAAA,CAAO;QACxBJ,OAAAA,EAAS;AACPK,YAAAA,SAAAA,EAAW,IAAIC,IAAAA,EAAAA;AACfnB,YAAAA,OAAAA;YACAoB,MAAAA,EAAQ;AACV,SAAA;QACAC,IAAAA,EAAM;AACR,KAAA,CAAA;AACF;AAGA,SAAA,QAAA,CAAU,GAAGC,aAAuB,EAAA;AAClC,IAAA,MAAMC,MAAAA,GAAS3D,qBAAAA,CAAK4D,KAAK,CAAC9B,IAAI,CAAA,GAAI4B,aAAAA,CAAAA;IAClC,MAAMG,QAAAA,GAAWF,MAAAA,CAAOG,KAAK,CAAC9D,qBAAAA,CAAK4D,KAAK,CAAC1D,GAAG,CAAA,CAAE6D,MAAM,CAACC,OAAAA,CAAAA;IACrD,MAAMC,QAAAA,GAAWjE,sBAAKmD,OAAO,CAAC,gCAAA,IAAI,EAAC,eAAA,aAAA,CAAA,EAAA,GAAkBU,QAAAA,CAAAA;AACrD,IAAA,IAAI,CAACjE,gBAAAA,CAAiB,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,gBAAeqE,QAAAA,CAAAA,EAAW;AACnD,QAAA,MAAM,IAAIxD,qCAAAA,CAA4B,CAAC,cAAc,EAAEkD,MAAAA,CAAO,4BAA4B,CAAC,CAAA;AAC7F,IAAA;IACA,OAAOM,QAAAA;AACT;AA0BA,eAAA,YAAA,GAAA;AACE,IAAA,MAAMC,YAAAA,GAAe,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,CAAU1E,kBAAAA,CAAAA;AACpC,IAAA,IAAI,CAAE,MAAMe,mBAAAA,CAAGe,UAAU,CAAC4C,YAAAA,CAAAA,EAAgB;AACxC,QAAA,MAAM,IAAIzD,qCAAAA,CACR,CAAC,QAAQ,EAAEjB,kBAAAA,CAAmB,sBAAsB,EAAE,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,CAAc,EAAE,CAAC,CAAA;AAEhF,IAAA;AACA,IAAA,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,aAAY,MAAMe,mBAAAA,CAAG4D,QAAQ,CAACD,YAAAA,CAAAA;AACrC;AA2FA,eAAA,kBAAA,CAA0BvB,SAAsB,EAAEtB,UAAkB,EAAA;AAClE,IAAA,IAAI,CAAE,MAAMd,mBAAAA,CAAGe,UAAU,CAACD,UAAAA,CAAAA,EAAc;AACtCsB,QAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACb,QAAA;AACF,IAAA;IAEA,MAAM3C,KAAAA,GAAQ,CAAC,MAAMlB,oBAAGmB,OAAO,CAACL,UAAAA,CAAU,EAAGM,IAAI,EAAA;IACjD,KAAK,MAAMC,QAAQH,KAAAA,CAAO;AACxB,QAAA,MAAM4C,SAAAA,GAAYrE,qBAAAA,CAAK8B,IAAI,CAACT,UAAAA,EAAYO,IAAAA,CAAAA;AACxC,QAAA,MAAMtB,IAAAA,GAAO,MAAMC,mBAAAA,CAAGD,IAAI,CAAC+D,SAAAA,CAAAA;QAC3B,IAAI/D,IAAAA,CAAKyB,MAAM,EAAA,EAAI;AACjB,YAAA,IAAIP,WAAW,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,iBAAgB8C,GAAAA,CAAI1C,IAAAA,CAAAA;AACxC,YAAA,IAAI,CAACJ,QAAAA,EAAU;gBACb,IAAI;AACFA,oBAAAA,QAAAA,GAAW,MAAM,+BAAA,CAAA,IAAI,EAAC,oBAAA,kBAAA,CAAA,CAAmBI,IAAAA,CAAAA;AAC3C,gBAAA,CAAA,CAAE,OAAOK,KAAAA,EAAO;oBACdU,SAAAA,CAAUI,OAAO,CACf,IAAIC,+BAAAA,CAAsB,CAAC,4BAA4B,EAAEpB,MAAM,EAAE;wBAC/DqB,OAAAA,EAAS;AAAEhB,4BAAAA;AAAM;AACnB,qBAAA,CAAA,CAAA;AAEF,oBAAA;AACF,gBAAA;AACF,YAAA;YAEA,MAAMsC,cAAAA,GAAiBC,yBAAmBxE,qBAAAA,CAAK4D,KAAK,CAAC9B,IAAI,CAAC,UAAU,SAAA,EAAWF,IAAAA,CAAAA,CAAAA;AAC/E,YAAA,MAAM6C,KAAAA,GAAgB;AACpBjD,gBAAAA,QAAAA;gBACAkD,QAAAA,EAAU9C,IAAAA;gBACV+C,QAAAA,EAAUJ,cAAAA;gBACVK,KAAAA,EAAO;AAAEC,oBAAAA,IAAAA,EAAMvE,KAAKuE;AAAK,iBAAA;gBACzBC,MAAAA,EAAQvE,mBAAAA,CAAGwE,gBAAgB,CAACV,SAAAA;AAC9B,aAAA;AACA,YAAA,MAAMW,yBAAMrC,SAAAA,EAAW8B,KAAAA,CAAAA;AACzB,QAAA;AACF,IAAA;AACA9B,IAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACf;AAEA,eAAA,kBAAyBM,QAAgB,EAAA;IACvC,MAAMR,YAAAA,GAAe,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,CAAU,QAAA,EAAU,UAAA,EAAY,CAAA,EAAGQ,QAAAA,CAAS,KAAK,CAAC,CAAA;AAC5E,IAAA,OAAOO,2BAAAA,CAAsB,MAAM1E,mBAAAA,CAAG4D,QAAQ,CAACD,YAAAA,CAAAA,EAAeQ,QAAAA,CAAAA;AAChE;AAEA,eAAA,eAAsBQ,WAAmB,EAAA;IACvC,MAAMC,MAAAA,GAAS,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,CAAA,GAAaD,WAAAA,CAAYpB,KAAK,CAAC,GAAA,CAAA,CAAKC,MAAM,CAACC,OAAAA,CAAAA,CAAAA;AAC/D,IAAA,IAAI,CAAE,MAAMzD,mBAAAA,CAAGe,UAAU,CAAC6D,MAAAA,CAAAA,EAAU;AAClC,QAAA,OAAO,EAAE;AACX,IAAA;AACA,IAAA,MAAM1D,KAAAA,GAAQ,MAAMlB,mBAAAA,CAAGmB,OAAO,CAACyD,MAAAA,CAAAA;AAC/B,IAAA,OAAO1D,MACJsC,MAAM,CAAC,CAACqB,CAAAA,GAAMA,CAAAA,CAAEC,QAAQ,CAAC,QAAA,CAAA,CAAA,CACzB1D,IAAI,EAAA,CACJ2D,GAAG,CAAC,CAACF,IAAMpF,qBAAAA,CAAK8B,IAAI,CAACqD,MAAAA,EAAQC,CAAAA,CAAAA,CAAAA;AAClC;AAEA,SAAA,qBAAsBF,WAAmB,EAAA;IACvC,MAAMvC,SAAAA,GAAY,IAAIC,oBAAAA,CAAY;QAAEC,UAAAA,EAAY;AAAK,KAAA,CAAA;AACrD,IAAA,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,aAAY,CAAC,qBAAqB,EAAEqC,WAAAA,CAAAA,CAAa,CAAA;IAEtD,+BAAA,CAAA,IAAI,EAAC,2BAAA,CAAA,CAAA,2BAAA,CAAA,CAA4BvC,WAAWuC,WAAAA,CAAAA,CAAapC,KAAK,CAAC,CAACpC,CAAAA,GAAAA;QAC9DiC,SAAAA,CAAUI,OAAO,CACfrC,CAAAA,YAAayB,KAAAA,GAAQzB,IAAI,IAAIsC,+BAAAA,CAAsBX,OAAO3B,CAAAA,CAAAA,EAAI;YAAEuC,OAAAA,EAAS;gBAAEhB,KAAAA,EAAOvB;AAAE;AAAE,SAAA,CAAA,CAAA;AAE1F,IAAA,CAAA,CAAA;IAEA,OAAOiC,SAAAA;AACT;AAEA,eAAA,0BAAA,CAAkCA,SAAsB,EAAEuC,WAAmB,EAAA;AAC3E,IAAA,MAAMK,QAAQ,MAAM,+BAAA,CAAA,IAAI,EAAC,iBAAA,eAAA,CAAA,CAAgBL,WAAAA,CAAAA;IACzC,KAAK,MAAMM,WAAWD,KAAAA,CAAO;AAC3B,QAAA,MAAME,UAAAA,GAAa;YACjBC,aAAAA,CAAO;gBACLC,WAAAA,EAAa;AACf,aAAA,CAAA;YACA,CAACC,IAAAA,GAAyCA,KAAKC;AAChD,SAAA;AAED,QAAA,MAAMf,SAASvE,mBAAAA,CAAGwE,gBAAgB,CAACS,OAAAA,CAAAA,CAASM,IAAI,CAACC,iBAAAA,CAAMN,UAAAA,CAAAA,CAAAA;QAEvD,IAAI;YACF,WAAW,MAAMO,SAASlB,MAAAA,CAAQ;AAChC,gBAAA,MAAME,yBAAMrC,SAAAA,EAAWqD,KAAAA,CAAAA;AACzB,YAAA;AACF,QAAA,CAAA,CAAE,OAAOtF,CAAAA,EAAY;AACnBiC,YAAAA,SAAAA,CAAUI,OAAO,CACf,IAAIC,+BAAAA,CAAsB,CAAC,uBAAuB,EAAEwC,OAAAA,CAAQ,EAAE,EAAG9E,CAAAA,CAAY0B,OAAO,EAAE,EAAE;gBACtFa,OAAAA,EAAS;oBACPhB,KAAAA,EAAOvB;AACT;AACF,aAAA,CAAA,CAAA;AAEF,YAAA;AACF,QAAA;AACF,IAAA;AACAiC,IAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACf;;;;"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
import { PassThrough } from 'stream';
|
|
3
3
|
import fs__default from 'fs-extra';
|
|
4
|
-
import { isEmpty, keyBy } from 'lodash
|
|
4
|
+
import { isEmpty, keyBy } from 'lodash';
|
|
5
5
|
import { chain } from 'stream-chain';
|
|
6
6
|
import { parser } from 'stream-json/jsonl/Parser';
|
|
7
7
|
import 'crypto';
|
|
@@ -64,7 +64,7 @@ class LocalDirectorySourceProvider {
|
|
|
64
64
|
if (isEmpty(schemaCollection)) {
|
|
65
65
|
throw new ProviderInitializationError('Could not load schemas from Strapi data export.');
|
|
66
66
|
}
|
|
67
|
-
const schemas = keyBy('uid'
|
|
67
|
+
const schemas = keyBy(schemaCollection, 'uid');
|
|
68
68
|
return schemasToValidJSON(schemas);
|
|
69
69
|
}
|
|
70
70
|
async validateStage(stage) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","sources":["../../../../src/directory/providers/source/index.ts"],"sourcesContent":["import path from 'path';\nimport type { Readable } from 'stream';\nimport { PassThrough } from 'stream';\n\nimport fs from 'fs-extra';\nimport { isEmpty, keyBy } from 'lodash/fp';\nimport { chain } from 'stream-chain';\nimport { parser } from 'stream-json/jsonl/Parser';\nimport type { Struct } from '@strapi/types';\n\nimport type {\n IAsset,\n IMetadata,\n ISourceProvider,\n ProviderType,\n TransferStage,\n} from '../../../types';\nimport type { IDiagnosticReporter } from '../../../utils/diagnostic';\n\nimport * as utils from '../../../utils';\nimport { write } from '../../../utils/writable-async-write';\nimport {\n ProviderInitializationError,\n ProviderTransferError,\n ProviderValidationError,\n} from '../../../errors/providers';\nimport { unknownPathToPosix, validateAssetMetadata } from '../../../file/providers/source/utils';\n\nconst METADATA_FILE_PATH = 'metadata.json';\n\nexport interface ILocalDirectorySourceProviderOptions {\n directory: {\n /** Absolute or relative path to the root of an unpacked Strapi data-transfer export */\n path: string;\n };\n}\n\nexport const createLocalDirectorySourceProvider = (\n options: ILocalDirectorySourceProviderOptions\n) => {\n return new LocalDirectorySourceProvider(options);\n};\n\nconst isPathInsideRoot = (root: string, candidate: string): boolean => {\n const relative = path.relative(root, candidate);\n return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));\n};\n\nclass LocalDirectorySourceProvider implements ISourceProvider {\n type: ProviderType = 'source';\n\n name = 'source::local-directory';\n\n options: ILocalDirectorySourceProviderOptions;\n\n #rootResolved: string;\n\n #metadata?: IMetadata;\n\n #assetMetadata?: Map<string, IAsset['metadata']>;\n\n #diagnostics?: IDiagnosticReporter;\n\n constructor(options: ILocalDirectorySourceProviderOptions) {\n this.options = options;\n this.#rootResolved = path.resolve(options.directory.path);\n }\n\n #reportInfo(message: string) {\n this.#diagnostics?.report({\n details: {\n createdAt: new Date(),\n message,\n origin: 'directory-source-provider',\n },\n kind: 'info',\n });\n }\n\n /** Resolve a posix-style relative path under the export root; rejects escapes. */\n #safePath(...posixSegments: string[]): string {\n const joined = path.posix.join(...posixSegments);\n const segments = joined.split(path.posix.sep).filter(Boolean);\n const resolved = path.resolve(this.#rootResolved, ...segments);\n if (!isPathInsideRoot(this.#rootResolved, resolved)) {\n throw new ProviderInitializationError(`Invalid path \"${joined}\" — escapes backup directory`);\n }\n return resolved;\n }\n\n async bootstrap(diagnostics: IDiagnosticReporter) {\n this.#diagnostics = diagnostics;\n const root = this.#rootResolved;\n\n try {\n const stat = await fs.stat(root);\n if (!stat.isDirectory()) {\n throw new ProviderInitializationError(`Path '${root}' is not a directory.`);\n }\n await this.#loadMetadata();\n } catch (e) {\n if (e instanceof ProviderInitializationError) {\n throw e;\n }\n throw new ProviderInitializationError(\n `Directory '${root}' is not a valid Strapi data export.`\n );\n }\n\n if (!this.#metadata) {\n throw new ProviderInitializationError('Could not load metadata from Strapi data export.');\n }\n }\n\n async #loadMetadata() {\n const metadataPath = this.#safePath(METADATA_FILE_PATH);\n if (!(await fs.pathExists(metadataPath))) {\n throw new ProviderInitializationError(\n `Missing ${METADATA_FILE_PATH} in export directory '${this.#rootResolved}'.`\n );\n }\n this.#metadata = await fs.readJson(metadataPath);\n }\n\n async getMetadata() {\n this.#reportInfo('getting metadata');\n if (!this.#metadata) {\n await this.#loadMetadata();\n }\n return this.#metadata ?? null;\n }\n\n async getSchemas() {\n this.#reportInfo('getting schemas');\n const schemaCollection = await utils.stream.collect<Struct.Schema>(\n this.createSchemasReadStream()\n );\n\n if (isEmpty(schemaCollection)) {\n throw new ProviderInitializationError('Could not load schemas from Strapi data export.');\n }\n\n const schemas = keyBy('uid', schemaCollection);\n return utils.schema.schemasToValidJSON(schemas);\n }\n\n async validateStage(stage: TransferStage) {\n if (stage !== 'assets') {\n return;\n }\n\n const uploadsDir = this.#safePath('assets', 'uploads');\n if (!(await fs.pathExists(uploadsDir))) {\n this.#assetMetadata = new Map();\n return;\n }\n\n const metadata = new Map<string, IAsset['metadata']>();\n const names = (await fs.readdir(uploadsDir)).sort();\n for (const name of names) {\n const uploadPath = path.join(uploadsDir, name);\n if (!(await fs.stat(uploadPath)).isFile()) {\n continue;\n }\n\n try {\n metadata.set(name, await this.#readAssetMetadata(name));\n } catch (error) {\n const reason = error instanceof Error ? error.message : String(error);\n throw new ProviderValidationError(\n `Asset metadata preflight failed for \"${name}\": ${reason}. The destination was not modified; re-export and try again.`,\n { error }\n );\n }\n }\n\n this.#assetMetadata = metadata;\n }\n\n createEntitiesReadStream(): Readable {\n this.#reportInfo('creating entities read stream');\n return this.#streamJsonlDirectory('entities');\n }\n\n createSchemasReadStream(): Readable {\n this.#reportInfo('creating schemas read stream');\n return this.#streamJsonlDirectory('schemas');\n }\n\n createLinksReadStream(): Readable {\n this.#reportInfo('creating links read stream');\n return this.#streamJsonlDirectory('links');\n }\n\n createConfigurationReadStream(): Readable {\n this.#reportInfo('creating configuration read stream');\n return this.#streamJsonlDirectory('configuration');\n }\n\n createAssetsReadStream(): Readable {\n const outStream = new PassThrough({ objectMode: true });\n const uploadsDir = this.#safePath('assets', 'uploads');\n this.#reportInfo('creating assets read stream');\n\n this.#pipeAssetsToStream(outStream, uploadsDir).catch((e: unknown) => {\n outStream.destroy(\n e instanceof Error ? e : new ProviderTransferError(String(e), { details: { error: e } })\n );\n });\n\n return outStream;\n }\n\n async #pipeAssetsToStream(outStream: PassThrough, uploadsDir: string): Promise<void> {\n if (!(await fs.pathExists(uploadsDir))) {\n outStream.end();\n return;\n }\n\n const names = (await fs.readdir(uploadsDir)).sort();\n for (const name of names) {\n const absUpload = path.join(uploadsDir, name);\n const stat = await fs.stat(absUpload);\n if (stat.isFile()) {\n let metadata = this.#assetMetadata?.get(name);\n if (!metadata) {\n try {\n metadata = await this.#readAssetMetadata(name);\n } catch (error) {\n outStream.destroy(\n new ProviderTransferError(`Failed to read metadata for ${name}`, {\n details: { error },\n })\n );\n return;\n }\n }\n\n const normalizedPath = unknownPathToPosix(path.posix.join('assets', 'uploads', name));\n const asset: IAsset = {\n metadata,\n filename: name,\n filepath: normalizedPath,\n stats: { size: stat.size },\n stream: fs.createReadStream(absUpload),\n };\n await write(outStream, asset);\n }\n }\n outStream.end();\n }\n\n async #readAssetMetadata(filename: string): Promise<IAsset['metadata']> {\n const metadataPath = this.#safePath('assets', 'metadata', `${filename}.json`);\n return validateAssetMetadata(await fs.readJson(metadataPath), filename);\n }\n\n async #listJsonlFiles(posixSubdir: string): Promise<string[]> {\n const dirAbs = this.#safePath(...posixSubdir.split('/').filter(Boolean));\n if (!(await fs.pathExists(dirAbs))) {\n return [];\n }\n const names = await fs.readdir(dirAbs);\n return names\n .filter((n) => n.endsWith('.jsonl'))\n .sort()\n .map((n) => path.join(dirAbs, n));\n }\n\n #streamJsonlDirectory(posixSubdir: string): Readable {\n const outStream = new PassThrough({ objectMode: true });\n this.#reportInfo(`streaming jsonl from ${posixSubdir}`);\n\n this.#pipeJsonlDirectoryToStream(outStream, posixSubdir).catch((e: unknown) => {\n outStream.destroy(\n e instanceof Error ? e : new ProviderTransferError(String(e), { details: { error: e } })\n );\n });\n\n return outStream;\n }\n\n async #pipeJsonlDirectoryToStream(outStream: PassThrough, posixSubdir: string): Promise<void> {\n const files = await this.#listJsonlFiles(posixSubdir);\n for (const absPath of files) {\n const transforms = [\n parser({\n checkErrors: true,\n }),\n (line: { key: string; value: object }) => line.value,\n ];\n\n const stream = fs.createReadStream(absPath).pipe(chain(transforms));\n\n try {\n for await (const chunk of stream) {\n await write(outStream, chunk);\n }\n } catch (e: unknown) {\n outStream.destroy(\n new ProviderTransferError(`Error parsing JSONL in ${absPath}: ${(e as Error).message}`, {\n details: {\n error: e,\n },\n })\n );\n return;\n }\n }\n outStream.end();\n }\n}\n"],"names":["METADATA_FILE_PATH","createLocalDirectorySourceProvider","options","LocalDirectorySourceProvider","isPathInsideRoot","root","candidate","relative","path","startsWith","sep","isAbsolute","bootstrap","diagnostics","stat","fs","isDirectory","ProviderInitializationError","e","getMetadata","getSchemas","schemaCollection","utils","createSchemasReadStream","isEmpty","schemas","keyBy","validateStage","stage","uploadsDir","pathExists","Map","metadata","names","readdir","sort","name","uploadPath","join","isFile","set","error","reason","Error","message","String","ProviderValidationError","createEntitiesReadStream","createLinksReadStream","createConfigurationReadStream","createAssetsReadStream","outStream","PassThrough","objectMode","catch","destroy","ProviderTransferError","details","type","resolve","directory","report","createdAt","Date","origin","kind","posixSegments","joined","posix","segments","split","filter","Boolean","resolved","metadataPath","readJson","end","absUpload","get","normalizedPath","unknownPathToPosix","asset","filename","filepath","stats","size","stream","createReadStream","write","validateAssetMetadata","posixSubdir","dirAbs","n","endsWith","map","files","absPath","transforms","parser","checkErrors","line","value","pipe","chain","chunk"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAMA,kBAAAA,GAAqB,eAAA;AASpB,MAAMC,qCAAqC,CAChDC,OAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,4BAAAA,CAA6BD,OAAAA,CAAAA;AAC1C;AAEA,MAAME,gBAAAA,GAAmB,CAACC,IAAAA,EAAcC,SAAAA,GAAAA;AACtC,IAAA,MAAMC,QAAAA,GAAWC,IAAAA,CAAKD,QAAQ,CAACF,IAAAA,EAAMC,SAAAA,CAAAA;AACrC,IAAA,OAAOC,aAAa,EAAA,IAAO,CAACA,QAAAA,CAASE,UAAU,CAAC,CAAC,EAAE,EAAED,IAAAA,CAAKE,GAAG,CAAA,CAAE,CAAA,IAAK,CAACF,IAAAA,CAAKG,UAAU,CAACJ,QAAAA,CAAAA;AACvF,CAAA;IASE,aAAA,iBAAA,8BAAA,CAAA,eAAA,CAAA,EAEA,SAAA,iBAAA,8BAAA,CAAA,WAAA,CAAA,EAEA,cAAA,iBAAA,8BAAA,CAAA,gBAAA,CAAA,EAEA,YAAA,iBAAA,8BAAA,CAAA,cAAA,CAAA,EAOA,WAAA,iBAAA,8BAAA,CAAA,aAAA,CAAA,qFAYA,SAAA,iBAAA,8BAAA,CAAA,WAAA,CAAA,EAkCM,aAAA,iBAAA,8BAAA,CAAA,eAAA,CAAA,EAmGA,mBAAA,iBAAA,8BAAA,CAAA,qBAAA,CAAA,EAuCA,kBAAA,iBAAA,8BAAA,CAAA,oBAAA,CAAA,EAKA,eAAA,iBAAA,8BAAA,CAAA,iBAAA,CAAA,EAYN,qBAAA,iBAAA,8BAAA,CAAA,uBAAA,CAAA,EAaM,2BAAA,iBAAA,8BAAA,CAAA,6BAAA,CAAA;AA1OR,MAAMJ,4BAAAA,CAAAA;IA0CJ,MAAMS,SAAAA,CAAUC,WAAgC,EAAE;QAChD,+BAAA,CAAA,IAAI,EAAC,YAAA,CAAA,CAAA,YAAA,CAAA,GAAeA,WAAAA;AACpB,QAAA,MAAMR,IAAAA,GAAO,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA;QAElB,IAAI;AACF,YAAA,MAAMS,IAAAA,GAAO,MAAMC,WAAAA,CAAGD,IAAI,CAACT,IAAAA,CAAAA;YAC3B,IAAI,CAACS,IAAAA,CAAKE,WAAW,EAAA,EAAI;AACvB,gBAAA,MAAM,IAAIC,2BAAAA,CAA4B,CAAC,MAAM,EAAEZ,IAAAA,CAAK,qBAAqB,CAAC,CAAA;AAC5E,YAAA;YACA,MAAM,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,EAAA;AACb,QAAA,CAAA,CAAE,OAAOa,CAAAA,EAAG;AACV,YAAA,IAAIA,aAAaD,2BAAAA,EAA6B;gBAC5C,MAAMC,CAAAA;AACR,YAAA;AACA,YAAA,MAAM,IAAID,2BAAAA,CACR,CAAC,WAAW,EAAEZ,IAAAA,CAAK,oCAAoC,CAAC,CAAA;AAE5D,QAAA;AAEA,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,EAAW;AACnB,YAAA,MAAM,IAAIY,2BAAAA,CAA4B,kDAAA,CAAA;AACxC,QAAA;AACF,IAAA;AAYA,IAAA,MAAME,WAAAA,GAAc;QAClB,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,kBAAA,CAAA;AACjB,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,EAAW;YACnB,MAAM,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,EAAA;AACb,QAAA;AACA,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,IAAa,IAAA;AAC3B,IAAA;AAEA,IAAA,MAAMC,UAAAA,GAAa;QACjB,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,iBAAA,CAAA;QACjB,MAAMC,gBAAAA,GAAmB,MAAMC,OAAoB,CACjD,IAAI,CAACC,uBAAuB,EAAA,CAAA;AAG9B,QAAA,IAAIC,QAAQH,gBAAAA,CAAAA,EAAmB;AAC7B,YAAA,MAAM,IAAIJ,2BAAAA,CAA4B,iDAAA,CAAA;AACxC,QAAA;QAEA,MAAMQ,OAAAA,GAAUC,MAAM,KAAA,EAAOL,gBAAAA,CAAAA;AAC7B,QAAA,OAAOC,kBAA+B,CAACG,OAAAA,CAAAA;AACzC,IAAA;IAEA,MAAME,aAAAA,CAAcC,KAAoB,EAAE;AACxC,QAAA,IAAIA,UAAU,QAAA,EAAU;AACtB,YAAA;AACF,QAAA;AAEA,QAAA,MAAMC,aAAa,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,WAAU,QAAA,EAAU,SAAA,CAAA;AAC5C,QAAA,IAAI,CAAE,MAAMd,WAAAA,CAAGe,UAAU,CAACD,UAAAA,CAAAA,EAAc;AACtC,YAAA,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,cAAA,CAAA,GAAiB,IAAIE,GAAAA,EAAAA;AAC1B,YAAA;AACF,QAAA;AAEA,QAAA,MAAMC,WAAW,IAAID,GAAAA,EAAAA;QACrB,MAAME,KAAAA,GAAQ,CAAC,MAAMlB,YAAGmB,OAAO,CAACL,UAAAA,CAAU,EAAGM,IAAI,EAAA;QACjD,KAAK,MAAMC,QAAQH,KAAAA,CAAO;AACxB,YAAA,MAAMI,UAAAA,GAAa7B,IAAAA,CAAK8B,IAAI,CAACT,UAAAA,EAAYO,IAAAA,CAAAA;YACzC,IAAI,CAAC,CAAC,MAAMrB,WAAAA,CAAGD,IAAI,CAACuB,UAAAA,CAAU,EAAGE,MAAM,EAAA,EAAI;AACzC,gBAAA;AACF,YAAA;YAEA,IAAI;gBACFP,QAAAA,CAASQ,GAAG,CAACJ,IAAAA,EAAM,MAAM,gCAAA,IAAI,EAAC,oBAAA,kBAAA,CAAA,CAAmBA,IAAAA,CAAAA,CAAAA;AACnD,YAAA,CAAA,CAAE,OAAOK,KAAAA,EAAO;AACd,gBAAA,MAAMC,SAASD,KAAAA,YAAiBE,KAAAA,GAAQF,KAAAA,CAAMG,OAAO,GAAGC,MAAAA,CAAOJ,KAAAA,CAAAA;gBAC/D,MAAM,IAAIK,uBAAAA,CACR,CAAC,qCAAqC,EAAEV,IAAAA,CAAK,GAAG,EAAEM,MAAAA,CAAO,4DAA4D,CAAC,EACtH;AAAED,oBAAAA;AAAM,iBAAA,CAAA;AAEZ,YAAA;AACF,QAAA;QAEA,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,cAAA,CAAA,GAAiBT,QAAAA;AACxB,IAAA;IAEAe,wBAAAA,GAAqC;QACnC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,+BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,UAAA,CAAA;AACpC,IAAA;IAEAxB,uBAAAA,GAAoC;QAClC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,8BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,SAAA,CAAA;AACpC,IAAA;IAEAyB,qBAAAA,GAAkC;QAChC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,4BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,OAAA,CAAA;AACpC,IAAA;IAEAC,6BAAAA,GAA0C;QACxC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,oCAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,eAAA,CAAA;AACpC,IAAA;IAEAC,sBAAAA,GAAmC;QACjC,MAAMC,SAAAA,GAAY,IAAIC,WAAAA,CAAY;YAAEC,UAAAA,EAAY;AAAK,SAAA,CAAA;AACrD,QAAA,MAAMxB,aAAa,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,WAAU,QAAA,EAAU,SAAA,CAAA;QAC5C,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,6BAAA,CAAA;QAEjB,+BAAA,CAAA,IAAI,EAAC,mBAAA,CAAA,CAAA,mBAAA,CAAA,CAAoBsB,WAAWtB,UAAAA,CAAAA,CAAYyB,KAAK,CAAC,CAACpC,CAAAA,GAAAA;YACrDiC,SAAAA,CAAUI,OAAO,CACfrC,CAAAA,YAAayB,KAAAA,GAAQzB,IAAI,IAAIsC,qBAAAA,CAAsBX,OAAO3B,CAAAA,CAAAA,EAAI;gBAAEuC,OAAAA,EAAS;oBAAEhB,KAAAA,EAAOvB;AAAE;AAAE,aAAA,CAAA,CAAA;AAE1F,QAAA,CAAA,CAAA;QAEA,OAAOiC,SAAAA;AACT,IAAA;AApJA,IAAA,WAAA,CAAYjD,OAA6C,CAAE;QAK3D,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,WAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAYA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,SAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAkCA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,aAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAmGA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,mBAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAuCA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,kBAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAKA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,eAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAYA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,qBAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAaA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,2BAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAnOA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,aAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,SAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,cAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,YAAA,EAAA;;mBAAA;;aAZAwD,IAAAA,GAAqB,QAAA;aAErBtB,IAAAA,GAAO,yBAAA;QAaL,IAAI,CAAClC,OAAO,GAAGA,OAAAA;QACf,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,GAAgBM,IAAAA,CAAKmD,OAAO,CAACzD,OAAAA,CAAQ0D,SAAS,CAACpD,IAAI,CAAA;AAC1D,IAAA;AAqPF;AAnPE,SAAA,WAAYoC,OAAe,EAAA;AACzB,IAAA,+BAAA,CAAA,IAAI,EAAC,YAAA,CAAA,CAAA,YAAA,CAAA,EAAciB,MAAAA,CAAO;QACxBJ,OAAAA,EAAS;AACPK,YAAAA,SAAAA,EAAW,IAAIC,IAAAA,EAAAA;AACfnB,YAAAA,OAAAA;YACAoB,MAAAA,EAAQ;AACV,SAAA;QACAC,IAAAA,EAAM;AACR,KAAA,CAAA;AACF;AAGA,SAAA,QAAA,CAAU,GAAGC,aAAuB,EAAA;AAClC,IAAA,MAAMC,MAAAA,GAAS3D,IAAAA,CAAK4D,KAAK,CAAC9B,IAAI,CAAA,GAAI4B,aAAAA,CAAAA;IAClC,MAAMG,QAAAA,GAAWF,MAAAA,CAAOG,KAAK,CAAC9D,IAAAA,CAAK4D,KAAK,CAAC1D,GAAG,CAAA,CAAE6D,MAAM,CAACC,OAAAA,CAAAA;IACrD,MAAMC,QAAAA,GAAWjE,KAAKmD,OAAO,CAAC,gCAAA,IAAI,EAAC,eAAA,aAAA,CAAA,EAAA,GAAkBU,QAAAA,CAAAA;AACrD,IAAA,IAAI,CAACjE,gBAAAA,CAAiB,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,gBAAeqE,QAAAA,CAAAA,EAAW;AACnD,QAAA,MAAM,IAAIxD,2BAAAA,CAA4B,CAAC,cAAc,EAAEkD,MAAAA,CAAO,4BAA4B,CAAC,CAAA;AAC7F,IAAA;IACA,OAAOM,QAAAA;AACT;AA0BA,eAAA,YAAA,GAAA;AACE,IAAA,MAAMC,YAAAA,GAAe,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,CAAU1E,kBAAAA,CAAAA;AACpC,IAAA,IAAI,CAAE,MAAMe,WAAAA,CAAGe,UAAU,CAAC4C,YAAAA,CAAAA,EAAgB;AACxC,QAAA,MAAM,IAAIzD,2BAAAA,CACR,CAAC,QAAQ,EAAEjB,kBAAAA,CAAmB,sBAAsB,EAAE,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,CAAc,EAAE,CAAC,CAAA;AAEhF,IAAA;AACA,IAAA,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,aAAY,MAAMe,WAAAA,CAAG4D,QAAQ,CAACD,YAAAA,CAAAA;AACrC;AA2FA,eAAA,kBAAA,CAA0BvB,SAAsB,EAAEtB,UAAkB,EAAA;AAClE,IAAA,IAAI,CAAE,MAAMd,WAAAA,CAAGe,UAAU,CAACD,UAAAA,CAAAA,EAAc;AACtCsB,QAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACb,QAAA;AACF,IAAA;IAEA,MAAM3C,KAAAA,GAAQ,CAAC,MAAMlB,YAAGmB,OAAO,CAACL,UAAAA,CAAU,EAAGM,IAAI,EAAA;IACjD,KAAK,MAAMC,QAAQH,KAAAA,CAAO;AACxB,QAAA,MAAM4C,SAAAA,GAAYrE,IAAAA,CAAK8B,IAAI,CAACT,UAAAA,EAAYO,IAAAA,CAAAA;AACxC,QAAA,MAAMtB,IAAAA,GAAO,MAAMC,WAAAA,CAAGD,IAAI,CAAC+D,SAAAA,CAAAA;QAC3B,IAAI/D,IAAAA,CAAKyB,MAAM,EAAA,EAAI;AACjB,YAAA,IAAIP,WAAW,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,iBAAgB8C,GAAAA,CAAI1C,IAAAA,CAAAA;AACxC,YAAA,IAAI,CAACJ,QAAAA,EAAU;gBACb,IAAI;AACFA,oBAAAA,QAAAA,GAAW,MAAM,+BAAA,CAAA,IAAI,EAAC,oBAAA,kBAAA,CAAA,CAAmBI,IAAAA,CAAAA;AAC3C,gBAAA,CAAA,CAAE,OAAOK,KAAAA,EAAO;oBACdU,SAAAA,CAAUI,OAAO,CACf,IAAIC,qBAAAA,CAAsB,CAAC,4BAA4B,EAAEpB,MAAM,EAAE;wBAC/DqB,OAAAA,EAAS;AAAEhB,4BAAAA;AAAM;AACnB,qBAAA,CAAA,CAAA;AAEF,oBAAA;AACF,gBAAA;AACF,YAAA;YAEA,MAAMsC,cAAAA,GAAiBC,mBAAmBxE,IAAAA,CAAK4D,KAAK,CAAC9B,IAAI,CAAC,UAAU,SAAA,EAAWF,IAAAA,CAAAA,CAAAA;AAC/E,YAAA,MAAM6C,KAAAA,GAAgB;AACpBjD,gBAAAA,QAAAA;gBACAkD,QAAAA,EAAU9C,IAAAA;gBACV+C,QAAAA,EAAUJ,cAAAA;gBACVK,KAAAA,EAAO;AAAEC,oBAAAA,IAAAA,EAAMvE,KAAKuE;AAAK,iBAAA;gBACzBC,MAAAA,EAAQvE,WAAAA,CAAGwE,gBAAgB,CAACV,SAAAA;AAC9B,aAAA;AACA,YAAA,MAAMW,MAAMrC,SAAAA,EAAW8B,KAAAA,CAAAA;AACzB,QAAA;AACF,IAAA;AACA9B,IAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACf;AAEA,eAAA,kBAAyBM,QAAgB,EAAA;IACvC,MAAMR,YAAAA,GAAe,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,CAAU,QAAA,EAAU,UAAA,EAAY,CAAA,EAAGQ,QAAAA,CAAS,KAAK,CAAC,CAAA;AAC5E,IAAA,OAAOO,qBAAAA,CAAsB,MAAM1E,WAAAA,CAAG4D,QAAQ,CAACD,YAAAA,CAAAA,EAAeQ,QAAAA,CAAAA;AAChE;AAEA,eAAA,eAAsBQ,WAAmB,EAAA;IACvC,MAAMC,MAAAA,GAAS,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,CAAA,GAAaD,WAAAA,CAAYpB,KAAK,CAAC,GAAA,CAAA,CAAKC,MAAM,CAACC,OAAAA,CAAAA,CAAAA;AAC/D,IAAA,IAAI,CAAE,MAAMzD,WAAAA,CAAGe,UAAU,CAAC6D,MAAAA,CAAAA,EAAU;AAClC,QAAA,OAAO,EAAE;AACX,IAAA;AACA,IAAA,MAAM1D,KAAAA,GAAQ,MAAMlB,WAAAA,CAAGmB,OAAO,CAACyD,MAAAA,CAAAA;AAC/B,IAAA,OAAO1D,MACJsC,MAAM,CAAC,CAACqB,CAAAA,GAAMA,CAAAA,CAAEC,QAAQ,CAAC,QAAA,CAAA,CAAA,CACzB1D,IAAI,EAAA,CACJ2D,GAAG,CAAC,CAACF,IAAMpF,IAAAA,CAAK8B,IAAI,CAACqD,MAAAA,EAAQC,CAAAA,CAAAA,CAAAA;AAClC;AAEA,SAAA,qBAAsBF,WAAmB,EAAA;IACvC,MAAMvC,SAAAA,GAAY,IAAIC,WAAAA,CAAY;QAAEC,UAAAA,EAAY;AAAK,KAAA,CAAA;AACrD,IAAA,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,aAAY,CAAC,qBAAqB,EAAEqC,WAAAA,CAAAA,CAAa,CAAA;IAEtD,+BAAA,CAAA,IAAI,EAAC,2BAAA,CAAA,CAAA,2BAAA,CAAA,CAA4BvC,WAAWuC,WAAAA,CAAAA,CAAapC,KAAK,CAAC,CAACpC,CAAAA,GAAAA;QAC9DiC,SAAAA,CAAUI,OAAO,CACfrC,CAAAA,YAAayB,KAAAA,GAAQzB,IAAI,IAAIsC,qBAAAA,CAAsBX,OAAO3B,CAAAA,CAAAA,EAAI;YAAEuC,OAAAA,EAAS;gBAAEhB,KAAAA,EAAOvB;AAAE;AAAE,SAAA,CAAA,CAAA;AAE1F,IAAA,CAAA,CAAA;IAEA,OAAOiC,SAAAA;AACT;AAEA,eAAA,0BAAA,CAAkCA,SAAsB,EAAEuC,WAAmB,EAAA;AAC3E,IAAA,MAAMK,QAAQ,MAAM,+BAAA,CAAA,IAAI,EAAC,iBAAA,eAAA,CAAA,CAAgBL,WAAAA,CAAAA;IACzC,KAAK,MAAMM,WAAWD,KAAAA,CAAO;AAC3B,QAAA,MAAME,UAAAA,GAAa;YACjBC,MAAAA,CAAO;gBACLC,WAAAA,EAAa;AACf,aAAA,CAAA;YACA,CAACC,IAAAA,GAAyCA,KAAKC;AAChD,SAAA;AAED,QAAA,MAAMf,SAASvE,WAAAA,CAAGwE,gBAAgB,CAACS,OAAAA,CAAAA,CAASM,IAAI,CAACC,KAAAA,CAAMN,UAAAA,CAAAA,CAAAA;QAEvD,IAAI;YACF,WAAW,MAAMO,SAASlB,MAAAA,CAAQ;AAChC,gBAAA,MAAME,MAAMrC,SAAAA,EAAWqD,KAAAA,CAAAA;AACzB,YAAA;AACF,QAAA,CAAA,CAAE,OAAOtF,CAAAA,EAAY;AACnBiC,YAAAA,SAAAA,CAAUI,OAAO,CACf,IAAIC,qBAAAA,CAAsB,CAAC,uBAAuB,EAAEwC,OAAAA,CAAQ,EAAE,EAAG9E,CAAAA,CAAY0B,OAAO,EAAE,EAAE;gBACtFa,OAAAA,EAAS;oBACPhB,KAAAA,EAAOvB;AACT;AACF,aAAA,CAAA,CAAA;AAEF,YAAA;AACF,QAAA;AACF,IAAA;AACAiC,IAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACf;;;;"}
|
|
1
|
+
{"version":3,"file":"index.mjs","sources":["../../../../src/directory/providers/source/index.ts"],"sourcesContent":["import path from 'path';\nimport type { Readable } from 'stream';\nimport { PassThrough } from 'stream';\n\nimport fs from 'fs-extra';\nimport { isEmpty, keyBy } from 'lodash';\nimport { chain } from 'stream-chain';\nimport { parser } from 'stream-json/jsonl/Parser';\nimport type { Struct } from '@strapi/types';\n\nimport type {\n IAsset,\n IMetadata,\n ISourceProvider,\n ProviderType,\n TransferStage,\n} from '../../../types';\nimport type { IDiagnosticReporter } from '../../../utils/diagnostic';\n\nimport * as utils from '../../../utils';\nimport { write } from '../../../utils/writable-async-write';\nimport {\n ProviderInitializationError,\n ProviderTransferError,\n ProviderValidationError,\n} from '../../../errors/providers';\nimport { unknownPathToPosix, validateAssetMetadata } from '../../../file/providers/source/utils';\n\nconst METADATA_FILE_PATH = 'metadata.json';\n\nexport interface ILocalDirectorySourceProviderOptions {\n directory: {\n /** Absolute or relative path to the root of an unpacked Strapi data-transfer export */\n path: string;\n };\n}\n\nexport const createLocalDirectorySourceProvider = (\n options: ILocalDirectorySourceProviderOptions\n) => {\n return new LocalDirectorySourceProvider(options);\n};\n\nconst isPathInsideRoot = (root: string, candidate: string): boolean => {\n const relative = path.relative(root, candidate);\n return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));\n};\n\nclass LocalDirectorySourceProvider implements ISourceProvider {\n type: ProviderType = 'source';\n\n name = 'source::local-directory';\n\n options: ILocalDirectorySourceProviderOptions;\n\n #rootResolved: string;\n\n #metadata?: IMetadata;\n\n #assetMetadata?: Map<string, IAsset['metadata']>;\n\n #diagnostics?: IDiagnosticReporter;\n\n constructor(options: ILocalDirectorySourceProviderOptions) {\n this.options = options;\n this.#rootResolved = path.resolve(options.directory.path);\n }\n\n #reportInfo(message: string) {\n this.#diagnostics?.report({\n details: {\n createdAt: new Date(),\n message,\n origin: 'directory-source-provider',\n },\n kind: 'info',\n });\n }\n\n /** Resolve a posix-style relative path under the export root; rejects escapes. */\n #safePath(...posixSegments: string[]): string {\n const joined = path.posix.join(...posixSegments);\n const segments = joined.split(path.posix.sep).filter(Boolean);\n const resolved = path.resolve(this.#rootResolved, ...segments);\n if (!isPathInsideRoot(this.#rootResolved, resolved)) {\n throw new ProviderInitializationError(`Invalid path \"${joined}\" — escapes backup directory`);\n }\n return resolved;\n }\n\n async bootstrap(diagnostics: IDiagnosticReporter) {\n this.#diagnostics = diagnostics;\n const root = this.#rootResolved;\n\n try {\n const stat = await fs.stat(root);\n if (!stat.isDirectory()) {\n throw new ProviderInitializationError(`Path '${root}' is not a directory.`);\n }\n await this.#loadMetadata();\n } catch (e) {\n if (e instanceof ProviderInitializationError) {\n throw e;\n }\n throw new ProviderInitializationError(\n `Directory '${root}' is not a valid Strapi data export.`\n );\n }\n\n if (!this.#metadata) {\n throw new ProviderInitializationError('Could not load metadata from Strapi data export.');\n }\n }\n\n async #loadMetadata() {\n const metadataPath = this.#safePath(METADATA_FILE_PATH);\n if (!(await fs.pathExists(metadataPath))) {\n throw new ProviderInitializationError(\n `Missing ${METADATA_FILE_PATH} in export directory '${this.#rootResolved}'.`\n );\n }\n this.#metadata = await fs.readJson(metadataPath);\n }\n\n async getMetadata() {\n this.#reportInfo('getting metadata');\n if (!this.#metadata) {\n await this.#loadMetadata();\n }\n return this.#metadata ?? null;\n }\n\n async getSchemas() {\n this.#reportInfo('getting schemas');\n const schemaCollection = await utils.stream.collect<Struct.Schema>(\n this.createSchemasReadStream()\n );\n\n if (isEmpty(schemaCollection)) {\n throw new ProviderInitializationError('Could not load schemas from Strapi data export.');\n }\n\n const schemas = keyBy(schemaCollection, 'uid');\n return utils.schema.schemasToValidJSON(schemas);\n }\n\n async validateStage(stage: TransferStage) {\n if (stage !== 'assets') {\n return;\n }\n\n const uploadsDir = this.#safePath('assets', 'uploads');\n if (!(await fs.pathExists(uploadsDir))) {\n this.#assetMetadata = new Map();\n return;\n }\n\n const metadata = new Map<string, IAsset['metadata']>();\n const names = (await fs.readdir(uploadsDir)).sort();\n for (const name of names) {\n const uploadPath = path.join(uploadsDir, name);\n if (!(await fs.stat(uploadPath)).isFile()) {\n continue;\n }\n\n try {\n metadata.set(name, await this.#readAssetMetadata(name));\n } catch (error) {\n const reason = error instanceof Error ? error.message : String(error);\n throw new ProviderValidationError(\n `Asset metadata preflight failed for \"${name}\": ${reason}. The destination was not modified; re-export and try again.`,\n { error }\n );\n }\n }\n\n this.#assetMetadata = metadata;\n }\n\n createEntitiesReadStream(): Readable {\n this.#reportInfo('creating entities read stream');\n return this.#streamJsonlDirectory('entities');\n }\n\n createSchemasReadStream(): Readable {\n this.#reportInfo('creating schemas read stream');\n return this.#streamJsonlDirectory('schemas');\n }\n\n createLinksReadStream(): Readable {\n this.#reportInfo('creating links read stream');\n return this.#streamJsonlDirectory('links');\n }\n\n createConfigurationReadStream(): Readable {\n this.#reportInfo('creating configuration read stream');\n return this.#streamJsonlDirectory('configuration');\n }\n\n createAssetsReadStream(): Readable {\n const outStream = new PassThrough({ objectMode: true });\n const uploadsDir = this.#safePath('assets', 'uploads');\n this.#reportInfo('creating assets read stream');\n\n this.#pipeAssetsToStream(outStream, uploadsDir).catch((e: unknown) => {\n outStream.destroy(\n e instanceof Error ? e : new ProviderTransferError(String(e), { details: { error: e } })\n );\n });\n\n return outStream;\n }\n\n async #pipeAssetsToStream(outStream: PassThrough, uploadsDir: string): Promise<void> {\n if (!(await fs.pathExists(uploadsDir))) {\n outStream.end();\n return;\n }\n\n const names = (await fs.readdir(uploadsDir)).sort();\n for (const name of names) {\n const absUpload = path.join(uploadsDir, name);\n const stat = await fs.stat(absUpload);\n if (stat.isFile()) {\n let metadata = this.#assetMetadata?.get(name);\n if (!metadata) {\n try {\n metadata = await this.#readAssetMetadata(name);\n } catch (error) {\n outStream.destroy(\n new ProviderTransferError(`Failed to read metadata for ${name}`, {\n details: { error },\n })\n );\n return;\n }\n }\n\n const normalizedPath = unknownPathToPosix(path.posix.join('assets', 'uploads', name));\n const asset: IAsset = {\n metadata,\n filename: name,\n filepath: normalizedPath,\n stats: { size: stat.size },\n stream: fs.createReadStream(absUpload),\n };\n await write(outStream, asset);\n }\n }\n outStream.end();\n }\n\n async #readAssetMetadata(filename: string): Promise<IAsset['metadata']> {\n const metadataPath = this.#safePath('assets', 'metadata', `${filename}.json`);\n return validateAssetMetadata(await fs.readJson(metadataPath), filename);\n }\n\n async #listJsonlFiles(posixSubdir: string): Promise<string[]> {\n const dirAbs = this.#safePath(...posixSubdir.split('/').filter(Boolean));\n if (!(await fs.pathExists(dirAbs))) {\n return [];\n }\n const names = await fs.readdir(dirAbs);\n return names\n .filter((n) => n.endsWith('.jsonl'))\n .sort()\n .map((n) => path.join(dirAbs, n));\n }\n\n #streamJsonlDirectory(posixSubdir: string): Readable {\n const outStream = new PassThrough({ objectMode: true });\n this.#reportInfo(`streaming jsonl from ${posixSubdir}`);\n\n this.#pipeJsonlDirectoryToStream(outStream, posixSubdir).catch((e: unknown) => {\n outStream.destroy(\n e instanceof Error ? e : new ProviderTransferError(String(e), { details: { error: e } })\n );\n });\n\n return outStream;\n }\n\n async #pipeJsonlDirectoryToStream(outStream: PassThrough, posixSubdir: string): Promise<void> {\n const files = await this.#listJsonlFiles(posixSubdir);\n for (const absPath of files) {\n const transforms = [\n parser({\n checkErrors: true,\n }),\n (line: { key: string; value: object }) => line.value,\n ];\n\n const stream = fs.createReadStream(absPath).pipe(chain(transforms));\n\n try {\n for await (const chunk of stream) {\n await write(outStream, chunk);\n }\n } catch (e: unknown) {\n outStream.destroy(\n new ProviderTransferError(`Error parsing JSONL in ${absPath}: ${(e as Error).message}`, {\n details: {\n error: e,\n },\n })\n );\n return;\n }\n }\n outStream.end();\n }\n}\n"],"names":["METADATA_FILE_PATH","createLocalDirectorySourceProvider","options","LocalDirectorySourceProvider","isPathInsideRoot","root","candidate","relative","path","startsWith","sep","isAbsolute","bootstrap","diagnostics","stat","fs","isDirectory","ProviderInitializationError","e","getMetadata","getSchemas","schemaCollection","utils","createSchemasReadStream","isEmpty","schemas","keyBy","validateStage","stage","uploadsDir","pathExists","Map","metadata","names","readdir","sort","name","uploadPath","join","isFile","set","error","reason","Error","message","String","ProviderValidationError","createEntitiesReadStream","createLinksReadStream","createConfigurationReadStream","createAssetsReadStream","outStream","PassThrough","objectMode","catch","destroy","ProviderTransferError","details","type","resolve","directory","report","createdAt","Date","origin","kind","posixSegments","joined","posix","segments","split","filter","Boolean","resolved","metadataPath","readJson","end","absUpload","get","normalizedPath","unknownPathToPosix","asset","filename","filepath","stats","size","stream","createReadStream","write","validateAssetMetadata","posixSubdir","dirAbs","n","endsWith","map","files","absPath","transforms","parser","checkErrors","line","value","pipe","chain","chunk"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAMA,kBAAAA,GAAqB,eAAA;AASpB,MAAMC,qCAAqC,CAChDC,OAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,4BAAAA,CAA6BD,OAAAA,CAAAA;AAC1C;AAEA,MAAME,gBAAAA,GAAmB,CAACC,IAAAA,EAAcC,SAAAA,GAAAA;AACtC,IAAA,MAAMC,QAAAA,GAAWC,IAAAA,CAAKD,QAAQ,CAACF,IAAAA,EAAMC,SAAAA,CAAAA;AACrC,IAAA,OAAOC,aAAa,EAAA,IAAO,CAACA,QAAAA,CAASE,UAAU,CAAC,CAAC,EAAE,EAAED,IAAAA,CAAKE,GAAG,CAAA,CAAE,CAAA,IAAK,CAACF,IAAAA,CAAKG,UAAU,CAACJ,QAAAA,CAAAA;AACvF,CAAA;IASE,aAAA,iBAAA,8BAAA,CAAA,eAAA,CAAA,EAEA,SAAA,iBAAA,8BAAA,CAAA,WAAA,CAAA,EAEA,cAAA,iBAAA,8BAAA,CAAA,gBAAA,CAAA,EAEA,YAAA,iBAAA,8BAAA,CAAA,cAAA,CAAA,EAOA,WAAA,iBAAA,8BAAA,CAAA,aAAA,CAAA,qFAYA,SAAA,iBAAA,8BAAA,CAAA,WAAA,CAAA,EAkCM,aAAA,iBAAA,8BAAA,CAAA,eAAA,CAAA,EAmGA,mBAAA,iBAAA,8BAAA,CAAA,qBAAA,CAAA,EAuCA,kBAAA,iBAAA,8BAAA,CAAA,oBAAA,CAAA,EAKA,eAAA,iBAAA,8BAAA,CAAA,iBAAA,CAAA,EAYN,qBAAA,iBAAA,8BAAA,CAAA,uBAAA,CAAA,EAaM,2BAAA,iBAAA,8BAAA,CAAA,6BAAA,CAAA;AA1OR,MAAMJ,4BAAAA,CAAAA;IA0CJ,MAAMS,SAAAA,CAAUC,WAAgC,EAAE;QAChD,+BAAA,CAAA,IAAI,EAAC,YAAA,CAAA,CAAA,YAAA,CAAA,GAAeA,WAAAA;AACpB,QAAA,MAAMR,IAAAA,GAAO,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA;QAElB,IAAI;AACF,YAAA,MAAMS,IAAAA,GAAO,MAAMC,WAAAA,CAAGD,IAAI,CAACT,IAAAA,CAAAA;YAC3B,IAAI,CAACS,IAAAA,CAAKE,WAAW,EAAA,EAAI;AACvB,gBAAA,MAAM,IAAIC,2BAAAA,CAA4B,CAAC,MAAM,EAAEZ,IAAAA,CAAK,qBAAqB,CAAC,CAAA;AAC5E,YAAA;YACA,MAAM,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,EAAA;AACb,QAAA,CAAA,CAAE,OAAOa,CAAAA,EAAG;AACV,YAAA,IAAIA,aAAaD,2BAAAA,EAA6B;gBAC5C,MAAMC,CAAAA;AACR,YAAA;AACA,YAAA,MAAM,IAAID,2BAAAA,CACR,CAAC,WAAW,EAAEZ,IAAAA,CAAK,oCAAoC,CAAC,CAAA;AAE5D,QAAA;AAEA,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,EAAW;AACnB,YAAA,MAAM,IAAIY,2BAAAA,CAA4B,kDAAA,CAAA;AACxC,QAAA;AACF,IAAA;AAYA,IAAA,MAAME,WAAAA,GAAc;QAClB,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,kBAAA,CAAA;AACjB,QAAA,IAAI,CAAC,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,EAAW;YACnB,MAAM,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,EAAA;AACb,QAAA;AACA,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,IAAa,IAAA;AAC3B,IAAA;AAEA,IAAA,MAAMC,UAAAA,GAAa;QACjB,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,iBAAA,CAAA;QACjB,MAAMC,gBAAAA,GAAmB,MAAMC,OAAoB,CACjD,IAAI,CAACC,uBAAuB,EAAA,CAAA;AAG9B,QAAA,IAAIC,QAAQH,gBAAAA,CAAAA,EAAmB;AAC7B,YAAA,MAAM,IAAIJ,2BAAAA,CAA4B,iDAAA,CAAA;AACxC,QAAA;QAEA,MAAMQ,OAAAA,GAAUC,MAAML,gBAAAA,EAAkB,KAAA,CAAA;AACxC,QAAA,OAAOC,kBAA+B,CAACG,OAAAA,CAAAA;AACzC,IAAA;IAEA,MAAME,aAAAA,CAAcC,KAAoB,EAAE;AACxC,QAAA,IAAIA,UAAU,QAAA,EAAU;AACtB,YAAA;AACF,QAAA;AAEA,QAAA,MAAMC,aAAa,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,WAAU,QAAA,EAAU,SAAA,CAAA;AAC5C,QAAA,IAAI,CAAE,MAAMd,WAAAA,CAAGe,UAAU,CAACD,UAAAA,CAAAA,EAAc;AACtC,YAAA,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,cAAA,CAAA,GAAiB,IAAIE,GAAAA,EAAAA;AAC1B,YAAA;AACF,QAAA;AAEA,QAAA,MAAMC,WAAW,IAAID,GAAAA,EAAAA;QACrB,MAAME,KAAAA,GAAQ,CAAC,MAAMlB,YAAGmB,OAAO,CAACL,UAAAA,CAAU,EAAGM,IAAI,EAAA;QACjD,KAAK,MAAMC,QAAQH,KAAAA,CAAO;AACxB,YAAA,MAAMI,UAAAA,GAAa7B,IAAAA,CAAK8B,IAAI,CAACT,UAAAA,EAAYO,IAAAA,CAAAA;YACzC,IAAI,CAAC,CAAC,MAAMrB,WAAAA,CAAGD,IAAI,CAACuB,UAAAA,CAAU,EAAGE,MAAM,EAAA,EAAI;AACzC,gBAAA;AACF,YAAA;YAEA,IAAI;gBACFP,QAAAA,CAASQ,GAAG,CAACJ,IAAAA,EAAM,MAAM,gCAAA,IAAI,EAAC,oBAAA,kBAAA,CAAA,CAAmBA,IAAAA,CAAAA,CAAAA;AACnD,YAAA,CAAA,CAAE,OAAOK,KAAAA,EAAO;AACd,gBAAA,MAAMC,SAASD,KAAAA,YAAiBE,KAAAA,GAAQF,KAAAA,CAAMG,OAAO,GAAGC,MAAAA,CAAOJ,KAAAA,CAAAA;gBAC/D,MAAM,IAAIK,uBAAAA,CACR,CAAC,qCAAqC,EAAEV,IAAAA,CAAK,GAAG,EAAEM,MAAAA,CAAO,4DAA4D,CAAC,EACtH;AAAED,oBAAAA;AAAM,iBAAA,CAAA;AAEZ,YAAA;AACF,QAAA;QAEA,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,cAAA,CAAA,GAAiBT,QAAAA;AACxB,IAAA;IAEAe,wBAAAA,GAAqC;QACnC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,+BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,UAAA,CAAA;AACpC,IAAA;IAEAxB,uBAAAA,GAAoC;QAClC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,8BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,SAAA,CAAA;AACpC,IAAA;IAEAyB,qBAAAA,GAAkC;QAChC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,4BAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,OAAA,CAAA;AACpC,IAAA;IAEAC,6BAAAA,GAA0C;QACxC,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,oCAAA,CAAA;AACjB,QAAA,OAAO,+BAAA,CAAA,IAAI,EAAC,qBAAA,CAAA,CAAA,qBAAA,CAAA,CAAsB,eAAA,CAAA;AACpC,IAAA;IAEAC,sBAAAA,GAAmC;QACjC,MAAMC,SAAAA,GAAY,IAAIC,WAAAA,CAAY;YAAEC,UAAAA,EAAY;AAAK,SAAA,CAAA;AACrD,QAAA,MAAMxB,aAAa,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,WAAU,QAAA,EAAU,SAAA,CAAA;QAC5C,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,WAAA,CAAA,CAAY,6BAAA,CAAA;QAEjB,+BAAA,CAAA,IAAI,EAAC,mBAAA,CAAA,CAAA,mBAAA,CAAA,CAAoBsB,WAAWtB,UAAAA,CAAAA,CAAYyB,KAAK,CAAC,CAACpC,CAAAA,GAAAA;YACrDiC,SAAAA,CAAUI,OAAO,CACfrC,CAAAA,YAAayB,KAAAA,GAAQzB,IAAI,IAAIsC,qBAAAA,CAAsBX,OAAO3B,CAAAA,CAAAA,EAAI;gBAAEuC,OAAAA,EAAS;oBAAEhB,KAAAA,EAAOvB;AAAE;AAAE,aAAA,CAAA,CAAA;AAE1F,QAAA,CAAA,CAAA;QAEA,OAAOiC,SAAAA;AACT,IAAA;AApJA,IAAA,WAAA,CAAYjD,OAA6C,CAAE;QAK3D,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,WAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAYA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,SAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAkCA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,aAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAmGA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,mBAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAuCA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,kBAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAKA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,eAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAYA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,qBAAA,EAAA;AAAA,YAAA,KAAA,EAAA;;QAaA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAM,2BAAA,EAAA;AAAN,YAAA,KAAA,EAAA;;QAnOA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,aAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,SAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,cAAA,EAAA;;mBAAA;;QAEA,MAAA,CAAA,cAAA,CAAA,IAAA,EAAA,YAAA,EAAA;;mBAAA;;aAZAwD,IAAAA,GAAqB,QAAA;aAErBtB,IAAAA,GAAO,yBAAA;QAaL,IAAI,CAAClC,OAAO,GAAGA,OAAAA;QACf,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,GAAgBM,IAAAA,CAAKmD,OAAO,CAACzD,OAAAA,CAAQ0D,SAAS,CAACpD,IAAI,CAAA;AAC1D,IAAA;AAqPF;AAnPE,SAAA,WAAYoC,OAAe,EAAA;AACzB,IAAA,+BAAA,CAAA,IAAI,EAAC,YAAA,CAAA,CAAA,YAAA,CAAA,EAAciB,MAAAA,CAAO;QACxBJ,OAAAA,EAAS;AACPK,YAAAA,SAAAA,EAAW,IAAIC,IAAAA,EAAAA;AACfnB,YAAAA,OAAAA;YACAoB,MAAAA,EAAQ;AACV,SAAA;QACAC,IAAAA,EAAM;AACR,KAAA,CAAA;AACF;AAGA,SAAA,QAAA,CAAU,GAAGC,aAAuB,EAAA;AAClC,IAAA,MAAMC,MAAAA,GAAS3D,IAAAA,CAAK4D,KAAK,CAAC9B,IAAI,CAAA,GAAI4B,aAAAA,CAAAA;IAClC,MAAMG,QAAAA,GAAWF,MAAAA,CAAOG,KAAK,CAAC9D,IAAAA,CAAK4D,KAAK,CAAC1D,GAAG,CAAA,CAAE6D,MAAM,CAACC,OAAAA,CAAAA;IACrD,MAAMC,QAAAA,GAAWjE,KAAKmD,OAAO,CAAC,gCAAA,IAAI,EAAC,eAAA,aAAA,CAAA,EAAA,GAAkBU,QAAAA,CAAAA;AACrD,IAAA,IAAI,CAACjE,gBAAAA,CAAiB,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,gBAAeqE,QAAAA,CAAAA,EAAW;AACnD,QAAA,MAAM,IAAIxD,2BAAAA,CAA4B,CAAC,cAAc,EAAEkD,MAAAA,CAAO,4BAA4B,CAAC,CAAA;AAC7F,IAAA;IACA,OAAOM,QAAAA;AACT;AA0BA,eAAA,YAAA,GAAA;AACE,IAAA,MAAMC,YAAAA,GAAe,+BAAA,CAAA,IAAI,EAAC,WAAA,SAAA,CAAA,CAAU1E,kBAAAA,CAAAA;AACpC,IAAA,IAAI,CAAE,MAAMe,WAAAA,CAAGe,UAAU,CAAC4C,YAAAA,CAAAA,EAAgB;AACxC,QAAA,MAAM,IAAIzD,2BAAAA,CACR,CAAC,QAAQ,EAAEjB,kBAAAA,CAAmB,sBAAsB,EAAE,+BAAA,CAAA,IAAI,EAAC,aAAA,CAAA,CAAA,aAAA,CAAA,CAAc,EAAE,CAAC,CAAA;AAEhF,IAAA;AACA,IAAA,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,aAAY,MAAMe,WAAAA,CAAG4D,QAAQ,CAACD,YAAAA,CAAAA;AACrC;AA2FA,eAAA,kBAAA,CAA0BvB,SAAsB,EAAEtB,UAAkB,EAAA;AAClE,IAAA,IAAI,CAAE,MAAMd,WAAAA,CAAGe,UAAU,CAACD,UAAAA,CAAAA,EAAc;AACtCsB,QAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACb,QAAA;AACF,IAAA;IAEA,MAAM3C,KAAAA,GAAQ,CAAC,MAAMlB,YAAGmB,OAAO,CAACL,UAAAA,CAAU,EAAGM,IAAI,EAAA;IACjD,KAAK,MAAMC,QAAQH,KAAAA,CAAO;AACxB,QAAA,MAAM4C,SAAAA,GAAYrE,IAAAA,CAAK8B,IAAI,CAACT,UAAAA,EAAYO,IAAAA,CAAAA;AACxC,QAAA,MAAMtB,IAAAA,GAAO,MAAMC,WAAAA,CAAGD,IAAI,CAAC+D,SAAAA,CAAAA;QAC3B,IAAI/D,IAAAA,CAAKyB,MAAM,EAAA,EAAI;AACjB,YAAA,IAAIP,WAAW,+BAAA,CAAA,IAAI,EAAC,cAAA,CAAA,CAAA,iBAAgB8C,GAAAA,CAAI1C,IAAAA,CAAAA;AACxC,YAAA,IAAI,CAACJ,QAAAA,EAAU;gBACb,IAAI;AACFA,oBAAAA,QAAAA,GAAW,MAAM,+BAAA,CAAA,IAAI,EAAC,oBAAA,kBAAA,CAAA,CAAmBI,IAAAA,CAAAA;AAC3C,gBAAA,CAAA,CAAE,OAAOK,KAAAA,EAAO;oBACdU,SAAAA,CAAUI,OAAO,CACf,IAAIC,qBAAAA,CAAsB,CAAC,4BAA4B,EAAEpB,MAAM,EAAE;wBAC/DqB,OAAAA,EAAS;AAAEhB,4BAAAA;AAAM;AACnB,qBAAA,CAAA,CAAA;AAEF,oBAAA;AACF,gBAAA;AACF,YAAA;YAEA,MAAMsC,cAAAA,GAAiBC,mBAAmBxE,IAAAA,CAAK4D,KAAK,CAAC9B,IAAI,CAAC,UAAU,SAAA,EAAWF,IAAAA,CAAAA,CAAAA;AAC/E,YAAA,MAAM6C,KAAAA,GAAgB;AACpBjD,gBAAAA,QAAAA;gBACAkD,QAAAA,EAAU9C,IAAAA;gBACV+C,QAAAA,EAAUJ,cAAAA;gBACVK,KAAAA,EAAO;AAAEC,oBAAAA,IAAAA,EAAMvE,KAAKuE;AAAK,iBAAA;gBACzBC,MAAAA,EAAQvE,WAAAA,CAAGwE,gBAAgB,CAACV,SAAAA;AAC9B,aAAA;AACA,YAAA,MAAMW,MAAMrC,SAAAA,EAAW8B,KAAAA,CAAAA;AACzB,QAAA;AACF,IAAA;AACA9B,IAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACf;AAEA,eAAA,kBAAyBM,QAAgB,EAAA;IACvC,MAAMR,YAAAA,GAAe,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,CAAU,QAAA,EAAU,UAAA,EAAY,CAAA,EAAGQ,QAAAA,CAAS,KAAK,CAAC,CAAA;AAC5E,IAAA,OAAOO,qBAAAA,CAAsB,MAAM1E,WAAAA,CAAG4D,QAAQ,CAACD,YAAAA,CAAAA,EAAeQ,QAAAA,CAAAA;AAChE;AAEA,eAAA,eAAsBQ,WAAmB,EAAA;IACvC,MAAMC,MAAAA,GAAS,+BAAA,CAAA,IAAI,EAAC,SAAA,CAAA,CAAA,SAAA,CAAA,CAAA,GAAaD,WAAAA,CAAYpB,KAAK,CAAC,GAAA,CAAA,CAAKC,MAAM,CAACC,OAAAA,CAAAA,CAAAA;AAC/D,IAAA,IAAI,CAAE,MAAMzD,WAAAA,CAAGe,UAAU,CAAC6D,MAAAA,CAAAA,EAAU;AAClC,QAAA,OAAO,EAAE;AACX,IAAA;AACA,IAAA,MAAM1D,KAAAA,GAAQ,MAAMlB,WAAAA,CAAGmB,OAAO,CAACyD,MAAAA,CAAAA;AAC/B,IAAA,OAAO1D,MACJsC,MAAM,CAAC,CAACqB,CAAAA,GAAMA,CAAAA,CAAEC,QAAQ,CAAC,QAAA,CAAA,CAAA,CACzB1D,IAAI,EAAA,CACJ2D,GAAG,CAAC,CAACF,IAAMpF,IAAAA,CAAK8B,IAAI,CAACqD,MAAAA,EAAQC,CAAAA,CAAAA,CAAAA;AAClC;AAEA,SAAA,qBAAsBF,WAAmB,EAAA;IACvC,MAAMvC,SAAAA,GAAY,IAAIC,WAAAA,CAAY;QAAEC,UAAAA,EAAY;AAAK,KAAA,CAAA;AACrD,IAAA,+BAAA,CAAA,IAAI,EAAC,WAAA,CAAA,CAAA,aAAY,CAAC,qBAAqB,EAAEqC,WAAAA,CAAAA,CAAa,CAAA;IAEtD,+BAAA,CAAA,IAAI,EAAC,2BAAA,CAAA,CAAA,2BAAA,CAAA,CAA4BvC,WAAWuC,WAAAA,CAAAA,CAAapC,KAAK,CAAC,CAACpC,CAAAA,GAAAA;QAC9DiC,SAAAA,CAAUI,OAAO,CACfrC,CAAAA,YAAayB,KAAAA,GAAQzB,IAAI,IAAIsC,qBAAAA,CAAsBX,OAAO3B,CAAAA,CAAAA,EAAI;YAAEuC,OAAAA,EAAS;gBAAEhB,KAAAA,EAAOvB;AAAE;AAAE,SAAA,CAAA,CAAA;AAE1F,IAAA,CAAA,CAAA;IAEA,OAAOiC,SAAAA;AACT;AAEA,eAAA,0BAAA,CAAkCA,SAAsB,EAAEuC,WAAmB,EAAA;AAC3E,IAAA,MAAMK,QAAQ,MAAM,+BAAA,CAAA,IAAI,EAAC,iBAAA,eAAA,CAAA,CAAgBL,WAAAA,CAAAA;IACzC,KAAK,MAAMM,WAAWD,KAAAA,CAAO;AAC3B,QAAA,MAAME,UAAAA,GAAa;YACjBC,MAAAA,CAAO;gBACLC,WAAAA,EAAa;AACf,aAAA,CAAA;YACA,CAACC,IAAAA,GAAyCA,KAAKC;AAChD,SAAA;AAED,QAAA,MAAMf,SAASvE,WAAAA,CAAGwE,gBAAgB,CAACS,OAAAA,CAAAA,CAASM,IAAI,CAACC,KAAAA,CAAMN,UAAAA,CAAAA,CAAAA;QAEvD,IAAI;YACF,WAAW,MAAMO,SAASlB,MAAAA,CAAQ;AAChC,gBAAA,MAAME,MAAMrC,SAAAA,EAAWqD,KAAAA,CAAAA;AACzB,YAAA;AACF,QAAA,CAAA,CAAE,OAAOtF,CAAAA,EAAY;AACnBiC,YAAAA,SAAAA,CAAUI,OAAO,CACf,IAAIC,qBAAAA,CAAsB,CAAC,uBAAuB,EAAEwC,OAAAA,CAAQ,EAAE,EAAG9E,CAAAA,CAAY0B,OAAO,EAAE,EAAE;gBACtFa,OAAAA,EAAS;oBACPhB,KAAAA,EAAOvB;AACT;AACF,aAAA,CAAA,CAAA;AAEF,YAAA;AACF,QAAA;AACF,IAAA;AACAiC,IAAAA,SAAAA,CAAUyB,GAAG,EAAA;AACf;;;;"}
|
package/dist/engine/index.js
CHANGED
|
@@ -5,7 +5,7 @@ var promises = require('stream/promises');
|
|
|
5
5
|
var path = require('path');
|
|
6
6
|
var os = require('os');
|
|
7
7
|
var streamChain = require('stream-chain');
|
|
8
|
-
var
|
|
8
|
+
var _ = require('lodash');
|
|
9
9
|
var semver = require('semver');
|
|
10
10
|
var index = require('./validation/schemas/index.js');
|
|
11
11
|
var provider = require('./validation/provider.js');
|
|
@@ -156,7 +156,7 @@ class TransferEngine {
|
|
|
156
156
|
return false;
|
|
157
157
|
}
|
|
158
158
|
// everything is included by default unless 'only' has been set
|
|
159
|
-
let included =
|
|
159
|
+
let included = _.isEmpty(only);
|
|
160
160
|
if (only && only.length > 0) {
|
|
161
161
|
included = only.some((transferGroup)=>{
|
|
162
162
|
return TransferGroupPresets[transferGroup][stage];
|
|
@@ -236,7 +236,7 @@ class TransferEngine {
|
|
|
236
236
|
destination: this.destinationProvider
|
|
237
237
|
};
|
|
238
238
|
// if we don't have any handlers, throw the original error
|
|
239
|
-
if (
|
|
239
|
+
if (_.isEmpty(_class_private_field_loose_base(this, _handlers)[_handlers].schemaDiff)) {
|
|
240
240
|
throw error;
|
|
241
241
|
}
|
|
242
242
|
await middleware.runMiddleware(context, _class_private_field_loose_base(this, _handlers)[_handlers].schemaDiff);
|
|
@@ -278,7 +278,7 @@ class TransferEngine {
|
|
|
278
278
|
_class_private_field_loose_base(this, _emitTransferUpdate)[_emitTransferUpdate]('error', {
|
|
279
279
|
error: e
|
|
280
280
|
});
|
|
281
|
-
const lastDiagnostic =
|
|
281
|
+
const lastDiagnostic = this.diagnostics.stack.items.at(-1);
|
|
282
282
|
// Do not report an error diagnostic if the last one reported the same error
|
|
283
283
|
if (e instanceof Error && (!lastDiagnostic || lastDiagnostic.kind !== 'error' || lastDiagnostic.details.error !== e)) {
|
|
284
284
|
this.reportError(e, e.severity || 'fatal');
|
|
@@ -381,7 +381,10 @@ class TransferEngine {
|
|
|
381
381
|
const { type, data } = entity;
|
|
382
382
|
const attributes = schemas[type].attributes;
|
|
383
383
|
const attributesToKeep = Object.keys(attributes).concat('documentId');
|
|
384
|
-
const updatedEntity =
|
|
384
|
+
const updatedEntity = {
|
|
385
|
+
...entity,
|
|
386
|
+
data: _.pick(data, attributesToKeep)
|
|
387
|
+
};
|
|
385
388
|
callback(null, updatedEntity);
|
|
386
389
|
}
|
|
387
390
|
})
|
|
@@ -616,7 +619,7 @@ function createStageTransformStream(key, options = {}) {
|
|
|
616
619
|
if (includeGlobal) {
|
|
617
620
|
applyTransforms(globalTransforms);
|
|
618
621
|
}
|
|
619
|
-
if (
|
|
622
|
+
if (_.isNumber(throttle) && throttle > 0) {
|
|
620
623
|
stream$2 = stream$2.pipe(new stream.PassThrough({
|
|
621
624
|
objectMode: true,
|
|
622
625
|
async transform (data, _encoding, callback) {
|
|
@@ -785,7 +788,9 @@ function assertSchemasMatching(sourceSchemas, destinationSchemas) {
|
|
|
785
788
|
if (strategy === 'ignore') {
|
|
786
789
|
return;
|
|
787
790
|
}
|
|
788
|
-
const keys =
|
|
791
|
+
const keys = [
|
|
792
|
+
...new Set(Object.keys(sourceSchemas).concat(Object.keys(destinationSchemas)))
|
|
793
|
+
];
|
|
789
794
|
const diffs = {};
|
|
790
795
|
keys.forEach((key)=>{
|
|
791
796
|
const sourceSchema = sourceSchemas[key];
|
|
@@ -795,7 +800,7 @@ function assertSchemasMatching(sourceSchemas, destinationSchemas) {
|
|
|
795
800
|
diffs[key] = schemaDiffs;
|
|
796
801
|
}
|
|
797
802
|
});
|
|
798
|
-
if (!
|
|
803
|
+
if (!_.isEmpty(diffs)) {
|
|
799
804
|
const formattedDiffs = Object.entries(diffs).map(([uid, ctDiffs])=>{
|
|
800
805
|
let msg = `- ${uid}:${os.EOL}`;
|
|
801
806
|
msg += ctDiffs.sort((a, b)=>a.kind > b.kind ? -1 : 1).map((diff)=>{
|