@strapi/data-transfer 4.6.0-alpha.0 → 4.6.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/encryption/decrypt.js +4 -4
- package/dist/encryption/encrypt.js +4 -4
- package/dist/engine/index.d.ts +11 -10
- package/dist/engine/index.js +145 -166
- package/dist/providers/{local-file-destination-provider.d.ts → local-file-destination-provider/index.d.ts} +9 -5
- package/dist/providers/{local-file-destination-provider.js → local-file-destination-provider/index.js} +54 -82
- package/dist/providers/local-file-destination-provider/utils.d.ts +9 -0
- package/dist/providers/local-file-destination-provider/utils.js +61 -0
- package/dist/providers/{local-file-source-provider.d.ts → local-file-source-provider/index.d.ts} +18 -22
- package/dist/providers/{local-file-source-provider.js → local-file-source-provider/index.js} +91 -36
- package/dist/providers/local-strapi-destination-provider/index.d.ts +31 -0
- package/dist/providers/local-strapi-destination-provider/index.js +187 -0
- package/dist/providers/local-strapi-destination-provider/strategies/index.d.ts +1 -0
- package/dist/providers/local-strapi-destination-provider/strategies/index.js +28 -0
- package/dist/providers/local-strapi-destination-provider/strategies/restore/configuration.d.ts +5 -0
- package/dist/providers/local-strapi-destination-provider/strategies/restore/configuration.js +44 -0
- package/dist/providers/local-strapi-destination-provider/strategies/restore/entities.d.ts +9 -0
- package/dist/providers/local-strapi-destination-provider/strategies/restore/entities.js +88 -0
- package/dist/providers/local-strapi-destination-provider/strategies/restore/index.d.ts +32 -0
- package/dist/providers/local-strapi-destination-provider/strategies/restore/index.js +106 -0
- package/dist/providers/local-strapi-destination-provider/strategies/restore/links.d.ts +3 -0
- package/dist/providers/local-strapi-destination-provider/strategies/restore/links.js +29 -0
- package/dist/providers/local-strapi-source-provider/assets.d.ts +5 -0
- package/dist/providers/local-strapi-source-provider/assets.js +31 -0
- package/dist/providers/local-strapi-source-provider/configuration.js +1 -1
- package/dist/providers/local-strapi-source-provider/entities.js +29 -9
- package/dist/providers/local-strapi-source-provider/index.d.ts +8 -7
- package/dist/providers/local-strapi-source-provider/index.js +32 -2
- package/dist/providers/local-strapi-source-provider/{links/index.d.ts → links.d.ts} +1 -1
- package/dist/providers/local-strapi-source-provider/links.js +23 -0
- package/dist/providers/shared/index.d.ts +1 -0
- package/dist/providers/shared/index.js +28 -0
- package/dist/providers/shared/strapi/entity.d.ts +19 -0
- package/dist/providers/shared/strapi/entity.js +130 -0
- package/dist/providers/shared/strapi/index.d.ts +2 -0
- package/dist/providers/shared/strapi/index.js +29 -0
- package/dist/providers/shared/strapi/link.d.ts +6 -0
- package/dist/providers/shared/strapi/link.js +201 -0
- package/dist/providers/test-utils/index.d.ts +111 -0
- package/dist/providers/test-utils/index.js +64 -0
- package/dist/strategies/index.d.ts +3 -3
- package/dist/strategies/index.js +27 -4
- package/dist/utils/index.d.ts +3 -0
- package/dist/utils/index.js +30 -0
- package/dist/utils/json.d.ts +23 -0
- package/dist/{utils.js → utils/json.js} +9 -38
- package/dist/utils/schema.d.ts +3 -0
- package/dist/utils/schema.js +22 -0
- package/dist/utils/stream.d.ts +10 -0
- package/dist/utils/stream.js +39 -0
- package/jest.config.js +3 -1
- package/package.json +4 -4
- package/dist/providers/local-strapi-destination-provider.d.ts +0 -22
- package/dist/providers/local-strapi-destination-provider.js +0 -78
- package/dist/providers/local-strapi-source-provider/links/index.js +0 -37
- package/dist/providers/local-strapi-source-provider/links/utils.d.ts +0 -27
- package/dist/providers/local-strapi-source-provider/links/utils.js +0 -155
- package/dist/utils.d.ts +0 -10
|
@@ -5,25 +5,25 @@ const crypto_1 = require("crypto");
|
|
|
5
5
|
// different key values depending on algorithm chosen
|
|
6
6
|
const getDecryptionStrategy = (algorithm) => {
|
|
7
7
|
const strategies = {
|
|
8
|
-
'aes-128-ecb'
|
|
8
|
+
'aes-128-ecb'(key) {
|
|
9
9
|
const hashedKey = (0, crypto_1.scryptSync)(key, '', 16);
|
|
10
10
|
const initVector = null;
|
|
11
11
|
const securityKey = hashedKey;
|
|
12
12
|
return (0, crypto_1.createDecipheriv)(algorithm, securityKey, initVector);
|
|
13
13
|
},
|
|
14
|
-
aes128
|
|
14
|
+
aes128(key) {
|
|
15
15
|
const hashedKey = (0, crypto_1.scryptSync)(key, '', 32);
|
|
16
16
|
const initVector = hashedKey.slice(16);
|
|
17
17
|
const securityKey = hashedKey.slice(0, 16);
|
|
18
18
|
return (0, crypto_1.createDecipheriv)(algorithm, securityKey, initVector);
|
|
19
19
|
},
|
|
20
|
-
aes192
|
|
20
|
+
aes192(key) {
|
|
21
21
|
const hashedKey = (0, crypto_1.scryptSync)(key, '', 40);
|
|
22
22
|
const initVector = hashedKey.slice(24);
|
|
23
23
|
const securityKey = hashedKey.slice(0, 24);
|
|
24
24
|
return (0, crypto_1.createDecipheriv)(algorithm, securityKey, initVector);
|
|
25
25
|
},
|
|
26
|
-
aes256
|
|
26
|
+
aes256(key) {
|
|
27
27
|
const hashedKey = (0, crypto_1.scryptSync)(key, '', 48);
|
|
28
28
|
const initVector = hashedKey.slice(32);
|
|
29
29
|
const securityKey = hashedKey.slice(0, 32);
|
|
@@ -5,25 +5,25 @@ const crypto_1 = require("crypto");
|
|
|
5
5
|
// different key values depending on algorithm chosen
|
|
6
6
|
const getEncryptionStrategy = (algorithm) => {
|
|
7
7
|
const strategies = {
|
|
8
|
-
'aes-128-ecb'
|
|
8
|
+
'aes-128-ecb'(key) {
|
|
9
9
|
const hashedKey = (0, crypto_1.scryptSync)(key, '', 16);
|
|
10
10
|
const initVector = null;
|
|
11
11
|
const securityKey = hashedKey;
|
|
12
12
|
return (0, crypto_1.createCipheriv)(algorithm, securityKey, initVector);
|
|
13
13
|
},
|
|
14
|
-
aes128
|
|
14
|
+
aes128(key) {
|
|
15
15
|
const hashedKey = (0, crypto_1.scryptSync)(key, '', 32);
|
|
16
16
|
const initVector = hashedKey.slice(16);
|
|
17
17
|
const securityKey = hashedKey.slice(0, 16);
|
|
18
18
|
return (0, crypto_1.createCipheriv)(algorithm, securityKey, initVector);
|
|
19
19
|
},
|
|
20
|
-
aes192
|
|
20
|
+
aes192(key) {
|
|
21
21
|
const hashedKey = (0, crypto_1.scryptSync)(key, '', 40);
|
|
22
22
|
const initVector = hashedKey.slice(24);
|
|
23
23
|
const securityKey = hashedKey.slice(0, 24);
|
|
24
24
|
return (0, crypto_1.createCipheriv)(algorithm, securityKey, initVector);
|
|
25
25
|
},
|
|
26
|
-
aes256
|
|
26
|
+
aes256(key) {
|
|
27
27
|
const hashedKey = (0, crypto_1.scryptSync)(key, '', 48);
|
|
28
28
|
const initVector = hashedKey.slice(32);
|
|
29
29
|
const securityKey = hashedKey.slice(0, 32);
|
package/dist/engine/index.d.ts
CHANGED
|
@@ -1,28 +1,29 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { PassThrough } from 'stream
|
|
3
|
-
import type { IDestinationProvider, ISourceProvider, ITransferEngine, ITransferEngineOptions, ITransferResults } from '../../types';
|
|
4
|
-
declare
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
};
|
|
8
|
-
export declare const VALID_STRATEGIES: string[];
|
|
2
|
+
import { PassThrough } from 'stream';
|
|
3
|
+
import type { IDestinationProvider, ISourceProvider, ITransferEngine, ITransferEngineOptions, TransferProgress, ITransferResults, TransferStage } from '../../types';
|
|
4
|
+
export declare const TRANSFER_STAGES: ReadonlyArray<TransferStage>;
|
|
5
|
+
export declare const DEFAULT_VERSION_STRATEGY = "ignore";
|
|
6
|
+
export declare const DEFAULT_SCHEMA_STRATEGY = "strict";
|
|
9
7
|
declare class TransferEngine<S extends ISourceProvider = ISourceProvider, D extends IDestinationProvider = IDestinationProvider> implements ITransferEngine {
|
|
10
8
|
#private;
|
|
11
9
|
sourceProvider: ISourceProvider;
|
|
12
10
|
destinationProvider: IDestinationProvider;
|
|
13
11
|
options: ITransferEngineOptions;
|
|
14
|
-
|
|
12
|
+
progress: {
|
|
13
|
+
data: TransferProgress;
|
|
14
|
+
stream: PassThrough;
|
|
15
|
+
};
|
|
15
16
|
constructor(sourceProvider: ISourceProvider, destinationProvider: IDestinationProvider, options: ITransferEngineOptions);
|
|
16
17
|
init(): Promise<void>;
|
|
17
18
|
bootstrap(): Promise<void>;
|
|
18
19
|
close(): Promise<void>;
|
|
19
20
|
integrityCheck(): Promise<boolean>;
|
|
20
|
-
validateTransferOptions(): void;
|
|
21
21
|
transfer(): Promise<ITransferResults<S, D>>;
|
|
22
|
+
beforeTransfer(): Promise<void>;
|
|
22
23
|
transferSchemas(): Promise<void>;
|
|
23
24
|
transferEntities(): Promise<void>;
|
|
24
25
|
transferLinks(): Promise<void>;
|
|
25
|
-
|
|
26
|
+
transferAssets(): Promise<void>;
|
|
26
27
|
transferConfiguration(): Promise<void>;
|
|
27
28
|
}
|
|
28
29
|
export declare const createTransferEngine: <S extends ISourceProvider = ISourceProvider, D extends IDestinationProvider = IDestinationProvider>(sourceProvider: S, destinationProvider: D, options: ITransferEngineOptions) => TransferEngine<S, D>;
|
package/dist/engine/index.js
CHANGED
|
@@ -7,37 +7,28 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
7
7
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
8
8
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
9
9
|
};
|
|
10
|
-
var _TransferEngine_instances, _TransferEngine_metadata,
|
|
10
|
+
var _TransferEngine_instances, _TransferEngine_metadata, _TransferEngine_createStageTransformStream, _TransferEngine_updateTransferProgress, _TransferEngine_progressTracker, _TransferEngine_emitTransferUpdate, _TransferEngine_emitStageUpdate, _TransferEngine_assertStrapiVersionIntegrity, _TransferEngine_assertSchemasMatching, _TransferEngine_transferStage, _TransferEngine_resolveProviderResource;
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.createTransferEngine = exports.
|
|
13
|
-
const
|
|
12
|
+
exports.createTransferEngine = exports.DEFAULT_SCHEMA_STRATEGY = exports.DEFAULT_VERSION_STRATEGY = exports.TRANSFER_STAGES = void 0;
|
|
13
|
+
const stream_1 = require("stream");
|
|
14
|
+
const path_1 = require("path");
|
|
14
15
|
const fp_1 = require("lodash/fp");
|
|
15
|
-
const
|
|
16
|
+
const semver_1 = require("semver");
|
|
16
17
|
const strategies_1 = __importDefault(require("../strategies"));
|
|
17
|
-
|
|
18
|
+
const stream_2 = require("../utils/stream");
|
|
19
|
+
exports.TRANSFER_STAGES = Object.freeze([
|
|
20
|
+
'entities',
|
|
21
|
+
'links',
|
|
22
|
+
'assets',
|
|
23
|
+
'schemas',
|
|
24
|
+
'configuration',
|
|
25
|
+
]);
|
|
26
|
+
exports.DEFAULT_VERSION_STRATEGY = 'ignore';
|
|
27
|
+
exports.DEFAULT_SCHEMA_STRATEGY = 'strict';
|
|
18
28
|
class TransferEngine {
|
|
19
29
|
constructor(sourceProvider, destinationProvider, options) {
|
|
20
30
|
_TransferEngine_instances.add(this);
|
|
21
31
|
_TransferEngine_metadata.set(this, {});
|
|
22
|
-
_TransferEngine_transferProgress.set(this, {});
|
|
23
|
-
// TODO: Type the stream chunks. Doesn't seem trivial, especially since PassThrough doesn't provide a PassThroughOptions type
|
|
24
|
-
_TransferEngine_progressStream.set(this, new stream_chain_1.PassThrough({ objectMode: true }));
|
|
25
|
-
_TransferEngine_countRecorder.set(this, (transferStage, aggregateKey) => {
|
|
26
|
-
return new stream_chain_1.PassThrough({
|
|
27
|
-
objectMode: true,
|
|
28
|
-
transform: (data, _encoding, callback) => {
|
|
29
|
-
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_increaseTransferProgress).call(this, transferStage, data, aggregateKey);
|
|
30
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'progress', transferStage);
|
|
31
|
-
callback(null, data);
|
|
32
|
-
},
|
|
33
|
-
});
|
|
34
|
-
});
|
|
35
|
-
_TransferEngine_updateStage.set(this, (type, transferStage) => {
|
|
36
|
-
__classPrivateFieldGet(this, _TransferEngine_progressStream, "f").emit(type, {
|
|
37
|
-
data: __classPrivateFieldGet(this, _TransferEngine_transferProgress, "f"),
|
|
38
|
-
stage: transferStage,
|
|
39
|
-
});
|
|
40
|
-
});
|
|
41
32
|
if (sourceProvider.type !== 'source') {
|
|
42
33
|
throw new Error("SourceProvider does not have type 'source'");
|
|
43
34
|
}
|
|
@@ -47,12 +38,7 @@ class TransferEngine {
|
|
|
47
38
|
this.sourceProvider = sourceProvider;
|
|
48
39
|
this.destinationProvider = destinationProvider;
|
|
49
40
|
this.options = options;
|
|
50
|
-
|
|
51
|
-
get progress() {
|
|
52
|
-
return {
|
|
53
|
-
data: __classPrivateFieldGet(this, _TransferEngine_transferProgress, "f"),
|
|
54
|
-
stream: __classPrivateFieldGet(this, _TransferEngine_progressStream, "f"),
|
|
55
|
-
};
|
|
41
|
+
this.progress = { data: {}, stream: new stream_1.PassThrough({ objectMode: true }) };
|
|
56
42
|
}
|
|
57
43
|
async init() {
|
|
58
44
|
// Resolve providers' resource and store
|
|
@@ -77,8 +63,8 @@ class TransferEngine {
|
|
|
77
63
|
if (sourceMetadata && destinationMetadata) {
|
|
78
64
|
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_assertStrapiVersionIntegrity).call(this, sourceMetadata?.strapi?.version, destinationMetadata?.strapi?.version);
|
|
79
65
|
}
|
|
80
|
-
const sourceSchemas = await this.sourceProvider.getSchemas?.();
|
|
81
|
-
const destinationSchemas = await this.destinationProvider.getSchemas?.();
|
|
66
|
+
const sourceSchemas = (await this.sourceProvider.getSchemas?.());
|
|
67
|
+
const destinationSchemas = (await this.destinationProvider.getSchemas?.());
|
|
82
68
|
if (sourceSchemas && destinationSchemas) {
|
|
83
69
|
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_assertSchemasMatching).call(this, sourceSchemas, destinationSchemas);
|
|
84
70
|
}
|
|
@@ -88,30 +74,31 @@ class TransferEngine {
|
|
|
88
74
|
return false;
|
|
89
75
|
}
|
|
90
76
|
}
|
|
91
|
-
validateTransferOptions() {
|
|
92
|
-
if (!exports.VALID_STRATEGIES.includes(this.options.strategy)) {
|
|
93
|
-
throw new Error('Invalid stategy ' + this.options.strategy);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
77
|
async transfer() {
|
|
78
|
+
// reset data between transfers
|
|
79
|
+
this.progress.data = {};
|
|
80
|
+
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_emitTransferUpdate).call(this, 'start');
|
|
97
81
|
try {
|
|
98
|
-
this.validateTransferOptions();
|
|
99
82
|
await this.bootstrap();
|
|
100
83
|
await this.init();
|
|
101
84
|
const isValidTransfer = await this.integrityCheck();
|
|
102
85
|
if (!isValidTransfer) {
|
|
86
|
+
// TODO: provide the log from the integrity check
|
|
103
87
|
throw new Error(`Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.`);
|
|
104
88
|
}
|
|
89
|
+
await this.beforeTransfer();
|
|
105
90
|
// Run the transfer stages
|
|
106
91
|
await this.transferSchemas();
|
|
107
92
|
await this.transferEntities();
|
|
108
|
-
await this.
|
|
93
|
+
await this.transferAssets();
|
|
109
94
|
await this.transferLinks();
|
|
110
95
|
await this.transferConfiguration();
|
|
111
96
|
// Gracefully close the providers
|
|
112
97
|
await this.close();
|
|
98
|
+
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_emitTransferUpdate).call(this, 'finish');
|
|
113
99
|
}
|
|
114
100
|
catch (e) {
|
|
101
|
+
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_emitTransferUpdate).call(this, 'error', { error: e });
|
|
115
102
|
// Rollback the destination provider if an exception is thrown during the transfer
|
|
116
103
|
// Note: This will be configurable in the future
|
|
117
104
|
await this.destinationProvider.rollback?.(e);
|
|
@@ -120,147 +107,118 @@ class TransferEngine {
|
|
|
120
107
|
return {
|
|
121
108
|
source: this.sourceProvider.results,
|
|
122
109
|
destination: this.destinationProvider.results,
|
|
110
|
+
engine: this.progress.data,
|
|
123
111
|
};
|
|
124
112
|
}
|
|
113
|
+
async beforeTransfer() {
|
|
114
|
+
await this.sourceProvider.beforeTransfer?.();
|
|
115
|
+
await this.destinationProvider.beforeTransfer?.();
|
|
116
|
+
}
|
|
125
117
|
async transferSchemas() {
|
|
126
|
-
const
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
if (!outStream) {
|
|
133
|
-
return;
|
|
134
|
-
}
|
|
135
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'start', stageName);
|
|
136
|
-
return new Promise((resolve, reject) => {
|
|
137
|
-
inStream
|
|
138
|
-
// Throw on error in the source
|
|
139
|
-
.on('error', reject);
|
|
140
|
-
outStream
|
|
141
|
-
// Throw on error in the destination
|
|
142
|
-
.on('error', reject)
|
|
143
|
-
// Resolve the promise when the destination has finished reading all the data from the source
|
|
144
|
-
.on('close', () => {
|
|
145
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'complete', stageName);
|
|
146
|
-
resolve();
|
|
147
|
-
});
|
|
148
|
-
inStream.pipe(__classPrivateFieldGet(this, _TransferEngine_countRecorder, "f").call(this, stageName)).pipe(outStream);
|
|
149
|
-
});
|
|
118
|
+
const stage = 'schemas';
|
|
119
|
+
const source = await this.sourceProvider.streamSchemas?.();
|
|
120
|
+
const destination = await this.destinationProvider.getSchemasStream?.();
|
|
121
|
+
const transform = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_createStageTransformStream).call(this, stage);
|
|
122
|
+
const tracker = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_progressTracker).call(this, stage, { key: (value) => value.modelType });
|
|
123
|
+
await __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_transferStage).call(this, { stage, source, destination, transform, tracker });
|
|
150
124
|
}
|
|
151
125
|
async transferEntities() {
|
|
152
|
-
const
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
if (!outStream) {
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'start', stageName);
|
|
162
|
-
return new Promise((resolve, reject) => {
|
|
163
|
-
inStream
|
|
164
|
-
// Throw on error in the source
|
|
165
|
-
.on('error', (e) => {
|
|
166
|
-
reject(e);
|
|
167
|
-
});
|
|
168
|
-
outStream
|
|
169
|
-
// Throw on error in the destination
|
|
170
|
-
.on('error', (e) => {
|
|
171
|
-
reject(e);
|
|
172
|
-
})
|
|
173
|
-
// Resolve the promise when the destination has finished reading all the data from the source
|
|
174
|
-
.on('close', () => {
|
|
175
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'complete', stageName);
|
|
176
|
-
resolve();
|
|
177
|
-
});
|
|
178
|
-
inStream.pipe(__classPrivateFieldGet(this, _TransferEngine_countRecorder, "f").call(this, stageName, 'type')).pipe(outStream);
|
|
179
|
-
});
|
|
126
|
+
const stage = 'entities';
|
|
127
|
+
const source = await this.sourceProvider.streamEntities?.();
|
|
128
|
+
const destination = await this.destinationProvider.getEntitiesStream?.();
|
|
129
|
+
const transform = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_createStageTransformStream).call(this, stage);
|
|
130
|
+
const tracker = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_progressTracker).call(this, stage, { key: (value) => value.type });
|
|
131
|
+
await __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_transferStage).call(this, { stage, source, destination, transform, tracker });
|
|
180
132
|
}
|
|
181
133
|
async transferLinks() {
|
|
182
|
-
const
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
if (!outStream) {
|
|
189
|
-
return;
|
|
190
|
-
}
|
|
191
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'start', 'links');
|
|
192
|
-
return new Promise((resolve, reject) => {
|
|
193
|
-
inStream
|
|
194
|
-
// Throw on error in the source
|
|
195
|
-
.on('error', reject);
|
|
196
|
-
outStream
|
|
197
|
-
// Throw on error in the destination
|
|
198
|
-
.on('error', reject)
|
|
199
|
-
// Resolve the promise when the destination has finished reading all the data from the source
|
|
200
|
-
.on('close', () => {
|
|
201
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'complete', stageName);
|
|
202
|
-
resolve();
|
|
203
|
-
});
|
|
204
|
-
inStream.pipe(__classPrivateFieldGet(this, _TransferEngine_countRecorder, "f").call(this, stageName)).pipe(outStream);
|
|
205
|
-
});
|
|
134
|
+
const stage = 'links';
|
|
135
|
+
const source = await this.sourceProvider.streamLinks?.();
|
|
136
|
+
const destination = await this.destinationProvider.getLinksStream?.();
|
|
137
|
+
const transform = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_createStageTransformStream).call(this, stage);
|
|
138
|
+
const tracker = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_progressTracker).call(this, stage);
|
|
139
|
+
await __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_transferStage).call(this, { stage, source, destination, transform, tracker });
|
|
206
140
|
}
|
|
207
|
-
async
|
|
208
|
-
const
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
141
|
+
async transferAssets() {
|
|
142
|
+
const stage = 'assets';
|
|
143
|
+
const source = await this.sourceProvider.streamAssets?.();
|
|
144
|
+
const destination = await this.destinationProvider.getAssetsStream?.();
|
|
145
|
+
const transform = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_createStageTransformStream).call(this, stage);
|
|
146
|
+
const tracker = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_progressTracker).call(this, stage, {
|
|
147
|
+
size: (value) => value.stats.size,
|
|
148
|
+
key: (value) => (0, path_1.extname)(value.filename),
|
|
149
|
+
});
|
|
150
|
+
await __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_transferStage).call(this, { stage, source, destination, transform, tracker });
|
|
215
151
|
}
|
|
216
152
|
async transferConfiguration() {
|
|
217
|
-
const
|
|
218
|
-
const
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
if (!outStream) {
|
|
224
|
-
return;
|
|
225
|
-
}
|
|
226
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'start', stageName);
|
|
227
|
-
return new Promise((resolve, reject) => {
|
|
228
|
-
inStream
|
|
229
|
-
// Throw on error in the source
|
|
230
|
-
.on('error', reject);
|
|
231
|
-
outStream
|
|
232
|
-
// Throw on error in the destination
|
|
233
|
-
.on('error', reject)
|
|
234
|
-
// Resolve the promise when the destination has finished reading all the data from the source
|
|
235
|
-
.on('close', () => {
|
|
236
|
-
__classPrivateFieldGet(this, _TransferEngine_updateStage, "f").call(this, 'complete', stageName);
|
|
237
|
-
resolve();
|
|
238
|
-
});
|
|
239
|
-
inStream.pipe(__classPrivateFieldGet(this, _TransferEngine_countRecorder, "f").call(this, stageName)).pipe(outStream);
|
|
240
|
-
});
|
|
153
|
+
const stage = 'configuration';
|
|
154
|
+
const source = await this.sourceProvider.streamConfiguration?.();
|
|
155
|
+
const destination = await this.destinationProvider.getConfigurationStream?.();
|
|
156
|
+
const transform = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_createStageTransformStream).call(this, stage);
|
|
157
|
+
const tracker = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_progressTracker).call(this, stage);
|
|
158
|
+
await __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_transferStage).call(this, { stage, source, destination, transform, tracker });
|
|
241
159
|
}
|
|
242
160
|
}
|
|
243
|
-
_TransferEngine_metadata = new WeakMap(),
|
|
244
|
-
|
|
245
|
-
|
|
161
|
+
_TransferEngine_metadata = new WeakMap(), _TransferEngine_instances = new WeakSet(), _TransferEngine_createStageTransformStream = function _TransferEngine_createStageTransformStream(key, options = {}) {
|
|
162
|
+
const { includeGlobal = true } = options;
|
|
163
|
+
const { global: globalTransforms, [key]: stageTransforms } = this.options?.transforms ?? {};
|
|
164
|
+
let stream = new stream_1.PassThrough({ objectMode: true });
|
|
165
|
+
const applyTransforms = (transforms = []) => {
|
|
166
|
+
for (const transform of transforms) {
|
|
167
|
+
if ('filter' in transform) {
|
|
168
|
+
stream = stream.pipe((0, stream_2.filter)(transform.filter));
|
|
169
|
+
}
|
|
170
|
+
if ('map' in transform) {
|
|
171
|
+
stream = stream.pipe((0, stream_2.map)(transform.map));
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
if (includeGlobal) {
|
|
176
|
+
applyTransforms(globalTransforms);
|
|
177
|
+
}
|
|
178
|
+
applyTransforms(stageTransforms);
|
|
179
|
+
return stream;
|
|
180
|
+
}, _TransferEngine_updateTransferProgress = function _TransferEngine_updateTransferProgress(stage, data, aggregate) {
|
|
181
|
+
if (!this.progress.data[stage]) {
|
|
182
|
+
this.progress.data[stage] = { count: 0, bytes: 0 };
|
|
183
|
+
}
|
|
184
|
+
const stageProgress = this.progress.data[stage];
|
|
185
|
+
if (!stageProgress) {
|
|
186
|
+
return;
|
|
246
187
|
}
|
|
247
|
-
|
|
248
|
-
const
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
188
|
+
const size = aggregate?.size?.(data) ?? JSON.stringify(data).length;
|
|
189
|
+
const key = aggregate?.key?.(data);
|
|
190
|
+
stageProgress.count += 1;
|
|
191
|
+
stageProgress.bytes += size;
|
|
192
|
+
// Handle aggregate updates if necessary
|
|
193
|
+
if (key) {
|
|
194
|
+
if (!stageProgress.aggregates) {
|
|
195
|
+
stageProgress.aggregates = {};
|
|
254
196
|
}
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
197
|
+
const { aggregates } = stageProgress;
|
|
198
|
+
if (!aggregates[key]) {
|
|
199
|
+
aggregates[key] = { count: 0, bytes: 0 };
|
|
258
200
|
}
|
|
259
|
-
|
|
260
|
-
|
|
201
|
+
aggregates[key].count += 1;
|
|
202
|
+
aggregates[key].bytes += size;
|
|
261
203
|
}
|
|
204
|
+
}, _TransferEngine_progressTracker = function _TransferEngine_progressTracker(stage, aggregate) {
|
|
205
|
+
return new stream_1.PassThrough({
|
|
206
|
+
objectMode: true,
|
|
207
|
+
transform: (data, _encoding, callback) => {
|
|
208
|
+
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_updateTransferProgress).call(this, stage, data, aggregate);
|
|
209
|
+
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_emitStageUpdate).call(this, 'progress', stage);
|
|
210
|
+
callback(null, data);
|
|
211
|
+
},
|
|
212
|
+
});
|
|
213
|
+
}, _TransferEngine_emitTransferUpdate = function _TransferEngine_emitTransferUpdate(type, payload) {
|
|
214
|
+
this.progress.stream.emit(`transfer::${type}`, payload);
|
|
215
|
+
}, _TransferEngine_emitStageUpdate = function _TransferEngine_emitStageUpdate(type, transferStage) {
|
|
216
|
+
this.progress.stream.emit(`stage::${type}`, {
|
|
217
|
+
data: this.progress.data,
|
|
218
|
+
stage: transferStage,
|
|
219
|
+
});
|
|
262
220
|
}, _TransferEngine_assertStrapiVersionIntegrity = function _TransferEngine_assertStrapiVersionIntegrity(sourceVersion, destinationVersion) {
|
|
263
|
-
const strategy = this.options.
|
|
221
|
+
const strategy = this.options.versionStrategy || exports.DEFAULT_VERSION_STRATEGY;
|
|
264
222
|
if (!sourceVersion ||
|
|
265
223
|
!destinationVersion ||
|
|
266
224
|
strategy === 'ignore' ||
|
|
@@ -269,7 +227,7 @@ _TransferEngine_metadata = new WeakMap(), _TransferEngine_transferProgress = new
|
|
|
269
227
|
}
|
|
270
228
|
let diff;
|
|
271
229
|
try {
|
|
272
|
-
diff =
|
|
230
|
+
diff = (0, semver_1.diff)(sourceVersion, destinationVersion);
|
|
273
231
|
}
|
|
274
232
|
catch (e) {
|
|
275
233
|
throw new Error(`Strapi versions doesn't match (${strategy} check): ${sourceVersion} does not match with ${destinationVersion}`);
|
|
@@ -291,7 +249,10 @@ _TransferEngine_metadata = new WeakMap(), _TransferEngine_transferProgress = new
|
|
|
291
249
|
}
|
|
292
250
|
throw new Error(`Strapi versions doesn't match (${strategy} check): ${sourceVersion} does not match with ${destinationVersion}`);
|
|
293
251
|
}, _TransferEngine_assertSchemasMatching = function _TransferEngine_assertSchemasMatching(sourceSchemas, destinationSchemas) {
|
|
294
|
-
const strategy = this.options.
|
|
252
|
+
const strategy = this.options.schemaStrategy || exports.DEFAULT_SCHEMA_STRATEGY;
|
|
253
|
+
if (strategy === 'ignore') {
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
295
256
|
const keys = (0, fp_1.uniq)(Object.keys(sourceSchemas).concat(Object.keys(destinationSchemas)));
|
|
296
257
|
const diffs = {};
|
|
297
258
|
keys.forEach((key) => {
|
|
@@ -304,9 +265,27 @@ _TransferEngine_metadata = new WeakMap(), _TransferEngine_transferProgress = new
|
|
|
304
265
|
});
|
|
305
266
|
if (!(0, fp_1.isEmpty)(diffs)) {
|
|
306
267
|
throw new Error(`Import process failed because the project doesn't have a matching data structure
|
|
307
|
-
${JSON.stringify(diffs, null, 2)}
|
|
268
|
+
${JSON.stringify(diffs, null, 2)}
|
|
308
269
|
`);
|
|
309
270
|
}
|
|
271
|
+
}, _TransferEngine_transferStage = async function _TransferEngine_transferStage(options) {
|
|
272
|
+
const { stage, source, destination, transform, tracker } = options;
|
|
273
|
+
if (!source || !destination) {
|
|
274
|
+
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_emitStageUpdate).call(this, 'skip', stage);
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_emitStageUpdate).call(this, 'start', stage);
|
|
278
|
+
await new Promise((resolve, reject) => {
|
|
279
|
+
let stream = source;
|
|
280
|
+
if (transform) {
|
|
281
|
+
stream = stream.pipe(transform);
|
|
282
|
+
}
|
|
283
|
+
if (tracker) {
|
|
284
|
+
stream = stream.pipe(tracker);
|
|
285
|
+
}
|
|
286
|
+
stream.pipe(destination).on('error', reject).on('close', resolve);
|
|
287
|
+
});
|
|
288
|
+
__classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_emitStageUpdate).call(this, 'finish', stage);
|
|
310
289
|
}, _TransferEngine_resolveProviderResource = async function _TransferEngine_resolveProviderResource() {
|
|
311
290
|
const sourceMetadata = await this.sourceProvider.getMetadata();
|
|
312
291
|
const destinationMetadata = await this.destinationProvider.getMetadata();
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
/// <reference types="stream-chain" />
|
|
2
1
|
/// <reference types="node" />
|
|
3
|
-
|
|
2
|
+
/// <reference types="stream-chain" />
|
|
3
|
+
import zlib from 'zlib';
|
|
4
|
+
import { Writable } from 'stream';
|
|
5
|
+
import type { IDestinationProvider, IDestinationProviderTransferResults, IMetadata, ProviderType } from '../../../types';
|
|
4
6
|
export interface ILocalFileDestinationProviderOptions {
|
|
5
7
|
encryption: {
|
|
6
8
|
enabled: boolean;
|
|
@@ -29,13 +31,15 @@ declare class LocalFileDestinationProvider implements IDestinationProvider {
|
|
|
29
31
|
results: ILocalFileDestinationProviderTransferResults;
|
|
30
32
|
constructor(options: ILocalFileDestinationProviderOptions);
|
|
31
33
|
setMetadata(target: ProviderType, metadata: IMetadata): IDestinationProvider;
|
|
34
|
+
createGzip(): zlib.Gzip;
|
|
32
35
|
bootstrap(): void | Promise<void>;
|
|
33
36
|
close(): Promise<void>;
|
|
34
37
|
rollback(): Promise<void>;
|
|
35
38
|
getMetadata(): null;
|
|
36
39
|
getSchemasStream(): import("stream-chain");
|
|
37
|
-
getEntitiesStream():
|
|
38
|
-
getLinksStream():
|
|
39
|
-
getConfigurationStream():
|
|
40
|
+
getEntitiesStream(): Writable;
|
|
41
|
+
getLinksStream(): Writable;
|
|
42
|
+
getConfigurationStream(): Writable;
|
|
43
|
+
getAssetsStream(): Writable;
|
|
40
44
|
}
|
|
41
45
|
export {};
|