@strapi/data-transfer 4.6.0-alpha.0 → 4.6.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/jest.config.js +3 -1
- package/{dist → lib}/engine/index.d.ts +17 -10
- package/lib/engine/index.js +359 -0
- package/lib/engine/validation/index.d.ts +1 -0
- package/lib/engine/validation/index.js +28 -0
- package/lib/engine/validation/schemas/index.d.ts +7 -0
- package/lib/engine/validation/schemas/index.js +53 -0
- package/lib/file/index.d.ts +1 -0
- package/lib/file/index.js +28 -0
- package/{dist/providers/local-file-destination-provider.d.ts → lib/file/providers/destination/index.d.ts} +10 -6
- package/{dist/providers/local-file-destination-provider.js → lib/file/providers/destination/index.js} +59 -87
- package/lib/file/providers/destination/utils.d.ts +9 -0
- package/lib/file/providers/destination/utils.js +61 -0
- package/lib/file/providers/index.d.ts +2 -0
- package/{dist → lib/file}/providers/index.js +2 -6
- package/lib/file/providers/source/index.d.ts +40 -0
- package/{dist/providers/local-file-source-provider.js → lib/file/providers/source/index.js} +94 -51
- package/lib/index.d.ts +4 -0
- package/lib/index.js +31 -0
- package/lib/strapi/index.d.ts +4 -0
- package/lib/strapi/index.js +35 -0
- package/lib/strapi/providers/index.d.ts +3 -0
- package/lib/strapi/providers/index.js +22 -0
- package/lib/strapi/providers/local-destination/index.d.ts +32 -0
- package/lib/strapi/providers/local-destination/index.js +195 -0
- package/lib/strapi/providers/local-destination/strategies/index.d.ts +1 -0
- package/lib/strapi/providers/local-destination/strategies/index.js +28 -0
- package/lib/strapi/providers/local-destination/strategies/restore/configuration.d.ts +5 -0
- package/lib/strapi/providers/local-destination/strategies/restore/configuration.js +44 -0
- package/lib/strapi/providers/local-destination/strategies/restore/entities.d.ts +9 -0
- package/lib/strapi/providers/local-destination/strategies/restore/entities.js +100 -0
- package/lib/strapi/providers/local-destination/strategies/restore/index.d.ts +32 -0
- package/lib/strapi/providers/local-destination/strategies/restore/index.js +106 -0
- package/lib/strapi/providers/local-destination/strategies/restore/links.d.ts +3 -0
- package/lib/strapi/providers/local-destination/strategies/restore/links.js +29 -0
- package/lib/strapi/providers/local-source/assets.d.ts +5 -0
- package/lib/strapi/providers/local-source/assets.js +31 -0
- package/{dist/providers/local-strapi-source-provider → lib/strapi/providers/local-source}/configuration.d.ts +0 -0
- package/{dist/providers/local-strapi-source-provider → lib/strapi/providers/local-source}/configuration.js +13 -14
- package/{dist/providers/local-strapi-source-provider → lib/strapi/providers/local-source}/entities.d.ts +0 -0
- package/lib/strapi/providers/local-source/entities.js +85 -0
- package/{dist/providers/local-strapi-source-provider → lib/strapi/providers/local-source}/index.d.ts +9 -8
- package/{dist/providers/local-strapi-source-provider → lib/strapi/providers/local-source}/index.js +36 -6
- package/{dist/providers/local-strapi-source-provider/links/index.d.ts → lib/strapi/providers/local-source/links.d.ts} +1 -1
- package/lib/strapi/providers/local-source/links.js +23 -0
- package/lib/strapi/providers/remote-destination/index.d.ts +40 -0
- package/lib/strapi/providers/remote-destination/index.js +171 -0
- package/lib/strapi/providers/remote-destination/utils.d.ts +31 -0
- package/lib/strapi/providers/remote-destination/utils.js +72 -0
- package/lib/strapi/queries/entity.d.ts +19 -0
- package/lib/strapi/queries/entity.js +130 -0
- package/lib/strapi/queries/index.d.ts +2 -0
- package/lib/strapi/queries/index.js +29 -0
- package/lib/strapi/queries/link.d.ts +6 -0
- package/lib/strapi/queries/link.js +201 -0
- package/lib/strapi/register.d.ts +7 -0
- package/lib/strapi/register.js +13 -0
- package/lib/strapi/remote/constants.d.ts +1 -0
- package/lib/strapi/remote/constants.js +5 -0
- package/lib/strapi/remote/controllers/index.d.ts +1 -0
- package/{dist → lib/strapi/remote/controllers}/index.js +1 -2
- package/lib/strapi/remote/controllers/push.d.ts +25 -0
- package/lib/strapi/remote/controllers/push.js +95 -0
- package/lib/strapi/remote/handlers.d.ts +3 -0
- package/lib/strapi/remote/handlers.js +177 -0
- package/lib/strapi/remote/index.d.ts +3 -0
- package/lib/strapi/remote/index.js +30 -0
- package/lib/strapi/remote/routes.d.ts +21 -0
- package/lib/strapi/remote/routes.js +22 -0
- package/lib/utils/encryption/decrypt.d.ts +11 -0
- package/{dist → lib/utils}/encryption/decrypt.js +12 -4
- package/lib/utils/encryption/encrypt.d.ts +11 -0
- package/{dist → lib/utils}/encryption/encrypt.js +12 -4
- package/{dist → lib/utils}/encryption/index.d.ts +0 -0
- package/{dist → lib/utils}/encryption/index.js +0 -0
- package/lib/utils/index.d.ts +4 -0
- package/lib/utils/index.js +31 -0
- package/lib/utils/json.d.ts +30 -0
- package/{dist/utils.js → lib/utils/json.js} +16 -38
- package/lib/utils/schema.d.ts +7 -0
- package/lib/utils/schema.js +29 -0
- package/lib/utils/stream.d.ts +27 -0
- package/lib/utils/stream.js +59 -0
- package/package.json +19 -12
- package/dist/encryption/decrypt.d.ts +0 -3
- package/dist/encryption/encrypt.d.ts +0 -3
- package/dist/engine/index.js +0 -324
- package/dist/index.d.ts +0 -2
- package/dist/providers/index.d.ts +0 -4
- package/dist/providers/local-file-source-provider.d.ts +0 -43
- package/dist/providers/local-strapi-destination-provider.d.ts +0 -22
- package/dist/providers/local-strapi-destination-provider.js +0 -78
- package/dist/providers/local-strapi-source-provider/entities.js +0 -58
- package/dist/providers/local-strapi-source-provider/links/index.js +0 -37
- package/dist/providers/local-strapi-source-provider/links/utils.d.ts +0 -27
- package/dist/providers/local-strapi-source-provider/links/utils.js +0 -155
- package/dist/strategies/index.d.ts +0 -7
- package/dist/strategies/index.js +0 -29
- package/dist/utils.d.ts +0 -10
|
@@ -7,17 +7,18 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
7
7
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
8
8
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
9
9
|
};
|
|
10
|
-
var _LocalFileDestinationProvider_instances, _LocalFileDestinationProvider_providersMetadata, _LocalFileDestinationProvider_archive, _LocalFileDestinationProvider_archivePath_get,
|
|
10
|
+
var _LocalFileDestinationProvider_instances, _LocalFileDestinationProvider_providersMetadata, _LocalFileDestinationProvider_archive, _LocalFileDestinationProvider_archivePath_get, _LocalFileDestinationProvider_writeMetadata, _LocalFileDestinationProvider_getMetadataStream;
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.createLocalFileDestinationProvider = void 0;
|
|
13
|
-
const fs_extra_1 =
|
|
14
|
-
const path_1 = __importDefault(require("path"));
|
|
13
|
+
const fs_extra_1 = require("fs-extra");
|
|
15
14
|
const tar_stream_1 = __importDefault(require("tar-stream"));
|
|
15
|
+
const path_1 = __importDefault(require("path"));
|
|
16
16
|
const zlib_1 = __importDefault(require("zlib"));
|
|
17
|
-
const stream_1 = require("stream");
|
|
18
17
|
const Stringer_1 = require("stream-json/jsonl/Stringer");
|
|
19
18
|
const stream_chain_1 = require("stream-chain");
|
|
20
|
-
const
|
|
19
|
+
const stream_1 = require("stream");
|
|
20
|
+
const encryption_1 = require("../../../utils/encryption");
|
|
21
|
+
const utils_1 = require("./utils");
|
|
21
22
|
const createLocalFileDestinationProvider = (options) => {
|
|
22
23
|
return new LocalFileDestinationProvider(options);
|
|
23
24
|
};
|
|
@@ -36,19 +37,22 @@ class LocalFileDestinationProvider {
|
|
|
36
37
|
__classPrivateFieldGet(this, _LocalFileDestinationProvider_providersMetadata, "f")[target] = metadata;
|
|
37
38
|
return this;
|
|
38
39
|
}
|
|
40
|
+
createGzip() {
|
|
41
|
+
return zlib_1.default.createGzip();
|
|
42
|
+
}
|
|
39
43
|
bootstrap() {
|
|
40
44
|
const { compression, encryption } = this.options;
|
|
41
45
|
if (encryption.enabled && !encryption.key) {
|
|
42
46
|
throw new Error("Can't encrypt without a key");
|
|
43
47
|
}
|
|
44
48
|
__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream = tar_stream_1.default.pack();
|
|
45
|
-
const outStream = fs_extra_1.
|
|
49
|
+
const outStream = (0, fs_extra_1.createWriteStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_instances, "a", _LocalFileDestinationProvider_archivePath_get));
|
|
46
50
|
const archiveTransforms = [];
|
|
47
51
|
if (compression.enabled) {
|
|
48
|
-
archiveTransforms.push(
|
|
52
|
+
archiveTransforms.push(this.createGzip());
|
|
49
53
|
}
|
|
50
54
|
if (encryption.enabled && encryption.key) {
|
|
51
|
-
archiveTransforms.push((0,
|
|
55
|
+
archiveTransforms.push((0, encryption_1.createEncryptionCipher)(encryption.key));
|
|
52
56
|
}
|
|
53
57
|
__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").pipeline = (0, stream_chain_1.chain)([__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, ...archiveTransforms, outStream]);
|
|
54
58
|
this.results.file = { path: __classPrivateFieldGet(this, _LocalFileDestinationProvider_instances, "a", _LocalFileDestinationProvider_archivePath_get) };
|
|
@@ -68,62 +72,82 @@ class LocalFileDestinationProvider {
|
|
|
68
72
|
}
|
|
69
73
|
async rollback() {
|
|
70
74
|
await this.close();
|
|
71
|
-
fs_extra_1.
|
|
75
|
+
await (0, fs_extra_1.rm)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_instances, "a", _LocalFileDestinationProvider_archivePath_get), { force: true });
|
|
72
76
|
}
|
|
73
77
|
getMetadata() {
|
|
74
78
|
return null;
|
|
75
79
|
}
|
|
76
|
-
|
|
80
|
+
createSchemasWriteStream() {
|
|
77
81
|
if (!__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream) {
|
|
78
82
|
throw new Error('Archive stream is unavailable');
|
|
79
83
|
}
|
|
80
|
-
const filePathFactory = createFilePathFactory('schemas');
|
|
81
|
-
const entryStream = createTarEntryStream(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.
|
|
84
|
+
const filePathFactory = (0, utils_1.createFilePathFactory)('schemas');
|
|
85
|
+
const entryStream = (0, utils_1.createTarEntryStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.maxSizeJsonl);
|
|
82
86
|
return (0, stream_chain_1.chain)([(0, Stringer_1.stringer)(), entryStream]);
|
|
83
87
|
}
|
|
84
|
-
|
|
88
|
+
createEntitiesWriteStream() {
|
|
85
89
|
if (!__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream) {
|
|
86
90
|
throw new Error('Archive stream is unavailable');
|
|
87
91
|
}
|
|
88
|
-
const filePathFactory = createFilePathFactory('entities');
|
|
89
|
-
const entryStream = createTarEntryStream(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.
|
|
92
|
+
const filePathFactory = (0, utils_1.createFilePathFactory)('entities');
|
|
93
|
+
const entryStream = (0, utils_1.createTarEntryStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.maxSizeJsonl);
|
|
90
94
|
return (0, stream_chain_1.chain)([(0, Stringer_1.stringer)(), entryStream]);
|
|
91
95
|
}
|
|
92
|
-
|
|
96
|
+
createLinksWriteStream() {
|
|
93
97
|
if (!__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream) {
|
|
94
98
|
throw new Error('Archive stream is unavailable');
|
|
95
99
|
}
|
|
96
|
-
const filePathFactory = createFilePathFactory('links');
|
|
97
|
-
const entryStream = createTarEntryStream(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.
|
|
100
|
+
const filePathFactory = (0, utils_1.createFilePathFactory)('links');
|
|
101
|
+
const entryStream = (0, utils_1.createTarEntryStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.maxSizeJsonl);
|
|
98
102
|
return (0, stream_chain_1.chain)([(0, Stringer_1.stringer)(), entryStream]);
|
|
99
103
|
}
|
|
100
|
-
|
|
104
|
+
createConfigurationWriteStream() {
|
|
101
105
|
if (!__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream) {
|
|
102
106
|
throw new Error('Archive stream is unavailable');
|
|
103
107
|
}
|
|
104
|
-
const filePathFactory = createFilePathFactory('configuration');
|
|
105
|
-
const entryStream = createTarEntryStream(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.
|
|
108
|
+
const filePathFactory = (0, utils_1.createFilePathFactory)('configuration');
|
|
109
|
+
const entryStream = (0, utils_1.createTarEntryStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.maxSizeJsonl);
|
|
106
110
|
return (0, stream_chain_1.chain)([(0, Stringer_1.stringer)(), entryStream]);
|
|
107
111
|
}
|
|
112
|
+
createAssetsWriteStream() {
|
|
113
|
+
const { stream: archiveStream } = __classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f");
|
|
114
|
+
if (!archiveStream) {
|
|
115
|
+
throw new Error('Archive stream is unavailable');
|
|
116
|
+
}
|
|
117
|
+
return new stream_1.Writable({
|
|
118
|
+
objectMode: true,
|
|
119
|
+
write(data, _encoding, callback) {
|
|
120
|
+
const entryPath = path_1.default.join('assets', 'uploads', data.filename);
|
|
121
|
+
const entry = archiveStream.entry({
|
|
122
|
+
name: entryPath,
|
|
123
|
+
size: data.stats.size,
|
|
124
|
+
});
|
|
125
|
+
if (!entry) {
|
|
126
|
+
callback(new Error(`Failed to created a tar entry for ${entryPath}`));
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
data.stream.pipe(entry);
|
|
130
|
+
entry
|
|
131
|
+
.on('finish', () => {
|
|
132
|
+
callback(null);
|
|
133
|
+
})
|
|
134
|
+
.on('error', (error) => {
|
|
135
|
+
callback(error);
|
|
136
|
+
});
|
|
137
|
+
},
|
|
138
|
+
});
|
|
139
|
+
}
|
|
108
140
|
}
|
|
109
141
|
_LocalFileDestinationProvider_providersMetadata = new WeakMap(), _LocalFileDestinationProvider_archive = new WeakMap(), _LocalFileDestinationProvider_instances = new WeakSet(), _LocalFileDestinationProvider_archivePath_get = function _LocalFileDestinationProvider_archivePath_get() {
|
|
110
142
|
const { encryption, compression, file } = this.options;
|
|
111
|
-
let
|
|
143
|
+
let filePath = `${file.path}.tar`;
|
|
112
144
|
if (compression.enabled) {
|
|
113
|
-
|
|
145
|
+
filePath += '.gz';
|
|
114
146
|
}
|
|
115
147
|
if (encryption.enabled) {
|
|
116
|
-
|
|
117
|
-
}
|
|
118
|
-
return path;
|
|
119
|
-
}, _LocalFileDestinationProvider_getDataTransformers = function _LocalFileDestinationProvider_getDataTransformers(options = {}) {
|
|
120
|
-
const { jsonl = true } = options;
|
|
121
|
-
const transforms = [];
|
|
122
|
-
if (jsonl) {
|
|
123
|
-
// Convert to stringified JSON lines
|
|
124
|
-
transforms.push((0, Stringer_1.stringer)());
|
|
148
|
+
filePath += '.enc';
|
|
125
149
|
}
|
|
126
|
-
return
|
|
150
|
+
return filePath;
|
|
127
151
|
}, _LocalFileDestinationProvider_writeMetadata = async function _LocalFileDestinationProvider_writeMetadata() {
|
|
128
152
|
const metadata = __classPrivateFieldGet(this, _LocalFileDestinationProvider_providersMetadata, "f").source;
|
|
129
153
|
if (metadata) {
|
|
@@ -138,58 +162,6 @@ _LocalFileDestinationProvider_providersMetadata = new WeakMap(), _LocalFileDesti
|
|
|
138
162
|
if (!stream) {
|
|
139
163
|
throw new Error('Archive stream is unavailable');
|
|
140
164
|
}
|
|
141
|
-
return createTarEntryStream(stream, () => 'metadata.json');
|
|
142
|
-
};
|
|
143
|
-
/**
|
|
144
|
-
* Create a file path factory for a given path & prefix.
|
|
145
|
-
* Upon being called, the factory will return a file path for a given index
|
|
146
|
-
*/
|
|
147
|
-
const createFilePathFactory = (type) => (fileIndex = 0) => {
|
|
148
|
-
return path_1.default.join(
|
|
149
|
-
// "{type}" directory
|
|
150
|
-
type,
|
|
151
|
-
// "${type}_XXXXX.jsonl" file
|
|
152
|
-
`${type}_${String(fileIndex).padStart(5, '0')}.jsonl`);
|
|
153
|
-
};
|
|
154
|
-
const createTarEntryStream = (archive, pathFactory, maxSize = 2.56e8) => {
|
|
155
|
-
let fileIndex = 0;
|
|
156
|
-
let buffer = '';
|
|
157
|
-
const flush = async () => {
|
|
158
|
-
if (!buffer) {
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
const name = pathFactory(fileIndex++);
|
|
162
|
-
const size = buffer.length;
|
|
163
|
-
await new Promise((resolve, reject) => {
|
|
164
|
-
archive.entry({ name, size }, buffer, (err) => {
|
|
165
|
-
if (err) {
|
|
166
|
-
reject(err);
|
|
167
|
-
}
|
|
168
|
-
resolve();
|
|
169
|
-
});
|
|
170
|
-
});
|
|
171
|
-
buffer = '';
|
|
172
|
-
};
|
|
173
|
-
const push = (chunk) => {
|
|
174
|
-
buffer += chunk;
|
|
175
|
-
};
|
|
176
|
-
return new stream_1.Writable({
|
|
177
|
-
async destroy(err, callback) {
|
|
178
|
-
await flush();
|
|
179
|
-
callback(err);
|
|
180
|
-
},
|
|
181
|
-
async write(chunk, _encoding, callback) {
|
|
182
|
-
const size = chunk.length;
|
|
183
|
-
if (chunk.length > maxSize) {
|
|
184
|
-
callback(new Error(`payload too large: ${chunk.length}>${maxSize}`));
|
|
185
|
-
return;
|
|
186
|
-
}
|
|
187
|
-
if (buffer.length + size > maxSize) {
|
|
188
|
-
await flush();
|
|
189
|
-
}
|
|
190
|
-
push(chunk);
|
|
191
|
-
callback(null);
|
|
192
|
-
},
|
|
193
|
-
});
|
|
165
|
+
return (0, utils_1.createTarEntryStream)(stream, () => 'metadata.json');
|
|
194
166
|
};
|
|
195
|
-
//# sourceMappingURL=
|
|
167
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { Writable } from 'stream';
|
|
3
|
+
import tar from 'tar-stream';
|
|
4
|
+
/**
|
|
5
|
+
* Create a file path factory for a given path & prefix.
|
|
6
|
+
* Upon being called, the factory will return a file path for a given index
|
|
7
|
+
*/
|
|
8
|
+
export declare const createFilePathFactory: (type: string) => (fileIndex?: number) => string;
|
|
9
|
+
export declare const createTarEntryStream: (archive: tar.Pack, pathFactory: (index?: number | undefined) => string, maxSize?: number) => Writable;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createTarEntryStream = exports.createFilePathFactory = void 0;
|
|
4
|
+
const stream_1 = require("stream");
|
|
5
|
+
const path_1 = require("path");
|
|
6
|
+
/**
|
|
7
|
+
* Create a file path factory for a given path & prefix.
|
|
8
|
+
* Upon being called, the factory will return a file path for a given index
|
|
9
|
+
*/
|
|
10
|
+
const createFilePathFactory = (type) => (fileIndex = 0) => {
|
|
11
|
+
return (0, path_1.join)(
|
|
12
|
+
// "{type}" directory
|
|
13
|
+
type,
|
|
14
|
+
// "${type}_XXXXX.jsonl" file
|
|
15
|
+
`${type}_${String(fileIndex).padStart(5, '0')}.jsonl`);
|
|
16
|
+
};
|
|
17
|
+
exports.createFilePathFactory = createFilePathFactory;
|
|
18
|
+
const createTarEntryStream = (archive, pathFactory, maxSize = 2.56e8) => {
|
|
19
|
+
let fileIndex = 0;
|
|
20
|
+
let buffer = '';
|
|
21
|
+
const flush = async () => {
|
|
22
|
+
if (!buffer) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
fileIndex += 1;
|
|
26
|
+
const name = pathFactory(fileIndex);
|
|
27
|
+
const size = buffer.length;
|
|
28
|
+
await new Promise((resolve, reject) => {
|
|
29
|
+
archive.entry({ name, size }, buffer, (err) => {
|
|
30
|
+
if (err) {
|
|
31
|
+
reject(err);
|
|
32
|
+
}
|
|
33
|
+
resolve();
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
buffer = '';
|
|
37
|
+
};
|
|
38
|
+
const push = (chunk) => {
|
|
39
|
+
buffer += chunk;
|
|
40
|
+
};
|
|
41
|
+
return new stream_1.Writable({
|
|
42
|
+
async destroy(err, callback) {
|
|
43
|
+
await flush();
|
|
44
|
+
callback(err);
|
|
45
|
+
},
|
|
46
|
+
async write(chunk, _encoding, callback) {
|
|
47
|
+
const size = chunk.length;
|
|
48
|
+
if (chunk.length > maxSize) {
|
|
49
|
+
callback(new Error(`payload too large: ${chunk.length}>${maxSize}`));
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (buffer.length + size > maxSize) {
|
|
53
|
+
await flush();
|
|
54
|
+
}
|
|
55
|
+
push(chunk);
|
|
56
|
+
callback(null);
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
};
|
|
60
|
+
exports.createTarEntryStream = createTarEntryStream;
|
|
61
|
+
//# sourceMappingURL=utils.js.map
|
|
@@ -14,10 +14,6 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
|
|
18
|
-
__exportStar(require("./
|
|
19
|
-
__exportStar(require("./local-strapi-source-provider"), exports);
|
|
20
|
-
// destination providers
|
|
21
|
-
__exportStar(require("./local-file-destination-provider"), exports);
|
|
22
|
-
__exportStar(require("./local-strapi-destination-provider"), exports);
|
|
17
|
+
__exportStar(require("./source"), exports);
|
|
18
|
+
__exportStar(require("./destination"), exports);
|
|
23
19
|
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/// <reference types="lodash" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
import type { Readable } from 'stream';
|
|
4
|
+
import type { Schema } from '@strapi/strapi';
|
|
5
|
+
import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types';
|
|
6
|
+
/**
|
|
7
|
+
* Provider options
|
|
8
|
+
*/
|
|
9
|
+
export interface ILocalFileSourceProviderOptions {
|
|
10
|
+
file: {
|
|
11
|
+
path: string;
|
|
12
|
+
};
|
|
13
|
+
encryption: {
|
|
14
|
+
enabled: boolean;
|
|
15
|
+
key?: string;
|
|
16
|
+
};
|
|
17
|
+
compression: {
|
|
18
|
+
enabled: boolean;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export declare const createLocalFileSourceProvider: (options: ILocalFileSourceProviderOptions) => LocalFileSourceProvider;
|
|
22
|
+
declare class LocalFileSourceProvider implements ISourceProvider {
|
|
23
|
+
#private;
|
|
24
|
+
type: ProviderType;
|
|
25
|
+
name: string;
|
|
26
|
+
options: ILocalFileSourceProviderOptions;
|
|
27
|
+
constructor(options: ILocalFileSourceProviderOptions);
|
|
28
|
+
/**
|
|
29
|
+
* Pre flight checks regarding the provided options, making sure that the file can be opened (decrypted, decompressed), etc.
|
|
30
|
+
*/
|
|
31
|
+
bootstrap(): Promise<void>;
|
|
32
|
+
getMetadata(): Promise<IMetadata>;
|
|
33
|
+
getSchemas(): Promise<import("lodash").Dictionary<Schema>>;
|
|
34
|
+
createEntitiesReadStream(): Readable;
|
|
35
|
+
createSchemasReadStream(): Readable;
|
|
36
|
+
createLinksReadStream(): Readable;
|
|
37
|
+
createConfigurationReadStream(): Readable;
|
|
38
|
+
createAssetsReadStream(): Readable | Promise<Readable>;
|
|
39
|
+
}
|
|
40
|
+
export {};
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
3
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
4
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
5
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
6
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
7
|
+
};
|
|
2
8
|
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
3
9
|
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
4
10
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
@@ -7,18 +13,19 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
7
13
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
8
14
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
9
15
|
};
|
|
10
|
-
var _LocalFileSourceProvider_instances, _LocalFileSourceProvider_getBackupStream, _LocalFileSourceProvider_streamJsonlDirectory, _LocalFileSourceProvider_parseJSONFile;
|
|
16
|
+
var _LocalFileSourceProvider_instances, _LocalFileSourceProvider_metadata, _LocalFileSourceProvider_getBackupStream, _LocalFileSourceProvider_streamJsonlDirectory, _LocalFileSourceProvider_parseJSONFile;
|
|
11
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
18
|
exports.createLocalFileSourceProvider = void 0;
|
|
13
|
-
const
|
|
19
|
+
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
14
20
|
const zlib_1 = __importDefault(require("zlib"));
|
|
15
21
|
const tar_1 = __importDefault(require("tar"));
|
|
22
|
+
const path_1 = __importDefault(require("path"));
|
|
16
23
|
const fp_1 = require("lodash/fp");
|
|
17
24
|
const stream_chain_1 = require("stream-chain");
|
|
18
25
|
const stream_1 = require("stream");
|
|
19
26
|
const Parser_1 = require("stream-json/jsonl/Parser");
|
|
20
|
-
const encryption_1 = require("
|
|
21
|
-
const
|
|
27
|
+
const encryption_1 = require("../../../utils/encryption");
|
|
28
|
+
const stream_2 = require("../../../utils/stream");
|
|
22
29
|
/**
|
|
23
30
|
* Constant for the metadata file path
|
|
24
31
|
*/
|
|
@@ -32,20 +39,24 @@ class LocalFileSourceProvider {
|
|
|
32
39
|
_LocalFileSourceProvider_instances.add(this);
|
|
33
40
|
this.type = 'source';
|
|
34
41
|
this.name = 'source::local-file';
|
|
42
|
+
_LocalFileSourceProvider_metadata.set(this, void 0);
|
|
35
43
|
this.options = options;
|
|
36
|
-
|
|
44
|
+
const { encryption } = this.options;
|
|
45
|
+
if (encryption.enabled && encryption.key === undefined) {
|
|
37
46
|
throw new Error('Missing encryption key');
|
|
38
47
|
}
|
|
39
48
|
}
|
|
40
49
|
/**
|
|
41
|
-
* Pre flight checks regarding the provided options
|
|
50
|
+
* Pre flight checks regarding the provided options, making sure that the file can be opened (decrypted, decompressed), etc.
|
|
42
51
|
*/
|
|
43
|
-
bootstrap() {
|
|
44
|
-
const path = this.options.
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
52
|
+
async bootstrap() {
|
|
53
|
+
const { path: filePath } = this.options.file;
|
|
54
|
+
try {
|
|
55
|
+
// Read the metadata to ensure the file can be parsed
|
|
56
|
+
__classPrivateFieldSet(this, _LocalFileSourceProvider_metadata, await this.getMetadata(), "f");
|
|
57
|
+
}
|
|
58
|
+
catch (e) {
|
|
59
|
+
throw new Error(`Can't read file "${filePath}".`);
|
|
49
60
|
}
|
|
50
61
|
}
|
|
51
62
|
getMetadata() {
|
|
@@ -55,62 +66,90 @@ class LocalFileSourceProvider {
|
|
|
55
66
|
return __classPrivateFieldGet(this, _LocalFileSourceProvider_instances, "m", _LocalFileSourceProvider_parseJSONFile).call(this, backupStream, METADATA_FILE_PATH);
|
|
56
67
|
}
|
|
57
68
|
async getSchemas() {
|
|
58
|
-
const schemas = await (0,
|
|
69
|
+
const schemas = await (0, stream_2.collect)(this.createSchemasReadStream());
|
|
59
70
|
return (0, fp_1.keyBy)('uid', schemas);
|
|
60
71
|
}
|
|
61
|
-
|
|
72
|
+
createEntitiesReadStream() {
|
|
62
73
|
return __classPrivateFieldGet(this, _LocalFileSourceProvider_instances, "m", _LocalFileSourceProvider_streamJsonlDirectory).call(this, 'entities');
|
|
63
74
|
}
|
|
64
|
-
|
|
75
|
+
createSchemasReadStream() {
|
|
65
76
|
return __classPrivateFieldGet(this, _LocalFileSourceProvider_instances, "m", _LocalFileSourceProvider_streamJsonlDirectory).call(this, 'schemas');
|
|
66
77
|
}
|
|
67
|
-
|
|
78
|
+
createLinksReadStream() {
|
|
68
79
|
return __classPrivateFieldGet(this, _LocalFileSourceProvider_instances, "m", _LocalFileSourceProvider_streamJsonlDirectory).call(this, 'links');
|
|
69
80
|
}
|
|
70
|
-
|
|
81
|
+
createConfigurationReadStream() {
|
|
71
82
|
// NOTE: TBD
|
|
72
83
|
return __classPrivateFieldGet(this, _LocalFileSourceProvider_instances, "m", _LocalFileSourceProvider_streamJsonlDirectory).call(this, 'configuration');
|
|
73
84
|
}
|
|
85
|
+
createAssetsReadStream() {
|
|
86
|
+
const inStream = __classPrivateFieldGet(this, _LocalFileSourceProvider_instances, "m", _LocalFileSourceProvider_getBackupStream).call(this);
|
|
87
|
+
const outStream = new stream_1.PassThrough({ objectMode: true });
|
|
88
|
+
(0, stream_1.pipeline)([
|
|
89
|
+
inStream,
|
|
90
|
+
new tar_1.default.Parse({
|
|
91
|
+
filter(filePath, entry) {
|
|
92
|
+
if (entry.type !== 'File') {
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
const parts = filePath.split('/');
|
|
96
|
+
return parts[0] === 'assets' && parts[1] === 'uploads';
|
|
97
|
+
},
|
|
98
|
+
onentry(entry) {
|
|
99
|
+
const { path: filePath, size = 0 } = entry;
|
|
100
|
+
const file = path_1.default.basename(filePath);
|
|
101
|
+
const asset = {
|
|
102
|
+
filename: file,
|
|
103
|
+
filepath: filePath,
|
|
104
|
+
stats: { size },
|
|
105
|
+
stream: entry,
|
|
106
|
+
};
|
|
107
|
+
outStream.write(asset);
|
|
108
|
+
},
|
|
109
|
+
}),
|
|
110
|
+
], () => outStream.end());
|
|
111
|
+
return outStream;
|
|
112
|
+
}
|
|
74
113
|
}
|
|
75
|
-
_LocalFileSourceProvider_instances = new WeakSet(), _LocalFileSourceProvider_getBackupStream = function _LocalFileSourceProvider_getBackupStream(
|
|
76
|
-
const
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
114
|
+
_LocalFileSourceProvider_metadata = new WeakMap(), _LocalFileSourceProvider_instances = new WeakSet(), _LocalFileSourceProvider_getBackupStream = function _LocalFileSourceProvider_getBackupStream() {
|
|
115
|
+
const { file, encryption, compression } = this.options;
|
|
116
|
+
const streams = [];
|
|
117
|
+
try {
|
|
118
|
+
streams.push(fs_extra_1.default.createReadStream(file.path));
|
|
119
|
+
}
|
|
120
|
+
catch (e) {
|
|
121
|
+
throw new Error(`Could not read backup file path provided at "${this.options.file.path}"`);
|
|
122
|
+
}
|
|
123
|
+
if (encryption.enabled && encryption.key) {
|
|
124
|
+
streams.push((0, encryption_1.createDecryptionCipher)(encryption.key));
|
|
125
|
+
}
|
|
126
|
+
if (compression.enabled) {
|
|
81
127
|
streams.push(zlib_1.default.createGunzip());
|
|
82
128
|
}
|
|
83
129
|
return (0, stream_chain_1.chain)(streams);
|
|
84
130
|
}, _LocalFileSourceProvider_streamJsonlDirectory = function _LocalFileSourceProvider_streamJsonlDirectory(directory) {
|
|
85
|
-
const options = this.options;
|
|
86
131
|
const inStream = __classPrivateFieldGet(this, _LocalFileSourceProvider_instances, "m", _LocalFileSourceProvider_getBackupStream).call(this);
|
|
87
132
|
const outStream = new stream_1.PassThrough({ objectMode: true });
|
|
88
133
|
(0, stream_1.pipeline)([
|
|
89
134
|
inStream,
|
|
90
135
|
new tar_1.default.Parse({
|
|
91
|
-
filter(
|
|
136
|
+
filter(filePath, entry) {
|
|
92
137
|
if (entry.type !== 'File') {
|
|
93
138
|
return false;
|
|
94
139
|
}
|
|
95
|
-
const parts =
|
|
140
|
+
const parts = filePath.split('/');
|
|
96
141
|
if (parts.length !== 2) {
|
|
97
142
|
return false;
|
|
98
143
|
}
|
|
99
144
|
return parts[0] === directory;
|
|
100
145
|
},
|
|
101
146
|
onentry(entry) {
|
|
102
|
-
const transforms = [
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
}
|
|
109
|
-
transforms.push(
|
|
110
|
-
// JSONL parser to read the data chunks one by one (line by line)
|
|
111
|
-
(0, Parser_1.parser)(),
|
|
112
|
-
// The JSONL parser returns each line as key/value
|
|
113
|
-
(line) => line.value);
|
|
147
|
+
const transforms = [
|
|
148
|
+
// JSONL parser to read the data chunks one by one (line by line)
|
|
149
|
+
(0, Parser_1.parser)(),
|
|
150
|
+
// The JSONL parser returns each line as key/value
|
|
151
|
+
(line) => line.value,
|
|
152
|
+
];
|
|
114
153
|
entry
|
|
115
154
|
// Pipe transforms
|
|
116
155
|
.pipe((0, stream_chain_1.chain)(transforms))
|
|
@@ -135,28 +174,32 @@ _LocalFileSourceProvider_instances = new WeakSet(), _LocalFileSourceProvider_get
|
|
|
135
174
|
/**
|
|
136
175
|
* Filter the parsed entries to only keep the one that matches the given filepath
|
|
137
176
|
*/
|
|
138
|
-
filter(
|
|
139
|
-
return
|
|
177
|
+
filter(entryPath, entry) {
|
|
178
|
+
return entryPath === filePath && entry.type === 'File';
|
|
140
179
|
},
|
|
141
|
-
/**
|
|
142
|
-
* Whenever an entry passes the filter method, process it
|
|
143
|
-
*/
|
|
144
180
|
async onentry(entry) {
|
|
145
181
|
// Collect all the content of the entry file
|
|
146
182
|
const content = await entry.collect();
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
183
|
+
try {
|
|
184
|
+
// Parse from buffer to string to JSON
|
|
185
|
+
const parsedContent = JSON.parse(content.toString());
|
|
186
|
+
// Resolve the Promise with the parsed content
|
|
187
|
+
resolve(parsedContent);
|
|
188
|
+
}
|
|
189
|
+
catch (e) {
|
|
190
|
+
reject(e);
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
// Cleanup (close the stream associated to the entry)
|
|
194
|
+
entry.destroy();
|
|
195
|
+
}
|
|
153
196
|
},
|
|
154
197
|
}),
|
|
155
198
|
], () => {
|
|
156
199
|
// If the promise hasn't been resolved and we've parsed all
|
|
157
200
|
// the archive entries, then the file doesn't exist
|
|
158
|
-
reject(
|
|
201
|
+
reject(new Error(`File "${filePath}" not found`));
|
|
159
202
|
});
|
|
160
203
|
});
|
|
161
204
|
};
|
|
162
|
-
//# sourceMappingURL=
|
|
205
|
+
//# sourceMappingURL=index.js.map
|
package/lib/index.d.ts
ADDED
package/lib/index.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.utils = exports.file = exports.strapi = exports.engine = void 0;
|
|
27
|
+
exports.engine = __importStar(require("./engine"));
|
|
28
|
+
exports.strapi = __importStar(require("./strapi"));
|
|
29
|
+
exports.file = __importStar(require("./file"));
|
|
30
|
+
exports.utils = __importStar(require("./utils"));
|
|
31
|
+
//# sourceMappingURL=index.js.map
|