@hubspot/local-dev-lib 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/sandboxHubs.d.ts +1 -1
- package/config/CLIConfiguration.d.ts +4 -4
- package/config/CLIConfiguration.js +42 -33
- package/config/configFile.js +5 -4
- package/config/configUtils.js +3 -2
- package/config/config_DEPRECATED.d.ts +2 -1
- package/config/config_DEPRECATED.js +23 -3
- package/config/environment.js +5 -4
- package/config/index.d.ts +2 -1
- package/config/index.js +8 -1
- package/constants/config.d.ts +7 -0
- package/constants/config.js +8 -1
- package/errors/apiErrors.d.ts +7 -2
- package/errors/apiErrors.js +12 -11
- package/errors/standardErrors.d.ts +1 -1
- package/http/index.d.ts +2 -4
- package/http/index.js +6 -7
- package/lang/en.json +10 -10
- package/lang/lang/en.json +10 -10
- package/lib/archive.d.ts +1 -3
- package/lib/archive.js +14 -16
- package/lib/cms/functions.d.ts +1 -3
- package/lib/cms/functions.js +25 -32
- package/lib/cms/handleFieldsJS.js +6 -5
- package/lib/cms/modules.d.ts +2 -3
- package/lib/cms/modules.js +16 -9
- package/lib/cms/templates.d.ts +1 -3
- package/lib/cms/templates.js +6 -7
- package/lib/cms/uploadFolder.d.ts +1 -3
- package/lib/cms/uploadFolder.js +14 -16
- package/lib/cms/watch.d.ts +1 -4
- package/lib/cms/watch.js +34 -47
- package/lib/fileManager.d.ts +2 -6
- package/lib/fileManager.js +31 -42
- package/lib/fileMapper.d.ts +2 -4
- package/lib/fileMapper.js +24 -35
- package/lib/github.d.ts +3 -4
- package/lib/github.js +35 -14
- package/lib/oauth.d.ts +1 -4
- package/lib/oauth.js +7 -9
- package/lib/personalAccessKey.js +19 -2
- package/lib/sandboxes.d.ts +1 -1
- package/lib/sandboxes.js +7 -8
- package/lib/trackUsage.js +5 -4
- package/models/OAuth2Manager.d.ts +16 -13
- package/models/OAuth2Manager.js +30 -38
- package/package.json +3 -2
- package/types/Accounts.d.ts +5 -0
- package/types/Sandbox.d.ts +13 -12
- package/utils/PortManagerServer.js +7 -7
- package/types/LogCallbacks.d.ts +0 -7
- package/types/LogCallbacks.js +0 -2
- package/utils/logger.d.ts +0 -5
- package/utils/logger.js +0 -23
package/lib/cms/watch.js
CHANGED
|
@@ -18,35 +18,24 @@ const escapeRegExp_1 = require("../../utils/escapeRegExp");
|
|
|
18
18
|
const path_2 = require("../path");
|
|
19
19
|
const notify_1 = require("../../utils/notify");
|
|
20
20
|
const themes_1 = require("./themes");
|
|
21
|
-
const logger_1 = require("
|
|
22
|
-
const
|
|
21
|
+
const logger_1 = require("../logging/logger");
|
|
22
|
+
const lang_1 = require("../../utils/lang");
|
|
23
23
|
const i18nKey = 'lib.cms.watch';
|
|
24
|
-
const watchCallbackKeys = [
|
|
25
|
-
'notifyOfThemePreview',
|
|
26
|
-
'uploadSuccess',
|
|
27
|
-
'deleteSuccess',
|
|
28
|
-
'folderUploadSuccess',
|
|
29
|
-
'ready',
|
|
30
|
-
'deleteSuccessWithType',
|
|
31
|
-
];
|
|
32
|
-
const makeLogger = (logger_1.makeTypedLogger);
|
|
33
24
|
const queue = new p_queue_1.default({
|
|
34
25
|
concurrency: 10,
|
|
35
26
|
});
|
|
36
|
-
function _notifyOfThemePreview(filePath, accountId
|
|
37
|
-
const logger = makeLogger(logCallbacks);
|
|
27
|
+
function _notifyOfThemePreview(filePath, accountId) {
|
|
38
28
|
if (queue.size > 0)
|
|
39
29
|
return;
|
|
40
30
|
const previewUrl = (0, themes_1.getThemePreviewUrl)(filePath, accountId);
|
|
41
31
|
if (!previewUrl)
|
|
42
32
|
return;
|
|
43
|
-
logger(
|
|
33
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.notifyOfThemePreview`, {
|
|
44
34
|
previewUrl,
|
|
45
|
-
});
|
|
35
|
+
}));
|
|
46
36
|
}
|
|
47
37
|
const notifyOfThemePreview = (0, debounce_1.debounce)(_notifyOfThemePreview, 1000);
|
|
48
|
-
async function uploadFile(accountId, file, dest, options, mode = null
|
|
49
|
-
const logger = makeLogger(logCallbacks);
|
|
38
|
+
async function uploadFile(accountId, file, dest, options, mode = null) {
|
|
50
39
|
const src = options.src;
|
|
51
40
|
const absoluteSrcPath = path_1.default.resolve((0, path_2.getCwd)(), file);
|
|
52
41
|
const themeJsonPath = (0, themes_1.getThemeJSONPath)(absoluteSrcPath);
|
|
@@ -55,11 +44,11 @@ async function uploadFile(accountId, file, dest, options, mode = null, logCallba
|
|
|
55
44
|
: path_1.default.dirname((0, path_2.getCwd)());
|
|
56
45
|
const convertFields = (0, handleFieldsJS_1.isConvertableFieldJs)(src, file, options.commandOptions.convertFields);
|
|
57
46
|
if (!(0, path_2.isAllowedExtension)(file) && !convertFields) {
|
|
58
|
-
(0,
|
|
47
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.skipUnsupportedExtension`, { file }));
|
|
59
48
|
return;
|
|
60
49
|
}
|
|
61
50
|
if ((0, ignoreRules_1.shouldIgnoreFile)(file)) {
|
|
62
|
-
(0,
|
|
51
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.skipIgnoreRule`, { file }));
|
|
63
52
|
return;
|
|
64
53
|
}
|
|
65
54
|
let fieldsJs = undefined;
|
|
@@ -71,22 +60,22 @@ async function uploadFile(accountId, file, dest, options, mode = null, logCallba
|
|
|
71
60
|
dest = (0, path_2.convertToUnixPath)(path_1.default.join(path_1.default.dirname(dest), 'fields.json'));
|
|
72
61
|
}
|
|
73
62
|
const fileToUpload = convertFields && fieldsJs?.outputPath ? fieldsJs.outputPath : file;
|
|
74
|
-
(0,
|
|
63
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.uploadAttempt`, { file, dest }));
|
|
75
64
|
const apiOptions = (0, fileMapper_1.getFileMapperQueryValues)(mode, options);
|
|
76
65
|
queue.add(() => {
|
|
77
66
|
return (0, fileMapper_2.upload)(accountId, fileToUpload, dest, apiOptions)
|
|
78
67
|
.then(() => {
|
|
79
|
-
logger(
|
|
80
|
-
notifyOfThemePreview(file, accountId
|
|
68
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.uploadSuccess`, { file, dest }));
|
|
69
|
+
notifyOfThemePreview(file, accountId);
|
|
81
70
|
})
|
|
82
71
|
.catch(() => {
|
|
83
|
-
(0,
|
|
84
|
-
(0,
|
|
72
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.uploadFailed`, { file, dest }));
|
|
73
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.uploadRetry`, { file, dest }));
|
|
85
74
|
return (0, fileMapper_2.upload)(accountId, file, dest, apiOptions).catch((error) => {
|
|
86
|
-
(0,
|
|
75
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.uploadFailed`, {
|
|
87
76
|
file,
|
|
88
77
|
dest,
|
|
89
|
-
});
|
|
78
|
+
}));
|
|
90
79
|
(0, apiErrors_1.throwApiUploadError)(error, {
|
|
91
80
|
accountId,
|
|
92
81
|
request: dest,
|
|
@@ -96,23 +85,22 @@ async function uploadFile(accountId, file, dest, options, mode = null, logCallba
|
|
|
96
85
|
});
|
|
97
86
|
});
|
|
98
87
|
}
|
|
99
|
-
async function deleteRemoteFile(accountId, filePath, remoteFilePath
|
|
100
|
-
const logger = makeLogger(logCallbacks);
|
|
88
|
+
async function deleteRemoteFile(accountId, filePath, remoteFilePath) {
|
|
101
89
|
if ((0, ignoreRules_1.shouldIgnoreFile)(filePath)) {
|
|
102
|
-
(0,
|
|
90
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.skipIgnoreRule`, { file: filePath }));
|
|
103
91
|
return;
|
|
104
92
|
}
|
|
105
|
-
(0,
|
|
93
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.deleteAttempt`, { remoteFilePath }));
|
|
106
94
|
return queue.add(() => {
|
|
107
95
|
return (0, fileMapper_2.deleteFile)(accountId, remoteFilePath)
|
|
108
96
|
.then(() => {
|
|
109
|
-
logger(
|
|
110
|
-
notifyOfThemePreview(filePath, accountId
|
|
97
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.deleteSuccess`, { remoteFilePath }));
|
|
98
|
+
notifyOfThemePreview(filePath, accountId);
|
|
111
99
|
})
|
|
112
100
|
.catch((error) => {
|
|
113
|
-
(0,
|
|
101
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.deleteFailed`, {
|
|
114
102
|
remoteFilePath,
|
|
115
|
-
});
|
|
103
|
+
}));
|
|
116
104
|
(0, apiErrors_1.throwApiError)(error, {
|
|
117
105
|
accountId,
|
|
118
106
|
request: remoteFilePath,
|
|
@@ -120,8 +108,7 @@ async function deleteRemoteFile(accountId, filePath, remoteFilePath, logCallback
|
|
|
120
108
|
});
|
|
121
109
|
});
|
|
122
110
|
}
|
|
123
|
-
function watch(accountId, src, dest, { mode, remove, disableInitial, notify, commandOptions, filePaths, }, postInitialUploadCallback = null, onUploadFolderError, onQueueAddError
|
|
124
|
-
const logger = makeLogger(logCallbacks);
|
|
111
|
+
function watch(accountId, src, dest, { mode, remove, disableInitial, notify, commandOptions, filePaths, }, postInitialUploadCallback = null, onUploadFolderError, onQueueAddError) {
|
|
125
112
|
const regex = new RegExp(`^${(0, escapeRegExp_1.escapeRegExp)(src)}`);
|
|
126
113
|
if (notify) {
|
|
127
114
|
(0, ignoreRules_1.ignoreFile)(notify);
|
|
@@ -137,11 +124,11 @@ function watch(accountId, src, dest, { mode, remove, disableInitial, notify, com
|
|
|
137
124
|
if (!disableInitial) {
|
|
138
125
|
// Use uploadFolder so that failures of initial upload are retried
|
|
139
126
|
const uploadFolderPromise = (0, uploadFolder_1.uploadFolder)(accountId, src, dest, {}, commandOptions, filePaths, mode || null).then(result => {
|
|
140
|
-
logger(
|
|
127
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.folderUploadSuccess`, {
|
|
141
128
|
src,
|
|
142
129
|
dest,
|
|
143
130
|
accountId,
|
|
144
|
-
});
|
|
131
|
+
}));
|
|
145
132
|
if (postInitialUploadCallback) {
|
|
146
133
|
postInitialUploadCallback(result);
|
|
147
134
|
}
|
|
@@ -151,14 +138,14 @@ function watch(accountId, src, dest, { mode, remove, disableInitial, notify, com
|
|
|
151
138
|
}
|
|
152
139
|
}
|
|
153
140
|
watcher.on('ready', () => {
|
|
154
|
-
logger(
|
|
141
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.ready`, { src }));
|
|
155
142
|
});
|
|
156
143
|
watcher.on('add', async (filePath) => {
|
|
157
144
|
const destPath = getDesignManagerPath(filePath);
|
|
158
145
|
const uploadPromise = uploadFile(accountId, filePath, destPath, {
|
|
159
146
|
src,
|
|
160
147
|
commandOptions,
|
|
161
|
-
}, mode
|
|
148
|
+
}, mode);
|
|
162
149
|
(0, notify_1.triggerNotify)(notify, 'Added', filePath, uploadPromise);
|
|
163
150
|
});
|
|
164
151
|
if (remove) {
|
|
@@ -169,19 +156,19 @@ function watch(accountId, src, dest, { mode, remove, disableInitial, notify, com
|
|
|
169
156
|
}
|
|
170
157
|
const remotePath = getDesignManagerPath(filePath);
|
|
171
158
|
if ((0, ignoreRules_1.shouldIgnoreFile)(filePath)) {
|
|
172
|
-
(0,
|
|
159
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.skipIgnoreRule`, { file: filePath }));
|
|
173
160
|
return;
|
|
174
161
|
}
|
|
175
|
-
(0,
|
|
162
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.deleteAttemptWithType`, {
|
|
176
163
|
type,
|
|
177
164
|
remoteFilePath: remotePath,
|
|
178
|
-
});
|
|
165
|
+
}));
|
|
179
166
|
const queueAddPromise = queue.add(() => {
|
|
180
|
-
const deletePromise = deleteRemoteFile(accountId, filePath, remotePath
|
|
181
|
-
logger(
|
|
167
|
+
const deletePromise = deleteRemoteFile(accountId, filePath, remotePath).then(() => {
|
|
168
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.deleteSuccessWithType`, {
|
|
182
169
|
type,
|
|
183
170
|
remoteFilePath: remotePath,
|
|
184
|
-
});
|
|
171
|
+
}));
|
|
185
172
|
});
|
|
186
173
|
if (onQueueAddError) {
|
|
187
174
|
queueAddPromise.catch(onQueueAddError);
|
|
@@ -198,7 +185,7 @@ function watch(accountId, src, dest, { mode, remove, disableInitial, notify, com
|
|
|
198
185
|
const uploadPromise = uploadFile(accountId, filePath, destPath, {
|
|
199
186
|
src,
|
|
200
187
|
commandOptions,
|
|
201
|
-
}, mode
|
|
188
|
+
}, mode);
|
|
202
189
|
(0, notify_1.triggerNotify)(notify, 'Changed', filePath, uploadPromise);
|
|
203
190
|
});
|
|
204
191
|
return watcher;
|
package/lib/fileManager.d.ts
CHANGED
|
@@ -1,6 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
declare
|
|
3
|
-
declare const downloadCallbackKeys: readonly ["skippedExisting", "fetchFolderStarted", "fetchFolderSuccess", "fetchFileStarted", "fetchFileSuccess"];
|
|
4
|
-
export declare function uploadFolder(accountId: number, src: string, dest: string, logCallbacks?: LogCallbacksArg<typeof uploadCallbackKeys>): Promise<void>;
|
|
5
|
-
export declare function downloadFileOrFolder(accountId: number, src: string, dest: string, overwrite?: boolean, includeArchived?: boolean, logCallbacks?: LogCallbacksArg<typeof downloadCallbackKeys>): Promise<void>;
|
|
6
|
-
export {};
|
|
1
|
+
export declare function uploadFolder(accountId: number, src: string, dest: string): Promise<void>;
|
|
2
|
+
export declare function downloadFileOrFolder(accountId: number, src: string, dest: string, overwrite?: boolean, includeArchived?: boolean): Promise<void>;
|
package/lib/fileManager.js
CHANGED
|
@@ -9,7 +9,7 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
|
9
9
|
const path_1 = __importDefault(require("path"));
|
|
10
10
|
const fileManager_1 = require("../api/fileManager");
|
|
11
11
|
const fs_1 = require("./fs");
|
|
12
|
-
const logger_1 = require("
|
|
12
|
+
const logger_1 = require("./logging/logger");
|
|
13
13
|
const ignoreRules_1 = require("./ignoreRules");
|
|
14
14
|
const http_1 = __importDefault(require("../http"));
|
|
15
15
|
const escapeRegExp_1 = require("../utils/escapeRegExp");
|
|
@@ -17,17 +17,9 @@ const path_2 = require("./path");
|
|
|
17
17
|
const apiErrors_1 = require("../errors/apiErrors");
|
|
18
18
|
const standardErrors_1 = require("../errors/standardErrors");
|
|
19
19
|
const fileSystemErrors_1 = require("../errors/fileSystemErrors");
|
|
20
|
+
const lang_1 = require("../utils/lang");
|
|
20
21
|
const i18nKey = 'lib.fileManager';
|
|
21
|
-
|
|
22
|
-
const downloadCallbackKeys = [
|
|
23
|
-
'skippedExisting',
|
|
24
|
-
'fetchFolderStarted',
|
|
25
|
-
'fetchFolderSuccess',
|
|
26
|
-
'fetchFileStarted',
|
|
27
|
-
'fetchFileSuccess',
|
|
28
|
-
];
|
|
29
|
-
async function uploadFolder(accountId, src, dest, logCallbacks) {
|
|
30
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
22
|
+
async function uploadFolder(accountId, src, dest) {
|
|
31
23
|
const regex = new RegExp(`^${(0, escapeRegExp_1.escapeRegExp)(src)}`);
|
|
32
24
|
const files = await (0, fs_1.walk)(src);
|
|
33
25
|
const filesToUpload = files.filter((0, ignoreRules_1.createIgnoreFilter)(false));
|
|
@@ -36,14 +28,14 @@ async function uploadFolder(accountId, src, dest, logCallbacks) {
|
|
|
36
28
|
const file = filesToUpload[index];
|
|
37
29
|
const relativePath = file.replace(regex, '');
|
|
38
30
|
const destPath = (0, path_2.convertToUnixPath)(path_1.default.join(dest, relativePath));
|
|
39
|
-
(0,
|
|
31
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.uploadStarted`, {
|
|
40
32
|
file,
|
|
41
33
|
destPath,
|
|
42
34
|
accountId,
|
|
43
|
-
});
|
|
35
|
+
}));
|
|
44
36
|
try {
|
|
45
37
|
await (0, fileManager_1.uploadFile)(accountId, file, destPath);
|
|
46
|
-
logger(
|
|
38
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.uploadSuccess`, { file, destPath }));
|
|
47
39
|
}
|
|
48
40
|
catch (err) {
|
|
49
41
|
const error = err;
|
|
@@ -58,21 +50,20 @@ async function uploadFolder(accountId, src, dest, logCallbacks) {
|
|
|
58
50
|
}
|
|
59
51
|
}
|
|
60
52
|
exports.uploadFolder = uploadFolder;
|
|
61
|
-
async function skipExisting(overwrite, filepath
|
|
62
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
53
|
+
async function skipExisting(overwrite, filepath) {
|
|
63
54
|
if (overwrite) {
|
|
64
55
|
return false;
|
|
65
56
|
}
|
|
66
57
|
if (await fs_extra_1.default.pathExists(filepath)) {
|
|
67
|
-
logger(
|
|
58
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.skippedExisting`, { filepath }));
|
|
68
59
|
return true;
|
|
69
60
|
}
|
|
70
61
|
return false;
|
|
71
62
|
}
|
|
72
|
-
async function downloadFile(accountId, file, dest, overwrite
|
|
63
|
+
async function downloadFile(accountId, file, dest, overwrite) {
|
|
73
64
|
const fileName = `${file.name}.${file.extension}`;
|
|
74
65
|
const destPath = (0, path_2.convertToLocalFileSystemPath)(path_1.default.join(dest, fileName));
|
|
75
|
-
if (await skipExisting(overwrite || false, destPath
|
|
66
|
+
if (await skipExisting(overwrite || false, destPath)) {
|
|
76
67
|
return;
|
|
77
68
|
}
|
|
78
69
|
await http_1.default.getOctetStream(accountId, {
|
|
@@ -96,7 +87,7 @@ async function fetchAllPagedFiles(accountId, folderId, includeArchived) {
|
|
|
96
87
|
}
|
|
97
88
|
return files;
|
|
98
89
|
}
|
|
99
|
-
async function fetchFolderContents(accountId, folder, dest, overwrite, includeArchived
|
|
90
|
+
async function fetchFolderContents(accountId, folder, dest, overwrite, includeArchived) {
|
|
100
91
|
try {
|
|
101
92
|
await fs_extra_1.default.ensureDir(dest);
|
|
102
93
|
}
|
|
@@ -108,22 +99,21 @@ async function fetchFolderContents(accountId, folder, dest, overwrite, includeAr
|
|
|
108
99
|
});
|
|
109
100
|
}
|
|
110
101
|
const files = await fetchAllPagedFiles(accountId, folder.id, includeArchived);
|
|
111
|
-
(0,
|
|
102
|
+
logger_1.logger.debug((0, lang_1.i18n)(`${i18nKey}.fetchingFiles`, {
|
|
112
103
|
fileCount: files.length,
|
|
113
104
|
folderName: folder.name || '',
|
|
114
|
-
});
|
|
105
|
+
}));
|
|
115
106
|
for (const file of files) {
|
|
116
|
-
await downloadFile(accountId, file, dest, overwrite
|
|
107
|
+
await downloadFile(accountId, file, dest, overwrite);
|
|
117
108
|
}
|
|
118
109
|
const { objects: folders } = await (0, fileManager_1.fetchFolders)(accountId, folder.id);
|
|
119
110
|
for (const folder of folders) {
|
|
120
111
|
const nestedFolder = path_1.default.join(dest, folder.name);
|
|
121
|
-
await fetchFolderContents(accountId, folder, nestedFolder, overwrite, includeArchived
|
|
112
|
+
await fetchFolderContents(accountId, folder, nestedFolder, overwrite, includeArchived);
|
|
122
113
|
}
|
|
123
114
|
}
|
|
124
115
|
// Download a folder and write to local file system.
|
|
125
|
-
async function downloadFolder(accountId, src, dest, folder, overwrite, includeArchived
|
|
126
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
116
|
+
async function downloadFolder(accountId, src, dest, folder, overwrite, includeArchived) {
|
|
127
117
|
let absolutePath;
|
|
128
118
|
if (folder.name) {
|
|
129
119
|
absolutePath = (0, path_2.convertToLocalFileSystemPath)(path_1.default.resolve((0, path_2.getCwd)(), dest, folder.name));
|
|
@@ -131,52 +121,51 @@ async function downloadFolder(accountId, src, dest, folder, overwrite, includeAr
|
|
|
131
121
|
else {
|
|
132
122
|
absolutePath = (0, path_2.convertToLocalFileSystemPath)(path_1.default.resolve((0, path_2.getCwd)(), dest));
|
|
133
123
|
}
|
|
134
|
-
logger(
|
|
124
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.fetchFolderStarted`, {
|
|
135
125
|
src,
|
|
136
126
|
path: absolutePath,
|
|
137
127
|
accountId,
|
|
138
|
-
});
|
|
139
|
-
await fetchFolderContents(accountId, folder, absolutePath, overwrite, includeArchived
|
|
140
|
-
logger(
|
|
128
|
+
}));
|
|
129
|
+
await fetchFolderContents(accountId, folder, absolutePath, overwrite, includeArchived);
|
|
130
|
+
logger_1.logger.success((0, lang_1.i18n)(`${i18nKey}.fetchFolderSuccess`, {
|
|
141
131
|
src,
|
|
142
132
|
dest,
|
|
143
|
-
});
|
|
133
|
+
}));
|
|
144
134
|
}
|
|
145
135
|
// Download a single file and write to local file system.
|
|
146
|
-
async function downloadSingleFile(accountId, src, dest, file, overwrite, includeArchived
|
|
147
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
136
|
+
async function downloadSingleFile(accountId, src, dest, file, overwrite, includeArchived) {
|
|
148
137
|
if (!includeArchived && file.archived) {
|
|
149
138
|
(0, standardErrors_1.throwErrorWithMessage)(`${i18nKey}.errors.archivedFile`, { src });
|
|
150
139
|
}
|
|
151
140
|
if (file.hidden) {
|
|
152
141
|
(0, standardErrors_1.throwErrorWithMessage)(`${i18nKey}.errors.hiddenFile`, { src });
|
|
153
142
|
}
|
|
154
|
-
logger(
|
|
143
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.fetchFileStarted`, {
|
|
155
144
|
src,
|
|
156
145
|
dest,
|
|
157
146
|
accountId,
|
|
158
|
-
});
|
|
159
|
-
await downloadFile(accountId, file, dest, overwrite
|
|
160
|
-
logger(
|
|
147
|
+
}));
|
|
148
|
+
await downloadFile(accountId, file, dest, overwrite);
|
|
149
|
+
logger_1.logger.success((0, lang_1.i18n)(`${i18nKey}.fetchFileSuccess`, {
|
|
161
150
|
src,
|
|
162
151
|
dest,
|
|
163
|
-
});
|
|
152
|
+
}));
|
|
164
153
|
}
|
|
165
154
|
// Lookup path in file manager and initiate download
|
|
166
|
-
async function downloadFileOrFolder(accountId, src, dest, overwrite, includeArchived
|
|
155
|
+
async function downloadFileOrFolder(accountId, src, dest, overwrite, includeArchived) {
|
|
167
156
|
try {
|
|
168
157
|
if (src == '/') {
|
|
169
158
|
// Filemanager API treats 'None' as the root
|
|
170
159
|
const rootFolder = { id: 'None', name: '' };
|
|
171
|
-
await downloadFolder(accountId, src, dest, rootFolder, overwrite, includeArchived
|
|
160
|
+
await downloadFolder(accountId, src, dest, rootFolder, overwrite, includeArchived);
|
|
172
161
|
}
|
|
173
162
|
else {
|
|
174
163
|
const { file, folder } = await (0, fileManager_1.fetchStat)(accountId, src);
|
|
175
164
|
if (file) {
|
|
176
|
-
await downloadSingleFile(accountId, src, dest, file, overwrite, includeArchived
|
|
165
|
+
await downloadSingleFile(accountId, src, dest, file, overwrite, includeArchived);
|
|
177
166
|
}
|
|
178
167
|
else if (folder) {
|
|
179
|
-
await downloadFolder(accountId, src, dest, folder, overwrite, includeArchived
|
|
168
|
+
await downloadFolder(accountId, src, dest, folder, overwrite, includeArchived);
|
|
180
169
|
}
|
|
181
170
|
}
|
|
182
171
|
}
|
package/lib/fileMapper.d.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import { FileMapperNode, Mode, FileMapperOptions, FileMapperInputOptions } from '../types/Files';
|
|
2
|
-
import { LogCallbacksArg } from '../types/LogCallbacks';
|
|
3
|
-
declare const filemapperCallbackKeys: readonly ["skippedExisting", "wroteFolder", "completedFetch", "folderFetch", "completedFolderFetch"];
|
|
4
2
|
export declare function isPathToFile(filepath: string): boolean;
|
|
5
3
|
export declare function isPathToModule(filepath: string): boolean;
|
|
6
4
|
export declare function isPathToRoot(filepath: string): boolean;
|
|
@@ -17,7 +15,7 @@ export declare function getTypeDataFromPath(src: string): PathTypeData;
|
|
|
17
15
|
type RecursiveFileMapperCallback = (node: FileMapperNode, filepath?: string, depth?: number) => boolean;
|
|
18
16
|
export declare function recurseFolder(node: FileMapperNode, callback: RecursiveFileMapperCallback, filepath?: string, depth?: number): boolean;
|
|
19
17
|
export declare function writeUtimes(accountId: number, filepath: string, node: FileMapperNode): Promise<void>;
|
|
20
|
-
export declare function fetchFolderFromApi(accountId: number, src: string, mode?: Mode, options?: FileMapperInputOptions
|
|
18
|
+
export declare function fetchFolderFromApi(accountId: number, src: string, mode?: Mode, options?: FileMapperInputOptions): Promise<FileMapperNode>;
|
|
21
19
|
/**
|
|
22
20
|
* Fetch a file/folder and write to local file system.
|
|
23
21
|
*
|
|
@@ -25,5 +23,5 @@ export declare function fetchFolderFromApi(accountId: number, src: string, mode?
|
|
|
25
23
|
* @param {FileMapperInputArguments} input
|
|
26
24
|
* @returns {Promise}
|
|
27
25
|
*/
|
|
28
|
-
export declare function downloadFileOrFolder(accountId: number, src: string, dest: string, mode?: Mode, options?: FileMapperInputOptions
|
|
26
|
+
export declare function downloadFileOrFolder(accountId: number, src: string, dest: string, mode?: Mode, options?: FileMapperInputOptions): Promise<void>;
|
|
29
27
|
export {};
|
package/lib/fileMapper.js
CHANGED
|
@@ -9,20 +9,14 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
|
9
9
|
const path_1 = __importDefault(require("path"));
|
|
10
10
|
const p_queue_1 = __importDefault(require("p-queue"));
|
|
11
11
|
const path_2 = require("./path");
|
|
12
|
+
const logger_1 = require("./logging/logger");
|
|
12
13
|
const fileMapper_1 = require("../api/fileMapper");
|
|
13
14
|
const standardErrors_1 = require("../errors/standardErrors");
|
|
14
15
|
const extensions_1 = require("../constants/extensions");
|
|
15
16
|
const files_1 = require("../constants/files");
|
|
16
17
|
const fileSystemErrors_1 = require("../errors/fileSystemErrors");
|
|
17
|
-
const
|
|
18
|
+
const lang_1 = require("../utils/lang");
|
|
18
19
|
const i18nKey = 'lib.fileMapper';
|
|
19
|
-
const filemapperCallbackKeys = [
|
|
20
|
-
'skippedExisting',
|
|
21
|
-
'wroteFolder',
|
|
22
|
-
'completedFetch',
|
|
23
|
-
'folderFetch',
|
|
24
|
-
'completedFolderFetch',
|
|
25
|
-
];
|
|
26
20
|
const queue = new p_queue_1.default({
|
|
27
21
|
concurrency: 10,
|
|
28
22
|
});
|
|
@@ -147,13 +141,12 @@ async function skipExisting(filepath, overwrite = false) {
|
|
|
147
141
|
}
|
|
148
142
|
return false;
|
|
149
143
|
}
|
|
150
|
-
async function fetchAndWriteFileStream(accountId, srcPath, filepath, mode, options = {}
|
|
151
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
144
|
+
async function fetchAndWriteFileStream(accountId, srcPath, filepath, mode, options = {}) {
|
|
152
145
|
if (typeof srcPath !== 'string' || !srcPath.trim()) {
|
|
153
146
|
return;
|
|
154
147
|
}
|
|
155
148
|
if (await skipExisting(filepath, options.overwrite)) {
|
|
156
|
-
logger(
|
|
149
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.skippedExisting`, { filepath }));
|
|
157
150
|
return;
|
|
158
151
|
}
|
|
159
152
|
if (!(0, path_2.isAllowedExtension)(srcPath)) {
|
|
@@ -170,18 +163,17 @@ async function fetchAndWriteFileStream(accountId, srcPath, filepath, mode, optio
|
|
|
170
163
|
}
|
|
171
164
|
// Writes an individual file or folder (not recursive). If file source is missing, the
|
|
172
165
|
//file is fetched.
|
|
173
|
-
async function writeFileMapperNode(accountId, filepath, node, mode, options = {}
|
|
174
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
166
|
+
async function writeFileMapperNode(accountId, filepath, node, mode, options = {}) {
|
|
175
167
|
const localFilepath = (0, path_2.convertToLocalFileSystemPath)(path_1.default.resolve(filepath));
|
|
176
168
|
if (await skipExisting(localFilepath, options.overwrite)) {
|
|
177
|
-
logger(
|
|
169
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.skippedExisting`, {
|
|
178
170
|
filepath: localFilepath,
|
|
179
|
-
});
|
|
171
|
+
}));
|
|
180
172
|
return true;
|
|
181
173
|
}
|
|
182
174
|
if (!node.folder) {
|
|
183
175
|
try {
|
|
184
|
-
await fetchAndWriteFileStream(accountId, node.path, localFilepath, mode, options
|
|
176
|
+
await fetchAndWriteFileStream(accountId, node.path, localFilepath, mode, options);
|
|
185
177
|
return true;
|
|
186
178
|
}
|
|
187
179
|
catch (err) {
|
|
@@ -190,9 +182,9 @@ async function writeFileMapperNode(accountId, filepath, node, mode, options = {}
|
|
|
190
182
|
}
|
|
191
183
|
try {
|
|
192
184
|
await fs_extra_1.default.ensureDir(localFilepath);
|
|
193
|
-
logger(
|
|
185
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.wroteFolder`, {
|
|
194
186
|
filepath: localFilepath,
|
|
195
|
-
});
|
|
187
|
+
}));
|
|
196
188
|
}
|
|
197
189
|
catch (err) {
|
|
198
190
|
(0, fileSystemErrors_1.throwFileSystemError)(err, {
|
|
@@ -207,8 +199,7 @@ async function writeFileMapperNode(accountId, filepath, node, mode, options = {}
|
|
|
207
199
|
function isTimeout(err) {
|
|
208
200
|
return !!err && (err.status === 408 || err.code === 'ESOCKETTIMEDOUT');
|
|
209
201
|
}
|
|
210
|
-
async function downloadFile(accountId, src, destPath, mode, options = {}
|
|
211
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
202
|
+
async function downloadFile(accountId, src, destPath, mode, options = {}) {
|
|
212
203
|
const { isFile, isHubspot } = getTypeDataFromPath(src);
|
|
213
204
|
try {
|
|
214
205
|
if (!isFile) {
|
|
@@ -233,13 +224,13 @@ async function downloadFile(accountId, src, destPath, mode, options = {}, logCal
|
|
|
233
224
|
: path_1.default.resolve(cwd, dest, name);
|
|
234
225
|
}
|
|
235
226
|
const localFsPath = (0, path_2.convertToLocalFileSystemPath)(filepath);
|
|
236
|
-
await fetchAndWriteFileStream(accountId, src, localFsPath, mode, options
|
|
227
|
+
await fetchAndWriteFileStream(accountId, src, localFsPath, mode, options);
|
|
237
228
|
await queue.onIdle();
|
|
238
|
-
logger(
|
|
229
|
+
logger_1.logger.success((0, lang_1.i18n)(`${i18nKey}.completedFetch`, {
|
|
239
230
|
src,
|
|
240
231
|
version: getAssetVersionIdentifier(options.assetVersion, src),
|
|
241
232
|
dest,
|
|
242
|
-
});
|
|
233
|
+
}));
|
|
243
234
|
}
|
|
244
235
|
catch (err) {
|
|
245
236
|
const error = err;
|
|
@@ -251,8 +242,7 @@ async function downloadFile(accountId, src, destPath, mode, options = {}, logCal
|
|
|
251
242
|
}
|
|
252
243
|
}
|
|
253
244
|
}
|
|
254
|
-
async function fetchFolderFromApi(accountId, src, mode, options = {}
|
|
255
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
245
|
+
async function fetchFolderFromApi(accountId, src, mode, options = {}) {
|
|
256
246
|
const { isRoot, isFolder, isHubspot } = getTypeDataFromPath(src);
|
|
257
247
|
if (!isFolder) {
|
|
258
248
|
(0, standardErrors_1.throwErrorWithMessage)(`${i18nKey}.errors.invalidFetchFolderRequest`, {
|
|
@@ -265,7 +255,7 @@ async function fetchFolderFromApi(accountId, src, mode, options = {}, logCallbac
|
|
|
265
255
|
const node = isHubspot
|
|
266
256
|
? await (0, fileMapper_1.downloadDefault)(accountId, srcPath, queryValues)
|
|
267
257
|
: await (0, fileMapper_1.download)(accountId, srcPath, queryValues);
|
|
268
|
-
logger(
|
|
258
|
+
logger_1.logger.log((0, lang_1.i18n)(`${i18nKey}.folderFetch`, { src, accountId }));
|
|
269
259
|
return node;
|
|
270
260
|
}
|
|
271
261
|
catch (err) {
|
|
@@ -279,10 +269,9 @@ async function fetchFolderFromApi(accountId, src, mode, options = {}, logCallbac
|
|
|
279
269
|
}
|
|
280
270
|
}
|
|
281
271
|
exports.fetchFolderFromApi = fetchFolderFromApi;
|
|
282
|
-
async function downloadFolder(accountId, src, destPath, mode, options = {}
|
|
283
|
-
const logger = (0, logger_1.makeTypedLogger)(logCallbacks);
|
|
272
|
+
async function downloadFolder(accountId, src, destPath, mode, options = {}) {
|
|
284
273
|
try {
|
|
285
|
-
const node = await fetchFolderFromApi(accountId, src, mode, options
|
|
274
|
+
const node = await fetchFolderFromApi(accountId, src, mode, options);
|
|
286
275
|
if (!node) {
|
|
287
276
|
return;
|
|
288
277
|
}
|
|
@@ -293,7 +282,7 @@ async function downloadFolder(accountId, src, destPath, mode, options = {}, logC
|
|
|
293
282
|
let success = true;
|
|
294
283
|
recurseFolder(node, (childNode, filepath) => {
|
|
295
284
|
queue.add(async () => {
|
|
296
|
-
const succeeded = await writeFileMapperNode(accountId, filepath || '', childNode, mode, options
|
|
285
|
+
const succeeded = await writeFileMapperNode(accountId, filepath || '', childNode, mode, options);
|
|
297
286
|
if (succeeded === false) {
|
|
298
287
|
success = false;
|
|
299
288
|
}
|
|
@@ -302,11 +291,11 @@ async function downloadFolder(accountId, src, destPath, mode, options = {}, logC
|
|
|
302
291
|
}, rootPath);
|
|
303
292
|
await queue.onIdle();
|
|
304
293
|
if (success) {
|
|
305
|
-
logger(
|
|
294
|
+
logger_1.logger.success((0, lang_1.i18n)(`${i18nKey}.completedFolderFetch`, {
|
|
306
295
|
src,
|
|
307
296
|
version: getAssetVersionIdentifier(options.assetVersion, src),
|
|
308
297
|
dest,
|
|
309
|
-
});
|
|
298
|
+
}));
|
|
310
299
|
}
|
|
311
300
|
else {
|
|
312
301
|
(0, standardErrors_1.throwErrorWithMessage)(`${i18nKey}.errors.incompleteFetch`, { src });
|
|
@@ -323,16 +312,16 @@ async function downloadFolder(accountId, src, destPath, mode, options = {}, logC
|
|
|
323
312
|
* @param {FileMapperInputArguments} input
|
|
324
313
|
* @returns {Promise}
|
|
325
314
|
*/
|
|
326
|
-
async function downloadFileOrFolder(accountId, src, dest, mode, options = {}
|
|
315
|
+
async function downloadFileOrFolder(accountId, src, dest, mode, options = {}) {
|
|
327
316
|
if (!src) {
|
|
328
317
|
return;
|
|
329
318
|
}
|
|
330
319
|
const { isFile } = getTypeDataFromPath(src);
|
|
331
320
|
if (isFile) {
|
|
332
|
-
await downloadFile(accountId, src, dest, mode, options
|
|
321
|
+
await downloadFile(accountId, src, dest, mode, options);
|
|
333
322
|
}
|
|
334
323
|
else {
|
|
335
|
-
await downloadFolder(accountId, src, dest, mode, options
|
|
324
|
+
await downloadFolder(accountId, src, dest, mode, options);
|
|
336
325
|
}
|
|
337
326
|
}
|
|
338
327
|
exports.downloadFileOrFolder = downloadFileOrFolder;
|
package/lib/github.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { GithubReleaseData } from '../types/Github';
|
|
3
|
-
import { LogCallbacksArg } from '../types/LogCallbacks';
|
|
4
|
-
declare const cloneGithubRepoCallbackKeys: readonly ["success"];
|
|
2
|
+
import { GithubReleaseData, GithubRepoFile } from '../types/Github';
|
|
5
3
|
type RepoPath = `${string}/${string}`;
|
|
6
4
|
export declare function fetchFileFromRepository(repoPath: RepoPath, filePath: string, ref: string): Promise<Buffer>;
|
|
7
5
|
export declare function fetchReleaseData(repoPath: RepoPath, tag?: string): Promise<GithubReleaseData>;
|
|
@@ -12,6 +10,7 @@ type CloneGithubRepoOptions = {
|
|
|
12
10
|
tag?: string;
|
|
13
11
|
sourceDir?: string;
|
|
14
12
|
};
|
|
15
|
-
export declare function cloneGithubRepo(repoPath: RepoPath, dest: string, options?: CloneGithubRepoOptions
|
|
13
|
+
export declare function cloneGithubRepo(repoPath: RepoPath, dest: string, options?: CloneGithubRepoOptions): Promise<boolean>;
|
|
16
14
|
export declare function downloadGithubRepoContents(repoPath: RepoPath, contentPath: string, dest: string, ref?: string, filter?: (contentPiecePath: string, downloadPath: string) => boolean): Promise<void>;
|
|
15
|
+
export declare function listGithubRepoContents(repoPath: RepoPath, contentPath: string, fileFilter?: 'file' | 'dir'): Promise<GithubRepoFile[]>;
|
|
17
16
|
export {};
|