@cyberalien/deploy-utils 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ssh/upload-files.d.ts +2 -2
- package/lib/ssh/upload-files.js +22 -29
- package/package.json +1 -1
|
@@ -5,9 +5,9 @@ import { GitRepoFilesList } from "@cyberalien/git-utils/lib/git/index.js";
|
|
|
5
5
|
*
|
|
6
6
|
* For binary files use putSFTPFiles() instead
|
|
7
7
|
*/
|
|
8
|
-
declare function uploadSFTPFiles(client: Client, remoteRootDir: string, files: GitRepoFilesList): Promise<void>;
|
|
8
|
+
declare function uploadSFTPFiles(client: Client, remoteRootDir: string, files: GitRepoFilesList, queueSize?: number): Promise<void>;
|
|
9
9
|
/**
|
|
10
10
|
* Upload multiple local files
|
|
11
11
|
*/
|
|
12
|
-
declare function putSFTPFiles(client: Client, localRootDir: string, remoteRootDir: string, filenames: string[]): Promise<void>;
|
|
12
|
+
declare function putSFTPFiles(client: Client, localRootDir: string, remoteRootDir: string, filenames: string[], queueSize?: number): Promise<void>;
|
|
13
13
|
export { putSFTPFiles, uploadSFTPFiles };
|
package/lib/ssh/upload-files.js
CHANGED
|
@@ -30,25 +30,18 @@ async function createDirectories(client, rootDir, filenames) {
|
|
|
30
30
|
if (execResult.code !== 0) throw new Error(`Failed to create directories: ${execResult.stderr || execResult.stdout}`);
|
|
31
31
|
}
|
|
32
32
|
/**
|
|
33
|
-
*
|
|
33
|
+
* Process queue
|
|
34
34
|
*/
|
|
35
|
-
async function
|
|
36
|
-
const
|
|
35
|
+
async function processQueue(client, queue, queueSize, callback) {
|
|
36
|
+
const total = queue.length;
|
|
37
|
+
queueSize = queueSize || (total > 40 ? 5 : total > 20 ? 4 : total > 10 ? 3 : total > 3 ? 2 : 1);
|
|
38
|
+
const sessions = [await startSFTPSession(client)];
|
|
37
39
|
const addSession = async () => {
|
|
38
40
|
try {
|
|
39
|
-
|
|
41
|
+
sessions.push(await startSFTPSession(client));
|
|
40
42
|
} catch {}
|
|
41
43
|
};
|
|
42
|
-
|
|
43
|
-
if (total > 10) await addSession();
|
|
44
|
-
if (total > 20) await addSession();
|
|
45
|
-
return sftp;
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Process queue
|
|
49
|
-
*/
|
|
50
|
-
async function processQueue(client, queue, callback) {
|
|
51
|
-
const sessions = await createSessions(client, queue.length);
|
|
44
|
+
for (let i = 1; i < queueSize; i++) await addSession();
|
|
52
45
|
await new Promise((resolve, reject) => {
|
|
53
46
|
let activeSessions = 0;
|
|
54
47
|
const next = (sftp) => {
|
|
@@ -63,7 +56,9 @@ async function processQueue(client, queue, callback) {
|
|
|
63
56
|
};
|
|
64
57
|
for (const session of sessions) {
|
|
65
58
|
activeSessions++;
|
|
66
|
-
|
|
59
|
+
setTimeout(() => {
|
|
60
|
+
next(session);
|
|
61
|
+
});
|
|
67
62
|
}
|
|
68
63
|
});
|
|
69
64
|
}
|
|
@@ -72,39 +67,37 @@ async function processQueue(client, queue, callback) {
|
|
|
72
67
|
*
|
|
73
68
|
* For binary files use putSFTPFiles() instead
|
|
74
69
|
*/
|
|
75
|
-
async function uploadSFTPFiles(client, remoteRootDir, files) {
|
|
70
|
+
async function uploadSFTPFiles(client, remoteRootDir, files, queueSize = 0) {
|
|
71
|
+
let index = 0;
|
|
76
72
|
const filenames = [];
|
|
77
73
|
let filesCount = 0;
|
|
78
74
|
for (const file in files) {
|
|
79
75
|
filesCount++;
|
|
80
|
-
if (files[file] === null)
|
|
76
|
+
if (files[file] === null) {
|
|
77
|
+
console.log(`[${index} / ${filesCount}] Deleting ${file}`);
|
|
78
|
+
await execSFTPCommand(client, `rm -f "${join(remoteRootDir, file)}"`);
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
81
|
filenames.push(file);
|
|
82
82
|
}
|
|
83
|
+
if (!filenames.length) return;
|
|
83
84
|
await createDirectories(client, remoteRootDir, filenames);
|
|
84
|
-
|
|
85
|
-
let index = 0;
|
|
86
|
-
for (const file in files) {
|
|
85
|
+
await processQueue(client, filenames, queueSize, async (sftp, file) => {
|
|
87
86
|
index++;
|
|
88
87
|
const content = files[file];
|
|
89
88
|
const target = join(remoteRootDir, file);
|
|
90
|
-
if (content === null) {
|
|
91
|
-
console.log(`[${index} / ${filesCount}] Deleting ${file}`);
|
|
92
|
-
await execSFTPCommand(client, `rm -f "${target}"`);
|
|
93
|
-
continue;
|
|
94
|
-
}
|
|
95
89
|
console.log(`[${index} / ${filesCount}] Uploading ${file}`);
|
|
96
90
|
await uploadSFTPFile(sftp, target, typeof content === "string" ? content : JSON.stringify(content, null, 2) + "\n");
|
|
97
|
-
}
|
|
98
|
-
sftp.end();
|
|
91
|
+
});
|
|
99
92
|
}
|
|
100
93
|
/**
|
|
101
94
|
* Upload multiple local files
|
|
102
95
|
*/
|
|
103
|
-
async function putSFTPFiles(client, localRootDir, remoteRootDir, filenames) {
|
|
96
|
+
async function putSFTPFiles(client, localRootDir, remoteRootDir, filenames, queueSize = 0) {
|
|
104
97
|
await createDirectories(client, remoteRootDir, filenames);
|
|
105
98
|
let index = 0;
|
|
106
99
|
const filesCount = filenames.length;
|
|
107
|
-
await processQueue(client, [...filenames], async (sftp, file) => {
|
|
100
|
+
await processQueue(client, [...filenames], queueSize, async (sftp, file) => {
|
|
108
101
|
index++;
|
|
109
102
|
console.log(`[${index} / ${filesCount}] Uploading ${file}`);
|
|
110
103
|
await putSFTPFile(sftp, join(localRootDir, file), join(remoteRootDir, file));
|
package/package.json
CHANGED