@cyberalien/deploy-utils 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ssh/upload-files.d.ts +2 -2
- package/lib/ssh/upload-files.js +14 -19
- package/package.json +1 -1
|
@@ -5,9 +5,9 @@ import { GitRepoFilesList } from "@cyberalien/git-utils/lib/git/index.js";
|
|
|
5
5
|
*
|
|
6
6
|
* For binary files use putSFTPFiles() instead
|
|
7
7
|
*/
|
|
8
|
-
declare function uploadSFTPFiles(client: Client, remoteRootDir: string, files: GitRepoFilesList): Promise<void>;
|
|
8
|
+
declare function uploadSFTPFiles(client: Client, remoteRootDir: string, files: GitRepoFilesList, queueSize?: number): Promise<void>;
|
|
9
9
|
/**
|
|
10
10
|
* Upload multiple local files
|
|
11
11
|
*/
|
|
12
|
-
declare function putSFTPFiles(client: Client, localRootDir: string, remoteRootDir: string, filenames: string[]): Promise<void>;
|
|
12
|
+
declare function putSFTPFiles(client: Client, localRootDir: string, remoteRootDir: string, filenames: string[], queueSize?: number): Promise<void>;
|
|
13
13
|
export { putSFTPFiles, uploadSFTPFiles };
|
package/lib/ssh/upload-files.js
CHANGED
|
@@ -30,25 +30,18 @@ async function createDirectories(client, rootDir, filenames) {
|
|
|
30
30
|
if (execResult.code !== 0) throw new Error(`Failed to create directories: ${execResult.stderr || execResult.stdout}`);
|
|
31
31
|
}
|
|
32
32
|
/**
|
|
33
|
-
*
|
|
33
|
+
* Process queue
|
|
34
34
|
*/
|
|
35
|
-
async function
|
|
36
|
-
const
|
|
35
|
+
async function processQueue(client, queue, queueSize, callback) {
|
|
36
|
+
const total = queue.length;
|
|
37
|
+
queueSize = queueSize || (total > 40 ? 5 : total > 20 ? 4 : total > 10 ? 3 : total > 3 ? 2 : 1);
|
|
38
|
+
const sessions = [await startSFTPSession(client)];
|
|
37
39
|
const addSession = async () => {
|
|
38
40
|
try {
|
|
39
|
-
|
|
41
|
+
sessions.push(await startSFTPSession(client));
|
|
40
42
|
} catch {}
|
|
41
43
|
};
|
|
42
|
-
|
|
43
|
-
if (total > 10) await addSession();
|
|
44
|
-
if (total > 20) await addSession();
|
|
45
|
-
return sftp;
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Process queue
|
|
49
|
-
*/
|
|
50
|
-
async function processQueue(client, queue, callback) {
|
|
51
|
-
const sessions = await createSessions(client, queue.length);
|
|
44
|
+
for (let i = 1; i < queueSize; i++) await addSession();
|
|
52
45
|
await new Promise((resolve, reject) => {
|
|
53
46
|
let activeSessions = 0;
|
|
54
47
|
const next = (sftp) => {
|
|
@@ -63,7 +56,9 @@ async function processQueue(client, queue, callback) {
|
|
|
63
56
|
};
|
|
64
57
|
for (const session of sessions) {
|
|
65
58
|
activeSessions++;
|
|
66
|
-
|
|
59
|
+
setTimeout(() => {
|
|
60
|
+
next(session);
|
|
61
|
+
});
|
|
67
62
|
}
|
|
68
63
|
});
|
|
69
64
|
}
|
|
@@ -72,7 +67,7 @@ async function processQueue(client, queue, callback) {
|
|
|
72
67
|
*
|
|
73
68
|
* For binary files use putSFTPFiles() instead
|
|
74
69
|
*/
|
|
75
|
-
async function uploadSFTPFiles(client, remoteRootDir, files) {
|
|
70
|
+
async function uploadSFTPFiles(client, remoteRootDir, files, queueSize = 0) {
|
|
76
71
|
let index = 0;
|
|
77
72
|
const filenames = [];
|
|
78
73
|
let filesCount = 0;
|
|
@@ -87,7 +82,7 @@ async function uploadSFTPFiles(client, remoteRootDir, files) {
|
|
|
87
82
|
}
|
|
88
83
|
if (!filenames.length) return;
|
|
89
84
|
await createDirectories(client, remoteRootDir, filenames);
|
|
90
|
-
await processQueue(client, filenames, async (sftp, file) => {
|
|
85
|
+
await processQueue(client, filenames, queueSize, async (sftp, file) => {
|
|
91
86
|
index++;
|
|
92
87
|
const content = files[file];
|
|
93
88
|
const target = join(remoteRootDir, file);
|
|
@@ -98,11 +93,11 @@ async function uploadSFTPFiles(client, remoteRootDir, files) {
|
|
|
98
93
|
/**
|
|
99
94
|
* Upload multiple local files
|
|
100
95
|
*/
|
|
101
|
-
async function putSFTPFiles(client, localRootDir, remoteRootDir, filenames) {
|
|
96
|
+
async function putSFTPFiles(client, localRootDir, remoteRootDir, filenames, queueSize = 0) {
|
|
102
97
|
await createDirectories(client, remoteRootDir, filenames);
|
|
103
98
|
let index = 0;
|
|
104
99
|
const filesCount = filenames.length;
|
|
105
|
-
await processQueue(client, [...filenames], async (sftp, file) => {
|
|
100
|
+
await processQueue(client, [...filenames], queueSize, async (sftp, file) => {
|
|
106
101
|
index++;
|
|
107
102
|
console.log(`[${index} / ${filesCount}] Uploading ${file}`);
|
|
108
103
|
await putSFTPFile(sftp, join(localRootDir, file), join(remoteRootDir, file));
|
package/package.json
CHANGED