@leogps/file-uploader 2.0.3 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -10
- package/dist/client/index.html +1 -1
- package/dist/client/{main.5f2a69fd2944bb03bad4.bundle.js → main.5b512c69d6a84b326aa0.bundle.js} +395 -321
- package/dist/index.js +1 -1
- package/package.json +5 -2
- package/scripts/inject-version.js +23 -0
- package/src/globals.ts +57 -29
- package/src/index.ts +59 -37
- package/src/routes/config.ts +15 -0
- package/src/routes/upload.ts +6 -7
- package/src/routes/uploadChunk.ts +3 -3
- package/src/routes/uploadInit.ts +6 -7
- package/src-client/entrypoint.ts +127 -23
- package/tsconfig.dev.json +11 -0
- package/webpack-client.common.js +0 -5
- package/webpack-client.dev.js +8 -0
- package/webpack-client.prod.js +8 -0
- package/webpack.config.js +6 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@leogps/file-uploader",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.4",
|
|
4
4
|
"description": "Facilitates file uploader server.",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"repository": {
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
},
|
|
13
13
|
"scripts": {
|
|
14
14
|
"clean": "rimraf dist file-uploader.tar.gz",
|
|
15
|
-
"precompile": "eslint -c .eslintrc.js --fix --ext .ts src src-client",
|
|
15
|
+
"precompile": "eslint -c .eslintrc.js --fix --ext .ts src src-client && node scripts/inject-version.js",
|
|
16
16
|
"compile-server": "./node_modules/webpack-cli/bin/cli.js --config webpack.config.js",
|
|
17
17
|
"compile-client-dev": "./node_modules/webpack-cli/bin/cli.js --config webpack-client.dev.js",
|
|
18
18
|
"compile-client-prod": "./node_modules/webpack-cli/bin/cli.js --config webpack-client.prod.js",
|
|
@@ -105,10 +105,13 @@
|
|
|
105
105
|
"rimraf": "^6.1.2",
|
|
106
106
|
"sass": "^1.94.2",
|
|
107
107
|
"sass-loader": "^13.3.3",
|
|
108
|
+
"source-map-support": "^0.5.21",
|
|
108
109
|
"style-loader": "^3.3.1",
|
|
109
110
|
"targz": "^1.0.1",
|
|
110
111
|
"ts-loader": "^9.5.4",
|
|
112
|
+
"ts-node": "^10.9.2",
|
|
111
113
|
"tsc-watch": "^5.0.3",
|
|
114
|
+
"tsconfig-paths": "^4.2.0",
|
|
112
115
|
"typescript": "^5.9.3",
|
|
113
116
|
"webpack": "^5.74.0",
|
|
114
117
|
"webpack-cli": "^6.0.1",
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const pkg = require('../package.json');
|
|
4
|
+
|
|
5
|
+
const filePath = path.join(__dirname, '../src/globals.ts');
|
|
6
|
+
|
|
7
|
+
let content = fs.readFileSync(filePath, 'utf8');
|
|
8
|
+
|
|
9
|
+
const versionBlockRegex = /\/\/ version:start[\s\S]*?\/\/ version:end/;
|
|
10
|
+
|
|
11
|
+
const newBlock = `// version:start
|
|
12
|
+
export const APP_VERSION = '${pkg.version}';
|
|
13
|
+
// version:end`;
|
|
14
|
+
|
|
15
|
+
if (!versionBlockRegex.test(content)) {
|
|
16
|
+
throw new Error('Version block markers not found in globals.ts');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
content = content.replace(versionBlockRegex, newBlock);
|
|
20
|
+
|
|
21
|
+
fs.writeFileSync(filePath, content, 'utf8');
|
|
22
|
+
|
|
23
|
+
console.log(`APP_VERSION updated to ${pkg.version}`);
|
package/src/globals.ts
CHANGED
|
@@ -1,42 +1,70 @@
|
|
|
1
1
|
import { Progress } from './model/progress';
|
|
2
2
|
import { ProgressWriter } from './service/progress_writer';
|
|
3
3
|
import _ from "lodash";
|
|
4
|
+
import os from "os";
|
|
5
|
+
import path from "path";
|
|
4
6
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
let enableCompression = true;
|
|
9
|
-
let serverPort = 8082;
|
|
10
|
-
let maxFileSize = 100 * 1024 * 1024 * 1024; // 100Gb
|
|
7
|
+
// version:start
|
|
8
|
+
export const APP_VERSION = '2.0.4';
|
|
9
|
+
// version:end
|
|
11
10
|
|
|
12
|
-
export
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
export interface ServerConfig {
|
|
12
|
+
readonly uploadsDir: string;
|
|
13
|
+
readonly uploadChunkSize: number;
|
|
14
|
+
readonly maxParallelFileUploads: number;
|
|
15
|
+
readonly maxParallelChunkUploads: number;
|
|
16
|
+
readonly enableCompression: boolean;
|
|
17
|
+
readonly serverPort: number;
|
|
18
|
+
readonly maxFileSize: number;
|
|
19
|
+
readonly throttleWaitTimeInMillis: number;
|
|
20
|
+
readonly progressWriter?: ProgressWriter;
|
|
21
|
+
readonly version: string;
|
|
22
|
+
}
|
|
23
|
+
export type ServerConfigJson = Omit<ServerConfig, "progressWriter">;
|
|
16
24
|
|
|
17
|
-
export const
|
|
18
|
-
|
|
25
|
+
export const DEFAULT_SERVER_CONFIG: ServerConfig = Object.freeze({
|
|
26
|
+
uploadsDir: `${os.homedir()}${path.sep}uploads/`,
|
|
27
|
+
uploadChunkSize: 512 * 1024,
|
|
28
|
+
maxParallelFileUploads: 3,
|
|
29
|
+
maxParallelChunkUploads: 10,
|
|
30
|
+
enableCompression: true,
|
|
31
|
+
serverPort: 8082,
|
|
32
|
+
maxFileSize: 100 * 1024 * 1024 * 1024, // 100GB
|
|
33
|
+
throttleWaitTimeInMillis: 250,
|
|
34
|
+
version: APP_VERSION
|
|
35
|
+
});
|
|
19
36
|
|
|
20
|
-
|
|
21
|
-
export const getUploadChunkSize = () => uploadChunkSize;
|
|
37
|
+
const currentConfig: ServerConfig = { ...DEFAULT_SERVER_CONFIG };
|
|
22
38
|
|
|
23
|
-
export const
|
|
24
|
-
export const getMaxParallelChunkUploads = () => maxParallelChunkUploads;
|
|
39
|
+
export const getServerConfig = (): ServerConfig => currentConfig;
|
|
25
40
|
|
|
26
|
-
export const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
export const getServerPort = () => serverPort;
|
|
41
|
+
export const updateServerConfig = (overrides: Partial<ServerConfig>) => {
|
|
42
|
+
Object.assign(currentConfig, overrides);
|
|
43
|
+
return currentConfig;
|
|
44
|
+
};
|
|
31
45
|
|
|
32
|
-
export const
|
|
33
|
-
export const
|
|
46
|
+
export const progresses: Progress[] = [];
|
|
47
|
+
export const uploadsProgressMap: Map<string, Progress> = new Map();
|
|
34
48
|
|
|
35
|
-
export const
|
|
36
|
-
|
|
49
|
+
export const getProgressWriter = (): ProgressWriter => {
|
|
50
|
+
const writer = getServerConfig().progressWriter;
|
|
51
|
+
if (!writer) {
|
|
52
|
+
throw new Error('ProgressWriter not configured');
|
|
53
|
+
}
|
|
54
|
+
return writer;
|
|
37
55
|
};
|
|
38
|
-
export const getProgressWriter = (): ProgressWriter => progressWriter;
|
|
39
56
|
|
|
40
|
-
export
|
|
41
|
-
|
|
42
|
-
|
|
57
|
+
export let throttledBroadcaster: _.DebouncedFunc<() => void> =
|
|
58
|
+
_.throttle(() => {
|
|
59
|
+
// noop.
|
|
60
|
+
}, 0);
|
|
61
|
+
export const createThrottledBroadcaster = (): void => {
|
|
62
|
+
throttledBroadcaster?.cancel?.();
|
|
63
|
+
|
|
64
|
+
throttledBroadcaster = _.throttle(
|
|
65
|
+
() => {
|
|
66
|
+
getProgressWriter().writeProgress(progresses);
|
|
67
|
+
},
|
|
68
|
+
getServerConfig().throttleWaitTimeInMillis
|
|
69
|
+
)
|
|
70
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -6,83 +6,97 @@ import {router as uploadChunkRouter} from "./routes/uploadChunk";
|
|
|
6
6
|
import {router as uploadCompleteRouter} from "./routes/uploadComplete";
|
|
7
7
|
import {router as uploadStatusRouter} from "./routes/uploadStatus";
|
|
8
8
|
import {router as uploadRouter} from "./routes/upload";
|
|
9
|
+
import {router as configRouter} from "./routes/config";
|
|
9
10
|
import {ProgressWriter} from "./service/progress_writer";
|
|
10
11
|
import * as socketio from "socket.io";
|
|
11
12
|
import yargs from "yargs";
|
|
12
13
|
import {hideBin} from "yargs/helpers";
|
|
13
|
-
|
|
14
|
+
|
|
14
15
|
import {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
progresses, setEnableCompression, setMaxFileSize,
|
|
18
|
-
setMaxParallelChunkUploads, setProgressWriter, setServerPort, setUploadChunkSize, setUploadsDir, throttledBroadcaster
|
|
16
|
+
createThrottledBroadcaster,
|
|
17
|
+
getServerConfig, progresses, throttledBroadcaster, updateServerConfig,
|
|
19
18
|
} from "./globals";
|
|
20
19
|
import prettyBytes from "pretty-bytes";
|
|
20
|
+
import path from "path";
|
|
21
21
|
|
|
22
|
-
const
|
|
23
|
-
|
|
22
|
+
const serverConfig = getServerConfig();
|
|
23
|
+
console.log(`🚀file-uploader ${serverConfig.version}\n`);
|
|
24
24
|
const argv: any = yargs(hideBin(process.argv))
|
|
25
25
|
.option('upload-location', {
|
|
26
26
|
alias: 'l',
|
|
27
27
|
type: 'string',
|
|
28
28
|
description: 'upload location',
|
|
29
|
-
default: uploadsDir
|
|
29
|
+
default: serverConfig.uploadsDir,
|
|
30
30
|
})
|
|
31
31
|
.option('port', {
|
|
32
32
|
alias: 'p',
|
|
33
33
|
type: 'number',
|
|
34
|
-
default:
|
|
34
|
+
default: serverConfig.serverPort,
|
|
35
35
|
description: 'server port'
|
|
36
36
|
})
|
|
37
37
|
.option('chunk-size', {
|
|
38
38
|
alias: 's',
|
|
39
39
|
type: 'number',
|
|
40
40
|
description: 'chunk size in bytes',
|
|
41
|
-
default:
|
|
41
|
+
default: serverConfig.uploadChunkSize,
|
|
42
|
+
defaultDescription: prettyBytes(serverConfig.uploadChunkSize, {binary: true}),
|
|
43
|
+
})
|
|
44
|
+
.option('parallel-file-uploads', {
|
|
45
|
+
alias: 'N',
|
|
46
|
+
type: 'number',
|
|
47
|
+
description: 'number of simultaneous parallel file uploads',
|
|
48
|
+
default: serverConfig.maxParallelFileUploads
|
|
42
49
|
})
|
|
43
|
-
.option('parallel-uploads', {
|
|
50
|
+
.option('parallel-chunk-uploads', {
|
|
44
51
|
alias: 'n',
|
|
45
52
|
type: 'number',
|
|
46
53
|
description: 'number of simultaneous parallel chunk uploads (per file)',
|
|
47
|
-
default:
|
|
54
|
+
default: serverConfig.maxParallelChunkUploads
|
|
48
55
|
})
|
|
49
56
|
.option('enable-compression', {
|
|
50
57
|
alias: 'c',
|
|
51
58
|
type: 'boolean',
|
|
52
59
|
description: 'enable gzip compression (server to client responses)',
|
|
53
|
-
default:
|
|
60
|
+
default: serverConfig.enableCompression
|
|
54
61
|
})
|
|
55
62
|
.option('max-file-size', {
|
|
56
63
|
alias: 'm',
|
|
57
64
|
type: 'number',
|
|
58
65
|
description: 'maximum file size in bytes',
|
|
59
|
-
default:
|
|
66
|
+
default: serverConfig.maxFileSize,
|
|
67
|
+
defaultDescription: prettyBytes(serverConfig.maxFileSize, {binary: true}),
|
|
60
68
|
})
|
|
61
69
|
.help()
|
|
62
70
|
.argv
|
|
63
71
|
|
|
64
72
|
const uploadLocationArg = argv["upload-location"]
|
|
73
|
+
let uploadsDir: string = serverConfig.uploadsDir
|
|
65
74
|
if (uploadLocationArg) {
|
|
66
75
|
uploadsDir = uploadLocationArg.endsWith('/') ? uploadLocationArg: uploadLocationArg + '/'
|
|
67
76
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
console.log(`
|
|
78
|
-
console.log(`Parallel
|
|
79
|
-
console.log(`
|
|
80
|
-
console.log(`
|
|
77
|
+
updateServerConfig({
|
|
78
|
+
uploadsDir,
|
|
79
|
+
uploadChunkSize: argv["chunk-size"],
|
|
80
|
+
maxParallelFileUploads: argv["parallel-file-uploads"],
|
|
81
|
+
maxParallelChunkUploads: argv["parallel-chunk-uploads"],
|
|
82
|
+
enableCompression: argv["enable-compression"],
|
|
83
|
+
serverPort: argv.port,
|
|
84
|
+
maxFileSize: argv["max-file-size"],
|
|
85
|
+
})
|
|
86
|
+
console.log(`Upload location: ${serverConfig.uploadsDir}`)
|
|
87
|
+
console.log(`Max Parallel file uploads: ${serverConfig.maxParallelFileUploads}`)
|
|
88
|
+
console.log(`Max Parallel uploads per file: ${serverConfig.maxParallelChunkUploads}`)
|
|
89
|
+
console.log(`Parallel upload chunk size: ${prettyBytes(serverConfig.uploadChunkSize, {binary: true})}`)
|
|
90
|
+
console.log(`Compression: ${serverConfig.enableCompression ? "Enabled" : "Disabled"}`)
|
|
91
|
+
console.log(`Server port: ${serverConfig.serverPort}`)
|
|
92
|
+
console.log(`\n`)
|
|
81
93
|
|
|
94
|
+
console.info("Starting application server...")
|
|
82
95
|
const app: Express = express();
|
|
83
96
|
const httpServer: Server = createServer(app)
|
|
84
97
|
const io: socketio.Server = new socketio.Server(httpServer);
|
|
85
|
-
if (
|
|
98
|
+
if (serverConfig.enableCompression) {
|
|
99
|
+
console.debug("enabling compression...")
|
|
86
100
|
app.use(
|
|
87
101
|
compression({
|
|
88
102
|
threshold: 10 * 1024,
|
|
@@ -90,8 +104,10 @@ if (getEnableCompression()) {
|
|
|
90
104
|
})
|
|
91
105
|
)
|
|
92
106
|
}
|
|
93
|
-
|
|
94
|
-
|
|
107
|
+
updateServerConfig({
|
|
108
|
+
progressWriter: new ProgressWriter(io)
|
|
109
|
+
})
|
|
110
|
+
createThrottledBroadcaster();
|
|
95
111
|
|
|
96
112
|
app.use(express.json());
|
|
97
113
|
app.use(express.urlencoded({ extended: true }));
|
|
@@ -100,10 +116,20 @@ app.use('/upload/chunk', uploadChunkRouter);
|
|
|
100
116
|
app.use('/upload/complete', uploadCompleteRouter);
|
|
101
117
|
app.use('/upload/status', uploadStatusRouter);
|
|
102
118
|
app.use('/upload', uploadRouter);
|
|
119
|
+
app.use('/config', configRouter);
|
|
120
|
+
|
|
121
|
+
const isDev = process.env.NODE_ENV !== 'production';
|
|
122
|
+
|
|
123
|
+
const clientDir = isDev
|
|
124
|
+
? path.resolve(__dirname, "../dist/client") // dev: server in src/, client in dist/
|
|
125
|
+
: path.resolve(__dirname, "client"); // prod: server in dist/, client in dist/client
|
|
103
126
|
|
|
104
127
|
app.get('/', (_, res) => {
|
|
105
|
-
res.sendFile(
|
|
128
|
+
res.sendFile(path.join(clientDir, 'index.html'));
|
|
106
129
|
});
|
|
130
|
+
app.use('/', [
|
|
131
|
+
express.static(clientDir)
|
|
132
|
+
]);
|
|
107
133
|
|
|
108
134
|
app.get('/progresses', (_: Request, res: Response) => {
|
|
109
135
|
console.log("Progresses requested...");
|
|
@@ -124,10 +150,6 @@ io.on('connection', (socket: socketio.Socket) => {
|
|
|
124
150
|
});
|
|
125
151
|
});
|
|
126
152
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
]);
|
|
130
|
-
|
|
131
|
-
httpServer.listen(port, () => {
|
|
132
|
-
console.log('Server listening on ' + port + ' ...');
|
|
153
|
+
httpServer.listen(serverConfig.serverPort, () => {
|
|
154
|
+
console.log('Server listening on ' + serverConfig.serverPort + ' ...');
|
|
133
155
|
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { Router, Request, Response } from 'express';
|
|
2
|
+
import {getServerConfig, ServerConfig, ServerConfigJson} from "../globals";
|
|
3
|
+
|
|
4
|
+
const serverConfig = getServerConfig();
|
|
5
|
+
export const router = Router();
|
|
6
|
+
|
|
7
|
+
router.get('/', (_: Request, res: Response) => {
|
|
8
|
+
console.log("GET /config");
|
|
9
|
+
const toServerConfigJson = (config: ServerConfig): ServerConfigJson => {
|
|
10
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
11
|
+
const { progressWriter, ...json } = config;
|
|
12
|
+
return json;
|
|
13
|
+
};
|
|
14
|
+
res.json(toServerConfigJson(serverConfig));
|
|
15
|
+
})
|
package/src/routes/upload.ts
CHANGED
|
@@ -2,11 +2,9 @@ import { Router } from 'express';
|
|
|
2
2
|
import formidable, { File } from "formidable";
|
|
3
3
|
import { v4 as uuidv4 } from 'uuid';
|
|
4
4
|
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
progresses,
|
|
5
|
+
getServerConfig,
|
|
6
|
+
progresses, ServerConfig,
|
|
8
7
|
throttledBroadcaster,
|
|
9
|
-
throttleWaitTimeInMillis,
|
|
10
8
|
uploadsProgressMap
|
|
11
9
|
} from "../globals";
|
|
12
10
|
import {FileTransferProgress, Progress, UploadStatus} from "../model/progress";
|
|
@@ -14,11 +12,12 @@ import _ from "lodash";
|
|
|
14
12
|
import prettyBytes from "pretty-bytes";
|
|
15
13
|
import mv from "mv";
|
|
16
14
|
|
|
15
|
+
const serverConfig: ServerConfig = getServerConfig();
|
|
17
16
|
export const router = Router();
|
|
18
17
|
|
|
19
18
|
router.post('/', (req: any, res: any) => {
|
|
20
|
-
const maxFileSize =
|
|
21
|
-
const uploadsDir =
|
|
19
|
+
const maxFileSize = serverConfig.maxFileSize;
|
|
20
|
+
const uploadsDir = serverConfig.uploadsDir;
|
|
22
21
|
// parse a file upload
|
|
23
22
|
const form = formidable({
|
|
24
23
|
multiples: true,
|
|
@@ -47,7 +46,7 @@ router.post('/', (req: any, res: any) => {
|
|
|
47
46
|
console.warn("Progress not found in the map for uuid: " + uuid);
|
|
48
47
|
return;
|
|
49
48
|
}
|
|
50
|
-
}, throttleWaitTimeInMillis, {
|
|
49
|
+
}, serverConfig.throttleWaitTimeInMillis, {
|
|
51
50
|
leading: true
|
|
52
51
|
});
|
|
53
52
|
|
|
@@ -5,11 +5,11 @@ import path from "path";
|
|
|
5
5
|
import crypto from "crypto";
|
|
6
6
|
import {
|
|
7
7
|
uploadsProgressMap,
|
|
8
|
-
|
|
9
|
-
throttledBroadcaster
|
|
8
|
+
throttledBroadcaster, getServerConfig
|
|
10
9
|
} from "../globals";
|
|
11
10
|
import { FileTransferProgress } from "../model/progress";
|
|
12
11
|
|
|
12
|
+
const serverConfig = getServerConfig();
|
|
13
13
|
export const router = Router();
|
|
14
14
|
|
|
15
15
|
router.post("/", (req: Request, res: Response) => {
|
|
@@ -30,7 +30,7 @@ router.post("/", (req: Request, res: Response) => {
|
|
|
30
30
|
|
|
31
31
|
const progress = uploadsProgressMap.get(fileId)! as FileTransferProgress;
|
|
32
32
|
|
|
33
|
-
const uploadDir =
|
|
33
|
+
const uploadDir = serverConfig.uploadsDir;
|
|
34
34
|
await fs.mkdir(uploadDir, { recursive: true });
|
|
35
35
|
|
|
36
36
|
const form = formidable({
|
package/src/routes/uploadInit.ts
CHANGED
|
@@ -2,14 +2,13 @@ import { Router, Request, Response } from 'express';
|
|
|
2
2
|
import { v4 as uuidv4 } from 'uuid';
|
|
3
3
|
import {
|
|
4
4
|
uploadsProgressMap,
|
|
5
|
-
progresses,
|
|
6
|
-
getUploadsDir,
|
|
7
|
-
getUploadChunkSize, getMaxParallelChunkUploads
|
|
5
|
+
progresses, getServerConfig
|
|
8
6
|
} from '../globals';
|
|
9
7
|
import { FileTransferProgress } from '../model/progress';
|
|
10
8
|
import path from "path";
|
|
11
9
|
import fs from "fs";
|
|
12
10
|
|
|
11
|
+
const serverConfig = getServerConfig();
|
|
13
12
|
export const router = Router();
|
|
14
13
|
|
|
15
14
|
/**
|
|
@@ -34,7 +33,7 @@ router.post('/', (req: Request, res: Response) => {
|
|
|
34
33
|
return res.status(400).json({ msg: 'Missing or invalid fileName/fileSize' });
|
|
35
34
|
}
|
|
36
35
|
|
|
37
|
-
const finalPath = path.join(
|
|
36
|
+
const finalPath = path.join(serverConfig.uploadsDir, fileName);
|
|
38
37
|
let progress: FileTransferProgress;
|
|
39
38
|
let fileId: string;
|
|
40
39
|
|
|
@@ -53,7 +52,7 @@ router.post('/', (req: Request, res: Response) => {
|
|
|
53
52
|
progress = new FileTransferProgress(fileId, Date.now());
|
|
54
53
|
progress.fileName = fileName;
|
|
55
54
|
progress.bytesExpected = fileSize;
|
|
56
|
-
progress.chunkSize =
|
|
55
|
+
progress.chunkSize = serverConfig.uploadChunkSize;
|
|
57
56
|
progress.totalChunks = Math.ceil(fileSize / progress.chunkSize);
|
|
58
57
|
progress.bytesReceived = fs.statSync(finalPath).size; // resume
|
|
59
58
|
progress.resetUploadedChunks();
|
|
@@ -68,7 +67,7 @@ router.post('/', (req: Request, res: Response) => {
|
|
|
68
67
|
progress = new FileTransferProgress(fileId, Date.now());
|
|
69
68
|
progress.fileName = fileName;
|
|
70
69
|
progress.bytesExpected = fileSize;
|
|
71
|
-
progress.chunkSize =
|
|
70
|
+
progress.chunkSize = serverConfig.uploadChunkSize;
|
|
72
71
|
progress.totalChunks = Math.ceil(fileSize / progress.chunkSize);
|
|
73
72
|
progress.resetUploadedChunks();
|
|
74
73
|
uploadsProgressMap.set(fileId, progress);
|
|
@@ -82,7 +81,7 @@ router.post('/', (req: Request, res: Response) => {
|
|
|
82
81
|
fileId,
|
|
83
82
|
chunkSize: progress.chunkSize,
|
|
84
83
|
totalChunks: progress.totalChunks,
|
|
85
|
-
|
|
84
|
+
maxParallelChunkUploads: serverConfig.maxParallelChunkUploads,
|
|
86
85
|
bytesReceived: progress.bytesReceived || 0 // client can skip uploaded chunks
|
|
87
86
|
});
|
|
88
87
|
});
|
package/src-client/entrypoint.ts
CHANGED
|
@@ -3,6 +3,7 @@ import "jquery-blockui/jquery.blockUI.js";
|
|
|
3
3
|
import "./style.scss";
|
|
4
4
|
import Toastify from "toastify-js";
|
|
5
5
|
import {computeSHA1} from "./sha1";
|
|
6
|
+
import {ServerConfigJson} from "../src/globals";
|
|
6
7
|
|
|
7
8
|
const MAX_COMPLETE_CHECK_RETRIES = 20;
|
|
8
9
|
const COMPLETE_CHECK_RETRY_DELAY_MS = 1000;
|
|
@@ -15,6 +16,28 @@ jQuery(() => {
|
|
|
15
16
|
pageEventRegistrar.registerEvents();
|
|
16
17
|
});
|
|
17
18
|
|
|
19
|
+
const asyncPool = async <T>(
|
|
20
|
+
poolLimit: number,
|
|
21
|
+
array: T[],
|
|
22
|
+
iteratorFn: (item: T) => Promise<void>
|
|
23
|
+
): Promise<void> => {
|
|
24
|
+
const executing = new Set<Promise<void>>();
|
|
25
|
+
|
|
26
|
+
for (const item of array) {
|
|
27
|
+
const p = Promise.resolve().then(() => iteratorFn(item));
|
|
28
|
+
executing.add(p);
|
|
29
|
+
|
|
30
|
+
const clean = () => executing.delete(p);
|
|
31
|
+
p.then(clean).catch(clean);
|
|
32
|
+
|
|
33
|
+
if (executing.size >= poolLimit) {
|
|
34
|
+
await Promise.race(executing);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
await Promise.all(executing);
|
|
39
|
+
};
|
|
40
|
+
|
|
18
41
|
class PageEventRegistrar {
|
|
19
42
|
public registerEvents(): void {
|
|
20
43
|
this.registerThemeSelectionEventHandler();
|
|
@@ -95,37 +118,42 @@ class PageEventRegistrar {
|
|
|
95
118
|
return;
|
|
96
119
|
}
|
|
97
120
|
|
|
98
|
-
const disableChunked = (jQuery("#disableChunkedUpload").prop("checked") === true);
|
|
99
|
-
|
|
100
121
|
// Block form before uploading
|
|
101
122
|
$uploadForm.block({
|
|
102
123
|
message: '<h1 class="upload-block-modal p-2 m-0">Uploading...</h1>'
|
|
103
124
|
});
|
|
104
125
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
126
|
+
const disableChunked = (jQuery("#disableChunkedUpload").prop("checked") === true);
|
|
127
|
+
console.log("disableChunked?", disableChunked);
|
|
128
|
+
|
|
129
|
+
const serverConfigResponse = await this.retrieveConfig();
|
|
130
|
+
const errorMessage = serverConfigResponse.error
|
|
131
|
+
if (errorMessage) {
|
|
132
|
+
let errorText = errorMessage;
|
|
133
|
+
const errorObject = serverConfigResponse.errorObject;
|
|
134
|
+
const errorObjectMessage = errorObject instanceof Error ? errorObject.message : String(errorObject);
|
|
135
|
+
if (serverConfigResponse.errorObject) {
|
|
136
|
+
errorText = `${errorText}. ${errorObjectMessage}`;
|
|
113
137
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
this.onFilesChange($fileNameDiv, $fileInput);
|
|
122
|
-
|
|
123
|
-
$uploadForm.unblock();
|
|
138
|
+
console.error(errorText);
|
|
139
|
+
Toastify({
|
|
140
|
+
text: errorText,
|
|
141
|
+
duration: -1,
|
|
142
|
+
close: true,
|
|
143
|
+
style: { background: "linear-gradient(to right, #F39454, #FF6600)" }
|
|
144
|
+
}).showToast();
|
|
124
145
|
}
|
|
146
|
+
const serverConfigJson = serverConfigResponse.response;
|
|
147
|
+
await this.doUpload(files, {
|
|
148
|
+
$uploadForm,
|
|
149
|
+
serverConfig: serverConfigJson,
|
|
150
|
+
disableChunkedUpload: disableChunked,
|
|
151
|
+
});
|
|
125
152
|
})().catch(err => {
|
|
153
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
126
154
|
console.error("Error during upload:", err);
|
|
127
155
|
Toastify({
|
|
128
|
-
text: `Upload
|
|
156
|
+
text: `Upload failed: ${message}`,
|
|
129
157
|
duration: -1,
|
|
130
158
|
close: true,
|
|
131
159
|
style: { background: "linear-gradient(to right, #F39454, #FF6600)" }
|
|
@@ -134,6 +162,53 @@ class PageEventRegistrar {
|
|
|
134
162
|
});
|
|
135
163
|
}
|
|
136
164
|
|
|
165
|
+
private async doUpload(files: FileList, {
|
|
166
|
+
disableChunkedUpload = false,
|
|
167
|
+
serverConfig,
|
|
168
|
+
$uploadForm,
|
|
169
|
+
}: {
|
|
170
|
+
disableChunkedUpload: boolean,
|
|
171
|
+
serverConfig?: ServerConfigJson,
|
|
172
|
+
$uploadForm: JQuery<HTMLElement>
|
|
173
|
+
}):Promise<void> {
|
|
174
|
+
try {
|
|
175
|
+
let maxParallelFileUploads = 1;
|
|
176
|
+
if (serverConfig != null) {
|
|
177
|
+
maxParallelFileUploads = serverConfig.maxParallelFileUploads;
|
|
178
|
+
}
|
|
179
|
+
console.log(`Max parallel file uploads: ${maxParallelFileUploads}`);
|
|
180
|
+
await asyncPool(
|
|
181
|
+
maxParallelFileUploads,
|
|
182
|
+
Array.from(files),
|
|
183
|
+
async (file) => {
|
|
184
|
+
if (disableChunkedUpload) {
|
|
185
|
+
await this.uploadFileNonChunked(file);
|
|
186
|
+
} else {
|
|
187
|
+
await this.uploadFile(file);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
).catch(err => {
|
|
191
|
+
console.error("Error during upload:", err);
|
|
192
|
+
Toastify({
|
|
193
|
+
text: `Upload failed: ${err}`,
|
|
194
|
+
duration: -1,
|
|
195
|
+
close: true,
|
|
196
|
+
style: { background: "linear-gradient(to right, #F39454, #FF6600)" }
|
|
197
|
+
}).showToast();
|
|
198
|
+
});
|
|
199
|
+
} finally {
|
|
200
|
+
// Unblock and reset form after all files finish
|
|
201
|
+
$uploadForm.trigger("reset");
|
|
202
|
+
|
|
203
|
+
const $fileDiv = jQuery("#file-div");
|
|
204
|
+
const $fileNameDiv = $fileDiv.find("#file-name");
|
|
205
|
+
const $fileInput = jQuery("form#uploadForm input[name='file']");
|
|
206
|
+
this.onFilesChange($fileNameDiv, $fileInput);
|
|
207
|
+
|
|
208
|
+
$uploadForm.unblock();
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
137
212
|
private async uploadFileNonChunked(file: File): Promise<void> {
|
|
138
213
|
const formData = new FormData();
|
|
139
214
|
// Server-side uses formidable({ multiples: true }) so using the same field name is fine
|
|
@@ -175,7 +250,7 @@ class PageEventRegistrar {
|
|
|
175
250
|
const initData = await initResp.json();
|
|
176
251
|
const fileId: string = initData.fileId;
|
|
177
252
|
const chunkSize: number = initData.chunkSize;
|
|
178
|
-
const
|
|
253
|
+
const maxParallelChunkUploads: number = initData.maxParallelChunkUploads || 3;
|
|
179
254
|
const totalChunks: number = Math.ceil(file.size / chunkSize);
|
|
180
255
|
|
|
181
256
|
// Active upload pool
|
|
@@ -226,7 +301,7 @@ class PageEventRegistrar {
|
|
|
226
301
|
pool.push(taskPromise);
|
|
227
302
|
|
|
228
303
|
// If pool is full, wait for at least one to finish
|
|
229
|
-
if (pool.length >=
|
|
304
|
+
if (pool.length >= maxParallelChunkUploads) {
|
|
230
305
|
await Promise.race(pool).catch((err) => {
|
|
231
306
|
console.warn(`Pool full, but one task failed, err: ${err}`);
|
|
232
307
|
}); // don't block other tasks
|
|
@@ -306,4 +381,33 @@ class PageEventRegistrar {
|
|
|
306
381
|
}
|
|
307
382
|
}
|
|
308
383
|
}
|
|
384
|
+
|
|
385
|
+
private async retrieveConfig(): Promise<ResponseOrError<ServerConfigJson>> {
|
|
386
|
+
try {
|
|
387
|
+
const response = await fetch(`/config`)
|
|
388
|
+
if (response.ok) {
|
|
389
|
+
const serverConfigJson = await response.json();
|
|
390
|
+
return {
|
|
391
|
+
response: serverConfigJson as ServerConfigJson
|
|
392
|
+
};
|
|
393
|
+
}
|
|
394
|
+
return {
|
|
395
|
+
"error": `Could not retrieve config: ${response.status}`
|
|
396
|
+
};
|
|
397
|
+
} catch (error) {
|
|
398
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
399
|
+
console.error(`Could not retrieve config: ${errorMessage}`);
|
|
400
|
+
return {
|
|
401
|
+
error: `Failed to retrieve config: ${errorMessage}`,
|
|
402
|
+
errorObject: error,
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
interface ResponseOrError<T> {
|
|
410
|
+
response?: T;
|
|
411
|
+
error?: string,
|
|
412
|
+
errorObject?: any
|
|
309
413
|
}
|