@internxt/cli 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -27
- package/dist/commands/add-cert.js +2 -2
- package/dist/commands/create-folder.d.ts +3 -3
- package/dist/commands/delete-permanently-file.d.ts +2 -2
- package/dist/commands/delete-permanently-folder.d.ts +2 -2
- package/dist/commands/download-file.d.ts +4 -5
- package/dist/commands/download-file.js +22 -37
- package/dist/commands/list.d.ts +4 -4
- package/dist/commands/login-legacy.d.ts +5 -5
- package/dist/commands/login.d.ts +2 -2
- package/dist/commands/logs.js +3 -3
- package/dist/commands/move-file.d.ts +4 -4
- package/dist/commands/move-folder.d.ts +3 -3
- package/dist/commands/rename-file.d.ts +3 -3
- package/dist/commands/rename-folder.d.ts +3 -3
- package/dist/commands/trash-clear.d.ts +2 -2
- package/dist/commands/trash-file.d.ts +2 -2
- package/dist/commands/trash-folder.d.ts +2 -2
- package/dist/commands/trash-list.d.ts +2 -2
- package/dist/commands/trash-restore-file.d.ts +4 -4
- package/dist/commands/trash-restore-folder.d.ts +3 -3
- package/dist/commands/upload-file.d.ts +6 -7
- package/dist/commands/upload-file.js +61 -80
- package/dist/commands/upload-folder.d.ts +16 -0
- package/dist/commands/upload-folder.js +87 -0
- package/dist/commands/webdav-config.d.ts +6 -6
- package/dist/commands/webdav.d.ts +1 -1
- package/dist/constants/configs.d.ts +12 -0
- package/dist/constants/configs.js +20 -0
- package/dist/services/auth.service.js +11 -6
- package/dist/services/config.service.d.ts +0 -12
- package/dist/services/config.service.js +33 -38
- package/dist/services/drive/drive-folder.service.d.ts +1 -1
- package/dist/services/drive/trash.service.d.ts +1 -1
- package/dist/services/local-filesystem/local-filesystem.service.d.ts +6 -0
- package/dist/services/local-filesystem/local-filesystem.service.js +57 -0
- package/dist/services/local-filesystem/local-filesystem.types.d.ts +13 -0
- package/dist/services/local-filesystem/local-filesystem.types.js +2 -0
- package/dist/services/network/upload/upload-facade.service.d.ts +9 -0
- package/dist/services/network/upload/upload-facade.service.js +53 -0
- package/dist/services/network/upload/upload-file.service.d.ts +8 -0
- package/dist/services/network/upload/upload-file.service.js +134 -0
- package/dist/services/network/upload/upload-folder.service.d.ts +6 -0
- package/dist/services/network/upload/upload-folder.service.js +62 -0
- package/dist/services/network/upload/upload.types.d.ts +53 -0
- package/dist/services/network/upload/upload.types.js +6 -0
- package/dist/services/sdk-manager.service.d.ts +2 -2
- package/dist/services/sdk-manager.service.js +2 -2
- package/dist/services/thumbnail.service.js +1 -34
- package/dist/services/validation.service.d.ts +5 -0
- package/dist/services/validation.service.js +22 -18
- package/dist/types/command.types.d.ts +3 -0
- package/dist/types/command.types.js +8 -1
- package/dist/types/fast-xml-parser.types.d.ts +59 -0
- package/dist/types/fast-xml-parser.types.js +2 -0
- package/dist/utils/cli.utils.d.ts +16 -2
- package/dist/utils/cli.utils.js +84 -3
- package/dist/utils/errors.utils.d.ts +2 -0
- package/dist/utils/errors.utils.js +13 -2
- package/dist/utils/logger.utils.js +5 -5
- package/dist/utils/network.utils.d.ts +2 -2
- package/dist/utils/network.utils.js +24 -23
- package/dist/utils/thumbnail.utils.d.ts +17 -0
- package/dist/utils/thumbnail.utils.js +36 -8
- package/dist/utils/xml.utils.d.ts +2 -2
- package/dist/utils/xml.utils.js +1 -1
- package/dist/webdav/handlers/GET.handler.js +37 -26
- package/dist/webdav/handlers/PUT.handler.js +56 -41
- package/dist/webdav/middewares/errors.middleware.js +1 -1
- package/dist/webdav/webdav-server.js +4 -4
- package/oclif.manifest.json +62 -1
- package/package.json +24 -24
|
@@ -4,23 +4,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.ConfigService = void 0;
|
|
7
|
-
const node_path_1 = __importDefault(require("node:path"));
|
|
8
|
-
const os_1 = __importDefault(require("os"));
|
|
9
7
|
const promises_1 = __importDefault(require("node:fs/promises"));
|
|
10
8
|
const crypto_service_1 = require("./crypto.service");
|
|
9
|
+
const errors_utils_1 = require("../utils/errors.utils");
|
|
10
|
+
const configs_1 = require("../constants/configs");
|
|
11
11
|
class ConfigService {
|
|
12
|
-
static INTERNXT_CLI_DATA_DIR = node_path_1.default.join(os_1.default.homedir(), '.internxt-cli');
|
|
13
|
-
static INTERNXT_CLI_LOGS_DIR = node_path_1.default.join(this.INTERNXT_CLI_DATA_DIR, 'logs');
|
|
14
|
-
static INTERNXT_TMP_DIR = os_1.default.tmpdir();
|
|
15
|
-
static CREDENTIALS_FILE = node_path_1.default.join(this.INTERNXT_CLI_DATA_DIR, '.inxtcli');
|
|
16
|
-
static DRIVE_SQLITE_FILE = node_path_1.default.join(this.INTERNXT_CLI_DATA_DIR, 'internxt-cli-drive.sqlite');
|
|
17
|
-
static WEBDAV_SSL_CERTS_DIR = node_path_1.default.join(this.INTERNXT_CLI_DATA_DIR, 'certs');
|
|
18
|
-
static WEBDAV_CONFIGS_FILE = node_path_1.default.join(this.INTERNXT_CLI_DATA_DIR, 'config.webdav.inxt');
|
|
19
|
-
static WEBDAV_DEFAULT_HOST = '127.0.0.1';
|
|
20
|
-
static WEBDAV_DEFAULT_PORT = '3005';
|
|
21
|
-
static WEBDAV_DEFAULT_PROTOCOL = 'https';
|
|
22
|
-
static WEBDAV_DEFAULT_TIMEOUT = 0;
|
|
23
|
-
static WEBDAV_DEFAULT_CREATE_FULL_PATH = true;
|
|
24
12
|
static instance = new ConfigService();
|
|
25
13
|
get = (key) => {
|
|
26
14
|
const value = process.env[key];
|
|
@@ -32,17 +20,24 @@ class ConfigService {
|
|
|
32
20
|
await this.ensureInternxtCliDataDirExists();
|
|
33
21
|
const credentialsString = JSON.stringify(loginCredentials);
|
|
34
22
|
const encryptedCredentials = crypto_service_1.CryptoService.instance.encryptText(credentialsString);
|
|
35
|
-
await promises_1.default.writeFile(
|
|
23
|
+
await promises_1.default.writeFile(configs_1.CREDENTIALS_FILE, encryptedCredentials, 'utf8');
|
|
36
24
|
};
|
|
37
25
|
clearUser = async () => {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
26
|
+
try {
|
|
27
|
+
const stat = await promises_1.default.stat(configs_1.CREDENTIALS_FILE);
|
|
28
|
+
if (stat.size === 0)
|
|
29
|
+
return;
|
|
30
|
+
await promises_1.default.writeFile(configs_1.CREDENTIALS_FILE, '', 'utf8');
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
if (!(0, errors_utils_1.isFileNotFoundError)(error)) {
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
42
37
|
};
|
|
43
38
|
readUser = async () => {
|
|
44
39
|
try {
|
|
45
|
-
const encryptedCredentials = await promises_1.default.readFile(
|
|
40
|
+
const encryptedCredentials = await promises_1.default.readFile(configs_1.CREDENTIALS_FILE, 'utf8');
|
|
46
41
|
const credentialsString = crypto_service_1.CryptoService.instance.decryptText(encryptedCredentials);
|
|
47
42
|
const loginCredentials = JSON.parse(credentialsString);
|
|
48
43
|
return loginCredentials;
|
|
@@ -54,52 +49,52 @@ class ConfigService {
|
|
|
54
49
|
saveWebdavConfig = async (webdavConfig) => {
|
|
55
50
|
await this.ensureInternxtCliDataDirExists();
|
|
56
51
|
const configs = JSON.stringify(webdavConfig);
|
|
57
|
-
await promises_1.default.writeFile(
|
|
52
|
+
await promises_1.default.writeFile(configs_1.WEBDAV_CONFIGS_FILE, configs, 'utf8');
|
|
58
53
|
};
|
|
59
54
|
readWebdavConfig = async () => {
|
|
60
55
|
try {
|
|
61
|
-
const configsData = await promises_1.default.readFile(
|
|
56
|
+
const configsData = await promises_1.default.readFile(configs_1.WEBDAV_CONFIGS_FILE, 'utf8');
|
|
62
57
|
const configs = JSON.parse(configsData);
|
|
63
58
|
return {
|
|
64
|
-
host: configs?.host ??
|
|
65
|
-
port: configs?.port ??
|
|
66
|
-
protocol: configs?.protocol ??
|
|
67
|
-
timeoutMinutes: configs?.timeoutMinutes ??
|
|
68
|
-
createFullPath: configs?.createFullPath ??
|
|
59
|
+
host: configs?.host ?? configs_1.WEBDAV_DEFAULT_HOST,
|
|
60
|
+
port: configs?.port ?? configs_1.WEBDAV_DEFAULT_PORT,
|
|
61
|
+
protocol: configs?.protocol ?? configs_1.WEBDAV_DEFAULT_PROTOCOL,
|
|
62
|
+
timeoutMinutes: configs?.timeoutMinutes ?? configs_1.WEBDAV_DEFAULT_TIMEOUT,
|
|
63
|
+
createFullPath: configs?.createFullPath ?? configs_1.WEBDAV_DEFAULT_CREATE_FULL_PATH,
|
|
69
64
|
};
|
|
70
65
|
}
|
|
71
66
|
catch {
|
|
72
67
|
return {
|
|
73
|
-
host:
|
|
74
|
-
port:
|
|
75
|
-
protocol:
|
|
76
|
-
timeoutMinutes:
|
|
77
|
-
createFullPath:
|
|
68
|
+
host: configs_1.WEBDAV_DEFAULT_HOST,
|
|
69
|
+
port: configs_1.WEBDAV_DEFAULT_PORT,
|
|
70
|
+
protocol: configs_1.WEBDAV_DEFAULT_PROTOCOL,
|
|
71
|
+
timeoutMinutes: configs_1.WEBDAV_DEFAULT_TIMEOUT,
|
|
72
|
+
createFullPath: configs_1.WEBDAV_DEFAULT_CREATE_FULL_PATH,
|
|
78
73
|
};
|
|
79
74
|
}
|
|
80
75
|
};
|
|
81
76
|
ensureInternxtCliDataDirExists = async () => {
|
|
82
77
|
try {
|
|
83
|
-
await promises_1.default.access(
|
|
78
|
+
await promises_1.default.access(configs_1.INTERNXT_CLI_DATA_DIR);
|
|
84
79
|
}
|
|
85
80
|
catch {
|
|
86
|
-
await promises_1.default.mkdir(
|
|
81
|
+
await promises_1.default.mkdir(configs_1.INTERNXT_CLI_DATA_DIR);
|
|
87
82
|
}
|
|
88
83
|
};
|
|
89
84
|
ensureWebdavCertsDirExists = async () => {
|
|
90
85
|
try {
|
|
91
|
-
await promises_1.default.access(
|
|
86
|
+
await promises_1.default.access(configs_1.WEBDAV_SSL_CERTS_DIR);
|
|
92
87
|
}
|
|
93
88
|
catch {
|
|
94
|
-
await promises_1.default.mkdir(
|
|
89
|
+
await promises_1.default.mkdir(configs_1.WEBDAV_SSL_CERTS_DIR);
|
|
95
90
|
}
|
|
96
91
|
};
|
|
97
92
|
ensureInternxtLogsDirExists = async () => {
|
|
98
93
|
try {
|
|
99
|
-
await promises_1.default.access(
|
|
94
|
+
await promises_1.default.access(configs_1.INTERNXT_CLI_LOGS_DIR);
|
|
100
95
|
}
|
|
101
96
|
catch {
|
|
102
|
-
await promises_1.default.mkdir(
|
|
97
|
+
await promises_1.default.mkdir(configs_1.INTERNXT_CLI_LOGS_DIR);
|
|
103
98
|
}
|
|
104
99
|
};
|
|
105
100
|
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { FileSystemNode, ScanResult } from './local-filesystem.types';
|
|
2
|
+
export declare class LocalFilesystemService {
|
|
3
|
+
static readonly instance: LocalFilesystemService;
|
|
4
|
+
scanLocalDirectory(path: string): Promise<ScanResult>;
|
|
5
|
+
scanRecursive(currentPath: string, parentPath: string, folders: FileSystemNode[], files: FileSystemNode[]): Promise<number>;
|
|
6
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LocalFilesystemService = void 0;
|
|
4
|
+
const node_fs_1 = require("node:fs");
|
|
5
|
+
const node_path_1 = require("node:path");
|
|
6
|
+
const logger_utils_1 = require("../../utils/logger.utils");
|
|
7
|
+
class LocalFilesystemService {
|
|
8
|
+
static instance = new LocalFilesystemService();
|
|
9
|
+
async scanLocalDirectory(path) {
|
|
10
|
+
const folders = [];
|
|
11
|
+
const files = [];
|
|
12
|
+
const parentPath = (0, node_path_1.dirname)(path);
|
|
13
|
+
const totalBytes = await this.scanRecursive(path, parentPath, folders, files);
|
|
14
|
+
return {
|
|
15
|
+
folders,
|
|
16
|
+
files,
|
|
17
|
+
totalItems: folders.length + files.length,
|
|
18
|
+
totalBytes,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
async scanRecursive(currentPath, parentPath, folders, files) {
|
|
22
|
+
try {
|
|
23
|
+
const stats = await node_fs_1.promises.stat(currentPath);
|
|
24
|
+
const relativePath = (0, node_path_1.relative)(parentPath, currentPath);
|
|
25
|
+
if (stats.isFile() && stats.size > 0) {
|
|
26
|
+
const fileInfo = (0, node_path_1.parse)(currentPath);
|
|
27
|
+
files.push({
|
|
28
|
+
type: 'file',
|
|
29
|
+
name: fileInfo.name,
|
|
30
|
+
absolutePath: currentPath,
|
|
31
|
+
relativePath,
|
|
32
|
+
size: stats.size,
|
|
33
|
+
});
|
|
34
|
+
return stats.size;
|
|
35
|
+
}
|
|
36
|
+
if (stats.isDirectory()) {
|
|
37
|
+
folders.push({
|
|
38
|
+
type: 'folder',
|
|
39
|
+
name: (0, node_path_1.basename)(currentPath),
|
|
40
|
+
absolutePath: currentPath,
|
|
41
|
+
relativePath,
|
|
42
|
+
size: 0,
|
|
43
|
+
});
|
|
44
|
+
const entries = await node_fs_1.promises.readdir(currentPath, { withFileTypes: true });
|
|
45
|
+
const validEntries = entries.filter((e) => !e.isSymbolicLink());
|
|
46
|
+
const bytesArray = await Promise.all(validEntries.map((e) => this.scanRecursive((0, node_path_1.join)(currentPath, e.name), parentPath, folders, files)));
|
|
47
|
+
return bytesArray.reduce((sum, bytes) => sum + bytes, 0);
|
|
48
|
+
}
|
|
49
|
+
return 0;
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
logger_utils_1.logger.warn(`Error scanning path ${currentPath}: ${error.message} - skipping...`);
|
|
53
|
+
return 0;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
exports.LocalFilesystemService = LocalFilesystemService;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface FileSystemNode {
|
|
2
|
+
type: 'file' | 'folder';
|
|
3
|
+
name: string;
|
|
4
|
+
size: number;
|
|
5
|
+
absolutePath: string;
|
|
6
|
+
relativePath: string;
|
|
7
|
+
}
|
|
8
|
+
export interface ScanResult {
|
|
9
|
+
folders: FileSystemNode[];
|
|
10
|
+
files: FileSystemNode[];
|
|
11
|
+
totalItems: number;
|
|
12
|
+
totalBytes: number;
|
|
13
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { UploadFolderParams } from './upload.types';
|
|
2
|
+
export declare class UploadFacade {
|
|
3
|
+
static readonly instance: UploadFacade;
|
|
4
|
+
uploadFolder({ localPath, destinationFolderUuid, loginUserDetails, jsonFlag, onProgress }: UploadFolderParams): Promise<{
|
|
5
|
+
totalBytes: number;
|
|
6
|
+
rootFolderId: string;
|
|
7
|
+
uploadTimeMs: number;
|
|
8
|
+
}>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UploadFacade = void 0;
|
|
4
|
+
const node_path_1 = require("node:path");
|
|
5
|
+
const cli_utils_1 = require("../../../utils/cli.utils");
|
|
6
|
+
const logger_utils_1 = require("../../../utils/logger.utils");
|
|
7
|
+
const local_filesystem_service_1 = require("../../local-filesystem/local-filesystem.service");
|
|
8
|
+
const upload_folder_service_1 = require("./upload-folder.service");
|
|
9
|
+
const upload_file_service_1 = require("./upload-file.service");
|
|
10
|
+
const async_utils_1 = require("../../../utils/async.utils");
|
|
11
|
+
class UploadFacade {
|
|
12
|
+
static instance = new UploadFacade();
|
|
13
|
+
async uploadFolder({ localPath, destinationFolderUuid, loginUserDetails, jsonFlag, onProgress }) {
|
|
14
|
+
const timer = cli_utils_1.CLIUtils.timer();
|
|
15
|
+
const network = cli_utils_1.CLIUtils.prepareNetwork({ jsonFlag, loginUserDetails });
|
|
16
|
+
const scanResult = await local_filesystem_service_1.LocalFilesystemService.instance.scanLocalDirectory(localPath);
|
|
17
|
+
logger_utils_1.logger.info(`Scanned folder ${localPath}: found ${scanResult.totalItems} items, total size ${scanResult.totalBytes} bytes.`);
|
|
18
|
+
const currentProgress = { itemsUploaded: 0, bytesUploaded: 0 };
|
|
19
|
+
const emitProgress = () => {
|
|
20
|
+
const itemProgress = currentProgress.itemsUploaded / scanResult.totalItems;
|
|
21
|
+
const sizeProgress = currentProgress.bytesUploaded / scanResult.totalBytes;
|
|
22
|
+
const percentage = Math.floor((itemProgress * 0.5 + sizeProgress * 0.5) * 100);
|
|
23
|
+
onProgress({ percentage });
|
|
24
|
+
};
|
|
25
|
+
const folderMap = await upload_folder_service_1.UploadFolderService.instance.createFolders({
|
|
26
|
+
foldersToCreate: scanResult.folders,
|
|
27
|
+
destinationFolderUuid,
|
|
28
|
+
currentProgress,
|
|
29
|
+
emitProgress,
|
|
30
|
+
});
|
|
31
|
+
if (folderMap.size === 0) {
|
|
32
|
+
throw new Error('Failed to create folders, cannot upload files');
|
|
33
|
+
}
|
|
34
|
+
await async_utils_1.AsyncUtils.sleep(500);
|
|
35
|
+
const totalBytes = await upload_file_service_1.UploadFileService.instance.uploadFilesConcurrently({
|
|
36
|
+
network,
|
|
37
|
+
filesToUpload: scanResult.files,
|
|
38
|
+
folderMap,
|
|
39
|
+
bucket: loginUserDetails.bucket,
|
|
40
|
+
destinationFolderUuid,
|
|
41
|
+
currentProgress,
|
|
42
|
+
emitProgress,
|
|
43
|
+
});
|
|
44
|
+
const rootFolderName = (0, node_path_1.basename)(localPath);
|
|
45
|
+
const rootFolderId = folderMap.get(rootFolderName) ?? '';
|
|
46
|
+
return {
|
|
47
|
+
totalBytes,
|
|
48
|
+
rootFolderId,
|
|
49
|
+
uploadTimeMs: timer.stop(),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
exports.UploadFacade = UploadFacade;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { UploadFilesConcurrentlyParams, UploadFileWithRetryParams } from './upload.types';
|
|
2
|
+
import { DriveFileItem } from '../../../types/drive.types';
|
|
3
|
+
export declare class UploadFileService {
|
|
4
|
+
static readonly instance: UploadFileService;
|
|
5
|
+
uploadFilesConcurrently({ network, filesToUpload, folderMap, bucket, destinationFolderUuid, currentProgress, emitProgress, }: UploadFilesConcurrentlyParams): Promise<number>;
|
|
6
|
+
uploadFileWithRetry({ file, network, bucket, parentFolderUuid, }: UploadFileWithRetryParams): Promise<DriveFileItem | null>;
|
|
7
|
+
private concurrencyArray;
|
|
8
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UploadFileService = void 0;
|
|
4
|
+
const logger_utils_1 = require("../../../utils/logger.utils");
|
|
5
|
+
const upload_types_1 = require("./upload.types");
|
|
6
|
+
const drive_file_service_1 = require("../../drive/drive-file.service");
|
|
7
|
+
const node_path_1 = require("node:path");
|
|
8
|
+
const errors_utils_1 = require("../../../utils/errors.utils");
|
|
9
|
+
const promises_1 = require("node:fs/promises");
|
|
10
|
+
const types_1 = require("@internxt/sdk/dist/drive/storage/types");
|
|
11
|
+
const thumbnail_utils_1 = require("../../../utils/thumbnail.utils");
|
|
12
|
+
const cli_utils_1 = require("../../../utils/cli.utils");
|
|
13
|
+
class UploadFileService {
|
|
14
|
+
static instance = new UploadFileService();
|
|
15
|
+
async uploadFilesConcurrently({ network, filesToUpload, folderMap, bucket, destinationFolderUuid, currentProgress, emitProgress, }) {
|
|
16
|
+
let bytesUploaded = 0;
|
|
17
|
+
const concurrentFiles = this.concurrencyArray(filesToUpload, upload_types_1.MAX_CONCURRENT_UPLOADS);
|
|
18
|
+
for (const fileArray of concurrentFiles) {
|
|
19
|
+
await Promise.allSettled(fileArray.map(async (file) => {
|
|
20
|
+
const parentPath = (0, node_path_1.dirname)(file.relativePath);
|
|
21
|
+
const parentFolderUuid = parentPath === '.' || parentPath === '' ? destinationFolderUuid : folderMap.get(parentPath);
|
|
22
|
+
if (!parentFolderUuid) {
|
|
23
|
+
logger_utils_1.logger.warn(`Parent folder not found for ${file.relativePath}, skipping...`);
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
const createdFileUuid = await this.uploadFileWithRetry({
|
|
27
|
+
file,
|
|
28
|
+
network,
|
|
29
|
+
bucket,
|
|
30
|
+
parentFolderUuid,
|
|
31
|
+
});
|
|
32
|
+
if (createdFileUuid) {
|
|
33
|
+
bytesUploaded += file.size;
|
|
34
|
+
currentProgress.bytesUploaded += file.size;
|
|
35
|
+
currentProgress.itemsUploaded++;
|
|
36
|
+
}
|
|
37
|
+
emitProgress();
|
|
38
|
+
}));
|
|
39
|
+
}
|
|
40
|
+
return bytesUploaded;
|
|
41
|
+
}
|
|
42
|
+
async uploadFileWithRetry({ file, network, bucket, parentFolderUuid, }) {
|
|
43
|
+
for (let attempt = 0; attempt <= upload_types_1.MAX_RETRIES; attempt++) {
|
|
44
|
+
try {
|
|
45
|
+
const stats = await (0, promises_1.stat)(file.absolutePath);
|
|
46
|
+
const fileSize = stats.size ?? 0;
|
|
47
|
+
const fileType = (0, node_path_1.extname)(file.absolutePath).replaceAll('.', '');
|
|
48
|
+
let fileId;
|
|
49
|
+
let thumbnailStream;
|
|
50
|
+
const timings = {
|
|
51
|
+
networkUpload: 0,
|
|
52
|
+
driveUpload: 0,
|
|
53
|
+
thumbnailUpload: 0,
|
|
54
|
+
};
|
|
55
|
+
if (fileSize > 0) {
|
|
56
|
+
const { fileStream, bufferStream } = (0, thumbnail_utils_1.createFileStreamWithBuffer)({
|
|
57
|
+
path: file.absolutePath,
|
|
58
|
+
fileType,
|
|
59
|
+
});
|
|
60
|
+
const uploadTimer = cli_utils_1.CLIUtils.timer();
|
|
61
|
+
thumbnailStream = bufferStream;
|
|
62
|
+
fileId = await new Promise((resolve, reject) => {
|
|
63
|
+
network.uploadFile(fileStream, fileSize, bucket, (err, res) => {
|
|
64
|
+
if (err) {
|
|
65
|
+
return reject(err);
|
|
66
|
+
}
|
|
67
|
+
resolve(res);
|
|
68
|
+
}, () => { });
|
|
69
|
+
});
|
|
70
|
+
timings.networkUpload = uploadTimer.stop();
|
|
71
|
+
}
|
|
72
|
+
const driveTimer = cli_utils_1.CLIUtils.timer();
|
|
73
|
+
const createdDriveFile = await drive_file_service_1.DriveFileService.instance.createFile({
|
|
74
|
+
plainName: file.name,
|
|
75
|
+
type: fileType,
|
|
76
|
+
size: fileSize,
|
|
77
|
+
folderUuid: parentFolderUuid,
|
|
78
|
+
fileId,
|
|
79
|
+
bucket,
|
|
80
|
+
encryptVersion: types_1.EncryptionVersion.Aes03,
|
|
81
|
+
creationTime: stats.birthtime?.toISOString(),
|
|
82
|
+
modificationTime: stats.mtime?.toISOString(),
|
|
83
|
+
});
|
|
84
|
+
timings.driveUpload = driveTimer.stop();
|
|
85
|
+
const thumbnailTimer = cli_utils_1.CLIUtils.timer();
|
|
86
|
+
if (thumbnailStream && fileSize > 0) {
|
|
87
|
+
void (0, thumbnail_utils_1.tryUploadThumbnail)({
|
|
88
|
+
bufferStream: thumbnailStream,
|
|
89
|
+
fileType,
|
|
90
|
+
userBucket: bucket,
|
|
91
|
+
fileUuid: createdDriveFile.uuid,
|
|
92
|
+
networkFacade: network,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
timings.thumbnailUpload = thumbnailTimer.stop();
|
|
96
|
+
const totalTime = Object.values(timings).reduce((sum, time) => sum + time, 0);
|
|
97
|
+
const throughputMBps = cli_utils_1.CLIUtils.calculateThroughputMBps(stats.size, timings.networkUpload);
|
|
98
|
+
logger_utils_1.logger.info(`Uploaded '${file.name}' (${cli_utils_1.CLIUtils.formatBytesToString(stats.size)})`);
|
|
99
|
+
logger_utils_1.logger.info(`Timing breakdown:\n
|
|
100
|
+
Network upload: ${cli_utils_1.CLIUtils.formatDuration(timings.networkUpload)} (${throughputMBps.toFixed(2)} MB/s)\n
|
|
101
|
+
Drive upload: ${cli_utils_1.CLIUtils.formatDuration(timings.driveUpload)}\n
|
|
102
|
+
Thumbnail: ${cli_utils_1.CLIUtils.formatDuration(timings.thumbnailUpload)}\n
|
|
103
|
+
Total: ${cli_utils_1.CLIUtils.formatDuration(totalTime)}\n`);
|
|
104
|
+
return createdDriveFile;
|
|
105
|
+
}
|
|
106
|
+
catch (error) {
|
|
107
|
+
if ((0, errors_utils_1.isAlreadyExistsError)(error)) {
|
|
108
|
+
const msg = `File ${file.name} already exists, skipping...`;
|
|
109
|
+
logger_utils_1.logger.info(msg);
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
if (attempt < upload_types_1.MAX_RETRIES) {
|
|
113
|
+
const delay = upload_types_1.DELAYS_MS[attempt];
|
|
114
|
+
const retryMsg = `Failed to upload file ${file.name}, retrying in ${delay}ms...`;
|
|
115
|
+
logger_utils_1.logger.warn(`${retryMsg} (attempt ${attempt + 1}/${upload_types_1.MAX_RETRIES + 1})`);
|
|
116
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
logger_utils_1.logger.error(`Failed to upload file ${file.name} after ${upload_types_1.MAX_RETRIES + 1} attempts`);
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
concurrencyArray(array, arraySize) {
|
|
127
|
+
const arrays = [];
|
|
128
|
+
for (let i = 0; i < array.length; i += arraySize) {
|
|
129
|
+
arrays.push(array.slice(i, i + arraySize));
|
|
130
|
+
}
|
|
131
|
+
return arrays;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
exports.UploadFileService = UploadFileService;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { CreateFoldersParams, CreateFolderWithRetryParams } from './upload.types';
|
|
2
|
+
export declare class UploadFolderService {
|
|
3
|
+
static readonly instance: UploadFolderService;
|
|
4
|
+
createFolders({ foldersToCreate, destinationFolderUuid, currentProgress, emitProgress, }: CreateFoldersParams): Promise<Map<string, string>>;
|
|
5
|
+
createFolderWithRetry({ folderName, parentFolderUuid }: CreateFolderWithRetryParams): Promise<string | null>;
|
|
6
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UploadFolderService = void 0;
|
|
4
|
+
const node_path_1 = require("node:path");
|
|
5
|
+
const errors_utils_1 = require("../../../utils/errors.utils");
|
|
6
|
+
const logger_utils_1 = require("../../../utils/logger.utils");
|
|
7
|
+
const drive_folder_service_1 = require("../../drive/drive-folder.service");
|
|
8
|
+
const upload_types_1 = require("./upload.types");
|
|
9
|
+
class UploadFolderService {
|
|
10
|
+
static instance = new UploadFolderService();
|
|
11
|
+
async createFolders({ foldersToCreate, destinationFolderUuid, currentProgress, emitProgress, }) {
|
|
12
|
+
const folderMap = new Map();
|
|
13
|
+
for (const folder of foldersToCreate) {
|
|
14
|
+
const parentPath = (0, node_path_1.dirname)(folder.relativePath);
|
|
15
|
+
const parentUuid = parentPath === '.' ? destinationFolderUuid : folderMap.get(parentPath);
|
|
16
|
+
if (!parentUuid) {
|
|
17
|
+
logger_utils_1.logger.warn(`Parent folder not found for ${folder.relativePath}, skipping...`);
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
const createdFolderUuid = await this.createFolderWithRetry({
|
|
21
|
+
folderName: folder.name,
|
|
22
|
+
parentFolderUuid: parentUuid,
|
|
23
|
+
});
|
|
24
|
+
if (createdFolderUuid) {
|
|
25
|
+
folderMap.set(folder.relativePath, createdFolderUuid);
|
|
26
|
+
currentProgress.itemsUploaded++;
|
|
27
|
+
emitProgress();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return folderMap;
|
|
31
|
+
}
|
|
32
|
+
async createFolderWithRetry({ folderName, parentFolderUuid }) {
|
|
33
|
+
for (let attempt = 0; attempt <= upload_types_1.MAX_RETRIES; attempt++) {
|
|
34
|
+
try {
|
|
35
|
+
const [createFolderPromise] = drive_folder_service_1.DriveFolderService.instance.createFolder({
|
|
36
|
+
plainName: folderName,
|
|
37
|
+
parentFolderUuid,
|
|
38
|
+
});
|
|
39
|
+
const createdFolder = await createFolderPromise;
|
|
40
|
+
return createdFolder.uuid;
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
if ((0, errors_utils_1.isAlreadyExistsError)(error)) {
|
|
44
|
+
logger_utils_1.logger.info(`Folder ${folderName} already exists, skipping...`);
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
if (attempt < upload_types_1.MAX_RETRIES) {
|
|
48
|
+
const delay = upload_types_1.DELAYS_MS[attempt];
|
|
49
|
+
logger_utils_1.logger.warn(`Failed to create folder ${folderName},
|
|
50
|
+
retrying in ${delay}ms... (attempt ${attempt + 1}/${upload_types_1.MAX_RETRIES + 1})`);
|
|
51
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
logger_utils_1.logger.error(`Failed to create folder ${folderName} after ${upload_types_1.MAX_RETRIES + 1} attempts`);
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
exports.UploadFolderService = UploadFolderService;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { LoginUserDetails } from '../../../types/command.types';
|
|
2
|
+
import { FileSystemNode } from '../../local-filesystem/local-filesystem.types';
|
|
3
|
+
import { NetworkFacade } from '../network-facade.service';
|
|
4
|
+
export interface UploadResult {
|
|
5
|
+
totalBytes: number;
|
|
6
|
+
rootFolderId: string;
|
|
7
|
+
uploadTimeMs: number;
|
|
8
|
+
}
|
|
9
|
+
export interface UploadFolderParams {
|
|
10
|
+
localPath: string;
|
|
11
|
+
destinationFolderUuid: string;
|
|
12
|
+
loginUserDetails: LoginUserDetails;
|
|
13
|
+
jsonFlag?: boolean;
|
|
14
|
+
onProgress: (progress: UploadProgress) => void;
|
|
15
|
+
}
|
|
16
|
+
export interface UploadProgress {
|
|
17
|
+
percentage: number;
|
|
18
|
+
currentFile?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface CreateFoldersParams {
|
|
21
|
+
foldersToCreate: FileSystemNode[];
|
|
22
|
+
destinationFolderUuid: string;
|
|
23
|
+
currentProgress: {
|
|
24
|
+
itemsUploaded: number;
|
|
25
|
+
bytesUploaded: number;
|
|
26
|
+
};
|
|
27
|
+
emitProgress: () => void;
|
|
28
|
+
}
|
|
29
|
+
export interface CreateFolderWithRetryParams {
|
|
30
|
+
folderName: string;
|
|
31
|
+
parentFolderUuid: string;
|
|
32
|
+
}
|
|
33
|
+
export interface UploadFilesConcurrentlyParams {
|
|
34
|
+
network: NetworkFacade;
|
|
35
|
+
filesToUpload: FileSystemNode[];
|
|
36
|
+
folderMap: Map<string, string>;
|
|
37
|
+
bucket: string;
|
|
38
|
+
destinationFolderUuid: string;
|
|
39
|
+
currentProgress: {
|
|
40
|
+
itemsUploaded: number;
|
|
41
|
+
bytesUploaded: number;
|
|
42
|
+
};
|
|
43
|
+
emitProgress: () => void;
|
|
44
|
+
}
|
|
45
|
+
export interface UploadFileWithRetryParams {
|
|
46
|
+
file: FileSystemNode;
|
|
47
|
+
network: NetworkFacade;
|
|
48
|
+
bucket: string;
|
|
49
|
+
parentFolderUuid: string;
|
|
50
|
+
}
|
|
51
|
+
export declare const MAX_CONCURRENT_UPLOADS = 10;
|
|
52
|
+
export declare const DELAYS_MS: number[];
|
|
53
|
+
export declare const MAX_RETRIES = 2;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MAX_RETRIES = exports.DELAYS_MS = exports.MAX_CONCURRENT_UPLOADS = void 0;
|
|
4
|
+
exports.MAX_CONCURRENT_UPLOADS = 10;
|
|
5
|
+
exports.DELAYS_MS = [500, 1000, 2000];
|
|
6
|
+
exports.MAX_RETRIES = 2;
|
|
@@ -6,8 +6,8 @@ export declare class SdkManager {
|
|
|
6
6
|
private static apiSecurity?;
|
|
7
7
|
static readonly init: (apiSecurity: SdkManagerApiSecurity) => void;
|
|
8
8
|
static readonly clean: () => void;
|
|
9
|
-
static readonly getApiSecurity: (
|
|
10
|
-
throwErrorOnMissingCredentials
|
|
9
|
+
static readonly getApiSecurity: ({ throwErrorOnMissingCredentials }?: {
|
|
10
|
+
throwErrorOnMissingCredentials?: boolean | undefined;
|
|
11
11
|
}) => SdkManagerApiSecurity;
|
|
12
12
|
static readonly getAppDetails: () => AppDetails;
|
|
13
13
|
getAuth(): Auth;
|
|
@@ -18,8 +18,8 @@ class SdkManager {
|
|
|
18
18
|
static clean = () => {
|
|
19
19
|
SdkManager.apiSecurity = undefined;
|
|
20
20
|
};
|
|
21
|
-
static getApiSecurity = (
|
|
22
|
-
if (!SdkManager.apiSecurity &&
|
|
21
|
+
static getApiSecurity = ({ throwErrorOnMissingCredentials = true } = {}) => {
|
|
22
|
+
if (!SdkManager.apiSecurity && throwErrorOnMissingCredentials)
|
|
23
23
|
throw new Error('Api security properties not found in SdkManager');
|
|
24
24
|
return SdkManager.apiSecurity;
|
|
25
25
|
};
|
|
@@ -1,37 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
3
|
exports.ThumbnailService = void 0;
|
|
37
4
|
const node_stream_1 = require("node:stream");
|
|
@@ -42,7 +9,7 @@ let sharpDependency = null;
|
|
|
42
9
|
const getSharp = async () => {
|
|
43
10
|
if (!sharpDependency) {
|
|
44
11
|
try {
|
|
45
|
-
sharpDependency = (await
|
|
12
|
+
sharpDependency = (await import('sharp')).default;
|
|
46
13
|
}
|
|
47
14
|
catch {
|
|
48
15
|
return null;
|
|
@@ -9,6 +9,11 @@ export declare class ValidationService {
|
|
|
9
9
|
validateStringIsNotEmpty: (str: string) => boolean;
|
|
10
10
|
validateDirectoryExists: (path: string) => Promise<boolean>;
|
|
11
11
|
validateFileExists: (path: string) => Promise<boolean>;
|
|
12
|
+
validateJwtAndCheckExpiration: (token?: string) => number | null;
|
|
13
|
+
checkTokenExpiration: (expirationTimestamp: number) => {
|
|
14
|
+
expired: boolean;
|
|
15
|
+
refreshRequired: boolean;
|
|
16
|
+
};
|
|
12
17
|
validateTokenAndCheckExpiration: (token?: string) => {
|
|
13
18
|
isValid: boolean;
|
|
14
19
|
expiration: {
|