@blocklet/did-space-js 0.5.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -0
- package/dist/client/index.d.ts +40 -0
- package/dist/client/index.js +110 -0
- package/dist/commands/backup/backup-blocklet.d.ts +14 -0
- package/dist/commands/backup/backup-blocklet.js +83 -0
- package/dist/commands/backup/index.d.ts +3 -0
- package/dist/commands/backup/index.js +19 -0
- package/dist/commands/backup/post-app-backup.d.ts +9 -0
- package/dist/commands/backup/post-app-backup.js +49 -0
- package/dist/commands/backup/put-app-backup.d.ts +9 -0
- package/dist/commands/backup/put-app-backup.js +49 -0
- package/dist/commands/base.d.ts +21 -0
- package/dist/commands/base.js +47 -0
- package/dist/commands/incremental-backup/backup-blocklet.d.ts +14 -0
- package/dist/commands/incremental-backup/backup-blocklet.js +90 -0
- package/dist/commands/incremental-backup/index.d.ts +1 -0
- package/dist/commands/incremental-backup/index.js +17 -0
- package/dist/commands/incremental-sync/index.d.ts +2 -0
- package/dist/commands/incremental-sync/index.js +18 -0
- package/dist/commands/incremental-sync/sync-base.d.ts +76 -0
- package/dist/commands/incremental-sync/sync-base.js +236 -0
- package/dist/commands/incremental-sync/sync-push.d.ts +30 -0
- package/dist/commands/incremental-sync/sync-push.js +202 -0
- package/dist/commands/index.d.ts +9 -0
- package/dist/commands/index.js +25 -0
- package/dist/commands/nft/index.d.ts +1 -0
- package/dist/commands/nft/index.js +17 -0
- package/dist/commands/nft/put-nft-object.d.ts +20 -0
- package/dist/commands/nft/put-nft-object.js +143 -0
- package/dist/commands/object/delete-object.d.ts +7 -0
- package/dist/commands/object/delete-object.js +38 -0
- package/dist/commands/object/get-object.d.ts +7 -0
- package/dist/commands/object/get-object.js +43 -0
- package/dist/commands/object/index.d.ts +5 -0
- package/dist/commands/object/index.js +21 -0
- package/dist/commands/object/list-object.d.ts +7 -0
- package/dist/commands/object/list-object.js +31 -0
- package/dist/commands/object/list-objects.d.ts +7 -0
- package/dist/commands/object/list-objects.js +44 -0
- package/dist/commands/object/put-object.d.ts +7 -0
- package/dist/commands/object/put-object.js +61 -0
- package/dist/commands/restore/index.d.ts +3 -0
- package/dist/commands/restore/index.js +19 -0
- package/dist/commands/restore/post-app-restore.d.ts +7 -0
- package/dist/commands/restore/post-app-restore.js +45 -0
- package/dist/commands/restore/put-app-restore.d.ts +9 -0
- package/dist/commands/restore/put-app-restore.js +49 -0
- package/dist/commands/restore/restore-blocklet.d.ts +14 -0
- package/dist/commands/restore/restore-blocklet.js +80 -0
- package/dist/commands/space/head.d.ts +8 -0
- package/dist/commands/space/head.js +52 -0
- package/dist/commands/space/index.d.ts +1 -0
- package/dist/commands/space/index.js +17 -0
- package/dist/commands/sync/index.d.ts +3 -0
- package/dist/commands/sync/index.js +19 -0
- package/dist/commands/sync/sync-folder-base.d.ts +105 -0
- package/dist/commands/sync/sync-folder-base.js +319 -0
- package/dist/commands/sync/sync-folder-pull.d.ts +13 -0
- package/dist/commands/sync/sync-folder-pull.js +151 -0
- package/dist/commands/sync/sync-folder-push.d.ts +20 -0
- package/dist/commands/sync/sync-folder-push.js +166 -0
- package/dist/did-document/did-document.d.ts +29 -0
- package/dist/did-document/did-document.js +2 -0
- package/dist/did-document/index.d.ts +3 -0
- package/dist/did-document/index.js +19 -0
- package/dist/did-document/sign-document.d.ts +3 -0
- package/dist/did-document/sign-document.js +13 -0
- package/dist/did-document/verify-document.d.ts +10 -0
- package/dist/did-document/verify-document.js +35 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +23 -0
- package/dist/libs/api.d.ts +3 -0
- package/dist/libs/api.js +65 -0
- package/dist/libs/did.d.ts +9 -0
- package/dist/libs/did.js +34 -0
- package/dist/libs/encoding.d.ts +7 -0
- package/dist/libs/encoding.js +17 -0
- package/dist/libs/error.d.ts +4 -0
- package/dist/libs/error.js +27 -0
- package/dist/libs/hash.d.ts +1 -0
- package/dist/libs/hash.js +5 -0
- package/dist/libs/index.d.ts +11 -0
- package/dist/libs/index.js +27 -0
- package/dist/libs/logger.d.ts +2 -0
- package/dist/libs/logger.js +12 -0
- package/dist/libs/object-integrality.d.ts +1 -0
- package/dist/libs/object-integrality.js +13 -0
- package/dist/libs/performance.d.ts +7 -0
- package/dist/libs/performance.js +13 -0
- package/dist/libs/sleep.d.ts +1 -0
- package/dist/libs/sleep.js +9 -0
- package/dist/libs/space.d.ts +16 -0
- package/dist/libs/space.js +56 -0
- package/dist/libs/stream.d.ts +3 -0
- package/dist/libs/stream.js +10 -0
- package/dist/meta/backup/backup-blocklet-command.d.ts +50 -0
- package/dist/meta/backup/backup-blocklet-command.js +2 -0
- package/dist/meta/backup/index.d.ts +3 -0
- package/dist/meta/backup/index.js +19 -0
- package/dist/meta/backup/post-app-backup-command.d.ts +8 -0
- package/dist/meta/backup/post-app-backup-command.js +16 -0
- package/dist/meta/backup/put-app-backup-command.d.ts +7 -0
- package/dist/meta/backup/put-app-backup-command.js +18 -0
- package/dist/meta/incremental-backup/backup-blocklet-command.d.ts +50 -0
- package/dist/meta/incremental-backup/backup-blocklet-command.js +2 -0
- package/dist/meta/incremental-backup/index.d.ts +1 -0
- package/dist/meta/incremental-backup/index.js +17 -0
- package/dist/meta/index.d.ts +7 -0
- package/dist/meta/index.js +23 -0
- package/dist/meta/nft/index.d.ts +1 -0
- package/dist/meta/nft/index.js +17 -0
- package/dist/meta/nft/put-nft-object-command.d.ts +51 -0
- package/dist/meta/nft/put-nft-object-command.js +2 -0
- package/dist/meta/object/delete-object-command.d.ts +6 -0
- package/dist/meta/object/delete-object-command.js +2 -0
- package/dist/meta/object/get-object-command.d.ts +10 -0
- package/dist/meta/object/get-object-command.js +2 -0
- package/dist/meta/object/head-object-command.d.ts +13 -0
- package/dist/meta/object/head-object-command.js +2 -0
- package/dist/meta/object/index.d.ts +5 -0
- package/dist/meta/object/index.js +21 -0
- package/dist/meta/object/list-object-command.d.ts +6 -0
- package/dist/meta/object/list-object-command.js +2 -0
- package/dist/meta/object/list-objects-command.d.ts +8 -0
- package/dist/meta/object/list-objects-command.js +2 -0
- package/dist/meta/object/put-object-command.d.ts +29 -0
- package/dist/meta/object/put-object-command.js +2 -0
- package/dist/meta/restore/index.d.ts +3 -0
- package/dist/meta/restore/index.js +19 -0
- package/dist/meta/restore/post-app-restore-command.d.ts +8 -0
- package/dist/meta/restore/post-app-restore-command.js +13 -0
- package/dist/meta/restore/put-app-restore-command.d.ts +7 -0
- package/dist/meta/restore/put-app-restore-command.js +18 -0
- package/dist/meta/restore/restore-blocklet-command.d.ts +28 -0
- package/dist/meta/restore/restore-blocklet-command.js +2 -0
- package/dist/meta/spaces-client-options.d.ts +5 -0
- package/dist/meta/spaces-client-options.js +2 -0
- package/dist/meta/sync/incremental-sync-base-command.d.ts +7 -0
- package/dist/meta/sync/incremental-sync-base-command.js +2 -0
- package/dist/meta/sync/incremental-sync-push-command.d.ts +8 -0
- package/dist/meta/sync/incremental-sync-push-command.js +2 -0
- package/dist/meta/sync/index.d.ts +6 -0
- package/dist/meta/sync/index.js +22 -0
- package/dist/meta/sync/sync-base-command.d.ts +121 -0
- package/dist/meta/sync/sync-base-command.js +2 -0
- package/dist/meta/sync/sync-folder-base-command.d.ts +12 -0
- package/dist/meta/sync/sync-folder-base-command.js +2 -0
- package/dist/meta/sync/sync-folder-pull-command.d.ts +5 -0
- package/dist/meta/sync/sync-folder-pull-command.js +2 -0
- package/dist/meta/sync/sync-folder-push-command.d.ts +7 -0
- package/dist/meta/sync/sync-folder-push-command.js +2 -0
- package/dist/protocol/command.protocol.d.ts +68 -0
- package/dist/protocol/command.protocol.js +2 -0
- package/dist/protocol/index.d.ts +1 -0
- package/dist/protocol/index.js +17 -0
- package/dist/security/index.d.ts +1 -0
- package/dist/security/index.js +17 -0
- package/dist/security/request-signature.d.ts +32 -0
- package/dist/security/request-signature.js +140 -0
- package/package.json +78 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RestoreBlockletCommand = void 0;
|
|
4
|
+
const path_1 = require("path");
|
|
5
|
+
const core_1 = require("@did-space/core");
|
|
6
|
+
const base_1 = require("../base");
|
|
7
|
+
const sync_1 = require("../sync");
|
|
8
|
+
const post_app_restore_1 = require("./post-app-restore");
|
|
9
|
+
const put_app_restore_1 = require("./put-app-restore");
|
|
10
|
+
const libs_1 = require("../../libs");
|
|
11
|
+
const space_1 = require("../../libs/space");
|
|
12
|
+
/**
|
|
13
|
+
*
|
|
14
|
+
* FIXME: @yejianchao 优化点,需要支持 joi 校验
|
|
15
|
+
* @export
|
|
16
|
+
* @class RestoreBlockletCommand
|
|
17
|
+
* @extends {BaseCommand<RestoreBlockletCommandInput, RestoreBlockletCommandOutput>}
|
|
18
|
+
*/
|
|
19
|
+
class RestoreBlockletCommand extends base_1.BaseCommand {
|
|
20
|
+
getAxiosRequestConfig() {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
async resolveMiddleware() {
|
|
24
|
+
let appRestore;
|
|
25
|
+
try {
|
|
26
|
+
// @note: 备份还原的时候我们希望使用 service url 来完成,不希望走 CDN
|
|
27
|
+
this.context.endpoint = await (0, space_1.getSpaceServiceEndpoint)(this.context.endpoint);
|
|
28
|
+
appRestore = (await new post_app_restore_1.PostAppRestoreCommand({
|
|
29
|
+
userDid: this.input.userDid,
|
|
30
|
+
referrer: this.input.referrer,
|
|
31
|
+
serverDid: this.input.serverDid,
|
|
32
|
+
}, this.context).request()).data;
|
|
33
|
+
const res = await new sync_1.SyncFolderPullCommand({
|
|
34
|
+
source: (0, path_1.join)('.did-objects', this.input.appDid, '/'),
|
|
35
|
+
...this.input,
|
|
36
|
+
}, this.context).request();
|
|
37
|
+
if (res.data.errorCount !== 0) {
|
|
38
|
+
await new put_app_restore_1.PutAppRestoreCommand({
|
|
39
|
+
id: appRestore.id,
|
|
40
|
+
status: core_1.APP_BACKUP_STATUS.FAILED,
|
|
41
|
+
message: res?.statusMessage,
|
|
42
|
+
}, this.context)
|
|
43
|
+
.request()
|
|
44
|
+
.catch(console.error);
|
|
45
|
+
return res;
|
|
46
|
+
}
|
|
47
|
+
// 仅仅是触发一下接口,更新一下 updateAt
|
|
48
|
+
await new put_app_restore_1.PutAppRestoreCommand({
|
|
49
|
+
id: appRestore.id,
|
|
50
|
+
status: core_1.APP_BACKUP_STATUS.SUCCEEDED,
|
|
51
|
+
}, this.context).request();
|
|
52
|
+
return res;
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
console.error(err);
|
|
56
|
+
const error = err;
|
|
57
|
+
const message = (0, libs_1.getErrorMessage)(error);
|
|
58
|
+
if (appRestore) {
|
|
59
|
+
await new put_app_restore_1.PutAppRestoreCommand({
|
|
60
|
+
id: appRestore.id,
|
|
61
|
+
status: core_1.APP_BACKUP_STATUS.FAILED,
|
|
62
|
+
message,
|
|
63
|
+
}, this.context)
|
|
64
|
+
.request()
|
|
65
|
+
.catch(console.error);
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
statusCode: (0, libs_1.getErrorStatusCode)(error),
|
|
69
|
+
statusMessage: message,
|
|
70
|
+
stack: error.stack,
|
|
71
|
+
data: {
|
|
72
|
+
errorCount: -1,
|
|
73
|
+
count: -1,
|
|
74
|
+
duration: -1,
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
exports.RestoreBlockletCommand = RestoreBlockletCommand;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { AxiosRequestConfig } from 'axios';
|
|
2
|
+
import { BaseCommand } from '../base';
|
|
3
|
+
import type { HeadSpaceCommandInput, HeadSpaceCommandOutput } from '../../meta/object/head-object-command';
|
|
4
|
+
export declare class HeadSpaceCommand extends BaseCommand<HeadSpaceCommandInput, HeadSpaceCommandOutput> {
|
|
5
|
+
getAxiosRequestConfig(): Promise<AxiosRequestConfig<any>>;
|
|
6
|
+
private getUrl;
|
|
7
|
+
resolveMiddleware(): Promise<HeadSpaceCommandOutput>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HeadSpaceCommand = void 0;
|
|
4
|
+
const core_1 = require("@did-space/core");
|
|
5
|
+
const ufo_1 = require("ufo");
|
|
6
|
+
const libs_1 = require("../../libs");
|
|
7
|
+
const security_1 = require("../../security");
|
|
8
|
+
const base_1 = require("../base");
|
|
9
|
+
class HeadSpaceCommand extends base_1.BaseCommand {
|
|
10
|
+
// eslint-disable-next-line require-await
|
|
11
|
+
async getAxiosRequestConfig() {
|
|
12
|
+
return null;
|
|
13
|
+
}
|
|
14
|
+
getUrl() {
|
|
15
|
+
const array = this.context.endpoint.split('/');
|
|
16
|
+
const lastIndexOfApp = array.lastIndexOf('app');
|
|
17
|
+
const url = array.filter((_, i) => i < lastIndexOfApp).join('/');
|
|
18
|
+
return (0, ufo_1.withQuery)(url, {
|
|
19
|
+
...this.input,
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
async resolveMiddleware() {
|
|
23
|
+
const url = this.getUrl();
|
|
24
|
+
const method = 'HEAD';
|
|
25
|
+
const { headers } = (0, security_1.signRequest)({
|
|
26
|
+
url,
|
|
27
|
+
method,
|
|
28
|
+
data: {},
|
|
29
|
+
headers: {
|
|
30
|
+
[core_1.HEADERS.X_SIGNATURE_VERSION]: '2',
|
|
31
|
+
},
|
|
32
|
+
wallet: this.context.wallet,
|
|
33
|
+
delegation: this.context.delegation,
|
|
34
|
+
});
|
|
35
|
+
const optionsOfTextResponseBody = {
|
|
36
|
+
method,
|
|
37
|
+
headers: {
|
|
38
|
+
...headers,
|
|
39
|
+
},
|
|
40
|
+
throwHttpErrors: false,
|
|
41
|
+
};
|
|
42
|
+
// TODO: axios 会导致严重的内存泄漏, 之后可以考虑切换到 got, @see: https://github.com/axios/axios/issues/4763#issuecomment-1334953446
|
|
43
|
+
const response = await (0, libs_1.gotApi)(url, optionsOfTextResponseBody);
|
|
44
|
+
return {
|
|
45
|
+
statusCode: response.statusCode,
|
|
46
|
+
statusMessage: response.headers['x-message'] ?? response.statusMessage,
|
|
47
|
+
data: undefined,
|
|
48
|
+
response,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
exports.HeadSpaceCommand = HeadSpaceCommand;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './head';
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./head"), exports);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./sync-folder-base"), exports);
|
|
18
|
+
__exportStar(require("./sync-folder-pull"), exports);
|
|
19
|
+
__exportStar(require("./sync-folder-push"), exports);
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { Object } from '@did-space/core';
|
|
2
|
+
import { AxiosError } from 'axios';
|
|
3
|
+
import PQueue from 'p-queue';
|
|
4
|
+
import { ListObjectCommandOutput, OnAfterUploadInput, OnProgressInput, SyncFolderBaseCommandInput, SyncFolderBaseCommandOutput, SyncFolderPullCommandOutput } from '../../meta';
|
|
5
|
+
import { BaseCommand } from '../base';
|
|
6
|
+
import type { SpaceClientOptions } from '../../protocol';
|
|
7
|
+
export interface ObjectsMap {
|
|
8
|
+
[key: string]: ListObjectCommandOutput['data'];
|
|
9
|
+
}
|
|
10
|
+
export interface SyncObject extends Object {
|
|
11
|
+
action: 'PUT' | 'DELETE';
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* @see s3 sync https://stackoverflow.com/questions/27932345/downloading-folders-from-aws-s3-cp-or-sync
|
|
15
|
+
* @see Keep-alive 和连接重用的 NodeJS https://azureossd.github.io/2022/03/10/NodeJS-with-Keep-Alives-and-Connection-Reuse/index.html
|
|
16
|
+
* @export
|
|
17
|
+
* @class SyncFolderPushCommand
|
|
18
|
+
* @extends {BaseCommand<SyncFolderCommandInput, SyncFolderPushCommandOutput>}
|
|
19
|
+
*/
|
|
20
|
+
export declare abstract class SyncFolderBaseCommand<CommandInput extends SyncFolderBaseCommandInput, CommandOutput extends SyncFolderBaseCommandOutput> extends BaseCommand<CommandInput, CommandOutput> {
|
|
21
|
+
protected readonly queue: PQueue;
|
|
22
|
+
protected readonly startTime: number;
|
|
23
|
+
static readonly DEFAULT_RETRY_COUNT: number;
|
|
24
|
+
static readonly MAX_RETRY_COUNT: number;
|
|
25
|
+
static readonly DEFAULT_CONCURRENCY: number;
|
|
26
|
+
static readonly MAX_CONCURRENCY: number;
|
|
27
|
+
private retryCount;
|
|
28
|
+
constructor(input: SyncFolderBaseCommandInput, context?: SpaceClientOptions);
|
|
29
|
+
abstract getLocalObjectsMap(): Promise<ObjectsMap>;
|
|
30
|
+
abstract getRemoteObjectsMap(): Promise<ObjectsMap>;
|
|
31
|
+
/**
|
|
32
|
+
*
|
|
33
|
+
* @description 仅在开启调试模式时,输出调试信息,
|
|
34
|
+
* @protected
|
|
35
|
+
* @param {{
|
|
36
|
+
* completedCount: number;
|
|
37
|
+
* errorCount: number;
|
|
38
|
+
* totalCount: number;
|
|
39
|
+
* }} {
|
|
40
|
+
* completedCount,
|
|
41
|
+
* errorCount,
|
|
42
|
+
* totalCount,
|
|
43
|
+
* }
|
|
44
|
+
* @return {*} {Promise<void>}
|
|
45
|
+
* @memberof SyncFolderBaseCommand
|
|
46
|
+
*/
|
|
47
|
+
protected printDebugInfo({ direction, completedCount, errorCount, totalCount, }: {
|
|
48
|
+
direction: 'pull' | 'push';
|
|
49
|
+
completedCount: number;
|
|
50
|
+
errorCount: number;
|
|
51
|
+
totalCount: number;
|
|
52
|
+
}): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
*
|
|
55
|
+
*
|
|
56
|
+
* @protected
|
|
57
|
+
* @param {(Error | AxiosError)} error
|
|
58
|
+
* @return {*} {Promise<void>}
|
|
59
|
+
* @memberof SyncFolderBaseCommand
|
|
60
|
+
*/
|
|
61
|
+
protected retryRequest(error: Error | AxiosError): Promise<SyncFolderPullCommandOutput>;
|
|
62
|
+
/**
|
|
63
|
+
*
|
|
64
|
+
* @description 比较双方的文件到底谁更新鲜,更新鲜的数据才需要被同步上去
|
|
65
|
+
* @protected
|
|
66
|
+
* @param {ObjectsMap} fromObjectsMap
|
|
67
|
+
* @param {ObjectsMap} toObjectsMap
|
|
68
|
+
* @return {*} {Promise<string[]>}
|
|
69
|
+
* @memberof SyncFolderBaseCommand
|
|
70
|
+
*/
|
|
71
|
+
protected getWaitSyncObjects(fromObjectsMap: ObjectsMap, toObjectsMap: ObjectsMap): Promise<SyncObject[]>;
|
|
72
|
+
/**
|
|
73
|
+
* @see https://github.com/ArcBlock/did-spaces/issues/955#issuecomment-1873723050
|
|
74
|
+
* @description 备份的对象随时可能在进行写入操作,所以对于需要备份的文件,最佳实践是备份它们某一个时刻的副本,否则会引发错误 Error: Unexpected end of form
|
|
75
|
+
* @param {SyncObject[]} waitSyncObjects
|
|
76
|
+
* @return {*} {Promise<SyncObject[]>}
|
|
77
|
+
* @memberof SyncFolderBaseCommand
|
|
78
|
+
*/
|
|
79
|
+
copyObjects(waitSyncObjects: SyncObject[]): Promise<SyncObject[]>;
|
|
80
|
+
/**
|
|
81
|
+
* @description 是同一个对象吗?
|
|
82
|
+
* @param {string} source
|
|
83
|
+
* @param {string} target
|
|
84
|
+
* @return {*} {Promise<boolean>}
|
|
85
|
+
* @memberof SyncFolderBaseCommand
|
|
86
|
+
*/
|
|
87
|
+
diffObject(source: string, target: string): Promise<{
|
|
88
|
+
same: boolean;
|
|
89
|
+
sourceObject: Object;
|
|
90
|
+
targetObject: Object;
|
|
91
|
+
}>;
|
|
92
|
+
getObject(absolutePath: string): Promise<Object>;
|
|
93
|
+
/**
|
|
94
|
+
* @FIXME: 重复的代码,需要重构一下接口
|
|
95
|
+
* @description 判断一个对象是否应该被备份
|
|
96
|
+
* @protected
|
|
97
|
+
* @param {string} absolutePath
|
|
98
|
+
* @return {*} {Promise<boolean>}
|
|
99
|
+
* @memberof IncrementalSyncBaseCommand
|
|
100
|
+
*/
|
|
101
|
+
protected shouldBeBackup(absolutePath: string): Promise<boolean>;
|
|
102
|
+
protected triggerOnProgressHook(params: OnProgressInput): Promise<void>;
|
|
103
|
+
protected triggerOnAfterUploadHook(params: OnAfterUploadInput): Promise<void>;
|
|
104
|
+
destroy(): Promise<void>;
|
|
105
|
+
}
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.SyncFolderBaseCommand = void 0;
|
|
7
|
+
/* eslint-disable @typescript-eslint/indent */
|
|
8
|
+
const core_1 = require("@did-space/core");
|
|
9
|
+
const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
|
|
10
|
+
const isUndefined_1 = __importDefault(require("lodash/isUndefined"));
|
|
11
|
+
const keys_1 = __importDefault(require("lodash/keys"));
|
|
12
|
+
const p_queue_1 = __importDefault(require("p-queue"));
|
|
13
|
+
const debug_1 = __importDefault(require("debug"));
|
|
14
|
+
const fs_extra_1 = require("fs-extra");
|
|
15
|
+
const p_all_1 = __importDefault(require("p-all"));
|
|
16
|
+
const cloneDeep_1 = __importDefault(require("lodash/cloneDeep"));
|
|
17
|
+
const path_1 = require("path");
|
|
18
|
+
const mime_types_1 = __importDefault(require("mime-types"));
|
|
19
|
+
const promises_1 = require("fs/promises");
|
|
20
|
+
const os_1 = require("os");
|
|
21
|
+
const libs_1 = require("../../libs");
|
|
22
|
+
const logger_1 = require("../../libs/logger");
|
|
23
|
+
const base_1 = require("../base");
|
|
24
|
+
const { name, version } = require('../../../package.json');
|
|
25
|
+
const debug = (0, debug_1.default)(`${name}@${version}:SyncFolderBaseCommand`);
|
|
26
|
+
/**
|
|
27
|
+
* @see s3 sync https://stackoverflow.com/questions/27932345/downloading-folders-from-aws-s3-cp-or-sync
|
|
28
|
+
* @see Keep-alive 和连接重用的 NodeJS https://azureossd.github.io/2022/03/10/NodeJS-with-Keep-Alives-and-Connection-Reuse/index.html
|
|
29
|
+
* @export
|
|
30
|
+
* @class SyncFolderPushCommand
|
|
31
|
+
* @extends {BaseCommand<SyncFolderCommandInput, SyncFolderPushCommandOutput>}
|
|
32
|
+
*/
|
|
33
|
+
class SyncFolderBaseCommand extends base_1.BaseCommand {
|
|
34
|
+
queue;
|
|
35
|
+
startTime = Date.now();
|
|
36
|
+
static DEFAULT_RETRY_COUNT = 3;
|
|
37
|
+
static MAX_RETRY_COUNT = 10;
|
|
38
|
+
static DEFAULT_CONCURRENCY = 4;
|
|
39
|
+
static MAX_CONCURRENCY = (0, os_1.cpus)().length;
|
|
40
|
+
retryCount = 0;
|
|
41
|
+
constructor(input, context) {
|
|
42
|
+
// @ts-expect-error why?暂时没想通报错的原因
|
|
43
|
+
super(input, context);
|
|
44
|
+
if ((0, isUndefined_1.default)(input?.concurrency)) {
|
|
45
|
+
this.input.concurrency = SyncFolderBaseCommand.DEFAULT_CONCURRENCY;
|
|
46
|
+
}
|
|
47
|
+
else if (input.concurrency > SyncFolderBaseCommand.MAX_CONCURRENCY) {
|
|
48
|
+
this.input.concurrency = SyncFolderBaseCommand.MAX_CONCURRENCY;
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
this.input.concurrency = input.concurrency;
|
|
52
|
+
}
|
|
53
|
+
if ((0, isUndefined_1.default)(input?.retryCount)) {
|
|
54
|
+
this.input.retryCount = SyncFolderBaseCommand.DEFAULT_RETRY_COUNT;
|
|
55
|
+
}
|
|
56
|
+
else if (input.retryCount > SyncFolderBaseCommand.MAX_RETRY_COUNT) {
|
|
57
|
+
this.input.retryCount = SyncFolderBaseCommand.MAX_RETRY_COUNT;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
this.input.retryCount = input.retryCount;
|
|
61
|
+
}
|
|
62
|
+
if (!this.input.source.endsWith('/')) {
|
|
63
|
+
this.input.source = (0, path_1.join)(this.input.source, '/');
|
|
64
|
+
}
|
|
65
|
+
if (!this.input.target.endsWith('/')) {
|
|
66
|
+
this.input.target = (0, path_1.join)(this.input.target, '/');
|
|
67
|
+
}
|
|
68
|
+
if ((0, isEmpty_1.default)(this.input.tmpDir)) {
|
|
69
|
+
this.input.tmpDir = (0, path_1.join)(process.env.BLOCKLET_DATA_DIR || process.env.ABT_NODE_DATA_DIR, 'tmp', this.constructor.name, this.input.target);
|
|
70
|
+
(0, fs_extra_1.ensureDirSync)(this.input.tmpDir);
|
|
71
|
+
}
|
|
72
|
+
this.queue = new p_queue_1.default({ concurrency: this.input.concurrency });
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
*
|
|
76
|
+
* @description 仅在开启调试模式时,输出调试信息,
|
|
77
|
+
* @protected
|
|
78
|
+
* @param {{
|
|
79
|
+
* completedCount: number;
|
|
80
|
+
* errorCount: number;
|
|
81
|
+
* totalCount: number;
|
|
82
|
+
* }} {
|
|
83
|
+
* completedCount,
|
|
84
|
+
* errorCount,
|
|
85
|
+
* totalCount,
|
|
86
|
+
* }
|
|
87
|
+
* @return {*} {Promise<void>}
|
|
88
|
+
* @memberof SyncFolderBaseCommand
|
|
89
|
+
*/
|
|
90
|
+
printDebugInfo({ direction, completedCount, errorCount, totalCount, }) {
|
|
91
|
+
if (!this.input.debug) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const remaining = totalCount - errorCount - completedCount;
|
|
95
|
+
const progress = ((completedCount * 100) / totalCount).toFixed(2);
|
|
96
|
+
const duration = (Date.now() - this.startTime) / 1000;
|
|
97
|
+
logger_1.logger.info(`[${direction}] pending: ${this.queue.pending}, completed: ${completedCount}, error: ${errorCount}, remaining: ${remaining}, progress: ${progress}%, total: ${totalCount}, retryCount: ${this.input.retryCount}, duration: ${duration}(s)`);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
*
|
|
101
|
+
*
|
|
102
|
+
* @protected
|
|
103
|
+
* @param {(Error | AxiosError)} error
|
|
104
|
+
* @return {*} {Promise<void>}
|
|
105
|
+
* @memberof SyncFolderBaseCommand
|
|
106
|
+
*/
|
|
107
|
+
async retryRequest(error) {
|
|
108
|
+
// 减少重试次数
|
|
109
|
+
--this.input.retryCount;
|
|
110
|
+
// clear queue
|
|
111
|
+
this.queue.clear();
|
|
112
|
+
// 输出错误
|
|
113
|
+
const errorMessage = (0, libs_1.getErrorMessage)(error);
|
|
114
|
+
const statusCode = (0, libs_1.getErrorStatusCode)(error);
|
|
115
|
+
logger_1.logger.error('#retryRequest()', error);
|
|
116
|
+
logger_1.logger.error('#retryRequest()', {
|
|
117
|
+
input: this.input,
|
|
118
|
+
endpoint: this.context.endpoint,
|
|
119
|
+
errorMessage,
|
|
120
|
+
statusCode,
|
|
121
|
+
});
|
|
122
|
+
// FIXME: retryCount === 0 表示,重试次数刚好用完了,后续优化 server 显示错误的进度 @yejianchao
|
|
123
|
+
// 防呆控制,retryCount < 0 说明调用基础接口始终是失败的,遇到了预料之外的事情,因此需要停止运行
|
|
124
|
+
if (this.input.retryCount <= 0 || [400, 401, 403, 404, 413].includes(statusCode)) {
|
|
125
|
+
await this.destroy();
|
|
126
|
+
return {
|
|
127
|
+
statusCode,
|
|
128
|
+
statusMessage: (0, libs_1.getErrorMessage)(error),
|
|
129
|
+
stack: error.stack,
|
|
130
|
+
data: {
|
|
131
|
+
errorCount: -1,
|
|
132
|
+
count: -1,
|
|
133
|
+
duration: (Date.now() - this.startTime) / 1000,
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
// 目前我们的重试时间间隔是以 2 的指数增长的,第1次重试 1 s,第 2 次重试 2 s, 第 3 次重试间隔 4 s...
|
|
138
|
+
const retryDelayMs = 2 ** this.retryCount * 1000;
|
|
139
|
+
++this.retryCount;
|
|
140
|
+
await (0, libs_1.sleep)(retryDelayMs);
|
|
141
|
+
return this.request();
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
*
|
|
145
|
+
* @description 比较双方的文件到底谁更新鲜,更新鲜的数据才需要被同步上去
|
|
146
|
+
* @protected
|
|
147
|
+
* @param {ObjectsMap} fromObjectsMap
|
|
148
|
+
* @param {ObjectsMap} toObjectsMap
|
|
149
|
+
* @return {*} {Promise<string[]>}
|
|
150
|
+
* @memberof SyncFolderBaseCommand
|
|
151
|
+
*/
|
|
152
|
+
async getWaitSyncObjects(fromObjectsMap, toObjectsMap) {
|
|
153
|
+
const intersectionKeys = [...new Set((0, keys_1.default)(fromObjectsMap).concat((0, keys_1.default)(toObjectsMap)))];
|
|
154
|
+
const actions = intersectionKeys.map((key) => {
|
|
155
|
+
return async () => {
|
|
156
|
+
const fromObject = fromObjectsMap[key];
|
|
157
|
+
const toObject = toObjectsMap[key];
|
|
158
|
+
debug(`getWaitSyncObjects.$fromObjectsMap[${key}]`, fromObject);
|
|
159
|
+
debug(`getWaitSyncObjects.$toObjectsMap[${key}]`, toObject);
|
|
160
|
+
// 本地存在的文件 && 在远程不存在, 则修改远程的
|
|
161
|
+
if (fromObject && !toObject) {
|
|
162
|
+
return {
|
|
163
|
+
...fromObject,
|
|
164
|
+
action: 'PUT',
|
|
165
|
+
key,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
// 本地不存在 && 远程存在, 则删除远程的
|
|
169
|
+
if (!fromObject && toObject) {
|
|
170
|
+
return {
|
|
171
|
+
...toObject,
|
|
172
|
+
action: 'DELETE',
|
|
173
|
+
key,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
// 本地存在的文件 && 在远程存在 && 两者文件大小不一致, 则修改远程的
|
|
177
|
+
if (fromObject.size !== toObject.size) {
|
|
178
|
+
return {
|
|
179
|
+
...fromObject,
|
|
180
|
+
action: 'PUT',
|
|
181
|
+
key,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
// 本地存在的文件 && 在远程存在 && 本地日期不新鲜,则无需修改
|
|
185
|
+
if (fromObject.lastModified <= toObject.lastModified) {
|
|
186
|
+
return null;
|
|
187
|
+
}
|
|
188
|
+
if (!fromObject.hash) {
|
|
189
|
+
if (!(await this.shouldBeBackup(fromObject.absolutePath))) {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
// 此时计算 hash
|
|
193
|
+
fromObject.hash = await (0, core_1.getHash)((0, fs_extra_1.createReadStream)(fromObject.absolutePath));
|
|
194
|
+
}
|
|
195
|
+
// 本地存在的文件 && 在远程存在 && 两者文件内容不同, 则修改远程的
|
|
196
|
+
if (fromObject.hash !== toObject.hash) {
|
|
197
|
+
return {
|
|
198
|
+
...fromObject,
|
|
199
|
+
action: 'PUT',
|
|
200
|
+
key,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
return null;
|
|
204
|
+
};
|
|
205
|
+
});
|
|
206
|
+
const syncObjects = (await (0, p_all_1.default)(actions, { concurrency: this.input.concurrency })).filter(Boolean);
|
|
207
|
+
debug('getWaitSyncObjects.$syncObjects.length', syncObjects.length);
|
|
208
|
+
debug('getWaitSyncObjects.$syncObjects.slice(0, 3)', syncObjects.slice(0, 3));
|
|
209
|
+
return syncObjects;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* @see https://github.com/ArcBlock/did-spaces/issues/955#issuecomment-1873723050
|
|
213
|
+
* @description 备份的对象随时可能在进行写入操作,所以对于需要备份的文件,最佳实践是备份它们某一个时刻的副本,否则会引发错误 Error: Unexpected end of form
|
|
214
|
+
* @param {SyncObject[]} waitSyncObjects
|
|
215
|
+
* @return {*} {Promise<SyncObject[]>}
|
|
216
|
+
* @memberof SyncFolderBaseCommand
|
|
217
|
+
*/
|
|
218
|
+
async copyObjects(waitSyncObjects) {
|
|
219
|
+
const syncObjects = (0, cloneDeep_1.default)(waitSyncObjects);
|
|
220
|
+
await (0, p_all_1.default)(syncObjects.map((x) => {
|
|
221
|
+
return async () => {
|
|
222
|
+
const source = x.absolutePath;
|
|
223
|
+
const target = (0, path_1.join)(this.input.tmpDir, x.key);
|
|
224
|
+
if (x.action !== 'PUT' || !(await (0, fs_extra_1.pathExists)(source))) {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
if (!(await (0, fs_extra_1.pathExists)(target))) {
|
|
228
|
+
await (0, fs_extra_1.ensureDir)((0, path_1.dirname)(target));
|
|
229
|
+
await (0, promises_1.copyFile)(source, target);
|
|
230
|
+
}
|
|
231
|
+
const { same, targetObject } = await this.diffObject(source, target);
|
|
232
|
+
if (!same) {
|
|
233
|
+
x.size = targetObject.size;
|
|
234
|
+
x.lastModified = targetObject.lastModified;
|
|
235
|
+
x.hash = await (0, core_1.getHash)((0, fs_extra_1.createReadStream)(targetObject.absolutePath));
|
|
236
|
+
}
|
|
237
|
+
x.absolutePath = target;
|
|
238
|
+
};
|
|
239
|
+
}), {
|
|
240
|
+
concurrency: this.input.concurrency,
|
|
241
|
+
});
|
|
242
|
+
return syncObjects;
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* @description 是同一个对象吗?
|
|
246
|
+
* @param {string} source
|
|
247
|
+
* @param {string} target
|
|
248
|
+
* @return {*} {Promise<boolean>}
|
|
249
|
+
* @memberof SyncFolderBaseCommand
|
|
250
|
+
*/
|
|
251
|
+
async diffObject(source, target) {
|
|
252
|
+
const sourceObject = await this.getObject(source);
|
|
253
|
+
const targetObject = await this.getObject(target);
|
|
254
|
+
const same = sourceObject.lastModified <= targetObject.lastModified && sourceObject.size === targetObject.size;
|
|
255
|
+
return {
|
|
256
|
+
same,
|
|
257
|
+
sourceObject,
|
|
258
|
+
targetObject,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
async getObject(absolutePath) {
|
|
262
|
+
const fileStat = await (0, fs_extra_1.stat)(absolutePath);
|
|
263
|
+
const objectName = (0, path_1.basename)(absolutePath);
|
|
264
|
+
const directorySuffixes = fileStat.isDirectory() ? '/' : '';
|
|
265
|
+
return {
|
|
266
|
+
key: absolutePath,
|
|
267
|
+
name: (0, path_1.join)(objectName, directorySuffixes),
|
|
268
|
+
isDir: fileStat.isDirectory(),
|
|
269
|
+
size: fileStat.size,
|
|
270
|
+
lastModified: new Date(fileStat.mtime).getTime(),
|
|
271
|
+
editable: false,
|
|
272
|
+
mimeType: mime_types_1.default.lookup(objectName) || 'unknown',
|
|
273
|
+
absolutePath,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* @FIXME: 重复的代码,需要重构一下接口
|
|
278
|
+
* @description 判断一个对象是否应该被备份
|
|
279
|
+
* @protected
|
|
280
|
+
* @param {string} absolutePath
|
|
281
|
+
* @return {*} {Promise<boolean>}
|
|
282
|
+
* @memberof IncrementalSyncBaseCommand
|
|
283
|
+
*/
|
|
284
|
+
// eslint-disable-next-line require-await
|
|
285
|
+
async shouldBeBackup(absolutePath) {
|
|
286
|
+
if (typeof absolutePath !== 'string' || (0, isEmpty_1.default)(absolutePath)) {
|
|
287
|
+
return false;
|
|
288
|
+
}
|
|
289
|
+
if (absolutePath.endsWith('~')) {
|
|
290
|
+
// 临时文件不备份,因为随时可能被删除,从而引发错误
|
|
291
|
+
return false;
|
|
292
|
+
}
|
|
293
|
+
if (!(await (0, fs_extra_1.pathExists)(absolutePath))) {
|
|
294
|
+
// 不存在的文件不备份
|
|
295
|
+
return false;
|
|
296
|
+
}
|
|
297
|
+
return true;
|
|
298
|
+
}
|
|
299
|
+
// FIXME: @yejianchao 之后考虑使用事件驱动
|
|
300
|
+
async triggerOnProgressHook(params) {
|
|
301
|
+
if (typeof this.input.onProgress !== 'function') {
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
await this.input.onProgress(params);
|
|
305
|
+
}
|
|
306
|
+
// FIXME: @yejianchao 之后考虑使用事件驱动
|
|
307
|
+
async triggerOnAfterUploadHook(params) {
|
|
308
|
+
if (typeof this.input.onAfterUpload !== 'function') {
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
await this.input.onAfterUpload(params);
|
|
312
|
+
}
|
|
313
|
+
async destroy() {
|
|
314
|
+
if (await (0, fs_extra_1.pathExists)(this.input.tmpDir)) {
|
|
315
|
+
await (0, fs_extra_1.rm)(this.input.tmpDir, { recursive: true });
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
exports.SyncFolderBaseCommand = SyncFolderBaseCommand;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { AxiosRequestConfig } from 'axios';
|
|
2
|
+
import { SyncFolderPullCommandInput, SyncFolderPullCommandOutput } from '../../meta';
|
|
3
|
+
import { ObjectsMap, SyncFolderBaseCommand, SyncObject } from './sync-folder-base';
|
|
4
|
+
import type { SpaceClientOptions } from '../../protocol';
|
|
5
|
+
export declare class SyncFolderPullCommand extends SyncFolderBaseCommand<SyncFolderPullCommandInput, SyncFolderPullCommandOutput> {
|
|
6
|
+
protected readonly startTime: number;
|
|
7
|
+
constructor(input: SyncFolderPullCommandInput, context?: SpaceClientOptions);
|
|
8
|
+
getAxiosRequestConfig(): Promise<AxiosRequestConfig<any>>;
|
|
9
|
+
request(): Promise<SyncFolderPullCommandOutput>;
|
|
10
|
+
getRemoteObjectsMap(): Promise<ObjectsMap>;
|
|
11
|
+
getLocalObjectsMap(): Promise<ObjectsMap>;
|
|
12
|
+
sync(syncObjects: SyncObject[]): Promise<SyncFolderPullCommandOutput>;
|
|
13
|
+
}
|