@blocklet/did-space-js 0.5.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +77 -0
  2. package/dist/client/index.d.ts +40 -0
  3. package/dist/client/index.js +110 -0
  4. package/dist/commands/backup/backup-blocklet.d.ts +14 -0
  5. package/dist/commands/backup/backup-blocklet.js +83 -0
  6. package/dist/commands/backup/index.d.ts +3 -0
  7. package/dist/commands/backup/index.js +19 -0
  8. package/dist/commands/backup/post-app-backup.d.ts +9 -0
  9. package/dist/commands/backup/post-app-backup.js +49 -0
  10. package/dist/commands/backup/put-app-backup.d.ts +9 -0
  11. package/dist/commands/backup/put-app-backup.js +49 -0
  12. package/dist/commands/base.d.ts +21 -0
  13. package/dist/commands/base.js +47 -0
  14. package/dist/commands/incremental-backup/backup-blocklet.d.ts +14 -0
  15. package/dist/commands/incremental-backup/backup-blocklet.js +90 -0
  16. package/dist/commands/incremental-backup/index.d.ts +1 -0
  17. package/dist/commands/incremental-backup/index.js +17 -0
  18. package/dist/commands/incremental-sync/index.d.ts +2 -0
  19. package/dist/commands/incremental-sync/index.js +18 -0
  20. package/dist/commands/incremental-sync/sync-base.d.ts +76 -0
  21. package/dist/commands/incremental-sync/sync-base.js +236 -0
  22. package/dist/commands/incremental-sync/sync-push.d.ts +30 -0
  23. package/dist/commands/incremental-sync/sync-push.js +202 -0
  24. package/dist/commands/index.d.ts +9 -0
  25. package/dist/commands/index.js +25 -0
  26. package/dist/commands/nft/index.d.ts +1 -0
  27. package/dist/commands/nft/index.js +17 -0
  28. package/dist/commands/nft/put-nft-object.d.ts +20 -0
  29. package/dist/commands/nft/put-nft-object.js +143 -0
  30. package/dist/commands/object/delete-object.d.ts +7 -0
  31. package/dist/commands/object/delete-object.js +38 -0
  32. package/dist/commands/object/get-object.d.ts +7 -0
  33. package/dist/commands/object/get-object.js +43 -0
  34. package/dist/commands/object/index.d.ts +5 -0
  35. package/dist/commands/object/index.js +21 -0
  36. package/dist/commands/object/list-object.d.ts +7 -0
  37. package/dist/commands/object/list-object.js +31 -0
  38. package/dist/commands/object/list-objects.d.ts +7 -0
  39. package/dist/commands/object/list-objects.js +44 -0
  40. package/dist/commands/object/put-object.d.ts +7 -0
  41. package/dist/commands/object/put-object.js +61 -0
  42. package/dist/commands/restore/index.d.ts +3 -0
  43. package/dist/commands/restore/index.js +19 -0
  44. package/dist/commands/restore/post-app-restore.d.ts +7 -0
  45. package/dist/commands/restore/post-app-restore.js +45 -0
  46. package/dist/commands/restore/put-app-restore.d.ts +9 -0
  47. package/dist/commands/restore/put-app-restore.js +49 -0
  48. package/dist/commands/restore/restore-blocklet.d.ts +14 -0
  49. package/dist/commands/restore/restore-blocklet.js +80 -0
  50. package/dist/commands/space/head.d.ts +8 -0
  51. package/dist/commands/space/head.js +52 -0
  52. package/dist/commands/space/index.d.ts +1 -0
  53. package/dist/commands/space/index.js +17 -0
  54. package/dist/commands/sync/index.d.ts +3 -0
  55. package/dist/commands/sync/index.js +19 -0
  56. package/dist/commands/sync/sync-folder-base.d.ts +105 -0
  57. package/dist/commands/sync/sync-folder-base.js +319 -0
  58. package/dist/commands/sync/sync-folder-pull.d.ts +13 -0
  59. package/dist/commands/sync/sync-folder-pull.js +151 -0
  60. package/dist/commands/sync/sync-folder-push.d.ts +20 -0
  61. package/dist/commands/sync/sync-folder-push.js +166 -0
  62. package/dist/did-document/did-document.d.ts +29 -0
  63. package/dist/did-document/did-document.js +2 -0
  64. package/dist/did-document/index.d.ts +3 -0
  65. package/dist/did-document/index.js +19 -0
  66. package/dist/did-document/sign-document.d.ts +3 -0
  67. package/dist/did-document/sign-document.js +13 -0
  68. package/dist/did-document/verify-document.d.ts +10 -0
  69. package/dist/did-document/verify-document.js +35 -0
  70. package/dist/index.d.ts +7 -0
  71. package/dist/index.js +23 -0
  72. package/dist/libs/api.d.ts +3 -0
  73. package/dist/libs/api.js +65 -0
  74. package/dist/libs/did.d.ts +9 -0
  75. package/dist/libs/did.js +34 -0
  76. package/dist/libs/encoding.d.ts +7 -0
  77. package/dist/libs/encoding.js +17 -0
  78. package/dist/libs/error.d.ts +4 -0
  79. package/dist/libs/error.js +27 -0
  80. package/dist/libs/hash.d.ts +1 -0
  81. package/dist/libs/hash.js +5 -0
  82. package/dist/libs/index.d.ts +11 -0
  83. package/dist/libs/index.js +27 -0
  84. package/dist/libs/logger.d.ts +2 -0
  85. package/dist/libs/logger.js +12 -0
  86. package/dist/libs/object-integrality.d.ts +1 -0
  87. package/dist/libs/object-integrality.js +13 -0
  88. package/dist/libs/performance.d.ts +7 -0
  89. package/dist/libs/performance.js +13 -0
  90. package/dist/libs/sleep.d.ts +1 -0
  91. package/dist/libs/sleep.js +9 -0
  92. package/dist/libs/space.d.ts +16 -0
  93. package/dist/libs/space.js +56 -0
  94. package/dist/libs/stream.d.ts +3 -0
  95. package/dist/libs/stream.js +10 -0
  96. package/dist/meta/backup/backup-blocklet-command.d.ts +50 -0
  97. package/dist/meta/backup/backup-blocklet-command.js +2 -0
  98. package/dist/meta/backup/index.d.ts +3 -0
  99. package/dist/meta/backup/index.js +19 -0
  100. package/dist/meta/backup/post-app-backup-command.d.ts +8 -0
  101. package/dist/meta/backup/post-app-backup-command.js +16 -0
  102. package/dist/meta/backup/put-app-backup-command.d.ts +7 -0
  103. package/dist/meta/backup/put-app-backup-command.js +18 -0
  104. package/dist/meta/incremental-backup/backup-blocklet-command.d.ts +50 -0
  105. package/dist/meta/incremental-backup/backup-blocklet-command.js +2 -0
  106. package/dist/meta/incremental-backup/index.d.ts +1 -0
  107. package/dist/meta/incremental-backup/index.js +17 -0
  108. package/dist/meta/index.d.ts +7 -0
  109. package/dist/meta/index.js +23 -0
  110. package/dist/meta/nft/index.d.ts +1 -0
  111. package/dist/meta/nft/index.js +17 -0
  112. package/dist/meta/nft/put-nft-object-command.d.ts +51 -0
  113. package/dist/meta/nft/put-nft-object-command.js +2 -0
  114. package/dist/meta/object/delete-object-command.d.ts +6 -0
  115. package/dist/meta/object/delete-object-command.js +2 -0
  116. package/dist/meta/object/get-object-command.d.ts +10 -0
  117. package/dist/meta/object/get-object-command.js +2 -0
  118. package/dist/meta/object/head-object-command.d.ts +13 -0
  119. package/dist/meta/object/head-object-command.js +2 -0
  120. package/dist/meta/object/index.d.ts +5 -0
  121. package/dist/meta/object/index.js +21 -0
  122. package/dist/meta/object/list-object-command.d.ts +6 -0
  123. package/dist/meta/object/list-object-command.js +2 -0
  124. package/dist/meta/object/list-objects-command.d.ts +8 -0
  125. package/dist/meta/object/list-objects-command.js +2 -0
  126. package/dist/meta/object/put-object-command.d.ts +29 -0
  127. package/dist/meta/object/put-object-command.js +2 -0
  128. package/dist/meta/restore/index.d.ts +3 -0
  129. package/dist/meta/restore/index.js +19 -0
  130. package/dist/meta/restore/post-app-restore-command.d.ts +8 -0
  131. package/dist/meta/restore/post-app-restore-command.js +13 -0
  132. package/dist/meta/restore/put-app-restore-command.d.ts +7 -0
  133. package/dist/meta/restore/put-app-restore-command.js +18 -0
  134. package/dist/meta/restore/restore-blocklet-command.d.ts +28 -0
  135. package/dist/meta/restore/restore-blocklet-command.js +2 -0
  136. package/dist/meta/spaces-client-options.d.ts +5 -0
  137. package/dist/meta/spaces-client-options.js +2 -0
  138. package/dist/meta/sync/incremental-sync-base-command.d.ts +7 -0
  139. package/dist/meta/sync/incremental-sync-base-command.js +2 -0
  140. package/dist/meta/sync/incremental-sync-push-command.d.ts +8 -0
  141. package/dist/meta/sync/incremental-sync-push-command.js +2 -0
  142. package/dist/meta/sync/index.d.ts +6 -0
  143. package/dist/meta/sync/index.js +22 -0
  144. package/dist/meta/sync/sync-base-command.d.ts +121 -0
  145. package/dist/meta/sync/sync-base-command.js +2 -0
  146. package/dist/meta/sync/sync-folder-base-command.d.ts +12 -0
  147. package/dist/meta/sync/sync-folder-base-command.js +2 -0
  148. package/dist/meta/sync/sync-folder-pull-command.d.ts +5 -0
  149. package/dist/meta/sync/sync-folder-pull-command.js +2 -0
  150. package/dist/meta/sync/sync-folder-push-command.d.ts +7 -0
  151. package/dist/meta/sync/sync-folder-push-command.js +2 -0
  152. package/dist/protocol/command.protocol.d.ts +68 -0
  153. package/dist/protocol/command.protocol.js +2 -0
  154. package/dist/protocol/index.d.ts +1 -0
  155. package/dist/protocol/index.js +17 -0
  156. package/dist/security/index.d.ts +1 -0
  157. package/dist/security/index.js +17 -0
  158. package/dist/security/request-signature.d.ts +32 -0
  159. package/dist/security/request-signature.js +140 -0
  160. package/package.json +78 -0
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.RestoreBlockletCommand = void 0;
4
+ const path_1 = require("path");
5
+ const core_1 = require("@did-space/core");
6
+ const base_1 = require("../base");
7
+ const sync_1 = require("../sync");
8
+ const post_app_restore_1 = require("./post-app-restore");
9
+ const put_app_restore_1 = require("./put-app-restore");
10
+ const libs_1 = require("../../libs");
11
+ const space_1 = require("../../libs/space");
12
+ /**
13
+ *
14
+ * FIXME: @yejianchao 优化点,需要支持 joi 校验
15
+ * @export
16
+ * @class RestoreBlockletCommand
17
+ * @extends {BaseCommand<RestoreBlockletCommandInput, RestoreBlockletCommandOutput>}
18
+ */
19
+ class RestoreBlockletCommand extends base_1.BaseCommand {
20
+ getAxiosRequestConfig() {
21
+ return null;
22
+ }
23
+ async resolveMiddleware() {
24
+ let appRestore;
25
+ try {
26
+ // @note: 备份还原的时候我们希望使用 service url 来完成,不希望走 CDN
27
+ this.context.endpoint = await (0, space_1.getSpaceServiceEndpoint)(this.context.endpoint);
28
+ appRestore = (await new post_app_restore_1.PostAppRestoreCommand({
29
+ userDid: this.input.userDid,
30
+ referrer: this.input.referrer,
31
+ serverDid: this.input.serverDid,
32
+ }, this.context).request()).data;
33
+ const res = await new sync_1.SyncFolderPullCommand({
34
+ source: (0, path_1.join)('.did-objects', this.input.appDid, '/'),
35
+ ...this.input,
36
+ }, this.context).request();
37
+ if (res.data.errorCount !== 0) {
38
+ await new put_app_restore_1.PutAppRestoreCommand({
39
+ id: appRestore.id,
40
+ status: core_1.APP_BACKUP_STATUS.FAILED,
41
+ message: res?.statusMessage,
42
+ }, this.context)
43
+ .request()
44
+ .catch(console.error);
45
+ return res;
46
+ }
47
+ // 仅仅是触发一下接口,更新一下 updateAt
48
+ await new put_app_restore_1.PutAppRestoreCommand({
49
+ id: appRestore.id,
50
+ status: core_1.APP_BACKUP_STATUS.SUCCEEDED,
51
+ }, this.context).request();
52
+ return res;
53
+ }
54
+ catch (err) {
55
+ console.error(err);
56
+ const error = err;
57
+ const message = (0, libs_1.getErrorMessage)(error);
58
+ if (appRestore) {
59
+ await new put_app_restore_1.PutAppRestoreCommand({
60
+ id: appRestore.id,
61
+ status: core_1.APP_BACKUP_STATUS.FAILED,
62
+ message,
63
+ }, this.context)
64
+ .request()
65
+ .catch(console.error);
66
+ }
67
+ return {
68
+ statusCode: (0, libs_1.getErrorStatusCode)(error),
69
+ statusMessage: message,
70
+ stack: error.stack,
71
+ data: {
72
+ errorCount: -1,
73
+ count: -1,
74
+ duration: -1,
75
+ },
76
+ };
77
+ }
78
+ }
79
+ }
80
+ exports.RestoreBlockletCommand = RestoreBlockletCommand;
@@ -0,0 +1,8 @@
1
+ import { AxiosRequestConfig } from 'axios';
2
+ import { BaseCommand } from '../base';
3
+ import type { HeadSpaceCommandInput, HeadSpaceCommandOutput } from '../../meta/object/head-object-command';
4
+ export declare class HeadSpaceCommand extends BaseCommand<HeadSpaceCommandInput, HeadSpaceCommandOutput> {
5
+ getAxiosRequestConfig(): Promise<AxiosRequestConfig<any>>;
6
+ private getUrl;
7
+ resolveMiddleware(): Promise<HeadSpaceCommandOutput>;
8
+ }
@@ -0,0 +1,52 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.HeadSpaceCommand = void 0;
4
+ const core_1 = require("@did-space/core");
5
+ const ufo_1 = require("ufo");
6
+ const libs_1 = require("../../libs");
7
+ const security_1 = require("../../security");
8
+ const base_1 = require("../base");
9
+ class HeadSpaceCommand extends base_1.BaseCommand {
10
+ // eslint-disable-next-line require-await
11
+ async getAxiosRequestConfig() {
12
+ return null;
13
+ }
14
+ getUrl() {
15
+ const array = this.context.endpoint.split('/');
16
+ const lastIndexOfApp = array.lastIndexOf('app');
17
+ const url = array.filter((_, i) => i < lastIndexOfApp).join('/');
18
+ return (0, ufo_1.withQuery)(url, {
19
+ ...this.input,
20
+ });
21
+ }
22
+ async resolveMiddleware() {
23
+ const url = this.getUrl();
24
+ const method = 'HEAD';
25
+ const { headers } = (0, security_1.signRequest)({
26
+ url,
27
+ method,
28
+ data: {},
29
+ headers: {
30
+ [core_1.HEADERS.X_SIGNATURE_VERSION]: '2',
31
+ },
32
+ wallet: this.context.wallet,
33
+ delegation: this.context.delegation,
34
+ });
35
+ const optionsOfTextResponseBody = {
36
+ method,
37
+ headers: {
38
+ ...headers,
39
+ },
40
+ throwHttpErrors: false,
41
+ };
42
+ // TODO: axios 会导致严重的内存泄漏, 之后可以考虑切换到 got, @see: https://github.com/axios/axios/issues/4763#issuecomment-1334953446
43
+ const response = await (0, libs_1.gotApi)(url, optionsOfTextResponseBody);
44
+ return {
45
+ statusCode: response.statusCode,
46
+ statusMessage: response.headers['x-message'] ?? response.statusMessage,
47
+ data: undefined,
48
+ response,
49
+ };
50
+ }
51
+ }
52
+ exports.HeadSpaceCommand = HeadSpaceCommand;
@@ -0,0 +1 @@
1
+ export * from './head';
@@ -0,0 +1,17 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./head"), exports);
@@ -0,0 +1,3 @@
1
+ export * from './sync-folder-base';
2
+ export * from './sync-folder-pull';
3
+ export * from './sync-folder-push';
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./sync-folder-base"), exports);
18
+ __exportStar(require("./sync-folder-pull"), exports);
19
+ __exportStar(require("./sync-folder-push"), exports);
@@ -0,0 +1,105 @@
1
+ import { Object } from '@did-space/core';
2
+ import { AxiosError } from 'axios';
3
+ import PQueue from 'p-queue';
4
+ import { ListObjectCommandOutput, OnAfterUploadInput, OnProgressInput, SyncFolderBaseCommandInput, SyncFolderBaseCommandOutput, SyncFolderPullCommandOutput } from '../../meta';
5
+ import { BaseCommand } from '../base';
6
+ import type { SpaceClientOptions } from '../../protocol';
7
+ export interface ObjectsMap {
8
+ [key: string]: ListObjectCommandOutput['data'];
9
+ }
10
+ export interface SyncObject extends Object {
11
+ action: 'PUT' | 'DELETE';
12
+ }
13
+ /**
14
+ * @see s3 sync https://stackoverflow.com/questions/27932345/downloading-folders-from-aws-s3-cp-or-sync
15
+ * @see Keep-alive 和连接重用的 NodeJS https://azureossd.github.io/2022/03/10/NodeJS-with-Keep-Alives-and-Connection-Reuse/index.html
16
+ * @export
17
+ * @class SyncFolderPushCommand
18
+ * @extends {BaseCommand<SyncFolderCommandInput, SyncFolderPushCommandOutput>}
19
+ */
20
+ export declare abstract class SyncFolderBaseCommand<CommandInput extends SyncFolderBaseCommandInput, CommandOutput extends SyncFolderBaseCommandOutput> extends BaseCommand<CommandInput, CommandOutput> {
21
+ protected readonly queue: PQueue;
22
+ protected readonly startTime: number;
23
+ static readonly DEFAULT_RETRY_COUNT: number;
24
+ static readonly MAX_RETRY_COUNT: number;
25
+ static readonly DEFAULT_CONCURRENCY: number;
26
+ static readonly MAX_CONCURRENCY: number;
27
+ private retryCount;
28
+ constructor(input: SyncFolderBaseCommandInput, context?: SpaceClientOptions);
29
+ abstract getLocalObjectsMap(): Promise<ObjectsMap>;
30
+ abstract getRemoteObjectsMap(): Promise<ObjectsMap>;
31
+ /**
32
+ *
33
+ * @description 仅在开启调试模式时,输出调试信息,
34
+ * @protected
35
+ * @param {{
36
+ * completedCount: number;
37
+ * errorCount: number;
38
+ * totalCount: number;
39
+ * }} {
40
+ * completedCount,
41
+ * errorCount,
42
+ * totalCount,
43
+ * }
44
+ * @return {*} {Promise<void>}
45
+ * @memberof SyncFolderBaseCommand
46
+ */
47
+ protected printDebugInfo({ direction, completedCount, errorCount, totalCount, }: {
48
+ direction: 'pull' | 'push';
49
+ completedCount: number;
50
+ errorCount: number;
51
+ totalCount: number;
52
+ }): Promise<void>;
53
+ /**
54
+ *
55
+ *
56
+ * @protected
57
+ * @param {(Error | AxiosError)} error
58
+ * @return {*} {Promise<void>}
59
+ * @memberof SyncFolderBaseCommand
60
+ */
61
+ protected retryRequest(error: Error | AxiosError): Promise<SyncFolderPullCommandOutput>;
62
+ /**
63
+ *
64
+ * @description 比较双方的文件到底谁更新鲜,更新鲜的数据才需要被同步上去
65
+ * @protected
66
+ * @param {ObjectsMap} fromObjectsMap
67
+ * @param {ObjectsMap} toObjectsMap
68
+ * @return {*} {Promise<string[]>}
69
+ * @memberof SyncFolderBaseCommand
70
+ */
71
+ protected getWaitSyncObjects(fromObjectsMap: ObjectsMap, toObjectsMap: ObjectsMap): Promise<SyncObject[]>;
72
+ /**
73
+ * @see https://github.com/ArcBlock/did-spaces/issues/955#issuecomment-1873723050
74
+ * @description 备份的对象随时可能在进行写入操作,所以对于需要备份的文件,最佳实践是备份它们某一个时刻的副本,否则会引发错误 Error: Unexpected end of form
75
+ * @param {SyncObject[]} waitSyncObjects
76
+ * @return {*} {Promise<SyncObject[]>}
77
+ * @memberof SyncFolderBaseCommand
78
+ */
79
+ copyObjects(waitSyncObjects: SyncObject[]): Promise<SyncObject[]>;
80
+ /**
81
+ * @description 是同一个对象吗?
82
+ * @param {string} source
83
+ * @param {string} target
84
+ * @return {*} {Promise<boolean>}
85
+ * @memberof SyncFolderBaseCommand
86
+ */
87
+ diffObject(source: string, target: string): Promise<{
88
+ same: boolean;
89
+ sourceObject: Object;
90
+ targetObject: Object;
91
+ }>;
92
+ getObject(absolutePath: string): Promise<Object>;
93
+ /**
94
+ * @FIXME: 重复的代码,需要重构一下接口
95
+ * @description 判断一个对象是否应该被备份
96
+ * @protected
97
+ * @param {string} absolutePath
98
+ * @return {*} {Promise<boolean>}
99
+ * @memberof IncrementalSyncBaseCommand
100
+ */
101
+ protected shouldBeBackup(absolutePath: string): Promise<boolean>;
102
+ protected triggerOnProgressHook(params: OnProgressInput): Promise<void>;
103
+ protected triggerOnAfterUploadHook(params: OnAfterUploadInput): Promise<void>;
104
+ destroy(): Promise<void>;
105
+ }
@@ -0,0 +1,319 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.SyncFolderBaseCommand = void 0;
7
+ /* eslint-disable @typescript-eslint/indent */
8
+ const core_1 = require("@did-space/core");
9
+ const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
10
+ const isUndefined_1 = __importDefault(require("lodash/isUndefined"));
11
+ const keys_1 = __importDefault(require("lodash/keys"));
12
+ const p_queue_1 = __importDefault(require("p-queue"));
13
+ const debug_1 = __importDefault(require("debug"));
14
+ const fs_extra_1 = require("fs-extra");
15
+ const p_all_1 = __importDefault(require("p-all"));
16
+ const cloneDeep_1 = __importDefault(require("lodash/cloneDeep"));
17
+ const path_1 = require("path");
18
+ const mime_types_1 = __importDefault(require("mime-types"));
19
+ const promises_1 = require("fs/promises");
20
+ const os_1 = require("os");
21
+ const libs_1 = require("../../libs");
22
+ const logger_1 = require("../../libs/logger");
23
+ const base_1 = require("../base");
24
+ const { name, version } = require('../../../package.json');
25
+ const debug = (0, debug_1.default)(`${name}@${version}:SyncFolderBaseCommand`);
26
+ /**
27
+ * @see s3 sync https://stackoverflow.com/questions/27932345/downloading-folders-from-aws-s3-cp-or-sync
28
+ * @see Keep-alive 和连接重用的 NodeJS https://azureossd.github.io/2022/03/10/NodeJS-with-Keep-Alives-and-Connection-Reuse/index.html
29
+ * @export
30
+ * @class SyncFolderPushCommand
31
+ * @extends {BaseCommand<SyncFolderCommandInput, SyncFolderPushCommandOutput>}
32
+ */
33
+ class SyncFolderBaseCommand extends base_1.BaseCommand {
34
+ queue;
35
+ startTime = Date.now();
36
+ static DEFAULT_RETRY_COUNT = 3;
37
+ static MAX_RETRY_COUNT = 10;
38
+ static DEFAULT_CONCURRENCY = 4;
39
+ static MAX_CONCURRENCY = (0, os_1.cpus)().length;
40
+ retryCount = 0;
41
+ constructor(input, context) {
42
+ // @ts-expect-error why?暂时没想通报错的原因
43
+ super(input, context);
44
+ if ((0, isUndefined_1.default)(input?.concurrency)) {
45
+ this.input.concurrency = SyncFolderBaseCommand.DEFAULT_CONCURRENCY;
46
+ }
47
+ else if (input.concurrency > SyncFolderBaseCommand.MAX_CONCURRENCY) {
48
+ this.input.concurrency = SyncFolderBaseCommand.MAX_CONCURRENCY;
49
+ }
50
+ else {
51
+ this.input.concurrency = input.concurrency;
52
+ }
53
+ if ((0, isUndefined_1.default)(input?.retryCount)) {
54
+ this.input.retryCount = SyncFolderBaseCommand.DEFAULT_RETRY_COUNT;
55
+ }
56
+ else if (input.retryCount > SyncFolderBaseCommand.MAX_RETRY_COUNT) {
57
+ this.input.retryCount = SyncFolderBaseCommand.MAX_RETRY_COUNT;
58
+ }
59
+ else {
60
+ this.input.retryCount = input.retryCount;
61
+ }
62
+ if (!this.input.source.endsWith('/')) {
63
+ this.input.source = (0, path_1.join)(this.input.source, '/');
64
+ }
65
+ if (!this.input.target.endsWith('/')) {
66
+ this.input.target = (0, path_1.join)(this.input.target, '/');
67
+ }
68
+ if ((0, isEmpty_1.default)(this.input.tmpDir)) {
69
+ this.input.tmpDir = (0, path_1.join)(process.env.BLOCKLET_DATA_DIR || process.env.ABT_NODE_DATA_DIR, 'tmp', this.constructor.name, this.input.target);
70
+ (0, fs_extra_1.ensureDirSync)(this.input.tmpDir);
71
+ }
72
+ this.queue = new p_queue_1.default({ concurrency: this.input.concurrency });
73
+ }
74
+ /**
75
+ *
76
+ * @description 仅在开启调试模式时,输出调试信息,
77
+ * @protected
78
+ * @param {{
79
+ * completedCount: number;
80
+ * errorCount: number;
81
+ * totalCount: number;
82
+ * }} {
83
+ * completedCount,
84
+ * errorCount,
85
+ * totalCount,
86
+ * }
87
+ * @return {*} {Promise<void>}
88
+ * @memberof SyncFolderBaseCommand
89
+ */
90
+ printDebugInfo({ direction, completedCount, errorCount, totalCount, }) {
91
+ if (!this.input.debug) {
92
+ return;
93
+ }
94
+ const remaining = totalCount - errorCount - completedCount;
95
+ const progress = ((completedCount * 100) / totalCount).toFixed(2);
96
+ const duration = (Date.now() - this.startTime) / 1000;
97
+ logger_1.logger.info(`[${direction}] pending: ${this.queue.pending}, completed: ${completedCount}, error: ${errorCount}, remaining: ${remaining}, progress: ${progress}%, total: ${totalCount}, retryCount: ${this.input.retryCount}, duration: ${duration}(s)`);
98
+ }
99
+ /**
100
+ *
101
+ *
102
+ * @protected
103
+ * @param {(Error | AxiosError)} error
104
+ * @return {*} {Promise<void>}
105
+ * @memberof SyncFolderBaseCommand
106
+ */
107
+ async retryRequest(error) {
108
+ // 减少重试次数
109
+ --this.input.retryCount;
110
+ // clear queue
111
+ this.queue.clear();
112
+ // 输出错误
113
+ const errorMessage = (0, libs_1.getErrorMessage)(error);
114
+ const statusCode = (0, libs_1.getErrorStatusCode)(error);
115
+ logger_1.logger.error('#retryRequest()', error);
116
+ logger_1.logger.error('#retryRequest()', {
117
+ input: this.input,
118
+ endpoint: this.context.endpoint,
119
+ errorMessage,
120
+ statusCode,
121
+ });
122
+ // FIXME: retryCount === 0 表示,重试次数刚好用完了,后续优化 server 显示错误的进度 @yejianchao
123
+ // 防呆控制,retryCount < 0 说明调用基础接口始终是失败的,遇到了预料之外的事情,因此需要停止运行
124
+ if (this.input.retryCount <= 0 || [400, 401, 403, 404, 413].includes(statusCode)) {
125
+ await this.destroy();
126
+ return {
127
+ statusCode,
128
+ statusMessage: (0, libs_1.getErrorMessage)(error),
129
+ stack: error.stack,
130
+ data: {
131
+ errorCount: -1,
132
+ count: -1,
133
+ duration: (Date.now() - this.startTime) / 1000,
134
+ },
135
+ };
136
+ }
137
+ // 目前我们的重试时间间隔是以 2 的指数增长的,第1次重试 1 s,第 2 次重试 2 s, 第 3 次重试间隔 4 s...
138
+ const retryDelayMs = 2 ** this.retryCount * 1000;
139
+ ++this.retryCount;
140
+ await (0, libs_1.sleep)(retryDelayMs);
141
+ return this.request();
142
+ }
143
+ /**
144
+ *
145
+ * @description 比较双方的文件到底谁更新鲜,更新鲜的数据才需要被同步上去
146
+ * @protected
147
+ * @param {ObjectsMap} fromObjectsMap
148
+ * @param {ObjectsMap} toObjectsMap
149
+ * @return {*} {Promise<string[]>}
150
+ * @memberof SyncFolderBaseCommand
151
+ */
152
+ async getWaitSyncObjects(fromObjectsMap, toObjectsMap) {
153
+ const intersectionKeys = [...new Set((0, keys_1.default)(fromObjectsMap).concat((0, keys_1.default)(toObjectsMap)))];
154
+ const actions = intersectionKeys.map((key) => {
155
+ return async () => {
156
+ const fromObject = fromObjectsMap[key];
157
+ const toObject = toObjectsMap[key];
158
+ debug(`getWaitSyncObjects.$fromObjectsMap[${key}]`, fromObject);
159
+ debug(`getWaitSyncObjects.$toObjectsMap[${key}]`, toObject);
160
+ // 本地存在的文件 && 在远程不存在, 则修改远程的
161
+ if (fromObject && !toObject) {
162
+ return {
163
+ ...fromObject,
164
+ action: 'PUT',
165
+ key,
166
+ };
167
+ }
168
+ // 本地不存在 && 远程存在, 则删除远程的
169
+ if (!fromObject && toObject) {
170
+ return {
171
+ ...toObject,
172
+ action: 'DELETE',
173
+ key,
174
+ };
175
+ }
176
+ // 本地存在的文件 && 在远程存在 && 两者文件大小不一致, 则修改远程的
177
+ if (fromObject.size !== toObject.size) {
178
+ return {
179
+ ...fromObject,
180
+ action: 'PUT',
181
+ key,
182
+ };
183
+ }
184
+ // 本地存在的文件 && 在远程存在 && 本地日期不新鲜,则无需修改
185
+ if (fromObject.lastModified <= toObject.lastModified) {
186
+ return null;
187
+ }
188
+ if (!fromObject.hash) {
189
+ if (!(await this.shouldBeBackup(fromObject.absolutePath))) {
190
+ return null;
191
+ }
192
+ // 此时计算 hash
193
+ fromObject.hash = await (0, core_1.getHash)((0, fs_extra_1.createReadStream)(fromObject.absolutePath));
194
+ }
195
+ // 本地存在的文件 && 在远程存在 && 两者文件内容不同, 则修改远程的
196
+ if (fromObject.hash !== toObject.hash) {
197
+ return {
198
+ ...fromObject,
199
+ action: 'PUT',
200
+ key,
201
+ };
202
+ }
203
+ return null;
204
+ };
205
+ });
206
+ const syncObjects = (await (0, p_all_1.default)(actions, { concurrency: this.input.concurrency })).filter(Boolean);
207
+ debug('getWaitSyncObjects.$syncObjects.length', syncObjects.length);
208
+ debug('getWaitSyncObjects.$syncObjects.slice(0, 3)', syncObjects.slice(0, 3));
209
+ return syncObjects;
210
+ }
211
+ /**
212
+ * @see https://github.com/ArcBlock/did-spaces/issues/955#issuecomment-1873723050
213
+ * @description 备份的对象随时可能在进行写入操作,所以对于需要备份的文件,最佳实践是备份它们某一个时刻的副本,否则会引发错误 Error: Unexpected end of form
214
+ * @param {SyncObject[]} waitSyncObjects
215
+ * @return {*} {Promise<SyncObject[]>}
216
+ * @memberof SyncFolderBaseCommand
217
+ */
218
+ async copyObjects(waitSyncObjects) {
219
+ const syncObjects = (0, cloneDeep_1.default)(waitSyncObjects);
220
+ await (0, p_all_1.default)(syncObjects.map((x) => {
221
+ return async () => {
222
+ const source = x.absolutePath;
223
+ const target = (0, path_1.join)(this.input.tmpDir, x.key);
224
+ if (x.action !== 'PUT' || !(await (0, fs_extra_1.pathExists)(source))) {
225
+ return;
226
+ }
227
+ if (!(await (0, fs_extra_1.pathExists)(target))) {
228
+ await (0, fs_extra_1.ensureDir)((0, path_1.dirname)(target));
229
+ await (0, promises_1.copyFile)(source, target);
230
+ }
231
+ const { same, targetObject } = await this.diffObject(source, target);
232
+ if (!same) {
233
+ x.size = targetObject.size;
234
+ x.lastModified = targetObject.lastModified;
235
+ x.hash = await (0, core_1.getHash)((0, fs_extra_1.createReadStream)(targetObject.absolutePath));
236
+ }
237
+ x.absolutePath = target;
238
+ };
239
+ }), {
240
+ concurrency: this.input.concurrency,
241
+ });
242
+ return syncObjects;
243
+ }
244
+ /**
245
+ * @description 是同一个对象吗?
246
+ * @param {string} source
247
+ * @param {string} target
248
+ * @return {*} {Promise<boolean>}
249
+ * @memberof SyncFolderBaseCommand
250
+ */
251
+ async diffObject(source, target) {
252
+ const sourceObject = await this.getObject(source);
253
+ const targetObject = await this.getObject(target);
254
+ const same = sourceObject.lastModified <= targetObject.lastModified && sourceObject.size === targetObject.size;
255
+ return {
256
+ same,
257
+ sourceObject,
258
+ targetObject,
259
+ };
260
+ }
261
+ async getObject(absolutePath) {
262
+ const fileStat = await (0, fs_extra_1.stat)(absolutePath);
263
+ const objectName = (0, path_1.basename)(absolutePath);
264
+ const directorySuffixes = fileStat.isDirectory() ? '/' : '';
265
+ return {
266
+ key: absolutePath,
267
+ name: (0, path_1.join)(objectName, directorySuffixes),
268
+ isDir: fileStat.isDirectory(),
269
+ size: fileStat.size,
270
+ lastModified: new Date(fileStat.mtime).getTime(),
271
+ editable: false,
272
+ mimeType: mime_types_1.default.lookup(objectName) || 'unknown',
273
+ absolutePath,
274
+ };
275
+ }
276
+ /**
277
+ * @FIXME: 重复的代码,需要重构一下接口
278
+ * @description 判断一个对象是否应该被备份
279
+ * @protected
280
+ * @param {string} absolutePath
281
+ * @return {*} {Promise<boolean>}
282
+ * @memberof IncrementalSyncBaseCommand
283
+ */
284
+ // eslint-disable-next-line require-await
285
+ async shouldBeBackup(absolutePath) {
286
+ if (typeof absolutePath !== 'string' || (0, isEmpty_1.default)(absolutePath)) {
287
+ return false;
288
+ }
289
+ if (absolutePath.endsWith('~')) {
290
+ // 临时文件不备份,因为随时可能被删除,从而引发错误
291
+ return false;
292
+ }
293
+ if (!(await (0, fs_extra_1.pathExists)(absolutePath))) {
294
+ // 不存在的文件不备份
295
+ return false;
296
+ }
297
+ return true;
298
+ }
299
+ // FIXME: @yejianchao 之后考虑使用事件驱动
300
+ async triggerOnProgressHook(params) {
301
+ if (typeof this.input.onProgress !== 'function') {
302
+ return;
303
+ }
304
+ await this.input.onProgress(params);
305
+ }
306
+ // FIXME: @yejianchao 之后考虑使用事件驱动
307
+ async triggerOnAfterUploadHook(params) {
308
+ if (typeof this.input.onAfterUpload !== 'function') {
309
+ return;
310
+ }
311
+ await this.input.onAfterUpload(params);
312
+ }
313
+ async destroy() {
314
+ if (await (0, fs_extra_1.pathExists)(this.input.tmpDir)) {
315
+ await (0, fs_extra_1.rm)(this.input.tmpDir, { recursive: true });
316
+ }
317
+ }
318
+ }
319
+ exports.SyncFolderBaseCommand = SyncFolderBaseCommand;
@@ -0,0 +1,13 @@
1
+ import { AxiosRequestConfig } from 'axios';
2
+ import { SyncFolderPullCommandInput, SyncFolderPullCommandOutput } from '../../meta';
3
+ import { ObjectsMap, SyncFolderBaseCommand, SyncObject } from './sync-folder-base';
4
+ import type { SpaceClientOptions } from '../../protocol';
5
+ export declare class SyncFolderPullCommand extends SyncFolderBaseCommand<SyncFolderPullCommandInput, SyncFolderPullCommandOutput> {
6
+ protected readonly startTime: number;
7
+ constructor(input: SyncFolderPullCommandInput, context?: SpaceClientOptions);
8
+ getAxiosRequestConfig(): Promise<AxiosRequestConfig<any>>;
9
+ request(): Promise<SyncFolderPullCommandOutput>;
10
+ getRemoteObjectsMap(): Promise<ObjectsMap>;
11
+ getLocalObjectsMap(): Promise<ObjectsMap>;
12
+ sync(syncObjects: SyncObject[]): Promise<SyncFolderPullCommandOutput>;
13
+ }