@abtnode/core 1.8.65-beta-81d3340c → 1.8.65-beta-f7af64a4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,6 @@
1
1
  /* eslint-disable no-underscore-dangle */
2
2
  /* eslint-disable no-await-in-loop */
3
3
  const fs = require('fs-extra');
4
- const { fileURLToPath } = require('url');
5
4
  const path = require('path');
6
5
  const flat = require('flat');
7
6
  const get = require('lodash/get');
@@ -17,14 +16,12 @@ const didDocument = require('@abtnode/util/lib/did-document');
17
16
  const { sign } = require('@arcblock/jwt');
18
17
  const { isValid: isValidDid } = require('@arcblock/did');
19
18
  const { verifyPresentation } = require('@arcblock/vc');
20
- const { toBase58 } = require('@ocap/util');
21
19
  const { toSvg: createDidLogo } =
22
20
  process.env.NODE_ENV !== 'test' ? require('@arcblock/did-motif') : require('@arcblock/did-motif/dist/did-motif.cjs');
23
21
  const getBlockletInfo = require('@blocklet/meta/lib/info');
22
+ const sleep = require('@abtnode/util/lib/sleep');
24
23
 
25
24
  const logger = require('@abtnode/logger')('@abtnode/core:blocklet:manager');
26
- const downloadFile = require('@abtnode/util/lib/download-file');
27
- const Lock = require('@abtnode/util/lib/lock');
28
25
  const { getVcFromPresentation } = require('@abtnode/util/lib/vc');
29
26
  const {
30
27
  VC_TYPE_BLOCKLET_PURCHASE,
@@ -45,17 +42,16 @@ const {
45
42
  forEachChildSync,
46
43
  forEachBlocklet,
47
44
  getComponentId,
48
- getComponentBundleId,
49
45
  isPreferenceKey,
50
46
  getRolesFromAuthConfig,
51
47
  } = require('@blocklet/meta/lib/util');
52
48
  const getComponentProcessId = require('@blocklet/meta/lib/get-component-process-id');
53
- const validateBlockletEntry = require('@blocklet/meta/lib/entry');
54
49
  const toBlockletDid = require('@blocklet/meta/lib/did');
55
50
  const { validateMeta } = require('@blocklet/meta/lib/validate');
56
51
  const { update: updateMetaFile } = require('@blocklet/meta/lib/file');
57
52
  const { titleSchema, mountPointSchema, environmentNameSchema } = require('@blocklet/meta/lib/schema');
58
53
  const hasReservedKey = require('@blocklet/meta/lib/has-reserved-key');
54
+ const Lock = require('@abtnode/util/lib/lock');
59
55
 
60
56
  const { toExternalBlocklet } = toBlockletDid;
61
57
 
@@ -63,7 +59,6 @@ const {
63
59
  BlockletStatus,
64
60
  BlockletSource,
65
61
  BlockletEvents,
66
- BLOCKLET_BUNDLE_FOLDER,
67
62
  BLOCKLET_MODES,
68
63
  BlockletGroup,
69
64
  fromBlockletStatus,
@@ -86,7 +81,6 @@ const {
86
81
  } = require('../../util/get-accessible-external-node-ip');
87
82
  const {
88
83
  getBlockletMetaFromUrl,
89
- ensureBlockletExpanded,
90
84
  getAppSystemEnvironments,
91
85
  getComponentSystemEnvironments,
92
86
  getAppOverwrittenEnvironments,
@@ -99,8 +93,6 @@ const {
99
93
  checkBlockletProcessHealthy,
100
94
  validateBlocklet,
101
95
  statusMap,
102
- expandTarball,
103
- verifyIntegrity,
104
96
  pruneBlockletBundle,
105
97
  getDiskInfo,
106
98
  getUpdateMetaList,
@@ -110,7 +102,6 @@ const {
110
102
  checkDuplicateComponents,
111
103
  getDiffFiles,
112
104
  getBundleDir,
113
- needBlockletDownload,
114
105
  findAvailableDid,
115
106
  ensureMeta,
116
107
  getBlocklet,
@@ -131,12 +122,13 @@ const runMigrationScripts = require('../migration');
131
122
  const hooks = require('../hooks');
132
123
  const { formatName, getDidDomainForBlocklet } = require('../../util/get-domain-for-blocklet');
133
124
  const handleInstanceInStore = require('../../util/public-to-store');
134
- const { getNFTState, getServerDidDomain } = require('../../util');
135
125
  const { BlockletRuntimeMonitor } = require('../../monitor/blocklet-runtime-monitor');
136
126
  const getHistoryList = require('../../monitor/get-history-list');
137
127
  const { SpacesBackup } = require('../storage/backup/spaces');
138
128
  const { SpacesRestore } = require('../storage/restore/spaces');
139
129
  const installFromBackup = require('./helper/install-from-backup');
130
+ const { resolveDownload, resolveDiffDownload } = require('../downloader/resolve-download');
131
+ const BlockletDownloader = require('../downloader/blocklet-downloader');
140
132
 
141
133
  const {
142
134
  isInProgress,
@@ -150,8 +142,6 @@ const {
150
142
 
151
143
  const statusLock = new Lock('blocklet-status-lock');
152
144
 
153
- const asyncFs = fs.promises;
154
-
155
145
  const pm2StatusMap = {
156
146
  online: BlockletStatus.running,
157
147
  stop: BlockletStatus.stopped,
@@ -211,15 +201,6 @@ class BlockletManager extends BaseBlockletManager {
211
201
  this.installQueue = installQueue;
212
202
  this.teamManager = teamManager;
213
203
 
214
- /**
215
- * { did: Map({ <childDid>: <downloadFile.cancelCtrl> }) }
216
- */
217
- this.downloadCtrls = {};
218
- /**
219
- * { [download-did-version]: Lock }
220
- */
221
- this.downloadLocks = {};
222
-
223
204
  // cached installed blocklets for performance
224
205
  this.cachedBlocklets = null;
225
206
 
@@ -231,6 +212,12 @@ class BlockletManager extends BaseBlockletManager {
231
212
 
232
213
  this.runtimeMonitor = new BlockletRuntimeMonitor({ historyLength: MONITOR_HISTORY_LENGTH, states });
233
214
 
215
+ this.blockletDownloader = new BlockletDownloader({
216
+ installDir: this.installDir,
217
+ downloadDir: this.dataDirs.tmp,
218
+ cache: states.cache,
219
+ });
220
+
234
221
  if (daemon) {
235
222
  blockletPm2Events.on('online', (data) => this._syncPm2Status('online', data.blockletDid));
236
223
  blockletPm2Events.on('stop', (data) => this._syncPm2Status('stop', data.blockletDid));
@@ -457,6 +444,7 @@ class BlockletManager extends BaseBlockletManager {
457
444
 
458
445
  async start({ did, throwOnError, checkHealthImmediately = false, e2eMode = false }, context) {
459
446
  logger.info('start blocklet', { did });
447
+ // should check blocklet integrity
460
448
  const blocklet = await this.ensureBlocklet(did, { e2eMode });
461
449
 
462
450
  try {
@@ -570,7 +558,7 @@ class BlockletManager extends BaseBlockletManager {
570
558
  async stop({ did, updateStatus = true, silent = false }, context) {
571
559
  logger.info('stop blocklet', { did });
572
560
 
573
- const blocklet = await this.ensureBlocklet(did);
561
+ const blocklet = await this.getBlocklet(did);
574
562
  const { processId } = blocklet.env;
575
563
 
576
564
  if (updateStatus) {
@@ -668,7 +656,7 @@ class BlockletManager extends BaseBlockletManager {
668
656
 
669
657
  // eslint-disable-next-line no-unused-vars
670
658
  async reload({ did }, context) {
671
- const blocklet = await this.ensureBlocklet(did);
659
+ const blocklet = await this.getBlocklet(did);
672
660
 
673
661
  await states.blocklet.setBlockletStatus(did, BlockletStatus.stopping);
674
662
  await reloadBlockletProcess(blocklet);
@@ -684,13 +672,12 @@ class BlockletManager extends BaseBlockletManager {
684
672
  logger.info('delete blocklet', { did, keepData });
685
673
 
686
674
  try {
687
- const blocklet = await this.ensureBlocklet(did, { validateEnv: false });
675
+ const blocklet = await this.getBlocklet(did);
688
676
  if (isDeletableBlocklet(blocklet) === false) {
689
677
  throw new Error('Blocklet is protected from accidental deletion');
690
678
  }
691
679
 
692
680
  const nodeEnvironments = await states.node.getEnvironments();
693
-
694
681
  await deleteBlockletProcess(blocklet, {
695
682
  preDelete: (b, { ancestors }) =>
696
683
  hooks.preUninstall(b.env.processId, {
@@ -713,31 +700,27 @@ class BlockletManager extends BaseBlockletManager {
713
700
  severity: 'success',
714
701
  });
715
702
  return doc;
716
- } catch (err) {
703
+ } catch (error) {
717
704
  // If we installed a corrupted blocklet accidentally, just cleanup the disk and state db
718
- if (err.code === 'BLOCKLET_CORRUPTED') {
719
- logger.info('blocklet is corrupted, will delete again', { did });
720
- const doc = await this._deleteBlocklet({ did, keepData, keepLogsDir, keepConfigs }, context);
721
-
722
- this._createNotification(doc.meta.did, {
723
- title: 'Blocklet Deleted',
724
- description: `Blocklet ${doc.meta.name}@${doc.meta.version} is deleted.`,
725
- entityType: 'blocklet',
726
- entityId: doc.meta.did,
727
- severity: 'success',
728
- });
705
+ logger.error('blocklet delete failed, will delete again', { did, error });
706
+ const doc = await this._deleteBlocklet({ did, keepData, keepLogsDir, keepConfigs }, context);
729
707
 
730
- return doc;
731
- }
708
+ this._createNotification(doc.meta.did, {
709
+ title: 'Blocklet Deleted',
710
+ description: `Blocklet ${doc.meta.name}@${doc.meta.version} is deleted.`,
711
+ entityType: 'blocklet',
712
+ entityId: doc.meta.did,
713
+ severity: 'success',
714
+ });
732
715
 
733
- throw err;
716
+ return doc;
734
717
  }
735
718
  }
736
719
 
737
720
  async reset({ did, childDid }, context = {}) {
738
721
  logger.info('reset blocklet', { did, childDid });
739
722
 
740
- const blocklet = await this.ensureBlocklet(did);
723
+ const blocklet = await this.getBlocklet(did);
741
724
 
742
725
  if (isInProgress(blocklet.status || blocklet.status === BlockletStatus.running)) {
743
726
  throw new Error('Cannot reset when blocklet is in progress');
@@ -787,7 +770,7 @@ class BlockletManager extends BaseBlockletManager {
787
770
  async deleteComponent({ did, rootDid, keepData, keepState }, context) {
788
771
  logger.info('delete blocklet component', { did, rootDid, keepData });
789
772
 
790
- const blocklet = await this.ensureBlocklet(rootDid, { validateEnv: false });
773
+ const blocklet = await this.getBlocklet(rootDid);
791
774
  const child = blocklet.children.find((x) => x.meta.did === did);
792
775
  if (!child) {
793
776
  throw new Error('Component does not exist');
@@ -841,7 +824,7 @@ class BlockletManager extends BaseBlockletManager {
841
824
  await states.blockletExtras.delConfigs([blocklet.meta.did, child.meta.did]);
842
825
  }
843
826
 
844
- const newBlocklet = await this.ensureBlocklet(rootDid);
827
+ const newBlocklet = await this.getBlocklet(rootDid);
845
828
  this.emit(BlockletEvents.upgraded, { blocklet: newBlocklet, context: { ...context, createAuditLog: false } }); // trigger router refresh
846
829
 
847
830
  this._createNotification(newBlocklet.meta.did, {
@@ -877,7 +860,7 @@ class BlockletManager extends BaseBlockletManager {
877
860
  }
878
861
 
879
862
  if (blocklet.status === BlockletStatus.downloading) {
880
- await this._cancelDownload(blocklet.meta, context);
863
+ await this.blockletDownloader.cancelDownload(blocklet.meta.did);
881
864
  } else if (blocklet.status === BlockletStatus.waiting) {
882
865
  await this._cancelWaiting(blocklet.meta, context);
883
866
  }
@@ -894,7 +877,7 @@ class BlockletManager extends BaseBlockletManager {
894
877
 
895
878
  // eslint-disable-next-line no-unused-vars
896
879
  async deleteProcess({ did }, context) {
897
- const blocklet = await this.ensureBlocklet(did);
880
+ const blocklet = await this.getBlocklet(did);
898
881
 
899
882
  logger.info('delete blocklet process', { did });
900
883
 
@@ -917,17 +900,17 @@ class BlockletManager extends BaseBlockletManager {
917
900
 
918
901
  if (!attachRuntimeInfo) {
919
902
  try {
920
- const blocklet = await this.ensureBlocklet(did, { throwOnNotExist: false });
903
+ const blocklet = await this.getBlocklet(did, { throwOnNotExist: false });
921
904
  return blocklet;
922
905
  } catch (e) {
923
- logger.error('get blocklet detail error', { error: e.message });
924
- return null;
906
+ logger.error('get blocklet detail error', { error: e });
907
+ return states.blocklet.getBlocklet(did);
925
908
  }
926
909
  }
927
910
 
928
911
  const nodeInfo = await states.node.read();
929
912
 
930
- return this.attachRuntimeInfo({ did, nodeInfo, diskInfo: true, context });
913
+ return this._attachRuntimeInfo({ did, nodeInfo, diskInfo: true, context });
931
914
  }
932
915
 
933
916
  async attachBlockletListRuntimeInfo({ blocklets, useCache }, context) {
@@ -942,7 +925,7 @@ class BlockletManager extends BaseBlockletManager {
942
925
  const cachedBlocklet =
943
926
  useCache && this.cachedBlocklets ? this.cachedBlocklets.find((y) => y.meta.did === x.meta.did) : null;
944
927
 
945
- return this.attachRuntimeInfo({
928
+ return this._attachRuntimeInfo({
946
929
  did: x.meta.did,
947
930
  nodeInfo,
948
931
  diskInfo: false,
@@ -990,7 +973,7 @@ class BlockletManager extends BaseBlockletManager {
990
973
  const [rootDid, ...childDids] = dids;
991
974
  logger.info('config blocklet', { dids });
992
975
 
993
- let blocklet = await this.ensureBlocklet(rootDid);
976
+ let blocklet = await this.getBlocklet(rootDid);
994
977
  for (const childDid of childDids) {
995
978
  blocklet = blocklet.children.find((x) => x.meta.did === childDid);
996
979
  if (!blocklet) {
@@ -1048,20 +1031,20 @@ class BlockletManager extends BaseBlockletManager {
1048
1031
  await this.updateBlockletEnvironment(rootDid);
1049
1032
 
1050
1033
  // response
1051
- const newState = await this.ensureBlocklet(rootDid);
1034
+ const newState = await this.getBlocklet(rootDid);
1052
1035
  this.emit(BlockletEvents.updated, newState);
1053
1036
  return newState;
1054
1037
  }
1055
1038
 
1056
1039
  async configPublicToStore({ did, publicToStore = false }) {
1057
- const blocklet = await this.ensureBlocklet(did);
1040
+ const blocklet = await this.getBlocklet(did);
1058
1041
  // publicToStore 由用户传入
1059
1042
  // handleInstanceInStore 方法写在前面,保证向 store 操作成功后才会更改 blocklet 中的 publicToStore值
1060
1043
  // handleInstanceInStore 中会校验修改 publicToStore字段 的条件,不符合则会抛错,就不会执行下面更新 publicToStore 的逻辑
1061
1044
  await handleInstanceInStore(blocklet, { publicToStore });
1062
1045
  await states.blockletExtras.setSettings(did, { publicToStore });
1063
1046
 
1064
- const newState = await this.ensureBlocklet(did);
1047
+ const newState = await this.getBlocklet(did);
1065
1048
  return newState;
1066
1049
  }
1067
1050
 
@@ -1071,7 +1054,7 @@ class BlockletManager extends BaseBlockletManager {
1071
1054
  }
1072
1055
  await states.blockletExtras.setSettings(did, { navigations });
1073
1056
 
1074
- const newState = await this.ensureBlocklet(did);
1057
+ const newState = await this.getBlocklet(did);
1075
1058
  this.emit(BlockletEvents.updated, newState);
1076
1059
  return newState;
1077
1060
  }
@@ -1123,7 +1106,7 @@ class BlockletManager extends BaseBlockletManager {
1123
1106
  await states.blockletExtras.setConfigs(dids, configs);
1124
1107
  }
1125
1108
 
1126
- const blocklet = await this.ensureBlocklet(rootDid);
1109
+ const blocklet = await this.getBlocklet(rootDid);
1127
1110
 
1128
1111
  this.emit(BlockletEvents.updated, { meta: { did: blocklet.meta.did } });
1129
1112
 
@@ -1171,7 +1154,7 @@ class BlockletManager extends BaseBlockletManager {
1171
1154
  // trigger dashboard frontend refresh
1172
1155
  this.emit(BlockletEvents.updated, blocklet);
1173
1156
 
1174
- return this.ensureBlocklet(rootDid);
1157
+ return this.getBlocklet(rootDid);
1175
1158
  }
1176
1159
 
1177
1160
  async updateComponentMountPoint({ did, rootDid: inputRootDid, mountPoint }, context) {
@@ -1207,7 +1190,7 @@ class BlockletManager extends BaseBlockletManager {
1207
1190
 
1208
1191
  this.emit(BlockletEvents.upgraded, { blocklet, context: { ...context, createAuditLog: false } }); // trigger router refresh
1209
1192
 
1210
- return this.ensureBlocklet(rootDid);
1193
+ return this.getBlocklet(rootDid);
1211
1194
  }
1212
1195
 
1213
1196
  /**
@@ -1549,11 +1532,12 @@ class BlockletManager extends BaseBlockletManager {
1549
1532
  });
1550
1533
  logger.info('add blocklet for dev', { did, version, meta });
1551
1534
 
1552
- const oldBlocklet = { children: children.filter((x) => x.dynamic) }; // let downloader skip re-downloading dynamic blocklet
1553
- await this._downloadBlocklet(added, oldBlocklet);
1535
+ await this._downloadBlocklet(added);
1554
1536
 
1555
1537
  // Add Config
1556
1538
  await this._setConfigsFromMeta(did);
1539
+
1540
+ // should ensure blocklet integrity
1557
1541
  let blocklet = await this.ensureBlocklet(did);
1558
1542
 
1559
1543
  // pre install
@@ -1561,14 +1545,14 @@ class BlockletManager extends BaseBlockletManager {
1561
1545
 
1562
1546
  // Add environments
1563
1547
  await this.updateBlockletEnvironment(did);
1564
- blocklet = await this.ensureBlocklet(did);
1548
+ blocklet = await this.getBlocklet(did);
1565
1549
 
1566
1550
  // post install
1567
1551
  await this._runPostInstallHook(blocklet);
1568
1552
 
1569
1553
  await states.blocklet.setBlockletStatus(did, BlockletStatus.installed);
1570
1554
 
1571
- blocklet = await this.ensureBlocklet(did);
1555
+ blocklet = await this.getBlocklet(did);
1572
1556
 
1573
1557
  await fs.writeFile(path.join(blocklet.env.dataDir, 'logo.svg'), createDidLogo(blocklet.meta.did));
1574
1558
 
@@ -1614,13 +1598,15 @@ class BlockletManager extends BaseBlockletManager {
1614
1598
  await states.blocklet.addChildren(rootDid, [component]);
1615
1599
 
1616
1600
  const newBlocklet = await states.blocklet.getBlocklet(rootDid);
1617
- await this._downloadBlocklet(newBlocklet, existRoot);
1601
+ await this._downloadBlocklet(newBlocklet);
1618
1602
  await states.blocklet.setBlockletStatus(rootDid, BlockletStatus.stopped);
1619
1603
 
1620
1604
  await this._upsertDynamicNavigation(existRoot.meta.did, component, { skipNavigation });
1621
1605
 
1622
1606
  // Add Config
1623
1607
  await this._setConfigsFromMeta(rootDid);
1608
+
1609
+ // should ensure blocklet integrity
1624
1610
  let blocklet = await this.ensureBlocklet(rootDid);
1625
1611
 
1626
1612
  // pre install
@@ -1628,14 +1614,14 @@ class BlockletManager extends BaseBlockletManager {
1628
1614
 
1629
1615
  // Add environments
1630
1616
  await this.updateBlockletEnvironment(rootDid);
1631
- blocklet = await this.ensureBlocklet(rootDid);
1617
+ blocklet = await this.getBlocklet(rootDid);
1632
1618
 
1633
1619
  // post install
1634
1620
  await this._runPostInstallHook(blocklet);
1635
1621
 
1636
1622
  logger.info('add blocklet component for dev', { did, version, meta });
1637
1623
 
1638
- blocklet = await this.ensureBlocklet(rootDid);
1624
+ blocklet = await this.getBlocklet(rootDid);
1639
1625
 
1640
1626
  return blocklet;
1641
1627
  }
@@ -1659,7 +1645,11 @@ class BlockletManager extends BaseBlockletManager {
1659
1645
  }
1660
1646
 
1661
1647
  async ensureBlocklet(did, opts = {}) {
1662
- return getBlocklet({ ...opts, states, dataDirs: this.dataDirs, did });
1648
+ return getBlocklet({ ...opts, states, dataDirs: this.dataDirs, did, ensureIntegrity: true });
1649
+ }
1650
+
1651
+ async getBlocklet(did, opts = {}) {
1652
+ return getBlocklet({ ...opts, states, dataDirs: this.dataDirs, did, ensureIntegrity: false });
1663
1653
  }
1664
1654
 
1665
1655
  async hasBlocklet({ did }) {
@@ -1677,7 +1667,7 @@ class BlockletManager extends BaseBlockletManager {
1677
1667
 
1678
1668
  this.emit(BlockletEvents.updated, { meta: { did: blocklet.meta.did } });
1679
1669
 
1680
- return this.ensureBlocklet(did);
1670
+ return this.getBlocklet(did);
1681
1671
  }
1682
1672
 
1683
1673
  async status(did, { forceSync = false } = {}) {
@@ -1694,7 +1684,7 @@ class BlockletManager extends BaseBlockletManager {
1694
1684
  return res;
1695
1685
  };
1696
1686
 
1697
- const blocklet = await this.ensureBlocklet(did);
1687
+ const blocklet = await this.getBlocklet(did);
1698
1688
 
1699
1689
  let shouldUpdateStatus = forceSync || shouldUpdateBlockletStatus(blocklet.status);
1700
1690
  if (isInProgress(blocklet.status)) {
@@ -1723,80 +1713,6 @@ class BlockletManager extends BaseBlockletManager {
1723
1713
  }
1724
1714
  }
1725
1715
 
1726
- async attachRuntimeInfo({ did, nodeInfo, diskInfo = true, context, cachedBlocklet }) {
1727
- if (!did) {
1728
- throw new Error('did should not be empty');
1729
- }
1730
-
1731
- try {
1732
- const blocklet = await this.ensureBlocklet(did, { throwOnNotExist: false });
1733
-
1734
- if (!blocklet) {
1735
- return null;
1736
- }
1737
-
1738
- const fromCache = !!cachedBlocklet;
1739
-
1740
- // if from cached data, only use cache data of runtime info (engine, diskInfo, runtimeInfo...)
1741
- if (fromCache) {
1742
- const cached = {};
1743
- forEachBlockletSync(cachedBlocklet, (component, { id }) => {
1744
- cached[id] = component;
1745
- });
1746
-
1747
- Object.assign(blocklet, pick(cachedBlocklet, ['appRuntimeInfo', 'diskInfo']));
1748
-
1749
- forEachBlockletSync(blocklet, (component, { id }) => {
1750
- if (cached[id]) {
1751
- Object.assign(component, pick(cached[id], ['runtimeInfo']));
1752
- }
1753
- });
1754
- }
1755
-
1756
- // 处理 domainAliases#value SLOT_FOR_IP_DNS_SITE
1757
- if (blocklet?.site?.domainAliases?.length) {
1758
- const nodeIp = await getAccessibleExternalNodeIp(nodeInfo);
1759
- blocklet.site.domainAliases = blocklet.site.domainAliases.map((x) => ({
1760
- ...x,
1761
- value: util.replaceDomainSlot({ domain: x.value, context, nodeIp }),
1762
- }));
1763
- }
1764
-
1765
- // app runtime info, app status
1766
- blocklet.appRuntimeInfo = this.runtimeMonitor.getRuntimeInfo(blocklet.meta.did);
1767
-
1768
- if (!fromCache) {
1769
- // app disk info, component runtime info, component status, component engine
1770
- await forEachBlocklet(blocklet, async (component, { level }) => {
1771
- component.engine = getEngine(getBlockletEngineNameByPlatform(component.meta)).describe();
1772
-
1773
- if (level === 0) {
1774
- component.diskInfo = await getDiskInfo(component, {
1775
- useFakeDiskInfo: !diskInfo,
1776
- });
1777
- }
1778
-
1779
- component.runtimeInfo = this.runtimeMonitor.getRuntimeInfo(blocklet.meta.did, component.env.id);
1780
-
1781
- if (component.runtimeInfo?.status && shouldUpdateBlockletStatus(component.status)) {
1782
- component.status = statusMap[component.runtimeInfo.status];
1783
- }
1784
- });
1785
- }
1786
-
1787
- return blocklet;
1788
- } catch (err) {
1789
- const simpleState = await states.blocklet.getBlocklet(did);
1790
- logger.error('failed to get blocklet info', {
1791
- did,
1792
- name: get(simpleState, 'meta.name'),
1793
- status: get(simpleState, 'status'),
1794
- error: err,
1795
- });
1796
- return simpleState;
1797
- }
1798
- }
1799
-
1800
1716
  async refreshListCache() {
1801
1717
  this.list({ useCache: false }).catch((err) => {
1802
1718
  logger.error('refresh blocklet list failed', { error: err });
@@ -1862,7 +1778,7 @@ class BlockletManager extends BaseBlockletManager {
1862
1778
 
1863
1779
  // download bundle
1864
1780
  try {
1865
- const { isCancelled } = await this._downloadBlocklet(blocklet, oldBlocklet, context);
1781
+ const { isCancelled } = await this._downloadBlocklet(blocklet, context);
1866
1782
 
1867
1783
  if (isCancelled) {
1868
1784
  logger.info('Download was canceled', { name, did, version });
@@ -1995,7 +1911,7 @@ class BlockletManager extends BaseBlockletManager {
1995
1911
 
1996
1912
  async onCheckIfStarted(jobInfo, { throwOnError } = {}) {
1997
1913
  const { did, context, minConsecutiveTime = 5000, timeout } = jobInfo;
1998
- const blocklet = await this.ensureBlocklet(did);
1914
+ const blocklet = await this.getBlocklet(did);
1999
1915
 
2000
1916
  const { meta } = blocklet;
2001
1917
  const { name } = meta;
@@ -2036,7 +1952,7 @@ class BlockletManager extends BaseBlockletManager {
2036
1952
  }
2037
1953
 
2038
1954
  async updateBlockletEnvironment(did) {
2039
- const blockletWithEnv = await this.ensureBlocklet(did);
1955
+ const blockletWithEnv = await this.getBlocklet(did);
2040
1956
  const blocklet = await states.blocklet.getBlocklet(did);
2041
1957
  const nodeInfo = await states.node.read();
2042
1958
 
@@ -2388,7 +2304,7 @@ class BlockletManager extends BaseBlockletManager {
2388
2304
 
2389
2305
  async _installFromUpload({ file, did, diffVersion, deleteSet, context }) {
2390
2306
  logger.info('install blocklet', { from: 'upload file' });
2391
- const { cwd, tarFile } = await this._downloadFromUpload(file);
2307
+ const { tarFile } = await this._downloadFromUpload(file);
2392
2308
 
2393
2309
  // diff deploy
2394
2310
  if (did && diffVersion) {
@@ -2410,14 +2326,17 @@ class BlockletManager extends BaseBlockletManager {
2410
2326
  throw new Error(`Can not deploy blocklet when it is ${fromBlockletStatus(oldBlocklet.status)}`);
2411
2327
  }
2412
2328
 
2413
- const { meta } = await this._resolveDiffDownload(cwd, tarFile, deleteSet, oldBlocklet.meta);
2329
+ const { meta } = await resolveDiffDownload(tarFile, this.installDir, {
2330
+ deleteSet,
2331
+ meta: oldBlocklet.meta,
2332
+ });
2414
2333
  const newBlocklet = await states.blocklet.getBlocklet(did);
2415
2334
  newBlocklet.meta = ensureMeta(meta);
2416
2335
  newBlocklet.source = BlockletSource.upload;
2417
2336
  newBlocklet.deployedFrom = `Upload by ${context.user.fullName}`;
2418
2337
  newBlocklet.children = await this._getChildrenForInstallation(meta);
2419
2338
  await validateBlocklet(newBlocklet);
2420
- await this._downloadBlocklet(newBlocklet, oldBlocklet);
2339
+ await this._downloadBlocklet(newBlocklet);
2421
2340
 
2422
2341
  return this._upgradeBlocklet({
2423
2342
  oldBlocklet,
@@ -2427,7 +2346,7 @@ class BlockletManager extends BaseBlockletManager {
2427
2346
  }
2428
2347
 
2429
2348
  // full deploy
2430
- const { meta } = await this._resolveDownload(cwd, tarFile);
2349
+ const { meta } = await resolveDownload(tarFile, this.installDir);
2431
2350
  const oldBlocklet = await this._getBlockletForInstallation(meta.did);
2432
2351
 
2433
2352
  // full deploy - upgrade
@@ -2444,7 +2363,7 @@ class BlockletManager extends BaseBlockletManager {
2444
2363
  newBlocklet.children = await this._getChildrenForInstallation(meta);
2445
2364
 
2446
2365
  await validateBlocklet(newBlocklet);
2447
- await this._downloadBlocklet(newBlocklet, oldBlocklet);
2366
+ await this._downloadBlocklet(newBlocklet);
2448
2367
 
2449
2368
  return this._upgradeBlocklet({
2450
2369
  oldBlocklet,
@@ -2472,11 +2391,7 @@ class BlockletManager extends BaseBlockletManager {
2472
2391
  await checkDuplicateAppSk({ did: meta.did, states });
2473
2392
 
2474
2393
  // download
2475
- await this._downloadBlocklet(
2476
- blocklet,
2477
- // let downloader skip re-downloading dynamic blocklet
2478
- { children: children.filter((x) => x.dynamic) }
2479
- );
2394
+ await this._downloadBlocklet(blocklet);
2480
2395
 
2481
2396
  return this._installBlocklet({
2482
2397
  did: meta.did,
@@ -2501,7 +2416,7 @@ class BlockletManager extends BaseBlockletManager {
2501
2416
  }) {
2502
2417
  logger.info('install blocklet', { from: 'upload file' });
2503
2418
  // download
2504
- const { cwd, tarFile } = await this._downloadFromUpload(file);
2419
+ const { tarFile } = await this._downloadFromUpload(file);
2505
2420
 
2506
2421
  const oldBlocklet = await this._getBlockletForInstallation(rootDid);
2507
2422
  if (!oldBlocklet) {
@@ -2530,10 +2445,10 @@ class BlockletManager extends BaseBlockletManager {
2530
2445
  throw new Error('Blocklet version changed when diff deploying');
2531
2446
  }
2532
2447
 
2533
- meta = (await this._resolveDiffDownload(cwd, tarFile, deleteSet, oldChild.meta)).meta;
2448
+ meta = (await resolveDiffDownload(tarFile, this.installDir, { deleteSet, meta: oldChild.meta })).meta;
2534
2449
  } else {
2535
2450
  // full deploy
2536
- meta = (await this._resolveDownload(cwd, tarFile)).meta;
2451
+ meta = (await resolveDownload(tarFile, this.installDir)).meta;
2537
2452
  }
2538
2453
 
2539
2454
  if (meta.did === rootDid) {
@@ -2567,7 +2482,7 @@ class BlockletManager extends BaseBlockletManager {
2567
2482
 
2568
2483
  await this._upsertDynamicNavigation(newBlocklet.meta.did, newChild, { skipNavigation });
2569
2484
 
2570
- await this._downloadBlocklet(newBlocklet, oldBlocklet);
2485
+ await this._downloadBlocklet(newBlocklet);
2571
2486
 
2572
2487
  await validateBlocklet(newBlocklet);
2573
2488
 
@@ -2649,6 +2564,7 @@ class BlockletManager extends BaseBlockletManager {
2649
2564
  {
2650
2565
  name: 'delete-expired-external-blocklet',
2651
2566
  time: '0 */30 * * * *', // 30min
2567
+ options: { runOnInit: false },
2652
2568
  fn: () => this._deleteExpiredExternalBlocklet(),
2653
2569
  },
2654
2570
  {
@@ -2664,6 +2580,80 @@ class BlockletManager extends BaseBlockletManager {
2664
2580
  ];
2665
2581
  }
2666
2582
 
2583
+ async _attachRuntimeInfo({ did, nodeInfo, diskInfo = true, context, cachedBlocklet }) {
2584
+ if (!did) {
2585
+ throw new Error('did should not be empty');
2586
+ }
2587
+
2588
+ try {
2589
+ const blocklet = await this.getBlocklet(did, { throwOnNotExist: false });
2590
+
2591
+ if (!blocklet) {
2592
+ return null;
2593
+ }
2594
+
2595
+ const fromCache = !!cachedBlocklet;
2596
+
2597
+ // if from cached data, only use cache data of runtime info (engine, diskInfo, runtimeInfo...)
2598
+ if (fromCache) {
2599
+ const cached = {};
2600
+ forEachBlockletSync(cachedBlocklet, (component, { id }) => {
2601
+ cached[id] = component;
2602
+ });
2603
+
2604
+ Object.assign(blocklet, pick(cachedBlocklet, ['appRuntimeInfo', 'diskInfo']));
2605
+
2606
+ forEachBlockletSync(blocklet, (component, { id }) => {
2607
+ if (cached[id]) {
2608
+ Object.assign(component, pick(cached[id], ['runtimeInfo']));
2609
+ }
2610
+ });
2611
+ }
2612
+
2613
+ // 处理 domainAliases#value SLOT_FOR_IP_DNS_SITE
2614
+ if (blocklet?.site?.domainAliases?.length) {
2615
+ const nodeIp = await getAccessibleExternalNodeIp(nodeInfo);
2616
+ blocklet.site.domainAliases = blocklet.site.domainAliases.map((x) => ({
2617
+ ...x,
2618
+ value: util.replaceDomainSlot({ domain: x.value, context, nodeIp }),
2619
+ }));
2620
+ }
2621
+
2622
+ // app runtime info, app status
2623
+ blocklet.appRuntimeInfo = this.runtimeMonitor.getRuntimeInfo(blocklet.meta.did);
2624
+
2625
+ if (!fromCache) {
2626
+ // app disk info, component runtime info, component status, component engine
2627
+ await forEachBlocklet(blocklet, async (component, { level }) => {
2628
+ component.engine = getEngine(getBlockletEngineNameByPlatform(component.meta)).describe();
2629
+
2630
+ if (level === 0) {
2631
+ component.diskInfo = await getDiskInfo(component, {
2632
+ useFakeDiskInfo: !diskInfo,
2633
+ });
2634
+ }
2635
+
2636
+ component.runtimeInfo = this.runtimeMonitor.getRuntimeInfo(blocklet.meta.did, component.env.id);
2637
+
2638
+ if (component.runtimeInfo?.status && shouldUpdateBlockletStatus(component.status)) {
2639
+ component.status = statusMap[component.runtimeInfo.status];
2640
+ }
2641
+ });
2642
+ }
2643
+
2644
+ return blocklet;
2645
+ } catch (err) {
2646
+ const simpleState = await states.blocklet.getBlocklet(did);
2647
+ logger.error('failed to get blocklet info', {
2648
+ did,
2649
+ name: get(simpleState, 'meta.name'),
2650
+ status: get(simpleState, 'status'),
2651
+ error: err,
2652
+ });
2653
+ return simpleState;
2654
+ }
2655
+ }
2656
+
2667
2657
  async _syncBlockletStatus() {
2668
2658
  const run = async (blocklet) => {
2669
2659
  try {
@@ -2906,143 +2896,13 @@ class BlockletManager extends BaseBlockletManager {
2906
2896
  return newBlocklet;
2907
2897
  }
2908
2898
 
2909
- /**
2910
- * decompress file, format dir and move to installDir
2911
- * @param {string} cwd
2912
- * @param {string} tarFile
2913
- * @param {object} originalMeta for verification
2914
- * @param {object} option
2915
- */
2916
- async _resolveDownload(cwd, tarFile, originalMeta, { removeTarFile = true } = {}) {
2917
- const downloadDir = path.join(cwd, `${path.basename(tarFile, path.extname(tarFile))}`);
2918
- const tmp = `${downloadDir}-tmp`;
2919
- try {
2920
- await expandTarball({ source: tarFile, dest: tmp, strip: 0 });
2921
- } catch (error) {
2922
- logger.error('expand blocklet tar file error');
2923
- throw error;
2924
- } finally {
2925
- if (removeTarFile) {
2926
- fs.removeSync(tarFile);
2927
- }
2928
- }
2929
- let installDir;
2930
- let meta;
2931
- try {
2932
- // resolve dir
2933
- let dir = tmp;
2934
- const files = await asyncFs.readdir(dir);
2935
- if (files.includes('package')) {
2936
- dir = path.join(tmp, 'package');
2937
- } else if (files.length === 1) {
2938
- const d = path.join(dir, files[0]);
2939
- if ((await asyncFs.stat(d)).isDirectory()) {
2940
- dir = d;
2941
- }
2942
- }
2943
-
2944
- if (fs.existsSync(path.join(dir, BLOCKLET_BUNDLE_FOLDER))) {
2945
- dir = path.join(dir, BLOCKLET_BUNDLE_FOLDER);
2946
- }
2947
-
2948
- logger.info('Move downloadDir to installDir', { downloadDir });
2949
- await fs.move(dir, downloadDir, { overwrite: true });
2950
- fs.removeSync(tmp);
2951
-
2952
- meta = getBlockletMeta(downloadDir, { ensureMain: true });
2953
- const { did, name, version } = meta;
2954
-
2955
- // validate
2956
- if (
2957
- originalMeta &&
2958
- (originalMeta.did !== did || originalMeta.name !== name || originalMeta.version !== version)
2959
- ) {
2960
- logger.error('Meta has differences', { originalMeta, meta });
2961
- throw new Error('There are differences between the meta from tarball file and the original meta');
2962
- }
2963
- await validateBlockletEntry(downloadDir, meta);
2964
-
2965
- await ensureBlockletExpanded(meta, downloadDir);
2966
-
2967
- installDir = getBundleDir(this.installDir, meta);
2968
- if (fs.existsSync(installDir)) {
2969
- fs.removeSync(installDir);
2970
- logger.info('cleanup blocklet upgrade dir', { name, version, installDir });
2971
- }
2972
-
2973
- fs.mkdirSync(installDir, { recursive: true });
2974
-
2975
- await fs.move(downloadDir, installDir, { overwrite: true });
2976
- } catch (error) {
2977
- fs.removeSync(downloadDir);
2978
- fs.removeSync(tmp);
2979
- throw error;
2980
- }
2981
-
2982
- return { meta, installDir };
2983
- }
2984
-
2985
- async _resolveDiffDownload(cwd, tarFile, deleteSet, bundle) {
2986
- logger.info('Resolve diff download', { tarFile, cwd });
2987
- const downloadDir = path.join(cwd, `${path.basename(tarFile, path.extname(tarFile))}`);
2988
- const diffDir = `${downloadDir}-diff`;
2989
- try {
2990
- await expandTarball({ source: tarFile, dest: diffDir, strip: 0 });
2991
- fs.removeSync(tarFile);
2992
- } catch (error) {
2993
- fs.removeSync(tarFile);
2994
- logger.error('expand blocklet tar file error');
2995
- throw error;
2996
- }
2997
- logger.info('Copy installDir to downloadDir', { installDir: this.installDir, downloadDir });
2998
- await fs.copy(getBundleDir(this.installDir, bundle), downloadDir);
2999
- try {
3000
- // delete
3001
- logger.info('Delete files from downloadDir', { fileNum: deleteSet.length });
3002
- // eslint-disable-next-line no-restricted-syntax
3003
- for (const file of deleteSet) {
3004
- await fs.remove(path.join(downloadDir, file));
3005
- }
3006
- // walk & cover
3007
- logger.info('Move files from diffDir to downloadDir', { diffDir, downloadDir });
3008
- const walkDiff = async (dir) => {
3009
- const files = await asyncFs.readdir(dir);
3010
- // eslint-disable-next-line no-restricted-syntax
3011
- for (const file of files) {
3012
- const p = path.join(dir, file);
3013
- const stat = await asyncFs.stat(p);
3014
- if (stat.isDirectory()) {
3015
- await walkDiff(p);
3016
- } else if (stat.isFile()) {
3017
- await fs.move(p, path.join(downloadDir, path.relative(diffDir, p)), { overwrite: true });
3018
- }
3019
- }
3020
- };
3021
- await walkDiff(diffDir);
3022
- fs.removeSync(diffDir);
3023
- const meta = getBlockletMeta(downloadDir, { ensureMain: true });
3024
-
3025
- await ensureBlockletExpanded(meta, downloadDir);
3026
-
3027
- // move to installDir
3028
- const bundleDir = getBundleDir(this.installDir, meta);
3029
- logger.info('Move downloadDir to installDir', { downloadDir, bundleDir });
3030
- await fs.move(downloadDir, bundleDir, { overwrite: true });
3031
-
3032
- return { meta, installDir: bundleDir };
3033
- } catch (error) {
3034
- fs.removeSync(downloadDir);
3035
- fs.removeSync(diffDir);
3036
- throw error;
3037
- }
3038
- }
3039
-
3040
2899
  /**
3041
2900
  * @param {string} opt.did
3042
2901
  * @param {object} opt.context
3043
2902
  */
3044
2903
  async _installBlocklet({ did, oldBlocklet, context }) {
3045
2904
  try {
2905
+ // should ensure blocklet integrity
3046
2906
  let blocklet = await this.ensureBlocklet(did);
3047
2907
  const { meta, source, deployedFrom } = blocklet;
3048
2908
 
@@ -3059,13 +2919,13 @@ class BlockletManager extends BaseBlockletManager {
3059
2919
 
3060
2920
  // Add environments
3061
2921
  await this.updateBlockletEnvironment(meta.did);
3062
- blocklet = await this.ensureBlocklet(did);
2922
+ blocklet = await this.getBlocklet(did);
3063
2923
 
3064
2924
  // post install
3065
2925
  await this._runPostInstallHook(blocklet, context);
3066
2926
 
3067
2927
  await states.blocklet.setBlockletStatus(did, BlockletStatus.installed);
3068
- blocklet = await this.ensureBlocklet(did);
2928
+ blocklet = await this.getBlocklet(did);
3069
2929
  logger.info('blocklet installed', { source, did: meta.did });
3070
2930
 
3071
2931
  await fs.writeFile(path.join(blocklet.env.dataDir, 'logo.svg'), createDidLogo(blocklet.meta.did));
@@ -3150,6 +3010,7 @@ class BlockletManager extends BaseBlockletManager {
3150
3010
  await states.blocklet.upgradeBlocklet({ meta, source, deployedFrom, children });
3151
3011
  await this._setConfigsFromMeta(did);
3152
3012
 
3013
+ // should ensure blocklet integrity
3153
3014
  let blocklet = await this.ensureBlocklet(did);
3154
3015
 
3155
3016
  // pre install
@@ -3157,7 +3018,7 @@ class BlockletManager extends BaseBlockletManager {
3157
3018
 
3158
3019
  // Add environments
3159
3020
  await this.updateBlockletEnvironment(did);
3160
- blocklet = await this.ensureBlocklet(did);
3021
+ blocklet = await this.getBlocklet(did);
3161
3022
 
3162
3023
  // post install
3163
3024
  await this._runPostInstallHook(blocklet, context);
@@ -3197,7 +3058,7 @@ class BlockletManager extends BaseBlockletManager {
3197
3058
  logger.info('started blocklet for upgrading', { did, version });
3198
3059
  }
3199
3060
 
3200
- blocklet = await this.ensureBlocklet(did, context);
3061
+ blocklet = await this.getBlocklet(did, context);
3201
3062
 
3202
3063
  await fs.writeFile(path.join(blocklet.env.dataDir, 'logo.svg'), createDidLogo(blocklet.meta.did));
3203
3064
 
@@ -3267,275 +3128,33 @@ class BlockletManager extends BaseBlockletManager {
3267
3128
  await states.blockletExtras.setSettings(did, { children: dynamicChildren });
3268
3129
  }
3269
3130
 
3270
- /**
3271
- *
3272
- *
3273
- * @param {{
3274
- * url: string,
3275
- * cwd: string,
3276
- * tarball: string,
3277
- * did: string,
3278
- * integrity: string,
3279
- * verify: boolean = true,
3280
- * ctrlStore: {},
3281
- * rootDid: string,
3282
- * context: {} = {},
3283
- * }} { url, cwd, tarball, did, integrity, verify = true, ctrlStore = {}, rootDid, context = {} }
3284
- * @return {*}
3285
- * @memberof BlockletManager
3286
- */
3287
- async _downloadTarball({ url, cwd, tarball, did, integrity, verify = true, ctrlStore = {}, rootDid, context = {} }) {
3288
- fs.mkdirSync(cwd, { recursive: true });
3289
-
3290
- const tarballName = url.split('/').slice(-1)[0];
3291
-
3292
- const tarballPath = path.join(cwd, tarballName);
3293
-
3294
- const { protocol } = new URL(url);
3295
-
3296
- const cachedTarFile = await this._getCachedTarFile(integrity);
3297
- if (cachedTarFile) {
3298
- logger.info('found cache tarFile', { did, tarballName, integrity });
3299
- await fs.move(cachedTarFile, tarballPath, { overwrite: true });
3300
- } else if (protocol.startsWith('file')) {
3301
- await fs.copy(decodeURIComponent(fileURLToPath(url)), tarballPath);
3302
- } else {
3303
- const cancelCtrl = new downloadFile.CancelCtrl();
3304
-
3305
- if (!ctrlStore[rootDid]) {
3306
- ctrlStore[rootDid] = new Map();
3307
- }
3308
- ctrlStore[rootDid].set(did, cancelCtrl);
3309
-
3310
- const headers = context.headers ? cloneDeep(context.headers) : {};
3311
- const exist = (context.downloadTokenList || []).find((x) => x.did === did);
3312
- if (exist) {
3313
- headers['x-download-token'] = exist.token;
3314
- }
3315
-
3316
- await downloadFile(url, path.join(cwd, tarballName), { cancelCtrl }, { ...context, headers });
3317
-
3318
- if (ctrlStore[rootDid]) {
3319
- ctrlStore[rootDid].delete(did);
3320
- if (!ctrlStore[rootDid].size) {
3321
- delete ctrlStore[rootDid];
3322
- }
3323
- }
3324
-
3325
- if (cancelCtrl.isCancelled) {
3326
- return downloadFile.CANCEL;
3327
- }
3328
- }
3329
-
3330
- if (verify) {
3331
- try {
3332
- await verifyIntegrity({ file: tarballPath, integrity });
3333
- } catch (error) {
3334
- logger.error('verify integrity error', { error, tarball, url });
3335
- throw new Error(`${tarball} integrity check failed.`);
3336
- }
3337
- }
3338
-
3339
- return tarballPath;
3340
- }
3341
-
3342
- /**
3343
- * use LRU algorithm
3344
- */
3345
- async _addCacheTarFile(tarballPath, integrity) {
3346
- // eslint-disable-next-line no-param-reassign
3347
- integrity = toBase58(integrity);
3348
-
3349
- // move tarball to cache dir
3350
- const cwd = path.join(this.dataDirs.tmp, 'download-cache');
3351
- const cachePath = path.join(cwd, `${integrity}.tar.gz`);
3352
- await fs.ensureDir(cwd);
3353
- await fs.move(tarballPath, cachePath, { overwrite: true });
3354
-
3355
- const key = 'blocklet:manager:downloadCache';
3356
- const cacheList = (await states.cache.get(key)) || [];
3357
- const exist = cacheList.find((x) => x.integrity === integrity);
3358
-
3359
- // update
3360
- if (exist) {
3361
- logger.info('update cache tarFile', { base58: integrity });
3362
- exist.accessAt = Date.now();
3363
- await states.cache.set(key, cacheList);
3364
- return;
3365
- }
3366
-
3367
- // add
3368
- cacheList.push({ integrity, accessAt: Date.now() });
3369
- if (cacheList.length > 10) {
3370
- // find and remove
3371
- let minIndex = 0;
3372
- let min = cacheList[0];
3373
- cacheList.forEach((x, i) => {
3374
- if (x.accessAt < min.accessAt) {
3375
- minIndex = i;
3376
- min = x;
3377
- }
3378
- });
3379
-
3380
- cacheList.splice(minIndex, 1);
3381
- await fs.remove(path.join(cwd, `${min.integrity}.tar.gz`));
3382
- logger.info('remove cache tarFile', { base58: min.integrity });
3383
- }
3384
- logger.info('add cache tarFile', { base58: integrity });
3385
-
3386
- // update
3387
- await states.cache.set(key, cacheList);
3388
- }
3389
-
3390
- async _getCachedTarFile(integrity) {
3391
- // eslint-disable-next-line no-param-reassign
3392
- integrity = toBase58(integrity);
3393
-
3394
- const cwd = path.join(this.dataDirs.tmp, 'download-cache');
3395
- const cachePath = path.join(cwd, `${integrity}.tar.gz`);
3396
-
3397
- if (fs.existsSync(cachePath)) {
3398
- return cachePath;
3399
- }
3400
-
3401
- return null;
3402
- }
3403
-
3404
- /**
3405
- *
3406
- *
3407
- * @param {*} meta
3408
- * @param {*} rootDid
3409
- * @param {*} url
3410
- * @param {{}} [context={}]
3411
- * @return {*}
3412
- * @memberof BlockletManager
3413
- */
3414
- async _downloadBundle(meta, rootDid, url, context = {}) {
3415
- const { bundleName: name, bundleDid: did, version, dist = {} } = meta;
3416
- const { tarball, integrity } = dist;
3417
-
3418
- const lockName = `download-${did}-${version}`;
3419
- let lock = this.downloadLocks[lockName];
3420
- if (!lock) {
3421
- lock = new Lock(lockName);
3422
- this.downloadLocks[lockName] = lock;
3423
- }
3424
-
3425
- try {
3426
- await lock.acquire();
3427
- logger.info('downloaded blocklet for installing', { name, version, tarball, integrity });
3428
- const cwd = path.join(this.dataDirs.tmp, 'download', name);
3429
- await fs.ensureDir(cwd);
3430
- logger.info('start download blocklet', { name, version, cwd, tarball, integrity });
3431
- const tarballPath = await this._downloadTarball({
3432
- name,
3433
- did,
3434
- version,
3435
- cwd,
3436
- tarball,
3437
- integrity,
3438
- verify: true,
3439
- ctrlStore: this.downloadCtrls,
3440
- rootDid,
3441
- url,
3442
- context,
3443
- });
3444
- logger.info('downloaded blocklet tar file', { name, version, tarballPath });
3445
- if (tarballPath === downloadFile.CANCEL) {
3446
- lock.release();
3447
- return { isCancelled: true };
3448
- }
3449
-
3450
- // resolve tarball and mv tarball to cache after resolved
3451
- await this._resolveDownload(cwd, tarballPath, null, { removeTarFile: false });
3452
- await this._addCacheTarFile(tarballPath, integrity);
3453
-
3454
- logger.info('resolved blocklet tar file to install dir', { name, version });
3455
- lock.release();
3456
- return { isCancelled: false };
3457
- } catch (error) {
3458
- lock.release();
3459
- throw error;
3460
- }
3461
- }
3462
-
3463
3131
  /**
3464
3132
  *
3465
3133
  *
3466
3134
  * @param {{}} blocklet
3467
- * @param {{}} [oldBlocklet={}]
3468
3135
  * @param {{}} [context={}]
3469
3136
  * @return {*}
3470
3137
  * @memberof BlockletManager
3471
3138
  */
3472
- async _downloadBlocklet(blocklet, oldBlocklet = {}, context = {}) {
3139
+ async _downloadBlocklet(blocklet, context = {}) {
3473
3140
  const {
3474
- meta: { name, did },
3141
+ meta: { did },
3475
3142
  } = blocklet;
3476
3143
 
3477
- const oldComponentObj = {};
3478
- const downloadComponentIds = [];
3479
- const metaObj = {};
3480
- const metas = [];
3481
-
3482
- forEachBlockletSync(oldBlocklet, (oldComponent, { id }) => {
3483
- if (oldComponent.meta) {
3484
- oldComponentObj[id] = oldComponent;
3485
- metaObj[getComponentBundleId(oldComponent)] = oldComponent.meta;
3486
- }
3487
- });
3488
-
3489
- forEachBlockletSync(blocklet, (component, { id: componentId }) => {
3490
- if (needBlockletDownload(component, oldComponentObj[componentId])) {
3491
- const bundleId = getComponentBundleId(component);
3492
-
3493
- if (metaObj[bundleId]) {
3494
- if (get(component, 'meta.dist.integrity') !== get(metaObj[bundleId], 'dist.integrity')) {
3495
- // FIXME: what should we do if component bundle integrity not same in different apps
3496
- logger.error(`find duplicate bundle with different integrity when downloading ${blocklet.meta.title}`, {
3497
- bundleId,
3498
- });
3499
- }
3500
-
3501
- logger.info(`skip download duplicate bundle ${bundleId}`);
3502
- return;
3503
- }
3504
-
3505
- metaObj[bundleId] = component.meta;
3506
- metas.push(component.meta);
3507
- downloadComponentIds.push(componentId);
3508
- }
3509
- });
3510
-
3511
- // update children status
3512
- const blocklet1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.downloading, {
3513
- children: downloadComponentIds,
3144
+ return this.blockletDownloader.download(blocklet, {
3145
+ ...context,
3146
+ preDownload: async ({ downloadComponentIds }) => {
3147
+ // update children status
3148
+ const blocklet1 = await states.blocklet.setBlockletStatus(did, BlockletStatus.downloading, {
3149
+ children: downloadComponentIds,
3150
+ });
3151
+ this.emit(BlockletEvents.statusChange, blocklet1);
3152
+ },
3153
+ postDownload: async () => {
3154
+ // since preferences only exist in blocklet bundle, we need to populate then after downloaded
3155
+ await this._setConfigsFromMeta(did);
3156
+ },
3514
3157
  });
3515
- this.emit(BlockletEvents.statusChange, blocklet1);
3516
-
3517
- try {
3518
- logger.info('Download Blocklet', { name, did, bundles: metas.map((x) => get(x, 'dist.tarball')) });
3519
- const tasks = [];
3520
- for (const meta of metas) {
3521
- const url = meta.dist.tarball;
3522
- tasks.push(this._downloadBundle(meta, did, url, context));
3523
- }
3524
- const results = await Promise.all(tasks);
3525
-
3526
- // since preferences only exist in blocklet bundle, we need to populate then after downloaded
3527
- await this._setConfigsFromMeta(did);
3528
-
3529
- if (results.find((x) => x.isCancelled)) {
3530
- return { isCancelled: true };
3531
- }
3532
- } catch (error) {
3533
- logger.error('Download blocklet failed', { did, name, error });
3534
- await this._cancelDownload(blocklet.meta);
3535
- throw error;
3536
- }
3537
-
3538
- return { isCancelled: false };
3539
3158
  }
3540
3159
 
3541
3160
  async _syncPm2Status(pm2Status, did) {
@@ -3551,17 +3170,6 @@ class BlockletManager extends BaseBlockletManager {
3551
3170
  }
3552
3171
  }
3553
3172
 
3554
- // eslint-disable-next-line no-unused-vars
3555
- async _cancelDownload(blockletMeta, context) {
3556
- const { did } = blockletMeta;
3557
-
3558
- if (this.downloadCtrls[did]) {
3559
- for (const cancelCtrl of this.downloadCtrls[did].values()) {
3560
- cancelCtrl.cancel();
3561
- }
3562
- }
3563
- }
3564
-
3565
3173
  // eslint-disable-next-line no-unused-vars
3566
3174
  async _cancelWaiting(blockletMeta, context) {
3567
3175
  const { did } = blockletMeta;
@@ -3681,7 +3289,7 @@ class BlockletManager extends BaseBlockletManager {
3681
3289
  }
3682
3290
 
3683
3291
  async _setConfigsFromMeta(did, childDid) {
3684
- const blocklet = await getBlocklet({ states, dataDirs: this.dataDirs, did, validateEnv: false, ensureDirs: false });
3292
+ const blocklet = await getBlocklet({ states, dataDirs: this.dataDirs, did });
3685
3293
 
3686
3294
  if (!childDid) {
3687
3295
  await forEachBlocklet(blocklet, async (b, { ancestors }) => {
@@ -3848,48 +3456,55 @@ class BlockletManager extends BaseBlockletManager {
3848
3456
  async _deleteExpiredExternalBlocklet() {
3849
3457
  try {
3850
3458
  logger.info('start check expired external blocklet');
3851
- const blocklets = await states.blocklet.getBlocklets({
3852
- controller: {
3853
- $exists: true,
3459
+ const blockletExtras = await states.blockletExtras.find(
3460
+ {
3461
+ controller: {
3462
+ $exists: true,
3463
+ },
3464
+ 'controller.expiredAt': {
3465
+ $exists: false,
3466
+ },
3854
3467
  },
3855
- });
3856
-
3857
- const nodeInfo = await states.node.read();
3468
+ { did: 1, meta: 1, controller: 1 }
3469
+ );
3858
3470
 
3859
- const tasks = blocklets.map(async (blocklet) => {
3471
+ for (const data of blockletExtras) {
3860
3472
  try {
3861
- const assetState = await getNFTState(nodeInfo.launcher.chainHost, blocklet.controller.nftId);
3473
+ const assetState = await util.getNFTState(data.controller.chainHost, data.controller.nftId);
3862
3474
  const isExpired = isNFTExpired(assetState);
3475
+
3863
3476
  if (isExpired) {
3864
3477
  logger.info('the blocklet already expired', {
3865
- blockletId: blocklet._id,
3866
- nftId: blocklet.controller.nftId,
3478
+ blockletDid: data.meta.did,
3479
+ nftId: data.controller.nftId,
3867
3480
  });
3868
3481
 
3869
- await this.delete({ did: blocklet.meta.did, keepData: true, keepConfigs: true, keepLogsDir: true });
3482
+ await this.delete({ did: data.meta.did, keepData: true, keepConfigs: true, keepLogsDir: true });
3870
3483
  logger.info('the expired blocklet already deleted', {
3871
- blockletId: blocklet._id,
3872
- nftId: blocklet.controller.nftId,
3873
- did: blocklet.meta.did,
3484
+ blockletDid: data.meta.did,
3485
+ nftId: data.controller.nftId,
3874
3486
  });
3875
3487
 
3876
3488
  const expiredAt = getNftExpirationDate(assetState);
3877
- await states.blockletExtras.updateExpireInfo({ did: blocklet.meta.did, expiredAt });
3489
+ await states.blockletExtras.updateExpireInfo({ did: data.meta.did, expiredAt });
3878
3490
  logger.info('updated expired blocklet extra info', {
3879
- blockletId: blocklet._id,
3880
- nftId: blocklet.controller.nftId,
3881
- did: blocklet.meta.did,
3491
+ nftId: data.controller.nftId,
3492
+ blockletDid: data.meta.did,
3882
3493
  });
3494
+
3495
+ // 删除 blocklet 后会 reload nginx, 所以这里每次删除一个
3496
+ if (process.env.NODE_ENV !== 'test') {
3497
+ await sleep(10 * 1000);
3498
+ }
3883
3499
  }
3884
3500
  } catch (error) {
3885
- logger.error('get asset state failed when check expired external blocklet', {
3886
- blockletId: blocklet._id,
3887
- nftId: blocklet.controller.nftId,
3501
+ logger.error('delete expired blocklet failed', {
3502
+ blockletDid: data.meta.did,
3503
+ nftId: data.controller.nftId,
3504
+ error,
3888
3505
  });
3889
3506
  }
3890
- });
3891
-
3892
- await Promise.all(tasks);
3507
+ }
3893
3508
 
3894
3509
  logger.info('check expired external blocklet end');
3895
3510
  } catch (error) {
@@ -3954,13 +3569,13 @@ class BlockletManager extends BaseBlockletManager {
3954
3569
 
3955
3570
  this.emit(BlockletEvents.appDidChanged, blocklet);
3956
3571
 
3957
- const blockletWithEnv = await this.ensureBlocklet(blocklet.meta.did);
3572
+ const blockletWithEnv = await this.getBlocklet(blocklet.meta.did);
3958
3573
  const appSystemEnvironments = getAppOverwrittenEnvironments(blockletWithEnv, nodeInfo);
3959
3574
 
3960
3575
  await didDocument.updateBlockletDocument({
3961
3576
  wallet,
3962
3577
  blockletAppDid: appSystemEnvironments.BLOCKLET_APP_ID,
3963
- daemonDidDomain: getServerDidDomain(nodeInfo),
3578
+ daemonDidDomain: util.getServerDidDomain(nodeInfo),
3964
3579
  didRegistryUrl: nodeInfo.didRegistry,
3965
3580
  domain: nodeInfo.didDomain,
3966
3581
  });