@abtnode/core 1.16.13-beta-d17a7de4 → 1.16.13-beta-90ded76f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/blocklet/manager/disk.js +27 -25
- package/lib/index.js +3 -0
- package/lib/router/helper.js +97 -10
- package/lib/states/blocklet.js +63 -26
- package/lib/states/index.js +4 -0
- package/lib/states/traffic-insight.js +25 -0
- package/lib/util/blocklet.js +46 -2
- package/lib/util/index.js +23 -0
- package/lib/util/rotator.js +3 -3
- package/package.json +19 -20
|
@@ -116,6 +116,7 @@ const {
|
|
|
116
116
|
getBlockletStatus,
|
|
117
117
|
shouldSkipComponent,
|
|
118
118
|
exceedRedemptionPeriod,
|
|
119
|
+
ensureAppPortsNotOccupied,
|
|
119
120
|
} = require('../../util/blocklet');
|
|
120
121
|
const states = require('../../states');
|
|
121
122
|
const BaseBlockletManager = require('./base');
|
|
@@ -460,37 +461,36 @@ class BlockletManager extends BaseBlockletManager {
|
|
|
460
461
|
async start({ did, throwOnError, checkHealthImmediately = false, e2eMode = false, componentDids }, context) {
|
|
461
462
|
logger.info('start blocklet', { did, componentDids, throwOnError, checkHealthImmediately, e2eMode });
|
|
462
463
|
// should check blocklet integrity
|
|
463
|
-
const
|
|
464
|
+
const blocklet1 = await this.ensureBlocklet(did, { e2eMode });
|
|
464
465
|
|
|
465
|
-
await this.checkControllerStatus(
|
|
466
|
+
await this.checkControllerStatus(blocklet1, 'start');
|
|
467
|
+
|
|
468
|
+
// validate requirement and engine
|
|
469
|
+
await validateBlocklet(blocklet1);
|
|
470
|
+
await validateBlockletChainInfo(blocklet1);
|
|
471
|
+
|
|
472
|
+
if (!hasRunnableComponent(blocklet1)) {
|
|
473
|
+
throw new Error('No runnable component found');
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
// check required config
|
|
477
|
+
const missingProps = getAppMissingConfigs(blocklet1);
|
|
478
|
+
if (missingProps.length) {
|
|
479
|
+
throw new Error(
|
|
480
|
+
`Missing required configuration to start the blocklet: ${missingProps.map((x) => x.key).join(',')}`
|
|
481
|
+
);
|
|
482
|
+
}
|
|
466
483
|
|
|
467
484
|
try {
|
|
468
485
|
// blocklet may be manually stopped durning starting
|
|
469
486
|
// so error message would not be sent if blocklet is stopped
|
|
470
487
|
// so we need update status first
|
|
471
488
|
await states.blocklet.setBlockletStatus(did, BlockletStatus.starting, { componentDids });
|
|
472
|
-
|
|
489
|
+
blocklet1.status = BlockletStatus.starting;
|
|
490
|
+
this.emit(BlockletEvents.statusChange, blocklet1);
|
|
473
491
|
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
await validateBlockletChainInfo(blocklet);
|
|
477
|
-
|
|
478
|
-
if (!hasRunnableComponent(blocklet)) {
|
|
479
|
-
throw new Error('No runnable component found');
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
// check required config
|
|
483
|
-
const missingProps = getAppMissingConfigs(blocklet);
|
|
484
|
-
if (missingProps.length) {
|
|
485
|
-
throw new Error(
|
|
486
|
-
`Missing required configuration to start the blocklet: ${missingProps.map((x) => x.key).join(',')}`
|
|
487
|
-
);
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
this.emit(BlockletEvents.statusChange, blocklet);
|
|
491
|
-
|
|
492
|
-
if (blocklet.mode === BLOCKLET_MODES.DEVELOPMENT) {
|
|
493
|
-
const { logsDir } = blocklet.env;
|
|
492
|
+
if (blocklet1.mode === BLOCKLET_MODES.DEVELOPMENT) {
|
|
493
|
+
const { logsDir } = blocklet1.env;
|
|
494
494
|
|
|
495
495
|
try {
|
|
496
496
|
fs.removeSync(logsDir);
|
|
@@ -501,6 +501,8 @@ class BlockletManager extends BaseBlockletManager {
|
|
|
501
501
|
}
|
|
502
502
|
}
|
|
503
503
|
|
|
504
|
+
const blocklet = await ensureAppPortsNotOccupied({ blocklet: blocklet1, componentDids, states, manager: this });
|
|
505
|
+
|
|
504
506
|
const getHookFn =
|
|
505
507
|
(hookName) =>
|
|
506
508
|
(b, { env }) =>
|
|
@@ -554,8 +556,8 @@ class BlockletManager extends BaseBlockletManager {
|
|
|
554
556
|
}
|
|
555
557
|
|
|
556
558
|
const error = Array.isArray(err) ? err[0] : err;
|
|
557
|
-
logger.error('Failed to start blocklet', { error, did,
|
|
558
|
-
const description = `Start blocklet ${
|
|
559
|
+
logger.error('Failed to start blocklet', { error, did, title: blocklet1.meta.title });
|
|
560
|
+
const description = `Start blocklet ${blocklet1.meta.title} failed with error: ${error.message}`;
|
|
559
561
|
this._createNotification(did, {
|
|
560
562
|
title: 'Start Blocklet Failed',
|
|
561
563
|
description,
|
package/lib/index.js
CHANGED
|
@@ -411,6 +411,9 @@ function ABTNode(options) {
|
|
|
411
411
|
return states.auditLog.findPaginated.call(states.auditLog, params);
|
|
412
412
|
},
|
|
413
413
|
|
|
414
|
+
// Insights
|
|
415
|
+
getTrafficInsights: states.trafficInsight.findPaginated.bind(states.trafficInsight),
|
|
416
|
+
|
|
414
417
|
// Routing
|
|
415
418
|
routerManager,
|
|
416
419
|
addRoutingSite,
|
package/lib/router/helper.js
CHANGED
|
@@ -4,6 +4,7 @@ const fs = require('fs-extra');
|
|
|
4
4
|
const path = require('path');
|
|
5
5
|
const tar = require('tar');
|
|
6
6
|
const UUID = require('uuid');
|
|
7
|
+
const dayjs = require('@abtnode/util/lib/dayjs');
|
|
7
8
|
const isUrl = require('is-url');
|
|
8
9
|
const get = require('lodash/get');
|
|
9
10
|
const cloneDeep = require('lodash/cloneDeep');
|
|
@@ -18,6 +19,7 @@ const downloadFile = require('@abtnode/util/lib/download-file');
|
|
|
18
19
|
const { updateBlockletDocument } = require('@abtnode/util/lib/did-document');
|
|
19
20
|
const getBlockletInfo = require('@blocklet/meta/lib/info');
|
|
20
21
|
const { forEachBlockletSync } = require('@blocklet/meta/lib/util');
|
|
22
|
+
const { processLogByDate } = require('@abtnode/insights');
|
|
21
23
|
const {
|
|
22
24
|
DOMAIN_FOR_DEFAULT_SITE,
|
|
23
25
|
DOMAIN_FOR_IP_SITE_REGEXP,
|
|
@@ -39,6 +41,7 @@ const {
|
|
|
39
41
|
WELLKNOWN_PING_PREFIX,
|
|
40
42
|
LOG_RETAIN_IN_DAYS,
|
|
41
43
|
EVENTS,
|
|
44
|
+
DEFAULT_IP_DOMAIN,
|
|
42
45
|
} = require('@abtnode/constant');
|
|
43
46
|
const {
|
|
44
47
|
BLOCKLET_DYNAMIC_PATH_PREFIX,
|
|
@@ -69,6 +72,8 @@ const Router = require('./index');
|
|
|
69
72
|
const states = require('../states');
|
|
70
73
|
const { getBlockletDomainGroupName, getDidFromDomainGroupName } = require('../util/router');
|
|
71
74
|
const { getBlockletKnownAs, getBlockletDidDomainList } = require('../util/blocklet');
|
|
75
|
+
const { toCamelCase } = require('../util/index');
|
|
76
|
+
const { get: getIp } = require('../util/ip');
|
|
72
77
|
|
|
73
78
|
const isServiceFeDevelopment = process.env.ABT_NODE_SERVICE_FE_PORT;
|
|
74
79
|
|
|
@@ -77,31 +82,43 @@ const hasRuleByPrefix = (site, value) => site.rules.find((x) => x.isProtected &&
|
|
|
77
82
|
/**
|
|
78
83
|
* replace 888-888-888-888 with accessible ip for domain
|
|
79
84
|
*/
|
|
80
|
-
const attachRuntimeDomainAliases = async ({ sites = [], context = {}
|
|
85
|
+
const attachRuntimeDomainAliases = async ({ sites = [], context = {} }) => {
|
|
81
86
|
if (!sites) {
|
|
82
87
|
return [];
|
|
83
88
|
}
|
|
84
89
|
|
|
85
90
|
let ip;
|
|
86
91
|
const ipRegex = /\d+[-.]\d+[-.]\d+[-.]\d+/;
|
|
87
|
-
const match = ipRegex.exec(context.hostname);
|
|
92
|
+
const match = ipRegex.exec(context.hostname || '');
|
|
88
93
|
if (match) {
|
|
89
94
|
ip = match[0];
|
|
90
|
-
} else
|
|
91
|
-
const
|
|
92
|
-
const nodeIp = await getAccessibleExternalNodeIp(nodeInfo);
|
|
95
|
+
} else {
|
|
96
|
+
const nodeIp = await getAccessibleExternalNodeIp();
|
|
93
97
|
if (nodeIp) {
|
|
94
98
|
ip = nodeIp;
|
|
95
99
|
}
|
|
96
100
|
}
|
|
97
101
|
|
|
102
|
+
if (!ip) {
|
|
103
|
+
const result = await getIp();
|
|
104
|
+
if (result) {
|
|
105
|
+
ip = result.internal;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
logger.info('attachRuntimeDomainAliases', { ip });
|
|
110
|
+
|
|
98
111
|
const getDomainAliases = (site) =>
|
|
99
112
|
(site.domainAliases || []).map((domain) => {
|
|
100
113
|
if (!domain.value) {
|
|
101
114
|
return domain;
|
|
102
115
|
}
|
|
103
|
-
if (
|
|
104
|
-
|
|
116
|
+
if (ip) {
|
|
117
|
+
if (domain.value.includes(SLOT_FOR_IP_DNS_SITE)) {
|
|
118
|
+
domain.value = replaceSlotToIp(domain.value, ip);
|
|
119
|
+
} else if (domain.value === DEFAULT_IP_DOMAIN) {
|
|
120
|
+
domain.value = `${ip.split('.').join('-')}${domain.value.substring(2)}`;
|
|
121
|
+
}
|
|
105
122
|
}
|
|
106
123
|
return domain;
|
|
107
124
|
});
|
|
@@ -629,6 +646,7 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
|
|
|
629
646
|
const blockletState = states.blocklet;
|
|
630
647
|
const siteState = states.site;
|
|
631
648
|
const notification = states.notification;
|
|
649
|
+
const trafficInsight = states.trafficInsight;
|
|
632
650
|
|
|
633
651
|
// site level duplication detection
|
|
634
652
|
|
|
@@ -1265,6 +1283,71 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
|
|
|
1265
1283
|
}
|
|
1266
1284
|
};
|
|
1267
1285
|
|
|
1286
|
+
const analyzeRouterLog = async () => {
|
|
1287
|
+
const info = await nodeState.read();
|
|
1288
|
+
const sites = await getRoutingSites({});
|
|
1289
|
+
const providerName = get(info, 'routing.provider', null);
|
|
1290
|
+
if (!providerName || !providers[providerName]) {
|
|
1291
|
+
logger.warn('No router provider instance found');
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
const groups = [];
|
|
1296
|
+
const server = sites.find((x) => x.domainAliases.some((a) => a.value === DOMAIN_FOR_IP_SITE));
|
|
1297
|
+
if (server) {
|
|
1298
|
+
groups.push({
|
|
1299
|
+
did: info.did,
|
|
1300
|
+
type: 'server',
|
|
1301
|
+
hosts: server.domainAliases.map((d) => d.value).filter(Boolean),
|
|
1302
|
+
});
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
// blocklets
|
|
1306
|
+
sites
|
|
1307
|
+
.filter((x) => x.domain.endsWith(BLOCKLET_SITE_GROUP_SUFFIX))
|
|
1308
|
+
.forEach((site) => {
|
|
1309
|
+
groups.push({
|
|
1310
|
+
did: site.domain.replace(BLOCKLET_SITE_GROUP_SUFFIX, ''),
|
|
1311
|
+
type: 'blocklet',
|
|
1312
|
+
hosts: site.domainAliases.map((d) => d.value).filter(Boolean),
|
|
1313
|
+
});
|
|
1314
|
+
});
|
|
1315
|
+
|
|
1316
|
+
const logDir = providers[providerName].getLogDir();
|
|
1317
|
+
const doAnalyze = async (date) => {
|
|
1318
|
+
logger.info('Start analyze router logs', { date, groups, logDir });
|
|
1319
|
+
try {
|
|
1320
|
+
let results = await processLogByDate(logDir, dataDirs.tmp, dataDirs.data, date, groups);
|
|
1321
|
+
logger.info('Done analyze router logs', { date, results });
|
|
1322
|
+
|
|
1323
|
+
results = await Promise.all(
|
|
1324
|
+
results
|
|
1325
|
+
.filter((x) => x.result)
|
|
1326
|
+
.map((x) => ({ did: x.did, date, ...toCamelCase(x.result) }))
|
|
1327
|
+
.map((x) => trafficInsight.upsert({ did: x.did, date: x.date }, x))
|
|
1328
|
+
);
|
|
1329
|
+
logger.info('Done insert insight results', { date, results });
|
|
1330
|
+
} catch (err) {
|
|
1331
|
+
console.error(err);
|
|
1332
|
+
logger.error('Failed to analyze router logs', { date, error: err });
|
|
1333
|
+
}
|
|
1334
|
+
};
|
|
1335
|
+
|
|
1336
|
+
const analyzeLock = path.join(logDir, '.analyze.lock');
|
|
1337
|
+
if (fs.existsSync(analyzeLock)) {
|
|
1338
|
+
// FIXME: @wangshijun how do we support real time logs
|
|
1339
|
+
const date = dayjs().subtract(1, 'day').format('YYYY-MM-DD');
|
|
1340
|
+
await doAnalyze(date);
|
|
1341
|
+
} else {
|
|
1342
|
+
fs.writeFileSync(analyzeLock, '1');
|
|
1343
|
+
for (let i = 1; i <= 30; i++) {
|
|
1344
|
+
const date = dayjs().subtract(i, 'day').format('YYYY-MM-DD');
|
|
1345
|
+
// eslint-disable-next-line no-await-in-loop
|
|
1346
|
+
await doAnalyze(date);
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
};
|
|
1350
|
+
|
|
1268
1351
|
const updateNodeRouting = async (params, context = {}) => {
|
|
1269
1352
|
const info = await nodeState.read();
|
|
1270
1353
|
const { snapshotHash: oldSnapshotHash } = info.routing || {};
|
|
@@ -1396,7 +1479,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
|
|
|
1396
1479
|
return attachRuntimeDomainAliases({
|
|
1397
1480
|
sites: await ensureLatestInfo(sites, { withDefaultCors }),
|
|
1398
1481
|
context,
|
|
1399
|
-
node: nodeState,
|
|
1400
1482
|
});
|
|
1401
1483
|
};
|
|
1402
1484
|
|
|
@@ -1405,7 +1487,6 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
|
|
|
1405
1487
|
return attachRuntimeDomainAliases({
|
|
1406
1488
|
sites: await ensureLatestInfo(sites, { withDefaultCors }),
|
|
1407
1489
|
context,
|
|
1408
|
-
node: nodeState,
|
|
1409
1490
|
});
|
|
1410
1491
|
};
|
|
1411
1492
|
|
|
@@ -1490,7 +1571,13 @@ module.exports = function getRouterHelpers({ dataDirs, routingSnapshot, routerMa
|
|
|
1490
1571
|
name: 'rotate-log-files',
|
|
1491
1572
|
time: '1 0 0 * * *', // rotate at 00:00:01 every day
|
|
1492
1573
|
fn: rotateRouterLog,
|
|
1493
|
-
options: { runOnInit:
|
|
1574
|
+
options: { runOnInit: process.env.ABT_NODE_JOB_NAME === 'rotate-log-files' },
|
|
1575
|
+
},
|
|
1576
|
+
{
|
|
1577
|
+
name: 'analyze-log-files',
|
|
1578
|
+
time: '0 5 0 * * *', // analyze at 00:05:00 every day
|
|
1579
|
+
fn: analyzeRouterLog,
|
|
1580
|
+
options: { runOnInit: process.env.ABT_NODE_JOB_NAME === 'analyze-log-files' },
|
|
1494
1581
|
},
|
|
1495
1582
|
],
|
|
1496
1583
|
|
package/lib/states/blocklet.js
CHANGED
|
@@ -14,6 +14,7 @@ const {
|
|
|
14
14
|
getDisplayName,
|
|
15
15
|
forEachBlocklet,
|
|
16
16
|
forEachBlockletSync,
|
|
17
|
+
forEachComponentV2,
|
|
17
18
|
forEachComponentV2Sync,
|
|
18
19
|
getBlockletServices,
|
|
19
20
|
} = require('@blocklet/meta/lib/util');
|
|
@@ -24,6 +25,7 @@ const {
|
|
|
24
25
|
BLOCKLET_DEFAULT_PORT_NAME,
|
|
25
26
|
BlockletGroup,
|
|
26
27
|
} = require('@blocklet/constant');
|
|
28
|
+
const { refreshPorts } = require('@abtnode/util/lib/port');
|
|
27
29
|
const { APP_STRUCT_VERSION } = require('@abtnode/constant');
|
|
28
30
|
|
|
29
31
|
const logger = require('@abtnode/logger')('@abtnode/core:states:blocklet');
|
|
@@ -343,6 +345,9 @@ class BlockletState extends BaseState {
|
|
|
343
345
|
}
|
|
344
346
|
}
|
|
345
347
|
|
|
348
|
+
/**
|
|
349
|
+
* assign ports for blocklet during install/upgrade workflow
|
|
350
|
+
*/
|
|
346
351
|
async getBlockletPorts({
|
|
347
352
|
interfaces = [],
|
|
348
353
|
alreadyAssigned = {},
|
|
@@ -351,32 +356,7 @@ class BlockletState extends BaseState {
|
|
|
351
356
|
defaultPort = 0,
|
|
352
357
|
} = {}) {
|
|
353
358
|
try {
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
const occupiedExternalPorts = new Map();
|
|
357
|
-
const occupiedInternalPorts = new Map();
|
|
358
|
-
|
|
359
|
-
const calcPortsFromBlocklet = (blocklet) => {
|
|
360
|
-
occupiedInternalPorts.set(Number(blocklet.port), true);
|
|
361
|
-
|
|
362
|
-
if (blocklet.ports && typeof blocklet.ports === 'object') {
|
|
363
|
-
Object.keys(blocklet.ports).forEach((key) => {
|
|
364
|
-
occupiedInternalPorts.set(Number(blocklet.ports[key]), true);
|
|
365
|
-
});
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
if (Array.isArray(blocklet.meta.interfaces)) {
|
|
369
|
-
blocklet.meta.interfaces.forEach((x) => {
|
|
370
|
-
if (x.port && x.port.external) {
|
|
371
|
-
occupiedExternalPorts.set(Number(x.port.external), true);
|
|
372
|
-
}
|
|
373
|
-
});
|
|
374
|
-
}
|
|
375
|
-
};
|
|
376
|
-
|
|
377
|
-
for (const blocklet of blocklets) {
|
|
378
|
-
await forEachBlocklet(blocklet, calcPortsFromBlocklet);
|
|
379
|
-
}
|
|
359
|
+
const { occupiedExternalPorts, occupiedInternalPorts } = await this._getOccupiedPorts();
|
|
380
360
|
|
|
381
361
|
const wantedPorts = uniq(
|
|
382
362
|
interfaces
|
|
@@ -437,6 +417,30 @@ class BlockletState extends BaseState {
|
|
|
437
417
|
}
|
|
438
418
|
}
|
|
439
419
|
|
|
420
|
+
/**
|
|
421
|
+
* refresh ports for blocklet if occupied during starting workflow
|
|
422
|
+
*/
|
|
423
|
+
async refreshBlockletPorts(did, componentDids = []) {
|
|
424
|
+
const blocklet = await this.getBlocklet(did);
|
|
425
|
+
if (!blocklet) {
|
|
426
|
+
throw new Error('Blocklet does not exist');
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
const { occupiedExternalPorts, occupiedInternalPorts } = await this._getOccupiedPorts();
|
|
430
|
+
|
|
431
|
+
await forEachComponentV2(blocklet, async (component) => {
|
|
432
|
+
if (!shouldSkipComponent(component.meta.did, componentDids)) {
|
|
433
|
+
component.ports = await refreshPorts(component.ports, {
|
|
434
|
+
blackList: [...occupiedExternalPorts.keys(), ...occupiedInternalPorts.keys()],
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
return this.updateBlocklet(did, {
|
|
440
|
+
children: blocklet.children,
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
|
|
440
444
|
async getServices() {
|
|
441
445
|
const blocklets = await this.getBlocklets({}, { meta: 1, children: 1, ports: 1 });
|
|
442
446
|
const services = [];
|
|
@@ -662,6 +666,39 @@ class BlockletState extends BaseState {
|
|
|
662
666
|
|
|
663
667
|
return this.statusLocks[did];
|
|
664
668
|
}
|
|
669
|
+
|
|
670
|
+
async _getOccupiedPorts() {
|
|
671
|
+
const blocklets = await this.getBlocklets({}, { port: 1, ports: 1, meta: 1, children: 1 });
|
|
672
|
+
|
|
673
|
+
const occupiedExternalPorts = new Map();
|
|
674
|
+
const occupiedInternalPorts = new Map();
|
|
675
|
+
|
|
676
|
+
const calcPortsFromBlocklet = (blocklet) => {
|
|
677
|
+
occupiedInternalPorts.set(Number(blocklet.port), true);
|
|
678
|
+
|
|
679
|
+
if (blocklet.ports && typeof blocklet.ports === 'object') {
|
|
680
|
+
Object.keys(blocklet.ports).forEach((key) => {
|
|
681
|
+
occupiedInternalPorts.set(Number(blocklet.ports[key]), true);
|
|
682
|
+
});
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
if (Array.isArray(blocklet.meta.interfaces)) {
|
|
686
|
+
blocklet.meta.interfaces.forEach((x) => {
|
|
687
|
+
if (x.port && x.port.external) {
|
|
688
|
+
occupiedExternalPorts.set(Number(x.port.external), true);
|
|
689
|
+
}
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
};
|
|
693
|
+
|
|
694
|
+
for (const blocklet of blocklets) {
|
|
695
|
+
await forEachBlocklet(blocklet, calcPortsFromBlocklet);
|
|
696
|
+
}
|
|
697
|
+
return {
|
|
698
|
+
occupiedExternalPorts,
|
|
699
|
+
occupiedInternalPorts,
|
|
700
|
+
};
|
|
701
|
+
}
|
|
665
702
|
}
|
|
666
703
|
|
|
667
704
|
BlockletState.BlockletStatus = BlockletStatus;
|
package/lib/states/index.js
CHANGED
|
@@ -15,6 +15,8 @@ const CacheState = require('./cache');
|
|
|
15
15
|
const AuditLogState = require('./audit-log');
|
|
16
16
|
const JobState = require('./job');
|
|
17
17
|
const BackupState = require('./backup');
|
|
18
|
+
const TrafficInsightState = require('./traffic-insight');
|
|
19
|
+
|
|
18
20
|
const { getDbFilePath } = require('../util');
|
|
19
21
|
|
|
20
22
|
const models = getServerModels();
|
|
@@ -38,6 +40,7 @@ const init = (dataDirs, config = {}) => {
|
|
|
38
40
|
const auditLogState = new AuditLogState(models.AuditLog, config);
|
|
39
41
|
const jobState = new JobState(models.Job, config);
|
|
40
42
|
const backupState = new BackupState(models.Backup, config);
|
|
43
|
+
const trafficInsight = new TrafficInsightState(models.TrafficInsight, config);
|
|
41
44
|
|
|
42
45
|
return {
|
|
43
46
|
node: nodeState,
|
|
@@ -53,6 +56,7 @@ const init = (dataDirs, config = {}) => {
|
|
|
53
56
|
auditLog: auditLogState,
|
|
54
57
|
job: jobState,
|
|
55
58
|
backup: backupState,
|
|
59
|
+
trafficInsight,
|
|
56
60
|
};
|
|
57
61
|
};
|
|
58
62
|
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
const { Op } = require('sequelize');
|
|
2
|
+
const BaseState = require('./base');
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @extends BaseState<import('@abtnode/models').TrafficInsightState>
|
|
6
|
+
*/
|
|
7
|
+
class TrafficInsight extends BaseState {
|
|
8
|
+
findPaginated({ did = '', startDate = '', endDate = '', paging = { pageSize: 30 } } = {}) {
|
|
9
|
+
const where = {};
|
|
10
|
+
if (did) {
|
|
11
|
+
where.did = did;
|
|
12
|
+
}
|
|
13
|
+
if (startDate) {
|
|
14
|
+
where.date = { [Op.gte]: startDate };
|
|
15
|
+
}
|
|
16
|
+
if (endDate) {
|
|
17
|
+
where.date = where.date || {};
|
|
18
|
+
where.date[Op.lte] = endDate;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
return super.paginate({ where }, { date: -1 }, paging);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
module.exports = TrafficInsight;
|
package/lib/util/blocklet.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
const fs = require('fs-extra');
|
|
4
4
|
const path = require('path');
|
|
5
|
-
const dayjs = require('dayjs');
|
|
5
|
+
const dayjs = require('@abtnode/util/lib/dayjs');
|
|
6
6
|
const shelljs = require('shelljs');
|
|
7
7
|
const os = require('os');
|
|
8
8
|
const tar = require('tar');
|
|
@@ -26,7 +26,6 @@ const logger = require('@abtnode/logger')('@abtnode/core:util:blocklet');
|
|
|
26
26
|
const pm2 = require('@abtnode/util/lib/async-pm2');
|
|
27
27
|
const sleep = require('@abtnode/util/lib/sleep');
|
|
28
28
|
const getPm2ProcessInfo = require('@abtnode/util/lib/get-pm2-process-info');
|
|
29
|
-
const killProcessOccupiedPorts = require('@abtnode/util/lib/kill-process-occupied-ports');
|
|
30
29
|
const { formatEnv } = require('@abtnode/util/lib/security');
|
|
31
30
|
const ensureEndpointHealthy = require('@abtnode/util/lib/ensure-endpoint-healthy');
|
|
32
31
|
const CustomError = require('@abtnode/util/lib/custom-error');
|
|
@@ -49,6 +48,7 @@ const {
|
|
|
49
48
|
getComponentConfig,
|
|
50
49
|
} = require('@blocklet/resolver');
|
|
51
50
|
const formatBackSlash = require('@abtnode/util/lib/format-back-slash');
|
|
51
|
+
const { isPortsOccupiedByOtherProcess, killProcessOccupiedPorts } = require('@abtnode/util/lib/port');
|
|
52
52
|
const { toSvg: createDidLogo } =
|
|
53
53
|
process.env.NODE_ENV !== 'test' ? require('@arcblock/did-motif') : require('@arcblock/did-motif/dist/did-motif.cjs');
|
|
54
54
|
const { createBlockiesSvg } = require('@blocklet/meta/lib/blockies');
|
|
@@ -80,6 +80,7 @@ const {
|
|
|
80
80
|
findWebInterface,
|
|
81
81
|
forEachBlockletSync,
|
|
82
82
|
forEachChildSync,
|
|
83
|
+
forEachComponentV2,
|
|
83
84
|
forEachComponentV2Sync,
|
|
84
85
|
getSharedConfigObj,
|
|
85
86
|
getComponentName,
|
|
@@ -1768,6 +1769,48 @@ const exceedRedemptionPeriod = (expirationDate) => {
|
|
|
1768
1769
|
return dayjs().diff(dayjs(expirationDate), 'day') > EXPIRED_BLOCKLET_DATA_RETENTION_DAYS;
|
|
1769
1770
|
};
|
|
1770
1771
|
|
|
1772
|
+
const ensureAppPortsNotOccupied = async ({
|
|
1773
|
+
blocklet,
|
|
1774
|
+
componentDids: inputDids,
|
|
1775
|
+
states,
|
|
1776
|
+
manager,
|
|
1777
|
+
checkPortsFn = isPortsOccupiedByOtherProcess,
|
|
1778
|
+
}) => {
|
|
1779
|
+
const { did } = blocklet.meta;
|
|
1780
|
+
const componentDids = [];
|
|
1781
|
+
await forEachComponentV2(blocklet, async (b) => {
|
|
1782
|
+
try {
|
|
1783
|
+
if (shouldSkipComponent(b.meta.did, inputDids)) {
|
|
1784
|
+
return;
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
const { processId } = b.env;
|
|
1788
|
+
const { ports } = b;
|
|
1789
|
+
if (
|
|
1790
|
+
await checkPortsFn({
|
|
1791
|
+
ports,
|
|
1792
|
+
pm2ProcessId: processId,
|
|
1793
|
+
printError: logger.error.bind(logger),
|
|
1794
|
+
})
|
|
1795
|
+
) {
|
|
1796
|
+
componentDids.push(b.meta.did);
|
|
1797
|
+
}
|
|
1798
|
+
} catch (error) {
|
|
1799
|
+
logger.error('Failed to check ports occupied', { error });
|
|
1800
|
+
}
|
|
1801
|
+
});
|
|
1802
|
+
|
|
1803
|
+
let newBlocklet = blocklet;
|
|
1804
|
+
if (componentDids.length) {
|
|
1805
|
+
await states.blocklet.refreshBlockletPorts(did, componentDids);
|
|
1806
|
+
logger.info('refresh component ports', { did, componentDids });
|
|
1807
|
+
await manager._updateBlockletEnvironment(did);
|
|
1808
|
+
newBlocklet = await manager.ensureBlocklet(did);
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
return newBlocklet;
|
|
1812
|
+
};
|
|
1813
|
+
|
|
1771
1814
|
module.exports = {
|
|
1772
1815
|
updateBlockletFallbackLogo,
|
|
1773
1816
|
forEachBlocklet,
|
|
@@ -1829,4 +1872,5 @@ module.exports = {
|
|
|
1829
1872
|
shouldSkipComponent,
|
|
1830
1873
|
getBlockletURLForLauncher,
|
|
1831
1874
|
exceedRedemptionPeriod,
|
|
1875
|
+
ensureAppPortsNotOccupied,
|
|
1832
1876
|
};
|
package/lib/util/index.js
CHANGED
|
@@ -4,6 +4,7 @@ const path = require('path');
|
|
|
4
4
|
const dns = require('dns');
|
|
5
5
|
const crypto = require('crypto');
|
|
6
6
|
const shell = require('shelljs');
|
|
7
|
+
const camelCase = require('lodash/camelCase');
|
|
7
8
|
const get = require('lodash/get');
|
|
8
9
|
const pickBy = require('lodash/pickBy');
|
|
9
10
|
const { isFromPublicKey } = require('@arcblock/did');
|
|
@@ -469,6 +470,27 @@ const isServerSite = (domain) =>
|
|
|
469
470
|
|
|
470
471
|
const getDbFilePath = (filePath) => (process.env.NODE_ENV === 'test' ? `${filePath}:memory:` : filePath);
|
|
471
472
|
|
|
473
|
+
const toCamelCase = (obj) => {
|
|
474
|
+
if (typeof obj !== 'object' || obj === null) {
|
|
475
|
+
return obj;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
if (Array.isArray(obj)) {
|
|
479
|
+
return obj.map(toCamelCase);
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
const converted = {};
|
|
483
|
+
|
|
484
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
485
|
+
for (const key in obj) {
|
|
486
|
+
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
487
|
+
converted[camelCase(key)] = toCamelCase(obj[key]);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
return converted;
|
|
492
|
+
};
|
|
493
|
+
|
|
472
494
|
const lib = {
|
|
473
495
|
validateOwner,
|
|
474
496
|
getProviderFromNodeInfo,
|
|
@@ -505,6 +527,7 @@ const lib = {
|
|
|
505
527
|
isBlockletSite,
|
|
506
528
|
isServerSite,
|
|
507
529
|
getDbFilePath,
|
|
530
|
+
toCamelCase,
|
|
508
531
|
};
|
|
509
532
|
|
|
510
533
|
module.exports = lib;
|
package/lib/util/rotator.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
const fs = require('fs-extra');
|
|
5
5
|
const path = require('path');
|
|
6
|
-
const
|
|
6
|
+
const dayjs = require('@abtnode/util/lib/dayjs');
|
|
7
7
|
const zlib = require('zlib');
|
|
8
8
|
const log = require('@abtnode/logger');
|
|
9
9
|
|
|
@@ -70,11 +70,11 @@ module.exports = class LogRotate {
|
|
|
70
70
|
*/
|
|
71
71
|
proceed(file, callback = () => {}) {
|
|
72
72
|
// set default final time
|
|
73
|
-
let finalTime =
|
|
73
|
+
let finalTime = dayjs().subtract(1, 'day').format(this.dateFormat);
|
|
74
74
|
// check for a timezone
|
|
75
75
|
if (this.tz) {
|
|
76
76
|
try {
|
|
77
|
-
finalTime =
|
|
77
|
+
finalTime = dayjs().tz(this.tz).subtract(1, 'day').format(this.dateFormat);
|
|
78
78
|
} catch (err) {
|
|
79
79
|
// use default
|
|
80
80
|
}
|
package/package.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"publishConfig": {
|
|
4
4
|
"access": "public"
|
|
5
5
|
},
|
|
6
|
-
"version": "1.16.13-beta-
|
|
6
|
+
"version": "1.16.13-beta-90ded76f",
|
|
7
7
|
"description": "",
|
|
8
8
|
"main": "lib/index.js",
|
|
9
9
|
"files": [
|
|
@@ -19,18 +19,19 @@
|
|
|
19
19
|
"author": "wangshijun <wangshijun2010@gmail.com> (http://github.com/wangshijun)",
|
|
20
20
|
"license": "Apache-2.0",
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@abtnode/auth": "1.16.13-beta-
|
|
23
|
-
"@abtnode/certificate-manager": "1.16.13-beta-
|
|
24
|
-
"@abtnode/constant": "1.16.13-beta-
|
|
25
|
-
"@abtnode/cron": "1.16.13-beta-
|
|
26
|
-
"@abtnode/
|
|
27
|
-
"@abtnode/
|
|
28
|
-
"@abtnode/
|
|
29
|
-
"@abtnode/
|
|
30
|
-
"@abtnode/
|
|
31
|
-
"@abtnode/
|
|
32
|
-
"@abtnode/
|
|
33
|
-
"@abtnode/
|
|
22
|
+
"@abtnode/auth": "1.16.13-beta-90ded76f",
|
|
23
|
+
"@abtnode/certificate-manager": "1.16.13-beta-90ded76f",
|
|
24
|
+
"@abtnode/constant": "1.16.13-beta-90ded76f",
|
|
25
|
+
"@abtnode/cron": "1.16.13-beta-90ded76f",
|
|
26
|
+
"@abtnode/insights": "1.16.13-beta-90ded76f",
|
|
27
|
+
"@abtnode/logger": "1.16.13-beta-90ded76f",
|
|
28
|
+
"@abtnode/models": "1.16.13-beta-90ded76f",
|
|
29
|
+
"@abtnode/queue": "1.16.13-beta-90ded76f",
|
|
30
|
+
"@abtnode/rbac": "1.16.13-beta-90ded76f",
|
|
31
|
+
"@abtnode/router-provider": "1.16.13-beta-90ded76f",
|
|
32
|
+
"@abtnode/static-server": "1.16.13-beta-90ded76f",
|
|
33
|
+
"@abtnode/timemachine": "1.16.13-beta-90ded76f",
|
|
34
|
+
"@abtnode/util": "1.16.13-beta-90ded76f",
|
|
34
35
|
"@arcblock/did": "1.18.84",
|
|
35
36
|
"@arcblock/did-auth": "1.18.84",
|
|
36
37
|
"@arcblock/did-ext": "^1.18.84",
|
|
@@ -41,10 +42,10 @@
|
|
|
41
42
|
"@arcblock/pm2-events": "^0.0.5",
|
|
42
43
|
"@arcblock/validator": "^1.18.84",
|
|
43
44
|
"@arcblock/vc": "1.18.84",
|
|
44
|
-
"@blocklet/constant": "1.16.13-beta-
|
|
45
|
-
"@blocklet/meta": "1.16.13-beta-
|
|
46
|
-
"@blocklet/resolver": "1.16.13-beta-
|
|
47
|
-
"@blocklet/sdk": "1.16.13-beta-
|
|
45
|
+
"@blocklet/constant": "1.16.13-beta-90ded76f",
|
|
46
|
+
"@blocklet/meta": "1.16.13-beta-90ded76f",
|
|
47
|
+
"@blocklet/resolver": "1.16.13-beta-90ded76f",
|
|
48
|
+
"@blocklet/sdk": "1.16.13-beta-90ded76f",
|
|
48
49
|
"@did-space/client": "^0.2.117",
|
|
49
50
|
"@fidm/x509": "^1.2.1",
|
|
50
51
|
"@ocap/mcrypto": "1.18.84",
|
|
@@ -56,7 +57,6 @@
|
|
|
56
57
|
"axon": "^2.0.3",
|
|
57
58
|
"chalk": "^4.1.2",
|
|
58
59
|
"cross-spawn": "^7.0.3",
|
|
59
|
-
"dayjs": "^1.11.7",
|
|
60
60
|
"deep-diff": "^1.0.2",
|
|
61
61
|
"detect-port": "^1.5.1",
|
|
62
62
|
"escape-string-regexp": "^4.0.0",
|
|
@@ -73,7 +73,6 @@
|
|
|
73
73
|
"kill-port": "^2.0.1",
|
|
74
74
|
"lodash": "^4.17.21",
|
|
75
75
|
"lru-cache": "^6.0.0",
|
|
76
|
-
"moment-timezone": "^0.5.37",
|
|
77
76
|
"node-stream-zip": "^1.15.0",
|
|
78
77
|
"p-limit": "^3.1.0",
|
|
79
78
|
"p-retry": "4.6.1",
|
|
@@ -97,5 +96,5 @@
|
|
|
97
96
|
"express": "^4.18.2",
|
|
98
97
|
"jest": "^27.5.1"
|
|
99
98
|
},
|
|
100
|
-
"gitHead": "
|
|
99
|
+
"gitHead": "e6d4b88fe2cb45e00e5d745e1cd148dc76a019a4"
|
|
101
100
|
}
|