@kne/fastify-app-manager 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +215 -17
- package/index.js +18 -0
- package/libs/controllers/app.js +185 -0
- package/libs/controllers/gateway.js +11 -0
- package/libs/services/app.js +160 -27
- package/libs/services/bootstrap.js +160 -3
- package/libs/utils/env.js +15 -2
- package/libs/utils/loadMonitor.js +73 -0
- package/libs/utils/logFiles.js +168 -20
- package/libs/utils/logRotate.js +162 -0
- package/libs/utils/logTime.js +132 -0
- package/libs/utils/pm2.js +7 -0
- package/libs/utils/requestMetrics.js +132 -0
- package/package.json +3 -2
package/libs/services/app.js
CHANGED
|
@@ -3,10 +3,12 @@ const path = require('node:path');
|
|
|
3
3
|
const crypto = require('node:crypto');
|
|
4
4
|
const fs = require('fs-extra');
|
|
5
5
|
const portPool = require('../utils/portPool');
|
|
6
|
-
const { mergeEnv, maskEnvForResponse, applyEnvPatch, mergePm2Config, normalizeSecretKeys, collectSecretKeys } = require('../utils/env');
|
|
6
|
+
const { mergeEnv, maskEnvForResponse, applyEnvPatch, mergePm2Config, normalizeSecretKeys, collectSecretKeys, isSystemEnvKey, omitSystemEnv } = require('../utils/env');
|
|
7
7
|
const { prepareVersionArtifact } = require('../utils/version');
|
|
8
8
|
const { injectEntryHtml } = require('../utils/entryInject');
|
|
9
|
-
const { ensureLogFiles, readLogTail, readLastLines, appendLog } = require('../utils/logFiles');
|
|
9
|
+
const { ensureLogFiles, readLogTail, readLastLines, appendLog, countLines } = require('../utils/logFiles');
|
|
10
|
+
const { currentFileName, isCurrentFileName, resolveLogFile, listLogFiles } = require('../utils/logRotate');
|
|
11
|
+
const { toIsoInTz, convertLogText } = require('../utils/logTime');
|
|
10
12
|
const { runSqlMigrations, MIGRATION_ACTIONS } = require('../utils/migrate');
|
|
11
13
|
const { listSqlFiles } = require('../utils/validatePackage');
|
|
12
14
|
const pm2Util = require('../utils/pm2');
|
|
@@ -31,6 +33,18 @@ module.exports = fp(async (fastify, options) => {
|
|
|
31
33
|
const mountPrefix = name => `${options.pathPrefix.replace(/\/$/, '')}/${name}`;
|
|
32
34
|
|
|
33
35
|
const resolveSecretEnvKeys = appLike => normalizeSecretKeys(appLike?.options?.secretEnvKeys);
|
|
36
|
+
// 未设置视为公开(与远程组件默认公开一致)
|
|
37
|
+
const isAppPublic = appLike => appLike?.options?.isPublic !== false;
|
|
38
|
+
// 分组选择器的值可能是整条分组记录(含 id/children/options),只保留 code/name
|
|
39
|
+
const normalizeCategory = category => {
|
|
40
|
+
if (!category) {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
if (typeof category !== 'object') {
|
|
44
|
+
return String(category);
|
|
45
|
+
}
|
|
46
|
+
return category.code ? { code: category.code, name: category.name || category.code } : null;
|
|
47
|
+
};
|
|
34
48
|
|
|
35
49
|
const toPublicApp = app => {
|
|
36
50
|
if (!app) {
|
|
@@ -43,6 +57,8 @@ module.exports = fp(async (fastify, options) => {
|
|
|
43
57
|
json.passthroughEnvKeys = options.passthroughEnvKeys || [];
|
|
44
58
|
json.secretEnvKeys = collectSecretKeys(json.env || {}, envOpts);
|
|
45
59
|
json.pathUrl = `${mountPrefix(json.name)}/`;
|
|
60
|
+
json.category = json.options?.category || null;
|
|
61
|
+
json.isPublic = isAppPublic(json);
|
|
46
62
|
return json;
|
|
47
63
|
};
|
|
48
64
|
|
|
@@ -55,7 +71,7 @@ module.exports = fp(async (fastify, options) => {
|
|
|
55
71
|
};
|
|
56
72
|
|
|
57
73
|
const create = async data => {
|
|
58
|
-
const { name, label, domain, icon, description, env, pm2Config, options: appOptions } = data;
|
|
74
|
+
const { name, label, domain, icon, description, category, isPublic, env, pm2Config, options: appOptions } = data;
|
|
59
75
|
if (!name || !NAME_RE.test(name)) {
|
|
60
76
|
httpError(400, 'invalid name: use lowercase slug [a-z0-9-]');
|
|
61
77
|
}
|
|
@@ -89,6 +105,12 @@ module.exports = fp(async (fastify, options) => {
|
|
|
89
105
|
if (normalizedOptions.secretEnvKeys !== undefined) {
|
|
90
106
|
normalizedOptions.secretEnvKeys = normalizeSecretKeys(normalizedOptions.secretEnvKeys);
|
|
91
107
|
}
|
|
108
|
+
if (category !== undefined) {
|
|
109
|
+
normalizedOptions.category = normalizeCategory(category);
|
|
110
|
+
}
|
|
111
|
+
if (isPublic !== undefined) {
|
|
112
|
+
normalizedOptions.isPublic = !!isPublic;
|
|
113
|
+
}
|
|
92
114
|
|
|
93
115
|
const app = await models.app.create({
|
|
94
116
|
name,
|
|
@@ -96,7 +118,7 @@ module.exports = fp(async (fastify, options) => {
|
|
|
96
118
|
domain: domain || null,
|
|
97
119
|
icon: icon || null,
|
|
98
120
|
description: description || null,
|
|
99
|
-
env: env
|
|
121
|
+
env: omitSystemEnv(env),
|
|
100
122
|
pm2Config: pm2Config || {},
|
|
101
123
|
options: normalizedOptions,
|
|
102
124
|
port,
|
|
@@ -108,7 +130,7 @@ module.exports = fp(async (fastify, options) => {
|
|
|
108
130
|
return toPublicApp(app);
|
|
109
131
|
};
|
|
110
132
|
|
|
111
|
-
const save = async ({ name, ...data }) => {
|
|
133
|
+
const save = async ({ name, category, isPublic, ...data }) => {
|
|
112
134
|
const app = await getByName(name);
|
|
113
135
|
const omit = ['name', 'port', 'rootPath', 'pm2Name', 'status', 'currentVersionId'];
|
|
114
136
|
const patch = {};
|
|
@@ -130,19 +152,25 @@ module.exports = fp(async (fastify, options) => {
|
|
|
130
152
|
}
|
|
131
153
|
if (patch.options && patch.options.secretEnvKeys !== undefined) {
|
|
132
154
|
patch.options = Object.assign({}, patch.options, {
|
|
133
|
-
secretEnvKeys: normalizeSecretKeys(patch.options.secretEnvKeys)
|
|
155
|
+
secretEnvKeys: normalizeSecretKeys([...resolveSecretEnvKeys(app), ...patch.options.secretEnvKeys])
|
|
134
156
|
});
|
|
135
157
|
}
|
|
158
|
+
if (category !== undefined) {
|
|
159
|
+
patch.options = Object.assign({}, patch.options || app.options || {}, { category: normalizeCategory(category) });
|
|
160
|
+
}
|
|
161
|
+
if (isPublic !== undefined) {
|
|
162
|
+
patch.options = Object.assign({}, patch.options || app.options || {}, { isPublic: !!isPublic });
|
|
163
|
+
}
|
|
136
164
|
await app.update(patch);
|
|
137
165
|
return toPublicApp(app);
|
|
138
166
|
};
|
|
139
167
|
|
|
140
168
|
const saveEnv = async ({ name, env, secretEnvKeys: nextSecretKeys } = {}) => {
|
|
141
169
|
const app = await getByName(name);
|
|
142
|
-
|
|
143
|
-
const secretEnvKeys =
|
|
170
|
+
// 密钥类型不可撤销,只随变量删除而移除
|
|
171
|
+
const secretEnvKeys = normalizeSecretKeys([...resolveSecretEnvKeys(app), ...(nextSecretKeys || [])]);
|
|
144
172
|
const nextEnv = applyEnvPatch(app.env || {}, env || {}, Object.assign({}, options, { secretEnvKeys }));
|
|
145
|
-
const cleanedSecrets = secretEnvKeys.filter(key => Object.prototype.hasOwnProperty.call(nextEnv, key));
|
|
173
|
+
const cleanedSecrets = secretEnvKeys.filter(key => !isSystemEnvKey(key) && Object.prototype.hasOwnProperty.call(nextEnv, key));
|
|
146
174
|
await app.update({
|
|
147
175
|
env: nextEnv,
|
|
148
176
|
options: Object.assign({}, app.options || {}, { secretEnvKeys: cleanedSecrets })
|
|
@@ -172,6 +200,22 @@ module.exports = fp(async (fastify, options) => {
|
|
|
172
200
|
};
|
|
173
201
|
};
|
|
174
202
|
|
|
203
|
+
const centerList = async ({ publicOnly = false } = {}) => {
|
|
204
|
+
const rows = await models.app.findAll({ where: { status: 'running' }, order: [['createdAt', 'DESC']] });
|
|
205
|
+
const pageData = rows
|
|
206
|
+
.filter(app => !publicOnly || isAppPublic(app))
|
|
207
|
+
.map(app => ({
|
|
208
|
+
name: app.name,
|
|
209
|
+
label: app.label,
|
|
210
|
+
icon: app.icon,
|
|
211
|
+
description: app.description,
|
|
212
|
+
category: app.options?.category || null,
|
|
213
|
+
isPublic: isAppPublic(app),
|
|
214
|
+
pathUrl: `${mountPrefix(app.name)}/`
|
|
215
|
+
}));
|
|
216
|
+
return { pageData, totalCount: pageData.length };
|
|
217
|
+
};
|
|
218
|
+
|
|
175
219
|
const detail = async ({ name }) => {
|
|
176
220
|
const app = await getByName(name);
|
|
177
221
|
return toPublicApp(app);
|
|
@@ -629,10 +673,103 @@ module.exports = fp(async (fastify, options) => {
|
|
|
629
673
|
};
|
|
630
674
|
};
|
|
631
675
|
|
|
632
|
-
const
|
|
676
|
+
const logsDirOf = app => path.join(app.rootPath, 'logs');
|
|
677
|
+
const currentLogFile = (app, stream) => path.join(logsDirOf(app), currentFileName(stream));
|
|
678
|
+
|
|
679
|
+
const resolveExistingLogFile = async (app, fileName) => {
|
|
680
|
+
let filePath;
|
|
681
|
+
try {
|
|
682
|
+
filePath = resolveLogFile(logsDirOf(app), fileName);
|
|
683
|
+
} catch (e) {
|
|
684
|
+
httpError(400, e.message);
|
|
685
|
+
}
|
|
686
|
+
if (!(await fs.pathExists(filePath))) {
|
|
687
|
+
httpError(404, `log file ${fileName} not found`);
|
|
688
|
+
}
|
|
689
|
+
return filePath;
|
|
690
|
+
};
|
|
691
|
+
|
|
692
|
+
const logs = async ({ name, stream = 'out', file, perPage = 100, currentPage = 1, beforeLine } = {}) => {
|
|
633
693
|
const app = await getByName(name);
|
|
634
|
-
const
|
|
635
|
-
return readLogTail(
|
|
694
|
+
const filePath = file ? await resolveExistingLogFile(app, file) : currentLogFile(app, stream);
|
|
695
|
+
return readLogTail(filePath, { perPage, currentPage, beforeLine, timezone: options.logTimezone });
|
|
696
|
+
};
|
|
697
|
+
|
|
698
|
+
const logFiles = async ({ name }) => {
|
|
699
|
+
const app = await getByName(name);
|
|
700
|
+
const pageData = (await listLogFiles(logsDirOf(app))).map(item => Object.assign({}, item, { mtime: toIsoInTz(item.mtime, options.logTimezone) }));
|
|
701
|
+
return { pageData, totalCount: pageData.length };
|
|
702
|
+
};
|
|
703
|
+
|
|
704
|
+
const resolveAppLogFile = async ({ name, file }) => {
|
|
705
|
+
const app = await getByName(name);
|
|
706
|
+
const filePath = await resolveExistingLogFile(app, file);
|
|
707
|
+
const { size } = await fs.stat(filePath);
|
|
708
|
+
return { path: filePath, fileName: file, size, compressed: file.endsWith('.gz') };
|
|
709
|
+
};
|
|
710
|
+
|
|
711
|
+
const resolveAppLogFiles = async ({ name, files = [] }) => {
|
|
712
|
+
const app = await getByName(name);
|
|
713
|
+
const items = [];
|
|
714
|
+
for (const fileName of Array.from(new Set(files))) {
|
|
715
|
+
const filePath = await resolveExistingLogFile(app, fileName);
|
|
716
|
+
const { size, mtime } = await fs.stat(filePath);
|
|
717
|
+
items.push({ path: filePath, fileName, size, mtime, compressed: fileName.endsWith('.gz') });
|
|
718
|
+
}
|
|
719
|
+
return { name: app.name, files: items };
|
|
720
|
+
};
|
|
721
|
+
|
|
722
|
+
const removeLogFiles = async ({ name, files = [] }) => {
|
|
723
|
+
const app = await getByName(name);
|
|
724
|
+
const targets = [];
|
|
725
|
+
for (const fileName of files) {
|
|
726
|
+
if (isCurrentFileName(fileName)) {
|
|
727
|
+
httpError(400, `cannot remove current log file ${fileName}`);
|
|
728
|
+
}
|
|
729
|
+
try {
|
|
730
|
+
targets.push({ fileName, filePath: resolveLogFile(logsDirOf(app), fileName) });
|
|
731
|
+
} catch (e) {
|
|
732
|
+
httpError(400, e.message);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
const removed = [];
|
|
736
|
+
for (const { fileName, filePath } of targets) {
|
|
737
|
+
if (await fs.pathExists(filePath)) {
|
|
738
|
+
await fs.remove(filePath);
|
|
739
|
+
removed.push(fileName);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
return { removed };
|
|
743
|
+
};
|
|
744
|
+
|
|
745
|
+
const getLoadStore = () => fastify[options.name].loadStore;
|
|
746
|
+
|
|
747
|
+
const load = async ({ name }) => {
|
|
748
|
+
const app = await getByName(name);
|
|
749
|
+
const store = getLoadStore();
|
|
750
|
+
return {
|
|
751
|
+
intervalMs: options.loadSampleIntervalMs > 0 ? options.loadSampleIntervalMs : 0,
|
|
752
|
+
current: store ? store.latest(app.name) : null,
|
|
753
|
+
pageData: store ? store.history(app.name) : []
|
|
754
|
+
};
|
|
755
|
+
};
|
|
756
|
+
|
|
757
|
+
const emitAppLog = async (appName, stream, content) => {
|
|
758
|
+
const hub = getLogHub();
|
|
759
|
+
if (!hub) {
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
const app = await models.app.findOne({ where: { name: appName } });
|
|
763
|
+
if (!app) {
|
|
764
|
+
return;
|
|
765
|
+
}
|
|
766
|
+
hub.emit(`log:${appName}`, {
|
|
767
|
+
appName,
|
|
768
|
+
stream,
|
|
769
|
+
content: convertLogText(content, options.logTimezone),
|
|
770
|
+
line: await countLines(currentLogFile(app, stream)),
|
|
771
|
+
loggedAt: toIsoInTz(new Date(), options.logTimezone)
|
|
772
|
+
});
|
|
636
773
|
};
|
|
637
774
|
|
|
638
775
|
const findByHost = async (host, { running = true } = {}) => {
|
|
@@ -653,6 +790,7 @@ module.exports = fp(async (fastify, options) => {
|
|
|
653
790
|
save,
|
|
654
791
|
saveEnv,
|
|
655
792
|
list,
|
|
793
|
+
centerList,
|
|
656
794
|
detail,
|
|
657
795
|
uploadVersion,
|
|
658
796
|
listVersions,
|
|
@@ -664,34 +802,29 @@ module.exports = fp(async (fastify, options) => {
|
|
|
664
802
|
syncAllStatuses,
|
|
665
803
|
remove,
|
|
666
804
|
logs,
|
|
805
|
+
logFiles,
|
|
806
|
+
resolveAppLogFile,
|
|
807
|
+
resolveAppLogFiles,
|
|
808
|
+
removeLogFiles,
|
|
809
|
+
load,
|
|
810
|
+
getLoadStore,
|
|
667
811
|
toPublicApp,
|
|
668
812
|
mountPrefix,
|
|
669
813
|
findByHost,
|
|
670
814
|
findByPathName,
|
|
671
815
|
readLastLines: async (name, stream, n) => {
|
|
672
816
|
const app = await getByName(name);
|
|
673
|
-
|
|
674
|
-
return readLastLines(file, n);
|
|
817
|
+
return readLastLines(currentLogFile(app, stream), n, { timezone: options.logTimezone });
|
|
675
818
|
},
|
|
676
819
|
getLogHub,
|
|
820
|
+
emitAppLog,
|
|
677
821
|
appendAppLog: async (appName, stream, content) => {
|
|
678
822
|
const app = await models.app.findOne({ where: { name: appName } });
|
|
679
823
|
if (!app) {
|
|
680
824
|
return;
|
|
681
825
|
}
|
|
682
|
-
|
|
683
|
-
await
|
|
684
|
-
const hub = getLogHub();
|
|
685
|
-
if (hub) {
|
|
686
|
-
const { totalCount } = await readLogTail(file, { perPage: 1, currentPage: 1 });
|
|
687
|
-
hub.emit(`log:${appName}`, {
|
|
688
|
-
appName,
|
|
689
|
-
stream,
|
|
690
|
-
content,
|
|
691
|
-
line: totalCount,
|
|
692
|
-
loggedAt: new Date().toISOString()
|
|
693
|
-
});
|
|
694
|
-
}
|
|
826
|
+
await appendLog(currentLogFile(app, stream), content);
|
|
827
|
+
await emitAppLog(appName, stream, content);
|
|
695
828
|
}
|
|
696
829
|
}
|
|
697
830
|
});
|
|
@@ -1,14 +1,34 @@
|
|
|
1
1
|
const fp = require('fastify-plugin');
|
|
2
|
+
const path = require('node:path');
|
|
3
|
+
const fs = require('fs-extra');
|
|
2
4
|
const { EventEmitter } = require('node:events');
|
|
3
5
|
const pm2Util = require('../utils/pm2');
|
|
6
|
+
const { LOG_STREAMS, rotateIfNeeded, compressArchive, applyRetention } = require('../utils/logRotate');
|
|
7
|
+
const { summarizeProcesses, createLoadStore } = require('../utils/loadMonitor');
|
|
8
|
+
const { createRequestMetrics } = require('../utils/requestMetrics');
|
|
9
|
+
const { toIsoInTz } = require('../utils/logTime');
|
|
10
|
+
|
|
11
|
+
const OFFLINE_LOAD = { status: 'offline', cpu: 0, memory: 0, instances: 0, pids: [], uptime: 0, restarts: 0 };
|
|
4
12
|
|
|
5
13
|
module.exports = fp(async (fastify, options) => {
|
|
6
14
|
const logHub = new EventEmitter();
|
|
7
15
|
logHub.setMaxListeners(0);
|
|
8
16
|
fastify[options.name].logHub = logHub;
|
|
9
17
|
|
|
18
|
+
const loadStore = createLoadStore({
|
|
19
|
+
size: options.loadSampleIntervalMs > 0 ? Math.ceil((options.loadHistoryMinutes * 60 * 1000) / options.loadSampleIntervalMs) : 1
|
|
20
|
+
});
|
|
21
|
+
fastify[options.name].loadStore = loadStore;
|
|
22
|
+
|
|
23
|
+
const requestMetrics = createRequestMetrics({ intervalMs: options.loadSampleIntervalMs });
|
|
24
|
+
fastify[options.name].requestMetrics = requestMetrics;
|
|
25
|
+
|
|
10
26
|
let bus = null;
|
|
11
27
|
let connected = false;
|
|
28
|
+
let rotateTimer = null;
|
|
29
|
+
let rotating = null;
|
|
30
|
+
let loadTimer = null;
|
|
31
|
+
let sampling = null;
|
|
12
32
|
|
|
13
33
|
const mapPm2NameToApp = async pm2Name => {
|
|
14
34
|
const { models } = fastify[options.name];
|
|
@@ -41,7 +61,8 @@ module.exports = fp(async (fastify, options) => {
|
|
|
41
61
|
if (!content) {
|
|
42
62
|
return;
|
|
43
63
|
}
|
|
44
|
-
|
|
64
|
+
// PM2 already writes out_file/error_file; the bus only feeds SSE subscribers.
|
|
65
|
+
await fastify[options.name].services.app.emitAppLog(app.name, stream, content);
|
|
45
66
|
} catch (e) {
|
|
46
67
|
fastify.log.debug({ err: e }, 'log bus handler error');
|
|
47
68
|
}
|
|
@@ -52,16 +73,33 @@ module.exports = fp(async (fastify, options) => {
|
|
|
52
73
|
fastify.log.warn({ err: e }, 'pm2 launchBus failed');
|
|
53
74
|
}
|
|
54
75
|
|
|
76
|
+
if (options.logRotateIntervalMs > 0) {
|
|
77
|
+
rotateTimer = setInterval(() => {
|
|
78
|
+
rotateLogs().catch(err => fastify.log.warn({ err }, 'log rotation failed'));
|
|
79
|
+
}, options.logRotateIntervalMs);
|
|
80
|
+
rotateTimer.unref?.();
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (options.loadSampleIntervalMs > 0) {
|
|
84
|
+
loadTimer = setInterval(() => {
|
|
85
|
+
sampleLoad().catch(err => fastify.log.debug({ err }, 'load sampling failed'));
|
|
86
|
+
}, options.loadSampleIntervalMs);
|
|
87
|
+
loadTimer.unref?.();
|
|
88
|
+
}
|
|
89
|
+
|
|
55
90
|
// 父应用重启后不阻塞 ready:后台按 PM2 真实状态异步对齐 DB
|
|
56
91
|
setImmediate(() => {
|
|
57
92
|
reconcile().catch(err => {
|
|
58
93
|
fastify.log.warn({ err }, 'async reconcile/sync statuses failed');
|
|
59
94
|
});
|
|
95
|
+
rotateLogs().catch(err => fastify.log.warn({ err }, 'log rotation failed'));
|
|
96
|
+
if (options.loadSampleIntervalMs > 0) {
|
|
97
|
+
sampleLoad().catch(err => fastify.log.debug({ err }, 'load sampling failed'));
|
|
98
|
+
}
|
|
60
99
|
});
|
|
61
100
|
};
|
|
62
101
|
|
|
63
102
|
const fsEnsureAppsRoot = async () => {
|
|
64
|
-
const fs = require('fs-extra');
|
|
65
103
|
await fs.ensureDir(options.appsRoot);
|
|
66
104
|
};
|
|
67
105
|
|
|
@@ -73,7 +111,126 @@ module.exports = fp(async (fastify, options) => {
|
|
|
73
111
|
return [];
|
|
74
112
|
};
|
|
75
113
|
|
|
114
|
+
const runRotation = async ({ now = new Date() } = {}) => {
|
|
115
|
+
const { models } = fastify[options.name];
|
|
116
|
+
const apps = await models.app.findAll({ attributes: ['name', 'rootPath'] });
|
|
117
|
+
const targets = [];
|
|
118
|
+
for (const app of apps) {
|
|
119
|
+
const logsDir = path.join(app.rootPath, 'logs');
|
|
120
|
+
if (await fs.pathExists(logsDir)) {
|
|
121
|
+
targets.push({ name: app.name, logsDir });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const rotated = [];
|
|
126
|
+
for (const { name, logsDir } of targets) {
|
|
127
|
+
for (const stream of LOG_STREAMS) {
|
|
128
|
+
try {
|
|
129
|
+
const result = await rotateIfNeeded({
|
|
130
|
+
logsDir,
|
|
131
|
+
stream,
|
|
132
|
+
maxSize: options.logMaxSize,
|
|
133
|
+
daily: options.logRotateDaily,
|
|
134
|
+
timezone: options.logTimezone,
|
|
135
|
+
now
|
|
136
|
+
});
|
|
137
|
+
if (result) {
|
|
138
|
+
rotated.push(Object.assign({ name, stream }, result));
|
|
139
|
+
}
|
|
140
|
+
} catch (e) {
|
|
141
|
+
fastify.log.warn({ err: e, app: name, stream }, 'rotate log file failed');
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Until PM2 reopens its handles it keeps writing into the renamed file, so compress only afterwards.
|
|
147
|
+
if (rotated.length && connected) {
|
|
148
|
+
try {
|
|
149
|
+
await pm2Util.reloadLogs();
|
|
150
|
+
} catch (e) {
|
|
151
|
+
fastify.log.warn({ err: e }, 'pm2 reloadLogs failed');
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if (options.logCompress) {
|
|
156
|
+
for (const item of rotated) {
|
|
157
|
+
try {
|
|
158
|
+
item.archive = await compressArchive(item.archive);
|
|
159
|
+
} catch (e) {
|
|
160
|
+
fastify.log.warn({ err: e, app: item.name, archive: item.archive }, 'compress log archive failed');
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
for (const { name, logsDir } of targets) {
|
|
166
|
+
for (const stream of LOG_STREAMS) {
|
|
167
|
+
try {
|
|
168
|
+
await applyRetention({ logsDir, stream, maxFiles: options.logRetentionMaxFiles, maxDays: options.logRetentionDays, now });
|
|
169
|
+
} catch (e) {
|
|
170
|
+
fastify.log.warn({ err: e, app: name, stream }, 'log retention failed');
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return rotated.map(({ name, stream, archive, reason }) => ({ name, stream, fileName: path.basename(archive), reason }));
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const rotateLogs = async opts => {
|
|
179
|
+
if (rotating) {
|
|
180
|
+
return rotating;
|
|
181
|
+
}
|
|
182
|
+
rotating = runRotation(opts).finally(() => {
|
|
183
|
+
rotating = null;
|
|
184
|
+
});
|
|
185
|
+
return rotating;
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
const runSampling = async ({ now = new Date() } = {}) => {
|
|
189
|
+
if (!connected) {
|
|
190
|
+
return [];
|
|
191
|
+
}
|
|
192
|
+
const { models } = fastify[options.name];
|
|
193
|
+
const summary = summarizeProcesses(await pm2Util.list(), { now: now.getTime() });
|
|
194
|
+
const apps = await models.app.findAll({ attributes: ['name', 'pm2Name'] });
|
|
195
|
+
const base = { ts: now.getTime(), sampledAt: toIsoInTz(now, options.logTimezone) };
|
|
196
|
+
const samples = apps.map(app => {
|
|
197
|
+
const sample = Object.assign({ name: app.name }, base, summary.get(app.pm2Name) || OFFLINE_LOAD, {
|
|
198
|
+
requests: requestMetrics.snapshot(app.name, { now: base.ts })
|
|
199
|
+
});
|
|
200
|
+
loadStore.push(app.name, sample);
|
|
201
|
+
return sample;
|
|
202
|
+
});
|
|
203
|
+
const names = apps.map(app => app.name);
|
|
204
|
+
loadStore.retain(names);
|
|
205
|
+
requestMetrics.retain(names);
|
|
206
|
+
return samples;
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
const sampleLoad = async opts => {
|
|
210
|
+
if (sampling) {
|
|
211
|
+
return sampling;
|
|
212
|
+
}
|
|
213
|
+
sampling = runSampling(opts).finally(() => {
|
|
214
|
+
sampling = null;
|
|
215
|
+
});
|
|
216
|
+
return sampling;
|
|
217
|
+
};
|
|
218
|
+
|
|
76
219
|
const onClose = async () => {
|
|
220
|
+
if (rotateTimer) {
|
|
221
|
+
clearInterval(rotateTimer);
|
|
222
|
+
rotateTimer = null;
|
|
223
|
+
}
|
|
224
|
+
if (loadTimer) {
|
|
225
|
+
clearInterval(loadTimer);
|
|
226
|
+
loadTimer = null;
|
|
227
|
+
}
|
|
228
|
+
if (rotating) {
|
|
229
|
+
await rotating.catch(() => {});
|
|
230
|
+
}
|
|
231
|
+
if (sampling) {
|
|
232
|
+
await sampling.catch(() => {});
|
|
233
|
+
}
|
|
77
234
|
if (bus) {
|
|
78
235
|
try {
|
|
79
236
|
bus.close?.();
|
|
@@ -87,6 +244,6 @@ module.exports = fp(async (fastify, options) => {
|
|
|
87
244
|
};
|
|
88
245
|
|
|
89
246
|
Object.assign(fastify[options.name].services, {
|
|
90
|
-
bootstrap: { onReady, onClose, reconcile }
|
|
247
|
+
bootstrap: { onReady, onClose, reconcile, rotateLogs, sampleLoad }
|
|
91
248
|
});
|
|
92
249
|
});
|
package/libs/utils/env.js
CHANGED
|
@@ -105,10 +105,17 @@ const isSecretKey = (key, pattern, secretKeys = []) => {
|
|
|
105
105
|
return isPatternSecretKey(key, pattern);
|
|
106
106
|
};
|
|
107
107
|
|
|
108
|
+
// 由系统写入维护:不在接口中返回,也不接受外部新增 / 修改 / 删除
|
|
109
|
+
const SYSTEM_ENV_KEYS = ['DB_TABLE_PREFIX'];
|
|
110
|
+
|
|
111
|
+
const isSystemEnvKey = key => SYSTEM_ENV_KEYS.includes(key);
|
|
112
|
+
|
|
113
|
+
const omitSystemEnv = (appEnv = {}) => Object.fromEntries(Object.entries(appEnv || {}).filter(([key]) => !isSystemEnvKey(key)));
|
|
114
|
+
|
|
108
115
|
const maskEnvForResponse = (appEnv = {}, { secretEnvKeyPattern, secretEnvKeys = [] } = {}) => {
|
|
109
116
|
const masked = {};
|
|
110
117
|
const explicit = normalizeSecretKeys(secretEnvKeys);
|
|
111
|
-
for (const [key, value] of Object.entries(appEnv
|
|
118
|
+
for (const [key, value] of Object.entries(omitSystemEnv(appEnv))) {
|
|
112
119
|
if (isSecretKey(key, secretEnvKeyPattern, explicit)) {
|
|
113
120
|
masked[key] = value == null || value === '' ? null : SECRET_MASK;
|
|
114
121
|
} else {
|
|
@@ -128,6 +135,9 @@ const applyEnvPatch = (current = {}, patch = {}, { secretEnvKeyPattern, secretEn
|
|
|
128
135
|
const next = Object.assign({}, current);
|
|
129
136
|
const explicit = normalizeSecretKeys(secretEnvKeys);
|
|
130
137
|
for (const [key, value] of Object.entries(patch || {})) {
|
|
138
|
+
if (isSystemEnvKey(key)) {
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
131
141
|
if (value === null) {
|
|
132
142
|
delete next[key];
|
|
133
143
|
continue;
|
|
@@ -146,7 +156,7 @@ const applyEnvPatch = (current = {}, patch = {}, { secretEnvKeyPattern, secretEn
|
|
|
146
156
|
const collectSecretKeys = (appEnv = {}, { secretEnvKeyPattern, secretEnvKeys = [] } = {}) => {
|
|
147
157
|
const explicit = normalizeSecretKeys(secretEnvKeys);
|
|
148
158
|
const fromPattern = Object.keys(appEnv || {}).filter(key => isPatternSecretKey(key, secretEnvKeyPattern));
|
|
149
|
-
return normalizeSecretKeys([...explicit, ...fromPattern]);
|
|
159
|
+
return normalizeSecretKeys([...explicit, ...fromPattern]).filter(key => !isSystemEnvKey(key));
|
|
150
160
|
};
|
|
151
161
|
|
|
152
162
|
const PM2_CONFIG_KEYS = ['exec_mode', 'instances', 'autorestart', 'max_memory_restart', 'max_restarts', 'min_uptime', 'kill_timeout', 'merge_logs'];
|
|
@@ -171,6 +181,9 @@ module.exports = {
|
|
|
171
181
|
mergeEnv,
|
|
172
182
|
isPatternSecretKey,
|
|
173
183
|
isSecretKey,
|
|
184
|
+
SYSTEM_ENV_KEYS,
|
|
185
|
+
isSystemEnvKey,
|
|
186
|
+
omitSystemEnv,
|
|
174
187
|
maskEnvForResponse,
|
|
175
188
|
applyEnvPatch,
|
|
176
189
|
collectSecretKeys,
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
const { EventEmitter } = require('node:events');
|
|
2
|
+
|
|
3
|
+
// cluster 模式下同名进程有多个实例,按 PM2 进程名汇总为一条负载
|
|
4
|
+
const summarizeProcesses = (processes, { now = Date.now() } = {}) => {
|
|
5
|
+
const groups = new Map();
|
|
6
|
+
for (const proc of processes || []) {
|
|
7
|
+
const name = proc?.name || proc?.pm2_env?.name;
|
|
8
|
+
if (!name) {
|
|
9
|
+
continue;
|
|
10
|
+
}
|
|
11
|
+
if (!groups.has(name)) {
|
|
12
|
+
groups.set(name, []);
|
|
13
|
+
}
|
|
14
|
+
groups.get(name).push(proc);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const result = new Map();
|
|
18
|
+
for (const [name, list] of groups) {
|
|
19
|
+
const online = list.filter(proc => proc.pm2_env?.status === 'online');
|
|
20
|
+
const startedAt = online.map(proc => Number(proc.pm2_env?.pm_uptime)).filter(value => Number.isFinite(value) && value > 0);
|
|
21
|
+
const cpu = list.reduce((sum, proc) => sum + (Number(proc.monit?.cpu) || 0), 0);
|
|
22
|
+
result.set(name, {
|
|
23
|
+
status: online.length ? 'online' : list[0].pm2_env?.status || 'unknown',
|
|
24
|
+
cpu: Math.round(cpu * 10) / 10,
|
|
25
|
+
memory: list.reduce((sum, proc) => sum + (Number(proc.monit?.memory) || 0), 0),
|
|
26
|
+
instances: list.length,
|
|
27
|
+
pids: online.map(proc => proc.pid).filter(Boolean),
|
|
28
|
+
uptime: startedAt.length ? Math.max(0, now - Math.min(...startedAt)) : 0,
|
|
29
|
+
restarts: list.reduce((sum, proc) => sum + (Number(proc.pm2_env?.restart_time) || 0), 0)
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const createLoadStore = ({ size }) => {
|
|
36
|
+
const capacity = Math.max(1, Math.floor(size) || 1);
|
|
37
|
+
const buffers = new Map();
|
|
38
|
+
const emitter = new EventEmitter();
|
|
39
|
+
emitter.setMaxListeners(0);
|
|
40
|
+
|
|
41
|
+
const push = (name, sample) => {
|
|
42
|
+
const buffer = buffers.get(name) || [];
|
|
43
|
+
buffer.push(sample);
|
|
44
|
+
if (buffer.length > capacity) {
|
|
45
|
+
buffer.splice(0, buffer.length - capacity);
|
|
46
|
+
}
|
|
47
|
+
buffers.set(name, buffer);
|
|
48
|
+
emitter.emit(`load:${name}`, sample);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const history = name => (buffers.get(name) || []).slice();
|
|
52
|
+
|
|
53
|
+
const latest = name => {
|
|
54
|
+
const buffer = buffers.get(name);
|
|
55
|
+
return buffer && buffer.length ? buffer[buffer.length - 1] : null;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const retain = names => {
|
|
59
|
+
const keep = new Set(names);
|
|
60
|
+
for (const name of buffers.keys()) {
|
|
61
|
+
if (!keep.has(name)) {
|
|
62
|
+
buffers.delete(name);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
return { capacity, push, history, latest, retain, emitter };
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
module.exports = {
|
|
71
|
+
summarizeProcesses,
|
|
72
|
+
createLoadStore
|
|
73
|
+
};
|