@kne/fastify-app-manager 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +198 -21
- package/index.js +17 -0
- package/libs/controllers/app.js +167 -0
- package/libs/controllers/gateway.js +16 -2
- package/libs/services/app.js +175 -28
- package/libs/services/bootstrap.js +160 -3
- package/libs/services/dbops.js +2 -10
- package/libs/utils/dbIdentity.js +16 -7
- package/libs/utils/env.js +39 -7
- package/libs/utils/loadMonitor.js +73 -0
- package/libs/utils/logFiles.js +168 -20
- package/libs/utils/logRotate.js +162 -0
- package/libs/utils/logTime.js +132 -0
- package/libs/utils/pathRewrite.js +28 -3
- package/libs/utils/pm2.js +7 -0
- package/libs/utils/requestMetrics.js +132 -0
- package/package.json +3 -2
package/libs/utils/env.js
CHANGED
|
@@ -23,6 +23,22 @@ const hasAppDbConfig = (appEnv = {}) => {
|
|
|
23
23
|
return !!appEnv.DB_DATABASE;
|
|
24
24
|
};
|
|
25
25
|
|
|
26
|
+
const DB_DEDICATED_KEY = 'DB_DEDICATED';
|
|
27
|
+
const DB_CONNECTION_KEYS = ['DB_DIALECT', 'DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_USERNAME', 'DB_PASSWORD', 'DB_STORAGE'];
|
|
28
|
+
|
|
29
|
+
// 配好独立库后默认启用;DB_DEDICATED=false 时保留配置但回落共享库
|
|
30
|
+
const isAppDbEnabled = (appEnv = {}) =>
|
|
31
|
+
hasAppDbConfig(appEnv) &&
|
|
32
|
+
String(appEnv[DB_DEDICATED_KEY] ?? '')
|
|
33
|
+
.trim()
|
|
34
|
+
.toLowerCase() !== 'false';
|
|
35
|
+
|
|
36
|
+
const omitDbConnection = (appEnv = {}) => {
|
|
37
|
+
const next = Object.assign({}, appEnv);
|
|
38
|
+
DB_CONNECTION_KEYS.forEach(key => delete next[key]);
|
|
39
|
+
return next;
|
|
40
|
+
};
|
|
41
|
+
|
|
26
42
|
const buildDefaultAppDbEnv = (defaultAppDb = {}) => {
|
|
27
43
|
const dialect = defaultAppDb.dialect || 'sqlite';
|
|
28
44
|
const env = { DB_DIALECT: dialect };
|
|
@@ -59,10 +75,11 @@ const buildDefaultAppDbEnv = (defaultAppDb = {}) => {
|
|
|
59
75
|
*/
|
|
60
76
|
const resolveAppDbEnv = ({ passthroughEnvKeys = [], appEnv = {}, defaultAppDb = null, port = null, includePort = false, appName = null, injectTablePrefix = true } = {}) => {
|
|
61
77
|
const base = Object.assign({}, pickHostEnv(passthroughEnvKeys));
|
|
62
|
-
|
|
78
|
+
const useAppDb = isAppDbEnabled(appEnv || {});
|
|
79
|
+
if (defaultAppDb && !useAppDb) {
|
|
63
80
|
Object.assign(base, buildDefaultAppDbEnv(defaultAppDb));
|
|
64
81
|
}
|
|
65
|
-
Object.assign(base, appEnv || {});
|
|
82
|
+
Object.assign(base, useAppDb ? appEnv || {} : omitDbConnection(appEnv || {}));
|
|
66
83
|
if (includePort && port != null) {
|
|
67
84
|
base.PORT = String(port);
|
|
68
85
|
}
|
|
@@ -105,10 +122,17 @@ const isSecretKey = (key, pattern, secretKeys = []) => {
|
|
|
105
122
|
return isPatternSecretKey(key, pattern);
|
|
106
123
|
};
|
|
107
124
|
|
|
108
|
-
|
|
125
|
+
// 由系统写入维护:不在接口中返回,也不接受外部新增 / 修改 / 删除;宿主可通过 systemEnvKeys 选项追加
|
|
126
|
+
const SYSTEM_ENV_KEYS = ['DB_TABLE_PREFIX'];
|
|
127
|
+
|
|
128
|
+
const isSystemEnvKey = (key, systemEnvKeys = []) => SYSTEM_ENV_KEYS.includes(key) || (systemEnvKeys || []).includes(key);
|
|
129
|
+
|
|
130
|
+
const omitSystemEnv = (appEnv = {}, systemEnvKeys = []) => Object.fromEntries(Object.entries(appEnv || {}).filter(([key]) => !isSystemEnvKey(key, systemEnvKeys)));
|
|
131
|
+
|
|
132
|
+
const maskEnvForResponse = (appEnv = {}, { secretEnvKeyPattern, secretEnvKeys = [], systemEnvKeys = [] } = {}) => {
|
|
109
133
|
const masked = {};
|
|
110
134
|
const explicit = normalizeSecretKeys(secretEnvKeys);
|
|
111
|
-
for (const [key, value] of Object.entries(appEnv
|
|
135
|
+
for (const [key, value] of Object.entries(omitSystemEnv(appEnv, systemEnvKeys))) {
|
|
112
136
|
if (isSecretKey(key, secretEnvKeyPattern, explicit)) {
|
|
113
137
|
masked[key] = value == null || value === '' ? null : SECRET_MASK;
|
|
114
138
|
} else {
|
|
@@ -124,10 +148,13 @@ const maskEnvForResponse = (appEnv = {}, { secretEnvKeyPattern, secretEnvKeys =
|
|
|
124
148
|
* SECRET_MASK => keep existing (when key is secret by pattern or explicit list)
|
|
125
149
|
* other => set
|
|
126
150
|
*/
|
|
127
|
-
const applyEnvPatch = (current = {}, patch = {}, { secretEnvKeyPattern, secretEnvKeys = [] } = {}) => {
|
|
151
|
+
const applyEnvPatch = (current = {}, patch = {}, { secretEnvKeyPattern, secretEnvKeys = [], systemEnvKeys = [] } = {}) => {
|
|
128
152
|
const next = Object.assign({}, current);
|
|
129
153
|
const explicit = normalizeSecretKeys(secretEnvKeys);
|
|
130
154
|
for (const [key, value] of Object.entries(patch || {})) {
|
|
155
|
+
if (isSystemEnvKey(key, systemEnvKeys)) {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
131
158
|
if (value === null) {
|
|
132
159
|
delete next[key];
|
|
133
160
|
continue;
|
|
@@ -143,10 +170,10 @@ const applyEnvPatch = (current = {}, patch = {}, { secretEnvKeyPattern, secretEn
|
|
|
143
170
|
/**
|
|
144
171
|
* 响应中的密钥键列表 = 显式列表 ∪ 当前 env 中匹配 pattern 的键
|
|
145
172
|
*/
|
|
146
|
-
const collectSecretKeys = (appEnv = {}, { secretEnvKeyPattern, secretEnvKeys = [] } = {}) => {
|
|
173
|
+
const collectSecretKeys = (appEnv = {}, { secretEnvKeyPattern, secretEnvKeys = [], systemEnvKeys = [] } = {}) => {
|
|
147
174
|
const explicit = normalizeSecretKeys(secretEnvKeys);
|
|
148
175
|
const fromPattern = Object.keys(appEnv || {}).filter(key => isPatternSecretKey(key, secretEnvKeyPattern));
|
|
149
|
-
return normalizeSecretKeys([...explicit, ...fromPattern]);
|
|
176
|
+
return normalizeSecretKeys([...explicit, ...fromPattern]).filter(key => !isSystemEnvKey(key, systemEnvKeys));
|
|
150
177
|
};
|
|
151
178
|
|
|
152
179
|
const PM2_CONFIG_KEYS = ['exec_mode', 'instances', 'autorestart', 'max_memory_restart', 'max_restarts', 'min_uptime', 'kill_timeout', 'merge_logs'];
|
|
@@ -166,11 +193,16 @@ module.exports = {
|
|
|
166
193
|
normalizeSecretKeys,
|
|
167
194
|
pickHostEnv,
|
|
168
195
|
hasAppDbConfig,
|
|
196
|
+
DB_DEDICATED_KEY,
|
|
197
|
+
isAppDbEnabled,
|
|
169
198
|
buildDefaultAppDbEnv,
|
|
170
199
|
resolveAppDbEnv,
|
|
171
200
|
mergeEnv,
|
|
172
201
|
isPatternSecretKey,
|
|
173
202
|
isSecretKey,
|
|
203
|
+
SYSTEM_ENV_KEYS,
|
|
204
|
+
isSystemEnvKey,
|
|
205
|
+
omitSystemEnv,
|
|
174
206
|
maskEnvForResponse,
|
|
175
207
|
applyEnvPatch,
|
|
176
208
|
collectSecretKeys,
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
const { EventEmitter } = require('node:events');
|
|
2
|
+
|
|
3
|
+
// cluster 模式下同名进程有多个实例,按 PM2 进程名汇总为一条负载
|
|
4
|
+
const summarizeProcesses = (processes, { now = Date.now() } = {}) => {
|
|
5
|
+
const groups = new Map();
|
|
6
|
+
for (const proc of processes || []) {
|
|
7
|
+
const name = proc?.name || proc?.pm2_env?.name;
|
|
8
|
+
if (!name) {
|
|
9
|
+
continue;
|
|
10
|
+
}
|
|
11
|
+
if (!groups.has(name)) {
|
|
12
|
+
groups.set(name, []);
|
|
13
|
+
}
|
|
14
|
+
groups.get(name).push(proc);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const result = new Map();
|
|
18
|
+
for (const [name, list] of groups) {
|
|
19
|
+
const online = list.filter(proc => proc.pm2_env?.status === 'online');
|
|
20
|
+
const startedAt = online.map(proc => Number(proc.pm2_env?.pm_uptime)).filter(value => Number.isFinite(value) && value > 0);
|
|
21
|
+
const cpu = list.reduce((sum, proc) => sum + (Number(proc.monit?.cpu) || 0), 0);
|
|
22
|
+
result.set(name, {
|
|
23
|
+
status: online.length ? 'online' : list[0].pm2_env?.status || 'unknown',
|
|
24
|
+
cpu: Math.round(cpu * 10) / 10,
|
|
25
|
+
memory: list.reduce((sum, proc) => sum + (Number(proc.monit?.memory) || 0), 0),
|
|
26
|
+
instances: list.length,
|
|
27
|
+
pids: online.map(proc => proc.pid).filter(Boolean),
|
|
28
|
+
uptime: startedAt.length ? Math.max(0, now - Math.min(...startedAt)) : 0,
|
|
29
|
+
restarts: list.reduce((sum, proc) => sum + (Number(proc.pm2_env?.restart_time) || 0), 0)
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const createLoadStore = ({ size }) => {
|
|
36
|
+
const capacity = Math.max(1, Math.floor(size) || 1);
|
|
37
|
+
const buffers = new Map();
|
|
38
|
+
const emitter = new EventEmitter();
|
|
39
|
+
emitter.setMaxListeners(0);
|
|
40
|
+
|
|
41
|
+
const push = (name, sample) => {
|
|
42
|
+
const buffer = buffers.get(name) || [];
|
|
43
|
+
buffer.push(sample);
|
|
44
|
+
if (buffer.length > capacity) {
|
|
45
|
+
buffer.splice(0, buffer.length - capacity);
|
|
46
|
+
}
|
|
47
|
+
buffers.set(name, buffer);
|
|
48
|
+
emitter.emit(`load:${name}`, sample);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const history = name => (buffers.get(name) || []).slice();
|
|
52
|
+
|
|
53
|
+
const latest = name => {
|
|
54
|
+
const buffer = buffers.get(name);
|
|
55
|
+
return buffer && buffer.length ? buffer[buffer.length - 1] : null;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const retain = names => {
|
|
59
|
+
const keep = new Set(names);
|
|
60
|
+
for (const name of buffers.keys()) {
|
|
61
|
+
if (!keep.has(name)) {
|
|
62
|
+
buffers.delete(name);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
return { capacity, push, history, latest, retain, emitter };
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
module.exports = {
|
|
71
|
+
summarizeProcesses,
|
|
72
|
+
createLoadStore
|
|
73
|
+
};
|
package/libs/utils/logFiles.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
const path = require('node:path');
|
|
2
|
+
const zlib = require('node:zlib');
|
|
2
3
|
const fs = require('fs-extra');
|
|
3
4
|
const { EventEmitter } = require('node:events');
|
|
5
|
+
const { convertLinePrefix } = require('./logTime');
|
|
6
|
+
|
|
7
|
+
const NEWLINE = 0x0a;
|
|
8
|
+
const CHUNK_SIZE = 64 * 1024;
|
|
4
9
|
|
|
5
10
|
const createLogHub = () => {
|
|
6
11
|
const emitter = new EventEmitter();
|
|
@@ -22,38 +27,180 @@ const appendLog = async (filePath, content) => {
|
|
|
22
27
|
await fs.appendFile(filePath, content.endsWith('\n') ? content : `${content}\n`);
|
|
23
28
|
};
|
|
24
29
|
|
|
25
|
-
const
|
|
30
|
+
const lineCache = new Map();
|
|
31
|
+
const gzipLineCache = new Map();
|
|
32
|
+
|
|
33
|
+
const countNewlines = buffer => {
|
|
34
|
+
let count = 0;
|
|
35
|
+
for (let idx = buffer.indexOf(NEWLINE); idx !== -1; idx = buffer.indexOf(NEWLINE, idx + 1)) {
|
|
36
|
+
count++;
|
|
37
|
+
}
|
|
38
|
+
return count;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/** Counts lines incrementally; the file is only rescanned from 0 after rotation (inode change) or truncation. */
|
|
42
|
+
const getLineInfo = async filePath => {
|
|
43
|
+
let stat;
|
|
44
|
+
try {
|
|
45
|
+
stat = await fs.stat(filePath);
|
|
46
|
+
} catch (e) {
|
|
47
|
+
lineCache.delete(filePath);
|
|
48
|
+
return { count: 0, size: 0 };
|
|
49
|
+
}
|
|
50
|
+
const cached = lineCache.get(filePath);
|
|
51
|
+
let { newlines, lastByte, size } = cached && cached.ino === stat.ino && stat.size >= cached.size ? cached : { newlines: 0, lastByte: null, size: 0 };
|
|
52
|
+
if (stat.size > size) {
|
|
53
|
+
for await (const chunk of fs.createReadStream(filePath, { start: size, end: stat.size - 1 })) {
|
|
54
|
+
newlines += countNewlines(chunk);
|
|
55
|
+
lastByte = chunk[chunk.length - 1];
|
|
56
|
+
}
|
|
57
|
+
size = stat.size;
|
|
58
|
+
}
|
|
59
|
+
lineCache.set(filePath, { ino: stat.ino, size, newlines, lastByte });
|
|
60
|
+
return { count: newlines + (size > 0 && lastByte !== NEWLINE ? 1 : 0), size };
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const countLines = async filePath => (await getLineInfo(filePath)).count;
|
|
64
|
+
|
|
65
|
+
/** Reads `take` lines ending `skip` lines before the end of the first `size` bytes; newest first. */
|
|
66
|
+
const readLinesFromEnd = async (filePath, { size, skip, take }) => {
|
|
67
|
+
const collected = [];
|
|
68
|
+
let skipped = 0;
|
|
69
|
+
const accept = buffer => {
|
|
70
|
+
if (skipped < skip) {
|
|
71
|
+
skipped++;
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
collected.push(buffer.toString('utf8').replace(/\r$/, ''));
|
|
75
|
+
return collected.length < take;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const handle = await fs.promises.open(filePath, 'r');
|
|
79
|
+
try {
|
|
80
|
+
let pos = size;
|
|
81
|
+
let carry = Buffer.alloc(0);
|
|
82
|
+
let first = true;
|
|
83
|
+
let more = true;
|
|
84
|
+
while (pos > 0 && more) {
|
|
85
|
+
const length = Math.min(CHUNK_SIZE, pos);
|
|
86
|
+
pos -= length;
|
|
87
|
+
const chunk = Buffer.alloc(length);
|
|
88
|
+
await handle.read(chunk, 0, length, pos);
|
|
89
|
+
let data = carry.length ? Buffer.concat([chunk, carry]) : chunk;
|
|
90
|
+
if (first) {
|
|
91
|
+
first = false;
|
|
92
|
+
if (data[data.length - 1] === NEWLINE) {
|
|
93
|
+
data = data.subarray(0, data.length - 1);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
let end = data.length;
|
|
97
|
+
let idx = data.lastIndexOf(NEWLINE);
|
|
98
|
+
while (idx !== -1 && more) {
|
|
99
|
+
more = accept(data.subarray(idx + 1, end));
|
|
100
|
+
end = idx;
|
|
101
|
+
idx = idx > 0 ? data.lastIndexOf(NEWLINE, idx - 1) : -1;
|
|
102
|
+
}
|
|
103
|
+
carry = data.subarray(0, end);
|
|
104
|
+
}
|
|
105
|
+
if (more && pos === 0 && size > 0) {
|
|
106
|
+
accept(carry);
|
|
107
|
+
}
|
|
108
|
+
} finally {
|
|
109
|
+
await handle.close();
|
|
110
|
+
}
|
|
111
|
+
return collected;
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
// Splits on `\n` only (same rule as plain files); `onLine` returns false to stop early.
|
|
115
|
+
const scanGzipLines = async (filePath, onLine) => {
|
|
116
|
+
const input = fs.createReadStream(filePath);
|
|
117
|
+
const gunzip = zlib.createGunzip();
|
|
118
|
+
input.pipe(gunzip);
|
|
119
|
+
let index = 0;
|
|
120
|
+
let carry = Buffer.alloc(0);
|
|
121
|
+
try {
|
|
122
|
+
for await (const chunk of gunzip) {
|
|
123
|
+
const data = carry.length ? Buffer.concat([carry, chunk]) : chunk;
|
|
124
|
+
let begin = 0;
|
|
125
|
+
for (let idx = data.indexOf(NEWLINE); idx !== -1; idx = data.indexOf(NEWLINE, begin)) {
|
|
126
|
+
if (onLine(data.subarray(begin, idx), index++) === false) {
|
|
127
|
+
return index;
|
|
128
|
+
}
|
|
129
|
+
begin = idx + 1;
|
|
130
|
+
}
|
|
131
|
+
carry = data.subarray(begin);
|
|
132
|
+
}
|
|
133
|
+
if (carry.length) {
|
|
134
|
+
onLine(carry, index++);
|
|
135
|
+
}
|
|
136
|
+
return index;
|
|
137
|
+
} finally {
|
|
138
|
+
input.destroy();
|
|
139
|
+
gunzip.destroy();
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
const countGzipLines = async filePath => {
|
|
144
|
+
const stat = await fs.stat(filePath);
|
|
145
|
+
const key = `${stat.mtimeMs}:${stat.size}`;
|
|
146
|
+
const cached = gzipLineCache.get(filePath);
|
|
147
|
+
if (cached && cached.key === key) {
|
|
148
|
+
return cached.count;
|
|
149
|
+
}
|
|
150
|
+
const count = await scanGzipLines(filePath, () => true);
|
|
151
|
+
gzipLineCache.set(filePath, { key, count });
|
|
152
|
+
return count;
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
const readGzipRange = async (filePath, start, end) => {
|
|
156
|
+
const lines = [];
|
|
157
|
+
await scanGzipLines(filePath, (buffer, index) => {
|
|
158
|
+
if (index >= start) {
|
|
159
|
+
lines.push(buffer.toString('utf8').replace(/\r$/, ''));
|
|
160
|
+
}
|
|
161
|
+
return index + 1 < end;
|
|
162
|
+
});
|
|
163
|
+
return lines.reverse();
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const resolvePageRange = (totalCount, { perPage, currentPage, beforeLine }) => {
|
|
26
167
|
const size = Math.min(100, Math.max(1, Number(perPage) || 100));
|
|
27
168
|
const page = Math.max(1, Number(currentPage) || 1);
|
|
169
|
+
if (beforeLine != null && beforeLine !== '' && Number(beforeLine) > 0) {
|
|
170
|
+
// 取 line < beforeLine 的尾部 size 行(0-based slice end = beforeLine-1)
|
|
171
|
+
const end = Math.min(Math.max(0, Number(beforeLine) - 1), totalCount);
|
|
172
|
+
return { start: Math.max(0, end - size), end };
|
|
173
|
+
}
|
|
174
|
+
return {
|
|
175
|
+
start: Math.max(0, totalCount - size * page),
|
|
176
|
+
end: Math.max(0, totalCount - size * (page - 1))
|
|
177
|
+
};
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
const readLogTail = async (filePath, { perPage = 100, currentPage = 1, beforeLine, timezone } = {}) => {
|
|
28
181
|
if (!(await fs.pathExists(filePath))) {
|
|
29
182
|
return { pageData: [], totalCount: 0, hasMore: false };
|
|
30
183
|
}
|
|
31
|
-
const
|
|
32
|
-
const
|
|
33
|
-
const totalCount =
|
|
34
|
-
|
|
35
|
-
let
|
|
36
|
-
if (
|
|
37
|
-
|
|
38
|
-
end = Math.min(Math.max(0, Number(beforeLine) - 1), totalCount);
|
|
39
|
-
start = Math.max(0, end - size);
|
|
40
|
-
} else {
|
|
41
|
-
start = Math.max(0, totalCount - size * page);
|
|
42
|
-
end = Math.max(0, totalCount - size * (page - 1));
|
|
184
|
+
const compressed = filePath.endsWith('.gz');
|
|
185
|
+
const info = compressed ? { count: await countGzipLines(filePath) } : await getLineInfo(filePath);
|
|
186
|
+
const totalCount = info.count;
|
|
187
|
+
const { start, end } = resolvePageRange(totalCount, { perPage, currentPage, beforeLine });
|
|
188
|
+
let contents = [];
|
|
189
|
+
if (end > start) {
|
|
190
|
+
contents = compressed ? await readGzipRange(filePath, start, end) : await readLinesFromEnd(filePath, { size: info.size, skip: totalCount - end, take: end - start });
|
|
43
191
|
}
|
|
44
|
-
const pageData = lines.slice(start, end).map((content, i) => ({
|
|
45
|
-
line: start + i + 1,
|
|
46
|
-
content
|
|
47
|
-
}));
|
|
48
192
|
return {
|
|
49
|
-
pageData:
|
|
193
|
+
pageData: contents.map((content, i) => ({
|
|
194
|
+
line: end - i,
|
|
195
|
+
content: convertLinePrefix(content, timezone)
|
|
196
|
+
})),
|
|
50
197
|
totalCount,
|
|
51
198
|
hasMore: start > 0
|
|
52
199
|
};
|
|
53
200
|
};
|
|
54
201
|
|
|
55
|
-
const readLastLines = async (filePath, n = 100) => {
|
|
56
|
-
const { pageData } = await readLogTail(filePath, { perPage: n, currentPage: 1 });
|
|
202
|
+
const readLastLines = async (filePath, n = 100, { timezone } = {}) => {
|
|
203
|
+
const { pageData } = await readLogTail(filePath, { perPage: n, currentPage: 1, timezone });
|
|
57
204
|
return pageData.reverse();
|
|
58
205
|
};
|
|
59
206
|
|
|
@@ -61,6 +208,7 @@ module.exports = {
|
|
|
61
208
|
createLogHub,
|
|
62
209
|
ensureLogFiles,
|
|
63
210
|
appendLog,
|
|
211
|
+
countLines,
|
|
64
212
|
readLogTail,
|
|
65
213
|
readLastLines
|
|
66
214
|
};
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
const path = require('node:path');
|
|
2
|
+
const zlib = require('node:zlib');
|
|
3
|
+
const { pipeline } = require('node:stream/promises');
|
|
4
|
+
const fs = require('fs-extra');
|
|
5
|
+
const { formatInTz, dateKeyInTz } = require('./logTime');
|
|
6
|
+
|
|
7
|
+
const LOG_STREAMS = ['out', 'err'];
|
|
8
|
+
const STATE_FILE = '.rotate-state.json';
|
|
9
|
+
const LOG_FILE_RE = /^(out|err)(?:-\d{8}-\d{6}(?:-\d+)?)?\.log(\.gz)?$/;
|
|
10
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
11
|
+
|
|
12
|
+
const currentFileName = stream => `${stream === 'err' ? 'err' : 'out'}.log`;
|
|
13
|
+
|
|
14
|
+
const isCurrentFileName = fileName => LOG_STREAMS.some(stream => currentFileName(stream) === fileName);
|
|
15
|
+
|
|
16
|
+
const badRequest = message => {
|
|
17
|
+
const err = new Error(message);
|
|
18
|
+
err.statusCode = 400;
|
|
19
|
+
return err;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
const resolveLogFile = (logsDir, fileName) => {
|
|
23
|
+
if (typeof fileName !== 'string' || path.basename(fileName) !== fileName || !LOG_FILE_RE.test(fileName)) {
|
|
24
|
+
throw badRequest(`invalid log file: ${fileName}`);
|
|
25
|
+
}
|
|
26
|
+
return path.join(logsDir, fileName);
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const readState = async logsDir => {
|
|
30
|
+
try {
|
|
31
|
+
return (await fs.readJson(path.join(logsDir, STATE_FILE))) || {};
|
|
32
|
+
} catch (e) {
|
|
33
|
+
return {};
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const writeState = (logsDir, state) => fs.writeJson(path.join(logsDir, STATE_FILE), state);
|
|
38
|
+
|
|
39
|
+
const resolveStartTime = (state, stream, stat) => {
|
|
40
|
+
const recorded = state[stream] ? new Date(state[stream]) : null;
|
|
41
|
+
if (recorded && !Number.isNaN(recorded.getTime())) {
|
|
42
|
+
return recorded;
|
|
43
|
+
}
|
|
44
|
+
return stat.birthtimeMs > 0 ? stat.birthtime : stat.mtime;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const uniqueArchivePath = async (logsDir, baseName) => {
|
|
48
|
+
for (let n = 0; ; n++) {
|
|
49
|
+
const target = path.join(logsDir, n ? `${baseName}-${n}.log` : `${baseName}.log`);
|
|
50
|
+
if (!(await fs.pathExists(target)) && !(await fs.pathExists(`${target}.gz`))) {
|
|
51
|
+
return target;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Renames `{stream}.log` to `{stream}-{startTime}.log` when it exceeds maxSize or its start day
|
|
58
|
+
* (in `timezone`) is before `now`. Caller must reopen the PM2 log handles afterwards.
|
|
59
|
+
*/
|
|
60
|
+
const rotateIfNeeded = async ({ logsDir, stream, maxSize, daily = false, timezone, now = new Date() }) => {
|
|
61
|
+
const file = path.join(logsDir, currentFileName(stream));
|
|
62
|
+
let stat;
|
|
63
|
+
try {
|
|
64
|
+
stat = await fs.stat(file);
|
|
65
|
+
} catch (e) {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
const state = await readState(logsDir);
|
|
69
|
+
const startedAt = resolveStartTime(state, stream, stat);
|
|
70
|
+
const dayChanged = daily && dateKeyInTz(startedAt, timezone) !== dateKeyInTz(now, timezone);
|
|
71
|
+
|
|
72
|
+
let reason = null;
|
|
73
|
+
if (stat.size > 0 && maxSize > 0 && stat.size >= maxSize) {
|
|
74
|
+
reason = 'size';
|
|
75
|
+
} else if (stat.size > 0 && dayChanged) {
|
|
76
|
+
reason = 'daily';
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (!reason) {
|
|
80
|
+
// An empty file carries no content from the old day; restart its clock so today's logs get today's name.
|
|
81
|
+
if (!state[stream] || (stat.size === 0 && dayChanged)) {
|
|
82
|
+
state[stream] = (stat.size === 0 && dayChanged ? now : startedAt).toISOString();
|
|
83
|
+
await writeState(logsDir, state);
|
|
84
|
+
}
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const archive = await uniqueArchivePath(logsDir, `${stream}-${formatInTz(startedAt, timezone, 'YYYYMMDD-HHmmss')}`);
|
|
89
|
+
await fs.rename(file, archive);
|
|
90
|
+
await fs.ensureFile(file);
|
|
91
|
+
state[stream] = now.toISOString();
|
|
92
|
+
await writeState(logsDir, state);
|
|
93
|
+
return { archive, reason };
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
const compressArchive = async file => {
|
|
97
|
+
const target = `${file}.gz`;
|
|
98
|
+
const stat = await fs.stat(file);
|
|
99
|
+
await pipeline(fs.createReadStream(file), zlib.createGzip(), fs.createWriteStream(target));
|
|
100
|
+
await fs.utimes(target, stat.atime, stat.mtime);
|
|
101
|
+
await fs.remove(file);
|
|
102
|
+
return target;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const listLogFiles = async logsDir => {
|
|
106
|
+
let names = [];
|
|
107
|
+
try {
|
|
108
|
+
names = await fs.readdir(logsDir);
|
|
109
|
+
} catch (e) {
|
|
110
|
+
return [];
|
|
111
|
+
}
|
|
112
|
+
const files = [];
|
|
113
|
+
for (const fileName of names) {
|
|
114
|
+
const match = LOG_FILE_RE.exec(fileName);
|
|
115
|
+
if (!match) {
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
let stat;
|
|
119
|
+
try {
|
|
120
|
+
stat = await fs.stat(path.join(logsDir, fileName));
|
|
121
|
+
} catch (e) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (!stat.isFile()) {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
files.push({
|
|
128
|
+
fileName,
|
|
129
|
+
stream: match[1],
|
|
130
|
+
size: stat.size,
|
|
131
|
+
compressed: !!match[2],
|
|
132
|
+
current: isCurrentFileName(fileName),
|
|
133
|
+
mtime: stat.mtime
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
return files.sort((a, b) => Number(b.current) - Number(a.current) || b.mtime - a.mtime);
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const applyRetention = async ({ logsDir, stream, maxFiles, maxDays, now = new Date() }) => {
|
|
140
|
+
const archives = (await listLogFiles(logsDir)).filter(file => !file.current && (!stream || file.stream === stream));
|
|
141
|
+
const removed = [];
|
|
142
|
+
for (const [index, file] of archives.entries()) {
|
|
143
|
+
const tooMany = maxFiles > 0 && index >= maxFiles;
|
|
144
|
+
const tooOld = maxDays > 0 && now.getTime() - file.mtime.getTime() > maxDays * DAY_MS;
|
|
145
|
+
if (tooMany || tooOld) {
|
|
146
|
+
await fs.remove(path.join(logsDir, file.fileName));
|
|
147
|
+
removed.push(file.fileName);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return removed;
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
module.exports = {
|
|
154
|
+
LOG_STREAMS,
|
|
155
|
+
currentFileName,
|
|
156
|
+
isCurrentFileName,
|
|
157
|
+
resolveLogFile,
|
|
158
|
+
rotateIfNeeded,
|
|
159
|
+
compressArchive,
|
|
160
|
+
listLogFiles,
|
|
161
|
+
applyRetention
|
|
162
|
+
};
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
const OFFSET_RE = /^([+-])(\d{2}):?(\d{2})$/;
|
|
2
|
+
const LINE_PREFIX_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?=: )/;
|
|
3
|
+
|
|
4
|
+
const formatterCache = new Map();
|
|
5
|
+
|
|
6
|
+
const getFormatter = timeZone => {
|
|
7
|
+
if (!formatterCache.has(timeZone)) {
|
|
8
|
+
formatterCache.set(
|
|
9
|
+
timeZone,
|
|
10
|
+
new Intl.DateTimeFormat('en-US', {
|
|
11
|
+
timeZone,
|
|
12
|
+
hourCycle: 'h23',
|
|
13
|
+
year: 'numeric',
|
|
14
|
+
month: '2-digit',
|
|
15
|
+
day: '2-digit',
|
|
16
|
+
hour: '2-digit',
|
|
17
|
+
minute: '2-digit',
|
|
18
|
+
second: '2-digit'
|
|
19
|
+
})
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
return formatterCache.get(timeZone);
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
const parsedCache = new Map();
|
|
26
|
+
|
|
27
|
+
const parseTimezone = tz => {
|
|
28
|
+
if (tz && typeof tz === 'object' && (tz.type === 'offset' || tz.type === 'iana')) {
|
|
29
|
+
return tz;
|
|
30
|
+
}
|
|
31
|
+
const value = tz == null ? '' : String(tz).trim();
|
|
32
|
+
if (!parsedCache.has(value)) {
|
|
33
|
+
parsedCache.set(value, parseTimezoneValue(value));
|
|
34
|
+
}
|
|
35
|
+
return parsedCache.get(value);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const parseTimezoneValue = value => {
|
|
39
|
+
const match = OFFSET_RE.exec(value);
|
|
40
|
+
if (match) {
|
|
41
|
+
const hours = Number(match[2]);
|
|
42
|
+
const minutes = Number(match[3]);
|
|
43
|
+
if (hours > 14 || minutes > 59) {
|
|
44
|
+
throw new Error(`invalid logTimezone: ${value}`);
|
|
45
|
+
}
|
|
46
|
+
return { type: 'offset', offsetMinutes: (match[1] === '-' ? -1 : 1) * (hours * 60 + minutes) };
|
|
47
|
+
}
|
|
48
|
+
if (!value) {
|
|
49
|
+
throw new Error('invalid logTimezone: empty');
|
|
50
|
+
}
|
|
51
|
+
try {
|
|
52
|
+
getFormatter(value);
|
|
53
|
+
} catch (e) {
|
|
54
|
+
throw new Error(`invalid logTimezone: ${value}`);
|
|
55
|
+
}
|
|
56
|
+
return { type: 'iana', name: value };
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const getOffsetMinutes = (date, tz) => {
|
|
60
|
+
const info = parseTimezone(tz);
|
|
61
|
+
if (info.type === 'offset') {
|
|
62
|
+
return info.offsetMinutes;
|
|
63
|
+
}
|
|
64
|
+
const parts = {};
|
|
65
|
+
for (const { type, value } of getFormatter(info.name).formatToParts(date)) {
|
|
66
|
+
parts[type] = value;
|
|
67
|
+
}
|
|
68
|
+
const asUtc = Date.UTC(Number(parts.year), Number(parts.month) - 1, Number(parts.day), Number(parts.hour) % 24, Number(parts.minute), Number(parts.second));
|
|
69
|
+
return Math.round((asUtc - Math.floor(date.getTime() / 1000) * 1000) / 60000);
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const pad = (value, length = 2) => String(value).padStart(length, '0');
|
|
73
|
+
|
|
74
|
+
const formatOffset = minutes => {
|
|
75
|
+
const abs = Math.abs(minutes);
|
|
76
|
+
return `${minutes < 0 ? '-' : '+'}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}`;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const formatInTz = (date, tz, pattern = 'YYYY-MM-DDTHH:mm:ss') => {
|
|
80
|
+
const shifted = new Date(date.getTime() + getOffsetMinutes(date, tz) * 60000);
|
|
81
|
+
const tokens = {
|
|
82
|
+
YYYY: pad(shifted.getUTCFullYear(), 4),
|
|
83
|
+
MM: pad(shifted.getUTCMonth() + 1),
|
|
84
|
+
DD: pad(shifted.getUTCDate()),
|
|
85
|
+
HH: pad(shifted.getUTCHours()),
|
|
86
|
+
mm: pad(shifted.getUTCMinutes()),
|
|
87
|
+
ss: pad(shifted.getUTCSeconds())
|
|
88
|
+
};
|
|
89
|
+
return pattern.replace(/YYYY|MM|DD|HH|mm|ss/g, token => tokens[token]);
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const toIsoInTz = (date, tz) => `${formatInTz(date, tz)}${formatOffset(getOffsetMinutes(date, tz))}`;
|
|
93
|
+
|
|
94
|
+
const dateKeyInTz = (date, tz) => formatInTz(date, tz, 'YYYY-MM-DD');
|
|
95
|
+
|
|
96
|
+
// PM2 `time: true` prefixes lines in the daemon's local time without an offset; assume it matches this process.
|
|
97
|
+
const convertLinePrefix = (line, tz) => {
|
|
98
|
+
if (!tz || typeof line !== 'string') {
|
|
99
|
+
return line;
|
|
100
|
+
}
|
|
101
|
+
const match = LINE_PREFIX_RE.exec(line);
|
|
102
|
+
if (!match) {
|
|
103
|
+
return line;
|
|
104
|
+
}
|
|
105
|
+
const [, y, mo, d, h, mi, s] = match.map(Number);
|
|
106
|
+
const date = new Date(y, mo - 1, d, h, mi, s);
|
|
107
|
+
if (Number.isNaN(date.getTime()) || getOffsetMinutes(date, tz) === -date.getTimezoneOffset()) {
|
|
108
|
+
return line;
|
|
109
|
+
}
|
|
110
|
+
return `${formatInTz(date, tz)}${line.slice(match[0].length)}`;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
const convertLogText = (text, tz) => {
|
|
114
|
+
if (!tz || typeof text !== 'string') {
|
|
115
|
+
return text;
|
|
116
|
+
}
|
|
117
|
+
return text
|
|
118
|
+
.split('\n')
|
|
119
|
+
.map(line => convertLinePrefix(line, tz))
|
|
120
|
+
.join('\n');
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
module.exports = {
|
|
124
|
+
parseTimezone,
|
|
125
|
+
getOffsetMinutes,
|
|
126
|
+
formatOffset,
|
|
127
|
+
formatInTz,
|
|
128
|
+
toIsoInTz,
|
|
129
|
+
dateKeyInTz,
|
|
130
|
+
convertLinePrefix,
|
|
131
|
+
convertLogText
|
|
132
|
+
};
|