@kne/fastify-app-manager 0.1.8 → 0.1.9

package/libs/services/app.js

@@ -3,10 +3,12 @@ const path = require('node:path');
 const crypto = require('node:crypto');
 const fs = require('fs-extra');
 const portPool = require('../utils/portPool');
-const { mergeEnv, maskEnvForResponse, applyEnvPatch, mergePm2Config, normalizeSecretKeys, collectSecretKeys } = require('../utils/env');
+const { mergeEnv, maskEnvForResponse, applyEnvPatch, mergePm2Config, normalizeSecretKeys, collectSecretKeys, isSystemEnvKey, omitSystemEnv } = require('../utils/env');
 const { prepareVersionArtifact } = require('../utils/version');
 const { injectEntryHtml } = require('../utils/entryInject');
-const { ensureLogFiles, readLogTail, readLastLines, appendLog } = require('../utils/logFiles');
+const { ensureLogFiles, readLogTail, readLastLines, appendLog, countLines } = require('../utils/logFiles');
+const { currentFileName, isCurrentFileName, resolveLogFile, listLogFiles } = require('../utils/logRotate');
+const { toIsoInTz, convertLogText } = require('../utils/logTime');
 const { runSqlMigrations, MIGRATION_ACTIONS } = require('../utils/migrate');
 const { listSqlFiles } = require('../utils/validatePackage');
 const pm2Util = require('../utils/pm2');
@@ -116,7 +118,7 @@ module.exports = fp(async (fastify, options) => {
       domain: domain || null,
       icon: icon || null,
       description: description || null,
-      env: env || {},
+      env: omitSystemEnv(env),
       pm2Config: pm2Config || {},
       options: normalizedOptions,
       port,
@@ -150,7 +152,7 @@ module.exports = fp(async (fastify, options) => {
     }
     if (patch.options && patch.options.secretEnvKeys !== undefined) {
       patch.options = Object.assign({}, patch.options, {
-        secretEnvKeys: normalizeSecretKeys(patch.options.secretEnvKeys)
+        secretEnvKeys: normalizeSecretKeys([...resolveSecretEnvKeys(app), ...patch.options.secretEnvKeys])
       });
     }
     if (category !== undefined) {
@@ -165,10 +167,10 @@ module.exports = fp(async (fastify, options) => {
 
   const saveEnv = async ({ name, env, secretEnvKeys: nextSecretKeys } = {}) => {
     const app = await getByName(name);
-    const currentSecrets = resolveSecretEnvKeys(app);
-    const secretEnvKeys = nextSecretKeys !== undefined ? normalizeSecretKeys(nextSecretKeys) : currentSecrets;
+    // 密钥类型不可撤销，只随变量删除而移除
+    const secretEnvKeys = normalizeSecretKeys([...resolveSecretEnvKeys(app), ...(nextSecretKeys || [])]);
     const nextEnv = applyEnvPatch(app.env || {}, env || {}, Object.assign({}, options, { secretEnvKeys }));
-    const cleanedSecrets = secretEnvKeys.filter(key => Object.prototype.hasOwnProperty.call(nextEnv, key));
+    const cleanedSecrets = secretEnvKeys.filter(key => !isSystemEnvKey(key) && Object.prototype.hasOwnProperty.call(nextEnv, key));
     await app.update({
       env: nextEnv,
       options: Object.assign({}, app.options || {}, { secretEnvKeys: cleanedSecrets })
@@ -671,10 +673,103 @@ module.exports = fp(async (fastify, options) => {
     };
   };
 
-  const logs = async ({ name, stream = 'out', perPage = 100, currentPage = 1, beforeLine } = {}) => {
+  const logsDirOf = app => path.join(app.rootPath, 'logs');
+  const currentLogFile = (app, stream) => path.join(logsDirOf(app), currentFileName(stream));
+
+  const resolveExistingLogFile = async (app, fileName) => {
+    let filePath;
+    try {
+      filePath = resolveLogFile(logsDirOf(app), fileName);
+    } catch (e) {
+      httpError(400, e.message);
+    }
+    if (!(await fs.pathExists(filePath))) {
+      httpError(404, `log file ${fileName} not found`);
+    }
+    return filePath;
+  };
+
+  const logs = async ({ name, stream = 'out', file, perPage = 100, currentPage = 1, beforeLine } = {}) => {
     const app = await getByName(name);
-    const file = path.join(app.rootPath, 'logs', stream === 'err' ? 'err.log' : 'out.log');
-    return readLogTail(file, { perPage, currentPage, beforeLine });
+    const filePath = file ? await resolveExistingLogFile(app, file) : currentLogFile(app, stream);
+    return readLogTail(filePath, { perPage, currentPage, beforeLine, timezone: options.logTimezone });
+  };
+
+  const logFiles = async ({ name }) => {
+    const app = await getByName(name);
+    const pageData = (await listLogFiles(logsDirOf(app))).map(item => Object.assign({}, item, { mtime: toIsoInTz(item.mtime, options.logTimezone) }));
+    return { pageData, totalCount: pageData.length };
+  };
+
+  const resolveAppLogFile = async ({ name, file }) => {
+    const app = await getByName(name);
+    const filePath = await resolveExistingLogFile(app, file);
+    const { size } = await fs.stat(filePath);
+    return { path: filePath, fileName: file, size, compressed: file.endsWith('.gz') };
+  };
+
+  const resolveAppLogFiles = async ({ name, files = [] }) => {
+    const app = await getByName(name);
+    const items = [];
+    for (const fileName of Array.from(new Set(files))) {
+      const filePath = await resolveExistingLogFile(app, fileName);
+      const { size, mtime } = await fs.stat(filePath);
+      items.push({ path: filePath, fileName, size, mtime, compressed: fileName.endsWith('.gz') });
+    }
+    return { name: app.name, files: items };
+  };
+
+  const removeLogFiles = async ({ name, files = [] }) => {
+    const app = await getByName(name);
+    const targets = [];
+    for (const fileName of files) {
+      if (isCurrentFileName(fileName)) {
+        httpError(400, `cannot remove current log file ${fileName}`);
+      }
+      try {
+        targets.push({ fileName, filePath: resolveLogFile(logsDirOf(app), fileName) });
+      } catch (e) {
+        httpError(400, e.message);
+      }
+    }
+    const removed = [];
+    for (const { fileName, filePath } of targets) {
+      if (await fs.pathExists(filePath)) {
+        await fs.remove(filePath);
+        removed.push(fileName);
+      }
+    }
+    return { removed };
+  };
+
+  const getLoadStore = () => fastify[options.name].loadStore;
+
+  const load = async ({ name }) => {
+    const app = await getByName(name);
+    const store = getLoadStore();
+    return {
+      intervalMs: options.loadSampleIntervalMs > 0 ? options.loadSampleIntervalMs : 0,
+      current: store ? store.latest(app.name) : null,
+      pageData: store ? store.history(app.name) : []
+    };
+  };
+
+  const emitAppLog = async (appName, stream, content) => {
+    const hub = getLogHub();
+    if (!hub) {
+      return;
+    }
+    const app = await models.app.findOne({ where: { name: appName } });
+    if (!app) {
+      return;
+    }
+    hub.emit(`log:${appName}`, {
+      appName,
+      stream,
+      content: convertLogText(content, options.logTimezone),
+      line: await countLines(currentLogFile(app, stream)),
+      loggedAt: toIsoInTz(new Date(), options.logTimezone)
+    });
   };
 
   const findByHost = async (host, { running = true } = {}) => {
@@ -707,34 +802,29 @@ module.exports = fp(async (fastify, options) => {
       syncAllStatuses,
       remove,
       logs,
+      logFiles,
+      resolveAppLogFile,
+      resolveAppLogFiles,
+      removeLogFiles,
+      load,
+      getLoadStore,
       toPublicApp,
       mountPrefix,
       findByHost,
       findByPathName,
       readLastLines: async (name, stream, n) => {
         const app = await getByName(name);
-        const file = path.join(app.rootPath, 'logs', stream === 'err' ? 'err.log' : 'out.log');
-        return readLastLines(file, n);
+        return readLastLines(currentLogFile(app, stream), n, { timezone: options.logTimezone });
       },
       getLogHub,
+      emitAppLog,
       appendAppLog: async (appName, stream, content) => {
         const app = await models.app.findOne({ where: { name: appName } });
         if (!app) {
           return;
         }
-        const file = path.join(app.rootPath, 'logs', stream === 'err' ? 'err.log' : 'out.log');
-        await appendLog(file, content);
-        const hub = getLogHub();
-        if (hub) {
-          const { totalCount } = await readLogTail(file, { perPage: 1, currentPage: 1 });
-          hub.emit(`log:${appName}`, {
-            appName,
-            stream,
-            content,
-            line: totalCount,
-            loggedAt: new Date().toISOString()
-          });
-        }
+        await appendLog(currentLogFile(app, stream), content);
+        await emitAppLog(appName, stream, content);
       }
     }
   });

package/libs/services/bootstrap.js

@@ -1,14 +1,34 @@
 const fp = require('fastify-plugin');
+const path = require('node:path');
+const fs = require('fs-extra');
 const { EventEmitter } = require('node:events');
 const pm2Util = require('../utils/pm2');
+const { LOG_STREAMS, rotateIfNeeded, compressArchive, applyRetention } = require('../utils/logRotate');
+const { summarizeProcesses, createLoadStore } = require('../utils/loadMonitor');
+const { createRequestMetrics } = require('../utils/requestMetrics');
+const { toIsoInTz } = require('../utils/logTime');
+
+const OFFLINE_LOAD = { status: 'offline', cpu: 0, memory: 0, instances: 0, pids: [], uptime: 0, restarts: 0 };
 
 module.exports = fp(async (fastify, options) => {
   const logHub = new EventEmitter();
   logHub.setMaxListeners(0);
   fastify[options.name].logHub = logHub;
 
+  const loadStore = createLoadStore({
+    size: options.loadSampleIntervalMs > 0 ? Math.ceil((options.loadHistoryMinutes * 60 * 1000) / options.loadSampleIntervalMs) : 1
+  });
+  fastify[options.name].loadStore = loadStore;
+
+  const requestMetrics = createRequestMetrics({ intervalMs: options.loadSampleIntervalMs });
+  fastify[options.name].requestMetrics = requestMetrics;
+
   let bus = null;
   let connected = false;
+  let rotateTimer = null;
+  let rotating = null;
+  let loadTimer = null;
+  let sampling = null;
 
   const mapPm2NameToApp = async pm2Name => {
     const { models } = fastify[options.name];
@@ -41,7 +61,8 @@ module.exports = fp(async (fastify, options) => {
           if (!content) {
             return;
           }
-          await fastify[options.name].services.app.appendAppLog(app.name, stream, content);
+          // PM2 already writes out_file/error_file; the bus only feeds SSE subscribers.
+          await fastify[options.name].services.app.emitAppLog(app.name, stream, content);
         } catch (e) {
           fastify.log.debug({ err: e }, 'log bus handler error');
         }
@@ -52,16 +73,33 @@ module.exports = fp(async (fastify, options) => {
       fastify.log.warn({ err: e }, 'pm2 launchBus failed');
     }
 
+    if (options.logRotateIntervalMs > 0) {
+      rotateTimer = setInterval(() => {
+        rotateLogs().catch(err => fastify.log.warn({ err }, 'log rotation failed'));
+      }, options.logRotateIntervalMs);
+      rotateTimer.unref?.();
+    }
+
+    if (options.loadSampleIntervalMs > 0) {
+      loadTimer = setInterval(() => {
+        sampleLoad().catch(err => fastify.log.debug({ err }, 'load sampling failed'));
+      }, options.loadSampleIntervalMs);
+      loadTimer.unref?.();
+    }
+
     // 父应用重启后不阻塞 ready：后台按 PM2 真实状态异步对齐 DB
     setImmediate(() => {
       reconcile().catch(err => {
         fastify.log.warn({ err }, 'async reconcile/sync statuses failed');
       });
+      rotateLogs().catch(err => fastify.log.warn({ err }, 'log rotation failed'));
+      if (options.loadSampleIntervalMs > 0) {
+        sampleLoad().catch(err => fastify.log.debug({ err }, 'load sampling failed'));
+      }
     });
   };
 
   const fsEnsureAppsRoot = async () => {
-    const fs = require('fs-extra');
     await fs.ensureDir(options.appsRoot);
   };
 
@@ -73,7 +111,126 @@ module.exports = fp(async (fastify, options) => {
     return [];
   };
 
+  const runRotation = async ({ now = new Date() } = {}) => {
+    const { models } = fastify[options.name];
+    const apps = await models.app.findAll({ attributes: ['name', 'rootPath'] });
+    const targets = [];
+    for (const app of apps) {
+      const logsDir = path.join(app.rootPath, 'logs');
+      if (await fs.pathExists(logsDir)) {
+        targets.push({ name: app.name, logsDir });
+      }
+    }
+
+    const rotated = [];
+    for (const { name, logsDir } of targets) {
+      for (const stream of LOG_STREAMS) {
+        try {
+          const result = await rotateIfNeeded({
+            logsDir,
+            stream,
+            maxSize: options.logMaxSize,
+            daily: options.logRotateDaily,
+            timezone: options.logTimezone,
+            now
+          });
+          if (result) {
+            rotated.push(Object.assign({ name, stream }, result));
+          }
+        } catch (e) {
+          fastify.log.warn({ err: e, app: name, stream }, 'rotate log file failed');
+        }
+      }
+    }
+
+    // Until PM2 reopens its handles it keeps writing into the renamed file, so compress only afterwards.
+    if (rotated.length && connected) {
+      try {
+        await pm2Util.reloadLogs();
+      } catch (e) {
+        fastify.log.warn({ err: e }, 'pm2 reloadLogs failed');
+      }
+    }
+
+    if (options.logCompress) {
+      for (const item of rotated) {
+        try {
+          item.archive = await compressArchive(item.archive);
+        } catch (e) {
+          fastify.log.warn({ err: e, app: item.name, archive: item.archive }, 'compress log archive failed');
+        }
+      }
+    }
+
+    for (const { name, logsDir } of targets) {
+      for (const stream of LOG_STREAMS) {
+        try {
+          await applyRetention({ logsDir, stream, maxFiles: options.logRetentionMaxFiles, maxDays: options.logRetentionDays, now });
+        } catch (e) {
+          fastify.log.warn({ err: e, app: name, stream }, 'log retention failed');
+        }
+      }
+    }
+
+    return rotated.map(({ name, stream, archive, reason }) => ({ name, stream, fileName: path.basename(archive), reason }));
+  };
+
+  const rotateLogs = async opts => {
+    if (rotating) {
+      return rotating;
+    }
+    rotating = runRotation(opts).finally(() => {
+      rotating = null;
+    });
+    return rotating;
+  };
+
+  const runSampling = async ({ now = new Date() } = {}) => {
+    if (!connected) {
+      return [];
+    }
+    const { models } = fastify[options.name];
+    const summary = summarizeProcesses(await pm2Util.list(), { now: now.getTime() });
+    const apps = await models.app.findAll({ attributes: ['name', 'pm2Name'] });
+    const base = { ts: now.getTime(), sampledAt: toIsoInTz(now, options.logTimezone) };
+    const samples = apps.map(app => {
+      const sample = Object.assign({ name: app.name }, base, summary.get(app.pm2Name) || OFFLINE_LOAD, {
+        requests: requestMetrics.snapshot(app.name, { now: base.ts })
+      });
+      loadStore.push(app.name, sample);
+      return sample;
+    });
+    const names = apps.map(app => app.name);
+    loadStore.retain(names);
+    requestMetrics.retain(names);
+    return samples;
+  };
+
+  const sampleLoad = async opts => {
+    if (sampling) {
+      return sampling;
+    }
+    sampling = runSampling(opts).finally(() => {
+      sampling = null;
+    });
+    return sampling;
+  };
+
   const onClose = async () => {
+    if (rotateTimer) {
+      clearInterval(rotateTimer);
+      rotateTimer = null;
+    }
+    if (loadTimer) {
+      clearInterval(loadTimer);
+      loadTimer = null;
+    }
+    if (rotating) {
+      await rotating.catch(() => {});
+    }
+    if (sampling) {
+      await sampling.catch(() => {});
+    }
     if (bus) {
       try {
         bus.close?.();
@@ -87,6 +244,6 @@ module.exports = fp(async (fastify, options) => {
   };
 
   Object.assign(fastify[options.name].services, {
-    bootstrap: { onReady, onClose, reconcile }
+    bootstrap: { onReady, onClose, reconcile, rotateLogs, sampleLoad }
   });
 });

package/libs/utils/env.js

@@ -105,10 +105,17 @@ const isSecretKey = (key, pattern, secretKeys = []) => {
   return isPatternSecretKey(key, pattern);
 };
 
+// 由系统写入维护：不在接口中返回，也不接受外部新增 / 修改 / 删除
+const SYSTEM_ENV_KEYS = ['DB_TABLE_PREFIX'];
+
+const isSystemEnvKey = key => SYSTEM_ENV_KEYS.includes(key);
+
+const omitSystemEnv = (appEnv = {}) => Object.fromEntries(Object.entries(appEnv || {}).filter(([key]) => !isSystemEnvKey(key)));
+
 const maskEnvForResponse = (appEnv = {}, { secretEnvKeyPattern, secretEnvKeys = [] } = {}) => {
   const masked = {};
   const explicit = normalizeSecretKeys(secretEnvKeys);
-  for (const [key, value] of Object.entries(appEnv || {})) {
+  for (const [key, value] of Object.entries(omitSystemEnv(appEnv))) {
     if (isSecretKey(key, secretEnvKeyPattern, explicit)) {
       masked[key] = value == null || value === '' ? null : SECRET_MASK;
     } else {
@@ -128,6 +135,9 @@ const applyEnvPatch = (current = {}, patch = {}, { secretEnvKeyPattern, secretEn
   const next = Object.assign({}, current);
   const explicit = normalizeSecretKeys(secretEnvKeys);
   for (const [key, value] of Object.entries(patch || {})) {
+    if (isSystemEnvKey(key)) {
+      continue;
+    }
     if (value === null) {
       delete next[key];
       continue;
@@ -146,7 +156,7 @@ const applyEnvPatch = (current = {}, patch = {}, { secretEnvKeyPattern, secretEn
 const collectSecretKeys = (appEnv = {}, { secretEnvKeyPattern, secretEnvKeys = [] } = {}) => {
   const explicit = normalizeSecretKeys(secretEnvKeys);
   const fromPattern = Object.keys(appEnv || {}).filter(key => isPatternSecretKey(key, secretEnvKeyPattern));
-  return normalizeSecretKeys([...explicit, ...fromPattern]);
+  return normalizeSecretKeys([...explicit, ...fromPattern]).filter(key => !isSystemEnvKey(key));
 };
 
 const PM2_CONFIG_KEYS = ['exec_mode', 'instances', 'autorestart', 'max_memory_restart', 'max_restarts', 'min_uptime', 'kill_timeout', 'merge_logs'];
@@ -171,6 +181,9 @@ module.exports = {
   mergeEnv,
   isPatternSecretKey,
   isSecretKey,
+  SYSTEM_ENV_KEYS,
+  isSystemEnvKey,
+  omitSystemEnv,
   maskEnvForResponse,
   applyEnvPatch,
   collectSecretKeys,

package/libs/utils/loadMonitor.js

@@ -0,0 +1,73 @@
+const { EventEmitter } = require('node:events');
+
+// cluster 模式下同名进程有多个实例，按 PM2 进程名汇总为一条负载
+const summarizeProcesses = (processes, { now = Date.now() } = {}) => {
+  const groups = new Map();
+  for (const proc of processes || []) {
+    const name = proc?.name || proc?.pm2_env?.name;
+    if (!name) {
+      continue;
+    }
+    if (!groups.has(name)) {
+      groups.set(name, []);
+    }
+    groups.get(name).push(proc);
+  }
+
+  const result = new Map();
+  for (const [name, list] of groups) {
+    const online = list.filter(proc => proc.pm2_env?.status === 'online');
+    const startedAt = online.map(proc => Number(proc.pm2_env?.pm_uptime)).filter(value => Number.isFinite(value) && value > 0);
+    const cpu = list.reduce((sum, proc) => sum + (Number(proc.monit?.cpu) || 0), 0);
+    result.set(name, {
+      status: online.length ? 'online' : list[0].pm2_env?.status || 'unknown',
+      cpu: Math.round(cpu * 10) / 10,
+      memory: list.reduce((sum, proc) => sum + (Number(proc.monit?.memory) || 0), 0),
+      instances: list.length,
+      pids: online.map(proc => proc.pid).filter(Boolean),
+      uptime: startedAt.length ? Math.max(0, now - Math.min(...startedAt)) : 0,
+      restarts: list.reduce((sum, proc) => sum + (Number(proc.pm2_env?.restart_time) || 0), 0)
+    });
+  }
+  return result;
+};
+
+const createLoadStore = ({ size }) => {
+  const capacity = Math.max(1, Math.floor(size) || 1);
+  const buffers = new Map();
+  const emitter = new EventEmitter();
+  emitter.setMaxListeners(0);
+
+  const push = (name, sample) => {
+    const buffer = buffers.get(name) || [];
+    buffer.push(sample);
+    if (buffer.length > capacity) {
+      buffer.splice(0, buffer.length - capacity);
+    }
+    buffers.set(name, buffer);
+    emitter.emit(`load:${name}`, sample);
+  };
+
+  const history = name => (buffers.get(name) || []).slice();
+
+  const latest = name => {
+    const buffer = buffers.get(name);
+    return buffer && buffer.length ? buffer[buffer.length - 1] : null;
+  };
+
+  const retain = names => {
+    const keep = new Set(names);
+    for (const name of buffers.keys()) {
+      if (!keep.has(name)) {
+        buffers.delete(name);
+      }
+    }
+  };
+
+  return { capacity, push, history, latest, retain, emitter };
+};
+
+module.exports = {
+  summarizeProcesses,
+  createLoadStore
+};