@eggjs/logrotator 4.0.0

package/src/app/schedule/clean_log.ts

@@ -0,0 +1,74 @@
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import { exists } from 'utility';
+import moment from 'moment';
+import { EggCore } from '@eggjs/core';
+import { walkLoggerFile } from '../../lib/utils.js';
+
+// clean all xxx.log.YYYY-MM-DD before expired date.
+export default (app: EggCore) => ({
+  schedule: {
+    type: 'worker', // only one worker run this task
+    cron: '0 0 * * *', // run every day at 00:00
+  },
+
+  async task() {
+    const logger = app.coreLogger;
+    const logDirs = new Set<string>();
+    const loggerFiles = walkLoggerFile(app.loggers);
+    loggerFiles.forEach(file => {
+      const logDir = path.dirname(file);
+      logDirs.add(logDir);
+    });
+    const maxDays = app.config.logrotator.maxDays;
+    if (maxDays && maxDays > 0) {
+      try {
+        const tasks = Array.from(logDirs, logDir => removeExpiredLogFiles(logDir, maxDays, logger));
+        await Promise.all(tasks);
+      } catch (err) {
+        logger.error(err);
+      }
+    }
+
+    logger.info('[@eggjs/logrotator] clean all log before %s days', maxDays);
+  },
+});
+
+// remove expired log files: xxx.log.YYYY-MM-DD
+async function removeExpiredLogFiles(logDir: string, maxDays: number, logger: EggCore['coreLogger']) {
+  // ignore not exists dir
+  const stat = await exists(logDir);
+  if (!stat) {
+    logger.warn(`[@eggjs/logrotator] logDir ${logDir} not exists`);
+    return;
+  }
+
+  const files = await fs.readdir(logDir);
+  const expiredDate = moment().subtract(maxDays, 'days').startOf('date');
+  const names = files.filter(file => {
+    const name = path.extname(file).substring(1);
+    if (!/^\d{4}\-\d{2}\-\d{2}/.test(name)) {
+      return false;
+    }
+    const date = moment(name, 'YYYY-MM-DD').startOf('date');
+    if (!date.isValid()) {
+      return false;
+    }
+    return date.isBefore(expiredDate);
+  });
+  if (names.length === 0) {
+    return;
+  }
+
+  logger.info(`[@eggjs/logrotator] start remove ${logDir} files: ${names.join(', ')}`);
+
+  await Promise.all(names.map(async name => {
+    const logFile = path.join(logDir, name);
+    try {
+      await fs.unlink(logFile);
+    } catch (err: any) {
+      err.message = `[@eggjs/logrotator] remove logFile ${logFile} error, ${err.message}`;
+      logger.error(err);
+    }
+  }));
+}

package/src/app/schedule/rotate_by_file.ts

@@ -0,0 +1,18 @@
+import { EggCore } from '@eggjs/core';
+import { DayRotator } from '../../lib/day_rotator.js';
+
+export default (app: EggCore) => {
+  const rotator = new DayRotator({ app });
+
+  return {
+    schedule: {
+      type: 'worker', // only one worker run this task
+      cron: '1 0 0 * * *', // run every day at 00:00
+      disable: app.config.logrotator.disableRotateByDay,
+    },
+
+    async task() {
+      await rotator.rotate();
+    },
+  };
+};

package/src/app/schedule/rotate_by_hour.ts

@@ -0,0 +1,19 @@
+import { EggCore } from '@eggjs/core';
+import { HourRotator } from '../../lib/hour_rotator.js';
+
+export default (app: EggCore) => {
+  const rotator = new HourRotator({ app });
+
+  return {
+    schedule: {
+      type: 'worker', // only one worker run this task
+      cron: '1 * * * *', // run every hour at 01
+      disable: (app.config.logrotator.filesRotateByHour || []).length === 0,
+    },
+
+    async task() {
+      await rotator.rotate();
+    },
+
+  };
+};

package/src/app/schedule/rotate_by_size.ts

@@ -0,0 +1,18 @@
+import { EggCore } from '@eggjs/core';
+import { SizeRotator } from '../../lib/size_rotator.js';
+
+export default (app: EggCore) => {
+  const rotator = new SizeRotator({ app });
+
+  return {
+    schedule: {
+      type: 'worker',
+      interval: app.config.logrotator.rotateDuration,
+      disable: (app.config.logrotator.filesRotateBySize || []).length === 0,
+    },
+
+    async task() {
+      await rotator.rotate();
+    },
+  };
+};

package/src/app.ts

@@ -0,0 +1,3 @@
+import { Boot } from './boot.js';
+
+export default Boot;

package/src/boot.ts

@@ -0,0 +1,13 @@
+import type { EggCore, ILifecycleBoot } from '@eggjs/core';
+
+export class Boot implements ILifecycleBoot {
+  constructor(private readonly app: EggCore) {}
+
+  async didLoad() {
+    // reload logger to new fd after rotating
+    this.app.messenger.on('log-reload', () => {
+      this.app.loggers.reload();
+      this.app.coreLogger.info('[@eggjs/logrotator] %s logger reload: got log-reload message', this.app.type);
+    });
+  }
+}

package/src/config/config.default.ts

@@ -0,0 +1,74 @@
+/**
+ * logrotator options
+ * @member Config#logrotator
+ */
+export interface LogrotatorConfig {
+  /**
+   * Disable rotate by day
+   *
+   * Default: `false`
+   */
+  disableRotateByDay: boolean;
+  /**
+   * List of files that will be rotated by hour
+   *
+   * Default: `null`
+   */
+  filesRotateByHour: string[] | null;
+  /**
+   * Hour delimiter
+   *
+   * Default: `-`
+   */
+  hourDelimiter: string;
+  /**
+   * List of files that will be rotated by size
+   *
+   * Default: `null`
+   */
+  filesRotateBySize: string[] | null;
+  /**
+   * Max file size to judge if any file need rotate
+   *
+   * Default: `50 * 1024 * 1024`
+   */
+  maxFileSize: number;
+  /**
+   * Max files to keep
+   *
+   * Default: `10`
+   */
+  maxFiles: number;
+  /**
+   * Time interval to judge if any file need rotate
+   *
+   * Default: `60000`
+   */
+  rotateDuration: number;
+  /**
+   * Max days to keep log files, set `0` to keep all logs.
+   *
+   * Default: `31`
+   */
+  maxDays: number;
+  /**
+   * Enable gzip compression for rotated files
+   *
+   * Default: `false`
+   */
+  gzip: boolean;
+}
+
+export default {
+  logrotator: {
+    disableRotateByDay: false,
+    filesRotateByHour: null,
+    hourDelimiter: '-',
+    filesRotateBySize: null,
+    maxFileSize: 50 * 1024 * 1024,
+    maxFiles: 10,
+    rotateDuration: 60000,
+    maxDays: 31,
+    gzip: false,
+  } as LogrotatorConfig,
+};

package/src/index.ts

@@ -0,0 +1,3 @@
+import './types.js';
+
+export * from './lib/rotator.js';

package/src/lib/day_rotator.ts

@@ -0,0 +1,91 @@
+import path from 'node:path';
+import moment from 'moment';
+import fs from 'node:fs/promises';
+import { debuglog } from 'node:util';
+import { exists } from 'utility';
+import { LogRotator, RotateFile, RotatorOptions } from './rotator.js';
+import { walkLoggerFile } from './utils.js';
+
+const debug = debuglog('@eggjs/logrotator/lib/day_rotator');
+
+// rotate log by day
+// rename from foo.log to foo.log.YYYY-MM-DD
+export class DayRotator extends LogRotator {
+  private filesRotateBySize: string[];
+  private filesRotateByHour: string[];
+
+  constructor(options: RotatorOptions) {
+    super(options);
+    this.filesRotateBySize = this.app.config.logrotator.filesRotateBySize || [];
+    this.filesRotateByHour = this.app.config.logrotator.filesRotateByHour || [];
+  }
+
+  async getRotateFiles() {
+    const files = new Map<string, RotateFile>();
+    const logDir = this.app.config.logger.dir;
+    const loggers = this.app.loggers;
+    const loggerFiles = walkLoggerFile(loggers);
+    loggerFiles.forEach(file => {
+      // support relative path
+      if (!path.isAbsolute(file)) {
+        file = path.join(logDir, file);
+      }
+      this._setFile(file, files);
+    });
+
+    // Should rotate agent log, because schedule is running under app worker,
+    // agent log is the only difference between app worker and agent worker.
+    // - app worker -> egg-web.log
+    // - agent worker -> egg-agent.log
+    const agentLogName = this.app.config.logger.agentLogName;
+    this._setFile(path.join(logDir, agentLogName), files);
+
+    // rotateLogDirs is deprecated
+    const rotateLogDirs = this.app.config.logger.rotateLogDirs;
+    if (rotateLogDirs && rotateLogDirs.length > 0) {
+      this.app.deprecate('[egg-logrotator] Do not use app.config.logger.rotateLogDirs, only rotate core loggers and custom loggers');
+
+      for (const dir of rotateLogDirs) {
+        const stat = await exists(dir);
+        if (!stat) continue;
+
+        try {
+          const names = await fs.readdir(dir);
+          for (const name of names) {
+            if (!name.endsWith('.log')) {
+              continue;
+            }
+            this._setFile(path.join(dir, name), files);
+          }
+        } catch (err) {
+          this.logger.error(err);
+        }
+      }
+    }
+
+    return files;
+  }
+
+  _setFile(srcPath: string, files: Map<string, RotateFile>) {
+    // don't rotate logPath in filesRotateBySize
+    if (this.filesRotateBySize.indexOf(srcPath) > -1) {
+      return;
+    }
+
+    // don't rotate logPath in filesRotateByHour
+    if (this.filesRotateByHour.indexOf(srcPath) > -1) {
+      return;
+    }
+
+    if (!files.has(srcPath)) {
+      const ext = this.app.config.logrotator.gzip === true ? '.gz' : '';
+      // allow 2 minutes deviation
+      const targetPath = srcPath + moment()
+        .subtract(23, 'hours')
+        .subtract(58, 'minutes')
+        .format('.YYYY-MM-DD') + ext;
+      debug('set file %s => %s', srcPath, targetPath);
+      files.set(srcPath, { srcPath, targetPath });
+    }
+  }
+}

package/src/lib/hour_rotator.ts

@@ -0,0 +1,44 @@
+import moment from 'moment';
+import path from 'node:path';
+import { debuglog } from 'node:util';
+import { exists } from 'utility';
+import { LogRotator, RotateFile } from './rotator.js';
+
+const debug = debuglog('@eggjs/logrotator/lib/hour_rotator');
+
+// rotate log by hour
+// rename from foo.log to foo.log.YYYY-MM-DD-HH
+export class HourRotator extends LogRotator {
+  async getRotateFiles() {
+    const files = new Map<string, RotateFile>();
+    const logDir = this.app.config.logger.dir;
+    const filesRotateByHour = this.app.config.logrotator.filesRotateByHour || [];
+
+    for (let logPath of filesRotateByHour) {
+      // support relative path
+      if (!path.isAbsolute(logPath)) {
+        logPath = path.join(logDir, logPath);
+      }
+      const stat = await exists(logPath);
+      if (!stat) {
+        continue;
+      }
+      this._setFile(logPath, files);
+    }
+
+    return files;
+  }
+
+  get hourDelimiter() {
+    return this.app.config.logrotator.hourDelimiter;
+  }
+
+  _setFile(srcPath: string, files: Map<string, RotateFile>) {
+    if (!files.has(srcPath)) {
+      const ext = this.app.config.logrotator.gzip === true ? '.gz' : '';
+      const targetPath = srcPath + moment().subtract(1, 'hours').format(`.YYYY-MM-DD${this.hourDelimiter}HH`) + ext;
+      debug('set file %s => %s', srcPath, targetPath);
+      files.set(srcPath, { srcPath, targetPath });
+    }
+  }
+}

package/src/lib/rotator.ts

@@ -0,0 +1,88 @@
+import assert from 'node:assert';
+import { createWriteStream, createReadStream } from 'node:fs';
+import fs from 'node:fs/promises';
+import { pipeline } from 'node:stream/promises';
+import { createGzip } from 'node:zlib';
+import { debuglog } from 'node:util';
+import { exists } from 'utility';
+import { EggCore } from '@eggjs/core';
+
+const debug = debuglog('@eggjs/logrotator/lib/rotator');
+
+export interface RotatorOptions {
+  app: EggCore;
+}
+
+export interface RotateFile {
+  srcPath: string;
+  targetPath: string;
+}
+
+export abstract class LogRotator {
+  protected readonly options: RotatorOptions;
+  protected readonly app: EggCore;
+  protected readonly logger: EggCore['coreLogger'];
+
+  constructor(options: RotatorOptions) {
+    this.options = options;
+    assert(this.options.app, 'options.app is required');
+    this.app = this.options.app;
+    this.logger = this.app.coreLogger;
+  }
+
+  abstract getRotateFiles(): Promise<Map<string, RotateFile>>;
+
+  async rotate() {
+    const files = await this.getRotateFiles();
+    assert(files instanceof Map, 'getRotateFiles should return a Map');
+    const rotatedFiles: string[] = [];
+    for (const file of files.values()) {
+      try {
+        debug('rename from %s to %s', file.srcPath, file.targetPath);
+        await renameOrDelete(file.srcPath, file.targetPath, this.app.config.logrotator.gzip);
+        rotatedFiles.push(`${file.srcPath} -> ${file.targetPath}`);
+      } catch (err: any) {
+        err.message = `[@eggjs/logrotator] rename ${file.srcPath}, found exception: ` + err.message;
+        this.logger.error(err);
+      }
+    }
+
+    if (rotatedFiles.length) {
+      // tell every one to reload logger
+      this.logger.info('[@eggjs/logrotator] broadcast log-reload');
+      this.app.messenger.sendToApp('log-reload');
+      this.app.messenger.sendToAgent('log-reload');
+    }
+
+    this.logger.info('[@eggjs/logrotator] rotate files success by %s, files %j',
+      this.constructor.name, rotatedFiles);
+  }
+}
+
+// rename from srcPath to targetPath, for example foo.log.1 > foo.log.2
+// if gzip is true, then use gzip to compress the file, and delete the src file, for example foo.log.1 -> foo.log.2.gz
+async function renameOrDelete(srcPath: string, targetPath: string, gzip: boolean) {
+  if (srcPath === targetPath) {
+    return;
+  }
+  const srcExists = await exists(srcPath);
+  if (!srcExists) {
+    return;
+  }
+  const targetExists = await exists(targetPath);
+  // if target file exists, then throw
+  // because the target file always be renamed first.
+  if (targetExists) {
+    const err = new Error(`targetFile ${targetPath} exists!!!`);
+    throw err;
+  }
+  // if gzip is true, then use gzip
+  if (gzip === true) {
+    const tmpPath = `${targetPath}.tmp`;
+    await fs.rename(srcPath, tmpPath);
+    await pipeline(createReadStream(tmpPath), createGzip(), createWriteStream(targetPath));
+    await fs.unlink(tmpPath);
+  } else {
+    await fs.rename(srcPath, targetPath);
+  }
+}

package/src/lib/size_rotator.ts

@@ -0,0 +1,67 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { debuglog } from 'node:util';
+import { exists } from 'utility';
+import { LogRotator, RotateFile } from './rotator.js';
+
+const debug = debuglog('@eggjs/logrotator/lib/size_rotator');
+
+// rotate log by size, if the size of file over maxFileSize,
+// it will rename from foo.log to foo.log.1
+// if foo.log.1 exists, foo.log.1 will rename to foo.log.2
+export class SizeRotator extends LogRotator {
+  async getRotateFiles() {
+    const files = new Map<string, RotateFile>();
+    const logDir = this.app.config.logger.dir;
+    const filesRotateBySize = this.app.config.logrotator.filesRotateBySize || [];
+    const maxFileSize = this.app.config.logrotator.maxFileSize;
+    const maxFiles = this.app.config.logrotator.maxFiles;
+    for (let logPath of filesRotateBySize) {
+      // support relative path
+      if (!path.isAbsolute(logPath)) {
+        logPath = path.join(logDir, logPath);
+      }
+      const stat = await exists(logPath);
+      if (!stat) {
+        continue;
+      }
+      const size = stat.size;
+      try {
+        if (size >= maxFileSize) {
+          this.logger.info(`[@eggjs/logrotator] file ${logPath} reach the maximum file size, current size: ${size}, max size: ${maxFileSize}`);
+          // delete max log file if exists, otherwise will throw when rename
+          const maxFileName = `${logPath}.${maxFiles}`;
+          const stat = await exists(maxFileName);
+          if (stat) {
+            await fs.unlink(maxFileName);
+            this.logger.info(`[@eggjs/logrotator] delete max log file ${maxFileName}`);
+          }
+          this._setFile(logPath, files);
+        }
+      } catch (err: any) {
+        err.message = '[@eggjs/logrotator] ' + err.message;
+        this.logger.error(err);
+      }
+    }
+    return files;
+  }
+
+  _setFile(logPath: string, files: Map<string, RotateFile>) {
+    const maxFiles = this.app.config.logrotator.maxFiles;
+    if (files.has(logPath)) {
+      return;
+    }
+    const ext = this.app.config.logrotator.gzip === true ? '.gz' : '';
+    // foo.log.2 -> foo.log.3
+    // foo.log.1 -> foo.log.2
+    for (let i = maxFiles - 1; i >= 1; i--) {
+      const srcPath = `${logPath}.${i}`;
+      const targetPath = `${logPath}.${i + 1}${ext}`;
+      debug('set file %s => %s', srcPath, targetPath);
+      files.set(srcPath, { srcPath, targetPath });
+    }
+    // foo.log -> foo.log.1
+    debug('set file %s => %s', logPath, `${logPath}.1`);
+    files.set(logPath, { srcPath: logPath, targetPath: `${logPath}.1${ext}` });
+  }
+}

package/src/lib/utils.ts

@@ -0,0 +1,26 @@
+interface LoggerTransport {
+  options: {
+    file: string;
+  };
+}
+
+/**
+ * Walk all logger files from loggers
+ * @param loggers - The loggers to walk
+ */
+export function walkLoggerFile(loggers: Record<string, Map<string, LoggerTransport>>) {
+  const files: string[] = [];
+  for (const key in loggers) {
+    if (!loggers.hasOwnProperty(key)) {
+      continue;
+    }
+    const registeredLogger = loggers[key];
+    for (const transport of registeredLogger.values()) {
+      const file = transport.options.file;
+      if (file) {
+        files.push(file);
+      }
+    }
+  }
+  return files;
+}

package/src/types.ts

@@ -0,0 +1,15 @@
+import type { LogrotatorConfig } from './config/config.default.js';
+import type { LogRotator } from './lib/rotator.js';
+
+export type { LogrotatorConfig };
+
+declare module '@eggjs/core' {
+  // add EggAppConfig overrides types
+  interface EggAppConfig {
+    logrotator: LogrotatorConfig;
+  }
+
+  interface EggCore {
+    LogRotator: typeof LogRotator;
+  }
+}

package/src/typings/index.d.ts

@@ -0,0 +1,4 @@
+// make sure to import egg typings and let typescript know about it
+// @see https://github.com/whxaxes/blog/issues/11
+// and https://www.typescriptlang.org/docs/handbook/declaration-merging.html
+import 'egg';