@oh-my-pi/pi-utils 18.4.8 → 18.4.10

package/src/logger/rotating-file.ts

@@ -1,21 +1,16 @@
-/** Behavior-compatible reimplementation of winston-daily-rotate-file's used surface. */
+/** Process-local rotating file sink: local-day and size rotation with bounded retention. */
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { localDay } from "../dirs";
 import { openCloexecSync } from "../fs-open";
 
-interface AuditEntry {
-	readonly date: number;
-	readonly name: string;
-	readonly hash: string;
-}
-
-interface AuditState {
-	readonly keep: { readonly days: false; readonly amount: number };
-	readonly auditLog: string;
-	readonly files: AuditEntry[];
-	readonly hashType: "sha256";
+/** Write batching for {@link RotatingFileSink}. */
+export interface RotatingFileBatchOptions {
+	/** Longest a buffered record waits before an unref'd timer writes it. */
+	readonly intervalMs: number;
+	/** Buffered UTF-8 bytes at which the batch is written immediately. */
+	readonly maxBytes: number;
 }
 
 /** Configuration for a process-local rotating file sink. */
@@ -23,12 +18,16 @@ export interface RotatingFileOptions {
 	readonly directory: string;
 	readonly filenamePrefix: string;
 	readonly filenameSuffix: string;
-	readonly auditFile: string;
 	readonly maxBytes: number;
 	readonly maxFiles: number;
+	/**
+	 * Buffer records and write them together instead of one write per record.
+	 * Without it every record is written as it arrives.
+	 */
+	readonly batch?: RotatingFileBatchOptions;
 	/**
 	 * Called with the new active file path whenever the sink opens a different
-	 * file — construction, local-day rotation, and size rotation alike.
+	 * file — first write, local-day rotation, and size rotation alike.
 	 *
 	 * Consumers that hold their own descriptor on the active log (the macOS
 	 * stderr guard dup2s it onto fd 2) use this to follow the sink instead of
@@ -37,50 +36,53 @@ export interface RotatingFileOptions {
 	readonly onRotate?: (filePath: string) => void;
 }
 
-function isAuditEntry(value: unknown): value is AuditEntry {
-	if (value === null || typeof value !== "object") return false;
-	const entry = value as Record<string, unknown>;
-	return typeof entry.date === "number" && typeof entry.name === "string" && typeof entry.hash === "string";
-}
-
-/** Synchronous append sink with local-day and size rotation plus bounded retention. */
+/**
+ * Synchronous append sink with local-day and size rotation plus bounded retention.
+ *
+ * Construction performs no filesystem I/O: the first file is created when the
+ * first record is written out, so a process that never logs leaves nothing on
+ * disk. Every file name embeds the owning process's suffix (its PID), so the
+ * set of files this sink created is tracked in memory; files left by earlier
+ * processes are pruned by name pattern in the logger, not by this sink.
+ *
+ * With {@link RotatingFileOptions.batch}, records are buffered and written in
+ * one `write` per batch. Buffered bytes already count toward the active file's
+ * size, and the buffer is written to its file before any rotation, so file
+ * selection matches an unbatched sink record for record.
+ */
 export class RotatingFileSink {
 	readonly #directory: string;
 	readonly #filenamePrefix: string;
 	readonly #filenameSuffix: string;
-	readonly #auditFile: string;
 	readonly #maxBytes: number;
 	readonly #maxFiles: number;
+	readonly #batch: RotatingFileBatchOptions | undefined;
 	readonly #onRotate: ((filePath: string) => void) | undefined;
-	#files: AuditEntry[];
+	/** Files this sink created, oldest first. */
+	readonly #files: string[] = [];
 	#activeDay: string | undefined;
 	#activeIndex = 0;
 	#activePath: string | undefined;
+	/** Size of the active file including buffered records not yet written. */
 	#activeBytes = 0;
 	#closed = false;
-	// Held append fd: the old code did open+write+close per line
-	// (appendFileSync) plus a throwaway open in the constructor. A single fd
-	// per active file removes two syscalls per log line; it is reopened on
-	// rotation and closed on close()/rotation (required for Windows
-	// delete-on-prune semantics).
+	// Held append fd: one descriptor per active file instead of
+	// open+write+close per line. It is reopened on rotation and closed on
+	// close()/rotation (required for Windows delete-on-prune semantics).
 	#fd: number | undefined;
+	/** Records for the active file not yet written, each ending in EOL. */
+	#pending = "";
+	#pendingBytes = 0;
+	#flushTimer: NodeJS.Timeout | undefined;
 
 	constructor(options: RotatingFileOptions) {
 		this.#directory = options.directory;
 		this.#filenamePrefix = options.filenamePrefix;
 		this.#filenameSuffix = options.filenameSuffix;
-		this.#auditFile = options.auditFile;
 		this.#maxBytes = options.maxBytes;
 		this.#maxFiles = options.maxFiles;
+		this.#batch = options.batch;
 		this.#onRotate = options.onRotate;
-		this.#files = this.#readAudit();
-		const now = new Date();
-		this.#selectFile(localDay(now));
-		const activePath = this.#activePath;
-		if (activePath) {
-			this.#registerFile(activePath, now.getTime());
-			this.#openFd(activePath);
-		}
 	}
 
 	#openFd(filePath: string): void {
@@ -104,41 +106,84 @@ export class RotatingFileSink {
 		}
 	}
 
-	/** Append one already-formatted log record. */
-	write(line: string): void {
+	/**
+	 * Append one already-formatted log record. A batching sink buffers it unless
+	 * `flushNow` is set or the buffer reached its size threshold; either way the
+	 * whole buffer, this record included, is then written in order.
+	 */
+	write(line: string, flushNow = false): void {
 		if (this.#closed) return;
-		const prevPath = this.#activePath;
-		const now = new Date();
-		this.#selectFile(localDay(now));
-		const activePath = this.#activePath;
-		if (!activePath) return;
-		// Rotation moved the active path: close the old descriptor BEFORE
-		// registering the new path, because registration prunes beyond
-		// maxFiles — on Windows the pruned predecessor cannot be deleted
-		// while still open, which would leak it (audit entry removed, file
-		// left on disk, retention unbounded).
-		if (activePath !== prevPath) this.#closeFd();
-		this.#registerFile(activePath, now.getTime());
-		if (this.#fd === undefined) this.#openFd(activePath);
+		this.#selectFile(localDay(new Date()));
 		const record = `${line}${os.EOL}`;
-		const buf = Buffer.from(record, "utf8");
+		const bytes = Buffer.byteLength(record, "utf8");
+		this.#pending += record;
+		this.#pendingBytes += bytes;
+		this.#activeBytes += bytes;
+		const batch = this.#batch;
+		if (flushNow || !batch || this.#pendingBytes >= batch.maxBytes) {
+			this.flush();
+		} else if (this.#flushTimer === undefined) {
+			this.#flushTimer = setTimeout(this.#flushFromTimer, batch.intervalMs);
+			this.#flushTimer.unref();
+		}
+	}
+
+	/** Write every buffered record to the active file now. */
+	flush(): void {
+		if (this.#flushTimer !== undefined) {
+			clearTimeout(this.#flushTimer);
+			this.#flushTimer = undefined;
+		}
+		const activePath = this.#activePath;
+		if (this.#pendingBytes === 0 || !activePath) return;
+		const buf = Buffer.from(this.#pending, "utf8");
+		// Drop the batch even if the write below fails, so a broken file cannot
+		// make the buffer grow without bound.
+		this.#pending = "";
+		this.#pendingBytes = 0;
+		if (this.#fd === undefined) {
+			this.#registerFile(activePath);
+			this.#openFd(activePath);
+		}
 		let off = 0;
 		while (off < buf.length) {
 			const written = fs.writeSync(this.#fd!, buf, off);
 			if (written <= 0) break;
 			off += written;
 		}
-		this.#activeBytes += buf.length;
 	}
 
-	/** Stop accepting records. Synchronous writes require no drain phase. */
+	#flushFromTimer = (): void => {
+		this.#flushTimer = undefined;
+		try {
+			this.flush();
+		} catch {
+			// A background flush has no caller to report to; logging never throws.
+		}
+	};
+
+	/** Write buffered records, then stop accepting records. */
 	close(): void {
+		if (this.#closed) return;
+		try {
+			this.flush();
+		} catch {
+			// Best-effort: a failed final write must not keep the descriptor open.
+		}
 		this.#closed = true;
 		this.#closeFd();
 	}
 
 	#selectFile(day: string): void {
-		if (day !== this.#activeDay) {
+		const dayChanged = day !== this.#activeDay;
+		if (!dayChanged && this.#activeBytes <= this.#maxBytes) return;
+		// The buffered records belong to the current file: write them there, then
+		// close its descriptor BEFORE the next file is registered, because
+		// registration prunes beyond maxFiles — on Windows the pruned
+		// predecessor cannot be deleted while still open, which would leak it.
+		this.flush();
+		this.#closeFd();
+		if (dayChanged) {
 			this.#activeDay = day;
 			this.#activeIndex = 0;
 			this.#setActivePath(day, 0);
@@ -162,38 +207,17 @@ export class RotatingFileSink {
 		}
 	}
 
-	#registerFile(filePath: string, date: number): void {
-		if (this.#files.some(file => file.name === filePath)) return;
-		const hash = Bun.SHA256.hash(`${filePath}LOG_FILE${date}`, "hex");
-		this.#files.push({ date, name: filePath, hash });
+	#registerFile(filePath: string): void {
+		if (this.#files.includes(filePath)) return;
+		this.#files.push(filePath);
 		while (this.#files.length > this.#maxFiles) {
 			const removed = this.#files.shift();
 			if (!removed) break;
 			try {
-				fs.rmSync(removed.name, { force: true });
+				fs.rmSync(removed, { force: true });
 			} catch {
 				// Retention is best-effort; the current record must still be written.
 			}
 		}
-		this.#writeAudit();
-	}
-
-	#readAudit(): AuditEntry[] {
-		try {
-			const parsed = JSON.parse(fs.readFileSync(this.#auditFile, "utf8")) as { files?: unknown };
-			return Array.isArray(parsed.files) ? parsed.files.filter(isAuditEntry) : [];
-		} catch {
-			return [];
-		}
-	}
-
-	#writeAudit(): void {
-		const state: AuditState = {
-			keep: { days: false, amount: this.#maxFiles },
-			auditLog: this.#auditFile,
-			files: this.#files,
-			hashType: "sha256",
-		};
-		fs.writeFileSync(this.#auditFile, JSON.stringify(state, undefined, 4), "utf8");
 	}
 }

package/src/logger.ts

@@ -7,7 +7,11 @@
  * transport so a process supervisor (pm2, journald, k8s) captures the logs.
  *
  * Each entry includes `process.pid` so concurrent omp instances stay
- * traceable.
+ * traceable. The file is created on the first record written. Records are
+ * batched — one write per second or per 64 KiB — while `warn`/`error` records
+ * are written at once together with everything buffered before them; exit,
+ * fatal, and signal paths flush the rest. `OMP_LOG_LEVEL` optionally limits
+ * the levels written to the file (default: all).
  */
 import { AsyncLocalStorage } from "node:async_hooks";
 import * as fs from "node:fs";
@@ -55,9 +59,13 @@ function emitToSinks(level: LogLevel, message: string, context: Record<string, u
 }
 
 const PROCESS_LOG_PATTERN = /^omp\.(\d{4}-\d{2}-\d{2})\.(\d+)\.log(?:\.(\d+))?$/;
+/** Per-process audit files written by earlier releases; current sinks track rotations in memory. */
 const PROCESS_AUDIT_PATTERN = /^\.omp\.(\d+)-audit\.json$/;
+/** Shared daily logs (plain, size-rolled, or gzipped) written by the winston-era logger. */
+const LEGACY_LOG_PATTERN = /^omp\.(\d{4}-\d{2}-\d{2})\.log(?:\.\d+)?(?:\.gz)?$/;
+/** Hash-named audit files written by winston-daily-rotate-file. */
+const LEGACY_AUDIT_PATTERN = /^\.[0-9a-f]{8,}-audit\.json$/;
 const RETAINED_STALE_LOGS_PER_PROCESS_DAY = 1;
-const RETAINED_STALE_AUDIT_FILES = 0;
 const RETAINED_STALE_LOG_DAYS = 5;
 
 function processIsRunning(pid: number): boolean {
@@ -71,9 +79,11 @@ function processIsRunning(pid: number): boolean {
 
 /**
  * Retain one newest completed-process log per process/day within the current
- * and previous four local calendar days, and remove one-use audit files. Live
- * PID namespaces are never touched. The calendar-day boundary preserves daily
- * diagnostic coverage while bounding completed-process storage and scans.
+ * and previous four local calendar days, remove completed-process audit files
+ * left by earlier releases, and age out legacy shared daily logs and their
+ * hash-named audits past the same window. Live PID namespaces are never
+ * touched. The calendar-day boundary preserves daily diagnostic coverage while
+ * bounding completed-process storage and scans.
  */
 function pruneStaleProcessLogs(dir: string): void {
 	let entries: fs.Dirent[];
@@ -87,33 +97,38 @@ function pruneStaleProcessLogs(dir: string): void {
 	const cutoff = new Date(current);
 	cutoff.setDate(cutoff.getDate() - (RETAINED_STALE_LOG_DAYS - 1));
 	const cutoffDate = localDay(cutoff);
+	cutoff.setHours(0, 0, 0, 0);
+	const cutoffMs = cutoff.getTime();
 
 	const staleLogsByProcessDay = new Map<string, Array<{ path: string; rollover: number }>>();
 	for (const entry of entries) {
 		if (!entry.isFile()) continue;
+		const entryPath = path.join(dir, entry.name);
+		const legacyMatch = LEGACY_LOG_PATTERN.exec(entry.name);
+		if (legacyMatch?.[1]) {
+			if (legacyMatch[1] < cutoffDate) removeBestEffort(entryPath);
+			continue;
+		}
+		if (LEGACY_AUDIT_PATTERN.test(entry.name)) {
+			try {
+				if (fs.statSync(entryPath).mtimeMs < cutoffMs) removeBestEffort(entryPath);
+			} catch {
+				// Another process may have pruned the same legacy audit.
+			}
+			continue;
+		}
 		const logMatch = PROCESS_LOG_PATTERN.exec(entry.name);
 		const auditMatch = PROCESS_AUDIT_PATTERN.exec(entry.name);
 		const pidText = logMatch?.[2] ?? auditMatch?.[1];
 		if (!pidText || processIsRunning(Number(pidText))) continue;
-		const entryPath = path.join(dir, entry.name);
 
 		if (auditMatch) {
-			if (RETAINED_STALE_AUDIT_FILES === 0) {
-				try {
-					fs.rmSync(entryPath, { force: true });
-				} catch {
-					// Retention is best-effort; logging must still initialize.
-				}
-			}
+			removeBestEffort(entryPath);
 			continue;
 		}
 		if (!logMatch?.[1]) continue;
 		if (logMatch[1] < cutoffDate || logMatch[1] > currentDate) {
-			try {
-				fs.rmSync(entryPath, { force: true });
-			} catch {
-				// Another process may have pruned the same stale namespace.
-			}
+			removeBestEffort(entryPath);
 			continue;
 		}
 
@@ -139,13 +154,15 @@ function pruneStaleProcessLogs(dir: string): void {
 		ranked.sort(
 			(a, b) => b.mtimeMs - a.mtimeMs || b.rollover - a.rollover || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0),
 		);
-		for (const stale of ranked.slice(RETAINED_STALE_LOGS_PER_PROCESS_DAY)) {
-			try {
-				fs.rmSync(stale.path, { force: true });
-			} catch {
-				// Another process may have pruned the same stale namespace.
-			}
-		}
+		for (const stale of ranked.slice(RETAINED_STALE_LOGS_PER_PROCESS_DAY)) removeBestEffort(stale.path);
+	}
+}
+
+function removeBestEffort(filePath: string): void {
+	try {
+		fs.rmSync(filePath, { force: true });
+	} catch {
+		// Retention is best-effort; another process may have pruned the same file.
 	}
 }
 
@@ -244,22 +261,72 @@ function formatLogInfo(info: NormalizedLogInfo): string {
 	return JSON.stringify(entry, jsonReplacer) as string;
 }
 
+/** Longest a buffered file record waits before it is written. */
+const FILE_FLUSH_INTERVAL_MS = 1_000;
+/** Buffered bytes at which the file batch is written immediately. */
+const FILE_FLUSH_BYTES = 64 * 1024;
+
+let exitFlushInstalled = false;
+/** Set once the process is exiting: every later record is written immediately. */
+let exiting = false;
+
+function flushOnExit(): void {
+	exiting = true;
+	flush();
+}
+
 /** Build a rotating file sink with process-local rotation and shared retention. */
 function makeFileTransport(dir?: string): RotatingFileSink {
 	const logsDir = ensureDir(dir ?? getLogsDir());
 	schedulePruneStaleProcessLogs(logsDir);
+	if (!exitFlushInstalled) {
+		exitFlushInstalled = true;
+		// Natural exit and process.exit(); signal and fatal exits that bypass the
+		// `exit` event flush through postmortem's exitProcess().
+		process.on("exit", flushOnExit);
+	}
+	// The sink opens (and creates) its file on the first batch written.
 	return new RotatingFileSink({
 		directory: logsDir,
 		filenamePrefix: "omp",
 		filenameSuffix: String(process.pid),
 		maxBytes: 10 * 1024 * 1024,
 		maxFiles: 5,
-		auditFile: path.join(logsDir, `.omp.${process.pid}-audit.json`),
+		batch: { intervalMs: FILE_FLUSH_INTERVAL_MS, maxBytes: FILE_FLUSH_BYTES },
 		// Keep the stderr guard's fd 2 on the file this sink is writing.
 		onRotate: setStderrRedirectTarget,
 	});
 }
 
+const LOG_LEVEL_RANK: Record<LogLevel, number> = { error: 0, warn: 1, info: 2, debug: 3 };
+/** `warn` and `error` records are written at once, with everything buffered before them. */
+const IMMEDIATE_FLUSH_RANK = LOG_LEVEL_RANK.warn;
+
+/**
+ * Rank of the most verbose level the file transport persists: `OMP_LOG_LEVEL`
+ * (error|warn|info|debug, case-insensitive), default `debug` (everything).
+ */
+function resolveFileLevelRank(): number {
+	const requested = process.env.OMP_LOG_LEVEL?.trim().toLowerCase();
+	return requested && Object.hasOwn(LOG_LEVEL_RANK, requested)
+		? LOG_LEVEL_RANK[requested as LogLevel]
+		: LOG_LEVEL_RANK.debug;
+}
+
+/**
+ * Write buffered file records to disk now. The file transport batches records
+ * (one write per second or per 64 KiB) and writes `warn`/`error` records
+ * immediately; process exit flushes on its own. Call this before handing the
+ * log file to something that reads it in-process. Never throws.
+ */
+export function flush(): void {
+	try {
+		activeTransports?.file?.flush();
+	} catch {
+		// Flushing diagnostics must never mask the caller's own work or failure.
+	}
+}
+
 /**
  * Desired transport configuration, applied when local logging is initialized.
  * Default: file ON (TUI-safe), console OFF.
@@ -269,6 +336,8 @@ let transportOpts: { console?: boolean; file?: boolean | string } = { file: true
 interface LocalTransports {
 	readonly file: RotatingFileSink | undefined;
 	readonly console: boolean;
+	/** Rank of the most verbose level written to {@link file}. */
+	readonly fileLevelRank: number;
 }
 
 /** Local transports, constructed lazily on first log emission. */
@@ -278,6 +347,7 @@ function buildTransports(opts: { console?: boolean; file?: boolean | string }):
 	return {
 		file: opts.file ? makeFileTransport(typeof opts.file === "string" ? opts.file : undefined) : undefined,
 		console: opts.console === true,
+		fileLevelRank: resolveFileLevelRank(),
 	};
 }
 
@@ -288,10 +358,12 @@ function getLocalTransports(): LocalTransports {
 
 function emitLocally(level: LogLevel, message: string, context: Record<string, unknown> | undefined): void {
 	const transports = getLocalTransports();
-	if (!transports.file && !transports.console) return;
+	const rank = LOG_LEVEL_RANK[level];
+	const file = rank <= transports.fileLevelRank ? transports.file : undefined;
+	if (!file && !transports.console) return;
 	const info = normalizeLogInfo(level, message, context);
 	const line = formatLogInfo(info);
-	if (transports.file) transports.file.write(line);
+	file?.write(line, rank <= IMMEDIATE_FLUSH_RANK || exiting);
 	if (transports.console) fs.writeSync(1, `${line}${os.EOL}`);
 }
 
@@ -299,12 +371,14 @@ function emitLocally(level: LogLevel, message: string, context: Record<string, u
  * Replace the active log transports. Pass `console: true, file: false` for
  * long-running services (the auth broker, etc.) that want their structured
  * logs piped into a process supervisor instead of the rotating file.
+ * The previous file transport writes its buffered records before closing, and
+ * the file level (`OMP_LOG_LEVEL`) is re-read on each reconfiguration.
  */
 export function setTransports(opts: { console?: boolean; file?: boolean | string }): void {
 	transportOpts = opts;
 	if (!activeTransports) return; // applied lazily when local logging is first initialized
 	const previousTransports = activeTransports;
-	activeTransports = { file: undefined, console: false };
+	activeTransports = { file: undefined, console: false, fileLevelRank: previousTransports.fileLevelRank };
 	previousTransports.file?.close();
 	activeTransports = buildTransports(opts);
 }