@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/logger.ts

@@ -1,23 +1,46 @@
 /**
  * Logger — scoped file-based diagnostics
  *
- * Silent by default. When --verbose or PI_RESEARCH_VERBOSE=1 is set,
- * writes timestamped lines to {tmpdir}/pi-research-debug-{hash}.log where {hash}
- * is a random 4-character alphanumeric suffix (a-z, 0-9) to keep logs separate per run.
+ * Writes newline-delimited JSON records to a single consolidated log file. The
+ * path defaults to {os.tmpdir()}/pi-research.log and is overridable via the
+ * PI_RESEARCH_LOG_PATH env var (see buildDefaultDebugLogPath / log-utils). The
+ * file rotates at 10MB, keeping the last 10 archives.
+ * ERROR and WARN levels are always logged.
+ * INFO and DEBUG levels are only logged when PI_RESEARCH_DEBUG=true (or config.DEBUG=true).
  *
- * When NOT verbose: logFile is null, preventing all temp-dir writes.
  * This module never patches process-global console.* methods.
+ *
+ * Write durability / performance contract (see LogWriter):
+ *  - WARN/ERROR are written SYNCHRONOUSLY. They are the primary forensic signal
+ *    and must survive an immediate crash, so they are never buffered.
+ *  - INFO/DEBUG (only emitted when verbose) are buffered and flushed off the
+ *    event loop via fs.promises.appendFile, so high-frequency debug logging
+ *    never blocks the event loop or the TUI render path — regardless of whether
+ *    the log file lives on a fast (tmpfs/RAM) or slow (disk) filesystem. The
+ *    buffer is drained synchronously before every WARN/ERROR write (preserving
+ *    order) and on process 'exit', so graceful shutdowns lose nothing. Only a
+ *    hard SIGKILL can drop the sub-millisecond tail of buffered INFO/DEBUG,
+ *    which is acceptable for non-critical diagnostics.
  */
 
-import { appendFileSync } from 'node:fs';
+import { appendFileSync, existsSync, mkdirSync, openSync, closeSync } from 'node:fs';
+import { appendFile } from 'node:fs/promises';
 import { AsyncLocalStorage } from 'node:async_hooks';
-import { randomBytes } from 'node:crypto';
-import * as os from 'node:os';
 import * as path from 'node:path';
+import { errorTracker, type ErrorContext } from './utils/error-tracker.ts';
+import {
+  buildDefaultDebugLogPath,
+  isVerboseFromEnv,
+  getLogContext,
+  safeJsonStringify,
+  formatArg,
+  redactSecrets,
+  type LogContext,
+} from './utils/log-utils.ts';
+import { LogRotation } from './utils/log-rotation.ts';
+import { DiskSpaceChecker } from './utils/disk-space-checker.ts';
+import { captureStdio } from './utils/stdio-capture.ts';
 
-/**
- * Log level enum
- */
 export enum LogLevel {
   INFO = 'INFO',
   ERROR = 'ERROR',
@@ -25,142 +48,259 @@ export enum LogLevel {
   DEBUG = 'DEBUG',
 }
 
-/**
- * Logger interface for dependency injection
- */
 export interface ILogger {
   log(...args: unknown[]): void;
   info(...args: unknown[]): void;
   error(...args: unknown[]): void;
   warn(...args: unknown[]): void;
   debug(...args: unknown[]): void;
+  clear(): void;
+  flushSync(): void;
+  runCapturingStderr<T>(task: () => Promise<T>): Promise<T>;
 }
 
-/**
- * Logger configuration options
- */
 export interface LoggerOptions {
   verbose: boolean;
   logFilePath?: string;
-}
-
-export interface LogContext {
-  sessionId?: string;
-  sessionFile?: string;
-  cwd?: string;
   researchRunId?: string;
-  toolName?: string;
-  phase?: string;
-  eventName?: string;
+  consoleLog?: boolean;
 }
 
-const logContextStorage = new AsyncLocalStorage<LogContext>();
-
 /**
- * Generate a 4-character alphanumeric hash for uniqueness
+ * Per-file log writer, shared across every Logger instance that targets the
+ * same path (logging is consolidated to one file, so a single shared buffer is
+ * required to preserve global write order).
+ *
+ * INFO/DEBUG go through enqueue() — buffered, flushed asynchronously off the
+ * event loop. WARN/ERROR go through writeSync() — which first drains the buffer
+ * (keeping order) and then writes synchronously, so critical lines are durable.
  */
-function generateHash(): string {
-  const chars = 'abcdefghijklmnopqrstuvwxyz0123456789';
-  let hash = '';
-  for (let i = 0; i < 4; i++) {
-    hash += chars.charAt(Math.floor(Math.random() * chars.length));
+class LogWriter {
+  private buffer: string[] = [];
+  private bufferBytes = 0;
+  private flushScheduled = false;
+  private flushing = false;
+  // Flush eagerly once buffered output gets large, to bound memory and latency
+  // during sustained bursts instead of waiting for the scheduled tick.
+  private static readonly MAX_BUFFER_BYTES = 256 * 1024;
+
+  constructor(private readonly filePath: string) {}
+
+  /** Queue a line for asynchronous, non-blocking write (INFO/DEBUG path). */
+  enqueue(line: string): void {
+    this.buffer.push(line);
+    this.bufferBytes += line.length;
+    if (this.bufferBytes >= LogWriter.MAX_BUFFER_BYTES) {
+      void this.flush();
+      return;
+    }
+    this.scheduleFlush();
   }
-  return hash;
-}
-
-function buildDefaultDebugLogPath(hash: string): string {
-  return path.join(os.tmpdir(), `pi-research-debug-${hash}.log`);
-}
-
-export function getDefaultDebugLogPathTemplate(): string {
-  return buildDefaultDebugLogPath('{hash}');
-}
-
-/**
- * Check if verbose mode is enabled from environment
- */
-export function isVerboseFromEnv(): boolean {
-  return process.argv.includes('--verbose') || process.env['PI_RESEARCH_VERBOSE'] === '1';
-}
 
-export function createResearchRunId(): string {
-  return `run-${randomBytes(4).toString('hex')}`;
-}
+  private scheduleFlush(): void {
+    if (this.flushScheduled || this.flushing) return;
+    this.flushScheduled = true;
+    // setTimeout(0) runs the flush on the next event-loop turn — off the
+    // synchronous caller path — and coalesces a burst of lines into one write.
+    setTimeout(() => { void this.flush(); }, 0);
+  }
 
-export function getLogContext(): LogContext {
-  return logContextStorage.getStore() ?? {};
-}
+  /** Asynchronously write buffered lines. Never blocks the event loop. */
+  private async flush(): Promise<void> {
+    this.flushScheduled = false;
+    if (this.flushing || this.buffer.length === 0) return;
+    this.flushing = true;
+    try {
+      while (this.buffer.length > 0) {
+        const batch = this.buffer.join('');
+        this.buffer.length = 0;
+        this.bufferBytes = 0;
+        try {
+          await appendFile(this.filePath, batch);
+        } catch {
+          // Silently ignore file write errors (matches the synchronous path).
+        }
+      }
+    } finally {
+      this.flushing = false;
+      // A line enqueued after the loop drained but before this point would
+      // otherwise linger with no scheduled flush — reschedule defensively.
+      if (this.buffer.length > 0) this.scheduleFlush();
+    }
+  }
 
-export function runWithLogContext<T>(context: LogContext, callback: () => T): T {
-  const parent = logContextStorage.getStore() ?? {};
-  return logContextStorage.run({ ...parent, ...context }, callback);
-}
+  /**
+   * Drain buffered lines, then write `line`, all synchronously and in order.
+   * Used for WARN/ERROR so the critical line (and all lines queued in the buffer
+   * before it) are on disk before this call returns.
+   *
+   * Ordering caveat: if a large INFO/DEBUG burst already cleared the buffer into
+   * an in-flight async appendFile, drainSync() will find the buffer empty and
+   * the WARN line will be appended before that in-flight batch completes. In
+   * practice this means WARN/ERROR lines may appear slightly before the last
+   * flush of the preceding INFO/DEBUG burst. Acceptable for a diagnostic log.
+   */
+  writeSync(line: string): void {
+    this.drainSync();
+    try {
+      appendFileSync(this.filePath, line);
+    } catch {
+      // Silently ignore file write errors.
+    }
+  }
 
-function safeJsonStringify(value: unknown): string {
-  try {
-    return JSON.stringify(value, null, 0);
-  } catch {
-    return '[unserializable]';
+  /** Synchronously flush all buffered lines (graceful exit / tests / pre-sync-write). */
+  drainSync(): void {
+    if (this.buffer.length === 0) return;
+    const batch = this.buffer.join('');
+    this.buffer.length = 0;
+    this.bufferBytes = 0;
+    try {
+      appendFileSync(this.filePath, batch);
+    } catch {
+      // Silently ignore file write errors.
+    }
   }
 }
 
-function formatArg(arg: unknown): string {
-  if (arg instanceof Error) {
-    return arg.stack ?? arg.message;
+// One writer per target path → shared, ordered buffer for consolidated logging.
+const logWriters = new Map<string, LogWriter>();
+
+// Drain every writer synchronously on graceful process exit so no buffered
+// INFO/DEBUG line is lost. Registered exactly once, on first writer creation.
+let exitDrainRegistered = false;
+function getLogWriter(filePath: string): LogWriter {
+  let writer = logWriters.get(filePath);
+  if (!writer) {
+    writer = new LogWriter(filePath);
+    logWriters.set(filePath, writer);
   }
-  if (typeof arg === 'object' && arg !== null) {
-    return safeJsonStringify(arg);
+  if (!exitDrainRegistered) {
+    exitDrainRegistered = true;
+    process.on('exit', () => {
+      for (const w of logWriters.values()) {
+        try { w.drainSync(); } catch { /* best-effort on exit */ }
+      }
+    });
   }
-  return String(arg);
+  return writer;
 }
 
 /**
- * Logger implementation — writes to file when verbose, silent otherwise
+ * Symbol used to identify real Logger instances
  */
+const LOGGER_BRAND = Symbol.for('pi-research.Logger');
+
 export class Logger implements ILogger {
   private verbose: boolean;
-  private logFile: string | null;
+  private consoleLog: boolean;
+  private logFile: string;
+  private logDir: string;
+  private readonly sessionId: string | undefined;
+  readonly [LOGGER_BRAND] = true;
+
+  private readonly rotation: LogRotation;
+  private readonly diskSpaceChecker: DiskSpaceChecker;
 
   constructor(options: Partial<LoggerOptions> = {}) {
     this.verbose = options.verbose ?? isVerboseFromEnv();
-    // Only set logFile path if verbose mode is enabled
-    if (this.verbose) {
-      const hash = generateHash();
-      this.logFile = options.logFilePath ?? buildDefaultDebugLogPath(hash);
-    } else {
-      this.logFile = null;
+    this.consoleLog = options.consoleLog ?? (process.env['PI_RESEARCH_CONSOLE_LOG'] === 'true');
+    // Consolidated logging: always use the same file regardless of run ID
+    this.logFile = options.logFilePath ?? buildDefaultDebugLogPath();
+    this.logDir = path.dirname(this.logFile);
+    this.sessionId = options.researchRunId;
+
+    this.rotation = new LogRotation(this);
+    this.diskSpaceChecker = new DiskSpaceChecker();
+
+    // Ensure parent directory exists, and pre-create the log file owner-only.
+    // The default path lives in a world-traversable tmpdir; appendFile would create
+    // it at the umask default (often world-readable). Logs are redacted, but the
+    // file should still match the 0o600 posture used for state/lock/knowledge files.
+    try {
+      if (!existsSync(this.logDir)) {
+        mkdirSync(this.logDir, { recursive: true, mode: 0o700 });
+      }
+      // Atomically create the log file owner-only if it's absent. 'wx' = O_CREAT|
+      // O_EXCL: it either creates the file at 0o600 or fails with EEXIST when one
+      // already exists — no check-then-create window for another user to win.
+      try {
+        closeSync(openSync(this.logFile, 'wx', 0o600));
+      } catch {
+        // Already exists (or not creatable) — appendFile uses the existing file.
+      }
+    } catch {
+      // Ignore if we can't create the dir or file
     }
   }
 
-  /**
-   * Emit a log message to file (if verbose) or nowhere (if silent)
-   * @param level - Log level (INFO, ERROR, WARN, DEBUG)
-   * @param args - Arguments to log
-   */
   private emit(level: string, ...args: unknown[]): void {
-    if (!this.verbose || !this.logFile) {
+    // Only filter INFO and DEBUG if not verbose
+    if (!this.verbose && (level === LogLevel.INFO || level === LogLevel.DEBUG)) {
+      return;
+    }
+
+    // Check disk space before writing
+    if (!this.diskSpaceChecker.checkDiskSpace(this.logDir)) {
       return;
     }
 
+    // Rotate log file if needed (throttled to one stat per 60s — see LogRotation).
+    this.rotation.checkAndRotate(this.logFile, this.logDir);
+
     const timestamp = new Date().toISOString();
     const firstError = args.find((arg): arg is Error => arg instanceof Error);
+    // Redact credentials and bound size before the message reaches ANY sink, so
+    // neither the log file nor the console persists clear-text secrets or
+    // unbounded network/scraped payloads.
+    const message = redactSecrets(args.map(formatArg).join(' '));
     const entry = {
       timestamp,
       level,
       ...getLogContext(),
-      message: args.map(formatArg).join(' '),
+      message,
       ...(firstError
-        ? { errorMessage: firstError.message, errorStack: firstError.stack }
+        ? { errorMessage: redactSecrets(firstError.message), errorStack: redactSecrets(firstError.stack ?? '') }
         : {}),
     };
     const line = `${safeJsonStringify(entry)}\n`;
 
-    // Write to file
-    try {
-      appendFileSync(this.logFile, line);
-    } catch {
-      // Silently ignore file write errors
+    // Write to file. WARN/ERROR are written synchronously for crash-durability;
+    // INFO/DEBUG are buffered and flushed off the event loop so high-frequency
+    // logging never blocks the loop or the TUI render path (see LogWriter).
+    const writer = getLogWriter(this.logFile);
+    if (level === LogLevel.ERROR || level === LogLevel.WARN) {
+      writer.writeSync(line);
+    } else {
+      writer.enqueue(line);
     }
+
+    // Optional console output
+    if (this.consoleLog) {
+      const color = level === LogLevel.ERROR ? '\x1b[31m' :
+                    level === LogLevel.WARN ? '\x1b[33m' :
+                    level === LogLevel.DEBUG ? '\x1b[90m' : '\x1b[36m';
+      const reset = '\x1b[0m';
+      // Reuse the already-redacted message; additionally strip CR/LF so untrusted
+      // content cannot forge log lines (log-injection defense). The JSON file sink
+      // escapes control chars on its own. CodeQL (js/log-injection) StringReplaceSanitizer
+      // requires the replacement to be "" (empty string) — not ' ' — per its QL definition:
+      //   replaces(s, "") and s.regexpMatch("\\n")
+      const msg = message.replace(/[\r\n]/g, '');
+      const rawSid = this.sessionId ?? '';
+      const sid = rawSid.replace(/[\r\n]/g, '');
+      const prefix = sid ? `[${sid}] ` : '';
+      console.log(`${color}${timestamp} ${level} ${prefix}${reset}${msg}`);
+    }
+  }
+
+  async runCapturingStderr<T>(task: () => Promise<T>): Promise<T> {
+    return captureStdio(
+      this.logFile,
+      () => this.diskSpaceChecker.checkDiskSpace(this.logDir),
+      task,
+      this.sessionId
+    );
   }
 
   log(...args: unknown[]): void {
@@ -172,6 +312,11 @@ export class Logger implements ILogger {
   }
 
   error(...args: unknown[]): void {
+    const errArg = args.find((arg): arg is Error => arg instanceof Error);
+    if (errArg) {
+      const context: ErrorContext = getLogContext();
+      errorTracker.trackError(errArg, context);
+    }
     this.emit(LogLevel.ERROR, ...args);
   }
 
@@ -183,64 +328,113 @@ export class Logger implements ILogger {
     this.emit(LogLevel.DEBUG, ...args);
   }
 
+  clear(): void {
+    this.flushSync();
+    this.rotation.clearLogs(this.logFile, this.logDir);
+  }
+
   /**
-   * Get the log file path
+   * Synchronously flush this logger's buffered INFO/DEBUG output to disk.
+   * Use before reading the log file directly (tests) or during shutdown.
    */
+  flushSync(): void {
+    getLogWriter(this.logFile).drainSync();
+  }
+
   getLogFilePath(): string | null {
     return this.logFile;
   }
 
-  /**
-   * Check if verbose mode is enabled
-   */
   isVerbose(): boolean {
     return this.verbose;
   }
 }
 
-/**
- * Global logger instance
- */
-let globalLogger: Logger | null = null;
+// AsyncLocalStorage context for concurrent run isolation
+const loggerContext = new AsyncLocalStorage<Logger>();
 
 /**
- * Create a new logger instance
+ * Run a function with a specific logger bound to the async context.
+ * All calls to getLogger() within fn (and any async work it spawns) will
+ * return this logger instance, providing proper isolation for concurrent runs.
  */
-export function createLogger(options: Partial<LoggerOptions> = {}): Logger {
-  return new Logger(options);
+export function runWithLogger<T>(loggerInstance: Logger, fn: () => Promise<T>): Promise<T> {
+  return loggerContext.run(loggerInstance, fn);
 }
 
-/**
- * Get the global logger instance (singleton)
- */
-export function getLogger(): Logger {
-  if (!globalLogger) {
-    globalLogger = createLogger({ verbose: isVerboseFromEnv() });
+// Session-scoped logger storage to support concurrent research runs
+// Each session gets its own logger instance to prevent log bleeding
+const sessionLoggers = new Map<string, Logger>();
+let _globalLogger: Logger | null = null;
+
+export function getLogger(sessionId?: string): Logger {
+  // Check async context first (used for concurrent run isolation via runWithLogger)
+  const contextLogger = loggerContext.getStore();
+  if (contextLogger) return contextLogger;
+
+  // Return session-scoped logger if sessionId is provided
+  if (sessionId) {
+    let lg = sessionLoggers.get(sessionId);
+    if (!lg) {
+      lg = new Logger({ 
+        verbose: isVerboseFromEnv(), 
+        researchRunId: sessionId,
+        consoleLog: process.env['PI_RESEARCH_CONSOLE_LOG'] === 'true'
+      });
+      sessionLoggers.set(sessionId, lg);
+    }
+    return lg;
+  }
+  // Fall back to creating a global logger if no context-bound logger is available
+  if (!_globalLogger) {
+    _globalLogger = new Logger({ 
+      verbose: isVerboseFromEnv(),
+      consoleLog: process.env['PI_RESEARCH_CONSOLE_LOG'] === 'true'
+    });
   }
-  return globalLogger;
+  return _globalLogger;
 }
 
-/**
- * Set the global logger instance (for testing)
- */
-export function setLogger(logger: Logger): void {
-  globalLogger = logger;
+export function setLogger(loggerInstance: Logger, sessionId?: string): void {
+  if (loggerInstance && !(loggerInstance as any)[LOGGER_BRAND]) {
+    throw new Error('setLogger must be called with a Logger instance, not the wrapper.');
+  }
+  if (sessionId) {
+    sessionLoggers.set(sessionId, loggerInstance);
+  } else {
+    _globalLogger = loggerInstance;
+  }
 }
 
-/**
- * Reset the global logger instance (for testing)
- */
-export function resetLogger(): void {
-  globalLogger = null;
+export function resetLogger(sessionId?: string): void {
+  if (sessionId) {
+    sessionLoggers.delete(sessionId);
+  } else {
+    _globalLogger = null;
+    sessionLoggers.clear();
+  }
+}
+
+export function createLogger(options: Partial<LoggerOptions> = {}): Logger {
+  if (options.consoleLog === undefined) {
+    options.consoleLog = process.env['PI_RESEARCH_CONSOLE_LOG'] === 'true';
+  }
+  return new Logger(options);
 }
 
 /**
- * Logger singleton for backward compatibility
+ * Global default logger proxy
  */
-export const logger = {
+export const logger: ILogger = {
   log:   (...args: unknown[]) => getLogger().log(...args),
   info:  (...args: unknown[]) => getLogger().info(...args),
   error: (...args: unknown[]) => getLogger().error(...args),
   warn:  (...args: unknown[]) => getLogger().warn(...args),
   debug: (...args: unknown[]) => getLogger().debug(...args),
+  clear: () => getLogger().clear(),
+  flushSync: () => getLogger().flushSync(),
+  runCapturingStderr: <T>(task: () => Promise<T>) => getLogger().runCapturingStderr(task),
 };
+
+export { type LogContext };
+export { buildDefaultDebugLogPath, isVerboseFromEnv, createResearchRunId, getLogContext, runWithLogContext } from './utils/log-utils.ts';