@keyou007/agent-bot 0.1.2 → 0.1.3

package/src/supervision/SafeRestartScheduler.ts

@@ -4,7 +4,12 @@ export interface ServerActivityState {
   latestInboundAt?: string;
 }
 
-export type SafeRestartPhase = "waiting_tasks" | "waiting_delivery" | "countdown" | "restarting";
+export type SafeRestartPhase =
+  | "waiting_tasks"
+  | "waiting_delivery"
+  | "countdown"
+  | "restarting"
+  | "cancelled";
 
 export interface SafeRestartStatus {
   scheduleId: number;
@@ -56,6 +61,24 @@ export class SafeRestartScheduler {
   }
 
   cancel(): void {
+    this.clear();
+  }
+
+  async cancelScheduled(expectedScheduleId: number): Promise<boolean> {
+    const reason = this.reason;
+    if (!reason || this.scheduleId !== expectedScheduleId) return false;
+    const activity = this.options.readActivity();
+    this.clear();
+    await this.emitStatus({
+      scheduleId: expectedScheduleId,
+      reason,
+      phase: "cancelled",
+      activity,
+    });
+    return true;
+  }
+
+  private clear(): void {
     if (this.timer) clearInterval(this.timer);
     this.timer = undefined;
     this.reason = undefined;
@@ -117,8 +140,9 @@ export class SafeRestartScheduler {
       }
       return;
     }
-    await this.emitStatus({ scheduleId: this.scheduleId, reason, phase: "restarting", activity: confirmed, remainingMs: 0 });
-    this.cancel();
+    const scheduleId = this.scheduleId;
+    this.clear();
+    await this.emitStatus({ scheduleId, reason, phase: "restarting", activity: confirmed, remainingMs: 0 });
     await this.options.onReady(reason);
   }
 

package/src/supervision/SupervisorDiagnostics.ts

@@ -0,0 +1,268 @@
+import fs from "node:fs";
+import path from "node:path";
+import type { AppConfig } from "../config/schema.js";
+
+const MAX_LOG_BYTES = 10 * 1024 * 1024;
+const LOG_BACKUP_COUNT = 3;
+
+export interface SupervisorDiagnosticsPaths {
+  supervisorLogPath: string;
+  workerStderrPath: string;
+  crashReportDirectory: string;
+  lastCrashPath: string;
+}
+
+export interface WorkerCrashContext {
+  workerPid?: number;
+  exitCode: number | null;
+  signal: NodeJS.Signals | null;
+  startedAt: string;
+  exitedAt: string;
+  uptimeMs: number;
+  consecutiveFailures: number;
+  restartDelayMs: number;
+}
+
+export interface WorkerCrashRecord extends WorkerCrashContext {
+  version: 1;
+  supervisorPid: number;
+  nodeVersion: string;
+  executablePath: string;
+  cwd: string;
+  supervisorLogPath: string;
+  workerStderrPath: string;
+  crashReportDirectory: string;
+  diagnosticReports: string[];
+}
+
+type ExtendedProcessReport = typeof process.report & {
+  excludeEnv?: boolean;
+  excludeNetwork?: boolean;
+};
+
+export class SupervisorDiagnostics {
+  readonly paths: SupervisorDiagnosticsPaths;
+
+  constructor(config: Pick<AppConfig, "logging" | "storage">) {
+    this.paths = resolveSupervisorDiagnosticsPaths(config);
+  }
+
+  initialize(): void {
+    try {
+      ensurePrivateDirectory(path.dirname(this.paths.supervisorLogPath));
+      ensurePrivateDirectory(this.paths.crashReportDirectory);
+      rotateLogFile(this.paths.supervisorLogPath);
+      rotateLogFile(this.paths.workerStderrPath);
+      this.configureCurrentProcessReports();
+      this.writeEvent("diagnostics_initialized", {
+        nodeVersion: process.versions.node,
+        supervisorLogPath: this.paths.supervisorLogPath,
+        workerStderrPath: this.paths.workerStderrPath,
+        crashReportDirectory: this.paths.crashReportDirectory,
+      });
+    } catch (error) {
+      writeFallbackDiagnostic("initialize_failed", error);
+    }
+  }
+
+  writeEvent(event: string, data: Record<string, unknown> = {}): void {
+    const line = `${JSON.stringify({
+      component: "agent-bot-supervisor",
+      event,
+      time: new Date().toISOString(),
+      ...data,
+    })}\n`;
+    try {
+      fs.appendFileSync(this.paths.supervisorLogPath, line, { encoding: "utf8", mode: 0o600 });
+      restrictFilePermissions(this.paths.supervisorLogPath);
+    } catch (error) {
+      writeFallbackDiagnostic(event, error);
+    }
+  }
+
+  openWorkerStderr(): number | "ignore" {
+    try {
+      rotateLogFile(this.paths.workerStderrPath);
+      fs.appendFileSync(
+        this.paths.workerStderrPath,
+        `${JSON.stringify({ event: "worker_started", time: new Date().toISOString() })}\n`,
+        { encoding: "utf8", mode: 0o600 },
+      );
+      restrictFilePermissions(this.paths.workerStderrPath);
+      return fs.openSync(this.paths.workerStderrPath, "a", 0o600);
+    } catch (error) {
+      this.writeEvent("worker_stderr_open_failed", { error: errorMessage(error) });
+      return "ignore";
+    }
+  }
+
+  closeWorkerStderr(target: number | "ignore"): void {
+    if (typeof target !== "number") return;
+    try {
+      fs.closeSync(target);
+    } catch (error) {
+      this.writeEvent("worker_stderr_close_failed", { error: errorMessage(error) });
+    }
+  }
+
+  recordCrash(context: WorkerCrashContext): WorkerCrashRecord | undefined {
+    try {
+      const record: WorkerCrashRecord = {
+        version: 1,
+        ...context,
+        supervisorPid: process.pid,
+        nodeVersion: process.versions.node,
+        executablePath: process.execPath,
+        cwd: process.cwd(),
+        supervisorLogPath: this.paths.supervisorLogPath,
+        workerStderrPath: this.paths.workerStderrPath,
+        crashReportDirectory: this.paths.crashReportDirectory,
+        diagnosticReports: findDiagnosticReports(
+          this.paths.crashReportDirectory,
+          context.workerPid,
+          Date.parse(context.startedAt),
+        ),
+      };
+      const crashPath = path.join(
+        this.paths.crashReportDirectory,
+        crashRecordFilename(context.exitedAt, context.workerPid),
+      );
+      writePrivateJson(crashPath, record);
+      writePrivateJson(this.paths.lastCrashPath, record);
+      this.writeEvent("crash_context_saved", {
+        crashPath,
+        lastCrashPath: this.paths.lastCrashPath,
+        diagnosticReports: record.diagnosticReports,
+      });
+      return record;
+    } catch (error) {
+      this.writeEvent("crash_context_save_failed", { error: errorMessage(error) });
+      return undefined;
+    }
+  }
+
+  private configureCurrentProcessReports(): void {
+    const report = process.report as ExtendedProcessReport;
+    report.directory = this.paths.crashReportDirectory;
+    report.reportOnFatalError = true;
+    report.reportOnUncaughtException = true;
+    if ("excludeEnv" in report) report.excludeEnv = true;
+    if ("excludeNetwork" in report) report.excludeNetwork = true;
+  }
+}
+
+export function resolveSupervisorDiagnosticsPaths(
+  config: Pick<AppConfig, "logging" | "storage">,
+): SupervisorDiagnosticsPaths {
+  const logDirectory = path.dirname(config.logging.path);
+  const dataDirectory = path.dirname(config.storage.sqlitePath);
+  return {
+    supervisorLogPath: path.join(logDirectory, "supervisor.log"),
+    workerStderrPath: path.join(logDirectory, "worker.stderr.log"),
+    crashReportDirectory: path.join(dataDirectory, "crash-reports"),
+    lastCrashPath: path.join(dataDirectory, "last-crash.json"),
+  };
+}
+
+export function nodeDiagnosticReportArguments(
+  reportDirectory: string,
+  allowedFlags: ReadonlySet<string> = process.allowedNodeEnvironmentFlags,
+): string[] {
+  const args = [
+    "--report-on-fatalerror",
+    "--report-uncaught-exception",
+    `--report-directory=${reportDirectory}`,
+  ];
+  if (allowedFlags.has("--report-exclude-env")) args.push("--report-exclude-env");
+  if (allowedFlags.has("--report-exclude-network")) args.push("--report-exclude-network");
+  return args;
+}
+
+export function prepareCrashReportDirectory(reportDirectory: string): boolean {
+  try {
+    ensurePrivateDirectory(reportDirectory);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function ensurePrivateDirectory(directoryPath: string): void {
+  fs.mkdirSync(directoryPath, { recursive: true, mode: 0o700 });
+  try {
+    fs.chmodSync(directoryPath, 0o700);
+  } catch {
+    // Windows and some mounted filesystems do not implement POSIX modes.
+  }
+}
+
+function rotateLogFile(filePath: string): void {
+  const size = fs.statSync(filePath, { throwIfNoEntry: false })?.size ?? 0;
+  if (size < MAX_LOG_BYTES) return;
+  for (let index = LOG_BACKUP_COUNT; index >= 1; index -= 1) {
+    const source = index === 1 ? filePath : `${filePath}.${index - 1}`;
+    const target = `${filePath}.${index}`;
+    if (!fs.existsSync(source)) continue;
+    fs.rmSync(target, { force: true });
+    fs.renameSync(source, target);
+  }
+}
+
+function findDiagnosticReports(
+  reportDirectory: string,
+  workerPid: number | undefined,
+  startedAtMs: number,
+): string[] {
+  if (!fs.existsSync(reportDirectory)) return [];
+  const pidToken = workerPid === undefined ? undefined : `.${workerPid}.`;
+  return fs.readdirSync(reportDirectory, { withFileTypes: true })
+    .filter((entry) => entry.isFile() && /^report\..+\.json$/i.test(entry.name))
+    .filter((entry) => {
+      if (pidToken && entry.name.includes(pidToken)) return true;
+      const modifiedAt = fs.statSync(path.join(reportDirectory, entry.name)).mtimeMs;
+      return workerPid === undefined && modifiedAt >= startedAtMs - 1_000;
+    })
+    .map((entry) => path.join(reportDirectory, entry.name))
+    .sort();
+}
+
+function crashRecordFilename(exitedAt: string, workerPid: number | undefined): string {
+  const timestamp = exitedAt.replace(/\D/g, "");
+  return `crash-${timestamp}-${workerPid ?? "unknown"}.json`;
+}
+
+function writePrivateJson(filePath: string, value: unknown): void {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
+  fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, {
+    encoding: "utf8",
+    mode: 0o600,
+  });
+  restrictFilePermissions(filePath);
+}
+
+function restrictFilePermissions(filePath: string): void {
+  try {
+    fs.chmodSync(filePath, 0o600);
+  } catch {
+    // Windows and some mounted filesystems do not implement POSIX modes.
+  }
+}
+
+function writeFallbackDiagnostic(event: string, error: unknown): void {
+  try {
+    process.stderr.write(
+      `${JSON.stringify({
+        component: "agent-bot-supervisor",
+        event,
+        time: new Date().toISOString(),
+        error: errorMessage(error),
+      })}\n`,
+    );
+  } catch {
+    // Diagnostics must never prevent the supervisor from keeping the worker alive.
+  }
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/supervisor.ts

@@ -12,12 +12,15 @@ import {
   STABLE_UPTIME_MS,
   STOP_EXIT_CODE,
 } from "./supervision/restartPolicy.js";
+import { SupervisorDiagnostics, nodeDiagnosticReportArguments } from "./supervision/SupervisorDiagnostics.js";
 
 const childEntry = fileURLToPath(new URL("./index.js", import.meta.url));
 const config = loadConfig();
 requireServerFeishuTransport(config.feishu);
 const sqlitePath = config.storage.sqlitePath;
 const serverEndpoint = controlEndpoint(sqlitePath);
+const diagnostics = new SupervisorDiagnostics(config);
+diagnostics.initialize();
 let child: ChildProcess | undefined;
 let restartTimer: NodeJS.Timeout | undefined;
 let stopping = false;
@@ -34,19 +37,33 @@ async function startChild(): Promise<void> {
   const startedAt = Date.now();
   const restartReason = nextStartReason;
   nextStartReason = "Supervisor 重新拉起进程";
-  child = spawn(process.execPath, [childEntry], {
-    cwd: process.cwd(),
-    env: { ...process.env, AGENT_BOT_SUPERVISED: "1", AGENT_BOT_RESTART_REASON: restartReason },
-    stdio: "inherit",
-    windowsHide: true,
+  const workerStderr = diagnostics.openWorkerStderr();
+  try {
+    child = spawn(process.execPath, [
+      ...nodeDiagnosticReportArguments(diagnostics.paths.crashReportDirectory),
+      childEntry,
+    ], {
+      cwd: process.cwd(),
+      env: { ...process.env, AGENT_BOT_SUPERVISED: "1", AGENT_BOT_RESTART_REASON: restartReason },
+      stdio: ["ignore", "ignore", workerStderr],
+      windowsHide: true,
+    });
+  } finally {
+    diagnostics.closeWorkerStderr(workerStderr);
+  }
+  const workerPid = child.pid;
+  writeSupervisorLog("started", {
+    pid: child.pid,
+    restartReason,
+    workerStderrPath: diagnostics.paths.workerStderrPath,
+    crashReportDirectory: diagnostics.paths.crashReportDirectory,
   });
-  writeSupervisorLog("started", { pid: child.pid, restartReason });
 
   child.once("error", (error) => {
-    writeSupervisorLog("spawn_error", { error: error.message });
+    writeSupervisorLog("spawn_error", { pid: workerPid, error: error.message });
   });
   child.once("exit", (code, signal) => {
-    void handleChildExit(code, signal, startedAt);
+    void handleChildExit(code, signal, startedAt, workerPid);
   });
 }
 
@@ -54,10 +71,11 @@ async function handleChildExit(
   code: number | null,
   signal: NodeJS.Signals | null,
   startedAt: number,
+  workerPid: number | undefined,
 ): Promise<void> {
   child = undefined;
   const uptimeMs = Date.now() - startedAt;
-  writeSupervisorLog("exited", { code, signal, uptimeMs });
+  writeSupervisorLog("exited", { pid: workerPid, code, signal, uptimeMs });
   if (stopping) {
     process.exit(0);
     return;
@@ -80,7 +98,24 @@ async function handleChildExit(
   const delayMs = intentional
     ? INTENTIONAL_RESTART_DELAY_MS
     : crashRestartDelayMs(consecutiveFailures);
-  writeSupervisorLog("restarting", { delayMs, consecutiveFailures, intentional });
+  if (!intentional) {
+    diagnostics.recordCrash({
+      workerPid,
+      exitCode: code,
+      signal,
+      startedAt: new Date(startedAt).toISOString(),
+      exitedAt: new Date().toISOString(),
+      uptimeMs,
+      consecutiveFailures,
+      restartDelayMs: delayMs,
+    });
+  }
+  writeSupervisorLog("restarting", {
+    previousPid: workerPid,
+    delayMs,
+    consecutiveFailures,
+    intentional,
+  });
   restartTimer = setTimeout(() => void startChild(), delayMs);
 }
 
@@ -96,7 +131,7 @@ function stop(signal: NodeJS.Signals): void {
 }
 
 function writeSupervisorLog(event: string, data: Record<string, unknown>): void {
-  process.stdout.write(`${JSON.stringify({ component: "agent-bot-supervisor", event, time: new Date().toISOString(), ...data })}\n`);
+  diagnostics.writeEvent(event, data);
 }
 
 process.on("SIGINT", () => stop("SIGINT"));