@ronkovic/aad 0.3.0 → 0.3.2

package/src/shared/memory-check.ts

@@ -0,0 +1,155 @@
+/**
+ * Memory Check Utility
+ * Provides memory status checking for AAD pipeline execution
+ */
+
+import type pino from "pino";
+import { MemoryError } from "./errors";
+
+export interface MemoryStatus {
+  totalGB: number;
+  usedGB: number;
+  freeGB: number;
+  usedPercent: number;
+  isLowMemory: boolean;
+  recommendedAction?: string;
+}
+
+/**
+ * Get current system memory status (macOS only)
+ */
+export async function getMemoryStatus(): Promise<MemoryStatus> {
+  const { stdout } = await Bun.spawn(["vm_stat"], {
+    stdout: "pipe",
+  }).exited.then(async (code) => {
+    if (code !== 0) throw new Error("vm_stat failed");
+    return {
+      stdout: await new Response(Bun.spawn(["vm_stat"], { stdout: "pipe" }).stdout).text(),
+    };
+  });
+
+  // Parse vm_stat output
+  const pageSizeMatch = stdout.match(/page size of (\d+) bytes/);
+  const freeMatch = stdout.match(/Pages free:\s+(\d+)/);
+  const activeMatch = stdout.match(/Pages active:\s+(\d+)/);
+  const inactiveMatch = stdout.match(/Pages inactive:\s+(\d+)/);
+  const speculativeMatch = stdout.match(/Pages speculative:\s+(\d+)/);
+  const wiredMatch = stdout.match(/Pages wired down:\s+(\d+)/);
+
+  if (!pageSizeMatch || !freeMatch || !activeMatch || !inactiveMatch || !wiredMatch) {
+    throw new Error("Failed to parse vm_stat output");
+  }
+
+  const pageSize = parseInt(pageSizeMatch[1]!, 10);
+  const freePages = parseInt(freeMatch[1]!, 10);
+  const activePages = parseInt(activeMatch[1]!, 10);
+  const inactivePages = parseInt(inactiveMatch[1]!, 10);
+  const speculativePages = parseInt(speculativeMatch?.[1] ?? "0", 10);
+  const wiredPages = parseInt(wiredMatch[1]!, 10);
+
+  // Calculate memory in GB
+  const bytesToGB = (bytes: number) => bytes / (1024 * 1024 * 1024);
+  const pagesToGB = (pages: number) => bytesToGB(pages * pageSize);
+
+  const freeGB = pagesToGB(freePages + speculativePages);
+  const usedGB = pagesToGB(activePages + inactivePages + wiredPages);
+  const totalGB = freeGB + usedGB;
+  const usedPercent = (usedGB / totalGB) * 100;
+
+  // Determine if memory is low
+  const isLowMemory = freeGB < 2.0; // Less than 2GB free
+  let recommendedAction: string | undefined;
+
+  if (freeGB < 1.5) {
+    recommendedAction = "Critical: Free memory < 1.5GB. Close heavy applications (Chrome, Docker) before running AAD.";
+  } else if (freeGB < 2.5) {
+    recommendedAction = "Warning: Free memory < 2.5GB. Consider closing unnecessary applications.";
+  }
+
+  return {
+    totalGB: Math.round(totalGB * 10) / 10,
+    usedGB: Math.round(usedGB * 10) / 10,
+    freeGB: Math.round(freeGB * 10) / 10,
+    usedPercent: Math.round(usedPercent),
+    isLowMemory,
+    recommendedAction,
+  };
+}
+
+export interface WaitForMemoryOptions {
+  minFreeGB?: number;
+  retryIntervalMs?: number;
+  maxRetries?: number;
+}
+
+export interface WaitForMemoryResult {
+  shouldReduceWorkers: boolean;
+  freeGB: number;
+}
+
+/**
+ * Wait for sufficient memory before dispatching a task.
+ * - freeGB < 1.5GB → wait and retry (default 30s, max 5 retries)
+ * - freeGB < 1.0GB after all retries → throw MemoryError
+ * - freeGB < 3.0GB → return { shouldReduceWorkers: true }
+ */
+export async function waitForMemory(
+  logger: pino.Logger,
+  options?: WaitForMemoryOptions,
+  _getMemoryStatus?: () => Promise<MemoryStatus>,
+  _sleep?: (ms: number) => Promise<void>,
+): Promise<WaitForMemoryResult> {
+  const minFreeGB = options?.minFreeGB ?? 1.5;
+  const retryIntervalMs = options?.retryIntervalMs ?? 30_000;
+  const maxRetries = options?.maxRetries ?? 5;
+  const getStatus = _getMemoryStatus ?? getMemoryStatus;
+  const sleep = _sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
+
+  let status = await getStatus();
+  let retries = 0;
+
+  while (status.freeGB < minFreeGB && retries < maxRetries) {
+    retries++;
+    logger.warn(
+      { freeGB: status.freeGB, retry: retries, maxRetries },
+      `Low memory (${status.freeGB}GB free). Waiting ${retryIntervalMs / 1000}s before retry ${retries}/${maxRetries}...`,
+    );
+    await sleep(retryIntervalMs);
+    status = await getStatus();
+  }
+
+  if (status.freeGB < 1.0) {
+    throw new MemoryError(`Insufficient memory: ${status.freeGB}GB free (need at least 1.0GB)`, {
+      freeGB: status.freeGB,
+      retries,
+    });
+  }
+
+  return {
+    shouldReduceWorkers: status.freeGB < 3.0,
+    freeGB: status.freeGB,
+  };
+}
+
+/**
+ * Check memory and log warning if low
+ */
+export async function checkMemoryAndWarn(logger: pino.Logger): Promise<void> {
+  try {
+    const status = await getMemoryStatus();
+    
+    logger.info({
+      totalGB: status.totalGB,
+      usedGB: status.usedGB,
+      freeGB: status.freeGB,
+      usedPercent: status.usedPercent,
+    }, "Memory status");
+
+    if (status.isLowMemory && status.recommendedAction) {
+      logger.warn({ freeGB: status.freeGB }, status.recommendedAction);
+      console.warn(`\n⚠️  ${status.recommendedAction}\n`);
+    }
+  } catch (error) {
+    logger.debug({ error }, "Memory check failed (non-critical)");
+  }
+}

package/src/shared/memory-monitor.ts

@@ -0,0 +1,69 @@
+/**
+ * Memory Monitor
+ * Polls system memory during SDK execution and fires callback when critical.
+ */
+
+import type pino from "pino";
+import { getMemoryStatus, type MemoryStatus } from "./memory-check";
+
+export interface MemoryMonitorOptions {
+  logger: pino.Logger;
+  checkIntervalMs?: number;
+  criticalFreeGB?: number;
+  /** For testing: inject a custom getMemoryStatus */
+  _getMemoryStatus?: () => Promise<MemoryStatus>;
+}
+
+export interface MemoryMonitor {
+  start(): void;
+  stop(): void;
+  onCritical(callback: () => void): void;
+}
+
+export function createMemoryMonitor(options: MemoryMonitorOptions): MemoryMonitor {
+  const {
+    logger,
+    checkIntervalMs = 2000,
+    criticalFreeGB = 0.8,
+    _getMemoryStatus = getMemoryStatus,
+  } = options;
+
+  let intervalId: ReturnType<typeof setInterval> | null = null;
+  let criticalCallback: (() => void) | null = null;
+  let fired = false;
+
+  return {
+    start() {
+      if (intervalId) return;
+      fired = false;
+      intervalId = setInterval(() => {
+        void (async () => {
+          try {
+            const status = await _getMemoryStatus();
+            if (status.freeGB < criticalFreeGB && !fired) {
+              fired = true;
+              logger.error(
+                { freeGB: status.freeGB, criticalFreeGB },
+                "Critical memory threshold reached"
+              );
+              criticalCallback?.();
+            }
+          } catch (err) {
+            logger.debug({ err }, "Memory monitor check failed");
+          }
+        })();
+      }, checkIntervalMs);
+    },
+
+    stop() {
+      if (intervalId) {
+        clearInterval(intervalId);
+        intervalId = null;
+      }
+    },
+
+    onCritical(callback: () => void) {
+      criticalCallback = callback;
+    },
+  };
+}

package/src/shared/shutdown-handler.ts

@@ -0,0 +1,115 @@
+/**
+ * Graceful Shutdown Handler
+ * Saves run state on SIGTERM/SIGINT/uncaughtException so --resume can pick up.
+ */
+
+import type pino from "pino";
+import type { RunStore, TaskStore } from "../modules/persistence/stores.port";
+import type { RunId } from "./types";
+
+export interface ShutdownHandlerOptions {
+  runId: RunId;
+  stores: {
+    runStore: RunStore;
+    taskStore: TaskStore;
+  };
+  logger: pino.Logger;
+  exitFn?: (code: number) => void;
+}
+
+let _shuttingDown = false;
+let _lastSignalTime = 0;
+let _installed = false;
+let _cleanup: (() => void) | null = null;
+
+/**
+ * Check if the process is shutting down.
+ */
+export function isShuttingDown(): boolean {
+  return _shuttingDown;
+}
+
+/**
+ * Reset internal state (for testing only).
+ */
+export function _resetShutdownState(): void {
+  _shuttingDown = false;
+  _lastSignalTime = 0;
+  if (_cleanup) {
+    _cleanup();
+    _cleanup = null;
+  }
+  _installed = false;
+}
+
+/**
+ * Install signal handlers for graceful shutdown with state persistence.
+ */
+export function installShutdownHandler(options: ShutdownHandlerOptions): void {
+  if (_installed) return;
+  _installed = true;
+
+  const { runId, stores, logger, exitFn = (code: number) => process.exit(code) } = options;
+  let saving = false;
+
+  const handleShutdown = async (reason: string) => {
+    const now = Date.now();
+    // Debounce: ignore signals within 1s
+    if (now - _lastSignalTime < 1000) return;
+    _lastSignalTime = now;
+
+    if (saving) return;
+    saving = true;
+    _shuttingDown = true;
+
+    logger.warn({ runId, reason }, "Graceful shutdown initiated, saving state...");
+
+    try {
+      // Update running tasks to pending so resume can re-dispatch them
+      const tasks = await stores.taskStore.getAll();
+      for (const task of tasks) {
+        if (task.status === "running") {
+          await stores.taskStore.save({ ...task, status: "pending", workerId: undefined });
+        }
+      }
+
+      // Update run state counts
+      const allTasks = await stores.taskStore.getAll();
+      const pending = allTasks.filter((t) => t.status === "pending").length;
+      const completed = allTasks.filter((t) => t.status === "completed").length;
+      const failed = allTasks.filter((t) => t.status === "failed").length;
+
+      const runState = await stores.runStore.get(runId);
+      if (runState) {
+        await stores.runStore.save({
+          ...runState,
+          pending,
+          running: 0,
+          completed,
+          failed,
+          endTime: new Date().toISOString(),
+        });
+      }
+
+      logger.info({ runId, pending, completed, failed }, "State saved. Use --resume to continue.");
+    } catch (err) {
+      logger.error({ err }, "Failed to save state during shutdown");
+    }
+
+    exitFn(1);
+  };
+
+  const onSigterm = () => { void handleShutdown("SIGTERM"); };
+  const onSigint = () => { void handleShutdown("SIGINT"); };
+  const onUncaught = (err: Error) => { void handleShutdown(`uncaughtException: ${err.message}`); };
+
+  process.on("SIGTERM", onSigterm);
+  process.on("SIGINT", onSigint);
+  process.on("uncaughtException", onUncaught);
+
+  _cleanup = () => {
+    process.removeListener("SIGTERM", onSigterm);
+    process.removeListener("SIGINT", onSigint);
+    process.removeListener("uncaughtException", onUncaught);
+  };
+}

package/src/shared/utils.ts

@@ -0,0 +1,49 @@
+// Utility functions for shared functionality
+
+/**
+ * Capitalizes the first character of a string.
+ * @param str - The string to capitalize
+ * @returns The string with the first character capitalized
+ */
+export function capitalize(str: string): string {
+  if (str.length === 0) {
+    return str;
+  }
+  return str.charAt(0).toUpperCase() + str.slice(1);
+}
+
+/**
+ * Converts all characters in a string to lowercase.
+ * @param str - The string to convert
+ * @returns The string with all characters in lowercase
+ */
+export function toLowerCase(str: string): string {
+  return str.toLowerCase();
+}
+
+/**
+ * Removes leading and trailing whitespace from a string.
+ * @param str - The string to trim
+ * @returns The string with leading and trailing whitespace removed
+ */
+export function trim(str: string): string {
+  return str.trim();
+}
+
+/**
+ * Reverses a string.
+ * @param str - The string to reverse
+ * @returns The reversed string
+ */
+export function reverse(str: string): string {
+  return str.split("").reverse().join("");
+}
+
+/**
+ * Converts all characters in a string to uppercase.
+ * @param str - The string to convert
+ * @returns The string with all characters in uppercase
+ */
+export function toUpperCase(str: string): string {
+  return str.toUpperCase();
+}