@tonyclaw/agent-inspector 3.1.32 → 4.0.2

package/src/proxy/runtimeHealth.ts

@@ -1,221 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { writeFile, rm, statfs } from "node:fs/promises";
-import { join } from "node:path";
-import { getDataDir } from "./dataDir";
-import { getFinalizerRuntimeHealth } from "./logFinalizer";
-import { ensurePrivateDirectorySync, PRIVATE_FILE_MODE } from "./privateDataPath";
-import { isSqliteLogIndexReady } from "./sqliteLogIndex";
-import { recoverCapturedLogStorage } from "./jsonlRecovery";
-import { resolveLogDir } from "./logger";
-import { loadIndex } from "./logIndex";
-import { getSessionRuntimeQueueHealth } from "./sessionRuntime";
-
-const DEFAULT_CACHE_MS = 2_000;
-const DEFAULT_MIN_DISK_HEADROOM_BYTES = 256 * 1024 * 1024;
-const DEFAULT_MAX_FINALIZER_QUEUE = 256;
-
-export type ReadinessLevel = "ready" | "degraded" | "unavailable";
-export type ReadinessCheckLevel = "ok" | "degraded" | "unavailable";
-
-export type SafeReadiness = {
-  status: ReadinessLevel;
-  checks: {
-    storage: ReadinessCheckLevel;
-    disk: ReadinessCheckLevel;
-    workers: ReadinessCheckLevel;
-    index: ReadinessCheckLevel;
-    queue: ReadinessCheckLevel;
-    shutdown: ReadinessCheckLevel;
-  };
-  checkedAt: string;
-};
-
-export type ReadinessDependencies = {
-  checkStorage: () => Promise<ReadinessCheckLevel>;
-  checkDisk: () => Promise<ReadinessCheckLevel>;
-  checkWorkers: () => Promise<ReadinessCheckLevel>;
-  checkIndex: () => Promise<ReadinessCheckLevel>;
-  checkQueue: () => Promise<ReadinessCheckLevel>;
-  checkShutdown: () => Promise<ReadinessCheckLevel>;
-  now: () => number;
-};
-
-export type ReadinessService = {
-  get: () => Promise<SafeReadiness>;
-  invalidate: () => void;
-};
-
-let runtimeShuttingDown = false;
-
-function configuredPositiveInteger(name: string, fallback: number): number {
-  const raw = process.env[name];
-  if (raw === undefined || raw === "") return fallback;
-  const parsed = Number(raw);
-  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
-}
-
-async function checkStorageWritable(): Promise<ReadinessCheckLevel> {
-  const dataDir = getDataDir();
-  if (!ensurePrivateDirectorySync(dataDir)) return "unavailable";
-  const probePath = join(dataDir, `.readiness-${String(process.pid)}-${randomUUID()}`);
-  try {
-    await writeFile(probePath, "", { flag: "wx", mode: PRIVATE_FILE_MODE });
-    return "ok";
-  } catch {
-    return "unavailable";
-  } finally {
-    await rm(probePath, { force: true }).catch(() => undefined);
-  }
-}
-
-async function checkDiskHeadroom(): Promise<ReadinessCheckLevel> {
-  try {
-    const stats = await statfs(getDataDir());
-    const availableBytes = stats.bavail * stats.bsize;
-    const minimumBytes = configuredPositiveInteger(
-      "AGENT_INSPECTOR_MIN_DISK_HEADROOM_BYTES",
-      DEFAULT_MIN_DISK_HEADROOM_BYTES,
-    );
-    return availableBytes >= minimumBytes ? "ok" : "unavailable";
-  } catch {
-    return "degraded";
-  }
-}
-
-function finalizerChecks(): {
-  workers: ReadinessCheckLevel;
-  queue: ReadinessCheckLevel;
-} {
-  const health = getFinalizerRuntimeHealth();
-  const sessionQueue = getSessionRuntimeQueueHealth();
-  const runtimeArtifactMissing = health.runtime !== "inline" && !health.artifactAvailable;
-  const initializedWorkerMissing =
-    health.runtime === "worker" &&
-    health.activeWorkers !== null &&
-    health.activeWorkers < health.configuredWorkers;
-  const workers =
-    runtimeArtifactMissing || initializedWorkerMissing
-      ? "unavailable"
-      : health.runtime === "inline"
-        ? "degraded"
-        : "ok";
-
-  const queueLimit = configuredPositiveInteger(
-    "AGENT_INSPECTOR_READY_MAX_FINALIZER_QUEUE",
-    DEFAULT_MAX_FINALIZER_QUEUE,
-  );
-  const warningThreshold = Math.max(1, Math.floor(queueLimit * 0.8));
-  const queue =
-    health.pendingJobs + sessionQueue.queuedTasks >= queueLimit
-      ? "unavailable"
-      : health.pendingJobs + sessionQueue.queuedTasks >= warningThreshold
-        ? "degraded"
-        : "ok";
-  return { workers, queue };
-}
-
-function checkWorkers(): Promise<ReadinessCheckLevel> {
-  return Promise.resolve(finalizerChecks().workers);
-}
-
-function checkQueue(): Promise<ReadinessCheckLevel> {
-  return Promise.resolve(finalizerChecks().queue);
-}
-
-async function checkIndex(): Promise<ReadinessCheckLevel> {
-  const recovery = await recoverCapturedLogStorage(resolveLogDir());
-  if (recovery.level === "unavailable") return "unavailable";
-
-  try {
-    await loadIndex();
-  } catch {
-    return "unavailable";
-  }
-
-  const sqliteReady = await isSqliteLogIndexReady();
-  return recovery.level === "degraded" || !sqliteReady ? "degraded" : "ok";
-}
-
-function checkShutdown(): Promise<ReadinessCheckLevel> {
-  return Promise.resolve(runtimeShuttingDown ? "unavailable" : "ok");
-}
-
-function readinessLevel(checks: SafeReadiness["checks"]): ReadinessLevel {
-  const levels = Object.values(checks);
-  if (levels.some((level) => level === "unavailable")) return "unavailable";
-  return levels.some((level) => level === "degraded") ? "degraded" : "ready";
-}
-
-export function createReadinessService(
-  dependencies: ReadinessDependencies,
-  cacheMs = DEFAULT_CACHE_MS,
-): ReadinessService {
-  let cached: { expiresAt: number; value: SafeReadiness } | null = null;
-  let inFlight: Promise<SafeReadiness> | null = null;
-
-  async function evaluate(): Promise<SafeReadiness> {
-    // Establish the directory before checking its filesystem. On a clean install
-    // the data directory may not exist until the first write probe creates it.
-    const storage = await dependencies.checkStorage();
-    const [disk, workers, index, queue, shutdown] = await Promise.all([
-      dependencies.checkDisk(),
-      dependencies.checkWorkers(),
-      dependencies.checkIndex(),
-      dependencies.checkQueue(),
-      dependencies.checkShutdown(),
-    ]);
-    const checks = { storage, disk, workers, index, queue, shutdown };
-    return {
-      status: readinessLevel(checks),
-      checks,
-      checkedAt: new Date(dependencies.now()).toISOString(),
-    };
-  }
-
-  async function get(): Promise<SafeReadiness> {
-    const now = dependencies.now();
-    if (cached !== null && cached.expiresAt > now) return cached.value;
-    if (inFlight !== null) return await inFlight;
-
-    const evaluation = evaluate();
-    inFlight = evaluation;
-    try {
-      const value = await evaluation;
-      cached = { expiresAt: dependencies.now() + Math.max(0, cacheMs), value };
-      return value;
-    } finally {
-      if (inFlight === evaluation) inFlight = null;
-    }
-  }
-
-  function invalidate(): void {
-    cached = null;
-  }
-
-  return { get, invalidate };
-}
-
-const readinessService = createReadinessService({
-  checkStorage: checkStorageWritable,
-  checkDisk: checkDiskHeadroom,
-  checkWorkers,
-  checkIndex,
-  checkQueue,
-  checkShutdown,
-  now: Date.now,
-});
-
-export async function getRuntimeReadiness(): Promise<SafeReadiness> {
-  return await readinessService.get();
-}
-
-/** Shutdown coordination will call this before stopping admission. */
-export function setRuntimeShuttingDown(shuttingDown: boolean): void {
-  runtimeShuttingDown = shuttingDown;
-  readinessService.invalidate();
-}
-
-export function _resetRuntimeHealthForTests(): void {
-  runtimeShuttingDown = false;
-  readinessService.invalidate();
-}

package/src/proxy/runtimeMetrics.ts

@@ -1,149 +0,0 @@
-import { DEFAULT_RUNTIME_BUDGET_POLICY } from "../lib/resourceLimits";
-import { getFinalizerRuntimeHealth } from "./logFinalizer";
-import { getRuntimeAdmissionState } from "./runtimeAdmission";
-import { getRuntimeReadiness } from "./runtimeHealth";
-import { getSessionRuntimeQueueHealth } from "./sessionRuntime";
-import { listSqliteLogCursorEntries } from "./sqliteLogIndex";
-import { getStorageLifecycleInventory, getLastStorageLifecycleGcResult } from "./storageLifecycle";
-
-export const PERFORMANCE_METRICS_SCHEMA = "agent-inspector.performance-metrics/v1";
-export const OPERATIONAL_METRICS_SCHEMA = "agent-inspector.operational-metrics/v1";
-
-export type SseRuntimeMetrics = {
-  activeClients: number;
-  capacity: number;
-};
-
-export type SafeOperationalMetrics = {
-  schema: typeof PERFORMANCE_METRICS_SCHEMA;
-  activeUpstream: number;
-  queueDepth: number;
-  queueBytes: number;
-  incompleteCaptures: number;
-  finalizerLagMs: number;
-  persistenceLagMs: number;
-  activeLeases: number;
-  runtime: {
-    schema: typeof OPERATIONAL_METRICS_SCHEMA;
-    collectedAt: string;
-    readiness: Awaited<ReturnType<typeof getRuntimeReadiness>>;
-    requests: {
-      activeProxyRequests: number;
-      proxyCapacity: number;
-      acceptingRequests: boolean;
-    };
-    limits: typeof DEFAULT_RUNTIME_BUDGET_POLICY;
-    queues: {
-      finalizerPendingJobs: number;
-      sessionQueuedTasks: number;
-      sessionActiveQueues: number;
-      sessionQueueLimit: number;
-    };
-    workers: ReturnType<typeof getFinalizerRuntimeHealth>;
-    storage: {
-      totalFiles: number;
-      totalBytes: number;
-      reclaimableBytes: number;
-      lastGc: {
-        checkedAt: string;
-        filesDeleted: number;
-        bytesDeleted: number;
-        errors: number;
-      } | null;
-    };
-    sse: SseRuntimeMetrics;
-    truncation: {
-      captureIncompleteLogs: number;
-      droppedChunks: number;
-      droppedBytes: number;
-      source: "sqlite" | "unavailable";
-    };
-  };
-};
-
-async function getTruncationMetrics(): Promise<SafeOperationalMetrics["runtime"]["truncation"]> {
-  const entries = await listSqliteLogCursorEntries({});
-  if (entries === null) {
-    return {
-      captureIncompleteLogs: 0,
-      droppedChunks: 0,
-      droppedBytes: 0,
-      source: "unavailable",
-    };
-  }
-
-  return {
-    captureIncompleteLogs: entries.filter((entry) => entry.summary?.captureIncomplete === true)
-      .length,
-    droppedChunks: entries.reduce((sum, entry) => sum + (entry.summary?.droppedChunks ?? 0), 0),
-    droppedBytes: entries.reduce((sum, entry) => sum + (entry.summary?.droppedBytes ?? 0), 0),
-    source: "sqlite",
-  };
-}
-
-export async function getSafeOperationalMetrics(
-  sse: SseRuntimeMetrics,
-): Promise<SafeOperationalMetrics> {
-  const [readiness, truncation] = await Promise.all([
-    getRuntimeReadiness(),
-    getTruncationMetrics(),
-  ]);
-  const admission = getRuntimeAdmissionState();
-  const workers = getFinalizerRuntimeHealth();
-  const sessionQueue = getSessionRuntimeQueueHealth();
-  const storageInventory = getStorageLifecycleInventory();
-  const lastGc = getLastStorageLifecycleGcResult();
-  const reclaimableBytes = storageInventory.artifacts
-    .filter(
-      (artifact) =>
-        artifact.cleanupBehavior === "retention-gc" ||
-        artifact.cleanupBehavior === "orphan-reclaim",
-    )
-    .reduce((sum, artifact) => sum + artifact.bytes, 0);
-  const queueDepth = workers.pendingJobs + sessionQueue.queuedTasks;
-
-  return {
-    schema: PERFORMANCE_METRICS_SCHEMA,
-    activeUpstream: admission.activeRequests,
-    queueDepth,
-    queueBytes: 0,
-    incompleteCaptures: truncation.captureIncompleteLogs,
-    finalizerLagMs: 0,
-    persistenceLagMs: 0,
-    activeLeases: sse.activeClients,
-    runtime: {
-      schema: OPERATIONAL_METRICS_SCHEMA,
-      collectedAt: new Date().toISOString(),
-      readiness,
-      requests: {
-        activeProxyRequests: admission.activeRequests,
-        proxyCapacity: admission.capacity,
-        acceptingRequests: admission.acceptingRequests,
-      },
-      limits: DEFAULT_RUNTIME_BUDGET_POLICY,
-      queues: {
-        finalizerPendingJobs: workers.pendingJobs,
-        sessionQueuedTasks: sessionQueue.queuedTasks,
-        sessionActiveQueues: sessionQueue.activeQueues,
-        sessionQueueLimit: sessionQueue.queueLimit,
-      },
-      workers,
-      storage: {
-        totalFiles: storageInventory.totalFiles,
-        totalBytes: storageInventory.totalBytes,
-        reclaimableBytes,
-        lastGc:
-          lastGc === null
-            ? null
-            : {
-                checkedAt: lastGc.checkedAt,
-                filesDeleted: lastGc.filesDeleted,
-                bytesDeleted: lastGc.bytesDeleted,
-                errors: lastGc.errors.length,
-              },
-      },
-      sse,
-      truncation,
-    },
-  };
-}

package/src/proxy/runtimeShutdown.ts

@@ -1,157 +0,0 @@
-import { createHash, timingSafeEqual } from "node:crypto";
-import {
-  parseManagedInstanceShutdownMessage,
-  readManagedInstanceLaunch,
-} from "../lib/managedInstance";
-import { closeLogFinalizerWorkers, forceCloseLogFinalizerWorkers } from "./logFinalizer";
-import { flushIndex } from "./logIndex";
-import { flushApplicationLog, flushLogBuffer } from "./logger";
-import { flushAllRawStreamCaptures } from "./rawStreamCapture";
-import { setRuntimeShuttingDown } from "./runtimeHealth";
-import { stopRuntimeAdmission, waitForRuntimeTrafficDrain } from "./runtimeAdmission";
-import {
-  createShutdownCoordinator,
-  createShutdownSignalHandler,
-  type ShutdownResult,
-} from "./shutdownCoordinator";
-import { destroyAllSessionProcesses } from "./sessionProcess";
-import {
-  beginEcosystemTaskShutdown,
-  forceCloseEcosystemTasks,
-  waitForEcosystemTasksToClose,
-} from "./ecosystemTasks";
-import { flushSessionTasks } from "./sessionRuntime";
-import { closeSqliteLogIndex } from "./sqliteLogIndex";
-import { flushSessionArchiveWrites } from "./store";
-import { flushToolSchemaWarningAnalysis } from "./logFinalizer";
-
-const DEFAULT_SHUTDOWN_DEADLINE_MS = 15_000;
-
-function shutdownDeadlineMs(): number {
-  const raw = process.env["AGENT_INSPECTOR_SHUTDOWN_TIMEOUT_MS"];
-  if (raw === undefined || raw === "") return DEFAULT_SHUTDOWN_DEADLINE_MS;
-  const parsed = Number(raw);
-  return Number.isSafeInteger(parsed) && parsed > 0 && parsed <= 300_000
-    ? parsed
-    : DEFAULT_SHUTDOWN_DEADLINE_MS;
-}
-
-async function drainFinalizers(): Promise<void> {
-  await flushAllRawStreamCaptures();
-  await flushSessionTasks();
-  await flushToolSchemaWarningAnalysis();
-}
-
-async function flushPersistence(): Promise<void> {
-  await flushLogBuffer();
-  await flushSessionArchiveWrites();
-  await flushIndex();
-  await flushApplicationLog();
-}
-
-async function closeResources(): Promise<void> {
-  await waitForEcosystemTasksToClose();
-  await closeLogFinalizerWorkers();
-  destroyAllSessionProcesses();
-  closeSqliteLogIndex();
-  await flushApplicationLog();
-}
-
-function forceCloseResources(): void {
-  forceCloseEcosystemTasks();
-  forceCloseLogFinalizerWorkers();
-  destroyAllSessionProcesses();
-  closeSqliteLogIndex();
-}
-
-const runtimeShutdownCoordinator = createShutdownCoordinator(
-  {
-    markShuttingDown: () => {
-      setRuntimeShuttingDown(true);
-      beginEcosystemTaskShutdown();
-    },
-    stopAdmission: stopRuntimeAdmission,
-    drainTraffic: waitForRuntimeTrafficDrain,
-    drainFinalizers,
-    flushPersistence,
-    closeResources,
-    forceClose: forceCloseResources,
-  },
-  shutdownDeadlineMs(),
-);
-
-let signalHandlersInstalled = false;
-let managedInstanceHandlersInstalled = false;
-
-export type ManagedRuntimeShutdownController = {
-  handleDisconnect: () => void;
-  handleMessage: (message: unknown) => void;
-};
-
-type ManagedRuntimeShutdownControllerOptions = {
-  controlToken: string;
-  shutdown: () => Promise<ShutdownResult>;
-  exit: (code: number) => void;
-};
-
-function managedControlTokensMatch(received: string, expected: string): boolean {
-  if (received.length === 0 || expected.length === 0) return false;
-  const receivedDigest = createHash("sha256").update(received).digest();
-  const expectedDigest = createHash("sha256").update(expected).digest();
-  return timingSafeEqual(receivedDigest, expectedDigest);
-}
-
-export function createManagedRuntimeShutdownController(
-  options: ManagedRuntimeShutdownControllerOptions,
-): ManagedRuntimeShutdownController {
-  let shutdownRequested = false;
-  const requestShutdown = (): void => {
-    if (shutdownRequested) return;
-    shutdownRequested = true;
-    void options.shutdown().then(
-      (result) => options.exit(result.forced ? 1 : 0),
-      () => options.exit(1),
-    );
-  };
-  return {
-    handleDisconnect: requestShutdown,
-    handleMessage: (message) => {
-      const parsed = parseManagedInstanceShutdownMessage(message);
-      if (
-        parsed === null ||
-        !managedControlTokensMatch(parsed.controlToken, options.controlToken)
-      ) {
-        return;
-      }
-      requestShutdown();
-    },
-  };
-}
-
-/** Install lifecycle cleanup for the private supervisor/runtime IPC channel. */
-export function installRuntimeManagedInstanceShutdownHandlers(): void {
-  if (managedInstanceHandlersInstalled) return;
-  const launchResult = readManagedInstanceLaunch(process.env);
-  if (launchResult.kind !== "managed") return;
-  managedInstanceHandlersInstalled = true;
-  const controller = createManagedRuntimeShutdownController({
-    controlToken: launchResult.launch.controlToken,
-    shutdown: async () => await runtimeShutdownCoordinator.shutdown("manual"),
-    exit: (code) => process.exit(code),
-  });
-  process.on("message", controller.handleMessage);
-  process.once("disconnect", controller.handleDisconnect);
-  if (process.connected === false) process.nextTick(controller.handleDisconnect);
-}
-
-/** Install once in the server runtime; the CLI supervisor owns its own public-ingress signals. */
-export function installRuntimeShutdownSignalHandlers(): void {
-  installRuntimeManagedInstanceShutdownHandlers();
-  if (signalHandlersInstalled) return;
-  signalHandlersInstalled = true;
-  const handleSignal = createShutdownSignalHandler(runtimeShutdownCoordinator, (result) => {
-    process.exit(result.forced ? 1 : 0);
-  });
-  process.on("SIGINT", () => handleSignal("SIGINT"));
-  process.on("SIGTERM", () => handleSignal("SIGTERM"));
-}

package/src/proxy/schemas.ts

@@ -1,112 +0,0 @@
-import { z } from "zod";
-
-export {
-  CaptureIncompleteReasonSchema,
-  CapturedLogSchema,
-  JsonValueSchema,
-  StreamingChunkSchema,
-  StopReasonSchema,
-  ToolTraceEventSchema,
-} from "../contracts";
-export type {
-  CapturedLog,
-  ApiFormat,
-  CaptureIncompleteReason,
-  JsonValue,
-  StreamingChunk,
-  StopReason,
-  ToolTraceEvent,
-  TokenUsage,
-} from "../contracts";
-
-// ============================================================
-// Import types and schemas from shared contracts
-// ============================================================
-
-// Re-export types and schemas from shared contracts for backward compatibility
-export {
-  AnthropicRequestSchema,
-  AnthropicResponseSchema,
-  SseEventSchema,
-} from "../contracts/anthropic";
-export type { AnthropicResponse, ResponseContentBlockType } from "../contracts/anthropic";
-
-export {
-  OpenAIRequestSchema,
-  OpenAIResponsesRequestSchema,
-  OpenAIResponsesResponseSchema,
-  OpenAIResponsesSSEventSchema,
-  OpenAIResponsesUsageSchema,
-  OpenAIResponseSchema,
-  OpenAISSERawChunkSchema,
-  type OpenAIResponse,
-  type OpenAIResponsesResponse,
-  type OpenAIToolCall,
-  parseOpenAIResponsesResponse,
-  parseOpenAIResponse,
-} from "../contracts/openai";
-
-// Schema for model name in request body (used for routing)
-const RequestModelSchema = z.object({
-  model: z.string(),
-});
-
-export function extractModelFromBody(body: string): string | null {
-  try {
-    const json: unknown = JSON.parse(body);
-    const parsed = RequestModelSchema.safeParse(json);
-    if (parsed.success) {
-      return parsed.data.model;
-    }
-    return null;
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Loose request schema used for routing + log metadata. Tolerant of missing
- * fields so it can extract what it can from a partially-valid or unparseable
- * body without throwing.
- */
-export const LooseRequestSchema = z.object({
-  model: z.string().optional(),
-  metadata: z.object({ user_id: z.string().optional() }).passthrough().optional(),
-});
-
-export type RequestMetadata = {
-  model: string | null;
-  sessionId: string | null;
-};
-
-function extractSessionIdFromHeaders(headers: Headers): string | null {
-  return headers.get("x-agent-inspector-session-id") ?? headers.get("x-session-affinity");
-}
-
-/**
- * Parse a request body exactly once and pull out everything the handler needs
- * for routing + log creation: the model (used for provider selection) and the
- * session id (from `metadata.user_id` or the `x-session-affinity` header).
- *
- * Returning `null` for either field is a valid outcome; callers must handle
- * "unknown" gracefully. Callers that already have the body string should use
- * this instead of re-parsing with `extractModelFromBody` and a second pass
- * for the session id.
- */
-export function extractRequestMetadata(body: string | null, headers: Headers): RequestMetadata {
-  const headerSessionId = extractSessionIdFromHeaders(headers);
-  if (body === null) return { model: null, sessionId: headerSessionId };
-  try {
-    const json: unknown = JSON.parse(body);
-    const loose = LooseRequestSchema.safeParse(json);
-    if (loose.success) {
-      return {
-        model: loose.data.model ?? null,
-        sessionId: loose.data.metadata?.user_id ?? headerSessionId,
-      };
-    }
-  } catch {
-    // body not valid JSON
-  }
-  return { model: null, sessionId: headerSessionId };
-}