4runr-os 2.10.85 → 2.10.86

package/apps/os-server/src/devkit/routes.ts

@@ -0,0 +1,349 @@
+/**
+ * DevKit API Routes
+ * 
+ * Dev-only API namespace for tooling and UI support.
+ * Only available when DEVKIT_ENABLED=true and NODE_ENV !== "production"
+ */
+
+import { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
+import { env } from '@4runr/shared';
+import { requireDevKit, getDevKitStatus } from './middleware.js';
+import { performHealthChecks } from '../health/index.js';
+import { getPersistenceMode } from '../config/persistence.js';
+import { getMetrics } from '../metrics/index.js';
+import { getMetricsSummary } from './metrics-parser.js';
+import { registerAgentRoutes } from './agents-api.js';
+import { registerToolRoutes } from './tools-api.js';
+import { getPrismaClient } from '../db/prisma.js';
+
+/**
+ * Get gateway version from package.json
+ */
+function getGatewayVersion(): string {
+  try {
+    // In production, version might be baked in
+    return process.env['GATEWAY_VERSION'] || '1.0.0';
+  } catch {
+    return 'unknown';
+  }
+}
+
+/**
+ * Measure latency to a service
+ */
+async function measureLatency(check: () => Promise<any>): Promise<number> {
+  const start = Date.now();
+  try {
+    await check();
+    return Date.now() - start;
+  } catch {
+    return -1; // Return -1 for failed checks
+  }
+}
+
+/**
+ * Register DevKit API routes
+ */
+export function registerDevKitRoutes(fastify: FastifyInstance) {
+  // Scope DevKit middleware to only /devkit routes
+  fastify.addHook('onRequest', async (request, reply) => {
+    // Only apply DevKit check to /devkit/* routes
+    if (request.url.startsWith('/devkit')) {
+      await requireDevKit(request, reply);
+    }
+  });
+
+  // Register agent and tool management routes
+  registerAgentRoutes(fastify);
+  registerToolRoutes(fastify);
+
+  /**
+   * GET /devkit/api/status
+   * Returns system status information
+   */
+  fastify.get('/devkit/api/status', async (request: FastifyRequest, reply: FastifyReply) => {
+    // Perform health checks
+    const health = await performHealthChecks();
+    
+    // Get latency measurements (if available)
+    const dbCheck = health.checks.database as any;
+    const redisCheck = health.checks.redis as any;
+    const dbLatency = dbCheck?.latency || -1;
+    const redisLatency = redisCheck?.latency || -1;
+
+    // Get Sentinel status
+    const sentinelEnabled = env.SENTINEL_ENABLED;
+    const sentinelMode = env.SENTINEL_MODE;
+    const shieldMode = env.SENTINEL_SHIELD_MODE;
+
+    return {
+      gateway: {
+        version: getGatewayVersion(),
+        status: health.status,
+        uptime: process.uptime(),
+        timestamp: new Date().toISOString()
+      },
+      database: {
+        status: health.checks.database?.status || 'unknown',
+        latencyMs: dbLatency,
+        mode: getPersistenceMode()
+      },
+      redis: {
+        status: health.checks.redis?.status || 'unknown',
+        latencyMs: redisLatency
+      },
+      queue: {
+        status: health.checks.queue?.status || 'unknown'
+      },
+      sentinel: {
+        enabled: sentinelEnabled,
+        mode: sentinelMode,
+        shieldMode: shieldMode
+      },
+      auth: {
+        enabled: env.AUTH_ENABLED,
+        mode: env.AUTH_MODE
+      },
+      rateLimit: {
+        enabled: env.RATE_LIMIT_ENABLED,
+        max: env.RATE_LIMIT_MAX,
+        windowMs: env.RATE_LIMIT_WINDOW_MS
+      },
+      devkit: getDevKitStatus()
+    };
+  });
+
+  /**
+   * GET /devkit/api/config
+   * Returns non-sensitive configuration snapshot
+   */
+  fastify.get('/devkit/api/config', async (request: FastifyRequest, reply: FastifyReply) => {
+    // Return only non-sensitive configuration
+    return {
+      environment: {
+        NODE_ENV: env.NODE_ENV,
+        DEV_MODE: env.DEV_MODE,
+        DEVKIT_ENABLED: env.DEVKIT_ENABLED
+      },
+      server: {
+        PORT: env.PORT,
+        HOST: env.HOST
+      },
+      persistence: {
+        mode: getPersistenceMode(),
+        // Don't expose DATABASE_URL or connection strings
+        databaseConfigured: !!env.DATABASE_URL,
+        redisConfigured: !!env.REDIS_URL
+      },
+      auth: {
+        enabled: env.AUTH_ENABLED,
+        mode: env.AUTH_MODE
+      },
+      rateLimit: {
+        enabled: env.RATE_LIMIT_ENABLED,
+        max: env.RATE_LIMIT_MAX,
+        windowMs: env.RATE_LIMIT_WINDOW_MS
+      },
+      cors: {
+        enabled: env.CORS_ENABLED,
+        origin: env.CORS_ORIGIN
+      },
+      sentinel: {
+        enabled: env.SENTINEL_ENABLED,
+        mode: env.SENTINEL_MODE,
+        shieldMode: env.SENTINEL_SHIELD_MODE,
+        tokenCap: env.SHIELD_TOKEN_CAP || 500,
+        maxTokens: env.RUN_MAX_TOKENS || 50
+      },
+      idempotency: {
+        enabled: env.IDEMPOTENCY_ENABLED,
+        ttlSeconds: env.IDEMPOTENCY_TTL_SECONDS
+      },
+      cache: {
+        enabled: env.CACHE_ENABLED,
+        ttlSeconds: env.CACHE_TTL_SECONDS
+      },
+      metrics: {
+        enabled: env.METRICS_ENABLED,
+        port: env.METRICS_PORT,
+        path: env.METRICS_PATH
+      },
+      logging: {
+        level: env.LOG_LEVEL,
+        format: env.LOG_FORMAT,
+        requests: env.LOG_REQUESTS,
+        responses: env.LOG_RESPONSES
+      }
+    };
+  });
+
+  /**
+   * GET /devkit/api/metrics/summary
+   * Returns parsed metrics summary (easier to consume than raw Prometheus format)
+   */
+  fastify.get('/devkit/api/metrics/summary', async (request: FastifyRequest, reply: FastifyReply) => {
+    try {
+      // Get raw metrics
+      const metricsText = await getMetrics();
+      
+      // Parse into summary
+      const summary = getMetricsSummary(metricsText);
+      
+      return {
+        timestamp: new Date().toISOString(),
+        metrics: summary
+      };
+    } catch (error: any) {
+      return reply.status(500).send({
+        error: 'metrics_unavailable',
+        message: 'Failed to retrieve metrics',
+        details: error.message
+      });
+    }
+  });
+
+  /**
+   * GET /devkit/api/health
+   * Returns detailed health check results
+   */
+  fastify.get('/devkit/api/health', async (request: FastifyRequest, reply: FastifyReply) => {
+    const health = await performHealthChecks();
+    
+    return {
+      status: health.status,
+      timestamp: health.timestamp,
+      checks: health.checks
+    };
+  });
+
+  /**
+   * GET /devkit/api/runs
+   * Returns recent runs with optional filtering
+   */
+  fastify.get('/devkit/api/runs', async (request: FastifyRequest, reply: FastifyReply) => {
+    try {
+      const query = request.query as { limit?: string; agentId?: string; status?: string };
+      const limit = query.limit ? parseInt(query.limit) : 50;
+      
+      // Get runs from database
+      const where: any = {};
+      
+      // Filter by agent if specified (check tags for agent:slug)
+      if (query.agentId) {
+        where.tags = {
+          has: `agent:${query.agentId}`
+        };
+      }
+      
+      // Filter by status if specified
+      if (query.status) {
+        where.status = query.status.toUpperCase();
+      }
+      
+      const prisma = await getPrismaClient();
+      const runs = await prisma.run.findMany({
+        where,
+        take: limit,
+        orderBy: {
+          createdAt: 'desc'
+        },
+        select: {
+          id: true,
+          name: true,
+          status: true,
+          tags: true,
+          createdAt: true,
+          startedAt: true,
+          completedAt: true,
+          input: true,
+          output: true
+        }
+      });
+
+      // Extract agent info from tags or input
+      const runsWithAgent = runs.map((run: any) => {
+        const agentTag = run.tags?.find((tag: string) => tag.startsWith('agent:'));
+        const agentSlug = agentTag ? agentTag.replace('agent:', '') : null;
+        const agentInfo = run.input?._agent || null;
+        
+        return {
+          id: run.id,
+          name: run.name,
+          status: run.status,
+          createdAt: run.createdAt,
+          startedAt: run.startedAt,
+          completedAt: run.completedAt,
+          tags: run.tags,
+          agent: agentInfo ? {
+            id: agentInfo.id,
+            name: agentInfo.name,
+            slug: agentInfo.slug
+          } : (agentSlug ? { slug: agentSlug } : null),
+          inputSummary: run.input ? JSON.stringify(run.input).substring(0, 100) : null,
+          outputSummary: run.output ? JSON.stringify(run.output).substring(0, 100) : null,
+          latencyMs: run.startedAt && run.completedAt 
+            ? new Date(run.completedAt).getTime() - new Date(run.startedAt).getTime()
+            : null
+        };
+      });
+
+      return {
+        runs: runsWithAgent,
+        total: runs.length,
+        limit
+      };
+    } catch (error: any) {
+      return reply.status(500).send({
+        error: 'failed_to_fetch_runs',
+        message: error.message
+      });
+    }
+  });
+
+  /**
+   * GET /devkit/api/runs/:id
+   * Get detailed run information
+   */
+  fastify.get('/devkit/api/runs/:id', async (request: FastifyRequest, reply: FastifyReply) => {
+    try {
+      const { id } = request.params as { id: string };
+      
+      const prisma = await getPrismaClient();
+      const run = await prisma.run.findUnique({
+        where: { id }
+      });
+
+      if (!run) {
+        return reply.status(404).send({
+          error: 'run_not_found',
+          message: `Run with id '${id}' not found`
+        });
+      }
+
+      // Extract agent info
+      const agentTag = run.tags?.find((tag: string) => tag.startsWith('agent:'));
+      const agentSlug = agentTag ? agentTag.replace('agent:', '') : null;
+      const agentInfo = run.input?._agent || null;
+
+      return {
+        run: {
+          ...run,
+          agent: agentInfo ? {
+            id: agentInfo.id,
+            name: agentInfo.name,
+            slug: agentInfo.slug
+          } : (agentSlug ? { slug: agentSlug } : null),
+          latencyMs: run.startedAt && run.completedAt 
+            ? new Date(run.completedAt).getTime() - new Date(run.startedAt).getTime()
+            : null
+        }
+      };
+    } catch (error: any) {
+      return reply.status(500).send({
+        error: 'failed_to_fetch_run',
+        message: error.message
+      });
+    }
+  });
+}
+

package/apps/os-server/src/health/index.ts

@@ -0,0 +1,375 @@
+/**
+ * Health check utilities
+ * Provides comprehensive health checking for dependencies and system status
+ */
+
+import { getRedisClient, isRedisAvailable } from '../db/redis.js';
+import { getQueue } from '../queue/index.js';
+import { getPrismaClient } from '../db/prisma.js';
+import { Shield } from '@4runr/shield';
+import { getShieldMisconfigurationWarnings } from '../security/shield-config.js';
+import { env, createLogger } from '@4runr/shared';
+
+const logger = createLogger('Gateway:Health');
+
+function databaseUrlConfigured(): boolean {
+  return Boolean(env.DATABASE_URL || process.env['DATABASE_URL']);
+}
+
+function redisUrlConfigured(): boolean {
+  return Boolean(env.REDIS_URL || process.env['REDIS_URL']);
+}
+
+/** Best-effort stringify for Promise rejection reasons (Error, string, primitives, etc.). */
+function rejectionMessage(reason: unknown, fallback: string): string {
+  if (reason instanceof Error) return reason.message;
+  if (typeof reason === 'string') return reason;
+  if (reason === undefined || reason === null) return fallback;
+  return String(reason);
+}
+
+export interface HealthCheckResult {
+  status: 'healthy' | 'unhealthy' | 'degraded';
+  checks: {
+    database?: DependencyCheck;
+    redis?: DependencyCheck;
+    queue?: DependencyCheck;
+    shield?: ShieldCheck;
+  };
+  timestamp: string;
+}
+
+export interface DependencyCheck {
+  status: 'up' | 'down' | 'degraded';
+  responseTime?: number;
+  error?: string;
+  message?: string;
+}
+
+export interface ShieldCheck {
+  enabled: boolean;
+  mode: string;
+  detectors: {
+    pii: boolean;
+    injection: boolean;
+    hallucination: boolean;
+  };
+  status: 'operational' | 'misconfigured';
+  warnings?: string[];
+}
+
+/**
+ * Check database connectivity
+ */
+export async function checkDatabase(): Promise<DependencyCheck> {
+  const startTime = Date.now();
+
+  if (!databaseUrlConfigured()) {
+    return {
+      status: 'up',
+      responseTime: Date.now() - startTime,
+      message: 'DATABASE_URL not set — no Postgres configured for this process (skipping DB probe)',
+    };
+  }
+
+  try {
+    const prisma = await getPrismaClient();
+    await prisma.$queryRaw`SELECT 1`;
+    const responseTime = Date.now() - startTime;
+
+    return {
+      status: 'up',
+      responseTime,
+      message: 'Postgres reachable (SELECT 1)',
+    };
+  } catch (error) {
+    const responseTime = Date.now() - startTime;
+    logger.warn('Database health check failed', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    return {
+      status: 'down',
+      responseTime,
+      error: error instanceof Error ? error.message : String(error),
+      message: 'Postgres not reachable or Prisma client failed',
+    };
+  }
+}
+
+/**
+ * Check Redis connectivity
+ */
+export async function checkRedis(): Promise<DependencyCheck> {
+  const startTime = Date.now();
+  
+  try {
+    if (!redisUrlConfigured()) {
+      return {
+        status: 'up',
+        responseTime: Date.now() - startTime,
+        message: 'REDIS_URL not set — Redis not required for readiness',
+      };
+    }
+
+    const available = await isRedisAvailable();
+    if (!available) {
+      return {
+        status: 'down',
+        responseTime: Date.now() - startTime,
+        message: 'REDIS_URL set but Redis is unreachable (start redis-server or Docker, then retry)',
+      };
+    }
+
+    const redis = await getRedisClient();
+    if (!redis) {
+      return {
+        status: 'down',
+        responseTime: Date.now() - startTime,
+        message: 'Redis client not available',
+      };
+    }
+
+    // Simple ping test
+    const result = await redis.ping();
+    const responseTime = Date.now() - startTime;
+
+    if (result === 'PONG') {
+      return {
+        status: 'up',
+        responseTime,
+        message: 'Redis connection healthy',
+      };
+    }
+
+    return {
+      status: 'degraded',
+      responseTime,
+      message: 'Redis ping returned unexpected result',
+    };
+  } catch (error) {
+    const responseTime = Date.now() - startTime;
+    logger.warn('Redis health check failed', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    return {
+      status: 'down',
+      responseTime,
+      error: error instanceof Error ? error.message : String(error),
+      message: 'Redis connection failed',
+    };
+  }
+}
+
+/**
+ * Check queue system status
+ */
+export async function checkQueue(): Promise<DependencyCheck> {
+  const startTime = Date.now();
+
+  try {
+    const queue = getQueue();
+    if (!queue) {
+      if (!redisUrlConfigured()) {
+        return {
+          status: 'up',
+          responseTime: Date.now() - startTime,
+          message: 'No run queue (Redis not configured)',
+        };
+      }
+      return {
+        status: 'down',
+        responseTime: Date.now() - startTime,
+        message: 'Run queue not initialized — Redis configured but BullMQ queue unavailable',
+      };
+    }
+
+    const queueName = (queue as { name?: string }).name ?? 'unknown';
+    const responseTime = Date.now() - startTime;
+
+    return {
+      status: 'up',
+      responseTime,
+      message: `Run queue ready (${queueName})`,
+    };
+  } catch (error) {
+    const responseTime = Date.now() - startTime;
+    logger.warn('Queue health check failed', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    return {
+      status: 'down',
+      responseTime,
+      error: error instanceof Error ? error.message : String(error),
+      message: 'Queue system check failed',
+    };
+  }
+}
+
+/**
+ * Check Shield AI safety layer status (S5)
+ */
+export async function checkShield(): Promise<ShieldCheck> {
+  try {
+    const shield = Shield.getInstance();
+    const health = shield.getHealth();
+    const config = shield.getConfig();
+    
+    // S5 fix: use shared helper and same API key source as startup
+    const openAiApiKey = process.env['OPENAI_API_KEY'];
+    const warnings = getShieldMisconfigurationWarnings(config, openAiApiKey);
+
+    const status: ShieldCheck['status'] = warnings.length > 0 ? 'misconfigured' : 'operational';
+
+    return {
+      enabled: shield.isEnabled(), // Use isEnabled() for effective state
+      mode: health.mode,
+      detectors: health.detectors,
+      status,
+      ...(warnings.length > 0 && { warnings }),
+    };
+  } catch (error) {
+    logger.warn('Shield health check failed', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+
+    return {
+      enabled: false,
+      mode: 'unknown',
+      detectors: {
+        pii: false,
+        injection: false,
+        hallucination: false,
+      },
+      status: 'misconfigured',
+      warnings: ['Shield health check threw error'],
+    };
+  }
+}
+
+/**
+ * Perform all health checks (S5: includes Shield)
+ */
+export async function performHealthChecks(): Promise<HealthCheckResult> {
+  const checks: HealthCheckResult['checks'] = {};
+
+  // Run checks in parallel
+  const [databaseCheck, redisCheck, queueCheck, shieldCheck] = await Promise.allSettled([
+    checkDatabase(),
+    checkRedis(),
+    checkQueue(),
+    checkShield(),
+  ]);
+
+  if (databaseCheck.status === 'fulfilled') {
+    checks.database = databaseCheck.value;
+  } else {
+    const msg = rejectionMessage(databaseCheck.reason, 'Database check failed');
+    checks.database = {
+      status: 'down',
+      message: msg,
+      error: msg,
+    };
+  }
+
+  if (redisCheck.status === 'fulfilled') {
+    checks.redis = redisCheck.value;
+  } else {
+    const msg = rejectionMessage(redisCheck.reason, 'Redis check failed');
+    checks.redis = {
+      status: 'down',
+      message: msg,
+      error: msg,
+    };
+  }
+
+  if (queueCheck.status === 'fulfilled') {
+    checks.queue = queueCheck.value;
+  } else {
+    const msg = rejectionMessage(queueCheck.reason, 'Queue check failed');
+    checks.queue = {
+      status: 'down',
+      message: msg,
+      error: msg,
+    };
+  }
+
+  if (shieldCheck.status === 'fulfilled') {
+    checks.shield = shieldCheck.value;
+  } else {
+    const msg = rejectionMessage(shieldCheck.reason, 'Shield check failed');
+    checks.shield = {
+      enabled: false,
+      mode: 'unknown',
+      detectors: { pii: false, injection: false, hallucination: false },
+      status: 'misconfigured',
+      warnings: [msg],
+    };
+  }
+
+  const db = checks.database;
+  const redis = checks.redis;
+  const queue = checks.queue;
+
+  const dbRequired = databaseUrlConfigured();
+  const redisRequired = redisUrlConfigured();
+
+  const databaseDown = dbRequired && db?.status === 'down';
+  const redisDown = redisRequired && redis?.status === 'down';
+  const queueDown = redisRequired && queue?.status === 'down';
+
+  const hasDegraded = Object.values(checks).some(
+    (check) => check?.status === 'degraded'
+  );
+
+  // Queue failure alone must not mark "unhealthy": Gateway may still serve traffic and use sync
+  // execution when BullMQ did not initialize; DB + Redis are the hard readiness gates.
+  // Shield misconfiguration is reported but does not gate health status (S5).
+  let status: HealthCheckResult['status'] = 'healthy';
+  if (databaseDown || redisDown) {
+    status = 'unhealthy';
+  } else if (queueDown || hasDegraded) {
+    status = 'degraded';
+  }
+
+  return {
+    status,
+    checks,
+    timestamp: new Date().toISOString(),
+  };
+}
+
+/**
+ * Ready for traffic when configured Postgres and Redis pass probes.
+ * Queue (BullMQ) is reported in checks but does not gate readiness — startup races or a failed
+ * worker init would otherwise keep /ready at 503 while the process is otherwise usable.
+ */
+export function computeReady(health: HealthCheckResult): boolean {
+  const dbRequired = databaseUrlConfigured();
+  const redisRequired = redisUrlConfigured();
+
+  if (dbRequired && health.checks.database?.status !== 'up') {
+    return false;
+  }
+  if (redisRequired && health.checks.redis?.status !== 'up') {
+    return false;
+  }
+  return true;
+}
+
+export async function isReady(): Promise<boolean> {
+  const health = await performHealthChecks();
+  return computeReady(health);
+}
+
+/**
+ * Check if service is alive (simple liveness check)
+ */
+export function isAlive(): boolean {
+  // Liveness is simple - just check if process is running
+  // More complex checks could verify memory, CPU, etc.
+  return true;
+}
+