4runr-os 2.10.85 → 2.10.86

package/apps/gateway/src/__tests__/integration/sentinel.test.ts

@@ -1,364 +0,0 @@
-/**
- * Integration tests for Sentinel integration
- * Tests policy enforcement, kill switch, event streaming, and audit trails
- */
-
-import { describe, it, expect, beforeAll } from '@jest/globals';
-import {
-  initTestServer,
-  authenticatedRequest,
-  waitForServer,
-  pollRunUntilStatus,
-} from './helpers/test-server';
-
-const SENTINEL_SMOKE_TIMEOUT = 120000;
-
-describe('Sentinel Integration', () => {
-  let sentinelHealthy = false;
-
-  beforeAll(async () => {
-    initTestServer();
-    const isReady = await waitForServer(30000);
-    if (!isReady) {
-      throw new Error('Test server is not ready');
-    }
-
-    const healthResponse = await authenticatedRequest('/sentinel/health');
-    if (healthResponse.ok) {
-      const health = await healthResponse.json();
-      sentinelHealthy = health.healthy === true;
-      console.log(`Sentinel healthy: ${sentinelHealthy}`);
-    } else {
-      console.warn('⚠️  Sentinel health endpoint unavailable; policy smoke tests may skip.');
-    }
-  });
-
-  describe('Policy Enforcement', () => {
-    it('should enforce max duration policy', async () => {
-      // Create a run that might violate duration policy
-      const response = await authenticatedRequest('/api/runs', {
-        method: 'POST',
-        body: JSON.stringify({
-          name: 'Duration Policy Test',
-          input: { test: 'duration' },
-        }),
-      });
-
-      expect([201, 429]).toContain(response.status);
-      if (response.status === 429) {
-        return; // Skip if rate limited
-      }
-      const data = await response.json();
-      const runId = data.run.id;
-
-      // Start the run
-      const startResponse = await authenticatedRequest(`/api/runs/${runId}/start`, {
-        method: 'POST',
-        body: JSON.stringify({ priority: 'high' }),
-      });
-
-      expect(startResponse.status).toBe(200);
-
-      // Note: In a real test, you'd wait for the run to exceed max duration
-      // and verify Sentinel kills it. For now, we just verify the run started.
-    });
-
-    it('should enforce max tokens policy', async () => {
-      // This would require a run that generates tokens
-      // For now, we just verify Sentinel is integrated
-      const response = await authenticatedRequest('/api/runs', {
-        method: 'POST',
-        body: JSON.stringify({
-          name: 'Token Policy Test',
-          input: { test: 'tokens' },
-        }),
-      });
-
-      expect([201, 429]).toContain(response.status);
-      // Token enforcement would be tested with actual run execution
-    });
-  });
-
-  describe('Kill Switch', () => {
-    it('should cancel a run with stable Sentinel deny output (N2)', async () => {
-      const createResponse = await authenticatedRequest('/api/runs', {
-        method: 'POST',
-        body: JSON.stringify({
-          name: 'Kill Switch Test',
-          input: {
-            agent_id: 'test',
-            sleep_ms: 10000,
-          },
-        }),
-      });
-
-      expect([201, 429]).toContain(createResponse.status);
-      if (createResponse.status === 429) {
-        return;
-      }
-      const createData = await createResponse.json();
-      const runId = createData.run.id;
-
-      const startResponse = await authenticatedRequest(`/api/runs/${runId}/start`, {
-        method: 'POST',
-        body: JSON.stringify({ priority: 'high' }),
-      });
-      expect(startResponse.status).toBe(200);
-
-      const runningRun = await pollRunUntilStatus(runId, 'running', {
-        maxWaitMs: 15000,
-        intervalMs: 200,
-      });
-      expect(runningRun.status).toBe('running');
-
-      const cancelResponse = await authenticatedRequest(`/api/runs/${runId}/cancel`, {
-        method: 'POST',
-        body: JSON.stringify({ reason: 'manual_cancellation' }),
-      });
-      expect(cancelResponse.status).toBe(200);
-
-      const getResponse = await authenticatedRequest(`/api/runs/${runId}`);
-      expect(getResponse.status).toBe(200);
-      const run = await getResponse.json();
-
-      expect(run.status).toBe('killed');
-      expect(run.output).toMatchObject({
-        error: 'Run terminated by Sentinel',
-        source: 'sentinel',
-        action: 'deny',
-        reason: 'manual_cancellation',
-      });
-      expect(run.logs?.some((log: { message?: string }) =>
-        log.message?.includes('Sentinel denied run: manual_cancellation')
-      )).toBe(true);
-
-      const cancelAgain = await authenticatedRequest(`/api/runs/${runId}/cancel`, {
-        method: 'POST',
-        body: JSON.stringify({ reason: 'manual_cancellation' }),
-      });
-      expect(cancelAgain.status).toBe(200);
-      const againBody = await cancelAgain.json();
-      expect(againBody.idempotent).toBe(true);
-    });
-  });
-
-  describe('Event Streaming', () => {
-    it('should stream Sentinel events via SSE', async () => {
-      // Create a run
-      const createResponse = await authenticatedRequest('/api/runs', {
-        method: 'POST',
-        body: JSON.stringify({
-          name: 'SSE Event Test',
-          input: { test: 'sse' },
-        }),
-      });
-
-      expect([201, 429]).toContain(createResponse.status);
-      if (createResponse.status === 429) {
-        return; // Skip if rate limited
-      }
-      const createData = await createResponse.json();
-      const runId = createData.run.id;
-
-      // Connect to SSE endpoint
-      // Note: SSE testing requires special handling
-      // For now, we verify the run was created
-      expect(createData.run.id).toBe(runId);
-    });
-  });
-
-  describe('N5 Staging Smoke — strict policy idle kill', () => {
-    it(
-      'apply strict template, idle run is killed with stable Sentinel output',
-      async () => {
-        if (!sentinelHealthy) {
-          console.log('Skipping: Sentinel is not healthy (set SENTINEL_ENABLED=true and restart Gateway)');
-          return;
-        }
-
-        const applyResponse = await authenticatedRequest('/api/sentinel/policies/apply', {
-          method: 'POST',
-          body: JSON.stringify({ template: 'strict' }),
-        });
-        expect(applyResponse.status).toBe(200);
-        const applyData = await applyResponse.json();
-        expect(applyData.config.runIdleMs).toBe(15000);
-
-        const currentResponse = await authenticatedRequest('/api/sentinel/policies/current');
-        expect(currentResponse.status).toBe(200);
-        const currentData = await currentResponse.json();
-        expect(currentData.config.runIdleMs).toBe(15000);
-
-        const createResponse = await authenticatedRequest('/api/runs', {
-          method: 'POST',
-          body: JSON.stringify({
-            name: 'N5 Strict Idle Smoke',
-            input: {
-              agent_id: 'test',
-              sleep_ms: 18000,
-            },
-          }),
-        });
-
-        expect([201, 429]).toContain(createResponse.status);
-        if (createResponse.status === 429) {
-          console.log('Rate limited, skipping N5 smoke');
-          return;
-        }
-
-        const createData = await createResponse.json();
-        const runId = createData.run.id;
-
-        const startResponse = await authenticatedRequest(`/api/runs/${runId}/start`, {
-          method: 'POST',
-          body: JSON.stringify({ priority: 'high' }),
-        });
-        expect(startResponse.status).toBe(200);
-
-        const finalRun = await pollRunUntilStatus(runId, 'killed', {
-          maxWaitMs: 90000,
-          intervalMs: 1000,
-        });
-
-        expect(finalRun.status).toBe('killed');
-        expect(finalRun.output).toMatchObject({
-          error: 'Run terminated by Sentinel',
-          source: 'sentinel',
-          action: 'deny',
-          reason: 'policy_violation:idle',
-          policy: 'idle',
-        });
-
-        const logs = (finalRun.logs as Array<{ message?: string }>) || [];
-        expect(
-          logs.some((log) => log.message?.includes('Sentinel denied run: policy_violation:idle'))
-        ).toBe(true);
-
-        await authenticatedRequest('/api/sentinel/policies/apply', {
-          method: 'POST',
-          body: JSON.stringify({ template: 'development' }),
-        });
-      },
-      SENTINEL_SMOKE_TIMEOUT
-    );
-
-    it(
-      'custom runIdleMs from POST /policies/custom is enforced (same path as OS UI apply)',
-      async () => {
-        if (!sentinelHealthy) {
-          console.log('Skipping: Sentinel is not healthy');
-          return;
-        }
-
-        const currentResponse = await authenticatedRequest('/api/sentinel/policies/current');
-        expect(currentResponse.status).toBe(200);
-        const currentData = await currentResponse.json();
-        const previousConfig = currentData.config as Record<string, unknown>;
-
-        const customIdleMs = 10_000;
-        const sleepMs = 13_000;
-
-        const customResponse = await authenticatedRequest('/api/sentinel/policies/custom', {
-          method: 'POST',
-          body: JSON.stringify({
-            ...previousConfig,
-            enabled: true,
-            runIdleMs: customIdleMs,
-          }),
-        });
-        expect(customResponse.status).toBe(200);
-        const customData = await customResponse.json();
-        expect(customData.config.runIdleMs).toBe(customIdleMs);
-
-        const createResponse = await authenticatedRequest('/api/runs', {
-          method: 'POST',
-          body: JSON.stringify({
-            name: 'Custom Idle Enforcement Smoke',
-            input: {
-              agent_id: 'test',
-              sleep_ms: sleepMs,
-            },
-          }),
-        });
-
-        expect([201, 429]).toContain(createResponse.status);
-        if (createResponse.status === 429) {
-          console.log('Rate limited, skipping custom idle enforcement smoke');
-          return;
-        }
-
-        const createData = await createResponse.json();
-        const runId = createData.run.id;
-
-        const startResponse = await authenticatedRequest(`/api/runs/${runId}/start`, {
-          method: 'POST',
-          body: JSON.stringify({ priority: 'high' }),
-        });
-        expect(startResponse.status).toBe(200);
-
-        const startedAt = Date.now();
-        const finalRun = await pollRunUntilStatus(runId, 'killed', {
-          maxWaitMs: 25_000,
-          intervalMs: 500,
-        });
-        const elapsedMs = Date.now() - startedAt;
-
-        expect(finalRun.status).toBe('killed');
-        expect(finalRun.output).toMatchObject({
-          source: 'sentinel',
-          reason: 'policy_violation:idle',
-          policy: 'idle',
-        });
-        // Killed near custom 10s threshold — not strict (15s) or typical UI 60s idle.
-        expect(elapsedMs).toBeGreaterThan(customIdleMs - 3_000);
-        expect(elapsedMs).toBeLessThan(14_000);
-
-        await authenticatedRequest('/api/sentinel/policies/custom', {
-          method: 'POST',
-          body: JSON.stringify(previousConfig),
-        });
-      },
-      SENTINEL_SMOKE_TIMEOUT
-    );
-
-    it('GET /sentinel/health and /api/sentinel/policies/current respond for staging checks', async () => {
-      const healthResponse = await authenticatedRequest('/sentinel/health');
-      expect(healthResponse.status).toBe(200);
-      const health = await healthResponse.json();
-      expect(health).toMatchObject({
-        enabled: expect.any(Boolean),
-        healthy: expect.any(Boolean),
-        status: expect.any(String),
-      });
-
-      const currentResponse = await authenticatedRequest('/api/sentinel/policies/current');
-      expect(currentResponse.status).toBe(200);
-      const current = await currentResponse.json();
-      expect(current.success).toBe(true);
-      expect(current.config).toMatchObject({
-        enabled: expect.any(Boolean),
-        runMaxDurationMs: expect.any(Number),
-        runIdleMs: expect.any(Number),
-      });
-    });
-  });
-
-  describe('Audit Trails', () => {
-    it('should log Sentinel actions to audit trail', async () => {
-      // Create a run that triggers Sentinel
-      const response = await authenticatedRequest('/api/runs', {
-        method: 'POST',
-        body: JSON.stringify({
-          name: 'Audit Trail Test',
-          input: { test: 'audit' },
-        }),
-      });
-
-      expect([201, 429]).toContain(response.status);
-      
-      // Note: Audit trail verification would require checking audit logs
-      // This depends on SENTINEL_AUDIT_SINK configuration
-    });
-  });
-});
-

package/apps/gateway/src/adapters/redis-sentinel-publisher.ts

@@ -1,176 +0,0 @@
-/**
- * Redis publisher for Sentinel events
- * Publishes guard/audit events to Redis pub/sub channels
- */
-
-import { getRedisClient } from '../db/redis.js';
-import { createLogger } from '@4runr/shared';
-import {
-  buildSentinelFailureOutput,
-  parsePolicyFromKillReason,
-} from '../security/sentinel-run-failure.js';
-import {
-  sentinelKillsTotal,
-  sentinelPolicyViolationsTotal,
-} from '../metrics/index.js';
-
-const logger = createLogger('Gateway:Sentinel:RedisPublisher');
-
-export interface SentinelEvent {
-  type: 'shield.decision' | 'judge.verdict' | 'audit.log' | 'policy.violation' | 'kill';
-  runId: string;
-  timestamp: number;
-  data: Record<string, unknown>;
-}
-
-export interface PolicyViolationEventData {
-  policy: string;
-  reason: string;
-  threshold: number;
-  actual: number;
-  action: 'deny';
-}
-
-export type SentinelKillTrigger = 'policy' | 'manual' | 'operator';
-
-/**
- * Publish a Sentinel event to Redis
- */
-export async function publishSentinelEvent(event: SentinelEvent): Promise<void> {
-  try {
-    const redisClient = await getRedisClient();
-    if (!redisClient) {
-      return;
-    }
-
-    const channel = `sentinel:${event.type}`;
-    const message = JSON.stringify(event);
-
-    await redisClient.publish(channel, message);
-    logger.debug('Published Sentinel event to Redis', {
-      channel,
-      runId: event.runId,
-      type: event.type,
-    });
-  } catch (error) {
-    logger.error('Failed to publish Sentinel event to Redis', {
-      runId: event.runId,
-      type: event.type,
-      error: error instanceof Error ? error.message : String(error),
-    });
-  }
-}
-
-export function buildPolicyViolationEvent(
-  runId: string,
-  violation: PolicyViolationEventData,
-  timestamp = Date.now()
-): SentinelEvent {
-  return {
-    type: 'policy.violation',
-    runId,
-    timestamp,
-    data: { ...violation },
-  };
-}
-
-export function buildKillEvent(
-  runId: string,
-  reason: string,
-  trigger: SentinelKillTrigger,
-  timestamp = Date.now()
-): SentinelEvent {
-  return {
-    type: 'kill',
-    runId,
-    timestamp,
-    data: {
-      reason,
-      trigger,
-      source: 'sentinel',
-      output: buildSentinelFailureOutput(reason),
-      policy: parsePolicyFromKillReason(reason),
-    },
-  };
-}
-
-export function buildAuditLogEvent(
-  runId: string,
-  action: 'kill' | 'policy_violation',
-  payload: Record<string, unknown>,
-  timestamp = Date.now()
-): SentinelEvent {
-  return {
-    type: 'audit.log',
-    runId,
-    timestamp,
-    data: { action, ...payload },
-  };
-}
-
-function resolveKillTrigger(reason: string): SentinelKillTrigger {
-  if (reason === 'manual_cancellation') {
-    return 'manual';
-  }
-  if (reason.startsWith('policy_violation:')) {
-    return 'policy';
-  }
-  return 'operator';
-}
-
-/**
- * Publish kill (+ audit) after run store is updated. Policy violations are
- * published separately via setPolicyViolationPublisher when evaluation fires.
- */
-export async function publishSentinelRunKill(
-  runId: string,
-  reason: string
-): Promise<void> {
-  const trigger = resolveKillTrigger(reason);
-  sentinelKillsTotal.inc({ trigger });
-
-  await publishSentinelEvent(buildKillEvent(runId, reason, trigger));
-  await publishSentinelEvent(
-    buildAuditLogEvent(runId, 'kill', {
-      reason,
-      trigger,
-      output: buildSentinelFailureOutput(reason),
-    })
-  );
-}
-
-/**
- * Policy violation observed during Sentinel evaluation (before kill completes).
- */
-export async function publishSentinelPolicyViolation(
-  runId: string,
-  violation: PolicyViolationEventData
-): Promise<void> {
-  sentinelPolicyViolationsTotal.inc({ policy: violation.policy });
-
-  await publishSentinelEvent(buildPolicyViolationEvent(runId, violation));
-  await publishSentinelEvent(
-    buildAuditLogEvent(runId, 'policy_violation', {
-      policy: violation.policy,
-      reason: violation.reason,
-      threshold: violation.threshold,
-      actual: violation.actual,
-      action: 'deny',
-    })
-  );
-}
-
-/** Fire-and-forget wrapper for hot paths (Shield pattern). */
-export function publishSentinelRunKillFireAndForget(
-  runId: string,
-  reason: string
-): void {
-  void publishSentinelRunKill(runId, reason);
-}
-
-export function publishSentinelPolicyViolationFireAndForget(
-  runId: string,
-  violation: PolicyViolationEventData
-): void {
-  void publishSentinelPolicyViolation(runId, violation);
-}