4runr-os 2.10.65 → 2.10.67

package/apps/gateway/src/__tests__/sentinel-run-failure.test.ts

@@ -1,89 +1,89 @@
-import { describe, it, expect } from '@jest/globals';
-import {
-  appendRunLog,
-  buildSentinelFailureLog,
-  buildSentinelFailureOutput,
-  extractSentinelKillReason,
-  formatSentinelKillError,
-  isSentinelKillError,
-  parsePolicyFromKillReason,
-  SENTINEL_KILL_ERROR_PREFIX,
-} from '../security/sentinel-run-failure.js';
-
-describe('sentinel-run-failure', () => {
-  describe('parsePolicyFromKillReason', () => {
-    it('extracts policy name from policy_violation prefix', () => {
-      expect(parsePolicyFromKillReason('policy_violation:timeout')).toBe('timeout');
-      expect(parsePolicyFromKillReason('policy_violation:token_cap')).toBe('token_cap');
-    });
-
-    it('returns undefined for non-policy reasons', () => {
-      expect(parsePolicyFromKillReason('manual_cancellation')).toBeUndefined();
-      expect(parsePolicyFromKillReason('operator_stop')).toBeUndefined();
-    });
-  });
-
-  describe('buildSentinelFailureOutput', () => {
-    it('returns stable deny shape for policy violations', () => {
-      expect(buildSentinelFailureOutput('policy_violation:idle')).toEqual({
-        error: 'Run terminated by Sentinel',
-        source: 'sentinel',
-        action: 'deny',
-        reason: 'policy_violation:idle',
-        policy: 'idle',
-      });
-    });
-
-    it('returns stable deny shape for manual cancellation', () => {
-      expect(buildSentinelFailureOutput('manual_cancellation')).toEqual({
-        error: 'Run terminated by Sentinel',
-        source: 'sentinel',
-        action: 'deny',
-        reason: 'manual_cancellation',
-      });
-    });
-  });
-
-  describe('buildSentinelFailureLog', () => {
-    it('uses consistent log prefix', () => {
-      expect(buildSentinelFailureLog('policy_violation:cost')).toBe(
-        'Sentinel denied run: policy_violation:cost'
-      );
-    });
-  });
-
-  describe('extractSentinelKillReason', () => {
-    it('parses processor throw message', () => {
-      expect(
-        extractSentinelKillReason('Run killed by Sentinel: policy_violation:timeout')
-      ).toBe('policy_violation:timeout');
-    });
-
-    it('returns null for unrelated errors', () => {
-      expect(extractSentinelKillReason('Agent not found')).toBeNull();
-      expect(isSentinelKillError('Agent not found')).toBe(false);
-    });
-
-    it('returns null when reason after prefix is empty', () => {
-      expect(extractSentinelKillReason(SENTINEL_KILL_ERROR_PREFIX)).toBeNull();
-      expect(extractSentinelKillReason(`${SENTINEL_KILL_ERROR_PREFIX}   `)).toBeNull();
-    });
-  });
-
-  describe('formatSentinelKillError', () => {
-    it('round-trips with extractSentinelKillReason', () => {
-      const reason = 'policy_violation:idle';
-      const message = formatSentinelKillError(reason);
-      expect(extractSentinelKillReason(message)).toBe(reason);
-    });
-  });
-
-  describe('appendRunLog', () => {
-    it('preserves existing entries', () => {
-      const first = { timestamp: 't1', level: 'info', message: 'started' };
-      const second = { timestamp: 't2', level: 'error', message: 'denied' };
-      expect(appendRunLog([first], second)).toEqual([first, second]);
-      expect(appendRunLog(undefined, second)).toEqual([second]);
-    });
-  });
-});
+import { describe, it, expect } from '@jest/globals';
+import {
+  appendRunLog,
+  buildSentinelFailureLog,
+  buildSentinelFailureOutput,
+  extractSentinelKillReason,
+  formatSentinelKillError,
+  isSentinelKillError,
+  parsePolicyFromKillReason,
+  SENTINEL_KILL_ERROR_PREFIX,
+} from '../security/sentinel-run-failure.js';
+
+describe('sentinel-run-failure', () => {
+  describe('parsePolicyFromKillReason', () => {
+    it('extracts policy name from policy_violation prefix', () => {
+      expect(parsePolicyFromKillReason('policy_violation:timeout')).toBe('timeout');
+      expect(parsePolicyFromKillReason('policy_violation:token_cap')).toBe('token_cap');
+    });
+
+    it('returns undefined for non-policy reasons', () => {
+      expect(parsePolicyFromKillReason('manual_cancellation')).toBeUndefined();
+      expect(parsePolicyFromKillReason('operator_stop')).toBeUndefined();
+    });
+  });
+
+  describe('buildSentinelFailureOutput', () => {
+    it('returns stable deny shape for policy violations', () => {
+      expect(buildSentinelFailureOutput('policy_violation:idle')).toEqual({
+        error: 'Run terminated by Sentinel',
+        source: 'sentinel',
+        action: 'deny',
+        reason: 'policy_violation:idle',
+        policy: 'idle',
+      });
+    });
+
+    it('returns stable deny shape for manual cancellation', () => {
+      expect(buildSentinelFailureOutput('manual_cancellation')).toEqual({
+        error: 'Run terminated by Sentinel',
+        source: 'sentinel',
+        action: 'deny',
+        reason: 'manual_cancellation',
+      });
+    });
+  });
+
+  describe('buildSentinelFailureLog', () => {
+    it('uses consistent log prefix', () => {
+      expect(buildSentinelFailureLog('policy_violation:cost')).toBe(
+        'Sentinel denied run: policy_violation:cost'
+      );
+    });
+  });
+
+  describe('extractSentinelKillReason', () => {
+    it('parses processor throw message', () => {
+      expect(
+        extractSentinelKillReason('Run killed by Sentinel: policy_violation:timeout')
+      ).toBe('policy_violation:timeout');
+    });
+
+    it('returns null for unrelated errors', () => {
+      expect(extractSentinelKillReason('Agent not found')).toBeNull();
+      expect(isSentinelKillError('Agent not found')).toBe(false);
+    });
+
+    it('returns null when reason after prefix is empty', () => {
+      expect(extractSentinelKillReason(SENTINEL_KILL_ERROR_PREFIX)).toBeNull();
+      expect(extractSentinelKillReason(`${SENTINEL_KILL_ERROR_PREFIX}   `)).toBeNull();
+    });
+  });
+
+  describe('formatSentinelKillError', () => {
+    it('round-trips with extractSentinelKillReason', () => {
+      const reason = 'policy_violation:idle';
+      const message = formatSentinelKillError(reason);
+      expect(extractSentinelKillReason(message)).toBe(reason);
+    });
+  });
+
+  describe('appendRunLog', () => {
+    it('preserves existing entries', () => {
+      const first = { timestamp: 't1', level: 'info', message: 'started' };
+      const second = { timestamp: 't2', level: 'error', message: 'denied' };
+      expect(appendRunLog([first], second)).toEqual([first, second]);
+      expect(appendRunLog(undefined, second)).toEqual([second]);
+    });
+  });
+});

package/apps/gateway/src/adapters/gateway-cancel-adapter.ts

@@ -1,32 +1,32 @@
-/**
- * In-process cancel adapter for @4runr/sentinel — avoids authenticated HTTP loopback.
- */
-
-import type { CancelAdapter, CancelResult } from '@4runr/sentinel';
-import { applyRunKill } from '../runs/run-kill.js';
-
-export function createGatewayCancelAdapter(): CancelAdapter {
-  return {
-    async cancelRun(runId: string, reason: string): Promise<CancelResult> {
-      try {
-        const result = await applyRunKill(runId, reason);
-        return {
-          success: result.applied,
-          reason: result.idempotent
-            ? 'Run already terminal'
-            : result.applied
-              ? 'Run killed in store'
-              : 'Run kill not applied',
-          timestamp: Date.now(),
-        };
-      } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        return {
-          success: false,
-          reason: message,
-          timestamp: Date.now(),
-        };
-      }
-    },
-  };
-}
+/**
+ * In-process cancel adapter for @4runr/sentinel — avoids authenticated HTTP loopback.
+ */
+
+import type { CancelAdapter, CancelResult } from '@4runr/sentinel';
+import { applyRunKill } from '../runs/run-kill.js';
+
+export function createGatewayCancelAdapter(): CancelAdapter {
+  return {
+    async cancelRun(runId: string, reason: string): Promise<CancelResult> {
+      try {
+        const result = await applyRunKill(runId, reason);
+        return {
+          success: result.applied,
+          reason: result.idempotent
+            ? 'Run already terminal'
+            : result.applied
+              ? 'Run killed in store'
+              : 'Run kill not applied',
+          timestamp: Date.now(),
+        };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+          success: false,
+          reason: message,
+          timestamp: Date.now(),
+        };
+      }
+    },
+  };
+}

package/apps/gateway/src/queue/sentinel-execute-watch.ts

@@ -1,42 +1,42 @@
-/**
- * Poll Sentinel buffer during blocking agent.execute().
- * Does not abort the agent process — returns/rejects as soon as the run is marked killed.
- */
-
-import { formatSentinelKillError } from '../security/sentinel-run-failure.js';
-
-export const SENTINEL_KILL_CHECK_INTERVAL_MS = 250;
-
-export type SentinelRunStatusSnapshot = {
-  status?: string;
-  killReason?: string;
-};
-
-/**
- * Race execute() against periodic Sentinel kill checks.
- * Agent work may continue in the background until the executor finishes.
- */
-export async function executeWithSentinelWatch<T>(
-  runId: string,
-  execute: () => Promise<T>,
-  getRunStatus: (runId: string) => SentinelRunStatusSnapshot | undefined
-): Promise<T> {
-  let timer: ReturnType<typeof setInterval> | undefined;
-
-  const watchPromise = new Promise<never>((_, reject) => {
-    timer = setInterval(() => {
-      const state = getRunStatus(runId);
-      if (state?.status === 'killed') {
-        reject(new Error(formatSentinelKillError(state.killReason ?? 'unknown')));
-      }
-    }, SENTINEL_KILL_CHECK_INTERVAL_MS);
-  });
-
-  try {
-    return await Promise.race([execute(), watchPromise]);
-  } finally {
-    if (timer !== undefined) {
-      clearInterval(timer);
-    }
-  }
-}
+/**
+ * Poll Sentinel buffer during blocking agent.execute().
+ * Does not abort the agent process — returns/rejects as soon as the run is marked killed.
+ */
+
+import { formatSentinelKillError } from '../security/sentinel-run-failure.js';
+
+export const SENTINEL_KILL_CHECK_INTERVAL_MS = 250;
+
+export type SentinelRunStatusSnapshot = {
+  status?: string;
+  killReason?: string;
+};
+
+/**
+ * Race execute() against periodic Sentinel kill checks.
+ * Agent work may continue in the background until the executor finishes.
+ */
+export async function executeWithSentinelWatch<T>(
+  runId: string,
+  execute: () => Promise<T>,
+  getRunStatus: (runId: string) => SentinelRunStatusSnapshot | undefined
+): Promise<T> {
+  let timer: ReturnType<typeof setInterval> | undefined;
+
+  const watchPromise = new Promise<never>((_, reject) => {
+    timer = setInterval(() => {
+      const state = getRunStatus(runId);
+      if (state?.status === 'killed') {
+        reject(new Error(formatSentinelKillError(state.killReason ?? 'unknown')));
+      }
+    }, SENTINEL_KILL_CHECK_INTERVAL_MS);
+  });
+
+  try {
+    return await Promise.race([execute(), watchPromise]);
+  } finally {
+    if (timer !== undefined) {
+      clearInterval(timer);
+    }
+  }
+}

package/apps/gateway/src/routes/sentinel-policies.ts

@@ -175,7 +175,7 @@ export async function sentinelPolicyRoutes(fastify: FastifyInstance) {
       
       try {
         sentinel.updateConfig(config);
-        const savedTo = persistSentinelConfigAfterApply(sentinel, config);
+        const savedTo = persistSentinelConfigAfterApply(sentinel, config, fastify.log);
         return {
           success: true,
           template: templateName,
@@ -235,7 +235,7 @@ export async function sentinelPolicyRoutes(fastify: FastifyInstance) {
 
         try {
           sentinel.updateConfig(config);
-          const savedTo = persistSentinelConfigAfterApply(sentinel, config);
+          const savedTo = persistSentinelConfigAfterApply(sentinel, config, fastify.log);
           return {
             success: true,
             config,

package/apps/gateway/src/runs/run-kill.ts

@@ -1,117 +1,117 @@
-/**
- * In-process run kill — used by HTTP cancel and Sentinel policy kill (via cancel adapter).
- */
-
-import { getRunStore } from './index.js';
-import { removeRunExecutionJob } from '../queue/index.js';
-import {
-  appendRunLog,
-  buildSentinelFailureLog,
-  buildSentinelFailureOutput,
-  emitSentinelKilledSse,
-} from '../security/sentinel-run-failure.js';
-import { publishSentinelRunKillFireAndForget } from '../adapters/redis-sentinel-publisher.js';
-import type { Run, RunStatus } from './types.js';
-import { createLogger } from '@4runr/shared';
-
-const logger = createLogger('Gateway:RunKill');
-
-const TERMINAL_STATUSES: RunStatus[] = ['killed', 'completed', 'failed'];
-
-export interface RunKillSideEffects {
-  decRunsActive?: () => void;
-  getSseConnections?: (runId: string) => Set<{ write: (chunk: string) => void }> | undefined;
-  onSseMessage?: () => void;
-  deleteEventStream?: (runId: string) => void;
-}
-
-export interface ApplyRunKillResult {
-  applied: boolean;
-  idempotent: boolean;
-  runId: string;
-  status: RunStatus;
-}
-
-let sideEffects: RunKillSideEffects = {};
-
-export function setRunKillSideEffects(effects: RunKillSideEffects): void {
-  sideEffects = effects;
-}
-
-export function isTerminalRunStatus(status: RunStatus): boolean {
-  return TERMINAL_STATUSES.includes(status);
-}
-
-/**
- * Persist killed status, append deny log, notify SSE, remove queued job.
- * Idempotent when the run is already terminal.
- */
-export async function applyRunKill(
-  runId: string,
-  reason: string
-): Promise<ApplyRunKillResult> {
-  const store = getRunStore();
-  const run = await store.getRunById(runId);
-
-  if (!run) {
-    throw new Error(`Run not found: ${runId}`);
-  }
-
-  if (isTerminalRunStatus(run.status)) {
-    logger.info('Run kill skipped — already terminal', { runId, status: run.status });
-    return {
-      applied: false,
-      idempotent: true,
-      runId,
-      status: run.status,
-    };
-  }
-
-  if (run.status === 'queued') {
-    try {
-      await removeRunExecutionJob(runId);
-    } catch (error) {
-      logger.warn('Failed to remove queued job during kill', {
-        runId,
-        error: error instanceof Error ? error.message : String(error),
-      });
-    }
-  }
-
-  const logEntry = {
-    timestamp: new Date().toISOString(),
-    level: 'error',
-    message: buildSentinelFailureLog(reason),
-  };
-
-  await store.updateRunStatus(runId, 'killed', {
-    completedAt: new Date().toISOString(),
-    output: buildSentinelFailureOutput(reason),
-    logs: appendRunLog(run.logs, logEntry),
-  });
-
-  sideEffects.decRunsActive?.();
-
-  const connections = sideEffects.getSseConnections?.(runId);
-  if (connections) {
-    connections.forEach((connection) => {
-      emitSentinelKilledSse(connection, runId, reason);
-      sideEffects.onSseMessage?.();
-    });
-  }
-
-  sideEffects.deleteEventStream?.(runId);
-
-  publishSentinelRunKillFireAndForget(runId, reason);
-
-  logger.info('Run killed in store', { runId, reason });
-
-  return {
-    applied: true,
-    idempotent: false,
-    runId,
-    status: 'killed',
-  };
-}
-
-export type { Run };
+/**
+ * In-process run kill — used by HTTP cancel and Sentinel policy kill (via cancel adapter).
+ */
+
+import { getRunStore } from './index.js';
+import { removeRunExecutionJob } from '../queue/index.js';
+import {
+  appendRunLog,
+  buildSentinelFailureLog,
+  buildSentinelFailureOutput,
+  emitSentinelKilledSse,
+} from '../security/sentinel-run-failure.js';
+import { publishSentinelRunKillFireAndForget } from '../adapters/redis-sentinel-publisher.js';
+import type { Run, RunStatus } from './types.js';
+import { createLogger } from '@4runr/shared';
+
+const logger = createLogger('Gateway:RunKill');
+
+const TERMINAL_STATUSES: RunStatus[] = ['killed', 'completed', 'failed'];
+
+export interface RunKillSideEffects {
+  decRunsActive?: () => void;
+  getSseConnections?: (runId: string) => Set<{ write: (chunk: string) => void }> | undefined;
+  onSseMessage?: () => void;
+  deleteEventStream?: (runId: string) => void;
+}
+
+export interface ApplyRunKillResult {
+  applied: boolean;
+  idempotent: boolean;
+  runId: string;
+  status: RunStatus;
+}
+
+let sideEffects: RunKillSideEffects = {};
+
+export function setRunKillSideEffects(effects: RunKillSideEffects): void {
+  sideEffects = effects;
+}
+
+export function isTerminalRunStatus(status: RunStatus): boolean {
+  return TERMINAL_STATUSES.includes(status);
+}
+
+/**
+ * Persist killed status, append deny log, notify SSE, remove queued job.
+ * Idempotent when the run is already terminal.
+ */
+export async function applyRunKill(
+  runId: string,
+  reason: string
+): Promise<ApplyRunKillResult> {
+  const store = getRunStore();
+  const run = await store.getRunById(runId);
+
+  if (!run) {
+    throw new Error(`Run not found: ${runId}`);
+  }
+
+  if (isTerminalRunStatus(run.status)) {
+    logger.info('Run kill skipped — already terminal', { runId, status: run.status });
+    return {
+      applied: false,
+      idempotent: true,
+      runId,
+      status: run.status,
+    };
+  }
+
+  if (run.status === 'queued') {
+    try {
+      await removeRunExecutionJob(runId);
+    } catch (error) {
+      logger.warn('Failed to remove queued job during kill', {
+        runId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
+  const logEntry = {
+    timestamp: new Date().toISOString(),
+    level: 'error',
+    message: buildSentinelFailureLog(reason),
+  };
+
+  await store.updateRunStatus(runId, 'killed', {
+    completedAt: new Date().toISOString(),
+    output: buildSentinelFailureOutput(reason),
+    logs: appendRunLog(run.logs, logEntry),
+  });
+
+  sideEffects.decRunsActive?.();
+
+  const connections = sideEffects.getSseConnections?.(runId);
+  if (connections) {
+    connections.forEach((connection) => {
+      emitSentinelKilledSse(connection, runId, reason);
+      sideEffects.onSseMessage?.();
+    });
+  }
+
+  sideEffects.deleteEventStream?.(runId);
+
+  publishSentinelRunKillFireAndForget(runId, reason);
+
+  logger.info('Run killed in store', { runId, reason });
+
+  return {
+    applied: true,
+    idempotent: false,
+    runId,
+    status: 'killed',
+  };
+}
+
+export type { Run };

package/apps/gateway/src/security/sentinel-config-store.ts

@@ -119,7 +119,7 @@ export function mergePersistedSentinelConfig(sentinel: Sentinel): boolean {
 
   for (const { field } of ENV_FIELD_KEYS) {
     if (envDefinesField(field)) continue;
-    (merged as Record<string, unknown>)[field] = saved[field];
+    (merged as any)[field] = saved[field];
   }
 
   try {
@@ -137,12 +137,49 @@ export function applyPersistedSentinelConfig(sentinel: Sentinel, logger?: {
   info: (msg: string, meta?: Record<string, unknown>) => void;
   warn: (msg: string, meta?: Record<string, unknown>) => void;
 }): boolean {
+  const filePath = getSentinelLimitsFilePath();
+  const saved = loadSentinelLimitsFromDisk();
+  
+  if (!saved) {
+    logger?.info('No Sentinel limits file found on disk (will use env defaults)', {
+      expectedPath: filePath,
+    });
+    return false;
+  }
+
+  // Check which env vars are set and will override the disk values
+  const envOverrides: string[] = [];
+  const diskApplied: string[] = [];
+  
+  for (const { field, envKeys } of ENV_FIELD_KEYS) {
+    if (envDefinesField(field)) {
+      const activeEnv = envKeys.find(k => process.env[k] !== undefined && process.env[k]?.trim() !== '');
+      if (activeEnv) {
+        envOverrides.push(`${field} (via ${activeEnv}=${process.env[activeEnv]})`);
+      }
+    } else {
+      diskApplied.push(`${field}=${saved[field]}`);
+    }
+  }
+
   const ok = mergePersistedSentinelConfig(sentinel);
+  
   if (ok) {
-    logger?.info('Sentinel limits loaded from disk', {
-      path: getSentinelLimitsFilePath(),
+    const finalConfig = sentinel.getConfig();
+    logger?.info('✅ Sentinel limits loaded from disk and merged', {
+      diskPath: filePath,
+      diskValues: saved,
+      envOverrides: envOverrides.length > 0 ? envOverrides : 'none',
+      diskAppliedFields: diskApplied.length > 0 ? diskApplied : 'none (all env-overridden)',
+      finalActiveConfig: finalConfig,
+    });
+  } else {
+    logger?.warn('Failed to apply Sentinel limits from disk', {
+      diskPath: filePath,
+      savedConfig: saved,
     });
   }
+  
   return ok;
 }
 
@@ -153,8 +190,19 @@ export function hydrateSentinelConfigFromDisk(sentinel: Sentinel): void {
 
 export function persistSentinelConfigAfterApply(
   sentinel: Sentinel,
-  config: SentinelConfig
+  config: SentinelConfig,
+  logger?: {
+    info: (msg: string, meta?: Record<string, unknown>) => void;
+  }
 ): string {
+  const filePath = getSentinelLimitsFilePath();
   saveSentinelLimitsToDisk(config);
-  return getSentinelLimitsFilePath();
+  
+  logger?.info('✅ Sentinel limits saved to disk', {
+    path: filePath,
+    config: config,
+    note: 'Survives Gateway restart. Env vars in docker-compose override individual fields when set.',
+  });
+  
+  return filePath;
 }