palaryn 0.1.0 → 0.3.2

package/src/config/validate.ts

@@ -0,0 +1,88 @@
+import * as fs from 'fs';
+import { GatewayConfig } from '../types/config';
+
+export interface ConfigIssue {
+  level: 'fatal' | 'warning';
+  message: string;
+}
+
+export interface ConfigValidationResult {
+  valid: boolean;
+  issues: ConfigIssue[];
+}
+
+/**
+ * Validate gateway configuration before startup.
+ * Returns fatal errors (should prevent startup) and warnings (log and continue).
+ */
+export function validateConfig(config: GatewayConfig): ConfigValidationResult {
+  const issues: ConfigIssue[] = [];
+  const isProduction = process.env.NODE_ENV === 'production';
+
+  // Port validation
+  if (!Number.isInteger(config.port) || config.port < 1 || config.port > 65535) {
+    issues.push({ level: 'fatal', message: `Invalid port: ${config.port}. Must be between 1 and 65535.` });
+  }
+
+  // Policy file must be readable
+  try {
+    fs.accessSync(config.policy.pack_path, fs.constants.R_OK);
+  } catch {
+    issues.push({ level: 'fatal', message: `Policy file not readable: ${config.policy.pack_path}` });
+  }
+
+  // Weak secrets in production
+  if (isProduction) {
+    if (!process.env.JWT_SECRET) {
+      issues.push({ level: 'fatal', message: 'JWT_SECRET must be set in production.' });
+    }
+    if (!process.env.APPROVAL_SECRET) {
+      issues.push({ level: 'fatal', message: 'APPROVAL_SECRET must be set in production.' });
+    }
+  }
+
+  // Warnings: no persistent storage
+  if (!process.env.REDIS_URL && !process.env.DATABASE_URL) {
+    issues.push({ level: 'warning', message: 'No persistent storage configured (REDIS_URL or DATABASE_URL). Data will be lost on restart.' });
+  }
+
+  // Auth enabled but no keys
+  if (config.auth.enabled && Object.keys(config.auth.api_keys).length === 0 && !config.auth.jwt?.enabled) {
+    issues.push({ level: 'warning', message: 'Auth is enabled but no API keys or JWT configured. All authenticated requests will fail.' });
+  }
+
+  // Audit log directory
+  if (config.audit.enabled && config.audit.log_dir) {
+    try {
+      fs.accessSync(config.audit.log_dir, fs.constants.W_OK);
+    } catch {
+      issues.push({ level: 'warning', message: `Audit log directory not writable: ${config.audit.log_dir}` });
+    }
+  }
+
+  // Auth must be enabled — MCP HTTP transport requires authentication
+  if (!config.auth.enabled) {
+    issues.push({ level: 'fatal', message: 'Auth is disabled. MCP HTTP transport requires authentication. Set AUTH_ENABLED=true.' });
+  }
+
+  // RBAC default_role must not be admin or operator
+  if (config.auth.rbac?.default_role) {
+    const dr = config.auth.rbac.default_role;
+    if (dr === 'admin' || dr === 'operator') {
+      issues.push({ level: 'fatal', message: `RBAC default_role '${dr}' is a privileged role. Use 'agent', 'readonly', or a custom role.` });
+    }
+  }
+
+  // Proxy port validation
+  if (config.proxy?.enabled) {
+    if (!Number.isInteger(config.proxy.port) || config.proxy.port < 1 || config.proxy.port > 65535) {
+      issues.push({ level: 'fatal', message: `Invalid proxy port: ${config.proxy.port}. Must be between 1 and 65535.` });
+    }
+    if (config.proxy.port === config.port) {
+      issues.push({ level: 'fatal', message: `Proxy port (${config.proxy.port}) conflicts with server port (${config.port}).` });
+    }
+  }
+
+  const hasFatal = issues.some(i => i.level === 'fatal');
+  return { valid: !hasFatal, issues };
+}

package/src/dlp/composite-scanner.ts

@@ -0,0 +1,213 @@
+import { DLPReport, DLPRedaction, DLPSeverity } from '../types/tool-result';
+import { DLPConfig } from '../types/config';
+import { DLPScanner } from './scanner';
+import { DLPBackend, DLPDetection } from './interfaces';
+
+/** Maximum recursion depth for extracting string values from nested structures. */
+const MAX_EXTRACT_DEPTH = 32;
+
+/**
+ * CompositeDLPScanner wraps the existing DLPScanner and augments it with
+ * additional pluggable DLP backends (e.g. TruffleHog, regex backend).
+ *
+ * The scan flow:
+ *   1. Run the standard DLPScanner.scan() for full existing behavior
+ *   2. Extract all string values from the data object recursively
+ *   3. For each additional backend, call scanString on each extracted string
+ *   4. Merge backend detections into the DLPReport (deduplicate by pattern_name)
+ *   5. Recalculate severity based on all combined findings
+ *
+ * This design preserves full backward compatibility -- the existing DLPScanner
+ * continues to handle redaction, policy integration, and all current features.
+ * The backends provide supplemental detections that are merged into the report.
+ */
+export class CompositeDLPScanner {
+  private readonly dlpScanner: DLPScanner;
+  private readonly backends: DLPBackend[];
+  private readonly config: DLPConfig;
+
+  constructor(config: DLPConfig, backends: DLPBackend[]) {
+    this.config = config;
+    this.dlpScanner = new DLPScanner(config);
+    this.backends = backends;
+  }
+
+  /**
+   * Get the underlying DLPScanner instance (for redaction operations, etc.).
+   */
+  get scanner(): DLPScanner {
+    return this.dlpScanner;
+  }
+
+  /**
+   * Scan data for secrets and PII using the built-in scanner and all backends.
+   *
+   * @param data - The data to scan (object, array, string, or primitive).
+   * @param basePath - Dot-notation prefix for paths within the data structure.
+   * @returns A merged DLPReport with detections from all sources.
+   */
+  scan(data: unknown, basePath: string = ''): DLPReport {
+    // Step 1: Run the standard DLPScanner for existing behavior
+    const baseReport = this.dlpScanner.scan(data, basePath);
+
+    // If DLP is disabled or there are no backends, return the base report as-is
+    if (!this.config.enabled || this.backends.length === 0) {
+      return baseReport;
+    }
+
+    // Step 2: Extract all string values with their paths from the data,
+    // including fields that the default recursive walker would miss
+    // (context.purpose, context.labels, actor.display).
+    const stringEntries = this.extractStrings(data, basePath, 0);
+
+    // Step 2.5: Explicitly extract commonly-missed ToolCall text fields.
+    // The generic walker handles args.* but context and actor are top-level
+    // fields that may not be traversed if 'data' is the full ToolCall object
+    // or if basePath filtering skips them.
+    if (data && typeof data === 'object' && !Array.isArray(data)) {
+      const obj = data as Record<string, unknown>;
+      // context.purpose
+      if (obj.context && typeof obj.context === 'object') {
+        const ctx = obj.context as Record<string, unknown>;
+        if (typeof ctx.purpose === 'string') {
+          const p = basePath ? `${basePath}.context.purpose` : 'context.purpose';
+          if (!stringEntries.some(e => e.path === p)) {
+            stringEntries.push({ value: ctx.purpose, path: p });
+          }
+        }
+        // context.labels[]
+        if (Array.isArray(ctx.labels)) {
+          for (let i = 0; i < ctx.labels.length; i++) {
+            if (typeof ctx.labels[i] === 'string') {
+              const p = basePath ? `${basePath}.context.labels[${i}]` : `context.labels[${i}]`;
+              if (!stringEntries.some(e => e.path === p)) {
+                stringEntries.push({ value: ctx.labels[i] as string, path: p });
+              }
+            }
+          }
+        }
+      }
+      // actor.display
+      if (obj.actor && typeof obj.actor === 'object') {
+        const actor = obj.actor as Record<string, unknown>;
+        if (typeof actor.display === 'string') {
+          const p = basePath ? `${basePath}.actor.display` : 'actor.display';
+          if (!stringEntries.some(e => e.path === p)) {
+            stringEntries.push({ value: actor.display, path: p });
+          }
+        }
+      }
+    }
+
+    // Step 3: Run each backend on each string value
+    const backendDetections: Array<{ detection: DLPDetection; path: string }> = [];
+
+    for (const backend of this.backends) {
+      for (const entry of stringEntries) {
+        try {
+          const detections = backend.scanString(entry.value);
+          for (const detection of detections) {
+            backendDetections.push({ detection, path: entry.path });
+          }
+        } catch (err: unknown) {
+          // Graceful degradation: log and continue with other backends/strings
+          const message = err instanceof Error ? err.message : String(err);
+          console.warn(`[CompositeDLPScanner] backend '${backend.name}' failed on path '${entry.path}': ${message}`);
+        }
+      }
+    }
+
+    // Step 4: Merge backend detections into the base report (deduplicate by pattern_name)
+    const existingPatterns = new Set(baseReport.detected);
+    const additionalDetected: string[] = [];
+    const additionalRedactions: DLPRedaction[] = [];
+
+    for (const { detection, path } of backendDetections) {
+      if (!existingPatterns.has(detection.pattern_name)) {
+        existingPatterns.add(detection.pattern_name);
+        additionalDetected.push(detection.pattern_name);
+        additionalRedactions.push({
+          path,
+          method: this.config.default_redaction_method,
+          original_type: detection.pattern_name,
+        });
+      }
+    }
+
+    const mergedDetected = [...baseReport.detected, ...additionalDetected];
+    const mergedRedactions = [...baseReport.redactions, ...additionalRedactions];
+
+    // Step 5: Recalculate severity based on all findings
+    const allSeverities: DLPSeverity[] = [baseReport.severity];
+    for (const { detection } of backendDetections) {
+      allSeverities.push(detection.severity);
+    }
+    const severity = this.highestSeverity(allSeverities);
+
+    return {
+      detected: mergedDetected,
+      redactions: mergedRedactions,
+      severity,
+    };
+  }
+
+  /**
+   * Recursively extract all string values and their dot-notation paths
+   * from a nested data structure.
+   */
+  private extractStrings(
+    data: unknown,
+    path: string,
+    depth: number,
+  ): Array<{ value: string; path: string }> {
+    if (depth > MAX_EXTRACT_DEPTH) {
+      return [];
+    }
+
+    if (data === null || data === undefined) {
+      return [];
+    }
+
+    if (typeof data === 'string') {
+      return [{ value: data, path }];
+    }
+
+    if (Array.isArray(data)) {
+      const results: Array<{ value: string; path: string }> = [];
+      for (let i = 0; i < data.length; i++) {
+        const childPath = path ? `${path}[${i}]` : `[${i}]`;
+        results.push(...this.extractStrings(data[i], childPath, depth + 1));
+      }
+      return results;
+    }
+
+    if (typeof data === 'object') {
+      const results: Array<{ value: string; path: string }> = [];
+      for (const [key, value] of Object.entries(data as Record<string, unknown>)) {
+        const childPath = path ? `${path}.${key}` : key;
+        results.push(...this.extractStrings(value, childPath, depth + 1));
+      }
+      return results;
+    }
+
+    // Non-string primitives are ignored
+    return [];
+  }
+
+  /**
+   * Return the highest severity from an array of severities.
+   */
+  private highestSeverity(severities: DLPSeverity[]): DLPSeverity {
+    const rank: Record<DLPSeverity, number> = { low: 0, medium: 1, high: 2 };
+    let highest: DLPSeverity = 'low';
+
+    for (const sev of severities) {
+      if (rank[sev] > rank[highest]) {
+        highest = sev;
+      }
+      if (highest === 'high') break; // Short-circuit
+    }
+
+    return highest;
+  }
+}

package/src/dlp/index.ts

@@ -0,0 +1,9 @@
+export { DLPScanner } from './scanner';
+export { DLPBackend, DLPDetection } from './interfaces';
+export { DLPPattern, SECRET_PATTERNS, PII_PATTERNS } from './patterns';
+export { RegexDLPBackend, RegexBackendConfig } from './regex-backend';
+export { TruffleHogBackend, TruffleHogConfig } from './trufflehog-backend';
+export { CompositeDLPScanner } from './composite-scanner';
+export { PROMPT_INJECTION_PATTERNS, OUTPUT_INJECTION_PATTERNS } from './prompt-injection-patterns';
+export { PromptInjectionBackend, PromptInjectionConfig } from './prompt-injection-backend';
+export { normalizeText, normalizeLeetspeak, ZERO_WIDTH_REGEX, HOMOGLYPH_MAP, LEETSPEAK_MAP } from './text-normalizer';

package/src/dlp/interfaces.ts

@@ -0,0 +1,34 @@
+import { DLPSeverity } from '../types/tool-result';
+
+/**
+ * A detection returned by a DLP backend scanner.
+ *
+ * Each detection identifies a single sensitive value found within a string,
+ * including its location so that upstream consumers can apply redactions.
+ */
+export interface DLPDetection {
+  /** Name of the pattern or detector that matched (e.g. 'aws_access_key', 'GitHubToken'). */
+  pattern_name: string;
+  /** Severity of the detection. */
+  severity: DLPSeverity;
+  /** The matched text. */
+  match: string;
+  /** Start index within the scanned string. */
+  start: number;
+  /** End index (exclusive) within the scanned string. */
+  end: number;
+}
+
+/**
+ * Pluggable backend interface for DLP secret/PII scanning.
+ *
+ * Implementations scan raw string values and return structured detections.
+ * The gateway can compose multiple backends (regex, trufflehog, etc.) via
+ * the CompositeDLPScanner.
+ */
+export interface DLPBackend {
+  /** Scan a string for secrets/PII. Returns detected pattern names and their locations. */
+  scanString(value: string): DLPDetection[];
+  /** Name of this backend for logging. */
+  readonly name: string;
+}

package/src/dlp/patterns.ts

@@ -0,0 +1,30 @@
+import { DLPSeverity } from '../types/tool-result';
+
+export interface DLPPattern {
+  name: string;
+  pattern: RegExp;
+  severity: DLPSeverity;
+}
+
+// Detection patterns for secrets
+export const SECRET_PATTERNS: DLPPattern[] = [
+  { name: 'aws_access_key', pattern: /AKIA[0-9A-Z]{16}/g, severity: 'high' },
+  { name: 'aws_secret_key', pattern: /(?:aws_secret_access_key|secret_key)\s*[=:]\s*[A-Za-z0-9/+=]{40}/gi, severity: 'high' },
+  { name: 'github_token', pattern: /gh[pousr]_[A-Za-z0-9_]{36,255}/g, severity: 'high' },
+  { name: 'generic_api_key', pattern: /(?:api[_-]?key|apikey)\s*[=:]\s*['"]?[A-Za-z0-9_\-]{20,}['"]?/gi, severity: 'medium' },
+  { name: 'bearer_token', pattern: /Bearer\s+[A-Za-z0-9\-._~+/]+=*/g, severity: 'high' },
+  { name: 'jwt_token', pattern: /eyJ[A-Za-z0-9_-]*\.eyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_\-]+/g, severity: 'high' },
+  { name: 'private_key', pattern: /-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----/g, severity: 'high' },
+  { name: 'password_field', pattern: /(?:password|passwd|pwd)\s*[=:]\s*['"]?[^\s'"]{8,}['"]?/gi, severity: 'high' },
+  { name: 'slack_token', pattern: /xox[baprs]-[0-9a-zA-Z-]{10,}/g, severity: 'high' },
+  { name: 'generic_secret', pattern: /(?:secret|token|credential)\s*[=:]\s*['"]?[A-Za-z0-9_\-]{16,}['"]?/gi, severity: 'medium' },
+];
+
+// PII patterns
+export const PII_PATTERNS: DLPPattern[] = [
+  { name: 'email', pattern: /[a-zA-Z0-9._%+\-]+@(?!(?:\d{1,3}\.){3}\d{1,3}\b)[a-zA-Z0-9\-]+(?:\.[a-zA-Z0-9\-]+)*\.[a-zA-Z]{2,}/g, severity: 'medium' },
+  { name: 'phone_us', pattern: /(?:\+1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}/g, severity: 'medium' },
+  { name: 'ssn', pattern: /\b\d{3}-\d{2}-\d{4}\b/g, severity: 'high' },
+  { name: 'credit_card', pattern: /\b(?:\d{4}[-\s]?){3}\d{4}\b/g, severity: 'high' },
+  { name: 'ip_address', pattern: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g, severity: 'low' },
+];

package/src/dlp/prompt-injection-backend.ts

@@ -0,0 +1,181 @@
+import { DLPBackend, DLPDetection } from './interfaces';
+import { DLPPattern } from './patterns';
+import { DLPSeverity } from '../types/tool-result';
+import { PROMPT_INJECTION_PATTERNS, OUTPUT_INJECTION_PATTERNS } from './prompt-injection-patterns';
+import { normalizeText, normalizeLeetspeak } from './text-normalizer';
+
+export interface PromptInjectionConfig {
+  /** Enable prompt injection detection. Default true. */
+  enabled?: boolean;
+  /** Additional custom patterns to scan alongside the built-in set. */
+  custom_patterns?: DLPPattern[];
+  /** Enable output-side pattern scanning. Default false. */
+  scan_output?: boolean;
+}
+
+/**
+ * DLP backend that detects common prompt injection patterns in text.
+ *
+ * Scans for 13 categories of prompt injection:
+ *   - Direct instruction overrides
+ *   - Role manipulation
+ *   - Jailbreak keywords
+ *   - System prompt extraction attempts
+ *   - Delimiter injection
+ *   - Context manipulation
+ *   - Authority impersonation
+ *   - Indirect injection
+ *   - Encoding indicators
+ *   - Synonym variants
+ *   - Prompt leakage requests
+ *   - Obfuscation markers
+ *   - Multi-turn manipulation
+ *
+ * All text is normalized before pattern matching to resist evasion via
+ * zero-width chars, HTML entities, homoglyphs, URL encoding, etc.
+ *
+ * Combination severity scoring: 3+ medium = effective high, 2+ high = effective critical.
+ */
+export class PromptInjectionBackend implements DLPBackend {
+  readonly name = 'prompt_injection';
+
+  private readonly enabled: boolean;
+  private readonly patterns: DLPPattern[];
+  private readonly outputPatterns: DLPPattern[];
+  private readonly scanOutput: boolean;
+
+  constructor(config?: PromptInjectionConfig) {
+    this.enabled = config?.enabled ?? true;
+    this.scanOutput = config?.scan_output ?? false;
+    this.patterns = [
+      ...PROMPT_INJECTION_PATTERNS,
+      ...(config?.custom_patterns ?? []),
+    ];
+    this.outputPatterns = OUTPUT_INJECTION_PATTERNS;
+  }
+
+  /**
+   * Scan a string for prompt injection patterns.
+   *
+   * Text is normalized before matching to defeat evasion techniques.
+   * Patterns are matched against both the standard-normalized text and
+   * the leetspeak-normalized variant. Positions in detections refer to
+   * the original (un-normalized) string for accurate redaction.
+   */
+  scanString(value: string): DLPDetection[] {
+    if (!this.enabled) {
+      return [];
+    }
+
+    const detections: DLPDetection[] = [];
+
+    // Normalize text for bypass resistance
+    const normalized = normalizeText(value);
+    const leetNormalized = normalizeLeetspeak(normalized);
+
+    // Choose pattern set: include output patterns when configured
+    const patternsToScan = this.scanOutput
+      ? [...this.patterns, ...this.outputPatterns]
+      : this.patterns;
+
+    // Scan standard-normalized text
+    this.matchPatterns(patternsToScan, normalized, detections);
+
+    // Scan leetspeak-normalized text (only if it differs from standard)
+    if (leetNormalized !== normalized) {
+      const existingNames = new Set(detections.map(d => d.pattern_name + ':' + d.match));
+      const leetDetections: DLPDetection[] = [];
+      this.matchPatterns(patternsToScan, leetNormalized, leetDetections);
+
+      // Deduplicate — only add leet detections not already found
+      for (const ld of leetDetections) {
+        const key = ld.pattern_name + ':' + ld.match;
+        if (!existingNames.has(key)) {
+          existingNames.add(key);
+          detections.push(ld);
+        }
+      }
+    }
+
+    // Apply combination severity scoring
+    this.applyCombinationScoring(detections);
+
+    return detections;
+  }
+
+  /**
+   * Scan output text specifically for output-side injection patterns.
+   * Called during postExecute DLP scanning.
+   */
+  scanOutputText(value: string): DLPDetection[] {
+    if (!this.enabled) {
+      return [];
+    }
+
+    const normalized = normalizeText(value);
+    const detections: DLPDetection[] = [];
+    this.matchPatterns(this.outputPatterns, normalized, detections);
+    return detections;
+  }
+
+  /**
+   * Run all patterns against a text value and append matches to the detections array.
+   */
+  private matchPatterns(
+    patterns: DLPPattern[],
+    text: string,
+    detections: DLPDetection[],
+  ): void {
+    for (const pat of patterns) {
+      pat.pattern.lastIndex = 0;
+      let m: RegExpExecArray | null;
+      while ((m = pat.pattern.exec(text)) !== null) {
+        detections.push({
+          pattern_name: pat.name,
+          severity: pat.severity,
+          match: m[0],
+          start: m.index,
+          end: m.index + m[0].length,
+        });
+        // Prevent infinite loops on zero-length matches
+        if (m[0].length === 0) {
+          pat.pattern.lastIndex++;
+        }
+      }
+      pat.pattern.lastIndex = 0;
+    }
+  }
+
+  /**
+   * Apply combination severity escalation:
+   * - 3+ medium detections -> add effective_severity 'high' metadata
+   * - 2+ high detections -> add effective_severity 'critical' metadata
+   *
+   * Mutates detections in place by upgrading severity where applicable.
+   */
+  private applyCombinationScoring(detections: DLPDetection[]): void {
+    if (detections.length < 2) return;
+
+    const mediumCount = detections.filter(d => d.severity === 'medium').length;
+    const highCount = detections.filter(d => d.severity === 'high').length;
+
+    // 3+ medium -> escalate all mediums to high
+    if (mediumCount >= 3) {
+      for (const d of detections) {
+        if (d.severity === 'medium') {
+          (d as any).effective_severity = 'high';
+          d.severity = 'high' as DLPSeverity;
+        }
+      }
+    }
+
+    // 2+ high -> mark effective severity as critical on all highs
+    if (highCount >= 2) {
+      for (const d of detections) {
+        if (d.severity === 'high') {
+          (d as any).effective_severity = 'critical';
+        }
+      }
+    }
+  }
+}