palaryn 0.5.11 → 0.6.0

package/src/policy/engine.ts

@@ -1,6 +1,9 @@
 import * as fs from 'fs';
 import * as net from 'net';
 import * as yaml from 'js-yaml';
+import { logger } from '../server/logger';
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const punycode = require('punycode/');
 import { ToolCall } from '../types/tool-call';
 import {
   PolicyDecision,
@@ -353,13 +356,13 @@ export class PolicyEngine {
     }
     // ReDoS protection: block patterns with nested quantifiers or excessive length
     if (!PolicyEngine.isSafeRegex(pattern)) {
-      console.warn(`PolicyEngine: rejecting potentially unsafe regex pattern (ReDoS risk): "${pattern.slice(0, 100)}..."`);
+      logger.warn('Rejecting potentially unsafe regex pattern (ReDoS risk)', { component: 'policy-engine', pattern: pattern.slice(0, 100) });
       return null;
     }
     try {
       re = new RegExp(pattern);
     } catch (err) {
-      console.warn(`PolicyEngine: invalid regex pattern "${pattern}": ${err instanceof Error ? err.message : String(err)}`);
+      logger.warn('Invalid regex pattern', { component: 'policy-engine', pattern, error: err instanceof Error ? err.message : String(err) });
       return null;
     }
     // Evict oldest entry if at capacity
@@ -775,6 +778,36 @@ export class PolicyEngine {
       if (!conditions.provider_tool_types.includes(toolType as string)) return false;
     }
 
+    // --- Referrer/provenance conditions ---
+    if (conditions.has_referrer !== undefined) {
+      const hasReferrer = !!toolCall.context?.referrer_url;
+      if (conditions.has_referrer !== hasReferrer) {
+        return false;
+      }
+    }
+
+    if (conditions.referrer_domains && conditions.referrer_domains.length > 0) {
+      const referrerUrl = toolCall.context?.referrer_url;
+      if (!referrerUrl) {
+        // Rule requires specific referrer domain but no referrer is present — no match.
+        return false;
+      }
+      const referrerDomain = this.extractDomain(referrerUrl);
+      if (!referrerDomain || !this.isDomainInList(referrerDomain, conditions.referrer_domains)) {
+        return false;
+      }
+    }
+
+    if (conditions.referrer_domains_blocklist && conditions.referrer_domains_blocklist.length > 0) {
+      const referrerUrl = toolCall.context?.referrer_url;
+      if (referrerUrl) {
+        const referrerDomain = this.extractDomain(referrerUrl);
+        if (referrerDomain && this.isDomainInList(referrerDomain, conditions.referrer_domains_blocklist)) {
+          return false;
+        }
+      }
+    }
+
     // --- DLP conditions (only evaluated when dlpContext is provided) ---
     if (conditions.dlp_detected !== undefined) {
       if (!dlpContext) return false; // DLP conditions require DLP context
@@ -812,7 +845,10 @@ export class PolicyEngine {
   private extractDomain(url: string): string | null {
     try {
       const parsed = new URL(url);
-      return parsed.hostname.toLowerCase();
+      let hostname = parsed.hostname.toLowerCase();
+      // Decode Punycode (xn--) domains to Unicode for consistent matching
+      try { hostname = punycode.toUnicode(hostname); } catch { /* leave as-is */ }
+      return hostname;
     } catch {
       return null;
     }

package/src/policy/opa-engine.ts

@@ -3,6 +3,7 @@ import * as https from 'https';
 import { URL } from 'url';
 import { ToolCall } from '../types/tool-call';
 import { PolicyDecision, PolicyEvalResult, PolicyTransformation } from '../types/policy';
+import { logger } from '../server/logger';
 import { OPAConfig } from '../types/config';
 
 /**
@@ -79,7 +80,7 @@ export class OPAEngine {
             throw new Error(`OPA server URL must not use private/reserved IP address in production: "${hostname}"`);
           }
           // In development, allow but warn
-          console.warn(`[OPAEngine] WARNING: OPA server URL points to private address "${hostname}". This would be blocked in production.`);
+          logger.warn('OPA server URL points to private address — blocked in production', { component: 'opa-engine', hostname });
         }
 
         // Normalize: remove trailing slashes for consistency

package/src/server/app.ts

@@ -43,7 +43,7 @@ import { StripeClient, WebhookHandler, createWebhookRouter, createBillingRouter,
 import { createPlanEnforcerMiddleware } from '../billing/plan-enforcer';
 import { FilePersistedStores } from '../storage/file-persistence';
 import { hashPassword } from '../auth/password';
-import { log as devLog } from './logger';
+import { log as devLog, logger } from './logger';
 
 const MAX_TRACKED_IPS = 10000;
 
@@ -262,6 +262,12 @@ export function createApp(config: GatewayConfig, externalSaaSStores?: Partial<Sa
     gateway.setStores({ policyStore: saasStores.policyStore });
   }
 
+  // Inject workspace store for plan enforcement inside the gateway pipeline
+  // (covers MCP bridge and proxy entry points, not just Express middleware)
+  if (saasStores.workspaceStore) {
+    gateway.setStores({ workspaceStore: saasStores.workspaceStore });
+  }
+
   // Register noop executors for non-HTTP tools (e.g. pre-flight validation from Android)
   gateway.registerExecutor('web_search', new NoopExecutor({ body: { validated: true, tool: 'web_search' } }));
   gateway.registerExecutor('chat.completion', new NoopExecutor({ body: { validated: true, tool: 'chat.completion' } }));
@@ -500,10 +506,13 @@ export function createApp(config: GatewayConfig, externalSaaSStores?: Partial<Sa
   const rbacPolicyWrite = createRBACMiddleware(config.auth, 'policy:write');
 
   // Admin routes (require auth + admin:full permission)
-  // Auth + RBAC middleware mounted directly with the router to prevent route decoupling
-  const adminRouter = createAdminRouter(gateway, config);
+  // Deprecated: use React SPA frontend instead (FRONTEND_ENABLED=true).
+  // Server-rendered admin is kept as fallback when frontend is disabled.
   const rbacAdmin = createRBACMiddleware(config.auth, 'admin:full');
-  app.use('/admin', authMiddleware, rbacAdmin, adminRouter);
+  if (!config.frontend?.enabled) {
+    const adminRouter = createAdminRouter(gateway, config);
+    app.use('/admin', authMiddleware, rbacAdmin, adminRouter);
+  }
 
   // Plan enforcer middleware — blocks calls when workspace exceeds plan limits
   const planEnforcerMiddleware = createPlanEnforcerMiddleware({
@@ -568,13 +577,13 @@ export function createApp(config: GatewayConfig, externalSaaSStores?: Partial<Sa
       }
       // S3: Never leak raw error messages to clients (may contain internal paths, IPs, secrets)
       if (result.error) {
-        console.error('[tool-execute] Gateway error:', result.error);
+        logger.error('Gateway error', { component: 'tool-execute', error: result.error });
         result.error = 'Tool execution failed';
       }
       res.status(httpStatus).json(result);
     } catch (err) {
       const errorMessage = err instanceof Error ? err.message : 'Unknown error';
-      console.error('[tool-execute] Execution error:', errorMessage);
+      logger.error('Execution error', { component: 'tool-execute', error: errorMessage });
       sendError(res, 500, ErrorCode.TOOL_EXECUTION_ERROR, 'Tool execution failed');
     }
   });
@@ -635,13 +644,13 @@ export function createApp(config: GatewayConfig, externalSaaSStores?: Partial<Sa
             headers: proxyResult.headers,
           }, pre);
         } catch (postErr) {
-          console.error('[post-execute] Failed:', postErr instanceof Error ? postErr.message : postErr);
+          logger.error('Post-execute failed', { component: 'stream-proxy', error: postErr instanceof Error ? postErr.message : postErr });
           // Don't fail the response — it's already sent for streaming
         }
 
       } catch (err) {
         const errorMessage = err instanceof Error ? err.message : 'Unknown error';
-        console.error('[stream-proxy] Execution error:', errorMessage);
+        logger.error('Execution error', { component: 'stream-proxy', error: errorMessage });
         if (!res.headersSent) {
           sendError(res, 502, ErrorCode.TOOL_EXECUTION_ERROR, 'Stream proxy failed');
         } else {
@@ -684,7 +693,7 @@ export function createApp(config: GatewayConfig, externalSaaSStores?: Partial<Sa
       }
     } catch (err) {
       const errorMessage = err instanceof Error ? err.message : 'Unknown error';
-      console.error('[tool-approve] Error:', errorMessage);
+      logger.error('Approval error', { component: 'tool-approve', error: errorMessage });
       sendError(res, 500, ErrorCode.INTERNAL_ERROR, 'Approval processing failed');
     }
   });
@@ -751,7 +760,7 @@ export function createApp(config: GatewayConfig, externalSaaSStores?: Partial<Sa
       res.json({ status: 'ok' });
     } catch (err) {
       const errorMessage = err instanceof Error ? err.message : 'Unknown error';
-      console.error('[usage-report] Error:', errorMessage);
+      logger.error('Usage report error', { component: 'usage-report', error: errorMessage });
       sendError(res, 500, ErrorCode.INTERNAL_ERROR, 'Usage reporting failed');
     }
   });

package/src/server/gateway.ts

@@ -14,6 +14,9 @@ import { HeuristicScorerBackend } from '../dlp/heuristic-scorer';
 import { scorePromptInjection } from '../dlp/heuristic-scorer';
 import { TruffleHogBackend } from '../dlp/trufflehog-backend';
 import { ExfiltrationDetectionBackend } from '../dlp/exfiltration-backend';
+import { NavigationInstructionBackend } from '../dlp/navigation-instruction-backend';
+import { SensitiveFileBackend } from '../dlp/tool-patterns';
+import { extractMultipartTextParts } from '../dlp/multipart-extractor';
 import { NemoGuardrailsBackend } from '../dlp/nemo-backend';
 import { DeBERTaBackend } from '../dlp/deberta-backend';
 import { BudgetManager, CostRecord } from '../budget/manager';
@@ -37,6 +40,9 @@ import { GatewayMetrics } from '../metrics';
 import { GatewayTracer } from '../tracing';
 import { AnomalyDetector } from '../anomaly';
 import { LlmPromptInjectionClassifier } from '../dlp/llm-classifier';
+import { PlanEnforcer } from '../billing/plan-enforcer';
+import { WorkspaceStore } from '../storage/interfaces';
+import { PlanTier } from '../types/subscription';
 import { log as devLog, logger } from './logger';
 
 export interface PreExecuteResult {
@@ -169,6 +175,7 @@ export class Gateway {
    * the in-flight result instead of executing a second time.
    */
   private inFlightCalls = new Map<string, InFlightEntry>();
+  private workspaceStore?: WorkspaceStore;
 
   constructor(config: GatewayConfig, metrics?: GatewayMetrics, tracer?: GatewayTracer) {
     this.config = config;
@@ -183,6 +190,8 @@ export class Gateway {
       }));
       dlpBackends.push(new HeuristicScorerBackend());
       dlpBackends.push(new ExfiltrationDetectionBackend());
+      dlpBackends.push(new NavigationInstructionBackend());
+      dlpBackends.push(new SensitiveFileBackend());
     }
     if (config.dlp.deberta?.enabled) {
       dlpBackends.push(new DeBERTaBackend({
@@ -263,7 +272,7 @@ export class Gateway {
     // A1: Periodic cleanup of stale inFlightCalls entries (every 60s, remove entries older than 5min)
     this.inFlightCleanupInterval = setInterval(() => {
       const now = Date.now();
-      const MAX_AGE_MS = 5 * 60 * 1000;
+      const MAX_AGE_MS = this.config.idempotency_cache_ttl_ms ?? 300_000;
       for (const [key, entry] of this.inFlightCalls) {
         if (now - entry.createdAt > MAX_AGE_MS) {
           clearTimeout(entry.timeout);
@@ -368,6 +377,38 @@ export class Gateway {
     }
     devLog.anomaly(anomalyAlerts.length, false);
 
+    // Plan enforcement — block calls when workspace exceeds plan limits.
+    // Covers ALL entry points (REST, MCP bridge, MCP HTTP, proxy).
+    if (this.workspaceStore && toolCall.workspace_id) {
+      stepStart = Date.now();
+      const workspace = this.workspaceStore.getById(toolCall.workspace_id);
+      if (workspace) {
+        const plan = (workspace.plan || 'free') as PlanTier;
+        const now = new Date();
+        const monthStart = new Date(now.getFullYear(), now.getMonth(), 1).toISOString();
+        const events = this.auditLogger.getAllEvents();
+        const monthlyCount = events.filter(
+          e => e.workspace_id === toolCall.workspace_id
+            && e.event_type === 'TOOL_CALL_RECEIVED'
+            && e.timestamp >= monthStart
+        ).length;
+        const planResult = PlanEnforcer.enforce(plan, monthlyCount);
+        if (!planResult.allowed) {
+          const durationSec = (Date.now() - startTime) / 1000;
+          this.metrics?.recordRequest('blocked', toolCall.tool.name, toolCall.tool.capability, durationSec);
+          const result = this.buildResult(toolCall, 'blocked', {
+            decision: 'deny',
+            rule_id: 'plan_limit',
+            rule_name: 'Plan limit',
+            reasons: [planResult.reason || 'Plan limit exceeded'],
+          }, startTime, undefined, planResult.reason);
+          devLog.pipelineEnd('blocked', Date.now() - startTime);
+          return { allowed: false, result, stepTimings, startTime };
+        }
+      }
+      stepTimings.plan_enforcement = Date.now() - stepStart;
+    }
+
     // DLP scan on full toolCall (args + context + actor fields)
     // Runs BEFORE policy evaluation so secrets are always detected regardless of policy outcome
     stepStart = Date.now();
@@ -628,6 +669,8 @@ export class Gateway {
     stepTimings.policy = Date.now() - stepStart;
     this.auditLogger.logPolicyDecided(toolCall, policyResult.decision, policyResult.rule_id, policyResult.reasons);
     this.metrics?.recordPolicyDecision(policyResult.decision, policyResult.rule_id || 'unknown');
+    const policyEngine_ = usedWorkspacePolicy ? 'workspace' : (this.opaEngine ? 'opa' : 'yaml');
+    this.metrics?.recordPolicyEvaluation(policyEngine_, stepTimings.policy / 1000);
     devLog.policy(policyResult.decision, policyResult.rule_id, policyResult.reasons);
 
     // Policy: deny
@@ -741,12 +784,51 @@ export class Gateway {
     this.anomalyDetector?.recordResult(toolCall, executionDuration, false);
     this.anomalyDetector?.analyzeResult(toolCall, executionDuration, false);
 
+    // Content-type inspection: determine if output can be meaningfully text-scanned
+    const responseContentType = (output.headers?.['content-type'] || '').toLowerCase();
+    const OPAQUE_PREFIXES = ['image/', 'audio/', 'video/', 'application/pdf', 'application/zip',
+      'application/gzip', 'application/octet-stream', 'application/wasm', 'font/'];
+    const isOpaqueContent = OPAQUE_PREFIXES.some(p => responseContentType.startsWith(p))
+      && !responseContentType.includes('image/svg+xml'); // SVGs are text-based XML
+
+    // For multipart responses, extract text parts for scanning
+    let multipartTextParts: string[] | undefined;
+    if (responseContentType.includes('multipart/') && typeof output.body === 'string') {
+      const boundaryMatch = responseContentType.match(/boundary=([^\s;]+)/);
+      if (boundaryMatch) {
+        multipartTextParts = extractMultipartTextParts(output.body, boundaryMatch[1]);
+      }
+    }
+
     // DLP scan on output
     let stepStart = Date.now();
     let outputDlp: { detected: string[]; redactions: any[]; severity: DLPSeverity } = { detected: [], redactions: [], severity: 'low' };
     if (this.config.dlp.scan_output && output.body) {
       outputDlp = childSpanWithAttrs(otel, SPAN.DLP_OUTPUT, (s) => {
-        const result = this.dlpScanner.scan(output, 'output');
+        let result;
+        if (isOpaqueContent) {
+          // Binary content cannot be text-scanned — flag it as unscannable
+          result = {
+            detected: ['opaque_content_type_unscannable'],
+            redactions: [],
+            severity: 'medium' as DLPSeverity,
+          };
+          devLog.pipelineStep('⚠️', 'DLP_OPAQUE_CONTENT',
+            `Response content-type ${responseContentType} cannot be text-scanned`);
+        } else if (multipartTextParts && multipartTextParts.length > 0) {
+          // Scan extracted text parts from multipart response
+          result = this.dlpScanner.scan(output, 'output');
+          for (const part of multipartTextParts) {
+            const partReport = this.dlpScanner.scan({ content: part }, 'output.multipart');
+            for (const det of partReport.detected) {
+              if (!result.detected.includes(det)) result.detected.push(det);
+            }
+            result.redactions.push(...partReport.redactions);
+            result.severity = this.maxSeverity(result.severity, partReport.severity);
+          }
+        } else {
+          result = this.dlpScanner.scan(output, 'output');
+        }
         s.setAttribute(ATTR.DLP_DETECTION_COUNT, result.detected.length);
         s.setAttribute(ATTR.DLP_SEVERITY, result.severity);
         if (result.detected.length > 0) s.setAttribute(ATTR.DLP_TYPES, result.detected.join(','));
@@ -847,6 +929,70 @@ export class Gateway {
       }
     }
 
+    // Navigation instruction handling (runs after prompt injection output check)
+    const navAction = this.config.dlp.navigation_instruction_action || 'log';
+    if (navAction !== 'log' && outputDlp.detected.length > 0) {
+      const navDetections = outputDlp.detected.filter((d: string) =>
+        d.startsWith('navigation_instruction_')
+      );
+      if (navDetections.length > 0) {
+        if (navAction === 'block') {
+          // Release budget reservation since we're blocking
+          if (reservationKey) {
+            this.budgetManager.commitReservation(reservationKey, 0);
+          }
+          devLog.pipelineStep('🧭', 'NAVIGATION_INSTRUCTION_BLOCK',
+            `Blocked output: ${navDetections.join(', ')}`);
+          const durationSec = (Date.now() - startTime) / 1000;
+          this.metrics?.recordRequest('blocked', toolCall.tool.name, toolCall.tool.capability, durationSec);
+          const defaultPolicy = policyResult || { decision: 'allow' as const, rule_id: 'passthrough', rule_name: 'Passthrough', reasons: [] };
+          const argsDlpSafe = argsDlp || { detected: [], redactions: [], severity: 'low' as DLPSeverity };
+          const mergedDlp = {
+            detected: [...new Set([...argsDlpSafe.detected, ...outputDlp.detected])],
+            redactions: [...argsDlpSafe.redactions, ...outputDlp.redactions],
+            severity: this.maxSeverity(argsDlpSafe.severity, outputDlp.severity),
+          };
+          const result = this.buildResult(toolCall, 'blocked', {
+            decision: 'deny',
+            rule_id: 'navigation_instruction_output_block',
+            rule_name: 'Navigation instructions detected in output',
+            reasons: [`Navigation instructions detected in output: ${navDetections.join(', ')}`],
+          }, startTime, undefined,
+            `Blocked by navigation instruction detection in output: ${navDetections.join(', ')}`,
+            undefined, mergedDlp);
+          this.auditLogger.logToolResultReturned(toolCall, 'blocked', Date.now() - startTime, { navigation_instruction_blocked: true, detections: navDetections });
+          devLog.pipelineEnd('blocked', Date.now() - startTime);
+          return result;
+        } else if (navAction === 'strip' && output.body) {
+          // Re-scan the body string with NavigationInstructionBackend to get
+          // actual match positions (the DLPReport only carries pattern names).
+          let bodyStr = typeof output.body === 'string' ? output.body : JSON.stringify(output.body);
+          const navBackend = new NavigationInstructionBackend();
+          const matches = navBackend.scanString(bodyStr);
+
+          if (matches.length > 0) {
+            // Sort by start position descending so splicing from the end
+            // doesn't invalidate earlier indices.
+            matches.sort((a, b) => b.start - a.start);
+            for (const m of matches) {
+              const before = bodyStr.slice(0, m.start);
+              const after = bodyStr.slice(m.end);
+              bodyStr = before + '[NAVIGATION_INSTRUCTION_REDACTED]' + after;
+            }
+          }
+
+          // Re-parse if original was JSON
+          if (typeof output.body !== 'string') {
+            try { output.body = JSON.parse(bodyStr); } catch { output.body = bodyStr; }
+          } else {
+            output.body = bodyStr;
+          }
+          devLog.pipelineStep('🧭', 'NAVIGATION_INSTRUCTION_STRIP',
+            `Stripped ${matches.length} navigation instruction(s) from output`);
+        }
+      }
+    }
+
     // Extract usage data from response
     const headerUsage = this.usageExtractor.extractFromHeaders(output.headers);
     const bodyUsage = this.usageExtractor.extractFromBody(output.body);
@@ -1084,8 +1230,8 @@ export class Gateway {
       // Run post-execution pipeline (DLP output, usage, budget recording, metrics, audit)
       const result = await this.postExecute(toolCall, output, pre, otel);
 
-      // Cache result for idempotency (5 minute TTL) using cache key with body hash
-      this.idempotencyStore.set(cacheKey, result, 300000);
+      // Cache result for idempotency using cache key with body hash
+      this.idempotencyStore.set(cacheKey, result, this.config.idempotency_cache_ttl_ms ?? 300_000);
       if (this.idempotencyStore.flush) await this.idempotencyStore.flush();
 
       return result;
@@ -1260,7 +1406,11 @@ export class Gateway {
     policyStore?: PolicyStore;
     rateLimitConfigStore?: RateLimitConfigStore;
     budgetConfigStore?: BudgetConfigStore;
+    workspaceStore?: WorkspaceStore;
   }): void {
+    if (stores.workspaceStore) {
+      this.workspaceStore = stores.workspaceStore;
+    }
     if (stores.idempotencyStore) {
       this.idempotencyStore = stores.idempotencyStore;
     }
@@ -1305,6 +1455,7 @@ export class Gateway {
   getPolicyStore(): PolicyStore | undefined { return this.policyStore; }
   getRateLimitConfigStore(): RateLimitConfigStore | undefined { return this.rateLimitConfigStore; }
   getBudgetConfigStore(): BudgetConfigStore | undefined { return this.budgetConfigStore; }
+  getWorkspaceStore(): WorkspaceStore | undefined { return this.workspaceStore; }
 
   /** Return the workspace-specific policy if one exists, otherwise the global policy. */
   getWorkspacePolicy(workspaceId: string): { policy: PolicyPack; is_custom: boolean } {

package/src/types/config.ts

@@ -37,6 +37,8 @@ export interface GatewayConfig {
   mcp_oauth?: MCPOAuthConfig;
   /** Webhook alerting for governance events (DLP, budget, anomaly, policy denials) */
   alerting?: AlertingConfig;
+  /** Idempotency cache TTL in milliseconds. Default: 300000 (5 minutes). */
+  idempotency_cache_ttl_ms?: number;
 }
 
 export interface MCPOAuthConfig {
@@ -190,6 +192,13 @@ export interface DLPConfig {
     api_url: string;
     timeout_ms?: number;
   };
+  /**
+   * Action when navigation instructions are detected in output responses.
+   *   - 'log'   (default): detect and log, pass response through unchanged
+   *   - 'strip': redact detected navigation instructions from the response body
+   *   - 'block': block the entire response if navigation instructions are found
+   */
+  navigation_instruction_action?: 'log' | 'strip' | 'block';
 }
 
 export interface AuditConfig {

package/src/types/policy.ts

@@ -57,6 +57,11 @@ export interface PolicyConditions {
   dlp_severity?: string[];      // e.g., ['high', 'critical'] — match if DLP max severity is in this list
   dlp_detected?: boolean;       // true = rule matches only when DLP detections exist
   dlp_pattern_names?: string[]; // match if any of these pattern names were detected
+
+  /** Referrer/provenance conditions — match based on context.referrer_url */
+  has_referrer?: boolean;              // true = match only when referrer_url is present; false = only when absent
+  referrer_domains?: string[];         // allowlist: match only if referrer URL domain is in this list (supports glob, e.g. "*.github.com")
+  referrer_domains_blocklist?: string[]; // blocklist: match if referrer URL domain is in this list
 }
 
 export interface PolicyPack {

package/src/types/tool-call.ts

@@ -31,6 +31,10 @@ export interface Constraints {
 export interface CallContext {
   purpose?: string;
   labels?: string[];
+  /** URL that triggered this request (set by agent/SDK when following a link from a previous response). */
+  referrer_url?: string;
+  /** Tool call ID that produced the response containing the referrer URL. */
+  referrer_tool_call_id?: string;
 }
 
 export interface ToolCall {