@st-gr/sail-proxy 0.9.2 → 0.9.4

package/bundled/gateway/services/gateway/src/controllers/awsBedrockController.js

@@ -13,11 +13,15 @@ const pluginExecutor_1 = require("../services/pluginExecutor");
 const logger_1 = require("@libs/logger");
 const logger = (0, logger_1.getDefaultLogger)();
 const usageTracker_1 = require("../utils/usageTracker");
+const payloadLogger_1 = require("../utils/payloadLogger");
 /**
  * Handle AWS Bedrock requests for all model operations
  */
 const handleBedrockRequest = async (req, res, _next) => {
-    const debugRequestId = process.env.DEBUG === 'true' ? (0, uuid_1.v4)() : undefined;
+    // Generate a debug request ID whenever payload logging can occur — via the
+    // legacy DEBUG env or the dynamic config switch (hot-reloadable per request).
+    const payloadLoggingActive = process.env.DEBUG === 'true' || (0, payloadLogger_1.isPayloadLoggingEnabled)();
+    const debugRequestId = payloadLoggingActive ? (req.debugRequestId || (0, uuid_1.v4)()) : undefined;
     req.debugRequestId = debugRequestId;
     logger.info('AwsBedrockController', `Processing request for model: ${req.params.modelId}, subpath: ${req.params.subpath}, Debug ID: ${debugRequestId}`);
     // Initialize usage tracking

package/bundled/gateway/services/gateway/src/index.js

@@ -379,6 +379,20 @@ initializeGatewayService().then(() => {
         logger.info('Gateway Service', '');
         logger.info('Gateway Service', '💡 Press Ctrl+C to gracefully shutdown the service');
     });
+    // Fail loudly if the port is taken: otherwise a stale gateway instance keeps serving
+    // requests while this (newer) process dies quietly — old code masquerades as the
+    // current build, which makes debugging almost impossible.
+    server.on('error', (error) => {
+        if (error.code === 'EADDRINUSE') {
+            logger.error('Gateway Service', `FATAL: ${config.host}:${config.port} is already in use — ANOTHER gateway instance is running. ` +
+                `Requests are being served by THAT process (possibly stale code), not this one. ` +
+                `Find it with: lsof -nP -iTCP:${config.port} -sTCP:LISTEN`);
+        }
+        else {
+            logger.error('Gateway Service', 'HTTP server error:', error instanceof Error ? error : new Error(String(error)));
+        }
+        process.exit(1);
+    });
     // Store server reference for cleanup
     cleanupResources.server = server;
 }).catch((error) => {

package/bundled/gateway/services/gateway/src/plugins/pseudonymization/detectors/index.js

@@ -12,6 +12,23 @@ const customDetector_1 = require("./customDetector");
 const regexDetectors_1 = require("./regexDetectors");
 const nerDetector_1 = require("./nerDetector");
 const dictionaryDetector_1 = require("./dictionaryDetector");
+/**
+ * Matches placeholders that ALREADY exist in the text so they are never re-masked.
+ * Covers the `MASKED_<TYPE>_<id>` family and the URL-shaped `masked-url-<id>.invalid`
+ * form. Re-masking a placeholder produces layered tokens ("keeps getting re-masked")
+ * that can never be unmasked back to the original value.
+ */
+const EXISTING_PLACEHOLDER = /MASKED_[A-Z_]+_[0-9a-f]+|(?:(?:https?|wss?|ftps?):\/\/)?masked-url-\d+\.invalid/g;
+/** Spans in `text` occupied by placeholders that must be excluded from detection. */
+function placeholderSpans(text) {
+    const spans = [];
+    EXISTING_PLACEHOLDER.lastIndex = 0;
+    let m;
+    while ((m = EXISTING_PLACEHOLDER.exec(text)) !== null) {
+        spans.push({ start: m.index, end: m.index + m[0].length });
+    }
+    return spans;
+}
 /**
  * Check if a match is in the allow list
  */
@@ -65,8 +82,13 @@ function detectEntities(text, config) {
     allMatches.push(...(0, nerDetector_1.detectNerEntities)(text, config.entities));
     // Tier 3: Dictionary
     allMatches.push(...(0, dictionaryDetector_1.detectDictionaryEntities)(text, config.entities));
+    // Never re-mask an existing placeholder: drop any match overlapping one.
+    const reserved = placeholderSpans(text);
+    const notPlaceholder = reserved.length === 0
+        ? allMatches
+        : allMatches.filter(m => !reserved.some(s => m.start < s.end && m.end > s.start));
     // Filter allow-list
-    const filtered = allMatches.filter(m => !isAllowed(m.original, config.allow_list));
+    const filtered = notPlaceholder.filter(m => !isAllowed(m.original, config.allow_list));
     // Resolve overlaps
     return resolveOverlaps(filtered);
 }

package/bundled/gateway/services/gateway/src/plugins/pseudonymization/detectors/regexDetectors.js

@@ -8,6 +8,51 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.detectRegexEntities = detectRegexEntities;
+/**
+ * Decide whether a URL is worth masking. We SKIP:
+ *  - loopback / private / link-local hosts (localhost, 127.x, ::1, 10.x,
+ *    192.168.x, 172.16-31.x, *.local, *.internal) — these are dev/infra
+ *    endpoints with ~no privacy value, and masking them breaks agent edit
+ *    workflows (the model must reproduce an opaque token to match file text);
+ *  - template / non-literal URLs containing <...>, {...}, ${...} or an
+ *    unresolved-looking host — masking these fragments one logical endpoint
+ *    into many bogus tokens.
+ * Real external hosts still get masked.
+ */
+function isMaskableUrl(fullMatch) {
+    // Template markers anywhere in the matched URL → not a real value.
+    if (/[<>{}]|\$\{/.test(fullMatch))
+        return false;
+    // Trailing colon = incomplete authority, i.e. the port was a template variable
+    // (`http://host:{PORT}` matches only up to `http://host:` since `{` is excluded).
+    if (/:$/.test(fullMatch))
+        return false;
+    const m = fullMatch.match(/^(?:(?:https?|wss?|ftps?):\/\/)?([^/?#]+)/);
+    const authority = (m ? m[1] : fullMatch).toLowerCase();
+    // Authority must be a well-formed host[:port] (or [ipv6][:port]). Anything else —
+    // e.g. a leftover `http://host:$` from `${PORT}`, or a bare scheme — is a template
+    // fragment, not a real endpoint.
+    if (!/^(?:\[[0-9a-f:]+\]|[a-z0-9._-]+)(?::\d+)?$/.test(authority))
+        return false;
+    const host = authority.replace(/:\d*$/, '').replace(/^\[|\]$/g, '');
+    if (host === '' || host === 'localhost' || host.endsWith('.local') || host.endsWith('.internal'))
+        return false;
+    if (host === '::1' || host.startsWith('127.') || host.startsWith('0.'))
+        return false;
+    if (host.startsWith('10.') || host.startsWith('192.168.'))
+        return false;
+    // 172.16.0.0 – 172.31.255.255
+    const m172 = host.match(/^172\.(\d{1,3})\./);
+    if (m172) {
+        const o = Number(m172[1]);
+        if (o >= 16 && o <= 31)
+            return false;
+    }
+    // 169.254.0.0/16 link-local
+    if (host.startsWith('169.254.'))
+        return false;
+    return true;
+}
 // Luhn algorithm for credit card validation
 function luhnCheck(digits) {
     const nums = digits.split('').map(Number);
@@ -54,6 +99,13 @@ function phoneValidate(match) {
     const digits = match.replace(/\D/g, '');
     return digits.length >= 7 && digits.length <= 15;
 }
+// Credential value validation: reject template placeholders (`<token>`, `${VAR}`,
+// `$VAR`) and values too short to be real secrets. Applied to the captured VALUE.
+function credentialValueValidate(value) {
+    if (/[<>{}]|\$\{|^\$/.test(value))
+        return false;
+    return value.length >= 4;
+}
 const DETECTORS = [
     {
         type: 'profile-email',
@@ -88,7 +140,7 @@ const DETECTORS = [
     },
     {
         type: 'profile-url',
-        pattern: /(?:https?:\/\/|www\.)[^\s<>"{}|\\^`\[\]]+/g,
+        pattern: /(?:(?:https?|wss?|ftps?):\/\/|www\.)[^\s<>"{}|\\^`\[\]]+/g,
     },
     {
         type: 'profile-nationalid',
@@ -104,20 +156,55 @@ const DETECTORS = [
         type: 'profile-nationalid',
         // Generic context-anchored
         pattern: /(?:national\s*id|identification\s*(?:number|no\.?))\s*[:=]?\s*([A-Z0-9\-]{6,20})/gi,
+        captureValue: true,
     },
     {
         type: 'profile-passport',
         // Context-anchored
         pattern: /(?:passport\s*(?:number|no\.?|#))\s*[:=]?\s*([A-Z]{0,2}\d{6,9})/gi,
+        captureValue: true,
     },
     {
         type: 'profile-driverlicense',
         // Context-anchored generic
         pattern: /(?:driver'?s?\s*licen[sc]e|DL)\s*(?:number|no\.?|#)?\s*[:=]?\s*([A-Z0-9\-]{5,15})/gi,
+        captureValue: true,
     },
     {
         type: 'profile-username-password',
-        pattern: /(?:user(?:name)?|login|pass(?:word|wd)?|pwd|secret|token|api[_\-]?key)\s*[:=]\s*\S+/gi,
+        // key=value / key: value credentials. The leading (?<![A-Za-z0-9_]) + greedy
+        // identifier prefix make the match consume the WHOLE identifier ending in a
+        // credential word — so `$env:SSHPASS="x"` masks only `x` (not `SSHMASKED_…`),
+        // and `SSH_ASKPASS_REQUIRE=force` does not match (identifier ends in REQUIRE).
+        // Only the value is captured, preserving surrounding script structure.
+        pattern: /(?<![A-Za-z0-9_])[A-Za-z0-9_$-]*(?:user(?:name)?|login|pass(?:word|wd)?|pwd|secret|token|api[_\-]?key)\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|(\S+))/gi,
+        validate: credentialValueValidate,
+        captureValue: true,
+    },
+    {
+        type: 'profile-username-password',
+        // JSON-style credentials: "token": "value" — the quoted key defeats the pattern
+        // above. Capture ONLY the value so masking preserves the JSON structure.
+        pattern: /"(?:user(?:name)?|login|pass(?:word|wd)?|pwd|secret|token|api[_\-]?key|access[_\-]?token|auth[_\-]?token|bearer)"\s*:\s*"([^"]+)"/gi,
+        validate: credentialValueValidate,
+        captureValue: true,
+    },
+    {
+        type: 'profile-username-password',
+        // HTTP auth headers: `Authorization: Bearer <token>` and siblings. This is the
+        // gap behind the 28551619 incident — the secret rode through in four Bearer
+        // headers unmasked. Capture the opaque credential value only.
+        pattern: /(?:Authorization|Proxy-Authorization|X-Auth-Token|X-Api-Key)["']?\s*[:=]\s*["']?(?:(?:Bearer|Basic|Token|OAuth)\s+)?([A-Za-z0-9][A-Za-z0-9._~+\/=-]{7,})/gi,
+        validate: credentialValueValidate,
+        captureValue: true,
+    },
+    {
+        type: 'profile-username-password',
+        // Standalone `Bearer <opaque-token>` not preceded by a header name. Min length
+        // 12 keeps ordinary prose words after "Bearer" from matching.
+        pattern: /\bBearer\s+([A-Za-z0-9][A-Za-z0-9._~+\/=-]{11,})/g,
+        validate: credentialValueValidate,
+        captureValue: true,
     },
     {
         type: 'profile-address',
@@ -128,6 +215,7 @@ const DETECTORS = [
         type: 'profile-pronouns-gender',
         // Context-anchored pronoun declarations
         pattern: /(?:pronouns?|goes\s+by)\s*[:=]?\s*((?:he|she|they|ze|xe|ey|fae)(?:\s*\/\s*(?:him|her|them|zir|xem|em|faer))+)/gi,
+        captureValue: true,
     },
 ];
 /**
@@ -142,16 +230,67 @@ function detectRegexEntities(text, enabledEntities) {
         if (!enabledTypes.has(detector.type) && !sensitiveDataEnabled) {
             continue;
         }
-        // Reset lastIndex for global regex
-        detector.pattern.lastIndex = 0;
+        // Reset lastIndex for global regex. Ensure the `d` (hasIndices) flag so we can
+        // locate a captured group by its true offset instead of a fragile indexOf.
+        const pattern = detector.pattern.flags.includes('d')
+            ? detector.pattern
+            : new RegExp(detector.pattern.source, detector.pattern.flags + 'd');
+        pattern.lastIndex = 0;
         let match;
-        while ((match = detector.pattern.exec(text)) !== null) {
+        while ((match = pattern.exec(text)) !== null) {
             const fullMatch = match[0];
-            const capturedGroup = match[1]; // Some patterns use capture groups
-            // Use captured group if available (for context-anchored patterns)
-            const entityText = capturedGroup || fullMatch;
-            const start = capturedGroup ? match.index + fullMatch.indexOf(capturedGroup) : match.index;
-            const end = start + entityText.length;
+            // Extract a capture group ONLY for value-capturing detectors (captureValue).
+            // Structural detectors (phone/SSN/credit-card) have format sub-groups that are
+            // NOT the entity — extracting one there feeds e.g. an area code to the validator
+            // and drops the match. For value-capturing patterns the value can land in any of
+            // several alternation groups, so use the first DEFINED group.
+            let groupIdx = -1;
+            if (detector.captureValue) {
+                for (let g = 1; g < match.length; g++) {
+                    if (match[g] !== undefined) {
+                        groupIdx = g;
+                        break;
+                    }
+                }
+            }
+            const capturedGroup = groupIdx >= 0 ? match[groupIdx] : undefined;
+            // Use captured group if available (for context-anchored / value-only patterns).
+            // Locate it via match.indices (the `d` flag) rather than fullMatch.indexOf,
+            // which mis-locates a value that also occurs inside the key (e.g. token=token123).
+            let entityText = capturedGroup ?? fullMatch;
+            const indices = match.indices;
+            const start = (capturedGroup !== undefined && indices && indices[groupIdx])
+                ? indices[groupIdx][0]
+                : match.index;
+            let end = start + entityText.length;
+            // Credential values captured by a greedy `\S+` (or the Bearer char class) can
+            // swallow trailing sentence punctuation — e.g. `token: <secret>.` grabs the
+            // period, minting a DIFFERENT content-derived token than the same secret seen
+            // as `Bearer <secret>'`. Trim trailing punctuation so one secret → one token
+            // (and the punctuation stays in the surrounding text).
+            if (detector.type === 'profile-username-password' && capturedGroup !== undefined) {
+                const trimmed = entityText.replace(/[.,;:!?)\]}>'"]+$/, '');
+                if (trimmed.length >= 4 && trimmed.length !== entityText.length) {
+                    end -= entityText.length - trimmed.length;
+                    entityText = trimmed;
+                }
+            }
+            // URLs: mask only the ORIGIN (scheme://host[:port]), never the path/query.
+            // The placeholder is URL-shaped (see ReplacementMap), so the model can still
+            // compose variants (append paths, add query params) and the composed URL
+            // remains unmaskable — masking the full URL made composition impossible and
+            // pushed models into fabricating placeholder ids.
+            if (detector.type === 'profile-url') {
+                // Skip loopback/private/template URLs entirely (see isMaskableUrl).
+                if (!isMaskableUrl(entityText)) {
+                    continue;
+                }
+                const originMatch = entityText.match(/^(?:(?:https?|wss?|ftps?):\/\/)?[^/?#]+/);
+                if (originMatch) {
+                    entityText = originMatch[0];
+                    end = start + entityText.length;
+                }
+            }
             // Run post-detection validation if defined
             if (detector.validate && !detector.validate(entityText)) {
                 continue;

package/bundled/gateway/services/gateway/src/plugins/pseudonymization/entityToggles.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Entity-category toggles for the pseudonymizationPlugin.
+ *
+ * api_config.json can enable/disable masking categories with boolean maps
+ * ({ "<category>": true|false }) at three layers: global
+ * (api_config.pseudonymization.entities), per endpoint
+ * (defaultHooks.<endpoint>.pseudonymization.entities), and per model
+ * (model_list_changes.<model>.pseudonymization.entities). Later layers win.
+ *
+ * These toggles configure the plugin's DEFAULT entity set (force-config and
+ * triggerword activation) only. They do not activate masking by themselves —
+ * the plugin must be wired into the hook chain and an activation source must
+ * fire — and an explicit caller-supplied body `masking` config is used as-is.
+ */
+import { EntityConfig } from './types';
+/** All category types the plugin understands, for toggle validation. */
+export declare function buildKnownEntityTypes(defaults: EntityConfig[]): Set<string>;
+/**
+ * Apply a toggle map to a base entity list. `false` removes the category,
+ * `true` adds it if absent; categories not mentioned keep their current state.
+ * Unknown category names are ignored with a single WARN — a typo can never
+ * crash request handling or silently widen masking. Pure — returns a new array.
+ */
+export declare function applyEntityToggles(base: EntityConfig[], toggles: Record<string, boolean> | null | undefined, knownTypes: Set<string>): EntityConfig[];

package/bundled/gateway/services/gateway/src/plugins/pseudonymization/entityToggles.js

@@ -0,0 +1,54 @@
+"use strict";
+/**
+ * Entity-category toggles for the pseudonymizationPlugin.
+ *
+ * api_config.json can enable/disable masking categories with boolean maps
+ * ({ "<category>": true|false }) at three layers: global
+ * (api_config.pseudonymization.entities), per endpoint
+ * (defaultHooks.<endpoint>.pseudonymization.entities), and per model
+ * (model_list_changes.<model>.pseudonymization.entities). Later layers win.
+ *
+ * These toggles configure the plugin's DEFAULT entity set (force-config and
+ * triggerword activation) only. They do not activate masking by themselves —
+ * the plugin must be wired into the hook chain and an activation source must
+ * fire — and an explicit caller-supplied body `masking` config is used as-is.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildKnownEntityTypes = buildKnownEntityTypes;
+exports.applyEntityToggles = applyEntityToggles;
+const logger_1 = require("@libs/logger");
+/** All category types the plugin understands, for toggle validation. */
+function buildKnownEntityTypes(defaults) {
+    return new Set([...defaults.map(e => e.type), 'profile-sensitive-data']);
+}
+// Unknown toggle keys already warned about (one WARN per key per process, not per request).
+const warnedUnknownToggles = new Set();
+/**
+ * Apply a toggle map to a base entity list. `false` removes the category,
+ * `true` adds it if absent; categories not mentioned keep their current state.
+ * Unknown category names are ignored with a single WARN — a typo can never
+ * crash request handling or silently widen masking. Pure — returns a new array.
+ */
+function applyEntityToggles(base, toggles, knownTypes) {
+    if (!toggles || typeof toggles !== 'object')
+        return base;
+    let result = base.slice();
+    for (const [type, enabled] of Object.entries(toggles)) {
+        if (typeof enabled !== 'boolean')
+            continue;
+        if (!knownTypes.has(type)) {
+            if (!warnedUnknownToggles.has(type)) {
+                warnedUnknownToggles.add(type);
+                (0, logger_1.getDefaultLogger)().warn('Pseudonymization', `Ignoring unknown entity toggle '${type}' in api_config.json pseudonymization config (known types: ${Array.from(knownTypes).join(', ')})`);
+            }
+            continue;
+        }
+        if (enabled === false) {
+            result = result.filter(e => e.type !== type);
+        }
+        else if (!result.some(e => e.type === type)) {
+            result.push({ type });
+        }
+    }
+    return result;
+}

package/bundled/gateway/services/gateway/src/plugins/pseudonymization/index.d.ts

@@ -15,7 +15,8 @@
  *
  * Strategies: before + after + stream
  *
- * @see pseudonymization-proxy-prompt.md - Full specification
+ * @see ../pseudonymization.md - Usage, activation methods, and category configuration
+ * @see ../../../../../docs/security/pseudonymization-security-assessment.md - Security assessment
  */
 import { Request, Response } from 'express';
 interface PluginContext {