@st-gr/sail-proxy 0.9.3 → 0.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/gateway/node_modules/.package-lock.json +17 -17
- package/bundled/gateway/node_modules/form-data/CHANGELOG.md +29 -2
- package/bundled/gateway/node_modules/form-data/README.md +4 -4
- package/bundled/gateway/node_modules/form-data/lib/form_data.js +14 -2
- package/bundled/gateway/node_modules/form-data/package.json +7 -7
- package/bundled/gateway/node_modules/qs/CHANGELOG.md +16 -0
- package/bundled/gateway/node_modules/qs/README.md +1 -1
- package/bundled/gateway/node_modules/qs/dist/qs.js +15 -15
- package/bundled/gateway/node_modules/qs/eslint.config.mjs +1 -0
- package/bundled/gateway/node_modules/qs/lib/parse.js +54 -24
- package/bundled/gateway/node_modules/qs/lib/stringify.js +11 -4
- package/bundled/gateway/node_modules/qs/package.json +3 -3
- package/bundled/gateway/node_modules/qs/test/parse.js +135 -0
- package/bundled/gateway/node_modules/qs/test/stringify.js +138 -0
- package/bundled/gateway/node_modules/qs/test/utils.js +31 -3
- package/bundled/gateway/node_modules/semver/classes/range.js +17 -4
- package/bundled/gateway/node_modules/semver/package.json +2 -2
- package/bundled/gateway/node_modules/side-channel/CHANGELOG.md +10 -0
- package/bundled/gateway/node_modules/side-channel/README.md +1 -1
- package/bundled/gateway/node_modules/side-channel/index.js +5 -2
- package/bundled/gateway/node_modules/side-channel/package.json +10 -10
- package/bundled/gateway/node_modules/side-channel/test/index.js +16 -0
- package/bundled/gateway/package-lock.json +22 -22
- package/bundled/gateway/package.json +13 -5
- package/bundled/gateway/services/gateway/src/controllers/anthropicController.js +1 -1
- package/bundled/gateway/services/gateway/src/controllers/awsBedrockController.js +5 -1
- package/bundled/gateway/services/gateway/src/index.js +14 -0
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/detectors/index.js +23 -1
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/detectors/regexDetectors.js +149 -10
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/entityToggles.d.ts +24 -0
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/entityToggles.js +54 -0
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/index.d.ts +2 -1
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/index.js +441 -92
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/replacementMap.d.ts +24 -0
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/replacementMap.js +93 -4
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/replacer.d.ts +22 -0
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/replacer.js +77 -0
- package/bundled/gateway/services/gateway/src/plugins/pseudonymization/streamBuffer.js +26 -8
- package/bundled/gateway/services/gateway/src/services/awsBedrockService.js +118 -46
- package/bundled/gateway/services/gateway/src/services/betaFlagQuarantine.d.ts +18 -0
- package/bundled/gateway/services/gateway/src/services/betaFlagQuarantine.js +111 -0
- package/bundled/gateway/services/gateway/src/services/configService.d.ts +8 -0
- package/bundled/gateway/services/gateway/src/services/configService.js +61 -53
- package/bundled/gateway/services/gateway/src/utils/bedrockStreamParser.d.ts +9 -0
- package/bundled/gateway/services/gateway/src/utils/bedrockStreamParser.js +69 -3
- package/bundled/gateway/services/gateway/src/utils/betaFeatureFilter.d.ts +31 -0
- package/bundled/gateway/services/gateway/src/utils/betaFeatureFilter.js +58 -0
- package/bundled/gateway/services/gateway/src/utils/payloadLogger.d.ts +6 -0
- package/bundled/gateway/services/gateway/src/utils/payloadLogger.js +11 -7
- package/bundled/gateway/services/gateway/src/utils/pluginCacheSelector.d.ts +15 -0
- package/bundled/gateway/services/gateway/src/utils/pluginCacheSelector.js +24 -0
- package/bundled/gateway/services/gateway/src/utils/sseWriter.js +4 -2
- package/bundled/gateway/services/gateway/src/utils/upstreamErrorBody.d.ts +10 -0
- package/bundled/gateway/services/gateway/src/utils/upstreamErrorBody.js +71 -0
- package/bundled/ollama/node_modules/.package-lock.json +14 -14
- package/bundled/ollama/node_modules/form-data/CHANGELOG.md +29 -2
- package/bundled/ollama/node_modules/form-data/README.md +4 -4
- package/bundled/ollama/node_modules/form-data/lib/form_data.js +14 -2
- package/bundled/ollama/node_modules/form-data/package.json +7 -7
- package/bundled/ollama/node_modules/qs/CHANGELOG.md +16 -0
- package/bundled/ollama/node_modules/qs/README.md +1 -1
- package/bundled/ollama/node_modules/qs/dist/qs.js +15 -15
- package/bundled/ollama/node_modules/qs/eslint.config.mjs +1 -0
- package/bundled/ollama/node_modules/qs/lib/parse.js +54 -24
- package/bundled/ollama/node_modules/qs/lib/stringify.js +11 -4
- package/bundled/ollama/node_modules/qs/package.json +3 -3
- package/bundled/ollama/node_modules/qs/test/parse.js +135 -0
- package/bundled/ollama/node_modules/qs/test/stringify.js +138 -0
- package/bundled/ollama/node_modules/qs/test/utils.js +31 -3
- package/bundled/ollama/node_modules/side-channel/CHANGELOG.md +10 -0
- package/bundled/ollama/node_modules/side-channel/README.md +1 -1
- package/bundled/ollama/node_modules/side-channel/index.js +5 -2
- package/bundled/ollama/node_modules/side-channel/package.json +10 -10
- package/bundled/ollama/node_modules/side-channel/test/index.js +16 -0
- package/bundled/ollama/package-lock.json +15 -15
- package/bundled/ollama/package.json +13 -5
- package/dist/templates/api_config.template.json +266 -18
- package/node_modules/@sap-llm-gateway/service-key-parser/package.json +1 -1
- package/package.json +2 -2
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gateway",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.4",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "gateway",
|
|
9
|
-
"version": "0.9.
|
|
9
|
+
"version": "0.9.4",
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"axios": "1.16.1",
|
|
12
12
|
"body-parser": "1.20.3",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"express": "4.20.0",
|
|
17
17
|
"gpt-tokenizer": "^3.4.0",
|
|
18
18
|
"morgan": "^1.10.0",
|
|
19
|
-
"qs": "6.15.
|
|
19
|
+
"qs": "6.15.2",
|
|
20
20
|
"sharp": "^0.33.5",
|
|
21
21
|
"uuid": "^11.1.0",
|
|
22
22
|
"wink-eng-lite-web-model": "^1.8.1",
|
|
@@ -51,9 +51,9 @@
|
|
|
51
51
|
}
|
|
52
52
|
},
|
|
53
53
|
"node_modules/@emnapi/runtime": {
|
|
54
|
-
"version": "1.
|
|
55
|
-
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.
|
|
56
|
-
"integrity": "sha512-
|
|
54
|
+
"version": "1.11.2",
|
|
55
|
+
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz",
|
|
56
|
+
"integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==",
|
|
57
57
|
"license": "MIT",
|
|
58
58
|
"optional": true,
|
|
59
59
|
"dependencies": {
|
|
@@ -1065,16 +1065,16 @@
|
|
|
1065
1065
|
}
|
|
1066
1066
|
},
|
|
1067
1067
|
"node_modules/form-data": {
|
|
1068
|
-
"version": "4.0.
|
|
1069
|
-
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.
|
|
1070
|
-
"integrity": "sha512-
|
|
1068
|
+
"version": "4.0.6",
|
|
1069
|
+
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz",
|
|
1070
|
+
"integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==",
|
|
1071
1071
|
"license": "MIT",
|
|
1072
1072
|
"dependencies": {
|
|
1073
1073
|
"asynckit": "^0.4.0",
|
|
1074
1074
|
"combined-stream": "^1.0.8",
|
|
1075
1075
|
"es-set-tostringtag": "^2.1.0",
|
|
1076
|
-
"hasown": "^2.0.
|
|
1077
|
-
"mime-types": "^2.1.
|
|
1076
|
+
"hasown": "^2.0.4",
|
|
1077
|
+
"mime-types": "^2.1.35"
|
|
1078
1078
|
},
|
|
1079
1079
|
"engines": {
|
|
1080
1080
|
"node": ">= 6"
|
|
@@ -1586,9 +1586,9 @@
|
|
|
1586
1586
|
}
|
|
1587
1587
|
},
|
|
1588
1588
|
"node_modules/qs": {
|
|
1589
|
-
"version": "6.15.
|
|
1590
|
-
"resolved": "https://registry.npmjs.org/qs/-/qs-6.15.
|
|
1591
|
-
"integrity": "sha512-
|
|
1589
|
+
"version": "6.15.2",
|
|
1590
|
+
"resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz",
|
|
1591
|
+
"integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==",
|
|
1592
1592
|
"license": "BSD-3-Clause",
|
|
1593
1593
|
"dependencies": {
|
|
1594
1594
|
"side-channel": "^1.1.0"
|
|
@@ -1697,9 +1697,9 @@
|
|
|
1697
1697
|
"license": "MIT"
|
|
1698
1698
|
},
|
|
1699
1699
|
"node_modules/semver": {
|
|
1700
|
-
"version": "7.8.
|
|
1701
|
-
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.
|
|
1702
|
-
"integrity": "sha512-
|
|
1700
|
+
"version": "7.8.5",
|
|
1701
|
+
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz",
|
|
1702
|
+
"integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==",
|
|
1703
1703
|
"license": "ISC",
|
|
1704
1704
|
"bin": {
|
|
1705
1705
|
"semver": "bin/semver.js"
|
|
@@ -1838,14 +1838,14 @@
|
|
|
1838
1838
|
}
|
|
1839
1839
|
},
|
|
1840
1840
|
"node_modules/side-channel": {
|
|
1841
|
-
"version": "1.1.
|
|
1842
|
-
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.
|
|
1843
|
-
"integrity": "sha512-
|
|
1841
|
+
"version": "1.1.1",
|
|
1842
|
+
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz",
|
|
1843
|
+
"integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==",
|
|
1844
1844
|
"license": "MIT",
|
|
1845
1845
|
"dependencies": {
|
|
1846
1846
|
"es-errors": "^1.3.0",
|
|
1847
|
-
"object-inspect": "^1.13.
|
|
1848
|
-
"side-channel-list": "^1.0.
|
|
1847
|
+
"object-inspect": "^1.13.4",
|
|
1848
|
+
"side-channel-list": "^1.0.1",
|
|
1849
1849
|
"side-channel-map": "^1.0.1",
|
|
1850
1850
|
"side-channel-weakmap": "^1.0.2"
|
|
1851
1851
|
},
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gateway",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.4",
|
|
4
4
|
"description": "SAP AI Core Multi-Provider LLM Gateway Service",
|
|
5
5
|
"private": true,
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"gpt-tokenizer": "^3.4.0",
|
|
18
18
|
"iovalkey": "^0.3.3",
|
|
19
19
|
"morgan": "^1.10.0",
|
|
20
|
-
"qs": "6.15.
|
|
20
|
+
"qs": "6.15.2",
|
|
21
21
|
"sharp": "^0.33.5",
|
|
22
22
|
"uuid": "^11.1.0",
|
|
23
23
|
"wink-eng-lite-web-model": "^1.8.1",
|
|
@@ -30,19 +30,27 @@
|
|
|
30
30
|
"overrides": {
|
|
31
31
|
"path-to-regexp": "0.1.13",
|
|
32
32
|
"router": "1.3.8",
|
|
33
|
+
"form-data": ">=4.0.6 <5.0.0",
|
|
34
|
+
"http-proxy-middleware": ">=3.0.7 <4.0.0",
|
|
33
35
|
"cookie": "0.7.0",
|
|
34
36
|
"send": "0.19.0",
|
|
35
37
|
"serve-static": "1.16.0",
|
|
36
38
|
"tmp": "0.2.4",
|
|
37
39
|
"got": ">=11.8.5",
|
|
38
|
-
"tar": ">=7.5.
|
|
39
|
-
"js-yaml": ">=4.
|
|
40
|
+
"tar": ">=7.5.3 <8.0.0",
|
|
41
|
+
"js-yaml": ">=4.2.0 <5.0.0",
|
|
42
|
+
"lodash": ">=4.17.23 <5.0.0",
|
|
43
|
+
"underscore": ">=1.13.8 <2.0.0",
|
|
44
|
+
"ws": ">=8.20.1 <9.0.0",
|
|
45
|
+
"flatted": ">=3.4.2 <4.0.0",
|
|
46
|
+
"undici": ">=6.27.0 <7.0.0",
|
|
40
47
|
"minimatch": ">=9.0.6",
|
|
41
48
|
"follow-redirects": ">=1.16.0",
|
|
42
49
|
"@cap-js/sqlite": "2.0.4",
|
|
43
50
|
"@cap-js/postgres": "1.14.2",
|
|
44
51
|
"@cap-js/db-service": "<=2.6.0",
|
|
45
52
|
"jws": "3.2.3",
|
|
46
|
-
"fast-uri": "3.1.2"
|
|
53
|
+
"fast-uri": "3.1.2",
|
|
54
|
+
"shell-quote": ">=1.8.4"
|
|
47
55
|
}
|
|
48
56
|
}
|
|
@@ -501,7 +501,7 @@ function logStructuredEvent(event) {
|
|
|
501
501
|
* Save SSE stream data directly to a text file
|
|
502
502
|
*/
|
|
503
503
|
function saveSSEStreamToFile(requestId, sseEvents) {
|
|
504
|
-
if (!requestId || !
|
|
504
|
+
if (!requestId || !payloadLogger.isPayloadLoggingEnabled()) {
|
|
505
505
|
return;
|
|
506
506
|
}
|
|
507
507
|
try {
|
|
@@ -13,11 +13,15 @@ const pluginExecutor_1 = require("../services/pluginExecutor");
|
|
|
13
13
|
const logger_1 = require("@libs/logger");
|
|
14
14
|
const logger = (0, logger_1.getDefaultLogger)();
|
|
15
15
|
const usageTracker_1 = require("../utils/usageTracker");
|
|
16
|
+
const payloadLogger_1 = require("../utils/payloadLogger");
|
|
16
17
|
/**
|
|
17
18
|
* Handle AWS Bedrock requests for all model operations
|
|
18
19
|
*/
|
|
19
20
|
const handleBedrockRequest = async (req, res, _next) => {
|
|
20
|
-
|
|
21
|
+
// Generate a debug request ID whenever payload logging can occur — via the
|
|
22
|
+
// legacy DEBUG env or the dynamic config switch (hot-reloadable per request).
|
|
23
|
+
const payloadLoggingActive = process.env.DEBUG === 'true' || (0, payloadLogger_1.isPayloadLoggingEnabled)();
|
|
24
|
+
const debugRequestId = payloadLoggingActive ? (req.debugRequestId || (0, uuid_1.v4)()) : undefined;
|
|
21
25
|
req.debugRequestId = debugRequestId;
|
|
22
26
|
logger.info('AwsBedrockController', `Processing request for model: ${req.params.modelId}, subpath: ${req.params.subpath}, Debug ID: ${debugRequestId}`);
|
|
23
27
|
// Initialize usage tracking
|
|
@@ -379,6 +379,20 @@ initializeGatewayService().then(() => {
|
|
|
379
379
|
logger.info('Gateway Service', '');
|
|
380
380
|
logger.info('Gateway Service', '💡 Press Ctrl+C to gracefully shutdown the service');
|
|
381
381
|
});
|
|
382
|
+
// Fail loudly if the port is taken: otherwise a stale gateway instance keeps serving
|
|
383
|
+
// requests while this (newer) process dies quietly — old code masquerades as the
|
|
384
|
+
// current build, which makes debugging almost impossible.
|
|
385
|
+
server.on('error', (error) => {
|
|
386
|
+
if (error.code === 'EADDRINUSE') {
|
|
387
|
+
logger.error('Gateway Service', `FATAL: ${config.host}:${config.port} is already in use — ANOTHER gateway instance is running. ` +
|
|
388
|
+
`Requests are being served by THAT process (possibly stale code), not this one. ` +
|
|
389
|
+
`Find it with: lsof -nP -iTCP:${config.port} -sTCP:LISTEN`);
|
|
390
|
+
}
|
|
391
|
+
else {
|
|
392
|
+
logger.error('Gateway Service', 'HTTP server error:', error instanceof Error ? error : new Error(String(error)));
|
|
393
|
+
}
|
|
394
|
+
process.exit(1);
|
|
395
|
+
});
|
|
382
396
|
// Store server reference for cleanup
|
|
383
397
|
cleanupResources.server = server;
|
|
384
398
|
}).catch((error) => {
|
|
@@ -12,6 +12,23 @@ const customDetector_1 = require("./customDetector");
|
|
|
12
12
|
const regexDetectors_1 = require("./regexDetectors");
|
|
13
13
|
const nerDetector_1 = require("./nerDetector");
|
|
14
14
|
const dictionaryDetector_1 = require("./dictionaryDetector");
|
|
15
|
+
/**
|
|
16
|
+
* Matches placeholders that ALREADY exist in the text so they are never re-masked.
|
|
17
|
+
* Covers the `MASKED_<TYPE>_<id>` family and the URL-shaped `masked-url-<id>.invalid`
|
|
18
|
+
* form. Re-masking a placeholder produces layered tokens ("keeps getting re-masked")
|
|
19
|
+
* that can never be unmasked back to the original value.
|
|
20
|
+
*/
|
|
21
|
+
const EXISTING_PLACEHOLDER = /MASKED_[A-Z_]+_[0-9a-f]+|(?:(?:https?|wss?|ftps?):\/\/)?masked-url-\d+\.invalid/g;
|
|
22
|
+
/** Spans in `text` occupied by placeholders that must be excluded from detection. */
|
|
23
|
+
function placeholderSpans(text) {
|
|
24
|
+
const spans = [];
|
|
25
|
+
EXISTING_PLACEHOLDER.lastIndex = 0;
|
|
26
|
+
let m;
|
|
27
|
+
while ((m = EXISTING_PLACEHOLDER.exec(text)) !== null) {
|
|
28
|
+
spans.push({ start: m.index, end: m.index + m[0].length });
|
|
29
|
+
}
|
|
30
|
+
return spans;
|
|
31
|
+
}
|
|
15
32
|
/**
|
|
16
33
|
* Check if a match is in the allow list
|
|
17
34
|
*/
|
|
@@ -65,8 +82,13 @@ function detectEntities(text, config) {
|
|
|
65
82
|
allMatches.push(...(0, nerDetector_1.detectNerEntities)(text, config.entities));
|
|
66
83
|
// Tier 3: Dictionary
|
|
67
84
|
allMatches.push(...(0, dictionaryDetector_1.detectDictionaryEntities)(text, config.entities));
|
|
85
|
+
// Never re-mask an existing placeholder: drop any match overlapping one.
|
|
86
|
+
const reserved = placeholderSpans(text);
|
|
87
|
+
const notPlaceholder = reserved.length === 0
|
|
88
|
+
? allMatches
|
|
89
|
+
: allMatches.filter(m => !reserved.some(s => m.start < s.end && m.end > s.start));
|
|
68
90
|
// Filter allow-list
|
|
69
|
-
const filtered =
|
|
91
|
+
const filtered = notPlaceholder.filter(m => !isAllowed(m.original, config.allow_list));
|
|
70
92
|
// Resolve overlaps
|
|
71
93
|
return resolveOverlaps(filtered);
|
|
72
94
|
}
|
package/bundled/gateway/services/gateway/src/plugins/pseudonymization/detectors/regexDetectors.js
CHANGED
|
@@ -8,6 +8,51 @@
|
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
10
|
exports.detectRegexEntities = detectRegexEntities;
|
|
11
|
+
/**
|
|
12
|
+
* Decide whether a URL is worth masking. We SKIP:
|
|
13
|
+
* - loopback / private / link-local hosts (localhost, 127.x, ::1, 10.x,
|
|
14
|
+
* 192.168.x, 172.16-31.x, *.local, *.internal) — these are dev/infra
|
|
15
|
+
* endpoints with ~no privacy value, and masking them breaks agent edit
|
|
16
|
+
* workflows (the model must reproduce an opaque token to match file text);
|
|
17
|
+
* - template / non-literal URLs containing <...>, {...}, ${...} or an
|
|
18
|
+
* unresolved-looking host — masking these fragments one logical endpoint
|
|
19
|
+
* into many bogus tokens.
|
|
20
|
+
* Real external hosts still get masked.
|
|
21
|
+
*/
|
|
22
|
+
function isMaskableUrl(fullMatch) {
|
|
23
|
+
// Template markers anywhere in the matched URL → not a real value.
|
|
24
|
+
if (/[<>{}]|\$\{/.test(fullMatch))
|
|
25
|
+
return false;
|
|
26
|
+
// Trailing colon = incomplete authority, i.e. the port was a template variable
|
|
27
|
+
// (`http://host:{PORT}` matches only up to `http://host:` since `{` is excluded).
|
|
28
|
+
if (/:$/.test(fullMatch))
|
|
29
|
+
return false;
|
|
30
|
+
const m = fullMatch.match(/^(?:(?:https?|wss?|ftps?):\/\/)?([^/?#]+)/);
|
|
31
|
+
const authority = (m ? m[1] : fullMatch).toLowerCase();
|
|
32
|
+
// Authority must be a well-formed host[:port] (or [ipv6][:port]). Anything else —
|
|
33
|
+
// e.g. a leftover `http://host:$` from `${PORT}`, or a bare scheme — is a template
|
|
34
|
+
// fragment, not a real endpoint.
|
|
35
|
+
if (!/^(?:\[[0-9a-f:]+\]|[a-z0-9._-]+)(?::\d+)?$/.test(authority))
|
|
36
|
+
return false;
|
|
37
|
+
const host = authority.replace(/:\d*$/, '').replace(/^\[|\]$/g, '');
|
|
38
|
+
if (host === '' || host === 'localhost' || host.endsWith('.local') || host.endsWith('.internal'))
|
|
39
|
+
return false;
|
|
40
|
+
if (host === '::1' || host.startsWith('127.') || host.startsWith('0.'))
|
|
41
|
+
return false;
|
|
42
|
+
if (host.startsWith('10.') || host.startsWith('192.168.'))
|
|
43
|
+
return false;
|
|
44
|
+
// 172.16.0.0 – 172.31.255.255
|
|
45
|
+
const m172 = host.match(/^172\.(\d{1,3})\./);
|
|
46
|
+
if (m172) {
|
|
47
|
+
const o = Number(m172[1]);
|
|
48
|
+
if (o >= 16 && o <= 31)
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
// 169.254.0.0/16 link-local
|
|
52
|
+
if (host.startsWith('169.254.'))
|
|
53
|
+
return false;
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
11
56
|
// Luhn algorithm for credit card validation
|
|
12
57
|
function luhnCheck(digits) {
|
|
13
58
|
const nums = digits.split('').map(Number);
|
|
@@ -54,6 +99,13 @@ function phoneValidate(match) {
|
|
|
54
99
|
const digits = match.replace(/\D/g, '');
|
|
55
100
|
return digits.length >= 7 && digits.length <= 15;
|
|
56
101
|
}
|
|
102
|
+
// Credential value validation: reject template placeholders (`<token>`, `${VAR}`,
|
|
103
|
+
// `$VAR`) and values too short to be real secrets. Applied to the captured VALUE.
|
|
104
|
+
function credentialValueValidate(value) {
|
|
105
|
+
if (/[<>{}]|\$\{|^\$/.test(value))
|
|
106
|
+
return false;
|
|
107
|
+
return value.length >= 4;
|
|
108
|
+
}
|
|
57
109
|
const DETECTORS = [
|
|
58
110
|
{
|
|
59
111
|
type: 'profile-email',
|
|
@@ -88,7 +140,7 @@ const DETECTORS = [
|
|
|
88
140
|
},
|
|
89
141
|
{
|
|
90
142
|
type: 'profile-url',
|
|
91
|
-
pattern: /(?:https
|
|
143
|
+
pattern: /(?:(?:https?|wss?|ftps?):\/\/|www\.)[^\s<>"{}|\\^`\[\]]+/g,
|
|
92
144
|
},
|
|
93
145
|
{
|
|
94
146
|
type: 'profile-nationalid',
|
|
@@ -104,20 +156,55 @@ const DETECTORS = [
|
|
|
104
156
|
type: 'profile-nationalid',
|
|
105
157
|
// Generic context-anchored
|
|
106
158
|
pattern: /(?:national\s*id|identification\s*(?:number|no\.?))\s*[:=]?\s*([A-Z0-9\-]{6,20})/gi,
|
|
159
|
+
captureValue: true,
|
|
107
160
|
},
|
|
108
161
|
{
|
|
109
162
|
type: 'profile-passport',
|
|
110
163
|
// Context-anchored
|
|
111
164
|
pattern: /(?:passport\s*(?:number|no\.?|#))\s*[:=]?\s*([A-Z]{0,2}\d{6,9})/gi,
|
|
165
|
+
captureValue: true,
|
|
112
166
|
},
|
|
113
167
|
{
|
|
114
168
|
type: 'profile-driverlicense',
|
|
115
169
|
// Context-anchored generic
|
|
116
170
|
pattern: /(?:driver'?s?\s*licen[sc]e|DL)\s*(?:number|no\.?|#)?\s*[:=]?\s*([A-Z0-9\-]{5,15})/gi,
|
|
171
|
+
captureValue: true,
|
|
117
172
|
},
|
|
118
173
|
{
|
|
119
174
|
type: 'profile-username-password',
|
|
120
|
-
|
|
175
|
+
// key=value / key: value credentials. The leading (?<![A-Za-z0-9_]) + greedy
|
|
176
|
+
// identifier prefix make the match consume the WHOLE identifier ending in a
|
|
177
|
+
// credential word — so `$env:SSHPASS="x"` masks only `x` (not `SSHMASKED_…`),
|
|
178
|
+
// and `SSH_ASKPASS_REQUIRE=force` does not match (identifier ends in REQUIRE).
|
|
179
|
+
// Only the value is captured, preserving surrounding script structure.
|
|
180
|
+
pattern: /(?<![A-Za-z0-9_])[A-Za-z0-9_$-]*(?:user(?:name)?|login|pass(?:word|wd)?|pwd|secret|token|api[_\-]?key)\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|(\S+))/gi,
|
|
181
|
+
validate: credentialValueValidate,
|
|
182
|
+
captureValue: true,
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
type: 'profile-username-password',
|
|
186
|
+
// JSON-style credentials: "token": "value" — the quoted key defeats the pattern
|
|
187
|
+
// above. Capture ONLY the value so masking preserves the JSON structure.
|
|
188
|
+
pattern: /"(?:user(?:name)?|login|pass(?:word|wd)?|pwd|secret|token|api[_\-]?key|access[_\-]?token|auth[_\-]?token|bearer)"\s*:\s*"([^"]+)"/gi,
|
|
189
|
+
validate: credentialValueValidate,
|
|
190
|
+
captureValue: true,
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
type: 'profile-username-password',
|
|
194
|
+
// HTTP auth headers: `Authorization: Bearer <token>` and siblings. This is the
|
|
195
|
+
// gap behind the 28551619 incident — the secret rode through in four Bearer
|
|
196
|
+
// headers unmasked. Capture the opaque credential value only.
|
|
197
|
+
pattern: /(?:Authorization|Proxy-Authorization|X-Auth-Token|X-Api-Key)["']?\s*[:=]\s*["']?(?:(?:Bearer|Basic|Token|OAuth)\s+)?([A-Za-z0-9][A-Za-z0-9._~+\/=-]{7,})/gi,
|
|
198
|
+
validate: credentialValueValidate,
|
|
199
|
+
captureValue: true,
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
type: 'profile-username-password',
|
|
203
|
+
// Standalone `Bearer <opaque-token>` not preceded by a header name. Min length
|
|
204
|
+
// 12 keeps ordinary prose words after "Bearer" from matching.
|
|
205
|
+
pattern: /\bBearer\s+([A-Za-z0-9][A-Za-z0-9._~+\/=-]{11,})/g,
|
|
206
|
+
validate: credentialValueValidate,
|
|
207
|
+
captureValue: true,
|
|
121
208
|
},
|
|
122
209
|
{
|
|
123
210
|
type: 'profile-address',
|
|
@@ -128,6 +215,7 @@ const DETECTORS = [
|
|
|
128
215
|
type: 'profile-pronouns-gender',
|
|
129
216
|
// Context-anchored pronoun declarations
|
|
130
217
|
pattern: /(?:pronouns?|goes\s+by)\s*[:=]?\s*((?:he|she|they|ze|xe|ey|fae)(?:\s*\/\s*(?:him|her|them|zir|xem|em|faer))+)/gi,
|
|
218
|
+
captureValue: true,
|
|
131
219
|
},
|
|
132
220
|
];
|
|
133
221
|
/**
|
|
@@ -142,16 +230,67 @@ function detectRegexEntities(text, enabledEntities) {
|
|
|
142
230
|
if (!enabledTypes.has(detector.type) && !sensitiveDataEnabled) {
|
|
143
231
|
continue;
|
|
144
232
|
}
|
|
145
|
-
// Reset lastIndex for global regex
|
|
146
|
-
|
|
233
|
+
// Reset lastIndex for global regex. Ensure the `d` (hasIndices) flag so we can
|
|
234
|
+
// locate a captured group by its true offset instead of a fragile indexOf.
|
|
235
|
+
const pattern = detector.pattern.flags.includes('d')
|
|
236
|
+
? detector.pattern
|
|
237
|
+
: new RegExp(detector.pattern.source, detector.pattern.flags + 'd');
|
|
238
|
+
pattern.lastIndex = 0;
|
|
147
239
|
let match;
|
|
148
|
-
while ((match =
|
|
240
|
+
while ((match = pattern.exec(text)) !== null) {
|
|
149
241
|
const fullMatch = match[0];
|
|
150
|
-
|
|
151
|
-
//
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
242
|
+
// Extract a capture group ONLY for value-capturing detectors (captureValue).
|
|
243
|
+
// Structural detectors (phone/SSN/credit-card) have format sub-groups that are
|
|
244
|
+
// NOT the entity — extracting one there feeds e.g. an area code to the validator
|
|
245
|
+
// and drops the match. For value-capturing patterns the value can land in any of
|
|
246
|
+
// several alternation groups, so use the first DEFINED group.
|
|
247
|
+
let groupIdx = -1;
|
|
248
|
+
if (detector.captureValue) {
|
|
249
|
+
for (let g = 1; g < match.length; g++) {
|
|
250
|
+
if (match[g] !== undefined) {
|
|
251
|
+
groupIdx = g;
|
|
252
|
+
break;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
const capturedGroup = groupIdx >= 0 ? match[groupIdx] : undefined;
|
|
257
|
+
// Use captured group if available (for context-anchored / value-only patterns).
|
|
258
|
+
// Locate it via match.indices (the `d` flag) rather than fullMatch.indexOf,
|
|
259
|
+
// which mis-locates a value that also occurs inside the key (e.g. token=token123).
|
|
260
|
+
let entityText = capturedGroup ?? fullMatch;
|
|
261
|
+
const indices = match.indices;
|
|
262
|
+
const start = (capturedGroup !== undefined && indices && indices[groupIdx])
|
|
263
|
+
? indices[groupIdx][0]
|
|
264
|
+
: match.index;
|
|
265
|
+
let end = start + entityText.length;
|
|
266
|
+
// Credential values captured by a greedy `\S+` (or the Bearer char class) can
|
|
267
|
+
// swallow trailing sentence punctuation — e.g. `token: <secret>.` grabs the
|
|
268
|
+
// period, minting a DIFFERENT content-derived token than the same secret seen
|
|
269
|
+
// as `Bearer <secret>'`. Trim trailing punctuation so one secret → one token
|
|
270
|
+
// (and the punctuation stays in the surrounding text).
|
|
271
|
+
if (detector.type === 'profile-username-password' && capturedGroup !== undefined) {
|
|
272
|
+
const trimmed = entityText.replace(/[.,;:!?)\]}>'"]+$/, '');
|
|
273
|
+
if (trimmed.length >= 4 && trimmed.length !== entityText.length) {
|
|
274
|
+
end -= entityText.length - trimmed.length;
|
|
275
|
+
entityText = trimmed;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
// URLs: mask only the ORIGIN (scheme://host[:port]), never the path/query.
|
|
279
|
+
// The placeholder is URL-shaped (see ReplacementMap), so the model can still
|
|
280
|
+
// compose variants (append paths, add query params) and the composed URL
|
|
281
|
+
// remains unmaskable — masking the full URL made composition impossible and
|
|
282
|
+
// pushed models into fabricating placeholder ids.
|
|
283
|
+
if (detector.type === 'profile-url') {
|
|
284
|
+
// Skip loopback/private/template URLs entirely (see isMaskableUrl).
|
|
285
|
+
if (!isMaskableUrl(entityText)) {
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
const originMatch = entityText.match(/^(?:(?:https?|wss?|ftps?):\/\/)?[^/?#]+/);
|
|
289
|
+
if (originMatch) {
|
|
290
|
+
entityText = originMatch[0];
|
|
291
|
+
end = start + entityText.length;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
155
294
|
// Run post-detection validation if defined
|
|
156
295
|
if (detector.validate && !detector.validate(entityText)) {
|
|
157
296
|
continue;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Entity-category toggles for the pseudonymizationPlugin.
|
|
3
|
+
*
|
|
4
|
+
* api_config.json can enable/disable masking categories with boolean maps
|
|
5
|
+
* ({ "<category>": true|false }) at three layers: global
|
|
6
|
+
* (api_config.pseudonymization.entities), per endpoint
|
|
7
|
+
* (defaultHooks.<endpoint>.pseudonymization.entities), and per model
|
|
8
|
+
* (model_list_changes.<model>.pseudonymization.entities). Later layers win.
|
|
9
|
+
*
|
|
10
|
+
* These toggles configure the plugin's DEFAULT entity set (force-config and
|
|
11
|
+
* triggerword activation) only. They do not activate masking by themselves —
|
|
12
|
+
* the plugin must be wired into the hook chain and an activation source must
|
|
13
|
+
* fire — and an explicit caller-supplied body `masking` config is used as-is.
|
|
14
|
+
*/
|
|
15
|
+
import { EntityConfig } from './types';
|
|
16
|
+
/** All category types the plugin understands, for toggle validation. */
|
|
17
|
+
export declare function buildKnownEntityTypes(defaults: EntityConfig[]): Set<string>;
|
|
18
|
+
/**
|
|
19
|
+
* Apply a toggle map to a base entity list. `false` removes the category,
|
|
20
|
+
* `true` adds it if absent; categories not mentioned keep their current state.
|
|
21
|
+
* Unknown category names are ignored with a single WARN — a typo can never
|
|
22
|
+
* crash request handling or silently widen masking. Pure — returns a new array.
|
|
23
|
+
*/
|
|
24
|
+
export declare function applyEntityToggles(base: EntityConfig[], toggles: Record<string, boolean> | null | undefined, knownTypes: Set<string>): EntityConfig[];
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Entity-category toggles for the pseudonymizationPlugin.
|
|
4
|
+
*
|
|
5
|
+
* api_config.json can enable/disable masking categories with boolean maps
|
|
6
|
+
* ({ "<category>": true|false }) at three layers: global
|
|
7
|
+
* (api_config.pseudonymization.entities), per endpoint
|
|
8
|
+
* (defaultHooks.<endpoint>.pseudonymization.entities), and per model
|
|
9
|
+
* (model_list_changes.<model>.pseudonymization.entities). Later layers win.
|
|
10
|
+
*
|
|
11
|
+
* These toggles configure the plugin's DEFAULT entity set (force-config and
|
|
12
|
+
* triggerword activation) only. They do not activate masking by themselves —
|
|
13
|
+
* the plugin must be wired into the hook chain and an activation source must
|
|
14
|
+
* fire — and an explicit caller-supplied body `masking` config is used as-is.
|
|
15
|
+
*/
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.buildKnownEntityTypes = buildKnownEntityTypes;
|
|
18
|
+
exports.applyEntityToggles = applyEntityToggles;
|
|
19
|
+
const logger_1 = require("@libs/logger");
|
|
20
|
+
/** All category types the plugin understands, for toggle validation. */
|
|
21
|
+
function buildKnownEntityTypes(defaults) {
|
|
22
|
+
return new Set([...defaults.map(e => e.type), 'profile-sensitive-data']);
|
|
23
|
+
}
|
|
24
|
+
// Unknown toggle keys already warned about (one WARN per key per process, not per request).
|
|
25
|
+
const warnedUnknownToggles = new Set();
|
|
26
|
+
/**
|
|
27
|
+
* Apply a toggle map to a base entity list. `false` removes the category,
|
|
28
|
+
* `true` adds it if absent; categories not mentioned keep their current state.
|
|
29
|
+
* Unknown category names are ignored with a single WARN — a typo can never
|
|
30
|
+
* crash request handling or silently widen masking. Pure — returns a new array.
|
|
31
|
+
*/
|
|
32
|
+
function applyEntityToggles(base, toggles, knownTypes) {
|
|
33
|
+
if (!toggles || typeof toggles !== 'object')
|
|
34
|
+
return base;
|
|
35
|
+
let result = base.slice();
|
|
36
|
+
for (const [type, enabled] of Object.entries(toggles)) {
|
|
37
|
+
if (typeof enabled !== 'boolean')
|
|
38
|
+
continue;
|
|
39
|
+
if (!knownTypes.has(type)) {
|
|
40
|
+
if (!warnedUnknownToggles.has(type)) {
|
|
41
|
+
warnedUnknownToggles.add(type);
|
|
42
|
+
(0, logger_1.getDefaultLogger)().warn('Pseudonymization', `Ignoring unknown entity toggle '${type}' in api_config.json pseudonymization config (known types: ${Array.from(knownTypes).join(', ')})`);
|
|
43
|
+
}
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
if (enabled === false) {
|
|
47
|
+
result = result.filter(e => e.type !== type);
|
|
48
|
+
}
|
|
49
|
+
else if (!result.some(e => e.type === type)) {
|
|
50
|
+
result.push({ type });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return result;
|
|
54
|
+
}
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
*
|
|
16
16
|
* Strategies: before + after + stream
|
|
17
17
|
*
|
|
18
|
-
* @see pseudonymization
|
|
18
|
+
* @see ../pseudonymization.md - Usage, activation methods, and category configuration
|
|
19
|
+
* @see ../../../../../docs/security/pseudonymization-security-assessment.md - Security assessment
|
|
19
20
|
*/
|
|
20
21
|
import { Request, Response } from 'express';
|
|
21
22
|
interface PluginContext {
|