@reproapp/node-sdk 0.0.52 → 0.0.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +18 -5
- package/dist/privacy.js +46 -11
- package/dist/trace-materializer-worker.js +14 -3
- package/package.json +5 -4
- package/src/index.ts +24 -6
- package/src/privacy.ts +54 -11
- package/src/trace-materializer-worker.ts +13 -3
- package/test/capture-latency.testcontainers.test.js +1 -0
- package/test/fixtures/capture-latency-business.js +1 -1
- package/test/fixtures/capture-latency-server.js +7 -4
- package/test/object-id-shapes.test.js +36 -0
- package/test/privacy-policy-index.test.js +15 -0
- package/test/trace-throughput.testcontainers.test.js +110 -0
- package/test/trace-worker-policy-reference.test.js +43 -0
- package/test/twelve-element-capture.testcontainers.test.js +116 -0
package/dist/index.js
CHANGED
|
@@ -1116,6 +1116,7 @@ let traceMaterializerWorkerDisabled = !TRACE_MATERIALIZE_OFFTHREAD_ENABLED;
|
|
|
1116
1116
|
let traceMaterializerTaskSeq = 0;
|
|
1117
1117
|
let traceMaterializerIdleTimer = null;
|
|
1118
1118
|
const traceMaterializerExpectedTerminations = new WeakSet();
|
|
1119
|
+
const traceMaterializerWorkerPolicies = new WeakMap();
|
|
1119
1120
|
const traceMaterializerPending = new Map();
|
|
1120
1121
|
function hasActiveClientRequests() {
|
|
1121
1122
|
return activeClientRequestCount > 0;
|
|
@@ -1332,12 +1333,22 @@ async function materializePendingTraceEventsOffThread(events, payload) {
|
|
|
1332
1333
|
timeout,
|
|
1333
1334
|
});
|
|
1334
1335
|
try {
|
|
1336
|
+
const previousPolicy = traceMaterializerWorkerPolicies.get(worker);
|
|
1337
|
+
const sendPolicy = !previousPolicy || previousPolicy.policy !== payload.privacy;
|
|
1338
|
+
const policyState = sendPolicy ? { policy: payload.privacy, id: taskId } : previousPolicy;
|
|
1339
|
+
const { policy: _inlinePolicy, ...privacyConfig } = payload.cfg.privacy ?? {};
|
|
1335
1340
|
const request = {
|
|
1336
1341
|
taskId,
|
|
1337
1342
|
type: 'materialize-trace-events',
|
|
1338
|
-
payload
|
|
1343
|
+
payload: {
|
|
1344
|
+
...payload,
|
|
1345
|
+
cfg: { ...payload.cfg, privacy: privacyConfig },
|
|
1346
|
+
privacy: sendPolicy ? payload.privacy : undefined,
|
|
1347
|
+
policyId: policyState.id,
|
|
1348
|
+
},
|
|
1339
1349
|
};
|
|
1340
1350
|
worker.postMessage(request);
|
|
1351
|
+
traceMaterializerWorkerPolicies.set(worker, policyState);
|
|
1341
1352
|
}
|
|
1342
1353
|
catch (err) {
|
|
1343
1354
|
traceMaterializerPending.delete(taskId);
|
|
@@ -2340,12 +2351,15 @@ function coerceMongoId(value) {
|
|
|
2340
2351
|
if (typeof value !== 'object') {
|
|
2341
2352
|
return null;
|
|
2342
2353
|
}
|
|
2354
|
+
// Argument lists and query-result arrays are business data, not BSON ids.
|
|
2355
|
+
if (Array.isArray(value))
|
|
2356
|
+
return null;
|
|
2343
2357
|
const bsonType = typeof value?._bsontype === 'string' ? String(value._bsontype).toLowerCase() : '';
|
|
2344
2358
|
const ctorName = typeof value?.constructor?.name === 'string' ? String(value.constructor.name).toLowerCase() : '';
|
|
2345
2359
|
const looksLikeObjectId = bsonType.includes('objectid') ||
|
|
2346
2360
|
ctorName === 'objectid' ||
|
|
2347
2361
|
typeof value.toHexString === 'function';
|
|
2348
|
-
const fromBufferShape = coerceMongoIdFromBufferShape(value,
|
|
2362
|
+
const fromBufferShape = looksLikeObjectId ? coerceMongoIdFromBufferShape(value, true) : null;
|
|
2349
2363
|
if (fromBufferShape) {
|
|
2350
2364
|
return fromBufferShape;
|
|
2351
2365
|
}
|
|
@@ -2422,12 +2436,11 @@ function coerceMongoIdFromBufferShape(value, includeIdCandidate = false) {
|
|
|
2422
2436
|
if (!Array.isArray(bytes) || bytes.length !== 12) {
|
|
2423
2437
|
return null;
|
|
2424
2438
|
}
|
|
2425
|
-
|
|
2426
|
-
if (normalized.some((byte) => !Number.isInteger(byte) || byte < 0 || byte > 255)) {
|
|
2439
|
+
if (bytes.some((byte) => typeof byte !== 'number' || !Number.isInteger(byte) || byte < 0 || byte > 255)) {
|
|
2427
2440
|
return null;
|
|
2428
2441
|
}
|
|
2429
2442
|
try {
|
|
2430
|
-
return Buffer.from(
|
|
2443
|
+
return Buffer.from(bytes).toString('hex');
|
|
2431
2444
|
}
|
|
2432
2445
|
catch {
|
|
2433
2446
|
return null;
|
package/dist/privacy.js
CHANGED
|
@@ -11,6 +11,7 @@ const EMPTY_RUNTIME_PRIVACY_POLICY = {
|
|
|
11
11
|
fingerprint: 'empty',
|
|
12
12
|
};
|
|
13
13
|
const runtimePrivacyStatementIndexes = new WeakMap();
|
|
14
|
+
const comparableTokenCache = new Map();
|
|
14
15
|
const RUNTIME_PRIVACY_CANDIDATE_CACHE_MAX_ENTRIES = 2048;
|
|
15
16
|
const RUNTIME_PRIVACY_PROVENANCE_MAX_ENTRIES = 1000;
|
|
16
17
|
const RUNTIME_PRIVACY_PROVENANCE_MAX_SERIALIZED_LENGTH = 200000;
|
|
@@ -453,6 +454,7 @@ function buildRuntimePrivacyStatementIndex(statements) {
|
|
|
453
454
|
functionStatements,
|
|
454
455
|
candidateCache: new Map(),
|
|
455
456
|
hasNumericTargetSegments,
|
|
457
|
+
scoredCandidateCache: new Map(),
|
|
456
458
|
};
|
|
457
459
|
}
|
|
458
460
|
function getRuntimePrivacyStatementIndex(policy) {
|
|
@@ -874,6 +876,8 @@ function rememberSerializedRuntimePrivacyProvenance(value, provenance) {
|
|
|
874
876
|
}
|
|
875
877
|
}
|
|
876
878
|
function lookupSerializedRuntimePrivacyProvenance(value) {
|
|
879
|
+
if (serializedRuntimePrivacyProvenance.size === 0)
|
|
880
|
+
return null;
|
|
877
881
|
for (const serialized of runtimePrivacyProvenanceSerializations(value)) {
|
|
878
882
|
const provenance = serializedRuntimePrivacyProvenance.get(hashRuntimePrivacyProvenance(serialized));
|
|
879
883
|
if (provenance) {
|
|
@@ -1228,6 +1232,26 @@ async function transformSerializedStructuredStringAsChildrenAsync(policy, surfac
|
|
|
1228
1232
|
return JSON.stringify(transformed);
|
|
1229
1233
|
}
|
|
1230
1234
|
function matchStatement(policy, surface, value, context, path) {
|
|
1235
|
+
const index = getRuntimePrivacyStatementIndex(policy);
|
|
1236
|
+
// Scores and non-pattern selectors depend on the context/path, not the
|
|
1237
|
+
// observed value. Preserve numeric positions here even for wildcard rules.
|
|
1238
|
+
const key = JSON.stringify([surface, context.trace?.fn, context.routeKey,
|
|
1239
|
+
context.db?.type, context.db?.collection, context.db?.op, path]);
|
|
1240
|
+
let scored = index.scoredCandidateCache.get(key);
|
|
1241
|
+
if (!scored) {
|
|
1242
|
+
scored = scoreStatementCandidates(policy, surface, context, path);
|
|
1243
|
+
if (index.scoredCandidateCache.size >= RUNTIME_PRIVACY_CANDIDATE_CACHE_MAX_ENTRIES) {
|
|
1244
|
+
index.scoredCandidateCache.clear();
|
|
1245
|
+
}
|
|
1246
|
+
index.scoredCandidateCache.set(key, scored);
|
|
1247
|
+
}
|
|
1248
|
+
const leaf = path[path.length - 1] ?? '';
|
|
1249
|
+
const joinedPath = normalizeComparablePath(path);
|
|
1250
|
+
const candidates = scored.filter(({ statement }) => statement.scope !== 'PATTERN'
|
|
1251
|
+
|| patternMatches(statement.pattern, leaf, joinedPath, value));
|
|
1252
|
+
return selectStatementCandidate(candidates, surface, value, path);
|
|
1253
|
+
}
|
|
1254
|
+
function scoreStatementCandidates(policy, surface, context, path) {
|
|
1231
1255
|
let candidates = [];
|
|
1232
1256
|
for (const statement of candidateStatementsForMatch(policy, surface, context, path)) {
|
|
1233
1257
|
if (shouldIgnoreBroadContainerRuleOnDescendant(statement, surface, path)) {
|
|
@@ -1240,7 +1264,7 @@ function matchStatement(policy, surface, value, context, path) {
|
|
|
1240
1264
|
if (targetScore < 0) {
|
|
1241
1265
|
continue;
|
|
1242
1266
|
}
|
|
1243
|
-
if (!matchesScope(statement, surface,
|
|
1267
|
+
if (statement.scope !== 'PATTERN' && !matchesScope(statement, surface, undefined, context, path)) {
|
|
1244
1268
|
continue;
|
|
1245
1269
|
}
|
|
1246
1270
|
const selectorScore = matchSelectorSpecificity(statement, surface, context, path);
|
|
@@ -1254,10 +1278,9 @@ function matchStatement(policy, surface, value, context, path) {
|
|
|
1254
1278
|
statement.priority,
|
|
1255
1279
|
});
|
|
1256
1280
|
}
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
candidates = candidates.filter(({ statement }) => !isBroadContextKeepExact(statement, surface, path));
|
|
1281
|
+
return candidates.filter(({ statement }) => !isBroadContextKeepExact(statement, surface, path));
|
|
1282
|
+
}
|
|
1283
|
+
function selectStatementCandidate(candidates, surface, value, path) {
|
|
1261
1284
|
if (!candidates.length) {
|
|
1262
1285
|
return null;
|
|
1263
1286
|
}
|
|
@@ -1632,7 +1655,10 @@ function normalizeComparablePath(path) {
|
|
|
1632
1655
|
.join('.');
|
|
1633
1656
|
}
|
|
1634
1657
|
function normalizeComparableToken(value) {
|
|
1635
|
-
|
|
1658
|
+
const cached = comparableTokenCache.get(value);
|
|
1659
|
+
if (cached !== undefined)
|
|
1660
|
+
return cached;
|
|
1661
|
+
const normalized = value
|
|
1636
1662
|
.trim()
|
|
1637
1663
|
.replace(/\s+/g, '')
|
|
1638
1664
|
.replace(/\[\*\]/g, '')
|
|
@@ -1641,6 +1667,12 @@ function normalizeComparableToken(value) {
|
|
|
1641
1667
|
.replace(/^.*->/, '')
|
|
1642
1668
|
.replace(/\|/g, '')
|
|
1643
1669
|
.toLowerCase();
|
|
1670
|
+
if (value.length <= 1024) {
|
|
1671
|
+
if (comparableTokenCache.size >= 2048)
|
|
1672
|
+
comparableTokenCache.clear();
|
|
1673
|
+
comparableTokenCache.set(value, normalized);
|
|
1674
|
+
}
|
|
1675
|
+
return normalized;
|
|
1644
1676
|
}
|
|
1645
1677
|
function relaxedFieldTargetMatchScore(statement, surface, path) {
|
|
1646
1678
|
if (!path.length) {
|
|
@@ -2379,7 +2411,9 @@ function partiallyMaskUnknownString(value) {
|
|
|
2379
2411
|
return normalizePrivacyReplacementSpacing(output);
|
|
2380
2412
|
}
|
|
2381
2413
|
function normalizePrivacyReplacementSpacing(value) {
|
|
2382
|
-
|
|
2414
|
+
// Most captured strings contain no replacement markers. The token regex
|
|
2415
|
+
// backtracks over long ordinary words even though no token can match.
|
|
2416
|
+
if (!value || (!value.includes('[') && !/_tok_/i.test(value))) {
|
|
2383
2417
|
return value;
|
|
2384
2418
|
}
|
|
2385
2419
|
const bracketMarker = String.raw `(?:\[(?:dropped|secret-token|connection-string|auth-token|session-secret|private-key|basic-auth|jwt|credit-card|phone|person|ssn|masked)\])`;
|
|
@@ -2891,12 +2925,14 @@ function coercePrivacyObjectId(value) {
|
|
|
2891
2925
|
if (typeof value !== 'object') {
|
|
2892
2926
|
return null;
|
|
2893
2927
|
}
|
|
2928
|
+
if (Array.isArray(value))
|
|
2929
|
+
return null;
|
|
2894
2930
|
const bsonType = typeof value?._bsontype === 'string' ? String(value._bsontype).toLowerCase() : '';
|
|
2895
2931
|
const ctorName = typeof value?.constructor?.name === 'string' ? String(value.constructor.name).toLowerCase() : '';
|
|
2896
2932
|
const looksLikeObjectId = bsonType.includes('objectid') ||
|
|
2897
2933
|
ctorName === 'objectid' ||
|
|
2898
2934
|
typeof value.toHexString === 'function';
|
|
2899
|
-
const fromBufferShape = coercePrivacyObjectIdFromBufferShape(value,
|
|
2935
|
+
const fromBufferShape = looksLikeObjectId ? coercePrivacyObjectIdFromBufferShape(value, true) : null;
|
|
2900
2936
|
if (fromBufferShape) {
|
|
2901
2937
|
return fromBufferShape;
|
|
2902
2938
|
}
|
|
@@ -2970,12 +3006,11 @@ function coercePrivacyObjectIdFromBufferShape(value, includeIdCandidate = false)
|
|
|
2970
3006
|
if (!Array.isArray(bytes) || bytes.length !== 12) {
|
|
2971
3007
|
return null;
|
|
2972
3008
|
}
|
|
2973
|
-
|
|
2974
|
-
if (normalized.some((byte) => !Number.isInteger(byte) || byte < 0 || byte > 255)) {
|
|
3009
|
+
if (bytes.some((byte) => typeof byte !== 'number' || !Number.isInteger(byte) || byte < 0 || byte > 255)) {
|
|
2975
3010
|
return null;
|
|
2976
3011
|
}
|
|
2977
3012
|
try {
|
|
2978
|
-
return Buffer.from(
|
|
3013
|
+
return Buffer.from(bytes).toString('hex');
|
|
2979
3014
|
}
|
|
2980
3015
|
catch {
|
|
2981
3016
|
return null;
|
|
@@ -9,13 +9,24 @@ function isTraceMaterializationWorkerRequest(value) {
|
|
|
9
9
|
const message = value;
|
|
10
10
|
return Number.isFinite(message.taskId) && message.type === 'materialize-trace-events' && !!message.payload;
|
|
11
11
|
}
|
|
12
|
+
let policyId;
|
|
13
|
+
let privacy;
|
|
14
|
+
let queue = Promise.resolve();
|
|
12
15
|
node_worker_threads_1.parentPort?.on('message', (message) => {
|
|
13
16
|
if (!isTraceMaterializationWorkerRequest(message)) {
|
|
14
17
|
return;
|
|
15
18
|
}
|
|
16
|
-
|
|
19
|
+
// FIFO also isolates the materializer's full-value-capture flag across tasks.
|
|
20
|
+
queue = queue.then(async () => {
|
|
17
21
|
try {
|
|
18
|
-
|
|
22
|
+
if (message.payload.privacy !== undefined) {
|
|
23
|
+
policyId = message.payload.policyId;
|
|
24
|
+
privacy = message.payload.privacy;
|
|
25
|
+
}
|
|
26
|
+
else if (policyId === undefined || policyId !== message.payload.policyId) {
|
|
27
|
+
throw new Error('trace materialization policy reference is unavailable');
|
|
28
|
+
}
|
|
29
|
+
const events = await (0, index_1.__materializePendingTraceEventsForWorker)({ ...message.payload, privacy });
|
|
19
30
|
node_worker_threads_1.parentPort?.postMessage({
|
|
20
31
|
taskId: message.taskId,
|
|
21
32
|
ok: true,
|
|
@@ -29,5 +40,5 @@ node_worker_threads_1.parentPort?.on('message', (message) => {
|
|
|
29
40
|
error: err instanceof Error ? err.message : String(err),
|
|
30
41
|
});
|
|
31
42
|
}
|
|
32
|
-
})
|
|
43
|
+
});
|
|
33
44
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@reproapp/node-sdk",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.54",
|
|
4
4
|
"description": "Repro Nest SDK",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -9,14 +9,15 @@
|
|
|
9
9
|
"node": ">=18"
|
|
10
10
|
},
|
|
11
11
|
"scripts": {
|
|
12
|
+
"test:throughput": "npm run build && node test/trace-worker-policy-reference.test.js && node test/privacy-policy-index.test.js && node test/trace-throughput.testcontainers.test.js",
|
|
12
13
|
"test:latency": "npm run build && node test/privacy-fetch-timeout.test.js && node test/capture-latency.testcontainers.test.js",
|
|
13
|
-
"test:live-capture": "npm run build && node test/live-capture-limits.test.js && node test/ingest-throttle-retention.test.js && node test/live-capture-limits.testcontainers.test.js",
|
|
14
|
+
"test:live-capture": "npm run build && node test/twelve-element-capture.testcontainers.test.js && node test/live-capture-limits.test.js && node test/ingest-throttle-retention.test.js && node test/live-capture-limits.testcontainers.test.js",
|
|
14
15
|
"test:live-brokers": "npm run build && node test/live-broker-limits.testcontainers.test.js",
|
|
15
16
|
"build": "tsc -p tsconfig.json",
|
|
16
17
|
"dev": "tsc -p tsconfig.json --watch --preserveWatchOutput",
|
|
17
18
|
"prepublishOnly": "npm run build",
|
|
18
|
-
"test": "npm run build && node test/semantic-serialization.test.js && node test/semantic-serialization-trace.test.js && node test/content-addressed-trace-values.test.js && node test/trace-observation-snapshots.test.js && node test/session-storage-budget.test.js && node test/trace-source-exclude.test.js && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/request-capture-background.test.js && node test/init-auto-http-server.test.js && node test/backend-live-mode.test.js && node test/client-performance-regression.test.js && node test/request-manifest-priority.test.js && node test/runtime-no-double-invoke.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node test/sdk-background-priority.test.js && node test/sdk-background-timeout-isolation.test.js && node test/runtime-service-registration.test.js && node test/ingest-queue-retention.test.js && node test/ingest-high-priority-retention.test.js && node test/ingest-high-priority-fairness.test.js && node test/ingest-critical-retention.test.js && node test/ingest-critical-retry-retention.test.js && node test/ingest-batching.test.js && node test/ingest-timeout.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/wrap-plugin-super-call.test.js && node test/function-arg-identity.test.js && node test/privacy-service-scope.test.js && node test/privacy-policy-index.test.js && node test/policy-capture-attestation.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js && node test/amqplib-integration.test.js && node test/mongodb-integration.test.js && node test/mongodb-esm-import-hook.test.js && node test/mongodb-suppression-cleanup.test.js && node test/postgresql-integration.test.js",
|
|
19
|
-
"test:containers": "npm run build && node test/background-continuous-traffic.testcontainers.test.js && node test/runtime-no-double-invoke.testcontainers.test.js && node test/wrap-plugin-super-call.testcontainers.test.js && node test/function-arg-identity.testcontainers.test.js && node test/mongodb-mongoose-native-suppression.testcontainers.test.js && node test/content-addressed-mongodb.testcontainers.test.js && node test/policy-session-completion.testcontainers.test.js && node test/postgresql-integrations.testcontainers.test.js"
|
|
19
|
+
"test": "npm run build && node test/semantic-serialization.test.js && node test/semantic-serialization-trace.test.js && node test/content-addressed-trace-values.test.js && node test/trace-observation-snapshots.test.js && node test/session-storage-budget.test.js && node test/trace-source-exclude.test.js && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/request-capture-background.test.js && node test/init-auto-http-server.test.js && node test/backend-live-mode.test.js && node test/client-performance-regression.test.js && node test/request-manifest-priority.test.js && node test/runtime-no-double-invoke.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node test/sdk-background-priority.test.js && node test/sdk-background-timeout-isolation.test.js && node test/runtime-service-registration.test.js && node test/ingest-queue-retention.test.js && node test/ingest-high-priority-retention.test.js && node test/ingest-high-priority-fairness.test.js && node test/ingest-critical-retention.test.js && node test/ingest-critical-retry-retention.test.js && node test/ingest-batching.test.js && node test/ingest-timeout.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/wrap-plugin-super-call.test.js && node test/function-arg-identity.test.js && node test/privacy-service-scope.test.js && node test/privacy-policy-index.test.js && node test/policy-capture-attestation.test.js && node test/object-id-shapes.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js && node test/amqplib-integration.test.js && node test/mongodb-integration.test.js && node test/mongodb-esm-import-hook.test.js && node test/mongodb-suppression-cleanup.test.js && node test/postgresql-integration.test.js",
|
|
20
|
+
"test:containers": "npm run build && node test/trace-worker-policy-reference.test.js && node test/trace-throughput.testcontainers.test.js && node test/twelve-element-capture.testcontainers.test.js && node test/background-continuous-traffic.testcontainers.test.js && node test/runtime-no-double-invoke.testcontainers.test.js && node test/wrap-plugin-super-call.testcontainers.test.js && node test/function-arg-identity.testcontainers.test.js && node test/mongodb-mongoose-native-suppression.testcontainers.test.js && node test/content-addressed-mongodb.testcontainers.test.js && node test/policy-session-completion.testcontainers.test.js && node test/postgresql-integrations.testcontainers.test.js"
|
|
20
21
|
},
|
|
21
22
|
"peerDependencies": {
|
|
22
23
|
"express": "^5.1.0",
|
package/src/index.ts
CHANGED
|
@@ -445,7 +445,10 @@ type TraceMaterializationWorkerPayload = {
|
|
|
445
445
|
type TraceMaterializationWorkerRequest = {
|
|
446
446
|
taskId: number;
|
|
447
447
|
type: 'materialize-trace-events';
|
|
448
|
-
payload: TraceMaterializationWorkerPayload
|
|
448
|
+
payload: Omit<TraceMaterializationWorkerPayload, 'privacy'> & {
|
|
449
|
+
privacy?: NormalizedRuntimePrivacyPolicy | null;
|
|
450
|
+
policyId: number;
|
|
451
|
+
};
|
|
449
452
|
};
|
|
450
453
|
|
|
451
454
|
type TraceMaterializationWorkerResponse =
|
|
@@ -1564,6 +1567,10 @@ let traceMaterializerWorkerDisabled = !TRACE_MATERIALIZE_OFFTHREAD_ENABLED;
|
|
|
1564
1567
|
let traceMaterializerTaskSeq = 0;
|
|
1565
1568
|
let traceMaterializerIdleTimer: NodeJS.Timeout | null = null;
|
|
1566
1569
|
const traceMaterializerExpectedTerminations = new WeakSet<object>();
|
|
1570
|
+
const traceMaterializerWorkerPolicies = new WeakMap<object, {
|
|
1571
|
+
policy: NormalizedRuntimePrivacyPolicy | null;
|
|
1572
|
+
id: number;
|
|
1573
|
+
}>();
|
|
1567
1574
|
const traceMaterializerPending = new Map<number, {
|
|
1568
1575
|
resolve: (events: PendingTraceEventRecord[]) => void;
|
|
1569
1576
|
reject: (error: Error) => void;
|
|
@@ -1771,12 +1778,22 @@ async function materializePendingTraceEventsOffThread(
|
|
|
1771
1778
|
});
|
|
1772
1779
|
|
|
1773
1780
|
try {
|
|
1781
|
+
const previousPolicy = traceMaterializerWorkerPolicies.get(worker);
|
|
1782
|
+
const sendPolicy = !previousPolicy || previousPolicy.policy !== payload.privacy;
|
|
1783
|
+
const policyState = sendPolicy ? { policy: payload.privacy, id: taskId } : previousPolicy!;
|
|
1784
|
+
const { policy: _inlinePolicy, ...privacyConfig } = payload.cfg.privacy ?? {};
|
|
1774
1785
|
const request: TraceMaterializationWorkerRequest = {
|
|
1775
1786
|
taskId,
|
|
1776
1787
|
type: 'materialize-trace-events',
|
|
1777
|
-
payload
|
|
1788
|
+
payload: {
|
|
1789
|
+
...payload,
|
|
1790
|
+
cfg: { ...payload.cfg, privacy: privacyConfig },
|
|
1791
|
+
privacy: sendPolicy ? payload.privacy : undefined,
|
|
1792
|
+
policyId: policyState.id,
|
|
1793
|
+
},
|
|
1778
1794
|
};
|
|
1779
1795
|
worker.postMessage(request);
|
|
1796
|
+
traceMaterializerWorkerPolicies.set(worker, policyState);
|
|
1780
1797
|
} catch (err) {
|
|
1781
1798
|
traceMaterializerPending.delete(taskId);
|
|
1782
1799
|
try { clearTimeout(timeout); } catch {}
|
|
@@ -2906,6 +2923,8 @@ function coerceMongoId(value: any): string | null {
|
|
|
2906
2923
|
if (typeof value !== 'object') {
|
|
2907
2924
|
return null;
|
|
2908
2925
|
}
|
|
2926
|
+
// Argument lists and query-result arrays are business data, not BSON ids.
|
|
2927
|
+
if (Array.isArray(value)) return null;
|
|
2909
2928
|
|
|
2910
2929
|
const bsonType = typeof (value as any)?._bsontype === 'string' ? String((value as any)._bsontype).toLowerCase() : '';
|
|
2911
2930
|
const ctorName = typeof value?.constructor?.name === 'string' ? String(value.constructor.name).toLowerCase() : '';
|
|
@@ -2914,7 +2933,7 @@ function coerceMongoId(value: any): string | null {
|
|
|
2914
2933
|
ctorName === 'objectid' ||
|
|
2915
2934
|
typeof (value as any).toHexString === 'function';
|
|
2916
2935
|
|
|
2917
|
-
const fromBufferShape = coerceMongoIdFromBufferShape(value,
|
|
2936
|
+
const fromBufferShape = looksLikeObjectId ? coerceMongoIdFromBufferShape(value, true) : null;
|
|
2918
2937
|
if (fromBufferShape) {
|
|
2919
2938
|
return fromBufferShape;
|
|
2920
2939
|
}
|
|
@@ -2994,13 +3013,12 @@ function coerceMongoIdFromBufferShape(value: any, includeIdCandidate: boolean =
|
|
|
2994
3013
|
return null;
|
|
2995
3014
|
}
|
|
2996
3015
|
|
|
2997
|
-
|
|
2998
|
-
if (normalized.some((byte) => !Number.isInteger(byte) || byte < 0 || byte > 255)) {
|
|
3016
|
+
if (bytes.some((byte) => typeof byte !== 'number' || !Number.isInteger(byte) || byte < 0 || byte > 255)) {
|
|
2999
3017
|
return null;
|
|
3000
3018
|
}
|
|
3001
3019
|
|
|
3002
3020
|
try {
|
|
3003
|
-
return Buffer.from(
|
|
3021
|
+
return Buffer.from(bytes).toString('hex');
|
|
3004
3022
|
} catch {
|
|
3005
3023
|
return null;
|
|
3006
3024
|
}
|
package/src/privacy.ts
CHANGED
|
@@ -97,6 +97,7 @@ type RuntimePrivacyStatementIndex = {
|
|
|
97
97
|
functionStatements: FunctionStatementTrieNode;
|
|
98
98
|
candidateCache: Map<string, NormalizedStatement[]>;
|
|
99
99
|
hasNumericTargetSegments: boolean;
|
|
100
|
+
scoredCandidateCache: Map<string, Array<{ statement: NormalizedStatement; targetScore: number; score: number }>>;
|
|
100
101
|
};
|
|
101
102
|
|
|
102
103
|
type TransformFallback = {
|
|
@@ -150,6 +151,7 @@ const EMPTY_RUNTIME_PRIVACY_POLICY: NormalizedRuntimePrivacyPolicy = {
|
|
|
150
151
|
};
|
|
151
152
|
|
|
152
153
|
const runtimePrivacyStatementIndexes = new WeakMap<object, RuntimePrivacyStatementIndex>();
|
|
154
|
+
const comparableTokenCache = new Map<string, string>();
|
|
153
155
|
const RUNTIME_PRIVACY_CANDIDATE_CACHE_MAX_ENTRIES = 2048;
|
|
154
156
|
const RUNTIME_PRIVACY_PROVENANCE_MAX_ENTRIES = 1000;
|
|
155
157
|
const RUNTIME_PRIVACY_PROVENANCE_MAX_SERIALIZED_LENGTH = 200_000;
|
|
@@ -685,6 +687,7 @@ function buildRuntimePrivacyStatementIndex(
|
|
|
685
687
|
functionStatements,
|
|
686
688
|
candidateCache: new Map<string, NormalizedStatement[]>(),
|
|
687
689
|
hasNumericTargetSegments,
|
|
690
|
+
scoredCandidateCache: new Map(),
|
|
688
691
|
};
|
|
689
692
|
}
|
|
690
693
|
|
|
@@ -1222,6 +1225,7 @@ function rememberSerializedRuntimePrivacyProvenance(value: any, provenance: Runt
|
|
|
1222
1225
|
}
|
|
1223
1226
|
|
|
1224
1227
|
function lookupSerializedRuntimePrivacyProvenance(value: any): TransformFallback {
|
|
1228
|
+
if (serializedRuntimePrivacyProvenance.size === 0) return null;
|
|
1225
1229
|
for (const serialized of runtimePrivacyProvenanceSerializations(value)) {
|
|
1226
1230
|
const provenance = serializedRuntimePrivacyProvenance.get(hashRuntimePrivacyProvenance(serialized));
|
|
1227
1231
|
if (provenance) {
|
|
@@ -1691,6 +1695,32 @@ function matchStatement(
|
|
|
1691
1695
|
context: RuntimePrivacyApplyContext,
|
|
1692
1696
|
path: string[],
|
|
1693
1697
|
): NormalizedStatement | null {
|
|
1698
|
+
const index = getRuntimePrivacyStatementIndex(policy);
|
|
1699
|
+
// Scores and non-pattern selectors depend on the context/path, not the
|
|
1700
|
+
// observed value. Preserve numeric positions here even for wildcard rules.
|
|
1701
|
+
const key = JSON.stringify([surface, context.trace?.fn, context.routeKey,
|
|
1702
|
+
context.db?.type, context.db?.collection, context.db?.op, path]);
|
|
1703
|
+
let scored = index.scoredCandidateCache.get(key);
|
|
1704
|
+
if (!scored) {
|
|
1705
|
+
scored = scoreStatementCandidates(policy, surface, context, path);
|
|
1706
|
+
if (index.scoredCandidateCache.size >= RUNTIME_PRIVACY_CANDIDATE_CACHE_MAX_ENTRIES) {
|
|
1707
|
+
index.scoredCandidateCache.clear();
|
|
1708
|
+
}
|
|
1709
|
+
index.scoredCandidateCache.set(key, scored);
|
|
1710
|
+
}
|
|
1711
|
+
const leaf = path[path.length - 1] ?? '';
|
|
1712
|
+
const joinedPath = normalizeComparablePath(path);
|
|
1713
|
+
const candidates = scored.filter(({ statement }) => statement.scope !== 'PATTERN'
|
|
1714
|
+
|| patternMatches(statement.pattern, leaf, joinedPath, value));
|
|
1715
|
+
return selectStatementCandidate(candidates, surface, value, path);
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
function scoreStatementCandidates(
|
|
1719
|
+
policy: NormalizedRuntimePrivacyPolicy,
|
|
1720
|
+
surface: RuntimePrivacySurface,
|
|
1721
|
+
context: RuntimePrivacyApplyContext,
|
|
1722
|
+
path: string[],
|
|
1723
|
+
): Array<{ statement: NormalizedStatement; targetScore: number; score: number }> {
|
|
1694
1724
|
let candidates: Array<{ statement: NormalizedStatement; targetScore: number; score: number }> = [];
|
|
1695
1725
|
for (const statement of candidateStatementsForMatch(policy, surface, context, path)) {
|
|
1696
1726
|
if (shouldIgnoreBroadContainerRuleOnDescendant(statement, surface, path)) {
|
|
@@ -1703,7 +1733,7 @@ function matchStatement(
|
|
|
1703
1733
|
if (targetScore < 0) {
|
|
1704
1734
|
continue;
|
|
1705
1735
|
}
|
|
1706
|
-
if (!matchesScope(statement, surface,
|
|
1736
|
+
if (statement.scope !== 'PATTERN' && !matchesScope(statement, surface, undefined, context, path)) {
|
|
1707
1737
|
continue;
|
|
1708
1738
|
}
|
|
1709
1739
|
const selectorScore = matchSelectorSpecificity(statement, surface, context, path);
|
|
@@ -1718,11 +1748,15 @@ function matchStatement(
|
|
|
1718
1748
|
statement.priority,
|
|
1719
1749
|
});
|
|
1720
1750
|
}
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
}
|
|
1751
|
+
return candidates.filter(({ statement }) => !isBroadContextKeepExact(statement, surface, path));
|
|
1752
|
+
}
|
|
1724
1753
|
|
|
1725
|
-
|
|
1754
|
+
function selectStatementCandidate(
|
|
1755
|
+
candidates: Array<{ statement: NormalizedStatement; targetScore: number; score: number }>,
|
|
1756
|
+
surface: RuntimePrivacySurface,
|
|
1757
|
+
value: any,
|
|
1758
|
+
path: string[],
|
|
1759
|
+
): NormalizedStatement | null {
|
|
1726
1760
|
if (!candidates.length) {
|
|
1727
1761
|
return null;
|
|
1728
1762
|
}
|
|
@@ -2197,7 +2231,9 @@ function normalizeComparablePath(path: string[]): string {
|
|
|
2197
2231
|
}
|
|
2198
2232
|
|
|
2199
2233
|
function normalizeComparableToken(value: string): string {
|
|
2200
|
-
|
|
2234
|
+
const cached = comparableTokenCache.get(value);
|
|
2235
|
+
if (cached !== undefined) return cached;
|
|
2236
|
+
const normalized = value
|
|
2201
2237
|
.trim()
|
|
2202
2238
|
.replace(/\s+/g, '')
|
|
2203
2239
|
.replace(/\[\*\]/g, '')
|
|
@@ -2206,6 +2242,11 @@ function normalizeComparableToken(value: string): string {
|
|
|
2206
2242
|
.replace(/^.*->/, '')
|
|
2207
2243
|
.replace(/\|/g, '')
|
|
2208
2244
|
.toLowerCase();
|
|
2245
|
+
if (value.length <= 1024) {
|
|
2246
|
+
if (comparableTokenCache.size >= 2048) comparableTokenCache.clear();
|
|
2247
|
+
comparableTokenCache.set(value, normalized);
|
|
2248
|
+
}
|
|
2249
|
+
return normalized;
|
|
2209
2250
|
}
|
|
2210
2251
|
|
|
2211
2252
|
function relaxedFieldTargetMatchScore(
|
|
@@ -3082,7 +3123,9 @@ function partiallyMaskUnknownString(value: string): string {
|
|
|
3082
3123
|
}
|
|
3083
3124
|
|
|
3084
3125
|
function normalizePrivacyReplacementSpacing(value: string): string {
|
|
3085
|
-
|
|
3126
|
+
// Most captured strings contain no replacement markers. The token regex
|
|
3127
|
+
// backtracks over long ordinary words even though no token can match.
|
|
3128
|
+
if (!value || (!value.includes('[') && !/_tok_/i.test(value))) {
|
|
3086
3129
|
return value;
|
|
3087
3130
|
}
|
|
3088
3131
|
const bracketMarker = String.raw`(?:\[(?:dropped|secret-token|connection-string|auth-token|session-secret|private-key|basic-auth|jwt|credit-card|phone|person|ssn|masked)\])`;
|
|
@@ -3685,6 +3728,7 @@ function coercePrivacyObjectId(value: any): string | null {
|
|
|
3685
3728
|
if (typeof value !== 'object') {
|
|
3686
3729
|
return null;
|
|
3687
3730
|
}
|
|
3731
|
+
if (Array.isArray(value)) return null;
|
|
3688
3732
|
|
|
3689
3733
|
const bsonType = typeof (value as any)?._bsontype === 'string' ? String((value as any)._bsontype).toLowerCase() : '';
|
|
3690
3734
|
const ctorName = typeof value?.constructor?.name === 'string' ? String(value.constructor.name).toLowerCase() : '';
|
|
@@ -3693,7 +3737,7 @@ function coercePrivacyObjectId(value: any): string | null {
|
|
|
3693
3737
|
ctorName === 'objectid' ||
|
|
3694
3738
|
typeof value.toHexString === 'function';
|
|
3695
3739
|
|
|
3696
|
-
const fromBufferShape = coercePrivacyObjectIdFromBufferShape(value,
|
|
3740
|
+
const fromBufferShape = looksLikeObjectId ? coercePrivacyObjectIdFromBufferShape(value, true) : null;
|
|
3697
3741
|
if (fromBufferShape) {
|
|
3698
3742
|
return fromBufferShape;
|
|
3699
3743
|
}
|
|
@@ -3774,13 +3818,12 @@ function coercePrivacyObjectIdFromBufferShape(value: any, includeIdCandidate: bo
|
|
|
3774
3818
|
return null;
|
|
3775
3819
|
}
|
|
3776
3820
|
|
|
3777
|
-
|
|
3778
|
-
if (normalized.some((byte) => !Number.isInteger(byte) || byte < 0 || byte > 255)) {
|
|
3821
|
+
if (bytes.some((byte) => typeof byte !== 'number' || !Number.isInteger(byte) || byte < 0 || byte > 255)) {
|
|
3779
3822
|
return null;
|
|
3780
3823
|
}
|
|
3781
3824
|
|
|
3782
3825
|
try {
|
|
3783
|
-
return Buffer.from(
|
|
3826
|
+
return Buffer.from(bytes).toString('hex');
|
|
3784
3827
|
} catch {
|
|
3785
3828
|
return null;
|
|
3786
3829
|
}
|
|
@@ -15,14 +15,24 @@ function isTraceMaterializationWorkerRequest(value: unknown): value is TraceMate
|
|
|
15
15
|
return Number.isFinite(message.taskId) && message.type === 'materialize-trace-events' && !!message.payload;
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
let policyId: number | undefined;
|
|
19
|
+
let privacy: any;
|
|
20
|
+
let queue = Promise.resolve();
|
|
18
21
|
parentPort?.on('message', (message: unknown) => {
|
|
19
22
|
if (!isTraceMaterializationWorkerRequest(message)) {
|
|
20
23
|
return;
|
|
21
24
|
}
|
|
22
25
|
|
|
23
|
-
|
|
26
|
+
// FIFO also isolates the materializer's full-value-capture flag across tasks.
|
|
27
|
+
queue = queue.then(async () => {
|
|
24
28
|
try {
|
|
25
|
-
|
|
29
|
+
if (message.payload.privacy !== undefined) {
|
|
30
|
+
policyId = message.payload.policyId;
|
|
31
|
+
privacy = message.payload.privacy;
|
|
32
|
+
} else if (policyId === undefined || policyId !== message.payload.policyId) {
|
|
33
|
+
throw new Error('trace materialization policy reference is unavailable');
|
|
34
|
+
}
|
|
35
|
+
const events = await __materializePendingTraceEventsForWorker({ ...message.payload, privacy });
|
|
26
36
|
parentPort?.postMessage({
|
|
27
37
|
taskId: message.taskId,
|
|
28
38
|
ok: true,
|
|
@@ -35,5 +45,5 @@ parentPort?.on('message', (message: unknown) => {
|
|
|
35
45
|
error: err instanceof Error ? err.message : String(err),
|
|
36
46
|
});
|
|
37
47
|
}
|
|
38
|
-
})
|
|
48
|
+
});
|
|
39
49
|
});
|
|
@@ -70,6 +70,7 @@ async function main() {
|
|
|
70
70
|
const capture = scoped();
|
|
71
71
|
const manifest = capture.find(e => e.event_type === 'request_capture_manifest')?.payload.requestCaptureManifest;
|
|
72
72
|
return manifest && ['backend_request', 'db_change'].every(t => capture.some(e => e.event_type === t))
|
|
73
|
+
&& capture.flatMap(e => e.payload.db || []).length === manifest.expected.dbChange
|
|
73
74
|
&& capture.filter(e => e.event_type === 'trace_batch').length >= manifest.expected.traceBatch;
|
|
74
75
|
};
|
|
75
76
|
while (!complete() && Date.now() < deadline) await sleep(25);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
function observeRows(document) { return document; }
|
|
2
2
|
async function loadRows(Model) {
|
|
3
3
|
const document = await Model.findOne({ reference: 'LATENCY' }).lean().exec();
|
|
4
|
-
for (let i = 0; i < 12; i++) {
|
|
4
|
+
for (let i = 0; i < Number(process.env.CAPTURE_TEST_OBSERVATIONS || 12); i++) {
|
|
5
5
|
document.phase = i;
|
|
6
6
|
if (observeRows(document) !== document) throw new Error('return identity changed');
|
|
7
7
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const workerThreads = require('node:worker_threads');
|
|
2
|
-
const metrics = { workers: 0, tasks: 0 };
|
|
2
|
+
const metrics = { workers: 0, tasks: 0, policyTransfers: 0 };
|
|
3
3
|
const RealWorker = workerThreads.Worker;
|
|
4
4
|
workerThreads.Worker = class extends RealWorker {
|
|
5
5
|
constructor(...args) {
|
|
@@ -10,13 +10,17 @@ workerThreads.Worker = class extends RealWorker {
|
|
|
10
10
|
if (String(args[0]).includes('trace-materializer-worker')) {
|
|
11
11
|
metrics.workers++;
|
|
12
12
|
const post = this.postMessage.bind(this);
|
|
13
|
-
this.postMessage = (...messages) => {
|
|
13
|
+
this.postMessage = (...messages) => {
|
|
14
|
+
metrics.tasks++;
|
|
15
|
+
if (messages[0]?.payload?.privacy !== undefined) metrics.policyTransfers++;
|
|
16
|
+
return post(...messages);
|
|
17
|
+
};
|
|
14
18
|
}
|
|
15
19
|
}
|
|
16
20
|
};
|
|
17
21
|
const http = require('node:http');
|
|
18
22
|
const mode = process.env.CAPTURE_TEST_MODE;
|
|
19
|
-
const { initRepro
|
|
23
|
+
const { initRepro } = require(process.env.CAPTURE_TEST_SDK || '../../dist');
|
|
20
24
|
async function main() {
|
|
21
25
|
const config = {
|
|
22
26
|
tenantId: 'TENANT_latency', appId: 'APP_latency', appSecret: 'test-only',
|
|
@@ -31,7 +35,6 @@ async function main() {
|
|
|
31
35
|
const express = require('express');
|
|
32
36
|
const connection = await mongoose.createConnection(process.env.CAPTURE_TEST_MONGO).asPromise();
|
|
33
37
|
const schema = new mongoose.Schema({ reference: String, phase: Number, rows: [Object] });
|
|
34
|
-
if (mode !== 'off') schema.plugin(reproMongoosePlugin, config);
|
|
35
38
|
const Model = connection.model('Latency', schema);
|
|
36
39
|
await Model.create({ reference: 'LATENCY', phase: -1, rows: Array.from({ length: 120 }, (_, id) => ({
|
|
37
40
|
id, status: 'available', internalSecret: `secret-${id}`, description: 'Operational depot description '.repeat(32),
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
const assert = require('node:assert/strict');
|
|
2
|
+
const { ObjectId } = require('mongodb');
|
|
3
|
+
const { __reproTestHooks } = require('../dist');
|
|
4
|
+
const { applyRuntimePrivacyPolicy, applyRuntimePrivacyPolicyAsync } = require('../dist/privacy');
|
|
5
|
+
|
|
6
|
+
async function main() {
|
|
7
|
+
const serializers = {
|
|
8
|
+
trace: value => __reproTestHooks.sanitizeTraceValueForPrivacyForTest(value),
|
|
9
|
+
policy: value => applyRuntimePrivacyPolicy(null, 'trace.args', value, { appId: 'APP_test' }),
|
|
10
|
+
asyncPolicy: value => applyRuntimePrivacyPolicyAsync(null, 'db.resultMeta', value, { appId: 'APP_test' }),
|
|
11
|
+
};
|
|
12
|
+
const bytes = Array.from({ length: 12 }, (_, i) => i);
|
|
13
|
+
const hex = '000102030405060708090a0b';
|
|
14
|
+
for (const [name, serialize] of Object.entries(serializers)) {
|
|
15
|
+
const rows = bytes.map(ordinal => Object.assign(Object.create(null), { ordinal, status: 'active' }));
|
|
16
|
+
const numericKeys = Object.fromEntries(bytes.map(i => [i, i]));
|
|
17
|
+
for (const input of [bytes, rows, numericKeys, { buffer: bytes }, { type: 'Buffer', data: bytes }]) {
|
|
18
|
+
const result = await serialize(input);
|
|
19
|
+
assert.deepEqual(JSON.parse(JSON.stringify(result)), JSON.parse(JSON.stringify(input)),
|
|
20
|
+
`${name}: ordinary data must not be collapsed to a BSON id`);
|
|
21
|
+
}
|
|
22
|
+
for (const input of [new ObjectId(hex), { $oid: hex }, { _bsontype: 'ObjectId', buffer: bytes }]) {
|
|
23
|
+
assert.equal(await serialize(input), hex, `${name}: genuine ObjectId representation must remain supported`);
|
|
24
|
+
}
|
|
25
|
+
let coercions = 0;
|
|
26
|
+
const invalidByte = { valueOf() { coercions += 1; throw new Error('must not coerce application objects'); } };
|
|
27
|
+
for (const invalid of [invalidByte, Object.create(null), '1', -1, 256, 1.5, NaN]) {
|
|
28
|
+
const input = { _bsontype: 'ObjectId', buffer: [...bytes.slice(0, 11), invalid] };
|
|
29
|
+
assert.equal(await serialize(input), '[mongo-id]', `${name}: invalid bytes must not become a fabricated ObjectId`);
|
|
30
|
+
}
|
|
31
|
+
assert.equal(coercions, 0);
|
|
32
|
+
}
|
|
33
|
+
console.log('ObjectId discrimination passed for trace, synchronous and asynchronous privacy serialization');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
main().catch(error => { console.error(error); process.exitCode = 1; });
|
|
@@ -185,6 +185,21 @@ async function main() {
|
|
|
185
185
|
);
|
|
186
186
|
assert.match(transformedPositionalTrace[4], /^4_tok_[a-f0-9]{12}$/);
|
|
187
187
|
|
|
188
|
+
// Cached selector scores must not cache value-dependent PATTERN decisions,
|
|
189
|
+
// numeric targets, or leak a previous route/function/database context.
|
|
190
|
+
const contexts = [
|
|
191
|
+
{ trace: { fn: 'CapturedService.loadNames' }, routeKey: 'GET /api/v1/depots/:id/names' },
|
|
192
|
+
{ trace: { fn: 'OtherService.load' }, routeKey: 'POST /other', db: { type: 'mongo', collection: 'other', op: 'insert' } },
|
|
193
|
+
];
|
|
194
|
+
for (let i = 0; i < 8; i++) {
|
|
195
|
+
const value = { data: [{ notes: i % 2 ? 'ordinary note' : 'secret-value', privateNote: 'private' }] };
|
|
196
|
+
const context = { appId: 'APP_test', appSecret: 'secret', ...contexts[i % 2] };
|
|
197
|
+
const warm = await applyRuntimePrivacyPolicyAsync(policy, 'response.body', value, context);
|
|
198
|
+
const cold = await applyRuntimePrivacyPolicyAsync(structuredClone(policy), 'response.body', value, context);
|
|
199
|
+
assert.deepEqual(warm, cold);
|
|
200
|
+
assert.equal(warm.data[0].notes, i % 2 ? 'ordinary note' : '[dropped]');
|
|
201
|
+
assert.equal(value.data[0].privateNote, 'private');
|
|
202
|
+
}
|
|
188
203
|
console.log(`privacy policy index OK (${elapsedMs}ms)`);
|
|
189
204
|
}
|
|
190
205
|
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
const assert = require('node:assert/strict');
|
|
2
|
+
const http = require('node:http');
|
|
3
|
+
const { fork } = require('node:child_process');
|
|
4
|
+
const { once } = require('node:events');
|
|
5
|
+
const { performance } = require('node:perf_hooks');
|
|
6
|
+
const { GenericContainer, Wait } = require('testcontainers');
|
|
7
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
8
|
+
|
|
9
|
+
const policy = { statements: [
|
|
10
|
+
...Array.from({ length: 5000 }, (_, i) => ({ sid: `fn-${i}`, when: {
|
|
11
|
+
scope: 'FUNCTION', selector: `Service${i}.method`, target: 'trace.args|trace.returnValue',
|
|
12
|
+
}, apply: { action: 'KEEP_EXACT' } })),
|
|
13
|
+
...Array.from({ length: 9 }, (_, i) => ({ sid: `pattern-${i}`, when: {
|
|
14
|
+
scope: 'PATTERN', selector: `pattern-${i}`, pattern: `sensitive-value-${i}`,
|
|
15
|
+
}, apply: { action: 'DROP' } })),
|
|
16
|
+
{ sid: 'secret', when: { scope: 'FIELD', selector: 'field:internalSecret' }, apply: { action: 'DROP' } },
|
|
17
|
+
] };
|
|
18
|
+
|
|
19
|
+
async function main() {
|
|
20
|
+
let events = [], child, mongo;
|
|
21
|
+
const sink = http.createServer((req, res) => {
|
|
22
|
+
const chunks = [];
|
|
23
|
+
req.on('data', chunk => chunks.push(chunk));
|
|
24
|
+
req.on('end', () => {
|
|
25
|
+
events.push(...JSON.parse(Buffer.concat(chunks)).events);
|
|
26
|
+
setTimeout(() => { res.writeHead(202); res.end('{}'); }, 10);
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
await new Promise(resolve => sink.listen(0, '127.0.0.1', resolve));
|
|
30
|
+
try {
|
|
31
|
+
mongo = await new GenericContainer('mongo:7').withExposedPorts(27017)
|
|
32
|
+
.withWaitStrategy(Wait.forListeningPorts()).start();
|
|
33
|
+
for (const mode of ['live', 'staging']) {
|
|
34
|
+
events = [];
|
|
35
|
+
child = fork(require.resolve('./fixtures/capture-latency-server'), [], {
|
|
36
|
+
env: { ...process.env, CAPTURE_TEST_MODE: mode, CAPTURE_TEST_OBSERVATIONS: '128',
|
|
37
|
+
CAPTURE_TEST_MONGO: `mongodb://${mongo.getHost()}:${mongo.getMappedPort(27017)}/throughput_${mode}`,
|
|
38
|
+
CAPTURE_TEST_SINK: `http://127.0.0.1:${sink.address().port}`, CAPTURE_TEST_POLICY: JSON.stringify(policy),
|
|
39
|
+
REPRO_SDK_BACKGROUND_IDLE_DELAY_MS: '0', REPRO_SDK_BACKGROUND_BUSY_IDLE_DELAY_MS: '0',
|
|
40
|
+
REPRO_SDK_BACKGROUND_MAX_DEFER_MS: '0', TRACE_IDLE_FLUSH_MS: '10', TRACE_LINGER_AFTER_FINISH_MS: '0',
|
|
41
|
+
}, stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
|
42
|
+
});
|
|
43
|
+
let stderr = '';
|
|
44
|
+
child.stderr.on('data', chunk => { stderr += chunk; });
|
|
45
|
+
const ready = await Promise.race([once(child, 'message').then(([v]) => v),
|
|
46
|
+
once(child, 'exit').then(() => { throw new Error(stderr); })]);
|
|
47
|
+
const base = `http://127.0.0.1:${ready.port}`;
|
|
48
|
+
const sessionIds = [0, 1].map(i => `S_throughput_${mode}_${i}`);
|
|
49
|
+
const started = performance.now();
|
|
50
|
+
const responses = await Promise.all(sessionIds.map((sid, i) =>
|
|
51
|
+
fetch(`${base}/names?__repro_sid=${sid}&__repro_aid=A_${i}&__repro_mode=${mode}`).then(r => r.json())));
|
|
52
|
+
const responseMs = performance.now() - started;
|
|
53
|
+
for (const response of responses) {
|
|
54
|
+
assert.equal(response.phase, 127);
|
|
55
|
+
assert.equal(response.rows.length, 120);
|
|
56
|
+
assert.equal(response.rows[0].internalSecret, 'secret-0');
|
|
57
|
+
}
|
|
58
|
+
const complete = sid => {
|
|
59
|
+
const scoped = events.filter(e => e.session_id === sid);
|
|
60
|
+
const manifest = scoped.find(e => e.event_type === 'request_capture_manifest')?.payload.requestCaptureManifest;
|
|
61
|
+
return manifest && scoped.filter(e => e.event_type === 'trace_batch').length === manifest.expected.traceBatch
|
|
62
|
+
&& scoped.filter(e => e.event_type === 'backend_request').length === manifest.expected.backendRequest
|
|
63
|
+
&& scoped.reduce((sum, e) => sum + (Array.isArray(e.payload.db) ? e.payload.db.length : 0), 0) === manifest.expected.dbChange;
|
|
64
|
+
};
|
|
65
|
+
let maxHealthMs = 0;
|
|
66
|
+
while (!sessionIds.every(complete) && performance.now() - started < 60000) {
|
|
67
|
+
const t = performance.now();
|
|
68
|
+
await (await fetch(`${base}/health`)).json();
|
|
69
|
+
maxHealthMs = Math.max(maxHealthMs, performance.now() - t);
|
|
70
|
+
await sleep(20);
|
|
71
|
+
}
|
|
72
|
+
assert(sessionIds.every(complete), `missing evidence: ${JSON.stringify(sessionIds.map(sid => {
|
|
73
|
+
const scoped = events.filter(e => e.session_id === sid);
|
|
74
|
+
return { sid, counts: scoped.reduce((out, e) => { out[e.event_type] = (out[e.event_type] || 0) + 1; return out; }, {}),
|
|
75
|
+
manifest: scoped.find(e => e.event_type === 'request_capture_manifest')?.payload.requestCaptureManifest };
|
|
76
|
+
}))} ${stderr.slice(-2000)}`);
|
|
77
|
+
const completeMs = performance.now() - started;
|
|
78
|
+
for (const sid of sessionIds) {
|
|
79
|
+
const scoped = events.filter(e => e.session_id === sid);
|
|
80
|
+
const manifest = scoped.find(e => e.event_type === 'request_capture_manifest').payload.requestCaptureManifest;
|
|
81
|
+
assert.equal(manifest.privacy.status, 'applied');
|
|
82
|
+
assert(!manifest.captureFailures?.length);
|
|
83
|
+
const values = new Map(scoped.flatMap(e => [...(e.payload.traceValues || []), ...(e.payload.requestValues || []), ...(e.payload.dbValues || [])]).map(v => [v.id, v.value]));
|
|
84
|
+
const trace = scoped.flatMap(e => e.payload.trace || []);
|
|
85
|
+
const observations = [...new Map(trace.filter(e => e.fn === 'observeRows' && e.line === 6 && e.type === 'enter')
|
|
86
|
+
.map(e => [e.spanId, e])).values()];
|
|
87
|
+
assert.equal(observations.length, 128);
|
|
88
|
+
const args = observations.map(e => values.get(e.argsValueCapture?.valueId) ?? e.args);
|
|
89
|
+
assert.deepEqual(args.map(v => v[0].phase), Array.from({ length: 128 }, (_, i) => i));
|
|
90
|
+
for (const [document] of args) {
|
|
91
|
+
assert.equal(document.rows.length, mode === 'live' ? 101 : 120);
|
|
92
|
+
assert.equal(document.rows[0].internalSecret, '[dropped]');
|
|
93
|
+
}
|
|
94
|
+
assert(!JSON.stringify(scoped).includes('secret-0'), 'capture leaked a protected value');
|
|
95
|
+
}
|
|
96
|
+
const metrics = await (await fetch(`${base}/health`)).json();
|
|
97
|
+
assert(metrics.tasks > 4);
|
|
98
|
+
assert(metrics.policyTransfers <= metrics.workers, 'full policy was transferred again for every trace batch');
|
|
99
|
+
assert(completeMs < 25000, `${mode}: post-response capture backlog took ${completeMs}ms`);
|
|
100
|
+
assert(maxHealthMs < 500, `${mode}: background capture blocked HTTP for ${maxHealthMs}ms`);
|
|
101
|
+
console.log(JSON.stringify({ mode, responseMs: Math.round(responseMs), completeMs: Math.round(completeMs), maxHealthMs: Math.round(maxHealthMs), ...metrics }));
|
|
102
|
+
const exited = once(child, 'exit'); child.send('stop'); await exited; child = null;
|
|
103
|
+
}
|
|
104
|
+
} finally {
|
|
105
|
+
if (child) { const exited = once(child, 'exit'); child.kill(); await exited; }
|
|
106
|
+
await new Promise(resolve => sink.close(resolve));
|
|
107
|
+
if (mongo) await mongo.stop();
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
main().catch(error => { console.error(error); process.exitCode = 1; });
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
const assert = require('node:assert/strict');
|
|
2
|
+
const { Worker } = require('node:worker_threads');
|
|
3
|
+
const { normalizeRuntimePrivacyPolicy } = require('../dist/privacy');
|
|
4
|
+
|
|
5
|
+
async function main() {
|
|
6
|
+
const worker = new Worker(require.resolve('../dist/trace-materializer-worker'));
|
|
7
|
+
let taskId = 0;
|
|
8
|
+
const pending = new Map();
|
|
9
|
+
worker.on('message', message => { pending.get(message.taskId)(message); pending.delete(message.taskId); });
|
|
10
|
+
const policy = action => normalizeRuntimePrivacyPolicy({ statements: [
|
|
11
|
+
{ sid: action, when: { scope: 'FIELD', selector: 'field:businessNote' }, apply: { action } },
|
|
12
|
+
] });
|
|
13
|
+
const call = (policyId, privacy, tenant = 'T', raw = false) => new Promise(resolve => {
|
|
14
|
+
const id = ++taskId;
|
|
15
|
+
pending.set(id, resolve);
|
|
16
|
+
worker.postMessage({ taskId: id, type: 'materialize-trace-events', payload: {
|
|
17
|
+
policyId, privacy, fullValueCaptureEnabled: true,
|
|
18
|
+
cfg: { tenantId: tenant, appId: tenant, appSecret: tenant, privacy: { captureRaw: raw } },
|
|
19
|
+
maskReq: { key: 'GET /names' }, masking: null, valueNamespace: tenant,
|
|
20
|
+
events: [{ type: 'enter', fn: 'load', __reproPending: { argsRaw: [{ businessNote: 'operational-value' }] } }],
|
|
21
|
+
} });
|
|
22
|
+
});
|
|
23
|
+
try {
|
|
24
|
+
const missing = await call(1);
|
|
25
|
+
assert.equal(missing.ok, false);
|
|
26
|
+
assert.match(missing.error, /policy reference/);
|
|
27
|
+
// Queue interleaved policies before any result arrives. Old references must
|
|
28
|
+
// not accidentally use the new policy or a concurrent task's raw flag.
|
|
29
|
+
const responses = await Promise.all([
|
|
30
|
+
call(1, policy('DROP')), call(1), call(2, policy('KEEP_EXACT'), 'Other'),
|
|
31
|
+
call(2, undefined, 'Other'), call(1), call(3, null, 'Raw', true),
|
|
32
|
+
call(4, policy('DROP')), call(4),
|
|
33
|
+
]);
|
|
34
|
+
for (const i of [0, 1, 6, 7]) {
|
|
35
|
+
assert.equal(responses[i].ok, true);
|
|
36
|
+
assert.equal(responses[i].events[0].args[0].businessNote, '[dropped]');
|
|
37
|
+
}
|
|
38
|
+
for (const i of [2, 3, 5]) assert.equal(responses[i].events[0].args[0].businessNote, 'operational-value');
|
|
39
|
+
assert.equal(responses[4].ok, false, 'stale reference silently used another policy');
|
|
40
|
+
} finally { await worker.terminate(); }
|
|
41
|
+
console.log('trace worker policy references, updates, missing refs, and raw/policy isolation OK');
|
|
42
|
+
}
|
|
43
|
+
main().catch(error => { console.error(error); process.exitCode = 1; });
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
process.env.REPRO_SDK_BACKGROUND_IDLE_DELAY_MS = '0';
|
|
2
|
+
process.env.REPRO_SDK_BACKGROUND_BUSY_IDLE_DELAY_MS = '0';
|
|
3
|
+
process.env.REPRO_SDK_BACKGROUND_MAX_DEFER_MS = '30';
|
|
4
|
+
process.env.TRACE_IDLE_FLUSH_MS = '10';
|
|
5
|
+
process.env.TRACE_LINGER_AFTER_FINISH_MS = '0';
|
|
6
|
+
const assert = require('node:assert/strict');
|
|
7
|
+
const http = require('node:http');
|
|
8
|
+
const express = require('express');
|
|
9
|
+
const mongoose = require('mongoose');
|
|
10
|
+
const { GenericContainer, Wait } = require('testcontainers');
|
|
11
|
+
const { initReproTracing, reproMiddleware, reproMongoosePlugin } = require('../dist');
|
|
12
|
+
const { flushIngestQueue } = require('../dist/ingest/client');
|
|
13
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
14
|
+
|
|
15
|
+
async function main() {
|
|
16
|
+
let container, connection, server, sink;
|
|
17
|
+
const posts = [];
|
|
18
|
+
try {
|
|
19
|
+
sink = http.createServer(async (req, res) => {
|
|
20
|
+
const parts = []; for await (const part of req) parts.push(part);
|
|
21
|
+
posts.push(JSON.parse(Buffer.concat(parts).toString()));
|
|
22
|
+
res.writeHead(202); res.end('{}');
|
|
23
|
+
});
|
|
24
|
+
await new Promise(resolve => sink.listen(0, '127.0.0.1', resolve));
|
|
25
|
+
container = await new GenericContainer('mongo:7').withExposedPorts(27017)
|
|
26
|
+
.withWaitStrategy(Wait.forListeningPorts()).start();
|
|
27
|
+
connection = await mongoose.createConnection(`mongodb://${container.getHost()}:${container.getMappedPort(27017)}/twelve_elements`).asPromise();
|
|
28
|
+
initReproTracing({ instrument: false, logFunctionCalls: false, fullValueCapture: true });
|
|
29
|
+
const app = express();
|
|
30
|
+
const policy = { environment: 'dev', statements: [{ sid: 'drop-secret',
|
|
31
|
+
when: { scope: 'FIELD', selector: 'field:internalSecret', target: '**.internalSecret' },
|
|
32
|
+
apply: { action: 'DROP' }, meta: { priority: 1 } }] };
|
|
33
|
+
for (const privacy of ['policy', 'raw']) {
|
|
34
|
+
const config = { tenantId: 'TENANT_twelve', appId: 'APP_twelve', appSecret: 'test',
|
|
35
|
+
serviceName: `depot-${privacy}`, captureHeaders: false,
|
|
36
|
+
privacy: { environment: 'dev', captureRaw: privacy === 'raw', ...(privacy === 'policy' ? { policy } : {}) },
|
|
37
|
+
ingestBase: `http://127.0.0.1:${sink.address().port}` };
|
|
38
|
+
const schema = new mongoose.Schema({ ordinal: Number, location: String, internalSecret: String });
|
|
39
|
+
schema.plugin(reproMongoosePlugin(config));
|
|
40
|
+
const Depot = connection.model(`Depot_${privacy}`, schema);
|
|
41
|
+
await Depot.insertMany(Array.from({ length: 13 }, (_, ordinal) => ({ ordinal, location: `Depot ${ordinal}`, internalSecret: 'private-fixture-secret' })));
|
|
42
|
+
app.use(`/${privacy}`, reproMiddleware(config));
|
|
43
|
+
app.get(`/${privacy}/:count`, async (req, res, next) => {
|
|
44
|
+
try {
|
|
45
|
+
const rows = await Depot.find({}).sort({ ordinal: 1 }).limit(Number(req.params.count)).lean().exec();
|
|
46
|
+
function projectDepots(input) { return input; }
|
|
47
|
+
const observed = global.__repro_call(projectDepots, null, [rows], __filename, 1, 'projectDepots', false);
|
|
48
|
+
function numericArguments(...values) { return values; }
|
|
49
|
+
const numbers = global.__repro_call(numericArguments, null, Array.from({ length: 12 }, (_, i) => i), __filename, 2, 'numericArguments', false);
|
|
50
|
+
res.json({ rows: observed, numbers });
|
|
51
|
+
} catch (error) { next(error); }
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
server = await new Promise(resolve => { const listener = app.listen(0, '127.0.0.1', () => resolve(listener)); });
|
|
55
|
+
const scenarios = [];
|
|
56
|
+
for (const mode of ['live', 'staging']) for (const privacy of ['policy', 'raw']) for (const count of [12, 11, 13]) {
|
|
57
|
+
const sid = `S_${mode}_${privacy}_${count}`;
|
|
58
|
+
const response = await fetch(`http://127.0.0.1:${server.address().port}/${privacy}/${count}`, { headers: {
|
|
59
|
+
'x-bug-session-id': sid, 'x-bug-action-id': `A_${sid}`, 'x-repro-capture-mode': mode,
|
|
60
|
+
} });
|
|
61
|
+
assert.equal(response.status, 200);
|
|
62
|
+
const data = await response.json();
|
|
63
|
+
assert.equal(data.rows.length, count);
|
|
64
|
+
assert.equal(data.rows[0].internalSecret, 'private-fixture-secret', 'SDK must not mutate business results');
|
|
65
|
+
assert.deepEqual(data.numbers, Array.from({ length: 12 }, (_, i) => i));
|
|
66
|
+
scenarios.push({ sid, mode, privacy, count });
|
|
67
|
+
}
|
|
68
|
+
let events;
|
|
69
|
+
const deadline = Date.now() + 30000;
|
|
70
|
+
do {
|
|
71
|
+
await flushIngestQueue();
|
|
72
|
+
events = posts.flatMap(post => post.events || []);
|
|
73
|
+
if (events.filter(e => e.event_type === 'request_capture_manifest').length >= scenarios.length) break;
|
|
74
|
+
await sleep(50);
|
|
75
|
+
} while (Date.now() < deadline);
|
|
76
|
+
for (const { sid, privacy, count } of scenarios) {
|
|
77
|
+
const scoped = events.filter(e => e.session_id === sid);
|
|
78
|
+
const manifest = scoped.find(e => e.event_type === 'request_capture_manifest')?.payload.requestCaptureManifest;
|
|
79
|
+
assert(manifest, `missing manifest ${sid}`);
|
|
80
|
+
assert.equal(manifest.captureFailures, undefined, `${sid}: ${JSON.stringify(manifest.captureFailures)}`);
|
|
81
|
+
if (privacy === 'policy') assert.equal(manifest.privacy.status, 'applied');
|
|
82
|
+
const values = scoped.flatMap(e => [...(e.payload.traceValues || []), ...(e.payload.requestValues || []), ...(e.payload.dbValues || [])]);
|
|
83
|
+
const resolve = (record, ref, field) => values.find(v => v.id === record?.[ref]?.valueId)?.value ?? record?.[field];
|
|
84
|
+
const trace = scoped.flatMap(e => e.payload.trace || []);
|
|
85
|
+
const args = resolve(trace.find(e => e.fn === 'projectDepots' && e.type === 'enter'), 'argsValueCapture', 'args');
|
|
86
|
+
const result = resolve(trace.find(e => e.fn === 'projectDepots' && e.type === 'exit'), 'returnValueCapture', 'returnValue');
|
|
87
|
+
for (const rows of [args?.[0], result]) {
|
|
88
|
+
assert.equal(rows?.length, count, `missing business array ${sid}`);
|
|
89
|
+
assert.equal(rows.at(-1).ordinal, count - 1);
|
|
90
|
+
assert.equal(rows[0].internalSecret, privacy === 'policy' ? '[dropped]' : 'private-fixture-secret');
|
|
91
|
+
}
|
|
92
|
+
const numbers = resolve(trace.find(e => e.fn === 'numericArguments' && e.type === 'enter'), 'argsValueCapture', 'args');
|
|
93
|
+
assert.deepEqual(numbers, Array.from({ length: 12 }, (_, i) => i), 'numeric argument list must not become an ObjectId');
|
|
94
|
+
const request = scoped.find(e => e.event_type === 'backend_request')?.payload.request;
|
|
95
|
+
const body = resolve(request, 'respBodyValueCapture', 'respBody');
|
|
96
|
+
assert.equal(body.rows.length, count);
|
|
97
|
+
const dbRead = scoped.flatMap(e => e.payload.db || []).find(e => e.op === 'find');
|
|
98
|
+
assert(dbRead, `missing Mongo read ${sid}`);
|
|
99
|
+
const dbResult = resolve(dbRead, 'resultMetaValueCapture', 'resultMeta');
|
|
100
|
+
const rows = Array.isArray(dbResult) ? dbResult : dbResult?.result;
|
|
101
|
+
assert.equal(rows?.length, count, `missing Mongo result ${sid}: ${JSON.stringify({ dbRead, dbResult })}`);
|
|
102
|
+
assert.equal(rows[0].internalSecret, privacy === 'policy' ? '[dropped]' : 'private-fixture-secret');
|
|
103
|
+
assert.equal(scoped.filter(e => e.event_type === 'trace_batch').length, manifest.expected.traceBatch);
|
|
104
|
+
assert.equal(scoped.flatMap(e => e.payload.db || []).length, manifest.expected.dbChange);
|
|
105
|
+
if (privacy === 'policy') assert(!JSON.stringify(scoped).includes('private-fixture-secret'), 'secret leaked into captured evidence');
|
|
106
|
+
}
|
|
107
|
+
console.log('12 live/staging policy/raw scenarios: business arrays, numeric arguments, Mongo results and manifests passed');
|
|
108
|
+
} finally {
|
|
109
|
+
if (server) await new Promise(resolve => server.close(resolve));
|
|
110
|
+
if (connection) await connection.close();
|
|
111
|
+
await flushIngestQueue();
|
|
112
|
+
if (sink) await new Promise(resolve => sink.close(resolve));
|
|
113
|
+
if (container) await container.stop();
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
main().catch(error => { console.error(error); process.exitCode = 1; });
|