@reproapp/node-sdk 0.0.48 → 0.0.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -1
- package/dist/index.d.ts +17 -0
- package/dist/index.js +140 -23
- package/dist/ingest/client.js +27 -2
- package/dist/live-capture-limits.d.ts +19 -0
- package/dist/live-capture-limits.js +207 -0
- package/dist/semantic-serialization.d.ts +1 -0
- package/dist/semantic-serialization.js +33 -1
- package/package.json +7 -2
- package/src/index.ts +211 -24
- package/src/ingest/client.ts +27 -3
- package/src/live-capture-limits.ts +177 -0
- package/src/semantic-serialization.ts +30 -0
- package/test/content-addressed-mongodb.testcontainers.test.js +23 -4
- package/test/fixtures/live-capture-limits-server.js +21 -0
- package/test/ingest-throttle-retention.test.js +41 -0
- package/test/kafka-runtime-privacy-policy.test.js +26 -2
- package/test/live-broker-limits.testcontainers.test.js +86 -0
- package/test/live-capture-limits.test.js +99 -0
- package/test/live-capture-limits.testcontainers.test.js +78 -0
- package/test/policy-session-completion.testcontainers.test.js +18 -1
- package/test/trace-observation-snapshots.test.js +403 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { IncomingMessage, ServerResponse } from 'http';
|
|
2
|
+
import { mongooseDocumentData, projectSemanticInfrastructureValue } from './semantic-serialization';
|
|
3
|
+
|
|
4
|
+
export type CaptureOmission = {
|
|
5
|
+
reason: 'max-depth' | 'max-keys' | 'max-entries' | 'accessor' | 'uninspectable';
|
|
6
|
+
path: string;
|
|
7
|
+
limit?: number;
|
|
8
|
+
originalCount?: number;
|
|
9
|
+
capturedCount: number;
|
|
10
|
+
};
|
|
11
|
+
export const LIVE_CAPTURE_LIMITS = Object.freeze({ maxDepth: 10, maxKeys: 100, maxEntries: 100 });
|
|
12
|
+
const marker = (omission: CaptureOmission) => ({ __kind: 'repro-omitted', complete: false, ...omission });
|
|
13
|
+
const childPath = (path: string, key: string | number) => `${path}[${JSON.stringify(key)}]`;
|
|
14
|
+
function parseStructuredString(value: unknown): any {
|
|
15
|
+
if (typeof value !== 'string' || !/^[\s]*[\[{]/.test(value)) return undefined;
|
|
16
|
+
try { const parsed = JSON.parse(value); return parsed && typeof parsed === 'object' ? parsed : undefined; }
|
|
17
|
+
catch { return undefined; }
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Snapshot only data descriptors: capture must never execute application getters. */
|
|
21
|
+
export function limitLiveCapture(value: unknown, rootPath = '$'): { value: any; omissions: CaptureOmission[] } {
|
|
22
|
+
const omissions: CaptureOmission[] = [];
|
|
23
|
+
const seen = new WeakMap<object, { path: string; depth: number }>();
|
|
24
|
+
const omit = (detail: CaptureOmission) => { omissions.push(detail); return marker(detail); };
|
|
25
|
+
const walk = (input: any, depth: number, path: string): any => {
|
|
26
|
+
if (input === null || input === undefined) return input;
|
|
27
|
+
if (typeof input === 'string') {
|
|
28
|
+
const parsed = parseStructuredString(input);
|
|
29
|
+
if (parsed === undefined) return input;
|
|
30
|
+
const before = omissions.length;
|
|
31
|
+
const bounded = walk(parsed, depth, path);
|
|
32
|
+
// Preserve the original string exactly unless its structured data exceeded a limit.
|
|
33
|
+
return omissions.length === before ? input : JSON.stringify(bounded);
|
|
34
|
+
}
|
|
35
|
+
if (typeof input === 'bigint' || typeof input === 'symbol') return String(input);
|
|
36
|
+
if (typeof input === 'function') return '[Function]';
|
|
37
|
+
if (typeof input !== 'object') return input;
|
|
38
|
+
const previous = seen.get(input);
|
|
39
|
+
if (previous && previous.depth <= depth) return { __reproCircularRef: previous.path };
|
|
40
|
+
seen.set(input, { path, depth });
|
|
41
|
+
const descriptor = (key: string) => Object.getOwnPropertyDescriptor(input, key);
|
|
42
|
+
const data = (key: string) => descriptor(key)?.value;
|
|
43
|
+
if (data('__kind') === 'repro-omitted') {
|
|
44
|
+
const detail = { reason: data('reason'), path: data('path'), limit: data('limit'), originalCount: data('originalCount'), capturedCount: data('capturedCount') };
|
|
45
|
+
omissions.push(detail);
|
|
46
|
+
return marker(detail);
|
|
47
|
+
}
|
|
48
|
+
if (input instanceof Date) return Number.isNaN(Date.prototype.getTime.call(input)) ? 'Invalid Date' : Date.prototype.toISOString.call(input);
|
|
49
|
+
if (input instanceof RegExp) return RegExp.prototype.toString.call(input);
|
|
50
|
+
if (input instanceof IncomingMessage) {
|
|
51
|
+
// The business body is its own root, not depth one in a socket graph.
|
|
52
|
+
return { __kind: 'http-request', method: data('method'), url: data('originalUrl') ?? data('url'),
|
|
53
|
+
headers: walk(data('headers') ?? Object.getOwnPropertyDescriptor(IncomingMessage.prototype, 'headers')?.get?.call(input), 0, childPath(path, 'headers')),
|
|
54
|
+
query: walk(data('query'), 0, childPath(path, 'query')),
|
|
55
|
+
body: walk(data('body'), 0, childPath(path, 'body')),
|
|
56
|
+
params: walk(data('params'), 0, childPath(path, 'params')) };
|
|
57
|
+
}
|
|
58
|
+
if (input instanceof ServerResponse) return { __kind: 'http-response', statusCode: data('statusCode') };
|
|
59
|
+
const projected = projectSemanticInfrastructureValue(input);
|
|
60
|
+
if (projected) return projected.value;
|
|
61
|
+
// Mongoose stores business fields in _doc; never recurse through $__ state.
|
|
62
|
+
const documentData = mongooseDocumentData(input);
|
|
63
|
+
if (documentData) return walk(documentData, depth, path);
|
|
64
|
+
if (depth > LIVE_CAPTURE_LIMITS.maxDepth) return omit({ reason: 'max-depth', path, limit: LIVE_CAPTURE_LIMITS.maxDepth, capturedCount: 0 });
|
|
65
|
+
if (input instanceof Error) {
|
|
66
|
+
const details: Record<string, any> = Object.create(null);
|
|
67
|
+
for (const key of ['message', 'stack', 'cause', 'errors', ...Object.keys(input)]) {
|
|
68
|
+
const item = descriptor(key);
|
|
69
|
+
if (item) Object.defineProperty(details, key, { ...item, enumerable: true, configurable: true });
|
|
70
|
+
}
|
|
71
|
+
return walk(details, depth, path);
|
|
72
|
+
}
|
|
73
|
+
if (input instanceof ArrayBuffer || (typeof SharedArrayBuffer !== 'undefined' && input instanceof SharedArrayBuffer) || input instanceof DataView) {
|
|
74
|
+
let bytes: Uint8Array;
|
|
75
|
+
if (input instanceof DataView) {
|
|
76
|
+
const read = (key: string) => Object.getOwnPropertyDescriptor(DataView.prototype, key)!.get!.call(input);
|
|
77
|
+
bytes = new Uint8Array(read('buffer'), read('byteOffset'), read('byteLength'));
|
|
78
|
+
} else bytes = new Uint8Array(input);
|
|
79
|
+
return { __type: input instanceof DataView ? 'DataView' : 'ArrayBuffer', values: walk(bytes, depth, path).values };
|
|
80
|
+
}
|
|
81
|
+
if (input instanceof Map) {
|
|
82
|
+
const entries: any[] = [];
|
|
83
|
+
const size = Object.getOwnPropertyDescriptor(Map.prototype, 'size')!.get!.call(input);
|
|
84
|
+
for (const [key, item] of Map.prototype.entries.call(input)) {
|
|
85
|
+
if (entries.length >= LIVE_CAPTURE_LIMITS.maxEntries) break;
|
|
86
|
+
const index = entries.length;
|
|
87
|
+
entries.push([walk(key, depth + 1, childPath(path, `key:${index}`)), walk(item, depth + 1, childPath(path, index))]);
|
|
88
|
+
}
|
|
89
|
+
if (size > entries.length) entries.push(omit({ reason: 'max-entries', path, limit: 100, originalCount: size, capturedCount: entries.length }));
|
|
90
|
+
return { __type: 'Map', entries };
|
|
91
|
+
}
|
|
92
|
+
if (input instanceof Set) {
|
|
93
|
+
const values: any[] = [];
|
|
94
|
+
const size = Object.getOwnPropertyDescriptor(Set.prototype, 'size')!.get!.call(input);
|
|
95
|
+
for (const item of Set.prototype.values.call(input)) {
|
|
96
|
+
if (values.length >= 100) break;
|
|
97
|
+
values.push(walk(item, depth + 1, childPath(path, values.length)));
|
|
98
|
+
}
|
|
99
|
+
if (size > values.length) values.push(omit({ reason: 'max-entries', path, limit: 100, originalCount: size, capturedCount: values.length }));
|
|
100
|
+
return { __type: 'Set', values };
|
|
101
|
+
}
|
|
102
|
+
if (Array.isArray(input) || (ArrayBuffer.isView(input) && !(input instanceof DataView))) {
|
|
103
|
+
const length = (input as any).length;
|
|
104
|
+
const previousMarker = Array.isArray(input) && length === 101 ? descriptor('100')?.value : null;
|
|
105
|
+
const prior = previousMarker && Object.getOwnPropertyDescriptors(previousMarker);
|
|
106
|
+
const hasPriorLimit = prior?.__kind?.value === 'repro-omitted' && prior?.reason?.value === 'max-entries';
|
|
107
|
+
const values: any[] = [];
|
|
108
|
+
for (let i = 0; i < Math.min(length, 100); i++) {
|
|
109
|
+
const itemDescriptor = descriptor(String(i));
|
|
110
|
+
values.push(itemDescriptor && !('value' in itemDescriptor)
|
|
111
|
+
? omit({ reason: 'accessor', path: childPath(path, i), capturedCount: 0 })
|
|
112
|
+
: walk(itemDescriptor?.value, depth + 1, childPath(path, i)));
|
|
113
|
+
}
|
|
114
|
+
if (length > 100) values.push(hasPriorLimit
|
|
115
|
+
? walk(previousMarker, depth + 1, path)
|
|
116
|
+
: omit({ reason: 'max-entries', path, limit: 100, originalCount: length, capturedCount: 100 }));
|
|
117
|
+
return Array.isArray(input) ? values : { __type: 'TypedArray', values };
|
|
118
|
+
}
|
|
119
|
+
const keys = Object.keys(input);
|
|
120
|
+
const previousMetadataKey = keys.find(key => {
|
|
121
|
+
if (!/^__reproCapture\$*$/.test(key)) return false;
|
|
122
|
+
const candidate = descriptor(key)?.value;
|
|
123
|
+
return candidate && Object.getOwnPropertyDescriptor(candidate, '__kind')?.value === 'repro-omitted'
|
|
124
|
+
&& Object.getOwnPropertyDescriptor(candidate, 'reason')?.value === 'max-keys';
|
|
125
|
+
});
|
|
126
|
+
const businessKeys = keys.filter(key => key !== previousMetadataKey);
|
|
127
|
+
const out: Record<string, any> = Object.create(null);
|
|
128
|
+
for (const key of businessKeys.slice(0, 100)) {
|
|
129
|
+
const itemDescriptor = descriptor(key)!;
|
|
130
|
+
out[key] = 'value' in itemDescriptor
|
|
131
|
+
? walk(itemDescriptor.value, depth + 1, childPath(path, key))
|
|
132
|
+
: omit({ reason: 'accessor', path: childPath(path, key), capturedCount: 0 });
|
|
133
|
+
}
|
|
134
|
+
if (businessKeys.length > 100 || previousMetadataKey) {
|
|
135
|
+
const detail = previousMetadataKey
|
|
136
|
+
? walk(descriptor(previousMetadataKey)?.value, depth + 1, path)
|
|
137
|
+
: omit({ reason: 'max-keys', path, limit: 100, originalCount: businessKeys.length, capturedCount: 100 });
|
|
138
|
+
let metadataKey = '__reproCapture';
|
|
139
|
+
while (Object.prototype.hasOwnProperty.call(out, metadataKey)) metadataKey += '$';
|
|
140
|
+
out[metadataKey] = detail;
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
};
|
|
144
|
+
try { return { value: walk(value, 0, rootPath), omissions }; }
|
|
145
|
+
catch { return { value: omit({ reason: 'uninspectable', path: rootPath, capturedCount: 0 }), omissions }; }
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function collectCaptureOmissions(value: any): CaptureOmission[] {
|
|
149
|
+
const result: CaptureOmission[] = [];
|
|
150
|
+
const seen = new WeakSet();
|
|
151
|
+
const pending = [value];
|
|
152
|
+
while (pending.length) {
|
|
153
|
+
const item = pending.pop();
|
|
154
|
+
const node = typeof item === 'string' ? parseStructuredString(item) : item;
|
|
155
|
+
if (!node || typeof node !== 'object' || seen.has(node)) continue;
|
|
156
|
+
seen.add(node);
|
|
157
|
+
const descriptors = Object.getOwnPropertyDescriptors(node);
|
|
158
|
+
if (descriptors.__kind?.value === 'repro-omitted') {
|
|
159
|
+
result.push({ reason: descriptors.reason?.value, path: descriptors.path?.value, limit: descriptors.limit?.value,
|
|
160
|
+
originalCount: descriptors.originalCount?.value, capturedCount: descriptors.capturedCount?.value });
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
for (const item of Object.values(descriptors)) if ('value' in item) pending.push(item.value);
|
|
164
|
+
}
|
|
165
|
+
return result;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export function limitLiveArguments(args: any[]): any[] {
|
|
169
|
+
const values = args.slice(0, 100).map((arg, index) => limitLiveCapture(arg, `$[${index}]`).value);
|
|
170
|
+
if (args.length > 100) {
|
|
171
|
+
const prior = Object.getOwnPropertyDescriptor(args, '100')?.value;
|
|
172
|
+
values.push(args.length === 101 && prior && Object.getOwnPropertyDescriptor(prior, '__kind')?.value === 'repro-omitted'
|
|
173
|
+
? limitLiveCapture(prior).value
|
|
174
|
+
: marker({ reason: 'max-entries', path: '$', limit: 100, originalCount: args.length, capturedCount: 100 }));
|
|
175
|
+
}
|
|
176
|
+
return values;
|
|
177
|
+
}
|
|
@@ -52,6 +52,12 @@ function method(value: unknown, key: string): boolean {
|
|
|
52
52
|
}
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
export function mongooseDocumentData(value: object): object | null {
|
|
56
|
+
const doc = Object.getOwnPropertyDescriptor(value, '_doc')?.value;
|
|
57
|
+
if (!doc || typeof doc !== 'object' || !Object.getOwnPropertyDescriptor(value, '$__')?.value) return null;
|
|
58
|
+
return method(value, '$__init') && method(value, '$toObject') ? doc : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
55
61
|
function primitiveStateValue(value: unknown): string | number | boolean | null | undefined {
|
|
56
62
|
if (value === null) return null;
|
|
57
63
|
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
|
@@ -86,6 +92,7 @@ function projectConfigurationService(value: object, className: string): Semantic
|
|
|
86
92
|
|
|
87
93
|
function projectLogger(value: object, className: string): SemanticProjection | null {
|
|
88
94
|
if (!/(Logger|LoggerService)$/i.test(className)) return null;
|
|
95
|
+
if (!method(value, 'log') && !method(value, 'info')) return null;
|
|
89
96
|
const descriptors = ownDataDescriptors(value);
|
|
90
97
|
const context = primitiveStateValue(descriptors.context?.value);
|
|
91
98
|
return semanticValue('logger', className, context !== undefined ? { context } : {});
|
|
@@ -95,6 +102,14 @@ function projectNestInfrastructure(value: object, className: string): SemanticPr
|
|
|
95
102
|
if (!/(ModuleRef|ModulesContainer|InstanceWrapper|NestContainer|Injector|DiscoveryService|ExecutionContextHost|ArgumentsHost|HttpAdapterHost)$/i.test(className)) {
|
|
96
103
|
return null;
|
|
97
104
|
}
|
|
105
|
+
const signatures: Record<string, string[]> = {
|
|
106
|
+
ModuleRef: ['get', 'resolve'], ModulesContainer: ['get', 'set'],
|
|
107
|
+
InstanceWrapper: ['getInstanceByContextId'], NestContainer: ['getModules'],
|
|
108
|
+
Injector: ['loadInstance'], DiscoveryService: ['getProviders'],
|
|
109
|
+
ExecutionContextHost: ['switchToHttp', 'getArgs'], ArgumentsHost: ['switchToHttp', 'getArgs'],
|
|
110
|
+
HttpAdapterHost: ['listen$'],
|
|
111
|
+
};
|
|
112
|
+
if (!(signatures[className] ?? []).some(key => method(value, key))) return null;
|
|
98
113
|
return semanticValue('nestjs-infrastructure', className);
|
|
99
114
|
}
|
|
100
115
|
|
|
@@ -159,6 +174,20 @@ function projectTypeOrm(value: object, className: string): SemanticProjection |
|
|
|
159
174
|
return null;
|
|
160
175
|
}
|
|
161
176
|
|
|
177
|
+
function projectMongoInfrastructure(value: object, className: string): SemanticProjection | null {
|
|
178
|
+
const signatures: Record<string, string[]> = {
|
|
179
|
+
MongoClient: ['connect', 'db', 'close'],
|
|
180
|
+
ClientSession: ['startTransaction', 'endSession'],
|
|
181
|
+
ConnectionPool: ['checkOut', 'checkIn'],
|
|
182
|
+
NativeConnection: ['openUri', 'model', 'close'],
|
|
183
|
+
Schema: ['add', 'path', 'plugin'],
|
|
184
|
+
};
|
|
185
|
+
const required = signatures[className];
|
|
186
|
+
return required?.every(key => method(value, key))
|
|
187
|
+
? semanticValue('mongodb-infrastructure', className)
|
|
188
|
+
: null;
|
|
189
|
+
}
|
|
190
|
+
|
|
162
191
|
/**
|
|
163
192
|
* Projects known runtime machinery into stable, non-recursive descriptions.
|
|
164
193
|
* Unknown application values deliberately return null and remain fully captured.
|
|
@@ -176,6 +205,7 @@ export function projectSemanticInfrastructureValue(value: unknown): SemanticProj
|
|
|
176
205
|
projectRedis(value, className) ||
|
|
177
206
|
projectKafka(value, className) ||
|
|
178
207
|
projectAmqp(value, className) ||
|
|
208
|
+
projectMongoInfrastructure(value, className) ||
|
|
179
209
|
projectPostgres(value, className) ||
|
|
180
210
|
projectTypeOrm(value, className)
|
|
181
211
|
);
|
|
@@ -215,6 +215,7 @@ async function main() {
|
|
|
215
215
|
const middleware = reproMiddleware(config);
|
|
216
216
|
const { req, res } = makeReqRes('S_content_addressed_mongo', 'A_content_addressed_mongo');
|
|
217
217
|
const repository = new SubjectRepository(Subject, new ConfigService());
|
|
218
|
+
const queryInput = { subjectNumber: 'SUBJECT-CONTENT-1001' };
|
|
218
219
|
let operationResult;
|
|
219
220
|
|
|
220
221
|
await new Promise((resolve, reject) => {
|
|
@@ -227,12 +228,14 @@ async function main() {
|
|
|
227
228
|
operationResult = await global.__repro_call(
|
|
228
229
|
loadSubjectRepeatedly,
|
|
229
230
|
null,
|
|
230
|
-
[repository,
|
|
231
|
+
[repository, queryInput],
|
|
231
232
|
'/app/src/subjects.repository.ts',
|
|
232
233
|
120,
|
|
233
234
|
'loadSubjectRepeatedly',
|
|
234
235
|
false,
|
|
235
236
|
);
|
|
237
|
+
queryInput.subjectNumber = 'MUTATED-AFTER-TRACE-EXIT';
|
|
238
|
+
operationResult.docs[0].status = 'mutated-after-trace-exit';
|
|
236
239
|
res.end();
|
|
237
240
|
resolve();
|
|
238
241
|
} catch (operationError) {
|
|
@@ -313,15 +316,31 @@ async function main() {
|
|
|
313
316
|
const argsDefinition = traceDefinitions.find((entry) =>
|
|
314
317
|
entry.id === enter.argsValueCapture?.valueId
|
|
315
318
|
);
|
|
316
|
-
|
|
317
|
-
assert.
|
|
319
|
+
const capturedArgs = argsDefinition?.value ?? enter.args;
|
|
320
|
+
assert(Array.isArray(capturedArgs), JSON.stringify(enter));
|
|
321
|
+
assert.deepEqual(capturedArgs[0].configService, {
|
|
318
322
|
__kind: 'repro-semantic',
|
|
319
323
|
category: 'configuration-service',
|
|
320
324
|
className: 'ConfigService',
|
|
321
325
|
});
|
|
322
|
-
assert.equal(
|
|
326
|
+
assert.equal(capturedArgs[1].subjectNumber, 'SUBJECT-CONTENT-1001');
|
|
327
|
+
|
|
328
|
+
const exit = traceEvents.find((event) =>
|
|
329
|
+
event.fn === 'loadSubjectRepeatedly' && event.type === 'exit'
|
|
330
|
+
);
|
|
331
|
+
assert(exit, JSON.stringify(traceEvents));
|
|
332
|
+
const returnDefinition = traceDefinitions.find((entry) =>
|
|
333
|
+
entry.id === exit.returnValueCapture?.valueId
|
|
334
|
+
);
|
|
335
|
+
assert(returnDefinition, JSON.stringify({
|
|
336
|
+
exit,
|
|
337
|
+
definitions: traceDefinitions.map((entry) => ({ id: entry.id, target: entry.target })),
|
|
338
|
+
}));
|
|
339
|
+
assert.equal(returnDefinition.value.docs[0].status, 'screened');
|
|
323
340
|
|
|
324
341
|
const capturedJson = JSON.stringify(posts);
|
|
342
|
+
assert(!capturedJson.includes('MUTATED-AFTER-TRACE-EXIT'));
|
|
343
|
+
assert(!capturedJson.includes('mutated-after-trace-exit'));
|
|
325
344
|
assert(!capturedJson.includes('captured-config-secret'));
|
|
326
345
|
assert(!capturedJson.includes('mongo-config-credential-999'));
|
|
327
346
|
assert(!capturedJson.includes('mongo-test-logger-buffer'));
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
const express = require('express');
|
|
2
|
+
const mongoose = require('mongoose');
|
|
3
|
+
const { reproMiddleware, reproMongoosePlugin } = require('../../dist');
|
|
4
|
+
|
|
5
|
+
function inspectShipment(shipment) { return shipment; }
|
|
6
|
+
async function start(config, uri) {
|
|
7
|
+
const connection = await mongoose.createConnection(uri).asPromise();
|
|
8
|
+
const schema = new mongoose.Schema({ reference: String, rows: [Object] });
|
|
9
|
+
schema.plugin(reproMongoosePlugin, config);
|
|
10
|
+
const Shipment = connection.model('LiveCaptureShipment', schema);
|
|
11
|
+
await Shipment.create({ reference: 'SHIP-LIMIT', rows: Array.from({ length: 157 }, (_, id) => ({ id, status: 'available' })) });
|
|
12
|
+
const app = express();
|
|
13
|
+
app.use(reproMiddleware(config));
|
|
14
|
+
app.get('/shipment', async (_req, res) => {
|
|
15
|
+
const shipment = await Shipment.findOne({ reference: 'SHIP-LIMIT' }).lean().exec();
|
|
16
|
+
res.json(inspectShipment(shipment));
|
|
17
|
+
});
|
|
18
|
+
const server = await new Promise(resolve => { const server = app.listen(0, '127.0.0.1', () => resolve(server)); });
|
|
19
|
+
return { server, close: async () => { await new Promise(resolve => server.close(resolve)); await connection.close(); } };
|
|
20
|
+
}
|
|
21
|
+
module.exports = { start };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
const assert = require('node:assert/strict');
|
|
2
|
+
const http = require('node:http');
|
|
3
|
+
const { once } = require('node:events');
|
|
4
|
+
const { enqueueIngestEntries, flushIngestQueue } = require('../dist/ingest/client');
|
|
5
|
+
|
|
6
|
+
async function main() {
|
|
7
|
+
let attempts = 0;
|
|
8
|
+
const accepted = [];
|
|
9
|
+
const terminalAttempts = new Map();
|
|
10
|
+
const app = http.createServer(async (req, res) => {
|
|
11
|
+
const chunks = []; for await (const chunk of req) chunks.push(chunk);
|
|
12
|
+
const body = JSON.parse(Buffer.concat(chunks));
|
|
13
|
+
const sid = body.events[0]?.session_id;
|
|
14
|
+
if (sid === 'S_quota_denied' || sid === 'S_deleted') {
|
|
15
|
+
terminalAttempts.set(sid, (terminalAttempts.get(sid) ?? 0) + 1);
|
|
16
|
+
res.writeHead(sid === 'S_deleted' ? 410 : 402); res.end('{}'); return;
|
|
17
|
+
}
|
|
18
|
+
attempts++;
|
|
19
|
+
if (attempts === 1) { res.writeHead(429, { 'Retry-After': '1' }); res.end('{}'); }
|
|
20
|
+
else { accepted.push(body); res.writeHead(202); res.end('{}'); }
|
|
21
|
+
});
|
|
22
|
+
app.listen(0, '127.0.0.1'); await once(app, 'listening');
|
|
23
|
+
try {
|
|
24
|
+
const cfg = { tenantId: 'T', appId: 'A', ingestBase: `http://127.0.0.1:${app.address().port}` };
|
|
25
|
+
enqueueIngestEntries(cfg, 'S_throttle', [{ request: { rid: 'R1', method: 'POST', url: '/shipment', status: 200 }, t: Date.now() }]);
|
|
26
|
+
await flushIngestQueue();
|
|
27
|
+
for (let i = 0; i < 8; i++) await flushIngestQueue();
|
|
28
|
+
assert.equal(attempts, 1, 'forced flushes must not hammer the collector or use up retries during cooldown');
|
|
29
|
+
await new Promise(resolve => setTimeout(resolve, 1100));
|
|
30
|
+
await flushIngestQueue();
|
|
31
|
+
assert.equal(attempts, 2);
|
|
32
|
+
assert.equal(accepted.flatMap(batch => batch.events).filter(event => event.event_type === 'backend_request').length, 1);
|
|
33
|
+
for (const sid of ['S_quota_denied', 'S_deleted']) {
|
|
34
|
+
enqueueIngestEntries(cfg, sid, [{ requestCaptureManifest: { requestRid: sid, closedAt: Date.now() }, t: Date.now() }]);
|
|
35
|
+
for (let attempt = 0; attempt < 4; attempt++) await flushIngestQueue();
|
|
36
|
+
assert.equal(terminalAttempts.get(sid), 1, 'terminal refusal must not retry critical manifests indefinitely');
|
|
37
|
+
}
|
|
38
|
+
console.log('HTTP 429 respects Retry-After and retains the exact pending request');
|
|
39
|
+
} finally { app.closeAllConnections(); await new Promise(resolve => app.close(resolve)); }
|
|
40
|
+
}
|
|
41
|
+
main().catch(error => { console.error(error); process.exitCode = 1; });
|
|
@@ -145,10 +145,19 @@ async function testKafkaConsumerTraceUsesConsumeTraceId() {
|
|
|
145
145
|
privacy: { environment: 'dev' },
|
|
146
146
|
ingestBase: 'http://127.0.0.1:65535',
|
|
147
147
|
};
|
|
148
|
+
const consumerPayload = {
|
|
149
|
+
marker: 'before-consumer-work',
|
|
150
|
+
repeated: 'consumer-trace-value'.repeat(1024),
|
|
151
|
+
};
|
|
148
152
|
const wrapped = __reproTestHooks.wrapKafkaEachMessageHandlerForTest(
|
|
149
153
|
async () => {
|
|
150
|
-
tracer.tracer.enter('consumerWork', { file: '/app/consumer.ts', line: 10 }, { args: [
|
|
151
|
-
|
|
154
|
+
tracer.tracer.enter('consumerWork', { file: '/app/consumer.ts', line: 10 }, { args: [consumerPayload] });
|
|
155
|
+
consumerPayload.marker = 'at-consumer-exit';
|
|
156
|
+
tracer.tracer.exit(
|
|
157
|
+
{ fn: 'consumerWork', file: '/app/consumer.ts', line: 10 },
|
|
158
|
+
{ args: [consumerPayload], returnValue: consumerPayload },
|
|
159
|
+
);
|
|
160
|
+
consumerPayload.marker = 'after-consumer-exit';
|
|
152
161
|
|
|
153
162
|
tracer.withTrace('parent-http-trace', () => {
|
|
154
163
|
tracer.tracer.enter('parentWork', { file: '/app/controller.ts', line: 20 }, { args: ['wrong'] });
|
|
@@ -207,6 +216,21 @@ async function testKafkaConsumerTraceUsesConsumeTraceId() {
|
|
|
207
216
|
.map((traceEvent) => traceEvent.fn);
|
|
208
217
|
assert(traceFns.includes('consumerWork'), traceFns.join(','));
|
|
209
218
|
assert(!traceFns.includes('parentWork'), traceFns.join(','));
|
|
219
|
+
|
|
220
|
+
const traceBatches = events.filter((event) => event.event_type === 'trace_batch');
|
|
221
|
+
const traceEvents = traceBatches.flatMap((event) => event.payload?.trace || []);
|
|
222
|
+
const definitions = traceBatches.flatMap((event) => event.payload?.traceValues || []);
|
|
223
|
+
const definitionsById = new Map(definitions.map((definition) => [definition.id, definition]));
|
|
224
|
+
const consumerEnter = traceEvents.find((event) => event.fn === 'consumerWork' && event.type === 'enter');
|
|
225
|
+
const consumerExit = traceEvents.find((event) => event.fn === 'consumerWork' && event.type === 'exit');
|
|
226
|
+
assert(consumerEnter && consumerExit, JSON.stringify(traceEvents));
|
|
227
|
+
const enterValue = definitionsById.get(consumerEnter.argsValueCapture?.valueId)?.value;
|
|
228
|
+
const exitArgsValue = definitionsById.get(consumerExit.argsValueCapture?.valueId)?.value;
|
|
229
|
+
const returnValue = definitionsById.get(consumerExit.returnValueCapture?.valueId)?.value;
|
|
230
|
+
assert.equal(enterValue?.[0]?.marker, 'before-consumer-work');
|
|
231
|
+
assert.equal(exitArgsValue?.[0]?.marker, 'at-consumer-exit');
|
|
232
|
+
assert.equal(returnValue?.marker, 'at-consumer-exit');
|
|
233
|
+
assert(!JSON.stringify(definitions).includes('after-consumer-exit'));
|
|
210
234
|
} finally {
|
|
211
235
|
global.fetch = originalFetch;
|
|
212
236
|
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
process.env.REPRO_SDK_BACKGROUND_MAX_DEFER_MS = '20';
|
|
2
|
+
process.env.REPRO_SDK_BACKGROUND_IDLE_DELAY_MS = '0';
|
|
3
|
+
process.env.REPRO_SDK_BACKGROUND_BUSY_IDLE_DELAY_MS = '0';
|
|
4
|
+
const assert = require('node:assert/strict');
|
|
5
|
+
const http = require('node:http');
|
|
6
|
+
const { once } = require('node:events');
|
|
7
|
+
const { GenericContainer, Wait } = require('testcontainers');
|
|
8
|
+
const { KafkaContainer } = require('@testcontainers/kafka');
|
|
9
|
+
const { Kafka, logLevel } = require('kafkajs');
|
|
10
|
+
const amqp = require('amqplib');
|
|
11
|
+
const { __reproTestHooks } = require('../dist');
|
|
12
|
+
const { flushIngestQueue } = require('../dist/ingest/client');
|
|
13
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
14
|
+
|
|
15
|
+
async function main() {
|
|
16
|
+
const posts = [];
|
|
17
|
+
const sink = http.createServer(async (req, res) => {
|
|
18
|
+
const buffers = [];
|
|
19
|
+
for await (const chunk of req) buffers.push(chunk);
|
|
20
|
+
posts.push(JSON.parse(Buffer.concat(buffers)));
|
|
21
|
+
res.writeHead(202); res.end('{}');
|
|
22
|
+
});
|
|
23
|
+
sink.listen(0, '127.0.0.1'); await once(sink, 'listening');
|
|
24
|
+
let kafkaContainer, rabbitContainer, consumer, producer, connection, channel;
|
|
25
|
+
let consumed = 0;
|
|
26
|
+
const rows = Array.from({ length: 157 }, (_, id) => ({ id, state: 'available' }));
|
|
27
|
+
const payload = Buffer.from(JSON.stringify({ rows }));
|
|
28
|
+
const cfg = { tenantId: 'TENANT_broker_limits', appId: 'APP_limits', appSecret: 'test',
|
|
29
|
+
privacy: { captureRaw: true, environment: 'dev' }, ingestBase: `http://127.0.0.1:${sink.address().port}` };
|
|
30
|
+
const headers = mode => ({ 'x-bug-session-id': `S_broker_${mode}`, 'x-bug-action-id': `A_${mode}`, 'x-repro-capture-mode': mode });
|
|
31
|
+
async function businessWork(buffer) {
|
|
32
|
+
await sleep(15);
|
|
33
|
+
const value = JSON.parse(buffer.toString());
|
|
34
|
+
assert.equal(value.rows.length, 157, 'instrumentation must not modify the delivered message');
|
|
35
|
+
assert.equal(value.rows[156].id, 156);
|
|
36
|
+
consumed++;
|
|
37
|
+
}
|
|
38
|
+
try {
|
|
39
|
+
kafkaContainer = await new KafkaContainer('confluentinc/cp-kafka:7.6.1').withKraft().start();
|
|
40
|
+
const kafka = new Kafka({ brokers: [`${kafkaContainer.getHost()}:${kafkaContainer.getMappedPort(9093)}`], logLevel: logLevel.NOTHING });
|
|
41
|
+
producer = kafka.producer(); consumer = kafka.consumer({ groupId: 'repro-live-limit-test' });
|
|
42
|
+
await producer.connect(); await consumer.connect();
|
|
43
|
+
const admin = kafka.admin(); await admin.connect();
|
|
44
|
+
try { await admin.createTopics({ topics: [{ topic: 'capture-limits', numPartitions: 1, replicationFactor: 1 }] }); }
|
|
45
|
+
finally { await admin.disconnect(); }
|
|
46
|
+
await consumer.subscribe({ topic: 'capture-limits', fromBeginning: true });
|
|
47
|
+
await consumer.run({ eachMessage: __reproTestHooks.wrapKafkaEachMessageHandlerForTest(
|
|
48
|
+
async ({ message }) => businessWork(message.value), cfg, { __repro_group_id: 'repro-live-limit-test' },
|
|
49
|
+
) });
|
|
50
|
+
for (const mode of ['live', 'staging']) await producer.send({ topic: 'capture-limits', messages: [{ key: mode, value: payload, headers: headers(mode) }] });
|
|
51
|
+
|
|
52
|
+
rabbitContainer = await new GenericContainer('rabbitmq:3.13-alpine')
|
|
53
|
+
.withEnvironment({ RABBITMQ_DEFAULT_USER: 'repro', RABBITMQ_DEFAULT_PASS: 'test' })
|
|
54
|
+
.withExposedPorts(5672).withWaitStrategy(Wait.forLogMessage('Server startup complete')).start();
|
|
55
|
+
connection = await amqp.connect(`amqp://repro:test@${rabbitContainer.getHost()}:${rabbitContainer.getMappedPort(5672)}`);
|
|
56
|
+
channel = await connection.createChannel(); await channel.assertQueue('capture-limits', { durable: false });
|
|
57
|
+
await channel.consume('capture-limits', __reproTestHooks.wrapAmqplibConsumeHandlerForTest(async message => {
|
|
58
|
+
await businessWork(message.content); channel.ack(message);
|
|
59
|
+
}, cfg, 'capture-limits'));
|
|
60
|
+
for (const mode of ['live', 'staging']) channel.sendToQueue('capture-limits', payload, { headers: headers(mode) });
|
|
61
|
+
const deadline = Date.now() + 30_000;
|
|
62
|
+
let requests;
|
|
63
|
+
do {
|
|
64
|
+
await __reproTestHooks.drainSdkBackgroundQueueForTest(); await flushIngestQueue();
|
|
65
|
+
requests = posts.flatMap(post => post.events || []).filter(event => event.event_type === 'backend_request');
|
|
66
|
+
if (requests.length >= 4 && consumed === 4) break;
|
|
67
|
+
await sleep(50);
|
|
68
|
+
} while (Date.now() < deadline);
|
|
69
|
+
assert.equal(consumed, 4); assert.equal(requests.length, 4);
|
|
70
|
+
for (const method of ['KAFKA_CONSUME', 'AMQP_CONSUME']) for (const mode of ['live', 'staging']) {
|
|
71
|
+
const request = requests.find(event => event.session_id === `S_broker_${mode}` && event.payload.request.method === method)?.payload.request;
|
|
72
|
+
assert(request, `${method} ${mode} missing`);
|
|
73
|
+
assert.equal(request.body.value.rows.length, mode === 'live' ? 101 : 157, `${method} ${mode}`);
|
|
74
|
+
assert.equal(JSON.parse(request.body.rawValue).rows.length, mode === 'live' ? 101 : 157, 'raw JSON copy must not bypass live limits');
|
|
75
|
+
if (mode === 'live') assert.equal(request.body.value.rows[100].originalCount, 157);
|
|
76
|
+
}
|
|
77
|
+
console.log('real Kafka and RabbitMQ: live bounds after asynchronous consumers; staging and broker values unchanged');
|
|
78
|
+
} finally {
|
|
79
|
+
if (consumer) await consumer.disconnect(); if (producer) await producer.disconnect();
|
|
80
|
+
if (channel) await channel.close(); if (connection) await connection.close();
|
|
81
|
+
await __reproTestHooks.drainSdkBackgroundQueueForTest(); await flushIngestQueue();
|
|
82
|
+
sink.closeAllConnections(); await new Promise(resolve => sink.close(resolve));
|
|
83
|
+
if (rabbitContainer) await rabbitContainer.stop(); if (kafkaContainer) await kafkaContainer.stop();
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
main().catch(error => { console.error(error); process.exitCode = 1; });
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
const assert = require('node:assert/strict');
|
|
2
|
+
const { limitLiveCapture, limitLiveArguments, collectCaptureOmissions } = require('../dist/live-capture-limits');
|
|
3
|
+
const { projectSemanticInfrastructureValue } = require('../dist/semantic-serialization');
|
|
4
|
+
const { IncomingMessage } = require('node:http');
|
|
5
|
+
const { Socket } = require('node:net');
|
|
6
|
+
const mongoose = require('mongoose');
|
|
7
|
+
|
|
8
|
+
let getterCalls = 0;
|
|
9
|
+
const payload = { nested: { rows: Array.from({ length: 157 }, (_, id) => ({ id, secret: `value-${id}` })) } };
|
|
10
|
+
Object.defineProperty(payload, 'danger', { enumerable: true, get() { getterCalls++; throw new Error('not safe'); } });
|
|
11
|
+
payload.self = payload;
|
|
12
|
+
payload.alias = payload.nested;
|
|
13
|
+
const limited = limitLiveCapture(payload);
|
|
14
|
+
assert.equal(getterCalls, 0);
|
|
15
|
+
assert.equal(payload.nested.rows.length, 157);
|
|
16
|
+
assert.equal(limited.value.nested.rows.length, 101);
|
|
17
|
+
assert.equal(limited.value.nested.rows[99].id, 99);
|
|
18
|
+
assert.equal(limited.value.nested.rows[100].__kind, 'repro-omitted');
|
|
19
|
+
assert.equal(limited.value.nested.rows[100].originalCount, 157);
|
|
20
|
+
assert.equal(limited.value.alias.__reproCircularRef, '$["nested"]');
|
|
21
|
+
assert.equal(limited.value.self.__reproCircularRef, '$');
|
|
22
|
+
assert.equal(limited.omissions.length, 2);
|
|
23
|
+
assert.equal(limitLiveCapture(limited.value).value.nested.rows[100].originalCount, 157);
|
|
24
|
+
assert.equal(collectCaptureOmissions(limited.value).length, 2);
|
|
25
|
+
|
|
26
|
+
const native = limitLiveCapture({
|
|
27
|
+
map: new Map(Array.from({ length: 150 }, (_, i) => [{ id: i }, { value: i }])),
|
|
28
|
+
set: new Set(Array.from({ length: 101 }, (_, i) => i)),
|
|
29
|
+
bytes: new Uint8Array(130).fill(7),
|
|
30
|
+
}).value;
|
|
31
|
+
assert.deepEqual(native.map.entries[99], [{ id: 99 }, { value: 99 }].map(x => Object.assign(Object.create(null), x)));
|
|
32
|
+
assert.equal(native.map.entries[100].originalCount, 150);
|
|
33
|
+
assert.equal(native.set.values[100].originalCount, 101);
|
|
34
|
+
assert.equal(native.bytes.values[100].originalCount, 130);
|
|
35
|
+
const dataBuffer = new Uint8Array(200).fill(8).buffer;
|
|
36
|
+
const dataView = limitLiveCapture(new DataView(dataBuffer, 10, 140)).value;
|
|
37
|
+
assert.equal(dataView.values[99], 8);
|
|
38
|
+
assert.equal(dataView.values[100].originalCount, 140);
|
|
39
|
+
assert.equal(limitLiveCapture(dataBuffer).value.values[100].originalCount, 200);
|
|
40
|
+
const error = new Error('observed failure', { cause: { rows: payload.nested.rows } });
|
|
41
|
+
const capturedError = limitLiveCapture(error).value;
|
|
42
|
+
assert.equal(capturedError.message, 'observed failure');
|
|
43
|
+
assert.equal(capturedError.cause.rows[100].originalCount, 157);
|
|
44
|
+
const deep = {};
|
|
45
|
+
let node = deep;
|
|
46
|
+
for (let i = 0; i < 12; i++) node = node.next = { level: i + 1 };
|
|
47
|
+
const depth = limitLiveCapture(deep);
|
|
48
|
+
assert.equal(depth.omissions[0].reason, 'max-depth');
|
|
49
|
+
assert.equal(depth.omissions[0].limit, 10);
|
|
50
|
+
assert.equal(depth.omissions[0].path.split('["next"]').length - 1, 11);
|
|
51
|
+
const aliasTarget = { important: 'must remain visible through the shallower alias' };
|
|
52
|
+
node.next = aliasTarget;
|
|
53
|
+
deep.shallowAlias = aliasTarget;
|
|
54
|
+
assert.equal(limitLiveCapture(deep).value.shallowAlias.important, aliasTarget.important);
|
|
55
|
+
const wide = Object.fromEntries(Array.from({ length: 150 }, (_, i) => [`key${i}`, i]));
|
|
56
|
+
wide.__reproCapture = 'business field';
|
|
57
|
+
const width = limitLiveCapture(wide);
|
|
58
|
+
assert.equal(width.value.key99, 99);
|
|
59
|
+
assert.equal(width.value.key100, undefined);
|
|
60
|
+
assert.equal(width.omissions[0].originalCount, 151);
|
|
61
|
+
assert.equal(limitLiveCapture(width.value).omissions[0].originalCount, 151);
|
|
62
|
+
const tooManyArgs = Array.from({ length: 157 }, (_, i) => i);
|
|
63
|
+
assert.equal(limitLiveArguments(tooManyArgs)[100].originalCount, 157);
|
|
64
|
+
assert.equal(limitLiveArguments(limitLiveArguments(tooManyArgs))[100].originalCount, 157);
|
|
65
|
+
const rawJson = JSON.stringify({ rows: payload.nested.rows });
|
|
66
|
+
const boundedJson = limitLiveCapture(rawJson).value;
|
|
67
|
+
assert.equal(typeof boundedJson, 'string');
|
|
68
|
+
assert.equal(JSON.parse(boundedJson).rows[100].originalCount, 157);
|
|
69
|
+
assert.equal(collectCaptureOmissions(boundedJson)[0].originalCount, 157);
|
|
70
|
+
const smallJson = '{ "business": [1, 2, 3] }';
|
|
71
|
+
assert.equal(limitLiveCapture(smallJson).value, smallJson, 'unlimited strings retain exact formatting');
|
|
72
|
+
const MongoClient = class MongoClient { connect() {} db() {} close() {} };
|
|
73
|
+
const actualInfrastructure = new MongoClient();
|
|
74
|
+
actualInfrastructure.password = 'never-serialize-runtime-config';
|
|
75
|
+
assert.equal(limitLiveCapture(actualInfrastructure).value.category, 'mongodb-infrastructure');
|
|
76
|
+
assert.equal(limitLiveCapture(actualInfrastructure).value.password, undefined);
|
|
77
|
+
const unknownMongoClass = class MongoClient { constructor() { this.orderId = 'business'; } };
|
|
78
|
+
assert.equal(limitLiveCapture(new unknownMongoClass()).value.orderId, 'business');
|
|
79
|
+
const Logger = class Logger { constructor() { this.importantBusinessField = true; } };
|
|
80
|
+
const ModuleRef = class ModuleRef { constructor() { this.invoice = 'INV-1'; } };
|
|
81
|
+
assert.equal(projectSemanticInfrastructureValue(new Logger()), null);
|
|
82
|
+
assert.equal(projectSemanticInfrastructureValue(new ModuleRef()), null);
|
|
83
|
+
const collision = { $__: { business: true }, _doc: { id: 'BUSINESS_DOC' }, invoice: 'INV-2' };
|
|
84
|
+
assert.equal(limitLiveCapture(collision).value.invoice, 'INV-2', 'business property names must not trigger infrastructure removal');
|
|
85
|
+
const Shipment = mongoose.model('LiveLimitShipment', new mongoose.Schema({ status: String, rows: [Number] }));
|
|
86
|
+
const shipment = new Shipment({ status: 'received', rows: Array.from({ length: 157 }, (_, i) => i) });
|
|
87
|
+
assert.equal(limitLiveCapture(shipment).value.status, 'received');
|
|
88
|
+
assert.equal(limitLiveCapture(shipment).value.rows[100].originalCount, 157);
|
|
89
|
+
assert.equal(limitLiveCapture(shipment).value.$__, undefined);
|
|
90
|
+
const req = new IncomingMessage(new Socket());
|
|
91
|
+
req.headers = { 'x-shipment': 'SHIP_1' }; req.body = { rows: [1, 2, 3] };
|
|
92
|
+
Object.defineProperty(req, 'dangerous', { get() { throw new Error('must not execute'); } });
|
|
93
|
+
assert.equal(limitLiveCapture(req).value.headers['x-shipment'], 'SHIP_1');
|
|
94
|
+
assert.deepEqual(limitLiveCapture(req).value.body.rows, [1, 2, 3]);
|
|
95
|
+
req.socket.destroy();
|
|
96
|
+
const withPrototypeKey = JSON.parse('{"__proto__":{"polluted":true},"valid":1}');
|
|
97
|
+
assert.equal(limitLiveCapture(withPrototypeKey).value.__proto__.polluted, true);
|
|
98
|
+
assert.equal({}.polluted, undefined);
|
|
99
|
+
console.log('live capture limits: native collections, depth, keys, getters, references and metadata passed');
|