@reproapp/node-sdk 0.0.7 → 0.0.9

package/dist/index.d.ts

@@ -283,6 +283,8 @@ type InlinePrivacyMaterializationResult = {
     };
 };
 declare function materializeInlinePrivacyValueAsync(target: RuntimePrivacySurface, value: any, cfg: ReproMiddlewareConfig, req: MaskRequestContext, trace: TraceEventForFilter | null, masking: NormalizedMaskingConfig | null, privacy: NormalizedRuntimePrivacyPolicy | null, db?: RuntimePrivacyDbContext | null): Promise<InlinePrivacyMaterializationResult>;
+declare function estimateTraceBatchSerializedBytes(batch: TraceEventRecord[], requestRid: string, actionId: string | null | undefined, batchIndex: number): number;
+declare function chunkTraceEventsForTransport(events: TraceEventRecord[], requestRid: string, actionId: string | null | undefined): TraceEventRecord[][];
 export type ReproMiddlewareConfig = IngestClientConfig & {
     /** Configure header capture/masking. Defaults to capturing with sensitive headers masked. */
     captureHeaders?: boolean | HeaderCaptureOptions;
@@ -348,5 +350,7 @@ export declare const __reproTestHooks: {
     materializeInlinePrivacyValueAsyncForTest: typeof materializeInlinePrivacyValueAsync;
     patchKafkaProducerInstanceForTest: typeof patchKafkaProducerInstance;
     wrapKafkaEachMessageHandlerForTest: typeof wrapKafkaEachMessageHandler;
+    chunkTraceEventsForTransportForTest: typeof chunkTraceEventsForTransport;
+    estimateTraceBatchSerializedBytesForTest: typeof estimateTraceBatchSerializedBytes;
 };
 export {};

package/dist/index.js

@@ -1657,6 +1657,12 @@ const TRACE_INLINE_VALUE_MAX_SERIALIZED_CHARS = (() => {
 })();
 const TRACE_VALUE_SIZE_EXCEEDED = Symbol('trace-value-size-exceeded');
 const TRACE_BATCH_SIZE = 100;
+const TRACE_BATCH_MAX_SERIALIZED_BYTES = (() => {
+    const env = Number(process.env.REPRO_TRACE_BATCH_MAX_SERIALIZED_BYTES);
+    if (Number.isFinite(env) && env > 0)
+        return Math.trunc(env);
+    return 512 * 1024;
+})();
 const TRACE_FLUSH_DELAY_MS = 20;
 // Choose how to order trace events in payloads.
 //  - "chronological" (default): preserve event arrival order (no reshuffle).
@@ -1929,6 +1935,50 @@ function sanitizeTraceValue(value, depth = 0, seen = new WeakMap(), options = {}
     const mongoId = coerceMongoId(value);
     if (mongoId !== null)
         return mongoId;
+    if (isHttpRequestLike(value)) {
+        const projected = {
+            __kind: 'http-request',
+        };
+        if (typeof value.method === 'string') {
+            projected.method = value.method;
+        }
+        const url = typeof value.originalUrl === 'string'
+            ? value.originalUrl
+            : typeof value.url === 'string'
+                ? value.url
+                : undefined;
+        if (url) {
+            projected.url = url;
+        }
+        const headers = sanitizeHeaders(value.headers, true);
+        if (headers !== undefined) {
+            projected.headers = sanitizeTraceValue(headers, depth + 1, seen, options, childCapturePath(valuePath, 'headers'));
+        }
+        if (value.params !== undefined) {
+            projected.params = sanitizeTraceValue(value.params, depth + 1, seen, options, childCapturePath(valuePath, 'params'));
+        }
+        if (value.query !== undefined) {
+            projected.query = sanitizeTraceValue(value.query, depth + 1, seen, options, childCapturePath(valuePath, 'query'));
+        }
+        if (value.body !== undefined) {
+            projected.body = sanitizeTraceValue(value.body, depth + 1, seen, options, childCapturePath(valuePath, 'body'));
+        }
+        return projected;
+    }
+    if (isHttpResponseLike(value)) {
+        const projected = {
+            __kind: 'http-response',
+            statusCode: Number(value.statusCode) || 0,
+        };
+        const rawHeaders = typeof value.getHeaders === 'function'
+            ? value.getHeaders()
+            : value._headers;
+        const headers = sanitizeHeaders(rawHeaders, true);
+        if (headers !== undefined) {
+            projected.headers = sanitizeTraceValue(headers, depth + 1, seen, options, childCapturePath(valuePath, 'headers'));
+        }
+        return projected;
+    }
     if (isMongooseQueryLike(value)) {
         const captured = value.__repro_result;
         if (captured !== undefined) {
@@ -3482,6 +3532,51 @@ function collectBatchTraceValueEntries(batch, batchIndex) {
     });
     return collected;
 }
+function serializedByteLength(value) {
+    try {
+        return Buffer.byteLength(JSON.stringify(value), 'utf8');
+    }
+    catch {
+        return Number.MAX_SAFE_INTEGER;
+    }
+}
+function estimateTraceBatchSerializedBytes(batch, requestRid, actionId, batchIndex) {
+    const traceValues = collectBatchTraceValueEntries(batch, batchIndex);
+    return serializedByteLength({
+        actionId: actionId ?? null,
+        trace: batch,
+        traceValues: traceValues.length ? traceValues : undefined,
+        traceBatch: {
+            rid: requestRid,
+            index: batchIndex,
+            total: 0,
+        },
+        t: 0,
+    });
+}
+function chunkTraceEventsForTransport(events, requestRid, actionId) {
+    if (!Array.isArray(events) || events.length === 0)
+        return [];
+    const countBatches = chunkArray(events, TRACE_BATCH_SIZE);
+    const sizedBatches = [];
+    let current = [];
+    for (const event of countBatches.flat()) {
+        const candidate = current.concat(event);
+        const batchIndex = sizedBatches.length;
+        const estimatedBytes = estimateTraceBatchSerializedBytes(candidate, requestRid, actionId, batchIndex);
+        if (current.length > 0 &&
+            estimatedBytes > TRACE_BATCH_MAX_SERIALIZED_BYTES) {
+            sizedBatches.push(current);
+            current = [event];
+            continue;
+        }
+        current = candidate;
+    }
+    if (current.length > 0) {
+        sizedBatches.push(current);
+    }
+    return sizedBatches;
+}
 function createCapturedValueEntry(params) {
     if (!__FULL_VALUE_CAPTURE_ENABLED)
         return undefined;
@@ -4065,7 +4160,7 @@ function reproMiddleware(cfg) {
                             const orderedEvents = TRACE_ORDER_MODE === 'tree'
                                 ? reorderTraceEvents(baseEvents)
                                 : sortTraceEventsChronologically(baseEvents);
-                            const traceBatches = chunkArray(orderedEvents, TRACE_BATCH_SIZE);
+                            const traceBatches = chunkTraceEventsForTransport(orderedEvents, rid, aid);
                             if (traceBatches.length) {
                                 for (let i = 0; i < traceBatches.length; i++) {
                                     const batch = traceBatches[i];
@@ -5268,7 +5363,7 @@ function buildKafkaTraceEntries(actionId, requestRid, events) {
     const orderedEvents = TRACE_ORDER_MODE === 'tree'
         ? reorderTraceEvents(baseEvents)
         : sortTraceEventsChronologically(baseEvents);
-    const batches = chunkArray(orderedEvents, TRACE_BATCH_SIZE);
+    const batches = chunkTraceEventsForTransport(orderedEvents, requestRid, actionId);
     return batches.map((batch, index) => ({
         actionId: actionId ?? null,
         trace: batch,
@@ -6582,4 +6677,6 @@ exports.__reproTestHooks = {
     materializeInlinePrivacyValueAsyncForTest: materializeInlinePrivacyValueAsync,
     patchKafkaProducerInstanceForTest: patchKafkaProducerInstance,
     wrapKafkaEachMessageHandlerForTest: wrapKafkaEachMessageHandler,
+    chunkTraceEventsForTransportForTest: chunkTraceEventsForTransport,
+    estimateTraceBatchSerializedBytesForTest: estimateTraceBatchSerializedBytes,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@reproapp/node-sdk",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "description": "Repro Nest SDK",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,7 +12,7 @@
     "build": "tsc -p tsconfig.json",
     "dev": "tsc -p tsconfig.json --watch --preserveWatchOutput",
     "prepublishOnly": "npm run build",
-    "test": "npm run build && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js"
+    "test": "npm run build && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js"
   },
   "peerDependencies": {
     "express": "^5.1.0",

package/src/index.ts

@@ -2028,6 +2028,11 @@ const TRACE_INLINE_VALUE_MAX_SERIALIZED_CHARS = (() => {
 })();
 const TRACE_VALUE_SIZE_EXCEEDED = Symbol('trace-value-size-exceeded');
 const TRACE_BATCH_SIZE = 100;
+const TRACE_BATCH_MAX_SERIALIZED_BYTES = (() => {
+    const env = Number(process.env.REPRO_TRACE_BATCH_MAX_SERIALIZED_BYTES);
+    if (Number.isFinite(env) && env > 0) return Math.trunc(env);
+    return 512 * 1024;
+})();
 const TRACE_FLUSH_DELAY_MS = 20;
 // Choose how to order trace events in payloads.
 //  - "chronological" (default): preserve event arrival order (no reshuffle).
@@ -2318,6 +2323,55 @@ function sanitizeTraceValue(
     const mongoId = coerceMongoId(value);
     if (mongoId !== null) return mongoId;
 
+    if (isHttpRequestLike(value)) {
+        const projected: Record<string, any> = {
+            __kind: 'http-request',
+        };
+        if (typeof (value as any).method === 'string') {
+            projected.method = (value as any).method;
+        }
+        const url =
+            typeof (value as any).originalUrl === 'string'
+                ? (value as any).originalUrl
+                : typeof (value as any).url === 'string'
+                    ? (value as any).url
+                    : undefined;
+        if (url) {
+            projected.url = url;
+        }
+
+        const headers = sanitizeHeaders((value as any).headers, true);
+        if (headers !== undefined) {
+            projected.headers = sanitizeTraceValue(headers, depth + 1, seen, options, childCapturePath(valuePath, 'headers'));
+        }
+        if ((value as any).params !== undefined) {
+            projected.params = sanitizeTraceValue((value as any).params, depth + 1, seen, options, childCapturePath(valuePath, 'params'));
+        }
+        if ((value as any).query !== undefined) {
+            projected.query = sanitizeTraceValue((value as any).query, depth + 1, seen, options, childCapturePath(valuePath, 'query'));
+        }
+        if ((value as any).body !== undefined) {
+            projected.body = sanitizeTraceValue((value as any).body, depth + 1, seen, options, childCapturePath(valuePath, 'body'));
+        }
+        return projected;
+    }
+
+    if (isHttpResponseLike(value)) {
+        const projected: Record<string, any> = {
+            __kind: 'http-response',
+            statusCode: Number((value as any).statusCode) || 0,
+        };
+        const rawHeaders =
+            typeof (value as any).getHeaders === 'function'
+                ? (value as any).getHeaders()
+                : (value as any)._headers;
+        const headers = sanitizeHeaders(rawHeaders, true);
+        if (headers !== undefined) {
+            projected.headers = sanitizeTraceValue(headers, depth + 1, seen, options, childCapturePath(valuePath, 'headers'));
+        }
+        return projected;
+    }
+
     if (isMongooseQueryLike(value)) {
         const captured = (value as any).__repro_result;
         if (captured !== undefined) {
@@ -4166,6 +4220,74 @@ function collectBatchTraceValueEntries(batch: TraceEventRecord[], batchIndex: nu
     return collected;
 }
 
+function serializedByteLength(value: any): number {
+    try {
+        return Buffer.byteLength(JSON.stringify(value), 'utf8');
+    } catch {
+        return Number.MAX_SAFE_INTEGER;
+    }
+}
+
+function estimateTraceBatchSerializedBytes(
+    batch: TraceEventRecord[],
+    requestRid: string,
+    actionId: string | null | undefined,
+    batchIndex: number,
+): number {
+    const traceValues = collectBatchTraceValueEntries(batch, batchIndex);
+    return serializedByteLength({
+        actionId: actionId ?? null,
+        trace: batch,
+        traceValues: traceValues.length ? traceValues : undefined,
+        traceBatch: {
+            rid: requestRid,
+            index: batchIndex,
+            total: 0,
+        },
+        t: 0,
+    });
+}
+
+function chunkTraceEventsForTransport(
+    events: TraceEventRecord[],
+    requestRid: string,
+    actionId: string | null | undefined,
+): TraceEventRecord[][] {
+    if (!Array.isArray(events) || events.length === 0) return [];
+
+    const countBatches = chunkArray(events, TRACE_BATCH_SIZE);
+    const sizedBatches: TraceEventRecord[][] = [];
+    let current: TraceEventRecord[] = [];
+
+    for (const event of countBatches.flat()) {
+        const candidate = current.concat(event);
+        const batchIndex = sizedBatches.length;
+        const estimatedBytes = estimateTraceBatchSerializedBytes(
+            candidate,
+            requestRid,
+            actionId,
+            batchIndex,
+        );
+
+        if (
+            current.length > 0 &&
+            estimatedBytes > TRACE_BATCH_MAX_SERIALIZED_BYTES
+        ) {
+            sizedBatches.push(current);
+            current = [event];
+            continue;
+        }
+
+        current = candidate;
+    }
+
+    if (current.length > 0) {
+        sizedBatches.push(current);
+    }
+
+    return sizedBatches;
+}
+
 function createCapturedValueEntry(
     params: {
         target:
@@ -4904,7 +5026,11 @@ export function reproMiddleware(cfg: ReproMiddlewareConfig) {
                                 const orderedEvents = TRACE_ORDER_MODE === 'tree'
                                     ? reorderTraceEvents(baseEvents)
                                     : sortTraceEventsChronologically(baseEvents);
-                                const traceBatches = chunkArray(orderedEvents, TRACE_BATCH_SIZE);
+                                const traceBatches = chunkTraceEventsForTransport(
+                                    orderedEvents,
+                                    rid,
+                                    aid,
+                                );
 
                                 if (traceBatches.length) {
                                     for (let i = 0; i < traceBatches.length; i++) {
@@ -6204,7 +6330,11 @@ function buildKafkaTraceEntries(
         TRACE_ORDER_MODE === 'tree'
             ? reorderTraceEvents(baseEvents)
             : sortTraceEventsChronologically(baseEvents);
-    const batches = chunkArray(orderedEvents, TRACE_BATCH_SIZE);
+    const batches = chunkTraceEventsForTransport(
+        orderedEvents,
+        requestRid,
+        actionId,
+    );
     return batches.map((batch, index) => ({
         actionId: actionId ?? null,
         trace: batch,
@@ -7569,4 +7699,6 @@ export const __reproTestHooks = {
     materializeInlinePrivacyValueAsyncForTest: materializeInlinePrivacyValueAsync,
     patchKafkaProducerInstanceForTest: patchKafkaProducerInstance,
     wrapKafkaEachMessageHandlerForTest: wrapKafkaEachMessageHandler,
+    chunkTraceEventsForTransportForTest: chunkTraceEventsForTransport,
+    estimateTraceBatchSerializedBytesForTest: estimateTraceBatchSerializedBytes,
 };

package/test/express-trace-http-args.test.js

@@ -0,0 +1,126 @@
+const assert = require('assert');
+const http = require('http');
+
+const originalFetch = global.fetch;
+const capturedBodies = [];
+
+global.fetch = async (url, init = {}) => {
+    const target = String(url || '');
+    if (!target.includes('/v1/ingest/events')) {
+        throw new Error(`unexpected fetch target: ${target}`);
+    }
+    capturedBodies.push({
+        at: Date.now(),
+        url: target,
+        body: JSON.parse(String(init.body || '{}')),
+    });
+    return {
+        ok: true,
+        status: 200,
+        json: async () => ({ ok: true }),
+        text: async () => '{"ok":true}',
+    };
+};
+
+const { initRepro } = require('../dist');
+const { flushIngestQueue } = require('../dist/ingest/client');
+
+function findEvents(eventType) {
+    return capturedBodies.flatMap((entry) => Array.isArray(entry.body?.events) ? entry.body.events : [])
+        .filter((event) => event?.event_type === eventType);
+}
+
+function sendGet(url) {
+    return new Promise((resolve, reject) => {
+        const request = http.get(url, (response) => {
+            let text = '';
+            response.setEncoding('utf8');
+            response.on('data', (chunk) => {
+                text += chunk;
+            });
+            response.on('end', () => resolve({
+                statusCode: response.statusCode,
+                body: text,
+            }));
+        });
+        request.on('error', reject);
+    });
+}
+
+async function main() {
+    await initRepro({
+        tenantId: 'TENANT_express_trace_http_args',
+        appId: 'APP_express_trace_http_args',
+        appSecret: 'secret',
+        appName: 'express-trace-http-args',
+        serviceName: 'express-trace-http-args',
+        ingestBase: 'http://127.0.0.1:65535',
+        tracing: {
+            disableFunctionTypes: ['constructor'],
+            logFunctionCalls: false,
+        },
+    });
+
+    const { startServer } = require('./fixtures/express-trace-http-args-server');
+    const server = await startServer({
+        tenantId: 'TENANT_express_trace_http_args',
+        appId: 'APP_express_trace_http_args',
+        appSecret: 'secret',
+        appName: 'express-trace-http-args',
+        serviceName: 'express-trace-http-args',
+        ingestBase: 'http://127.0.0.1:65535',
+    });
+
+    try {
+        const port = server.address().port;
+        const requestUrl = `http://127.0.0.1:${port}/ping?name=Avery&__repro_sid=S_express_trace_http_args&__repro_aid=A_express_trace_http_args&__repro_start=${Date.now()}`;
+        const response = await sendGet(requestUrl);
+        assert.equal(response.statusCode, 200);
+
+        await new Promise((resolve) => setTimeout(resolve, 2500));
+        await flushIngestQueue();
+
+        const requestEvents = findEvents('backend_request');
+        const traceEvents = findEvents('trace_batch');
+
+        assert.equal(requestEvents.length, 1, JSON.stringify(capturedBodies));
+        assert.equal(traceEvents.length, 1, JSON.stringify(capturedBodies));
+
+        const batch = traceEvents[0]?.payload?.trace;
+        assert(Array.isArray(batch) && batch.length > 0, JSON.stringify(traceEvents[0]));
+
+        const handlePingEnter = batch.find((event) => event?.type === 'enter' && event?.fn === 'handlePing');
+        assert(handlePingEnter, JSON.stringify(batch));
+        assert(Array.isArray(handlePingEnter.args), JSON.stringify(handlePingEnter));
+        assert.equal(handlePingEnter.args[0]?.__kind, 'http-request', JSON.stringify(handlePingEnter.args[0]));
+        assert.equal(handlePingEnter.args[1]?.__kind, 'http-response', JSON.stringify(handlePingEnter.args[1]));
+        assert.equal(handlePingEnter.args[0]?.url, '/ping?name=Avery');
+        assert.equal(handlePingEnter.args[0]?.query?.name, 'Avery');
+
+        const serializedBatch = JSON.stringify(traceEvents[0]);
+        assert(serializedBatch.length < 250000, `trace batch still too large: ${serializedBatch.length}`);
+
+        // eslint-disable-next-line no-console
+        console.log('express trace http arg projection OK');
+    } finally {
+        await new Promise((resolve, reject) => {
+            server.close((error) => {
+                if (!error || error.code === 'ERR_SERVER_NOT_RUNNING') {
+                    resolve();
+                    return;
+                }
+                reject(error);
+            });
+        });
+    }
+}
+
+main()
+    .catch((error) => {
+        // eslint-disable-next-line no-console
+        console.error(error);
+        process.exitCode = 1;
+    })
+    .finally(() => {
+        global.fetch = originalFetch;
+    });

package/test/fixtures/express-trace-http-args-controller.js

@@ -0,0 +1,29 @@
+async function buildPayload(name) {
+    const normalized = normalizeName(name);
+    const emphasized = emphasize(normalized);
+    return {
+        original: name,
+        normalized,
+        emphasized,
+    };
+}
+
+function normalizeName(name) {
+    return String(name || 'anonymous').trim().toLowerCase();
+}
+
+function emphasize(name) {
+    return `${name.toUpperCase()}!`;
+}
+
+async function handlePing(req, res) {
+    const payload = await buildPayload(req.query?.name || 'Avery Debugson');
+    res.json({
+        ok: true,
+        payload,
+    });
+}
+
+module.exports = {
+    handlePing,
+};

package/test/fixtures/express-trace-http-args-server.js

@@ -0,0 +1,21 @@
+const express = require('express');
+const { reproMiddleware } = require('../../dist');
+const { handlePing } = require('./express-trace-http-args-controller');
+
+async function startServer(cfg) {
+    const app = express();
+    app.use(reproMiddleware(cfg));
+    app.get('/ping', handlePing);
+
+    const server = await new Promise((resolve, reject) => {
+        const instance = app.listen(0);
+        instance.once('listening', () => resolve(instance));
+        instance.once('error', reject);
+    });
+
+    return server;
+}
+
+module.exports = {
+    startServer,
+};

package/test/trace-batch-size.test.js

@@ -0,0 +1,58 @@
+const assert = require('assert');
+const { __reproTestHooks } = require('../dist');
+
+function makeEvent(index, payload) {
+    return {
+        t: index,
+        type: 'enter',
+        fn: `fn${index}`,
+        file: '/app/src/controllers/subjects/index.ts',
+        line: index + 1,
+        depth: 1,
+        spanId: index + 1,
+        parentSpanId: null,
+        args: [payload],
+    };
+}
+
+function main() {
+    const payload = {
+        __kind: 'http-request',
+        method: 'POST',
+        url: '/api/v1/subject_visits/697001aaac70cc3d60f21273/subjects/createNewSubject?tenantId=tgtherapeutics',
+        headers: {
+            x_http_user: 'x'.repeat(12000),
+            authorization: '[dropped]',
+        },
+        body: {
+            value: 'y'.repeat(12000),
+        },
+    };
+
+    const events = Array.from({ length: 120 }, (_, index) => makeEvent(index, payload));
+    const batches = __reproTestHooks.chunkTraceEventsForTransportForTest(
+        events,
+        'RID_test_trace_batch_size',
+        'A_test_trace_batch_size',
+    );
+
+    assert(batches.length > 1, `expected more than one batch, got ${batches.length}`);
+
+    batches.forEach((batch, index) => {
+        const size = __reproTestHooks.estimateTraceBatchSerializedBytesForTest(
+            batch,
+            'RID_test_trace_batch_size',
+            'A_test_trace_batch_size',
+            index,
+        );
+        assert(
+            size <= 512 * 1024,
+            `batch ${index} too large: ${size}`,
+        );
+    });
+
+    // eslint-disable-next-line no-console
+    console.log('trace batch size chunking OK');
+}
+
+main();