@reproapp/node-sdk 0.0.11 → 0.0.13

package/dist/index.d.ts

@@ -67,9 +67,12 @@ type TraceEventRecord = {
     spanId?: string | number | null;
     parentSpanId?: string | number | null;
     args?: any;
+    argsValueCapture?: TraceValueCaptureRef;
     returnValue?: any;
+    returnValueCapture?: TraceValueCaptureRef;
     threw?: boolean;
     error?: any;
+    errorValueCapture?: TraceValueCaptureRef;
     unawaited?: boolean;
 };
 type PendingTraceEventRecord = TraceEventRecord & {
@@ -82,9 +85,14 @@ type PendingTraceEventRecord = TraceEventRecord & {
     __reproSourceFile?: string | null;
     __reproTraceValueEntries?: TraceValueBatchEntry[];
 };
+type TraceValueCaptureRef = {
+    status: 'stored' | 'truncated-for-storage' | 'skipped-non-meaningful';
+    valueId?: string;
+    projectedKind?: string;
+};
 type TraceValueBatchEntry = {
     id: string;
-    target: 'db.pk' | 'db.before' | 'db.after' | 'db.query' | 'db.resultMeta' | 'db.error' | 'request.headers' | 'request.body' | 'request.params' | 'request.query' | 'response.body';
+    target: 'db.pk' | 'db.before' | 'db.after' | 'db.query' | 'db.resultMeta' | 'db.error' | 'request.headers' | 'request.body' | 'request.params' | 'request.query' | 'response.body' | 'trace.args' | 'trace.returnValue' | 'trace.error';
     value: any;
     truncatedForStorage?: boolean;
     projectedKind?: string;
@@ -285,6 +293,7 @@ type InlinePrivacyMaterializationResult = {
     };
 };
 declare function materializeInlinePrivacyValueAsync(target: RuntimePrivacySurface, value: any, cfg: ReproMiddlewareConfig, req: MaskRequestContext, trace: TraceEventForFilter | null, masking: NormalizedMaskingConfig | null, privacy: NormalizedRuntimePrivacyPolicy | null, db?: RuntimePrivacyDbContext | null): Promise<InlinePrivacyMaterializationResult>;
+declare function getEventTraceValueEntries(event: TraceEventRecord | PendingTraceEventRecord): TraceValueBatchEntry[] | undefined;
 declare function estimateTraceBatchSerializedBytes(batch: TraceEventRecord[], requestRid: string, actionId: string | null | undefined, batchIndex: number): number;
 declare function chunkTraceEventsForTransport(events: TraceEventRecord[], requestRid: string, actionId: string | null | undefined): TraceEventRecord[][];
 export type ReproMiddlewareConfig = IngestClientConfig & {
@@ -362,5 +371,6 @@ export declare const __reproTestHooks: {
     estimateTraceBatchSerializedBytesForTest: typeof estimateTraceBatchSerializedBytes;
     sanitizeTraceValueForPrivacyForTest: typeof sanitizeTraceValueForPrivacy;
     sanitizeMaterializedTraceValueForTest: typeof sanitizeMaterializedTraceValue;
+    getEventTraceValueEntriesForTest: typeof getEventTraceValueEntries;
 };
 export {};

package/dist/index.js

@@ -2129,6 +2129,56 @@ function sanitizeInlinePrivacyValue(value) {
 function sanitizeMaterializedTraceValue(value) {
     return sanitizeTraceValue(value, 0, new WeakMap(), { preserveLongStrings: true, disableTruncation: true });
 }
+function compactMaterializedTracePreview(value, depth = 0) {
+    if (value === undefined || value === null)
+        return value;
+    if (depth >= TRACE_VALUE_MAX_DEPTH) {
+        return makeOmittedValue(`depth>${TRACE_VALUE_MAX_DEPTH}`);
+    }
+    const type = typeof value;
+    if (type === 'string') {
+        if (value.length <= TRACE_VALUE_MAX_STRING)
+            return value;
+        return value.slice(0, TRACE_VALUE_MAX_STRING);
+    }
+    if (type === 'number' || type === 'boolean')
+        return value;
+    if (type === 'bigint')
+        return value.toString();
+    if (type === 'symbol')
+        return value.toString();
+    if (type === 'function')
+        return `[Function${value.name ? ` ${value.name}` : ''}]`;
+    if (type !== 'object')
+        return String(value);
+    if (Array.isArray(value)) {
+        return value
+            .slice(0, TRACE_VALUE_MAX_ITEMS)
+            .map((item) => compactMaterializedTracePreview(item, depth + 1));
+    }
+    const out = {};
+    const keys = Object.keys(value).slice(0, TRACE_VALUE_MAX_KEYS);
+    for (const key of keys) {
+        try {
+            out[key] = compactMaterializedTracePreview(value[key], depth + 1);
+        }
+        catch (err) {
+            out[key] = `[Cannot serialize: ${err?.message || 'unknown error'}]`;
+        }
+    }
+    return out;
+}
+function limitMaterializedTraceInlineValue(value) {
+    const inlineValue = sanitizeMaterializedTraceValue(value);
+    if (!traceInlineValueExceedsLimit(inlineValue)) {
+        return inlineValue;
+    }
+    const compactPreview = compactMaterializedTracePreview(inlineValue);
+    if (!traceInlineValueExceedsLimit(compactPreview)) {
+        return compactPreview;
+    }
+    return makeOmittedValue('inline-size-limit');
+}
 function sanitizeTraceArgsForPrivacy(values) {
     if (!Array.isArray(values))
         return [sanitizeTraceValueForPrivacy(values)];
@@ -2142,6 +2192,14 @@ function normalizeRawTraceArgs(values) {
 function hasMeaningfulRawTraceError(error) {
     return error !== undefined && error !== null;
 }
+function attachTraceValueEntry(event, entry) {
+    const current = event[TRACE_FULL_VALUE_ENTRY_SYMBOL];
+    if (Array.isArray(current)) {
+        current.push(entry);
+        return;
+    }
+    event[TRACE_FULL_VALUE_ENTRY_SYMBOL] = [entry];
+}
 async function materializePendingTraceEvents(events, params) {
     for (const evt of events) {
         const pending = evt.__reproPending;
@@ -2151,14 +2209,71 @@ async function materializePendingTraceEvents(events, params) {
         const candidate = pending.candidate ?? traceEventCandidateFromRecord(evt);
         const omitJsonBuiltinValues = shouldOmitJsonBuiltinTraceValues(candidate);
         const activePrivacy = params.resolvePrivacy();
+        const captureContext = {
+            runtimeConfig: params.cfg,
+            captureHeaders: params.cfg.captureHeaders,
+            maskReq: params.maskReq,
+            trace: candidate,
+            masking: params.masking,
+            privacy: activePrivacy,
+        };
+        const traceValueMetadata = {
+            fn: evt.fn,
+            file: evt.file,
+            line: evt.line ?? null,
+            spanId: evt.spanId ?? null,
+            parentSpanId: evt.parentSpanId ?? null,
+        };
         if (!omitJsonBuiltinValues && pending.argsRaw !== undefined) {
-            evt.args = sanitizeMaterializedTraceValue(await materializeTracePrivacyValueAsync('trace.args', sanitizeTraceArgsForPrivacy(pending.argsRaw), params.cfg, params.maskReq, candidate, params.masking, activePrivacy));
+            const argsMaterialized = await materializeTracePrivacyValueAsync('trace.args', sanitizeTraceArgsForPrivacy(pending.argsRaw), params.cfg, params.maskReq, candidate, params.masking, activePrivacy);
+            evt.args = limitMaterializedTraceInlineValue(argsMaterialized);
+            const argsValueCapture = await maybeCaptureTraceValueAsync({
+                target: 'trace.args',
+                rawValue: pending.argsRaw,
+                previewValue: evt.args,
+                capture: captureContext,
+                metadata: traceValueMetadata,
+            });
+            if (argsValueCapture?.entry) {
+                attachTraceValueEntry(evt, argsValueCapture.entry);
+            }
+            if (argsValueCapture?.captureRef) {
+                evt.argsValueCapture = argsValueCapture.captureRef;
+            }
         }
         if (!omitJsonBuiltinValues && pending.returnValueRaw !== undefined) {
-            evt.returnValue = sanitizeMaterializedTraceValue(await materializeTracePrivacyValueAsync('trace.returnValue', sanitizeTraceValueForPrivacy(pending.returnValueRaw), params.cfg, params.maskReq, candidate, params.masking, activePrivacy));
+            const returnValueMaterialized = await materializeTracePrivacyValueAsync('trace.returnValue', sanitizeTraceValueForPrivacy(pending.returnValueRaw), params.cfg, params.maskReq, candidate, params.masking, activePrivacy);
+            evt.returnValue = limitMaterializedTraceInlineValue(returnValueMaterialized);
+            const returnValueCapture = await maybeCaptureTraceValueAsync({
+                target: 'trace.returnValue',
+                rawValue: pending.returnValueRaw,
+                previewValue: evt.returnValue,
+                capture: captureContext,
+                metadata: traceValueMetadata,
+            });
+            if (returnValueCapture?.entry) {
+                attachTraceValueEntry(evt, returnValueCapture.entry);
+            }
+            if (returnValueCapture?.captureRef) {
+                evt.returnValueCapture = returnValueCapture.captureRef;
+            }
         }
         if (pending.errorRaw !== undefined) {
-            evt.error = sanitizeMaterializedTraceValue(await materializeTracePrivacyValueAsync('trace.error', sanitizeTraceValueForPrivacy(pending.errorRaw), params.cfg, params.maskReq, candidate, params.masking, activePrivacy));
+            const errorMaterialized = await materializeTracePrivacyValueAsync('trace.error', sanitizeTraceValueForPrivacy(pending.errorRaw), params.cfg, params.maskReq, candidate, params.masking, activePrivacy);
+            evt.error = limitMaterializedTraceInlineValue(errorMaterialized);
+            const errorValueCapture = await maybeCaptureTraceValueAsync({
+                target: 'trace.error',
+                rawValue: pending.errorRaw,
+                previewValue: evt.error,
+                capture: captureContext,
+                metadata: traceValueMetadata,
+            });
+            if (errorValueCapture?.entry) {
+                attachTraceValueEntry(evt, errorValueCapture.entry);
+            }
+            if (errorValueCapture?.captureRef) {
+                evt.errorValueCapture = errorValueCapture.captureRef;
+            }
         }
     }
     return events;
@@ -3710,6 +3825,12 @@ async function maybeCaptureRequestValueAsync(params, sink) {
     }
     return result.captureRef;
 }
+function maybeCaptureTraceValue(params) {
+    return createCapturedValueEntry(params);
+}
+async function maybeCaptureTraceValueAsync(params) {
+    return await createCapturedValueEntryAsync(params);
+}
 function maybeCaptureDbValue(params, sink) {
     const result = createCapturedValueEntry(params);
     if (!result)
@@ -5288,14 +5409,71 @@ function recordKafkaTraceEvent(raw, sink, cfg, maskReq) {
     };
     const privacy = getRuntimePrivacyState(cfg).policy ?? null;
     const masking = normalizeMaskingConfig(cfg.masking);
+    const captureContext = {
+        runtimeConfig: cfg,
+        captureHeaders: cfg.captureHeaders,
+        maskReq,
+        trace: candidate,
+        masking,
+        privacy,
+    };
+    const traceValueMetadata = {
+        fn: evt.fn,
+        file: evt.file,
+        line: evt.line ?? null,
+        spanId: evt.spanId ?? null,
+        parentSpanId: evt.parentSpanId ?? null,
+    };
     if (!omitJsonBuiltinValues && raw.args !== undefined) {
-        evt.args = sanitizeMaterializedTraceValue(materializeTracePrivacyValue('trace.args', sanitizeTraceArgsForPrivacy(raw.args), cfg, maskReq, candidate, masking, privacy));
+        const argsMaterialized = materializeTracePrivacyValue('trace.args', sanitizeTraceArgsForPrivacy(raw.args), cfg, maskReq, candidate, masking, privacy);
+        evt.args = limitMaterializedTraceInlineValue(argsMaterialized);
+        const argsValueCapture = maybeCaptureTraceValue({
+            target: 'trace.args',
+            rawValue: normalizeRawTraceArgs(raw.args),
+            previewValue: evt.args,
+            capture: captureContext,
+            metadata: traceValueMetadata,
+        });
+        if (argsValueCapture?.entry) {
+            attachTraceValueEntry(evt, argsValueCapture.entry);
+        }
+        if (argsValueCapture?.captureRef) {
+            evt.argsValueCapture = argsValueCapture.captureRef;
+        }
     }
     if (!omitJsonBuiltinValues && raw.returnValue !== undefined) {
-        evt.returnValue = sanitizeMaterializedTraceValue(materializeTracePrivacyValue('trace.returnValue', sanitizeTraceValueForPrivacy(raw.returnValue), cfg, maskReq, candidate, masking, privacy));
+        const returnValueMaterialized = materializeTracePrivacyValue('trace.returnValue', sanitizeTraceValueForPrivacy(raw.returnValue), cfg, maskReq, candidate, masking, privacy);
+        evt.returnValue = limitMaterializedTraceInlineValue(returnValueMaterialized);
+        const returnValueCapture = maybeCaptureTraceValue({
+            target: 'trace.returnValue',
+            rawValue: raw.returnValue,
+            previewValue: evt.returnValue,
+            capture: captureContext,
+            metadata: traceValueMetadata,
+        });
+        if (returnValueCapture?.entry) {
+            attachTraceValueEntry(evt, returnValueCapture.entry);
+        }
+        if (returnValueCapture?.captureRef) {
+            evt.returnValueCapture = returnValueCapture.captureRef;
+        }
     }
     if (hasMeaningfulRawTraceError(raw.error)) {
-        evt.error = sanitizeMaterializedTraceValue(materializeTracePrivacyValue('trace.error', sanitizeTraceValueForPrivacy(raw.error), cfg, maskReq, candidate, masking, privacy));
+        const errorMaterialized = materializeTracePrivacyValue('trace.error', sanitizeTraceValueForPrivacy(raw.error), cfg, maskReq, candidate, masking, privacy);
+        evt.error = limitMaterializedTraceInlineValue(errorMaterialized);
+        const errorValueCapture = maybeCaptureTraceValue({
+            target: 'trace.error',
+            rawValue: raw.error,
+            previewValue: evt.error,
+            capture: captureContext,
+            metadata: traceValueMetadata,
+        });
+        if (errorValueCapture?.entry) {
+            attachTraceValueEntry(evt, errorValueCapture.entry);
+        }
+        if (errorValueCapture?.captureRef) {
+            evt.errorValueCapture = errorValueCapture.captureRef;
+        }
     }
     if (raw.threw !== undefined)
         evt.threw = raw.threw === true;
@@ -5388,6 +5566,7 @@ function buildKafkaTraceEntries(actionId, requestRid, events) {
     return batches.map((batch, index) => ({
         actionId: actionId ?? null,
         trace: batch,
+        traceValues: collectBatchTraceValueEntries(batch, index),
         traceBatch: {
             rid: requestRid,
             index,
@@ -6727,4 +6906,5 @@ exports.__reproTestHooks = {
     estimateTraceBatchSerializedBytesForTest: estimateTraceBatchSerializedBytes,
     sanitizeTraceValueForPrivacyForTest: sanitizeTraceValueForPrivacy,
     sanitizeMaterializedTraceValueForTest: sanitizeMaterializedTraceValue,
+    getEventTraceValueEntriesForTest: getEventTraceValueEntries,
 };

package/dist/ingest/client.js

@@ -4,7 +4,13 @@ exports.flushIngestQueue = exports.enqueueIngestEntries = exports.postIngestEntr
 const mapper_1 = require("./mapper");
 const DEFAULT_INGEST_BASE = 'http://localhost:8080';
 const DEFAULT_INGEST_PATH = '/v1/ingest/events';
-const DEFAULT_QUEUE_MAX_ITEMS = 1000;
+const DEFAULT_QUEUE_MAX_ITEMS = (() => {
+    const env = Number(typeof process !== 'undefined' ? process?.env?.REPRO_INGEST_QUEUE_MAX_ITEMS : undefined);
+    if (Number.isFinite(env) && env > 0) {
+        return Math.trunc(env);
+    }
+    return 0;
+})();
 const DEFAULT_QUEUE_MAX_ATTEMPTS = 5;
 const DEFAULT_QUEUE_BACKOFF_MAX_MS = 4000;
 const DEFAULT_DEFER_DRAIN_MS = 25;
@@ -142,7 +148,7 @@ const enqueueIngestEntries = (config, sessionId, entries) => {
         entries: entries.slice(),
         attempts: 0,
     });
-    if (ingestQueue.length > DEFAULT_QUEUE_MAX_ITEMS) {
+    if (DEFAULT_QUEUE_MAX_ITEMS > 0 && ingestQueue.length > DEFAULT_QUEUE_MAX_ITEMS) {
         ingestQueue.splice(0, ingestQueue.length - DEFAULT_QUEUE_MAX_ITEMS);
     }
     scheduleDrain(shouldDeferDrain?.() ? deferDrainMs : 0);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@reproapp/node-sdk",
-  "version": "0.0.11",
+  "version": "0.0.13",
   "description": "Repro Nest SDK",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,7 +12,7 @@
     "build": "tsc -p tsconfig.json",
     "dev": "tsc -p tsconfig.json --watch --preserveWatchOutput",
     "prepublishOnly": "npm run build",
-    "test": "npm run build && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node test/sdk-background-priority.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js"
+    "test": "npm run build && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node test/sdk-background-priority.test.js && node test/ingest-queue-retention.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js"
   },
   "peerDependencies": {
     "express": "^5.1.0",

package/src/index.ts

@@ -351,9 +351,12 @@ type TraceEventRecord = {
     spanId?: string | number | null;
     parentSpanId?: string | number | null;
     args?: any;
+    argsValueCapture?: TraceValueCaptureRef;
     returnValue?: any;
+    returnValueCapture?: TraceValueCaptureRef;
     threw?: boolean;
     error?: any;
+    errorValueCapture?: TraceValueCaptureRef;
     unawaited?: boolean;
 };
 
@@ -387,7 +390,10 @@ type TraceValueBatchEntry = {
         | 'request.body'
         | 'request.params'
         | 'request.query'
-        | 'response.body';
+        | 'response.body'
+        | 'trace.args'
+        | 'trace.returnValue'
+        | 'trace.error';
     value: any;
     truncatedForStorage?: boolean;
     projectedKind?: string;
@@ -2533,6 +2539,53 @@ function sanitizeMaterializedTraceValue(value: any): any {
     return sanitizeTraceValue(value, 0, new WeakMap(), { preserveLongStrings: true, disableTruncation: true });
 }
 
+function compactMaterializedTracePreview(value: any, depth = 0): any {
+    if (value === undefined || value === null) return value;
+    if (depth >= TRACE_VALUE_MAX_DEPTH) {
+        return makeOmittedValue(`depth>${TRACE_VALUE_MAX_DEPTH}`);
+    }
+
+    const type = typeof value;
+    if (type === 'string') {
+        if (value.length <= TRACE_VALUE_MAX_STRING) return value;
+        return value.slice(0, TRACE_VALUE_MAX_STRING);
+    }
+    if (type === 'number' || type === 'boolean') return value;
+    if (type === 'bigint') return value.toString();
+    if (type === 'symbol') return value.toString();
+    if (type === 'function') return `[Function${value.name ? ` ${value.name}` : ''}]`;
+    if (type !== 'object') return String(value);
+
+    if (Array.isArray(value)) {
+        return value
+            .slice(0, TRACE_VALUE_MAX_ITEMS)
+            .map((item) => compactMaterializedTracePreview(item, depth + 1));
+    }
+
+    const out: Record<string, any> = {};
+    const keys = Object.keys(value).slice(0, TRACE_VALUE_MAX_KEYS);
+    for (const key of keys) {
+        try {
+            out[key] = compactMaterializedTracePreview((value as Record<string, any>)[key], depth + 1);
+        } catch (err) {
+            out[key] = `[Cannot serialize: ${(err as Error)?.message || 'unknown error'}]`;
+        }
+    }
+    return out;
+}
+
+function limitMaterializedTraceInlineValue(value: any): any {
+    const inlineValue = sanitizeMaterializedTraceValue(value);
+    if (!traceInlineValueExceedsLimit(inlineValue)) {
+        return inlineValue;
+    }
+    const compactPreview = compactMaterializedTracePreview(inlineValue);
+    if (!traceInlineValueExceedsLimit(compactPreview)) {
+        return compactPreview;
+    }
+    return makeOmittedValue('inline-size-limit');
+}
+
 function sanitizeTraceArgsForPrivacy(values: any): any {
     if (!Array.isArray(values)) return [sanitizeTraceValueForPrivacy(values)];
     return values.map(v => sanitizeTraceValueForPrivacy(v));
@@ -2547,6 +2600,18 @@ function hasMeaningfulRawTraceError(error: any): boolean {
     return error !== undefined && error !== null;
 }
 
+function attachTraceValueEntry(
+    event: TraceEventRecord | PendingTraceEventRecord,
+    entry: TraceValueBatchEntry,
+): void {
+    const current = (event as any)[TRACE_FULL_VALUE_ENTRY_SYMBOL] as TraceValueBatchEntry[] | undefined;
+    if (Array.isArray(current)) {
+        current.push(entry);
+        return;
+    }
+    (event as any)[TRACE_FULL_VALUE_ENTRY_SYMBOL] = [entry];
+}
+
 async function materializePendingTraceEvents(
     events: PendingTraceEventRecord[],
     params: {
@@ -2564,9 +2629,24 @@ async function materializePendingTraceEvents(
         const candidate = pending.candidate ?? traceEventCandidateFromRecord(evt);
         const omitJsonBuiltinValues = shouldOmitJsonBuiltinTraceValues(candidate);
         const activePrivacy = params.resolvePrivacy();
+        const captureContext: FullValueCaptureContext = {
+            runtimeConfig: params.cfg,
+            captureHeaders: params.cfg.captureHeaders,
+            maskReq: params.maskReq,
+            trace: candidate,
+            masking: params.masking,
+            privacy: activePrivacy,
+        };
+        const traceValueMetadata = {
+            fn: evt.fn,
+            file: evt.file,
+            line: evt.line ?? null,
+            spanId: evt.spanId ?? null,
+            parentSpanId: evt.parentSpanId ?? null,
+        };
 
         if (!omitJsonBuiltinValues && pending.argsRaw !== undefined) {
-            evt.args = sanitizeMaterializedTraceValue(await materializeTracePrivacyValueAsync(
+            const argsMaterialized = await materializeTracePrivacyValueAsync(
                 'trace.args',
                 sanitizeTraceArgsForPrivacy(pending.argsRaw),
                 params.cfg,
@@ -2574,10 +2654,24 @@ async function materializePendingTraceEvents(
                 candidate,
                 params.masking,
                 activePrivacy,
-            ));
+            );
+            evt.args = limitMaterializedTraceInlineValue(argsMaterialized);
+            const argsValueCapture = await maybeCaptureTraceValueAsync({
+                target: 'trace.args',
+                rawValue: pending.argsRaw,
+                previewValue: evt.args,
+                capture: captureContext,
+                metadata: traceValueMetadata,
+            });
+            if (argsValueCapture?.entry) {
+                attachTraceValueEntry(evt, argsValueCapture.entry);
+            }
+            if (argsValueCapture?.captureRef) {
+                evt.argsValueCapture = argsValueCapture.captureRef;
+            }
         }
         if (!omitJsonBuiltinValues && pending.returnValueRaw !== undefined) {
-            evt.returnValue = sanitizeMaterializedTraceValue(await materializeTracePrivacyValueAsync(
+            const returnValueMaterialized = await materializeTracePrivacyValueAsync(
                 'trace.returnValue',
                 sanitizeTraceValueForPrivacy(pending.returnValueRaw),
                 params.cfg,
@@ -2585,10 +2679,24 @@ async function materializePendingTraceEvents(
                 candidate,
                 params.masking,
                 activePrivacy,
-            ));
+            );
+            evt.returnValue = limitMaterializedTraceInlineValue(returnValueMaterialized);
+            const returnValueCapture = await maybeCaptureTraceValueAsync({
+                target: 'trace.returnValue',
+                rawValue: pending.returnValueRaw,
+                previewValue: evt.returnValue,
+                capture: captureContext,
+                metadata: traceValueMetadata,
+            });
+            if (returnValueCapture?.entry) {
+                attachTraceValueEntry(evt, returnValueCapture.entry);
+            }
+            if (returnValueCapture?.captureRef) {
+                evt.returnValueCapture = returnValueCapture.captureRef;
+            }
         }
         if (pending.errorRaw !== undefined) {
-            evt.error = sanitizeMaterializedTraceValue(await materializeTracePrivacyValueAsync(
+            const errorMaterialized = await materializeTracePrivacyValueAsync(
                 'trace.error',
                 sanitizeTraceValueForPrivacy(pending.errorRaw),
                 params.cfg,
@@ -2596,7 +2704,21 @@ async function materializePendingTraceEvents(
                 candidate,
                 params.masking,
                 activePrivacy,
-            ));
+            );
+            evt.error = limitMaterializedTraceInlineValue(errorMaterialized);
+            const errorValueCapture = await maybeCaptureTraceValueAsync({
+                target: 'trace.error',
+                rawValue: pending.errorRaw,
+                previewValue: evt.error,
+                capture: captureContext,
+                metadata: traceValueMetadata,
+            });
+            if (errorValueCapture?.entry) {
+                attachTraceValueEntry(evt, errorValueCapture.entry);
+            }
+            if (errorValueCapture?.captureRef) {
+                evt.errorValueCapture = errorValueCapture.captureRef;
+            }
         }
     }
     return events;
@@ -4325,7 +4447,10 @@ function createCapturedValueEntry(
             | 'request.body'
             | 'request.params'
             | 'request.query'
-            | 'response.body';
+            | 'response.body'
+            | 'trace.args'
+            | 'trace.returnValue'
+            | 'trace.error';
         rawValue: any;
         previewValue: any;
         capture: FullValueCaptureContext;
@@ -4404,7 +4529,10 @@ async function createCapturedValueEntryAsync(
             | 'request.body'
             | 'request.params'
             | 'request.query'
-            | 'response.body';
+            | 'response.body'
+            | 'trace.args'
+            | 'trace.returnValue'
+            | 'trace.error';
         rawValue: any;
         previewValue: any;
         capture: FullValueCaptureContext;
@@ -4504,6 +4632,30 @@ async function maybeCaptureRequestValueAsync(
     return result.captureRef;
 }
 
+function maybeCaptureTraceValue(
+    params: {
+        target: 'trace.args' | 'trace.returnValue' | 'trace.error';
+        rawValue: any;
+        previewValue: any;
+        capture: FullValueCaptureContext;
+        metadata?: Partial<TraceValueBatchEntry>;
+    },
+): { captureRef: TraceValueCaptureRef; entry?: TraceValueBatchEntry } | undefined {
+    return createCapturedValueEntry(params);
+}
+
+async function maybeCaptureTraceValueAsync(
+    params: {
+        target: 'trace.args' | 'trace.returnValue' | 'trace.error';
+        rawValue: any;
+        previewValue: any;
+        capture: FullValueCaptureContext;
+        metadata?: Partial<TraceValueBatchEntry>;
+    },
+): Promise<{ captureRef: TraceValueCaptureRef; entry?: TraceValueBatchEntry } | undefined> {
+    return await createCapturedValueEntryAsync(params);
+}
+
 function maybeCaptureDbValue(
     params: {
         target: 'db.pk' | 'db.before' | 'db.after' | 'db.query' | 'db.resultMeta' | 'db.error';
@@ -6222,8 +6374,23 @@ function recordKafkaTraceEvent(
 
     const privacy = getRuntimePrivacyState(cfg).policy ?? null;
     const masking = normalizeMaskingConfig(cfg.masking);
+    const captureContext: FullValueCaptureContext = {
+        runtimeConfig: cfg,
+        captureHeaders: cfg.captureHeaders,
+        maskReq,
+        trace: candidate,
+        masking,
+        privacy,
+    };
+    const traceValueMetadata = {
+        fn: evt.fn,
+        file: evt.file,
+        line: evt.line ?? null,
+        spanId: evt.spanId ?? null,
+        parentSpanId: evt.parentSpanId ?? null,
+    };
     if (!omitJsonBuiltinValues && raw.args !== undefined) {
-        evt.args = sanitizeMaterializedTraceValue(materializeTracePrivacyValue(
+        const argsMaterialized = materializeTracePrivacyValue(
             'trace.args',
             sanitizeTraceArgsForPrivacy(raw.args),
             cfg,
@@ -6231,10 +6398,24 @@ function recordKafkaTraceEvent(
             candidate,
             masking,
             privacy,
-        ));
+        );
+        evt.args = limitMaterializedTraceInlineValue(argsMaterialized);
+        const argsValueCapture = maybeCaptureTraceValue({
+            target: 'trace.args',
+            rawValue: normalizeRawTraceArgs(raw.args),
+            previewValue: evt.args,
+            capture: captureContext,
+            metadata: traceValueMetadata,
+        });
+        if (argsValueCapture?.entry) {
+            attachTraceValueEntry(evt, argsValueCapture.entry);
+        }
+        if (argsValueCapture?.captureRef) {
+            evt.argsValueCapture = argsValueCapture.captureRef;
+        }
     }
     if (!omitJsonBuiltinValues && raw.returnValue !== undefined) {
-        evt.returnValue = sanitizeMaterializedTraceValue(materializeTracePrivacyValue(
+        const returnValueMaterialized = materializeTracePrivacyValue(
             'trace.returnValue',
             sanitizeTraceValueForPrivacy(raw.returnValue),
             cfg,
@@ -6242,10 +6423,24 @@ function recordKafkaTraceEvent(
             candidate,
             masking,
             privacy,
-        ));
+        );
+        evt.returnValue = limitMaterializedTraceInlineValue(returnValueMaterialized);
+        const returnValueCapture = maybeCaptureTraceValue({
+            target: 'trace.returnValue',
+            rawValue: raw.returnValue,
+            previewValue: evt.returnValue,
+            capture: captureContext,
+            metadata: traceValueMetadata,
+        });
+        if (returnValueCapture?.entry) {
+            attachTraceValueEntry(evt, returnValueCapture.entry);
+        }
+        if (returnValueCapture?.captureRef) {
+            evt.returnValueCapture = returnValueCapture.captureRef;
+        }
     }
     if (hasMeaningfulRawTraceError(raw.error)) {
-        evt.error = sanitizeMaterializedTraceValue(materializeTracePrivacyValue(
+        const errorMaterialized = materializeTracePrivacyValue(
             'trace.error',
             sanitizeTraceValueForPrivacy(raw.error),
             cfg,
@@ -6253,7 +6448,21 @@ function recordKafkaTraceEvent(
             candidate,
             masking,
             privacy,
-        ));
+        );
+        evt.error = limitMaterializedTraceInlineValue(errorMaterialized);
+        const errorValueCapture = maybeCaptureTraceValue({
+            target: 'trace.error',
+            rawValue: raw.error,
+            previewValue: evt.error,
+            capture: captureContext,
+            metadata: traceValueMetadata,
+        });
+        if (errorValueCapture?.entry) {
+            attachTraceValueEntry(evt, errorValueCapture.entry);
+        }
+        if (errorValueCapture?.captureRef) {
+            evt.errorValueCapture = errorValueCapture.captureRef;
+        }
     }
     if (raw.threw !== undefined) evt.threw = raw.threw === true;
     if (raw.unawaited !== undefined) evt.unawaited = raw.unawaited === true;
@@ -6362,6 +6571,7 @@ function buildKafkaTraceEntries(
     return batches.map((batch, index) => ({
         actionId: actionId ?? null,
         trace: batch,
+        traceValues: collectBatchTraceValueEntries(batch, index),
         traceBatch: {
             rid: requestRid,
             index,
@@ -7752,4 +7962,5 @@ export const __reproTestHooks = {
     estimateTraceBatchSerializedBytesForTest: estimateTraceBatchSerializedBytes,
     sanitizeTraceValueForPrivacyForTest: sanitizeTraceValueForPrivacy,
     sanitizeMaterializedTraceValueForTest: sanitizeMaterializedTraceValue,
+    getEventTraceValueEntriesForTest: getEventTraceValueEntries,
 };

package/src/ingest/client.ts

@@ -3,7 +3,13 @@ import type { IngestClientConfig, LegacyEntry } from './types';
 
 const DEFAULT_INGEST_BASE = 'http://localhost:8080';
 const DEFAULT_INGEST_PATH = '/v1/ingest/events';
-const DEFAULT_QUEUE_MAX_ITEMS = 1000;
+const DEFAULT_QUEUE_MAX_ITEMS = (() => {
+    const env = Number(typeof process !== 'undefined' ? (process as any)?.env?.REPRO_INGEST_QUEUE_MAX_ITEMS : undefined);
+    if (Number.isFinite(env) && env > 0) {
+        return Math.trunc(env);
+    }
+    return 0;
+})();
 const DEFAULT_QUEUE_MAX_ATTEMPTS = 5;
 const DEFAULT_QUEUE_BACKOFF_MAX_MS = 4000;
 const DEFAULT_DEFER_DRAIN_MS = 25;
@@ -177,7 +183,7 @@ export const enqueueIngestEntries = (
         entries: entries.slice(),
         attempts: 0,
     });
-    if (ingestQueue.length > DEFAULT_QUEUE_MAX_ITEMS) {
+    if (DEFAULT_QUEUE_MAX_ITEMS > 0 && ingestQueue.length > DEFAULT_QUEUE_MAX_ITEMS) {
         ingestQueue.splice(0, ingestQueue.length - DEFAULT_QUEUE_MAX_ITEMS);
     }
     scheduleDrain(shouldDeferDrain?.() ? deferDrainMs : 0);

package/test/ingest-queue-retention.test.js

@@ -0,0 +1,82 @@
+const assert = require('assert');
+
+const originalFetch = global.fetch;
+const capturedBodies = [];
+
+global.fetch = async (_url, init = {}) => {
+    capturedBodies.push(JSON.parse(String(init.body || '{}')));
+    return {
+        ok: true,
+        status: 200,
+        json: async () => ({ ok: true }),
+        text: async () => '{"ok":true}',
+    };
+};
+
+const { enqueueIngestEntries, flushIngestQueue } = require('../dist/ingest/client');
+
+async function main() {
+    const cfg = {
+        tenantId: 'TENANT_test',
+        appId: 'APP_test',
+        appSecret: 'secret',
+        serviceName: 'queue-retention-test',
+        ingestBase: 'http://127.0.0.1:65535',
+    };
+
+    const sessionId = 'S_queue_retention';
+    const requestRid = 'R_queue_retention';
+    const totalBatches = 1414;
+
+    for (let index = 0; index < totalBatches; index += 1) {
+        enqueueIngestEntries(cfg, sessionId, [{
+            actionId: 'A_queue_retention',
+            trace: [{
+                t: Date.now(),
+                type: 'enter',
+                fn: `trace-${index}`,
+            }],
+            traceBatch: {
+                rid: requestRid,
+                index,
+                total: totalBatches,
+            },
+            t: Date.now(),
+        }]);
+    }
+
+    await flushIngestQueue();
+
+    const traceEvents = capturedBodies.flatMap((body) => Array.isArray(body?.events) ? body.events : [])
+        .filter((event) => event?.event_type === 'trace_batch');
+
+    assert.strictEqual(traceEvents.length, totalBatches, JSON.stringify({
+        got: traceEvents.length,
+        expected: totalBatches,
+    }));
+
+    const indexes = traceEvents
+        .map((event) => Number(event?.payload?.traceBatch?.index))
+        .filter((value) => Number.isFinite(value))
+        .sort((left, right) => left - right);
+
+    assert.strictEqual(indexes.length, totalBatches, JSON.stringify({
+        got: indexes.length,
+        expected: totalBatches,
+    }));
+    assert.strictEqual(indexes[0], 0);
+    assert.strictEqual(indexes[indexes.length - 1], totalBatches - 1);
+
+    // eslint-disable-next-line no-console
+    console.log('ingest queue retention OK');
+}
+
+main()
+    .catch((error) => {
+        // eslint-disable-next-line no-console
+        console.error(error);
+        process.exitCode = 1;
+    })
+    .finally(() => {
+        global.fetch = originalFetch;
+    });

package/test/kafka-runtime-privacy-policy.test.js

@@ -419,7 +419,19 @@ async function testKafkaProducerCapturesFullPublishedMessageValue() {
 }
 
 function assertNoLargeJsonPreviewArtifacts(value) {
-    const text = JSON.stringify(value);
+    const text = JSON.stringify(value, (key, currentValue) => {
+        if (
+            key === 'argsValueCapture' ||
+            key === 'returnValueCapture' ||
+            key === 'errorValueCapture' ||
+            key === 'argsMaterialization' ||
+            key === 'returnValueMaterialization' ||
+            key === 'errorMaterialization'
+        ) {
+            return undefined;
+        }
+        return currentValue;
+    });
     assert(!text.includes('__type":"json-string'), text);
     assert(!text.includes('__truncatedKeys'), text);
     assert(!text.includes('__truncatedItems'), text);
@@ -540,7 +552,7 @@ async function testJsonBuiltinTracePayloadsAreOmitted() {
     assertNoLargeJsonPreviewArtifacts(consumedSink[0]);
 }
 
-async function testLargeStringTraceValuesAreCapturedInlineWithoutTruncation() {
+async function testLargeStringTraceValuesStayIngestibleAndCaptureFullValueOutOfBand() {
     const hugeMarker = 'HUGE_INLINE_VALUE_MARKER';
     const hugeJson = JSON.stringify({
         eventId: 'evt-huge',
@@ -586,13 +598,18 @@ async function testLargeStringTraceValuesAreCapturedInlineWithoutTruncation() {
     assert.strictEqual(sink[0].fn, 'parseGatewayPayload');
     assert.strictEqual(typeof sink[0].args[0], 'string');
     assert(sink[0].args[0].includes(hugeMarker), JSON.stringify(sink[0]));
-    assert(sink[0].args[0].includes('x'.repeat(140 * 1024)), JSON.stringify(sink[0]));
-    assert.strictEqual(sink[0].argsMaterialization, undefined, JSON.stringify(sink[0]));
-    assert.strictEqual(sink[0].argsValueCapture, undefined, JSON.stringify(sink[0]));
-    assertNoLargeJsonPreviewArtifacts(sink[0]);
+    assert(sink[0].args[0].length < 4096, JSON.stringify(sink[0]));
+    assert.strictEqual(sink[0].argsValueCapture?.status, 'stored', JSON.stringify(sink[0]));
+    const entries = __reproTestHooks.getEventTraceValueEntriesForTest(sink[0]);
+    assert(Array.isArray(entries) && entries.length === 1, JSON.stringify(entries));
+    assert.strictEqual(entries[0].target, 'trace.args');
+    assert.strictEqual(typeof entries[0].value?.[0], 'string');
+    assert(entries[0].value[0].includes(hugeMarker), JSON.stringify(entries[0]));
+    assert(entries[0].value[0].includes('x'.repeat(140 * 1024)), JSON.stringify(entries[0]));
+    assertNoLargeJsonPreviewArtifacts(sink[0].args);
 }
 
-async function testLargeStructuredTraceValuesAreCapturedInlineWithoutTruncation() {
+async function testLargeStructuredTraceValuesStayIngestibleAndCaptureFullValueOutOfBand() {
     const hugeMarker = 'HUGE_STRUCTURED_TRACE_VALUE_MARKER';
     const hugeEnvelope = {
         eventId: 'evt-huge-structured',
@@ -641,19 +658,29 @@ async function testLargeStructuredTraceValuesAreCapturedInlineWithoutTruncation(
 
     assert.strictEqual(sink.length, 1);
     assert.strictEqual(sink[0].fn, 'recordConsumedEvent');
-    assert(sink[0].args, 'expected args to be stored inline');
-    assert.strictEqual(sink[0].args[0].providerResponse.rawNarrative, `${hugeMarker}:${'x'.repeat(40 * 1024)}`);
+    assert(sink[0].args, 'expected args preview to be stored inline');
+    assert(JSON.stringify(sink[0].args).length < 9000, JSON.stringify(sink[0]).slice(0, 12000));
+    assert.strictEqual(
+        sink[0].args[0]?.providerResponse?.rawNarrative?.__kind,
+        'omitted',
+        JSON.stringify(sink[0]),
+    );
     assert.strictEqual(typeof sink[0].args[2], 'string');
-    const parsedRawArg = JSON.parse(sink[0].args[2]);
+    assert(sink[0].args[2].includes('providerResponse'), JSON.stringify(sink[0]));
+    assert.strictEqual(sink[0].argsValueCapture?.status, 'stored', JSON.stringify(sink[0]));
+    const entries = __reproTestHooks.getEventTraceValueEntriesForTest(sink[0]);
+    assert(Array.isArray(entries) && entries.length === 1, JSON.stringify(entries));
+    assert.strictEqual(entries[0].target, 'trace.args');
+    assert.strictEqual(entries[0].value?.[0]?.providerResponse?.rawNarrative, `${hugeMarker}:${'x'.repeat(40 * 1024)}`);
+    assert.strictEqual(typeof entries[0].value?.[2], 'string');
+    const parsedRawArg = JSON.parse(entries[0].value[2]);
     assert.strictEqual(parsedRawArg.providerResponse.rawNarrative, `${hugeMarker}:${'x'.repeat(40 * 1024)}`);
     assert.strictEqual(parsedRawArg.providerResponse.credentials.apiKey, '[dropped]');
     assert.strictEqual(parsedRawArg.providerResponse.credentials.webhookSecret, '[dropped]');
-    assert.strictEqual(sink[0].argsMaterialization, undefined, JSON.stringify(sink[0]));
-    assert.strictEqual(sink[0].argsValueCapture, undefined, JSON.stringify(sink[0]));
-    assertNoLargeJsonPreviewArtifacts(sink[0]);
+    assertNoLargeJsonPreviewArtifacts(sink[0].args);
 }
 
-async function testLargeTraceReturnValueIsCapturedInlineWithoutTruncation() {
+async function testLargeTraceReturnValueStaysIngestibleAndCapturesFullValueOutOfBand() {
     const hugeMarker = 'HUGE_STRUCTURED_TRACE_RETURN_MARKER';
     const hugeResponse = {
         id: 'drill-1',
@@ -698,11 +725,17 @@ async function testLargeTraceReturnValueIsCapturedInlineWithoutTruncation() {
 
     assert.strictEqual(sink.length, 1);
     assert.strictEqual(sink[0].fn, 'toDrillResponse');
-    assert(sink[0].returnValue, 'expected return value to be stored inline');
-    assert.strictEqual(sink[0].returnValue.structuredSnapshot.payload, `${hugeMarker}:${'x'.repeat(40 * 1024)}`);
-    assert.strictEqual(sink[0].returnValueMaterialization, undefined, JSON.stringify(sink[0]));
-    assert.strictEqual(sink[0].returnValueCapture, undefined, JSON.stringify(sink[0]));
-    assertNoLargeJsonPreviewArtifacts(sink[0]);
+    assert(sink[0].returnValue, 'expected return value preview to be stored inline');
+    assert(JSON.stringify(sink[0].returnValue).length < 9000, JSON.stringify(sink[0]).slice(0, 12000));
+    assert.strictEqual(typeof sink[0].returnValue?.structuredSnapshot?.payload, 'string');
+    assert(sink[0].returnValue.structuredSnapshot.payload.includes(hugeMarker), JSON.stringify(sink[0]));
+    assert(sink[0].returnValue.structuredSnapshot.payload.length < 4096, JSON.stringify(sink[0]));
+    assert.strictEqual(sink[0].returnValueCapture?.status, 'stored', JSON.stringify(sink[0]));
+    const entries = __reproTestHooks.getEventTraceValueEntriesForTest(sink[0]);
+    assert(Array.isArray(entries) && entries.length === 1, JSON.stringify(entries));
+    assert.strictEqual(entries[0].target, 'trace.returnValue');
+    assert.strictEqual(entries[0].value?.structuredSnapshot?.payload, `${hugeMarker}:${'x'.repeat(40 * 1024)}`);
+    assertNoLargeJsonPreviewArtifacts(sink[0].returnValue);
 }
 
 async function main() {
@@ -712,9 +745,9 @@ async function main() {
     await testKafkaTraceFlushIsNotBlockedForeverByActiveHttpRequest();
     await testLargeHttpResponseBodyIsCapturedWithoutDroppingRequest();
     await testJsonBuiltinTracePayloadsAreOmitted();
-    await testLargeStringTraceValuesAreCapturedInlineWithoutTruncation();
-    await testLargeStructuredTraceValuesAreCapturedInlineWithoutTruncation();
-    await testLargeTraceReturnValueIsCapturedInlineWithoutTruncation();
+    await testLargeStringTraceValuesStayIngestibleAndCaptureFullValueOutOfBand();
+    await testLargeStructuredTraceValuesStayIngestibleAndCaptureFullValueOutOfBand();
+    await testLargeTraceReturnValueStaysIngestibleAndCapturesFullValueOutOfBand();
     console.log('kafka runtime privacy policy wiring OK');
 }