@reproapp/node-sdk 0.0.39 → 0.0.42

package/dist/ingest/client.js

@@ -14,6 +14,22 @@ const DEFAULT_QUEUE_MAX_ITEMS = (() => {
 const DEFAULT_QUEUE_MAX_ATTEMPTS = 5;
 const DEFAULT_QUEUE_BACKOFF_MAX_MS = 4000;
 const DEFAULT_DEFER_DRAIN_MS = 25;
+const DEFAULT_BATCH_MAX_ENTRIES = (() => {
+    const env = Number(typeof process !== 'undefined' ? process?.env?.REPRO_INGEST_BATCH_MAX_ENTRIES : undefined);
+    if (Number.isFinite(env) && env > 0) {
+        return Math.trunc(env);
+    }
+    return 100;
+})();
+const DEFAULT_BATCH_MAX_SERIALIZED_BYTES = (() => {
+    const env = Number(typeof process !== 'undefined'
+        ? process?.env?.REPRO_INGEST_BATCH_MAX_SERIALIZED_BYTES
+        : undefined);
+    if (Number.isFinite(env) && env > 0) {
+        return Math.trunc(env);
+    }
+    return 1024 * 1024;
+})();
 const DEFAULT_INGEST_REQUEST_TIMEOUT_MS = (() => {
     const env = Number(typeof process !== 'undefined' ? process?.env?.REPRO_INGEST_REQUEST_TIMEOUT_MS : undefined);
     if (Number.isFinite(env) && env >= 0) {
@@ -22,9 +38,13 @@ const DEFAULT_INGEST_REQUEST_TIMEOUT_MS = (() => {
     return 120000;
 })();
 const ingestQueue = [];
+const criticalIngestQueue = [];
 let drainTimer = null;
+let criticalDrainTimer = null;
 let draining = false;
+let criticalDraining = false;
 let backoffMs = 0;
+let criticalBackoffMs = 0;
 let shouldDeferDrain = null;
 let deferDrainMs = DEFAULT_DEFER_DRAIN_MS;
 const configureIngestQueueDrain = (options) => {
@@ -35,22 +55,88 @@ const configureIngestQueueDrain = (options) => {
 };
 exports.configureIngestQueueDrain = configureIngestQueueDrain;
 const kickIngestQueueDrain = () => {
+    if (criticalIngestQueue.length > 0 && !criticalDrainTimer) {
+        scheduleCriticalDrain();
+    }
     if (ingestQueue.length > 0 && !drainTimer) {
         scheduleDrain(shouldDeferDrain?.() ? deferDrainMs : 0);
     }
 };
 exports.kickIngestQueueDrain = kickIngestQueueDrain;
-const isCriticalIngestEntry = (entry) => (entry.requestCaptureManifest !== undefined ||
+const isHighPriorityIngestEntry = (entry) => (entry.requestCaptureStart !== undefined ||
+    entry.requestCaptureManifest !== undefined ||
     entry.sessionCaptureManifest !== undefined ||
     entry.request !== undefined ||
     entry.db !== undefined ||
     entry.trace !== undefined ||
     entry.email !== undefined);
 const resolveBatchPriority = (entries, options) => {
-    if (options.priority === 'high') {
-        return 'high';
+    if (options.priority === 'high' || options.priority === 'critical') {
+        return options.priority;
+    }
+    return entries.some(isHighPriorityIngestEntry) ? 'high' : 'normal';
+};
+const sameIngestDestination = (left, right) => (left.sessionId === right.sessionId &&
+    left.priority === right.priority &&
+    left.config.appId === right.config.appId &&
+    left.config.tenantId === right.config.tenantId &&
+    left.config.appSecret === right.config.appSecret &&
+    left.config.appName === right.config.appName &&
+    left.config.serviceName === right.config.serviceName &&
+    left.config.ingestBase === right.config.ingestBase &&
+    left.config.ingestPath === right.config.ingestPath &&
+    left.config.actorId === right.config.actorId &&
+    left.config.actorType === right.config.actorType &&
+    JSON.stringify(left.config.actorLabels ?? null) ===
+        JSON.stringify(right.config.actorLabels ?? null) &&
+    left.config.source === right.config.source &&
+    left.config.schemaVersion === right.config.schemaVersion);
+const serializedBytes = (value) => {
+    try {
+        return Buffer.byteLength(JSON.stringify(value), 'utf8');
+    }
+    catch {
+        return DEFAULT_BATCH_MAX_SERIALIZED_BYTES + 1;
     }
-    return entries.some(isCriticalIngestEntry) ? 'high' : 'normal';
+};
+const selectSendBatch = (queue) => {
+    const first = queue[0];
+    if (!first) {
+        return null;
+    }
+    if (first.priority === 'normal') {
+        return { batch: first, queueItems: 1 };
+    }
+    const entries = first.entries.slice();
+    let bytes = serializedBytes(entries);
+    let queueItems = 1;
+    for (let index = 1; index < queue.length && entries.length < DEFAULT_BATCH_MAX_ENTRIES; index += 1) {
+        const candidate = queue[index];
+        if (!candidate || !sameIngestDestination(first, candidate)) {
+            break;
+        }
+        if (entries.length + candidate.entries.length > DEFAULT_BATCH_MAX_ENTRIES) {
+            break;
+        }
+        const candidateBytes = serializedBytes(candidate.entries);
+        if (bytes + candidateBytes > DEFAULT_BATCH_MAX_SERIALIZED_BYTES) {
+            break;
+        }
+        entries.push(...candidate.entries);
+        bytes += candidateBytes;
+        queueItems += 1;
+    }
+    if (queueItems === 1) {
+        return { batch: first, queueItems };
+    }
+    return {
+        batch: {
+            ...first,
+            entries,
+            attempts: Math.max(...queue.slice(0, queueItems).map((item) => item.attempts)),
+        },
+        queueItems,
+    };
 };
 const resolveIngestUrl = (config) => {
     const envBase = typeof process !== 'undefined' ? process?.env?.REPRO_INGEST_BASE : undefined;
@@ -143,6 +229,51 @@ const scheduleDrain = (delayMs = 0) => {
     }
     catch { }
 };
+const scheduleCriticalDrain = (delayMs = 0) => {
+    if (criticalDrainTimer) {
+        return;
+    }
+    criticalDrainTimer = setTimeout(() => {
+        criticalDrainTimer = null;
+        void drainCriticalIngestQueue();
+    }, Math.max(0, delayMs));
+    try {
+        criticalDrainTimer.unref?.();
+    }
+    catch { }
+};
+const drainCriticalIngestQueue = async () => {
+    if (criticalDraining) {
+        return;
+    }
+    criticalDraining = true;
+    try {
+        while (criticalIngestQueue.length > 0) {
+            const selected = selectSendBatch(criticalIngestQueue);
+            if (!selected) {
+                return;
+            }
+            const ok = await sendIngestEntries(selected.batch.config, selected.batch.sessionId, selected.batch.entries);
+            if (ok) {
+                criticalIngestQueue.splice(0, selected.queueItems);
+                criticalBackoffMs = 0;
+                continue;
+            }
+            for (let index = 0; index < selected.queueItems; index += 1) {
+                criticalIngestQueue[index].attempts += 1;
+            }
+            criticalBackoffMs = Math.min(DEFAULT_QUEUE_BACKOFF_MAX_MS, criticalBackoffMs ? criticalBackoffMs * 2 : 250);
+            scheduleCriticalDrain(criticalBackoffMs);
+            return;
+        }
+    }
+    finally {
+        criticalDraining = false;
+        if (criticalIngestQueue.length > 0 && !criticalDrainTimer) {
+            scheduleCriticalDrain(criticalBackoffMs);
+        }
+    }
+};
 const drainIngestQueue = async () => {
     if (draining) {
         return;
@@ -158,15 +289,22 @@ const drainIngestQueue = async () => {
                 scheduleDrain(deferDrainMs);
                 return;
             }
-            const batch = ingestQueue[0];
+            const selected = selectSendBatch(ingestQueue);
+            if (!selected) {
+                return;
+            }
+            const batch = selected.batch;
             const ok = await sendIngestEntries(batch.config, batch.sessionId, batch.entries);
             if (ok) {
-                ingestQueue.shift();
+                ingestQueue.splice(0, selected.queueItems);
                 backoffMs = 0;
                 continue;
             }
-            batch.attempts += 1;
-            if (batch.attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS && batch.priority !== 'high') {
+            for (let index = 0; index < selected.queueItems; index += 1) {
+                ingestQueue[index].attempts += 1;
+            }
+            if (ingestQueue[0].attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS &&
+                ingestQueue[0].priority === 'normal') {
                 ingestQueue.shift();
                 backoffMs = 0;
                 continue;
@@ -195,6 +333,11 @@ const enqueueIngestEntries = (config, sessionId, entries, options = {}) => {
         attempts: 0,
         priority,
     };
+    if (priority === 'critical') {
+        criticalIngestQueue.push(batch);
+        scheduleCriticalDrain();
+        return true;
+    }
     if (priority === 'high') {
         const firstNormalIndex = ingestQueue.findIndex((queued) => queued.priority !== 'high');
         if (firstNormalIndex === -1) {
@@ -227,10 +370,15 @@ const enqueueIngestEntries = (config, sessionId, entries, options = {}) => {
 };
 exports.enqueueIngestEntries = enqueueIngestEntries;
 const flushIngestQueue = async () => {
+    if (criticalDrainTimer) {
+        clearTimeout(criticalDrainTimer);
+        criticalDrainTimer = null;
+    }
     if (drainTimer) {
         clearTimeout(drainTimer);
         drainTimer = null;
     }
+    await drainCriticalIngestQueue();
     await drainIngestQueue();
 };
 exports.flushIngestQueue = flushIngestQueue;

package/dist/ingest/mapper.js

@@ -6,6 +6,8 @@ const toIso = (timestamp) => {
     return new Date(numeric).toISOString();
 };
 const inferEventType = (entry) => {
+    if (entry.requestCaptureStart !== undefined)
+        return 'request_capture_start';
     if (entry.requestCaptureManifest !== undefined)
         return 'request_capture_manifest';
     if (entry.sessionCaptureManifest !== undefined)

package/dist/ingest/types.d.ts

@@ -17,6 +17,7 @@ export interface LegacyEntry {
     requestRid?: string | null;
     request?: unknown;
     requestValues?: unknown[];
+    requestCaptureStart?: unknown;
     requestCaptureManifest?: unknown;
     sessionCaptureManifest?: unknown;
     db?: unknown;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@reproapp/node-sdk",
-  "version": "0.0.39",
+  "version": "0.0.42",
   "description": "Repro Nest SDK",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,15 +12,21 @@
     "build": "tsc -p tsconfig.json",
     "dev": "tsc -p tsconfig.json --watch --preserveWatchOutput",
     "prepublishOnly": "npm run build",
-    "test": "npm run build && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/request-capture-background.test.js && node test/init-auto-http-server.test.js && node test/backend-live-mode.test.js && node test/client-performance-regression.test.js && node test/request-manifest-priority.test.js && node test/runtime-no-double-invoke.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node test/sdk-background-priority.test.js && node test/ingest-queue-retention.test.js && node test/ingest-high-priority-retention.test.js && node test/ingest-critical-retention.test.js && node test/ingest-critical-retry-retention.test.js && node test/ingest-timeout.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/wrap-plugin-super-call.test.js && node test/function-arg-identity.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js && node test/amqplib-integration.test.js && node test/mongodb-integration.test.js && node test/mongodb-esm-import-hook.test.js && node test/mongodb-suppression-cleanup.test.js",
-    "test:containers": "npm run build && node test/runtime-no-double-invoke.testcontainers.test.js && node test/wrap-plugin-super-call.testcontainers.test.js && node test/function-arg-identity.testcontainers.test.js && node test/mongodb-mongoose-native-suppression.testcontainers.test.js"
+    "test": "npm run build && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/request-capture-background.test.js && node test/init-auto-http-server.test.js && node test/backend-live-mode.test.js && node test/client-performance-regression.test.js && node test/request-manifest-priority.test.js && node test/runtime-no-double-invoke.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node test/sdk-background-priority.test.js && node test/ingest-queue-retention.test.js && node test/ingest-high-priority-retention.test.js && node test/ingest-critical-retention.test.js && node test/ingest-critical-retry-retention.test.js && node test/ingest-batching.test.js && node test/ingest-timeout.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/wrap-plugin-super-call.test.js && node test/function-arg-identity.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js && node test/amqplib-integration.test.js && node test/mongodb-integration.test.js && node test/mongodb-esm-import-hook.test.js && node test/mongodb-suppression-cleanup.test.js && node test/postgresql-integration.test.js",
+    "test:containers": "npm run build && node test/runtime-no-double-invoke.testcontainers.test.js && node test/wrap-plugin-super-call.testcontainers.test.js && node test/function-arg-identity.testcontainers.test.js && node test/mongodb-mongoose-native-suppression.testcontainers.test.js && node test/postgresql-integrations.testcontainers.test.js"
   },
   "peerDependencies": {
     "express": "^5.1.0",
     "mongoose": ">=6"
   },
   "devDependencies": {
-    "@types/express": "^5.0.3"
+    "@nestjs/typeorm": "^11.0.0",
+    "@types/express": "^5.0.3",
+    "@types/pg": "^8.20.0",
+    "mongodb": "^6.21.0",
+    "pg": "^8.22.0",
+    "postgres": "^3.4.9",
+    "typeorm": "^0.3.28"
   },
   "publishConfig": {
     "access": "public"