@reproapp/node-sdk 0.0.39 → 0.0.42

package/src/ingest/client.ts

@@ -13,6 +13,24 @@ const DEFAULT_QUEUE_MAX_ITEMS = (() => {
 const DEFAULT_QUEUE_MAX_ATTEMPTS = 5;
 const DEFAULT_QUEUE_BACKOFF_MAX_MS = 4000;
 const DEFAULT_DEFER_DRAIN_MS = 25;
+const DEFAULT_BATCH_MAX_ENTRIES = (() => {
+    const env = Number(typeof process !== 'undefined' ? (process as any)?.env?.REPRO_INGEST_BATCH_MAX_ENTRIES : undefined);
+    if (Number.isFinite(env) && env > 0) {
+        return Math.trunc(env);
+    }
+    return 100;
+})();
+const DEFAULT_BATCH_MAX_SERIALIZED_BYTES = (() => {
+    const env = Number(
+        typeof process !== 'undefined'
+            ? (process as any)?.env?.REPRO_INGEST_BATCH_MAX_SERIALIZED_BYTES
+            : undefined,
+    );
+    if (Number.isFinite(env) && env > 0) {
+        return Math.trunc(env);
+    }
+    return 1024 * 1024;
+})();
 const DEFAULT_INGEST_REQUEST_TIMEOUT_MS = (() => {
     const env = Number(typeof process !== 'undefined' ? (process as any)?.env?.REPRO_INGEST_REQUEST_TIMEOUT_MS : undefined);
     if (Number.isFinite(env) && env >= 0) {
@@ -26,17 +44,21 @@ type QueuedIngestBatch = {
     sessionId: string;
     entries: LegacyEntry[];
     attempts: number;
-    priority: 'normal' | 'high';
+    priority: 'normal' | 'high' | 'critical';
 };
 
 export type EnqueueIngestOptions = {
-    priority?: 'normal' | 'high';
+    priority?: 'normal' | 'high' | 'critical';
 };
 
 const ingestQueue: QueuedIngestBatch[] = [];
+const criticalIngestQueue: QueuedIngestBatch[] = [];
 let drainTimer: ReturnType<typeof setTimeout> | null = null;
+let criticalDrainTimer: ReturnType<typeof setTimeout> | null = null;
 let draining = false;
+let criticalDraining = false;
 let backoffMs = 0;
+let criticalBackoffMs = 0;
 let shouldDeferDrain: (() => boolean) | null = null;
 let deferDrainMs = DEFAULT_DEFER_DRAIN_MS;
 
@@ -51,12 +73,16 @@ export const configureIngestQueueDrain = (options: {
 };
 
 export const kickIngestQueueDrain = (): void => {
+    if (criticalIngestQueue.length > 0 && !criticalDrainTimer) {
+        scheduleCriticalDrain();
+    }
     if (ingestQueue.length > 0 && !drainTimer) {
         scheduleDrain(shouldDeferDrain?.() ? deferDrainMs : 0);
     }
 };
 
-const isCriticalIngestEntry = (entry: LegacyEntry): boolean => (
+const isHighPriorityIngestEntry = (entry: LegacyEntry): boolean => (
+    entry.requestCaptureStart !== undefined ||
     entry.requestCaptureManifest !== undefined ||
     entry.sessionCaptureManifest !== undefined ||
     entry.request !== undefined ||
@@ -69,10 +95,89 @@ const resolveBatchPriority = (
     entries: LegacyEntry[],
     options: EnqueueIngestOptions,
 ): QueuedIngestBatch['priority'] => {
-    if (options.priority === 'high') {
-        return 'high';
+    if (options.priority === 'high' || options.priority === 'critical') {
+        return options.priority;
+    }
+    return entries.some(isHighPriorityIngestEntry) ? 'high' : 'normal';
+};
+
+const sameIngestDestination = (
+    left: QueuedIngestBatch,
+    right: QueuedIngestBatch,
+): boolean => (
+    left.sessionId === right.sessionId &&
+    left.priority === right.priority &&
+    left.config.appId === right.config.appId &&
+    left.config.tenantId === right.config.tenantId &&
+    left.config.appSecret === right.config.appSecret &&
+    left.config.appName === right.config.appName &&
+    left.config.serviceName === right.config.serviceName &&
+    left.config.ingestBase === right.config.ingestBase &&
+    left.config.ingestPath === right.config.ingestPath &&
+    left.config.actorId === right.config.actorId &&
+    left.config.actorType === right.config.actorType &&
+    JSON.stringify(left.config.actorLabels ?? null) ===
+        JSON.stringify(right.config.actorLabels ?? null) &&
+    left.config.source === right.config.source &&
+    left.config.schemaVersion === right.config.schemaVersion
+);
+
+const serializedBytes = (value: unknown): number => {
+    try {
+        return Buffer.byteLength(JSON.stringify(value), 'utf8');
+    } catch {
+        return DEFAULT_BATCH_MAX_SERIALIZED_BYTES + 1;
     }
-    return entries.some(isCriticalIngestEntry) ? 'high' : 'normal';
+};
+
+const selectSendBatch = (
+    queue: QueuedIngestBatch[],
+): { batch: QueuedIngestBatch; queueItems: number } | null => {
+    const first = queue[0];
+    if (!first) {
+        return null;
+    }
+    if (first.priority === 'normal') {
+        return { batch: first, queueItems: 1 };
+    }
+
+    const entries = first.entries.slice();
+    let bytes = serializedBytes(entries);
+    let queueItems = 1;
+    for (
+        let index = 1;
+        index < queue.length && entries.length < DEFAULT_BATCH_MAX_ENTRIES;
+        index += 1
+    ) {
+        const candidate = queue[index];
+        if (!candidate || !sameIngestDestination(first, candidate)) {
+            break;
+        }
+        if (entries.length + candidate.entries.length > DEFAULT_BATCH_MAX_ENTRIES) {
+            break;
+        }
+        const candidateBytes = serializedBytes(candidate.entries);
+        if (bytes + candidateBytes > DEFAULT_BATCH_MAX_SERIALIZED_BYTES) {
+            break;
+        }
+        entries.push(...candidate.entries);
+        bytes += candidateBytes;
+        queueItems += 1;
+    }
+
+    if (queueItems === 1) {
+        return { batch: first, queueItems };
+    }
+    return {
+        batch: {
+            ...first,
+            entries,
+            attempts: Math.max(
+                ...queue.slice(0, queueItems).map((item) => item.attempts),
+            ),
+        },
+        queueItems,
+    };
 };
 
 const resolveIngestUrl = (config: IngestClientConfig): string => {
@@ -180,6 +285,59 @@ const scheduleDrain = (delayMs = 0): void => {
     } catch {}
 };
 
+const scheduleCriticalDrain = (delayMs = 0): void => {
+    if (criticalDrainTimer) {
+        return;
+    }
+    criticalDrainTimer = setTimeout(() => {
+        criticalDrainTimer = null;
+        void drainCriticalIngestQueue();
+    }, Math.max(0, delayMs));
+    try {
+        (criticalDrainTimer as any).unref?.();
+    } catch {}
+};
+
+const drainCriticalIngestQueue = async (): Promise<void> => {
+    if (criticalDraining) {
+        return;
+    }
+    criticalDraining = true;
+    try {
+        while (criticalIngestQueue.length > 0) {
+            const selected = selectSendBatch(criticalIngestQueue);
+            if (!selected) {
+                return;
+            }
+            const ok = await sendIngestEntries(
+                selected.batch.config,
+                selected.batch.sessionId,
+                selected.batch.entries,
+            );
+            if (ok) {
+                criticalIngestQueue.splice(0, selected.queueItems);
+                criticalBackoffMs = 0;
+                continue;
+            }
+
+            for (let index = 0; index < selected.queueItems; index += 1) {
+                criticalIngestQueue[index].attempts += 1;
+            }
+            criticalBackoffMs = Math.min(
+                DEFAULT_QUEUE_BACKOFF_MAX_MS,
+                criticalBackoffMs ? criticalBackoffMs * 2 : 250,
+            );
+            scheduleCriticalDrain(criticalBackoffMs);
+            return;
+        }
+    } finally {
+        criticalDraining = false;
+        if (criticalIngestQueue.length > 0 && !criticalDrainTimer) {
+            scheduleCriticalDrain(criticalBackoffMs);
+        }
+    }
+};
+
 const drainIngestQueue = async (): Promise<void> => {
     if (draining) {
         return;
@@ -195,16 +353,25 @@ const drainIngestQueue = async (): Promise<void> => {
                 scheduleDrain(deferDrainMs);
                 return;
             }
-            const batch = ingestQueue[0];
+            const selected = selectSendBatch(ingestQueue);
+            if (!selected) {
+                return;
+            }
+            const batch = selected.batch;
             const ok = await sendIngestEntries(batch.config, batch.sessionId, batch.entries);
             if (ok) {
-                ingestQueue.shift();
+                ingestQueue.splice(0, selected.queueItems);
                 backoffMs = 0;
                 continue;
             }
 
-            batch.attempts += 1;
-            if (batch.attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS && batch.priority !== 'high') {
+            for (let index = 0; index < selected.queueItems; index += 1) {
+                ingestQueue[index].attempts += 1;
+            }
+            if (
+                ingestQueue[0].attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS &&
+                ingestQueue[0].priority === 'normal'
+            ) {
                 ingestQueue.shift();
                 backoffMs = 0;
                 continue;
@@ -242,6 +409,11 @@ export const enqueueIngestEntries = (
         attempts: 0,
         priority,
     };
+    if (priority === 'critical') {
+        criticalIngestQueue.push(batch);
+        scheduleCriticalDrain();
+        return true;
+    }
     if (priority === 'high') {
         const firstNormalIndex = ingestQueue.findIndex((queued) => queued.priority !== 'high');
         if (firstNormalIndex === -1) {
@@ -272,10 +444,15 @@ export const enqueueIngestEntries = (
 };
 
 export const flushIngestQueue = async (): Promise<void> => {
+    if (criticalDrainTimer) {
+        clearTimeout(criticalDrainTimer);
+        criticalDrainTimer = null;
+    }
     if (drainTimer) {
         clearTimeout(drainTimer);
         drainTimer = null;
     }
+    await drainCriticalIngestQueue();
     await drainIngestQueue();
 };
 

package/src/ingest/mapper.ts

@@ -7,6 +7,7 @@ const toIso = (timestamp: unknown): string => {
 };
 
 const inferEventType = (entry: LegacyEntry): string => {
+  if (entry.requestCaptureStart !== undefined) return 'request_capture_start';
   if (entry.requestCaptureManifest !== undefined) return 'request_capture_manifest';
   if (entry.sessionCaptureManifest !== undefined) return 'session_capture_manifest';
   if (entry.request !== undefined) return 'backend_request';

package/src/ingest/types.ts

@@ -18,6 +18,7 @@ export interface LegacyEntry {
     requestRid?: string | null;
     request?: unknown;
     requestValues?: unknown[];
+    requestCaptureStart?: unknown;
     requestCaptureManifest?: unknown;
     sessionCaptureManifest?: unknown;
     db?: unknown;

package/test/backend-live-mode.test.js

@@ -262,8 +262,18 @@ async function main() {
         .filter((post) => post.url === 'http://log-collector.test/v1/ingest/events')
         .flatMap((post) => Array.isArray(post.body?.events) ? post.body.events : []);
       return claims.length >= 1 &&
-        events.some((event) => event.event_type === 'backend_request') &&
-        events.some((event) => event.event_type === 'request_capture_manifest');
+        events.some((event) =>
+          event.event_type === 'request_capture_start' &&
+          event.session_id === 'S_claim_child_actor-1'
+        ) &&
+        events.some((event) =>
+          event.event_type === 'backend_request' &&
+          event.session_id === 'S_claim_child_actor-1'
+        ) &&
+        events.some((event) =>
+          event.event_type === 'request_capture_manifest' &&
+          event.session_id === 'S_claim_child_actor-1'
+        );
     }, 2000);
 
     const claims = posts
@@ -277,6 +287,14 @@ async function main() {
     const ingestEvents = posts
       .filter((post) => post.url === 'http://log-collector.test/v1/ingest/events')
       .flatMap((post) => Array.isArray(post.body?.events) ? post.body.events : []);
+    assert(
+      ingestEvents.some((event) =>
+        event.event_type === 'request_capture_start' &&
+        event.session_id === 'S_claim_child_actor-1' &&
+        event.actor_id === 'actor-1'
+      ),
+      'request capture start should fence the claimed backend session',
+    );
     assert(
       ingestEvents.some((event) =>
         event.event_type === 'backend_request' &&

package/test/client-performance-regression.test.js

@@ -119,6 +119,11 @@ async function main() {
         });
 
         await sleep(2200);
+        assert.equal(
+            eventsOf(capturedBodies, 'request_capture_start').length,
+            1,
+            `request-open fence was deferred with background work: ${JSON.stringify(capturedBodies)}`,
+        );
         assert.equal(
             eventsOf(capturedBodies, 'backend_request').length,
             0,

package/test/ingest-batching.test.js

@@ -0,0 +1,67 @@
+process.env.REPRO_INGEST_BATCH_MAX_ENTRIES = '100';
+process.env.REPRO_INGEST_BATCH_MAX_SERIALIZED_BYTES = String(1024 * 1024);
+
+const assert = require('node:assert/strict');
+
+const originalFetch = global.fetch;
+const capturedBodies = [];
+
+global.fetch = async (_url, init = {}) => {
+    capturedBodies.push(JSON.parse(String(init.body || '{}')));
+    return {
+        ok: true,
+        status: 202,
+        json: async () => ({ ok: true }),
+        text: async () => '{"ok":true}',
+    };
+};
+
+const { enqueueIngestEntries, flushIngestQueue } = require('../dist/ingest/client');
+
+async function main() {
+    const cfg = {
+        tenantId: 'TENANT_test',
+        appId: 'APP_test',
+        appSecret: 'secret',
+        serviceName: 'batching-test',
+        ingestBase: 'http://127.0.0.1:65535',
+    };
+    const sessionId = 'S_ingest_batching';
+
+    for (let index = 0; index < 209; index += 1) {
+        enqueueIngestEntries(cfg, sessionId, [{
+            actionId: 'A_ingest_batching',
+            requestRid: 'R_ingest_batching',
+            db: [{
+                collection: 'subjects',
+                op: 'findOne',
+                query: { index },
+            }],
+            t: Date.now() + index,
+        }], { priority: 'high' });
+    }
+
+    await flushIngestQueue();
+
+    const events = capturedBodies.flatMap((body) =>
+        Array.isArray(body?.events) ? body.events : [],
+    );
+    assert.equal(events.length, 209);
+    assert(events.every((event) => event.event_type === 'db_change'));
+    assert.equal(
+        capturedBodies.length,
+        3,
+        `209 small evidence entries should use three bounded requests, got ${capturedBodies.length}`,
+    );
+
+    console.log('ingest batching OK');
+}
+
+main()
+    .catch((error) => {
+        console.error(error);
+        process.exitCode = 1;
+    })
+    .finally(() => {
+        global.fetch = originalFetch;
+    });