@reproapp/node-sdk 0.0.41 → 0.0.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -841,7 +841,7 @@ function emitRequestCaptureStart(cfg, sessionId, actionId, scope, openedAtMs) {
841
841
  },
842
842
  t: openedAt,
843
843
  }],
844
- }, { priority: 'high' });
844
+ }, { priority: 'critical' });
845
845
  }
846
846
  const als = new async_hooks_1.AsyncLocalStorage();
847
847
  const getCtx = () => als.getStore() || {};
@@ -985,7 +985,7 @@ const SDK_BACKGROUND_BUSY_IDLE_DELAY_MS = (() => {
985
985
  const SDK_BACKGROUND_MAX_DEFER_MS = (() => {
986
986
  const raw = String(process.env.REPRO_SDK_BACKGROUND_MAX_DEFER_MS ?? '').trim().toLowerCase();
987
987
  if (!raw) {
988
- return Number.POSITIVE_INFINITY;
988
+ return 30000;
989
989
  }
990
990
  if (raw === 'off' || raw === 'never' || raw === 'infinite' || raw === 'infinity' || raw === '-1') {
991
991
  return Number.POSITIVE_INFINITY;
@@ -1,6 +1,6 @@
1
1
  import type { IngestClientConfig, LegacyEntry } from './types';
2
2
  export type EnqueueIngestOptions = {
3
- priority?: 'normal' | 'high';
3
+ priority?: 'normal' | 'high' | 'critical';
4
4
  };
5
5
  export declare const configureIngestQueueDrain: (options: {
6
6
  shouldDefer?: (() => boolean) | null;
@@ -14,6 +14,22 @@ const DEFAULT_QUEUE_MAX_ITEMS = (() => {
14
14
  const DEFAULT_QUEUE_MAX_ATTEMPTS = 5;
15
15
  const DEFAULT_QUEUE_BACKOFF_MAX_MS = 4000;
16
16
  const DEFAULT_DEFER_DRAIN_MS = 25;
17
+ const DEFAULT_BATCH_MAX_ENTRIES = (() => {
18
+ const env = Number(typeof process !== 'undefined' ? process?.env?.REPRO_INGEST_BATCH_MAX_ENTRIES : undefined);
19
+ if (Number.isFinite(env) && env > 0) {
20
+ return Math.trunc(env);
21
+ }
22
+ return 100;
23
+ })();
24
+ const DEFAULT_BATCH_MAX_SERIALIZED_BYTES = (() => {
25
+ const env = Number(typeof process !== 'undefined'
26
+ ? process?.env?.REPRO_INGEST_BATCH_MAX_SERIALIZED_BYTES
27
+ : undefined);
28
+ if (Number.isFinite(env) && env > 0) {
29
+ return Math.trunc(env);
30
+ }
31
+ return 1024 * 1024;
32
+ })();
17
33
  const DEFAULT_INGEST_REQUEST_TIMEOUT_MS = (() => {
18
34
  const env = Number(typeof process !== 'undefined' ? process?.env?.REPRO_INGEST_REQUEST_TIMEOUT_MS : undefined);
19
35
  if (Number.isFinite(env) && env >= 0) {
@@ -22,9 +38,13 @@ const DEFAULT_INGEST_REQUEST_TIMEOUT_MS = (() => {
22
38
  return 120000;
23
39
  })();
24
40
  const ingestQueue = [];
41
+ const criticalIngestQueue = [];
25
42
  let drainTimer = null;
43
+ let criticalDrainTimer = null;
26
44
  let draining = false;
45
+ let criticalDraining = false;
27
46
  let backoffMs = 0;
47
+ let criticalBackoffMs = 0;
28
48
  let shouldDeferDrain = null;
29
49
  let deferDrainMs = DEFAULT_DEFER_DRAIN_MS;
30
50
  const configureIngestQueueDrain = (options) => {
@@ -35,12 +55,15 @@ const configureIngestQueueDrain = (options) => {
35
55
  };
36
56
  exports.configureIngestQueueDrain = configureIngestQueueDrain;
37
57
  const kickIngestQueueDrain = () => {
58
+ if (criticalIngestQueue.length > 0 && !criticalDrainTimer) {
59
+ scheduleCriticalDrain();
60
+ }
38
61
  if (ingestQueue.length > 0 && !drainTimer) {
39
62
  scheduleDrain(shouldDeferDrain?.() ? deferDrainMs : 0);
40
63
  }
41
64
  };
42
65
  exports.kickIngestQueueDrain = kickIngestQueueDrain;
43
- const isCriticalIngestEntry = (entry) => (entry.requestCaptureStart !== undefined ||
66
+ const isHighPriorityIngestEntry = (entry) => (entry.requestCaptureStart !== undefined ||
44
67
  entry.requestCaptureManifest !== undefined ||
45
68
  entry.sessionCaptureManifest !== undefined ||
46
69
  entry.request !== undefined ||
@@ -48,10 +71,72 @@ const isCriticalIngestEntry = (entry) => (entry.requestCaptureStart !== undefine
48
71
  entry.trace !== undefined ||
49
72
  entry.email !== undefined);
50
73
  const resolveBatchPriority = (entries, options) => {
51
- if (options.priority === 'high') {
52
- return 'high';
74
+ if (options.priority === 'high' || options.priority === 'critical') {
75
+ return options.priority;
76
+ }
77
+ return entries.some(isHighPriorityIngestEntry) ? 'high' : 'normal';
78
+ };
79
+ const sameIngestDestination = (left, right) => (left.sessionId === right.sessionId &&
80
+ left.priority === right.priority &&
81
+ left.config.appId === right.config.appId &&
82
+ left.config.tenantId === right.config.tenantId &&
83
+ left.config.appSecret === right.config.appSecret &&
84
+ left.config.appName === right.config.appName &&
85
+ left.config.serviceName === right.config.serviceName &&
86
+ left.config.ingestBase === right.config.ingestBase &&
87
+ left.config.ingestPath === right.config.ingestPath &&
88
+ left.config.actorId === right.config.actorId &&
89
+ left.config.actorType === right.config.actorType &&
90
+ JSON.stringify(left.config.actorLabels ?? null) ===
91
+ JSON.stringify(right.config.actorLabels ?? null) &&
92
+ left.config.source === right.config.source &&
93
+ left.config.schemaVersion === right.config.schemaVersion);
94
+ const serializedBytes = (value) => {
95
+ try {
96
+ return Buffer.byteLength(JSON.stringify(value), 'utf8');
97
+ }
98
+ catch {
99
+ return DEFAULT_BATCH_MAX_SERIALIZED_BYTES + 1;
53
100
  }
54
- return entries.some(isCriticalIngestEntry) ? 'high' : 'normal';
101
+ };
102
+ const selectSendBatch = (queue) => {
103
+ const first = queue[0];
104
+ if (!first) {
105
+ return null;
106
+ }
107
+ if (first.priority === 'normal') {
108
+ return { batch: first, queueItems: 1 };
109
+ }
110
+ const entries = first.entries.slice();
111
+ let bytes = serializedBytes(entries);
112
+ let queueItems = 1;
113
+ for (let index = 1; index < queue.length && entries.length < DEFAULT_BATCH_MAX_ENTRIES; index += 1) {
114
+ const candidate = queue[index];
115
+ if (!candidate || !sameIngestDestination(first, candidate)) {
116
+ break;
117
+ }
118
+ if (entries.length + candidate.entries.length > DEFAULT_BATCH_MAX_ENTRIES) {
119
+ break;
120
+ }
121
+ const candidateBytes = serializedBytes(candidate.entries);
122
+ if (bytes + candidateBytes > DEFAULT_BATCH_MAX_SERIALIZED_BYTES) {
123
+ break;
124
+ }
125
+ entries.push(...candidate.entries);
126
+ bytes += candidateBytes;
127
+ queueItems += 1;
128
+ }
129
+ if (queueItems === 1) {
130
+ return { batch: first, queueItems };
131
+ }
132
+ return {
133
+ batch: {
134
+ ...first,
135
+ entries,
136
+ attempts: Math.max(...queue.slice(0, queueItems).map((item) => item.attempts)),
137
+ },
138
+ queueItems,
139
+ };
55
140
  };
56
141
  const resolveIngestUrl = (config) => {
57
142
  const envBase = typeof process !== 'undefined' ? process?.env?.REPRO_INGEST_BASE : undefined;
@@ -144,6 +229,51 @@ const scheduleDrain = (delayMs = 0) => {
144
229
  }
145
230
  catch { }
146
231
  };
232
+ const scheduleCriticalDrain = (delayMs = 0) => {
233
+ if (criticalDrainTimer) {
234
+ return;
235
+ }
236
+ criticalDrainTimer = setTimeout(() => {
237
+ criticalDrainTimer = null;
238
+ void drainCriticalIngestQueue();
239
+ }, Math.max(0, delayMs));
240
+ try {
241
+ criticalDrainTimer.unref?.();
242
+ }
243
+ catch { }
244
+ };
245
+ const drainCriticalIngestQueue = async () => {
246
+ if (criticalDraining) {
247
+ return;
248
+ }
249
+ criticalDraining = true;
250
+ try {
251
+ while (criticalIngestQueue.length > 0) {
252
+ const selected = selectSendBatch(criticalIngestQueue);
253
+ if (!selected) {
254
+ return;
255
+ }
256
+ const ok = await sendIngestEntries(selected.batch.config, selected.batch.sessionId, selected.batch.entries);
257
+ if (ok) {
258
+ criticalIngestQueue.splice(0, selected.queueItems);
259
+ criticalBackoffMs = 0;
260
+ continue;
261
+ }
262
+ for (let index = 0; index < selected.queueItems; index += 1) {
263
+ criticalIngestQueue[index].attempts += 1;
264
+ }
265
+ criticalBackoffMs = Math.min(DEFAULT_QUEUE_BACKOFF_MAX_MS, criticalBackoffMs ? criticalBackoffMs * 2 : 250);
266
+ scheduleCriticalDrain(criticalBackoffMs);
267
+ return;
268
+ }
269
+ }
270
+ finally {
271
+ criticalDraining = false;
272
+ if (criticalIngestQueue.length > 0 && !criticalDrainTimer) {
273
+ scheduleCriticalDrain(criticalBackoffMs);
274
+ }
275
+ }
276
+ };
147
277
  const drainIngestQueue = async () => {
148
278
  if (draining) {
149
279
  return;
@@ -159,15 +289,22 @@ const drainIngestQueue = async () => {
159
289
  scheduleDrain(deferDrainMs);
160
290
  return;
161
291
  }
162
- const batch = ingestQueue[0];
292
+ const selected = selectSendBatch(ingestQueue);
293
+ if (!selected) {
294
+ return;
295
+ }
296
+ const batch = selected.batch;
163
297
  const ok = await sendIngestEntries(batch.config, batch.sessionId, batch.entries);
164
298
  if (ok) {
165
- ingestQueue.shift();
299
+ ingestQueue.splice(0, selected.queueItems);
166
300
  backoffMs = 0;
167
301
  continue;
168
302
  }
169
- batch.attempts += 1;
170
- if (batch.attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS && batch.priority !== 'high') {
303
+ for (let index = 0; index < selected.queueItems; index += 1) {
304
+ ingestQueue[index].attempts += 1;
305
+ }
306
+ if (ingestQueue[0].attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS &&
307
+ ingestQueue[0].priority === 'normal') {
171
308
  ingestQueue.shift();
172
309
  backoffMs = 0;
173
310
  continue;
@@ -196,6 +333,11 @@ const enqueueIngestEntries = (config, sessionId, entries, options = {}) => {
196
333
  attempts: 0,
197
334
  priority,
198
335
  };
336
+ if (priority === 'critical') {
337
+ criticalIngestQueue.push(batch);
338
+ scheduleCriticalDrain();
339
+ return true;
340
+ }
199
341
  if (priority === 'high') {
200
342
  const firstNormalIndex = ingestQueue.findIndex((queued) => queued.priority !== 'high');
201
343
  if (firstNormalIndex === -1) {
@@ -228,10 +370,15 @@ const enqueueIngestEntries = (config, sessionId, entries, options = {}) => {
228
370
  };
229
371
  exports.enqueueIngestEntries = enqueueIngestEntries;
230
372
  const flushIngestQueue = async () => {
373
+ if (criticalDrainTimer) {
374
+ clearTimeout(criticalDrainTimer);
375
+ criticalDrainTimer = null;
376
+ }
231
377
  if (drainTimer) {
232
378
  clearTimeout(drainTimer);
233
379
  drainTimer = null;
234
380
  }
381
+ await drainCriticalIngestQueue();
235
382
  await drainIngestQueue();
236
383
  };
237
384
  exports.flushIngestQueue = flushIngestQueue;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@reproapp/node-sdk",
3
- "version": "0.0.41",
3
+ "version": "0.0.42",
4
4
  "description": "Repro Nest SDK",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -12,7 +12,7 @@
12
12
  "build": "tsc -p tsconfig.json",
13
13
  "dev": "tsc -p tsconfig.json --watch --preserveWatchOutput",
14
14
  "prepublishOnly": "npm run build",
15
- "test": "npm run build && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/request-capture-background.test.js && node test/init-auto-http-server.test.js && node test/backend-live-mode.test.js && node test/client-performance-regression.test.js && node test/request-manifest-priority.test.js && node test/runtime-no-double-invoke.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node test/sdk-background-priority.test.js && node test/ingest-queue-retention.test.js && node test/ingest-high-priority-retention.test.js && node test/ingest-critical-retention.test.js && node test/ingest-critical-retry-retention.test.js && node test/ingest-timeout.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/wrap-plugin-super-call.test.js && node test/function-arg-identity.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js && node test/amqplib-integration.test.js && node test/mongodb-integration.test.js && node test/mongodb-esm-import-hook.test.js && node test/mongodb-suppression-cleanup.test.js && node test/postgresql-integration.test.js",
15
+ "test": "npm run build && node test/unawaited.test.js && node test/integration-unawaited.js && node test/request-flush-timing.test.js && node test/request-capture-background.test.js && node test/init-auto-http-server.test.js && node test/backend-live-mode.test.js && node test/client-performance-regression.test.js && node test/request-manifest-priority.test.js && node test/runtime-no-double-invoke.test.js && node test/express-trace-http-args.test.js && node test/trace-batch-size.test.js && node test/sdk-background-priority.test.js && node test/ingest-queue-retention.test.js && node test/ingest-high-priority-retention.test.js && node test/ingest-critical-retention.test.js && node test/ingest-critical-retry-retention.test.js && node test/ingest-batching.test.js && node test/ingest-timeout.test.js && node -r ./tracer/register test/promise-map.test.js && node test/disable-subtree.test.js && node test/circular-capture.test.js && node test/wrap-plugin-arrow-args.test.js && node test/wrap-plugin-super-call.test.js && node test/function-arg-identity.test.js && node test/privacy-runtime-policy.test.js && node test/runtime-privacy-materialization.test.js && node test/kafka-runtime-privacy-policy.test.js && node test/amqplib-integration.test.js && node test/mongodb-integration.test.js && node test/mongodb-esm-import-hook.test.js && node test/mongodb-suppression-cleanup.test.js && node test/postgresql-integration.test.js",
16
16
  "test:containers": "npm run build && node test/runtime-no-double-invoke.testcontainers.test.js && node test/wrap-plugin-super-call.testcontainers.test.js && node test/function-arg-identity.testcontainers.test.js && node test/mongodb-mongoose-native-suppression.testcontainers.test.js && node test/postgresql-integrations.testcontainers.test.js"
17
17
  },
18
18
  "peerDependencies": {
package/src/index.ts CHANGED
@@ -1249,7 +1249,7 @@ function emitRequestCaptureStart(
1249
1249
  },
1250
1250
  t: openedAt,
1251
1251
  }],
1252
- }, { priority: 'high' });
1252
+ }, { priority: 'critical' });
1253
1253
  }
1254
1254
 
1255
1255
  const als = new AsyncLocalStorage<Ctx>();
@@ -1393,7 +1393,7 @@ const SDK_BACKGROUND_BUSY_IDLE_DELAY_MS = (() => {
1393
1393
  const SDK_BACKGROUND_MAX_DEFER_MS = (() => {
1394
1394
  const raw = String(process.env.REPRO_SDK_BACKGROUND_MAX_DEFER_MS ?? '').trim().toLowerCase();
1395
1395
  if (!raw) {
1396
- return Number.POSITIVE_INFINITY;
1396
+ return 30000;
1397
1397
  }
1398
1398
  if (raw === 'off' || raw === 'never' || raw === 'infinite' || raw === 'infinity' || raw === '-1') {
1399
1399
  return Number.POSITIVE_INFINITY;
@@ -13,6 +13,24 @@ const DEFAULT_QUEUE_MAX_ITEMS = (() => {
13
13
  const DEFAULT_QUEUE_MAX_ATTEMPTS = 5;
14
14
  const DEFAULT_QUEUE_BACKOFF_MAX_MS = 4000;
15
15
  const DEFAULT_DEFER_DRAIN_MS = 25;
16
+ const DEFAULT_BATCH_MAX_ENTRIES = (() => {
17
+ const env = Number(typeof process !== 'undefined' ? (process as any)?.env?.REPRO_INGEST_BATCH_MAX_ENTRIES : undefined);
18
+ if (Number.isFinite(env) && env > 0) {
19
+ return Math.trunc(env);
20
+ }
21
+ return 100;
22
+ })();
23
+ const DEFAULT_BATCH_MAX_SERIALIZED_BYTES = (() => {
24
+ const env = Number(
25
+ typeof process !== 'undefined'
26
+ ? (process as any)?.env?.REPRO_INGEST_BATCH_MAX_SERIALIZED_BYTES
27
+ : undefined,
28
+ );
29
+ if (Number.isFinite(env) && env > 0) {
30
+ return Math.trunc(env);
31
+ }
32
+ return 1024 * 1024;
33
+ })();
16
34
  const DEFAULT_INGEST_REQUEST_TIMEOUT_MS = (() => {
17
35
  const env = Number(typeof process !== 'undefined' ? (process as any)?.env?.REPRO_INGEST_REQUEST_TIMEOUT_MS : undefined);
18
36
  if (Number.isFinite(env) && env >= 0) {
@@ -26,17 +44,21 @@ type QueuedIngestBatch = {
26
44
  sessionId: string;
27
45
  entries: LegacyEntry[];
28
46
  attempts: number;
29
- priority: 'normal' | 'high';
47
+ priority: 'normal' | 'high' | 'critical';
30
48
  };
31
49
 
32
50
  export type EnqueueIngestOptions = {
33
- priority?: 'normal' | 'high';
51
+ priority?: 'normal' | 'high' | 'critical';
34
52
  };
35
53
 
36
54
  const ingestQueue: QueuedIngestBatch[] = [];
55
+ const criticalIngestQueue: QueuedIngestBatch[] = [];
37
56
  let drainTimer: ReturnType<typeof setTimeout> | null = null;
57
+ let criticalDrainTimer: ReturnType<typeof setTimeout> | null = null;
38
58
  let draining = false;
59
+ let criticalDraining = false;
39
60
  let backoffMs = 0;
61
+ let criticalBackoffMs = 0;
40
62
  let shouldDeferDrain: (() => boolean) | null = null;
41
63
  let deferDrainMs = DEFAULT_DEFER_DRAIN_MS;
42
64
 
@@ -51,12 +73,15 @@ export const configureIngestQueueDrain = (options: {
51
73
  };
52
74
 
53
75
  export const kickIngestQueueDrain = (): void => {
76
+ if (criticalIngestQueue.length > 0 && !criticalDrainTimer) {
77
+ scheduleCriticalDrain();
78
+ }
54
79
  if (ingestQueue.length > 0 && !drainTimer) {
55
80
  scheduleDrain(shouldDeferDrain?.() ? deferDrainMs : 0);
56
81
  }
57
82
  };
58
83
 
59
- const isCriticalIngestEntry = (entry: LegacyEntry): boolean => (
84
+ const isHighPriorityIngestEntry = (entry: LegacyEntry): boolean => (
60
85
  entry.requestCaptureStart !== undefined ||
61
86
  entry.requestCaptureManifest !== undefined ||
62
87
  entry.sessionCaptureManifest !== undefined ||
@@ -70,10 +95,89 @@ const resolveBatchPriority = (
70
95
  entries: LegacyEntry[],
71
96
  options: EnqueueIngestOptions,
72
97
  ): QueuedIngestBatch['priority'] => {
73
- if (options.priority === 'high') {
74
- return 'high';
98
+ if (options.priority === 'high' || options.priority === 'critical') {
99
+ return options.priority;
100
+ }
101
+ return entries.some(isHighPriorityIngestEntry) ? 'high' : 'normal';
102
+ };
103
+
104
+ const sameIngestDestination = (
105
+ left: QueuedIngestBatch,
106
+ right: QueuedIngestBatch,
107
+ ): boolean => (
108
+ left.sessionId === right.sessionId &&
109
+ left.priority === right.priority &&
110
+ left.config.appId === right.config.appId &&
111
+ left.config.tenantId === right.config.tenantId &&
112
+ left.config.appSecret === right.config.appSecret &&
113
+ left.config.appName === right.config.appName &&
114
+ left.config.serviceName === right.config.serviceName &&
115
+ left.config.ingestBase === right.config.ingestBase &&
116
+ left.config.ingestPath === right.config.ingestPath &&
117
+ left.config.actorId === right.config.actorId &&
118
+ left.config.actorType === right.config.actorType &&
119
+ JSON.stringify(left.config.actorLabels ?? null) ===
120
+ JSON.stringify(right.config.actorLabels ?? null) &&
121
+ left.config.source === right.config.source &&
122
+ left.config.schemaVersion === right.config.schemaVersion
123
+ );
124
+
125
+ const serializedBytes = (value: unknown): number => {
126
+ try {
127
+ return Buffer.byteLength(JSON.stringify(value), 'utf8');
128
+ } catch {
129
+ return DEFAULT_BATCH_MAX_SERIALIZED_BYTES + 1;
75
130
  }
76
- return entries.some(isCriticalIngestEntry) ? 'high' : 'normal';
131
+ };
132
+
133
+ const selectSendBatch = (
134
+ queue: QueuedIngestBatch[],
135
+ ): { batch: QueuedIngestBatch; queueItems: number } | null => {
136
+ const first = queue[0];
137
+ if (!first) {
138
+ return null;
139
+ }
140
+ if (first.priority === 'normal') {
141
+ return { batch: first, queueItems: 1 };
142
+ }
143
+
144
+ const entries = first.entries.slice();
145
+ let bytes = serializedBytes(entries);
146
+ let queueItems = 1;
147
+ for (
148
+ let index = 1;
149
+ index < queue.length && entries.length < DEFAULT_BATCH_MAX_ENTRIES;
150
+ index += 1
151
+ ) {
152
+ const candidate = queue[index];
153
+ if (!candidate || !sameIngestDestination(first, candidate)) {
154
+ break;
155
+ }
156
+ if (entries.length + candidate.entries.length > DEFAULT_BATCH_MAX_ENTRIES) {
157
+ break;
158
+ }
159
+ const candidateBytes = serializedBytes(candidate.entries);
160
+ if (bytes + candidateBytes > DEFAULT_BATCH_MAX_SERIALIZED_BYTES) {
161
+ break;
162
+ }
163
+ entries.push(...candidate.entries);
164
+ bytes += candidateBytes;
165
+ queueItems += 1;
166
+ }
167
+
168
+ if (queueItems === 1) {
169
+ return { batch: first, queueItems };
170
+ }
171
+ return {
172
+ batch: {
173
+ ...first,
174
+ entries,
175
+ attempts: Math.max(
176
+ ...queue.slice(0, queueItems).map((item) => item.attempts),
177
+ ),
178
+ },
179
+ queueItems,
180
+ };
77
181
  };
78
182
 
79
183
  const resolveIngestUrl = (config: IngestClientConfig): string => {
@@ -181,6 +285,59 @@ const scheduleDrain = (delayMs = 0): void => {
181
285
  } catch {}
182
286
  };
183
287
 
288
+ const scheduleCriticalDrain = (delayMs = 0): void => {
289
+ if (criticalDrainTimer) {
290
+ return;
291
+ }
292
+ criticalDrainTimer = setTimeout(() => {
293
+ criticalDrainTimer = null;
294
+ void drainCriticalIngestQueue();
295
+ }, Math.max(0, delayMs));
296
+ try {
297
+ (criticalDrainTimer as any).unref?.();
298
+ } catch {}
299
+ };
300
+
301
+ const drainCriticalIngestQueue = async (): Promise<void> => {
302
+ if (criticalDraining) {
303
+ return;
304
+ }
305
+ criticalDraining = true;
306
+ try {
307
+ while (criticalIngestQueue.length > 0) {
308
+ const selected = selectSendBatch(criticalIngestQueue);
309
+ if (!selected) {
310
+ return;
311
+ }
312
+ const ok = await sendIngestEntries(
313
+ selected.batch.config,
314
+ selected.batch.sessionId,
315
+ selected.batch.entries,
316
+ );
317
+ if (ok) {
318
+ criticalIngestQueue.splice(0, selected.queueItems);
319
+ criticalBackoffMs = 0;
320
+ continue;
321
+ }
322
+
323
+ for (let index = 0; index < selected.queueItems; index += 1) {
324
+ criticalIngestQueue[index].attempts += 1;
325
+ }
326
+ criticalBackoffMs = Math.min(
327
+ DEFAULT_QUEUE_BACKOFF_MAX_MS,
328
+ criticalBackoffMs ? criticalBackoffMs * 2 : 250,
329
+ );
330
+ scheduleCriticalDrain(criticalBackoffMs);
331
+ return;
332
+ }
333
+ } finally {
334
+ criticalDraining = false;
335
+ if (criticalIngestQueue.length > 0 && !criticalDrainTimer) {
336
+ scheduleCriticalDrain(criticalBackoffMs);
337
+ }
338
+ }
339
+ };
340
+
184
341
  const drainIngestQueue = async (): Promise<void> => {
185
342
  if (draining) {
186
343
  return;
@@ -196,16 +353,25 @@ const drainIngestQueue = async (): Promise<void> => {
196
353
  scheduleDrain(deferDrainMs);
197
354
  return;
198
355
  }
199
- const batch = ingestQueue[0];
356
+ const selected = selectSendBatch(ingestQueue);
357
+ if (!selected) {
358
+ return;
359
+ }
360
+ const batch = selected.batch;
200
361
  const ok = await sendIngestEntries(batch.config, batch.sessionId, batch.entries);
201
362
  if (ok) {
202
- ingestQueue.shift();
363
+ ingestQueue.splice(0, selected.queueItems);
203
364
  backoffMs = 0;
204
365
  continue;
205
366
  }
206
367
 
207
- batch.attempts += 1;
208
- if (batch.attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS && batch.priority !== 'high') {
368
+ for (let index = 0; index < selected.queueItems; index += 1) {
369
+ ingestQueue[index].attempts += 1;
370
+ }
371
+ if (
372
+ ingestQueue[0].attempts >= DEFAULT_QUEUE_MAX_ATTEMPTS &&
373
+ ingestQueue[0].priority === 'normal'
374
+ ) {
209
375
  ingestQueue.shift();
210
376
  backoffMs = 0;
211
377
  continue;
@@ -243,6 +409,11 @@ export const enqueueIngestEntries = (
243
409
  attempts: 0,
244
410
  priority,
245
411
  };
412
+ if (priority === 'critical') {
413
+ criticalIngestQueue.push(batch);
414
+ scheduleCriticalDrain();
415
+ return true;
416
+ }
246
417
  if (priority === 'high') {
247
418
  const firstNormalIndex = ingestQueue.findIndex((queued) => queued.priority !== 'high');
248
419
  if (firstNormalIndex === -1) {
@@ -273,10 +444,15 @@ export const enqueueIngestEntries = (
273
444
  };
274
445
 
275
446
  export const flushIngestQueue = async (): Promise<void> => {
447
+ if (criticalDrainTimer) {
448
+ clearTimeout(criticalDrainTimer);
449
+ criticalDrainTimer = null;
450
+ }
276
451
  if (drainTimer) {
277
452
  clearTimeout(drainTimer);
278
453
  drainTimer = null;
279
454
  }
455
+ await drainCriticalIngestQueue();
280
456
  await drainIngestQueue();
281
457
  };
282
458
 
@@ -119,6 +119,11 @@ async function main() {
119
119
  });
120
120
 
121
121
  await sleep(2200);
122
+ assert.equal(
123
+ eventsOf(capturedBodies, 'request_capture_start').length,
124
+ 1,
125
+ `request-open fence was deferred with background work: ${JSON.stringify(capturedBodies)}`,
126
+ );
122
127
  assert.equal(
123
128
  eventsOf(capturedBodies, 'backend_request').length,
124
129
  0,
@@ -0,0 +1,67 @@
1
+ process.env.REPRO_INGEST_BATCH_MAX_ENTRIES = '100';
2
+ process.env.REPRO_INGEST_BATCH_MAX_SERIALIZED_BYTES = String(1024 * 1024);
3
+
4
+ const assert = require('node:assert/strict');
5
+
6
+ const originalFetch = global.fetch;
7
+ const capturedBodies = [];
8
+
9
+ global.fetch = async (_url, init = {}) => {
10
+ capturedBodies.push(JSON.parse(String(init.body || '{}')));
11
+ return {
12
+ ok: true,
13
+ status: 202,
14
+ json: async () => ({ ok: true }),
15
+ text: async () => '{"ok":true}',
16
+ };
17
+ };
18
+
19
+ const { enqueueIngestEntries, flushIngestQueue } = require('../dist/ingest/client');
20
+
21
+ async function main() {
22
+ const cfg = {
23
+ tenantId: 'TENANT_test',
24
+ appId: 'APP_test',
25
+ appSecret: 'secret',
26
+ serviceName: 'batching-test',
27
+ ingestBase: 'http://127.0.0.1:65535',
28
+ };
29
+ const sessionId = 'S_ingest_batching';
30
+
31
+ for (let index = 0; index < 209; index += 1) {
32
+ enqueueIngestEntries(cfg, sessionId, [{
33
+ actionId: 'A_ingest_batching',
34
+ requestRid: 'R_ingest_batching',
35
+ db: [{
36
+ collection: 'subjects',
37
+ op: 'findOne',
38
+ query: { index },
39
+ }],
40
+ t: Date.now() + index,
41
+ }], { priority: 'high' });
42
+ }
43
+
44
+ await flushIngestQueue();
45
+
46
+ const events = capturedBodies.flatMap((body) =>
47
+ Array.isArray(body?.events) ? body.events : [],
48
+ );
49
+ assert.equal(events.length, 209);
50
+ assert(events.every((event) => event.event_type === 'db_change'));
51
+ assert.equal(
52
+ capturedBodies.length,
53
+ 3,
54
+ `209 small evidence entries should use three bounded requests, got ${capturedBodies.length}`,
55
+ );
56
+
57
+ console.log('ingest batching OK');
58
+ }
59
+
60
+ main()
61
+ .catch((error) => {
62
+ console.error(error);
63
+ process.exitCode = 1;
64
+ })
65
+ .finally(() => {
66
+ global.fetch = originalFetch;
67
+ });
@@ -107,14 +107,19 @@ async function main() {
107
107
  });
108
108
 
109
109
  await sleep(100);
110
+ assert.equal(
111
+ countEvents(capturedBodies, 'request_capture_start'),
112
+ 1,
113
+ `request-open fence must bypass client workload deferral: ${JSON.stringify(capturedBodies)}`,
114
+ );
110
115
  assert.equal(
111
116
  countEvents(capturedBodies, 'backend_request'),
112
117
  0,
113
118
  `request capture should defer while another client request is active: ${JSON.stringify(capturedBodies)}`,
114
119
  );
115
120
 
116
- held.res.emit('finish');
117
121
  await waitFor(() => countEvents(capturedBodies, 'backend_request') === 1);
122
+ held.res.emit('finish');
118
123
  await flushIngestQueue();
119
124
 
120
125
  assert.equal(countEvents(capturedBodies, 'backend_request'), 1, JSON.stringify(capturedBodies));