@loadstrike/loadstrike-sdk 1.0.30001 → 1.0.30401
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/cjs/cluster.js +2417 -7
- package/dist/cjs/index.js +13 -2
- package/dist/cjs/iteration-observations.js +711 -0
- package/dist/cjs/load-engine-v2.js +966 -0
- package/dist/cjs/local.js +73 -20
- package/dist/cjs/reporting.js +123 -13
- package/dist/cjs/runtime.js +2368 -129
- package/dist/cjs/sinks.js +471 -17
- package/dist/cjs/transports.js +84 -146
- package/dist/esm/cluster.js +2386 -7
- package/dist/esm/index.js +1 -0
- package/dist/esm/iteration-observations.js +698 -0
- package/dist/esm/load-engine-v2.js +942 -0
- package/dist/esm/local.js +73 -20
- package/dist/esm/reporting.js +123 -13
- package/dist/esm/runtime.js +2369 -130
- package/dist/esm/sinks.js +471 -17
- package/dist/esm/transports.js +84 -146
- package/dist/types/cluster.d.ts +379 -1
- package/dist/types/index.d.ts +3 -1
- package/dist/types/iteration-observations.d.ts +225 -0
- package/dist/types/load-engine-v2.d.ts +147 -0
- package/dist/types/runtime.d.ts +214 -8
- package/dist/types/sinks.d.ts +67 -0
- package/dist/types/transports.d.ts +2 -8
- package/package.json +3 -4
package/dist/esm/runtime.js
CHANGED
|
@@ -3,11 +3,15 @@ import { randomBytes, randomUUID } from "node:crypto";
|
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import { resolve } from "node:path";
|
|
5
5
|
import { LoadStrikeLocalClient } from "./local.js";
|
|
6
|
-
import { DistributedClusterAgent, DistributedClusterCoordinator } from "./cluster.js";
|
|
6
|
+
import { DistributedClusterAgent, DistributedClusterCoordinator, buildLoadEngineV2ReservedStepOtherIdentityKey, buildLoadEngineV2GlobalInvocationId, buildLoadEngineV2ScenarioIdentityKey, buildLoadEngineV2SchedulerIdentityKey, buildLoadEngineV2StatusIdentityKey, buildLoadEngineV2StepIdentityKey, resolveLoadEngineV2StepIdentityKey, buildLoadEngineV2Plan } from "./cluster.js";
|
|
7
7
|
import { CorrelationStoreConfiguration, CrossPlatformTrackingRuntime, RedisCorrelationStore, RedisCorrelationStoreOptions, TrackingFieldSelector } from "./correlation.js";
|
|
8
8
|
import { EndpointAdapterFactory, LOADSTRIKE_TRACE_ID_TRACKING_FIELD } from "./transports.js";
|
|
9
9
|
import { buildDotnetCsvReport, buildDotnetHtmlReport, buildDotnetMarkdownReport, buildDotnetTxtReport } from "./reporting.js";
|
|
10
10
|
import { PortalReportingSink, cloneReportingSinkForRun } from "./sinks.js";
|
|
11
|
+
import { LoadEngineV2ExecutionBudget, buildLoadEngineV2TrafficMixSeedId, LoadStrikeHistogramV1, parseLoadEngineV2HistogramArtifact, serializeLoadEngineV2HistogramArtifact, classifyLoadEngineV2Arrival, loadEngineV2FixedArrivalCount, loadEngineV2FixedDeadlineNs, loadEngineV2LatenessToleranceNs, loadEngineV2TrafficMixLaneUnitCount, loadEngineV2TrafficMixOwnedUnits, planRampingInjectionDeadlines, planRampingConstantDeadlines, planRandomInjectionDeadlines, fnv1a32 } from "./load-engine-v2.js";
|
|
12
|
+
import { DEFAULT_ITERATION_OBSERVATION_SETTINGS, IterationObservationReporter, createIterationObservation, createIterationStepObservation, utcNowNs, validateIterationObservationSettings } from "./iteration-observations.js";
|
|
13
|
+
const LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS = Symbol("loadstrike.load-engine-v2.scheduler-distributions");
|
|
14
|
+
const PORTAL_RUN_TOKEN_PROVIDER = Symbol.for("loadstrike.internal.portal-run-token-provider");
|
|
11
15
|
export const LoadStrikeNodeType = {
|
|
12
16
|
SingleNode: "SingleNode",
|
|
13
17
|
Coordinator: "Coordinator",
|
|
@@ -139,12 +143,20 @@ export class LoadStrikePluginData {
|
|
|
139
143
|
}
|
|
140
144
|
}
|
|
141
145
|
class MeasurementAccumulator {
|
|
142
|
-
constructor() {
|
|
146
|
+
constructor(useHistogram = false) {
|
|
147
|
+
this.useHistogram = useHistogram;
|
|
143
148
|
this.count = 0;
|
|
144
149
|
this.allBytes = 0;
|
|
145
150
|
this.latenciesMs = [];
|
|
146
151
|
this.sizesBytes = [];
|
|
152
|
+
this.latencyLessOrEq800 = 0;
|
|
153
|
+
this.latencyMore800Less1200 = 0;
|
|
154
|
+
this.latencyMoreOrEq1200 = 0;
|
|
147
155
|
this.statusCodes = new Map();
|
|
156
|
+
if (useHistogram) {
|
|
157
|
+
this.latencyHistogram = new LoadStrikeHistogramV1();
|
|
158
|
+
this.sizeHistogram = new LoadStrikeHistogramV1();
|
|
159
|
+
}
|
|
148
160
|
}
|
|
149
161
|
get Count() {
|
|
150
162
|
return this.count;
|
|
@@ -161,8 +173,20 @@ class MeasurementAccumulator {
|
|
|
161
173
|
const key = `${statusCode}|${message}|${reply.isSuccess ? "ok" : "fail"}`;
|
|
162
174
|
this.count += 1;
|
|
163
175
|
this.allBytes += sizeBytes;
|
|
164
|
-
this.
|
|
165
|
-
|
|
176
|
+
if (this.useHistogram) {
|
|
177
|
+
this.latencyHistogram.record(normalizeLatencyMicroseconds(latencyMs));
|
|
178
|
+
this.sizeHistogram.record(normalizeHistogramInteger(sizeBytes, "Response size"));
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
this.latenciesMs.push(latencyMs);
|
|
182
|
+
this.sizesBytes.push(sizeBytes);
|
|
183
|
+
}
|
|
184
|
+
if (latencyMs <= 800)
|
|
185
|
+
this.latencyLessOrEq800 += 1;
|
|
186
|
+
else if (latencyMs < 1200)
|
|
187
|
+
this.latencyMore800Less1200 += 1;
|
|
188
|
+
else
|
|
189
|
+
this.latencyMoreOrEq1200 += 1;
|
|
166
190
|
const existing = this.statusCodes.get(key);
|
|
167
191
|
if (existing) {
|
|
168
192
|
existing.count += 1;
|
|
@@ -180,6 +204,9 @@ class MeasurementAccumulator {
|
|
|
180
204
|
* Use this when all builder inputs are ready to be materialized.
|
|
181
205
|
*/
|
|
182
206
|
build(allRequestCount, durationMs) {
|
|
207
|
+
if (this.useHistogram) {
|
|
208
|
+
return buildHistogramMeasurement(this.histogramSnapshot(), allRequestCount, durationMs);
|
|
209
|
+
}
|
|
183
210
|
const count = this.count;
|
|
184
211
|
const totalDurationMs = Math.max(durationMs, 0);
|
|
185
212
|
const latencyValues = [...this.latenciesMs];
|
|
@@ -228,14 +255,274 @@ class MeasurementAccumulator {
|
|
|
228
255
|
statusCodes
|
|
229
256
|
};
|
|
230
257
|
}
|
|
258
|
+
buildCombined(other, allRequestCount, durationMs) {
|
|
259
|
+
if (!this.useHistogram || !other.useHistogram) {
|
|
260
|
+
throw new Error("Combined measurements require Load Engine V2 histograms.");
|
|
261
|
+
}
|
|
262
|
+
const left = this.histogramSnapshot();
|
|
263
|
+
const right = other.histogramSnapshot();
|
|
264
|
+
left.latency.merge(right.latency);
|
|
265
|
+
left.size.merge(right.size);
|
|
266
|
+
for (const [key, value] of right.statusCodes) {
|
|
267
|
+
const existing = left.statusCodes.get(key);
|
|
268
|
+
if (existing)
|
|
269
|
+
existing.count += value.count;
|
|
270
|
+
else
|
|
271
|
+
left.statusCodes.set(key, { ...value });
|
|
272
|
+
}
|
|
273
|
+
return buildHistogramMeasurement({
|
|
274
|
+
count: left.count + right.count,
|
|
275
|
+
allBytes: left.allBytes + right.allBytes,
|
|
276
|
+
latency: left.latency,
|
|
277
|
+
size: left.size,
|
|
278
|
+
statusCodes: left.statusCodes,
|
|
279
|
+
lessOrEq800: left.lessOrEq800 + right.lessOrEq800,
|
|
280
|
+
more800Less1200: left.more800Less1200 + right.more800Less1200,
|
|
281
|
+
moreOrEq1200: left.moreOrEq1200 + right.moreOrEq1200
|
|
282
|
+
}, allRequestCount, durationMs);
|
|
283
|
+
}
|
|
284
|
+
histogramSnapshot() {
|
|
285
|
+
return {
|
|
286
|
+
count: this.count,
|
|
287
|
+
allBytes: this.allBytes,
|
|
288
|
+
latency: this.latencyHistogram.clone(),
|
|
289
|
+
size: this.sizeHistogram.clone(),
|
|
290
|
+
statusCodes: new Map(Array.from(this.statusCodes, ([key, value]) => [key, { ...value }])),
|
|
291
|
+
lessOrEq800: this.latencyLessOrEq800,
|
|
292
|
+
more800Less1200: this.latencyMore800Less1200,
|
|
293
|
+
moreOrEq1200: this.latencyMoreOrEq1200
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
function buildHistogramMeasurement(snapshot, allRequestCount, durationMs) {
|
|
298
|
+
const count = snapshot.count;
|
|
299
|
+
const totalDurationMs = Math.max(durationMs, 0);
|
|
300
|
+
const latency = snapshot.latency;
|
|
301
|
+
const size = snapshot.size;
|
|
302
|
+
return {
|
|
303
|
+
count64: latency.count.toString(),
|
|
304
|
+
distributionMode: latency.mode === "quantized-v1" || size.mode === "quantized-v1"
|
|
305
|
+
? "quantized-v1"
|
|
306
|
+
: "exact-normalized",
|
|
307
|
+
maxRelativeError: Math.max(latency.maxRelativeError, size.maxRelativeError),
|
|
308
|
+
histogramSidecar: {
|
|
309
|
+
latency: latency.toSidecar(),
|
|
310
|
+
size: size.toSidecar(),
|
|
311
|
+
allBytes64: size.exactTotal.toString(),
|
|
312
|
+
lessOrEq80064: snapshot.lessOrEq800.toString(),
|
|
313
|
+
more800Less120064: snapshot.more800Less1200.toString(),
|
|
314
|
+
moreOrEq120064: snapshot.moreOrEq1200.toString()
|
|
315
|
+
},
|
|
316
|
+
request: {
|
|
317
|
+
count,
|
|
318
|
+
percent: allRequestCount <= 0 ? 0 : Math.round((100 * count) / allRequestCount),
|
|
319
|
+
rps: totalDurationMs <= 0 ? 0 : count / (totalDurationMs / 1000)
|
|
320
|
+
},
|
|
321
|
+
dataTransfer: {
|
|
322
|
+
allBytes: snapshot.allBytes,
|
|
323
|
+
allBytes64: size.exactTotal.toString(),
|
|
324
|
+
minBytes: Number(size.minimum),
|
|
325
|
+
maxBytes: Number(size.maximum),
|
|
326
|
+
meanBytes: Math.round(size.mean),
|
|
327
|
+
percent50: Number(size.percentile(0.5)),
|
|
328
|
+
percent75: Number(size.percentile(0.75)),
|
|
329
|
+
percent95: Number(size.percentile(0.95)),
|
|
330
|
+
percent99: Number(size.percentile(0.99)),
|
|
331
|
+
percent100: Number(size.percentile(1)),
|
|
332
|
+
stdDev: size.populationStandardDeviation
|
|
333
|
+
},
|
|
334
|
+
latency: {
|
|
335
|
+
latencyCount: {
|
|
336
|
+
lessOrEq800: snapshot.lessOrEq800,
|
|
337
|
+
more800Less1200: snapshot.more800Less1200,
|
|
338
|
+
moreOrEq1200: snapshot.moreOrEq1200
|
|
339
|
+
},
|
|
340
|
+
minMs: Number(latency.minimum) / 1000,
|
|
341
|
+
maxMs: Number(latency.maximum) / 1000,
|
|
342
|
+
meanMs: latency.mean / 1000,
|
|
343
|
+
percent50: Number(latency.percentile(0.5)) / 1000,
|
|
344
|
+
percent75: Number(latency.percentile(0.75)) / 1000,
|
|
345
|
+
percent95: Number(latency.percentile(0.95)) / 1000,
|
|
346
|
+
percent99: Number(latency.percentile(0.99)) / 1000,
|
|
347
|
+
percent100: Number(latency.percentile(1)) / 1000,
|
|
348
|
+
stdDev: latency.populationStandardDeviation / 1000
|
|
349
|
+
},
|
|
350
|
+
statusCodes: Array.from(snapshot.statusCodes.values())
|
|
351
|
+
.sort((left, right) => right.count - left.count)
|
|
352
|
+
.map((value) => ({
|
|
353
|
+
count: value.count,
|
|
354
|
+
isError: value.isError,
|
|
355
|
+
message: value.message,
|
|
356
|
+
percent: count <= 0 ? 0 : Math.round((100 * value.count) / count),
|
|
357
|
+
statusCode: value.statusCode
|
|
358
|
+
}))
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
function normalizeLatencyMicroseconds(latencyMs) {
|
|
362
|
+
return normalizeHistogramInteger(Math.max(latencyMs, 0) * 1000, "Latency");
|
|
363
|
+
}
|
|
364
|
+
function normalizeRawObservationLatencyMicroseconds(latencyMs) {
|
|
365
|
+
const maximum = 9223372036854775807n;
|
|
366
|
+
if (!Number.isFinite(latencyMs) || latencyMs <= 0) {
|
|
367
|
+
return 0n;
|
|
368
|
+
}
|
|
369
|
+
const microseconds = latencyMs * 1000;
|
|
370
|
+
if (!Number.isFinite(microseconds) || microseconds >= Number(maximum)) {
|
|
371
|
+
return maximum;
|
|
372
|
+
}
|
|
373
|
+
return BigInt(Math.max(0, Math.round(microseconds)));
|
|
374
|
+
}
|
|
375
|
+
function normalizeHistogramInteger(value, name) {
|
|
376
|
+
if (!Number.isFinite(value) || value > Number(9223372036854775807n)) {
|
|
377
|
+
throw new RangeError(`${name} is outside the supported histogram range.`);
|
|
378
|
+
}
|
|
379
|
+
return BigInt(Math.max(0, Math.round(value)));
|
|
380
|
+
}
|
|
381
|
+
class LoadEngineV2Telemetry {
|
|
382
|
+
constructor(budget) {
|
|
383
|
+
this.budget = budget;
|
|
384
|
+
this.mutableSegments = [];
|
|
385
|
+
this.warnings = new Map();
|
|
386
|
+
}
|
|
387
|
+
createSegment(scenarioName, scenarioIndex, simulationIndex, kind, shardIndex, shardCount) {
|
|
388
|
+
const segment = {
|
|
389
|
+
scenarioName,
|
|
390
|
+
scenarioIndex,
|
|
391
|
+
simulationIndex,
|
|
392
|
+
kind,
|
|
393
|
+
shardIndex,
|
|
394
|
+
shardCount,
|
|
395
|
+
planned: 0n,
|
|
396
|
+
due: 0n,
|
|
397
|
+
started: 0n,
|
|
398
|
+
completed: 0n,
|
|
399
|
+
dropped: 0n,
|
|
400
|
+
unreached: 0n,
|
|
401
|
+
requestedWorkers: 0n,
|
|
402
|
+
startedWorkers: 0n,
|
|
403
|
+
unavailableWorkers: 0n,
|
|
404
|
+
dropReasons: new Map(),
|
|
405
|
+
unavailableWorkerReasons: new Map(),
|
|
406
|
+
decisionLag: new LoadStrikeHistogramV1(),
|
|
407
|
+
startLag: new LoadStrikeHistogramV1(),
|
|
408
|
+
accountingComplete: false
|
|
409
|
+
};
|
|
410
|
+
this.mutableSegments.push(segment);
|
|
411
|
+
return segment;
|
|
412
|
+
}
|
|
413
|
+
recordWarning(code, segment, count) {
|
|
414
|
+
if (count <= 0n)
|
|
415
|
+
return;
|
|
416
|
+
const key = `${code}\n${segment.scenarioIndex}\n${segment.simulationIndex}`;
|
|
417
|
+
const nowNs = BigInt(Date.now()) * 1000000n;
|
|
418
|
+
const existing = this.warnings.get(key);
|
|
419
|
+
if (existing) {
|
|
420
|
+
existing.count += count;
|
|
421
|
+
existing.lastObservedUtcNs = nowNs;
|
|
422
|
+
}
|
|
423
|
+
else {
|
|
424
|
+
this.warnings.set(key, {
|
|
425
|
+
code,
|
|
426
|
+
scenarioName: segment.scenarioName,
|
|
427
|
+
scenarioIndex: segment.scenarioIndex,
|
|
428
|
+
simulationIndex: segment.simulationIndex,
|
|
429
|
+
simulationKind: segment.kind,
|
|
430
|
+
count,
|
|
431
|
+
firstObservedUtcNs: nowNs,
|
|
432
|
+
lastObservedUtcNs: nowNs
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
recordDecisionLag(segment, lagNs) {
|
|
437
|
+
segment.decisionLag.record(maxBigInt(lagNs, 0n) / 1000n);
|
|
438
|
+
}
|
|
439
|
+
recordStartLag(segment, lagNs) {
|
|
440
|
+
segment.startLag.record(maxBigInt(lagNs, 0n) / 1000n);
|
|
441
|
+
}
|
|
442
|
+
buildSchedulerDistributions() {
|
|
443
|
+
return this.mutableSegments.flatMap((segment) => {
|
|
444
|
+
const scenarioIndex64 = segment.scenarioIndex.toString();
|
|
445
|
+
const simulationIndex64 = segment.simulationIndex.toString();
|
|
446
|
+
return [
|
|
447
|
+
["scheduler-decision-lag", "decision", segment.decisionLag],
|
|
448
|
+
["scheduler-start-lag", "start", segment.startLag]
|
|
449
|
+
].map(([seriesKind, identityKind, histogram]) => ({
|
|
450
|
+
seriesKind,
|
|
451
|
+
scenarioIndex64,
|
|
452
|
+
scenarioName: segment.scenarioName,
|
|
453
|
+
identityKeyHex: buildLoadEngineV2SchedulerIdentityKey(identityKind, scenarioIndex64, simulationIndex64).toString("hex"),
|
|
454
|
+
outcome: "none",
|
|
455
|
+
unit: "microseconds",
|
|
456
|
+
histogram: histogram.toSidecar(),
|
|
457
|
+
exactTotalDecimalOrEmpty: histogram.toSidecar().exactTotal64
|
|
458
|
+
}));
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
buildWarnings() {
|
|
462
|
+
return Array.from(this.warnings.values())
|
|
463
|
+
.sort((left, right) => left.code.localeCompare(right.code)
|
|
464
|
+
|| left.scenarioName.localeCompare(right.scenarioName)
|
|
465
|
+
|| left.simulationIndex - right.simulationIndex)
|
|
466
|
+
.map((value) => ({
|
|
467
|
+
code: value.code,
|
|
468
|
+
scenarioName: value.scenarioName,
|
|
469
|
+
scenarioIndex: value.scenarioIndex,
|
|
470
|
+
simulationIndex: value.simulationIndex,
|
|
471
|
+
simulationKind: value.simulationKind,
|
|
472
|
+
count64: value.count.toString(),
|
|
473
|
+
message: value.code,
|
|
474
|
+
firstObservedUtcNs: value.firstObservedUtcNs.toString(),
|
|
475
|
+
lastObservedUtcNs: value.lastObservedUtcNs.toString()
|
|
476
|
+
}));
|
|
477
|
+
}
|
|
478
|
+
buildSegments() {
|
|
479
|
+
return this.mutableSegments.map((segment) => ({
|
|
480
|
+
scenarioName: segment.scenarioName,
|
|
481
|
+
scenarioIndex: segment.scenarioIndex,
|
|
482
|
+
simulationIndex: segment.simulationIndex,
|
|
483
|
+
kind: segment.kind,
|
|
484
|
+
shardIndex: segment.shardIndex,
|
|
485
|
+
shardCount: segment.shardCount,
|
|
486
|
+
plannedIterations64: segment.planned.toString(),
|
|
487
|
+
dueIterations64: segment.due.toString(),
|
|
488
|
+
startedIterations64: segment.started.toString(),
|
|
489
|
+
completedIterations64: segment.completed.toString(),
|
|
490
|
+
droppedIterations64: segment.dropped.toString(),
|
|
491
|
+
unreachedIterations64: segment.unreached.toString(),
|
|
492
|
+
requestedWorkerSlots64: segment.requestedWorkers.toString(),
|
|
493
|
+
startedWorkerSlots64: segment.startedWorkers.toString(),
|
|
494
|
+
unavailableWorkerSlots64: segment.unavailableWorkers.toString(),
|
|
495
|
+
dropReasons: Object.fromEntries(Array.from(segment.dropReasons, ([key, count]) => [key, count.toString()])),
|
|
496
|
+
unavailableWorkerReasons: Object.fromEntries(Array.from(segment.unavailableWorkerReasons, ([key, count]) => [key, count.toString()])),
|
|
497
|
+
deliveryPercent: segment.due === 0n ? 100 : Number(segment.started * 10000n / segment.due) / 100,
|
|
498
|
+
accountingComplete: segment.accountingComplete
|
|
499
|
+
}));
|
|
500
|
+
}
|
|
501
|
+
buildStats() {
|
|
502
|
+
return {
|
|
503
|
+
configuredMaxInFlight: this.budget.maxInFlight,
|
|
504
|
+
maxInFlightObserved: this.budget.highWater,
|
|
505
|
+
currentInFlight: this.budget.current,
|
|
506
|
+
segments: this.buildSegments()
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
function incrementReason(reasons, code, count = 1n) {
|
|
511
|
+
reasons.set(code, (reasons.get(code) ?? 0n) + count);
|
|
512
|
+
}
|
|
513
|
+
function ownedV2OrdinalCount(total, shardIndex, shardCount) {
|
|
514
|
+
if (total <= BigInt(shardIndex))
|
|
515
|
+
return 0n;
|
|
516
|
+
return (total - 1n - BigInt(shardIndex)) / BigInt(shardCount) + 1n;
|
|
231
517
|
}
|
|
232
518
|
class StepStatsAccumulator {
|
|
233
|
-
constructor(scenarioName, stepName, sortIndex) {
|
|
519
|
+
constructor(scenarioName, stepName, sortIndex, useHistogram = false) {
|
|
234
520
|
this.scenarioName = scenarioName;
|
|
235
521
|
this.stepName = stepName;
|
|
236
522
|
this.sortIndex = sortIndex;
|
|
237
|
-
this.
|
|
238
|
-
this.
|
|
523
|
+
this.useHistogram = useHistogram;
|
|
524
|
+
this.ok = new MeasurementAccumulator(useHistogram);
|
|
525
|
+
this.fail = new MeasurementAccumulator(useHistogram);
|
|
239
526
|
}
|
|
240
527
|
/**
|
|
241
528
|
* Exposes the public record operation.
|
|
@@ -274,6 +561,7 @@ class StepStatsAccumulator {
|
|
|
274
561
|
minLatencyMs: minCandidates.length ? Math.min(...minCandidates) : 0,
|
|
275
562
|
maxLatencyMs: maxCandidates.length ? Math.max(...maxCandidates) : 0,
|
|
276
563
|
statusCodes,
|
|
564
|
+
allMeasurement: this.useHistogram ? this.ok.buildCombined(this.fail, requestCount, durationMs) : undefined,
|
|
277
565
|
ok,
|
|
278
566
|
fail,
|
|
279
567
|
sortIndex: this.sortIndex
|
|
@@ -281,15 +569,16 @@ class StepStatsAccumulator {
|
|
|
281
569
|
}
|
|
282
570
|
}
|
|
283
571
|
class ScenarioStatsAccumulator {
|
|
284
|
-
constructor(scenarioName, sortIndex) {
|
|
572
|
+
constructor(scenarioName, sortIndex, useHistogram = false) {
|
|
285
573
|
this.scenarioName = scenarioName;
|
|
286
574
|
this.sortIndex = sortIndex;
|
|
287
|
-
this.
|
|
288
|
-
this.fail = new MeasurementAccumulator();
|
|
575
|
+
this.useHistogram = useHistogram;
|
|
289
576
|
this.steps = new Map();
|
|
290
577
|
this.nextStepSortIndex = 0;
|
|
291
578
|
this.loadSimulationStats = { simulationName: "", value: 0 };
|
|
292
579
|
this.currentOperation = "None";
|
|
580
|
+
this.ok = new MeasurementAccumulator(useHistogram);
|
|
581
|
+
this.fail = new MeasurementAccumulator(useHistogram);
|
|
293
582
|
}
|
|
294
583
|
/**
|
|
295
584
|
* Exposes the public setLoadSimulation operation.
|
|
@@ -330,13 +619,18 @@ class ScenarioStatsAccumulator {
|
|
|
330
619
|
* Exposes the public recordStep operation.
|
|
331
620
|
* Use this when the surrounding wrapper type makes this operation the clearest way to express your intent.
|
|
332
621
|
*/
|
|
333
|
-
recordStep(stepName, reply, observedLatencyMs) {
|
|
622
|
+
recordStep(stepName, reply, observedLatencyMs, sortIndex) {
|
|
334
623
|
const existing = this.steps.get(stepName);
|
|
335
|
-
const
|
|
624
|
+
const resolvedSortIndex = sortIndex === undefined
|
|
625
|
+
? this.nextStepSortIndex + 1
|
|
626
|
+
: Math.max(Math.trunc(sortIndex), 0);
|
|
627
|
+
const step = existing ?? new StepStatsAccumulator(this.scenarioName, stepName, resolvedSortIndex, this.useHistogram);
|
|
336
628
|
if (!existing) {
|
|
629
|
+
this.nextStepSortIndex = Math.max(this.nextStepSortIndex, resolvedSortIndex);
|
|
337
630
|
this.steps.set(stepName, step);
|
|
338
631
|
}
|
|
339
632
|
step.record(reply, observedLatencyMs);
|
|
633
|
+
return step.sortIndex;
|
|
340
634
|
}
|
|
341
635
|
/**
|
|
342
636
|
* Builds the configured payload or helper object.
|
|
@@ -365,6 +659,7 @@ class ScenarioStatsAccumulator {
|
|
|
365
659
|
minLatencyMs: minCandidates.length ? Math.min(...minCandidates) : 0,
|
|
366
660
|
maxLatencyMs: maxCandidates.length ? Math.max(...maxCandidates) : 0,
|
|
367
661
|
statusCodes,
|
|
662
|
+
allMeasurement: this.useHistogram ? this.ok.buildCombined(this.fail, totalRequests, durationMs) : undefined,
|
|
368
663
|
allBytes,
|
|
369
664
|
currentOperation: this.currentOperation,
|
|
370
665
|
durationMs: Math.max(durationMs, 0),
|
|
@@ -459,7 +754,8 @@ export class LoadStrikeStep {
|
|
|
459
754
|
const internal = context;
|
|
460
755
|
await internal.invokeBeforeStep(stepName);
|
|
461
756
|
let reply;
|
|
462
|
-
const
|
|
757
|
+
const startedUtcNs = utcNowNs();
|
|
758
|
+
const startedAtNs = process.hrtime.bigint();
|
|
463
759
|
try {
|
|
464
760
|
reply = normalizeReply(await run());
|
|
465
761
|
}
|
|
@@ -467,8 +763,21 @@ export class LoadStrikeStep {
|
|
|
467
763
|
reply = LoadStrikeResponse.fail("step_exception", resolveRuntimeErrorMessage(error, "step failed"), 0);
|
|
468
764
|
}
|
|
469
765
|
reply = attachReplyProjection(reply);
|
|
470
|
-
const
|
|
471
|
-
|
|
766
|
+
const observedLatencyNs = maxBigInt(process.hrtime.bigint() - startedAtNs, 0n);
|
|
767
|
+
const completedUtcNs = startedUtcNs + observedLatencyNs;
|
|
768
|
+
const observedLatencyMs = Number(observedLatencyNs) / 1000000;
|
|
769
|
+
const recordedSortIndex = internal.recordStep(stepName, reply, observedLatencyMs);
|
|
770
|
+
internal.recordStepObservation?.(createIterationStepObservation({
|
|
771
|
+
stepName,
|
|
772
|
+
sortIndex: typeof recordedSortIndex === "number" ? recordedSortIndex : 0,
|
|
773
|
+
startedUtcNs,
|
|
774
|
+
completedUtcNs,
|
|
775
|
+
observedLatencyUs64: observedLatencyNs / 1000n,
|
|
776
|
+
reportedLatencyUs64: normalizeRawObservationLatencyMicroseconds(resolveRecordedLatency(reply.customLatencyMs, observedLatencyMs)),
|
|
777
|
+
isSuccess: reply.isSuccess,
|
|
778
|
+
statusCode: normalizeStatusCode(reply.statusCode, reply.isSuccess),
|
|
779
|
+
sizeBytes64: normalizeHistogramInteger(Math.max(toNumber(reply.sizeBytes), 0), "Step response bytes")
|
|
780
|
+
}));
|
|
472
781
|
await internal.invokeAfterStep(stepName, reply);
|
|
473
782
|
return reply;
|
|
474
783
|
}
|
|
@@ -999,10 +1308,14 @@ export class LoadStrikeContext {
|
|
|
999
1308
|
const normalizedValues = normalizeRunContextCollectionShapes(this.values);
|
|
1000
1309
|
return {
|
|
1001
1310
|
displayConsoleMetrics: normalizedValues.ConsoleMetricsEnabled,
|
|
1311
|
+
loadEngineContractVersion: normalizedValues.LoadEngineContractVersion,
|
|
1312
|
+
maxInFlight: normalizedValues.MaxInFlight,
|
|
1002
1313
|
nodeType: normalizedValues.NodeType,
|
|
1003
1314
|
localDevClusterEnabled: normalizedValues.LocalDevClusterEnabled,
|
|
1004
1315
|
agentGroup: normalizedValues.AgentGroup,
|
|
1005
1316
|
agentsCount: normalizedValues.AgentsCount,
|
|
1317
|
+
agentId: normalizedValues.AgentId,
|
|
1318
|
+
expectedAgentIds: normalizedValues.ExpectedAgentIds,
|
|
1006
1319
|
targetScenarios: normalizedValues.TargetScenarios,
|
|
1007
1320
|
agentTargetScenarios: normalizedValues.AgentTargetScenarios,
|
|
1008
1321
|
coordinatorTargetScenarios: normalizedValues.CoordinatorTargetScenarios,
|
|
@@ -1021,6 +1334,13 @@ export class LoadStrikeContext {
|
|
|
1021
1334
|
reportFolderPath: normalizedValues.ReportFolderPath,
|
|
1022
1335
|
reportFormats: normalizedValues.ReportFormats,
|
|
1023
1336
|
reportingIntervalSeconds: normalizedValues.ReportingIntervalSeconds,
|
|
1337
|
+
iterationObservationFlushIntervalSeconds: normalizedValues.IterationObservationFlushIntervalSeconds,
|
|
1338
|
+
maxIterationObservationBufferBytes: normalizedValues.MaxIterationObservationBufferBytes,
|
|
1339
|
+
maxIterationObservationsPerBatch: normalizedValues.MaxIterationObservationsPerBatch,
|
|
1340
|
+
maxIterationObservationBatchBytes: normalizedValues.MaxIterationObservationBatchBytes,
|
|
1341
|
+
iterationObservationSinkQueueDepth: normalizedValues.IterationObservationSinkQueueDepth,
|
|
1342
|
+
iterationObservationSinkParallelism: normalizedValues.IterationObservationSinkParallelism,
|
|
1343
|
+
iterationObservationDrainTimeoutSeconds: normalizedValues.IterationObservationDrainTimeoutSeconds,
|
|
1024
1344
|
minimumLogLevel: normalizedValues.MinimumLogLevel,
|
|
1025
1345
|
loggerConfig: normalizedValues.LoggerConfig,
|
|
1026
1346
|
reportingSinks: normalizedValues.ReportingSinks,
|
|
@@ -1034,6 +1354,8 @@ export class LoadStrikeContext {
|
|
|
1034
1354
|
workerPlugins: normalizedValues.WorkerPlugins,
|
|
1035
1355
|
customSettings: normalizedValues.CustomSettings,
|
|
1036
1356
|
globalCustomSettings: normalizedValues.GlobalCustomSettings,
|
|
1357
|
+
clusterShardIndex: normalizedValues.ClusterShardIndex,
|
|
1358
|
+
clusterShardCount: normalizedValues.ClusterShardCount,
|
|
1037
1359
|
runArgs: this.runArgs.length ? [...this.runArgs] : undefined
|
|
1038
1360
|
};
|
|
1039
1361
|
}
|
|
@@ -1095,6 +1417,19 @@ export class LoadStrikeContext {
|
|
|
1095
1417
|
DisplayConsoleMetrics(enable) {
|
|
1096
1418
|
return this.mergeValues({ ConsoleMetricsEnabled: Boolean(enable) });
|
|
1097
1419
|
}
|
|
1420
|
+
useLoadEngineV2() {
|
|
1421
|
+
return this.UseLoadEngineV2();
|
|
1422
|
+
}
|
|
1423
|
+
UseLoadEngineV2() {
|
|
1424
|
+
return this.mergeValues({ LoadEngineContractVersion: 2 });
|
|
1425
|
+
}
|
|
1426
|
+
withMaxInFlight(maxInFlight) {
|
|
1427
|
+
return this.WithMaxInFlight(maxInFlight);
|
|
1428
|
+
}
|
|
1429
|
+
WithMaxInFlight(maxInFlight) {
|
|
1430
|
+
validateV2MaxInFlight(this.values.LoadEngineContractVersion, maxInFlight);
|
|
1431
|
+
return this.mergeValues({ MaxInFlight: maxInFlight });
|
|
1432
|
+
}
|
|
1098
1433
|
/**
|
|
1099
1434
|
* Toggles local development cluster mode.
|
|
1100
1435
|
* Use this when you want to simulate coordinator and agent behavior on a single machine.
|
|
@@ -1161,6 +1496,26 @@ export class LoadStrikeContext {
|
|
|
1161
1496
|
WithAgentGroup(agentGroup) {
|
|
1162
1497
|
return this.mergeValues({ AgentGroup: requireNonEmpty(agentGroup, "Agent group must be provided.") });
|
|
1163
1498
|
}
|
|
1499
|
+
/** Sets the stable identity required by a remote Load Engine V2 agent. */
|
|
1500
|
+
withAgentId(agentId) {
|
|
1501
|
+
return this.WithAgentId(agentId);
|
|
1502
|
+
}
|
|
1503
|
+
/** Sets the stable identity required by a remote Load Engine V2 agent. */
|
|
1504
|
+
WithAgentId(agentId) {
|
|
1505
|
+
return this.mergeValues({ AgentId: requireNonEmpty(agentId, "Agent id must be provided.") });
|
|
1506
|
+
}
|
|
1507
|
+
/** Sets the exact remote agent identities required by a Load Engine V2 coordinator. */
|
|
1508
|
+
withExpectedAgentIds(...agentIds) {
|
|
1509
|
+
return this.WithExpectedAgentIds(...agentIds);
|
|
1510
|
+
}
|
|
1511
|
+
/** Sets the exact remote agent identities required by a Load Engine V2 coordinator. */
|
|
1512
|
+
WithExpectedAgentIds(...agentIds) {
|
|
1513
|
+
const normalized = validateScenarioNames(agentIds);
|
|
1514
|
+
if (new Set(normalized).size !== normalized.length) {
|
|
1515
|
+
throw new Error("Expected agent ids must be unique.");
|
|
1516
|
+
}
|
|
1517
|
+
return this.mergeValues({ ExpectedAgentIds: normalized });
|
|
1518
|
+
}
|
|
1164
1519
|
/**
|
|
1165
1520
|
* Sets the requested agent count.
|
|
1166
1521
|
* Use this when a coordinator should fan work out across a specific number of agents.
|
|
@@ -1997,7 +2352,7 @@ function firstWebVitalViolation(...values) {
|
|
|
1997
2352
|
return "";
|
|
1998
2353
|
}
|
|
1999
2354
|
export class LoadStrikeScenario {
|
|
2000
|
-
constructor(name, runHandler, initHandler, cleanHandler, loadSimulations, thresholds, trackingConfiguration, maxFailCount, withoutWarmUpValue, warmUpDurationSeconds, weight, restartIterationOnFail, internalLicenseFeatures = []) {
|
|
2355
|
+
constructor(name, runHandler, initHandler, cleanHandler, loadSimulations, thresholds, trackingConfiguration, maxFailCount, withoutWarmUpValue, warmUpDurationSeconds, weight, restartIterationOnFail, internalLicenseFeatures = [], declaredStepNames = []) {
|
|
2001
2356
|
this.name = name;
|
|
2002
2357
|
this.runHandler = runHandler;
|
|
2003
2358
|
this.initHandler = initHandler;
|
|
@@ -2011,6 +2366,7 @@ export class LoadStrikeScenario {
|
|
|
2011
2366
|
this.weight = weight;
|
|
2012
2367
|
this.restartIterationOnFail = restartIterationOnFail;
|
|
2013
2368
|
this.internalLicenseFeatures = normalizeStringArray(internalLicenseFeatures);
|
|
2369
|
+
this.declaredStepNames = normalizeDeclaredStepNames(declaredStepNames);
|
|
2014
2370
|
}
|
|
2015
2371
|
static create(name, runHandler) {
|
|
2016
2372
|
const scenarioName = requireNonEmpty(name, "Scenario name must be provided.");
|
|
@@ -2053,7 +2409,7 @@ export class LoadStrikeScenario {
|
|
|
2053
2409
|
if (typeof handler !== "function") {
|
|
2054
2410
|
throw new TypeError("Init handler must be provided.");
|
|
2055
2411
|
}
|
|
2056
|
-
return new LoadStrikeScenario(this.name, this.runHandler, handler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2412
|
+
return new LoadStrikeScenario(this.name, this.runHandler, handler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2057
2413
|
}
|
|
2058
2414
|
/**
|
|
2059
2415
|
* Configures init async for this SDK object.
|
|
@@ -2070,7 +2426,7 @@ export class LoadStrikeScenario {
|
|
|
2070
2426
|
if (typeof handler !== "function") {
|
|
2071
2427
|
throw new TypeError("Clean handler must be provided.");
|
|
2072
2428
|
}
|
|
2073
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, handler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2429
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, handler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2074
2430
|
}
|
|
2075
2431
|
/**
|
|
2076
2432
|
* Configures clean async for this SDK object.
|
|
@@ -2087,14 +2443,14 @@ export class LoadStrikeScenario {
|
|
|
2087
2443
|
if (!Number.isFinite(maxFailCount)) {
|
|
2088
2444
|
throw new RangeError("maxFailCount should be a finite number.");
|
|
2089
2445
|
}
|
|
2090
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, Math.trunc(maxFailCount), this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2446
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, Math.trunc(maxFailCount), this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2091
2447
|
}
|
|
2092
2448
|
/**
|
|
2093
2449
|
* Configures out warm up for this SDK object.
|
|
2094
2450
|
* Use this when out warm up should be set explicitly before the run starts.
|
|
2095
2451
|
*/
|
|
2096
2452
|
withoutWarmUp() {
|
|
2097
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, true, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2453
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, true, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2098
2454
|
}
|
|
2099
2455
|
/**
|
|
2100
2456
|
* Configures warm up duration for this SDK object.
|
|
@@ -2104,7 +2460,7 @@ export class LoadStrikeScenario {
|
|
|
2104
2460
|
if (!Number.isFinite(durationSeconds)) {
|
|
2105
2461
|
throw new RangeError("Warmup duration should be a finite number.");
|
|
2106
2462
|
}
|
|
2107
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, durationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2463
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, durationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2108
2464
|
}
|
|
2109
2465
|
/**
|
|
2110
2466
|
* Configures weight for this SDK object.
|
|
@@ -2114,14 +2470,14 @@ export class LoadStrikeScenario {
|
|
|
2114
2470
|
if (!Number.isFinite(weight)) {
|
|
2115
2471
|
throw new RangeError("Weight should be a finite number.");
|
|
2116
2472
|
}
|
|
2117
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, Math.trunc(weight), this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2473
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, Math.trunc(weight), this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2118
2474
|
}
|
|
2119
2475
|
/**
|
|
2120
2476
|
* Configures restart iteration on fail for this SDK object.
|
|
2121
2477
|
* Use this when restart iteration on fail should be set explicitly before the run starts.
|
|
2122
2478
|
*/
|
|
2123
2479
|
withRestartIterationOnFail(shouldRestart) {
|
|
2124
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, Boolean(shouldRestart), this.internalLicenseFeatures);
|
|
2480
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, Boolean(shouldRestart), this.internalLicenseFeatures, this.declaredStepNames);
|
|
2125
2481
|
}
|
|
2126
2482
|
/**
|
|
2127
2483
|
* Configures cross platform tracking for this SDK object.
|
|
@@ -2146,7 +2502,7 @@ export class LoadStrikeScenario {
|
|
|
2146
2502
|
if (isCorrelateExistingTrafficTracking(copied) && this.loadSimulations.length > 0) {
|
|
2147
2503
|
throw new Error("CorrelateExistingTraffic uses ForDuration and cannot be combined with WithLoadSimulations.");
|
|
2148
2504
|
}
|
|
2149
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, copied, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2505
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, copied, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2150
2506
|
}
|
|
2151
2507
|
/**
|
|
2152
2508
|
* Configures load simulations for this SDK object.
|
|
@@ -2159,7 +2515,7 @@ export class LoadStrikeScenario {
|
|
|
2159
2515
|
if (isCorrelateExistingTrafficTracking(this.trackingConfiguration)) {
|
|
2160
2516
|
throw new Error("CorrelateExistingTraffic uses ForDuration and cannot be combined with WithLoadSimulations.");
|
|
2161
2517
|
}
|
|
2162
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, simulations.map((simulation) => attachLoadSimulationProjection({ ...simulation })), this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2518
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, simulations.map((simulation) => attachLoadSimulationProjection({ ...simulation })), this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2163
2519
|
}
|
|
2164
2520
|
/**
|
|
2165
2521
|
* Configures thresholds for this SDK object.
|
|
@@ -2169,7 +2525,7 @@ export class LoadStrikeScenario {
|
|
|
2169
2525
|
if (!thresholds.length) {
|
|
2170
2526
|
throw new Error("At least one threshold should be provided.");
|
|
2171
2527
|
}
|
|
2172
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, thresholds.map((threshold) => ({ ...threshold })), this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2528
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, thresholds.map((threshold) => ({ ...threshold })), this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2173
2529
|
}
|
|
2174
2530
|
/**
|
|
2175
2531
|
* Returns simulations.
|
|
@@ -2230,6 +2586,26 @@ export class LoadStrikeScenario {
|
|
|
2230
2586
|
__loadStrikeInternalLicenseFeatures() {
|
|
2231
2587
|
return [...this.internalLicenseFeatures];
|
|
2232
2588
|
}
|
|
2589
|
+
/** Freezes the named step identities that may be reported by this scenario. */
|
|
2590
|
+
withDeclaredSteps(...stepNames) {
|
|
2591
|
+
if (!stepNames.length) {
|
|
2592
|
+
throw new Error("At least one declared step name should be provided.");
|
|
2593
|
+
}
|
|
2594
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, stepNames);
|
|
2595
|
+
}
|
|
2596
|
+
/** Returns the immutable declared-step names in declaration order. */
|
|
2597
|
+
getDeclaredSteps() {
|
|
2598
|
+
return [...this.declaredStepNames];
|
|
2599
|
+
}
|
|
2600
|
+
__loadStrikeSetTrafficMixV2Metadata(metadata) {
|
|
2601
|
+
this.trafficMixV2Metadata = cloneLoadEngineV2TrafficMixMetadata(metadata);
|
|
2602
|
+
return this;
|
|
2603
|
+
}
|
|
2604
|
+
__loadStrikeTrafficMixV2Metadata() {
|
|
2605
|
+
return this.trafficMixV2Metadata
|
|
2606
|
+
? cloneLoadEngineV2TrafficMixMetadata(this.trafficMixV2Metadata)
|
|
2607
|
+
: undefined;
|
|
2608
|
+
}
|
|
2233
2609
|
__loadStrikeScenarioSourceAnalysis() {
|
|
2234
2610
|
const source = this.runHandler.toString();
|
|
2235
2611
|
const lines = source
|
|
@@ -2252,7 +2628,7 @@ export class LoadStrikeScenario {
|
|
|
2252
2628
|
}
|
|
2253
2629
|
__loadStrikeWithInternalLicenseFeatures(...features) {
|
|
2254
2630
|
const merged = Array.from(new Set([...this.internalLicenseFeatures, ...normalizeStringArray(features)]));
|
|
2255
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, merged);
|
|
2631
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, merged, this.declaredStepNames);
|
|
2256
2632
|
}
|
|
2257
2633
|
async invokeInit(context) {
|
|
2258
2634
|
if (this.initHandler) {
|
|
@@ -2273,6 +2649,9 @@ export class LoadStrikeScenario {
|
|
|
2273
2649
|
return normalizeReply(result);
|
|
2274
2650
|
}
|
|
2275
2651
|
catch (error) {
|
|
2652
|
+
if (error instanceof RuntimePolicyCallbackError) {
|
|
2653
|
+
throw error;
|
|
2654
|
+
}
|
|
2276
2655
|
return LoadStrikeResponse.fail("exception", resolveRuntimeErrorMessage(error, "scenario failed"), 0);
|
|
2277
2656
|
}
|
|
2278
2657
|
}
|
|
@@ -2318,6 +2697,10 @@ export class LoadStrikeScenario {
|
|
|
2318
2697
|
WithLoadSimulations(...simulations) {
|
|
2319
2698
|
return this.withLoadSimulations(...simulations);
|
|
2320
2699
|
}
|
|
2700
|
+
/** Freezes the named step identities that may be reported by this scenario. */
|
|
2701
|
+
WithDeclaredSteps(...stepNames) {
|
|
2702
|
+
return this.withDeclaredSteps(...stepNames);
|
|
2703
|
+
}
|
|
2321
2704
|
/**
|
|
2322
2705
|
* Configures max fail count for this SDK object.
|
|
2323
2706
|
* Use this when max fail count should be set explicitly before the run starts.
|
|
@@ -2446,7 +2829,7 @@ export class LoadStrikeTrafficMix {
|
|
|
2446
2829
|
return this.withScenarioMix(...scenarioMix);
|
|
2447
2830
|
}
|
|
2448
2831
|
expandScenarios() {
|
|
2449
|
-
return expandTrafficMixScenarios(this);
|
|
2832
|
+
return expandTrafficMixScenarios(this, 0);
|
|
2450
2833
|
}
|
|
2451
2834
|
ExpandScenarios() {
|
|
2452
2835
|
return this.expandScenarios();
|
|
@@ -2459,10 +2842,11 @@ export class LoadStrikeTrafficMix {
|
|
|
2459
2842
|
}
|
|
2460
2843
|
}
|
|
2461
2844
|
export class LoadStrikeRunner {
|
|
2462
|
-
constructor(scenarios, options, contextConfigurators = []) {
|
|
2845
|
+
constructor(scenarios, options, contextConfigurators = [], internalOptions = {}) {
|
|
2463
2846
|
this.scenarios = scenarios;
|
|
2464
2847
|
this.options = normalizeRunnerOptionCollectionShapes(options);
|
|
2465
2848
|
this.contextConfigurators = [...contextConfigurators];
|
|
2849
|
+
this.internalOptions = internalOptions;
|
|
2466
2850
|
}
|
|
2467
2851
|
/**
|
|
2468
2852
|
* Creates a new instance of this public SDK type.
|
|
@@ -2498,7 +2882,7 @@ export class LoadStrikeRunner {
|
|
|
2498
2882
|
* Use this when one total load profile should be split across weighted scenario lanes.
|
|
2499
2883
|
*/
|
|
2500
2884
|
static registerTrafficMix(trafficMix) {
|
|
2501
|
-
return LoadStrikeRunner.registerScenarios(...expandTrafficMixScenarios(trafficMix));
|
|
2885
|
+
return LoadStrikeRunner.registerScenarios(...expandTrafficMixScenarios(trafficMix, 0));
|
|
2502
2886
|
}
|
|
2503
2887
|
/**
|
|
2504
2888
|
* Registers a traffic mix on a fresh runnable context.
|
|
@@ -2514,6 +2898,12 @@ export class LoadStrikeRunner {
|
|
|
2514
2898
|
static DisplayConsoleMetrics(context, enable) {
|
|
2515
2899
|
return context.DisplayConsoleMetrics(enable);
|
|
2516
2900
|
}
|
|
2901
|
+
static UseLoadEngineV2(context) {
|
|
2902
|
+
return context.UseLoadEngineV2();
|
|
2903
|
+
}
|
|
2904
|
+
static WithMaxInFlight(context, maxInFlight) {
|
|
2905
|
+
return context.WithMaxInFlight(maxInFlight);
|
|
2906
|
+
}
|
|
2517
2907
|
/**
|
|
2518
2908
|
* Toggles local development cluster mode.
|
|
2519
2909
|
* Use this when you want to simulate coordinator and agent behavior on a single machine.
|
|
@@ -2549,6 +2939,12 @@ export class LoadStrikeRunner {
|
|
|
2549
2939
|
static WithAgentGroup(context, agentGroup) {
|
|
2550
2940
|
return context.WithAgentGroup(agentGroup);
|
|
2551
2941
|
}
|
|
2942
|
+
static WithAgentId(context, agentId) {
|
|
2943
|
+
return context.WithAgentId(agentId);
|
|
2944
|
+
}
|
|
2945
|
+
static WithExpectedAgentIds(context, ...agentIds) {
|
|
2946
|
+
return context.WithExpectedAgentIds(...agentIds);
|
|
2947
|
+
}
|
|
2552
2948
|
/**
|
|
2553
2949
|
* Sets the requested agent count.
|
|
2554
2950
|
* Use this when a coordinator should fan work out across a specific number of agents.
|
|
@@ -2766,7 +3162,10 @@ export class LoadStrikeRunner {
|
|
|
2766
3162
|
* Use this when one total load profile should be split across weighted scenario lanes.
|
|
2767
3163
|
*/
|
|
2768
3164
|
addTrafficMix(trafficMix) {
|
|
2769
|
-
this.scenarios = [
|
|
3165
|
+
this.scenarios = [
|
|
3166
|
+
...this.scenarios,
|
|
3167
|
+
...expandTrafficMixScenarios(trafficMix, nextTrafficMixDeclarationIndex(this.scenarios))
|
|
3168
|
+
];
|
|
2770
3169
|
return this;
|
|
2771
3170
|
}
|
|
2772
3171
|
/**
|
|
@@ -2899,6 +3298,19 @@ export class LoadStrikeRunner {
|
|
|
2899
3298
|
WithReportingInterval(intervalSeconds) {
|
|
2900
3299
|
return this.withReportingInterval(intervalSeconds);
|
|
2901
3300
|
}
|
|
3301
|
+
useLoadEngineV2() {
|
|
3302
|
+
return this.configure({ loadEngineContractVersion: 2 });
|
|
3303
|
+
}
|
|
3304
|
+
UseLoadEngineV2() {
|
|
3305
|
+
return this.useLoadEngineV2();
|
|
3306
|
+
}
|
|
3307
|
+
withMaxInFlight(maxInFlight) {
|
|
3308
|
+
validateV2MaxInFlight(this.options.loadEngineContractVersion, maxInFlight);
|
|
3309
|
+
return this.configure({ maxInFlight });
|
|
3310
|
+
}
|
|
3311
|
+
WithMaxInFlight(maxInFlight) {
|
|
3312
|
+
return this.withMaxInFlight(maxInFlight);
|
|
3313
|
+
}
|
|
2902
3314
|
withReportingSinks(...sinks) {
|
|
2903
3315
|
if (!sinks.length) {
|
|
2904
3316
|
throw new Error("At least one reporting sink should be provided.");
|
|
@@ -2997,7 +3409,7 @@ export class LoadStrikeRunner {
|
|
|
2997
3409
|
}
|
|
2998
3410
|
async run(args = []) {
|
|
2999
3411
|
if (this.contextConfigurators.length) {
|
|
3000
|
-
return new LoadStrikeRunner(this.scenarios, this.buildContext().toRunnerOptions()).run(args);
|
|
3412
|
+
return new LoadStrikeRunner(this.scenarios, this.buildContext().toRunnerOptions(), [], this.internalOptions).run(args);
|
|
3001
3413
|
}
|
|
3002
3414
|
if (args.length) {
|
|
3003
3415
|
return this.buildContext().run(args);
|
|
@@ -3045,10 +3457,15 @@ export class LoadStrikeRunner {
|
|
|
3045
3457
|
let licenseClient = null;
|
|
3046
3458
|
let licensePayload = null;
|
|
3047
3459
|
let licenseSession = null;
|
|
3460
|
+
let iterationObservationsFinalized = false;
|
|
3048
3461
|
const clusterMode = resolveClusterExecutionMode(this.options);
|
|
3049
3462
|
const selectedScenarios = clusterMode === "local-coordinator" || clusterMode === "nats-coordinator"
|
|
3050
3463
|
? await this.filterScenariosWithPolicies(this.scenarios, policies, policyErrors, runtimePolicyErrorMode)
|
|
3051
3464
|
: await this.selectScenarios(policies, policyErrors, runtimePolicyErrorMode);
|
|
3465
|
+
if (this.options.loadEngineContractVersion === 2
|
|
3466
|
+
&& (clusterMode === "nats-coordinator" || clusterMode === "nats-agent")) {
|
|
3467
|
+
validateLoadEngineV2ScenarioFeatures(selectedScenarios);
|
|
3468
|
+
}
|
|
3052
3469
|
if (clusterMode === "nats-agent") {
|
|
3053
3470
|
return this.runAgentWithNats(createdUtc, testInfo, nodeInfo);
|
|
3054
3471
|
}
|
|
@@ -3113,6 +3530,38 @@ export class LoadStrikeRunner {
|
|
|
3113
3530
|
}
|
|
3114
3531
|
}
|
|
3115
3532
|
await this.startSinks(sinkStates, sessionInfo, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
|
|
3533
|
+
const iterationObservationRunId = String(this.internalOptions.iterationObservationRunId
|
|
3534
|
+
?? sessionInfo.portalReportingRunId
|
|
3535
|
+
?? sessionInfo.PortalReportingRunId
|
|
3536
|
+
?? testInfo.sessionId);
|
|
3537
|
+
const iterationObservationResultOwnerId = String(nodeInfo.nodeType).toLowerCase() === "agent"
|
|
3538
|
+
? String(this.options.agentCommandId
|
|
3539
|
+
?? `${nodeInfo.machineName}:${Math.max(Math.trunc(this.options.clusterShardIndex ?? 0), 0)}`)
|
|
3540
|
+
: "";
|
|
3541
|
+
const iterationObservationProcessGroup = 0;
|
|
3542
|
+
const iterationObservationExpectedResultOwnerCount64 = Math.max(Math.trunc(this.options.clusterShardCount ?? 1), 1).toString();
|
|
3543
|
+
const initializedIterationObservationSinks = sinkStates
|
|
3544
|
+
.filter((state) => !state.disabled)
|
|
3545
|
+
.map((state) => ({
|
|
3546
|
+
name: state.name,
|
|
3547
|
+
iterationObservationPortalSink: Boolean(state.sink.iterationObservationPortalSink),
|
|
3548
|
+
iterationObservationShapeLimited: Boolean(state.sink.iterationObservationShapeLimited),
|
|
3549
|
+
saveIterationBatch: resolveSinkSaveIterationBatch(state.sink),
|
|
3550
|
+
completeIterationObservationStream: resolveSinkCompleteIterationObservationStream(state.sink)
|
|
3551
|
+
}));
|
|
3552
|
+
const reporterIterationObservationSinks = this.internalOptions.iterationObservationSinks
|
|
3553
|
+
?? (clusterMode === "local-coordinator" || clusterMode === "nats-coordinator"
|
|
3554
|
+
? []
|
|
3555
|
+
: initializedIterationObservationSinks);
|
|
3556
|
+
const iterationObservationReporter = new IterationObservationReporter({
|
|
3557
|
+
runId: iterationObservationRunId,
|
|
3558
|
+
sessionId: testInfo.sessionId,
|
|
3559
|
+
resultOwnerId: iterationObservationResultOwnerId,
|
|
3560
|
+
expectedResultOwnerCount64: iterationObservationExpectedResultOwnerCount64,
|
|
3561
|
+
processGroup: iterationObservationProcessGroup,
|
|
3562
|
+
settings: resolveIterationObservationSettings(this.options),
|
|
3563
|
+
sinks: reporterIterationObservationSinks
|
|
3564
|
+
});
|
|
3116
3565
|
const emitRealtimeSnapshot = async () => {
|
|
3117
3566
|
if (realtimeInFlight) {
|
|
3118
3567
|
return;
|
|
@@ -3179,21 +3628,30 @@ export class LoadStrikeRunner {
|
|
|
3179
3628
|
let result;
|
|
3180
3629
|
let metricStats;
|
|
3181
3630
|
if (clusterMode === "local-coordinator") {
|
|
3182
|
-
const aggregated = await this.runCoordinatorWithLocalAgents(selectedScenarios, testInfo, nodeInfo, licenseClient, licenseSession);
|
|
3631
|
+
const aggregated = await this.runCoordinatorWithLocalAgents(selectedScenarios, testInfo, nodeInfo, licenseClient, licenseSession, iterationObservationRunId, initializedIterationObservationSinks);
|
|
3183
3632
|
metricStats = aggregated.metrics;
|
|
3184
3633
|
result = toDetailedRunResultFromNodeStats(aggregated, started.toISOString(), sinkErrors, policyErrors);
|
|
3185
3634
|
}
|
|
3186
3635
|
else if (clusterMode === "nats-coordinator") {
|
|
3187
|
-
const aggregated = await this.runCoordinatorWithNats(selectedScenarios, testInfo, nodeInfo, licenseClient, licenseSession);
|
|
3636
|
+
const aggregated = await this.runCoordinatorWithNats(selectedScenarios, testInfo, nodeInfo, licenseClient, licenseSession, iterationObservationRunId, initializedIterationObservationSinks);
|
|
3188
3637
|
metricStats = aggregated.metrics;
|
|
3189
3638
|
result = toDetailedRunResultFromNodeStats(aggregated, started.toISOString(), sinkErrors, policyErrors);
|
|
3190
3639
|
}
|
|
3191
3640
|
else {
|
|
3192
3641
|
const testAbortController = new AbortController();
|
|
3193
3642
|
const stopTestState = { value: false, reason: undefined };
|
|
3194
|
-
|
|
3643
|
+
const loadEngineV2Budget = this.options.loadEngineContractVersion === 2
|
|
3644
|
+
? (this.options.loadEngineV2BudgetOverride
|
|
3645
|
+
?? new LoadEngineV2ExecutionBudget(this.options.maxInFlight ?? 10000))
|
|
3646
|
+
: undefined;
|
|
3647
|
+
const loadEngineV2Telemetry = loadEngineV2Budget
|
|
3648
|
+
? new LoadEngineV2Telemetry(loadEngineV2Budget)
|
|
3649
|
+
: undefined;
|
|
3650
|
+
const executeSelectedScenario = (scenario, selectedScenarioIndex) => executeScenarioRuntime({
|
|
3195
3651
|
scenario,
|
|
3196
|
-
scenarioIndex
|
|
3652
|
+
scenarioIndex: this.options.loadEngineContractVersion === 2
|
|
3653
|
+
? this.scenarios.indexOf(scenario)
|
|
3654
|
+
: selectedScenarioIndex,
|
|
3197
3655
|
scenarioCount: selectedScenarios.length,
|
|
3198
3656
|
options: this.options,
|
|
3199
3657
|
logger: runLogger,
|
|
@@ -3208,12 +3666,26 @@ export class LoadStrikeRunner {
|
|
|
3208
3666
|
scenarioDurationsMs,
|
|
3209
3667
|
stopTestState,
|
|
3210
3668
|
testAbortController,
|
|
3669
|
+
loadEngineV2Budget,
|
|
3670
|
+
loadEngineV2Telemetry,
|
|
3671
|
+
iterationObservationReporter,
|
|
3672
|
+
iterationObservationRunId,
|
|
3673
|
+
iterationObservationResultOwnerId,
|
|
3674
|
+
iterationObservationProcessGroup,
|
|
3211
3675
|
executeScenarioInvocation: (targetScenario, context, operation) => this.executeScenarioInvocation(targetScenario, context, operation),
|
|
3212
3676
|
invokeBeforeScenario: (runtimePolicies, scenarioName) => this.invokeBeforeScenario(runtimePolicies, scenarioName, policyErrors, runtimePolicyErrorMode),
|
|
3213
3677
|
invokeAfterScenario: (runtimePolicies, scenarioName, stats) => this.invokeAfterScenario(runtimePolicies, scenarioName, stats, policyErrors, runtimePolicyErrorMode),
|
|
3214
3678
|
invokeBeforeStep: (runtimePolicies, scenarioName, stepName) => this.invokeBeforeStep(runtimePolicies, scenarioName, stepName, policyErrors, runtimePolicyErrorMode),
|
|
3215
3679
|
invokeAfterStep: (runtimePolicies, scenarioName, stepName, reply) => this.invokeAfterStep(runtimePolicies, scenarioName, stepName, reply, policyErrors, runtimePolicyErrorMode)
|
|
3216
|
-
})
|
|
3680
|
+
});
|
|
3681
|
+
if (this.options.loadEngineV2SegmentLifecycleOverride) {
|
|
3682
|
+
for (let scenarioIndex = 0; scenarioIndex < selectedScenarios.length; scenarioIndex += 1) {
|
|
3683
|
+
await executeSelectedScenario(selectedScenarios[scenarioIndex], scenarioIndex);
|
|
3684
|
+
}
|
|
3685
|
+
}
|
|
3686
|
+
else {
|
|
3687
|
+
await Promise.all(selectedScenarios.map(executeSelectedScenario));
|
|
3688
|
+
}
|
|
3217
3689
|
nodeInfo.currentOperation = stopTestState.value ? "Stop" : "Complete";
|
|
3218
3690
|
const scenarioStatList = Array.from(scenarioAccumulators.values())
|
|
3219
3691
|
.map((value) => value.build(scenarioDurationsMs.get(value.scenarioName) ?? 0))
|
|
@@ -3253,10 +3725,39 @@ export class LoadStrikeRunner {
|
|
|
3253
3725
|
reportFiles: [],
|
|
3254
3726
|
logFiles: [...loggerSetup.logFiles],
|
|
3255
3727
|
correlationRows: buildDetailedCorrelationRows(),
|
|
3256
|
-
failedCorrelationRows: buildDetailedFailedCorrelationRows()
|
|
3728
|
+
failedCorrelationRows: buildDetailedFailedCorrelationRows(),
|
|
3729
|
+
...(loadEngineV2Telemetry
|
|
3730
|
+
? {
|
|
3731
|
+
generatorWarnings: loadEngineV2Telemetry.buildWarnings(),
|
|
3732
|
+
schedulerSegments: loadEngineV2Telemetry.buildSegments(),
|
|
3733
|
+
schedulerStats: loadEngineV2Telemetry.buildStats(),
|
|
3734
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: loadEngineV2Telemetry.buildSchedulerDistributions()
|
|
3735
|
+
}
|
|
3736
|
+
: {})
|
|
3257
3737
|
};
|
|
3258
3738
|
}
|
|
3259
3739
|
await stopRealtimeReporting();
|
|
3740
|
+
const observationDelivery = await iterationObservationReporter.sealAndDrain();
|
|
3741
|
+
iterationObservationsFinalized = true;
|
|
3742
|
+
if (clusterMode !== "local-coordinator"
|
|
3743
|
+
&& clusterMode !== "nats-coordinator") {
|
|
3744
|
+
result.observationDeliveryStats = {
|
|
3745
|
+
lastBatchSequence64: observationDelivery.lastBatchSequence64,
|
|
3746
|
+
capturedCount64: observationDelivery.capturedCount64,
|
|
3747
|
+
deliveredCount64: observationDelivery.deliveredCount64,
|
|
3748
|
+
droppedBufferCount64: observationDelivery.droppedBufferCount64,
|
|
3749
|
+
droppedSinkCount64: observationDelivery.droppedSinkCount64
|
|
3750
|
+
};
|
|
3751
|
+
result.reportingComplete = observationDelivery.reportingComplete;
|
|
3752
|
+
}
|
|
3753
|
+
else {
|
|
3754
|
+
result.observationDeliveryStats ?? (result.observationDeliveryStats = emptyObservationDeliveryStats());
|
|
3755
|
+
result.reportingComplete ?? (result.reportingComplete = observationDelivery.reportingComplete);
|
|
3756
|
+
}
|
|
3757
|
+
result.generatorWarnings = [
|
|
3758
|
+
...(result.generatorWarnings ?? []),
|
|
3759
|
+
...iterationObservationReporter.buildWarnings()
|
|
3760
|
+
];
|
|
3260
3761
|
result.pluginsData = mergePluginData(result.pluginsData, await this.collectPluginData(plugins, attachRunResultAliases(result), pluginLifecycleErrors));
|
|
3261
3762
|
const finalizedResult = attachRunResultAliases(result);
|
|
3262
3763
|
finalizedResult.logFiles = mergeStringArrays(finalizedResult.logFiles, loggerSetup.logFiles);
|
|
@@ -3272,6 +3773,9 @@ export class LoadStrikeRunner {
|
|
|
3272
3773
|
}
|
|
3273
3774
|
finally {
|
|
3274
3775
|
await stopRealtimeReporting();
|
|
3776
|
+
if (!iterationObservationsFinalized) {
|
|
3777
|
+
await iterationObservationReporter.sealAndDrain().catch(() => { });
|
|
3778
|
+
}
|
|
3275
3779
|
if (!sinksStopped) {
|
|
3276
3780
|
await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
|
|
3277
3781
|
}
|
|
@@ -3308,7 +3812,7 @@ export class LoadStrikeRunner {
|
|
|
3308
3812
|
}
|
|
3309
3813
|
return filtered;
|
|
3310
3814
|
}
|
|
3311
|
-
async runClusterChildNode(targetScenarios, nodeType, machineName, includeWorkerExtensions, overrides = {}) {
|
|
3815
|
+
async runClusterChildNode(targetScenarios, nodeType, machineName, includeWorkerExtensions, overrides = {}, internalOptions = {}) {
|
|
3312
3816
|
if (!targetScenarios.length) {
|
|
3313
3817
|
return buildEmptyNodeStats({
|
|
3314
3818
|
startedUtc: new Date().toISOString(),
|
|
@@ -3336,7 +3840,7 @@ export class LoadStrikeRunner {
|
|
|
3336
3840
|
displayConsoleMetrics: false,
|
|
3337
3841
|
reportingSinks: includeWorkerExtensions ? this.options.reportingSinks : [],
|
|
3338
3842
|
workerPlugins: includeWorkerExtensions ? this.options.workerPlugins : []
|
|
3339
|
-
});
|
|
3843
|
+
}, [], internalOptions);
|
|
3340
3844
|
const childResult = await childRunner.run();
|
|
3341
3845
|
const childStats = detailedToNodeStats(childResult);
|
|
3342
3846
|
return {
|
|
@@ -3349,70 +3853,127 @@ export class LoadStrikeRunner {
|
|
|
3349
3853
|
logFiles: [...(childResult.logFiles ?? [])]
|
|
3350
3854
|
};
|
|
3351
3855
|
}
|
|
3352
|
-
async runCoordinatorWithLocalAgents(scenarios, testInfo, nodeInfo, licenseClient, licenseSession) {
|
|
3856
|
+
async runCoordinatorWithLocalAgents(scenarios, testInfo, nodeInfo, licenseClient, licenseSession, iterationObservationRunId, iterationObservationSinks) {
|
|
3353
3857
|
if (!licenseClient) {
|
|
3354
3858
|
throw new Error("Coordinator agent execution authorization requires an initialized licensing client.");
|
|
3355
3859
|
}
|
|
3356
|
-
const controllerRunToken =
|
|
3860
|
+
const controllerRunToken = currentLicenseSessionRunToken(licenseSession);
|
|
3357
3861
|
if (!controllerRunToken) {
|
|
3358
3862
|
throw new Error("Coordinator agent execution authorization requires an active controller run token.");
|
|
3359
3863
|
}
|
|
3360
|
-
const assignments = planRuntimeClusterAssignments(scenarios, Math.max(this.options.agentsCount ?? 1, 1), this.options.targetScenarios ?? [], this.options.agentTargetScenarios ?? []);
|
|
3864
|
+
const assignments = planRuntimeClusterAssignments(scenarios, Math.max(this.options.agentsCount ?? 1, 1), this.options.targetScenarios ?? [], this.options.agentTargetScenarios ?? [], this.options.loadEngineContractVersion === 2, this.options.coordinatorTargetScenarios ?? []);
|
|
3865
|
+
const sharedV2Budget = this.options.loadEngineContractVersion === 2
|
|
3866
|
+
? new LoadEngineV2ExecutionBudget(this.options.maxInFlight ?? 10000)
|
|
3867
|
+
: undefined;
|
|
3361
3868
|
const nodeResults = await Promise.all(assignments.map(async (targetScenarios, index) => {
|
|
3362
3869
|
const commandId = randomBytes(16).toString("hex");
|
|
3363
|
-
const agentExecutionToken = await licenseClient.createAgentExecutionToken(
|
|
3870
|
+
const agentExecutionToken = await licenseClient.createAgentExecutionToken(currentLicenseSessionRunToken(licenseSession), testInfo.sessionId, commandId, index, assignments.length, targetScenarios);
|
|
3364
3871
|
return this.runClusterChildNode(targetScenarios, "Agent", `local-agent-${index + 1}`, false, {
|
|
3365
3872
|
sessionId: testInfo.sessionId,
|
|
3366
3873
|
testSuite: testInfo.testSuite,
|
|
3367
3874
|
testName: testInfo.testName,
|
|
3368
3875
|
agentCommandId: commandId,
|
|
3369
|
-
agentExecutionToken
|
|
3876
|
+
agentExecutionToken,
|
|
3877
|
+
clusterShardIndex: index,
|
|
3878
|
+
clusterShardCount: assignments.length,
|
|
3879
|
+
loadEngineV2BudgetOverride: sharedV2Budget
|
|
3880
|
+
}, {
|
|
3881
|
+
iterationObservationRunId,
|
|
3882
|
+
iterationObservationSinks
|
|
3370
3883
|
});
|
|
3371
3884
|
}));
|
|
3372
3885
|
const coordinatorTargets = [...(this.options.coordinatorTargetScenarios ?? [])];
|
|
3373
3886
|
if (coordinatorTargets.length) {
|
|
3374
|
-
nodeResults.push(await this.runClusterChildNode(coordinatorTargets, "Coordinator", nodeInfo.machineName, false
|
|
3887
|
+
nodeResults.push(await this.runClusterChildNode(coordinatorTargets, "Coordinator", nodeInfo.machineName, false, {}, {
|
|
3888
|
+
iterationObservationRunId,
|
|
3889
|
+
iterationObservationSinks
|
|
3890
|
+
}));
|
|
3375
3891
|
}
|
|
3376
|
-
return aggregateNodeStats(testInfo, { ...nodeInfo, nodeType: "Coordinator" }, nodeResults);
|
|
3892
|
+
return aggregateNodeStats(testInfo, { ...nodeInfo, nodeType: "Coordinator" }, nodeResults, this.options.loadEngineContractVersion === 2);
|
|
3377
3893
|
}
|
|
3378
|
-
async runCoordinatorWithNats(scenarios, testInfo, nodeInfo, licenseClient, licenseSession) {
|
|
3894
|
+
async runCoordinatorWithNats(scenarios, testInfo, nodeInfo, licenseClient, licenseSession, iterationObservationRunId, iterationObservationSinks) {
|
|
3379
3895
|
if (!licenseClient) {
|
|
3380
3896
|
throw new Error("Coordinator agent execution authorization requires an initialized licensing client.");
|
|
3381
3897
|
}
|
|
3382
|
-
const controllerRunToken =
|
|
3898
|
+
const controllerRunToken = currentLicenseSessionRunToken(licenseSession);
|
|
3383
3899
|
if (!controllerRunToken) {
|
|
3384
3900
|
throw new Error("Coordinator agent execution authorization requires an active controller run token.");
|
|
3385
3901
|
}
|
|
3386
|
-
const assignments = planRuntimeClusterAssignments(scenarios, Math.max(this.options.agentsCount ?? 1, 1), this.options.targetScenarios ?? [], this.options.agentTargetScenarios ?? []);
|
|
3902
|
+
const assignments = planRuntimeClusterAssignments(scenarios, Math.max(this.options.agentsCount ?? 1, 1), this.options.targetScenarios ?? [], this.options.agentTargetScenarios ?? [], this.options.loadEngineContractVersion === 2, this.options.coordinatorTargetScenarios ?? []);
|
|
3903
|
+
const expectedAgentIds = this.options.loadEngineContractVersion === 2
|
|
3904
|
+
? normalizeRequiredV2AgentIds(this.options.expectedAgentIds, assignments.length)
|
|
3905
|
+
: undefined;
|
|
3387
3906
|
const coordinator = new DistributedClusterCoordinator({
|
|
3388
3907
|
clusterId: this.options.clusterId ?? "local",
|
|
3389
3908
|
sessionId: testInfo.sessionId,
|
|
3390
3909
|
testSuite: testInfo.testSuite,
|
|
3391
3910
|
testName: testInfo.testName,
|
|
3392
3911
|
expectedAgentResults: assignments.length,
|
|
3912
|
+
expectedAgentIds,
|
|
3913
|
+
loadEngineContractVersion: this.options.loadEngineContractVersion ?? 1,
|
|
3393
3914
|
agentGroup: this.options.agentGroup,
|
|
3394
3915
|
commandTimeoutMs: Math.max(Math.trunc((this.options.clusterCommandTimeoutSeconds ?? 120) * 1000), 1),
|
|
3395
3916
|
nats: this.options.natsServerUrl
|
|
3396
3917
|
? { ServerUrl: this.options.natsServerUrl }
|
|
3397
3918
|
: undefined
|
|
3398
3919
|
});
|
|
3399
|
-
const
|
|
3400
|
-
const
|
|
3920
|
+
const tokenFactory = (command) => licenseClient.createAgentExecutionToken(currentLicenseSessionRunToken(licenseSession), testInfo.sessionId, command.commandId, command.agentIndex, command.agentCount, command.targetScenarios);
|
|
3921
|
+
const dispatch = this.options.loadEngineContractVersion === 2
|
|
3922
|
+
? await coordinator.dispatchV2(assignments, buildRuntimeLoadEngineV2Plan(scenarios, this.options, testInfo, expectedAgentIds), tokenFactory)
|
|
3923
|
+
: await coordinator.dispatch(assignments, tokenFactory);
|
|
3924
|
+
const nodes = dispatch.nodeResults.map((value) => clusterNodeResultToNodeStats(value, testInfo, { ...nodeInfo, nodeType: "Agent" }, this.options.loadEngineContractVersion === 2));
|
|
3401
3925
|
const coordinatorTargets = [...(this.options.coordinatorTargetScenarios ?? [])];
|
|
3402
3926
|
if (coordinatorTargets.length) {
|
|
3403
|
-
nodes.push(await this.runClusterChildNode(coordinatorTargets, "Coordinator", nodeInfo.machineName, false
|
|
3927
|
+
nodes.push(await this.runClusterChildNode(coordinatorTargets, "Coordinator", nodeInfo.machineName, false, {}, {
|
|
3928
|
+
iterationObservationRunId,
|
|
3929
|
+
iterationObservationSinks
|
|
3930
|
+
}));
|
|
3404
3931
|
}
|
|
3405
|
-
let aggregated = aggregateNodeStats(testInfo, { ...nodeInfo, nodeType: "Coordinator" }, nodes);
|
|
3932
|
+
let aggregated = aggregateNodeStats(testInfo, { ...nodeInfo, nodeType: "Coordinator" }, nodes, this.options.loadEngineContractVersion === 2);
|
|
3406
3933
|
if (dispatch.missingNodes > 0) {
|
|
3407
3934
|
aggregated = appendClusterPluginHint(aggregated, `Timed out waiting for ${dispatch.missingNodes} agent node result(s).`);
|
|
3935
|
+
aggregated = attachNodeStatsAliases({
|
|
3936
|
+
...aggregated,
|
|
3937
|
+
reportingComplete: false,
|
|
3938
|
+
schedulerSegments: [
|
|
3939
|
+
...(aggregated.schedulerSegments ?? []),
|
|
3940
|
+
...(dispatch.ownerLoss?.schedulerSegments ?? [])
|
|
3941
|
+
],
|
|
3942
|
+
generatorWarnings: [
|
|
3943
|
+
...aggregated.generatorWarnings,
|
|
3944
|
+
...(dispatch.ownerLoss?.generatorWarnings ?? []).map((warning) => ({
|
|
3945
|
+
code: warning.code,
|
|
3946
|
+
scenarioName: "cluster",
|
|
3947
|
+
simulationIndex: -1,
|
|
3948
|
+
count64: warning.count64,
|
|
3949
|
+
message: `Result owner ${warning.agentId} was lost after assignment; application failures were not fabricated.`,
|
|
3950
|
+
firstObservedUtcNs: "0",
|
|
3951
|
+
lastObservedUtcNs: "0"
|
|
3952
|
+
}))
|
|
3953
|
+
]
|
|
3954
|
+
});
|
|
3408
3955
|
}
|
|
3409
3956
|
return aggregated;
|
|
3410
3957
|
}
|
|
3411
3958
|
async runAgentWithNats(startedUtc, testInfo, nodeInfo) {
|
|
3959
|
+
const v2AgentId = this.options.loadEngineContractVersion === 2
|
|
3960
|
+
? requireNonEmpty(this.options.agentId ?? "", "Remote Load Engine V2 agents require an explicit stable AgentId.")
|
|
3961
|
+
: `${nodeInfo.machineName}-${generateRuntimeSessionId()}`;
|
|
3412
3962
|
const agent = new DistributedClusterAgent({
|
|
3413
3963
|
clusterId: this.options.clusterId ?? "local",
|
|
3414
3964
|
sessionId: testInfo.sessionId,
|
|
3415
|
-
agentId:
|
|
3965
|
+
agentId: v2AgentId,
|
|
3966
|
+
loadEngineContractVersion: this.options.loadEngineContractVersion ?? 1,
|
|
3967
|
+
validateV2Plan: this.options.loadEngineContractVersion === 2
|
|
3968
|
+
? (received) => {
|
|
3969
|
+
const local = buildRuntimeLoadEngineV2Plan(this.scenarios, this.options, { ...testInfo, sessionId: received.sessionId }, received.expectedAgentIds);
|
|
3970
|
+
local.runId = received.runId;
|
|
3971
|
+
local.registrationNonce = received.registrationNonce;
|
|
3972
|
+
if (buildLoadEngineV2Plan(local).hash !== buildLoadEngineV2Plan(received).hash) {
|
|
3973
|
+
throw new Error("Load Engine V2 signed plan descriptors do not match immutable local scenario declarations.");
|
|
3974
|
+
}
|
|
3975
|
+
}
|
|
3976
|
+
: undefined,
|
|
3416
3977
|
agentGroup: this.options.agentGroup,
|
|
3417
3978
|
nats: this.options.natsServerUrl
|
|
3418
3979
|
? { ServerUrl: this.options.natsServerUrl }
|
|
@@ -3422,13 +3983,16 @@ export class LoadStrikeRunner {
|
|
|
3422
3983
|
const deadline = Date.now() + Math.max(Math.trunc((this.options.clusterCommandTimeoutSeconds ?? 120) * 1000), 1);
|
|
3423
3984
|
while (Date.now() < deadline) {
|
|
3424
3985
|
let handledStats = null;
|
|
3425
|
-
const
|
|
3986
|
+
const execute = async (dispatch) => {
|
|
3426
3987
|
handledStats = await this.runClusterChildNode(dispatch.scenarioNames, "Agent", nodeInfo.machineName, true, {
|
|
3427
3988
|
sessionId: testInfo.sessionId,
|
|
3428
3989
|
testSuite: testInfo.testSuite,
|
|
3429
3990
|
testName: testInfo.testName,
|
|
3430
3991
|
agentCommandId: dispatch.commandId,
|
|
3431
|
-
agentExecutionToken: dispatch.agentRunToken
|
|
3992
|
+
agentExecutionToken: dispatch.agentRunToken,
|
|
3993
|
+
clusterShardIndex: dispatch.agentIndex ?? 0,
|
|
3994
|
+
clusterShardCount: dispatch.agentCount ?? 1,
|
|
3995
|
+
loadEngineV2SegmentLifecycleOverride: dispatch.segmentLifecycle
|
|
3432
3996
|
});
|
|
3433
3997
|
return {
|
|
3434
3998
|
nodeId: handledStats.nodeInfo.machineName,
|
|
@@ -3436,9 +4000,12 @@ export class LoadStrikeRunner {
|
|
|
3436
4000
|
allRequestCount: handledStats.allRequestCount,
|
|
3437
4001
|
allOkCount: handledStats.allOkCount,
|
|
3438
4002
|
allFailCount: handledStats.allFailCount,
|
|
3439
|
-
stats: nodeStatsToClusterPayload(handledStats)
|
|
4003
|
+
stats: nodeStatsToClusterPayload(handledStats, this.scenarios.filter((scenario) => dispatch.scenarioNames.includes(scenario.name)), this.options.loadEngineContractVersion === 2)
|
|
3440
4004
|
};
|
|
3441
|
-
}
|
|
4005
|
+
};
|
|
4006
|
+
const handled = this.options.loadEngineContractVersion === 2
|
|
4007
|
+
? await agent.pollAndExecuteV2Once(execute)
|
|
4008
|
+
: await agent.pollAndExecuteOnce(execute);
|
|
3442
4009
|
if (handled && handledStats) {
|
|
3443
4010
|
return toDetailedRunResultFromNodeStats(handledStats, startedUtc, [], []);
|
|
3444
4011
|
}
|
|
@@ -3717,13 +4284,148 @@ function hasPluginRows(value) {
|
|
|
3717
4284
|
}
|
|
3718
4285
|
return value.tables.some((table) => Array.isArray(table.rows) && table.rows.length > 0);
|
|
3719
4286
|
}
|
|
4287
|
+
async function executeV2FixedArrivals(args) {
|
|
4288
|
+
const { rate, intervalNs, totalArrivals, budget, cancellationToken, shouldStopNow, nextInstanceInfo, runBombingInvocation, logger, scenarioName, deadlineOffsetsNs, tolerancesNs, ownedOrdinals, shardIndex = 0, shardCount = 1, telemetry, segment } = args;
|
|
4289
|
+
const segmentStartNs = process.hrtime.bigint();
|
|
4290
|
+
const toleranceNs = loadEngineV2LatenessToleranceNs(rate, intervalNs);
|
|
4291
|
+
const active = new Set();
|
|
4292
|
+
let schedulerLate = 0n;
|
|
4293
|
+
let maxInFlight = 0n;
|
|
4294
|
+
let executionError;
|
|
4295
|
+
const normalizedShardCount = Math.max(Math.trunc(shardCount), 1);
|
|
4296
|
+
const normalizedShardIndex = Math.min(Math.max(Math.trunc(shardIndex), 0), normalizedShardCount - 1);
|
|
4297
|
+
const ordinals = ownedOrdinals
|
|
4298
|
+
? [...ownedOrdinals]
|
|
4299
|
+
: (() => {
|
|
4300
|
+
const values = [];
|
|
4301
|
+
for (let ordinal = BigInt(normalizedShardIndex); ordinal < totalArrivals; ordinal += BigInt(normalizedShardCount)) {
|
|
4302
|
+
values.push(ordinal);
|
|
4303
|
+
}
|
|
4304
|
+
return values;
|
|
4305
|
+
})();
|
|
4306
|
+
if (ordinals.some((ordinal, index) => ordinal < 0n || ordinal >= totalArrivals
|
|
4307
|
+
|| (index > 0 && ordinal <= ordinals[index - 1]))) {
|
|
4308
|
+
throw new Error("Load Engine V2 owned arrival ordinals must be sorted, unique, and in range.");
|
|
4309
|
+
}
|
|
4310
|
+
if (segment) {
|
|
4311
|
+
segment.planned = BigInt(ordinals.length);
|
|
4312
|
+
}
|
|
4313
|
+
for (const ordinal of ordinals) {
|
|
4314
|
+
if (shouldStopNow())
|
|
4315
|
+
break;
|
|
4316
|
+
const index = Number(ordinal);
|
|
4317
|
+
const deadlineNs = segmentStartNs
|
|
4318
|
+
+ (deadlineOffsetsNs?.[index] ?? loadEngineV2FixedDeadlineNs(ordinal, rate, intervalNs));
|
|
4319
|
+
await delayUntilMonotonicDeadline(deadlineNs, cancellationToken);
|
|
4320
|
+
if (shouldStopNow()) {
|
|
4321
|
+
break;
|
|
4322
|
+
}
|
|
4323
|
+
const nowNs = process.hrtime.bigint();
|
|
4324
|
+
if (segment)
|
|
4325
|
+
segment.due += 1n;
|
|
4326
|
+
if (segment)
|
|
4327
|
+
telemetry?.recordDecisionLag(segment, nowNs - deadlineNs);
|
|
4328
|
+
if (classifyLoadEngineV2Arrival(nowNs, deadlineNs, tolerancesNs?.[index] ?? toleranceNs, true) === "scheduler_late") {
|
|
4329
|
+
schedulerLate += 1n;
|
|
4330
|
+
if (segment) {
|
|
4331
|
+
segment.dropped += 1n;
|
|
4332
|
+
incrementReason(segment.dropReasons, "scheduler_late");
|
|
4333
|
+
}
|
|
4334
|
+
continue;
|
|
4335
|
+
}
|
|
4336
|
+
const release = budget.tryAcquire();
|
|
4337
|
+
if (!release) {
|
|
4338
|
+
maxInFlight += 1n;
|
|
4339
|
+
if (segment) {
|
|
4340
|
+
segment.dropped += 1n;
|
|
4341
|
+
incrementReason(segment.dropReasons, "max_in_flight");
|
|
4342
|
+
}
|
|
4343
|
+
continue;
|
|
4344
|
+
}
|
|
4345
|
+
if (segment)
|
|
4346
|
+
segment.started += 1n;
|
|
4347
|
+
const instanceInfo = nextInstanceInfo();
|
|
4348
|
+
let task;
|
|
4349
|
+
task = (async () => {
|
|
4350
|
+
if (segment)
|
|
4351
|
+
telemetry?.recordStartLag(segment, process.hrtime.bigint() - deadlineNs);
|
|
4352
|
+
await runBombingInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, ordinal);
|
|
4353
|
+
})().catch((error) => {
|
|
4354
|
+
executionError ?? (executionError = error);
|
|
4355
|
+
}).finally(() => {
|
|
4356
|
+
if (segment)
|
|
4357
|
+
segment.completed += 1n;
|
|
4358
|
+
release();
|
|
4359
|
+
active.delete(task);
|
|
4360
|
+
});
|
|
4361
|
+
active.add(task);
|
|
4362
|
+
}
|
|
4363
|
+
while (active.size > 0) {
|
|
4364
|
+
await Promise.race(active);
|
|
4365
|
+
}
|
|
4366
|
+
if (segment) {
|
|
4367
|
+
segment.unreached = segment.planned > segment.due ? segment.planned - segment.due : 0n;
|
|
4368
|
+
segment.accountingComplete = segment.due + segment.unreached === segment.planned
|
|
4369
|
+
&& segment.started + segment.dropped === segment.due
|
|
4370
|
+
&& segment.completed === segment.started;
|
|
4371
|
+
}
|
|
4372
|
+
if (telemetry && segment) {
|
|
4373
|
+
telemetry.recordWarning("scheduler_late", segment, schedulerLate);
|
|
4374
|
+
telemetry.recordWarning("max_in_flight", segment, maxInFlight);
|
|
4375
|
+
}
|
|
4376
|
+
if (schedulerLate > 0n) {
|
|
4377
|
+
logger.warn(`Load Engine V2 warning scheduler_late: dropped ${schedulerLate.toString()} overdue arrivals for scenario ${scenarioName}.`);
|
|
4378
|
+
}
|
|
4379
|
+
if (maxInFlight > 0n) {
|
|
4380
|
+
logger.warn(`Load Engine V2 warning max_in_flight: dropped ${maxInFlight.toString()} arrivals for scenario ${scenarioName}; the process limit is ${budget.maxInFlight}.`);
|
|
4381
|
+
}
|
|
4382
|
+
if (executionError !== undefined) {
|
|
4383
|
+
throw executionError;
|
|
4384
|
+
}
|
|
4385
|
+
}
|
|
4386
|
+
async function delayUntilMonotonicDeadline(deadlineNs, cancellationToken) {
|
|
4387
|
+
while (!cancellationToken.aborted) {
|
|
4388
|
+
const remainingNs = deadlineNs - process.hrtime.bigint();
|
|
4389
|
+
if (remainingNs <= 0n) {
|
|
4390
|
+
return;
|
|
4391
|
+
}
|
|
4392
|
+
const remainingMs = Number((remainingNs + 999999n) / 1000000n);
|
|
4393
|
+
await delayWithAbort(Math.max(1, Math.min(remainingMs, 50)), cancellationToken);
|
|
4394
|
+
}
|
|
4395
|
+
}
|
|
4396
|
+
function secondsToNanoseconds(seconds, name) {
|
|
4397
|
+
if (!Number.isFinite(seconds) || seconds <= 0) {
|
|
4398
|
+
throw new RangeError(`${name} must be greater than zero.`);
|
|
4399
|
+
}
|
|
4400
|
+
const nanoseconds = Math.trunc(seconds * 1000000000);
|
|
4401
|
+
if (!Number.isSafeInteger(nanoseconds) || nanoseconds <= 0) {
|
|
4402
|
+
throw new RangeError(`${name} is outside the supported nanosecond range.`);
|
|
4403
|
+
}
|
|
4404
|
+
return BigInt(nanoseconds);
|
|
4405
|
+
}
|
|
4406
|
+
function buildV2RampTolerances(offsets, durationNs) {
|
|
4407
|
+
return offsets.map((offset, index) => {
|
|
4408
|
+
const quantum = offsets.length === 1
|
|
4409
|
+
? durationNs
|
|
4410
|
+
: index === 0
|
|
4411
|
+
? offsets[1] - offset
|
|
4412
|
+
: offset - offsets[index - 1];
|
|
4413
|
+
return minBigInt(100000000n, maxBigInt(2000000n, maxBigInt(1n, quantum) * 4n));
|
|
4414
|
+
});
|
|
4415
|
+
}
|
|
4416
|
+
function minBigInt(left, right) {
|
|
4417
|
+
return left < right ? left : right;
|
|
4418
|
+
}
|
|
4419
|
+
function maxBigInt(left, right) {
|
|
4420
|
+
return left > right ? left : right;
|
|
4421
|
+
}
|
|
3720
4422
|
async function executeScenarioRuntime(args) {
|
|
3721
|
-
const { scenario, scenarioIndex, scenarioCount, options, logger, nodeInfo, testInfo, policies, restartIterationMaxAttempts, allRegisteredMetrics, scenarioRuntimes, stepRuntimes, scenarioAccumulators, scenarioDurationsMs, stopTestState, testAbortController, executeScenarioInvocation, invokeBeforeScenario, invokeAfterScenario, invokeBeforeStep, invokeAfterStep } = args;
|
|
4423
|
+
const { scenario, scenarioIndex, scenarioCount, options, logger, nodeInfo, testInfo, policies, restartIterationMaxAttempts, allRegisteredMetrics, scenarioRuntimes, stepRuntimes, scenarioAccumulators, scenarioDurationsMs, stopTestState, testAbortController, loadEngineV2Budget, loadEngineV2Telemetry, iterationObservationReporter, iterationObservationRunId = testInfo.sessionId, iterationObservationResultOwnerId = "", iterationObservationProcessGroup = 0, executeScenarioInvocation, invokeBeforeScenario, invokeAfterScenario, invokeBeforeStep, invokeAfterStep } = args;
|
|
3722
4424
|
const scenarioStartedMs = Date.now();
|
|
3723
4425
|
const scenarioContextData = {};
|
|
3724
4426
|
const registeredMetrics = [];
|
|
3725
4427
|
const runtime = ensureScenarioRuntime(scenarioRuntimes, scenario.name);
|
|
3726
|
-
const accumulator = new ScenarioStatsAccumulator(scenario.name, scenarioIndex);
|
|
4428
|
+
const accumulator = new ScenarioStatsAccumulator(scenario.name, scenarioIndex, options.loadEngineContractVersion === 2);
|
|
3727
4429
|
scenarioAccumulators.set(scenario.name, accumulator);
|
|
3728
4430
|
const scenarioAbortController = new AbortController();
|
|
3729
4431
|
const scenarioCancellationToken = combineAbortSignals(testAbortController.signal, scenarioAbortController.signal);
|
|
@@ -3732,12 +4434,25 @@ async function executeScenarioRuntime(args) {
|
|
|
3732
4434
|
? Date.now() + Math.trunc(scenarioCompletionTimeoutSeconds * 1000)
|
|
3733
4435
|
: Number.POSITIVE_INFINITY;
|
|
3734
4436
|
const scenarioPartition = attachScenarioPartitionAliases({
|
|
3735
|
-
number: 0,
|
|
3736
|
-
count: 1
|
|
4437
|
+
number: options.loadEngineContractVersion === 2 ? Math.max(Math.trunc(options.clusterShardIndex ?? 0), 0) : 0,
|
|
4438
|
+
count: options.loadEngineContractVersion === 2 ? Math.max(Math.trunc(options.clusterShardCount ?? 1), 1) : 1
|
|
3737
4439
|
});
|
|
4440
|
+
const trafficMixV2 = options.loadEngineContractVersion === 2
|
|
4441
|
+
? scenario.__loadStrikeTrafficMixV2Metadata()
|
|
4442
|
+
: undefined;
|
|
4443
|
+
if (trafficMixV2) {
|
|
4444
|
+
const expectedSeedId = buildLoadEngineV2TrafficMixSeedId(trafficMixV2.declarationIndex, trafficMixV2.name);
|
|
4445
|
+
if (trafficMixV2.seedId !== expectedSeedId) {
|
|
4446
|
+
throw new Error(`Load Engine V2 traffic-mix seed metadata differs for scenario ${scenario.name}.`);
|
|
4447
|
+
}
|
|
4448
|
+
}
|
|
3738
4449
|
let stopScenario = false;
|
|
3739
4450
|
let invocationNumber = 0;
|
|
3740
4451
|
let instanceCounter = 0;
|
|
4452
|
+
let nextV1ObservationOrdinal = 0n;
|
|
4453
|
+
let nextV2FallbackOrdinal = BigInt(scenarioPartition.number);
|
|
4454
|
+
let nextObservationStepSortIndex = 0;
|
|
4455
|
+
const observationStepSortIndexes = new Map();
|
|
3741
4456
|
const shouldStopNow = () => stopTestState.value
|
|
3742
4457
|
|| stopScenario
|
|
3743
4458
|
|| scenarioCancellationToken.aborted
|
|
@@ -3771,7 +4486,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3771
4486
|
runtime.maxLatencyMs = Math.max(runtime.maxLatencyMs, latencyMs);
|
|
3772
4487
|
accumulator.recordScenario(reply, observedLatencyMs);
|
|
3773
4488
|
};
|
|
3774
|
-
const recordStepReply = (stepName, reply, observedLatencyMs) => {
|
|
4489
|
+
const recordStepReply = (stepName, reply, observedLatencyMs, sortIndex) => {
|
|
3775
4490
|
const key = `${scenario.name}::${stepName}`;
|
|
3776
4491
|
const stepRuntime = ensureStepRuntime(stepRuntimes, key, scenario.name, stepName);
|
|
3777
4492
|
if (reply.isSuccess) {
|
|
@@ -3794,11 +4509,35 @@ async function executeScenarioRuntime(args) {
|
|
|
3794
4509
|
stepRuntime.maxLatencyMs = Math.max(stepRuntime.maxLatencyMs, latencyMs);
|
|
3795
4510
|
const statusCode = normalizeStatusCode(reply.statusCode, reply.isSuccess);
|
|
3796
4511
|
stepRuntime.statusCodes[statusCode] = (stepRuntime.statusCodes[statusCode] ?? 0) + 1;
|
|
3797
|
-
accumulator.recordStep(stepName, reply, observedLatencyMs);
|
|
4512
|
+
return accumulator.recordStep(stepName, reply, observedLatencyMs, sortIndex);
|
|
4513
|
+
};
|
|
4514
|
+
const resolveObservationStepSortIndex = (stepName) => {
|
|
4515
|
+
const existing = observationStepSortIndexes.get(stepName);
|
|
4516
|
+
if (existing !== undefined) {
|
|
4517
|
+
return existing;
|
|
4518
|
+
}
|
|
4519
|
+
nextObservationStepSortIndex += 1;
|
|
4520
|
+
observationStepSortIndexes.set(stepName, nextObservationStepSortIndex);
|
|
4521
|
+
return nextObservationStepSortIndex;
|
|
4522
|
+
};
|
|
4523
|
+
const nextObservationOrdinal = (explicit) => {
|
|
4524
|
+
if (explicit !== undefined) {
|
|
4525
|
+
return explicit;
|
|
4526
|
+
}
|
|
4527
|
+
if (options.loadEngineContractVersion === 2) {
|
|
4528
|
+
const ordinal = nextV2FallbackOrdinal;
|
|
4529
|
+
nextV2FallbackOrdinal += BigInt(scenarioPartition.count);
|
|
4530
|
+
return ordinal;
|
|
4531
|
+
}
|
|
4532
|
+
const ordinal = nextV1ObservationOrdinal;
|
|
4533
|
+
nextV1ObservationOrdinal += 1n;
|
|
4534
|
+
return ordinal;
|
|
3798
4535
|
};
|
|
3799
4536
|
const runSingleInvocation = async (operation, instanceData, instanceNumber, instanceId, recordScenarioResult) => {
|
|
3800
4537
|
invocationNumber += 1;
|
|
3801
4538
|
const runtimeRandom = createRuntimeRandom();
|
|
4539
|
+
const attemptSteps = [];
|
|
4540
|
+
const recordedSteps = [];
|
|
3802
4541
|
const context = {
|
|
3803
4542
|
scenarioName: scenario.name,
|
|
3804
4543
|
data: scenarioContextData,
|
|
@@ -3827,7 +4566,12 @@ async function executeScenarioRuntime(args) {
|
|
|
3827
4566
|
testAbortController.abort(stopTestState.reason);
|
|
3828
4567
|
},
|
|
3829
4568
|
recordStep: (stepName, reply, observedLatencyMs) => {
|
|
3830
|
-
|
|
4569
|
+
const sortIndex = resolveObservationStepSortIndex(stepName);
|
|
4570
|
+
recordedSteps.push({ stepName, reply, observedLatencyMs, sortIndex });
|
|
4571
|
+
return sortIndex;
|
|
4572
|
+
},
|
|
4573
|
+
recordStepObservation: (observation) => {
|
|
4574
|
+
attemptSteps.push(observation);
|
|
3831
4575
|
},
|
|
3832
4576
|
shouldStopScenario: () => stopScenario || scenarioCancellationToken.aborted,
|
|
3833
4577
|
shouldStopTest: () => stopTestState.value || scenarioCancellationToken.aborted,
|
|
@@ -3835,28 +4579,92 @@ async function executeScenarioRuntime(args) {
|
|
|
3835
4579
|
invokeAfterStep: async (stepName, reply) => invokeAfterStep(policies, scenario.name, stepName, reply)
|
|
3836
4580
|
};
|
|
3837
4581
|
attachScenarioContextAliases(context);
|
|
3838
|
-
const
|
|
4582
|
+
const startedUtcNs = utcNowNs();
|
|
4583
|
+
const startedAtNs = process.hrtime.bigint();
|
|
3839
4584
|
const reply = await executeScenarioInvocation(scenario, context, operation);
|
|
3840
|
-
const
|
|
4585
|
+
const observedLatencyNs = maxBigInt(process.hrtime.bigint() - startedAtNs, 0n);
|
|
4586
|
+
const completedUtcNs = startedUtcNs + observedLatencyNs;
|
|
4587
|
+
const observedLatencyMs = Number(observedLatencyNs) / 1000000;
|
|
3841
4588
|
if (recordScenarioResult) {
|
|
3842
4589
|
recordScenarioReply(reply, observedLatencyMs);
|
|
3843
4590
|
}
|
|
3844
|
-
return {
|
|
4591
|
+
return {
|
|
4592
|
+
reply,
|
|
4593
|
+
observedLatencyMs,
|
|
4594
|
+
startedUtcNs,
|
|
4595
|
+
completedUtcNs,
|
|
4596
|
+
observedLatencyUs: observedLatencyNs / 1000n,
|
|
4597
|
+
reportedLatencyUs: normalizeRawObservationLatencyMicroseconds(resolveRecordedLatency(reply.customLatencyMs, observedLatencyMs)),
|
|
4598
|
+
steps: attemptSteps,
|
|
4599
|
+
recordedSteps
|
|
4600
|
+
};
|
|
4601
|
+
};
|
|
4602
|
+
const captureAttemptObservation = (operation, globalOrdinal, attemptIndex, isFinalAttempt, attempt, simulationIndex, simulationKind, iterationId, globalSecondaryOrdinal = 0n) => {
|
|
4603
|
+
if (!iterationObservationReporter?.enabled) {
|
|
4604
|
+
return;
|
|
4605
|
+
}
|
|
4606
|
+
iterationObservationReporter.capture(createIterationObservation({
|
|
4607
|
+
runId: iterationObservationRunId,
|
|
4608
|
+
sessionId: testInfo.sessionId,
|
|
4609
|
+
resultOwnerId: iterationObservationResultOwnerId,
|
|
4610
|
+
processGroup: iterationObservationProcessGroup,
|
|
4611
|
+
scenarioName: scenario.name,
|
|
4612
|
+
scenarioIndex,
|
|
4613
|
+
simulationIndex,
|
|
4614
|
+
simulationKind,
|
|
4615
|
+
phase: operation === "WarmUp" ? "warmup" : "bombing",
|
|
4616
|
+
globalOrdinal64: globalOrdinal,
|
|
4617
|
+
globalSecondaryOrdinal64: globalSecondaryOrdinal,
|
|
4618
|
+
...(iterationId ? { iterationId } : {}),
|
|
4619
|
+
shardIndex: scenarioPartition.number,
|
|
4620
|
+
shardCount: scenarioPartition.count,
|
|
4621
|
+
attemptIndex,
|
|
4622
|
+
isFinalAttempt,
|
|
4623
|
+
startedUtcNs: attempt.startedUtcNs,
|
|
4624
|
+
completedUtcNs: attempt.completedUtcNs,
|
|
4625
|
+
observedLatencyUs64: attempt.observedLatencyUs,
|
|
4626
|
+
reportedLatencyUs64: attempt.reportedLatencyUs,
|
|
4627
|
+
isSuccess: attempt.reply.isSuccess,
|
|
4628
|
+
statusCode: normalizeStatusCode(attempt.reply.statusCode, attempt.reply.isSuccess),
|
|
4629
|
+
sizeBytes64: normalizeHistogramInteger(Math.max(toNumber(attempt.reply.sizeBytes), 0), "Scenario response bytes"),
|
|
4630
|
+
steps: attempt.steps
|
|
4631
|
+
}));
|
|
3845
4632
|
};
|
|
3846
|
-
const runBombingInvocation = async (instanceData, instanceNumber, instanceId) => {
|
|
4633
|
+
const runBombingInvocation = async (instanceData, instanceNumber, instanceId, explicitGlobalOrdinal, simulationIndex = -1, simulationKind = "SingleInvocation", explicitSecondaryOrdinal = 0n) => {
|
|
3847
4634
|
if (shouldStopNow()) {
|
|
3848
4635
|
return;
|
|
3849
4636
|
}
|
|
4637
|
+
const globalOrdinal = nextObservationOrdinal(explicitGlobalOrdinal);
|
|
4638
|
+
const identityKind = canonicalLoadEngineV2InvocationIdentityKind(simulationKind);
|
|
4639
|
+
const iterationId = options.loadEngineContractVersion === 2
|
|
4640
|
+
&& explicitGlobalOrdinal !== undefined
|
|
4641
|
+
&& identityKind
|
|
4642
|
+
? buildLoadEngineV2GlobalInvocationId(iterationObservationRunId, scenarioIndex, simulationIndex, identityKind, globalOrdinal, explicitSecondaryOrdinal)
|
|
4643
|
+
: undefined;
|
|
3850
4644
|
let attempts = 0;
|
|
3851
4645
|
const maxAttempts = 1 + (scenario.shouldRestartIterationOnFail() ? restartIterationMaxAttempts : 0);
|
|
3852
4646
|
while (attempts < maxAttempts && !shouldStopNow()) {
|
|
3853
4647
|
attempts += 1;
|
|
3854
|
-
|
|
4648
|
+
let attempt;
|
|
4649
|
+
try {
|
|
4650
|
+
attempt = await runSingleInvocation("Bombing", instanceData, instanceNumber, instanceId, false);
|
|
4651
|
+
}
|
|
4652
|
+
catch (error) {
|
|
4653
|
+
if (error instanceof RuntimePolicyCallbackError) {
|
|
4654
|
+
stopScenario = true;
|
|
4655
|
+
scenarioAbortController.abort(error);
|
|
4656
|
+
}
|
|
4657
|
+
throw error;
|
|
4658
|
+
}
|
|
3855
4659
|
const shouldRetry = !attempt.reply.isSuccess
|
|
3856
4660
|
&& scenario.shouldRestartIterationOnFail()
|
|
3857
4661
|
&& attempts < maxAttempts
|
|
3858
4662
|
&& !shouldStopNow();
|
|
4663
|
+
captureAttemptObservation("Bombing", globalOrdinal, attempts - 1, !shouldRetry, attempt, simulationIndex, simulationKind, iterationId, explicitSecondaryOrdinal);
|
|
3859
4664
|
if (!shouldRetry) {
|
|
4665
|
+
for (const step of attempt.recordedSteps) {
|
|
4666
|
+
recordStepReply(step.stepName, step.reply, step.observedLatencyMs, step.sortIndex);
|
|
4667
|
+
}
|
|
3860
4668
|
recordScenarioReply(attempt.reply, attempt.observedLatencyMs);
|
|
3861
4669
|
if (scenario.getMaxFailCount() > 0 && runtime.allFailCount >= scenario.getMaxFailCount()) {
|
|
3862
4670
|
stopScenario = true;
|
|
@@ -3876,30 +4684,282 @@ async function executeScenarioRuntime(args) {
|
|
|
3876
4684
|
const endMs = Date.now() + Math.trunc(warmUpDurationSeconds * 1000);
|
|
3877
4685
|
const instanceInfo = nextInstanceInfo();
|
|
3878
4686
|
while (Date.now() < endMs && !shouldStopNow()) {
|
|
3879
|
-
|
|
4687
|
+
const globalOrdinal = nextObservationOrdinal();
|
|
4688
|
+
const attempt = await runSingleInvocation("WarmUp", instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, false);
|
|
4689
|
+
captureAttemptObservation("WarmUp", globalOrdinal, 0, true, attempt, -1, "SingleInvocation");
|
|
3880
4690
|
}
|
|
3881
4691
|
};
|
|
3882
|
-
const
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
const
|
|
3887
|
-
const
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
4692
|
+
const executeV2TimedConstant = async (copies, durationNs, ramping, runSimulationInvocation, segment) => {
|
|
4693
|
+
if (!loadEngineV2Budget) {
|
|
4694
|
+
return;
|
|
4695
|
+
}
|
|
4696
|
+
const offsets = ramping ? planRampingConstantDeadlines(copies, durationNs) : Array(copies).fill(0n);
|
|
4697
|
+
const ownedWorkerSlots = trafficMixV2
|
|
4698
|
+
? loadEngineV2TrafficMixOwnedUnits(BigInt(copies), trafficMixV2.shareWeights, trafficMixV2.laneIndex, scenarioPartition.number, scenarioPartition.count)
|
|
4699
|
+
: offsets.flatMap((_offset, workerSlot) => workerSlot % scenarioPartition.count === scenarioPartition.number
|
|
4700
|
+
? [{ laneOrdinal: BigInt(workerSlot), globalRank: BigInt(workerSlot) }]
|
|
4701
|
+
: []);
|
|
4702
|
+
const quantum = ramping ? maxBigInt(1n, durationNs / BigInt(copies)) : 1n;
|
|
4703
|
+
const toleranceNs = minBigInt(100000000n, maxBigInt(2000000n, quantum * 4n));
|
|
4704
|
+
const startNs = process.hrtime.bigint();
|
|
4705
|
+
const endNs = startNs + durationNs;
|
|
4706
|
+
const active = new Set();
|
|
4707
|
+
let schedulerLate = 0;
|
|
4708
|
+
let unavailable = 0;
|
|
4709
|
+
let executionError;
|
|
4710
|
+
if (segment) {
|
|
4711
|
+
segment.requestedWorkers = BigInt(ownedWorkerSlots.length);
|
|
4712
|
+
}
|
|
4713
|
+
for (const unit of ownedWorkerSlots) {
|
|
4714
|
+
const offsetNs = offsets[Number(unit.globalRank)];
|
|
4715
|
+
await delayUntilMonotonicDeadline(startNs + offsetNs, scenarioCancellationToken);
|
|
4716
|
+
if (shouldStopNow()) {
|
|
4717
|
+
break;
|
|
4718
|
+
}
|
|
4719
|
+
const decisionNowNs = process.hrtime.bigint();
|
|
4720
|
+
if (segment)
|
|
4721
|
+
loadEngineV2Telemetry?.recordDecisionLag(segment, decisionNowNs - (startNs + offsetNs));
|
|
4722
|
+
if (decisionNowNs - (startNs + offsetNs) > toleranceNs) {
|
|
4723
|
+
schedulerLate += 1;
|
|
4724
|
+
if (segment) {
|
|
4725
|
+
segment.unavailableWorkers += 1n;
|
|
4726
|
+
incrementReason(segment.unavailableWorkerReasons, "scheduler_late");
|
|
4727
|
+
}
|
|
4728
|
+
continue;
|
|
4729
|
+
}
|
|
4730
|
+
const release = loadEngineV2Budget.tryAcquire();
|
|
4731
|
+
if (!release) {
|
|
4732
|
+
unavailable += 1;
|
|
4733
|
+
if (segment) {
|
|
4734
|
+
segment.unavailableWorkers += 1n;
|
|
4735
|
+
incrementReason(segment.unavailableWorkerReasons, "max_in_flight");
|
|
4736
|
+
}
|
|
4737
|
+
continue;
|
|
4738
|
+
}
|
|
4739
|
+
if (segment)
|
|
4740
|
+
segment.startedWorkers += 1n;
|
|
4741
|
+
const instanceInfo = nextInstanceInfo();
|
|
4742
|
+
let task;
|
|
4743
|
+
task = (async () => {
|
|
4744
|
+
if (segment) {
|
|
4745
|
+
loadEngineV2Telemetry?.recordStartLag(segment, process.hrtime.bigint() - (startNs + offsetNs));
|
|
4746
|
+
}
|
|
4747
|
+
let completedSinceYield = 0;
|
|
4748
|
+
let workerIterationIndex = 0n;
|
|
4749
|
+
while (process.hrtime.bigint() < endNs && !shouldStopNow()) {
|
|
4750
|
+
if (segment) {
|
|
4751
|
+
segment.planned += 1n;
|
|
4752
|
+
segment.due += 1n;
|
|
4753
|
+
segment.started += 1n;
|
|
4754
|
+
}
|
|
4755
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, unit.globalRank, workerIterationIndex);
|
|
4756
|
+
workerIterationIndex += 1n;
|
|
4757
|
+
if (segment)
|
|
4758
|
+
segment.completed += 1n;
|
|
4759
|
+
completedSinceYield += 1;
|
|
4760
|
+
if (completedSinceYield >= 64) {
|
|
4761
|
+
completedSinceYield = 0;
|
|
4762
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
4763
|
+
}
|
|
4764
|
+
}
|
|
4765
|
+
})().catch((error) => {
|
|
4766
|
+
executionError ?? (executionError = error);
|
|
4767
|
+
}).finally(() => {
|
|
4768
|
+
release();
|
|
4769
|
+
active.delete(task);
|
|
4770
|
+
});
|
|
4771
|
+
active.add(task);
|
|
4772
|
+
}
|
|
4773
|
+
while (active.size > 0) {
|
|
4774
|
+
await Promise.race(active);
|
|
4775
|
+
}
|
|
4776
|
+
if (segment) {
|
|
4777
|
+
segment.accountingComplete = segment.completed === segment.started
|
|
4778
|
+
&& segment.started === segment.due
|
|
4779
|
+
&& segment.due === segment.planned
|
|
4780
|
+
&& segment.startedWorkers + segment.unavailableWorkers === segment.requestedWorkers;
|
|
4781
|
+
loadEngineV2Telemetry?.recordWarning("scheduler_late", segment, BigInt(schedulerLate));
|
|
4782
|
+
loadEngineV2Telemetry?.recordWarning("max_in_flight", segment, BigInt(unavailable));
|
|
4783
|
+
}
|
|
4784
|
+
if (schedulerLate > 0) {
|
|
4785
|
+
logger.warn(`Load Engine V2 warning scheduler_late: ${schedulerLate} constant worker slots were unavailable for scenario ${scenario.name}.`);
|
|
4786
|
+
}
|
|
4787
|
+
if (unavailable > 0) {
|
|
4788
|
+
logger.warn(`Load Engine V2 warning max_in_flight: ${unavailable} constant worker slots were unavailable for scenario ${scenario.name}.`);
|
|
4789
|
+
}
|
|
4790
|
+
if (executionError !== undefined) {
|
|
4791
|
+
throw executionError;
|
|
4792
|
+
}
|
|
4793
|
+
};
|
|
4794
|
+
const executeV2IterationsConstant = async (copies, iterations, runSimulationInvocation, segment) => {
|
|
4795
|
+
if (!loadEngineV2Budget || iterations <= 0) {
|
|
4796
|
+
return;
|
|
4797
|
+
}
|
|
4798
|
+
let activeShardCount;
|
|
4799
|
+
let ownedWorkerSlots;
|
|
4800
|
+
let ownedIterations;
|
|
4801
|
+
if (trafficMixV2) {
|
|
4802
|
+
const laneCopies = loadEngineV2TrafficMixLaneUnitCount(BigInt(copies), trafficMixV2.shareWeights, trafficMixV2.laneIndex);
|
|
4803
|
+
const laneIterations = loadEngineV2TrafficMixLaneUnitCount(BigInt(iterations), trafficMixV2.shareWeights, trafficMixV2.laneIndex);
|
|
4804
|
+
if (laneIterations === 0n) {
|
|
4805
|
+
if (segment) {
|
|
4806
|
+
segment.requestedWorkers = 0n;
|
|
4807
|
+
segment.accountingComplete = true;
|
|
4808
|
+
}
|
|
4809
|
+
return;
|
|
4810
|
+
}
|
|
4811
|
+
if (laneCopies === 0n) {
|
|
4812
|
+
throw new Error("Load Engine V2 traffic-mix lane owns iterations but no constant worker slot.");
|
|
4813
|
+
}
|
|
4814
|
+
activeShardCount = Math.min(scenarioPartition.count, Number(laneCopies), Number(laneIterations));
|
|
4815
|
+
if (scenarioPartition.number >= activeShardCount) {
|
|
4816
|
+
if (segment) {
|
|
4817
|
+
segment.requestedWorkers = 0n;
|
|
4818
|
+
segment.accountingComplete = true;
|
|
4819
|
+
}
|
|
4820
|
+
return;
|
|
4821
|
+
}
|
|
4822
|
+
ownedWorkerSlots = loadEngineV2TrafficMixOwnedUnits(BigInt(copies), trafficMixV2.shareWeights, trafficMixV2.laneIndex, scenarioPartition.number, activeShardCount);
|
|
4823
|
+
ownedIterations = loadEngineV2TrafficMixOwnedUnits(BigInt(iterations), trafficMixV2.shareWeights, trafficMixV2.laneIndex, scenarioPartition.number, activeShardCount);
|
|
4824
|
+
}
|
|
4825
|
+
else {
|
|
4826
|
+
activeShardCount = Math.min(scenarioPartition.count, copies, iterations);
|
|
4827
|
+
if (scenarioPartition.number >= activeShardCount) {
|
|
4828
|
+
return;
|
|
4829
|
+
}
|
|
4830
|
+
ownedWorkerSlots = Array.from({ length: copies }, (_value, slot) => slot)
|
|
4831
|
+
.filter((slot) => slot % activeShardCount === scenarioPartition.number)
|
|
4832
|
+
.map((slot) => ({ laneOrdinal: BigInt(slot), globalRank: BigInt(slot) }));
|
|
4833
|
+
ownedIterations = Array.from({ length: iterations }, (_value, ordinal) => ordinal)
|
|
4834
|
+
.filter((ordinal) => ordinal % activeShardCount === scenarioPartition.number)
|
|
4835
|
+
.map((ordinal) => ({ laneOrdinal: BigInt(ordinal), globalRank: BigInt(ordinal) }));
|
|
4836
|
+
}
|
|
4837
|
+
if (scenarioPartition.number >= activeShardCount) {
|
|
4838
|
+
return;
|
|
4839
|
+
}
|
|
4840
|
+
const targetWorkers = Math.min(ownedWorkerSlots.length, ownedIterations.length);
|
|
4841
|
+
if (targetWorkers <= 0) {
|
|
4842
|
+
if (segment) {
|
|
4843
|
+
segment.requestedWorkers = 0n;
|
|
4844
|
+
segment.accountingComplete = true;
|
|
4845
|
+
}
|
|
4846
|
+
return;
|
|
4847
|
+
}
|
|
4848
|
+
if (segment) {
|
|
4849
|
+
segment.planned = BigInt(ownedIterations.length);
|
|
4850
|
+
segment.requestedWorkers = BigInt(targetWorkers);
|
|
4851
|
+
}
|
|
4852
|
+
const releases = [];
|
|
4853
|
+
let firstRelease;
|
|
4854
|
+
while (!firstRelease && !shouldStopNow()) {
|
|
4855
|
+
firstRelease = loadEngineV2Budget.tryAcquire();
|
|
4856
|
+
if (!firstRelease) {
|
|
4857
|
+
await delayWithAbort(1, scenarioCancellationToken);
|
|
4858
|
+
}
|
|
4859
|
+
}
|
|
4860
|
+
if (!firstRelease) {
|
|
4861
|
+
if (segment) {
|
|
4862
|
+
segment.unreached = segment.planned;
|
|
4863
|
+
segment.unavailableWorkers = segment.requestedWorkers;
|
|
4864
|
+
if (segment.unavailableWorkers > 0n) {
|
|
4865
|
+
incrementReason(segment.unavailableWorkerReasons, "cancelled", segment.unavailableWorkers);
|
|
4866
|
+
}
|
|
4867
|
+
segment.accountingComplete = true;
|
|
4868
|
+
}
|
|
4869
|
+
return;
|
|
4870
|
+
}
|
|
4871
|
+
releases.push(firstRelease);
|
|
4872
|
+
for (let worker = 1; worker < targetWorkers; worker += 1) {
|
|
4873
|
+
const release = loadEngineV2Budget.tryAcquire();
|
|
4874
|
+
if (release) {
|
|
4875
|
+
releases.push(release);
|
|
4876
|
+
}
|
|
4877
|
+
}
|
|
4878
|
+
if (releases.length < targetWorkers) {
|
|
4879
|
+
logger.warn(`Load Engine V2 warning max_in_flight: requested ${targetWorkers} constant workers but started ${releases.length} for scenario ${scenario.name}; all iterations will still run.`);
|
|
4880
|
+
}
|
|
4881
|
+
if (segment) {
|
|
4882
|
+
segment.startedWorkers = BigInt(releases.length);
|
|
4883
|
+
segment.unavailableWorkers = BigInt(targetWorkers - releases.length);
|
|
4884
|
+
if (segment.unavailableWorkers > 0n) {
|
|
4885
|
+
incrementReason(segment.unavailableWorkerReasons, "max_in_flight", segment.unavailableWorkers);
|
|
4886
|
+
loadEngineV2Telemetry?.recordWarning("max_in_flight", segment, segment.unavailableWorkers);
|
|
4887
|
+
}
|
|
4888
|
+
}
|
|
4889
|
+
let nextIterationIndex = 0;
|
|
4890
|
+
const tasks = releases.map(async (release) => {
|
|
4891
|
+
const instanceInfo = nextInstanceInfo();
|
|
4892
|
+
try {
|
|
4893
|
+
while (!shouldStopNow()) {
|
|
4894
|
+
const unit = ownedIterations[nextIterationIndex];
|
|
4895
|
+
nextIterationIndex += 1;
|
|
4896
|
+
if (!unit) {
|
|
4897
|
+
break;
|
|
4898
|
+
}
|
|
4899
|
+
if (segment) {
|
|
4900
|
+
segment.due += 1n;
|
|
4901
|
+
segment.started += 1n;
|
|
4902
|
+
}
|
|
4903
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, unit.globalRank, 0n);
|
|
4904
|
+
if (segment)
|
|
4905
|
+
segment.completed += 1n;
|
|
4906
|
+
}
|
|
4907
|
+
}
|
|
4908
|
+
finally {
|
|
4909
|
+
release();
|
|
4910
|
+
}
|
|
4911
|
+
});
|
|
4912
|
+
await waitForScenarioTasks(tasks, scenario.name, scenarioCompletionTimeoutSeconds, logger, scenarioCancellationToken);
|
|
4913
|
+
if (segment) {
|
|
4914
|
+
segment.unreached = segment.planned > segment.due ? segment.planned - segment.due : 0n;
|
|
4915
|
+
segment.accountingComplete = segment.due + segment.unreached === segment.planned
|
|
4916
|
+
&& segment.started === segment.due
|
|
4917
|
+
&& segment.completed === segment.started
|
|
4918
|
+
&& segment.startedWorkers + segment.unavailableWorkers === segment.requestedWorkers;
|
|
4919
|
+
}
|
|
4920
|
+
};
|
|
4921
|
+
const executeSimulationAsync = async (simulation, simulationIndex) => {
|
|
4922
|
+
const descriptor = trafficMixV2
|
|
4923
|
+
? trafficMixV2.originalSimulations[simulationIndex]
|
|
4924
|
+
: simulation;
|
|
4925
|
+
if (!descriptor) {
|
|
4926
|
+
throw new Error(`Load Engine V2 traffic-mix phase ${simulationIndex} is missing its global descriptor.`);
|
|
4927
|
+
}
|
|
4928
|
+
const kind = String(descriptor.Kind ?? "");
|
|
4929
|
+
const simulationKind = kind || "SingleInvocation";
|
|
4930
|
+
const runSimulationInvocation = (instanceData, instanceNumber, instanceId, explicitGlobalOrdinal, explicitSecondaryOrdinal = 0n) => runBombingInvocation(instanceData, instanceNumber, instanceId, explicitGlobalOrdinal, simulationIndex, simulationKind, explicitSecondaryOrdinal);
|
|
4931
|
+
if (options.loadEngineContractVersion === 2) {
|
|
4932
|
+
nextV2FallbackOrdinal = BigInt(scenarioPartition.number);
|
|
4933
|
+
}
|
|
4934
|
+
const weight = trafficMixV2 ? 1 : Math.max(scenario.getWeight(), 1);
|
|
4935
|
+
const rate = applyScenarioWeight(toInt(descriptor.Rate), weight);
|
|
4936
|
+
const minRate = applyScenarioWeight(toInt(descriptor.MinRate), weight);
|
|
4937
|
+
const maxRate = applyScenarioWeight(toInt(descriptor.MaxRate), weight);
|
|
4938
|
+
const copies = Math.max(applyScenarioWeight(Math.max(toInt(descriptor.Copies), 1), weight), 1);
|
|
4939
|
+
const iterations = applyScenarioWeight(Math.max(toInt(descriptor.Iterations), 0), weight);
|
|
4940
|
+
const intervalMs = Math.max(Math.trunc(Math.max(toNumber(descriptor.IntervalSeconds), 0) * 1000), 0);
|
|
4941
|
+
const duringMs = Math.max(Math.trunc(Math.max(toNumber(descriptor.DuringSeconds), 0) * 1000), 0);
|
|
4942
|
+
accumulator.setLoadSimulation(kind, resolveLoadSimulationValue(descriptor, weight));
|
|
3893
4943
|
accumulator.setCurrentOperation("Bombing");
|
|
4944
|
+
const schedulerSegment = options.loadEngineContractVersion === 2
|
|
4945
|
+
? loadEngineV2Telemetry?.createSegment(scenario.name, scenarioIndex, simulationIndex, kind, scenarioPartition.number, scenarioPartition.count)
|
|
4946
|
+
: undefined;
|
|
4947
|
+
const trafficMixOwnedOrdinals = (total) => trafficMixV2
|
|
4948
|
+
? loadEngineV2TrafficMixOwnedUnits(total, trafficMixV2.shareWeights, trafficMixV2.laneIndex, scenarioPartition.number, scenarioPartition.count).map((unit) => unit.globalRank)
|
|
4949
|
+
: undefined;
|
|
3894
4950
|
if (kind === "KeepConstant") {
|
|
3895
4951
|
if (duringMs <= 0) {
|
|
3896
4952
|
return;
|
|
3897
4953
|
}
|
|
4954
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
4955
|
+
await executeV2TimedConstant(copies, secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "KeepConstant duration"), false, runSimulationInvocation, schedulerSegment);
|
|
4956
|
+
return;
|
|
4957
|
+
}
|
|
3898
4958
|
const endMs = Date.now() + duringMs;
|
|
3899
4959
|
const tasks = Array.from({ length: copies }, async () => {
|
|
3900
4960
|
const instanceInfo = nextInstanceInfo();
|
|
3901
4961
|
while (Date.now() < endMs && !shouldStopNow()) {
|
|
3902
|
-
await
|
|
4962
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId);
|
|
3903
4963
|
}
|
|
3904
4964
|
});
|
|
3905
4965
|
await waitForScenarioTasks(tasks, scenario.name, scenarioCompletionTimeoutSeconds, logger, scenarioCancellationToken);
|
|
@@ -3909,6 +4969,10 @@ async function executeScenarioRuntime(args) {
|
|
|
3909
4969
|
if (duringMs <= 0) {
|
|
3910
4970
|
return;
|
|
3911
4971
|
}
|
|
4972
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
4973
|
+
await executeV2TimedConstant(copies, secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "RampingConstant duration"), true, runSimulationInvocation, schedulerSegment);
|
|
4974
|
+
return;
|
|
4975
|
+
}
|
|
3912
4976
|
const tasks = [];
|
|
3913
4977
|
const endMs = Date.now() + duringMs;
|
|
3914
4978
|
const startIntervalMs = copies <= 1 ? 0 : Math.max(Math.trunc(duringMs / copies), 0);
|
|
@@ -3916,7 +4980,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3916
4980
|
const instanceInfo = nextInstanceInfo();
|
|
3917
4981
|
tasks.push((async () => {
|
|
3918
4982
|
while (Date.now() < endMs && !shouldStopNow()) {
|
|
3919
|
-
await
|
|
4983
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId);
|
|
3920
4984
|
}
|
|
3921
4985
|
})());
|
|
3922
4986
|
if (copy < copies && startIntervalMs > 0) {
|
|
@@ -3930,12 +4994,34 @@ async function executeScenarioRuntime(args) {
|
|
|
3930
4994
|
if (duringMs <= 0 || rate <= 0 || intervalMs <= 0) {
|
|
3931
4995
|
return;
|
|
3932
4996
|
}
|
|
4997
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
4998
|
+
const intervalNs = secondsToNanoseconds(toNumber(descriptor.IntervalSeconds), "Inject interval");
|
|
4999
|
+
const durationNs = secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "Inject duration");
|
|
5000
|
+
await executeV2FixedArrivals({
|
|
5001
|
+
rate,
|
|
5002
|
+
intervalNs,
|
|
5003
|
+
totalArrivals: loadEngineV2FixedArrivalCount(rate, intervalNs, durationNs),
|
|
5004
|
+
budget: loadEngineV2Budget,
|
|
5005
|
+
cancellationToken: scenarioCancellationToken,
|
|
5006
|
+
shouldStopNow,
|
|
5007
|
+
nextInstanceInfo,
|
|
5008
|
+
runBombingInvocation: runSimulationInvocation,
|
|
5009
|
+
logger,
|
|
5010
|
+
scenarioName: scenario.name,
|
|
5011
|
+
shardIndex: scenarioPartition.number,
|
|
5012
|
+
shardCount: scenarioPartition.count,
|
|
5013
|
+
ownedOrdinals: trafficMixOwnedOrdinals(loadEngineV2FixedArrivalCount(rate, intervalNs, durationNs)),
|
|
5014
|
+
telemetry: loadEngineV2Telemetry,
|
|
5015
|
+
segment: schedulerSegment
|
|
5016
|
+
});
|
|
5017
|
+
return;
|
|
5018
|
+
}
|
|
3933
5019
|
const pending = [];
|
|
3934
5020
|
const endMs = Date.now() + duringMs;
|
|
3935
5021
|
while (Date.now() < endMs && !shouldStopNow()) {
|
|
3936
5022
|
for (let index = 0; index < rate; index += 1) {
|
|
3937
5023
|
const instanceInfo = nextInstanceInfo();
|
|
3938
|
-
pending.push(
|
|
5024
|
+
pending.push(runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId));
|
|
3939
5025
|
}
|
|
3940
5026
|
await delayWithAbort(intervalMs, scenarioCancellationToken);
|
|
3941
5027
|
}
|
|
@@ -3946,6 +5032,31 @@ async function executeScenarioRuntime(args) {
|
|
|
3946
5032
|
if (duringMs <= 0 || rate <= 0 || intervalMs <= 0) {
|
|
3947
5033
|
return;
|
|
3948
5034
|
}
|
|
5035
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5036
|
+
const intervalNs = secondsToNanoseconds(toNumber(descriptor.IntervalSeconds), "RampingInject interval");
|
|
5037
|
+
const durationNs = secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "RampingInject duration");
|
|
5038
|
+
const offsets = planRampingInjectionDeadlines(rate, intervalNs, durationNs);
|
|
5039
|
+
await executeV2FixedArrivals({
|
|
5040
|
+
rate,
|
|
5041
|
+
intervalNs,
|
|
5042
|
+
totalArrivals: BigInt(offsets.length),
|
|
5043
|
+
deadlineOffsetsNs: offsets,
|
|
5044
|
+
tolerancesNs: buildV2RampTolerances(offsets, durationNs),
|
|
5045
|
+
budget: loadEngineV2Budget,
|
|
5046
|
+
cancellationToken: scenarioCancellationToken,
|
|
5047
|
+
shouldStopNow,
|
|
5048
|
+
nextInstanceInfo,
|
|
5049
|
+
runBombingInvocation: runSimulationInvocation,
|
|
5050
|
+
logger,
|
|
5051
|
+
scenarioName: scenario.name,
|
|
5052
|
+
shardIndex: scenarioPartition.number,
|
|
5053
|
+
shardCount: scenarioPartition.count,
|
|
5054
|
+
ownedOrdinals: trafficMixOwnedOrdinals(BigInt(offsets.length)),
|
|
5055
|
+
telemetry: loadEngineV2Telemetry,
|
|
5056
|
+
segment: schedulerSegment
|
|
5057
|
+
});
|
|
5058
|
+
return;
|
|
5059
|
+
}
|
|
3949
5060
|
const pending = [];
|
|
3950
5061
|
const startedAtMs = Date.now();
|
|
3951
5062
|
const endMs = startedAtMs + duringMs;
|
|
@@ -3955,7 +5066,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3955
5066
|
const currentRate = Math.max(1, Math.ceil(rate * progress));
|
|
3956
5067
|
for (let index = 0; index < currentRate; index += 1) {
|
|
3957
5068
|
const instanceInfo = nextInstanceInfo();
|
|
3958
|
-
pending.push(
|
|
5069
|
+
pending.push(runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId));
|
|
3959
5070
|
}
|
|
3960
5071
|
await delayWithAbort(intervalMs, scenarioCancellationToken);
|
|
3961
5072
|
}
|
|
@@ -3966,6 +5077,37 @@ async function executeScenarioRuntime(args) {
|
|
|
3966
5077
|
if (duringMs <= 0 || maxRate <= 0 || intervalMs <= 0) {
|
|
3967
5078
|
return;
|
|
3968
5079
|
}
|
|
5080
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5081
|
+
const intervalNs = secondsToNanoseconds(toNumber(descriptor.IntervalSeconds), "InjectRandom interval");
|
|
5082
|
+
const durationNs = secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "InjectRandom duration");
|
|
5083
|
+
const descriptorSeed = fnv1a32(trafficMixV2
|
|
5084
|
+
? `traffic-mix\n${trafficMixV2.seedId}\n${simulationIndex}`
|
|
5085
|
+
: `${scenario.name}\n${simulationIndex}`);
|
|
5086
|
+
const rotated = ((descriptorSeed << 13) | (descriptorSeed >>> 19)) >>> 0;
|
|
5087
|
+
const seed = (fnv1a32(testInfo.sessionId) ^ rotated) >>> 0;
|
|
5088
|
+
const offsets = planRandomInjectionDeadlines(Math.max(0, minRate), maxRate, intervalNs, durationNs, seed);
|
|
5089
|
+
const tolerance = loadEngineV2LatenessToleranceNs(Math.max(1, maxRate), intervalNs);
|
|
5090
|
+
await executeV2FixedArrivals({
|
|
5091
|
+
rate: Math.max(1, maxRate),
|
|
5092
|
+
intervalNs,
|
|
5093
|
+
totalArrivals: BigInt(offsets.length),
|
|
5094
|
+
deadlineOffsetsNs: offsets,
|
|
5095
|
+
tolerancesNs: offsets.map(() => tolerance),
|
|
5096
|
+
budget: loadEngineV2Budget,
|
|
5097
|
+
cancellationToken: scenarioCancellationToken,
|
|
5098
|
+
shouldStopNow,
|
|
5099
|
+
nextInstanceInfo,
|
|
5100
|
+
runBombingInvocation: runSimulationInvocation,
|
|
5101
|
+
logger,
|
|
5102
|
+
scenarioName: scenario.name,
|
|
5103
|
+
shardIndex: scenarioPartition.number,
|
|
5104
|
+
shardCount: scenarioPartition.count,
|
|
5105
|
+
ownedOrdinals: trafficMixOwnedOrdinals(BigInt(offsets.length)),
|
|
5106
|
+
telemetry: loadEngineV2Telemetry,
|
|
5107
|
+
segment: schedulerSegment
|
|
5108
|
+
});
|
|
5109
|
+
return;
|
|
5110
|
+
}
|
|
3969
5111
|
const pending = [];
|
|
3970
5112
|
const normalizedMinRate = Math.max(1, minRate);
|
|
3971
5113
|
const normalizedMaxRate = Math.max(normalizedMinRate, maxRate);
|
|
@@ -3974,7 +5116,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3974
5116
|
const currentRate = randomIntInclusive(normalizedMinRate, normalizedMaxRate);
|
|
3975
5117
|
for (let index = 0; index < currentRate; index += 1) {
|
|
3976
5118
|
const instanceInfo = nextInstanceInfo();
|
|
3977
|
-
pending.push(
|
|
5119
|
+
pending.push(runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId));
|
|
3978
5120
|
}
|
|
3979
5121
|
await delayWithAbort(intervalMs, scenarioCancellationToken);
|
|
3980
5122
|
}
|
|
@@ -3985,6 +5127,10 @@ async function executeScenarioRuntime(args) {
|
|
|
3985
5127
|
if (iterations <= 0) {
|
|
3986
5128
|
return;
|
|
3987
5129
|
}
|
|
5130
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5131
|
+
await executeV2IterationsConstant(copies, iterations, runSimulationInvocation, schedulerSegment);
|
|
5132
|
+
return;
|
|
5133
|
+
}
|
|
3988
5134
|
let remaining = iterations;
|
|
3989
5135
|
const tasks = Array.from({ length: copies }, async () => {
|
|
3990
5136
|
const instanceInfo = nextInstanceInfo();
|
|
@@ -3993,7 +5139,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3993
5139
|
if (remaining < 0) {
|
|
3994
5140
|
break;
|
|
3995
5141
|
}
|
|
3996
|
-
await
|
|
5142
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId);
|
|
3997
5143
|
}
|
|
3998
5144
|
});
|
|
3999
5145
|
await waitForScenarioTasks(tasks, scenario.name, scenarioCompletionTimeoutSeconds, logger, scenarioCancellationToken);
|
|
@@ -4003,13 +5149,33 @@ async function executeScenarioRuntime(args) {
|
|
|
4003
5149
|
if (iterations <= 0 || rate <= 0 || intervalMs <= 0) {
|
|
4004
5150
|
return;
|
|
4005
5151
|
}
|
|
5152
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5153
|
+
await executeV2FixedArrivals({
|
|
5154
|
+
rate,
|
|
5155
|
+
intervalNs: secondsToNanoseconds(toNumber(descriptor.IntervalSeconds), "IterationsForInject interval"),
|
|
5156
|
+
totalArrivals: BigInt(iterations),
|
|
5157
|
+
budget: loadEngineV2Budget,
|
|
5158
|
+
cancellationToken: scenarioCancellationToken,
|
|
5159
|
+
shouldStopNow,
|
|
5160
|
+
nextInstanceInfo,
|
|
5161
|
+
runBombingInvocation: runSimulationInvocation,
|
|
5162
|
+
logger,
|
|
5163
|
+
scenarioName: scenario.name,
|
|
5164
|
+
shardIndex: scenarioPartition.number,
|
|
5165
|
+
shardCount: scenarioPartition.count,
|
|
5166
|
+
ownedOrdinals: trafficMixOwnedOrdinals(BigInt(iterations)),
|
|
5167
|
+
telemetry: loadEngineV2Telemetry,
|
|
5168
|
+
segment: schedulerSegment
|
|
5169
|
+
});
|
|
5170
|
+
return;
|
|
5171
|
+
}
|
|
4006
5172
|
const pending = [];
|
|
4007
5173
|
let remaining = iterations;
|
|
4008
5174
|
while (remaining > 0 && !shouldStopNow()) {
|
|
4009
5175
|
const count = Math.min(rate, remaining);
|
|
4010
5176
|
for (let index = 0; index < count; index += 1) {
|
|
4011
5177
|
const instanceInfo = nextInstanceInfo();
|
|
4012
|
-
pending.push(
|
|
5178
|
+
pending.push(runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId));
|
|
4013
5179
|
}
|
|
4014
5180
|
remaining -= count;
|
|
4015
5181
|
if (remaining > 0) {
|
|
@@ -4021,10 +5187,19 @@ async function executeScenarioRuntime(args) {
|
|
|
4021
5187
|
}
|
|
4022
5188
|
if (kind === "Pause") {
|
|
4023
5189
|
if (duringMs > 0) {
|
|
4024
|
-
|
|
5190
|
+
if (options.loadEngineContractVersion === 2) {
|
|
5191
|
+
await delayUntilMonotonicDeadline(process.hrtime.bigint() + secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "Pause duration"), scenarioCancellationToken);
|
|
5192
|
+
}
|
|
5193
|
+
else {
|
|
5194
|
+
await delayWithAbort(duringMs, scenarioCancellationToken);
|
|
5195
|
+
}
|
|
4025
5196
|
}
|
|
5197
|
+
if (schedulerSegment)
|
|
5198
|
+
schedulerSegment.accountingComplete = true;
|
|
4026
5199
|
return;
|
|
4027
5200
|
}
|
|
5201
|
+
if (schedulerSegment)
|
|
5202
|
+
schedulerSegment.accountingComplete = true;
|
|
4028
5203
|
};
|
|
4029
5204
|
const initContext = {
|
|
4030
5205
|
customSettings: { ...(options.customSettings ?? {}) },
|
|
@@ -4056,15 +5231,33 @@ async function executeScenarioRuntime(args) {
|
|
|
4056
5231
|
accumulator.setCurrentOperation("Bombing");
|
|
4057
5232
|
const simulations = scenario.getSimulations();
|
|
4058
5233
|
if (!simulations.length) {
|
|
4059
|
-
|
|
4060
|
-
|
|
5234
|
+
if (options.loadEngineContractVersion === 2) {
|
|
5235
|
+
await options.loadEngineV2SegmentLifecycleOverride?.beforeSegment(scenarioIndex, 0);
|
|
5236
|
+
try {
|
|
5237
|
+
await executeSimulationAsync(LoadStrikeSimulation.iterationsForConstant(1, 1), 0);
|
|
5238
|
+
}
|
|
5239
|
+
finally {
|
|
5240
|
+
await options.loadEngineV2SegmentLifecycleOverride?.afterSegment(scenarioIndex, 0);
|
|
5241
|
+
}
|
|
5242
|
+
}
|
|
5243
|
+
else {
|
|
5244
|
+
const instanceInfo = nextInstanceInfo();
|
|
5245
|
+
await runBombingInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId);
|
|
5246
|
+
}
|
|
4061
5247
|
}
|
|
4062
5248
|
else {
|
|
4063
|
-
for (
|
|
5249
|
+
for (let simulationIndex = 0; simulationIndex < simulations.length; simulationIndex += 1) {
|
|
5250
|
+
const simulation = simulations[simulationIndex];
|
|
4064
5251
|
if (shouldStopNow()) {
|
|
4065
5252
|
break;
|
|
4066
5253
|
}
|
|
4067
|
-
await
|
|
5254
|
+
await options.loadEngineV2SegmentLifecycleOverride?.beforeSegment(scenarioIndex, simulationIndex);
|
|
5255
|
+
try {
|
|
5256
|
+
await executeSimulationAsync(simulation, simulationIndex);
|
|
5257
|
+
}
|
|
5258
|
+
finally {
|
|
5259
|
+
await options.loadEngineV2SegmentLifecycleOverride?.afterSegment(scenarioIndex, simulationIndex);
|
|
5260
|
+
}
|
|
4068
5261
|
}
|
|
4069
5262
|
}
|
|
4070
5263
|
accumulator.setCurrentOperation(stopScenario || stopTestState.value ? "Stop" : "Complete");
|
|
@@ -4153,18 +5346,29 @@ async function waitForScenarioTasks(tasks, scenarioName, timeoutSeconds, logger,
|
|
|
4153
5346
|
if (!tasks.length) {
|
|
4154
5347
|
return;
|
|
4155
5348
|
}
|
|
4156
|
-
const
|
|
5349
|
+
const settled = Promise.allSettled(tasks);
|
|
5350
|
+
const throwPolicyFailure = (results) => {
|
|
5351
|
+
const failure = results.find((result) => result.status === "rejected" && result.reason instanceof RuntimePolicyCallbackError);
|
|
5352
|
+
if (failure) {
|
|
5353
|
+
throw failure.reason;
|
|
5354
|
+
}
|
|
5355
|
+
};
|
|
4157
5356
|
if (timeoutSeconds <= 0) {
|
|
4158
|
-
await
|
|
5357
|
+
throwPolicyFailure(await settled);
|
|
4159
5358
|
return;
|
|
4160
5359
|
}
|
|
4161
5360
|
const completed = await Promise.race([
|
|
4162
|
-
|
|
5361
|
+
settled.then(() => true),
|
|
4163
5362
|
delayWithAbort(Math.trunc(timeoutSeconds * 1000), signal).then(() => false)
|
|
4164
5363
|
]);
|
|
4165
5364
|
if (!completed) {
|
|
5365
|
+
if (signal.reason instanceof RuntimePolicyCallbackError) {
|
|
5366
|
+
throw signal.reason;
|
|
5367
|
+
}
|
|
4166
5368
|
logger.warn(`Scenario ${scenarioName} timed out while waiting for completion (${timeoutSeconds}s).`);
|
|
5369
|
+
return;
|
|
4167
5370
|
}
|
|
5371
|
+
throwPolicyFailure(await settled);
|
|
4168
5372
|
}
|
|
4169
5373
|
function randomIntInclusive(minValue, maxValue) {
|
|
4170
5374
|
const min = Math.trunc(Math.min(minValue, maxValue));
|
|
@@ -4391,6 +5595,9 @@ function normalizeDataTransferStatsValue(value) {
|
|
|
4391
5595
|
const source = asAliasRecord(value);
|
|
4392
5596
|
return {
|
|
4393
5597
|
allBytes: pickAliasNumber(source, "allBytes", "AllBytes"),
|
|
5598
|
+
...(hasAliasValue(source, "allBytes64", "AllBytes64")
|
|
5599
|
+
? { allBytes64: pickAliasString(source, "allBytes64", "AllBytes64") }
|
|
5600
|
+
: {}),
|
|
4394
5601
|
maxBytes: pickAliasNumber(source, "maxBytes", "MaxBytes"),
|
|
4395
5602
|
meanBytes: pickAliasNumber(source, "meanBytes", "MeanBytes"),
|
|
4396
5603
|
minBytes: pickAliasNumber(source, "minBytes", "MinBytes"),
|
|
@@ -4398,6 +5605,9 @@ function normalizeDataTransferStatsValue(value) {
|
|
|
4398
5605
|
percent75: pickAliasNumber(source, "percent75", "Percent75"),
|
|
4399
5606
|
percent95: pickAliasNumber(source, "percent95", "Percent95"),
|
|
4400
5607
|
percent99: pickAliasNumber(source, "percent99", "Percent99"),
|
|
5608
|
+
...(hasAliasValue(source, "percent100", "Percent100")
|
|
5609
|
+
? { percent100: pickAliasNumber(source, "percent100", "Percent100") }
|
|
5610
|
+
: {}),
|
|
4401
5611
|
stdDev: pickAliasNumber(source, "stdDev", "StdDev")
|
|
4402
5612
|
};
|
|
4403
5613
|
}
|
|
@@ -4420,6 +5630,9 @@ function normalizeLatencyStatsValue(value) {
|
|
|
4420
5630
|
percent75: pickAliasNumber(source, "percent75", "Percent75"),
|
|
4421
5631
|
percent95: pickAliasNumber(source, "percent95", "Percent95"),
|
|
4422
5632
|
percent99: pickAliasNumber(source, "percent99", "Percent99"),
|
|
5633
|
+
...(hasAliasValue(source, "percent100", "Percent100")
|
|
5634
|
+
? { percent100: pickAliasNumber(source, "percent100", "Percent100") }
|
|
5635
|
+
: {}),
|
|
4423
5636
|
stdDev: pickAliasNumber(source, "stdDev", "StdDev")
|
|
4424
5637
|
};
|
|
4425
5638
|
}
|
|
@@ -4435,13 +5648,42 @@ function normalizeStatusCodeStatsValue(value) {
|
|
|
4435
5648
|
}
|
|
4436
5649
|
function normalizeMeasurementStatsValue(value) {
|
|
4437
5650
|
const source = asAliasRecord(value);
|
|
4438
|
-
|
|
5651
|
+
const projected = {
|
|
5652
|
+
...(hasAliasValue(source, "count64", "Count64")
|
|
5653
|
+
? { count64: pickAliasString(source, "count64", "Count64") }
|
|
5654
|
+
: {}),
|
|
5655
|
+
...(hasAliasValue(source, "distributionMode", "DistributionMode")
|
|
5656
|
+
? { distributionMode: pickAliasString(source, "distributionMode", "DistributionMode") }
|
|
5657
|
+
: {}),
|
|
5658
|
+
...(hasAliasValue(source, "maxRelativeError", "MaxRelativeError")
|
|
5659
|
+
? { maxRelativeError: pickAliasNumber(source, "maxRelativeError", "MaxRelativeError") }
|
|
5660
|
+
: {}),
|
|
4439
5661
|
dataTransfer: normalizeDataTransferStatsValue(pickAliasValue(source, "dataTransfer", "DataTransfer")),
|
|
4440
5662
|
latency: normalizeLatencyStatsValue(pickAliasValue(source, "latency", "Latency")),
|
|
4441
5663
|
request: normalizeRequestStatsValue(pickAliasValue(source, "request", "Request")),
|
|
4442
5664
|
statusCodes: pickAliasArray(source, "statusCodes", "StatusCodes")
|
|
4443
5665
|
.map((entry) => normalizeStatusCodeStatsValue(entry))
|
|
4444
5666
|
};
|
|
5667
|
+
const sidecarValue = pickAliasValue(source, "histogramSidecar", "HistogramSidecar");
|
|
5668
|
+
if (sidecarValue && typeof sidecarValue === "object" && !Array.isArray(sidecarValue)) {
|
|
5669
|
+
const sidecar = sidecarValue;
|
|
5670
|
+
if (sidecar?.latency && sidecar.size) {
|
|
5671
|
+
Object.defineProperty(projected, "histogramSidecar", {
|
|
5672
|
+
value: {
|
|
5673
|
+
latency: LoadStrikeHistogramV1.fromSidecar(sidecar.latency).toSidecar(),
|
|
5674
|
+
size: LoadStrikeHistogramV1.fromSidecar(sidecar.size).toSidecar(),
|
|
5675
|
+
allBytes64: String(sidecar.allBytes64 ?? "0"),
|
|
5676
|
+
lessOrEq80064: String(sidecar.lessOrEq80064 ?? "0"),
|
|
5677
|
+
more800Less120064: String(sidecar.more800Less120064 ?? "0"),
|
|
5678
|
+
moreOrEq120064: String(sidecar.moreOrEq120064 ?? "0")
|
|
5679
|
+
},
|
|
5680
|
+
enumerable: false,
|
|
5681
|
+
configurable: false,
|
|
5682
|
+
writable: false
|
|
5683
|
+
});
|
|
5684
|
+
}
|
|
5685
|
+
}
|
|
5686
|
+
return projected;
|
|
4445
5687
|
}
|
|
4446
5688
|
function normalizeLoadSimulationStatsValue(value) {
|
|
4447
5689
|
const source = asAliasRecord(value);
|
|
@@ -4541,6 +5783,9 @@ function normalizeStepStatsValue(value, index = 0) {
|
|
|
4541
5783
|
statusCodes: normalizeAliasNumberRecord(pickAliasValue(source, "statusCodes", "StatusCodes")),
|
|
4542
5784
|
ok: normalizeMeasurementStatsValue(pickAliasValue(source, "ok", "Ok")),
|
|
4543
5785
|
fail: normalizeMeasurementStatsValue(pickAliasValue(source, "fail", "Fail")),
|
|
5786
|
+
...(hasAliasValue(source, "allMeasurement", "AllMeasurement")
|
|
5787
|
+
? { allMeasurement: normalizeMeasurementStatsValue(pickAliasValue(source, "allMeasurement", "AllMeasurement")) }
|
|
5788
|
+
: {}),
|
|
4544
5789
|
sortIndex: hasAliasValue(source, "sortIndex", "SortIndex")
|
|
4545
5790
|
? pickAliasNumber(source, "sortIndex", "SortIndex")
|
|
4546
5791
|
: index
|
|
@@ -4580,6 +5825,9 @@ function normalizeScenarioStatsValue(value, index = 0) {
|
|
|
4580
5825
|
durationMs: pickAliasNumber(source, "durationMs", "DurationMs", "Duration"),
|
|
4581
5826
|
ok: normalizeMeasurementStatsValue(pickAliasValue(source, "ok", "Ok")),
|
|
4582
5827
|
fail: normalizeMeasurementStatsValue(pickAliasValue(source, "fail", "Fail")),
|
|
5828
|
+
...(hasAliasValue(source, "allMeasurement", "AllMeasurement")
|
|
5829
|
+
? { allMeasurement: normalizeMeasurementStatsValue(pickAliasValue(source, "allMeasurement", "AllMeasurement")) }
|
|
5830
|
+
: {}),
|
|
4583
5831
|
loadSimulationStats: normalizeLoadSimulationStatsValue(pickAliasValue(source, "loadSimulationStats", "LoadSimulationStats")),
|
|
4584
5832
|
sortIndex: hasAliasValue(source, "sortIndex", "SortIndex")
|
|
4585
5833
|
? pickAliasNumber(source, "sortIndex", "SortIndex")
|
|
@@ -4666,7 +5914,7 @@ function attachSessionStartInfoAliases(session) {
|
|
|
4666
5914
|
return session;
|
|
4667
5915
|
}
|
|
4668
5916
|
function attachPortalReportingSession(sinkSession, sessionInfo, licenseClient, licenseSession) {
|
|
4669
|
-
const runToken =
|
|
5917
|
+
const runToken = currentLicenseSessionRunToken(licenseSession);
|
|
4670
5918
|
if (!runToken || !licenseClient) {
|
|
4671
5919
|
return;
|
|
4672
5920
|
}
|
|
@@ -4676,9 +5924,17 @@ function attachPortalReportingSession(sinkSession, sessionInfo, licenseClient, l
|
|
|
4676
5924
|
sinkSession.portalReportingIngestUrl = ingestUrl;
|
|
4677
5925
|
sinkSession.portalReportingRunId = runId;
|
|
4678
5926
|
sessionInfo.runToken = runToken;
|
|
5927
|
+
Object.defineProperty(sessionInfo, PORTAL_RUN_TOKEN_PROVIDER, {
|
|
5928
|
+
configurable: true,
|
|
5929
|
+
enumerable: false,
|
|
5930
|
+
value: () => currentLicenseSessionRunToken(licenseSession)
|
|
5931
|
+
});
|
|
4679
5932
|
sessionInfo.portalReportingIngestUrl = ingestUrl;
|
|
4680
5933
|
sessionInfo.portalReportingRunId = runId;
|
|
4681
5934
|
}
|
|
5935
|
+
function currentLicenseSessionRunToken(licenseSession) {
|
|
5936
|
+
return stringValueOrDefault(licenseSession?.runToken, "").trim();
|
|
5937
|
+
}
|
|
4682
5938
|
function buildPortalReportingRunId(sessionId) {
|
|
4683
5939
|
const sessionPart = String(sessionId ?? "")
|
|
4684
5940
|
.replace(/[^A-Za-z0-9._-]+/g, "-")
|
|
@@ -4763,6 +6019,7 @@ function attachDataTransferStatsAliases(stats) {
|
|
|
4763
6019
|
const projected = normalizeDataTransferStatsValue(stats);
|
|
4764
6020
|
return attachAliasMap(projected, {
|
|
4765
6021
|
AllBytes: "allBytes",
|
|
6022
|
+
AllBytes64: "allBytes64",
|
|
4766
6023
|
MaxBytes: "maxBytes",
|
|
4767
6024
|
MeanBytes: "meanBytes",
|
|
4768
6025
|
MinBytes: "minBytes",
|
|
@@ -4770,6 +6027,7 @@ function attachDataTransferStatsAliases(stats) {
|
|
|
4770
6027
|
Percent75: "percent75",
|
|
4771
6028
|
Percent95: "percent95",
|
|
4772
6029
|
Percent99: "percent99",
|
|
6030
|
+
Percent100: "percent100",
|
|
4773
6031
|
StdDev: "stdDev"
|
|
4774
6032
|
});
|
|
4775
6033
|
}
|
|
@@ -4793,6 +6051,7 @@ function attachLatencyStatsAliases(stats) {
|
|
|
4793
6051
|
Percent75: "percent75",
|
|
4794
6052
|
Percent95: "percent95",
|
|
4795
6053
|
Percent99: "percent99",
|
|
6054
|
+
Percent100: "percent100",
|
|
4796
6055
|
StdDev: "stdDev"
|
|
4797
6056
|
});
|
|
4798
6057
|
return projected;
|
|
@@ -4814,6 +6073,9 @@ function attachMeasurementStatsAliases(stats) {
|
|
|
4814
6073
|
projected.latency = attachLatencyStatsAliases(projected.latency);
|
|
4815
6074
|
projected.statusCodes = projected.statusCodes.map((value) => attachStatusCodeStatsAliases(value));
|
|
4816
6075
|
attachAliasMap(projected, {
|
|
6076
|
+
Count64: "count64",
|
|
6077
|
+
DistributionMode: "distributionMode",
|
|
6078
|
+
MaxRelativeError: "maxRelativeError",
|
|
4817
6079
|
Request: "request",
|
|
4818
6080
|
DataTransfer: "dataTransfer",
|
|
4819
6081
|
Latency: "latency",
|
|
@@ -4875,6 +6137,8 @@ function attachStepStatsAliases(step) {
|
|
|
4875
6137
|
const projected = normalizeStepStatsValue(step);
|
|
4876
6138
|
projected.ok = attachMeasurementStatsAliases(projected.ok);
|
|
4877
6139
|
projected.fail = attachMeasurementStatsAliases(projected.fail);
|
|
6140
|
+
if (projected.allMeasurement)
|
|
6141
|
+
projected.allMeasurement = attachMeasurementStatsAliases(projected.allMeasurement);
|
|
4878
6142
|
attachAliasMap(projected, {
|
|
4879
6143
|
ScenarioName: "scenarioName",
|
|
4880
6144
|
StepName: "stepName",
|
|
@@ -4889,6 +6153,7 @@ function attachStepStatsAliases(step) {
|
|
|
4889
6153
|
StatusCodes: "statusCodes",
|
|
4890
6154
|
Ok: "ok",
|
|
4891
6155
|
Fail: "fail",
|
|
6156
|
+
AllMeasurement: "allMeasurement",
|
|
4892
6157
|
SortIndex: "sortIndex"
|
|
4893
6158
|
});
|
|
4894
6159
|
return projected;
|
|
@@ -4927,7 +6192,20 @@ function attachLoadSimulationProjection(simulation) {
|
|
|
4927
6192
|
});
|
|
4928
6193
|
return simulation;
|
|
4929
6194
|
}
|
|
4930
|
-
function
|
|
6195
|
+
function cloneLoadEngineV2TrafficMixMetadata(metadata) {
|
|
6196
|
+
return {
|
|
6197
|
+
...metadata,
|
|
6198
|
+
shareWeights: [...metadata.shareWeights],
|
|
6199
|
+
originalSimulations: metadata.originalSimulations.map((simulation) => attachLoadSimulationProjection({ ...simulation }))
|
|
6200
|
+
};
|
|
6201
|
+
}
|
|
6202
|
+
function nextTrafficMixDeclarationIndex(scenarios) {
|
|
6203
|
+
return scenarios.reduce((next, scenario) => {
|
|
6204
|
+
const metadata = scenario.__loadStrikeTrafficMixV2Metadata();
|
|
6205
|
+
return metadata ? Math.max(next, metadata.declarationIndex + 1) : next;
|
|
6206
|
+
}, 0);
|
|
6207
|
+
}
|
|
6208
|
+
function expandTrafficMixScenarios(trafficMix, declarationIndex = 0) {
|
|
4931
6209
|
if (!(trafficMix instanceof LoadStrikeTrafficMix)) {
|
|
4932
6210
|
throw new TypeError("Traffic mix must be provided.");
|
|
4933
6211
|
}
|
|
@@ -4940,16 +6218,30 @@ function expandTrafficMixScenarios(trafficMix) {
|
|
|
4940
6218
|
throw new Error("Traffic mix scenario shares must be configured before registration.");
|
|
4941
6219
|
}
|
|
4942
6220
|
const weights = scenarioMix.map((share) => share.weight);
|
|
6221
|
+
const seedId = buildLoadEngineV2TrafficMixSeedId(declarationIndex, trafficMix.name);
|
|
4943
6222
|
return scenarioMix.map((share, index) => {
|
|
4944
|
-
const splitSimulations = totalLoad
|
|
4945
|
-
|
|
4946
|
-
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
6223
|
+
const splitSimulations = totalLoad.map((simulation) => splitTrafficSimulation(simulation, weights, index)
|
|
6224
|
+
?? trafficMixNoWorkSimulation(simulation));
|
|
6225
|
+
return share.scenario
|
|
6226
|
+
.withLoadSimulations(...splitSimulations)
|
|
6227
|
+
.__loadStrikeWithInternalLicenseFeatures(TRAFFIC_MIX_FEATURE)
|
|
6228
|
+
.__loadStrikeSetTrafficMixV2Metadata({
|
|
6229
|
+
declarationIndex,
|
|
6230
|
+
name: trafficMix.name,
|
|
6231
|
+
laneIndex: index,
|
|
6232
|
+
shareWeight: share.weight,
|
|
6233
|
+
shareWeights: weights,
|
|
6234
|
+
seedId,
|
|
6235
|
+
originalSimulations: totalLoad
|
|
6236
|
+
});
|
|
4951
6237
|
});
|
|
4952
6238
|
}
|
|
6239
|
+
function trafficMixNoWorkSimulation(simulation) {
|
|
6240
|
+
const kind = String(simulation.Kind ?? "");
|
|
6241
|
+
return kind === "IterationsForInject" || kind === "IterationsForConstant"
|
|
6242
|
+
? LoadStrikeSimulation.pause(0)
|
|
6243
|
+
: LoadStrikeSimulation.pause(Math.max(readFiniteSimulationNumber(simulation, "DuringSeconds"), 0));
|
|
6244
|
+
}
|
|
4953
6245
|
function splitTrafficSimulation(simulation, weights, index) {
|
|
4954
6246
|
const kind = String(simulation.Kind ?? "");
|
|
4955
6247
|
const duringSeconds = readFiniteSimulationNumber(simulation, "DuringSeconds");
|
|
@@ -5036,6 +6328,8 @@ function attachScenarioStatsAliases(scenario) {
|
|
|
5036
6328
|
const normalized = normalizeScenarioStatsValue(scenario);
|
|
5037
6329
|
normalized.ok = attachMeasurementStatsAliases(normalized.ok);
|
|
5038
6330
|
normalized.fail = attachMeasurementStatsAliases(normalized.fail);
|
|
6331
|
+
if (normalized.allMeasurement)
|
|
6332
|
+
normalized.allMeasurement = attachMeasurementStatsAliases(normalized.allMeasurement);
|
|
5039
6333
|
normalized.loadSimulationStats = attachLoadSimulationStatsAliases(normalized.loadSimulationStats);
|
|
5040
6334
|
normalized.stepStats = normalized.stepStats.map((value) => attachStepStatsAliases(value));
|
|
5041
6335
|
const findStepStats = scenario.findStepStats ?? ((stepName) => normalized.stepStats.find((value) => value.stepName === stepName));
|
|
@@ -5069,6 +6363,7 @@ function attachScenarioStatsAliases(scenario) {
|
|
|
5069
6363
|
DurationMs: "durationMs",
|
|
5070
6364
|
Ok: "ok",
|
|
5071
6365
|
Fail: "fail",
|
|
6366
|
+
AllMeasurement: "allMeasurement",
|
|
5072
6367
|
LoadSimulationStats: "loadSimulationStats",
|
|
5073
6368
|
SortIndex: "sortIndex",
|
|
5074
6369
|
StepStats: "stepStats"
|
|
@@ -5089,6 +6384,10 @@ function attachNodeStatsAliases(stats) {
|
|
|
5089
6384
|
: stats.scenarioStats.flatMap((value) => value.stepStats)).map((value) => attachStepStatsAliases(value));
|
|
5090
6385
|
stats.pluginsData = stats.pluginsData.map((value) => normalizePluginData(value.pluginName ?? value.PluginName ?? "", value));
|
|
5091
6386
|
stats.sinkErrors = stats.sinkErrors.map((value) => attachSinkErrorAliases(value));
|
|
6387
|
+
stats.generatorWarnings = (stats.generatorWarnings ?? []).map(attachGeneratorWarningAliases);
|
|
6388
|
+
stats.schedulerSegments = (stats.schedulerSegments ?? []).map((value) => normalizeSchedulerSegment(value));
|
|
6389
|
+
stats.observationDeliveryStats = normalizeObservationDeliveryStats(stats.observationDeliveryStats ?? emptyObservationDeliveryStats());
|
|
6390
|
+
stats.reportingComplete ?? (stats.reportingComplete = false);
|
|
5092
6391
|
const findScenarioStats = stats.findScenarioStats ?? ((scenarioName) => stats.scenarioStats.find((value) => value.scenarioName === scenarioName));
|
|
5093
6392
|
const getScenarioStats = stats.getScenarioStats ?? ((scenarioName) => {
|
|
5094
6393
|
const value = findScenarioStats(scenarioName);
|
|
@@ -5123,7 +6422,11 @@ function attachNodeStatsAliases(stats) {
|
|
|
5123
6422
|
DisabledSinks: "disabledSinks",
|
|
5124
6423
|
SinkErrors: "sinkErrors",
|
|
5125
6424
|
ReportFiles: "reportFiles",
|
|
5126
|
-
LogFiles: "logFiles"
|
|
6425
|
+
LogFiles: "logFiles",
|
|
6426
|
+
GeneratorWarnings: "generatorWarnings",
|
|
6427
|
+
SchedulerSegments: "schedulerSegments",
|
|
6428
|
+
ObservationDeliveryStats: "observationDeliveryStats",
|
|
6429
|
+
ReportingComplete: "reportingComplete"
|
|
5127
6430
|
});
|
|
5128
6431
|
defineAliasProperty(projected, "StartedUtc", () => parseAliasDate(stats.startedUtc));
|
|
5129
6432
|
defineAliasProperty(projected, "CompletedUtc", () => parseAliasDate(stats.completedUtc));
|
|
@@ -5179,11 +6482,43 @@ function attachRunResultAliases(result) {
|
|
|
5179
6482
|
.map((value) => ({ ...asAliasRecord(value) })),
|
|
5180
6483
|
failedCorrelationRows: pickAliasArray(source, "failedCorrelationRows", "FailedCorrelationRows")
|
|
5181
6484
|
.map((value) => ({ ...asAliasRecord(value) })),
|
|
6485
|
+
...(hasAliasValue(source, "generatorWarnings", "GeneratorWarnings")
|
|
6486
|
+
? {
|
|
6487
|
+
generatorWarnings: pickAliasArray(source, "generatorWarnings", "GeneratorWarnings")
|
|
6488
|
+
.map(attachGeneratorWarningAliases)
|
|
6489
|
+
}
|
|
6490
|
+
: {}),
|
|
6491
|
+
...(hasAliasValue(source, "schedulerSegments", "SchedulerSegments")
|
|
6492
|
+
? {
|
|
6493
|
+
schedulerSegments: pickAliasArray(source, "schedulerSegments", "SchedulerSegments")
|
|
6494
|
+
.map(normalizeSchedulerSegment)
|
|
6495
|
+
}
|
|
6496
|
+
: {}),
|
|
6497
|
+
...(hasAliasValue(source, "schedulerStats", "SchedulerStats")
|
|
6498
|
+
? { schedulerStats: normalizeSchedulerStats(pickAliasValue(source, "schedulerStats", "SchedulerStats")) }
|
|
6499
|
+
: {}),
|
|
6500
|
+
...(hasAliasValue(source, "observationDeliveryStats", "ObservationDeliveryStats")
|
|
6501
|
+
? {
|
|
6502
|
+
observationDeliveryStats: normalizeObservationDeliveryStats(pickAliasValue(source, "observationDeliveryStats", "ObservationDeliveryStats"))
|
|
6503
|
+
}
|
|
6504
|
+
: {}),
|
|
6505
|
+
...(hasAliasValue(source, "reportingComplete", "ReportingComplete")
|
|
6506
|
+
? { reportingComplete: pickAliasBoolean(source, "reportingComplete", "ReportingComplete") }
|
|
6507
|
+
: {}),
|
|
5182
6508
|
findScenarioStats,
|
|
5183
6509
|
getScenarioStats,
|
|
5184
6510
|
FindScenarioStats: findScenarioStats,
|
|
5185
6511
|
GetScenarioStats: getScenarioStats
|
|
5186
6512
|
};
|
|
6513
|
+
const schedulerDistributions = result[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS];
|
|
6514
|
+
if (schedulerDistributions) {
|
|
6515
|
+
Object.defineProperty(projected, LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS, {
|
|
6516
|
+
value: schedulerDistributions.map(cloneLoadEngineV2DistributionRecord),
|
|
6517
|
+
enumerable: false,
|
|
6518
|
+
configurable: false,
|
|
6519
|
+
writable: false
|
|
6520
|
+
});
|
|
6521
|
+
}
|
|
5187
6522
|
attachAliasMap(projected, {
|
|
5188
6523
|
AllBytes: "allBytes",
|
|
5189
6524
|
AllRequestCount: "allRequestCount",
|
|
@@ -5207,7 +6542,12 @@ function attachRunResultAliases(result) {
|
|
|
5207
6542
|
ReportFiles: "reportFiles",
|
|
5208
6543
|
LogFiles: "logFiles",
|
|
5209
6544
|
CorrelationRows: "correlationRows",
|
|
5210
|
-
FailedCorrelationRows: "failedCorrelationRows"
|
|
6545
|
+
FailedCorrelationRows: "failedCorrelationRows",
|
|
6546
|
+
GeneratorWarnings: "generatorWarnings",
|
|
6547
|
+
SchedulerSegments: "schedulerSegments",
|
|
6548
|
+
SchedulerStats: "schedulerStats",
|
|
6549
|
+
ObservationDeliveryStats: "observationDeliveryStats",
|
|
6550
|
+
ReportingComplete: "reportingComplete"
|
|
5211
6551
|
});
|
|
5212
6552
|
defineAliasProperty(projected, "StartedUtc", () => parseAliasDate(projected.startedUtc));
|
|
5213
6553
|
defineAliasProperty(projected, "CompletedUtc", () => parseAliasDate(projected.completedUtc));
|
|
@@ -5815,6 +7155,14 @@ function detailedToNodeStats(result, metricStats) {
|
|
|
5815
7155
|
sinkErrors: (result.sinkErrors ?? []).map((sinkError) => ({ ...sinkError })),
|
|
5816
7156
|
reportFiles: [...(result.reportFiles ?? [])],
|
|
5817
7157
|
logFiles: [...(result.logFiles ?? [])],
|
|
7158
|
+
generatorWarnings: (result.generatorWarnings ?? []).map(attachGeneratorWarningAliases),
|
|
7159
|
+
schedulerSegments: result.schedulerSegments?.map((segment) => ({ ...segment })),
|
|
7160
|
+
schedulerStats: result.schedulerStats
|
|
7161
|
+
? normalizeSchedulerStats(result.schedulerStats)
|
|
7162
|
+
: undefined,
|
|
7163
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: result[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]?.map(cloneLoadEngineV2DistributionRecord),
|
|
7164
|
+
observationDeliveryStats: normalizeObservationDeliveryStats(result.observationDeliveryStats ?? emptyObservationDeliveryStats()),
|
|
7165
|
+
reportingComplete: result.reportingComplete ?? true,
|
|
5818
7166
|
findScenarioStats: (scenarioName) => scenarioStats.find((scenario) => scenario.scenarioName === scenarioName),
|
|
5819
7167
|
getScenarioStats: (scenarioName) => {
|
|
5820
7168
|
const value = scenarioStats.find((scenario) => scenario.scenarioName === scenarioName);
|
|
@@ -5882,22 +7230,35 @@ function buildEmptyNodeStats(args) {
|
|
|
5882
7230
|
sinkErrors: [],
|
|
5883
7231
|
reportFiles: [],
|
|
5884
7232
|
logFiles: [],
|
|
7233
|
+
generatorWarnings: [],
|
|
7234
|
+
observationDeliveryStats: emptyObservationDeliveryStats(),
|
|
7235
|
+
reportingComplete: true,
|
|
5885
7236
|
findScenarioStats: (scenarioName) => undefined,
|
|
5886
7237
|
getScenarioStats: (scenarioName) => {
|
|
5887
7238
|
throw new Error(`Scenario stats not found: ${scenarioName}`);
|
|
5888
7239
|
}
|
|
5889
7240
|
});
|
|
5890
7241
|
}
|
|
5891
|
-
function planRuntimeClusterAssignments(scenarios, agentsCount, targetScenarios, agentTargetScenarios) {
|
|
7242
|
+
function planRuntimeClusterAssignments(scenarios, agentsCount, targetScenarios, agentTargetScenarios, globalV2 = false, coordinatorTargetScenarios = []) {
|
|
5892
7243
|
const resolvedAgentCount = Math.max(agentsCount, 1);
|
|
5893
|
-
|
|
7244
|
+
let selectedScenarios = targetScenarios.length
|
|
5894
7245
|
? scenarios.filter((scenario) => targetScenarios.includes(scenario.name))
|
|
5895
7246
|
: [...scenarios];
|
|
7247
|
+
if (globalV2 && coordinatorTargetScenarios.length) {
|
|
7248
|
+
const coordinatorNames = new Set(coordinatorTargetScenarios);
|
|
7249
|
+
selectedScenarios = selectedScenarios.filter((scenario) => !coordinatorNames.has(scenario.name));
|
|
7250
|
+
}
|
|
5896
7251
|
if (!selectedScenarios.length) {
|
|
5897
7252
|
return Array.from({ length: resolvedAgentCount }, () => []);
|
|
5898
7253
|
}
|
|
5899
7254
|
if (agentTargetScenarios.length) {
|
|
5900
|
-
|
|
7255
|
+
const coordinatorNames = new Set(coordinatorTargetScenarios);
|
|
7256
|
+
const names = agentTargetScenarios.filter((name) => !coordinatorNames.has(name));
|
|
7257
|
+
return Array.from({ length: resolvedAgentCount }, () => [...names]);
|
|
7258
|
+
}
|
|
7259
|
+
if (globalV2) {
|
|
7260
|
+
const scenarioNames = selectedScenarios.map((scenario) => scenario.name);
|
|
7261
|
+
return Array.from({ length: resolvedAgentCount }, () => [...scenarioNames]);
|
|
5901
7262
|
}
|
|
5902
7263
|
const weightedNames = [];
|
|
5903
7264
|
for (const scenario of selectedScenarios) {
|
|
@@ -5917,7 +7278,320 @@ function planRuntimeClusterAssignments(scenarios, agentsCount, targetScenarios,
|
|
|
5917
7278
|
}
|
|
5918
7279
|
return assignments.map((entry) => [...entry]);
|
|
5919
7280
|
}
|
|
5920
|
-
function
|
|
7281
|
+
function buildRuntimeLoadEngineV2Plan(scenarios, options, testInfo, expectedAgentIds) {
|
|
7282
|
+
validateLoadEngineV2ScenarioFeatures(scenarios);
|
|
7283
|
+
const kindToken = (value) => {
|
|
7284
|
+
const normalized = String(value ?? "").replace(/[^a-z0-9]/gi, "").toLowerCase();
|
|
7285
|
+
const tokens = {
|
|
7286
|
+
inject: "inject", injectrandom: "inject-random", rampinginject: "ramping-inject",
|
|
7287
|
+
keepconstant: "keep-constant", rampingconstant: "ramping-constant",
|
|
7288
|
+
iterationsforinject: "iterations-for-inject", iterationsforconstant: "iterations-for-constant",
|
|
7289
|
+
pause: "pause"
|
|
7290
|
+
};
|
|
7291
|
+
const token = tokens[normalized];
|
|
7292
|
+
if (!token)
|
|
7293
|
+
throw new Error(`Load Engine V2 simulation kind is unsupported: ${String(value ?? "")}`);
|
|
7294
|
+
return token;
|
|
7295
|
+
};
|
|
7296
|
+
const integer = (value) => Math.max(Math.trunc(toNumber(value)), 0).toString();
|
|
7297
|
+
const ns = (value) => Math.max(Math.round(toNumber(value) * 1000000000), 0).toString();
|
|
7298
|
+
const coordinatorNames = new Set(options.coordinatorTargetScenarios ?? []);
|
|
7299
|
+
const explicitAgentNames = new Set(options.agentTargetScenarios ?? []);
|
|
7300
|
+
const selectedNames = new Set(options.targetScenarios ?? []);
|
|
7301
|
+
const plannedScenarios = scenarios
|
|
7302
|
+
.map((scenario, declarationIndex) => ({ scenario, declarationIndex }))
|
|
7303
|
+
.filter(({ scenario }) => (selectedNames.size === 0 || selectedNames.has(scenario.name))
|
|
7304
|
+
&& (explicitAgentNames.size > 0
|
|
7305
|
+
? explicitAgentNames.has(scenario.name)
|
|
7306
|
+
: !coordinatorNames.has(scenario.name)));
|
|
7307
|
+
const implicitSingleInvocation = {
|
|
7308
|
+
simulationIndex64: "0",
|
|
7309
|
+
kind: "iterations-for-constant",
|
|
7310
|
+
rate64: "0",
|
|
7311
|
+
minRate64: "0",
|
|
7312
|
+
maxRate64: "0",
|
|
7313
|
+
copies64: "1",
|
|
7314
|
+
iterations64: "1",
|
|
7315
|
+
intervalNs64: "0",
|
|
7316
|
+
durationNs64: "0"
|
|
7317
|
+
};
|
|
7318
|
+
return {
|
|
7319
|
+
runId: testInfo.sessionId,
|
|
7320
|
+
sessionId: testInfo.sessionId,
|
|
7321
|
+
registrationNonce: randomBytes(16).toString("hex"),
|
|
7322
|
+
maxInFlight64: Math.max(options.maxInFlight ?? 10000, 1).toString(),
|
|
7323
|
+
reportingIntervalNs64: ns(options.reportingIntervalSeconds ?? 1),
|
|
7324
|
+
schedulerVisibleProcessorCount64: Math.max(os.cpus().length, 1).toString(),
|
|
7325
|
+
expectedAgentIds: [...expectedAgentIds],
|
|
7326
|
+
scenarios: plannedScenarios.map(({ scenario, declarationIndex }) => ({
|
|
7327
|
+
scenarioIndex64: declarationIndex.toString(),
|
|
7328
|
+
scenarioName: scenario.name,
|
|
7329
|
+
target: "agent",
|
|
7330
|
+
callbackExecutionMode: "async",
|
|
7331
|
+
declaredStepNames: scenario.getDeclaredSteps(),
|
|
7332
|
+
simulations: scenario.getSimulations().length
|
|
7333
|
+
? scenario.getSimulations().map((simulation, simulationIndex) => ({
|
|
7334
|
+
simulationIndex64: simulationIndex.toString(),
|
|
7335
|
+
kind: kindToken(simulation.Kind ?? simulation.kind),
|
|
7336
|
+
rate64: integer(simulation.Rate ?? simulation.rate),
|
|
7337
|
+
minRate64: integer(simulation.MinRate ?? simulation.minRate),
|
|
7338
|
+
maxRate64: integer(simulation.MaxRate ?? simulation.maxRate),
|
|
7339
|
+
copies64: integer(simulation.Copies ?? simulation.copies),
|
|
7340
|
+
iterations64: integer(simulation.Iterations ?? simulation.iterations),
|
|
7341
|
+
intervalNs64: ns(simulation.IntervalSeconds ?? simulation.intervalSeconds),
|
|
7342
|
+
durationNs64: ns(simulation.DuringSeconds ?? simulation.duringSeconds)
|
|
7343
|
+
}))
|
|
7344
|
+
: [{ ...implicitSingleInvocation }]
|
|
7345
|
+
}))
|
|
7346
|
+
};
|
|
7347
|
+
}
|
|
7348
|
+
function normalizeRequiredV2AgentIds(values, expectedCount) {
|
|
7349
|
+
const ids = normalizeOptionalStringArray(values) ?? [];
|
|
7350
|
+
if (ids.length !== expectedCount || new Set(ids).size !== ids.length) {
|
|
7351
|
+
throw new Error("Remote Load Engine V2 coordinators require an exact unique ExpectedAgentIds set matching AgentsCount.");
|
|
7352
|
+
}
|
|
7353
|
+
return ids;
|
|
7354
|
+
}
|
|
7355
|
+
function validateLoadEngineV2ScenarioFeatures(scenarios) {
|
|
7356
|
+
for (const scenario of scenarios) {
|
|
7357
|
+
if (scenario.getTrackingConfiguration()) {
|
|
7358
|
+
throw new Error(`Load Engine V2 correlation is not available in the supported non-correlation profile. Scenario=${scenario.name}.`);
|
|
7359
|
+
}
|
|
7360
|
+
}
|
|
7361
|
+
}
|
|
7362
|
+
function canonicalLoadEngineV2InvocationIdentityKind(simulationKind) {
|
|
7363
|
+
const normalized = simulationKind.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
|
7364
|
+
if (["inject", "injectrandom", "rampinginject", "iterationsforinject"].includes(normalized)) {
|
|
7365
|
+
return "arrival";
|
|
7366
|
+
}
|
|
7367
|
+
if (["keepconstant", "rampingconstant"].includes(normalized)) {
|
|
7368
|
+
return "worker-iteration";
|
|
7369
|
+
}
|
|
7370
|
+
if (normalized === "iterationsforconstant") {
|
|
7371
|
+
return "constant-iteration";
|
|
7372
|
+
}
|
|
7373
|
+
return "";
|
|
7374
|
+
}
|
|
7375
|
+
function buildRuntimeLoadEngineV2HistogramArtifact(result, scenarioDeclarations) {
|
|
7376
|
+
const emptyHistogram = () => new LoadStrikeHistogramV1().toSidecar();
|
|
7377
|
+
const distributions = [];
|
|
7378
|
+
const measurementSummaries = [];
|
|
7379
|
+
const statusBody = (measurement, outcome) => {
|
|
7380
|
+
const named = measurement.statusCodes
|
|
7381
|
+
.filter((row) => Boolean(row.statusCode || row.message))
|
|
7382
|
+
.map((row) => {
|
|
7383
|
+
const key = buildLoadEngineV2StatusIdentityKey(row.statusCode, row.message);
|
|
7384
|
+
if (!key)
|
|
7385
|
+
throw new Error("Load Engine V2 status identity unexpectedly resolved empty.");
|
|
7386
|
+
return {
|
|
7387
|
+
statusIdentityKeyHex: key.identity.toString("hex"),
|
|
7388
|
+
display: key.display,
|
|
7389
|
+
count64: Math.max(Math.trunc(row.count), 0).toString(),
|
|
7390
|
+
aggregatedObservationCount64: "0",
|
|
7391
|
+
hasAggregatedIdentities: false
|
|
7392
|
+
};
|
|
7393
|
+
})
|
|
7394
|
+
.sort((left, right) => Buffer.compare(Buffer.from(left.statusIdentityKeyHex, "hex"), Buffer.from(right.statusIdentityKeyHex, "hex")));
|
|
7395
|
+
let statuses = named;
|
|
7396
|
+
if (named.length > 64) {
|
|
7397
|
+
const retained = named.slice(0, 63);
|
|
7398
|
+
const aggregated = named.slice(63).reduce((sum, row) => sum + BigInt(row.count64), 0n);
|
|
7399
|
+
retained.push({
|
|
7400
|
+
statusIdentityKeyHex: Buffer.concat([Buffer.from("LS-ID1\n", "ascii"), Buffer.from([0x0d])]).toString("hex"),
|
|
7401
|
+
display: "<other>", count64: aggregated.toString(),
|
|
7402
|
+
aggregatedObservationCount64: aggregated.toString(), hasAggregatedIdentities: aggregated > 0n
|
|
7403
|
+
});
|
|
7404
|
+
statuses = retained;
|
|
7405
|
+
}
|
|
7406
|
+
return {
|
|
7407
|
+
outcome,
|
|
7408
|
+
statusObservationCount64: statuses.reduce((sum, row) => sum + BigInt(row.count64), 0n).toString(),
|
|
7409
|
+
statuses
|
|
7410
|
+
};
|
|
7411
|
+
};
|
|
7412
|
+
const appendMeasurement = (seriesKind, scenarioIndex64, scenarioName, identity, display, ok, fail, all, reservedOther) => {
|
|
7413
|
+
const emptyMeasurement = () => ({
|
|
7414
|
+
count64: "0",
|
|
7415
|
+
histogramSidecar: {
|
|
7416
|
+
latency: emptyHistogram(), size: emptyHistogram(), allBytes64: "0",
|
|
7417
|
+
lessOrEq80064: "0", more800Less120064: "0", moreOrEq120064: "0"
|
|
7418
|
+
},
|
|
7419
|
+
request: { count: 0, percent: 0, rps: 0 },
|
|
7420
|
+
dataTransfer: { allBytes: 0, minBytes: 0, maxBytes: 0, meanBytes: 0, percent50: 0,
|
|
7421
|
+
percent75: 0, percent95: 0, percent99: 0, percent100: 0, stdDev: 0 },
|
|
7422
|
+
latency: { latencyCount: { lessOrEq800: 0, more800Less1200: 0, moreOrEq1200: 0 },
|
|
7423
|
+
minMs: 0, maxMs: 0, meanMs: 0, percent50: 0, percent75: 0, percent95: 0,
|
|
7424
|
+
percent99: 0, stdDev: 0 },
|
|
7425
|
+
statusCodes: []
|
|
7426
|
+
});
|
|
7427
|
+
const okValue = ok ?? emptyMeasurement();
|
|
7428
|
+
const failValue = fail ?? emptyMeasurement();
|
|
7429
|
+
const allValue = all ?? emptyMeasurement();
|
|
7430
|
+
for (const [outcome, measurement] of [
|
|
7431
|
+
["ok", okValue], ["fail", failValue], ["all", allValue]
|
|
7432
|
+
]) {
|
|
7433
|
+
const sidecar = measurement.histogramSidecar;
|
|
7434
|
+
if (!sidecar)
|
|
7435
|
+
throw new Error("Load Engine V2 assigned measurement is missing histogram state.");
|
|
7436
|
+
distributions.push({
|
|
7437
|
+
seriesKind, scenarioIndex64, scenarioName, identityKeyHex: identity.toString("hex"),
|
|
7438
|
+
outcome, unit: "microseconds", histogram: sidecar.latency,
|
|
7439
|
+
exactTotalDecimalOrEmpty: sidecar.latency.exactTotal64,
|
|
7440
|
+
bands: [
|
|
7441
|
+
{ bandId64: "0", count64: sidecar.lessOrEq80064 },
|
|
7442
|
+
{ bandId64: "1", count64: sidecar.more800Less120064 },
|
|
7443
|
+
{ bandId64: "2", count64: sidecar.moreOrEq120064 }
|
|
7444
|
+
]
|
|
7445
|
+
});
|
|
7446
|
+
distributions.push({
|
|
7447
|
+
seriesKind, scenarioIndex64, scenarioName, identityKeyHex: identity.toString("hex"),
|
|
7448
|
+
outcome, unit: "bytes", histogram: sidecar.size,
|
|
7449
|
+
exactTotalDecimalOrEmpty: sidecar.size.exactTotal64
|
|
7450
|
+
});
|
|
7451
|
+
}
|
|
7452
|
+
const observation = BigInt(allValue.count64 ?? allValue.histogramSidecar?.latency.count64 ?? "0");
|
|
7453
|
+
const success = BigInt(okValue.count64 ?? okValue.histogramSidecar?.latency.count64 ?? "0");
|
|
7454
|
+
const failure = BigInt(failValue.count64 ?? failValue.histogramSidecar?.latency.count64 ?? "0");
|
|
7455
|
+
measurementSummaries.push({
|
|
7456
|
+
seriesKind, scenarioIndex64, scenarioName, identityKeyHex: identity.toString("hex"), display,
|
|
7457
|
+
observationCount64: observation.toString(), successCount64: success.toString(),
|
|
7458
|
+
failureCount64: failure.toString(),
|
|
7459
|
+
aggregatedObservationCount64: reservedOther ? observation.toString() : "0",
|
|
7460
|
+
hasAggregatedIdentities: reservedOther && observation > 0n,
|
|
7461
|
+
outcomes: [statusBody(okValue, "ok"), statusBody(failValue, "fail")]
|
|
7462
|
+
});
|
|
7463
|
+
};
|
|
7464
|
+
for (const scenario of result.scenarioStats) {
|
|
7465
|
+
const scenarioIndex64 = Math.max(Math.trunc(scenario.sortIndex), 0).toString();
|
|
7466
|
+
appendMeasurement("scenario", scenarioIndex64, scenario.scenarioName, buildLoadEngineV2ScenarioIdentityKey(scenarioIndex64), scenario.scenarioName, scenario.ok, scenario.fail, scenario.allMeasurement, false);
|
|
7467
|
+
const declaration = scenarioDeclarations?.find((value) => value.name === scenario.scenarioName);
|
|
7468
|
+
const declaredNames = declaration?.getDeclaredSteps()
|
|
7469
|
+
?? (scenarioDeclarations ? [] : undefined);
|
|
7470
|
+
if (declaredNames === undefined) {
|
|
7471
|
+
for (const step of scenario.stepStats) {
|
|
7472
|
+
const key = buildLoadEngineV2StepIdentityKey(scenarioIndex64, step.stepName);
|
|
7473
|
+
appendMeasurement("step", scenarioIndex64, scenario.scenarioName, key.identity, key.display, step.ok, step.fail, step.allMeasurement, false);
|
|
7474
|
+
}
|
|
7475
|
+
}
|
|
7476
|
+
else {
|
|
7477
|
+
const requiredStepAllMeasurement = (step) => {
|
|
7478
|
+
if (!step.allMeasurement) {
|
|
7479
|
+
throw new Error("Load Engine V2 assigned step is missing its all-outcome histogram state.");
|
|
7480
|
+
}
|
|
7481
|
+
return step.allMeasurement;
|
|
7482
|
+
};
|
|
7483
|
+
const groups = new Map();
|
|
7484
|
+
for (const declaredName of declaredNames) {
|
|
7485
|
+
const key = buildLoadEngineV2StepIdentityKey(scenarioIndex64, declaredName);
|
|
7486
|
+
groups.set(key.identity.toString("hex"), {
|
|
7487
|
+
identity: key.identity, display: key.display, routedToOther: false, steps: []
|
|
7488
|
+
});
|
|
7489
|
+
}
|
|
7490
|
+
for (const step of scenario.stepStats) {
|
|
7491
|
+
const resolved = resolveLoadEngineV2StepIdentityKey(scenarioIndex64, step.stepName, declaredNames);
|
|
7492
|
+
const key = resolved.identity.toString("hex");
|
|
7493
|
+
const group = groups.get(key) ?? {
|
|
7494
|
+
identity: resolved.identity,
|
|
7495
|
+
display: resolved.display,
|
|
7496
|
+
routedToOther: resolved.routedToOther,
|
|
7497
|
+
steps: []
|
|
7498
|
+
};
|
|
7499
|
+
group.steps.push(step);
|
|
7500
|
+
groups.set(key, group);
|
|
7501
|
+
}
|
|
7502
|
+
const reservedHex = buildLoadEngineV2ReservedStepOtherIdentityKey(scenarioIndex64).toString("hex");
|
|
7503
|
+
const declaredGroups = [...groups.entries()]
|
|
7504
|
+
.filter(([key]) => key !== reservedHex)
|
|
7505
|
+
.map(([, value]) => value)
|
|
7506
|
+
.sort((left, right) => Buffer.compare(left.identity, right.identity));
|
|
7507
|
+
for (const group of declaredGroups) {
|
|
7508
|
+
const ok = group.steps.length
|
|
7509
|
+
? aggregateMeasurementStats(group.steps.map((step) => step.ok), scenario.allRequestCount, scenario.durationMs, true)
|
|
7510
|
+
: undefined;
|
|
7511
|
+
const fail = group.steps.length
|
|
7512
|
+
? aggregateMeasurementStats(group.steps.map((step) => step.fail), scenario.allRequestCount, scenario.durationMs, true)
|
|
7513
|
+
: undefined;
|
|
7514
|
+
const all = group.steps.length
|
|
7515
|
+
? aggregateMeasurementStats(group.steps.map(requiredStepAllMeasurement), scenario.allRequestCount, scenario.durationMs, true)
|
|
7516
|
+
: undefined;
|
|
7517
|
+
appendMeasurement("step", scenarioIndex64, scenario.scenarioName, group.identity, group.display, ok, fail, all, false);
|
|
7518
|
+
}
|
|
7519
|
+
const other = groups.get(reservedHex);
|
|
7520
|
+
if (other?.steps.length) {
|
|
7521
|
+
appendMeasurement("step", scenarioIndex64, scenario.scenarioName, other.identity, "<other>", aggregateMeasurementStats(other.steps.map((step) => step.ok), scenario.allRequestCount, scenario.durationMs, true), aggregateMeasurementStats(other.steps.map((step) => step.fail), scenario.allRequestCount, scenario.durationMs, true), aggregateMeasurementStats(other.steps.map(requiredStepAllMeasurement), scenario.allRequestCount, scenario.durationMs, true), true);
|
|
7522
|
+
continue;
|
|
7523
|
+
}
|
|
7524
|
+
}
|
|
7525
|
+
appendMeasurement("step", scenarioIndex64, scenario.scenarioName, buildLoadEngineV2ReservedStepOtherIdentityKey(scenarioIndex64), "<other>", undefined, undefined, undefined, true);
|
|
7526
|
+
}
|
|
7527
|
+
for (const segment of result.schedulerSegments ?? []) {
|
|
7528
|
+
const scenarioIndex64 = segment.scenarioIndex.toString();
|
|
7529
|
+
const simulationIndex64 = segment.simulationIndex.toString();
|
|
7530
|
+
for (const kind of ["decision", "start"]) {
|
|
7531
|
+
const seriesKind = kind === "decision" ? "scheduler-decision-lag" : "scheduler-start-lag";
|
|
7532
|
+
const identityKeyHex = buildLoadEngineV2SchedulerIdentityKey(kind, scenarioIndex64, simulationIndex64).toString("hex");
|
|
7533
|
+
const signed = result[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]?.find((record) => record.seriesKind === seriesKind
|
|
7534
|
+
&& record.scenarioIndex64 === scenarioIndex64
|
|
7535
|
+
&& record.identityKeyHex === identityKeyHex
|
|
7536
|
+
&& record.outcome === "none"
|
|
7537
|
+
&& record.unit === "microseconds");
|
|
7538
|
+
if (!distributions.some((record) => record.seriesKind === seriesKind
|
|
7539
|
+
&& record.scenarioIndex64 === scenarioIndex64
|
|
7540
|
+
&& record.identityKeyHex === identityKeyHex)) {
|
|
7541
|
+
distributions.push(signed ? cloneLoadEngineV2DistributionRecord(signed) : {
|
|
7542
|
+
seriesKind,
|
|
7543
|
+
scenarioIndex64, scenarioName: segment.scenarioName,
|
|
7544
|
+
identityKeyHex,
|
|
7545
|
+
outcome: "none", unit: "microseconds", histogram: emptyHistogram(),
|
|
7546
|
+
exactTotalDecimalOrEmpty: "0"
|
|
7547
|
+
});
|
|
7548
|
+
}
|
|
7549
|
+
}
|
|
7550
|
+
}
|
|
7551
|
+
return serializeLoadEngineV2HistogramArtifact({ distributions, measurementSummaries });
|
|
7552
|
+
}
|
|
7553
|
+
function cloneLoadEngineV2DistributionRecord(record) {
|
|
7554
|
+
return {
|
|
7555
|
+
...record,
|
|
7556
|
+
histogram: {
|
|
7557
|
+
...record.histogram,
|
|
7558
|
+
exactSamples64: [...record.histogram.exactSamples64],
|
|
7559
|
+
buckets: record.histogram.buckets.map((bucket) => ({ ...bucket }))
|
|
7560
|
+
},
|
|
7561
|
+
bands: record.bands?.map((band) => ({ ...band }))
|
|
7562
|
+
};
|
|
7563
|
+
}
|
|
7564
|
+
function mergeLoadEngineV2SchedulerDistributions(records) {
|
|
7565
|
+
const merged = new Map();
|
|
7566
|
+
for (const input of records) {
|
|
7567
|
+
if (input.seriesKind !== "scheduler-decision-lag" && input.seriesKind !== "scheduler-start-lag") {
|
|
7568
|
+
continue;
|
|
7569
|
+
}
|
|
7570
|
+
const key = [input.seriesKind, input.scenarioIndex64, input.identityKeyHex, input.outcome, input.unit].join("\0");
|
|
7571
|
+
const current = merged.get(key);
|
|
7572
|
+
if (!current) {
|
|
7573
|
+
merged.set(key, cloneLoadEngineV2DistributionRecord(input));
|
|
7574
|
+
continue;
|
|
7575
|
+
}
|
|
7576
|
+
if (current.scenarioName !== input.scenarioName) {
|
|
7577
|
+
throw new Error("Load Engine V2 scheduler histogram identities disagree across agents.");
|
|
7578
|
+
}
|
|
7579
|
+
const histogram = LoadStrikeHistogramV1.fromSidecar(current.histogram);
|
|
7580
|
+
histogram.merge(LoadStrikeHistogramV1.fromSidecar(input.histogram));
|
|
7581
|
+
current.histogram = histogram.toSidecar();
|
|
7582
|
+
current.exactTotalDecimalOrEmpty = histogram.toSidecar().exactTotal64;
|
|
7583
|
+
const bands = new Map();
|
|
7584
|
+
for (const band of [...(current.bands ?? []), ...(input.bands ?? [])]) {
|
|
7585
|
+
bands.set(band.bandId64, (bands.get(band.bandId64) ?? 0n) + BigInt(band.count64));
|
|
7586
|
+
}
|
|
7587
|
+
current.bands = Array.from(bands, ([bandId64, count]) => ({ bandId64, count64: count.toString() }));
|
|
7588
|
+
}
|
|
7589
|
+
return Array.from(merged.values());
|
|
7590
|
+
}
|
|
7591
|
+
function nodeStatsToClusterPayload(result, scenarioDeclarations, requireLoadEngineV2Histogram = false) {
|
|
7592
|
+
const histogramArtifactBase64 = requireLoadEngineV2Histogram
|
|
7593
|
+
? buildRuntimeLoadEngineV2HistogramArtifact(result, scenarioDeclarations).toString("base64")
|
|
7594
|
+
: undefined;
|
|
5921
7595
|
return {
|
|
5922
7596
|
allBytes: result.allBytes,
|
|
5923
7597
|
allRequestCount: result.allRequestCount,
|
|
@@ -5930,7 +7604,13 @@ function nodeStatsToClusterPayload(result) {
|
|
|
5930
7604
|
pluginsData: result.pluginsData,
|
|
5931
7605
|
nodeInfo: result.nodeInfo,
|
|
5932
7606
|
testInfo: result.testInfo,
|
|
5933
|
-
logFiles: [...(result.logFiles ?? [])]
|
|
7607
|
+
logFiles: [...(result.logFiles ?? [])],
|
|
7608
|
+
generatorWarnings: result.generatorWarnings,
|
|
7609
|
+
observationDeliveryStats: result.observationDeliveryStats,
|
|
7610
|
+
schedulerSegments: result.schedulerSegments,
|
|
7611
|
+
schedulerStats: result.schedulerStats,
|
|
7612
|
+
...(histogramArtifactBase64 ? { histogramArtifactBase64 } : {}),
|
|
7613
|
+
reportingComplete: result.reportingComplete
|
|
5934
7614
|
};
|
|
5935
7615
|
}
|
|
5936
7616
|
function toDetailedRunResultFromNodeStats(result, startedUtc, sinkErrors, policyErrors = []) {
|
|
@@ -5958,6 +7638,14 @@ function toDetailedRunResultFromNodeStats(result, startedUtc, sinkErrors, policy
|
|
|
5958
7638
|
policyErrors: policyErrors.map((value) => attachRuntimePolicyErrorAliases({ ...value })),
|
|
5959
7639
|
reportFiles: [...result.reportFiles],
|
|
5960
7640
|
logFiles: [...(result.logFiles ?? [])],
|
|
7641
|
+
generatorWarnings: (result.generatorWarnings ?? []).map(attachGeneratorWarningAliases),
|
|
7642
|
+
schedulerSegments: result.schedulerSegments?.map((segment) => ({ ...segment })),
|
|
7643
|
+
schedulerStats: result.schedulerStats
|
|
7644
|
+
? normalizeSchedulerStats(result.schedulerStats)
|
|
7645
|
+
: undefined,
|
|
7646
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: result[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]?.map(cloneLoadEngineV2DistributionRecord),
|
|
7647
|
+
observationDeliveryStats: normalizeObservationDeliveryStats(result.observationDeliveryStats ?? emptyObservationDeliveryStats()),
|
|
7648
|
+
reportingComplete: result.reportingComplete ?? false,
|
|
5961
7649
|
correlationRows: buildDetailedCorrelationRows(),
|
|
5962
7650
|
failedCorrelationRows: buildDetailedFailedCorrelationRows()
|
|
5963
7651
|
});
|
|
@@ -5980,7 +7668,198 @@ function flattenMetricValues(metricStats) {
|
|
|
5980
7668
|
}))
|
|
5981
7669
|
];
|
|
5982
7670
|
}
|
|
5983
|
-
function
|
|
7671
|
+
function projectLoadEngineV2MeasurementsFromArtifact(sourceScenarios, artifact) {
|
|
7672
|
+
const distributions = new Map();
|
|
7673
|
+
for (const distribution of artifact.distributions) {
|
|
7674
|
+
const key = loadEngineV2DistributionProjectionKey(distribution.seriesKind, distribution.scenarioIndex64, distribution.identityKeyHex, distribution.outcome, distribution.unit);
|
|
7675
|
+
if (distributions.has(key)) {
|
|
7676
|
+
throw new Error("Load Engine V2 histogram artifact contains a duplicate distribution identity.");
|
|
7677
|
+
}
|
|
7678
|
+
distributions.set(key, distribution);
|
|
7679
|
+
}
|
|
7680
|
+
const scenarioSummaries = artifact.measurementSummaries
|
|
7681
|
+
.filter((summary) => summary.seriesKind === "scenario")
|
|
7682
|
+
.sort((left, right) => compareLoadEngineV2Decimal(left.scenarioIndex64, right.scenarioIndex64));
|
|
7683
|
+
if (scenarioSummaries.length !== sourceScenarios.length) {
|
|
7684
|
+
throw new Error("Load Engine V2 histogram artifact scenario summaries do not reconcile with the scheduler snapshot.");
|
|
7685
|
+
}
|
|
7686
|
+
const sourceByName = new Map();
|
|
7687
|
+
for (const scenario of sourceScenarios) {
|
|
7688
|
+
if (!scenario.scenarioName || sourceByName.has(scenario.scenarioName)) {
|
|
7689
|
+
throw new Error("Load Engine V2 scheduler snapshot scenario identities are empty or duplicated.");
|
|
7690
|
+
}
|
|
7691
|
+
sourceByName.set(scenario.scenarioName, scenario);
|
|
7692
|
+
}
|
|
7693
|
+
return scenarioSummaries.map((summary) => {
|
|
7694
|
+
const source = sourceByName.get(summary.scenarioName);
|
|
7695
|
+
if (!source) {
|
|
7696
|
+
throw new Error("Load Engine V2 histogram artifact scenario identity is absent from the scheduler snapshot.");
|
|
7697
|
+
}
|
|
7698
|
+
const expectedIdentity = buildLoadEngineV2ScenarioIdentityKey(summary.scenarioIndex64).toString("hex");
|
|
7699
|
+
if (summary.identityKeyHex !== expectedIdentity) {
|
|
7700
|
+
throw new Error("Load Engine V2 histogram artifact scenario identity is not canonical.");
|
|
7701
|
+
}
|
|
7702
|
+
const observationCount = loadEngineV2SafeNumber(summary.observationCount64, "scenario observation count");
|
|
7703
|
+
const successCount = loadEngineV2SafeNumber(summary.successCount64, "scenario success count");
|
|
7704
|
+
const failureCount = loadEngineV2SafeNumber(summary.failureCount64, "scenario failure count");
|
|
7705
|
+
if (source.allRequestCount !== observationCount
|
|
7706
|
+
|| source.allOkCount !== successCount
|
|
7707
|
+
|| source.allFailCount !== failureCount) {
|
|
7708
|
+
throw new Error("Load Engine V2 histogram artifact scenario counts do not reconcile with the scheduler snapshot.");
|
|
7709
|
+
}
|
|
7710
|
+
const ok = projectLoadEngineV2Measurement(summary, "ok", distributions, observationCount, source.durationMs, [source.ok]);
|
|
7711
|
+
const fail = projectLoadEngineV2Measurement(summary, "fail", distributions, observationCount, source.durationMs, [source.fail]);
|
|
7712
|
+
const allMeasurement = projectLoadEngineV2Measurement(summary, "all", distributions, observationCount, source.durationMs, [source.ok, source.fail]);
|
|
7713
|
+
const stepSummaries = artifact.measurementSummaries
|
|
7714
|
+
.filter((candidate) => candidate.seriesKind === "step"
|
|
7715
|
+
&& candidate.scenarioIndex64 === summary.scenarioIndex64
|
|
7716
|
+
&& candidate.scenarioName === summary.scenarioName)
|
|
7717
|
+
.sort((left, right) => Buffer.compare(Buffer.from(left.identityKeyHex, "hex"), Buffer.from(right.identityKeyHex, "hex")));
|
|
7718
|
+
const stepIdentitySet = new Set(stepSummaries.map((candidate) => candidate.identityKeyHex));
|
|
7719
|
+
const reservedOtherIdentity = buildLoadEngineV2ReservedStepOtherIdentityKey(summary.scenarioIndex64).toString("hex");
|
|
7720
|
+
const stepStats = stepSummaries.map((stepSummary, sortIndex) => {
|
|
7721
|
+
const matchingSourceSteps = source.stepStats.filter((step) => {
|
|
7722
|
+
const observedIdentity = buildLoadEngineV2StepIdentityKey(summary.scenarioIndex64, step.stepName).identity.toString("hex");
|
|
7723
|
+
return stepSummary.identityKeyHex === reservedOtherIdentity
|
|
7724
|
+
? !stepIdentitySet.has(observedIdentity)
|
|
7725
|
+
: observedIdentity === stepSummary.identityKeyHex;
|
|
7726
|
+
});
|
|
7727
|
+
const stepObservationCount = loadEngineV2SafeNumber(stepSummary.observationCount64, "step observation count");
|
|
7728
|
+
const stepOk = projectLoadEngineV2Measurement(stepSummary, "ok", distributions, observationCount, source.durationMs, matchingSourceSteps.map((step) => step.ok));
|
|
7729
|
+
const stepFail = projectLoadEngineV2Measurement(stepSummary, "fail", distributions, observationCount, source.durationMs, matchingSourceSteps.map((step) => step.fail));
|
|
7730
|
+
const stepAll = projectLoadEngineV2Measurement(stepSummary, "all", distributions, observationCount, source.durationMs, matchingSourceSteps.flatMap((step) => [step.ok, step.fail]));
|
|
7731
|
+
const totalLatencyMs = stepAll.latency.meanMs * stepObservationCount;
|
|
7732
|
+
return {
|
|
7733
|
+
scenarioName: summary.scenarioName,
|
|
7734
|
+
stepName: stepSummary.display,
|
|
7735
|
+
okCount: stepOk.request.count,
|
|
7736
|
+
failCount: stepFail.request.count,
|
|
7737
|
+
requestCount: stepObservationCount,
|
|
7738
|
+
totalBytes: stepAll.dataTransfer.allBytes,
|
|
7739
|
+
totalLatencyMs,
|
|
7740
|
+
avgLatencyMs: stepObservationCount > 0 ? totalLatencyMs / stepObservationCount : 0,
|
|
7741
|
+
minLatencyMs: stepAll.latency.minMs,
|
|
7742
|
+
maxLatencyMs: stepAll.latency.maxMs,
|
|
7743
|
+
statusCodes: aggregateStatusCodeCounts(stepOk.statusCodes, stepFail.statusCodes),
|
|
7744
|
+
ok: stepOk,
|
|
7745
|
+
fail: stepFail,
|
|
7746
|
+
allMeasurement: stepAll,
|
|
7747
|
+
sortIndex
|
|
7748
|
+
};
|
|
7749
|
+
});
|
|
7750
|
+
const totalLatencyMs = allMeasurement.latency.meanMs * observationCount;
|
|
7751
|
+
const projected = {
|
|
7752
|
+
scenarioName: summary.scenarioName,
|
|
7753
|
+
allRequestCount: observationCount,
|
|
7754
|
+
allOkCount: successCount,
|
|
7755
|
+
allFailCount: failureCount,
|
|
7756
|
+
totalBytes: allMeasurement.dataTransfer.allBytes,
|
|
7757
|
+
totalLatencyMs,
|
|
7758
|
+
avgLatencyMs: observationCount > 0 ? totalLatencyMs / observationCount : 0,
|
|
7759
|
+
minLatencyMs: allMeasurement.latency.minMs,
|
|
7760
|
+
maxLatencyMs: allMeasurement.latency.maxMs,
|
|
7761
|
+
statusCodes: aggregateStatusCodeCounts(ok.statusCodes, fail.statusCodes),
|
|
7762
|
+
allMeasurement,
|
|
7763
|
+
allBytes: allMeasurement.dataTransfer.allBytes,
|
|
7764
|
+
currentOperation: source.currentOperation,
|
|
7765
|
+
durationMs: source.durationMs,
|
|
7766
|
+
ok,
|
|
7767
|
+
fail,
|
|
7768
|
+
loadSimulationStats: { ...source.loadSimulationStats },
|
|
7769
|
+
sortIndex: loadEngineV2SafeNumber(summary.scenarioIndex64, "scenario index"),
|
|
7770
|
+
stepStats,
|
|
7771
|
+
findStepStats: (stepName) => stepStats.find((step) => step.stepName === stepName),
|
|
7772
|
+
getStepStats: (stepName) => {
|
|
7773
|
+
const step = stepStats.find((candidate) => candidate.stepName === stepName);
|
|
7774
|
+
if (!step)
|
|
7775
|
+
throw new Error(`Step stats not found: ${stepName}`);
|
|
7776
|
+
return step;
|
|
7777
|
+
}
|
|
7778
|
+
};
|
|
7779
|
+
return attachScenarioStatsAliases(projected);
|
|
7780
|
+
});
|
|
7781
|
+
}
|
|
7782
|
+
function projectLoadEngineV2Measurement(summary, outcome, distributions, allRequestCount, durationMs, sourceMeasurements) {
|
|
7783
|
+
const expectedCount64 = outcome === "ok"
|
|
7784
|
+
? summary.successCount64
|
|
7785
|
+
: outcome === "fail"
|
|
7786
|
+
? summary.failureCount64
|
|
7787
|
+
: summary.observationCount64;
|
|
7788
|
+
const latency = requireLoadEngineV2MeasurementDistribution(summary, outcome, "microseconds", distributions);
|
|
7789
|
+
const size = requireLoadEngineV2MeasurementDistribution(summary, outcome, "bytes", distributions);
|
|
7790
|
+
if (latency.histogram.count64 !== expectedCount64 || size.histogram.count64 !== expectedCount64) {
|
|
7791
|
+
throw new Error("Load Engine V2 histogram distribution counts do not reconcile with its measurement summary.");
|
|
7792
|
+
}
|
|
7793
|
+
const bands = new Map();
|
|
7794
|
+
for (const band of latency.bands ?? []) {
|
|
7795
|
+
if (bands.has(band.bandId64)) {
|
|
7796
|
+
throw new Error("Load Engine V2 latency distribution contains a duplicate band.");
|
|
7797
|
+
}
|
|
7798
|
+
bands.set(band.bandId64, BigInt(band.count64));
|
|
7799
|
+
}
|
|
7800
|
+
if (bands.size !== 3 || !bands.has("0") || !bands.has("1") || !bands.has("2")
|
|
7801
|
+
|| [...bands.values()].reduce((sum, value) => sum + value, 0n) !== BigInt(expectedCount64)) {
|
|
7802
|
+
throw new Error("Load Engine V2 latency distribution bands do not reconcile with its count.");
|
|
7803
|
+
}
|
|
7804
|
+
const sourceStatuses = new Map();
|
|
7805
|
+
for (const measurement of sourceMeasurements) {
|
|
7806
|
+
for (const status of measurement.statusCodes) {
|
|
7807
|
+
const identity = buildLoadEngineV2StatusIdentityKey(status.statusCode, status.message);
|
|
7808
|
+
if (identity)
|
|
7809
|
+
sourceStatuses.set(identity.identity.toString("hex"), status);
|
|
7810
|
+
}
|
|
7811
|
+
}
|
|
7812
|
+
const statusCodes = new Map();
|
|
7813
|
+
const outcomeSummaries = outcome === "all"
|
|
7814
|
+
? summary.outcomes
|
|
7815
|
+
: summary.outcomes.filter((candidate) => candidate.outcome === outcome);
|
|
7816
|
+
for (const outcomeSummary of outcomeSummaries) {
|
|
7817
|
+
for (const status of outcomeSummary.statuses) {
|
|
7818
|
+
const source = sourceStatuses.get(status.statusIdentityKeyHex);
|
|
7819
|
+
statusCodes.set(`${outcomeSummary.outcome}\0${status.statusIdentityKeyHex}`, {
|
|
7820
|
+
statusCode: source?.statusCode ?? status.display,
|
|
7821
|
+
message: source?.message ?? "",
|
|
7822
|
+
isError: source?.isError ?? outcomeSummary.outcome === "fail",
|
|
7823
|
+
count: loadEngineV2SafeNumber(status.count64, "status count")
|
|
7824
|
+
});
|
|
7825
|
+
}
|
|
7826
|
+
}
|
|
7827
|
+
const expectedCount = loadEngineV2SafeNumber(expectedCount64, "measurement count");
|
|
7828
|
+
const sizeHistogram = LoadStrikeHistogramV1.fromSidecar(size.histogram);
|
|
7829
|
+
return buildHistogramMeasurement({
|
|
7830
|
+
count: expectedCount,
|
|
7831
|
+
allBytes: loadEngineV2SafeNumber(sizeHistogram.exactTotal.toString(), "measurement byte total"),
|
|
7832
|
+
latency: LoadStrikeHistogramV1.fromSidecar(latency.histogram),
|
|
7833
|
+
size: sizeHistogram,
|
|
7834
|
+
statusCodes,
|
|
7835
|
+
lessOrEq800: loadEngineV2SafeNumber((bands.get("0") ?? 0n).toString(), "latency band count"),
|
|
7836
|
+
more800Less1200: loadEngineV2SafeNumber((bands.get("1") ?? 0n).toString(), "latency band count"),
|
|
7837
|
+
moreOrEq1200: loadEngineV2SafeNumber((bands.get("2") ?? 0n).toString(), "latency band count")
|
|
7838
|
+
}, allRequestCount, durationMs);
|
|
7839
|
+
}
|
|
7840
|
+
function requireLoadEngineV2MeasurementDistribution(summary, outcome, unit, distributions) {
|
|
7841
|
+
const distribution = distributions.get(loadEngineV2DistributionProjectionKey(summary.seriesKind, summary.scenarioIndex64, summary.identityKeyHex, outcome, unit));
|
|
7842
|
+
if (!distribution || distribution.scenarioName !== summary.scenarioName) {
|
|
7843
|
+
throw new Error("Load Engine V2 measurement summary is missing its canonical histogram distribution.");
|
|
7844
|
+
}
|
|
7845
|
+
return distribution;
|
|
7846
|
+
}
|
|
7847
|
+
function loadEngineV2DistributionProjectionKey(seriesKind, scenarioIndex64, identityKeyHex, outcome, unit) {
|
|
7848
|
+
return [seriesKind, scenarioIndex64, identityKeyHex, outcome, unit].join("\0");
|
|
7849
|
+
}
|
|
7850
|
+
function compareLoadEngineV2Decimal(left, right) {
|
|
7851
|
+
const a = BigInt(left);
|
|
7852
|
+
const b = BigInt(right);
|
|
7853
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
7854
|
+
}
|
|
7855
|
+
function loadEngineV2SafeNumber(value, field) {
|
|
7856
|
+
const parsed = BigInt(value);
|
|
7857
|
+
if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
7858
|
+
throw new Error(`Load Engine V2 ${field} exceeds the JavaScript safe integer range.`);
|
|
7859
|
+
}
|
|
7860
|
+
return Number(parsed);
|
|
7861
|
+
}
|
|
7862
|
+
function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo, requireHistogramArtifact = false) {
|
|
5984
7863
|
const completedUtc = new Date().toISOString();
|
|
5985
7864
|
if (!result.success || !result.stats) {
|
|
5986
7865
|
return attachNodeStatsAliases({
|
|
@@ -6013,12 +7892,12 @@ function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo) {
|
|
|
6013
7892
|
isFailed: true,
|
|
6014
7893
|
errorCount: 1,
|
|
6015
7894
|
exceptionMessage: result.error ?? "Agent execution failed."
|
|
6016
|
-
}]
|
|
7895
|
+
}],
|
|
7896
|
+
reportingComplete: false
|
|
6017
7897
|
});
|
|
6018
7898
|
}
|
|
6019
7899
|
const metrics = normalizeMetricStatsPayload(result.stats.metrics, result.stats.durationMs ?? 0);
|
|
6020
|
-
|
|
6021
|
-
const stepStats = scenarioStats.flatMap((value) => value.stepStats);
|
|
7900
|
+
let scenarioStats = normalizeScenarioStatsPayload(result.stats.scenarioStats);
|
|
6022
7901
|
const thresholds = normalizeThresholdPayload(result.stats.thresholds);
|
|
6023
7902
|
const pluginsData = normalizePluginsPayload(result.stats.pluginsData);
|
|
6024
7903
|
const nodeInfo = {
|
|
@@ -6031,6 +7910,17 @@ function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo) {
|
|
|
6031
7910
|
...testInfo,
|
|
6032
7911
|
...(result.stats.testInfo ?? {})
|
|
6033
7912
|
};
|
|
7913
|
+
const histogramArtifactBase64 = String(result.stats.histogramArtifactBase64 ?? "");
|
|
7914
|
+
if (requireHistogramArtifact && !histogramArtifactBase64) {
|
|
7915
|
+
throw new Error("Load Engine V2 result omits the mandatory LS-H1 histogram artifact.");
|
|
7916
|
+
}
|
|
7917
|
+
const histogramArtifact = histogramArtifactBase64
|
|
7918
|
+
? parseLoadEngineV2HistogramArtifact(Buffer.from(histogramArtifactBase64, "base64"))
|
|
7919
|
+
: undefined;
|
|
7920
|
+
if (histogramArtifact) {
|
|
7921
|
+
scenarioStats = projectLoadEngineV2MeasurementsFromArtifact(scenarioStats, histogramArtifact);
|
|
7922
|
+
}
|
|
7923
|
+
const stepStats = scenarioStats.flatMap((value) => value.stepStats);
|
|
6034
7924
|
return attachNodeStatsAliases({
|
|
6035
7925
|
startedUtc: testInfo.createdUtc,
|
|
6036
7926
|
completedUtc,
|
|
@@ -6053,6 +7943,17 @@ function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo) {
|
|
|
6053
7943
|
sinkErrors: [],
|
|
6054
7944
|
reportFiles: [],
|
|
6055
7945
|
logFiles: normalizeAliasStringArray(result.stats.logFiles),
|
|
7946
|
+
generatorWarnings: (result.stats.generatorWarnings ?? []).map(attachGeneratorWarningAliases),
|
|
7947
|
+
schedulerSegments: (result.stats.schedulerSegments ?? []).map(normalizeSchedulerSegment),
|
|
7948
|
+
schedulerStats: result.stats.schedulerStats
|
|
7949
|
+
? normalizeSchedulerStats(result.stats.schedulerStats)
|
|
7950
|
+
: undefined,
|
|
7951
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: histogramArtifact?.distributions
|
|
7952
|
+
.filter((record) => record.seriesKind === "scheduler-decision-lag"
|
|
7953
|
+
|| record.seriesKind === "scheduler-start-lag")
|
|
7954
|
+
.map(cloneLoadEngineV2DistributionRecord),
|
|
7955
|
+
observationDeliveryStats: normalizeObservationDeliveryStats(result.stats.observationDeliveryStats ?? emptyObservationDeliveryStats()),
|
|
7956
|
+
reportingComplete: result.stats.reportingComplete ?? false,
|
|
6056
7957
|
findScenarioStats: (scenarioName) => scenarioStats.find((value) => value.scenarioName === scenarioName),
|
|
6057
7958
|
getScenarioStats: (scenarioName) => {
|
|
6058
7959
|
const value = scenarioStats.find((scenario) => scenario.scenarioName === scenarioName);
|
|
@@ -6063,7 +7964,46 @@ function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo) {
|
|
|
6063
7964
|
}
|
|
6064
7965
|
});
|
|
6065
7966
|
}
|
|
6066
|
-
function
|
|
7967
|
+
function emptyObservationDeliveryStats() {
|
|
7968
|
+
return normalizeObservationDeliveryStats({
|
|
7969
|
+
lastBatchSequence64: "-1",
|
|
7970
|
+
capturedCount64: "0",
|
|
7971
|
+
deliveredCount64: "0",
|
|
7972
|
+
droppedBufferCount64: "0",
|
|
7973
|
+
droppedSinkCount64: "0"
|
|
7974
|
+
});
|
|
7975
|
+
}
|
|
7976
|
+
function aggregateObservationDeliveryStats(nodes) {
|
|
7977
|
+
let lastBatchSequence = -1n;
|
|
7978
|
+
let captured = 0n;
|
|
7979
|
+
let delivered = 0n;
|
|
7980
|
+
let droppedBuffer = 0n;
|
|
7981
|
+
let droppedSink = 0n;
|
|
7982
|
+
for (const node of nodes) {
|
|
7983
|
+
const stats = node.observationDeliveryStats ?? emptyObservationDeliveryStats();
|
|
7984
|
+
lastBatchSequence = maxBigInt(lastBatchSequence, parseObservationDecimal(stats.lastBatchSequence64, -1n));
|
|
7985
|
+
captured += parseObservationDecimal(stats.capturedCount64);
|
|
7986
|
+
delivered += parseObservationDecimal(stats.deliveredCount64);
|
|
7987
|
+
droppedBuffer += parseObservationDecimal(stats.droppedBufferCount64);
|
|
7988
|
+
droppedSink += parseObservationDecimal(stats.droppedSinkCount64);
|
|
7989
|
+
}
|
|
7990
|
+
return normalizeObservationDeliveryStats({
|
|
7991
|
+
lastBatchSequence64: lastBatchSequence.toString(),
|
|
7992
|
+
capturedCount64: captured.toString(),
|
|
7993
|
+
deliveredCount64: delivered.toString(),
|
|
7994
|
+
droppedBufferCount64: droppedBuffer.toString(),
|
|
7995
|
+
droppedSinkCount64: droppedSink.toString()
|
|
7996
|
+
});
|
|
7997
|
+
}
|
|
7998
|
+
function parseObservationDecimal(value, fallback = 0n) {
|
|
7999
|
+
try {
|
|
8000
|
+
return BigInt(value);
|
|
8001
|
+
}
|
|
8002
|
+
catch {
|
|
8003
|
+
return fallback;
|
|
8004
|
+
}
|
|
8005
|
+
}
|
|
8006
|
+
function aggregateNodeStats(testInfo, coordinatorNodeInfo, nodes, requireHistograms = false) {
|
|
6067
8007
|
if (!nodes.length) {
|
|
6068
8008
|
return buildEmptyNodeStats({
|
|
6069
8009
|
startedUtc: testInfo.createdUtc,
|
|
@@ -6072,12 +8012,22 @@ function aggregateNodeStats(testInfo, coordinatorNodeInfo, nodes) {
|
|
|
6072
8012
|
testInfo
|
|
6073
8013
|
});
|
|
6074
8014
|
}
|
|
6075
|
-
const scenarioStats = aggregateScenarioStats(nodes);
|
|
8015
|
+
const scenarioStats = aggregateScenarioStats(nodes, requireHistograms);
|
|
6076
8016
|
const stepStats = scenarioStats.flatMap((value) => value.stepStats);
|
|
6077
8017
|
const metrics = aggregateMetricStats(nodes);
|
|
6078
8018
|
const thresholds = aggregateThresholds(nodes);
|
|
6079
8019
|
const pluginsData = aggregatePluginsData(nodes);
|
|
6080
8020
|
const completedUtc = new Date().toISOString();
|
|
8021
|
+
const schedulerSegments = nodes.flatMap((value) => value.schedulerSegments ?? []);
|
|
8022
|
+
const schedulerStatsRows = nodes.flatMap((value) => value.schedulerStats ? [value.schedulerStats] : []);
|
|
8023
|
+
const schedulerStats = schedulerSegments.length || schedulerStatsRows.length
|
|
8024
|
+
? {
|
|
8025
|
+
configuredMaxInFlight: schedulerStatsRows.reduce((maximum, value) => Math.max(maximum, value.configuredMaxInFlight), 0),
|
|
8026
|
+
maxInFlightObserved: schedulerStatsRows.reduce((maximum, value) => Math.max(maximum, value.maxInFlightObserved), 0),
|
|
8027
|
+
currentInFlight: schedulerStatsRows.reduce((sum, value) => sum + value.currentInFlight, 0),
|
|
8028
|
+
segments: schedulerSegments
|
|
8029
|
+
}
|
|
8030
|
+
: undefined;
|
|
6081
8031
|
return attachNodeStatsAliases({
|
|
6082
8032
|
startedUtc: testInfo.createdUtc,
|
|
6083
8033
|
completedUtc,
|
|
@@ -6100,6 +8050,12 @@ function aggregateNodeStats(testInfo, coordinatorNodeInfo, nodes) {
|
|
|
6100
8050
|
sinkErrors: [],
|
|
6101
8051
|
reportFiles: [],
|
|
6102
8052
|
logFiles: mergeStringArrays(...nodes.map((value) => value.logFiles ?? [])),
|
|
8053
|
+
generatorWarnings: nodes.flatMap((value) => value.generatorWarnings ?? []),
|
|
8054
|
+
...(schedulerSegments.length ? { schedulerSegments } : {}),
|
|
8055
|
+
...(schedulerStats ? { schedulerStats } : {}),
|
|
8056
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: mergeLoadEngineV2SchedulerDistributions(nodes.flatMap((value) => value[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS] ?? [])),
|
|
8057
|
+
observationDeliveryStats: aggregateObservationDeliveryStats(nodes),
|
|
8058
|
+
reportingComplete: nodes.every((value) => value.reportingComplete ?? false),
|
|
6103
8059
|
findScenarioStats: (scenarioName) => scenarioStats.find((value) => value.scenarioName === scenarioName),
|
|
6104
8060
|
getScenarioStats: (scenarioName) => {
|
|
6105
8061
|
const value = scenarioStats.find((scenario) => scenario.scenarioName === scenarioName);
|
|
@@ -6110,7 +8066,7 @@ function aggregateNodeStats(testInfo, coordinatorNodeInfo, nodes) {
|
|
|
6110
8066
|
}
|
|
6111
8067
|
});
|
|
6112
8068
|
}
|
|
6113
|
-
function aggregateScenarioStats(nodes) {
|
|
8069
|
+
function aggregateScenarioStats(nodes, requireHistograms = false) {
|
|
6114
8070
|
const grouped = new Map();
|
|
6115
8071
|
for (const node of nodes) {
|
|
6116
8072
|
for (const scenario of node.scenarioStats) {
|
|
@@ -6124,9 +8080,12 @@ function aggregateScenarioStats(nodes) {
|
|
|
6124
8080
|
.map((items) => {
|
|
6125
8081
|
const allRequestCount = items.reduce((sum, value) => sum + value.allRequestCount, 0);
|
|
6126
8082
|
const durationMs = items.reduce((max, value) => Math.max(max, value.durationMs), 0);
|
|
6127
|
-
const ok = aggregateMeasurementStats(items.map((value) => value.ok), allRequestCount, durationMs);
|
|
6128
|
-
const fail = aggregateMeasurementStats(items.map((value) => value.fail), allRequestCount, durationMs);
|
|
6129
|
-
const
|
|
8083
|
+
const ok = aggregateMeasurementStats(items.map((value) => value.ok), allRequestCount, durationMs, requireHistograms);
|
|
8084
|
+
const fail = aggregateMeasurementStats(items.map((value) => value.fail), allRequestCount, durationMs, requireHistograms);
|
|
8085
|
+
const allMeasurement = requireHistograms
|
|
8086
|
+
? aggregateMeasurementStats([ok, fail], allRequestCount, durationMs, true)
|
|
8087
|
+
: undefined;
|
|
8088
|
+
const stepStats = aggregateStepStats(items, requireHistograms);
|
|
6130
8089
|
const scenarioName = items[0]?.scenarioName ?? "";
|
|
6131
8090
|
const currentOperation = selectScenarioOperation(items.map((value) => value.currentOperation));
|
|
6132
8091
|
const loadSimulationStats = items.find((value) => value.loadSimulationStats.simulationName)?.loadSimulationStats ?? {
|
|
@@ -6151,6 +8110,7 @@ function aggregateScenarioStats(nodes) {
|
|
|
6151
8110
|
durationMs,
|
|
6152
8111
|
ok,
|
|
6153
8112
|
fail,
|
|
8113
|
+
...(allMeasurement ? { allMeasurement } : {}),
|
|
6154
8114
|
loadSimulationStats,
|
|
6155
8115
|
sortIndex: Math.min(...items.map((value) => value.sortIndex)),
|
|
6156
8116
|
stepStats,
|
|
@@ -6166,7 +8126,7 @@ function aggregateScenarioStats(nodes) {
|
|
|
6166
8126
|
return attachScenarioStatsAliases(scenario);
|
|
6167
8127
|
});
|
|
6168
8128
|
}
|
|
6169
|
-
function aggregateStepStats(scenarios) {
|
|
8129
|
+
function aggregateStepStats(scenarios, requireHistograms = false) {
|
|
6170
8130
|
const grouped = new Map();
|
|
6171
8131
|
for (const scenario of scenarios) {
|
|
6172
8132
|
for (const step of scenario.stepStats) {
|
|
@@ -6180,8 +8140,11 @@ function aggregateStepStats(scenarios) {
|
|
|
6180
8140
|
return Array.from(grouped.values())
|
|
6181
8141
|
.sort((left, right) => Math.min(...left.map((value) => value.sortIndex)) - Math.min(...right.map((value) => value.sortIndex)))
|
|
6182
8142
|
.map((items) => {
|
|
6183
|
-
const ok = aggregateMeasurementStats(items.map((value) => value.ok), allScenarioRequests, scenarioDurationMs);
|
|
6184
|
-
const fail = aggregateMeasurementStats(items.map((value) => value.fail), allScenarioRequests, scenarioDurationMs);
|
|
8143
|
+
const ok = aggregateMeasurementStats(items.map((value) => value.ok), allScenarioRequests, scenarioDurationMs, requireHistograms);
|
|
8144
|
+
const fail = aggregateMeasurementStats(items.map((value) => value.fail), allScenarioRequests, scenarioDurationMs, requireHistograms);
|
|
8145
|
+
const allMeasurement = requireHistograms
|
|
8146
|
+
? aggregateMeasurementStats([ok, fail], allScenarioRequests, scenarioDurationMs, true)
|
|
8147
|
+
: undefined;
|
|
6185
8148
|
const requestCount = ok.request.count + fail.request.count;
|
|
6186
8149
|
return {
|
|
6187
8150
|
scenarioName: items[0]?.scenarioName ?? "",
|
|
@@ -6197,14 +8160,53 @@ function aggregateStepStats(scenarios) {
|
|
|
6197
8160
|
statusCodes: aggregateStatusCodeCounts(ok.statusCodes, fail.statusCodes),
|
|
6198
8161
|
ok,
|
|
6199
8162
|
fail,
|
|
8163
|
+
...(allMeasurement ? { allMeasurement } : {}),
|
|
6200
8164
|
sortIndex: Math.min(...items.map((value) => value.sortIndex))
|
|
6201
8165
|
};
|
|
6202
8166
|
});
|
|
6203
8167
|
}
|
|
6204
|
-
function aggregateMeasurementStats(measurements, allRequestCount, durationMs) {
|
|
8168
|
+
function aggregateMeasurementStats(measurements, allRequestCount, durationMs, requireHistograms = false) {
|
|
6205
8169
|
if (!measurements.length) {
|
|
6206
8170
|
return buildMeasurementPlaceholder(0, allRequestCount, durationMs);
|
|
6207
8171
|
}
|
|
8172
|
+
if (measurements.every((measurement) => measurement.histogramSidecar)) {
|
|
8173
|
+
const latency = LoadStrikeHistogramV1.fromSidecar(measurements[0].histogramSidecar.latency);
|
|
8174
|
+
const size = LoadStrikeHistogramV1.fromSidecar(measurements[0].histogramSidecar.size);
|
|
8175
|
+
for (const measurement of measurements.slice(1)) {
|
|
8176
|
+
latency.merge(LoadStrikeHistogramV1.fromSidecar(measurement.histogramSidecar.latency));
|
|
8177
|
+
size.merge(LoadStrikeHistogramV1.fromSidecar(measurement.histogramSidecar.size));
|
|
8178
|
+
}
|
|
8179
|
+
const statusCodes = new Map();
|
|
8180
|
+
for (const measurement of measurements) {
|
|
8181
|
+
for (const status of measurement.statusCodes) {
|
|
8182
|
+
const key = `${status.statusCode}|${status.message}|${status.isError ? "1" : "0"}`;
|
|
8183
|
+
const current = statusCodes.get(key);
|
|
8184
|
+
if (current)
|
|
8185
|
+
current.count += status.count;
|
|
8186
|
+
else
|
|
8187
|
+
statusCodes.set(key, {
|
|
8188
|
+
statusCode: status.statusCode,
|
|
8189
|
+
message: status.message,
|
|
8190
|
+
isError: status.isError,
|
|
8191
|
+
count: status.count
|
|
8192
|
+
});
|
|
8193
|
+
}
|
|
8194
|
+
}
|
|
8195
|
+
const sumSidecar = (key) => Number(measurements.reduce((sum, measurement) => sum + BigInt(measurement.histogramSidecar[key]), 0n));
|
|
8196
|
+
return buildHistogramMeasurement({
|
|
8197
|
+
count: Number(latency.count),
|
|
8198
|
+
allBytes: measurements.reduce((sum, measurement) => sum + measurement.dataTransfer.allBytes, 0),
|
|
8199
|
+
latency,
|
|
8200
|
+
size,
|
|
8201
|
+
statusCodes,
|
|
8202
|
+
lessOrEq800: sumSidecar("lessOrEq80064"),
|
|
8203
|
+
more800Less1200: sumSidecar("more800Less120064"),
|
|
8204
|
+
moreOrEq1200: sumSidecar("moreOrEq120064")
|
|
8205
|
+
}, allRequestCount, durationMs);
|
|
8206
|
+
}
|
|
8207
|
+
if (requireHistograms) {
|
|
8208
|
+
throw new Error("Load Engine V2 aggregation requires canonical histogram state for every node measurement.");
|
|
8209
|
+
}
|
|
6208
8210
|
const weights = measurements.map((value) => value.request.count);
|
|
6209
8211
|
const totalCount = measurements.reduce((sum, value) => sum + value.request.count, 0);
|
|
6210
8212
|
return {
|
|
@@ -8026,6 +10028,68 @@ function normalizeOptionalReportFormats(value) {
|
|
|
8026
10028
|
const normalized = normalizeReportFormats(value);
|
|
8027
10029
|
return normalized.length ? normalized : undefined;
|
|
8028
10030
|
}
|
|
10031
|
+
function normalizeDeclaredStepNames(value) {
|
|
10032
|
+
if (!Array.isArray(value)) {
|
|
10033
|
+
throw new TypeError("Declared step names must be provided as text values.");
|
|
10034
|
+
}
|
|
10035
|
+
const seen = new Set();
|
|
10036
|
+
const normalized = [];
|
|
10037
|
+
for (const entry of value) {
|
|
10038
|
+
if (typeof entry !== "string" || !entry.trim()) {
|
|
10039
|
+
throw new Error("Declared step name must be non-empty text.");
|
|
10040
|
+
}
|
|
10041
|
+
const stepName = entry.trim();
|
|
10042
|
+
for (let index = 0; index < stepName.length; index += 1) {
|
|
10043
|
+
const code = stepName.charCodeAt(index);
|
|
10044
|
+
if (code >= 0xd800 && code <= 0xdbff) {
|
|
10045
|
+
const next = stepName.charCodeAt(index + 1);
|
|
10046
|
+
if (!(next >= 0xdc00 && next <= 0xdfff)) {
|
|
10047
|
+
throw new Error("Declared step name contains an invalid Unicode scalar.");
|
|
10048
|
+
}
|
|
10049
|
+
index += 1;
|
|
10050
|
+
}
|
|
10051
|
+
else if (code >= 0xdc00 && code <= 0xdfff) {
|
|
10052
|
+
throw new Error("Declared step name contains an invalid Unicode scalar.");
|
|
10053
|
+
}
|
|
10054
|
+
}
|
|
10055
|
+
if (!seen.has(stepName)) {
|
|
10056
|
+
seen.add(stepName);
|
|
10057
|
+
normalized.push(stepName);
|
|
10058
|
+
}
|
|
10059
|
+
}
|
|
10060
|
+
return normalized;
|
|
10061
|
+
}
|
|
10062
|
+
function resolveIterationObservationSettings(options) {
|
|
10063
|
+
return {
|
|
10064
|
+
flushIntervalMs: options.iterationObservationFlushIntervalSeconds === undefined
|
|
10065
|
+
? DEFAULT_ITERATION_OBSERVATION_SETTINGS.flushIntervalMs
|
|
10066
|
+
: options.iterationObservationFlushIntervalSeconds * 1000,
|
|
10067
|
+
maxBufferBytes: options.maxIterationObservationBufferBytes
|
|
10068
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.maxBufferBytes,
|
|
10069
|
+
maxObservationsPerBatch: options.maxIterationObservationsPerBatch
|
|
10070
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.maxObservationsPerBatch,
|
|
10071
|
+
maxBatchBytes: options.maxIterationObservationBatchBytes
|
|
10072
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.maxBatchBytes,
|
|
10073
|
+
sinkQueueDepth: options.iterationObservationSinkQueueDepth
|
|
10074
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.sinkQueueDepth,
|
|
10075
|
+
sinkParallelism: options.iterationObservationSinkParallelism
|
|
10076
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.sinkParallelism,
|
|
10077
|
+
drainTimeoutMs: options.iterationObservationDrainTimeoutSeconds === undefined
|
|
10078
|
+
? DEFAULT_ITERATION_OBSERVATION_SETTINGS.drainTimeoutMs
|
|
10079
|
+
: options.iterationObservationDrainTimeoutSeconds * 1000
|
|
10080
|
+
};
|
|
10081
|
+
}
|
|
10082
|
+
function validateRunContextIterationObservationSettings(values) {
|
|
10083
|
+
validateIterationObservationSettings(resolveIterationObservationSettings({
|
|
10084
|
+
iterationObservationFlushIntervalSeconds: values.IterationObservationFlushIntervalSeconds,
|
|
10085
|
+
maxIterationObservationBufferBytes: values.MaxIterationObservationBufferBytes,
|
|
10086
|
+
maxIterationObservationsPerBatch: values.MaxIterationObservationsPerBatch,
|
|
10087
|
+
maxIterationObservationBatchBytes: values.MaxIterationObservationBatchBytes,
|
|
10088
|
+
iterationObservationSinkQueueDepth: values.IterationObservationSinkQueueDepth,
|
|
10089
|
+
iterationObservationSinkParallelism: values.IterationObservationSinkParallelism,
|
|
10090
|
+
iterationObservationDrainTimeoutSeconds: values.IterationObservationDrainTimeoutSeconds
|
|
10091
|
+
}));
|
|
10092
|
+
}
|
|
8029
10093
|
function assertNoDisableLicenseEnforcementOption(value, source) {
|
|
8030
10094
|
if (value == null || typeof value !== "object" || Array.isArray(value)) {
|
|
8031
10095
|
return;
|
|
@@ -8044,12 +10108,121 @@ function normalizeRunContextCollectionShapes(values) {
|
|
|
8044
10108
|
TargetScenarios: normalizeOptionalStringArray(values.TargetScenarios),
|
|
8045
10109
|
AgentTargetScenarios: normalizeOptionalStringArray(values.AgentTargetScenarios),
|
|
8046
10110
|
CoordinatorTargetScenarios: normalizeOptionalStringArray(values.CoordinatorTargetScenarios),
|
|
10111
|
+
ExpectedAgentIds: normalizeOptionalStringArray(values.ExpectedAgentIds),
|
|
8047
10112
|
ReportFormats: normalizeOptionalReportFormats(values.ReportFormats)
|
|
8048
10113
|
};
|
|
8049
10114
|
validateNamedReportingSinks(normalized.ReportingSinks ?? []);
|
|
8050
10115
|
validateNamedWorkerPlugins(normalized.WorkerPlugins ?? []);
|
|
10116
|
+
validateLoadEngineV2Options(normalized.LoadEngineContractVersion, normalized.MaxInFlight);
|
|
10117
|
+
validateRunContextIterationObservationSettings(normalized);
|
|
8051
10118
|
return normalized;
|
|
8052
10119
|
}
|
|
10120
|
+
function normalizeAliasStringRecord(value) {
|
|
10121
|
+
const source = asAliasRecord(value);
|
|
10122
|
+
const output = {};
|
|
10123
|
+
for (const [key, entry] of Object.entries(source)) {
|
|
10124
|
+
output[key] = String(entry);
|
|
10125
|
+
}
|
|
10126
|
+
return output;
|
|
10127
|
+
}
|
|
10128
|
+
function attachGeneratorWarningAliases(value) {
|
|
10129
|
+
const source = asAliasRecord(value);
|
|
10130
|
+
const projected = {
|
|
10131
|
+
code: pickAliasString(source, "code", "Code"),
|
|
10132
|
+
...(hasAliasValue(source, "sinkName", "SinkName")
|
|
10133
|
+
? { sinkName: pickAliasString(source, "sinkName", "SinkName") }
|
|
10134
|
+
: {}),
|
|
10135
|
+
scenarioName: pickAliasString(source, "scenarioName", "ScenarioName"),
|
|
10136
|
+
...(hasAliasValue(source, "scenarioIndex", "ScenarioIndex")
|
|
10137
|
+
? { scenarioIndex: pickAliasNumber(source, "scenarioIndex", "ScenarioIndex") }
|
|
10138
|
+
: {}),
|
|
10139
|
+
simulationIndex: pickAliasNumber(source, "simulationIndex", "SimulationIndex"),
|
|
10140
|
+
...(hasAliasValue(source, "simulationKind", "SimulationKind")
|
|
10141
|
+
? { simulationKind: pickAliasString(source, "simulationKind", "SimulationKind") }
|
|
10142
|
+
: {}),
|
|
10143
|
+
count64: pickAliasString(source, "count64", "Count64"),
|
|
10144
|
+
message: pickAliasString(source, "message", "Message"),
|
|
10145
|
+
firstObservedUtcNs: pickAliasString(source, "firstObservedUtcNs", "FirstObservedUtcNs"),
|
|
10146
|
+
lastObservedUtcNs: pickAliasString(source, "lastObservedUtcNs", "LastObservedUtcNs")
|
|
10147
|
+
};
|
|
10148
|
+
return attachAliasMap(projected, {
|
|
10149
|
+
Code: "code",
|
|
10150
|
+
SinkName: "sinkName",
|
|
10151
|
+
ScenarioName: "scenarioName",
|
|
10152
|
+
ScenarioIndex: "scenarioIndex",
|
|
10153
|
+
SimulationIndex: "simulationIndex",
|
|
10154
|
+
SimulationKind: "simulationKind",
|
|
10155
|
+
Count64: "count64",
|
|
10156
|
+
Message: "message",
|
|
10157
|
+
FirstObservedUtcNs: "firstObservedUtcNs",
|
|
10158
|
+
LastObservedUtcNs: "lastObservedUtcNs"
|
|
10159
|
+
});
|
|
10160
|
+
}
|
|
10161
|
+
function normalizeSchedulerSegment(value) {
|
|
10162
|
+
const source = asAliasRecord(value);
|
|
10163
|
+
return {
|
|
10164
|
+
scenarioName: pickAliasString(source, "scenarioName", "ScenarioName"),
|
|
10165
|
+
scenarioIndex: pickAliasNumber(source, "scenarioIndex", "ScenarioIndex"),
|
|
10166
|
+
simulationIndex: pickAliasNumber(source, "simulationIndex", "SimulationIndex"),
|
|
10167
|
+
kind: pickAliasString(source, "kind", "Kind"),
|
|
10168
|
+
shardIndex: pickAliasNumber(source, "shardIndex", "ShardIndex"),
|
|
10169
|
+
shardCount: Math.max(pickAliasNumber(source, "shardCount", "ShardCount"), 1),
|
|
10170
|
+
plannedIterations64: pickAliasString(source, "plannedIterations64", "PlannedIterations64"),
|
|
10171
|
+
dueIterations64: pickAliasString(source, "dueIterations64", "DueIterations64"),
|
|
10172
|
+
startedIterations64: pickAliasString(source, "startedIterations64", "StartedIterations64"),
|
|
10173
|
+
completedIterations64: pickAliasString(source, "completedIterations64", "CompletedIterations64"),
|
|
10174
|
+
droppedIterations64: pickAliasString(source, "droppedIterations64", "DroppedIterations64"),
|
|
10175
|
+
unreachedIterations64: pickAliasString(source, "unreachedIterations64", "UnreachedIterations64"),
|
|
10176
|
+
requestedWorkerSlots64: pickAliasString(source, "requestedWorkerSlots64", "RequestedWorkerSlots64"),
|
|
10177
|
+
startedWorkerSlots64: pickAliasString(source, "startedWorkerSlots64", "StartedWorkerSlots64"),
|
|
10178
|
+
unavailableWorkerSlots64: pickAliasString(source, "unavailableWorkerSlots64", "UnavailableWorkerSlots64"),
|
|
10179
|
+
dropReasons: normalizeAliasStringRecord(pickAliasValue(source, "dropReasons", "DropReasons")),
|
|
10180
|
+
unavailableWorkerReasons: normalizeAliasStringRecord(pickAliasValue(source, "unavailableWorkerReasons", "UnavailableWorkerReasons")),
|
|
10181
|
+
deliveryPercent: pickAliasNumber(source, "deliveryPercent", "DeliveryPercent"),
|
|
10182
|
+
accountingComplete: pickAliasBoolean(source, "accountingComplete", "AccountingComplete")
|
|
10183
|
+
};
|
|
10184
|
+
}
|
|
10185
|
+
function normalizeSchedulerStats(value) {
|
|
10186
|
+
const source = asAliasRecord(value);
|
|
10187
|
+
return {
|
|
10188
|
+
configuredMaxInFlight: pickAliasNumber(source, "configuredMaxInFlight", "ConfiguredMaxInFlight"),
|
|
10189
|
+
maxInFlightObserved: pickAliasNumber(source, "maxInFlightObserved", "MaxInFlightObserved"),
|
|
10190
|
+
currentInFlight: pickAliasNumber(source, "currentInFlight", "CurrentInFlight"),
|
|
10191
|
+
segments: pickAliasArray(source, "segments", "Segments").map(normalizeSchedulerSegment)
|
|
10192
|
+
};
|
|
10193
|
+
}
|
|
10194
|
+
function normalizeObservationDeliveryStats(value) {
|
|
10195
|
+
const source = asAliasRecord(value);
|
|
10196
|
+
return attachAliasMap({
|
|
10197
|
+
lastBatchSequence64: pickAliasString(source, "lastBatchSequence64", "LastBatchSequence64"),
|
|
10198
|
+
capturedCount64: pickAliasString(source, "capturedCount64", "CapturedCount64"),
|
|
10199
|
+
deliveredCount64: pickAliasString(source, "deliveredCount64", "DeliveredCount64"),
|
|
10200
|
+
droppedBufferCount64: pickAliasString(source, "droppedBufferCount64", "DroppedBufferCount64"),
|
|
10201
|
+
droppedSinkCount64: pickAliasString(source, "droppedSinkCount64", "DroppedSinkCount64")
|
|
10202
|
+
}, {
|
|
10203
|
+
LastBatchSequence64: "lastBatchSequence64",
|
|
10204
|
+
CapturedCount64: "capturedCount64",
|
|
10205
|
+
DeliveredCount64: "deliveredCount64",
|
|
10206
|
+
DroppedBufferCount64: "droppedBufferCount64",
|
|
10207
|
+
DroppedSinkCount64: "droppedSinkCount64"
|
|
10208
|
+
});
|
|
10209
|
+
}
|
|
10210
|
+
function validateLoadEngineV2Options(contractVersion, maxInFlight) {
|
|
10211
|
+
if (contractVersion !== undefined && contractVersion !== 1 && contractVersion !== 2) {
|
|
10212
|
+
throw new RangeError("Load engine contract version must be either 1 or 2.");
|
|
10213
|
+
}
|
|
10214
|
+
if (maxInFlight !== undefined) {
|
|
10215
|
+
validateV2MaxInFlight(contractVersion, maxInFlight);
|
|
10216
|
+
}
|
|
10217
|
+
}
|
|
10218
|
+
function validateV2MaxInFlight(contractVersion, maxInFlight) {
|
|
10219
|
+
if (contractVersion !== 2) {
|
|
10220
|
+
throw new Error("MaxInFlight is available only when Load Engine V2 is selected.");
|
|
10221
|
+
}
|
|
10222
|
+
if (!Number.isSafeInteger(maxInFlight) || maxInFlight < 1 || maxInFlight > 1000000) {
|
|
10223
|
+
throw new RangeError("MaxInFlight must be an integer from 1 through 1000000.");
|
|
10224
|
+
}
|
|
10225
|
+
}
|
|
8053
10226
|
function normalizeRunnerOptionCollectionShapes(options) {
|
|
8054
10227
|
assertNoDisableLicenseEnforcementOption(options, "LoadStrikeRunner");
|
|
8055
10228
|
const normalized = {
|
|
@@ -8057,10 +10230,13 @@ function normalizeRunnerOptionCollectionShapes(options) {
|
|
|
8057
10230
|
targetScenarios: normalizeOptionalStringArray(options.targetScenarios),
|
|
8058
10231
|
agentTargetScenarios: normalizeOptionalStringArray(options.agentTargetScenarios),
|
|
8059
10232
|
coordinatorTargetScenarios: normalizeOptionalStringArray(options.coordinatorTargetScenarios),
|
|
10233
|
+
expectedAgentIds: normalizeOptionalStringArray(options.expectedAgentIds),
|
|
8060
10234
|
reportFormats: normalizeOptionalReportFormats(options.reportFormats)
|
|
8061
10235
|
};
|
|
8062
10236
|
validateNamedReportingSinks(normalized.reportingSinks ?? []);
|
|
8063
10237
|
validateNamedWorkerPlugins(normalized.workerPlugins ?? []);
|
|
10238
|
+
validateLoadEngineV2Options(normalized.loadEngineContractVersion, normalized.maxInFlight);
|
|
10239
|
+
validateIterationObservationSettings(resolveIterationObservationSettings(normalized));
|
|
8064
10240
|
return normalized;
|
|
8065
10241
|
}
|
|
8066
10242
|
function normalizedRuntimePolicyErrorMode(value) {
|
|
@@ -8124,6 +10300,7 @@ function extractContextOverridesFromConfig(config) {
|
|
|
8124
10300
|
setString("ReportFileName", "ReportFileName", "LoadStrike:ReportFileName");
|
|
8125
10301
|
setString("ClusterId", "ClusterId", "LoadStrike:ClusterId");
|
|
8126
10302
|
setString("AgentGroup", "AgentGroup", "LoadStrike:AgentGroup");
|
|
10303
|
+
setString("AgentId", "AgentId", "LoadStrike:AgentId");
|
|
8127
10304
|
setString("NatsServerUrl", "NatsServerUrl", "LoadStrike:NatsServerUrl");
|
|
8128
10305
|
setString("RunnerKey", "RunnerKey", "LoadStrike:RunnerKey");
|
|
8129
10306
|
setString("RuntimePolicyErrorMode", "RuntimePolicyErrorMode", "LoadStrike:RuntimePolicyErrorMode");
|
|
@@ -8153,6 +10330,13 @@ function extractContextOverridesFromConfig(config) {
|
|
|
8153
10330
|
}
|
|
8154
10331
|
}
|
|
8155
10332
|
setPositiveNumber("ReportingIntervalSeconds", "ReportingIntervalSeconds", "LoadStrike:ReportingIntervalSeconds");
|
|
10333
|
+
setPositiveNumber("IterationObservationFlushIntervalSeconds", "IterationObservationFlushInterval", "IterationObservationFlushIntervalSeconds", "LoadStrike:IterationObservationFlushInterval");
|
|
10334
|
+
setPositiveNumber("MaxIterationObservationBufferBytes", "MaxIterationObservationBufferBytes", "LoadStrike:MaxIterationObservationBufferBytes");
|
|
10335
|
+
setPositiveNumber("MaxIterationObservationsPerBatch", "MaxIterationObservationsPerBatch", "LoadStrike:MaxIterationObservationsPerBatch");
|
|
10336
|
+
setPositiveNumber("MaxIterationObservationBatchBytes", "MaxIterationObservationBatchBytes", "LoadStrike:MaxIterationObservationBatchBytes");
|
|
10337
|
+
setPositiveNumber("IterationObservationSinkQueueDepth", "IterationObservationSinkQueueDepth", "LoadStrike:IterationObservationSinkQueueDepth");
|
|
10338
|
+
setPositiveNumber("IterationObservationSinkParallelism", "IterationObservationSinkParallelism", "LoadStrike:IterationObservationSinkParallelism");
|
|
10339
|
+
setPositiveNumber("IterationObservationDrainTimeoutSeconds", "IterationObservationDrainTimeout", "IterationObservationDrainTimeoutSeconds", "LoadStrike:IterationObservationDrainTimeout");
|
|
8156
10340
|
setPositiveNumber("ScenarioCompletionTimeoutSeconds", "ScenarioCompletionTimeoutSeconds", "LoadStrike:ScenarioCompletionTimeoutSeconds");
|
|
8157
10341
|
setPositiveNumber("ClusterCommandTimeoutSeconds", "ClusterCommandTimeoutSeconds", "LoadStrike:ClusterCommandTimeoutSeconds");
|
|
8158
10342
|
setPositiveNumber("LicenseValidationTimeoutSeconds", "LicenseValidationTimeoutSeconds", "LoadStrike:LicenseValidation:TimeoutSeconds");
|
|
@@ -8160,6 +10344,14 @@ function extractContextOverridesFromConfig(config) {
|
|
|
8160
10344
|
if (reportingIntervalMs > 0) {
|
|
8161
10345
|
patch.ReportingIntervalSeconds = reportingIntervalMs / 1000;
|
|
8162
10346
|
}
|
|
10347
|
+
const observationFlushIntervalMs = toNumber(pick("IterationObservationFlushIntervalMs", "LoadStrike:IterationObservationFlushIntervalMs"));
|
|
10348
|
+
if (observationFlushIntervalMs > 0) {
|
|
10349
|
+
patch.IterationObservationFlushIntervalSeconds = observationFlushIntervalMs / 1000;
|
|
10350
|
+
}
|
|
10351
|
+
const observationDrainTimeoutMs = toNumber(pick("IterationObservationDrainTimeoutMs", "LoadStrike:IterationObservationDrainTimeoutMs"));
|
|
10352
|
+
if (observationDrainTimeoutMs > 0) {
|
|
10353
|
+
patch.IterationObservationDrainTimeoutSeconds = observationDrainTimeoutMs / 1000;
|
|
10354
|
+
}
|
|
8163
10355
|
const scenarioCompletionTimeoutMs = toNumber(pick("ScenarioCompletionTimeoutMs", "LoadStrike:ScenarioCompletionTimeoutMs"));
|
|
8164
10356
|
if (scenarioCompletionTimeoutMs > 0) {
|
|
8165
10357
|
patch.ScenarioCompletionTimeoutSeconds = scenarioCompletionTimeoutMs / 1000;
|
|
@@ -8204,6 +10396,10 @@ function extractContextOverridesFromConfig(config) {
|
|
|
8204
10396
|
if (agentTargetScenarios.length) {
|
|
8205
10397
|
patch.AgentTargetScenarios = agentTargetScenarios;
|
|
8206
10398
|
}
|
|
10399
|
+
const expectedAgentIds = normalizeStringArray(pick("ExpectedAgentIds", "LoadStrike:ExpectedAgentIds"));
|
|
10400
|
+
if (expectedAgentIds.length) {
|
|
10401
|
+
patch.ExpectedAgentIds = expectedAgentIds;
|
|
10402
|
+
}
|
|
8207
10403
|
const coordinatorTargetScenarios = normalizeStringArray(pick("CoordinatorTargetScenarios", "LoadStrike:CoordinatorTargetScenarios"));
|
|
8208
10404
|
if (coordinatorTargetScenarios.length) {
|
|
8209
10405
|
patch.CoordinatorTargetScenarios = coordinatorTargetScenarios;
|
|
@@ -8266,10 +10462,14 @@ function toRunContext(options) {
|
|
|
8266
10462
|
const normalized = normalizeRunnerOptionCollectionShapes(options);
|
|
8267
10463
|
return {
|
|
8268
10464
|
ConsoleMetricsEnabled: normalized.displayConsoleMetrics,
|
|
10465
|
+
LoadEngineContractVersion: normalized.loadEngineContractVersion,
|
|
10466
|
+
MaxInFlight: normalized.maxInFlight,
|
|
8269
10467
|
NodeType: normalized.nodeType,
|
|
8270
10468
|
LocalDevClusterEnabled: normalized.localDevClusterEnabled,
|
|
8271
10469
|
AgentGroup: normalized.agentGroup,
|
|
8272
10470
|
AgentsCount: normalized.agentsCount,
|
|
10471
|
+
AgentId: normalized.agentId,
|
|
10472
|
+
ExpectedAgentIds: normalized.expectedAgentIds,
|
|
8273
10473
|
TargetScenarios: normalized.targetScenarios,
|
|
8274
10474
|
AgentTargetScenarios: normalized.agentTargetScenarios,
|
|
8275
10475
|
CoordinatorTargetScenarios: normalized.coordinatorTargetScenarios,
|
|
@@ -8288,6 +10488,13 @@ function toRunContext(options) {
|
|
|
8288
10488
|
ReportFolderPath: normalized.reportFolderPath,
|
|
8289
10489
|
ReportFormats: normalized.reportFormats,
|
|
8290
10490
|
ReportingIntervalSeconds: normalized.reportingIntervalSeconds,
|
|
10491
|
+
IterationObservationFlushIntervalSeconds: normalized.iterationObservationFlushIntervalSeconds,
|
|
10492
|
+
MaxIterationObservationBufferBytes: normalized.maxIterationObservationBufferBytes,
|
|
10493
|
+
MaxIterationObservationsPerBatch: normalized.maxIterationObservationsPerBatch,
|
|
10494
|
+
MaxIterationObservationBatchBytes: normalized.maxIterationObservationBatchBytes,
|
|
10495
|
+
IterationObservationSinkQueueDepth: normalized.iterationObservationSinkQueueDepth,
|
|
10496
|
+
IterationObservationSinkParallelism: normalized.iterationObservationSinkParallelism,
|
|
10497
|
+
IterationObservationDrainTimeoutSeconds: normalized.iterationObservationDrainTimeoutSeconds,
|
|
8291
10498
|
MinimumLogLevel: normalized.minimumLogLevel,
|
|
8292
10499
|
LoggerConfig: normalized.loggerConfig,
|
|
8293
10500
|
ReportingSinks: normalized.reportingSinks,
|
|
@@ -8302,7 +10509,9 @@ function toRunContext(options) {
|
|
|
8302
10509
|
CustomSettings: normalized.customSettings,
|
|
8303
10510
|
GlobalCustomSettings: normalized.globalCustomSettings,
|
|
8304
10511
|
AgentExecutionToken: normalized.agentExecutionToken,
|
|
8305
|
-
AgentCommandId: normalized.agentCommandId
|
|
10512
|
+
AgentCommandId: normalized.agentCommandId,
|
|
10513
|
+
ClusterShardIndex: normalized.clusterShardIndex,
|
|
10514
|
+
ClusterShardCount: normalized.clusterShardCount
|
|
8306
10515
|
};
|
|
8307
10516
|
}
|
|
8308
10517
|
class RuntimePolicyCallbackError extends Error {
|
|
@@ -8390,11 +10599,15 @@ function looksLikeRunContext(value) {
|
|
|
8390
10599
|
const keys = new Set(Object.keys(value));
|
|
8391
10600
|
return [
|
|
8392
10601
|
"ConsoleMetricsEnabled",
|
|
10602
|
+
"LoadEngineContractVersion",
|
|
10603
|
+
"MaxInFlight",
|
|
8393
10604
|
"LocalDevClusterEnabled",
|
|
8394
10605
|
"ConfigPath",
|
|
8395
10606
|
"InfraConfigPath",
|
|
8396
10607
|
"AgentGroup",
|
|
8397
10608
|
"AgentsCount",
|
|
10609
|
+
"AgentId",
|
|
10610
|
+
"ExpectedAgentIds",
|
|
8398
10611
|
"AgentTargetScenarios",
|
|
8399
10612
|
"ClusterId",
|
|
8400
10613
|
"CoordinatorTargetScenarios",
|
|
@@ -8409,6 +10622,13 @@ function looksLikeRunContext(value) {
|
|
|
8409
10622
|
"ReportFolderPath",
|
|
8410
10623
|
"ReportFormats",
|
|
8411
10624
|
"ReportingIntervalSeconds",
|
|
10625
|
+
"IterationObservationFlushIntervalSeconds",
|
|
10626
|
+
"MaxIterationObservationBufferBytes",
|
|
10627
|
+
"MaxIterationObservationsPerBatch",
|
|
10628
|
+
"MaxIterationObservationBatchBytes",
|
|
10629
|
+
"IterationObservationSinkQueueDepth",
|
|
10630
|
+
"IterationObservationSinkParallelism",
|
|
10631
|
+
"IterationObservationDrainTimeoutSeconds",
|
|
8412
10632
|
"ReportingSinks",
|
|
8413
10633
|
"SinkRetryCount",
|
|
8414
10634
|
"SinkRetryBackoffMs",
|
|
@@ -8550,6 +10770,18 @@ function resolveSinkSaveRunResult(sink) {
|
|
|
8550
10770
|
? method.bind(sink)
|
|
8551
10771
|
: undefined;
|
|
8552
10772
|
}
|
|
10773
|
+
function resolveSinkSaveIterationBatch(sink) {
|
|
10774
|
+
const method = sink.saveIterationBatch ?? sink.SaveIterationBatch;
|
|
10775
|
+
return typeof method === "function"
|
|
10776
|
+
? method.bind(sink)
|
|
10777
|
+
: undefined;
|
|
10778
|
+
}
|
|
10779
|
+
function resolveSinkCompleteIterationObservationStream(sink) {
|
|
10780
|
+
const method = sink.completeIterationObservationStream ?? sink.CompleteIterationObservationStream;
|
|
10781
|
+
return typeof method === "function"
|
|
10782
|
+
? method.bind(sink)
|
|
10783
|
+
: undefined;
|
|
10784
|
+
}
|
|
8553
10785
|
function resolveSinkStop(sink) {
|
|
8554
10786
|
const method = sink.stop ?? sink.Stop;
|
|
8555
10787
|
return typeof method === "function"
|
|
@@ -8967,13 +11199,17 @@ export const __loadstrikeTestExports = {
|
|
|
8967
11199
|
ManagedScenarioTrackingRuntime,
|
|
8968
11200
|
ScenarioStatsAccumulator,
|
|
8969
11201
|
StepStatsAccumulator,
|
|
11202
|
+
planRuntimeClusterAssignments,
|
|
8970
11203
|
TrackingFieldSelector,
|
|
8971
11204
|
addCorrelationRow,
|
|
8972
11205
|
addFailedResponseRow,
|
|
8973
11206
|
aggregateNodeStats,
|
|
11207
|
+
aggregateMeasurementStats,
|
|
8974
11208
|
asRecord,
|
|
8975
11209
|
assertNoDisableLicenseEnforcementOption,
|
|
8976
11210
|
buildEmptyNodeStats,
|
|
11211
|
+
buildRuntimeLoadEngineV2HistogramArtifact,
|
|
11212
|
+
buildRuntimeLoadEngineV2Plan,
|
|
8977
11213
|
buildGroupedCorrelationRows,
|
|
8978
11214
|
buildMeasurementPlaceholder,
|
|
8979
11215
|
buildRichHtmlReport,
|
|
@@ -9046,7 +11282,9 @@ export const __loadstrikeTestExports = {
|
|
|
9046
11282
|
resolveSinkName,
|
|
9047
11283
|
resolveSinkSaveRealtimeMetrics,
|
|
9048
11284
|
resolveSinkSaveRealtimeStats,
|
|
11285
|
+
resolveSinkSaveIterationBatch,
|
|
9049
11286
|
resolveSinkSaveRunResult,
|
|
11287
|
+
resolveSinkCompleteIterationObservationStream,
|
|
9050
11288
|
resolveSinkStart,
|
|
9051
11289
|
resolveSinkStop,
|
|
9052
11290
|
resolveWorkerPlugins,
|
|
@@ -9067,6 +11305,7 @@ export const __loadstrikeTestExports = {
|
|
|
9067
11305
|
tryParseNodeTypeToken,
|
|
9068
11306
|
tryReadConfigValue,
|
|
9069
11307
|
validateNamedReportingSinks,
|
|
11308
|
+
validateLoadEngineV2ScenarioFeatures,
|
|
9070
11309
|
validateRegisteredScenarios,
|
|
9071
11310
|
validateRuntimeRedisCorrelationStoreConfiguration,
|
|
9072
11311
|
validateRuntimeTrackingConfiguration,
|