@loadstrike/loadstrike-sdk 1.0.30201 → 1.0.31001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/cjs/cluster.js +2417 -7
- package/dist/cjs/index.js +13 -2
- package/dist/cjs/iteration-observation-diagnostics.js +513 -0
- package/dist/cjs/iteration-observations.js +884 -0
- package/dist/cjs/load-engine-v2.js +966 -0
- package/dist/cjs/local.js +128 -30
- package/dist/cjs/reporting.js +148 -19
- package/dist/cjs/runtime.js +2514 -196
- package/dist/cjs/sink-retry-policy.js +52 -0
- package/dist/cjs/sinks.js +580 -23
- package/dist/cjs/transports.js +154 -163
- package/dist/esm/cluster.js +2386 -7
- package/dist/esm/index.js +1 -0
- package/dist/esm/iteration-observation-diagnostics.js +508 -0
- package/dist/esm/iteration-observations.js +871 -0
- package/dist/esm/load-engine-v2.js +942 -0
- package/dist/esm/local.js +128 -30
- package/dist/esm/reporting.js +148 -19
- package/dist/esm/runtime.js +2515 -197
- package/dist/esm/sink-retry-policy.js +44 -0
- package/dist/esm/sinks.js +580 -23
- package/dist/esm/transports.js +154 -163
- package/dist/types/cluster.d.ts +379 -1
- package/dist/types/index.d.ts +3 -1
- package/dist/types/iteration-observation-diagnostics.d.ts +21 -0
- package/dist/types/iteration-observations.d.ts +230 -0
- package/dist/types/load-engine-v2.d.ts +147 -0
- package/dist/types/runtime.d.ts +216 -8
- package/dist/types/sink-retry-policy.d.ts +9 -0
- package/dist/types/sinks.d.ts +73 -0
- package/dist/types/transports.d.ts +6 -8
- package/package.json +3 -4
package/dist/esm/runtime.js
CHANGED
|
@@ -3,11 +3,17 @@ import { randomBytes, randomUUID } from "node:crypto";
|
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import { resolve } from "node:path";
|
|
5
5
|
import { LoadStrikeLocalClient } from "./local.js";
|
|
6
|
-
import { DistributedClusterAgent, DistributedClusterCoordinator } from "./cluster.js";
|
|
6
|
+
import { DistributedClusterAgent, DistributedClusterCoordinator, buildLoadEngineV2ReservedStepOtherIdentityKey, buildLoadEngineV2GlobalInvocationId, buildLoadEngineV2ScenarioIdentityKey, buildLoadEngineV2SchedulerIdentityKey, buildLoadEngineV2StatusIdentityKey, buildLoadEngineV2StepIdentityKey, resolveLoadEngineV2StepIdentityKey, buildLoadEngineV2Plan } from "./cluster.js";
|
|
7
7
|
import { CorrelationStoreConfiguration, CrossPlatformTrackingRuntime, RedisCorrelationStore, RedisCorrelationStoreOptions, TrackingFieldSelector } from "./correlation.js";
|
|
8
8
|
import { EndpointAdapterFactory, LOADSTRIKE_TRACE_ID_TRACKING_FIELD } from "./transports.js";
|
|
9
9
|
import { buildDotnetCsvReport, buildDotnetHtmlReport, buildDotnetMarkdownReport, buildDotnetTxtReport } from "./reporting.js";
|
|
10
10
|
import { PortalReportingSink, cloneReportingSinkForRun } from "./sinks.js";
|
|
11
|
+
import { LoadEngineV2ExecutionBudget, buildLoadEngineV2TrafficMixSeedId, LoadStrikeHistogramV1, parseLoadEngineV2HistogramArtifact, serializeLoadEngineV2HistogramArtifact, classifyLoadEngineV2Arrival, loadEngineV2FixedArrivalCount, loadEngineV2FixedDeadlineNs, loadEngineV2LatenessToleranceNs, loadEngineV2TrafficMixLaneUnitCount, loadEngineV2TrafficMixOwnedUnits, planRampingInjectionDeadlines, planRampingConstantDeadlines, planRandomInjectionDeadlines, fnv1a32 } from "./load-engine-v2.js";
|
|
12
|
+
import { DEFAULT_ITERATION_OBSERVATION_SETTINGS, IterationObservationReporter, createIterationObservation, createIterationStepObservation, utcNowNs, validateIterationObservationSettings } from "./iteration-observations.js";
|
|
13
|
+
import { logIterationObservationFailure, logIterationObservationRecovery } from "./iteration-observation-diagnostics.js";
|
|
14
|
+
import { normalizeSinkRetryBackoffMs, normalizeSinkRetryCount, sinkRetryDelayMs, waitForSinkRetryDelay } from "./sink-retry-policy.js";
|
|
15
|
+
const LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS = Symbol("loadstrike.load-engine-v2.scheduler-distributions");
|
|
16
|
+
const PORTAL_RUN_TOKEN_PROVIDER = Symbol.for("loadstrike.internal.portal-run-token-provider");
|
|
11
17
|
export const LoadStrikeNodeType = {
|
|
12
18
|
SingleNode: "SingleNode",
|
|
13
19
|
Coordinator: "Coordinator",
|
|
@@ -139,12 +145,20 @@ export class LoadStrikePluginData {
|
|
|
139
145
|
}
|
|
140
146
|
}
|
|
141
147
|
class MeasurementAccumulator {
|
|
142
|
-
constructor() {
|
|
148
|
+
constructor(useHistogram = false) {
|
|
149
|
+
this.useHistogram = useHistogram;
|
|
143
150
|
this.count = 0;
|
|
144
151
|
this.allBytes = 0;
|
|
145
152
|
this.latenciesMs = [];
|
|
146
153
|
this.sizesBytes = [];
|
|
154
|
+
this.latencyLessOrEq800 = 0;
|
|
155
|
+
this.latencyMore800Less1200 = 0;
|
|
156
|
+
this.latencyMoreOrEq1200 = 0;
|
|
147
157
|
this.statusCodes = new Map();
|
|
158
|
+
if (useHistogram) {
|
|
159
|
+
this.latencyHistogram = new LoadStrikeHistogramV1();
|
|
160
|
+
this.sizeHistogram = new LoadStrikeHistogramV1();
|
|
161
|
+
}
|
|
148
162
|
}
|
|
149
163
|
get Count() {
|
|
150
164
|
return this.count;
|
|
@@ -161,8 +175,20 @@ class MeasurementAccumulator {
|
|
|
161
175
|
const key = `${statusCode}|${message}|${reply.isSuccess ? "ok" : "fail"}`;
|
|
162
176
|
this.count += 1;
|
|
163
177
|
this.allBytes += sizeBytes;
|
|
164
|
-
this.
|
|
165
|
-
|
|
178
|
+
if (this.useHistogram) {
|
|
179
|
+
this.latencyHistogram.record(normalizeLatencyMicroseconds(latencyMs));
|
|
180
|
+
this.sizeHistogram.record(normalizeHistogramInteger(sizeBytes, "Response size"));
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
this.latenciesMs.push(latencyMs);
|
|
184
|
+
this.sizesBytes.push(sizeBytes);
|
|
185
|
+
}
|
|
186
|
+
if (latencyMs <= 800)
|
|
187
|
+
this.latencyLessOrEq800 += 1;
|
|
188
|
+
else if (latencyMs < 1200)
|
|
189
|
+
this.latencyMore800Less1200 += 1;
|
|
190
|
+
else
|
|
191
|
+
this.latencyMoreOrEq1200 += 1;
|
|
166
192
|
const existing = this.statusCodes.get(key);
|
|
167
193
|
if (existing) {
|
|
168
194
|
existing.count += 1;
|
|
@@ -180,6 +206,9 @@ class MeasurementAccumulator {
|
|
|
180
206
|
* Use this when all builder inputs are ready to be materialized.
|
|
181
207
|
*/
|
|
182
208
|
build(allRequestCount, durationMs) {
|
|
209
|
+
if (this.useHistogram) {
|
|
210
|
+
return buildHistogramMeasurement(this.histogramSnapshot(), allRequestCount, durationMs);
|
|
211
|
+
}
|
|
183
212
|
const count = this.count;
|
|
184
213
|
const totalDurationMs = Math.max(durationMs, 0);
|
|
185
214
|
const latencyValues = [...this.latenciesMs];
|
|
@@ -228,14 +257,274 @@ class MeasurementAccumulator {
|
|
|
228
257
|
statusCodes
|
|
229
258
|
};
|
|
230
259
|
}
|
|
260
|
+
buildCombined(other, allRequestCount, durationMs) {
|
|
261
|
+
if (!this.useHistogram || !other.useHistogram) {
|
|
262
|
+
throw new Error("Combined measurements require Load Engine V2 histograms.");
|
|
263
|
+
}
|
|
264
|
+
const left = this.histogramSnapshot();
|
|
265
|
+
const right = other.histogramSnapshot();
|
|
266
|
+
left.latency.merge(right.latency);
|
|
267
|
+
left.size.merge(right.size);
|
|
268
|
+
for (const [key, value] of right.statusCodes) {
|
|
269
|
+
const existing = left.statusCodes.get(key);
|
|
270
|
+
if (existing)
|
|
271
|
+
existing.count += value.count;
|
|
272
|
+
else
|
|
273
|
+
left.statusCodes.set(key, { ...value });
|
|
274
|
+
}
|
|
275
|
+
return buildHistogramMeasurement({
|
|
276
|
+
count: left.count + right.count,
|
|
277
|
+
allBytes: left.allBytes + right.allBytes,
|
|
278
|
+
latency: left.latency,
|
|
279
|
+
size: left.size,
|
|
280
|
+
statusCodes: left.statusCodes,
|
|
281
|
+
lessOrEq800: left.lessOrEq800 + right.lessOrEq800,
|
|
282
|
+
more800Less1200: left.more800Less1200 + right.more800Less1200,
|
|
283
|
+
moreOrEq1200: left.moreOrEq1200 + right.moreOrEq1200
|
|
284
|
+
}, allRequestCount, durationMs);
|
|
285
|
+
}
|
|
286
|
+
histogramSnapshot() {
|
|
287
|
+
return {
|
|
288
|
+
count: this.count,
|
|
289
|
+
allBytes: this.allBytes,
|
|
290
|
+
latency: this.latencyHistogram.clone(),
|
|
291
|
+
size: this.sizeHistogram.clone(),
|
|
292
|
+
statusCodes: new Map(Array.from(this.statusCodes, ([key, value]) => [key, { ...value }])),
|
|
293
|
+
lessOrEq800: this.latencyLessOrEq800,
|
|
294
|
+
more800Less1200: this.latencyMore800Less1200,
|
|
295
|
+
moreOrEq1200: this.latencyMoreOrEq1200
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
function buildHistogramMeasurement(snapshot, allRequestCount, durationMs) {
|
|
300
|
+
const count = snapshot.count;
|
|
301
|
+
const totalDurationMs = Math.max(durationMs, 0);
|
|
302
|
+
const latency = snapshot.latency;
|
|
303
|
+
const size = snapshot.size;
|
|
304
|
+
return {
|
|
305
|
+
count64: latency.count.toString(),
|
|
306
|
+
distributionMode: latency.mode === "quantized-v1" || size.mode === "quantized-v1"
|
|
307
|
+
? "quantized-v1"
|
|
308
|
+
: "exact-normalized",
|
|
309
|
+
maxRelativeError: Math.max(latency.maxRelativeError, size.maxRelativeError),
|
|
310
|
+
histogramSidecar: {
|
|
311
|
+
latency: latency.toSidecar(),
|
|
312
|
+
size: size.toSidecar(),
|
|
313
|
+
allBytes64: size.exactTotal.toString(),
|
|
314
|
+
lessOrEq80064: snapshot.lessOrEq800.toString(),
|
|
315
|
+
more800Less120064: snapshot.more800Less1200.toString(),
|
|
316
|
+
moreOrEq120064: snapshot.moreOrEq1200.toString()
|
|
317
|
+
},
|
|
318
|
+
request: {
|
|
319
|
+
count,
|
|
320
|
+
percent: allRequestCount <= 0 ? 0 : Math.round((100 * count) / allRequestCount),
|
|
321
|
+
rps: totalDurationMs <= 0 ? 0 : count / (totalDurationMs / 1000)
|
|
322
|
+
},
|
|
323
|
+
dataTransfer: {
|
|
324
|
+
allBytes: snapshot.allBytes,
|
|
325
|
+
allBytes64: size.exactTotal.toString(),
|
|
326
|
+
minBytes: Number(size.minimum),
|
|
327
|
+
maxBytes: Number(size.maximum),
|
|
328
|
+
meanBytes: Math.round(size.mean),
|
|
329
|
+
percent50: Number(size.percentile(0.5)),
|
|
330
|
+
percent75: Number(size.percentile(0.75)),
|
|
331
|
+
percent95: Number(size.percentile(0.95)),
|
|
332
|
+
percent99: Number(size.percentile(0.99)),
|
|
333
|
+
percent100: Number(size.percentile(1)),
|
|
334
|
+
stdDev: size.populationStandardDeviation
|
|
335
|
+
},
|
|
336
|
+
latency: {
|
|
337
|
+
latencyCount: {
|
|
338
|
+
lessOrEq800: snapshot.lessOrEq800,
|
|
339
|
+
more800Less1200: snapshot.more800Less1200,
|
|
340
|
+
moreOrEq1200: snapshot.moreOrEq1200
|
|
341
|
+
},
|
|
342
|
+
minMs: Number(latency.minimum) / 1000,
|
|
343
|
+
maxMs: Number(latency.maximum) / 1000,
|
|
344
|
+
meanMs: latency.mean / 1000,
|
|
345
|
+
percent50: Number(latency.percentile(0.5)) / 1000,
|
|
346
|
+
percent75: Number(latency.percentile(0.75)) / 1000,
|
|
347
|
+
percent95: Number(latency.percentile(0.95)) / 1000,
|
|
348
|
+
percent99: Number(latency.percentile(0.99)) / 1000,
|
|
349
|
+
percent100: Number(latency.percentile(1)) / 1000,
|
|
350
|
+
stdDev: latency.populationStandardDeviation / 1000
|
|
351
|
+
},
|
|
352
|
+
statusCodes: Array.from(snapshot.statusCodes.values())
|
|
353
|
+
.sort((left, right) => right.count - left.count)
|
|
354
|
+
.map((value) => ({
|
|
355
|
+
count: value.count,
|
|
356
|
+
isError: value.isError,
|
|
357
|
+
message: value.message,
|
|
358
|
+
percent: count <= 0 ? 0 : Math.round((100 * value.count) / count),
|
|
359
|
+
statusCode: value.statusCode
|
|
360
|
+
}))
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
function normalizeLatencyMicroseconds(latencyMs) {
|
|
364
|
+
return normalizeHistogramInteger(Math.max(latencyMs, 0) * 1000, "Latency");
|
|
365
|
+
}
|
|
366
|
+
function normalizeRawObservationLatencyMicroseconds(latencyMs) {
|
|
367
|
+
const maximum = 9223372036854775807n;
|
|
368
|
+
if (!Number.isFinite(latencyMs) || latencyMs <= 0) {
|
|
369
|
+
return 0n;
|
|
370
|
+
}
|
|
371
|
+
const microseconds = latencyMs * 1000;
|
|
372
|
+
if (!Number.isFinite(microseconds) || microseconds >= Number(maximum)) {
|
|
373
|
+
return maximum;
|
|
374
|
+
}
|
|
375
|
+
return BigInt(Math.max(0, Math.round(microseconds)));
|
|
376
|
+
}
|
|
377
|
+
function normalizeHistogramInteger(value, name) {
|
|
378
|
+
if (!Number.isFinite(value) || value > Number(9223372036854775807n)) {
|
|
379
|
+
throw new RangeError(`${name} is outside the supported histogram range.`);
|
|
380
|
+
}
|
|
381
|
+
return BigInt(Math.max(0, Math.round(value)));
|
|
382
|
+
}
|
|
383
|
+
class LoadEngineV2Telemetry {
|
|
384
|
+
constructor(budget) {
|
|
385
|
+
this.budget = budget;
|
|
386
|
+
this.mutableSegments = [];
|
|
387
|
+
this.warnings = new Map();
|
|
388
|
+
}
|
|
389
|
+
createSegment(scenarioName, scenarioIndex, simulationIndex, kind, shardIndex, shardCount) {
|
|
390
|
+
const segment = {
|
|
391
|
+
scenarioName,
|
|
392
|
+
scenarioIndex,
|
|
393
|
+
simulationIndex,
|
|
394
|
+
kind,
|
|
395
|
+
shardIndex,
|
|
396
|
+
shardCount,
|
|
397
|
+
planned: 0n,
|
|
398
|
+
due: 0n,
|
|
399
|
+
started: 0n,
|
|
400
|
+
completed: 0n,
|
|
401
|
+
dropped: 0n,
|
|
402
|
+
unreached: 0n,
|
|
403
|
+
requestedWorkers: 0n,
|
|
404
|
+
startedWorkers: 0n,
|
|
405
|
+
unavailableWorkers: 0n,
|
|
406
|
+
dropReasons: new Map(),
|
|
407
|
+
unavailableWorkerReasons: new Map(),
|
|
408
|
+
decisionLag: new LoadStrikeHistogramV1(),
|
|
409
|
+
startLag: new LoadStrikeHistogramV1(),
|
|
410
|
+
accountingComplete: false
|
|
411
|
+
};
|
|
412
|
+
this.mutableSegments.push(segment);
|
|
413
|
+
return segment;
|
|
414
|
+
}
|
|
415
|
+
recordWarning(code, segment, count) {
|
|
416
|
+
if (count <= 0n)
|
|
417
|
+
return;
|
|
418
|
+
const key = `${code}\n${segment.scenarioIndex}\n${segment.simulationIndex}`;
|
|
419
|
+
const nowNs = BigInt(Date.now()) * 1000000n;
|
|
420
|
+
const existing = this.warnings.get(key);
|
|
421
|
+
if (existing) {
|
|
422
|
+
existing.count += count;
|
|
423
|
+
existing.lastObservedUtcNs = nowNs;
|
|
424
|
+
}
|
|
425
|
+
else {
|
|
426
|
+
this.warnings.set(key, {
|
|
427
|
+
code,
|
|
428
|
+
scenarioName: segment.scenarioName,
|
|
429
|
+
scenarioIndex: segment.scenarioIndex,
|
|
430
|
+
simulationIndex: segment.simulationIndex,
|
|
431
|
+
simulationKind: segment.kind,
|
|
432
|
+
count,
|
|
433
|
+
firstObservedUtcNs: nowNs,
|
|
434
|
+
lastObservedUtcNs: nowNs
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
recordDecisionLag(segment, lagNs) {
|
|
439
|
+
segment.decisionLag.record(maxBigInt(lagNs, 0n) / 1000n);
|
|
440
|
+
}
|
|
441
|
+
recordStartLag(segment, lagNs) {
|
|
442
|
+
segment.startLag.record(maxBigInt(lagNs, 0n) / 1000n);
|
|
443
|
+
}
|
|
444
|
+
buildSchedulerDistributions() {
|
|
445
|
+
return this.mutableSegments.flatMap((segment) => {
|
|
446
|
+
const scenarioIndex64 = segment.scenarioIndex.toString();
|
|
447
|
+
const simulationIndex64 = segment.simulationIndex.toString();
|
|
448
|
+
return [
|
|
449
|
+
["scheduler-decision-lag", "decision", segment.decisionLag],
|
|
450
|
+
["scheduler-start-lag", "start", segment.startLag]
|
|
451
|
+
].map(([seriesKind, identityKind, histogram]) => ({
|
|
452
|
+
seriesKind,
|
|
453
|
+
scenarioIndex64,
|
|
454
|
+
scenarioName: segment.scenarioName,
|
|
455
|
+
identityKeyHex: buildLoadEngineV2SchedulerIdentityKey(identityKind, scenarioIndex64, simulationIndex64).toString("hex"),
|
|
456
|
+
outcome: "none",
|
|
457
|
+
unit: "microseconds",
|
|
458
|
+
histogram: histogram.toSidecar(),
|
|
459
|
+
exactTotalDecimalOrEmpty: histogram.toSidecar().exactTotal64
|
|
460
|
+
}));
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
buildWarnings() {
|
|
464
|
+
return Array.from(this.warnings.values())
|
|
465
|
+
.sort((left, right) => left.code.localeCompare(right.code)
|
|
466
|
+
|| left.scenarioName.localeCompare(right.scenarioName)
|
|
467
|
+
|| left.simulationIndex - right.simulationIndex)
|
|
468
|
+
.map((value) => ({
|
|
469
|
+
code: value.code,
|
|
470
|
+
scenarioName: value.scenarioName,
|
|
471
|
+
scenarioIndex: value.scenarioIndex,
|
|
472
|
+
simulationIndex: value.simulationIndex,
|
|
473
|
+
simulationKind: value.simulationKind,
|
|
474
|
+
count64: value.count.toString(),
|
|
475
|
+
message: value.code,
|
|
476
|
+
firstObservedUtcNs: value.firstObservedUtcNs.toString(),
|
|
477
|
+
lastObservedUtcNs: value.lastObservedUtcNs.toString()
|
|
478
|
+
}));
|
|
479
|
+
}
|
|
480
|
+
buildSegments() {
|
|
481
|
+
return this.mutableSegments.map((segment) => ({
|
|
482
|
+
scenarioName: segment.scenarioName,
|
|
483
|
+
scenarioIndex: segment.scenarioIndex,
|
|
484
|
+
simulationIndex: segment.simulationIndex,
|
|
485
|
+
kind: segment.kind,
|
|
486
|
+
shardIndex: segment.shardIndex,
|
|
487
|
+
shardCount: segment.shardCount,
|
|
488
|
+
plannedIterations64: segment.planned.toString(),
|
|
489
|
+
dueIterations64: segment.due.toString(),
|
|
490
|
+
startedIterations64: segment.started.toString(),
|
|
491
|
+
completedIterations64: segment.completed.toString(),
|
|
492
|
+
droppedIterations64: segment.dropped.toString(),
|
|
493
|
+
unreachedIterations64: segment.unreached.toString(),
|
|
494
|
+
requestedWorkerSlots64: segment.requestedWorkers.toString(),
|
|
495
|
+
startedWorkerSlots64: segment.startedWorkers.toString(),
|
|
496
|
+
unavailableWorkerSlots64: segment.unavailableWorkers.toString(),
|
|
497
|
+
dropReasons: Object.fromEntries(Array.from(segment.dropReasons, ([key, count]) => [key, count.toString()])),
|
|
498
|
+
unavailableWorkerReasons: Object.fromEntries(Array.from(segment.unavailableWorkerReasons, ([key, count]) => [key, count.toString()])),
|
|
499
|
+
deliveryPercent: segment.due === 0n ? 100 : Number(segment.started * 10000n / segment.due) / 100,
|
|
500
|
+
accountingComplete: segment.accountingComplete
|
|
501
|
+
}));
|
|
502
|
+
}
|
|
503
|
+
buildStats() {
|
|
504
|
+
return {
|
|
505
|
+
configuredMaxInFlight: this.budget.maxInFlight,
|
|
506
|
+
maxInFlightObserved: this.budget.highWater,
|
|
507
|
+
currentInFlight: this.budget.current,
|
|
508
|
+
segments: this.buildSegments()
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
function incrementReason(reasons, code, count = 1n) {
|
|
513
|
+
reasons.set(code, (reasons.get(code) ?? 0n) + count);
|
|
514
|
+
}
|
|
515
|
+
function ownedV2OrdinalCount(total, shardIndex, shardCount) {
|
|
516
|
+
if (total <= BigInt(shardIndex))
|
|
517
|
+
return 0n;
|
|
518
|
+
return (total - 1n - BigInt(shardIndex)) / BigInt(shardCount) + 1n;
|
|
231
519
|
}
|
|
232
520
|
class StepStatsAccumulator {
|
|
233
|
-
constructor(scenarioName, stepName, sortIndex) {
|
|
521
|
+
constructor(scenarioName, stepName, sortIndex, useHistogram = false) {
|
|
234
522
|
this.scenarioName = scenarioName;
|
|
235
523
|
this.stepName = stepName;
|
|
236
524
|
this.sortIndex = sortIndex;
|
|
237
|
-
this.
|
|
238
|
-
this.
|
|
525
|
+
this.useHistogram = useHistogram;
|
|
526
|
+
this.ok = new MeasurementAccumulator(useHistogram);
|
|
527
|
+
this.fail = new MeasurementAccumulator(useHistogram);
|
|
239
528
|
}
|
|
240
529
|
/**
|
|
241
530
|
* Exposes the public record operation.
|
|
@@ -274,6 +563,7 @@ class StepStatsAccumulator {
|
|
|
274
563
|
minLatencyMs: minCandidates.length ? Math.min(...minCandidates) : 0,
|
|
275
564
|
maxLatencyMs: maxCandidates.length ? Math.max(...maxCandidates) : 0,
|
|
276
565
|
statusCodes,
|
|
566
|
+
allMeasurement: this.useHistogram ? this.ok.buildCombined(this.fail, requestCount, durationMs) : undefined,
|
|
277
567
|
ok,
|
|
278
568
|
fail,
|
|
279
569
|
sortIndex: this.sortIndex
|
|
@@ -281,15 +571,16 @@ class StepStatsAccumulator {
|
|
|
281
571
|
}
|
|
282
572
|
}
|
|
283
573
|
class ScenarioStatsAccumulator {
|
|
284
|
-
constructor(scenarioName, sortIndex) {
|
|
574
|
+
constructor(scenarioName, sortIndex, useHistogram = false) {
|
|
285
575
|
this.scenarioName = scenarioName;
|
|
286
576
|
this.sortIndex = sortIndex;
|
|
287
|
-
this.
|
|
288
|
-
this.fail = new MeasurementAccumulator();
|
|
577
|
+
this.useHistogram = useHistogram;
|
|
289
578
|
this.steps = new Map();
|
|
290
579
|
this.nextStepSortIndex = 0;
|
|
291
580
|
this.loadSimulationStats = { simulationName: "", value: 0 };
|
|
292
581
|
this.currentOperation = "None";
|
|
582
|
+
this.ok = new MeasurementAccumulator(useHistogram);
|
|
583
|
+
this.fail = new MeasurementAccumulator(useHistogram);
|
|
293
584
|
}
|
|
294
585
|
/**
|
|
295
586
|
* Exposes the public setLoadSimulation operation.
|
|
@@ -330,13 +621,18 @@ class ScenarioStatsAccumulator {
|
|
|
330
621
|
* Exposes the public recordStep operation.
|
|
331
622
|
* Use this when the surrounding wrapper type makes this operation the clearest way to express your intent.
|
|
332
623
|
*/
|
|
333
|
-
recordStep(stepName, reply, observedLatencyMs) {
|
|
624
|
+
recordStep(stepName, reply, observedLatencyMs, sortIndex) {
|
|
334
625
|
const existing = this.steps.get(stepName);
|
|
335
|
-
const
|
|
626
|
+
const resolvedSortIndex = sortIndex === undefined
|
|
627
|
+
? this.nextStepSortIndex + 1
|
|
628
|
+
: Math.max(Math.trunc(sortIndex), 0);
|
|
629
|
+
const step = existing ?? new StepStatsAccumulator(this.scenarioName, stepName, resolvedSortIndex, this.useHistogram);
|
|
336
630
|
if (!existing) {
|
|
631
|
+
this.nextStepSortIndex = Math.max(this.nextStepSortIndex, resolvedSortIndex);
|
|
337
632
|
this.steps.set(stepName, step);
|
|
338
633
|
}
|
|
339
634
|
step.record(reply, observedLatencyMs);
|
|
635
|
+
return step.sortIndex;
|
|
340
636
|
}
|
|
341
637
|
/**
|
|
342
638
|
* Builds the configured payload or helper object.
|
|
@@ -365,6 +661,7 @@ class ScenarioStatsAccumulator {
|
|
|
365
661
|
minLatencyMs: minCandidates.length ? Math.min(...minCandidates) : 0,
|
|
366
662
|
maxLatencyMs: maxCandidates.length ? Math.max(...maxCandidates) : 0,
|
|
367
663
|
statusCodes,
|
|
664
|
+
allMeasurement: this.useHistogram ? this.ok.buildCombined(this.fail, totalRequests, durationMs) : undefined,
|
|
368
665
|
allBytes,
|
|
369
666
|
currentOperation: this.currentOperation,
|
|
370
667
|
durationMs: Math.max(durationMs, 0),
|
|
@@ -459,7 +756,8 @@ export class LoadStrikeStep {
|
|
|
459
756
|
const internal = context;
|
|
460
757
|
await internal.invokeBeforeStep(stepName);
|
|
461
758
|
let reply;
|
|
462
|
-
const
|
|
759
|
+
const startedUtcNs = utcNowNs();
|
|
760
|
+
const startedAtNs = process.hrtime.bigint();
|
|
463
761
|
try {
|
|
464
762
|
reply = normalizeReply(await run());
|
|
465
763
|
}
|
|
@@ -467,8 +765,21 @@ export class LoadStrikeStep {
|
|
|
467
765
|
reply = LoadStrikeResponse.fail("step_exception", resolveRuntimeErrorMessage(error, "step failed"), 0);
|
|
468
766
|
}
|
|
469
767
|
reply = attachReplyProjection(reply);
|
|
470
|
-
const
|
|
471
|
-
|
|
768
|
+
const observedLatencyNs = maxBigInt(process.hrtime.bigint() - startedAtNs, 0n);
|
|
769
|
+
const completedUtcNs = startedUtcNs + observedLatencyNs;
|
|
770
|
+
const observedLatencyMs = Number(observedLatencyNs) / 1000000;
|
|
771
|
+
const recordedSortIndex = internal.recordStep(stepName, reply, observedLatencyMs);
|
|
772
|
+
internal.recordStepObservation?.(createIterationStepObservation({
|
|
773
|
+
stepName,
|
|
774
|
+
sortIndex: typeof recordedSortIndex === "number" ? recordedSortIndex : 0,
|
|
775
|
+
startedUtcNs,
|
|
776
|
+
completedUtcNs,
|
|
777
|
+
observedLatencyUs64: observedLatencyNs / 1000n,
|
|
778
|
+
reportedLatencyUs64: normalizeRawObservationLatencyMicroseconds(resolveRecordedLatency(reply.customLatencyMs, observedLatencyMs)),
|
|
779
|
+
isSuccess: reply.isSuccess,
|
|
780
|
+
statusCode: normalizeStatusCode(reply.statusCode, reply.isSuccess),
|
|
781
|
+
sizeBytes64: normalizeHistogramInteger(Math.max(toNumber(reply.sizeBytes), 0), "Step response bytes")
|
|
782
|
+
}));
|
|
472
783
|
await internal.invokeAfterStep(stepName, reply);
|
|
473
784
|
return reply;
|
|
474
785
|
}
|
|
@@ -999,10 +1310,14 @@ export class LoadStrikeContext {
|
|
|
999
1310
|
const normalizedValues = normalizeRunContextCollectionShapes(this.values);
|
|
1000
1311
|
return {
|
|
1001
1312
|
displayConsoleMetrics: normalizedValues.ConsoleMetricsEnabled,
|
|
1313
|
+
loadEngineContractVersion: normalizedValues.LoadEngineContractVersion,
|
|
1314
|
+
maxInFlight: normalizedValues.MaxInFlight,
|
|
1002
1315
|
nodeType: normalizedValues.NodeType,
|
|
1003
1316
|
localDevClusterEnabled: normalizedValues.LocalDevClusterEnabled,
|
|
1004
1317
|
agentGroup: normalizedValues.AgentGroup,
|
|
1005
1318
|
agentsCount: normalizedValues.AgentsCount,
|
|
1319
|
+
agentId: normalizedValues.AgentId,
|
|
1320
|
+
expectedAgentIds: normalizedValues.ExpectedAgentIds,
|
|
1006
1321
|
targetScenarios: normalizedValues.TargetScenarios,
|
|
1007
1322
|
agentTargetScenarios: normalizedValues.AgentTargetScenarios,
|
|
1008
1323
|
coordinatorTargetScenarios: normalizedValues.CoordinatorTargetScenarios,
|
|
@@ -1021,6 +1336,13 @@ export class LoadStrikeContext {
|
|
|
1021
1336
|
reportFolderPath: normalizedValues.ReportFolderPath,
|
|
1022
1337
|
reportFormats: normalizedValues.ReportFormats,
|
|
1023
1338
|
reportingIntervalSeconds: normalizedValues.ReportingIntervalSeconds,
|
|
1339
|
+
iterationObservationFlushIntervalSeconds: normalizedValues.IterationObservationFlushIntervalSeconds,
|
|
1340
|
+
maxIterationObservationBufferBytes: normalizedValues.MaxIterationObservationBufferBytes,
|
|
1341
|
+
maxIterationObservationsPerBatch: normalizedValues.MaxIterationObservationsPerBatch,
|
|
1342
|
+
maxIterationObservationBatchBytes: normalizedValues.MaxIterationObservationBatchBytes,
|
|
1343
|
+
iterationObservationSinkQueueDepth: normalizedValues.IterationObservationSinkQueueDepth,
|
|
1344
|
+
iterationObservationSinkParallelism: normalizedValues.IterationObservationSinkParallelism,
|
|
1345
|
+
iterationObservationDrainTimeoutSeconds: normalizedValues.IterationObservationDrainTimeoutSeconds,
|
|
1024
1346
|
minimumLogLevel: normalizedValues.MinimumLogLevel,
|
|
1025
1347
|
loggerConfig: normalizedValues.LoggerConfig,
|
|
1026
1348
|
reportingSinks: normalizedValues.ReportingSinks,
|
|
@@ -1034,6 +1356,8 @@ export class LoadStrikeContext {
|
|
|
1034
1356
|
workerPlugins: normalizedValues.WorkerPlugins,
|
|
1035
1357
|
customSettings: normalizedValues.CustomSettings,
|
|
1036
1358
|
globalCustomSettings: normalizedValues.GlobalCustomSettings,
|
|
1359
|
+
clusterShardIndex: normalizedValues.ClusterShardIndex,
|
|
1360
|
+
clusterShardCount: normalizedValues.ClusterShardCount,
|
|
1037
1361
|
runArgs: this.runArgs.length ? [...this.runArgs] : undefined
|
|
1038
1362
|
};
|
|
1039
1363
|
}
|
|
@@ -1095,6 +1419,19 @@ export class LoadStrikeContext {
|
|
|
1095
1419
|
DisplayConsoleMetrics(enable) {
|
|
1096
1420
|
return this.mergeValues({ ConsoleMetricsEnabled: Boolean(enable) });
|
|
1097
1421
|
}
|
|
1422
|
+
useLoadEngineV2() {
|
|
1423
|
+
return this.UseLoadEngineV2();
|
|
1424
|
+
}
|
|
1425
|
+
UseLoadEngineV2() {
|
|
1426
|
+
return this.mergeValues({ LoadEngineContractVersion: 2 });
|
|
1427
|
+
}
|
|
1428
|
+
withMaxInFlight(maxInFlight) {
|
|
1429
|
+
return this.WithMaxInFlight(maxInFlight);
|
|
1430
|
+
}
|
|
1431
|
+
WithMaxInFlight(maxInFlight) {
|
|
1432
|
+
validateV2MaxInFlight(this.values.LoadEngineContractVersion, maxInFlight);
|
|
1433
|
+
return this.mergeValues({ MaxInFlight: maxInFlight });
|
|
1434
|
+
}
|
|
1098
1435
|
/**
|
|
1099
1436
|
* Toggles local development cluster mode.
|
|
1100
1437
|
* Use this when you want to simulate coordinator and agent behavior on a single machine.
|
|
@@ -1161,6 +1498,26 @@ export class LoadStrikeContext {
|
|
|
1161
1498
|
WithAgentGroup(agentGroup) {
|
|
1162
1499
|
return this.mergeValues({ AgentGroup: requireNonEmpty(agentGroup, "Agent group must be provided.") });
|
|
1163
1500
|
}
|
|
1501
|
+
/** Sets the stable identity required by a remote Load Engine V2 agent. */
|
|
1502
|
+
withAgentId(agentId) {
|
|
1503
|
+
return this.WithAgentId(agentId);
|
|
1504
|
+
}
|
|
1505
|
+
/** Sets the stable identity required by a remote Load Engine V2 agent. */
|
|
1506
|
+
WithAgentId(agentId) {
|
|
1507
|
+
return this.mergeValues({ AgentId: requireNonEmpty(agentId, "Agent id must be provided.") });
|
|
1508
|
+
}
|
|
1509
|
+
/** Sets the exact remote agent identities required by a Load Engine V2 coordinator. */
|
|
1510
|
+
withExpectedAgentIds(...agentIds) {
|
|
1511
|
+
return this.WithExpectedAgentIds(...agentIds);
|
|
1512
|
+
}
|
|
1513
|
+
/** Sets the exact remote agent identities required by a Load Engine V2 coordinator. */
|
|
1514
|
+
WithExpectedAgentIds(...agentIds) {
|
|
1515
|
+
const normalized = validateScenarioNames(agentIds);
|
|
1516
|
+
if (new Set(normalized).size !== normalized.length) {
|
|
1517
|
+
throw new Error("Expected agent ids must be unique.");
|
|
1518
|
+
}
|
|
1519
|
+
return this.mergeValues({ ExpectedAgentIds: normalized });
|
|
1520
|
+
}
|
|
1164
1521
|
/**
|
|
1165
1522
|
* Sets the requested agent count.
|
|
1166
1523
|
* Use this when a coordinator should fan work out across a specific number of agents.
|
|
@@ -1997,7 +2354,7 @@ function firstWebVitalViolation(...values) {
|
|
|
1997
2354
|
return "";
|
|
1998
2355
|
}
|
|
1999
2356
|
export class LoadStrikeScenario {
|
|
2000
|
-
constructor(name, runHandler, initHandler, cleanHandler, loadSimulations, thresholds, trackingConfiguration, maxFailCount, withoutWarmUpValue, warmUpDurationSeconds, weight, restartIterationOnFail, internalLicenseFeatures = []) {
|
|
2357
|
+
constructor(name, runHandler, initHandler, cleanHandler, loadSimulations, thresholds, trackingConfiguration, maxFailCount, withoutWarmUpValue, warmUpDurationSeconds, weight, restartIterationOnFail, internalLicenseFeatures = [], declaredStepNames = []) {
|
|
2001
2358
|
this.name = name;
|
|
2002
2359
|
this.runHandler = runHandler;
|
|
2003
2360
|
this.initHandler = initHandler;
|
|
@@ -2011,6 +2368,7 @@ export class LoadStrikeScenario {
|
|
|
2011
2368
|
this.weight = weight;
|
|
2012
2369
|
this.restartIterationOnFail = restartIterationOnFail;
|
|
2013
2370
|
this.internalLicenseFeatures = normalizeStringArray(internalLicenseFeatures);
|
|
2371
|
+
this.declaredStepNames = normalizeDeclaredStepNames(declaredStepNames);
|
|
2014
2372
|
}
|
|
2015
2373
|
static create(name, runHandler) {
|
|
2016
2374
|
const scenarioName = requireNonEmpty(name, "Scenario name must be provided.");
|
|
@@ -2053,7 +2411,7 @@ export class LoadStrikeScenario {
|
|
|
2053
2411
|
if (typeof handler !== "function") {
|
|
2054
2412
|
throw new TypeError("Init handler must be provided.");
|
|
2055
2413
|
}
|
|
2056
|
-
return new LoadStrikeScenario(this.name, this.runHandler, handler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2414
|
+
return new LoadStrikeScenario(this.name, this.runHandler, handler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2057
2415
|
}
|
|
2058
2416
|
/**
|
|
2059
2417
|
* Configures init async for this SDK object.
|
|
@@ -2070,7 +2428,7 @@ export class LoadStrikeScenario {
|
|
|
2070
2428
|
if (typeof handler !== "function") {
|
|
2071
2429
|
throw new TypeError("Clean handler must be provided.");
|
|
2072
2430
|
}
|
|
2073
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, handler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2431
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, handler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2074
2432
|
}
|
|
2075
2433
|
/**
|
|
2076
2434
|
* Configures clean async for this SDK object.
|
|
@@ -2087,14 +2445,14 @@ export class LoadStrikeScenario {
|
|
|
2087
2445
|
if (!Number.isFinite(maxFailCount)) {
|
|
2088
2446
|
throw new RangeError("maxFailCount should be a finite number.");
|
|
2089
2447
|
}
|
|
2090
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, Math.trunc(maxFailCount), this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2448
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, Math.trunc(maxFailCount), this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2091
2449
|
}
|
|
2092
2450
|
/**
|
|
2093
2451
|
* Configures out warm up for this SDK object.
|
|
2094
2452
|
* Use this when out warm up should be set explicitly before the run starts.
|
|
2095
2453
|
*/
|
|
2096
2454
|
withoutWarmUp() {
|
|
2097
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, true, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2455
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, true, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2098
2456
|
}
|
|
2099
2457
|
/**
|
|
2100
2458
|
* Configures warm up duration for this SDK object.
|
|
@@ -2104,7 +2462,7 @@ export class LoadStrikeScenario {
|
|
|
2104
2462
|
if (!Number.isFinite(durationSeconds)) {
|
|
2105
2463
|
throw new RangeError("Warmup duration should be a finite number.");
|
|
2106
2464
|
}
|
|
2107
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, durationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2465
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, durationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2108
2466
|
}
|
|
2109
2467
|
/**
|
|
2110
2468
|
* Configures weight for this SDK object.
|
|
@@ -2114,14 +2472,14 @@ export class LoadStrikeScenario {
|
|
|
2114
2472
|
if (!Number.isFinite(weight)) {
|
|
2115
2473
|
throw new RangeError("Weight should be a finite number.");
|
|
2116
2474
|
}
|
|
2117
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, Math.trunc(weight), this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2475
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, Math.trunc(weight), this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2118
2476
|
}
|
|
2119
2477
|
/**
|
|
2120
2478
|
* Configures restart iteration on fail for this SDK object.
|
|
2121
2479
|
* Use this when restart iteration on fail should be set explicitly before the run starts.
|
|
2122
2480
|
*/
|
|
2123
2481
|
withRestartIterationOnFail(shouldRestart) {
|
|
2124
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, Boolean(shouldRestart), this.internalLicenseFeatures);
|
|
2482
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, Boolean(shouldRestart), this.internalLicenseFeatures, this.declaredStepNames);
|
|
2125
2483
|
}
|
|
2126
2484
|
/**
|
|
2127
2485
|
* Configures cross platform tracking for this SDK object.
|
|
@@ -2146,7 +2504,7 @@ export class LoadStrikeScenario {
|
|
|
2146
2504
|
if (isCorrelateExistingTrafficTracking(copied) && this.loadSimulations.length > 0) {
|
|
2147
2505
|
throw new Error("CorrelateExistingTraffic uses ForDuration and cannot be combined with WithLoadSimulations.");
|
|
2148
2506
|
}
|
|
2149
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, copied, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2507
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, copied, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2150
2508
|
}
|
|
2151
2509
|
/**
|
|
2152
2510
|
* Configures load simulations for this SDK object.
|
|
@@ -2159,7 +2517,7 @@ export class LoadStrikeScenario {
|
|
|
2159
2517
|
if (isCorrelateExistingTrafficTracking(this.trackingConfiguration)) {
|
|
2160
2518
|
throw new Error("CorrelateExistingTraffic uses ForDuration and cannot be combined with WithLoadSimulations.");
|
|
2161
2519
|
}
|
|
2162
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, simulations.map((simulation) => attachLoadSimulationProjection({ ...simulation })), this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2520
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, simulations.map((simulation) => attachLoadSimulationProjection({ ...simulation })), this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2163
2521
|
}
|
|
2164
2522
|
/**
|
|
2165
2523
|
* Configures thresholds for this SDK object.
|
|
@@ -2169,7 +2527,7 @@ export class LoadStrikeScenario {
|
|
|
2169
2527
|
if (!thresholds.length) {
|
|
2170
2528
|
throw new Error("At least one threshold should be provided.");
|
|
2171
2529
|
}
|
|
2172
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, thresholds.map((threshold) => ({ ...threshold })), this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures);
|
|
2530
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, thresholds.map((threshold) => ({ ...threshold })), this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, this.declaredStepNames);
|
|
2173
2531
|
}
|
|
2174
2532
|
/**
|
|
2175
2533
|
* Returns simulations.
|
|
@@ -2230,6 +2588,26 @@ export class LoadStrikeScenario {
|
|
|
2230
2588
|
__loadStrikeInternalLicenseFeatures() {
|
|
2231
2589
|
return [...this.internalLicenseFeatures];
|
|
2232
2590
|
}
|
|
2591
|
+
/** Freezes the named step identities that may be reported by this scenario. */
|
|
2592
|
+
withDeclaredSteps(...stepNames) {
|
|
2593
|
+
if (!stepNames.length) {
|
|
2594
|
+
throw new Error("At least one declared step name should be provided.");
|
|
2595
|
+
}
|
|
2596
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, this.internalLicenseFeatures, stepNames);
|
|
2597
|
+
}
|
|
2598
|
+
/** Returns the immutable declared-step names in declaration order. */
|
|
2599
|
+
getDeclaredSteps() {
|
|
2600
|
+
return [...this.declaredStepNames];
|
|
2601
|
+
}
|
|
2602
|
+
__loadStrikeSetTrafficMixV2Metadata(metadata) {
|
|
2603
|
+
this.trafficMixV2Metadata = cloneLoadEngineV2TrafficMixMetadata(metadata);
|
|
2604
|
+
return this;
|
|
2605
|
+
}
|
|
2606
|
+
__loadStrikeTrafficMixV2Metadata() {
|
|
2607
|
+
return this.trafficMixV2Metadata
|
|
2608
|
+
? cloneLoadEngineV2TrafficMixMetadata(this.trafficMixV2Metadata)
|
|
2609
|
+
: undefined;
|
|
2610
|
+
}
|
|
2233
2611
|
__loadStrikeScenarioSourceAnalysis() {
|
|
2234
2612
|
const source = this.runHandler.toString();
|
|
2235
2613
|
const lines = source
|
|
@@ -2252,7 +2630,7 @@ export class LoadStrikeScenario {
|
|
|
2252
2630
|
}
|
|
2253
2631
|
__loadStrikeWithInternalLicenseFeatures(...features) {
|
|
2254
2632
|
const merged = Array.from(new Set([...this.internalLicenseFeatures, ...normalizeStringArray(features)]));
|
|
2255
|
-
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, merged);
|
|
2633
|
+
return new LoadStrikeScenario(this.name, this.runHandler, this.initHandler, this.cleanHandler, this.loadSimulations, this.thresholds, this.trackingConfiguration, this.maxFailCount, this.withoutWarmUpValue, this.warmUpDurationSeconds, this.weight, this.restartIterationOnFail, merged, this.declaredStepNames);
|
|
2256
2634
|
}
|
|
2257
2635
|
async invokeInit(context) {
|
|
2258
2636
|
if (this.initHandler) {
|
|
@@ -2273,6 +2651,9 @@ export class LoadStrikeScenario {
|
|
|
2273
2651
|
return normalizeReply(result);
|
|
2274
2652
|
}
|
|
2275
2653
|
catch (error) {
|
|
2654
|
+
if (error instanceof RuntimePolicyCallbackError) {
|
|
2655
|
+
throw error;
|
|
2656
|
+
}
|
|
2276
2657
|
return LoadStrikeResponse.fail("exception", resolveRuntimeErrorMessage(error, "scenario failed"), 0);
|
|
2277
2658
|
}
|
|
2278
2659
|
}
|
|
@@ -2318,6 +2699,10 @@ export class LoadStrikeScenario {
|
|
|
2318
2699
|
WithLoadSimulations(...simulations) {
|
|
2319
2700
|
return this.withLoadSimulations(...simulations);
|
|
2320
2701
|
}
|
|
2702
|
+
/** Freezes the named step identities that may be reported by this scenario. */
|
|
2703
|
+
WithDeclaredSteps(...stepNames) {
|
|
2704
|
+
return this.withDeclaredSteps(...stepNames);
|
|
2705
|
+
}
|
|
2321
2706
|
/**
|
|
2322
2707
|
* Configures max fail count for this SDK object.
|
|
2323
2708
|
* Use this when max fail count should be set explicitly before the run starts.
|
|
@@ -2446,7 +2831,7 @@ export class LoadStrikeTrafficMix {
|
|
|
2446
2831
|
return this.withScenarioMix(...scenarioMix);
|
|
2447
2832
|
}
|
|
2448
2833
|
expandScenarios() {
|
|
2449
|
-
return expandTrafficMixScenarios(this);
|
|
2834
|
+
return expandTrafficMixScenarios(this, 0);
|
|
2450
2835
|
}
|
|
2451
2836
|
ExpandScenarios() {
|
|
2452
2837
|
return this.expandScenarios();
|
|
@@ -2459,10 +2844,11 @@ export class LoadStrikeTrafficMix {
|
|
|
2459
2844
|
}
|
|
2460
2845
|
}
|
|
2461
2846
|
export class LoadStrikeRunner {
|
|
2462
|
-
constructor(scenarios, options, contextConfigurators = []) {
|
|
2847
|
+
constructor(scenarios, options, contextConfigurators = [], internalOptions = {}) {
|
|
2463
2848
|
this.scenarios = scenarios;
|
|
2464
2849
|
this.options = normalizeRunnerOptionCollectionShapes(options);
|
|
2465
2850
|
this.contextConfigurators = [...contextConfigurators];
|
|
2851
|
+
this.internalOptions = internalOptions;
|
|
2466
2852
|
}
|
|
2467
2853
|
/**
|
|
2468
2854
|
* Creates a new instance of this public SDK type.
|
|
@@ -2498,7 +2884,7 @@ export class LoadStrikeRunner {
|
|
|
2498
2884
|
* Use this when one total load profile should be split across weighted scenario lanes.
|
|
2499
2885
|
*/
|
|
2500
2886
|
static registerTrafficMix(trafficMix) {
|
|
2501
|
-
return LoadStrikeRunner.registerScenarios(...expandTrafficMixScenarios(trafficMix));
|
|
2887
|
+
return LoadStrikeRunner.registerScenarios(...expandTrafficMixScenarios(trafficMix, 0));
|
|
2502
2888
|
}
|
|
2503
2889
|
/**
|
|
2504
2890
|
* Registers a traffic mix on a fresh runnable context.
|
|
@@ -2514,6 +2900,12 @@ export class LoadStrikeRunner {
|
|
|
2514
2900
|
static DisplayConsoleMetrics(context, enable) {
|
|
2515
2901
|
return context.DisplayConsoleMetrics(enable);
|
|
2516
2902
|
}
|
|
2903
|
+
static UseLoadEngineV2(context) {
|
|
2904
|
+
return context.UseLoadEngineV2();
|
|
2905
|
+
}
|
|
2906
|
+
static WithMaxInFlight(context, maxInFlight) {
|
|
2907
|
+
return context.WithMaxInFlight(maxInFlight);
|
|
2908
|
+
}
|
|
2517
2909
|
/**
|
|
2518
2910
|
* Toggles local development cluster mode.
|
|
2519
2911
|
* Use this when you want to simulate coordinator and agent behavior on a single machine.
|
|
@@ -2549,6 +2941,12 @@ export class LoadStrikeRunner {
|
|
|
2549
2941
|
static WithAgentGroup(context, agentGroup) {
|
|
2550
2942
|
return context.WithAgentGroup(agentGroup);
|
|
2551
2943
|
}
|
|
2944
|
+
static WithAgentId(context, agentId) {
|
|
2945
|
+
return context.WithAgentId(agentId);
|
|
2946
|
+
}
|
|
2947
|
+
static WithExpectedAgentIds(context, ...agentIds) {
|
|
2948
|
+
return context.WithExpectedAgentIds(...agentIds);
|
|
2949
|
+
}
|
|
2552
2950
|
/**
|
|
2553
2951
|
* Sets the requested agent count.
|
|
2554
2952
|
* Use this when a coordinator should fan work out across a specific number of agents.
|
|
@@ -2766,7 +3164,10 @@ export class LoadStrikeRunner {
|
|
|
2766
3164
|
* Use this when one total load profile should be split across weighted scenario lanes.
|
|
2767
3165
|
*/
|
|
2768
3166
|
addTrafficMix(trafficMix) {
|
|
2769
|
-
this.scenarios = [
|
|
3167
|
+
this.scenarios = [
|
|
3168
|
+
...this.scenarios,
|
|
3169
|
+
...expandTrafficMixScenarios(trafficMix, nextTrafficMixDeclarationIndex(this.scenarios))
|
|
3170
|
+
];
|
|
2770
3171
|
return this;
|
|
2771
3172
|
}
|
|
2772
3173
|
/**
|
|
@@ -2899,6 +3300,19 @@ export class LoadStrikeRunner {
|
|
|
2899
3300
|
WithReportingInterval(intervalSeconds) {
|
|
2900
3301
|
return this.withReportingInterval(intervalSeconds);
|
|
2901
3302
|
}
|
|
3303
|
+
useLoadEngineV2() {
|
|
3304
|
+
return this.configure({ loadEngineContractVersion: 2 });
|
|
3305
|
+
}
|
|
3306
|
+
UseLoadEngineV2() {
|
|
3307
|
+
return this.useLoadEngineV2();
|
|
3308
|
+
}
|
|
3309
|
+
withMaxInFlight(maxInFlight) {
|
|
3310
|
+
validateV2MaxInFlight(this.options.loadEngineContractVersion, maxInFlight);
|
|
3311
|
+
return this.configure({ maxInFlight });
|
|
3312
|
+
}
|
|
3313
|
+
WithMaxInFlight(maxInFlight) {
|
|
3314
|
+
return this.withMaxInFlight(maxInFlight);
|
|
3315
|
+
}
|
|
2902
3316
|
withReportingSinks(...sinks) {
|
|
2903
3317
|
if (!sinks.length) {
|
|
2904
3318
|
throw new Error("At least one reporting sink should be provided.");
|
|
@@ -2997,7 +3411,7 @@ export class LoadStrikeRunner {
|
|
|
2997
3411
|
}
|
|
2998
3412
|
async run(args = []) {
|
|
2999
3413
|
if (this.contextConfigurators.length) {
|
|
3000
|
-
return new LoadStrikeRunner(this.scenarios, this.buildContext().toRunnerOptions()).run(args);
|
|
3414
|
+
return new LoadStrikeRunner(this.scenarios, this.buildContext().toRunnerOptions(), [], this.internalOptions).run(args);
|
|
3001
3415
|
}
|
|
3002
3416
|
if (args.length) {
|
|
3003
3417
|
return this.buildContext().run(args);
|
|
@@ -3015,8 +3429,8 @@ export class LoadStrikeRunner {
|
|
|
3015
3429
|
}));
|
|
3016
3430
|
const sinkErrors = [];
|
|
3017
3431
|
const policyErrors = [];
|
|
3018
|
-
const sinkRetryCount =
|
|
3019
|
-
const sinkRetryBackoffMs =
|
|
3432
|
+
const sinkRetryCount = normalizeSinkRetryCount(this.options.sinkRetryCount);
|
|
3433
|
+
const sinkRetryBackoffMs = normalizeSinkRetryBackoffMs(this.options.sinkRetryBackoffMs);
|
|
3020
3434
|
const policies = this.options.runtimePolicies ?? [];
|
|
3021
3435
|
const runtimePolicyErrorMode = normalizedRuntimePolicyErrorMode(this.options.runtimePolicyErrorMode);
|
|
3022
3436
|
const plugins = this.options.reportingSinks === undefined && this.options.workerPlugins === undefined &&
|
|
@@ -3045,10 +3459,15 @@ export class LoadStrikeRunner {
|
|
|
3045
3459
|
let licenseClient = null;
|
|
3046
3460
|
let licensePayload = null;
|
|
3047
3461
|
let licenseSession = null;
|
|
3462
|
+
let iterationObservationsFinalized = false;
|
|
3048
3463
|
const clusterMode = resolveClusterExecutionMode(this.options);
|
|
3049
3464
|
const selectedScenarios = clusterMode === "local-coordinator" || clusterMode === "nats-coordinator"
|
|
3050
3465
|
? await this.filterScenariosWithPolicies(this.scenarios, policies, policyErrors, runtimePolicyErrorMode)
|
|
3051
3466
|
: await this.selectScenarios(policies, policyErrors, runtimePolicyErrorMode);
|
|
3467
|
+
if (this.options.loadEngineContractVersion === 2
|
|
3468
|
+
&& (clusterMode === "nats-coordinator" || clusterMode === "nats-agent")) {
|
|
3469
|
+
validateLoadEngineV2ScenarioFeatures(selectedScenarios);
|
|
3470
|
+
}
|
|
3052
3471
|
if (clusterMode === "nats-agent") {
|
|
3053
3472
|
return this.runAgentWithNats(createdUtc, testInfo, nodeInfo);
|
|
3054
3473
|
}
|
|
@@ -3105,14 +3524,49 @@ export class LoadStrikeRunner {
|
|
|
3105
3524
|
await init(baseContext, this.options.infraConfig ?? {});
|
|
3106
3525
|
}
|
|
3107
3526
|
}
|
|
3108
|
-
await this.initializeSinks(sinkStates, baseContext, this.options.infraConfig ?? {}, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
|
|
3527
|
+
await this.initializeSinks(sinkStates, baseContext, this.options.infraConfig ?? {}, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
|
|
3109
3528
|
for (const plugin of plugins) {
|
|
3110
3529
|
const start = resolveWorkerPluginStart(plugin);
|
|
3111
3530
|
if (start) {
|
|
3112
3531
|
await start(sessionInfo);
|
|
3113
3532
|
}
|
|
3114
3533
|
}
|
|
3115
|
-
await this.startSinks(sinkStates, sessionInfo, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
|
|
3534
|
+
await this.startSinks(sinkStates, sessionInfo, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
|
|
3535
|
+
const iterationObservationRunId = String(this.internalOptions.iterationObservationRunId
|
|
3536
|
+
?? sessionInfo.portalReportingRunId
|
|
3537
|
+
?? sessionInfo.PortalReportingRunId
|
|
3538
|
+
?? testInfo.sessionId);
|
|
3539
|
+
const iterationObservationResultOwnerId = String(nodeInfo.nodeType).toLowerCase() === "agent"
|
|
3540
|
+
? String(this.options.agentCommandId
|
|
3541
|
+
?? `${nodeInfo.machineName}:${Math.max(Math.trunc(this.options.clusterShardIndex ?? 0), 0)}`)
|
|
3542
|
+
: "";
|
|
3543
|
+
const iterationObservationProcessGroup = 0;
|
|
3544
|
+
const iterationObservationExpectedResultOwnerCount64 = Math.max(Math.trunc(this.options.clusterShardCount ?? 1), 1).toString();
|
|
3545
|
+
const initializedIterationObservationSinks = sinkStates
|
|
3546
|
+
.filter((state) => !state.disabled)
|
|
3547
|
+
.map((state) => ({
|
|
3548
|
+
name: state.name,
|
|
3549
|
+
iterationObservationPortalSink: Boolean(state.sink.iterationObservationPortalSink),
|
|
3550
|
+
iterationObservationShapeLimited: Boolean(state.sink.iterationObservationShapeLimited),
|
|
3551
|
+
saveIterationBatch: resolveSinkSaveIterationBatch(state.sink),
|
|
3552
|
+
completeIterationObservationStream: resolveSinkCompleteIterationObservationStream(state.sink)
|
|
3553
|
+
}));
|
|
3554
|
+
const reporterIterationObservationSinks = this.internalOptions.iterationObservationSinks
|
|
3555
|
+
?? (clusterMode === "local-coordinator" || clusterMode === "nats-coordinator"
|
|
3556
|
+
? []
|
|
3557
|
+
: initializedIterationObservationSinks);
|
|
3558
|
+
const iterationObservationReporter = new IterationObservationReporter({
|
|
3559
|
+
runId: iterationObservationRunId,
|
|
3560
|
+
sessionId: testInfo.sessionId,
|
|
3561
|
+
resultOwnerId: iterationObservationResultOwnerId,
|
|
3562
|
+
expectedResultOwnerCount64: iterationObservationExpectedResultOwnerCount64,
|
|
3563
|
+
processGroup: iterationObservationProcessGroup,
|
|
3564
|
+
settings: resolveIterationObservationSettings(this.options),
|
|
3565
|
+
sinks: reporterIterationObservationSinks,
|
|
3566
|
+
sinkRetryCount,
|
|
3567
|
+
sinkRetryBackoffMs,
|
|
3568
|
+
logger: runLogger
|
|
3569
|
+
});
|
|
3116
3570
|
const emitRealtimeSnapshot = async () => {
|
|
3117
3571
|
if (realtimeInFlight) {
|
|
3118
3572
|
return;
|
|
@@ -3123,7 +3577,7 @@ export class LoadStrikeRunner {
|
|
|
3123
3577
|
.map((value) => value.build(scenarioDurationsMs.get(value.scenarioName) ?? Math.max(Date.now() - started.getTime(), 0)))
|
|
3124
3578
|
.sort((left, right) => left.sortIndex - right.sortIndex);
|
|
3125
3579
|
const metricsSnapshot = collectMetricStats(allRegisteredMetrics, Date.now() - started.getTime());
|
|
3126
|
-
await this.emitRealtimeStats(sinkStates, snapshot, metricsSnapshot, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
|
|
3580
|
+
await this.emitRealtimeStats(sinkStates, snapshot, metricsSnapshot, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
|
|
3127
3581
|
if (toBoolean(this.options.displayConsoleMetrics, true)) {
|
|
3128
3582
|
const requestCount = snapshot.reduce((sum, value) => sum + value.allRequestCount, 0);
|
|
3129
3583
|
const okCount = snapshot.reduce((sum, value) => sum + value.allOkCount, 0);
|
|
@@ -3179,21 +3633,30 @@ export class LoadStrikeRunner {
|
|
|
3179
3633
|
let result;
|
|
3180
3634
|
let metricStats;
|
|
3181
3635
|
if (clusterMode === "local-coordinator") {
|
|
3182
|
-
const aggregated = await this.runCoordinatorWithLocalAgents(selectedScenarios, testInfo, nodeInfo, licenseClient, licenseSession);
|
|
3636
|
+
const aggregated = await this.runCoordinatorWithLocalAgents(selectedScenarios, testInfo, nodeInfo, licenseClient, licenseSession, iterationObservationRunId, initializedIterationObservationSinks);
|
|
3183
3637
|
metricStats = aggregated.metrics;
|
|
3184
3638
|
result = toDetailedRunResultFromNodeStats(aggregated, started.toISOString(), sinkErrors, policyErrors);
|
|
3185
3639
|
}
|
|
3186
3640
|
else if (clusterMode === "nats-coordinator") {
|
|
3187
|
-
const aggregated = await this.runCoordinatorWithNats(selectedScenarios, testInfo, nodeInfo, licenseClient, licenseSession);
|
|
3641
|
+
const aggregated = await this.runCoordinatorWithNats(selectedScenarios, testInfo, nodeInfo, licenseClient, licenseSession, iterationObservationRunId, initializedIterationObservationSinks);
|
|
3188
3642
|
metricStats = aggregated.metrics;
|
|
3189
3643
|
result = toDetailedRunResultFromNodeStats(aggregated, started.toISOString(), sinkErrors, policyErrors);
|
|
3190
3644
|
}
|
|
3191
3645
|
else {
|
|
3192
3646
|
const testAbortController = new AbortController();
|
|
3193
3647
|
const stopTestState = { value: false, reason: undefined };
|
|
3194
|
-
|
|
3648
|
+
const loadEngineV2Budget = this.options.loadEngineContractVersion === 2
|
|
3649
|
+
? (this.options.loadEngineV2BudgetOverride
|
|
3650
|
+
?? new LoadEngineV2ExecutionBudget(this.options.maxInFlight ?? 10000))
|
|
3651
|
+
: undefined;
|
|
3652
|
+
const loadEngineV2Telemetry = loadEngineV2Budget
|
|
3653
|
+
? new LoadEngineV2Telemetry(loadEngineV2Budget)
|
|
3654
|
+
: undefined;
|
|
3655
|
+
const executeSelectedScenario = (scenario, selectedScenarioIndex) => executeScenarioRuntime({
|
|
3195
3656
|
scenario,
|
|
3196
|
-
scenarioIndex
|
|
3657
|
+
scenarioIndex: this.options.loadEngineContractVersion === 2
|
|
3658
|
+
? this.scenarios.indexOf(scenario)
|
|
3659
|
+
: selectedScenarioIndex,
|
|
3197
3660
|
scenarioCount: selectedScenarios.length,
|
|
3198
3661
|
options: this.options,
|
|
3199
3662
|
logger: runLogger,
|
|
@@ -3208,12 +3671,26 @@ export class LoadStrikeRunner {
|
|
|
3208
3671
|
scenarioDurationsMs,
|
|
3209
3672
|
stopTestState,
|
|
3210
3673
|
testAbortController,
|
|
3674
|
+
loadEngineV2Budget,
|
|
3675
|
+
loadEngineV2Telemetry,
|
|
3676
|
+
iterationObservationReporter,
|
|
3677
|
+
iterationObservationRunId,
|
|
3678
|
+
iterationObservationResultOwnerId,
|
|
3679
|
+
iterationObservationProcessGroup,
|
|
3211
3680
|
executeScenarioInvocation: (targetScenario, context, operation) => this.executeScenarioInvocation(targetScenario, context, operation),
|
|
3212
3681
|
invokeBeforeScenario: (runtimePolicies, scenarioName) => this.invokeBeforeScenario(runtimePolicies, scenarioName, policyErrors, runtimePolicyErrorMode),
|
|
3213
3682
|
invokeAfterScenario: (runtimePolicies, scenarioName, stats) => this.invokeAfterScenario(runtimePolicies, scenarioName, stats, policyErrors, runtimePolicyErrorMode),
|
|
3214
3683
|
invokeBeforeStep: (runtimePolicies, scenarioName, stepName) => this.invokeBeforeStep(runtimePolicies, scenarioName, stepName, policyErrors, runtimePolicyErrorMode),
|
|
3215
3684
|
invokeAfterStep: (runtimePolicies, scenarioName, stepName, reply) => this.invokeAfterStep(runtimePolicies, scenarioName, stepName, reply, policyErrors, runtimePolicyErrorMode)
|
|
3216
|
-
})
|
|
3685
|
+
});
|
|
3686
|
+
if (this.options.loadEngineV2SegmentLifecycleOverride) {
|
|
3687
|
+
for (let scenarioIndex = 0; scenarioIndex < selectedScenarios.length; scenarioIndex += 1) {
|
|
3688
|
+
await executeSelectedScenario(selectedScenarios[scenarioIndex], scenarioIndex);
|
|
3689
|
+
}
|
|
3690
|
+
}
|
|
3691
|
+
else {
|
|
3692
|
+
await Promise.all(selectedScenarios.map(executeSelectedScenario));
|
|
3693
|
+
}
|
|
3217
3694
|
nodeInfo.currentOperation = stopTestState.value ? "Stop" : "Complete";
|
|
3218
3695
|
const scenarioStatList = Array.from(scenarioAccumulators.values())
|
|
3219
3696
|
.map((value) => value.build(scenarioDurationsMs.get(value.scenarioName) ?? 0))
|
|
@@ -3253,17 +3730,46 @@ export class LoadStrikeRunner {
|
|
|
3253
3730
|
reportFiles: [],
|
|
3254
3731
|
logFiles: [...loggerSetup.logFiles],
|
|
3255
3732
|
correlationRows: buildDetailedCorrelationRows(),
|
|
3256
|
-
failedCorrelationRows: buildDetailedFailedCorrelationRows()
|
|
3733
|
+
failedCorrelationRows: buildDetailedFailedCorrelationRows(),
|
|
3734
|
+
...(loadEngineV2Telemetry
|
|
3735
|
+
? {
|
|
3736
|
+
generatorWarnings: loadEngineV2Telemetry.buildWarnings(),
|
|
3737
|
+
schedulerSegments: loadEngineV2Telemetry.buildSegments(),
|
|
3738
|
+
schedulerStats: loadEngineV2Telemetry.buildStats(),
|
|
3739
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: loadEngineV2Telemetry.buildSchedulerDistributions()
|
|
3740
|
+
}
|
|
3741
|
+
: {})
|
|
3257
3742
|
};
|
|
3258
3743
|
}
|
|
3259
3744
|
await stopRealtimeReporting();
|
|
3745
|
+
const observationDelivery = await iterationObservationReporter.sealAndDrain();
|
|
3746
|
+
iterationObservationsFinalized = true;
|
|
3747
|
+
if (clusterMode !== "local-coordinator"
|
|
3748
|
+
&& clusterMode !== "nats-coordinator") {
|
|
3749
|
+
result.observationDeliveryStats = {
|
|
3750
|
+
lastBatchSequence64: observationDelivery.lastBatchSequence64,
|
|
3751
|
+
capturedCount64: observationDelivery.capturedCount64,
|
|
3752
|
+
deliveredCount64: observationDelivery.deliveredCount64,
|
|
3753
|
+
droppedBufferCount64: observationDelivery.droppedBufferCount64,
|
|
3754
|
+
droppedSinkCount64: observationDelivery.droppedSinkCount64
|
|
3755
|
+
};
|
|
3756
|
+
result.reportingComplete = observationDelivery.reportingComplete;
|
|
3757
|
+
}
|
|
3758
|
+
else {
|
|
3759
|
+
result.observationDeliveryStats ?? (result.observationDeliveryStats = emptyObservationDeliveryStats());
|
|
3760
|
+
result.reportingComplete ?? (result.reportingComplete = observationDelivery.reportingComplete);
|
|
3761
|
+
}
|
|
3762
|
+
result.generatorWarnings = [
|
|
3763
|
+
...(result.generatorWarnings ?? []),
|
|
3764
|
+
...iterationObservationReporter.buildWarnings()
|
|
3765
|
+
];
|
|
3260
3766
|
result.pluginsData = mergePluginData(result.pluginsData, await this.collectPluginData(plugins, attachRunResultAliases(result), pluginLifecycleErrors));
|
|
3261
3767
|
const finalizedResult = attachRunResultAliases(result);
|
|
3262
3768
|
finalizedResult.logFiles = mergeStringArrays(finalizedResult.logFiles, loggerSetup.logFiles);
|
|
3263
3769
|
finalizedResult.reportFiles = this.writeReports(finalizedResult);
|
|
3264
3770
|
finalizedResult.disabledSinks = sinkStates.filter((x) => x.disabled).map((x) => x.name);
|
|
3265
|
-
await this.emitRunResult(sinkStates, finalizedResult, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
|
|
3266
|
-
await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
|
|
3771
|
+
await this.emitRunResult(sinkStates, finalizedResult, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
|
|
3772
|
+
await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
|
|
3267
3773
|
sinksStopped = true;
|
|
3268
3774
|
finalizedResult.disabledSinks = sinkStates.filter((x) => x.disabled).map((x) => x.name);
|
|
3269
3775
|
finalizedResult.sinkErrors = sinkErrors
|
|
@@ -3272,8 +3778,11 @@ export class LoadStrikeRunner {
|
|
|
3272
3778
|
}
|
|
3273
3779
|
finally {
|
|
3274
3780
|
await stopRealtimeReporting();
|
|
3781
|
+
if (!iterationObservationsFinalized) {
|
|
3782
|
+
await iterationObservationReporter.sealAndDrain().catch(() => { });
|
|
3783
|
+
}
|
|
3275
3784
|
if (!sinksStopped) {
|
|
3276
|
-
await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
|
|
3785
|
+
await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
|
|
3277
3786
|
}
|
|
3278
3787
|
if (!pluginsStopped) {
|
|
3279
3788
|
await this.stopPlugins(plugins, pluginLifecycleErrors, runLogger);
|
|
@@ -3308,7 +3817,7 @@ export class LoadStrikeRunner {
|
|
|
3308
3817
|
}
|
|
3309
3818
|
return filtered;
|
|
3310
3819
|
}
|
|
3311
|
-
async runClusterChildNode(targetScenarios, nodeType, machineName, includeWorkerExtensions, overrides = {}) {
|
|
3820
|
+
async runClusterChildNode(targetScenarios, nodeType, machineName, includeWorkerExtensions, overrides = {}, internalOptions = {}) {
|
|
3312
3821
|
if (!targetScenarios.length) {
|
|
3313
3822
|
return buildEmptyNodeStats({
|
|
3314
3823
|
startedUtc: new Date().toISOString(),
|
|
@@ -3336,7 +3845,7 @@ export class LoadStrikeRunner {
|
|
|
3336
3845
|
displayConsoleMetrics: false,
|
|
3337
3846
|
reportingSinks: includeWorkerExtensions ? this.options.reportingSinks : [],
|
|
3338
3847
|
workerPlugins: includeWorkerExtensions ? this.options.workerPlugins : []
|
|
3339
|
-
});
|
|
3848
|
+
}, [], internalOptions);
|
|
3340
3849
|
const childResult = await childRunner.run();
|
|
3341
3850
|
const childStats = detailedToNodeStats(childResult);
|
|
3342
3851
|
return {
|
|
@@ -3349,70 +3858,127 @@ export class LoadStrikeRunner {
|
|
|
3349
3858
|
logFiles: [...(childResult.logFiles ?? [])]
|
|
3350
3859
|
};
|
|
3351
3860
|
}
|
|
3352
|
-
async runCoordinatorWithLocalAgents(scenarios, testInfo, nodeInfo, licenseClient, licenseSession) {
|
|
3861
|
+
async runCoordinatorWithLocalAgents(scenarios, testInfo, nodeInfo, licenseClient, licenseSession, iterationObservationRunId, iterationObservationSinks) {
|
|
3353
3862
|
if (!licenseClient) {
|
|
3354
3863
|
throw new Error("Coordinator agent execution authorization requires an initialized licensing client.");
|
|
3355
3864
|
}
|
|
3356
|
-
const controllerRunToken =
|
|
3865
|
+
const controllerRunToken = currentLicenseSessionRunToken(licenseSession);
|
|
3357
3866
|
if (!controllerRunToken) {
|
|
3358
3867
|
throw new Error("Coordinator agent execution authorization requires an active controller run token.");
|
|
3359
3868
|
}
|
|
3360
|
-
const assignments = planRuntimeClusterAssignments(scenarios, Math.max(this.options.agentsCount ?? 1, 1), this.options.targetScenarios ?? [], this.options.agentTargetScenarios ?? []);
|
|
3869
|
+
const assignments = planRuntimeClusterAssignments(scenarios, Math.max(this.options.agentsCount ?? 1, 1), this.options.targetScenarios ?? [], this.options.agentTargetScenarios ?? [], this.options.loadEngineContractVersion === 2, this.options.coordinatorTargetScenarios ?? []);
|
|
3870
|
+
const sharedV2Budget = this.options.loadEngineContractVersion === 2
|
|
3871
|
+
? new LoadEngineV2ExecutionBudget(this.options.maxInFlight ?? 10000)
|
|
3872
|
+
: undefined;
|
|
3361
3873
|
const nodeResults = await Promise.all(assignments.map(async (targetScenarios, index) => {
|
|
3362
3874
|
const commandId = randomBytes(16).toString("hex");
|
|
3363
|
-
const agentExecutionToken = await licenseClient.createAgentExecutionToken(
|
|
3875
|
+
const agentExecutionToken = await licenseClient.createAgentExecutionToken(currentLicenseSessionRunToken(licenseSession), testInfo.sessionId, commandId, index, assignments.length, targetScenarios);
|
|
3364
3876
|
return this.runClusterChildNode(targetScenarios, "Agent", `local-agent-${index + 1}`, false, {
|
|
3365
3877
|
sessionId: testInfo.sessionId,
|
|
3366
3878
|
testSuite: testInfo.testSuite,
|
|
3367
3879
|
testName: testInfo.testName,
|
|
3368
3880
|
agentCommandId: commandId,
|
|
3369
|
-
agentExecutionToken
|
|
3881
|
+
agentExecutionToken,
|
|
3882
|
+
clusterShardIndex: index,
|
|
3883
|
+
clusterShardCount: assignments.length,
|
|
3884
|
+
loadEngineV2BudgetOverride: sharedV2Budget
|
|
3885
|
+
}, {
|
|
3886
|
+
iterationObservationRunId,
|
|
3887
|
+
iterationObservationSinks
|
|
3370
3888
|
});
|
|
3371
3889
|
}));
|
|
3372
3890
|
const coordinatorTargets = [...(this.options.coordinatorTargetScenarios ?? [])];
|
|
3373
3891
|
if (coordinatorTargets.length) {
|
|
3374
|
-
nodeResults.push(await this.runClusterChildNode(coordinatorTargets, "Coordinator", nodeInfo.machineName, false
|
|
3892
|
+
nodeResults.push(await this.runClusterChildNode(coordinatorTargets, "Coordinator", nodeInfo.machineName, false, {}, {
|
|
3893
|
+
iterationObservationRunId,
|
|
3894
|
+
iterationObservationSinks
|
|
3895
|
+
}));
|
|
3375
3896
|
}
|
|
3376
|
-
return aggregateNodeStats(testInfo, { ...nodeInfo, nodeType: "Coordinator" }, nodeResults);
|
|
3897
|
+
return aggregateNodeStats(testInfo, { ...nodeInfo, nodeType: "Coordinator" }, nodeResults, this.options.loadEngineContractVersion === 2);
|
|
3377
3898
|
}
|
|
3378
|
-
async runCoordinatorWithNats(scenarios, testInfo, nodeInfo, licenseClient, licenseSession) {
|
|
3899
|
+
async runCoordinatorWithNats(scenarios, testInfo, nodeInfo, licenseClient, licenseSession, iterationObservationRunId, iterationObservationSinks) {
|
|
3379
3900
|
if (!licenseClient) {
|
|
3380
3901
|
throw new Error("Coordinator agent execution authorization requires an initialized licensing client.");
|
|
3381
3902
|
}
|
|
3382
|
-
const controllerRunToken =
|
|
3903
|
+
const controllerRunToken = currentLicenseSessionRunToken(licenseSession);
|
|
3383
3904
|
if (!controllerRunToken) {
|
|
3384
3905
|
throw new Error("Coordinator agent execution authorization requires an active controller run token.");
|
|
3385
3906
|
}
|
|
3386
|
-
const assignments = planRuntimeClusterAssignments(scenarios, Math.max(this.options.agentsCount ?? 1, 1), this.options.targetScenarios ?? [], this.options.agentTargetScenarios ?? []);
|
|
3907
|
+
const assignments = planRuntimeClusterAssignments(scenarios, Math.max(this.options.agentsCount ?? 1, 1), this.options.targetScenarios ?? [], this.options.agentTargetScenarios ?? [], this.options.loadEngineContractVersion === 2, this.options.coordinatorTargetScenarios ?? []);
|
|
3908
|
+
const expectedAgentIds = this.options.loadEngineContractVersion === 2
|
|
3909
|
+
? normalizeRequiredV2AgentIds(this.options.expectedAgentIds, assignments.length)
|
|
3910
|
+
: undefined;
|
|
3387
3911
|
const coordinator = new DistributedClusterCoordinator({
|
|
3388
3912
|
clusterId: this.options.clusterId ?? "local",
|
|
3389
3913
|
sessionId: testInfo.sessionId,
|
|
3390
3914
|
testSuite: testInfo.testSuite,
|
|
3391
3915
|
testName: testInfo.testName,
|
|
3392
3916
|
expectedAgentResults: assignments.length,
|
|
3917
|
+
expectedAgentIds,
|
|
3918
|
+
loadEngineContractVersion: this.options.loadEngineContractVersion ?? 1,
|
|
3393
3919
|
agentGroup: this.options.agentGroup,
|
|
3394
3920
|
commandTimeoutMs: Math.max(Math.trunc((this.options.clusterCommandTimeoutSeconds ?? 120) * 1000), 1),
|
|
3395
3921
|
nats: this.options.natsServerUrl
|
|
3396
3922
|
? { ServerUrl: this.options.natsServerUrl }
|
|
3397
3923
|
: undefined
|
|
3398
3924
|
});
|
|
3399
|
-
const
|
|
3400
|
-
const
|
|
3925
|
+
const tokenFactory = (command) => licenseClient.createAgentExecutionToken(currentLicenseSessionRunToken(licenseSession), testInfo.sessionId, command.commandId, command.agentIndex, command.agentCount, command.targetScenarios);
|
|
3926
|
+
const dispatch = this.options.loadEngineContractVersion === 2
|
|
3927
|
+
? await coordinator.dispatchV2(assignments, buildRuntimeLoadEngineV2Plan(scenarios, this.options, testInfo, expectedAgentIds), tokenFactory)
|
|
3928
|
+
: await coordinator.dispatch(assignments, tokenFactory);
|
|
3929
|
+
const nodes = dispatch.nodeResults.map((value) => clusterNodeResultToNodeStats(value, testInfo, { ...nodeInfo, nodeType: "Agent" }, this.options.loadEngineContractVersion === 2));
|
|
3401
3930
|
const coordinatorTargets = [...(this.options.coordinatorTargetScenarios ?? [])];
|
|
3402
3931
|
if (coordinatorTargets.length) {
|
|
3403
|
-
nodes.push(await this.runClusterChildNode(coordinatorTargets, "Coordinator", nodeInfo.machineName, false
|
|
3932
|
+
nodes.push(await this.runClusterChildNode(coordinatorTargets, "Coordinator", nodeInfo.machineName, false, {}, {
|
|
3933
|
+
iterationObservationRunId,
|
|
3934
|
+
iterationObservationSinks
|
|
3935
|
+
}));
|
|
3404
3936
|
}
|
|
3405
|
-
let aggregated = aggregateNodeStats(testInfo, { ...nodeInfo, nodeType: "Coordinator" }, nodes);
|
|
3937
|
+
let aggregated = aggregateNodeStats(testInfo, { ...nodeInfo, nodeType: "Coordinator" }, nodes, this.options.loadEngineContractVersion === 2);
|
|
3406
3938
|
if (dispatch.missingNodes > 0) {
|
|
3407
3939
|
aggregated = appendClusterPluginHint(aggregated, `Timed out waiting for ${dispatch.missingNodes} agent node result(s).`);
|
|
3940
|
+
aggregated = attachNodeStatsAliases({
|
|
3941
|
+
...aggregated,
|
|
3942
|
+
reportingComplete: false,
|
|
3943
|
+
schedulerSegments: [
|
|
3944
|
+
...(aggregated.schedulerSegments ?? []),
|
|
3945
|
+
...(dispatch.ownerLoss?.schedulerSegments ?? [])
|
|
3946
|
+
],
|
|
3947
|
+
generatorWarnings: [
|
|
3948
|
+
...aggregated.generatorWarnings,
|
|
3949
|
+
...(dispatch.ownerLoss?.generatorWarnings ?? []).map((warning) => ({
|
|
3950
|
+
code: warning.code,
|
|
3951
|
+
scenarioName: "cluster",
|
|
3952
|
+
simulationIndex: -1,
|
|
3953
|
+
count64: warning.count64,
|
|
3954
|
+
message: `Result owner ${warning.agentId} was lost after assignment; application failures were not fabricated.`,
|
|
3955
|
+
firstObservedUtcNs: "0",
|
|
3956
|
+
lastObservedUtcNs: "0"
|
|
3957
|
+
}))
|
|
3958
|
+
]
|
|
3959
|
+
});
|
|
3408
3960
|
}
|
|
3409
3961
|
return aggregated;
|
|
3410
3962
|
}
|
|
3411
3963
|
async runAgentWithNats(startedUtc, testInfo, nodeInfo) {
|
|
3964
|
+
const v2AgentId = this.options.loadEngineContractVersion === 2
|
|
3965
|
+
? requireNonEmpty(this.options.agentId ?? "", "Remote Load Engine V2 agents require an explicit stable AgentId.")
|
|
3966
|
+
: `${nodeInfo.machineName}-${generateRuntimeSessionId()}`;
|
|
3412
3967
|
const agent = new DistributedClusterAgent({
|
|
3413
3968
|
clusterId: this.options.clusterId ?? "local",
|
|
3414
3969
|
sessionId: testInfo.sessionId,
|
|
3415
|
-
agentId:
|
|
3970
|
+
agentId: v2AgentId,
|
|
3971
|
+
loadEngineContractVersion: this.options.loadEngineContractVersion ?? 1,
|
|
3972
|
+
validateV2Plan: this.options.loadEngineContractVersion === 2
|
|
3973
|
+
? (received) => {
|
|
3974
|
+
const local = buildRuntimeLoadEngineV2Plan(this.scenarios, this.options, { ...testInfo, sessionId: received.sessionId }, received.expectedAgentIds);
|
|
3975
|
+
local.runId = received.runId;
|
|
3976
|
+
local.registrationNonce = received.registrationNonce;
|
|
3977
|
+
if (buildLoadEngineV2Plan(local).hash !== buildLoadEngineV2Plan(received).hash) {
|
|
3978
|
+
throw new Error("Load Engine V2 signed plan descriptors do not match immutable local scenario declarations.");
|
|
3979
|
+
}
|
|
3980
|
+
}
|
|
3981
|
+
: undefined,
|
|
3416
3982
|
agentGroup: this.options.agentGroup,
|
|
3417
3983
|
nats: this.options.natsServerUrl
|
|
3418
3984
|
? { ServerUrl: this.options.natsServerUrl }
|
|
@@ -3422,13 +3988,16 @@ export class LoadStrikeRunner {
|
|
|
3422
3988
|
const deadline = Date.now() + Math.max(Math.trunc((this.options.clusterCommandTimeoutSeconds ?? 120) * 1000), 1);
|
|
3423
3989
|
while (Date.now() < deadline) {
|
|
3424
3990
|
let handledStats = null;
|
|
3425
|
-
const
|
|
3991
|
+
const execute = async (dispatch) => {
|
|
3426
3992
|
handledStats = await this.runClusterChildNode(dispatch.scenarioNames, "Agent", nodeInfo.machineName, true, {
|
|
3427
3993
|
sessionId: testInfo.sessionId,
|
|
3428
3994
|
testSuite: testInfo.testSuite,
|
|
3429
3995
|
testName: testInfo.testName,
|
|
3430
3996
|
agentCommandId: dispatch.commandId,
|
|
3431
|
-
agentExecutionToken: dispatch.agentRunToken
|
|
3997
|
+
agentExecutionToken: dispatch.agentRunToken,
|
|
3998
|
+
clusterShardIndex: dispatch.agentIndex ?? 0,
|
|
3999
|
+
clusterShardCount: dispatch.agentCount ?? 1,
|
|
4000
|
+
loadEngineV2SegmentLifecycleOverride: dispatch.segmentLifecycle
|
|
3432
4001
|
});
|
|
3433
4002
|
return {
|
|
3434
4003
|
nodeId: handledStats.nodeInfo.machineName,
|
|
@@ -3436,9 +4005,12 @@ export class LoadStrikeRunner {
|
|
|
3436
4005
|
allRequestCount: handledStats.allRequestCount,
|
|
3437
4006
|
allOkCount: handledStats.allOkCount,
|
|
3438
4007
|
allFailCount: handledStats.allFailCount,
|
|
3439
|
-
stats: nodeStatsToClusterPayload(handledStats)
|
|
4008
|
+
stats: nodeStatsToClusterPayload(handledStats, this.scenarios.filter((scenario) => dispatch.scenarioNames.includes(scenario.name)), this.options.loadEngineContractVersion === 2)
|
|
3440
4009
|
};
|
|
3441
|
-
}
|
|
4010
|
+
};
|
|
4011
|
+
const handled = this.options.loadEngineContractVersion === 2
|
|
4012
|
+
? await agent.pollAndExecuteV2Once(execute)
|
|
4013
|
+
: await agent.pollAndExecuteOnce(execute);
|
|
3442
4014
|
if (handled && handledStats) {
|
|
3443
4015
|
return toDetailedRunResultFromNodeStats(handledStats, startedUtc, [], []);
|
|
3444
4016
|
}
|
|
@@ -3518,9 +4090,9 @@ export class LoadStrikeRunner {
|
|
|
3518
4090
|
}
|
|
3519
4091
|
return filtered;
|
|
3520
4092
|
}
|
|
3521
|
-
async initializeSinks(sinkStates, context, infraConfig, retryCount, retryBackoffMs, sinkErrors) {
|
|
4093
|
+
async initializeSinks(sinkStates, context, infraConfig, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
|
|
3522
4094
|
for (const state of sinkStates) {
|
|
3523
|
-
await this.invokeSinkAction(state, "init", retryCount, retryBackoffMs, sinkErrors, false, true, async () => {
|
|
4095
|
+
await this.invokeSinkAction(state, "init", retryCount, retryBackoffMs, sinkErrors, runId, logger, false, true, async () => {
|
|
3524
4096
|
const init = resolveSinkInit(state.sink);
|
|
3525
4097
|
if (init) {
|
|
3526
4098
|
await init(context, infraConfig);
|
|
@@ -3528,9 +4100,9 @@ export class LoadStrikeRunner {
|
|
|
3528
4100
|
});
|
|
3529
4101
|
}
|
|
3530
4102
|
}
|
|
3531
|
-
async startSinks(sinkStates, session, retryCount, retryBackoffMs, sinkErrors) {
|
|
4103
|
+
async startSinks(sinkStates, session, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
|
|
3532
4104
|
for (const state of sinkStates) {
|
|
3533
|
-
await this.invokeSinkAction(state, "start", retryCount, retryBackoffMs, sinkErrors, false, true, async () => {
|
|
4105
|
+
await this.invokeSinkAction(state, "start", retryCount, retryBackoffMs, sinkErrors, runId, logger, false, true, async () => {
|
|
3534
4106
|
const start = resolveSinkStart(state.sink);
|
|
3535
4107
|
if (start) {
|
|
3536
4108
|
await start(session);
|
|
@@ -3538,9 +4110,9 @@ export class LoadStrikeRunner {
|
|
|
3538
4110
|
});
|
|
3539
4111
|
}
|
|
3540
4112
|
}
|
|
3541
|
-
async emitRealtimeStats(sinkStates, scenarioStats, metrics, retryCount, retryBackoffMs, sinkErrors) {
|
|
4113
|
+
async emitRealtimeStats(sinkStates, scenarioStats, metrics, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
|
|
3542
4114
|
for (const state of sinkStates) {
|
|
3543
|
-
await this.invokeSinkAction(state, "realtime", retryCount, retryBackoffMs, sinkErrors, false, true, async () => {
|
|
4115
|
+
await this.invokeSinkAction(state, "realtime", retryCount, retryBackoffMs, sinkErrors, runId, logger, false, true, async () => {
|
|
3544
4116
|
const saveRealtimeStats = resolveSinkSaveRealtimeStats(state.sink);
|
|
3545
4117
|
if (saveRealtimeStats) {
|
|
3546
4118
|
await saveRealtimeStats(scenarioStats);
|
|
@@ -3552,10 +4124,9 @@ export class LoadStrikeRunner {
|
|
|
3552
4124
|
});
|
|
3553
4125
|
}
|
|
3554
4126
|
}
|
|
3555
|
-
async stopSinks(sinkStates, retryCount, retryBackoffMs, sinkErrors) {
|
|
3556
|
-
const shutdownRetryCount = 0;
|
|
4127
|
+
async stopSinks(sinkStates, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
|
|
3557
4128
|
for (const state of sinkStates) {
|
|
3558
|
-
await this.invokeSinkAction(state, "stop",
|
|
4129
|
+
await this.invokeSinkAction(state, "stop", retryCount, retryBackoffMs, sinkErrors, runId, logger, true, true, async () => {
|
|
3559
4130
|
const stop = resolveSinkStop(state.sink);
|
|
3560
4131
|
if (stop) {
|
|
3561
4132
|
await stop();
|
|
@@ -3563,15 +4134,15 @@ export class LoadStrikeRunner {
|
|
|
3563
4134
|
});
|
|
3564
4135
|
const dispose = resolveSinkDispose(state.sink);
|
|
3565
4136
|
if (dispose) {
|
|
3566
|
-
await this.invokeSinkAction(state, "dispose",
|
|
4137
|
+
await this.invokeSinkAction(state, "dispose", retryCount, retryBackoffMs, sinkErrors, runId, logger, true, true, async () => {
|
|
3567
4138
|
await dispose();
|
|
3568
4139
|
});
|
|
3569
4140
|
}
|
|
3570
4141
|
}
|
|
3571
4142
|
}
|
|
3572
|
-
async emitRunResult(sinkStates, result, retryCount, retryBackoffMs, sinkErrors) {
|
|
4143
|
+
async emitRunResult(sinkStates, result, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
|
|
3573
4144
|
for (const state of sinkStates) {
|
|
3574
|
-
await this.invokeSinkAction(state, "run-result", retryCount, retryBackoffMs, sinkErrors, false, false, async () => {
|
|
4145
|
+
await this.invokeSinkAction(state, "run-result", retryCount, retryBackoffMs, sinkErrors, runId, logger, false, false, async () => {
|
|
3575
4146
|
const saveRunResult = resolveSinkSaveRunResult(state.sink);
|
|
3576
4147
|
if (saveRunResult) {
|
|
3577
4148
|
await saveRunResult(result);
|
|
@@ -3579,23 +4150,49 @@ export class LoadStrikeRunner {
|
|
|
3579
4150
|
});
|
|
3580
4151
|
}
|
|
3581
4152
|
}
|
|
3582
|
-
async invokeSinkAction(state, phase, retryCount, retryBackoffMs, sinkErrors, ignoreDisabled, disableOnFailure, action) {
|
|
4153
|
+
async invokeSinkAction(state, phase, retryCount, retryBackoffMs, sinkErrors, runId, logger, ignoreDisabled, disableOnFailure, action) {
|
|
3583
4154
|
if (state.disabled && !ignoreDisabled) {
|
|
3584
4155
|
return;
|
|
3585
4156
|
}
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
4157
|
+
const maximumAttempts = normalizeSinkRetryCount(retryCount) + 1;
|
|
4158
|
+
const backoffMs = normalizeSinkRetryBackoffMs(retryBackoffMs);
|
|
4159
|
+
for (let attempts = 1; attempts <= maximumAttempts; attempts += 1) {
|
|
3589
4160
|
try {
|
|
3590
4161
|
await action();
|
|
4162
|
+
if (attempts > 1) {
|
|
4163
|
+
logIterationObservationRecovery(logger, {
|
|
4164
|
+
sinkName: state.name,
|
|
4165
|
+
operation: "reporting-sink-action",
|
|
4166
|
+
phase,
|
|
4167
|
+
runId,
|
|
4168
|
+
resultOwnerId: "",
|
|
4169
|
+
observationCount: 0,
|
|
4170
|
+
attempt: attempts,
|
|
4171
|
+
maximumAttempts,
|
|
4172
|
+
nextDelayMs: 0
|
|
4173
|
+
});
|
|
4174
|
+
}
|
|
3591
4175
|
return;
|
|
3592
4176
|
}
|
|
3593
4177
|
catch (error) {
|
|
3594
|
-
|
|
4178
|
+
const exhausted = attempts >= maximumAttempts;
|
|
4179
|
+
const nextDelayMs = exhausted ? 0 : sinkRetryDelayMs(backoffMs, attempts);
|
|
4180
|
+
logIterationObservationFailure(logger, exhausted ? "error" : "warn", {
|
|
4181
|
+
sinkName: state.name,
|
|
4182
|
+
operation: "reporting-sink-action",
|
|
4183
|
+
phase,
|
|
4184
|
+
runId,
|
|
4185
|
+
resultOwnerId: "",
|
|
4186
|
+
observationCount: 0,
|
|
4187
|
+
attempt: attempts,
|
|
4188
|
+
maximumAttempts,
|
|
4189
|
+
nextDelayMs
|
|
4190
|
+
}, error);
|
|
4191
|
+
if (exhausted) {
|
|
3595
4192
|
sinkErrors.push({
|
|
3596
4193
|
sinkName: state.name,
|
|
3597
4194
|
phase,
|
|
3598
|
-
message:
|
|
4195
|
+
message: "The reporting sink action failed after retries.",
|
|
3599
4196
|
attempts
|
|
3600
4197
|
});
|
|
3601
4198
|
if (disableOnFailure) {
|
|
@@ -3603,9 +4200,7 @@ export class LoadStrikeRunner {
|
|
|
3603
4200
|
}
|
|
3604
4201
|
return;
|
|
3605
4202
|
}
|
|
3606
|
-
|
|
3607
|
-
await sleep(retryBackoffMs * attempts);
|
|
3608
|
-
}
|
|
4203
|
+
await waitForSinkRetryDelay(nextDelayMs);
|
|
3609
4204
|
}
|
|
3610
4205
|
}
|
|
3611
4206
|
}
|
|
@@ -3717,13 +4312,148 @@ function hasPluginRows(value) {
|
|
|
3717
4312
|
}
|
|
3718
4313
|
return value.tables.some((table) => Array.isArray(table.rows) && table.rows.length > 0);
|
|
3719
4314
|
}
|
|
4315
|
+
async function executeV2FixedArrivals(args) {
|
|
4316
|
+
const { rate, intervalNs, totalArrivals, budget, cancellationToken, shouldStopNow, nextInstanceInfo, runBombingInvocation, logger, scenarioName, deadlineOffsetsNs, tolerancesNs, ownedOrdinals, shardIndex = 0, shardCount = 1, telemetry, segment } = args;
|
|
4317
|
+
const segmentStartNs = process.hrtime.bigint();
|
|
4318
|
+
const toleranceNs = loadEngineV2LatenessToleranceNs(rate, intervalNs);
|
|
4319
|
+
const active = new Set();
|
|
4320
|
+
let schedulerLate = 0n;
|
|
4321
|
+
let maxInFlight = 0n;
|
|
4322
|
+
let executionError;
|
|
4323
|
+
const normalizedShardCount = Math.max(Math.trunc(shardCount), 1);
|
|
4324
|
+
const normalizedShardIndex = Math.min(Math.max(Math.trunc(shardIndex), 0), normalizedShardCount - 1);
|
|
4325
|
+
const ordinals = ownedOrdinals
|
|
4326
|
+
? [...ownedOrdinals]
|
|
4327
|
+
: (() => {
|
|
4328
|
+
const values = [];
|
|
4329
|
+
for (let ordinal = BigInt(normalizedShardIndex); ordinal < totalArrivals; ordinal += BigInt(normalizedShardCount)) {
|
|
4330
|
+
values.push(ordinal);
|
|
4331
|
+
}
|
|
4332
|
+
return values;
|
|
4333
|
+
})();
|
|
4334
|
+
if (ordinals.some((ordinal, index) => ordinal < 0n || ordinal >= totalArrivals
|
|
4335
|
+
|| (index > 0 && ordinal <= ordinals[index - 1]))) {
|
|
4336
|
+
throw new Error("Load Engine V2 owned arrival ordinals must be sorted, unique, and in range.");
|
|
4337
|
+
}
|
|
4338
|
+
if (segment) {
|
|
4339
|
+
segment.planned = BigInt(ordinals.length);
|
|
4340
|
+
}
|
|
4341
|
+
for (const ordinal of ordinals) {
|
|
4342
|
+
if (shouldStopNow())
|
|
4343
|
+
break;
|
|
4344
|
+
const index = Number(ordinal);
|
|
4345
|
+
const deadlineNs = segmentStartNs
|
|
4346
|
+
+ (deadlineOffsetsNs?.[index] ?? loadEngineV2FixedDeadlineNs(ordinal, rate, intervalNs));
|
|
4347
|
+
await delayUntilMonotonicDeadline(deadlineNs, cancellationToken);
|
|
4348
|
+
if (shouldStopNow()) {
|
|
4349
|
+
break;
|
|
4350
|
+
}
|
|
4351
|
+
const nowNs = process.hrtime.bigint();
|
|
4352
|
+
if (segment)
|
|
4353
|
+
segment.due += 1n;
|
|
4354
|
+
if (segment)
|
|
4355
|
+
telemetry?.recordDecisionLag(segment, nowNs - deadlineNs);
|
|
4356
|
+
if (classifyLoadEngineV2Arrival(nowNs, deadlineNs, tolerancesNs?.[index] ?? toleranceNs, true) === "scheduler_late") {
|
|
4357
|
+
schedulerLate += 1n;
|
|
4358
|
+
if (segment) {
|
|
4359
|
+
segment.dropped += 1n;
|
|
4360
|
+
incrementReason(segment.dropReasons, "scheduler_late");
|
|
4361
|
+
}
|
|
4362
|
+
continue;
|
|
4363
|
+
}
|
|
4364
|
+
const release = budget.tryAcquire();
|
|
4365
|
+
if (!release) {
|
|
4366
|
+
maxInFlight += 1n;
|
|
4367
|
+
if (segment) {
|
|
4368
|
+
segment.dropped += 1n;
|
|
4369
|
+
incrementReason(segment.dropReasons, "max_in_flight");
|
|
4370
|
+
}
|
|
4371
|
+
continue;
|
|
4372
|
+
}
|
|
4373
|
+
if (segment)
|
|
4374
|
+
segment.started += 1n;
|
|
4375
|
+
const instanceInfo = nextInstanceInfo();
|
|
4376
|
+
let task;
|
|
4377
|
+
task = (async () => {
|
|
4378
|
+
if (segment)
|
|
4379
|
+
telemetry?.recordStartLag(segment, process.hrtime.bigint() - deadlineNs);
|
|
4380
|
+
await runBombingInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, ordinal);
|
|
4381
|
+
})().catch((error) => {
|
|
4382
|
+
executionError ?? (executionError = error);
|
|
4383
|
+
}).finally(() => {
|
|
4384
|
+
if (segment)
|
|
4385
|
+
segment.completed += 1n;
|
|
4386
|
+
release();
|
|
4387
|
+
active.delete(task);
|
|
4388
|
+
});
|
|
4389
|
+
active.add(task);
|
|
4390
|
+
}
|
|
4391
|
+
while (active.size > 0) {
|
|
4392
|
+
await Promise.race(active);
|
|
4393
|
+
}
|
|
4394
|
+
if (segment) {
|
|
4395
|
+
segment.unreached = segment.planned > segment.due ? segment.planned - segment.due : 0n;
|
|
4396
|
+
segment.accountingComplete = segment.due + segment.unreached === segment.planned
|
|
4397
|
+
&& segment.started + segment.dropped === segment.due
|
|
4398
|
+
&& segment.completed === segment.started;
|
|
4399
|
+
}
|
|
4400
|
+
if (telemetry && segment) {
|
|
4401
|
+
telemetry.recordWarning("scheduler_late", segment, schedulerLate);
|
|
4402
|
+
telemetry.recordWarning("max_in_flight", segment, maxInFlight);
|
|
4403
|
+
}
|
|
4404
|
+
if (schedulerLate > 0n) {
|
|
4405
|
+
logger.warn(`Load Engine V2 warning scheduler_late: dropped ${schedulerLate.toString()} overdue arrivals for scenario ${scenarioName}.`);
|
|
4406
|
+
}
|
|
4407
|
+
if (maxInFlight > 0n) {
|
|
4408
|
+
logger.warn(`Load Engine V2 warning max_in_flight: dropped ${maxInFlight.toString()} arrivals for scenario ${scenarioName}; the process limit is ${budget.maxInFlight}.`);
|
|
4409
|
+
}
|
|
4410
|
+
if (executionError !== undefined) {
|
|
4411
|
+
throw executionError;
|
|
4412
|
+
}
|
|
4413
|
+
}
|
|
4414
|
+
async function delayUntilMonotonicDeadline(deadlineNs, cancellationToken) {
|
|
4415
|
+
while (!cancellationToken.aborted) {
|
|
4416
|
+
const remainingNs = deadlineNs - process.hrtime.bigint();
|
|
4417
|
+
if (remainingNs <= 0n) {
|
|
4418
|
+
return;
|
|
4419
|
+
}
|
|
4420
|
+
const remainingMs = Number((remainingNs + 999999n) / 1000000n);
|
|
4421
|
+
await delayWithAbort(Math.max(1, Math.min(remainingMs, 50)), cancellationToken);
|
|
4422
|
+
}
|
|
4423
|
+
}
|
|
4424
|
+
function secondsToNanoseconds(seconds, name) {
|
|
4425
|
+
if (!Number.isFinite(seconds) || seconds <= 0) {
|
|
4426
|
+
throw new RangeError(`${name} must be greater than zero.`);
|
|
4427
|
+
}
|
|
4428
|
+
const nanoseconds = Math.trunc(seconds * 1000000000);
|
|
4429
|
+
if (!Number.isSafeInteger(nanoseconds) || nanoseconds <= 0) {
|
|
4430
|
+
throw new RangeError(`${name} is outside the supported nanosecond range.`);
|
|
4431
|
+
}
|
|
4432
|
+
return BigInt(nanoseconds);
|
|
4433
|
+
}
|
|
4434
|
+
function buildV2RampTolerances(offsets, durationNs) {
|
|
4435
|
+
return offsets.map((offset, index) => {
|
|
4436
|
+
const quantum = offsets.length === 1
|
|
4437
|
+
? durationNs
|
|
4438
|
+
: index === 0
|
|
4439
|
+
? offsets[1] - offset
|
|
4440
|
+
: offset - offsets[index - 1];
|
|
4441
|
+
return minBigInt(100000000n, maxBigInt(2000000n, maxBigInt(1n, quantum) * 4n));
|
|
4442
|
+
});
|
|
4443
|
+
}
|
|
4444
|
+
function minBigInt(left, right) {
|
|
4445
|
+
return left < right ? left : right;
|
|
4446
|
+
}
|
|
4447
|
+
function maxBigInt(left, right) {
|
|
4448
|
+
return left > right ? left : right;
|
|
4449
|
+
}
|
|
3720
4450
|
async function executeScenarioRuntime(args) {
|
|
3721
|
-
const { scenario, scenarioIndex, scenarioCount, options, logger, nodeInfo, testInfo, policies, restartIterationMaxAttempts, allRegisteredMetrics, scenarioRuntimes, stepRuntimes, scenarioAccumulators, scenarioDurationsMs, stopTestState, testAbortController, executeScenarioInvocation, invokeBeforeScenario, invokeAfterScenario, invokeBeforeStep, invokeAfterStep } = args;
|
|
4451
|
+
const { scenario, scenarioIndex, scenarioCount, options, logger, nodeInfo, testInfo, policies, restartIterationMaxAttempts, allRegisteredMetrics, scenarioRuntimes, stepRuntimes, scenarioAccumulators, scenarioDurationsMs, stopTestState, testAbortController, loadEngineV2Budget, loadEngineV2Telemetry, iterationObservationReporter, iterationObservationRunId = testInfo.sessionId, iterationObservationResultOwnerId = "", iterationObservationProcessGroup = 0, executeScenarioInvocation, invokeBeforeScenario, invokeAfterScenario, invokeBeforeStep, invokeAfterStep } = args;
|
|
3722
4452
|
const scenarioStartedMs = Date.now();
|
|
3723
4453
|
const scenarioContextData = {};
|
|
3724
4454
|
const registeredMetrics = [];
|
|
3725
4455
|
const runtime = ensureScenarioRuntime(scenarioRuntimes, scenario.name);
|
|
3726
|
-
const accumulator = new ScenarioStatsAccumulator(scenario.name, scenarioIndex);
|
|
4456
|
+
const accumulator = new ScenarioStatsAccumulator(scenario.name, scenarioIndex, options.loadEngineContractVersion === 2);
|
|
3727
4457
|
scenarioAccumulators.set(scenario.name, accumulator);
|
|
3728
4458
|
const scenarioAbortController = new AbortController();
|
|
3729
4459
|
const scenarioCancellationToken = combineAbortSignals(testAbortController.signal, scenarioAbortController.signal);
|
|
@@ -3732,12 +4462,25 @@ async function executeScenarioRuntime(args) {
|
|
|
3732
4462
|
? Date.now() + Math.trunc(scenarioCompletionTimeoutSeconds * 1000)
|
|
3733
4463
|
: Number.POSITIVE_INFINITY;
|
|
3734
4464
|
const scenarioPartition = attachScenarioPartitionAliases({
|
|
3735
|
-
number: 0,
|
|
3736
|
-
count: 1
|
|
4465
|
+
number: options.loadEngineContractVersion === 2 ? Math.max(Math.trunc(options.clusterShardIndex ?? 0), 0) : 0,
|
|
4466
|
+
count: options.loadEngineContractVersion === 2 ? Math.max(Math.trunc(options.clusterShardCount ?? 1), 1) : 1
|
|
3737
4467
|
});
|
|
4468
|
+
const trafficMixV2 = options.loadEngineContractVersion === 2
|
|
4469
|
+
? scenario.__loadStrikeTrafficMixV2Metadata()
|
|
4470
|
+
: undefined;
|
|
4471
|
+
if (trafficMixV2) {
|
|
4472
|
+
const expectedSeedId = buildLoadEngineV2TrafficMixSeedId(trafficMixV2.declarationIndex, trafficMixV2.name);
|
|
4473
|
+
if (trafficMixV2.seedId !== expectedSeedId) {
|
|
4474
|
+
throw new Error(`Load Engine V2 traffic-mix seed metadata differs for scenario ${scenario.name}.`);
|
|
4475
|
+
}
|
|
4476
|
+
}
|
|
3738
4477
|
let stopScenario = false;
|
|
3739
4478
|
let invocationNumber = 0;
|
|
3740
4479
|
let instanceCounter = 0;
|
|
4480
|
+
let nextV1ObservationOrdinal = 0n;
|
|
4481
|
+
let nextV2FallbackOrdinal = BigInt(scenarioPartition.number);
|
|
4482
|
+
let nextObservationStepSortIndex = 0;
|
|
4483
|
+
const observationStepSortIndexes = new Map();
|
|
3741
4484
|
const shouldStopNow = () => stopTestState.value
|
|
3742
4485
|
|| stopScenario
|
|
3743
4486
|
|| scenarioCancellationToken.aborted
|
|
@@ -3771,7 +4514,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3771
4514
|
runtime.maxLatencyMs = Math.max(runtime.maxLatencyMs, latencyMs);
|
|
3772
4515
|
accumulator.recordScenario(reply, observedLatencyMs);
|
|
3773
4516
|
};
|
|
3774
|
-
const recordStepReply = (stepName, reply, observedLatencyMs) => {
|
|
4517
|
+
const recordStepReply = (stepName, reply, observedLatencyMs, sortIndex) => {
|
|
3775
4518
|
const key = `${scenario.name}::${stepName}`;
|
|
3776
4519
|
const stepRuntime = ensureStepRuntime(stepRuntimes, key, scenario.name, stepName);
|
|
3777
4520
|
if (reply.isSuccess) {
|
|
@@ -3794,11 +4537,35 @@ async function executeScenarioRuntime(args) {
|
|
|
3794
4537
|
stepRuntime.maxLatencyMs = Math.max(stepRuntime.maxLatencyMs, latencyMs);
|
|
3795
4538
|
const statusCode = normalizeStatusCode(reply.statusCode, reply.isSuccess);
|
|
3796
4539
|
stepRuntime.statusCodes[statusCode] = (stepRuntime.statusCodes[statusCode] ?? 0) + 1;
|
|
3797
|
-
accumulator.recordStep(stepName, reply, observedLatencyMs);
|
|
4540
|
+
return accumulator.recordStep(stepName, reply, observedLatencyMs, sortIndex);
|
|
4541
|
+
};
|
|
4542
|
+
const resolveObservationStepSortIndex = (stepName) => {
|
|
4543
|
+
const existing = observationStepSortIndexes.get(stepName);
|
|
4544
|
+
if (existing !== undefined) {
|
|
4545
|
+
return existing;
|
|
4546
|
+
}
|
|
4547
|
+
nextObservationStepSortIndex += 1;
|
|
4548
|
+
observationStepSortIndexes.set(stepName, nextObservationStepSortIndex);
|
|
4549
|
+
return nextObservationStepSortIndex;
|
|
4550
|
+
};
|
|
4551
|
+
const nextObservationOrdinal = (explicit) => {
|
|
4552
|
+
if (explicit !== undefined) {
|
|
4553
|
+
return explicit;
|
|
4554
|
+
}
|
|
4555
|
+
if (options.loadEngineContractVersion === 2) {
|
|
4556
|
+
const ordinal = nextV2FallbackOrdinal;
|
|
4557
|
+
nextV2FallbackOrdinal += BigInt(scenarioPartition.count);
|
|
4558
|
+
return ordinal;
|
|
4559
|
+
}
|
|
4560
|
+
const ordinal = nextV1ObservationOrdinal;
|
|
4561
|
+
nextV1ObservationOrdinal += 1n;
|
|
4562
|
+
return ordinal;
|
|
3798
4563
|
};
|
|
3799
4564
|
const runSingleInvocation = async (operation, instanceData, instanceNumber, instanceId, recordScenarioResult) => {
|
|
3800
4565
|
invocationNumber += 1;
|
|
3801
4566
|
const runtimeRandom = createRuntimeRandom();
|
|
4567
|
+
const attemptSteps = [];
|
|
4568
|
+
const recordedSteps = [];
|
|
3802
4569
|
const context = {
|
|
3803
4570
|
scenarioName: scenario.name,
|
|
3804
4571
|
data: scenarioContextData,
|
|
@@ -3827,7 +4594,12 @@ async function executeScenarioRuntime(args) {
|
|
|
3827
4594
|
testAbortController.abort(stopTestState.reason);
|
|
3828
4595
|
},
|
|
3829
4596
|
recordStep: (stepName, reply, observedLatencyMs) => {
|
|
3830
|
-
|
|
4597
|
+
const sortIndex = resolveObservationStepSortIndex(stepName);
|
|
4598
|
+
recordedSteps.push({ stepName, reply, observedLatencyMs, sortIndex });
|
|
4599
|
+
return sortIndex;
|
|
4600
|
+
},
|
|
4601
|
+
recordStepObservation: (observation) => {
|
|
4602
|
+
attemptSteps.push(observation);
|
|
3831
4603
|
},
|
|
3832
4604
|
shouldStopScenario: () => stopScenario || scenarioCancellationToken.aborted,
|
|
3833
4605
|
shouldStopTest: () => stopTestState.value || scenarioCancellationToken.aborted,
|
|
@@ -3835,28 +4607,100 @@ async function executeScenarioRuntime(args) {
|
|
|
3835
4607
|
invokeAfterStep: async (stepName, reply) => invokeAfterStep(policies, scenario.name, stepName, reply)
|
|
3836
4608
|
};
|
|
3837
4609
|
attachScenarioContextAliases(context);
|
|
3838
|
-
const
|
|
3839
|
-
const
|
|
3840
|
-
|
|
4610
|
+
const startedUtcNs = utcNowNs();
|
|
4611
|
+
const startedAtNs = process.hrtime.bigint();
|
|
4612
|
+
let policyFailure;
|
|
4613
|
+
let reply;
|
|
4614
|
+
try {
|
|
4615
|
+
reply = await executeScenarioInvocation(scenario, context, operation);
|
|
4616
|
+
}
|
|
4617
|
+
catch (error) {
|
|
4618
|
+
if (!(error instanceof RuntimePolicyCallbackError)) {
|
|
4619
|
+
throw error;
|
|
4620
|
+
}
|
|
4621
|
+
policyFailure = error;
|
|
4622
|
+
reply = LoadStrikeResponse.fail("runtime_policy_error", "", 0);
|
|
4623
|
+
}
|
|
4624
|
+
const observedLatencyNs = maxBigInt(process.hrtime.bigint() - startedAtNs, 0n);
|
|
4625
|
+
const completedUtcNs = startedUtcNs + observedLatencyNs;
|
|
4626
|
+
const observedLatencyMs = Number(observedLatencyNs) / 1000000;
|
|
3841
4627
|
if (recordScenarioResult) {
|
|
3842
4628
|
recordScenarioReply(reply, observedLatencyMs);
|
|
3843
4629
|
}
|
|
3844
|
-
return {
|
|
4630
|
+
return {
|
|
4631
|
+
reply,
|
|
4632
|
+
observedLatencyMs,
|
|
4633
|
+
startedUtcNs,
|
|
4634
|
+
completedUtcNs,
|
|
4635
|
+
observedLatencyUs: observedLatencyNs / 1000n,
|
|
4636
|
+
reportedLatencyUs: normalizeRawObservationLatencyMicroseconds(resolveRecordedLatency(reply.customLatencyMs, observedLatencyMs)),
|
|
4637
|
+
steps: attemptSteps,
|
|
4638
|
+
recordedSteps,
|
|
4639
|
+
...(policyFailure ? { policyFailure } : {})
|
|
4640
|
+
};
|
|
4641
|
+
};
|
|
4642
|
+
const captureAttemptObservation = (operation, globalOrdinal, attemptIndex, isFinalAttempt, attempt, simulationIndex, simulationKind, iterationId, globalSecondaryOrdinal = 0n) => {
|
|
4643
|
+
if (!iterationObservationReporter?.enabled) {
|
|
4644
|
+
return;
|
|
4645
|
+
}
|
|
4646
|
+
iterationObservationReporter.capture(createIterationObservation({
|
|
4647
|
+
runId: iterationObservationRunId,
|
|
4648
|
+
sessionId: testInfo.sessionId,
|
|
4649
|
+
resultOwnerId: iterationObservationResultOwnerId,
|
|
4650
|
+
processGroup: iterationObservationProcessGroup,
|
|
4651
|
+
scenarioName: scenario.name,
|
|
4652
|
+
scenarioIndex,
|
|
4653
|
+
simulationIndex,
|
|
4654
|
+
simulationKind,
|
|
4655
|
+
phase: operation === "WarmUp" ? "warmup" : "bombing",
|
|
4656
|
+
globalOrdinal64: globalOrdinal,
|
|
4657
|
+
globalSecondaryOrdinal64: globalSecondaryOrdinal,
|
|
4658
|
+
...(iterationId ? { iterationId } : {}),
|
|
4659
|
+
shardIndex: scenarioPartition.number,
|
|
4660
|
+
shardCount: scenarioPartition.count,
|
|
4661
|
+
attemptIndex,
|
|
4662
|
+
isFinalAttempt,
|
|
4663
|
+
startedUtcNs: attempt.startedUtcNs,
|
|
4664
|
+
completedUtcNs: attempt.completedUtcNs,
|
|
4665
|
+
observedLatencyUs64: attempt.observedLatencyUs,
|
|
4666
|
+
reportedLatencyUs64: attempt.reportedLatencyUs,
|
|
4667
|
+
isSuccess: attempt.reply.isSuccess,
|
|
4668
|
+
statusCode: normalizeStatusCode(attempt.reply.statusCode, attempt.reply.isSuccess),
|
|
4669
|
+
sizeBytes64: normalizeHistogramInteger(Math.max(toNumber(attempt.reply.sizeBytes), 0), "Scenario response bytes"),
|
|
4670
|
+
steps: attempt.steps
|
|
4671
|
+
}));
|
|
3845
4672
|
};
|
|
3846
|
-
const runBombingInvocation = async (instanceData, instanceNumber, instanceId) => {
|
|
4673
|
+
const runBombingInvocation = async (instanceData, instanceNumber, instanceId, explicitGlobalOrdinal, simulationIndex = -1, simulationKind = "SingleInvocation", explicitSecondaryOrdinal = 0n) => {
|
|
3847
4674
|
if (shouldStopNow()) {
|
|
3848
4675
|
return;
|
|
3849
4676
|
}
|
|
4677
|
+
const globalOrdinal = nextObservationOrdinal(explicitGlobalOrdinal);
|
|
4678
|
+
const identityKind = canonicalLoadEngineV2InvocationIdentityKind(simulationKind);
|
|
4679
|
+
const iterationId = options.loadEngineContractVersion === 2
|
|
4680
|
+
&& explicitGlobalOrdinal !== undefined
|
|
4681
|
+
&& identityKind
|
|
4682
|
+
? buildLoadEngineV2GlobalInvocationId(iterationObservationRunId, scenarioIndex, simulationIndex, identityKind, globalOrdinal, explicitSecondaryOrdinal)
|
|
4683
|
+
: undefined;
|
|
3850
4684
|
let attempts = 0;
|
|
3851
4685
|
const maxAttempts = 1 + (scenario.shouldRestartIterationOnFail() ? restartIterationMaxAttempts : 0);
|
|
3852
4686
|
while (attempts < maxAttempts && !shouldStopNow()) {
|
|
3853
4687
|
attempts += 1;
|
|
3854
4688
|
const attempt = await runSingleInvocation("Bombing", instanceData, instanceNumber, instanceId, false);
|
|
3855
|
-
const shouldRetry = !attempt.
|
|
4689
|
+
const shouldRetry = !attempt.policyFailure
|
|
4690
|
+
&& !attempt.reply.isSuccess
|
|
3856
4691
|
&& scenario.shouldRestartIterationOnFail()
|
|
3857
4692
|
&& attempts < maxAttempts
|
|
3858
4693
|
&& !shouldStopNow();
|
|
4694
|
+
captureAttemptObservation("Bombing", globalOrdinal, attempts - 1, !shouldRetry, attempt, simulationIndex, simulationKind, iterationId, explicitSecondaryOrdinal);
|
|
4695
|
+
if (attempt.policyFailure) {
|
|
4696
|
+
stopScenario = true;
|
|
4697
|
+
scenarioAbortController.abort(attempt.policyFailure);
|
|
4698
|
+
throw attempt.policyFailure;
|
|
4699
|
+
}
|
|
3859
4700
|
if (!shouldRetry) {
|
|
4701
|
+
for (const step of attempt.recordedSteps) {
|
|
4702
|
+
recordStepReply(step.stepName, step.reply, step.observedLatencyMs, step.sortIndex);
|
|
4703
|
+
}
|
|
3860
4704
|
recordScenarioReply(attempt.reply, attempt.observedLatencyMs);
|
|
3861
4705
|
if (scenario.getMaxFailCount() > 0 && runtime.allFailCount >= scenario.getMaxFailCount()) {
|
|
3862
4706
|
stopScenario = true;
|
|
@@ -3876,30 +4720,287 @@ async function executeScenarioRuntime(args) {
|
|
|
3876
4720
|
const endMs = Date.now() + Math.trunc(warmUpDurationSeconds * 1000);
|
|
3877
4721
|
const instanceInfo = nextInstanceInfo();
|
|
3878
4722
|
while (Date.now() < endMs && !shouldStopNow()) {
|
|
3879
|
-
|
|
4723
|
+
const globalOrdinal = nextObservationOrdinal();
|
|
4724
|
+
const attempt = await runSingleInvocation("WarmUp", instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, false);
|
|
4725
|
+
captureAttemptObservation("WarmUp", globalOrdinal, 0, true, attempt, -1, "SingleInvocation");
|
|
4726
|
+
if (attempt.policyFailure) {
|
|
4727
|
+
stopScenario = true;
|
|
4728
|
+
scenarioAbortController.abort(attempt.policyFailure);
|
|
4729
|
+
throw attempt.policyFailure;
|
|
4730
|
+
}
|
|
3880
4731
|
}
|
|
3881
4732
|
};
|
|
3882
|
-
const
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
const
|
|
3887
|
-
const
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
4733
|
+
const executeV2TimedConstant = async (copies, durationNs, ramping, runSimulationInvocation, segment) => {
|
|
4734
|
+
if (!loadEngineV2Budget) {
|
|
4735
|
+
return;
|
|
4736
|
+
}
|
|
4737
|
+
const offsets = ramping ? planRampingConstantDeadlines(copies, durationNs) : Array(copies).fill(0n);
|
|
4738
|
+
const ownedWorkerSlots = trafficMixV2
|
|
4739
|
+
? loadEngineV2TrafficMixOwnedUnits(BigInt(copies), trafficMixV2.shareWeights, trafficMixV2.laneIndex, scenarioPartition.number, scenarioPartition.count)
|
|
4740
|
+
: offsets.flatMap((_offset, workerSlot) => workerSlot % scenarioPartition.count === scenarioPartition.number
|
|
4741
|
+
? [{ laneOrdinal: BigInt(workerSlot), globalRank: BigInt(workerSlot) }]
|
|
4742
|
+
: []);
|
|
4743
|
+
const quantum = ramping ? maxBigInt(1n, durationNs / BigInt(copies)) : 1n;
|
|
4744
|
+
const toleranceNs = minBigInt(100000000n, maxBigInt(2000000n, quantum * 4n));
|
|
4745
|
+
const startNs = process.hrtime.bigint();
|
|
4746
|
+
const endNs = startNs + durationNs;
|
|
4747
|
+
const active = new Set();
|
|
4748
|
+
let schedulerLate = 0;
|
|
4749
|
+
let unavailable = 0;
|
|
4750
|
+
let executionError;
|
|
4751
|
+
if (segment) {
|
|
4752
|
+
segment.requestedWorkers = BigInt(ownedWorkerSlots.length);
|
|
4753
|
+
}
|
|
4754
|
+
for (const unit of ownedWorkerSlots) {
|
|
4755
|
+
const offsetNs = offsets[Number(unit.globalRank)];
|
|
4756
|
+
await delayUntilMonotonicDeadline(startNs + offsetNs, scenarioCancellationToken);
|
|
4757
|
+
if (shouldStopNow()) {
|
|
4758
|
+
break;
|
|
4759
|
+
}
|
|
4760
|
+
const decisionNowNs = process.hrtime.bigint();
|
|
4761
|
+
if (segment)
|
|
4762
|
+
loadEngineV2Telemetry?.recordDecisionLag(segment, decisionNowNs - (startNs + offsetNs));
|
|
4763
|
+
if (decisionNowNs - (startNs + offsetNs) > toleranceNs) {
|
|
4764
|
+
schedulerLate += 1;
|
|
4765
|
+
if (segment) {
|
|
4766
|
+
segment.unavailableWorkers += 1n;
|
|
4767
|
+
incrementReason(segment.unavailableWorkerReasons, "scheduler_late");
|
|
4768
|
+
}
|
|
4769
|
+
continue;
|
|
4770
|
+
}
|
|
4771
|
+
const release = loadEngineV2Budget.tryAcquire();
|
|
4772
|
+
if (!release) {
|
|
4773
|
+
unavailable += 1;
|
|
4774
|
+
if (segment) {
|
|
4775
|
+
segment.unavailableWorkers += 1n;
|
|
4776
|
+
incrementReason(segment.unavailableWorkerReasons, "max_in_flight");
|
|
4777
|
+
}
|
|
4778
|
+
continue;
|
|
4779
|
+
}
|
|
4780
|
+
if (segment)
|
|
4781
|
+
segment.startedWorkers += 1n;
|
|
4782
|
+
const instanceInfo = nextInstanceInfo();
|
|
4783
|
+
let task;
|
|
4784
|
+
task = (async () => {
|
|
4785
|
+
if (segment) {
|
|
4786
|
+
loadEngineV2Telemetry?.recordStartLag(segment, process.hrtime.bigint() - (startNs + offsetNs));
|
|
4787
|
+
}
|
|
4788
|
+
let completedSinceYield = 0;
|
|
4789
|
+
let workerIterationIndex = 0n;
|
|
4790
|
+
while (process.hrtime.bigint() < endNs && !shouldStopNow()) {
|
|
4791
|
+
if (segment) {
|
|
4792
|
+
segment.planned += 1n;
|
|
4793
|
+
segment.due += 1n;
|
|
4794
|
+
segment.started += 1n;
|
|
4795
|
+
}
|
|
4796
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, unit.globalRank, workerIterationIndex);
|
|
4797
|
+
workerIterationIndex += 1n;
|
|
4798
|
+
if (segment)
|
|
4799
|
+
segment.completed += 1n;
|
|
4800
|
+
completedSinceYield += 1;
|
|
4801
|
+
if (completedSinceYield >= 64) {
|
|
4802
|
+
completedSinceYield = 0;
|
|
4803
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
4804
|
+
}
|
|
4805
|
+
}
|
|
4806
|
+
})().catch((error) => {
|
|
4807
|
+
executionError ?? (executionError = error);
|
|
4808
|
+
}).finally(() => {
|
|
4809
|
+
release();
|
|
4810
|
+
active.delete(task);
|
|
4811
|
+
});
|
|
4812
|
+
active.add(task);
|
|
4813
|
+
}
|
|
4814
|
+
while (active.size > 0) {
|
|
4815
|
+
await Promise.race(active);
|
|
4816
|
+
}
|
|
4817
|
+
if (segment) {
|
|
4818
|
+
segment.accountingComplete = segment.completed === segment.started
|
|
4819
|
+
&& segment.started === segment.due
|
|
4820
|
+
&& segment.due === segment.planned
|
|
4821
|
+
&& segment.startedWorkers + segment.unavailableWorkers === segment.requestedWorkers;
|
|
4822
|
+
loadEngineV2Telemetry?.recordWarning("scheduler_late", segment, BigInt(schedulerLate));
|
|
4823
|
+
loadEngineV2Telemetry?.recordWarning("max_in_flight", segment, BigInt(unavailable));
|
|
4824
|
+
}
|
|
4825
|
+
if (schedulerLate > 0) {
|
|
4826
|
+
logger.warn(`Load Engine V2 warning scheduler_late: ${schedulerLate} constant worker slots were unavailable for scenario ${scenario.name}.`);
|
|
4827
|
+
}
|
|
4828
|
+
if (unavailable > 0) {
|
|
4829
|
+
logger.warn(`Load Engine V2 warning max_in_flight: ${unavailable} constant worker slots were unavailable for scenario ${scenario.name}.`);
|
|
4830
|
+
}
|
|
4831
|
+
if (executionError !== undefined) {
|
|
4832
|
+
throw executionError;
|
|
4833
|
+
}
|
|
4834
|
+
};
|
|
4835
|
+
const executeV2IterationsConstant = async (copies, iterations, runSimulationInvocation, segment) => {
|
|
4836
|
+
if (!loadEngineV2Budget || iterations <= 0) {
|
|
4837
|
+
return;
|
|
4838
|
+
}
|
|
4839
|
+
let activeShardCount;
|
|
4840
|
+
let ownedWorkerSlots;
|
|
4841
|
+
let ownedIterations;
|
|
4842
|
+
if (trafficMixV2) {
|
|
4843
|
+
const laneCopies = loadEngineV2TrafficMixLaneUnitCount(BigInt(copies), trafficMixV2.shareWeights, trafficMixV2.laneIndex);
|
|
4844
|
+
const laneIterations = loadEngineV2TrafficMixLaneUnitCount(BigInt(iterations), trafficMixV2.shareWeights, trafficMixV2.laneIndex);
|
|
4845
|
+
if (laneIterations === 0n) {
|
|
4846
|
+
if (segment) {
|
|
4847
|
+
segment.requestedWorkers = 0n;
|
|
4848
|
+
segment.accountingComplete = true;
|
|
4849
|
+
}
|
|
4850
|
+
return;
|
|
4851
|
+
}
|
|
4852
|
+
if (laneCopies === 0n) {
|
|
4853
|
+
throw new Error("Load Engine V2 traffic-mix lane owns iterations but no constant worker slot.");
|
|
4854
|
+
}
|
|
4855
|
+
activeShardCount = Math.min(scenarioPartition.count, Number(laneCopies), Number(laneIterations));
|
|
4856
|
+
if (scenarioPartition.number >= activeShardCount) {
|
|
4857
|
+
if (segment) {
|
|
4858
|
+
segment.requestedWorkers = 0n;
|
|
4859
|
+
segment.accountingComplete = true;
|
|
4860
|
+
}
|
|
4861
|
+
return;
|
|
4862
|
+
}
|
|
4863
|
+
ownedWorkerSlots = loadEngineV2TrafficMixOwnedUnits(BigInt(copies), trafficMixV2.shareWeights, trafficMixV2.laneIndex, scenarioPartition.number, activeShardCount);
|
|
4864
|
+
ownedIterations = loadEngineV2TrafficMixOwnedUnits(BigInt(iterations), trafficMixV2.shareWeights, trafficMixV2.laneIndex, scenarioPartition.number, activeShardCount);
|
|
4865
|
+
}
|
|
4866
|
+
else {
|
|
4867
|
+
activeShardCount = Math.min(scenarioPartition.count, copies, iterations);
|
|
4868
|
+
if (scenarioPartition.number >= activeShardCount) {
|
|
4869
|
+
return;
|
|
4870
|
+
}
|
|
4871
|
+
ownedWorkerSlots = Array.from({ length: copies }, (_value, slot) => slot)
|
|
4872
|
+
.filter((slot) => slot % activeShardCount === scenarioPartition.number)
|
|
4873
|
+
.map((slot) => ({ laneOrdinal: BigInt(slot), globalRank: BigInt(slot) }));
|
|
4874
|
+
ownedIterations = Array.from({ length: iterations }, (_value, ordinal) => ordinal)
|
|
4875
|
+
.filter((ordinal) => ordinal % activeShardCount === scenarioPartition.number)
|
|
4876
|
+
.map((ordinal) => ({ laneOrdinal: BigInt(ordinal), globalRank: BigInt(ordinal) }));
|
|
4877
|
+
}
|
|
4878
|
+
if (scenarioPartition.number >= activeShardCount) {
|
|
4879
|
+
return;
|
|
4880
|
+
}
|
|
4881
|
+
const targetWorkers = Math.min(ownedWorkerSlots.length, ownedIterations.length);
|
|
4882
|
+
if (targetWorkers <= 0) {
|
|
4883
|
+
if (segment) {
|
|
4884
|
+
segment.requestedWorkers = 0n;
|
|
4885
|
+
segment.accountingComplete = true;
|
|
4886
|
+
}
|
|
4887
|
+
return;
|
|
4888
|
+
}
|
|
4889
|
+
if (segment) {
|
|
4890
|
+
segment.planned = BigInt(ownedIterations.length);
|
|
4891
|
+
segment.requestedWorkers = BigInt(targetWorkers);
|
|
4892
|
+
}
|
|
4893
|
+
const releases = [];
|
|
4894
|
+
let firstRelease;
|
|
4895
|
+
while (!firstRelease && !shouldStopNow()) {
|
|
4896
|
+
firstRelease = loadEngineV2Budget.tryAcquire();
|
|
4897
|
+
if (!firstRelease) {
|
|
4898
|
+
await delayWithAbort(1, scenarioCancellationToken);
|
|
4899
|
+
}
|
|
4900
|
+
}
|
|
4901
|
+
if (!firstRelease) {
|
|
4902
|
+
if (segment) {
|
|
4903
|
+
segment.unreached = segment.planned;
|
|
4904
|
+
segment.unavailableWorkers = segment.requestedWorkers;
|
|
4905
|
+
if (segment.unavailableWorkers > 0n) {
|
|
4906
|
+
incrementReason(segment.unavailableWorkerReasons, "cancelled", segment.unavailableWorkers);
|
|
4907
|
+
}
|
|
4908
|
+
segment.accountingComplete = true;
|
|
4909
|
+
}
|
|
4910
|
+
return;
|
|
4911
|
+
}
|
|
4912
|
+
releases.push(firstRelease);
|
|
4913
|
+
for (let worker = 1; worker < targetWorkers; worker += 1) {
|
|
4914
|
+
const release = loadEngineV2Budget.tryAcquire();
|
|
4915
|
+
if (release) {
|
|
4916
|
+
releases.push(release);
|
|
4917
|
+
}
|
|
4918
|
+
}
|
|
4919
|
+
if (releases.length < targetWorkers) {
|
|
4920
|
+
logger.warn(`Load Engine V2 warning max_in_flight: requested ${targetWorkers} constant workers but started ${releases.length} for scenario ${scenario.name}; all iterations will still run.`);
|
|
4921
|
+
}
|
|
4922
|
+
if (segment) {
|
|
4923
|
+
segment.startedWorkers = BigInt(releases.length);
|
|
4924
|
+
segment.unavailableWorkers = BigInt(targetWorkers - releases.length);
|
|
4925
|
+
if (segment.unavailableWorkers > 0n) {
|
|
4926
|
+
incrementReason(segment.unavailableWorkerReasons, "max_in_flight", segment.unavailableWorkers);
|
|
4927
|
+
loadEngineV2Telemetry?.recordWarning("max_in_flight", segment, segment.unavailableWorkers);
|
|
4928
|
+
}
|
|
4929
|
+
}
|
|
4930
|
+
let nextIterationIndex = 0;
|
|
4931
|
+
const tasks = releases.map(async (release) => {
|
|
4932
|
+
const instanceInfo = nextInstanceInfo();
|
|
4933
|
+
try {
|
|
4934
|
+
while (!shouldStopNow()) {
|
|
4935
|
+
const unit = ownedIterations[nextIterationIndex];
|
|
4936
|
+
nextIterationIndex += 1;
|
|
4937
|
+
if (!unit) {
|
|
4938
|
+
break;
|
|
4939
|
+
}
|
|
4940
|
+
if (segment) {
|
|
4941
|
+
segment.due += 1n;
|
|
4942
|
+
segment.started += 1n;
|
|
4943
|
+
}
|
|
4944
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, unit.globalRank, 0n);
|
|
4945
|
+
if (segment)
|
|
4946
|
+
segment.completed += 1n;
|
|
4947
|
+
}
|
|
4948
|
+
}
|
|
4949
|
+
finally {
|
|
4950
|
+
release();
|
|
4951
|
+
}
|
|
4952
|
+
});
|
|
4953
|
+
await waitForScenarioTasks(tasks, scenario.name, scenarioCompletionTimeoutSeconds, logger, scenarioCancellationToken);
|
|
4954
|
+
if (segment) {
|
|
4955
|
+
segment.unreached = segment.planned > segment.due ? segment.planned - segment.due : 0n;
|
|
4956
|
+
segment.accountingComplete = segment.due + segment.unreached === segment.planned
|
|
4957
|
+
&& segment.started === segment.due
|
|
4958
|
+
&& segment.completed === segment.started
|
|
4959
|
+
&& segment.startedWorkers + segment.unavailableWorkers === segment.requestedWorkers;
|
|
4960
|
+
}
|
|
4961
|
+
};
|
|
4962
|
+
const executeSimulationAsync = async (simulation, simulationIndex) => {
|
|
4963
|
+
const descriptor = trafficMixV2
|
|
4964
|
+
? trafficMixV2.originalSimulations[simulationIndex]
|
|
4965
|
+
: simulation;
|
|
4966
|
+
if (!descriptor) {
|
|
4967
|
+
throw new Error(`Load Engine V2 traffic-mix phase ${simulationIndex} is missing its global descriptor.`);
|
|
4968
|
+
}
|
|
4969
|
+
const kind = String(descriptor.Kind ?? "");
|
|
4970
|
+
const simulationKind = kind || "SingleInvocation";
|
|
4971
|
+
const runSimulationInvocation = (instanceData, instanceNumber, instanceId, explicitGlobalOrdinal, explicitSecondaryOrdinal = 0n) => runBombingInvocation(instanceData, instanceNumber, instanceId, explicitGlobalOrdinal, simulationIndex, simulationKind, explicitSecondaryOrdinal);
|
|
4972
|
+
if (options.loadEngineContractVersion === 2) {
|
|
4973
|
+
nextV2FallbackOrdinal = BigInt(scenarioPartition.number);
|
|
4974
|
+
}
|
|
4975
|
+
const weight = trafficMixV2 ? 1 : Math.max(scenario.getWeight(), 1);
|
|
4976
|
+
const rate = applyScenarioWeight(toInt(descriptor.Rate), weight);
|
|
4977
|
+
const minRate = applyScenarioWeight(toInt(descriptor.MinRate), weight);
|
|
4978
|
+
const maxRate = applyScenarioWeight(toInt(descriptor.MaxRate), weight);
|
|
4979
|
+
const copies = Math.max(applyScenarioWeight(Math.max(toInt(descriptor.Copies), 1), weight), 1);
|
|
4980
|
+
const iterations = applyScenarioWeight(Math.max(toInt(descriptor.Iterations), 0), weight);
|
|
4981
|
+
const intervalMs = Math.max(Math.trunc(Math.max(toNumber(descriptor.IntervalSeconds), 0) * 1000), 0);
|
|
4982
|
+
const duringMs = Math.max(Math.trunc(Math.max(toNumber(descriptor.DuringSeconds), 0) * 1000), 0);
|
|
4983
|
+
accumulator.setLoadSimulation(kind, resolveLoadSimulationValue(descriptor, weight));
|
|
3893
4984
|
accumulator.setCurrentOperation("Bombing");
|
|
4985
|
+
const schedulerSegment = options.loadEngineContractVersion === 2
|
|
4986
|
+
? loadEngineV2Telemetry?.createSegment(scenario.name, scenarioIndex, simulationIndex, kind, scenarioPartition.number, scenarioPartition.count)
|
|
4987
|
+
: undefined;
|
|
4988
|
+
const trafficMixOwnedOrdinals = (total) => trafficMixV2
|
|
4989
|
+
? loadEngineV2TrafficMixOwnedUnits(total, trafficMixV2.shareWeights, trafficMixV2.laneIndex, scenarioPartition.number, scenarioPartition.count).map((unit) => unit.globalRank)
|
|
4990
|
+
: undefined;
|
|
3894
4991
|
if (kind === "KeepConstant") {
|
|
3895
4992
|
if (duringMs <= 0) {
|
|
3896
4993
|
return;
|
|
3897
4994
|
}
|
|
4995
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
4996
|
+
await executeV2TimedConstant(copies, secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "KeepConstant duration"), false, runSimulationInvocation, schedulerSegment);
|
|
4997
|
+
return;
|
|
4998
|
+
}
|
|
3898
4999
|
const endMs = Date.now() + duringMs;
|
|
3899
5000
|
const tasks = Array.from({ length: copies }, async () => {
|
|
3900
5001
|
const instanceInfo = nextInstanceInfo();
|
|
3901
5002
|
while (Date.now() < endMs && !shouldStopNow()) {
|
|
3902
|
-
await
|
|
5003
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId);
|
|
3903
5004
|
}
|
|
3904
5005
|
});
|
|
3905
5006
|
await waitForScenarioTasks(tasks, scenario.name, scenarioCompletionTimeoutSeconds, logger, scenarioCancellationToken);
|
|
@@ -3909,6 +5010,10 @@ async function executeScenarioRuntime(args) {
|
|
|
3909
5010
|
if (duringMs <= 0) {
|
|
3910
5011
|
return;
|
|
3911
5012
|
}
|
|
5013
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5014
|
+
await executeV2TimedConstant(copies, secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "RampingConstant duration"), true, runSimulationInvocation, schedulerSegment);
|
|
5015
|
+
return;
|
|
5016
|
+
}
|
|
3912
5017
|
const tasks = [];
|
|
3913
5018
|
const endMs = Date.now() + duringMs;
|
|
3914
5019
|
const startIntervalMs = copies <= 1 ? 0 : Math.max(Math.trunc(duringMs / copies), 0);
|
|
@@ -3916,7 +5021,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3916
5021
|
const instanceInfo = nextInstanceInfo();
|
|
3917
5022
|
tasks.push((async () => {
|
|
3918
5023
|
while (Date.now() < endMs && !shouldStopNow()) {
|
|
3919
|
-
await
|
|
5024
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId);
|
|
3920
5025
|
}
|
|
3921
5026
|
})());
|
|
3922
5027
|
if (copy < copies && startIntervalMs > 0) {
|
|
@@ -3930,12 +5035,34 @@ async function executeScenarioRuntime(args) {
|
|
|
3930
5035
|
if (duringMs <= 0 || rate <= 0 || intervalMs <= 0) {
|
|
3931
5036
|
return;
|
|
3932
5037
|
}
|
|
5038
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5039
|
+
const intervalNs = secondsToNanoseconds(toNumber(descriptor.IntervalSeconds), "Inject interval");
|
|
5040
|
+
const durationNs = secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "Inject duration");
|
|
5041
|
+
await executeV2FixedArrivals({
|
|
5042
|
+
rate,
|
|
5043
|
+
intervalNs,
|
|
5044
|
+
totalArrivals: loadEngineV2FixedArrivalCount(rate, intervalNs, durationNs),
|
|
5045
|
+
budget: loadEngineV2Budget,
|
|
5046
|
+
cancellationToken: scenarioCancellationToken,
|
|
5047
|
+
shouldStopNow,
|
|
5048
|
+
nextInstanceInfo,
|
|
5049
|
+
runBombingInvocation: runSimulationInvocation,
|
|
5050
|
+
logger,
|
|
5051
|
+
scenarioName: scenario.name,
|
|
5052
|
+
shardIndex: scenarioPartition.number,
|
|
5053
|
+
shardCount: scenarioPartition.count,
|
|
5054
|
+
ownedOrdinals: trafficMixOwnedOrdinals(loadEngineV2FixedArrivalCount(rate, intervalNs, durationNs)),
|
|
5055
|
+
telemetry: loadEngineV2Telemetry,
|
|
5056
|
+
segment: schedulerSegment
|
|
5057
|
+
});
|
|
5058
|
+
return;
|
|
5059
|
+
}
|
|
3933
5060
|
const pending = [];
|
|
3934
5061
|
const endMs = Date.now() + duringMs;
|
|
3935
5062
|
while (Date.now() < endMs && !shouldStopNow()) {
|
|
3936
5063
|
for (let index = 0; index < rate; index += 1) {
|
|
3937
5064
|
const instanceInfo = nextInstanceInfo();
|
|
3938
|
-
pending.push(
|
|
5065
|
+
pending.push(runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId));
|
|
3939
5066
|
}
|
|
3940
5067
|
await delayWithAbort(intervalMs, scenarioCancellationToken);
|
|
3941
5068
|
}
|
|
@@ -3946,6 +5073,31 @@ async function executeScenarioRuntime(args) {
|
|
|
3946
5073
|
if (duringMs <= 0 || rate <= 0 || intervalMs <= 0) {
|
|
3947
5074
|
return;
|
|
3948
5075
|
}
|
|
5076
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5077
|
+
const intervalNs = secondsToNanoseconds(toNumber(descriptor.IntervalSeconds), "RampingInject interval");
|
|
5078
|
+
const durationNs = secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "RampingInject duration");
|
|
5079
|
+
const offsets = planRampingInjectionDeadlines(rate, intervalNs, durationNs);
|
|
5080
|
+
await executeV2FixedArrivals({
|
|
5081
|
+
rate,
|
|
5082
|
+
intervalNs,
|
|
5083
|
+
totalArrivals: BigInt(offsets.length),
|
|
5084
|
+
deadlineOffsetsNs: offsets,
|
|
5085
|
+
tolerancesNs: buildV2RampTolerances(offsets, durationNs),
|
|
5086
|
+
budget: loadEngineV2Budget,
|
|
5087
|
+
cancellationToken: scenarioCancellationToken,
|
|
5088
|
+
shouldStopNow,
|
|
5089
|
+
nextInstanceInfo,
|
|
5090
|
+
runBombingInvocation: runSimulationInvocation,
|
|
5091
|
+
logger,
|
|
5092
|
+
scenarioName: scenario.name,
|
|
5093
|
+
shardIndex: scenarioPartition.number,
|
|
5094
|
+
shardCount: scenarioPartition.count,
|
|
5095
|
+
ownedOrdinals: trafficMixOwnedOrdinals(BigInt(offsets.length)),
|
|
5096
|
+
telemetry: loadEngineV2Telemetry,
|
|
5097
|
+
segment: schedulerSegment
|
|
5098
|
+
});
|
|
5099
|
+
return;
|
|
5100
|
+
}
|
|
3949
5101
|
const pending = [];
|
|
3950
5102
|
const startedAtMs = Date.now();
|
|
3951
5103
|
const endMs = startedAtMs + duringMs;
|
|
@@ -3955,7 +5107,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3955
5107
|
const currentRate = Math.max(1, Math.ceil(rate * progress));
|
|
3956
5108
|
for (let index = 0; index < currentRate; index += 1) {
|
|
3957
5109
|
const instanceInfo = nextInstanceInfo();
|
|
3958
|
-
pending.push(
|
|
5110
|
+
pending.push(runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId));
|
|
3959
5111
|
}
|
|
3960
5112
|
await delayWithAbort(intervalMs, scenarioCancellationToken);
|
|
3961
5113
|
}
|
|
@@ -3966,6 +5118,37 @@ async function executeScenarioRuntime(args) {
|
|
|
3966
5118
|
if (duringMs <= 0 || maxRate <= 0 || intervalMs <= 0) {
|
|
3967
5119
|
return;
|
|
3968
5120
|
}
|
|
5121
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5122
|
+
const intervalNs = secondsToNanoseconds(toNumber(descriptor.IntervalSeconds), "InjectRandom interval");
|
|
5123
|
+
const durationNs = secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "InjectRandom duration");
|
|
5124
|
+
const descriptorSeed = fnv1a32(trafficMixV2
|
|
5125
|
+
? `traffic-mix\n${trafficMixV2.seedId}\n${simulationIndex}`
|
|
5126
|
+
: `${scenario.name}\n${simulationIndex}`);
|
|
5127
|
+
const rotated = ((descriptorSeed << 13) | (descriptorSeed >>> 19)) >>> 0;
|
|
5128
|
+
const seed = (fnv1a32(testInfo.sessionId) ^ rotated) >>> 0;
|
|
5129
|
+
const offsets = planRandomInjectionDeadlines(Math.max(0, minRate), maxRate, intervalNs, durationNs, seed);
|
|
5130
|
+
const tolerance = loadEngineV2LatenessToleranceNs(Math.max(1, maxRate), intervalNs);
|
|
5131
|
+
await executeV2FixedArrivals({
|
|
5132
|
+
rate: Math.max(1, maxRate),
|
|
5133
|
+
intervalNs,
|
|
5134
|
+
totalArrivals: BigInt(offsets.length),
|
|
5135
|
+
deadlineOffsetsNs: offsets,
|
|
5136
|
+
tolerancesNs: offsets.map(() => tolerance),
|
|
5137
|
+
budget: loadEngineV2Budget,
|
|
5138
|
+
cancellationToken: scenarioCancellationToken,
|
|
5139
|
+
shouldStopNow,
|
|
5140
|
+
nextInstanceInfo,
|
|
5141
|
+
runBombingInvocation: runSimulationInvocation,
|
|
5142
|
+
logger,
|
|
5143
|
+
scenarioName: scenario.name,
|
|
5144
|
+
shardIndex: scenarioPartition.number,
|
|
5145
|
+
shardCount: scenarioPartition.count,
|
|
5146
|
+
ownedOrdinals: trafficMixOwnedOrdinals(BigInt(offsets.length)),
|
|
5147
|
+
telemetry: loadEngineV2Telemetry,
|
|
5148
|
+
segment: schedulerSegment
|
|
5149
|
+
});
|
|
5150
|
+
return;
|
|
5151
|
+
}
|
|
3969
5152
|
const pending = [];
|
|
3970
5153
|
const normalizedMinRate = Math.max(1, minRate);
|
|
3971
5154
|
const normalizedMaxRate = Math.max(normalizedMinRate, maxRate);
|
|
@@ -3974,7 +5157,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3974
5157
|
const currentRate = randomIntInclusive(normalizedMinRate, normalizedMaxRate);
|
|
3975
5158
|
for (let index = 0; index < currentRate; index += 1) {
|
|
3976
5159
|
const instanceInfo = nextInstanceInfo();
|
|
3977
|
-
pending.push(
|
|
5160
|
+
pending.push(runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId));
|
|
3978
5161
|
}
|
|
3979
5162
|
await delayWithAbort(intervalMs, scenarioCancellationToken);
|
|
3980
5163
|
}
|
|
@@ -3985,6 +5168,10 @@ async function executeScenarioRuntime(args) {
|
|
|
3985
5168
|
if (iterations <= 0) {
|
|
3986
5169
|
return;
|
|
3987
5170
|
}
|
|
5171
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5172
|
+
await executeV2IterationsConstant(copies, iterations, runSimulationInvocation, schedulerSegment);
|
|
5173
|
+
return;
|
|
5174
|
+
}
|
|
3988
5175
|
let remaining = iterations;
|
|
3989
5176
|
const tasks = Array.from({ length: copies }, async () => {
|
|
3990
5177
|
const instanceInfo = nextInstanceInfo();
|
|
@@ -3993,7 +5180,7 @@ async function executeScenarioRuntime(args) {
|
|
|
3993
5180
|
if (remaining < 0) {
|
|
3994
5181
|
break;
|
|
3995
5182
|
}
|
|
3996
|
-
await
|
|
5183
|
+
await runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId);
|
|
3997
5184
|
}
|
|
3998
5185
|
});
|
|
3999
5186
|
await waitForScenarioTasks(tasks, scenario.name, scenarioCompletionTimeoutSeconds, logger, scenarioCancellationToken);
|
|
@@ -4003,13 +5190,33 @@ async function executeScenarioRuntime(args) {
|
|
|
4003
5190
|
if (iterations <= 0 || rate <= 0 || intervalMs <= 0) {
|
|
4004
5191
|
return;
|
|
4005
5192
|
}
|
|
5193
|
+
if (options.loadEngineContractVersion === 2 && loadEngineV2Budget) {
|
|
5194
|
+
await executeV2FixedArrivals({
|
|
5195
|
+
rate,
|
|
5196
|
+
intervalNs: secondsToNanoseconds(toNumber(descriptor.IntervalSeconds), "IterationsForInject interval"),
|
|
5197
|
+
totalArrivals: BigInt(iterations),
|
|
5198
|
+
budget: loadEngineV2Budget,
|
|
5199
|
+
cancellationToken: scenarioCancellationToken,
|
|
5200
|
+
shouldStopNow,
|
|
5201
|
+
nextInstanceInfo,
|
|
5202
|
+
runBombingInvocation: runSimulationInvocation,
|
|
5203
|
+
logger,
|
|
5204
|
+
scenarioName: scenario.name,
|
|
5205
|
+
shardIndex: scenarioPartition.number,
|
|
5206
|
+
shardCount: scenarioPartition.count,
|
|
5207
|
+
ownedOrdinals: trafficMixOwnedOrdinals(BigInt(iterations)),
|
|
5208
|
+
telemetry: loadEngineV2Telemetry,
|
|
5209
|
+
segment: schedulerSegment
|
|
5210
|
+
});
|
|
5211
|
+
return;
|
|
5212
|
+
}
|
|
4006
5213
|
const pending = [];
|
|
4007
5214
|
let remaining = iterations;
|
|
4008
5215
|
while (remaining > 0 && !shouldStopNow()) {
|
|
4009
5216
|
const count = Math.min(rate, remaining);
|
|
4010
5217
|
for (let index = 0; index < count; index += 1) {
|
|
4011
5218
|
const instanceInfo = nextInstanceInfo();
|
|
4012
|
-
pending.push(
|
|
5219
|
+
pending.push(runSimulationInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId));
|
|
4013
5220
|
}
|
|
4014
5221
|
remaining -= count;
|
|
4015
5222
|
if (remaining > 0) {
|
|
@@ -4021,10 +5228,19 @@ async function executeScenarioRuntime(args) {
|
|
|
4021
5228
|
}
|
|
4022
5229
|
if (kind === "Pause") {
|
|
4023
5230
|
if (duringMs > 0) {
|
|
4024
|
-
|
|
5231
|
+
if (options.loadEngineContractVersion === 2) {
|
|
5232
|
+
await delayUntilMonotonicDeadline(process.hrtime.bigint() + secondsToNanoseconds(toNumber(descriptor.DuringSeconds), "Pause duration"), scenarioCancellationToken);
|
|
5233
|
+
}
|
|
5234
|
+
else {
|
|
5235
|
+
await delayWithAbort(duringMs, scenarioCancellationToken);
|
|
5236
|
+
}
|
|
4025
5237
|
}
|
|
5238
|
+
if (schedulerSegment)
|
|
5239
|
+
schedulerSegment.accountingComplete = true;
|
|
4026
5240
|
return;
|
|
4027
5241
|
}
|
|
5242
|
+
if (schedulerSegment)
|
|
5243
|
+
schedulerSegment.accountingComplete = true;
|
|
4028
5244
|
};
|
|
4029
5245
|
const initContext = {
|
|
4030
5246
|
customSettings: { ...(options.customSettings ?? {}) },
|
|
@@ -4056,15 +5272,33 @@ async function executeScenarioRuntime(args) {
|
|
|
4056
5272
|
accumulator.setCurrentOperation("Bombing");
|
|
4057
5273
|
const simulations = scenario.getSimulations();
|
|
4058
5274
|
if (!simulations.length) {
|
|
4059
|
-
|
|
4060
|
-
|
|
5275
|
+
if (options.loadEngineContractVersion === 2) {
|
|
5276
|
+
await options.loadEngineV2SegmentLifecycleOverride?.beforeSegment(scenarioIndex, 0);
|
|
5277
|
+
try {
|
|
5278
|
+
await executeSimulationAsync(LoadStrikeSimulation.iterationsForConstant(1, 1), 0);
|
|
5279
|
+
}
|
|
5280
|
+
finally {
|
|
5281
|
+
await options.loadEngineV2SegmentLifecycleOverride?.afterSegment(scenarioIndex, 0);
|
|
5282
|
+
}
|
|
5283
|
+
}
|
|
5284
|
+
else {
|
|
5285
|
+
const instanceInfo = nextInstanceInfo();
|
|
5286
|
+
await runBombingInvocation(instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId);
|
|
5287
|
+
}
|
|
4061
5288
|
}
|
|
4062
5289
|
else {
|
|
4063
|
-
for (
|
|
5290
|
+
for (let simulationIndex = 0; simulationIndex < simulations.length; simulationIndex += 1) {
|
|
5291
|
+
const simulation = simulations[simulationIndex];
|
|
4064
5292
|
if (shouldStopNow()) {
|
|
4065
5293
|
break;
|
|
4066
5294
|
}
|
|
4067
|
-
await
|
|
5295
|
+
await options.loadEngineV2SegmentLifecycleOverride?.beforeSegment(scenarioIndex, simulationIndex);
|
|
5296
|
+
try {
|
|
5297
|
+
await executeSimulationAsync(simulation, simulationIndex);
|
|
5298
|
+
}
|
|
5299
|
+
finally {
|
|
5300
|
+
await options.loadEngineV2SegmentLifecycleOverride?.afterSegment(scenarioIndex, simulationIndex);
|
|
5301
|
+
}
|
|
4068
5302
|
}
|
|
4069
5303
|
}
|
|
4070
5304
|
accumulator.setCurrentOperation(stopScenario || stopTestState.value ? "Stop" : "Complete");
|
|
@@ -4153,18 +5387,36 @@ async function waitForScenarioTasks(tasks, scenarioName, timeoutSeconds, logger,
|
|
|
4153
5387
|
if (!tasks.length) {
|
|
4154
5388
|
return;
|
|
4155
5389
|
}
|
|
4156
|
-
const
|
|
4157
|
-
|
|
4158
|
-
|
|
4159
|
-
|
|
5390
|
+
const settled = Promise.allSettled(tasks);
|
|
5391
|
+
const throwPolicyFailure = (results) => {
|
|
5392
|
+
const failure = results.find((result) => result.status === "rejected" && result.reason instanceof RuntimePolicyCallbackError);
|
|
5393
|
+
if (failure) {
|
|
5394
|
+
throw failure.reason;
|
|
5395
|
+
}
|
|
5396
|
+
};
|
|
5397
|
+
if (timeoutSeconds <= 0) {
|
|
5398
|
+
throwPolicyFailure(await settled);
|
|
5399
|
+
return;
|
|
5400
|
+
}
|
|
5401
|
+
const timeoutController = new AbortController();
|
|
5402
|
+
let completed;
|
|
5403
|
+
try {
|
|
5404
|
+
completed = await Promise.race([
|
|
5405
|
+
settled.then(() => true),
|
|
5406
|
+
delayWithAbort(Math.trunc(timeoutSeconds * 1000), combineAbortSignals(signal, timeoutController.signal)).then(() => false)
|
|
5407
|
+
]);
|
|
5408
|
+
}
|
|
5409
|
+
finally {
|
|
5410
|
+
timeoutController.abort();
|
|
4160
5411
|
}
|
|
4161
|
-
const completed = await Promise.race([
|
|
4162
|
-
all.then(() => true),
|
|
4163
|
-
delayWithAbort(Math.trunc(timeoutSeconds * 1000), signal).then(() => false)
|
|
4164
|
-
]);
|
|
4165
5412
|
if (!completed) {
|
|
5413
|
+
if (signal.reason instanceof RuntimePolicyCallbackError) {
|
|
5414
|
+
throw signal.reason;
|
|
5415
|
+
}
|
|
4166
5416
|
logger.warn(`Scenario ${scenarioName} timed out while waiting for completion (${timeoutSeconds}s).`);
|
|
5417
|
+
return;
|
|
4167
5418
|
}
|
|
5419
|
+
throwPolicyFailure(await settled);
|
|
4168
5420
|
}
|
|
4169
5421
|
function randomIntInclusive(minValue, maxValue) {
|
|
4170
5422
|
const min = Math.trunc(Math.min(minValue, maxValue));
|
|
@@ -4391,6 +5643,9 @@ function normalizeDataTransferStatsValue(value) {
|
|
|
4391
5643
|
const source = asAliasRecord(value);
|
|
4392
5644
|
return {
|
|
4393
5645
|
allBytes: pickAliasNumber(source, "allBytes", "AllBytes"),
|
|
5646
|
+
...(hasAliasValue(source, "allBytes64", "AllBytes64")
|
|
5647
|
+
? { allBytes64: pickAliasString(source, "allBytes64", "AllBytes64") }
|
|
5648
|
+
: {}),
|
|
4394
5649
|
maxBytes: pickAliasNumber(source, "maxBytes", "MaxBytes"),
|
|
4395
5650
|
meanBytes: pickAliasNumber(source, "meanBytes", "MeanBytes"),
|
|
4396
5651
|
minBytes: pickAliasNumber(source, "minBytes", "MinBytes"),
|
|
@@ -4398,6 +5653,9 @@ function normalizeDataTransferStatsValue(value) {
|
|
|
4398
5653
|
percent75: pickAliasNumber(source, "percent75", "Percent75"),
|
|
4399
5654
|
percent95: pickAliasNumber(source, "percent95", "Percent95"),
|
|
4400
5655
|
percent99: pickAliasNumber(source, "percent99", "Percent99"),
|
|
5656
|
+
...(hasAliasValue(source, "percent100", "Percent100")
|
|
5657
|
+
? { percent100: pickAliasNumber(source, "percent100", "Percent100") }
|
|
5658
|
+
: {}),
|
|
4401
5659
|
stdDev: pickAliasNumber(source, "stdDev", "StdDev")
|
|
4402
5660
|
};
|
|
4403
5661
|
}
|
|
@@ -4420,6 +5678,9 @@ function normalizeLatencyStatsValue(value) {
|
|
|
4420
5678
|
percent75: pickAliasNumber(source, "percent75", "Percent75"),
|
|
4421
5679
|
percent95: pickAliasNumber(source, "percent95", "Percent95"),
|
|
4422
5680
|
percent99: pickAliasNumber(source, "percent99", "Percent99"),
|
|
5681
|
+
...(hasAliasValue(source, "percent100", "Percent100")
|
|
5682
|
+
? { percent100: pickAliasNumber(source, "percent100", "Percent100") }
|
|
5683
|
+
: {}),
|
|
4423
5684
|
stdDev: pickAliasNumber(source, "stdDev", "StdDev")
|
|
4424
5685
|
};
|
|
4425
5686
|
}
|
|
@@ -4435,13 +5696,42 @@ function normalizeStatusCodeStatsValue(value) {
|
|
|
4435
5696
|
}
|
|
4436
5697
|
function normalizeMeasurementStatsValue(value) {
|
|
4437
5698
|
const source = asAliasRecord(value);
|
|
4438
|
-
|
|
5699
|
+
const projected = {
|
|
5700
|
+
...(hasAliasValue(source, "count64", "Count64")
|
|
5701
|
+
? { count64: pickAliasString(source, "count64", "Count64") }
|
|
5702
|
+
: {}),
|
|
5703
|
+
...(hasAliasValue(source, "distributionMode", "DistributionMode")
|
|
5704
|
+
? { distributionMode: pickAliasString(source, "distributionMode", "DistributionMode") }
|
|
5705
|
+
: {}),
|
|
5706
|
+
...(hasAliasValue(source, "maxRelativeError", "MaxRelativeError")
|
|
5707
|
+
? { maxRelativeError: pickAliasNumber(source, "maxRelativeError", "MaxRelativeError") }
|
|
5708
|
+
: {}),
|
|
4439
5709
|
dataTransfer: normalizeDataTransferStatsValue(pickAliasValue(source, "dataTransfer", "DataTransfer")),
|
|
4440
5710
|
latency: normalizeLatencyStatsValue(pickAliasValue(source, "latency", "Latency")),
|
|
4441
5711
|
request: normalizeRequestStatsValue(pickAliasValue(source, "request", "Request")),
|
|
4442
5712
|
statusCodes: pickAliasArray(source, "statusCodes", "StatusCodes")
|
|
4443
5713
|
.map((entry) => normalizeStatusCodeStatsValue(entry))
|
|
4444
5714
|
};
|
|
5715
|
+
const sidecarValue = pickAliasValue(source, "histogramSidecar", "HistogramSidecar");
|
|
5716
|
+
if (sidecarValue && typeof sidecarValue === "object" && !Array.isArray(sidecarValue)) {
|
|
5717
|
+
const sidecar = sidecarValue;
|
|
5718
|
+
if (sidecar?.latency && sidecar.size) {
|
|
5719
|
+
Object.defineProperty(projected, "histogramSidecar", {
|
|
5720
|
+
value: {
|
|
5721
|
+
latency: LoadStrikeHistogramV1.fromSidecar(sidecar.latency).toSidecar(),
|
|
5722
|
+
size: LoadStrikeHistogramV1.fromSidecar(sidecar.size).toSidecar(),
|
|
5723
|
+
allBytes64: String(sidecar.allBytes64 ?? "0"),
|
|
5724
|
+
lessOrEq80064: String(sidecar.lessOrEq80064 ?? "0"),
|
|
5725
|
+
more800Less120064: String(sidecar.more800Less120064 ?? "0"),
|
|
5726
|
+
moreOrEq120064: String(sidecar.moreOrEq120064 ?? "0")
|
|
5727
|
+
},
|
|
5728
|
+
enumerable: false,
|
|
5729
|
+
configurable: false,
|
|
5730
|
+
writable: false
|
|
5731
|
+
});
|
|
5732
|
+
}
|
|
5733
|
+
}
|
|
5734
|
+
return projected;
|
|
4445
5735
|
}
|
|
4446
5736
|
function normalizeLoadSimulationStatsValue(value) {
|
|
4447
5737
|
const source = asAliasRecord(value);
|
|
@@ -4541,6 +5831,9 @@ function normalizeStepStatsValue(value, index = 0) {
|
|
|
4541
5831
|
statusCodes: normalizeAliasNumberRecord(pickAliasValue(source, "statusCodes", "StatusCodes")),
|
|
4542
5832
|
ok: normalizeMeasurementStatsValue(pickAliasValue(source, "ok", "Ok")),
|
|
4543
5833
|
fail: normalizeMeasurementStatsValue(pickAliasValue(source, "fail", "Fail")),
|
|
5834
|
+
...(hasAliasValue(source, "allMeasurement", "AllMeasurement")
|
|
5835
|
+
? { allMeasurement: normalizeMeasurementStatsValue(pickAliasValue(source, "allMeasurement", "AllMeasurement")) }
|
|
5836
|
+
: {}),
|
|
4544
5837
|
sortIndex: hasAliasValue(source, "sortIndex", "SortIndex")
|
|
4545
5838
|
? pickAliasNumber(source, "sortIndex", "SortIndex")
|
|
4546
5839
|
: index
|
|
@@ -4580,6 +5873,9 @@ function normalizeScenarioStatsValue(value, index = 0) {
|
|
|
4580
5873
|
durationMs: pickAliasNumber(source, "durationMs", "DurationMs", "Duration"),
|
|
4581
5874
|
ok: normalizeMeasurementStatsValue(pickAliasValue(source, "ok", "Ok")),
|
|
4582
5875
|
fail: normalizeMeasurementStatsValue(pickAliasValue(source, "fail", "Fail")),
|
|
5876
|
+
...(hasAliasValue(source, "allMeasurement", "AllMeasurement")
|
|
5877
|
+
? { allMeasurement: normalizeMeasurementStatsValue(pickAliasValue(source, "allMeasurement", "AllMeasurement")) }
|
|
5878
|
+
: {}),
|
|
4583
5879
|
loadSimulationStats: normalizeLoadSimulationStatsValue(pickAliasValue(source, "loadSimulationStats", "LoadSimulationStats")),
|
|
4584
5880
|
sortIndex: hasAliasValue(source, "sortIndex", "SortIndex")
|
|
4585
5881
|
? pickAliasNumber(source, "sortIndex", "SortIndex")
|
|
@@ -4666,7 +5962,7 @@ function attachSessionStartInfoAliases(session) {
|
|
|
4666
5962
|
return session;
|
|
4667
5963
|
}
|
|
4668
5964
|
function attachPortalReportingSession(sinkSession, sessionInfo, licenseClient, licenseSession) {
|
|
4669
|
-
const runToken =
|
|
5965
|
+
const runToken = currentLicenseSessionRunToken(licenseSession);
|
|
4670
5966
|
if (!runToken || !licenseClient) {
|
|
4671
5967
|
return;
|
|
4672
5968
|
}
|
|
@@ -4676,9 +5972,17 @@ function attachPortalReportingSession(sinkSession, sessionInfo, licenseClient, l
|
|
|
4676
5972
|
sinkSession.portalReportingIngestUrl = ingestUrl;
|
|
4677
5973
|
sinkSession.portalReportingRunId = runId;
|
|
4678
5974
|
sessionInfo.runToken = runToken;
|
|
5975
|
+
Object.defineProperty(sessionInfo, PORTAL_RUN_TOKEN_PROVIDER, {
|
|
5976
|
+
configurable: true,
|
|
5977
|
+
enumerable: false,
|
|
5978
|
+
value: () => currentLicenseSessionRunToken(licenseSession)
|
|
5979
|
+
});
|
|
4679
5980
|
sessionInfo.portalReportingIngestUrl = ingestUrl;
|
|
4680
5981
|
sessionInfo.portalReportingRunId = runId;
|
|
4681
5982
|
}
|
|
5983
|
+
function currentLicenseSessionRunToken(licenseSession) {
|
|
5984
|
+
return stringValueOrDefault(licenseSession?.runToken, "").trim();
|
|
5985
|
+
}
|
|
4682
5986
|
function buildPortalReportingRunId(sessionId) {
|
|
4683
5987
|
const sessionPart = String(sessionId ?? "")
|
|
4684
5988
|
.replace(/[^A-Za-z0-9._-]+/g, "-")
|
|
@@ -4763,6 +6067,7 @@ function attachDataTransferStatsAliases(stats) {
|
|
|
4763
6067
|
const projected = normalizeDataTransferStatsValue(stats);
|
|
4764
6068
|
return attachAliasMap(projected, {
|
|
4765
6069
|
AllBytes: "allBytes",
|
|
6070
|
+
AllBytes64: "allBytes64",
|
|
4766
6071
|
MaxBytes: "maxBytes",
|
|
4767
6072
|
MeanBytes: "meanBytes",
|
|
4768
6073
|
MinBytes: "minBytes",
|
|
@@ -4770,6 +6075,7 @@ function attachDataTransferStatsAliases(stats) {
|
|
|
4770
6075
|
Percent75: "percent75",
|
|
4771
6076
|
Percent95: "percent95",
|
|
4772
6077
|
Percent99: "percent99",
|
|
6078
|
+
Percent100: "percent100",
|
|
4773
6079
|
StdDev: "stdDev"
|
|
4774
6080
|
});
|
|
4775
6081
|
}
|
|
@@ -4793,6 +6099,7 @@ function attachLatencyStatsAliases(stats) {
|
|
|
4793
6099
|
Percent75: "percent75",
|
|
4794
6100
|
Percent95: "percent95",
|
|
4795
6101
|
Percent99: "percent99",
|
|
6102
|
+
Percent100: "percent100",
|
|
4796
6103
|
StdDev: "stdDev"
|
|
4797
6104
|
});
|
|
4798
6105
|
return projected;
|
|
@@ -4814,6 +6121,9 @@ function attachMeasurementStatsAliases(stats) {
|
|
|
4814
6121
|
projected.latency = attachLatencyStatsAliases(projected.latency);
|
|
4815
6122
|
projected.statusCodes = projected.statusCodes.map((value) => attachStatusCodeStatsAliases(value));
|
|
4816
6123
|
attachAliasMap(projected, {
|
|
6124
|
+
Count64: "count64",
|
|
6125
|
+
DistributionMode: "distributionMode",
|
|
6126
|
+
MaxRelativeError: "maxRelativeError",
|
|
4817
6127
|
Request: "request",
|
|
4818
6128
|
DataTransfer: "dataTransfer",
|
|
4819
6129
|
Latency: "latency",
|
|
@@ -4875,6 +6185,8 @@ function attachStepStatsAliases(step) {
|
|
|
4875
6185
|
const projected = normalizeStepStatsValue(step);
|
|
4876
6186
|
projected.ok = attachMeasurementStatsAliases(projected.ok);
|
|
4877
6187
|
projected.fail = attachMeasurementStatsAliases(projected.fail);
|
|
6188
|
+
if (projected.allMeasurement)
|
|
6189
|
+
projected.allMeasurement = attachMeasurementStatsAliases(projected.allMeasurement);
|
|
4878
6190
|
attachAliasMap(projected, {
|
|
4879
6191
|
ScenarioName: "scenarioName",
|
|
4880
6192
|
StepName: "stepName",
|
|
@@ -4889,6 +6201,7 @@ function attachStepStatsAliases(step) {
|
|
|
4889
6201
|
StatusCodes: "statusCodes",
|
|
4890
6202
|
Ok: "ok",
|
|
4891
6203
|
Fail: "fail",
|
|
6204
|
+
AllMeasurement: "allMeasurement",
|
|
4892
6205
|
SortIndex: "sortIndex"
|
|
4893
6206
|
});
|
|
4894
6207
|
return projected;
|
|
@@ -4927,7 +6240,20 @@ function attachLoadSimulationProjection(simulation) {
|
|
|
4927
6240
|
});
|
|
4928
6241
|
return simulation;
|
|
4929
6242
|
}
|
|
4930
|
-
function
|
|
6243
|
+
function cloneLoadEngineV2TrafficMixMetadata(metadata) {
|
|
6244
|
+
return {
|
|
6245
|
+
...metadata,
|
|
6246
|
+
shareWeights: [...metadata.shareWeights],
|
|
6247
|
+
originalSimulations: metadata.originalSimulations.map((simulation) => attachLoadSimulationProjection({ ...simulation }))
|
|
6248
|
+
};
|
|
6249
|
+
}
|
|
6250
|
+
function nextTrafficMixDeclarationIndex(scenarios) {
|
|
6251
|
+
return scenarios.reduce((next, scenario) => {
|
|
6252
|
+
const metadata = scenario.__loadStrikeTrafficMixV2Metadata();
|
|
6253
|
+
return metadata ? Math.max(next, metadata.declarationIndex + 1) : next;
|
|
6254
|
+
}, 0);
|
|
6255
|
+
}
|
|
6256
|
+
function expandTrafficMixScenarios(trafficMix, declarationIndex = 0) {
|
|
4931
6257
|
if (!(trafficMix instanceof LoadStrikeTrafficMix)) {
|
|
4932
6258
|
throw new TypeError("Traffic mix must be provided.");
|
|
4933
6259
|
}
|
|
@@ -4940,16 +6266,30 @@ function expandTrafficMixScenarios(trafficMix) {
|
|
|
4940
6266
|
throw new Error("Traffic mix scenario shares must be configured before registration.");
|
|
4941
6267
|
}
|
|
4942
6268
|
const weights = scenarioMix.map((share) => share.weight);
|
|
6269
|
+
const seedId = buildLoadEngineV2TrafficMixSeedId(declarationIndex, trafficMix.name);
|
|
4943
6270
|
return scenarioMix.map((share, index) => {
|
|
4944
|
-
const splitSimulations = totalLoad
|
|
4945
|
-
|
|
4946
|
-
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
6271
|
+
const splitSimulations = totalLoad.map((simulation) => splitTrafficSimulation(simulation, weights, index)
|
|
6272
|
+
?? trafficMixNoWorkSimulation(simulation));
|
|
6273
|
+
return share.scenario
|
|
6274
|
+
.withLoadSimulations(...splitSimulations)
|
|
6275
|
+
.__loadStrikeWithInternalLicenseFeatures(TRAFFIC_MIX_FEATURE)
|
|
6276
|
+
.__loadStrikeSetTrafficMixV2Metadata({
|
|
6277
|
+
declarationIndex,
|
|
6278
|
+
name: trafficMix.name,
|
|
6279
|
+
laneIndex: index,
|
|
6280
|
+
shareWeight: share.weight,
|
|
6281
|
+
shareWeights: weights,
|
|
6282
|
+
seedId,
|
|
6283
|
+
originalSimulations: totalLoad
|
|
6284
|
+
});
|
|
4951
6285
|
});
|
|
4952
6286
|
}
|
|
6287
|
+
function trafficMixNoWorkSimulation(simulation) {
|
|
6288
|
+
const kind = String(simulation.Kind ?? "");
|
|
6289
|
+
return kind === "IterationsForInject" || kind === "IterationsForConstant"
|
|
6290
|
+
? LoadStrikeSimulation.pause(0)
|
|
6291
|
+
: LoadStrikeSimulation.pause(Math.max(readFiniteSimulationNumber(simulation, "DuringSeconds"), 0));
|
|
6292
|
+
}
|
|
4953
6293
|
function splitTrafficSimulation(simulation, weights, index) {
|
|
4954
6294
|
const kind = String(simulation.Kind ?? "");
|
|
4955
6295
|
const duringSeconds = readFiniteSimulationNumber(simulation, "DuringSeconds");
|
|
@@ -5036,6 +6376,8 @@ function attachScenarioStatsAliases(scenario) {
|
|
|
5036
6376
|
const normalized = normalizeScenarioStatsValue(scenario);
|
|
5037
6377
|
normalized.ok = attachMeasurementStatsAliases(normalized.ok);
|
|
5038
6378
|
normalized.fail = attachMeasurementStatsAliases(normalized.fail);
|
|
6379
|
+
if (normalized.allMeasurement)
|
|
6380
|
+
normalized.allMeasurement = attachMeasurementStatsAliases(normalized.allMeasurement);
|
|
5039
6381
|
normalized.loadSimulationStats = attachLoadSimulationStatsAliases(normalized.loadSimulationStats);
|
|
5040
6382
|
normalized.stepStats = normalized.stepStats.map((value) => attachStepStatsAliases(value));
|
|
5041
6383
|
const findStepStats = scenario.findStepStats ?? ((stepName) => normalized.stepStats.find((value) => value.stepName === stepName));
|
|
@@ -5069,6 +6411,7 @@ function attachScenarioStatsAliases(scenario) {
|
|
|
5069
6411
|
DurationMs: "durationMs",
|
|
5070
6412
|
Ok: "ok",
|
|
5071
6413
|
Fail: "fail",
|
|
6414
|
+
AllMeasurement: "allMeasurement",
|
|
5072
6415
|
LoadSimulationStats: "loadSimulationStats",
|
|
5073
6416
|
SortIndex: "sortIndex",
|
|
5074
6417
|
StepStats: "stepStats"
|
|
@@ -5089,6 +6432,10 @@ function attachNodeStatsAliases(stats) {
|
|
|
5089
6432
|
: stats.scenarioStats.flatMap((value) => value.stepStats)).map((value) => attachStepStatsAliases(value));
|
|
5090
6433
|
stats.pluginsData = stats.pluginsData.map((value) => normalizePluginData(value.pluginName ?? value.PluginName ?? "", value));
|
|
5091
6434
|
stats.sinkErrors = stats.sinkErrors.map((value) => attachSinkErrorAliases(value));
|
|
6435
|
+
stats.generatorWarnings = (stats.generatorWarnings ?? []).map(attachGeneratorWarningAliases);
|
|
6436
|
+
stats.schedulerSegments = (stats.schedulerSegments ?? []).map((value) => normalizeSchedulerSegment(value));
|
|
6437
|
+
stats.observationDeliveryStats = normalizeObservationDeliveryStats(stats.observationDeliveryStats ?? emptyObservationDeliveryStats());
|
|
6438
|
+
stats.reportingComplete ?? (stats.reportingComplete = false);
|
|
5092
6439
|
const findScenarioStats = stats.findScenarioStats ?? ((scenarioName) => stats.scenarioStats.find((value) => value.scenarioName === scenarioName));
|
|
5093
6440
|
const getScenarioStats = stats.getScenarioStats ?? ((scenarioName) => {
|
|
5094
6441
|
const value = findScenarioStats(scenarioName);
|
|
@@ -5123,7 +6470,11 @@ function attachNodeStatsAliases(stats) {
|
|
|
5123
6470
|
DisabledSinks: "disabledSinks",
|
|
5124
6471
|
SinkErrors: "sinkErrors",
|
|
5125
6472
|
ReportFiles: "reportFiles",
|
|
5126
|
-
LogFiles: "logFiles"
|
|
6473
|
+
LogFiles: "logFiles",
|
|
6474
|
+
GeneratorWarnings: "generatorWarnings",
|
|
6475
|
+
SchedulerSegments: "schedulerSegments",
|
|
6476
|
+
ObservationDeliveryStats: "observationDeliveryStats",
|
|
6477
|
+
ReportingComplete: "reportingComplete"
|
|
5127
6478
|
});
|
|
5128
6479
|
defineAliasProperty(projected, "StartedUtc", () => parseAliasDate(stats.startedUtc));
|
|
5129
6480
|
defineAliasProperty(projected, "CompletedUtc", () => parseAliasDate(stats.completedUtc));
|
|
@@ -5179,11 +6530,43 @@ function attachRunResultAliases(result) {
|
|
|
5179
6530
|
.map((value) => ({ ...asAliasRecord(value) })),
|
|
5180
6531
|
failedCorrelationRows: pickAliasArray(source, "failedCorrelationRows", "FailedCorrelationRows")
|
|
5181
6532
|
.map((value) => ({ ...asAliasRecord(value) })),
|
|
6533
|
+
...(hasAliasValue(source, "generatorWarnings", "GeneratorWarnings")
|
|
6534
|
+
? {
|
|
6535
|
+
generatorWarnings: pickAliasArray(source, "generatorWarnings", "GeneratorWarnings")
|
|
6536
|
+
.map(attachGeneratorWarningAliases)
|
|
6537
|
+
}
|
|
6538
|
+
: {}),
|
|
6539
|
+
...(hasAliasValue(source, "schedulerSegments", "SchedulerSegments")
|
|
6540
|
+
? {
|
|
6541
|
+
schedulerSegments: pickAliasArray(source, "schedulerSegments", "SchedulerSegments")
|
|
6542
|
+
.map(normalizeSchedulerSegment)
|
|
6543
|
+
}
|
|
6544
|
+
: {}),
|
|
6545
|
+
...(hasAliasValue(source, "schedulerStats", "SchedulerStats")
|
|
6546
|
+
? { schedulerStats: normalizeSchedulerStats(pickAliasValue(source, "schedulerStats", "SchedulerStats")) }
|
|
6547
|
+
: {}),
|
|
6548
|
+
...(hasAliasValue(source, "observationDeliveryStats", "ObservationDeliveryStats")
|
|
6549
|
+
? {
|
|
6550
|
+
observationDeliveryStats: normalizeObservationDeliveryStats(pickAliasValue(source, "observationDeliveryStats", "ObservationDeliveryStats"))
|
|
6551
|
+
}
|
|
6552
|
+
: {}),
|
|
6553
|
+
...(hasAliasValue(source, "reportingComplete", "ReportingComplete")
|
|
6554
|
+
? { reportingComplete: pickAliasBoolean(source, "reportingComplete", "ReportingComplete") }
|
|
6555
|
+
: {}),
|
|
5182
6556
|
findScenarioStats,
|
|
5183
6557
|
getScenarioStats,
|
|
5184
6558
|
FindScenarioStats: findScenarioStats,
|
|
5185
6559
|
GetScenarioStats: getScenarioStats
|
|
5186
6560
|
};
|
|
6561
|
+
const schedulerDistributions = result[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS];
|
|
6562
|
+
if (schedulerDistributions) {
|
|
6563
|
+
Object.defineProperty(projected, LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS, {
|
|
6564
|
+
value: schedulerDistributions.map(cloneLoadEngineV2DistributionRecord),
|
|
6565
|
+
enumerable: false,
|
|
6566
|
+
configurable: false,
|
|
6567
|
+
writable: false
|
|
6568
|
+
});
|
|
6569
|
+
}
|
|
5187
6570
|
attachAliasMap(projected, {
|
|
5188
6571
|
AllBytes: "allBytes",
|
|
5189
6572
|
AllRequestCount: "allRequestCount",
|
|
@@ -5207,7 +6590,12 @@ function attachRunResultAliases(result) {
|
|
|
5207
6590
|
ReportFiles: "reportFiles",
|
|
5208
6591
|
LogFiles: "logFiles",
|
|
5209
6592
|
CorrelationRows: "correlationRows",
|
|
5210
|
-
FailedCorrelationRows: "failedCorrelationRows"
|
|
6593
|
+
FailedCorrelationRows: "failedCorrelationRows",
|
|
6594
|
+
GeneratorWarnings: "generatorWarnings",
|
|
6595
|
+
SchedulerSegments: "schedulerSegments",
|
|
6596
|
+
SchedulerStats: "schedulerStats",
|
|
6597
|
+
ObservationDeliveryStats: "observationDeliveryStats",
|
|
6598
|
+
ReportingComplete: "reportingComplete"
|
|
5211
6599
|
});
|
|
5212
6600
|
defineAliasProperty(projected, "StartedUtc", () => parseAliasDate(projected.startedUtc));
|
|
5213
6601
|
defineAliasProperty(projected, "CompletedUtc", () => parseAliasDate(projected.completedUtc));
|
|
@@ -5815,6 +7203,14 @@ function detailedToNodeStats(result, metricStats) {
|
|
|
5815
7203
|
sinkErrors: (result.sinkErrors ?? []).map((sinkError) => ({ ...sinkError })),
|
|
5816
7204
|
reportFiles: [...(result.reportFiles ?? [])],
|
|
5817
7205
|
logFiles: [...(result.logFiles ?? [])],
|
|
7206
|
+
generatorWarnings: (result.generatorWarnings ?? []).map(attachGeneratorWarningAliases),
|
|
7207
|
+
schedulerSegments: result.schedulerSegments?.map((segment) => ({ ...segment })),
|
|
7208
|
+
schedulerStats: result.schedulerStats
|
|
7209
|
+
? normalizeSchedulerStats(result.schedulerStats)
|
|
7210
|
+
: undefined,
|
|
7211
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: result[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]?.map(cloneLoadEngineV2DistributionRecord),
|
|
7212
|
+
observationDeliveryStats: normalizeObservationDeliveryStats(result.observationDeliveryStats ?? emptyObservationDeliveryStats()),
|
|
7213
|
+
reportingComplete: result.reportingComplete ?? true,
|
|
5818
7214
|
findScenarioStats: (scenarioName) => scenarioStats.find((scenario) => scenario.scenarioName === scenarioName),
|
|
5819
7215
|
getScenarioStats: (scenarioName) => {
|
|
5820
7216
|
const value = scenarioStats.find((scenario) => scenario.scenarioName === scenarioName);
|
|
@@ -5882,22 +7278,35 @@ function buildEmptyNodeStats(args) {
|
|
|
5882
7278
|
sinkErrors: [],
|
|
5883
7279
|
reportFiles: [],
|
|
5884
7280
|
logFiles: [],
|
|
7281
|
+
generatorWarnings: [],
|
|
7282
|
+
observationDeliveryStats: emptyObservationDeliveryStats(),
|
|
7283
|
+
reportingComplete: true,
|
|
5885
7284
|
findScenarioStats: (scenarioName) => undefined,
|
|
5886
7285
|
getScenarioStats: (scenarioName) => {
|
|
5887
7286
|
throw new Error(`Scenario stats not found: ${scenarioName}`);
|
|
5888
7287
|
}
|
|
5889
7288
|
});
|
|
5890
7289
|
}
|
|
5891
|
-
function planRuntimeClusterAssignments(scenarios, agentsCount, targetScenarios, agentTargetScenarios) {
|
|
7290
|
+
function planRuntimeClusterAssignments(scenarios, agentsCount, targetScenarios, agentTargetScenarios, globalV2 = false, coordinatorTargetScenarios = []) {
|
|
5892
7291
|
const resolvedAgentCount = Math.max(agentsCount, 1);
|
|
5893
|
-
|
|
7292
|
+
let selectedScenarios = targetScenarios.length
|
|
5894
7293
|
? scenarios.filter((scenario) => targetScenarios.includes(scenario.name))
|
|
5895
7294
|
: [...scenarios];
|
|
7295
|
+
if (globalV2 && coordinatorTargetScenarios.length) {
|
|
7296
|
+
const coordinatorNames = new Set(coordinatorTargetScenarios);
|
|
7297
|
+
selectedScenarios = selectedScenarios.filter((scenario) => !coordinatorNames.has(scenario.name));
|
|
7298
|
+
}
|
|
5896
7299
|
if (!selectedScenarios.length) {
|
|
5897
7300
|
return Array.from({ length: resolvedAgentCount }, () => []);
|
|
5898
7301
|
}
|
|
5899
7302
|
if (agentTargetScenarios.length) {
|
|
5900
|
-
|
|
7303
|
+
const coordinatorNames = new Set(coordinatorTargetScenarios);
|
|
7304
|
+
const names = agentTargetScenarios.filter((name) => !coordinatorNames.has(name));
|
|
7305
|
+
return Array.from({ length: resolvedAgentCount }, () => [...names]);
|
|
7306
|
+
}
|
|
7307
|
+
if (globalV2) {
|
|
7308
|
+
const scenarioNames = selectedScenarios.map((scenario) => scenario.name);
|
|
7309
|
+
return Array.from({ length: resolvedAgentCount }, () => [...scenarioNames]);
|
|
5901
7310
|
}
|
|
5902
7311
|
const weightedNames = [];
|
|
5903
7312
|
for (const scenario of selectedScenarios) {
|
|
@@ -5917,7 +7326,320 @@ function planRuntimeClusterAssignments(scenarios, agentsCount, targetScenarios,
|
|
|
5917
7326
|
}
|
|
5918
7327
|
return assignments.map((entry) => [...entry]);
|
|
5919
7328
|
}
|
|
5920
|
-
function
|
|
7329
|
+
function buildRuntimeLoadEngineV2Plan(scenarios, options, testInfo, expectedAgentIds) {
|
|
7330
|
+
validateLoadEngineV2ScenarioFeatures(scenarios);
|
|
7331
|
+
const kindToken = (value) => {
|
|
7332
|
+
const normalized = String(value ?? "").replace(/[^a-z0-9]/gi, "").toLowerCase();
|
|
7333
|
+
const tokens = {
|
|
7334
|
+
inject: "inject", injectrandom: "inject-random", rampinginject: "ramping-inject",
|
|
7335
|
+
keepconstant: "keep-constant", rampingconstant: "ramping-constant",
|
|
7336
|
+
iterationsforinject: "iterations-for-inject", iterationsforconstant: "iterations-for-constant",
|
|
7337
|
+
pause: "pause"
|
|
7338
|
+
};
|
|
7339
|
+
const token = tokens[normalized];
|
|
7340
|
+
if (!token)
|
|
7341
|
+
throw new Error(`Load Engine V2 simulation kind is unsupported: ${String(value ?? "")}`);
|
|
7342
|
+
return token;
|
|
7343
|
+
};
|
|
7344
|
+
const integer = (value) => Math.max(Math.trunc(toNumber(value)), 0).toString();
|
|
7345
|
+
const ns = (value) => Math.max(Math.round(toNumber(value) * 1000000000), 0).toString();
|
|
7346
|
+
const coordinatorNames = new Set(options.coordinatorTargetScenarios ?? []);
|
|
7347
|
+
const explicitAgentNames = new Set(options.agentTargetScenarios ?? []);
|
|
7348
|
+
const selectedNames = new Set(options.targetScenarios ?? []);
|
|
7349
|
+
const plannedScenarios = scenarios
|
|
7350
|
+
.map((scenario, declarationIndex) => ({ scenario, declarationIndex }))
|
|
7351
|
+
.filter(({ scenario }) => (selectedNames.size === 0 || selectedNames.has(scenario.name))
|
|
7352
|
+
&& (explicitAgentNames.size > 0
|
|
7353
|
+
? explicitAgentNames.has(scenario.name)
|
|
7354
|
+
: !coordinatorNames.has(scenario.name)));
|
|
7355
|
+
const implicitSingleInvocation = {
|
|
7356
|
+
simulationIndex64: "0",
|
|
7357
|
+
kind: "iterations-for-constant",
|
|
7358
|
+
rate64: "0",
|
|
7359
|
+
minRate64: "0",
|
|
7360
|
+
maxRate64: "0",
|
|
7361
|
+
copies64: "1",
|
|
7362
|
+
iterations64: "1",
|
|
7363
|
+
intervalNs64: "0",
|
|
7364
|
+
durationNs64: "0"
|
|
7365
|
+
};
|
|
7366
|
+
return {
|
|
7367
|
+
runId: testInfo.sessionId,
|
|
7368
|
+
sessionId: testInfo.sessionId,
|
|
7369
|
+
registrationNonce: randomBytes(16).toString("hex"),
|
|
7370
|
+
maxInFlight64: Math.max(options.maxInFlight ?? 10000, 1).toString(),
|
|
7371
|
+
reportingIntervalNs64: ns(options.reportingIntervalSeconds ?? 1),
|
|
7372
|
+
schedulerVisibleProcessorCount64: Math.max(os.cpus().length, 1).toString(),
|
|
7373
|
+
expectedAgentIds: [...expectedAgentIds],
|
|
7374
|
+
scenarios: plannedScenarios.map(({ scenario, declarationIndex }) => ({
|
|
7375
|
+
scenarioIndex64: declarationIndex.toString(),
|
|
7376
|
+
scenarioName: scenario.name,
|
|
7377
|
+
target: "agent",
|
|
7378
|
+
callbackExecutionMode: "async",
|
|
7379
|
+
declaredStepNames: scenario.getDeclaredSteps(),
|
|
7380
|
+
simulations: scenario.getSimulations().length
|
|
7381
|
+
? scenario.getSimulations().map((simulation, simulationIndex) => ({
|
|
7382
|
+
simulationIndex64: simulationIndex.toString(),
|
|
7383
|
+
kind: kindToken(simulation.Kind ?? simulation.kind),
|
|
7384
|
+
rate64: integer(simulation.Rate ?? simulation.rate),
|
|
7385
|
+
minRate64: integer(simulation.MinRate ?? simulation.minRate),
|
|
7386
|
+
maxRate64: integer(simulation.MaxRate ?? simulation.maxRate),
|
|
7387
|
+
copies64: integer(simulation.Copies ?? simulation.copies),
|
|
7388
|
+
iterations64: integer(simulation.Iterations ?? simulation.iterations),
|
|
7389
|
+
intervalNs64: ns(simulation.IntervalSeconds ?? simulation.intervalSeconds),
|
|
7390
|
+
durationNs64: ns(simulation.DuringSeconds ?? simulation.duringSeconds)
|
|
7391
|
+
}))
|
|
7392
|
+
: [{ ...implicitSingleInvocation }]
|
|
7393
|
+
}))
|
|
7394
|
+
};
|
|
7395
|
+
}
|
|
7396
|
+
function normalizeRequiredV2AgentIds(values, expectedCount) {
|
|
7397
|
+
const ids = normalizeOptionalStringArray(values) ?? [];
|
|
7398
|
+
if (ids.length !== expectedCount || new Set(ids).size !== ids.length) {
|
|
7399
|
+
throw new Error("Remote Load Engine V2 coordinators require an exact unique ExpectedAgentIds set matching AgentsCount.");
|
|
7400
|
+
}
|
|
7401
|
+
return ids;
|
|
7402
|
+
}
|
|
7403
|
+
function validateLoadEngineV2ScenarioFeatures(scenarios) {
|
|
7404
|
+
for (const scenario of scenarios) {
|
|
7405
|
+
if (scenario.getTrackingConfiguration()) {
|
|
7406
|
+
throw new Error(`Load Engine V2 correlation is not available in the supported non-correlation profile. Scenario=${scenario.name}.`);
|
|
7407
|
+
}
|
|
7408
|
+
}
|
|
7409
|
+
}
|
|
7410
|
+
function canonicalLoadEngineV2InvocationIdentityKind(simulationKind) {
|
|
7411
|
+
const normalized = simulationKind.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
|
7412
|
+
if (["inject", "injectrandom", "rampinginject", "iterationsforinject"].includes(normalized)) {
|
|
7413
|
+
return "arrival";
|
|
7414
|
+
}
|
|
7415
|
+
if (["keepconstant", "rampingconstant"].includes(normalized)) {
|
|
7416
|
+
return "worker-iteration";
|
|
7417
|
+
}
|
|
7418
|
+
if (normalized === "iterationsforconstant") {
|
|
7419
|
+
return "constant-iteration";
|
|
7420
|
+
}
|
|
7421
|
+
return "";
|
|
7422
|
+
}
|
|
7423
|
+
function buildRuntimeLoadEngineV2HistogramArtifact(result, scenarioDeclarations) {
|
|
7424
|
+
const emptyHistogram = () => new LoadStrikeHistogramV1().toSidecar();
|
|
7425
|
+
const distributions = [];
|
|
7426
|
+
const measurementSummaries = [];
|
|
7427
|
+
const statusBody = (measurement, outcome) => {
|
|
7428
|
+
const named = measurement.statusCodes
|
|
7429
|
+
.filter((row) => Boolean(row.statusCode || row.message))
|
|
7430
|
+
.map((row) => {
|
|
7431
|
+
const key = buildLoadEngineV2StatusIdentityKey(row.statusCode, row.message);
|
|
7432
|
+
if (!key)
|
|
7433
|
+
throw new Error("Load Engine V2 status identity unexpectedly resolved empty.");
|
|
7434
|
+
return {
|
|
7435
|
+
statusIdentityKeyHex: key.identity.toString("hex"),
|
|
7436
|
+
display: key.display,
|
|
7437
|
+
count64: Math.max(Math.trunc(row.count), 0).toString(),
|
|
7438
|
+
aggregatedObservationCount64: "0",
|
|
7439
|
+
hasAggregatedIdentities: false
|
|
7440
|
+
};
|
|
7441
|
+
})
|
|
7442
|
+
.sort((left, right) => Buffer.compare(Buffer.from(left.statusIdentityKeyHex, "hex"), Buffer.from(right.statusIdentityKeyHex, "hex")));
|
|
7443
|
+
let statuses = named;
|
|
7444
|
+
if (named.length > 64) {
|
|
7445
|
+
const retained = named.slice(0, 63);
|
|
7446
|
+
const aggregated = named.slice(63).reduce((sum, row) => sum + BigInt(row.count64), 0n);
|
|
7447
|
+
retained.push({
|
|
7448
|
+
statusIdentityKeyHex: Buffer.concat([Buffer.from("LS-ID1\n", "ascii"), Buffer.from([0x0d])]).toString("hex"),
|
|
7449
|
+
display: "<other>", count64: aggregated.toString(),
|
|
7450
|
+
aggregatedObservationCount64: aggregated.toString(), hasAggregatedIdentities: aggregated > 0n
|
|
7451
|
+
});
|
|
7452
|
+
statuses = retained;
|
|
7453
|
+
}
|
|
7454
|
+
return {
|
|
7455
|
+
outcome,
|
|
7456
|
+
statusObservationCount64: statuses.reduce((sum, row) => sum + BigInt(row.count64), 0n).toString(),
|
|
7457
|
+
statuses
|
|
7458
|
+
};
|
|
7459
|
+
};
|
|
7460
|
+
const appendMeasurement = (seriesKind, scenarioIndex64, scenarioName, identity, display, ok, fail, all, reservedOther) => {
|
|
7461
|
+
const emptyMeasurement = () => ({
|
|
7462
|
+
count64: "0",
|
|
7463
|
+
histogramSidecar: {
|
|
7464
|
+
latency: emptyHistogram(), size: emptyHistogram(), allBytes64: "0",
|
|
7465
|
+
lessOrEq80064: "0", more800Less120064: "0", moreOrEq120064: "0"
|
|
7466
|
+
},
|
|
7467
|
+
request: { count: 0, percent: 0, rps: 0 },
|
|
7468
|
+
dataTransfer: { allBytes: 0, minBytes: 0, maxBytes: 0, meanBytes: 0, percent50: 0,
|
|
7469
|
+
percent75: 0, percent95: 0, percent99: 0, percent100: 0, stdDev: 0 },
|
|
7470
|
+
latency: { latencyCount: { lessOrEq800: 0, more800Less1200: 0, moreOrEq1200: 0 },
|
|
7471
|
+
minMs: 0, maxMs: 0, meanMs: 0, percent50: 0, percent75: 0, percent95: 0,
|
|
7472
|
+
percent99: 0, stdDev: 0 },
|
|
7473
|
+
statusCodes: []
|
|
7474
|
+
});
|
|
7475
|
+
const okValue = ok ?? emptyMeasurement();
|
|
7476
|
+
const failValue = fail ?? emptyMeasurement();
|
|
7477
|
+
const allValue = all ?? emptyMeasurement();
|
|
7478
|
+
for (const [outcome, measurement] of [
|
|
7479
|
+
["ok", okValue], ["fail", failValue], ["all", allValue]
|
|
7480
|
+
]) {
|
|
7481
|
+
const sidecar = measurement.histogramSidecar;
|
|
7482
|
+
if (!sidecar)
|
|
7483
|
+
throw new Error("Load Engine V2 assigned measurement is missing histogram state.");
|
|
7484
|
+
distributions.push({
|
|
7485
|
+
seriesKind, scenarioIndex64, scenarioName, identityKeyHex: identity.toString("hex"),
|
|
7486
|
+
outcome, unit: "microseconds", histogram: sidecar.latency,
|
|
7487
|
+
exactTotalDecimalOrEmpty: sidecar.latency.exactTotal64,
|
|
7488
|
+
bands: [
|
|
7489
|
+
{ bandId64: "0", count64: sidecar.lessOrEq80064 },
|
|
7490
|
+
{ bandId64: "1", count64: sidecar.more800Less120064 },
|
|
7491
|
+
{ bandId64: "2", count64: sidecar.moreOrEq120064 }
|
|
7492
|
+
]
|
|
7493
|
+
});
|
|
7494
|
+
distributions.push({
|
|
7495
|
+
seriesKind, scenarioIndex64, scenarioName, identityKeyHex: identity.toString("hex"),
|
|
7496
|
+
outcome, unit: "bytes", histogram: sidecar.size,
|
|
7497
|
+
exactTotalDecimalOrEmpty: sidecar.size.exactTotal64
|
|
7498
|
+
});
|
|
7499
|
+
}
|
|
7500
|
+
const observation = BigInt(allValue.count64 ?? allValue.histogramSidecar?.latency.count64 ?? "0");
|
|
7501
|
+
const success = BigInt(okValue.count64 ?? okValue.histogramSidecar?.latency.count64 ?? "0");
|
|
7502
|
+
const failure = BigInt(failValue.count64 ?? failValue.histogramSidecar?.latency.count64 ?? "0");
|
|
7503
|
+
measurementSummaries.push({
|
|
7504
|
+
seriesKind, scenarioIndex64, scenarioName, identityKeyHex: identity.toString("hex"), display,
|
|
7505
|
+
observationCount64: observation.toString(), successCount64: success.toString(),
|
|
7506
|
+
failureCount64: failure.toString(),
|
|
7507
|
+
aggregatedObservationCount64: reservedOther ? observation.toString() : "0",
|
|
7508
|
+
hasAggregatedIdentities: reservedOther && observation > 0n,
|
|
7509
|
+
outcomes: [statusBody(okValue, "ok"), statusBody(failValue, "fail")]
|
|
7510
|
+
});
|
|
7511
|
+
};
|
|
7512
|
+
for (const scenario of result.scenarioStats) {
|
|
7513
|
+
const scenarioIndex64 = Math.max(Math.trunc(scenario.sortIndex), 0).toString();
|
|
7514
|
+
appendMeasurement("scenario", scenarioIndex64, scenario.scenarioName, buildLoadEngineV2ScenarioIdentityKey(scenarioIndex64), scenario.scenarioName, scenario.ok, scenario.fail, scenario.allMeasurement, false);
|
|
7515
|
+
const declaration = scenarioDeclarations?.find((value) => value.name === scenario.scenarioName);
|
|
7516
|
+
const declaredNames = declaration?.getDeclaredSteps()
|
|
7517
|
+
?? (scenarioDeclarations ? [] : undefined);
|
|
7518
|
+
if (declaredNames === undefined) {
|
|
7519
|
+
for (const step of scenario.stepStats) {
|
|
7520
|
+
const key = buildLoadEngineV2StepIdentityKey(scenarioIndex64, step.stepName);
|
|
7521
|
+
appendMeasurement("step", scenarioIndex64, scenario.scenarioName, key.identity, key.display, step.ok, step.fail, step.allMeasurement, false);
|
|
7522
|
+
}
|
|
7523
|
+
}
|
|
7524
|
+
else {
|
|
7525
|
+
const requiredStepAllMeasurement = (step) => {
|
|
7526
|
+
if (!step.allMeasurement) {
|
|
7527
|
+
throw new Error("Load Engine V2 assigned step is missing its all-outcome histogram state.");
|
|
7528
|
+
}
|
|
7529
|
+
return step.allMeasurement;
|
|
7530
|
+
};
|
|
7531
|
+
const groups = new Map();
|
|
7532
|
+
for (const declaredName of declaredNames) {
|
|
7533
|
+
const key = buildLoadEngineV2StepIdentityKey(scenarioIndex64, declaredName);
|
|
7534
|
+
groups.set(key.identity.toString("hex"), {
|
|
7535
|
+
identity: key.identity, display: key.display, routedToOther: false, steps: []
|
|
7536
|
+
});
|
|
7537
|
+
}
|
|
7538
|
+
for (const step of scenario.stepStats) {
|
|
7539
|
+
const resolved = resolveLoadEngineV2StepIdentityKey(scenarioIndex64, step.stepName, declaredNames);
|
|
7540
|
+
const key = resolved.identity.toString("hex");
|
|
7541
|
+
const group = groups.get(key) ?? {
|
|
7542
|
+
identity: resolved.identity,
|
|
7543
|
+
display: resolved.display,
|
|
7544
|
+
routedToOther: resolved.routedToOther,
|
|
7545
|
+
steps: []
|
|
7546
|
+
};
|
|
7547
|
+
group.steps.push(step);
|
|
7548
|
+
groups.set(key, group);
|
|
7549
|
+
}
|
|
7550
|
+
const reservedHex = buildLoadEngineV2ReservedStepOtherIdentityKey(scenarioIndex64).toString("hex");
|
|
7551
|
+
const declaredGroups = [...groups.entries()]
|
|
7552
|
+
.filter(([key]) => key !== reservedHex)
|
|
7553
|
+
.map(([, value]) => value)
|
|
7554
|
+
.sort((left, right) => Buffer.compare(left.identity, right.identity));
|
|
7555
|
+
for (const group of declaredGroups) {
|
|
7556
|
+
const ok = group.steps.length
|
|
7557
|
+
? aggregateMeasurementStats(group.steps.map((step) => step.ok), scenario.allRequestCount, scenario.durationMs, true)
|
|
7558
|
+
: undefined;
|
|
7559
|
+
const fail = group.steps.length
|
|
7560
|
+
? aggregateMeasurementStats(group.steps.map((step) => step.fail), scenario.allRequestCount, scenario.durationMs, true)
|
|
7561
|
+
: undefined;
|
|
7562
|
+
const all = group.steps.length
|
|
7563
|
+
? aggregateMeasurementStats(group.steps.map(requiredStepAllMeasurement), scenario.allRequestCount, scenario.durationMs, true)
|
|
7564
|
+
: undefined;
|
|
7565
|
+
appendMeasurement("step", scenarioIndex64, scenario.scenarioName, group.identity, group.display, ok, fail, all, false);
|
|
7566
|
+
}
|
|
7567
|
+
const other = groups.get(reservedHex);
|
|
7568
|
+
if (other?.steps.length) {
|
|
7569
|
+
appendMeasurement("step", scenarioIndex64, scenario.scenarioName, other.identity, "<other>", aggregateMeasurementStats(other.steps.map((step) => step.ok), scenario.allRequestCount, scenario.durationMs, true), aggregateMeasurementStats(other.steps.map((step) => step.fail), scenario.allRequestCount, scenario.durationMs, true), aggregateMeasurementStats(other.steps.map(requiredStepAllMeasurement), scenario.allRequestCount, scenario.durationMs, true), true);
|
|
7570
|
+
continue;
|
|
7571
|
+
}
|
|
7572
|
+
}
|
|
7573
|
+
appendMeasurement("step", scenarioIndex64, scenario.scenarioName, buildLoadEngineV2ReservedStepOtherIdentityKey(scenarioIndex64), "<other>", undefined, undefined, undefined, true);
|
|
7574
|
+
}
|
|
7575
|
+
for (const segment of result.schedulerSegments ?? []) {
|
|
7576
|
+
const scenarioIndex64 = segment.scenarioIndex.toString();
|
|
7577
|
+
const simulationIndex64 = segment.simulationIndex.toString();
|
|
7578
|
+
for (const kind of ["decision", "start"]) {
|
|
7579
|
+
const seriesKind = kind === "decision" ? "scheduler-decision-lag" : "scheduler-start-lag";
|
|
7580
|
+
const identityKeyHex = buildLoadEngineV2SchedulerIdentityKey(kind, scenarioIndex64, simulationIndex64).toString("hex");
|
|
7581
|
+
const signed = result[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]?.find((record) => record.seriesKind === seriesKind
|
|
7582
|
+
&& record.scenarioIndex64 === scenarioIndex64
|
|
7583
|
+
&& record.identityKeyHex === identityKeyHex
|
|
7584
|
+
&& record.outcome === "none"
|
|
7585
|
+
&& record.unit === "microseconds");
|
|
7586
|
+
if (!distributions.some((record) => record.seriesKind === seriesKind
|
|
7587
|
+
&& record.scenarioIndex64 === scenarioIndex64
|
|
7588
|
+
&& record.identityKeyHex === identityKeyHex)) {
|
|
7589
|
+
distributions.push(signed ? cloneLoadEngineV2DistributionRecord(signed) : {
|
|
7590
|
+
seriesKind,
|
|
7591
|
+
scenarioIndex64, scenarioName: segment.scenarioName,
|
|
7592
|
+
identityKeyHex,
|
|
7593
|
+
outcome: "none", unit: "microseconds", histogram: emptyHistogram(),
|
|
7594
|
+
exactTotalDecimalOrEmpty: "0"
|
|
7595
|
+
});
|
|
7596
|
+
}
|
|
7597
|
+
}
|
|
7598
|
+
}
|
|
7599
|
+
return serializeLoadEngineV2HistogramArtifact({ distributions, measurementSummaries });
|
|
7600
|
+
}
|
|
7601
|
+
function cloneLoadEngineV2DistributionRecord(record) {
|
|
7602
|
+
return {
|
|
7603
|
+
...record,
|
|
7604
|
+
histogram: {
|
|
7605
|
+
...record.histogram,
|
|
7606
|
+
exactSamples64: [...record.histogram.exactSamples64],
|
|
7607
|
+
buckets: record.histogram.buckets.map((bucket) => ({ ...bucket }))
|
|
7608
|
+
},
|
|
7609
|
+
bands: record.bands?.map((band) => ({ ...band }))
|
|
7610
|
+
};
|
|
7611
|
+
}
|
|
7612
|
+
function mergeLoadEngineV2SchedulerDistributions(records) {
|
|
7613
|
+
const merged = new Map();
|
|
7614
|
+
for (const input of records) {
|
|
7615
|
+
if (input.seriesKind !== "scheduler-decision-lag" && input.seriesKind !== "scheduler-start-lag") {
|
|
7616
|
+
continue;
|
|
7617
|
+
}
|
|
7618
|
+
const key = [input.seriesKind, input.scenarioIndex64, input.identityKeyHex, input.outcome, input.unit].join("\0");
|
|
7619
|
+
const current = merged.get(key);
|
|
7620
|
+
if (!current) {
|
|
7621
|
+
merged.set(key, cloneLoadEngineV2DistributionRecord(input));
|
|
7622
|
+
continue;
|
|
7623
|
+
}
|
|
7624
|
+
if (current.scenarioName !== input.scenarioName) {
|
|
7625
|
+
throw new Error("Load Engine V2 scheduler histogram identities disagree across agents.");
|
|
7626
|
+
}
|
|
7627
|
+
const histogram = LoadStrikeHistogramV1.fromSidecar(current.histogram);
|
|
7628
|
+
histogram.merge(LoadStrikeHistogramV1.fromSidecar(input.histogram));
|
|
7629
|
+
current.histogram = histogram.toSidecar();
|
|
7630
|
+
current.exactTotalDecimalOrEmpty = histogram.toSidecar().exactTotal64;
|
|
7631
|
+
const bands = new Map();
|
|
7632
|
+
for (const band of [...(current.bands ?? []), ...(input.bands ?? [])]) {
|
|
7633
|
+
bands.set(band.bandId64, (bands.get(band.bandId64) ?? 0n) + BigInt(band.count64));
|
|
7634
|
+
}
|
|
7635
|
+
current.bands = Array.from(bands, ([bandId64, count]) => ({ bandId64, count64: count.toString() }));
|
|
7636
|
+
}
|
|
7637
|
+
return Array.from(merged.values());
|
|
7638
|
+
}
|
|
7639
|
+
function nodeStatsToClusterPayload(result, scenarioDeclarations, requireLoadEngineV2Histogram = false) {
|
|
7640
|
+
const histogramArtifactBase64 = requireLoadEngineV2Histogram
|
|
7641
|
+
? buildRuntimeLoadEngineV2HistogramArtifact(result, scenarioDeclarations).toString("base64")
|
|
7642
|
+
: undefined;
|
|
5921
7643
|
return {
|
|
5922
7644
|
allBytes: result.allBytes,
|
|
5923
7645
|
allRequestCount: result.allRequestCount,
|
|
@@ -5930,7 +7652,13 @@ function nodeStatsToClusterPayload(result) {
|
|
|
5930
7652
|
pluginsData: result.pluginsData,
|
|
5931
7653
|
nodeInfo: result.nodeInfo,
|
|
5932
7654
|
testInfo: result.testInfo,
|
|
5933
|
-
logFiles: [...(result.logFiles ?? [])]
|
|
7655
|
+
logFiles: [...(result.logFiles ?? [])],
|
|
7656
|
+
generatorWarnings: result.generatorWarnings,
|
|
7657
|
+
observationDeliveryStats: result.observationDeliveryStats,
|
|
7658
|
+
schedulerSegments: result.schedulerSegments,
|
|
7659
|
+
schedulerStats: result.schedulerStats,
|
|
7660
|
+
...(histogramArtifactBase64 ? { histogramArtifactBase64 } : {}),
|
|
7661
|
+
reportingComplete: result.reportingComplete
|
|
5934
7662
|
};
|
|
5935
7663
|
}
|
|
5936
7664
|
function toDetailedRunResultFromNodeStats(result, startedUtc, sinkErrors, policyErrors = []) {
|
|
@@ -5958,6 +7686,14 @@ function toDetailedRunResultFromNodeStats(result, startedUtc, sinkErrors, policy
|
|
|
5958
7686
|
policyErrors: policyErrors.map((value) => attachRuntimePolicyErrorAliases({ ...value })),
|
|
5959
7687
|
reportFiles: [...result.reportFiles],
|
|
5960
7688
|
logFiles: [...(result.logFiles ?? [])],
|
|
7689
|
+
generatorWarnings: (result.generatorWarnings ?? []).map(attachGeneratorWarningAliases),
|
|
7690
|
+
schedulerSegments: result.schedulerSegments?.map((segment) => ({ ...segment })),
|
|
7691
|
+
schedulerStats: result.schedulerStats
|
|
7692
|
+
? normalizeSchedulerStats(result.schedulerStats)
|
|
7693
|
+
: undefined,
|
|
7694
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: result[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]?.map(cloneLoadEngineV2DistributionRecord),
|
|
7695
|
+
observationDeliveryStats: normalizeObservationDeliveryStats(result.observationDeliveryStats ?? emptyObservationDeliveryStats()),
|
|
7696
|
+
reportingComplete: result.reportingComplete ?? false,
|
|
5961
7697
|
correlationRows: buildDetailedCorrelationRows(),
|
|
5962
7698
|
failedCorrelationRows: buildDetailedFailedCorrelationRows()
|
|
5963
7699
|
});
|
|
@@ -5980,7 +7716,198 @@ function flattenMetricValues(metricStats) {
|
|
|
5980
7716
|
}))
|
|
5981
7717
|
];
|
|
5982
7718
|
}
|
|
5983
|
-
function
|
|
7719
|
+
function projectLoadEngineV2MeasurementsFromArtifact(sourceScenarios, artifact) {
|
|
7720
|
+
const distributions = new Map();
|
|
7721
|
+
for (const distribution of artifact.distributions) {
|
|
7722
|
+
const key = loadEngineV2DistributionProjectionKey(distribution.seriesKind, distribution.scenarioIndex64, distribution.identityKeyHex, distribution.outcome, distribution.unit);
|
|
7723
|
+
if (distributions.has(key)) {
|
|
7724
|
+
throw new Error("Load Engine V2 histogram artifact contains a duplicate distribution identity.");
|
|
7725
|
+
}
|
|
7726
|
+
distributions.set(key, distribution);
|
|
7727
|
+
}
|
|
7728
|
+
const scenarioSummaries = artifact.measurementSummaries
|
|
7729
|
+
.filter((summary) => summary.seriesKind === "scenario")
|
|
7730
|
+
.sort((left, right) => compareLoadEngineV2Decimal(left.scenarioIndex64, right.scenarioIndex64));
|
|
7731
|
+
if (scenarioSummaries.length !== sourceScenarios.length) {
|
|
7732
|
+
throw new Error("Load Engine V2 histogram artifact scenario summaries do not reconcile with the scheduler snapshot.");
|
|
7733
|
+
}
|
|
7734
|
+
const sourceByName = new Map();
|
|
7735
|
+
for (const scenario of sourceScenarios) {
|
|
7736
|
+
if (!scenario.scenarioName || sourceByName.has(scenario.scenarioName)) {
|
|
7737
|
+
throw new Error("Load Engine V2 scheduler snapshot scenario identities are empty or duplicated.");
|
|
7738
|
+
}
|
|
7739
|
+
sourceByName.set(scenario.scenarioName, scenario);
|
|
7740
|
+
}
|
|
7741
|
+
return scenarioSummaries.map((summary) => {
|
|
7742
|
+
const source = sourceByName.get(summary.scenarioName);
|
|
7743
|
+
if (!source) {
|
|
7744
|
+
throw new Error("Load Engine V2 histogram artifact scenario identity is absent from the scheduler snapshot.");
|
|
7745
|
+
}
|
|
7746
|
+
const expectedIdentity = buildLoadEngineV2ScenarioIdentityKey(summary.scenarioIndex64).toString("hex");
|
|
7747
|
+
if (summary.identityKeyHex !== expectedIdentity) {
|
|
7748
|
+
throw new Error("Load Engine V2 histogram artifact scenario identity is not canonical.");
|
|
7749
|
+
}
|
|
7750
|
+
const observationCount = loadEngineV2SafeNumber(summary.observationCount64, "scenario observation count");
|
|
7751
|
+
const successCount = loadEngineV2SafeNumber(summary.successCount64, "scenario success count");
|
|
7752
|
+
const failureCount = loadEngineV2SafeNumber(summary.failureCount64, "scenario failure count");
|
|
7753
|
+
if (source.allRequestCount !== observationCount
|
|
7754
|
+
|| source.allOkCount !== successCount
|
|
7755
|
+
|| source.allFailCount !== failureCount) {
|
|
7756
|
+
throw new Error("Load Engine V2 histogram artifact scenario counts do not reconcile with the scheduler snapshot.");
|
|
7757
|
+
}
|
|
7758
|
+
const ok = projectLoadEngineV2Measurement(summary, "ok", distributions, observationCount, source.durationMs, [source.ok]);
|
|
7759
|
+
const fail = projectLoadEngineV2Measurement(summary, "fail", distributions, observationCount, source.durationMs, [source.fail]);
|
|
7760
|
+
const allMeasurement = projectLoadEngineV2Measurement(summary, "all", distributions, observationCount, source.durationMs, [source.ok, source.fail]);
|
|
7761
|
+
const stepSummaries = artifact.measurementSummaries
|
|
7762
|
+
.filter((candidate) => candidate.seriesKind === "step"
|
|
7763
|
+
&& candidate.scenarioIndex64 === summary.scenarioIndex64
|
|
7764
|
+
&& candidate.scenarioName === summary.scenarioName)
|
|
7765
|
+
.sort((left, right) => Buffer.compare(Buffer.from(left.identityKeyHex, "hex"), Buffer.from(right.identityKeyHex, "hex")));
|
|
7766
|
+
const stepIdentitySet = new Set(stepSummaries.map((candidate) => candidate.identityKeyHex));
|
|
7767
|
+
const reservedOtherIdentity = buildLoadEngineV2ReservedStepOtherIdentityKey(summary.scenarioIndex64).toString("hex");
|
|
7768
|
+
const stepStats = stepSummaries.map((stepSummary, sortIndex) => {
|
|
7769
|
+
const matchingSourceSteps = source.stepStats.filter((step) => {
|
|
7770
|
+
const observedIdentity = buildLoadEngineV2StepIdentityKey(summary.scenarioIndex64, step.stepName).identity.toString("hex");
|
|
7771
|
+
return stepSummary.identityKeyHex === reservedOtherIdentity
|
|
7772
|
+
? !stepIdentitySet.has(observedIdentity)
|
|
7773
|
+
: observedIdentity === stepSummary.identityKeyHex;
|
|
7774
|
+
});
|
|
7775
|
+
const stepObservationCount = loadEngineV2SafeNumber(stepSummary.observationCount64, "step observation count");
|
|
7776
|
+
const stepOk = projectLoadEngineV2Measurement(stepSummary, "ok", distributions, observationCount, source.durationMs, matchingSourceSteps.map((step) => step.ok));
|
|
7777
|
+
const stepFail = projectLoadEngineV2Measurement(stepSummary, "fail", distributions, observationCount, source.durationMs, matchingSourceSteps.map((step) => step.fail));
|
|
7778
|
+
const stepAll = projectLoadEngineV2Measurement(stepSummary, "all", distributions, observationCount, source.durationMs, matchingSourceSteps.flatMap((step) => [step.ok, step.fail]));
|
|
7779
|
+
const totalLatencyMs = stepAll.latency.meanMs * stepObservationCount;
|
|
7780
|
+
return {
|
|
7781
|
+
scenarioName: summary.scenarioName,
|
|
7782
|
+
stepName: stepSummary.display,
|
|
7783
|
+
okCount: stepOk.request.count,
|
|
7784
|
+
failCount: stepFail.request.count,
|
|
7785
|
+
requestCount: stepObservationCount,
|
|
7786
|
+
totalBytes: stepAll.dataTransfer.allBytes,
|
|
7787
|
+
totalLatencyMs,
|
|
7788
|
+
avgLatencyMs: stepObservationCount > 0 ? totalLatencyMs / stepObservationCount : 0,
|
|
7789
|
+
minLatencyMs: stepAll.latency.minMs,
|
|
7790
|
+
maxLatencyMs: stepAll.latency.maxMs,
|
|
7791
|
+
statusCodes: aggregateStatusCodeCounts(stepOk.statusCodes, stepFail.statusCodes),
|
|
7792
|
+
ok: stepOk,
|
|
7793
|
+
fail: stepFail,
|
|
7794
|
+
allMeasurement: stepAll,
|
|
7795
|
+
sortIndex
|
|
7796
|
+
};
|
|
7797
|
+
});
|
|
7798
|
+
const totalLatencyMs = allMeasurement.latency.meanMs * observationCount;
|
|
7799
|
+
const projected = {
|
|
7800
|
+
scenarioName: summary.scenarioName,
|
|
7801
|
+
allRequestCount: observationCount,
|
|
7802
|
+
allOkCount: successCount,
|
|
7803
|
+
allFailCount: failureCount,
|
|
7804
|
+
totalBytes: allMeasurement.dataTransfer.allBytes,
|
|
7805
|
+
totalLatencyMs,
|
|
7806
|
+
avgLatencyMs: observationCount > 0 ? totalLatencyMs / observationCount : 0,
|
|
7807
|
+
minLatencyMs: allMeasurement.latency.minMs,
|
|
7808
|
+
maxLatencyMs: allMeasurement.latency.maxMs,
|
|
7809
|
+
statusCodes: aggregateStatusCodeCounts(ok.statusCodes, fail.statusCodes),
|
|
7810
|
+
allMeasurement,
|
|
7811
|
+
allBytes: allMeasurement.dataTransfer.allBytes,
|
|
7812
|
+
currentOperation: source.currentOperation,
|
|
7813
|
+
durationMs: source.durationMs,
|
|
7814
|
+
ok,
|
|
7815
|
+
fail,
|
|
7816
|
+
loadSimulationStats: { ...source.loadSimulationStats },
|
|
7817
|
+
sortIndex: loadEngineV2SafeNumber(summary.scenarioIndex64, "scenario index"),
|
|
7818
|
+
stepStats,
|
|
7819
|
+
findStepStats: (stepName) => stepStats.find((step) => step.stepName === stepName),
|
|
7820
|
+
getStepStats: (stepName) => {
|
|
7821
|
+
const step = stepStats.find((candidate) => candidate.stepName === stepName);
|
|
7822
|
+
if (!step)
|
|
7823
|
+
throw new Error(`Step stats not found: ${stepName}`);
|
|
7824
|
+
return step;
|
|
7825
|
+
}
|
|
7826
|
+
};
|
|
7827
|
+
return attachScenarioStatsAliases(projected);
|
|
7828
|
+
});
|
|
7829
|
+
}
|
|
7830
|
+
function projectLoadEngineV2Measurement(summary, outcome, distributions, allRequestCount, durationMs, sourceMeasurements) {
|
|
7831
|
+
const expectedCount64 = outcome === "ok"
|
|
7832
|
+
? summary.successCount64
|
|
7833
|
+
: outcome === "fail"
|
|
7834
|
+
? summary.failureCount64
|
|
7835
|
+
: summary.observationCount64;
|
|
7836
|
+
const latency = requireLoadEngineV2MeasurementDistribution(summary, outcome, "microseconds", distributions);
|
|
7837
|
+
const size = requireLoadEngineV2MeasurementDistribution(summary, outcome, "bytes", distributions);
|
|
7838
|
+
if (latency.histogram.count64 !== expectedCount64 || size.histogram.count64 !== expectedCount64) {
|
|
7839
|
+
throw new Error("Load Engine V2 histogram distribution counts do not reconcile with its measurement summary.");
|
|
7840
|
+
}
|
|
7841
|
+
const bands = new Map();
|
|
7842
|
+
for (const band of latency.bands ?? []) {
|
|
7843
|
+
if (bands.has(band.bandId64)) {
|
|
7844
|
+
throw new Error("Load Engine V2 latency distribution contains a duplicate band.");
|
|
7845
|
+
}
|
|
7846
|
+
bands.set(band.bandId64, BigInt(band.count64));
|
|
7847
|
+
}
|
|
7848
|
+
if (bands.size !== 3 || !bands.has("0") || !bands.has("1") || !bands.has("2")
|
|
7849
|
+
|| [...bands.values()].reduce((sum, value) => sum + value, 0n) !== BigInt(expectedCount64)) {
|
|
7850
|
+
throw new Error("Load Engine V2 latency distribution bands do not reconcile with its count.");
|
|
7851
|
+
}
|
|
7852
|
+
const sourceStatuses = new Map();
|
|
7853
|
+
for (const measurement of sourceMeasurements) {
|
|
7854
|
+
for (const status of measurement.statusCodes) {
|
|
7855
|
+
const identity = buildLoadEngineV2StatusIdentityKey(status.statusCode, status.message);
|
|
7856
|
+
if (identity)
|
|
7857
|
+
sourceStatuses.set(identity.identity.toString("hex"), status);
|
|
7858
|
+
}
|
|
7859
|
+
}
|
|
7860
|
+
const statusCodes = new Map();
|
|
7861
|
+
const outcomeSummaries = outcome === "all"
|
|
7862
|
+
? summary.outcomes
|
|
7863
|
+
: summary.outcomes.filter((candidate) => candidate.outcome === outcome);
|
|
7864
|
+
for (const outcomeSummary of outcomeSummaries) {
|
|
7865
|
+
for (const status of outcomeSummary.statuses) {
|
|
7866
|
+
const source = sourceStatuses.get(status.statusIdentityKeyHex);
|
|
7867
|
+
statusCodes.set(`${outcomeSummary.outcome}\0${status.statusIdentityKeyHex}`, {
|
|
7868
|
+
statusCode: source?.statusCode ?? status.display,
|
|
7869
|
+
message: source?.message ?? "",
|
|
7870
|
+
isError: source?.isError ?? outcomeSummary.outcome === "fail",
|
|
7871
|
+
count: loadEngineV2SafeNumber(status.count64, "status count")
|
|
7872
|
+
});
|
|
7873
|
+
}
|
|
7874
|
+
}
|
|
7875
|
+
const expectedCount = loadEngineV2SafeNumber(expectedCount64, "measurement count");
|
|
7876
|
+
const sizeHistogram = LoadStrikeHistogramV1.fromSidecar(size.histogram);
|
|
7877
|
+
return buildHistogramMeasurement({
|
|
7878
|
+
count: expectedCount,
|
|
7879
|
+
allBytes: loadEngineV2SafeNumber(sizeHistogram.exactTotal.toString(), "measurement byte total"),
|
|
7880
|
+
latency: LoadStrikeHistogramV1.fromSidecar(latency.histogram),
|
|
7881
|
+
size: sizeHistogram,
|
|
7882
|
+
statusCodes,
|
|
7883
|
+
lessOrEq800: loadEngineV2SafeNumber((bands.get("0") ?? 0n).toString(), "latency band count"),
|
|
7884
|
+
more800Less1200: loadEngineV2SafeNumber((bands.get("1") ?? 0n).toString(), "latency band count"),
|
|
7885
|
+
moreOrEq1200: loadEngineV2SafeNumber((bands.get("2") ?? 0n).toString(), "latency band count")
|
|
7886
|
+
}, allRequestCount, durationMs);
|
|
7887
|
+
}
|
|
7888
|
+
function requireLoadEngineV2MeasurementDistribution(summary, outcome, unit, distributions) {
|
|
7889
|
+
const distribution = distributions.get(loadEngineV2DistributionProjectionKey(summary.seriesKind, summary.scenarioIndex64, summary.identityKeyHex, outcome, unit));
|
|
7890
|
+
if (!distribution || distribution.scenarioName !== summary.scenarioName) {
|
|
7891
|
+
throw new Error("Load Engine V2 measurement summary is missing its canonical histogram distribution.");
|
|
7892
|
+
}
|
|
7893
|
+
return distribution;
|
|
7894
|
+
}
|
|
7895
|
+
function loadEngineV2DistributionProjectionKey(seriesKind, scenarioIndex64, identityKeyHex, outcome, unit) {
|
|
7896
|
+
return [seriesKind, scenarioIndex64, identityKeyHex, outcome, unit].join("\0");
|
|
7897
|
+
}
|
|
7898
|
+
function compareLoadEngineV2Decimal(left, right) {
|
|
7899
|
+
const a = BigInt(left);
|
|
7900
|
+
const b = BigInt(right);
|
|
7901
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
7902
|
+
}
|
|
7903
|
+
function loadEngineV2SafeNumber(value, field) {
|
|
7904
|
+
const parsed = BigInt(value);
|
|
7905
|
+
if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
7906
|
+
throw new Error(`Load Engine V2 ${field} exceeds the JavaScript safe integer range.`);
|
|
7907
|
+
}
|
|
7908
|
+
return Number(parsed);
|
|
7909
|
+
}
|
|
7910
|
+
function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo, requireHistogramArtifact = false) {
|
|
5984
7911
|
const completedUtc = new Date().toISOString();
|
|
5985
7912
|
if (!result.success || !result.stats) {
|
|
5986
7913
|
return attachNodeStatsAliases({
|
|
@@ -6013,12 +7940,12 @@ function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo) {
|
|
|
6013
7940
|
isFailed: true,
|
|
6014
7941
|
errorCount: 1,
|
|
6015
7942
|
exceptionMessage: result.error ?? "Agent execution failed."
|
|
6016
|
-
}]
|
|
7943
|
+
}],
|
|
7944
|
+
reportingComplete: false
|
|
6017
7945
|
});
|
|
6018
7946
|
}
|
|
6019
7947
|
const metrics = normalizeMetricStatsPayload(result.stats.metrics, result.stats.durationMs ?? 0);
|
|
6020
|
-
|
|
6021
|
-
const stepStats = scenarioStats.flatMap((value) => value.stepStats);
|
|
7948
|
+
let scenarioStats = normalizeScenarioStatsPayload(result.stats.scenarioStats);
|
|
6022
7949
|
const thresholds = normalizeThresholdPayload(result.stats.thresholds);
|
|
6023
7950
|
const pluginsData = normalizePluginsPayload(result.stats.pluginsData);
|
|
6024
7951
|
const nodeInfo = {
|
|
@@ -6031,6 +7958,17 @@ function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo) {
|
|
|
6031
7958
|
...testInfo,
|
|
6032
7959
|
...(result.stats.testInfo ?? {})
|
|
6033
7960
|
};
|
|
7961
|
+
const histogramArtifactBase64 = String(result.stats.histogramArtifactBase64 ?? "");
|
|
7962
|
+
if (requireHistogramArtifact && !histogramArtifactBase64) {
|
|
7963
|
+
throw new Error("Load Engine V2 result omits the mandatory LS-H1 histogram artifact.");
|
|
7964
|
+
}
|
|
7965
|
+
const histogramArtifact = histogramArtifactBase64
|
|
7966
|
+
? parseLoadEngineV2HistogramArtifact(Buffer.from(histogramArtifactBase64, "base64"))
|
|
7967
|
+
: undefined;
|
|
7968
|
+
if (histogramArtifact) {
|
|
7969
|
+
scenarioStats = projectLoadEngineV2MeasurementsFromArtifact(scenarioStats, histogramArtifact);
|
|
7970
|
+
}
|
|
7971
|
+
const stepStats = scenarioStats.flatMap((value) => value.stepStats);
|
|
6034
7972
|
return attachNodeStatsAliases({
|
|
6035
7973
|
startedUtc: testInfo.createdUtc,
|
|
6036
7974
|
completedUtc,
|
|
@@ -6053,6 +7991,17 @@ function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo) {
|
|
|
6053
7991
|
sinkErrors: [],
|
|
6054
7992
|
reportFiles: [],
|
|
6055
7993
|
logFiles: normalizeAliasStringArray(result.stats.logFiles),
|
|
7994
|
+
generatorWarnings: (result.stats.generatorWarnings ?? []).map(attachGeneratorWarningAliases),
|
|
7995
|
+
schedulerSegments: (result.stats.schedulerSegments ?? []).map(normalizeSchedulerSegment),
|
|
7996
|
+
schedulerStats: result.stats.schedulerStats
|
|
7997
|
+
? normalizeSchedulerStats(result.stats.schedulerStats)
|
|
7998
|
+
: undefined,
|
|
7999
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: histogramArtifact?.distributions
|
|
8000
|
+
.filter((record) => record.seriesKind === "scheduler-decision-lag"
|
|
8001
|
+
|| record.seriesKind === "scheduler-start-lag")
|
|
8002
|
+
.map(cloneLoadEngineV2DistributionRecord),
|
|
8003
|
+
observationDeliveryStats: normalizeObservationDeliveryStats(result.stats.observationDeliveryStats ?? emptyObservationDeliveryStats()),
|
|
8004
|
+
reportingComplete: result.stats.reportingComplete ?? false,
|
|
6056
8005
|
findScenarioStats: (scenarioName) => scenarioStats.find((value) => value.scenarioName === scenarioName),
|
|
6057
8006
|
getScenarioStats: (scenarioName) => {
|
|
6058
8007
|
const value = scenarioStats.find((scenario) => scenario.scenarioName === scenarioName);
|
|
@@ -6063,7 +8012,46 @@ function clusterNodeResultToNodeStats(result, testInfo, fallbackNodeInfo) {
|
|
|
6063
8012
|
}
|
|
6064
8013
|
});
|
|
6065
8014
|
}
|
|
6066
|
-
function
|
|
8015
|
+
function emptyObservationDeliveryStats() {
|
|
8016
|
+
return normalizeObservationDeliveryStats({
|
|
8017
|
+
lastBatchSequence64: "-1",
|
|
8018
|
+
capturedCount64: "0",
|
|
8019
|
+
deliveredCount64: "0",
|
|
8020
|
+
droppedBufferCount64: "0",
|
|
8021
|
+
droppedSinkCount64: "0"
|
|
8022
|
+
});
|
|
8023
|
+
}
|
|
8024
|
+
function aggregateObservationDeliveryStats(nodes) {
|
|
8025
|
+
let lastBatchSequence = -1n;
|
|
8026
|
+
let captured = 0n;
|
|
8027
|
+
let delivered = 0n;
|
|
8028
|
+
let droppedBuffer = 0n;
|
|
8029
|
+
let droppedSink = 0n;
|
|
8030
|
+
for (const node of nodes) {
|
|
8031
|
+
const stats = node.observationDeliveryStats ?? emptyObservationDeliveryStats();
|
|
8032
|
+
lastBatchSequence = maxBigInt(lastBatchSequence, parseObservationDecimal(stats.lastBatchSequence64, -1n));
|
|
8033
|
+
captured += parseObservationDecimal(stats.capturedCount64);
|
|
8034
|
+
delivered += parseObservationDecimal(stats.deliveredCount64);
|
|
8035
|
+
droppedBuffer += parseObservationDecimal(stats.droppedBufferCount64);
|
|
8036
|
+
droppedSink += parseObservationDecimal(stats.droppedSinkCount64);
|
|
8037
|
+
}
|
|
8038
|
+
return normalizeObservationDeliveryStats({
|
|
8039
|
+
lastBatchSequence64: lastBatchSequence.toString(),
|
|
8040
|
+
capturedCount64: captured.toString(),
|
|
8041
|
+
deliveredCount64: delivered.toString(),
|
|
8042
|
+
droppedBufferCount64: droppedBuffer.toString(),
|
|
8043
|
+
droppedSinkCount64: droppedSink.toString()
|
|
8044
|
+
});
|
|
8045
|
+
}
|
|
8046
|
+
function parseObservationDecimal(value, fallback = 0n) {
|
|
8047
|
+
try {
|
|
8048
|
+
return BigInt(value);
|
|
8049
|
+
}
|
|
8050
|
+
catch {
|
|
8051
|
+
return fallback;
|
|
8052
|
+
}
|
|
8053
|
+
}
|
|
8054
|
+
function aggregateNodeStats(testInfo, coordinatorNodeInfo, nodes, requireHistograms = false) {
|
|
6067
8055
|
if (!nodes.length) {
|
|
6068
8056
|
return buildEmptyNodeStats({
|
|
6069
8057
|
startedUtc: testInfo.createdUtc,
|
|
@@ -6072,12 +8060,22 @@ function aggregateNodeStats(testInfo, coordinatorNodeInfo, nodes) {
|
|
|
6072
8060
|
testInfo
|
|
6073
8061
|
});
|
|
6074
8062
|
}
|
|
6075
|
-
const scenarioStats = aggregateScenarioStats(nodes);
|
|
8063
|
+
const scenarioStats = aggregateScenarioStats(nodes, requireHistograms);
|
|
6076
8064
|
const stepStats = scenarioStats.flatMap((value) => value.stepStats);
|
|
6077
8065
|
const metrics = aggregateMetricStats(nodes);
|
|
6078
8066
|
const thresholds = aggregateThresholds(nodes);
|
|
6079
8067
|
const pluginsData = aggregatePluginsData(nodes);
|
|
6080
8068
|
const completedUtc = new Date().toISOString();
|
|
8069
|
+
const schedulerSegments = nodes.flatMap((value) => value.schedulerSegments ?? []);
|
|
8070
|
+
const schedulerStatsRows = nodes.flatMap((value) => value.schedulerStats ? [value.schedulerStats] : []);
|
|
8071
|
+
const schedulerStats = schedulerSegments.length || schedulerStatsRows.length
|
|
8072
|
+
? {
|
|
8073
|
+
configuredMaxInFlight: schedulerStatsRows.reduce((maximum, value) => Math.max(maximum, value.configuredMaxInFlight), 0),
|
|
8074
|
+
maxInFlightObserved: schedulerStatsRows.reduce((maximum, value) => Math.max(maximum, value.maxInFlightObserved), 0),
|
|
8075
|
+
currentInFlight: schedulerStatsRows.reduce((sum, value) => sum + value.currentInFlight, 0),
|
|
8076
|
+
segments: schedulerSegments
|
|
8077
|
+
}
|
|
8078
|
+
: undefined;
|
|
6081
8079
|
return attachNodeStatsAliases({
|
|
6082
8080
|
startedUtc: testInfo.createdUtc,
|
|
6083
8081
|
completedUtc,
|
|
@@ -6100,6 +8098,12 @@ function aggregateNodeStats(testInfo, coordinatorNodeInfo, nodes) {
|
|
|
6100
8098
|
sinkErrors: [],
|
|
6101
8099
|
reportFiles: [],
|
|
6102
8100
|
logFiles: mergeStringArrays(...nodes.map((value) => value.logFiles ?? [])),
|
|
8101
|
+
generatorWarnings: nodes.flatMap((value) => value.generatorWarnings ?? []),
|
|
8102
|
+
...(schedulerSegments.length ? { schedulerSegments } : {}),
|
|
8103
|
+
...(schedulerStats ? { schedulerStats } : {}),
|
|
8104
|
+
[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS]: mergeLoadEngineV2SchedulerDistributions(nodes.flatMap((value) => value[LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS] ?? [])),
|
|
8105
|
+
observationDeliveryStats: aggregateObservationDeliveryStats(nodes),
|
|
8106
|
+
reportingComplete: nodes.every((value) => value.reportingComplete ?? false),
|
|
6103
8107
|
findScenarioStats: (scenarioName) => scenarioStats.find((value) => value.scenarioName === scenarioName),
|
|
6104
8108
|
getScenarioStats: (scenarioName) => {
|
|
6105
8109
|
const value = scenarioStats.find((scenario) => scenario.scenarioName === scenarioName);
|
|
@@ -6110,7 +8114,7 @@ function aggregateNodeStats(testInfo, coordinatorNodeInfo, nodes) {
|
|
|
6110
8114
|
}
|
|
6111
8115
|
});
|
|
6112
8116
|
}
|
|
6113
|
-
function aggregateScenarioStats(nodes) {
|
|
8117
|
+
function aggregateScenarioStats(nodes, requireHistograms = false) {
|
|
6114
8118
|
const grouped = new Map();
|
|
6115
8119
|
for (const node of nodes) {
|
|
6116
8120
|
for (const scenario of node.scenarioStats) {
|
|
@@ -6124,9 +8128,12 @@ function aggregateScenarioStats(nodes) {
|
|
|
6124
8128
|
.map((items) => {
|
|
6125
8129
|
const allRequestCount = items.reduce((sum, value) => sum + value.allRequestCount, 0);
|
|
6126
8130
|
const durationMs = items.reduce((max, value) => Math.max(max, value.durationMs), 0);
|
|
6127
|
-
const ok = aggregateMeasurementStats(items.map((value) => value.ok), allRequestCount, durationMs);
|
|
6128
|
-
const fail = aggregateMeasurementStats(items.map((value) => value.fail), allRequestCount, durationMs);
|
|
6129
|
-
const
|
|
8131
|
+
const ok = aggregateMeasurementStats(items.map((value) => value.ok), allRequestCount, durationMs, requireHistograms);
|
|
8132
|
+
const fail = aggregateMeasurementStats(items.map((value) => value.fail), allRequestCount, durationMs, requireHistograms);
|
|
8133
|
+
const allMeasurement = requireHistograms
|
|
8134
|
+
? aggregateMeasurementStats([ok, fail], allRequestCount, durationMs, true)
|
|
8135
|
+
: undefined;
|
|
8136
|
+
const stepStats = aggregateStepStats(items, requireHistograms);
|
|
6130
8137
|
const scenarioName = items[0]?.scenarioName ?? "";
|
|
6131
8138
|
const currentOperation = selectScenarioOperation(items.map((value) => value.currentOperation));
|
|
6132
8139
|
const loadSimulationStats = items.find((value) => value.loadSimulationStats.simulationName)?.loadSimulationStats ?? {
|
|
@@ -6151,6 +8158,7 @@ function aggregateScenarioStats(nodes) {
|
|
|
6151
8158
|
durationMs,
|
|
6152
8159
|
ok,
|
|
6153
8160
|
fail,
|
|
8161
|
+
...(allMeasurement ? { allMeasurement } : {}),
|
|
6154
8162
|
loadSimulationStats,
|
|
6155
8163
|
sortIndex: Math.min(...items.map((value) => value.sortIndex)),
|
|
6156
8164
|
stepStats,
|
|
@@ -6166,7 +8174,7 @@ function aggregateScenarioStats(nodes) {
|
|
|
6166
8174
|
return attachScenarioStatsAliases(scenario);
|
|
6167
8175
|
});
|
|
6168
8176
|
}
|
|
6169
|
-
function aggregateStepStats(scenarios) {
|
|
8177
|
+
function aggregateStepStats(scenarios, requireHistograms = false) {
|
|
6170
8178
|
const grouped = new Map();
|
|
6171
8179
|
for (const scenario of scenarios) {
|
|
6172
8180
|
for (const step of scenario.stepStats) {
|
|
@@ -6180,8 +8188,11 @@ function aggregateStepStats(scenarios) {
|
|
|
6180
8188
|
return Array.from(grouped.values())
|
|
6181
8189
|
.sort((left, right) => Math.min(...left.map((value) => value.sortIndex)) - Math.min(...right.map((value) => value.sortIndex)))
|
|
6182
8190
|
.map((items) => {
|
|
6183
|
-
const ok = aggregateMeasurementStats(items.map((value) => value.ok), allScenarioRequests, scenarioDurationMs);
|
|
6184
|
-
const fail = aggregateMeasurementStats(items.map((value) => value.fail), allScenarioRequests, scenarioDurationMs);
|
|
8191
|
+
const ok = aggregateMeasurementStats(items.map((value) => value.ok), allScenarioRequests, scenarioDurationMs, requireHistograms);
|
|
8192
|
+
const fail = aggregateMeasurementStats(items.map((value) => value.fail), allScenarioRequests, scenarioDurationMs, requireHistograms);
|
|
8193
|
+
const allMeasurement = requireHistograms
|
|
8194
|
+
? aggregateMeasurementStats([ok, fail], allScenarioRequests, scenarioDurationMs, true)
|
|
8195
|
+
: undefined;
|
|
6185
8196
|
const requestCount = ok.request.count + fail.request.count;
|
|
6186
8197
|
return {
|
|
6187
8198
|
scenarioName: items[0]?.scenarioName ?? "",
|
|
@@ -6197,14 +8208,53 @@ function aggregateStepStats(scenarios) {
|
|
|
6197
8208
|
statusCodes: aggregateStatusCodeCounts(ok.statusCodes, fail.statusCodes),
|
|
6198
8209
|
ok,
|
|
6199
8210
|
fail,
|
|
8211
|
+
...(allMeasurement ? { allMeasurement } : {}),
|
|
6200
8212
|
sortIndex: Math.min(...items.map((value) => value.sortIndex))
|
|
6201
8213
|
};
|
|
6202
8214
|
});
|
|
6203
8215
|
}
|
|
6204
|
-
function aggregateMeasurementStats(measurements, allRequestCount, durationMs) {
|
|
8216
|
+
function aggregateMeasurementStats(measurements, allRequestCount, durationMs, requireHistograms = false) {
|
|
6205
8217
|
if (!measurements.length) {
|
|
6206
8218
|
return buildMeasurementPlaceholder(0, allRequestCount, durationMs);
|
|
6207
8219
|
}
|
|
8220
|
+
if (measurements.every((measurement) => measurement.histogramSidecar)) {
|
|
8221
|
+
const latency = LoadStrikeHistogramV1.fromSidecar(measurements[0].histogramSidecar.latency);
|
|
8222
|
+
const size = LoadStrikeHistogramV1.fromSidecar(measurements[0].histogramSidecar.size);
|
|
8223
|
+
for (const measurement of measurements.slice(1)) {
|
|
8224
|
+
latency.merge(LoadStrikeHistogramV1.fromSidecar(measurement.histogramSidecar.latency));
|
|
8225
|
+
size.merge(LoadStrikeHistogramV1.fromSidecar(measurement.histogramSidecar.size));
|
|
8226
|
+
}
|
|
8227
|
+
const statusCodes = new Map();
|
|
8228
|
+
for (const measurement of measurements) {
|
|
8229
|
+
for (const status of measurement.statusCodes) {
|
|
8230
|
+
const key = `${status.statusCode}|${status.message}|${status.isError ? "1" : "0"}`;
|
|
8231
|
+
const current = statusCodes.get(key);
|
|
8232
|
+
if (current)
|
|
8233
|
+
current.count += status.count;
|
|
8234
|
+
else
|
|
8235
|
+
statusCodes.set(key, {
|
|
8236
|
+
statusCode: status.statusCode,
|
|
8237
|
+
message: status.message,
|
|
8238
|
+
isError: status.isError,
|
|
8239
|
+
count: status.count
|
|
8240
|
+
});
|
|
8241
|
+
}
|
|
8242
|
+
}
|
|
8243
|
+
const sumSidecar = (key) => Number(measurements.reduce((sum, measurement) => sum + BigInt(measurement.histogramSidecar[key]), 0n));
|
|
8244
|
+
return buildHistogramMeasurement({
|
|
8245
|
+
count: Number(latency.count),
|
|
8246
|
+
allBytes: measurements.reduce((sum, measurement) => sum + measurement.dataTransfer.allBytes, 0),
|
|
8247
|
+
latency,
|
|
8248
|
+
size,
|
|
8249
|
+
statusCodes,
|
|
8250
|
+
lessOrEq800: sumSidecar("lessOrEq80064"),
|
|
8251
|
+
more800Less1200: sumSidecar("more800Less120064"),
|
|
8252
|
+
moreOrEq1200: sumSidecar("moreOrEq120064")
|
|
8253
|
+
}, allRequestCount, durationMs);
|
|
8254
|
+
}
|
|
8255
|
+
if (requireHistograms) {
|
|
8256
|
+
throw new Error("Load Engine V2 aggregation requires canonical histogram state for every node measurement.");
|
|
8257
|
+
}
|
|
6208
8258
|
const weights = measurements.map((value) => value.request.count);
|
|
6209
8259
|
const totalCount = measurements.reduce((sum, value) => sum + value.request.count, 0);
|
|
6210
8260
|
return {
|
|
@@ -7420,11 +9470,23 @@ function readRuntimeTrackingId(payload, selector) {
|
|
|
7420
9470
|
return null;
|
|
7421
9471
|
}
|
|
7422
9472
|
let current = body;
|
|
7423
|
-
|
|
9473
|
+
const path = selector.slice("json:".length).trim().replace(/^\$\./, "");
|
|
9474
|
+
let segments;
|
|
9475
|
+
try {
|
|
9476
|
+
segments = runtimeSafeJsonPathSegments(path);
|
|
9477
|
+
}
|
|
9478
|
+
catch {
|
|
9479
|
+
return null;
|
|
9480
|
+
}
|
|
9481
|
+
for (const segment of segments) {
|
|
7424
9482
|
if (!current || typeof current !== "object" || Array.isArray(current)) {
|
|
7425
9483
|
return null;
|
|
7426
9484
|
}
|
|
7427
|
-
|
|
9485
|
+
const record = current;
|
|
9486
|
+
if (!Object.prototype.hasOwnProperty.call(record, segment)) {
|
|
9487
|
+
return null;
|
|
9488
|
+
}
|
|
9489
|
+
current = runtimeReadOwnJsonProperty(record, segment);
|
|
7428
9490
|
}
|
|
7429
9491
|
return current == null ? null : String(current);
|
|
7430
9492
|
}
|
|
@@ -7448,23 +9510,57 @@ function runtimeParseBodyAsObject(body) {
|
|
|
7448
9510
|
}
|
|
7449
9511
|
}
|
|
7450
9512
|
function setRuntimeJsonPathValue(body, path, value) {
|
|
7451
|
-
const target =
|
|
7452
|
-
|
|
7453
|
-
: {};
|
|
7454
|
-
const segments = path.split(".").filter(Boolean);
|
|
9513
|
+
const target = runtimeCloneJsonRecord(body);
|
|
9514
|
+
const segments = runtimeSafeJsonPathSegments(path);
|
|
7455
9515
|
if (!segments.length) {
|
|
7456
9516
|
return target;
|
|
7457
9517
|
}
|
|
7458
9518
|
let current = target;
|
|
7459
9519
|
for (let i = 0; i < segments.length - 1; i += 1) {
|
|
7460
9520
|
const segment = segments[i];
|
|
7461
|
-
const next = current
|
|
9521
|
+
const next = runtimeReadOwnJsonProperty(current, segment);
|
|
9522
|
+
let child;
|
|
7462
9523
|
if (!next || typeof next !== "object" || Array.isArray(next)) {
|
|
7463
|
-
|
|
9524
|
+
child = {};
|
|
9525
|
+
}
|
|
9526
|
+
else {
|
|
9527
|
+
child = runtimeCloneJsonRecord(next);
|
|
7464
9528
|
}
|
|
7465
|
-
current
|
|
9529
|
+
runtimeDefineJsonProperty(current, segment, child);
|
|
9530
|
+
current = child;
|
|
9531
|
+
}
|
|
9532
|
+
runtimeDefineJsonProperty(current, segments[segments.length - 1], value);
|
|
9533
|
+
return target;
|
|
9534
|
+
}
|
|
9535
|
+
const FORBIDDEN_RUNTIME_JSON_PATH_SEGMENTS = new Set(["__proto__", "prototype", "constructor"]);
|
|
9536
|
+
function runtimeSafeJsonPathSegments(path) {
|
|
9537
|
+
const segments = path.split(".").filter(Boolean);
|
|
9538
|
+
const forbidden = segments.find((segment) => FORBIDDEN_RUNTIME_JSON_PATH_SEGMENTS.has(segment));
|
|
9539
|
+
if (forbidden) {
|
|
9540
|
+
throw new Error(`Tracking selector contains forbidden JSON path segment '${forbidden}'.`);
|
|
9541
|
+
}
|
|
9542
|
+
return segments;
|
|
9543
|
+
}
|
|
9544
|
+
function runtimeDefineJsonProperty(target, key, value) {
|
|
9545
|
+
Object.defineProperty(target, key, {
|
|
9546
|
+
configurable: true,
|
|
9547
|
+
enumerable: true,
|
|
9548
|
+
value,
|
|
9549
|
+
writable: true
|
|
9550
|
+
});
|
|
9551
|
+
}
|
|
9552
|
+
function runtimeReadOwnJsonProperty(target, key) {
|
|
9553
|
+
const descriptor = Object.getOwnPropertyDescriptor(target, key);
|
|
9554
|
+
return descriptor && "value" in descriptor ? descriptor.value : undefined;
|
|
9555
|
+
}
|
|
9556
|
+
function runtimeCloneJsonRecord(value) {
|
|
9557
|
+
const target = {};
|
|
9558
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
9559
|
+
return target;
|
|
9560
|
+
}
|
|
9561
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
9562
|
+
runtimeDefineJsonProperty(target, key, entry);
|
|
7466
9563
|
}
|
|
7467
|
-
current[segments[segments.length - 1]] = value;
|
|
7468
9564
|
return target;
|
|
7469
9565
|
}
|
|
7470
9566
|
function asTrackingRecord(value) {
|
|
@@ -7474,8 +9570,8 @@ function asTrackingRecord(value) {
|
|
|
7474
9570
|
}
|
|
7475
9571
|
function pickTrackingValue(source, ...keys) {
|
|
7476
9572
|
for (const key of keys) {
|
|
7477
|
-
if (key
|
|
7478
|
-
return source
|
|
9573
|
+
if (Object.prototype.hasOwnProperty.call(source, key)) {
|
|
9574
|
+
return runtimeReadOwnJsonProperty(source, key);
|
|
7479
9575
|
}
|
|
7480
9576
|
}
|
|
7481
9577
|
return undefined;
|
|
@@ -7729,26 +9825,10 @@ function createDefaultLogger(logFilePath) {
|
|
|
7729
9825
|
function wrapLoggerWithMinimumLevel(baseLogger, minimumLogLevel) {
|
|
7730
9826
|
const threshold = logLevelOrder(minimumLogLevel);
|
|
7731
9827
|
return {
|
|
7732
|
-
debug: (message) =>
|
|
7733
|
-
|
|
7734
|
-
|
|
7735
|
-
|
|
7736
|
-
},
|
|
7737
|
-
info: (message) => {
|
|
7738
|
-
if (threshold <= 1) {
|
|
7739
|
-
baseLogger.info(message);
|
|
7740
|
-
}
|
|
7741
|
-
},
|
|
7742
|
-
warn: (message) => {
|
|
7743
|
-
if (threshold <= 2) {
|
|
7744
|
-
baseLogger.warn(message);
|
|
7745
|
-
}
|
|
7746
|
-
},
|
|
7747
|
-
error: (message) => {
|
|
7748
|
-
if (threshold <= 3) {
|
|
7749
|
-
baseLogger.error(message);
|
|
7750
|
-
}
|
|
7751
|
-
}
|
|
9828
|
+
debug: (message) => threshold <= 0 ? baseLogger.debug(message) : undefined,
|
|
9829
|
+
info: (message) => threshold <= 1 ? baseLogger.info(message) : undefined,
|
|
9830
|
+
warn: (message) => threshold <= 2 ? baseLogger.warn(message) : undefined,
|
|
9831
|
+
error: (message) => threshold <= 3 ? baseLogger.error(message) : undefined
|
|
7752
9832
|
};
|
|
7753
9833
|
}
|
|
7754
9834
|
function formatDefaultLoggerLine(level, message) {
|
|
@@ -8026,6 +10106,68 @@ function normalizeOptionalReportFormats(value) {
|
|
|
8026
10106
|
const normalized = normalizeReportFormats(value);
|
|
8027
10107
|
return normalized.length ? normalized : undefined;
|
|
8028
10108
|
}
|
|
10109
|
+
function normalizeDeclaredStepNames(value) {
|
|
10110
|
+
if (!Array.isArray(value)) {
|
|
10111
|
+
throw new TypeError("Declared step names must be provided as text values.");
|
|
10112
|
+
}
|
|
10113
|
+
const seen = new Set();
|
|
10114
|
+
const normalized = [];
|
|
10115
|
+
for (const entry of value) {
|
|
10116
|
+
if (typeof entry !== "string" || !entry.trim()) {
|
|
10117
|
+
throw new Error("Declared step name must be non-empty text.");
|
|
10118
|
+
}
|
|
10119
|
+
const stepName = entry.trim();
|
|
10120
|
+
for (let index = 0; index < stepName.length; index += 1) {
|
|
10121
|
+
const code = stepName.charCodeAt(index);
|
|
10122
|
+
if (code >= 0xd800 && code <= 0xdbff) {
|
|
10123
|
+
const next = stepName.charCodeAt(index + 1);
|
|
10124
|
+
if (!(next >= 0xdc00 && next <= 0xdfff)) {
|
|
10125
|
+
throw new Error("Declared step name contains an invalid Unicode scalar.");
|
|
10126
|
+
}
|
|
10127
|
+
index += 1;
|
|
10128
|
+
}
|
|
10129
|
+
else if (code >= 0xdc00 && code <= 0xdfff) {
|
|
10130
|
+
throw new Error("Declared step name contains an invalid Unicode scalar.");
|
|
10131
|
+
}
|
|
10132
|
+
}
|
|
10133
|
+
if (!seen.has(stepName)) {
|
|
10134
|
+
seen.add(stepName);
|
|
10135
|
+
normalized.push(stepName);
|
|
10136
|
+
}
|
|
10137
|
+
}
|
|
10138
|
+
return normalized;
|
|
10139
|
+
}
|
|
10140
|
+
function resolveIterationObservationSettings(options) {
|
|
10141
|
+
return {
|
|
10142
|
+
flushIntervalMs: options.iterationObservationFlushIntervalSeconds === undefined
|
|
10143
|
+
? DEFAULT_ITERATION_OBSERVATION_SETTINGS.flushIntervalMs
|
|
10144
|
+
: options.iterationObservationFlushIntervalSeconds * 1000,
|
|
10145
|
+
maxBufferBytes: options.maxIterationObservationBufferBytes
|
|
10146
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.maxBufferBytes,
|
|
10147
|
+
maxObservationsPerBatch: options.maxIterationObservationsPerBatch
|
|
10148
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.maxObservationsPerBatch,
|
|
10149
|
+
maxBatchBytes: options.maxIterationObservationBatchBytes
|
|
10150
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.maxBatchBytes,
|
|
10151
|
+
sinkQueueDepth: options.iterationObservationSinkQueueDepth
|
|
10152
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.sinkQueueDepth,
|
|
10153
|
+
sinkParallelism: options.iterationObservationSinkParallelism
|
|
10154
|
+
?? DEFAULT_ITERATION_OBSERVATION_SETTINGS.sinkParallelism,
|
|
10155
|
+
drainTimeoutMs: options.iterationObservationDrainTimeoutSeconds === undefined
|
|
10156
|
+
? DEFAULT_ITERATION_OBSERVATION_SETTINGS.drainTimeoutMs
|
|
10157
|
+
: options.iterationObservationDrainTimeoutSeconds * 1000
|
|
10158
|
+
};
|
|
10159
|
+
}
|
|
10160
|
+
function validateRunContextIterationObservationSettings(values) {
|
|
10161
|
+
validateIterationObservationSettings(resolveIterationObservationSettings({
|
|
10162
|
+
iterationObservationFlushIntervalSeconds: values.IterationObservationFlushIntervalSeconds,
|
|
10163
|
+
maxIterationObservationBufferBytes: values.MaxIterationObservationBufferBytes,
|
|
10164
|
+
maxIterationObservationsPerBatch: values.MaxIterationObservationsPerBatch,
|
|
10165
|
+
maxIterationObservationBatchBytes: values.MaxIterationObservationBatchBytes,
|
|
10166
|
+
iterationObservationSinkQueueDepth: values.IterationObservationSinkQueueDepth,
|
|
10167
|
+
iterationObservationSinkParallelism: values.IterationObservationSinkParallelism,
|
|
10168
|
+
iterationObservationDrainTimeoutSeconds: values.IterationObservationDrainTimeoutSeconds
|
|
10169
|
+
}));
|
|
10170
|
+
}
|
|
8029
10171
|
function assertNoDisableLicenseEnforcementOption(value, source) {
|
|
8030
10172
|
if (value == null || typeof value !== "object" || Array.isArray(value)) {
|
|
8031
10173
|
return;
|
|
@@ -8044,12 +10186,121 @@ function normalizeRunContextCollectionShapes(values) {
|
|
|
8044
10186
|
TargetScenarios: normalizeOptionalStringArray(values.TargetScenarios),
|
|
8045
10187
|
AgentTargetScenarios: normalizeOptionalStringArray(values.AgentTargetScenarios),
|
|
8046
10188
|
CoordinatorTargetScenarios: normalizeOptionalStringArray(values.CoordinatorTargetScenarios),
|
|
10189
|
+
ExpectedAgentIds: normalizeOptionalStringArray(values.ExpectedAgentIds),
|
|
8047
10190
|
ReportFormats: normalizeOptionalReportFormats(values.ReportFormats)
|
|
8048
10191
|
};
|
|
8049
10192
|
validateNamedReportingSinks(normalized.ReportingSinks ?? []);
|
|
8050
10193
|
validateNamedWorkerPlugins(normalized.WorkerPlugins ?? []);
|
|
10194
|
+
validateLoadEngineV2Options(normalized.LoadEngineContractVersion, normalized.MaxInFlight);
|
|
10195
|
+
validateRunContextIterationObservationSettings(normalized);
|
|
8051
10196
|
return normalized;
|
|
8052
10197
|
}
|
|
10198
|
+
function normalizeAliasStringRecord(value) {
|
|
10199
|
+
const source = asAliasRecord(value);
|
|
10200
|
+
const output = {};
|
|
10201
|
+
for (const [key, entry] of Object.entries(source)) {
|
|
10202
|
+
output[key] = String(entry);
|
|
10203
|
+
}
|
|
10204
|
+
return output;
|
|
10205
|
+
}
|
|
10206
|
+
function attachGeneratorWarningAliases(value) {
|
|
10207
|
+
const source = asAliasRecord(value);
|
|
10208
|
+
const projected = {
|
|
10209
|
+
code: pickAliasString(source, "code", "Code"),
|
|
10210
|
+
...(hasAliasValue(source, "sinkName", "SinkName")
|
|
10211
|
+
? { sinkName: pickAliasString(source, "sinkName", "SinkName") }
|
|
10212
|
+
: {}),
|
|
10213
|
+
scenarioName: pickAliasString(source, "scenarioName", "ScenarioName"),
|
|
10214
|
+
...(hasAliasValue(source, "scenarioIndex", "ScenarioIndex")
|
|
10215
|
+
? { scenarioIndex: pickAliasNumber(source, "scenarioIndex", "ScenarioIndex") }
|
|
10216
|
+
: {}),
|
|
10217
|
+
simulationIndex: pickAliasNumber(source, "simulationIndex", "SimulationIndex"),
|
|
10218
|
+
...(hasAliasValue(source, "simulationKind", "SimulationKind")
|
|
10219
|
+
? { simulationKind: pickAliasString(source, "simulationKind", "SimulationKind") }
|
|
10220
|
+
: {}),
|
|
10221
|
+
count64: pickAliasString(source, "count64", "Count64"),
|
|
10222
|
+
message: pickAliasString(source, "message", "Message"),
|
|
10223
|
+
firstObservedUtcNs: pickAliasString(source, "firstObservedUtcNs", "FirstObservedUtcNs"),
|
|
10224
|
+
lastObservedUtcNs: pickAliasString(source, "lastObservedUtcNs", "LastObservedUtcNs")
|
|
10225
|
+
};
|
|
10226
|
+
return attachAliasMap(projected, {
|
|
10227
|
+
Code: "code",
|
|
10228
|
+
SinkName: "sinkName",
|
|
10229
|
+
ScenarioName: "scenarioName",
|
|
10230
|
+
ScenarioIndex: "scenarioIndex",
|
|
10231
|
+
SimulationIndex: "simulationIndex",
|
|
10232
|
+
SimulationKind: "simulationKind",
|
|
10233
|
+
Count64: "count64",
|
|
10234
|
+
Message: "message",
|
|
10235
|
+
FirstObservedUtcNs: "firstObservedUtcNs",
|
|
10236
|
+
LastObservedUtcNs: "lastObservedUtcNs"
|
|
10237
|
+
});
|
|
10238
|
+
}
|
|
10239
|
+
function normalizeSchedulerSegment(value) {
|
|
10240
|
+
const source = asAliasRecord(value);
|
|
10241
|
+
return {
|
|
10242
|
+
scenarioName: pickAliasString(source, "scenarioName", "ScenarioName"),
|
|
10243
|
+
scenarioIndex: pickAliasNumber(source, "scenarioIndex", "ScenarioIndex"),
|
|
10244
|
+
simulationIndex: pickAliasNumber(source, "simulationIndex", "SimulationIndex"),
|
|
10245
|
+
kind: pickAliasString(source, "kind", "Kind"),
|
|
10246
|
+
shardIndex: pickAliasNumber(source, "shardIndex", "ShardIndex"),
|
|
10247
|
+
shardCount: Math.max(pickAliasNumber(source, "shardCount", "ShardCount"), 1),
|
|
10248
|
+
plannedIterations64: pickAliasString(source, "plannedIterations64", "PlannedIterations64"),
|
|
10249
|
+
dueIterations64: pickAliasString(source, "dueIterations64", "DueIterations64"),
|
|
10250
|
+
startedIterations64: pickAliasString(source, "startedIterations64", "StartedIterations64"),
|
|
10251
|
+
completedIterations64: pickAliasString(source, "completedIterations64", "CompletedIterations64"),
|
|
10252
|
+
droppedIterations64: pickAliasString(source, "droppedIterations64", "DroppedIterations64"),
|
|
10253
|
+
unreachedIterations64: pickAliasString(source, "unreachedIterations64", "UnreachedIterations64"),
|
|
10254
|
+
requestedWorkerSlots64: pickAliasString(source, "requestedWorkerSlots64", "RequestedWorkerSlots64"),
|
|
10255
|
+
startedWorkerSlots64: pickAliasString(source, "startedWorkerSlots64", "StartedWorkerSlots64"),
|
|
10256
|
+
unavailableWorkerSlots64: pickAliasString(source, "unavailableWorkerSlots64", "UnavailableWorkerSlots64"),
|
|
10257
|
+
dropReasons: normalizeAliasStringRecord(pickAliasValue(source, "dropReasons", "DropReasons")),
|
|
10258
|
+
unavailableWorkerReasons: normalizeAliasStringRecord(pickAliasValue(source, "unavailableWorkerReasons", "UnavailableWorkerReasons")),
|
|
10259
|
+
deliveryPercent: pickAliasNumber(source, "deliveryPercent", "DeliveryPercent"),
|
|
10260
|
+
accountingComplete: pickAliasBoolean(source, "accountingComplete", "AccountingComplete")
|
|
10261
|
+
};
|
|
10262
|
+
}
|
|
10263
|
+
function normalizeSchedulerStats(value) {
|
|
10264
|
+
const source = asAliasRecord(value);
|
|
10265
|
+
return {
|
|
10266
|
+
configuredMaxInFlight: pickAliasNumber(source, "configuredMaxInFlight", "ConfiguredMaxInFlight"),
|
|
10267
|
+
maxInFlightObserved: pickAliasNumber(source, "maxInFlightObserved", "MaxInFlightObserved"),
|
|
10268
|
+
currentInFlight: pickAliasNumber(source, "currentInFlight", "CurrentInFlight"),
|
|
10269
|
+
segments: pickAliasArray(source, "segments", "Segments").map(normalizeSchedulerSegment)
|
|
10270
|
+
};
|
|
10271
|
+
}
|
|
10272
|
+
function normalizeObservationDeliveryStats(value) {
|
|
10273
|
+
const source = asAliasRecord(value);
|
|
10274
|
+
return attachAliasMap({
|
|
10275
|
+
lastBatchSequence64: pickAliasString(source, "lastBatchSequence64", "LastBatchSequence64"),
|
|
10276
|
+
capturedCount64: pickAliasString(source, "capturedCount64", "CapturedCount64"),
|
|
10277
|
+
deliveredCount64: pickAliasString(source, "deliveredCount64", "DeliveredCount64"),
|
|
10278
|
+
droppedBufferCount64: pickAliasString(source, "droppedBufferCount64", "DroppedBufferCount64"),
|
|
10279
|
+
droppedSinkCount64: pickAliasString(source, "droppedSinkCount64", "DroppedSinkCount64")
|
|
10280
|
+
}, {
|
|
10281
|
+
LastBatchSequence64: "lastBatchSequence64",
|
|
10282
|
+
CapturedCount64: "capturedCount64",
|
|
10283
|
+
DeliveredCount64: "deliveredCount64",
|
|
10284
|
+
DroppedBufferCount64: "droppedBufferCount64",
|
|
10285
|
+
DroppedSinkCount64: "droppedSinkCount64"
|
|
10286
|
+
});
|
|
10287
|
+
}
|
|
10288
|
+
function validateLoadEngineV2Options(contractVersion, maxInFlight) {
|
|
10289
|
+
if (contractVersion !== undefined && contractVersion !== 1 && contractVersion !== 2) {
|
|
10290
|
+
throw new RangeError("Load engine contract version must be either 1 or 2.");
|
|
10291
|
+
}
|
|
10292
|
+
if (maxInFlight !== undefined) {
|
|
10293
|
+
validateV2MaxInFlight(contractVersion, maxInFlight);
|
|
10294
|
+
}
|
|
10295
|
+
}
|
|
10296
|
+
function validateV2MaxInFlight(contractVersion, maxInFlight) {
|
|
10297
|
+
if (contractVersion !== 2) {
|
|
10298
|
+
throw new Error("MaxInFlight is available only when Load Engine V2 is selected.");
|
|
10299
|
+
}
|
|
10300
|
+
if (!Number.isSafeInteger(maxInFlight) || maxInFlight < 1 || maxInFlight > 1000000) {
|
|
10301
|
+
throw new RangeError("MaxInFlight must be an integer from 1 through 1000000.");
|
|
10302
|
+
}
|
|
10303
|
+
}
|
|
8053
10304
|
function normalizeRunnerOptionCollectionShapes(options) {
|
|
8054
10305
|
assertNoDisableLicenseEnforcementOption(options, "LoadStrikeRunner");
|
|
8055
10306
|
const normalized = {
|
|
@@ -8057,10 +10308,13 @@ function normalizeRunnerOptionCollectionShapes(options) {
|
|
|
8057
10308
|
targetScenarios: normalizeOptionalStringArray(options.targetScenarios),
|
|
8058
10309
|
agentTargetScenarios: normalizeOptionalStringArray(options.agentTargetScenarios),
|
|
8059
10310
|
coordinatorTargetScenarios: normalizeOptionalStringArray(options.coordinatorTargetScenarios),
|
|
10311
|
+
expectedAgentIds: normalizeOptionalStringArray(options.expectedAgentIds),
|
|
8060
10312
|
reportFormats: normalizeOptionalReportFormats(options.reportFormats)
|
|
8061
10313
|
};
|
|
8062
10314
|
validateNamedReportingSinks(normalized.reportingSinks ?? []);
|
|
8063
10315
|
validateNamedWorkerPlugins(normalized.workerPlugins ?? []);
|
|
10316
|
+
validateLoadEngineV2Options(normalized.loadEngineContractVersion, normalized.maxInFlight);
|
|
10317
|
+
validateIterationObservationSettings(resolveIterationObservationSettings(normalized));
|
|
8064
10318
|
return normalized;
|
|
8065
10319
|
}
|
|
8066
10320
|
function normalizedRuntimePolicyErrorMode(value) {
|
|
@@ -8124,6 +10378,7 @@ function extractContextOverridesFromConfig(config) {
|
|
|
8124
10378
|
setString("ReportFileName", "ReportFileName", "LoadStrike:ReportFileName");
|
|
8125
10379
|
setString("ClusterId", "ClusterId", "LoadStrike:ClusterId");
|
|
8126
10380
|
setString("AgentGroup", "AgentGroup", "LoadStrike:AgentGroup");
|
|
10381
|
+
setString("AgentId", "AgentId", "LoadStrike:AgentId");
|
|
8127
10382
|
setString("NatsServerUrl", "NatsServerUrl", "LoadStrike:NatsServerUrl");
|
|
8128
10383
|
setString("RunnerKey", "RunnerKey", "LoadStrike:RunnerKey");
|
|
8129
10384
|
setString("RuntimePolicyErrorMode", "RuntimePolicyErrorMode", "LoadStrike:RuntimePolicyErrorMode");
|
|
@@ -8153,6 +10408,13 @@ function extractContextOverridesFromConfig(config) {
|
|
|
8153
10408
|
}
|
|
8154
10409
|
}
|
|
8155
10410
|
setPositiveNumber("ReportingIntervalSeconds", "ReportingIntervalSeconds", "LoadStrike:ReportingIntervalSeconds");
|
|
10411
|
+
setPositiveNumber("IterationObservationFlushIntervalSeconds", "IterationObservationFlushInterval", "IterationObservationFlushIntervalSeconds", "LoadStrike:IterationObservationFlushInterval");
|
|
10412
|
+
setPositiveNumber("MaxIterationObservationBufferBytes", "MaxIterationObservationBufferBytes", "LoadStrike:MaxIterationObservationBufferBytes");
|
|
10413
|
+
setPositiveNumber("MaxIterationObservationsPerBatch", "MaxIterationObservationsPerBatch", "LoadStrike:MaxIterationObservationsPerBatch");
|
|
10414
|
+
setPositiveNumber("MaxIterationObservationBatchBytes", "MaxIterationObservationBatchBytes", "LoadStrike:MaxIterationObservationBatchBytes");
|
|
10415
|
+
setPositiveNumber("IterationObservationSinkQueueDepth", "IterationObservationSinkQueueDepth", "LoadStrike:IterationObservationSinkQueueDepth");
|
|
10416
|
+
setPositiveNumber("IterationObservationSinkParallelism", "IterationObservationSinkParallelism", "LoadStrike:IterationObservationSinkParallelism");
|
|
10417
|
+
setPositiveNumber("IterationObservationDrainTimeoutSeconds", "IterationObservationDrainTimeout", "IterationObservationDrainTimeoutSeconds", "LoadStrike:IterationObservationDrainTimeout");
|
|
8156
10418
|
setPositiveNumber("ScenarioCompletionTimeoutSeconds", "ScenarioCompletionTimeoutSeconds", "LoadStrike:ScenarioCompletionTimeoutSeconds");
|
|
8157
10419
|
setPositiveNumber("ClusterCommandTimeoutSeconds", "ClusterCommandTimeoutSeconds", "LoadStrike:ClusterCommandTimeoutSeconds");
|
|
8158
10420
|
setPositiveNumber("LicenseValidationTimeoutSeconds", "LicenseValidationTimeoutSeconds", "LoadStrike:LicenseValidation:TimeoutSeconds");
|
|
@@ -8160,6 +10422,14 @@ function extractContextOverridesFromConfig(config) {
|
|
|
8160
10422
|
if (reportingIntervalMs > 0) {
|
|
8161
10423
|
patch.ReportingIntervalSeconds = reportingIntervalMs / 1000;
|
|
8162
10424
|
}
|
|
10425
|
+
const observationFlushIntervalMs = toNumber(pick("IterationObservationFlushIntervalMs", "LoadStrike:IterationObservationFlushIntervalMs"));
|
|
10426
|
+
if (observationFlushIntervalMs > 0) {
|
|
10427
|
+
patch.IterationObservationFlushIntervalSeconds = observationFlushIntervalMs / 1000;
|
|
10428
|
+
}
|
|
10429
|
+
const observationDrainTimeoutMs = toNumber(pick("IterationObservationDrainTimeoutMs", "LoadStrike:IterationObservationDrainTimeoutMs"));
|
|
10430
|
+
if (observationDrainTimeoutMs > 0) {
|
|
10431
|
+
patch.IterationObservationDrainTimeoutSeconds = observationDrainTimeoutMs / 1000;
|
|
10432
|
+
}
|
|
8163
10433
|
const scenarioCompletionTimeoutMs = toNumber(pick("ScenarioCompletionTimeoutMs", "LoadStrike:ScenarioCompletionTimeoutMs"));
|
|
8164
10434
|
if (scenarioCompletionTimeoutMs > 0) {
|
|
8165
10435
|
patch.ScenarioCompletionTimeoutSeconds = scenarioCompletionTimeoutMs / 1000;
|
|
@@ -8204,6 +10474,10 @@ function extractContextOverridesFromConfig(config) {
|
|
|
8204
10474
|
if (agentTargetScenarios.length) {
|
|
8205
10475
|
patch.AgentTargetScenarios = agentTargetScenarios;
|
|
8206
10476
|
}
|
|
10477
|
+
const expectedAgentIds = normalizeStringArray(pick("ExpectedAgentIds", "LoadStrike:ExpectedAgentIds"));
|
|
10478
|
+
if (expectedAgentIds.length) {
|
|
10479
|
+
patch.ExpectedAgentIds = expectedAgentIds;
|
|
10480
|
+
}
|
|
8207
10481
|
const coordinatorTargetScenarios = normalizeStringArray(pick("CoordinatorTargetScenarios", "LoadStrike:CoordinatorTargetScenarios"));
|
|
8208
10482
|
if (coordinatorTargetScenarios.length) {
|
|
8209
10483
|
patch.CoordinatorTargetScenarios = coordinatorTargetScenarios;
|
|
@@ -8266,10 +10540,14 @@ function toRunContext(options) {
|
|
|
8266
10540
|
const normalized = normalizeRunnerOptionCollectionShapes(options);
|
|
8267
10541
|
return {
|
|
8268
10542
|
ConsoleMetricsEnabled: normalized.displayConsoleMetrics,
|
|
10543
|
+
LoadEngineContractVersion: normalized.loadEngineContractVersion,
|
|
10544
|
+
MaxInFlight: normalized.maxInFlight,
|
|
8269
10545
|
NodeType: normalized.nodeType,
|
|
8270
10546
|
LocalDevClusterEnabled: normalized.localDevClusterEnabled,
|
|
8271
10547
|
AgentGroup: normalized.agentGroup,
|
|
8272
10548
|
AgentsCount: normalized.agentsCount,
|
|
10549
|
+
AgentId: normalized.agentId,
|
|
10550
|
+
ExpectedAgentIds: normalized.expectedAgentIds,
|
|
8273
10551
|
TargetScenarios: normalized.targetScenarios,
|
|
8274
10552
|
AgentTargetScenarios: normalized.agentTargetScenarios,
|
|
8275
10553
|
CoordinatorTargetScenarios: normalized.coordinatorTargetScenarios,
|
|
@@ -8288,6 +10566,13 @@ function toRunContext(options) {
|
|
|
8288
10566
|
ReportFolderPath: normalized.reportFolderPath,
|
|
8289
10567
|
ReportFormats: normalized.reportFormats,
|
|
8290
10568
|
ReportingIntervalSeconds: normalized.reportingIntervalSeconds,
|
|
10569
|
+
IterationObservationFlushIntervalSeconds: normalized.iterationObservationFlushIntervalSeconds,
|
|
10570
|
+
MaxIterationObservationBufferBytes: normalized.maxIterationObservationBufferBytes,
|
|
10571
|
+
MaxIterationObservationsPerBatch: normalized.maxIterationObservationsPerBatch,
|
|
10572
|
+
MaxIterationObservationBatchBytes: normalized.maxIterationObservationBatchBytes,
|
|
10573
|
+
IterationObservationSinkQueueDepth: normalized.iterationObservationSinkQueueDepth,
|
|
10574
|
+
IterationObservationSinkParallelism: normalized.iterationObservationSinkParallelism,
|
|
10575
|
+
IterationObservationDrainTimeoutSeconds: normalized.iterationObservationDrainTimeoutSeconds,
|
|
8291
10576
|
MinimumLogLevel: normalized.minimumLogLevel,
|
|
8292
10577
|
LoggerConfig: normalized.loggerConfig,
|
|
8293
10578
|
ReportingSinks: normalized.reportingSinks,
|
|
@@ -8302,7 +10587,9 @@ function toRunContext(options) {
|
|
|
8302
10587
|
CustomSettings: normalized.customSettings,
|
|
8303
10588
|
GlobalCustomSettings: normalized.globalCustomSettings,
|
|
8304
10589
|
AgentExecutionToken: normalized.agentExecutionToken,
|
|
8305
|
-
AgentCommandId: normalized.agentCommandId
|
|
10590
|
+
AgentCommandId: normalized.agentCommandId,
|
|
10591
|
+
ClusterShardIndex: normalized.clusterShardIndex,
|
|
10592
|
+
ClusterShardCount: normalized.clusterShardCount
|
|
8306
10593
|
};
|
|
8307
10594
|
}
|
|
8308
10595
|
class RuntimePolicyCallbackError extends Error {
|
|
@@ -8390,11 +10677,15 @@ function looksLikeRunContext(value) {
|
|
|
8390
10677
|
const keys = new Set(Object.keys(value));
|
|
8391
10678
|
return [
|
|
8392
10679
|
"ConsoleMetricsEnabled",
|
|
10680
|
+
"LoadEngineContractVersion",
|
|
10681
|
+
"MaxInFlight",
|
|
8393
10682
|
"LocalDevClusterEnabled",
|
|
8394
10683
|
"ConfigPath",
|
|
8395
10684
|
"InfraConfigPath",
|
|
8396
10685
|
"AgentGroup",
|
|
8397
10686
|
"AgentsCount",
|
|
10687
|
+
"AgentId",
|
|
10688
|
+
"ExpectedAgentIds",
|
|
8398
10689
|
"AgentTargetScenarios",
|
|
8399
10690
|
"ClusterId",
|
|
8400
10691
|
"CoordinatorTargetScenarios",
|
|
@@ -8409,6 +10700,13 @@ function looksLikeRunContext(value) {
|
|
|
8409
10700
|
"ReportFolderPath",
|
|
8410
10701
|
"ReportFormats",
|
|
8411
10702
|
"ReportingIntervalSeconds",
|
|
10703
|
+
"IterationObservationFlushIntervalSeconds",
|
|
10704
|
+
"MaxIterationObservationBufferBytes",
|
|
10705
|
+
"MaxIterationObservationsPerBatch",
|
|
10706
|
+
"MaxIterationObservationBatchBytes",
|
|
10707
|
+
"IterationObservationSinkQueueDepth",
|
|
10708
|
+
"IterationObservationSinkParallelism",
|
|
10709
|
+
"IterationObservationDrainTimeoutSeconds",
|
|
8412
10710
|
"ReportingSinks",
|
|
8413
10711
|
"SinkRetryCount",
|
|
8414
10712
|
"SinkRetryBackoffMs",
|
|
@@ -8550,6 +10848,18 @@ function resolveSinkSaveRunResult(sink) {
|
|
|
8550
10848
|
? method.bind(sink)
|
|
8551
10849
|
: undefined;
|
|
8552
10850
|
}
|
|
10851
|
+
function resolveSinkSaveIterationBatch(sink) {
|
|
10852
|
+
const method = sink.saveIterationBatch ?? sink.SaveIterationBatch;
|
|
10853
|
+
return typeof method === "function"
|
|
10854
|
+
? method.bind(sink)
|
|
10855
|
+
: undefined;
|
|
10856
|
+
}
|
|
10857
|
+
function resolveSinkCompleteIterationObservationStream(sink) {
|
|
10858
|
+
const method = sink.completeIterationObservationStream ?? sink.CompleteIterationObservationStream;
|
|
10859
|
+
return typeof method === "function"
|
|
10860
|
+
? method.bind(sink)
|
|
10861
|
+
: undefined;
|
|
10862
|
+
}
|
|
8553
10863
|
function resolveSinkStop(sink) {
|
|
8554
10864
|
const method = sink.stop ?? sink.Stop;
|
|
8555
10865
|
return typeof method === "function"
|
|
@@ -8967,13 +11277,17 @@ export const __loadstrikeTestExports = {
|
|
|
8967
11277
|
ManagedScenarioTrackingRuntime,
|
|
8968
11278
|
ScenarioStatsAccumulator,
|
|
8969
11279
|
StepStatsAccumulator,
|
|
11280
|
+
planRuntimeClusterAssignments,
|
|
8970
11281
|
TrackingFieldSelector,
|
|
8971
11282
|
addCorrelationRow,
|
|
8972
11283
|
addFailedResponseRow,
|
|
8973
11284
|
aggregateNodeStats,
|
|
11285
|
+
aggregateMeasurementStats,
|
|
8974
11286
|
asRecord,
|
|
8975
11287
|
assertNoDisableLicenseEnforcementOption,
|
|
8976
11288
|
buildEmptyNodeStats,
|
|
11289
|
+
buildRuntimeLoadEngineV2HistogramArtifact,
|
|
11290
|
+
buildRuntimeLoadEngineV2Plan,
|
|
8977
11291
|
buildGroupedCorrelationRows,
|
|
8978
11292
|
buildMeasurementPlaceholder,
|
|
8979
11293
|
buildRichHtmlReport,
|
|
@@ -9028,6 +11342,7 @@ export const __loadstrikeTestExports = {
|
|
|
9028
11342
|
parseStrictBooleanToken,
|
|
9029
11343
|
percentile,
|
|
9030
11344
|
pickOptionalTrackingSelectorString,
|
|
11345
|
+
pickTrackingValue,
|
|
9031
11346
|
pickTrackingNumber,
|
|
9032
11347
|
produceOrConsumeTrackingPayload,
|
|
9033
11348
|
readConfiguredSinkName,
|
|
@@ -9046,7 +11361,9 @@ export const __loadstrikeTestExports = {
|
|
|
9046
11361
|
resolveSinkName,
|
|
9047
11362
|
resolveSinkSaveRealtimeMetrics,
|
|
9048
11363
|
resolveSinkSaveRealtimeStats,
|
|
11364
|
+
resolveSinkSaveIterationBatch,
|
|
9049
11365
|
resolveSinkSaveRunResult,
|
|
11366
|
+
resolveSinkCompleteIterationObservationStream,
|
|
9050
11367
|
resolveSinkStart,
|
|
9051
11368
|
resolveSinkStop,
|
|
9052
11369
|
resolveWorkerPlugins,
|
|
@@ -9067,6 +11384,7 @@ export const __loadstrikeTestExports = {
|
|
|
9067
11384
|
tryParseNodeTypeToken,
|
|
9068
11385
|
tryReadConfigValue,
|
|
9069
11386
|
validateNamedReportingSinks,
|
|
11387
|
+
validateLoadEngineV2ScenarioFeatures,
|
|
9070
11388
|
validateRegisteredScenarios,
|
|
9071
11389
|
validateRuntimeRedisCorrelationStoreConfiguration,
|
|
9072
11390
|
validateRuntimeTrackingConfiguration,
|