@mark1russell7/harness 0.12.4 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/build/{cell-jEkGFgGL.d.ts → cell-D2QNOxE8.d.ts} +21 -2
- package/build/index.d.ts +248 -200
- package/build/index.js +91 -15
- package/build/index.js.map +1 -1
- package/build/schema.d.ts +1 -1
- package/image/native/src/bin/client.rs +7 -8
- package/image/native/src/bin/inject.rs +11 -10
- package/image/node/client-ioredis.mjs +4 -4
- package/image/node/client-sentinel.mjs +1 -1
- package/image/node-http/lines.mjs +2 -2
- package/package.json +1 -1
package/build/index.js
CHANGED
|
@@ -35,6 +35,37 @@ const INJECT = NOISE;
|
|
|
35
35
|
/** This function gives the δ_abs of a cell: its own, or `NOISE` for a cell that gives none. */
|
|
36
36
|
const noiseOf = (cell) => cell.noise ?? NOISE;
|
|
37
37
|
/**
|
|
38
|
+
* This function gives the pairing window of a run (SYS-25). On a runner of GitHub, the jitter of
|
|
39
|
+
* a program can be longer than the δ_abs of 1 ms. Thus each run measures it before its fault
|
|
40
|
+
* (`Run.jitter`), and its own measurement widens its window. A run without a jitter record is
|
|
41
|
+
* from before SYS-25. It reads as a bound of null, thus its window is δ_abs. The decision of
|
|
42
|
+
* 2026-10-19 keeps the rule of one miss: a pair that a window misses is still a finding.
|
|
43
|
+
*/
|
|
44
|
+
const pairingOf = (cell, run) => {
|
|
45
|
+
const noise = noiseOf(cell);
|
|
46
|
+
const bound = run.jitter?.bound ?? null;
|
|
47
|
+
return {
|
|
48
|
+
bound,
|
|
49
|
+
delta: bound !== null && valueIn(bound, "s") > valueIn(noise, "s") ? bound : noise
|
|
50
|
+
};
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* This function sums the pairings of the measurements of a cell. It gives the longest bound and
|
|
54
|
+
* the longest window. It also counts the pairings whose window is longer than δ_abs (`noise`). It
|
|
55
|
+
* is null when no measurement has a pairing.
|
|
56
|
+
*/
|
|
57
|
+
const pairingsOf = (runs, noise = NOISE) => {
|
|
58
|
+
const pairings = runs.flatMap((run) => run.pairing === void 0 ? [] : [run.pairing]);
|
|
59
|
+
if (pairings.length === 0) return null;
|
|
60
|
+
const longest = (values) => values.reduce((most, each) => valueIn(each, "s") > valueIn(most, "s") ? each : most);
|
|
61
|
+
const bounds = pairings.flatMap((pairing) => pairing.bound === null ? [] : [pairing.bound]);
|
|
62
|
+
return {
|
|
63
|
+
bound: bounds.length === 0 ? null : longest(bounds),
|
|
64
|
+
delta: longest(pairings.map((pairing) => pairing.delta)),
|
|
65
|
+
widened: pairings.filter((pairing) => valueIn(pairing.delta, "s") > valueIn(noise, "s")).length
|
|
66
|
+
};
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
38
69
|
* This function tells if a timer firing came late enough to void a run. A firing two jiffies or
|
|
39
70
|
* more behind its wheel slot came late because the host stopped a vCPU. With the HZ of the run,
|
|
40
71
|
* the rule is in time: the delay must also reach δ_abs, because each band already includes that
|
|
@@ -483,12 +514,12 @@ const verdictOfBand = (prediction, runs, delta = NOISE) => {
|
|
|
483
514
|
* These are the runs 39390222333 and 39903020657 of the client repository.
|
|
484
515
|
*
|
|
485
516
|
* The bound is 1.5 ms: the maximum and a margin of 0.48 ms. The pairing of a drop with its act
|
|
486
|
-
* uses it, or
|
|
487
|
-
* catalog of the cells). Read the records again after a change to the subject, to Node or to
|
|
517
|
+
* uses it, or the pairing window of the run when that window is longer. The band of a drop adds
|
|
518
|
+
* it to its high bound (the WS catalog of the cells). Read the records again after a change to the subject, to Node or to
|
|
488
519
|
* the runners.
|
|
489
520
|
*/
|
|
490
521
|
const DROP_HANDLING = quantity(1.5, "ms");
|
|
491
|
-
/** This function gives the window of the act of a drop
|
|
522
|
+
/** This function gives the window of the act of a drop, in seconds: `DROP_HANDLING`, or the pairing window (`delta`) when it is longer. */
|
|
492
523
|
const dropWindow = (delta) => Math.max(valueIn(DROP_HANDLING, "s"), valueIn(delta, "s"));
|
|
493
524
|
/** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */
|
|
494
525
|
const between$3 = (from, to) => quantity(Math.round((to - from) / 1e3) / 1e6, "s");
|
|
@@ -563,9 +594,9 @@ const closeOrExit = (line, left, exit, delta = NOISE) => {
|
|
|
563
594
|
};
|
|
564
595
|
/**
|
|
565
596
|
* This function tells how a drop differs from the way that its prediction gives, or gives null.
|
|
566
|
-
* A close closes the socket at once
|
|
567
|
-
*
|
|
568
|
-
* the socket at the drop. A peer that answers the close frame can close it, and that is no finding.
|
|
597
|
+
* A close closes the socket at once, in the handling time of the program (`DROP_HANDLING`). If
|
|
598
|
+
* the pairing window of the run (`delta`) is longer, that window is the limit. A close frame is a
|
|
599
|
+
* write, and the program does not close the socket at the drop. A peer that answers the close frame can close it, and that is no finding.
|
|
569
600
|
*/
|
|
570
601
|
const dropFinding = (expected, drop, left, delta = NOISE, exited) => {
|
|
571
602
|
const noise = dropWindow(delta);
|
|
@@ -578,7 +609,25 @@ const dropFinding = (expected, drop, left, delta = NOISE, exited) => {
|
|
|
578
609
|
return soon ? "the program closed its socket at the drop, where a close frame leaves it open" : null;
|
|
579
610
|
};
|
|
580
611
|
const isWrite = (record) => record.ev === "tx";
|
|
581
|
-
/**
|
|
612
|
+
/**
|
|
613
|
+
* This function measures the jitter of a program from its acts before `until`, the start of the
|
|
614
|
+
* fault. An act is a `write` line or a `ping` line on the connection. Its delay goes from the line
|
|
615
|
+
* to the first write of data on the socket after it, before the next act and before `until`. An
|
|
616
|
+
* act with no such write does not count. The bound is the longest delay, to the microsecond.
|
|
617
|
+
*/
|
|
618
|
+
const jitterOfLines = (lines, socket, local, until) => {
|
|
619
|
+
const acts = lines.filter((line) => (line.ev === "write" || line.ev === "ping") && line.t < until && onConnection(line, local));
|
|
620
|
+
const delays = acts.flatMap((line, at) => {
|
|
621
|
+
const next = Math.min(acts[at + 1]?.t ?? until, until);
|
|
622
|
+
const write = socket.find((record) => isWrite(record) && (record.seg?.len.value ?? 0) > 0 && record.mono >= line.t && record.mono < next);
|
|
623
|
+
return write === void 0 ? [] : [write.mono - line.t];
|
|
624
|
+
});
|
|
625
|
+
return {
|
|
626
|
+
acts: delays.length,
|
|
627
|
+
bound: delays.length === 0 ? null : quantity(Math.round(Math.max(...delays) / 1e3) / 1e3, "ms")
|
|
628
|
+
};
|
|
629
|
+
};
|
|
630
|
+
/** This function gives each ping of the program on the connection, with the kernel's stamp of its write in the pairing window (`delta`). */
|
|
582
631
|
const pingsOf = (lines, socket, local, delta = NOISE) => lines.filter((line) => line.ev === "ping" && onConnection(line, local)).map((line) => eventOf(socket, line, isWrite, delta));
|
|
583
632
|
/**
|
|
584
633
|
* This function gives the first ping that got no answer: no segment with data reached the socket
|
|
@@ -589,7 +638,7 @@ const firstUnanswered = (pings, socket, until = Number.POSITIVE_INFINITY) => pin
|
|
|
589
638
|
* This function gives the first drop of the connection by its program. The kernel stamps it as the
|
|
590
639
|
* write of a close frame. Or it stamps the change of the socket out of ESTABLISHED: a close or an
|
|
591
640
|
* abort. That stamp comes in the handling time of the program after the drop line
|
|
592
|
-
* (`DROP_HANDLING`)
|
|
641
|
+
* (`DROP_HANDLING`). If the pairing window of the run (`delta`) is longer, that window is the limit.
|
|
593
642
|
*/
|
|
594
643
|
const dropOf = (lines, socket, local, delta = NOISE) => {
|
|
595
644
|
const line = lines.find((each) => each.ev === "drop" && onConnection(each, local));
|
|
@@ -1680,7 +1729,11 @@ const crossingsOf = (run, hop, toClient, toTarget) => {
|
|
|
1680
1729
|
* than the one that the prediction gives is a finding.
|
|
1681
1730
|
*/
|
|
1682
1731
|
const measure = (cell, run) => {
|
|
1683
|
-
const
|
|
1732
|
+
const socketRead = measureSocket(cell, run);
|
|
1733
|
+
const read = run.jitter === void 0 ? socketRead : {
|
|
1734
|
+
...socketRead,
|
|
1735
|
+
pairing: pairingOf(cell, run)
|
|
1736
|
+
};
|
|
1684
1737
|
const expected = cell.prediction.request;
|
|
1685
1738
|
if (cell.probe.http === void 0 && expected === void 0) return read;
|
|
1686
1739
|
const request = requestOf(run.events);
|
|
@@ -1696,6 +1749,24 @@ const measure = (cell, run) => {
|
|
|
1696
1749
|
finding
|
|
1697
1750
|
};
|
|
1698
1751
|
};
|
|
1752
|
+
/**
|
|
1753
|
+
* This function measures the jitter of the program of the observer in a run, from its records
|
|
1754
|
+
* (`Jitter`). The acts count up to the start of the fault, or up to the reply of the client when
|
|
1755
|
+
* the fault has no stamp. A run with no lines of a program, or no socket of the observer, has no act.
|
|
1756
|
+
*/
|
|
1757
|
+
const jitterOf = (cell, run) => {
|
|
1758
|
+
const lines = programLines(cell, run);
|
|
1759
|
+
const ready = run.events.find((event) => event.ev === "ready");
|
|
1760
|
+
const place = lines.length === 0 || ready === void 0 ? void 0 : socketOf(cell, run);
|
|
1761
|
+
if (ready === void 0 || place === void 0) return {
|
|
1762
|
+
acts: 0,
|
|
1763
|
+
bound: null
|
|
1764
|
+
};
|
|
1765
|
+
const socket = run.kernel.filter((record) => record.netns === place.netns && record.cookie === place.cookie);
|
|
1766
|
+
const local = run.events.find((event) => event.ev === "connected")?.["local"];
|
|
1767
|
+
const named = cell.observer === "server" ? socket.find((record) => record.dst !== void 0)?.dst ?? local : local;
|
|
1768
|
+
return jitterOfLines(lines, socket, named, run.fault?.t0 ?? ready.t);
|
|
1769
|
+
};
|
|
1699
1770
|
/** This function tells how a request reading differs from its prediction, or gives null. */
|
|
1700
1771
|
const requestFinding = (expected, read) => {
|
|
1701
1772
|
if (read === null) return `the second request never went out, where it should meet: ${requestWords(expected)}`;
|
|
@@ -1847,7 +1918,8 @@ const measureSocket = (cell, run) => {
|
|
|
1847
1918
|
const lines = programLines(cell, run);
|
|
1848
1919
|
const named = server ? socket.find((record) => record.dst !== void 0)?.dst ?? local : local;
|
|
1849
1920
|
const kernelEnded = reported?.mono ?? closeWait?.mono ?? own?.mono ?? seen?.t;
|
|
1850
|
-
const
|
|
1921
|
+
const paired = pairingOf(cell, run).delta;
|
|
1922
|
+
const first = dropOf(lines, socket, named, paired);
|
|
1851
1923
|
const drop = first !== void 0 && (kernelEnded === void 0 || first.at <= kernelEnded) ? first : void 0;
|
|
1852
1924
|
const dropped = drop !== void 0;
|
|
1853
1925
|
const outcome = dropped ? { kind: "dropped" } : reported !== void 0 ? {
|
|
@@ -1860,7 +1932,7 @@ const measureSocket = (cell, run) => {
|
|
|
1860
1932
|
const ended = drop?.at ?? kernelEnded;
|
|
1861
1933
|
const noticed = reported !== void 0 && seen !== void 0 && !dropped ? between(reported.mono, seen.t) : null;
|
|
1862
1934
|
const { prediction } = cell;
|
|
1863
|
-
const pings = pingsOf(lines, socket, named,
|
|
1935
|
+
const pings = pingsOf(lines, socket, named, paired);
|
|
1864
1936
|
const before = run.fault?.t1 ?? ready.t;
|
|
1865
1937
|
const referenceAt = (from) => {
|
|
1866
1938
|
if (from === "request-sent") return request.t;
|
|
@@ -2052,7 +2124,7 @@ const measureSocket = (cell, run) => {
|
|
|
2052
2124
|
left: closed,
|
|
2053
2125
|
exited: void 0
|
|
2054
2126
|
} : closeOrExit(drop.line, closed, exitOf(run, cookie), noiseOf(cell));
|
|
2055
|
-
const how = prediction.drop === void 0 || drop === void 0 || left === void 0 ? null : dropFinding(prediction.drop, drop, left,
|
|
2127
|
+
const how = prediction.drop === void 0 || drop === void 0 || left === void 0 ? null : dropFinding(prediction.drop, drop, left, paired, exited);
|
|
2056
2128
|
const found = [
|
|
2057
2129
|
matches ? null : `it ended with ${words}, not ${prediction.errno}`,
|
|
2058
2130
|
late,
|
|
@@ -5581,14 +5653,18 @@ const inEnvironment = async (cell, environment, count, say) => {
|
|
|
5581
5653
|
if (latency !== null) await lab.netem(latency, "target");
|
|
5582
5654
|
await lab.warm();
|
|
5583
5655
|
const manifest = await lab.manifest(await lab.sysctl(cell.sysctls), settings, targetSysctls);
|
|
5656
|
+
const timed = (run) => ({
|
|
5657
|
+
...run,
|
|
5658
|
+
jitter: jitterOf(cell, run)
|
|
5659
|
+
});
|
|
5584
5660
|
const warmup = {
|
|
5585
|
-
...await once(lab, cell, environment, -1, endsEnvironment(cell)),
|
|
5661
|
+
...timed(await once(lab, cell, environment, -1, endsEnvironment(cell))),
|
|
5586
5662
|
warmup: true
|
|
5587
5663
|
};
|
|
5588
5664
|
say(line(cell, warmup, measure(cell, warmup)));
|
|
5589
5665
|
const runs = [];
|
|
5590
5666
|
for (let index = 0; index < count; index += 1) {
|
|
5591
|
-
const run = await once(lab, cell, environment, index);
|
|
5667
|
+
const run = timed(await once(lab, cell, environment, index));
|
|
5592
5668
|
runs.push(run);
|
|
5593
5669
|
say(line(cell, run, measure(cell, run)));
|
|
5594
5670
|
}
|
|
@@ -5740,6 +5816,6 @@ const runCell = async (cell, say = () => {}) => {
|
|
|
5740
5816
|
}
|
|
5741
5817
|
};
|
|
5742
5818
|
//#endregion
|
|
5743
|
-
export { CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DROP_HANDLING, GIVES_AWAY, INJECT, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, callOf, clockOffsetOf, closeOrExit, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, goHttpImage, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, judge, lagsOf, latestEndOf, leftAfter, logNameOf, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, onMonotonic, openClusterLab, openGroupLab, openLab, openReplicaSetLab, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, reconnectFinding, reconnectOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subjectClient, subjectProblems, subscriptionsOf, targetImageOf, targetSourceOf, templated, toolJsonOf, toxicBody, unlockedManifests, unpinnedBases, verdictOf, viewOf, writesOf };
|
|
5819
|
+
export { CLIENT_NAMES, CLUSTER_STAGES, CONVERGE_MS, DROP_HANDLING, GIVES_AWAY, INJECT, KEPT, LINGER_MS, LISTING, MEMBER_MARKS, MONITOR_DOWN, NODE_MARKS, NOISE, PINNED, PUMP, PUMP_SLACK, QUIET_MS, RELATIVE, REPLICA_SET_STAGES, REPOSITORY, SHIPPED_CLIENTS, SHIPPED_NAMES, STAGES, STATUS_SCRIPT, STEPS, STEP_DOWN, amqpClientImage, callOf, clockOffsetOf, closeOrExit, clusterArgs, clusterConf, clusterCutOf, clusterHostsOf, clusterOnce, clusterStagesOf, clusterViewOf, crossingsOf, cutOf, deadlineOf, delayVoids, deviceOf, dropFinding, dropOf, endpointAfter, errorsOf, exitOf, fieldOf, fieldsOf, firstUnanswered, goHttpImage, groupOnce, groupVerdictOf, grpcClientImage, harnessImage, hikariClientImage, hzOf, imageOf, imageProblems, inContext, infoOf, initiateScript, isHopPlace, jitterOf, jitterOfLines, judge, lagsOf, latestEndOf, leftAfter, logNameOf, matches, measure, measureDriver, measureGroup, medianInterval, memberCommand, memberConfig, memberLineOf, mongoClientImage, monotonicOf, nodeClientImage, nodeCommand, nodeHttpImage, nodeLineOf, nodeNameOf, nodesOnMonotonic, noiseOf, onMonotonic, openClusterLab, openGroupLab, openLab, openReplicaSetLab, pairingOf, pairingsOf, peersOf, pgClientImage, pingsOf, plainOf, planOf, programLines, programOf, quantileOf, reconnectFinding, reconnectOf, referenceOf, reorderingOf, replicaSetArgs, replicaSetOnce, replicaSetStagesOf, requestOf, requestWords, restamped, routerImage, routesOf, runCell, sentinelConfig, socketAt, socketOf, socketsOf, spread, spreadOf, stagesOf, statusViewOf, subjectClient, subjectProblems, subscriptionsOf, targetImageOf, targetSourceOf, templated, toolJsonOf, toxicBody, unlockedManifests, unpinnedBases, verdictOf, viewOf, writesOf };
|
|
5744
5820
|
|
|
5745
5821
|
//# sourceMappingURL=index.js.map
|