@mark1russell7/harness 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -12
- package/build/index.d.ts +311 -25
- package/build/index.js +870 -326
- package/build/index.js.map +1 -1
- package/image/native/src/bin/inject.rs +71 -1
- package/image/router/Dockerfile +8 -0
- package/image/router/ctwatch.py +66 -0
- package/image/router/mbx.sh +61 -0
- package/image/router/resetter.py +104 -0
- package/package.json +6 -2
package/build/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Quantity } from "@mark1russell7/quantity";
|
|
2
2
|
import { KernelRecord } from "@mark1russell7/prober";
|
|
3
|
+
import "testcontainers";
|
|
3
4
|
//#region src/cell.d.ts
|
|
4
5
|
/**
|
|
5
6
|
* A fault is a failure that the harness injects, by the name of its manifestation. Each name is
|
|
@@ -19,17 +20,99 @@ import { KernelRecord } from "@mark1russell7/prober";
|
|
|
19
20
|
* - `mtu-blackhole` drops each segment to the observer that is longer than `Cell.mtu` bytes.
|
|
20
21
|
* - `netem` is tc netem on the egress of the target (`Cell.netem`): loss, duplication,
|
|
21
22
|
* reordering or corruption. Its outcome is a distribution.
|
|
23
|
+
*
|
|
24
|
+
* These faults are of a topology (`Cell.topology`):
|
|
25
|
+
* - `middlebox-forget` injects nothing. The router of the topology forgets an idle flow at its own
|
|
26
|
+
* idle timeout, and its form tells what a late packet then meets.
|
|
27
|
+
* - `proxy-idle` injects nothing. The proxy of the topology closes an idle connection at its own
|
|
28
|
+
* timeout.
|
|
29
|
+
* - `toxic` adds a toxic of Toxiproxy (`Cell.toxic`) through its HTTP API.
|
|
30
|
+
* - `server-close` makes the server close the connection of its peer, with the command of the
|
|
31
|
+
* target (`Target.close`): `CLIENT KILL` in Redis.
|
|
32
|
+
*/
|
|
33
|
+
type Fault = "blackhole" | "freeze-process" | "rst" | "stall" | "fin" | "unreachable" | "simplex" | "mtu-blackhole" | "netem" | "middlebox-forget" | "proxy-idle" | "toxic" | "server-close";
|
|
34
|
+
/**
|
|
35
|
+
* A middlebox form tells what a router does with a flow that it forgot, when a late packet comes:
|
|
36
|
+
* - `reset-both` sends a reset to the two ends when it forgets the flow.
|
|
37
|
+
* - `reset-next` answers the next packet from inside with a reset to its sender, and drops a late
|
|
38
|
+
* packet from outside.
|
|
39
|
+
* - `silent-drop` drops each late packet, in each direction.
|
|
40
|
+
* - `recreate` lets a late packet from inside through, and the flow starts again. It drops a late
|
|
41
|
+
* packet from outside.
|
|
42
|
+
* - `none` is a plain NAT router. A late packet from inside starts the flow again, and the stack of
|
|
43
|
+
* the router answers a late packet from outside with a reset.
|
|
44
|
+
*/
|
|
45
|
+
type MiddleboxForm = "reset-both" | "reset-next" | "silent-drop" | "recreate" | "none";
|
|
46
|
+
/**
|
|
47
|
+
* A router is a hop that forwards and translates the addresses of the side of the client (NAT),
|
|
48
|
+
* with conntrack. It is the router image of the harness.
|
|
49
|
+
*/
|
|
50
|
+
interface RouterHop {
|
|
51
|
+
readonly name: string;
|
|
52
|
+
readonly kind: "router";
|
|
53
|
+
readonly form: MiddleboxForm;
|
|
54
|
+
/** This time is the idle timeout of an established flow: nf_conntrack_tcp_timeout_established, in whole seconds. */
|
|
55
|
+
readonly idle: Quantity;
|
|
56
|
+
/**
|
|
57
|
+
* This time is the interval of the listing that evicts each expired flow. Without it, the
|
|
58
|
+
* interval is 0.1 s. The DESTROY event of an expired flow waits for that listing.
|
|
59
|
+
*/
|
|
60
|
+
readonly pump?: Quantity;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* A proxy is a hop that ends each TCP connection from the side of the client and opens its own to
|
|
64
|
+
* the next hop. It is a stock image, pinned by digest.
|
|
65
|
+
*/
|
|
66
|
+
interface ProxyHop {
|
|
67
|
+
readonly name: string;
|
|
68
|
+
readonly kind: "proxy";
|
|
69
|
+
readonly image: string;
|
|
70
|
+
/** This port is the port that the proxy listens on for the side of the client. */
|
|
71
|
+
readonly port: number;
|
|
72
|
+
readonly command?: readonly string[];
|
|
73
|
+
/** These files go into the proxy before it starts, by their path. In their content, {{next}} is the address and port of the next hop. */
|
|
74
|
+
readonly files?: Readonly<Record<string, string>>;
|
|
75
|
+
/** This text is the log line that tells that the proxy serves. */
|
|
76
|
+
readonly ready?: string;
|
|
77
|
+
/** This record is the HTTP API of Toxiproxy: its port, and the name of the proxy that gets the toxics. */
|
|
78
|
+
readonly api?: {
|
|
79
|
+
readonly port: number;
|
|
80
|
+
readonly proxy: string;
|
|
81
|
+
};
|
|
82
|
+
/** These are the sysctls of the namespace of the proxy. A privileged sidecar sets them before each run. */
|
|
83
|
+
readonly sysctls?: Readonly<Record<string, string>>;
|
|
84
|
+
/** These are the settings of the proxy, for the manifest: for example its idle timeouts. */
|
|
85
|
+
readonly settings?: Readonly<Record<string, string>>;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* A hop is one box between the client and the target. Each hop is on two networks: the network
|
|
89
|
+
* of the side of the client, and the network of the side of the target.
|
|
22
90
|
*/
|
|
23
|
-
type
|
|
91
|
+
type Hop = RouterHop | ProxyHop;
|
|
92
|
+
/**
|
|
93
|
+
* A topology is the chain of hops between the client and the target, in sequence from the client.
|
|
94
|
+
* Without hops, the client and the target are a direct pair.
|
|
95
|
+
*/
|
|
96
|
+
interface Topology {
|
|
97
|
+
readonly name: string;
|
|
98
|
+
readonly hops: readonly Hop[];
|
|
99
|
+
}
|
|
100
|
+
/** A toxic is one toxic of Toxiproxy, as its API takes it. Its toxicity is 1. */
|
|
101
|
+
interface Toxic {
|
|
102
|
+
readonly type: "latency" | "timeout" | "reset_peer" | "slow_close" | "bandwidth" | "limit_data" | "slicer";
|
|
103
|
+
readonly stream: "downstream" | "upstream";
|
|
104
|
+
readonly attributes: Readonly<Record<string, number>>;
|
|
105
|
+
}
|
|
24
106
|
/**
|
|
25
107
|
* A reference is the event that the time of a detector counts from: the event that arms the
|
|
26
108
|
* timer in its machine document. Keepalive counts from the last received segment. The give-up of
|
|
27
109
|
* the retransmissions counts from the first retransmission of the RTO timer, not from the tail
|
|
28
110
|
* loss probe one RTO before it. The request itself meets a reset, thus a reset counts from the
|
|
29
111
|
* send of the client. A FIN counts from the change of state of the socket of the peer when its
|
|
30
|
-
* process dies (`peer-closed`).
|
|
112
|
+
* process dies (`peer-closed`). A toxic of Toxiproxy counts from the start of its commit, as the
|
|
113
|
+
* injector stamps it (`fault`).
|
|
31
114
|
*/
|
|
32
|
-
type Reference = "last-received" | "first-retransmission" | "request-sent" | "peer-closed";
|
|
115
|
+
type Reference = "last-received" | "first-retransmission" | "request-sent" | "peer-closed" | "fault";
|
|
33
116
|
/**
|
|
34
117
|
* This prediction tells when a connection moves again after a stall. That occurs at the first
|
|
35
118
|
* firing of its retransmission timer after the stall stops, and its reply comes back one round
|
|
@@ -52,14 +135,25 @@ interface Recovers {
|
|
|
52
135
|
readonly high: Quantity;
|
|
53
136
|
};
|
|
54
137
|
}
|
|
138
|
+
/**
|
|
139
|
+
* This prediction tells when the reply to the request comes back, from the send of the request.
|
|
140
|
+
* For example, the reply comes after the latency of a toxic, or after a forgotten flow starts again.
|
|
141
|
+
*/
|
|
142
|
+
interface Replies {
|
|
143
|
+
readonly low: Quantity;
|
|
144
|
+
readonly high: Quantity;
|
|
145
|
+
}
|
|
55
146
|
/**
|
|
56
147
|
* A prediction is what a cell predicts. Either a detection occurs in an interval [T_lo, T_hi]
|
|
57
148
|
* after the reference event, or nothing detects the fault in a window. Nothing detects a stall
|
|
58
149
|
* that is shorter than the give-up, and the connection recovers from it. The prediction then also
|
|
59
|
-
* tells when.
|
|
150
|
+
* tells when. A silent prediction can also tell when the reply to the request comes back
|
|
151
|
+
* (`replies`).
|
|
60
152
|
*
|
|
61
153
|
* A cell whose outcome is a distribution, for example under netem loss, has no band. Its
|
|
62
|
-
* prediction is `detects: null` with the reason, and its runs are measurements only.
|
|
154
|
+
* prediction is `detects: null` with the reason, and its runs are measurements only. A
|
|
155
|
+
* measurement-only cell that names a reference (`from`) also records the time from it to the end
|
|
156
|
+
* of the connection, or to the last data.
|
|
63
157
|
*/
|
|
64
158
|
type Prediction = {
|
|
65
159
|
readonly detects: true;
|
|
@@ -76,10 +170,12 @@ type Prediction = {
|
|
|
76
170
|
readonly detects: false;
|
|
77
171
|
readonly window: Quantity;
|
|
78
172
|
readonly recovers?: Recovers;
|
|
173
|
+
readonly replies?: Replies;
|
|
79
174
|
} | {
|
|
80
175
|
readonly detects: null;
|
|
81
176
|
readonly window: Quantity;
|
|
82
177
|
readonly why: string;
|
|
178
|
+
readonly from?: Reference;
|
|
83
179
|
};
|
|
84
180
|
/** A target is the server that a cell connects to: a stock image. */
|
|
85
181
|
interface Target {
|
|
@@ -104,16 +200,35 @@ interface Target {
|
|
|
104
200
|
readonly command?: readonly string[];
|
|
105
201
|
/** This command stops the server process in the target with SIGKILL, as argv, for a `fin` fault. */
|
|
106
202
|
readonly kill?: readonly string[];
|
|
203
|
+
/** This command makes the server close the connection of each client, as argv, for a `server-close` fault. */
|
|
204
|
+
readonly close?: readonly string[];
|
|
107
205
|
}
|
|
108
206
|
/** A client is what the runs of a cell use: the TCP client of the harness, or a driver on its socket. */
|
|
109
207
|
type Client = "tcp" | "ioredis";
|
|
208
|
+
/**
|
|
209
|
+
* A client image is a process that a lab starts as the client of each run, in the network
|
|
210
|
+
* namespace of the client. An example is an agent of an application on Node. Its command gets the
|
|
211
|
+
* arguments of the TCP client of the harness. These are the address and the port of the target, then
|
|
212
|
+
* the options of the cell. It writes JSON lines with an `ev` and a `t` (CLOCK_MONOTONIC in
|
|
213
|
+
* nanoseconds) on its standard output, as the client of the harness does.
|
|
214
|
+
*/
|
|
215
|
+
interface ClientImage {
|
|
216
|
+
readonly image: string;
|
|
217
|
+
readonly command: readonly string[];
|
|
218
|
+
}
|
|
110
219
|
/**
|
|
111
220
|
* An observer is the side whose detection a cell measures. The client is one observer. The
|
|
112
|
-
* server is
|
|
221
|
+
* server is another observer. It is the hold of the target on the connection of the client, at
|
|
113
222
|
* its server socket in its namespace. The fault is on the peer of the observer. Thus
|
|
114
223
|
* freeze-process freezes the target for the client, and the client for the server.
|
|
224
|
+
*
|
|
225
|
+
* A proxy of a topology is the third observer: its socket toward the client, or its socket toward
|
|
226
|
+
* the target.
|
|
115
227
|
*/
|
|
116
|
-
type Observer = "client" | "server"
|
|
228
|
+
type Observer = "client" | "server" | {
|
|
229
|
+
readonly hop: string;
|
|
230
|
+
readonly toward: "client" | "target";
|
|
231
|
+
};
|
|
117
232
|
/** A cell is one cell of the matrix: a topology, its settings, a workload, a fault and the prediction. */
|
|
118
233
|
interface Cell {
|
|
119
234
|
readonly name: string;
|
|
@@ -175,6 +290,16 @@ interface Cell {
|
|
|
175
290
|
readonly targetSettings?: Readonly<Record<string, string>>;
|
|
176
291
|
/** These are the sysctls of the network namespace of the target, for a cell on the server side: its tcp_retries2. */
|
|
177
292
|
readonly targetSysctls?: Readonly<Record<string, string>>;
|
|
293
|
+
/** This topology puts hops between the client and the target. Without it, the client and the target are a direct pair. */
|
|
294
|
+
readonly topology?: Topology;
|
|
295
|
+
/** This place is where the fault goes, when it is not the peer of the observer: for example one side of a proxy. */
|
|
296
|
+
readonly faultAt?: FaultPlace;
|
|
297
|
+
/** This time is how long an in-flight client stays idle after the fault holds, before it sends its request. */
|
|
298
|
+
readonly idleBefore?: Quantity;
|
|
299
|
+
/** This toxic is the toxic of a `toxic` fault, which the Toxiproxy hop gets. */
|
|
300
|
+
readonly toxic?: Toxic;
|
|
301
|
+
/** This image replaces the TCP client of the harness for each run. */
|
|
302
|
+
readonly client?: ClientImage;
|
|
178
303
|
/** This count is the number of runs of the cell (11), across new environments (two at least). */
|
|
179
304
|
readonly runs: number;
|
|
180
305
|
readonly environments: number;
|
|
@@ -185,6 +310,21 @@ interface Cell {
|
|
|
185
310
|
*/
|
|
186
311
|
readonly concurrency?: number;
|
|
187
312
|
}
|
|
313
|
+
/**
|
|
314
|
+
* A fault place is the namespace that gets the rules of a fault. "target" and "client" are the two
|
|
315
|
+
* ends. A hop place is a hop, with the side that the rules face. That side is the next hop toward
|
|
316
|
+
* the client, or the next hop toward the target.
|
|
317
|
+
*/
|
|
318
|
+
type FaultPlace = "target" | "client" | {
|
|
319
|
+
readonly hop: string;
|
|
320
|
+
readonly toward: "client" | "target";
|
|
321
|
+
};
|
|
322
|
+
/** A middlebox event is one line of the router: a change of a conntrack entry, or a reset that it sent. `t` is CLOCK_MONOTONIC in nanoseconds. */
|
|
323
|
+
interface MiddleboxEvent {
|
|
324
|
+
readonly ev: string;
|
|
325
|
+
readonly t: number;
|
|
326
|
+
readonly [field: string]: unknown;
|
|
327
|
+
}
|
|
188
328
|
/** A probe event is one line that the client wrote. `t` is CLOCK_MONOTONIC in nanoseconds: an instant, not an amount. */
|
|
189
329
|
interface ProbeEvent {
|
|
190
330
|
readonly ev: string;
|
|
@@ -195,6 +335,11 @@ interface ProbeEvent {
|
|
|
195
335
|
interface Run {
|
|
196
336
|
readonly environment: number;
|
|
197
337
|
readonly index: number;
|
|
338
|
+
/**
|
|
339
|
+
* This flag marks the warm-up of an environment. It occurs before the counted runs, and the
|
|
340
|
+
* verdict does not count it. On a loaded machine, a new environment is sometimes slow at first.
|
|
341
|
+
*/
|
|
342
|
+
readonly warmup?: true;
|
|
198
343
|
readonly events: readonly ProbeEvent[];
|
|
199
344
|
/** This bracket is the time of the injector around the commit of the fault, on the same clock. It is null for a freeze. */
|
|
200
345
|
readonly fault: {
|
|
@@ -217,6 +362,25 @@ interface Run {
|
|
|
217
362
|
* the run void. Null also makes the run void: no clock line came after the run to tell the count.
|
|
218
363
|
*/
|
|
219
364
|
readonly lost: number | null;
|
|
365
|
+
/** These are the addresses and the namespaces of a topology, which the measurement of a run reads. */
|
|
366
|
+
readonly topology?: RunTopology;
|
|
367
|
+
/** These are the lines of the routers of a topology from the start to the end of the run. */
|
|
368
|
+
readonly middlebox?: readonly MiddleboxEvent[];
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* A run topology tells where each socket of a run is, in a topology:
|
|
372
|
+
* - `targetPeer` is the address of the peer of the target, as the target sees it: the address of
|
|
373
|
+
* the last hop.
|
|
374
|
+
* - `hops` are the network namespaces of the hops, by their names, with the address of the peer of
|
|
375
|
+
* each hop on each side.
|
|
376
|
+
*/
|
|
377
|
+
interface RunTopology {
|
|
378
|
+
readonly targetPeer: string;
|
|
379
|
+
readonly hops: Readonly<Record<string, {
|
|
380
|
+
readonly netns: number;
|
|
381
|
+
readonly client: string;
|
|
382
|
+
readonly target: string;
|
|
383
|
+
}>>;
|
|
220
384
|
}
|
|
221
385
|
//#endregion
|
|
222
386
|
//#region src/observe.d.ts
|
|
@@ -264,14 +428,34 @@ interface Measured {
|
|
|
264
428
|
readonly retransmissions: number;
|
|
265
429
|
/** This count is the number of segments that the socket received after the request. */
|
|
266
430
|
readonly acks: number;
|
|
431
|
+
/** This count is the number of bytes that the client read after the request. */
|
|
432
|
+
readonly bytes?: number;
|
|
433
|
+
/**
|
|
434
|
+
* This time is when a router of the topology forgot the flow of the client. It counts from the
|
|
435
|
+
* last segment that the client received before the fault. It is null when no router forgot the flow.
|
|
436
|
+
*/
|
|
437
|
+
readonly forgot?: Quantity | null;
|
|
267
438
|
/** This text is the reason that the run cannot count, or null. A void run is not a pass and not a finding. */
|
|
268
439
|
readonly void: string | null;
|
|
269
440
|
/** This text tells what in the run disagrees with the prediction, other than its timing, or null. */
|
|
270
441
|
readonly finding: string | null;
|
|
271
442
|
}
|
|
443
|
+
/** A socket place is a socket of a run: its namespace and its cookie. */
|
|
444
|
+
interface SocketPlace {
|
|
445
|
+
readonly netns: number;
|
|
446
|
+
readonly cookie: number;
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* This function finds the socket of the observer of a cell in a run:
|
|
450
|
+
* - the client socket, by the address that it connected from, in its namespace.
|
|
451
|
+
* - the socket of the target for the same connection, in its namespace.
|
|
452
|
+
* - a socket of a proxy of the topology, toward the client or toward the target.
|
|
453
|
+
*/
|
|
454
|
+
export declare const socketOf: (cell: Cell, run: Run) => SocketPlace | undefined;
|
|
272
455
|
/**
|
|
273
456
|
* This function reads one run against its cell, from the records of the kernel first.
|
|
274
|
-
* - The socket of the run is the socket that the client connected
|
|
457
|
+
* - The socket of the run is the socket of its observer. It is the socket that the client connected
|
|
458
|
+
* from, in its namespace, or the socket of the target or of a proxy for that connection.
|
|
275
459
|
* - The fault must hold before the event that it must meet. That is before the request goes into
|
|
276
460
|
* it, and before the first keepalive probe. A live peer answers that probe, and the answer
|
|
277
461
|
* starts the idle time again. A run where the fault was late is void. A run whose records the
|
|
@@ -344,6 +528,20 @@ export declare const verdictOf: (prediction: Prediction, runs: readonly Measured
|
|
|
344
528
|
export declare const harnessImage: () => Promise<string>;
|
|
345
529
|
/** This function gives the image of the Node client. */
|
|
346
530
|
export declare const nodeClientImage: () => Promise<string>;
|
|
531
|
+
/** This function gives the image of the router. */
|
|
532
|
+
export declare const routerImage: () => Promise<string>;
|
|
533
|
+
/**
|
|
534
|
+
* This time is the interval of the listing of a router, when its hop does not set one. A listing
|
|
535
|
+
* evicts each expired flow that it walks past, and the DESTROY event of that flow follows.
|
|
536
|
+
*/
|
|
537
|
+
export declare const PUMP: Quantity;
|
|
538
|
+
/**
|
|
539
|
+
* This time is the longest listing of the conntrack table of a router that a prediction assumes.
|
|
540
|
+
* A DESTROY follows an expiry within the interval of the pump and two listings. On 2026-10-15,
|
|
541
|
+
* the slowest of 500 listings in an idle router took 19.4 ms, and the median took 5.5 ms. A
|
|
542
|
+
* longer listing writes a `pump-slow` line, and a run with one near its detection is void.
|
|
543
|
+
*/
|
|
544
|
+
export declare const LISTING: Quantity;
|
|
347
545
|
/**
|
|
348
546
|
* A network fault is a fault that the injector applies as a filter rule. It is a black hole (both
|
|
349
547
|
* directions, or one), or a reset on the next segment. It can also be an ICMP unreachable for each
|
|
@@ -367,6 +565,17 @@ interface FaultOptions {
|
|
|
367
565
|
* because it fires only for a retransmission that went out.
|
|
368
566
|
*/
|
|
369
567
|
type FaultSide = "target" | "client";
|
|
568
|
+
/** A hop of a manifest is one hop of a topology, as the lab started it. */
|
|
569
|
+
interface ManifestHop {
|
|
570
|
+
readonly name: string;
|
|
571
|
+
readonly kind: "router" | "proxy";
|
|
572
|
+
readonly image: string;
|
|
573
|
+
readonly id: string;
|
|
574
|
+
/** These are the settings of the hop: the form and the idle timeout of a router, the files and settings of a proxy. */
|
|
575
|
+
readonly settings: Readonly<Record<string, string>>;
|
|
576
|
+
/** These are the sysctls of the namespace of the hop, as read back after the harness set them. */
|
|
577
|
+
readonly sysctls: Readonly<Record<string, string>>;
|
|
578
|
+
}
|
|
370
579
|
/** A manifest is what a lab records about its machine, its images and its settings. */
|
|
371
580
|
interface Manifest {
|
|
372
581
|
readonly kernel: string;
|
|
@@ -388,18 +597,32 @@ interface Manifest {
|
|
|
388
597
|
readonly id: string;
|
|
389
598
|
readonly packages: string[];
|
|
390
599
|
};
|
|
391
|
-
/** This record is the client of each run: the TCP client of the harness,
|
|
600
|
+
/** This record is the client of each run: the TCP client of the harness, a driver, or an image that the cell gives. */
|
|
392
601
|
readonly client: {
|
|
393
|
-
readonly kind: Client;
|
|
602
|
+
readonly kind: Client | "image";
|
|
394
603
|
readonly image: string;
|
|
395
604
|
};
|
|
396
605
|
/** These are the namespaces that the prober watched, by inode. */
|
|
397
606
|
readonly netns: {
|
|
398
607
|
readonly client: number;
|
|
399
608
|
readonly target: number;
|
|
609
|
+
readonly hops?: Readonly<Record<string, number>>;
|
|
400
610
|
};
|
|
401
611
|
/** These are the sysctls of the client namespace, as read back after the harness set them. */
|
|
402
612
|
readonly sysctls: Readonly<Record<string, string>>;
|
|
613
|
+
/** These are the hops of a topology, in sequence from the client. */
|
|
614
|
+
readonly hops?: readonly ManifestHop[];
|
|
615
|
+
}
|
|
616
|
+
/** These are the JSON lines of a program, as they arrive. */
|
|
617
|
+
interface Lines<T extends {
|
|
618
|
+
readonly ev: string;
|
|
619
|
+
}> {
|
|
620
|
+
readonly seen: T[];
|
|
621
|
+
/** These are the other lines that it wrote, for example an error. They explain a program that stopped its output. */
|
|
622
|
+
readonly other: string[];
|
|
623
|
+
consume: (stream: NodeJS.ReadableStream) => void;
|
|
624
|
+
/** This method gives the first line of one of these kinds, or it rejects after `within` ms. */
|
|
625
|
+
until(kinds: readonly string[], within: number, who: string): Promise<T>;
|
|
403
626
|
}
|
|
404
627
|
/** A probe is the client container of one run. */
|
|
405
628
|
interface Probe {
|
|
@@ -415,14 +638,46 @@ interface Probe {
|
|
|
415
638
|
readonly events: readonly ProbeEvent[];
|
|
416
639
|
stop(): Promise<void>;
|
|
417
640
|
}
|
|
418
|
-
/**
|
|
641
|
+
/** An agent is a container that a lab starts in the namespaces of the target, for example a Node process that signals the server. */
|
|
642
|
+
interface Agent {
|
|
643
|
+
/** These are the JSON lines that it wrote, as they arrive. */
|
|
644
|
+
readonly lines: Lines<{
|
|
645
|
+
readonly ev: string;
|
|
646
|
+
}>;
|
|
647
|
+
/** This method starts a command in the agent, and gives its standard output. */
|
|
648
|
+
exec(argv: readonly string[]): Promise<string>;
|
|
649
|
+
stop(): Promise<void>;
|
|
650
|
+
}
|
|
651
|
+
/** A lab is one environment: the target, the client, the hops of a topology, the injectors and the prober. */
|
|
419
652
|
interface Lab {
|
|
653
|
+
/** This address is the address that the client connects to: the target, or the first proxy of a topology. */
|
|
420
654
|
readonly targetIp: string;
|
|
655
|
+
/** This port is the port that the client connects to. */
|
|
656
|
+
readonly targetPort: number;
|
|
421
657
|
readonly clientIp: string;
|
|
422
658
|
readonly netns: {
|
|
423
659
|
readonly client: number;
|
|
424
660
|
readonly target: number;
|
|
661
|
+
readonly hops?: Readonly<Record<string, number>>;
|
|
425
662
|
};
|
|
663
|
+
/** This record is where each socket of a topology is, or null for a direct pair. */
|
|
664
|
+
readonly topology: RunTopology | null;
|
|
665
|
+
/** These are the lines of the routers of a topology, in sequence, each with the name of its hop. A direct pair has none. */
|
|
666
|
+
readonly middlebox: readonly MiddleboxEvent[];
|
|
667
|
+
/** This method adds a toxic to the Toxiproxy hop of a topology, or removes it with null. The stamps of the injector bracket it. */
|
|
668
|
+
toxic(toxic: Toxic | null): Promise<{
|
|
669
|
+
t0: number;
|
|
670
|
+
t1: number;
|
|
671
|
+
}>;
|
|
672
|
+
/** This method makes the target close the connection of each client, with the command of the target. */
|
|
673
|
+
serverClose(): Promise<void>;
|
|
674
|
+
/**
|
|
675
|
+
* This method starts an agent in the network namespace of the target, and in its process
|
|
676
|
+
* namespace with `pid`. Its command keeps it up, for example `sleep infinity`.
|
|
677
|
+
*/
|
|
678
|
+
agent(image: string, command: readonly string[], options?: {
|
|
679
|
+
readonly pid?: boolean;
|
|
680
|
+
}): Promise<Agent>;
|
|
426
681
|
/** These are all the records that the prober wrote, in sequence. */
|
|
427
682
|
readonly kernel: readonly KernelRecord[];
|
|
428
683
|
/** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */
|
|
@@ -432,16 +687,17 @@ interface Lab {
|
|
|
432
687
|
/** This method sets the sysctls of the target namespace through a privileged sidecar there, and reads them back. */
|
|
433
688
|
targetSysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;
|
|
434
689
|
/**
|
|
435
|
-
* This method applies a fault between the client and the target, from the namespace
|
|
436
|
-
* target. The fault is a black hole, or a reset that answers the next segment of the
|
|
437
|
-
* the
|
|
690
|
+
* This method applies a fault between the client and the target, by default from the namespace
|
|
691
|
+
* of the target. The fault is a black hole, or a reset that answers the next segment of the
|
|
692
|
+
* client. At a proxy of a topology, the rules face the peer of the proxy on one side. The stamps
|
|
693
|
+
* of the injector bracket it.
|
|
438
694
|
*/
|
|
439
|
-
fault(kind: NetworkFault, at?:
|
|
695
|
+
fault(kind: NetworkFault, at?: FaultPlace, options?: FaultOptions): Promise<{
|
|
440
696
|
t0: number;
|
|
441
697
|
t1: number;
|
|
442
698
|
}>;
|
|
443
699
|
/** This method removes a fault. */
|
|
444
|
-
lift(kind: NetworkFault, at?:
|
|
700
|
+
lift(kind: NetworkFault, at?: FaultPlace, options?: FaultOptions): Promise<void>;
|
|
445
701
|
/**
|
|
446
702
|
* This method puts tc netem on the egress of a namespace, for example "delay 20ms" or "loss 5%",
|
|
447
703
|
* or removes it with null. The stamps of the injector bracket it.
|
|
@@ -470,7 +726,7 @@ interface Lab {
|
|
|
470
726
|
* waits for that instant in the VM, thus the length of a stall does not include the latency of
|
|
471
727
|
* a docker exec. Stamps bracket it, as for a fault.
|
|
472
728
|
*/
|
|
473
|
-
heal(at:
|
|
729
|
+
heal(at: FaultPlace, when: number): Promise<{
|
|
474
730
|
t0: number;
|
|
475
731
|
t1: number;
|
|
476
732
|
}>;
|
|
@@ -493,20 +749,46 @@ interface Lab {
|
|
|
493
749
|
manifest(sysctls: Readonly<Record<string, string>>, settings?: Readonly<Record<string, string>>, targetSysctls?: Readonly<Record<string, string>>): Promise<Manifest>;
|
|
494
750
|
close(): Promise<void>;
|
|
495
751
|
}
|
|
752
|
+
/** This function gives the content of a file of a proxy, with the address of the next hop for each {{next}}. */
|
|
753
|
+
export declare const templated: (content: string, next: string) => string;
|
|
754
|
+
/** This function gives the JSON body of a toxic for the API of Toxiproxy. Each toxic of a cell has the name `fault`. */
|
|
755
|
+
export declare const toxicBody: (toxic: Toxic) => string;
|
|
756
|
+
/**
|
|
757
|
+
* This function gives the routes of a chain: for each router, the networks after it, which the
|
|
758
|
+
* element before it reaches through it. Element 0 is the client, and element k is hop k - 1. Network
|
|
759
|
+
* j is the network between element j and element j + 1.
|
|
760
|
+
*/
|
|
761
|
+
export declare const routesOf: (hops: readonly Pick<Hop, "kind">[]) => Array<{
|
|
762
|
+
readonly from: number;
|
|
763
|
+
readonly via: number;
|
|
764
|
+
readonly networks: readonly number[];
|
|
765
|
+
}>;
|
|
766
|
+
/**
|
|
767
|
+
* This function gives the endpoint that an element connects to: the first element after it that
|
|
768
|
+
* is not a router. Element 0 is the client, element k is hop k - 1, and the last element is the target.
|
|
769
|
+
*/
|
|
770
|
+
export declare const endpointAfter: (hops: readonly Pick<Hop, "kind">[], element: number) => number;
|
|
496
771
|
/**
|
|
497
772
|
* This function opens one environment:
|
|
498
|
-
* - a new
|
|
773
|
+
* - a chain of new networks, and the stock target on the last one. A direct pair has one network.
|
|
499
774
|
* - a client that holds its own network namespace, with a privileged sidecar in it for its
|
|
500
|
-
* sysctls. Docker Desktop keeps /proc/sys read-only in a container that operates
|
|
775
|
+
* sysctls and its routes. Docker Desktop keeps /proc/sys read-only in a container that operates
|
|
776
|
+
* without privileges.
|
|
777
|
+
* - each hop of the topology, on the network of each of its sides. A hop is a router, or a stock
|
|
778
|
+
* proxy with an injector sidecar in its namespace.
|
|
501
779
|
* - an injector sidecar in the namespace of the target.
|
|
502
|
-
* - the prober, a privileged container that loads the kernel program and watches the
|
|
503
|
-
*
|
|
780
|
+
* - the prober, a privileged container that loads the kernel program and watches the namespace of
|
|
781
|
+
* the client, of the target and of each hop.
|
|
504
782
|
*
|
|
505
783
|
* The client of each run is its own container in the namespace of the client. It is the TCP
|
|
506
|
-
* client of the harness,
|
|
507
|
-
* lines.
|
|
784
|
+
* client of the harness, a driver from its own image (ioredis on Node), or an image that the
|
|
785
|
+
* caller gives. Each prints the same lines.
|
|
786
|
+
*
|
|
787
|
+
* The client reaches the networks after a router through that router, by a route that its
|
|
788
|
+
* privileged sidecar adds. A router translates the address of the side of the client, thus the
|
|
789
|
+
* target and each later hop see the router as the peer.
|
|
508
790
|
*/
|
|
509
|
-
export declare const openLab: (target: Target, kind?: Client) => Promise<Lab>;
|
|
791
|
+
export declare const openLab: (target: Target, kind?: Client | ClientImage, topology?: Topology) => Promise<Lab>;
|
|
510
792
|
//#endregion
|
|
511
793
|
//#region src/run.d.ts
|
|
512
794
|
/** A cell result is a cell with its manifests, its runs, their measurements and its verdict. */
|
|
@@ -514,6 +796,8 @@ interface CellResult {
|
|
|
514
796
|
readonly cell: Cell;
|
|
515
797
|
readonly manifests: readonly Manifest[];
|
|
516
798
|
readonly runs: readonly Run[];
|
|
799
|
+
/** These are the warm-up runs, one for each environment before its counted runs. The verdict does not count them. */
|
|
800
|
+
readonly warmups: readonly Run[];
|
|
517
801
|
readonly measured: readonly Measured[];
|
|
518
802
|
readonly verdict: Verdict;
|
|
519
803
|
/** These times are when its first environment opened and its last environment closed, on the wall clock. */
|
|
@@ -539,8 +823,10 @@ export declare const spread: (runs: number, environments: number) => number[];
|
|
|
539
823
|
* the cell from 11 runs to 22, and then to 44, each extension in new environments. With a
|
|
540
824
|
* concurrency of more than one, that many environments operate at the same time, each with its
|
|
541
825
|
* own prober. The runs then go across at least as many environments.
|
|
826
|
+
*
|
|
827
|
+
* Each environment does a warm-up first, and the result keeps it apart from the counted runs.
|
|
542
828
|
*/
|
|
543
829
|
export declare const runCell: (cell: Cell, say?: (line: string) => void) => Promise<CellResult>;
|
|
544
830
|
//#endregion
|
|
545
|
-
export type { Cell, CellResult, Client, Decision, Fault, FaultSide, Interval, KernelRecord, Lab, Manifest, Measured, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, Recovers, Reference, Run, Target, Verdict };
|
|
831
|
+
export type { Agent, Cell, CellResult, Client, ClientImage, Decision, Fault, FaultPlace, FaultSide, Hop, Interval, KernelRecord, Lab, Manifest, ManifestHop, Measured, MiddleboxEvent, MiddleboxForm, NetworkFault, Observer, Outcome, Prediction, Probe, ProbeEvent, ProxyHop, Recovers, Reference, Replies, RouterHop, Run, RunTopology, SocketPlace, Target, Topology, Toxic, Verdict };
|
|
546
832
|
//# sourceMappingURL=index.d.ts.map
|