@mark1russell7/harness 0.12.4 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/build/{cell-jEkGFgGL.d.ts → cell-D2QNOxE8.d.ts} +21 -2
- package/build/index.d.ts +248 -200
- package/build/index.js +91 -15
- package/build/index.js.map +1 -1
- package/build/schema.d.ts +1 -1
- package/image/native/src/bin/client.rs +7 -8
- package/image/native/src/bin/inject.rs +11 -10
- package/image/node/client-ioredis.mjs +4 -4
- package/image/node/client-sentinel.mjs +1 -1
- package/image/node-http/lines.mjs +2 -2
- package/package.json +1 -1
package/build/schema.d.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
//! beside the prober's records. It waits in poll(), never holding its socket's lock, so it never
|
|
9
9
|
//! puts off the socket's own keepalive timer.
|
|
10
10
|
//!
|
|
11
|
-
//!
|
|
11
|
+
//! Each mode sends one request and reads its reply first. That reply is the last segment received.
|
|
12
12
|
//! idle: then hold the connection idle until it fails.
|
|
13
13
|
//! inflight: then wait for the file --go to appear, send the request again, and wait. --request
|
|
14
14
|
//! names that second request, a Redis inline command: PING unless it says otherwise. BLPOP with a
|
|
@@ -29,7 +29,7 @@ use std::os::fd::AsRawFd;
|
|
|
29
29
|
use std::path::Path;
|
|
30
30
|
use std::time::Duration;
|
|
31
31
|
|
|
32
|
-
/// a Redis inline command
|
|
32
|
+
/// a Redis inline command. Any line-based server echoes it or gives an error, and that is a reply too
|
|
33
33
|
const REQUEST: &[u8] = b"PING\r\n";
|
|
34
34
|
|
|
35
35
|
fn emit(event: &str, fields: Value) {
|
|
@@ -45,7 +45,7 @@ fn emit_at(event: &str, t: u64, fields: Value) {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
/// Sends the request, stamped before the write: on a bridge, the kernel can take a reset to the
|
|
48
|
-
/// request before write() returns, and a stamp after it
|
|
48
|
+
/// request before write() returns, and a stamp after it comes later than the reset's own.
|
|
49
49
|
fn send(stream: &mut TcpStream, request: &[u8]) -> std::io::Result<()> {
|
|
50
50
|
let t = monotonic_ns();
|
|
51
51
|
stream.write_all(request)?;
|
|
@@ -211,15 +211,14 @@ fn run() -> Result<(), Box<dyn Error>> {
|
|
|
211
211
|
);
|
|
212
212
|
|
|
213
213
|
// the loop waits in poll(), which takes no lock on the socket, and reads only what is there. A
|
|
214
|
-
// read that waits takes the socket's lock each time it wakes
|
|
215
|
-
//
|
|
216
|
-
// it measures
|
|
214
|
+
// read that waits takes the socket's lock each time it wakes. A keepalive timer that finds the
|
|
215
|
+
// socket held puts itself off by HZ/20. Thus such a read moves the timer that the client measures
|
|
217
216
|
stream.set_nonblocking(true)?;
|
|
218
217
|
let tick = args.sample.as_millis().clamp(1, i32::MAX as u128) as i32;
|
|
219
218
|
let mut waiting = args.inflight;
|
|
220
219
|
let mut seen: Option<[u32; 7]> = None;
|
|
221
220
|
let mut age = info.tcpi_last_ack_recv;
|
|
222
|
-
// stream mode: requests sent, whether one waits for its reply, and when the next
|
|
221
|
+
// stream mode: requests sent, whether one waits for its reply, and when the next can go
|
|
223
222
|
let mut sent: u32 = 0;
|
|
224
223
|
let mut awaiting = false;
|
|
225
224
|
let mut reply = Reply::default();
|
|
@@ -240,7 +239,7 @@ fn run() -> Result<(), Box<dyn Error>> {
|
|
|
240
239
|
next_at = monotonic_ns() + args.interval.as_nanos() as u64;
|
|
241
240
|
}
|
|
242
241
|
}
|
|
243
|
-
// nothing after all
|
|
242
|
+
// nothing after all. ETIMEDOUT is the connection's own end
|
|
244
243
|
Err(error) if error.kind() == ErrorKind::WouldBlock || error.kind() == ErrorKind::Interrupted => {}
|
|
245
244
|
Err(error) => {
|
|
246
245
|
let code = error.raw_os_error().unwrap_or(0);
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
//! inject ACTION ARGS... [--at NS]
|
|
2
2
|
//!
|
|
3
|
-
//!
|
|
4
|
-
//!
|
|
5
|
-
//!
|
|
6
|
-
//!
|
|
3
|
+
//! This sidecar applies the fault of a cell. It applies the fault inside the network namespace
|
|
4
|
+
//! that it shares with the target (or, for the side of the server, with the client). Thus the
|
|
5
|
+
//! image of the target stays stock. Each fault lands in one commit, between two CLOCK_MONOTONIC
|
|
6
|
+
//! stamps that the sidecar takes in the VM. A docker exec from Windows lands 190 ms late at the
|
|
7
|
+
//! median, and this delay has no effect on the two stamps.
|
|
7
8
|
//!
|
|
8
9
|
//! The actions, each with its lift:
|
|
9
10
|
//!
|
|
@@ -29,8 +30,8 @@
|
|
|
29
30
|
//!
|
|
30
31
|
//! - `signal NAME SIGNAL`: SIGNAL (KILL, STOP, CONT, TERM or HUP) to each process whose name is
|
|
31
32
|
//! NAME, in the process namespace this sidecar shares with a member of a group or a hop. STOP and
|
|
32
|
-
//! CONT freeze a server and thaw it while its kernel stays up
|
|
33
|
-
//! closes its sockets
|
|
33
|
+
//! CONT freeze a server and thaw it while its kernel stays up. KILL stops it, and its kernel
|
|
34
|
+
//! closes its sockets. HUP asks a pooler to read its configuration again. The stamps bracket the
|
|
34
35
|
//! kill(2) calls.
|
|
35
36
|
//! - `write NAME PATH CONTENT`: CONTENT to the file PATH in the root of the process NAME, through
|
|
36
37
|
//! /proc/PID/root, by a rename in one step. A proxy that watches its configuration file reloads
|
|
@@ -119,7 +120,7 @@ fn options(args: &[String]) -> Result<Options, Box<dyn Error>> {
|
|
|
119
120
|
fn rules(action: &str, peer: Ipv4Addr, opts: &Options) -> Result<String, Box<dyn Error>> {
|
|
120
121
|
let op = |set: &str| if action == set { "-I" } else { "-D" };
|
|
121
122
|
// what comes from the peer, and what goes to it: at an end, the INPUT and OUTPUT of its own
|
|
122
|
-
// sockets
|
|
123
|
+
// sockets. At a router, what it forwards in and out of the device toward the peer
|
|
123
124
|
let (from, to) = match opts.via.as_deref() {
|
|
124
125
|
None => (format!("INPUT -s {peer}/32"), format!("OUTPUT -d {peer}/32")),
|
|
125
126
|
Some(dev) => (format!("FORWARD -i {dev} -s {peer}/32"), format!("FORWARD -o {dev} -d {peer}/32")),
|
|
@@ -152,8 +153,8 @@ fn rules(action: &str, peer: Ipv4Addr, opts: &Options) -> Result<String, Box<dyn
|
|
|
152
153
|
};
|
|
153
154
|
format!("{op} {from} -p tcp -j REJECT --reject-with {with}\n")
|
|
154
155
|
}
|
|
155
|
-
// the long segments drop where they arrive
|
|
156
|
-
//
|
|
156
|
+
// the long segments drop where they arrive. In the namespace of the sender, an OUTPUT drop
|
|
157
|
+
// fails its sends with EPERM, and it does not lose them on the way
|
|
157
158
|
"mtu" | "clear-mtu" => {
|
|
158
159
|
let op = op("mtu");
|
|
159
160
|
let above = opts.above.ok_or("--above N")?;
|
|
@@ -291,7 +292,7 @@ fn processes(name: &str) -> Result<Vec<libc::pid_t>, Box<dyn Error>> {
|
|
|
291
292
|
continue;
|
|
292
293
|
};
|
|
293
294
|
// a process can end between the listing and the read. A zombie keeps its name until its
|
|
294
|
-
// parent reaps it: a server that a fin
|
|
295
|
+
// parent reaps it: a server that a fin stopped is one, and no signal reaches it
|
|
295
296
|
let Ok(comm) = std::fs::read_to_string(entry.path().join("comm")) else { continue };
|
|
296
297
|
let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) else { continue };
|
|
297
298
|
if comm.trim_end() == name && !zombie(&stat) {
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
// beside them as `status` lines.
|
|
8
8
|
//
|
|
9
9
|
// ioredis sets the socket's keepalive itself, once it connects: setKeepAlive(true, keepAlive), and
|
|
10
|
-
// Node fixes the probe interval and count. It sets no other timer the kernel
|
|
11
|
-
//
|
|
10
|
+
// Node fixes the probe interval and count. It sets no other timer that the kernel sees. Each mode
|
|
11
|
+
// sends a PING and reads its reply first. That reply is the last segment received. idle: then hold
|
|
12
12
|
// the connection until it fails. inflight: then wait for the file --go to appear, send a PING into
|
|
13
13
|
// the fault, and wait.
|
|
14
14
|
|
|
@@ -40,7 +40,7 @@ emit("start", {
|
|
|
40
40
|
client: "ioredis",
|
|
41
41
|
});
|
|
42
42
|
|
|
43
|
-
// one connection, whose end is what is measured: ioredis
|
|
43
|
+
// one connection, whose end is what is measured: ioredis reconnects by itself, thus the client ends first
|
|
44
44
|
const redis = new Redis({ host, port: Number(port), keepAlive, lazyConnect: true });
|
|
45
45
|
for (const status of ["connect", "ready", "close", "reconnecting", "end"]) {
|
|
46
46
|
redis.on(status, () => emit("status", { status }));
|
|
@@ -69,7 +69,7 @@ setTimeout(() => {
|
|
|
69
69
|
end();
|
|
70
70
|
}, max);
|
|
71
71
|
|
|
72
|
-
/** a PING
|
|
72
|
+
/** This function sends a PING. It stamps the PING before the write, because the kernel can answer before the call ends. */
|
|
73
73
|
const ping = () => {
|
|
74
74
|
const at = now();
|
|
75
75
|
const reply = redis.ping();
|
|
@@ -49,7 +49,7 @@ emit("start", {
|
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
const redis = new Redis({ ...options, sentinels, name, lazyConnect: true });
|
|
52
|
-
/** the server that the connection is on
|
|
52
|
+
/** This function gives the server that the connection is on at this time, or null between connections. */
|
|
53
53
|
const remote = () => {
|
|
54
54
|
const stream = redis.stream;
|
|
55
55
|
return stream?.remoteAddress === undefined ? null : `${stream.remoteAddress}:${stream.remotePort}`;
|
|
@@ -6,10 +6,10 @@
|
|
|
6
6
|
|
|
7
7
|
import { existsSync, readFileSync } from "node:fs";
|
|
8
8
|
|
|
9
|
-
/** This function gives the time
|
|
9
|
+
/** This function gives the current time, in nanoseconds on CLOCK_MONOTONIC. */
|
|
10
10
|
export const now = () => process.hrtime.bigint();
|
|
11
11
|
|
|
12
|
-
/** This function writes one line, with the time `at` (the time
|
|
12
|
+
/** This function writes one line, with the time `at` (the current time when none is given). */
|
|
13
13
|
export const emit = (ev, fields = {}, at = now()) =>
|
|
14
14
|
process.stdout.write(`${JSON.stringify({ ev, t: Number(at), ...fields })}\n`);
|
|
15
15
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package",
|
|
3
3
|
"name": "@mark1russell7/harness",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.13.0",
|
|
5
5
|
"description": "A fault-injection harness for connection timers: a cell runs a client against a stock target in Docker, through a topology of routers and proxies, a replicated group under Sentinels or a replica set, injects a fault, measures the kernel's own records and judges a prediction. HTTP clients of Go, Node and undici race the keep-alive of a Node server, and the MongoDB driver meets the faults of a replica set. The WebSocket client of @procedures/client drops a silent connection on its own heartbeat, and the harness times that detection and the reconnect from the program's lines and the kernel's stamps.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"fault-injection",
|