@mark1russell7/harness 0.0.0-stage → 0.1.0

package/build/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":[],"sources":["../src/observe.ts","../src/verdict.ts","../src/lab.ts","../src/run.ts"],"sourcesContent":["import type { Quantity } from \"@mark1russell7/quantity\";\nimport { quantity, valueIn } from \"@mark1russell7/quantity\";\nimport type { Cell, KernelRecord, ProbeEvent, Run } from \"./cell.ts\";\n\n/**\n * An outcome tells how the wait of a run ended. The socket failed with an errno, or it read the\n * end of the file. Or nothing came, or the reply came back after a stall.\n */\nexport type Outcome =\n  | { readonly kind: \"error\"; readonly errno: string }\n  | { readonly kind: \"eof\" }\n  | { readonly kind: \"silent\" }\n  | { readonly kind: \"answered\" };\n\n/** A measurement is what one run measured, against the prediction of its cell. */\nexport interface Measured {\n  readonly environment: number;\n  readonly index: number;\n  readonly outcome: Outcome;\n  /**\n   * This time is from the reference event of the prediction to the detection, both as the kernel\n   * stamped them in the records of the prober. It is null when nothing detected the fault.\n   */\n  readonly elapsed: Quantity | null;\n  /** This time is the same interval from the TCP_INFO samples of the client, the second vantage. */\n  readonly socket: Quantity | null;\n  /** This time is how long the client took to see the error after the kernel reported it. */\n  readonly noticed: Quantity | null;\n  /**\n   * These are the steps of the kernel after the reference: each keepalive probe, or each firing of\n   * the RTO timer. After a stall, they are each retransmission from the send, the loss probe first.\n   */\n  readonly steps: readonly Quantity[];\n  /** This count is the number of keepalive probes that the kernel sent. */\n  readonly probes: number;\n  /** This count is the number of segments that the kernel sent again. */\n  readonly retransmissions: number;\n  /** This count is the number of segments that the socket received after the request. */\n  readonly acks: number;\n  /** This text is the reason that the run cannot count, or null. A void run is not a pass and not a finding. */\n  readonly void: string | null;\n  /** This text tells what in the run disagrees with the prediction, other than its timing, or null. */\n  readonly finding: string | null;\n}\n\n/** These are the errno numbers that the kernel reports to a socket, by the names that predictions use. */\nconst ERRNO: Readonly<Record<number, string>> = {\n  32: \"EPIPE\",\n  101: \"ENETUNREACH\",\n  103: \"ECONNABORTED\",\n  104: \"ECONNRESET\",\n  110: \"ETIMEDOUT\",\n  111: \"ECONNREFUSED\",\n  113: \"EHOSTUNREACH\",\n};\n\n/** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */\nconst between = (from: number, to: number): Quantity =>\n  quantity(Math.round((to - from) / 1e3) / 1e6, \"s\");\n\nconst count = (event: ProbeEvent | undefined, field: string): number =>\n  event === undefined ? 0 : Number(event[field] ?? 0);\n\n/** This function gives the view of the client on the same interval: TCP_INFO, with a sample each 2 ms. */\nconst socketVantage = (\n  cell: Cell,\n  events: readonly ProbeEvent[],\n  ready: ProbeEvent,\n  request: ProbeEvent,\n): Quantity | null => {\n  if (!cell.prediction.detects) return null;\n  const ended = events.find((event) => event.ev === \"error\" || event.ev === \"eof\");\n  const reference =\n    cell.prediction.from === \"last-received\"\n      ? ready\n      : cell.prediction.from === \"request-sent\"\n        ? request\n        : events.find(\n            (event) =>\n              event.ev === \"kernel\" && event.t >= request.t && count(event, \"retransmits\") > 0,\n          );\n  return ended === undefined || reference === undefined ? null : between(reference.t, ended.t);\n};\n\n/**\n * This function reads one run against its cell, from the records of the kernel first.\n * - The socket of the run is the socket that the client connected from, in its namespace.\n * - The fault must hold before the event that it must meet. That is before the request goes into\n *   it, and before the first keepalive probe. A live peer answers that probe, and the answer\n *   starts the idle time again. A run where the fault was late is void. A run whose records the\n *   prober lost is also void.\n */\nexport const measure = (cell: Cell, run: Run): Measured => {\n  const { events, kernel } = run;\n  const ready = events.find((event) => event.ev === \"ready\");\n  const local = events.find((event) => event.ev === \"connected\")?.[\"local\"];\n  const request =\n    cell.probe.mode === \"inflight\" ? events.filter((event) => event.ev === \"sent\")[1] : ready;\n  // the socket of the observer: the client socket, by the address that it connected from, or the\n  // socket of the target for the same connection, in its namespace, whose peer is that address\n  const server = cell.observer === \"server\";\n  const netns = server ? run.targetNetns : run.netns;\n  const cookie = kernel.find(\n    (record) => record.netns === netns && (server ? record.dst : record.src) === local,\n  )?.cookie;\n  const socket = kernel.filter((record) => record.netns === netns && record.cookie === cookie);\n  const keepalives = socket.filter(\n    (record) => record.ev === \"probe\" && record.what === \"keepalive\",\n  );\n  const retransmitted = socket.filter((record) => record.ev === \"retrans\");\n  /**\n   * This function examines the timer that drives the measurement: keepalive or retransmit. It\n   * finds the first firing in an interval that was two jiffies or more behind its wheel slot. The\n   * wheel does a slot in the tick when it is due, thus such a delay comes from the CPU: the host\n   * stopped a vCPU. The prediction is about the timing of the kernel, and that firing measured\n   * the timing of the host. One jiffy is a tick that occurred while the wheel worked, and δ_abs\n   * includes it. The function ignores the other timer: the retransmit timer fires one time, idle,\n   * after a reply, and it does not move a keepalive.\n   */\n  const delayed = (from: number, to: number, timer: \"keepalive\" | \"retransmit\"): string | null => {\n    const firings = socket.filter(\n      (record) =>\n        record.ev === \"timer\" && record.what === timer && record.mono >= from && record.mono <= to,\n    );\n    const late = firings.find((record) => (record.behind?.value ?? 0) >= 2);\n    if (late !== undefined)\n      return `its ${timer} timer ran ${late.behind!.value} jiffies behind its wheel slot: the VM delayed it, not the kernel`;\n    // a keepalive timer that finds its socket held by its owner sends nothing, and it tries again\n    // HZ/20 later (tcp_keepalive_timer). The retransmit timer only gives its work to the release\n    // by the owner, some microseconds later\n    const held = firings.find((record) => timer === \"keepalive\" && record.owned === true);\n    return held === undefined\n      ? null\n      : \"its keepalive timer found the socket held by its owner and put itself off by HZ/20: the run measured the owner, not the timer\";\n  };\n  const acks =\n    request === undefined\n      ? 0\n      : socket.filter((record) => record.ev === \"rx\" && record.mono > request.t).length;\n  const base = {\n    environment: run.environment,\n    index: run.index,\n    probes: keepalives.length,\n    retransmissions: retransmitted.length,\n    acks,\n  };\n  const nothing = {\n    outcome: { kind: \"silent\" } as const,\n    elapsed: null,\n    socket: null,\n    noticed: null,\n    steps: [],\n    finding: null,\n  };\n\n  if (ready === undefined) return { ...base, ...nothing, void: \"the client never had its reply\" };\n  if (request === undefined)\n    return { ...base, ...nothing, void: \"the request was never sent into the fault\" };\n  if (run.lost === null)\n    return {\n      ...base,\n      ...nothing,\n      void: \"the prober wrote no clock line after the run, so nothing says what it dropped\",\n    };\n  if (run.lost > 0) return { ...base, ...nothing, void: `the prober dropped ${run.lost} records` };\n  if (cookie === undefined)\n    return { ...base, ...nothing, void: \"the prober saw nothing of this connection\" };\n  let voided: string | null = null;\n  if (run.fault !== null) {\n    // the request of a client must go into the fault. The request of a cell on the server side must\n    // reach the server first, and its reply must not reach the client before the fault holds\n    const replied = events.some(\n      (event) => event.ev === \"data\" && event.t > request.t && event.t < run.fault!.t1,\n    );\n    if (cell.probe.mode === \"inflight\" && !server && run.fault.t1 > request.t)\n      voided = \"the request went out before the fault held\";\n    else if (cell.probe.mode === \"inflight\" && server && run.fault.t1 < request.t)\n      voided = \"the request went out into the fault, so the server never had it\";\n    else if (cell.probe.mode === \"inflight\" && server && replied)\n      voided = \"the server's reply arrived before the fault held\";\n    else if (keepalives[0] !== undefined && keepalives[0].mono < run.fault.t1)\n      voided = \"a keepalive probe went out before the fault held\";\n  }\n\n  // the detection as the kernel reported it to the owner of the socket, and as the client saw it.\n  // The process of a server tells nothing that the harness reads, thus its kernel tells all\n  const reported = socket.find((record) => record.ev === \"error\" && (record.err ?? 0) !== 0);\n  const seen = server\n    ? undefined\n    : events.find((event) => event.ev === \"error\" || event.ev === \"eof\");\n  const outcome: Outcome =\n    reported !== undefined\n      ? { kind: \"error\", errno: ERRNO[reported.err!] ?? `E${reported.err}` }\n      : seen?.ev === \"eof\"\n        ? { kind: \"eof\" }\n        : seen !== undefined\n          ? { kind: \"error\", errno: String(seen[\"errno\"]) }\n          : { kind: \"silent\" };\n  const ended = reported?.mono ?? seen?.t;\n  const noticed =\n    reported !== undefined && seen !== undefined ? between(reported.mono, seen.t) : null;\n  const { prediction } = cell;\n\n  // a stall: nothing must detect it, and the connection must move again at the first\n  // retransmission after it stops, timed from the send of the request\n  if (!prediction.detects && prediction.recovers !== undefined) {\n    const { recovers } = prediction;\n    const heal = run.heal ?? null;\n    if (voided === null && heal === null) voided = \"the stall never ended\";\n    else if (voided === null && heal !== null) {\n      const from = between(request.t, heal.t0);\n      const to = between(request.t, heal.t1);\n      if (\n        valueIn(from, \"s\") <= valueIn(recovers.heal.low, \"s\") ||\n        valueIn(to, \"s\") >= valueIn(recovers.heal.high, \"s\")\n      )\n        voided = `the stall ended ${from.value}–${to.value} s after the send, outside the ${recovers.heal.low.value}–${recovers.heal.high.value} s its prediction assumed`;\n    }\n    const steps = retransmitted\n      .filter((record) => record.mono > request.t)\n      .map((record) => between(request.t, record.mono));\n    if (ended !== undefined)\n      return {\n        ...base,\n        ...nothing,\n        outcome,\n        steps,\n        void: voided,\n        finding: `detected after ${between(request.t, ended).value} s, where it should recover`,\n      };\n    // the first segment back after the stall stopped: the answer of the server to the\n    // retransmission that got through, and the read of the reply by the client\n    const answer =\n      heal === null\n        ? undefined\n        : socket.find((record) => record.ev === \"rx\" && record.mono > heal.t0);\n    if (answer === undefined)\n      return {\n        ...base,\n        ...nothing,\n        steps,\n        void: voided,\n        finding: \"it never moved again: nothing came back after the stall ended\",\n      };\n    const reply = events.find((event) => event.ev === \"data\" && event.t > request.t);\n    voided ??= delayed(request.t, answer.mono, \"retransmit\");\n    return {\n      ...base,\n      outcome: { kind: \"answered\" },\n      elapsed: between(request.t, answer.mono),\n      socket: reply === undefined ? null : between(request.t, reply.t),\n      noticed: reply === undefined ? null : between(answer.mono, reply.t),\n      steps,\n      void: voided,\n      finding: null,\n    };\n  }\n\n  if (!prediction.detects) {\n    // the test of silence covers the window from the send of the request\n    const window = valueIn(prediction.window, \"s\") * 1e9;\n    if (ended === undefined || ended - request.t > window)\n      return { ...base, ...nothing, void: voided };\n    return {\n      ...base,\n      ...nothing,\n      outcome,\n      elapsed: between(request.t, ended),\n      void: voided,\n      finding: `detected after ${between(request.t, ended).value} s, where nothing should`,\n    };\n  }\n\n  if (ended === undefined)\n    return {\n      ...base,\n      ...nothing,\n      void: voided,\n      finding: `nothing detected it: ${prediction.timer} never fired`,\n    };\n  // keepalive counts from the last segment received before the fault. The budget of the give-up\n  // counts from the first retransmission of the RTO timer, not from the tail loss probe before it.\n  // A reset counts from the send of the client. The kernel has no record of that send, thus the\n  // stamp is the stamp of the client\n  const before = run.fault?.t1 ?? ready.t;\n  const kernelReference: KernelRecord | undefined =\n    prediction.from === \"last-received\"\n      ? socket.filter((record) => record.ev === \"rx\" && record.mono <= before).at(-1)\n      : prediction.from === \"first-retransmission\"\n        ? retransmitted.find((record) => record.why === \"rto\")\n        : undefined;\n  const reference: number | undefined =\n    prediction.from === \"request-sent\" ? request.t : kernelReference?.mono;\n  if (reference === undefined) {\n    return {\n      ...base,\n      ...nothing,\n      outcome,\n      noticed,\n      void: voided,\n      finding: `it ended without its reference event (${prediction.from})`,\n    };\n  }\n  const steps = (\n    prediction.from === \"last-received\"\n      ? keepalives\n      : prediction.from === \"first-retransmission\"\n        ? retransmitted.filter((record) => record.why === \"rto\" && record.mono > reference)\n        : []\n  ).map((record) => between(reference, record.mono));\n  const finding =\n    outcome.kind === \"error\" && outcome.errno === prediction.errno\n      ? null\n      : `it ended with ${outcome.kind === \"error\" ? outcome.errno : \"end of file\"}, not ${prediction.errno}`;\n  // a firing at the reference itself only moves the start of the interval. A reset meets the send\n  // with no timer\n  if (prediction.from !== \"request-sent\")\n    voided ??= delayed(\n      reference,\n      ended,\n      prediction.from === \"last-received\" ? \"keepalive\" : \"retransmit\",\n    );\n  return {\n    ...base,\n    outcome,\n    elapsed: between(reference, ended),\n    socket: server ? null : socketVantage(cell, events, ready, request),\n    noticed,\n    steps,\n    void: voided,\n    finding,\n  };\n};\n","import type { Quantity } from \"@mark1russell7/quantity\";\nimport { quantity, valueIn } from \"@mark1russell7/quantity\";\nimport type { Prediction } from \"./cell.ts\";\nimport type { Measured } from \"./observe.ts\";\n\n/**\n * This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is\n * the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its\n * minimum is one jiffy. On 2026-10-12 the worst of 60 resets at 1 and 4 environments took\n * 1.141 ms, thus the minimum sets it to 4 ms. The calibration occurs again after each change to\n * the harness, the kernel or Docker.\n */\nexport const NOISE: Quantity = quantity(4, \"ms\");\n/** This fraction, δ_rel, is how far the median of a cell can be outside the prediction and stay equivalent. */\nexport const RELATIVE = 0.05;\n/** These are the run counts of a cell: 11, then 22 and 44 while its median crosses the edge of the band. */\nexport const STEPS = [11, 22, 44] as const;\n\n/** A decision is the result of a verdict. */\nexport type Decision = \"holds\" | \"fails\" | \"extend\" | \"inconclusive\";\n\n/** An interval is a range of times. */\nexport interface Interval {\n  readonly low: Quantity;\n  readonly high: Quantity;\n}\n\n/** A verdict is the judgement of a cell on its runs. */\nexport interface Verdict {\n  readonly decision: Decision;\n  readonly counted: number;\n  readonly voided: number;\n  /** This band is where each run must land: [T_lo − δ_abs, T_hi + δ_abs]. It is null for a silent prediction. */\n  readonly band: Interval | null;\n  /** These are the runs outside the band, and the reason for each. Each is a finding. */\n  readonly outside: ReadonlyArray<{ readonly run: string; readonly why: string }>;\n  readonly median: Quantity | null;\n  /** This interval is the order-statistic interval of the median at 95 %, or the whole range for fewer than six runs. */\n  readonly medianInterval: Interval | null;\n  readonly why: string;\n}\n\nconst s = (value: number): Quantity => quantity(Math.round(value * 1e6) / 1e6, \"s\");\nconst name = (run: Measured): string => `env ${run.environment} run ${run.index}`;\n\n/** This function gives P(X ≤ j) for X ~ Binomial(n, 1/2), exactly. */\nconst below = (n: number, j: number): number => {\n  let term = 1;\n  let sum = 0;\n  for (let i = 0; i <= j; i += 1) {\n    sum += term;\n    term = (term * (n - i)) / (i + 1);\n  }\n  return sum / 2 ** n;\n};\n\n/**\n * This function gives the distribution-free interval of a median: [x_(k), x_(n+1−k)], for the\n * largest k whose two tails each hold 2.5 % or less. For eleven runs, it gives [x_(2), x_(10)],\n * at 98.8 %.\n */\nexport const medianInterval = (sorted: readonly number[]): [number, number] => {\n  const n = sorted.length;\n  let k = 0;\n  while (k + 1 <= n / 2 && below(n, k) <= 0.025) k += 1;\n  return k === 0 ? [sorted[0]!, sorted[n - 1]!] : [sorted[k - 1]!, sorted[n - k]!];\n};\n\nconst middle = (sorted: readonly number[]): number => {\n  const half = Math.floor(sorted.length / 2);\n  return sorted.length % 2 === 1 ? sorted[half]! : (sorted[half - 1]! + sorted[half]!) / 2;\n};\n\n/**\n * This function gives the verdict of a cell, from two tests:\n * - Containment: each run lands in [T_lo − δ_abs, T_hi + δ_abs]. A run outside is a finding.\n * - Median equivalence: the interval of the median is in the band, made wider by δ_rel. An\n *   interval outside the band fails. An interval across its edge extends the cell to 22 runs, then\n *   44, and then the verdict is inconclusive.\n *\n * A silent prediction holds only if no run detected the fault in its window. After a stall, the\n * return of the reply gets the same test as a detection, by its interval, and a detection is a\n * finding. The distribution test (a DKW band) needs a predicted distribution from a simulator,\n * which does not exist at this time.\n */\nexport const verdictOf = (prediction: Prediction, runs: readonly Measured[]): Verdict => {\n  const counted = runs.filter((run) => run.void === null);\n  const voided = runs.length - counted.length;\n  const outside = counted\n    .filter((run) => run.finding !== null)\n    .map((run) => ({ run: name(run), why: run.finding! }));\n  const empty = { counted: counted.length, voided, median: null, medianInterval: null };\n  if (counted.length === 0)\n    return { ...empty, decision: \"inconclusive\", band: null, outside, why: \"no run counted\" };\n\n  if (!prediction.detects && prediction.recovers === undefined) {\n    const decision = outside.length === 0 ? \"holds\" : \"fails\";\n    const why =\n      decision === \"holds\"\n        ? `nothing detected it in ${counted.length} runs of ${valueIn(prediction.window, \"s\")} s`\n        : `${outside.length} of ${counted.length} runs detected it`;\n    return { ...empty, decision, band: null, outside, why };\n  }\n\n  // the interval of a detection, or of the return of the reply after a stall\n  const timed = prediction.detects ? prediction : prediction.recovers!;\n  const noise = valueIn(NOISE, \"s\");\n  const low = valueIn(timed.low, \"s\");\n  const high = valueIn(timed.high, \"s\");\n  const band = { low: s(low - noise), high: s(high + noise) };\n  const values: number[] = [];\n  for (const run of counted) {\n    if (run.elapsed === null) continue;\n    const value = valueIn(run.elapsed, \"s\");\n    values.push(value);\n    if (value < low - noise || value > high + noise)\n      outside.push({\n        run: name(run),\n        why: `${value} s is outside [${band.low.value}, ${band.high.value}] s`,\n      });\n  }\n  values.sort((a, b) => a - b);\n  if (values.length === 0)\n    return { ...empty, decision: \"fails\", band, outside, why: \"no run measured its time\" };\n  const [from, to] = medianInterval(values);\n  const median = s(middle(values));\n  const interval = { low: s(from), high: s(to) };\n  const wide = { low: low * (1 - RELATIVE), high: high * (1 + RELATIVE) };\n  const inside = from >= wide.low && to <= wide.high;\n  const disjoint = to < wide.low || from > wide.high;\n  const counts = `${counted.length - outside.length} of ${counted.length} runs inside [${band.low.value}, ${band.high.value}] s`;\n  const centre = `median ${median.value} s, its interval [${interval.low.value}, ${interval.high.value}] s`;\n  const result = {\n    counted: counted.length,\n    voided,\n    band,\n    outside,\n    median,\n    medianInterval: interval,\n  };\n  if (outside.length > 0 || disjoint)\n    return { ...result, decision: \"fails\", why: `${counts}; ${centre}` };\n  if (inside) return { ...result, decision: \"holds\", why: `${counts}; ${centre}` };\n  const decision = counted.length < STEPS[2] ? \"extend\" : \"inconclusive\";\n  return { ...result, decision, why: `${counts}; ${centre} straddles the band` };\n};\n","import { createHash } from \"node:crypto\";\nimport {\n  copyFileSync,\n  mkdirSync,\n  mkdtempSync,\n  readFileSync,\n  readdirSync,\n  realpathSync,\n  rmSync,\n  statSync,\n} from \"node:fs\";\nimport { createRequire } from \"node:module\";\nimport { tmpdir } from \"node:os\";\nimport { dirname, join, relative } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport type { StartedNetwork, StartedTestContainer } from \"testcontainers\";\nimport type * as Testcontainers from \"testcontainers\";\nimport type { Client, KernelRecord, ProbeEvent, Target } from \"./cell.ts\";\n\n/**\n * These are the roots of the two packages that the images read: this package, and the prober\n * that Node resolves for it. In a workspace, they are two folders side by side. From npm, the\n * prober is a link, and a Docker build context does not follow a link. Thus each image builds\n * from a context that the harness copies from the two roots.\n */\nconst ROOTS: Readonly<Record<string, string>> = {\n  harness: fileURLToPath(new URL(\"..\", import.meta.url)),\n  prober: dirname(\n    realpathSync(createRequire(import.meta.url).resolve(\"@mark1russell7/prober/package.json\")),\n  ),\n};\n\n/** This function gives the path of a part of an image, which is `<package>/<path>` in the context. */\nconst pathOf = (part: string): string => {\n  const [root, ...rest] = part.split(\"/\");\n  const base = ROOTS[root!];\n  if (base === undefined) throw new Error(`${part} is in no package that the images read`);\n  return join(base, ...rest);\n};\n\n/** This function loads Testcontainers when a lab first opens. Code that only measures and judges does not load it. */\nconst containers = (): Promise<typeof Testcontainers> => import(\"testcontainers\");\n\n/** An image is an image that the harness builds: its Dockerfile, and each file or folder that the Dockerfile reads. */\ninterface Image {\n  readonly name: string;\n  readonly dockerfile: string;\n  readonly from: readonly string[];\n}\n\n/** This image holds the prober, the TCP client, the injector and the namespace admin. */\nconst HARNESS: Image = {\n  name: \"loom-harness\",\n  dockerfile: \"harness/image/Dockerfile\",\n  from: [\n    \"harness/image/Dockerfile\",\n    \"harness/image/native\",\n    \"prober/bpf\",\n    \"prober/native\",\n    \"prober/fixtures\",\n  ],\n};\n/** This image holds the Node client: ioredis 6.0.0 on Node 24.21.0. */\nconst NODE: Image = {\n  name: \"loom-harness-node\",\n  dockerfile: \"harness/image/node/Dockerfile\",\n  from: [\"harness/image/node\"],\n};\n\n/** This function gives each file under a path, in a stable sequence, but not the output of a local build. */\nconst sources = (path: string): string[] => {\n  if (statSync(path).isFile()) return [path];\n  const entries = readdirSync(path, { withFileTypes: true });\n  return entries\n    .filter((entry) => entry.name !== \"target\" && entry.name !== \"node_modules\")\n    .flatMap((entry) => sources(join(path, entry.name)))\n    .sort();\n};\n\n/** This function gives each file of an image, with its path in the build context. */\nconst filesOf = (image: Image): Array<{ readonly file: string; readonly at: string }> =>\n  image.from.flatMap((part) => {\n    const base = pathOf(part);\n    return sources(base).map((file) => ({\n      file,\n      at: join(part, relative(base, file)).replaceAll(\"\\\\\", \"/\"),\n    }));\n  });\n\n/** This function gives the tag of an image from its content. Thus a changed probe or client cannot operate as an old one. */\nconst imageTag = (image: Image): string => {\n  const hash = createHash(\"sha256\");\n  for (const { file, at } of filesOf(image)) {\n    hash.update(at);\n    hash.update(readFileSync(file));\n  }\n  return `${image.name}:${hash.digest(\"hex\").slice(0, 12)}`;\n};\n\n/** This function copies the files of an image into a new build context, and gives its path. */\nconst stage = (image: Image): string => {\n  const context = mkdtempSync(join(tmpdir(), `${image.name}-`));\n  for (const { file, at } of filesOf(image)) {\n    mkdirSync(dirname(join(context, at)), { recursive: true });\n    copyFileSync(file, join(context, at));\n  }\n  return context;\n};\n\nconst built = new Map<string, Promise<string>>();\n/** This function builds an image one time for each process. Docker keeps its layers between runs. */\nconst build = (image: Image): Promise<string> => {\n  let tag = built.get(image.name);\n  if (tag === undefined) {\n    tag = (async () => {\n      const name = imageTag(image);\n      const { GenericContainer } = await containers();\n      const context = stage(image);\n      try {\n        await GenericContainer.fromDockerfile(context, image.dockerfile).build(name, {\n          deleteOnExit: false,\n        });\n      } finally {\n        rmSync(context, { recursive: true, force: true });\n      }\n      return name;\n    })();\n    built.set(image.name, tag);\n  }\n  return tag;\n};\n\n/** This function gives the image of the harness. */\nexport const harnessImage = (): Promise<string> => build(HARNESS);\n/** This function gives the image of the Node client. */\nexport const nodeClientImage = (): Promise<string> => build(NODE);\n\n/** A network fault is a fault that the injector applies, by the name of its manifestation. */\nexport type NetworkFault = \"blackhole\" | \"rst\";\n\n/**\n * A fault side is the namespace that gets the rules of a fault: the namespace of the peer of the\n * observer. For the client, it is the namespace of the target (the injector). For the server, it\n * is the namespace of the client (its privileged admin). Rules in the namespace of the observer\n * make its sends fail locally, with EPERM, and the network does not lose them. Thus its\n * retransmissions do not go out. The retransmission tracepoint of the kernel then records none,\n * because it fires only for a retransmission that went out.\n */\nexport type FaultSide = \"target\" | \"client\";\n\n/** A manifest is what a lab records about its machine, its images and its settings. */\nexport interface Manifest {\n  readonly kernel: string;\n  /** This text is the architecture of the machine, as uname -m gives it: x86_64. */\n  readonly architecture: string;\n  readonly hz: string | null;\n  readonly boot_id: string;\n  readonly docker: string;\n  /** This record is the target, and its settings as it read them back. */\n  readonly target: {\n    readonly image: string;\n    readonly id: string;\n    readonly settings: Readonly<Record<string, string>>;\n    /** These are the sysctls of its namespace, as read back after the harness set them. */\n    readonly sysctls: Readonly<Record<string, string>>;\n  };\n  readonly harness: { readonly image: string; readonly id: string; readonly packages: string[] };\n  /** This record is the client of each run: the TCP client of the harness, or a driver from its own image. */\n  readonly client: { readonly kind: Client; readonly image: string };\n  /** These are the namespaces that the prober watched, by inode. */\n  readonly netns: { readonly client: number; readonly target: number };\n  /** These are the sysctls of the client namespace, as read back after the harness set them. */\n  readonly sysctls: Readonly<Record<string, string>>;\n}\n\n/** These are the JSON lines of a program, as they arrive. */\ninterface Lines<T extends { readonly ev: string }> {\n  readonly seen: T[];\n  /** These are the other lines that it wrote, for example an error. They explain a program that stopped its output. */\n  readonly other: string[];\n  consume: (stream: NodeJS.ReadableStream) => void;\n  /** This method gives the first line of one of these kinds, or it rejects after `within` ms. */\n  until(kinds: readonly string[], within: number, who: string): Promise<T>;\n}\n\nconst lines = <T extends { readonly ev: string }>(): Lines<T> => {\n  const seen: T[] = [];\n  const other: string[] = [];\n  const waiting: Array<{ kinds: readonly string[]; resolve: (line: T) => void }> = [];\n  let partial = \"\";\n  const take = (text: string) => {\n    if (!text.startsWith(\"{\")) {\n      if (text.length > 0) other.push(text);\n      return;\n    }\n    const line = JSON.parse(text) as T;\n    seen.push(line);\n    for (const wait of waiting.splice(0)) {\n      if (wait.kinds.includes(line.ev)) wait.resolve(line);\n      else waiting.push(wait);\n    }\n  };\n  return {\n    seen,\n    other,\n    consume: (stream) =>\n      stream.on(\"data\", (chunk: Buffer | string) => {\n        partial += chunk.toString();\n        const split = partial.split(\"\\n\");\n        partial = split.pop() ?? \"\";\n        for (const text of split) take(text.trim());\n      }),\n    until(kinds, within, who) {\n      const found = seen.find((line) => kinds.includes(line.ev));\n      if (found !== undefined) return Promise.resolve(found);\n      return new Promise((resolve, reject) => {\n        const timer = setTimeout(() => {\n          const said = other.length === 0 ? \"\" : `; it wrote: ${other.slice(-3).join(\" | \")}`;\n          reject(new Error(`no ${kinds.join(\" or \")} from the ${who} in ${within} ms${said}`));\n        }, within);\n        waiting.push({\n          kinds,\n          resolve: (line) => {\n            clearTimeout(timer);\n            resolve(line);\n          },\n        });\n      });\n    },\n  };\n};\n\n/** A probe is the client container of one run. */\nexport interface Probe {\n  /** This method gives the first event of one of these kinds, or it rejects after `within` ms. */\n  until(kinds: readonly string[], within: number): Promise<ProbeEvent>;\n  /** This method lets an in-flight probe send its request. */\n  go(): Promise<void>;\n  /** This method freezes the client with the cgroup freezer: its process stops, and its kernel continues to answer. */\n  freeze(): Promise<void>;\n  thaw(): Promise<void>;\n  readonly events: readonly ProbeEvent[];\n  stop(): Promise<void>;\n}\n\n/** A lab is one environment: the target, the client, the injector and the prober. */\nexport interface Lab {\n  readonly targetIp: string;\n  readonly clientIp: string;\n  readonly netns: { readonly client: number; readonly target: number };\n  /** These are all the records that the prober wrote, in sequence. */\n  readonly kernel: readonly KernelRecord[];\n  /** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */\n  sysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;\n  /** This method sets the settings of the target in it before a client connects, and reads them back. */\n  configure(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;\n  /** This method sets the sysctls of the target namespace through a privileged sidecar there, and reads them back. */\n  targetSysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;\n  /**\n   * This method applies a fault between the client and the target, from the namespace of the\n   * target. The fault is a black hole, or a reset that answers the next segment of the client. The stamps of\n   * the injector bracket it.\n   */\n  fault(kind: NetworkFault, at?: FaultSide): Promise<{ t0: number; t1: number }>;\n  /** This method removes a fault. */\n  lift(kind: NetworkFault, at?: FaultSide): Promise<void>;\n  /**\n   * This method stops a black hole at an instant on the monotonic clock of the VM. The injector\n   * waits for that instant in the VM, thus the length of a stall does not include the latency of\n   * a docker exec. Stamps bracket it, as for a fault.\n   */\n  heal(at: FaultSide, when: number): Promise<{ t0: number; t1: number }>;\n  /**\n   * This method removes what the two kernels learned from their earlier connections. These are\n   * the TCP metrics that each kernel keeps for a peer address when a connection closes. These metrics set the\n   * first round trip of the next connection, and thus its RTO. A run with these metrics is not\n   * independent of the run before it.\n   */\n  forget(): Promise<void>;\n  /** This method freezes the target with the cgroup freezer: its process stops, and its kernel continues to answer. */\n  freeze(): Promise<void>;\n  thaw(): Promise<void>;\n  probe(args: readonly string[]): Promise<Probe>;\n  /**\n   * This method waits for the next clock line of the prober, which gives the count of lost\n   * records. It gives null when no clock line comes in 5 s, because then nothing tells the count.\n   */\n  settle(): Promise<number | null>;\n  manifest(\n    sysctls: Readonly<Record<string, string>>,\n    settings?: Readonly<Record<string, string>>,\n    targetSysctls?: Readonly<Record<string, string>>,\n  ): Promise<Manifest>;\n  close(): Promise<void>;\n}\n\n// each container here is a holder, a probe or the prober: nothing drains, thus the stop is immediate\nconst STOP = { timeout: 0 };\n\nconst output = async (container: StartedTestContainer, argv: string[]): Promise<string> => {\n  const result = await container.exec(argv);\n  if (result.exitCode !== 0)\n    throw new Error(`${argv.join(\" \")} exited ${result.exitCode}: ${result.output.trim()}`);\n  return result.stdout.trim();\n};\n\n/** This function gives the inode of the network namespace of a container: net:[4026532583]. */\nconst netnsOf = async (container: StartedTestContainer): Promise<number> => {\n  const link = await output(container, [\"readlink\", \"/proc/self/ns/net\"]);\n  const inode = /^net:\\[(\\d+)\\]$/.exec(link)?.[1];\n  if (inode === undefined) throw new Error(`not a network namespace: ${link}`);\n  return Number(inode);\n};\n\n/**\n * This function opens one environment:\n * - a new network and the stock target.\n * - a client that holds its own network namespace, with a privileged sidecar in it for its\n *   sysctls. Docker Desktop keeps /proc/sys read-only in a container that operates.\n * - an injector sidecar in the namespace of the target.\n * - the prober, a privileged container that loads the kernel program and watches the two\n *   namespaces.\n *\n * The client of each run is its own container in the namespace of the client. It is the TCP\n * client of the harness, or a driver from its own image (ioredis on Node). The two print the same\n * lines.\n */\nexport const openLab = async (target: Target, kind: Client = \"tcp\"): Promise<Lab> => {\n  const image = await harnessImage();\n  const clientImage = kind === \"ioredis\" ? await nodeClientImage() : image;\n  const command =\n    kind === \"ioredis\" ? [\"node\", \"/opt/harness/node/client-ioredis.mjs\"] : [\"/opt/harness/client\"];\n  const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();\n  const opened: StartedTestContainer[] = [];\n  let network: StartedNetwork | undefined;\n  const close = async () => {\n    for (const container of opened.reverse()) await container.stop(STOP).catch(() => undefined);\n    await network?.stop().catch(() => undefined);\n  };\n  try {\n    network = await new Network().start();\n    const server = await new GenericContainer(target.image)\n      .withNetwork(network)\n      .withWaitStrategy(Wait.forLogMessage(target.ready))\n      .start();\n    opened.push(server);\n    const client = await new GenericContainer(image).withNetwork(network).start();\n    opened.push(client);\n    const admin = await new GenericContainer(image)\n      .withNetworkMode(`container:${client.getId()}`)\n      .withPrivilegedMode()\n      .start();\n    opened.push(admin);\n    const injector = await new GenericContainer(image)\n      .withNetworkMode(`container:${server.getId()}`)\n      .withAddedCapabilities(\"NET_ADMIN\", \"NET_RAW\")\n      .start();\n    opened.push(injector);\n    const netns = { client: await netnsOf(admin), target: await netnsOf(injector) };\n    const records = lines<KernelRecord>();\n    const prober = await new GenericContainer(image)\n      .withPrivilegedMode()\n      .withNetworkMode(\"none\")\n      .withCommand([\"/opt/harness/prober\", String(netns.client), String(netns.target)])\n      .withLogConsumer(records.consume)\n      .start();\n    opened.push(prober);\n    const begun = await records.until([\"begin\", \"fail\"], 30_000, \"prober\");\n    if (begun.ev === \"fail\") throw new Error(`the prober failed: ${String(begun[\"why\"])}`);\n    const targetIp = server.getIpAddress(network.getName());\n    const clientIp = client.getIpAddress(network.getName());\n    const runtime = await getContainerRuntimeClient();\n    const paused = runtime.container.getById(server.getId());\n    let targetAdmin: StartedTestContainer | undefined;\n\n    return {\n      targetIp,\n      clientIp,\n      netns,\n      kernel: records.seen,\n      async sysctl(values) {\n        const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);\n        if (settings.length > 0) await output(admin, [\"sysctl\", \"-q\", \"-w\", ...settings]);\n        const read: Record<string, string> = {};\n        for (const key of Object.keys(values))\n          read[key] = await output(admin, [\"sysctl\", \"-n\", key]);\n        return read;\n      },\n      async targetSysctl(values) {\n        const read: Record<string, string> = {};\n        if (Object.keys(values).length === 0) return read;\n        // privileged, and only when a cell asks for it: otherwise the namespace of the target stays as it is\n        targetAdmin ??= await new GenericContainer(image)\n          .withNetworkMode(`container:${server.getId()}`)\n          .withPrivilegedMode()\n          .start();\n        opened.push(targetAdmin);\n        const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);\n        await output(targetAdmin, [\"sysctl\", \"-q\", \"-w\", ...settings]);\n        for (const key of Object.keys(values))\n          read[key] = await output(targetAdmin, [\"sysctl\", \"-n\", key]);\n        return read;\n      },\n      async configure(values) {\n        const read: Record<string, string> = {};\n        if (Object.keys(values).length === 0) return read;\n        const how = target.settings;\n        if (how === undefined) throw new Error(`${target.image} names no way to set its settings`);\n        for (const [name, value] of Object.entries(values)) {\n          await output(server, [...how.set, name, value]);\n          read[name] = (await output(server, [...how.get, name])).split(\"\\n\").at(-1) ?? \"\";\n        }\n        return read;\n      },\n      async fault(kind, at = \"target\") {\n        const [where, peer] = at === \"client\" ? [admin, targetIp] : [injector, clientIp];\n        const stamp = JSON.parse(await output(where, [\"/opt/harness/inject\", kind, peer])) as {\n          t0: number;\n          t1: number;\n        };\n        return { t0: stamp.t0, t1: stamp.t1 };\n      },\n      async lift(kind, at = \"target\") {\n        const [where, peer] = at === \"client\" ? [admin, targetIp] : [injector, clientIp];\n        const action = kind === \"blackhole\" ? \"clear\" : \"clear-rst\";\n        await output(where, [\"/opt/harness/inject\", action, peer]);\n      },\n      async heal(at, when) {\n        const [where, peer] = at === \"client\" ? [admin, targetIp] : [injector, clientIp];\n        const args = [\"/opt/harness/inject\", \"clear\", peer, \"--at\", String(Math.round(when))];\n        const stamp = JSON.parse(await output(where, args)) as { t0: number; t1: number };\n        return { t0: stamp.t0, t1: stamp.t1 };\n      },\n      async forget() {\n        await output(admin, [\"ip\", \"tcp_metrics\", \"flush\", \"all\"]);\n        await output(injector, [\"ip\", \"tcp_metrics\", \"flush\", \"all\"]);\n      },\n      freeze: () => paused.pause().then(() => undefined),\n      thaw: () => paused.unpause().then(() => undefined),\n      async probe(args) {\n        const events = lines<ProbeEvent>();\n        const container = await new GenericContainer(clientImage)\n          .withNetworkMode(`container:${client.getId()}`)\n          .withCommand([...command, ...args])\n          .withLogConsumer(events.consume)\n          .start();\n        return {\n          events: events.seen,\n          until: (kinds, within) => events.until(kinds, within, \"client\"),\n          go: () => output(container, [\"touch\", \"/tmp/go\"]).then(() => undefined),\n          freeze: () =>\n            runtime.container\n              .getById(container.getId())\n              .pause()\n              .then(() => undefined),\n          thaw: () =>\n            runtime.container\n              .getById(container.getId())\n              .unpause()\n              .then(() => undefined),\n          stop: () => container.stop(STOP).then(() => undefined),\n        };\n      },\n      async settle() {\n        // the prober drains its ring each 20 ms, and it writes a clock line each second\n        const after = records.seen.length;\n        const clock = await new Promise<KernelRecord | null>((resolve) => {\n          const started = Date.now();\n          const look = () => {\n            const line = records.seen.slice(after).find((record) => record.ev === \"clock\");\n            if (line !== undefined) resolve(line);\n            else if (Date.now() - started > 5_000) resolve(null);\n            else setTimeout(look, 50);\n          };\n          look();\n        });\n        return clock === null ? null : Number(clock[\"lost\"] ?? 0);\n      },\n      async manifest(sysctls, settings = {}, targetSysctls = {}) {\n        const config = await output(admin, [\"zcat\", \"/proc/config.gz\"]).catch(() => \"\");\n        const inspect = async (container: StartedTestContainer) =>\n          runtime.container.inspect(runtime.container.getById(container.getId()));\n        return {\n          kernel: await output(client, [\"uname\", \"-r\"]),\n          architecture: await output(client, [\"uname\", \"-m\"]),\n          hz: /^CONFIG_HZ=(\\d+)$/m.exec(config)?.[1] ?? null,\n          boot_id: await output(client, [\"cat\", \"/proc/sys/kernel/random/boot_id\"]),\n          docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,\n          target: {\n            image: target.image,\n            id: (await inspect(server)).Image,\n            settings,\n            sysctls: targetSysctls,\n          },\n          harness: {\n            image,\n            id: (await inspect(client)).Image,\n            packages: (await output(client, [\"apk\", \"info\", \"-v\"])).split(\"\\n\").sort(),\n          },\n          client: { kind, image: clientImage },\n          netns,\n          sysctls,\n        };\n      },\n      close,\n    };\n  } catch (error) {\n    await close();\n    throw error;\n  }\n};\n","import { valueIn } from \"@mark1russell7/quantity\";\nimport type { Cell, KernelRecord, ProbeEvent, Run } from \"./cell.ts\";\nimport type { Lab, Manifest, Probe } from \"./lab.ts\";\nimport { openLab } from \"./lab.ts\";\nimport type { Measured } from \"./observe.ts\";\nimport { measure } from \"./observe.ts\";\nimport type { Verdict } from \"./verdict.ts\";\nimport { STEPS, verdictOf } from \"./verdict.ts\";\n\n/** A cell result is a cell with its manifests, its runs, their measurements and its verdict. */\nexport interface CellResult {\n  readonly cell: Cell;\n  readonly manifests: readonly Manifest[];\n  readonly runs: readonly Run[];\n  readonly measured: readonly Measured[];\n  readonly verdict: Verdict;\n  /** These times are when its first environment opened and its last environment closed, on the wall clock. */\n  readonly at: { readonly started: string; readonly ended: string };\n}\n\n/**\n * This function measures and judges the raw records of a cell, with the code of the harness at\n * this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it\n * again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.\n */\nexport const judge = (\n  cell: Cell,\n  runs: readonly Run[],\n): { measured: Measured[]; verdict: Verdict } => {\n  const measured = runs.map((run) => measure(cell, run));\n  return { measured, verdict: verdictOf(cell.prediction, measured) };\n};\n\n/** This function gives the count of lost records in the latest clock line of the prober, since it began. */\nconst droppedSoFar = (records: readonly KernelRecord[]): number => {\n  for (let at = records.length - 1; at >= 0; at -= 1)\n    if (records[at]!.ev === \"clock\") return Number(records[at]![\"lost\"] ?? 0);\n  return 0;\n};\n\n/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */\nconst probeArgs = (cell: Cell, lab: Lab): string[] => {\n  const { prediction, probe } = cell;\n  const longest = prediction.detects\n    ? valueIn(prediction.high, \"s\")\n    : valueIn(prediction.window, \"s\");\n  const args = [lab.targetIp, String(cell.target.port), \"--mode\", probe.mode];\n  if (probe.request !== undefined) args.push(\"--request\", probe.request);\n  if (probe.driver !== undefined) {\n    // a driver sets its socket options from its own options: the keepAlive of ioredis\n    args.push(\"--keepalive-ms\", String(valueIn(probe.driver.keepAlive, \"ms\")));\n    args.push(\"--max-s\", String(Math.ceil(longest * 2 + 10)));\n    return args;\n  }\n  if (probe.keepalive) args.push(\"--keepalive\");\n  if (probe.userTimeout !== undefined)\n    args.push(\"--user-timeout-ms\", String(valueIn(probe.userTimeout, \"ms\")));\n  args.push(\"--max-s\", String(Math.ceil(longest * 2 + 10)));\n  return args;\n};\n\n/**\n * This function waits for an error on the socket of the target for the connection of the\n * client. The prober records that error in the namespace of the target. A cell on the server side\n * waits for that. The promise resolves after `within` ms at the latest.\n */\nconst serverError = (lab: Lab, local: unknown, within: number): Promise<void> =>\n  new Promise((resolve) => {\n    const started = Date.now();\n    const look = () => {\n      const seen = lab.kernel.some(\n        (record) =>\n          record.ev === \"error\" &&\n          (record.err ?? 0) !== 0 &&\n          record.netns === lab.netns.target &&\n          record.dst === local,\n      );\n      if (seen || Date.now() - started > within) resolve();\n      else setTimeout(look, 50);\n    };\n    look();\n  });\n\n/**\n * This function waits for the request that an in-flight client sends on the signal: its second\n * send, after the PING that made it ready. Its stamp is when the request first met the fault.\n */\nconst requestSent = (probe: Probe, within: number): Promise<ProbeEvent> =>\n  new Promise((resolve, reject) => {\n    const started = Date.now();\n    const look = () => {\n      const sent = probe.events.filter((event) => event.ev === \"sent\")[1];\n      if (sent !== undefined) resolve(sent);\n      else if (Date.now() - started > within)\n        reject(new Error(`the client sent no request within ${within} ms of being told to`));\n      else setTimeout(look, 1);\n    };\n    look();\n  });\n\n/**\n * This function does one run. The client connects and gets its reply, the fault occurs, and an\n * in-flight client sends into it. The harness then waits for the observer to see the fault, or for\n * the silent window to end. The observer is the client, or the socket of the target for a cell on\n * the server side.\n *\n * A stall stops by itself, at its length after the send of the request, and the\n * run waits for the reply. Then the harness removes the fault, thus the environment can serve the\n * next run. The run keeps the records of the prober from its start to its end. The frozen process\n * is the peer of the observer: the target, or the client for the server.\n */\nconst once = async (lab: Lab, cell: Cell, environment: number, index: number): Promise<Run> => {\n  // each run starts from kernels that keep nothing from the runs before it\n  await lab.forget();\n  const from = lab.kernel.length;\n  const lostBefore = droppedSoFar(lab.kernel);\n  const probe = await lab.probe(probeArgs(cell, lab));\n  const server = cell.observer === \"server\";\n  const peer = server\n    ? { freeze: () => probe.freeze(), thaw: () => probe.thaw() }\n    : { freeze: () => lab.freeze(), thaw: () => lab.thaw() };\n  // a client that stopped before the signal to send tells why in its last lines\n  const go = () =>\n    probe.go().catch((error: unknown) => {\n      const last = probe.events.slice(-3).map((event) => JSON.stringify(event));\n      throw new Error(`${(error as Error).message}; the client's last lines: ${last.join(\" \")}`);\n    });\n  let fault: Run[\"fault\"];\n  let heal: { t0: number; t1: number } | null = null;\n  try {\n    await probe.until([\"ready\"], 15_000);\n    // the in-flight data of a run on the server side is the reply of the server. The request goes\n    // out first, and the fault occurs while the server holds it (the timeout of BLPOP)\n    const replyInFlight = server && cell.probe.mode === \"inflight\";\n    if (replyInFlight) {\n      // the client sends one sample after the file appears. The request reaches the server well\n      // in the pause, and the reply waits for the timeout of the request\n      await go();\n      await new Promise((resolve) => setTimeout(resolve, 200));\n    }\n    const faultAt = server ? \"client\" : \"target\";\n    // a stall is a black hole that stops by itself\n    const network = cell.fault === \"stall\" ? \"blackhole\" : cell.fault;\n    fault = network === \"freeze-process\" ? null : await lab.fault(network, faultAt);\n    if (network === \"freeze-process\") await peer.freeze();\n    try {\n      if (cell.probe.mode === \"inflight\" && !replyInFlight) await go();\n      const { prediction } = cell;\n      if (cell.fault === \"stall\") {\n        if (cell.stall === undefined) throw new Error(`${cell.name} says no stall's length`);\n        // the injector holds the end in the VM until the length of the stall after the send\n        const sent = await requestSent(probe, 5_000);\n        heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, \"s\") * 1e9);\n        const window = valueIn(prediction.detects ? prediction.high : prediction.window, \"s\");\n        await probe.until([\"data\", \"error\", \"eof\"], window * 1_000).catch(() => undefined);\n      } else if (prediction.detects) {\n        const within = valueIn(prediction.high, \"s\") * 2_000 + 10_000;\n        if (server) {\n          const local = probe.events.find((event) => event.ev === \"connected\")?.[\"local\"];\n          await serverError(lab, local, within);\n        } else {\n          await probe.until([\"error\", \"eof\", \"deadline\"], within).catch(() => undefined);\n        }\n      } else {\n        await new Promise((resolve) =>\n          setTimeout(resolve, valueIn(prediction.window, \"s\") * 1_000),\n        );\n      }\n    } finally {\n      if (network === \"freeze-process\") await peer.thaw();\n      else if (heal === null) await lab.lift(network, faultAt);\n    }\n    // a thawed target answers the request that it held: the client sees the late reply\n    if (cell.fault === \"freeze-process\" && !server)\n      await probe.until([\"data\"], 5_000).catch(() => undefined);\n  } finally {\n    await probe.stop();\n  }\n  // the prober drains its ring each 20 ms. Its next clock line closes the records of the run\n  const settled = await lab.settle();\n  const lost = settled === null ? null : settled - lostBefore;\n  return {\n    environment,\n    index,\n    events: [...probe.events],\n    fault,\n    ...(cell.fault === \"stall\" ? { heal } : {}),\n    kernel: lab.kernel.slice(from),\n    netns: lab.netns.client,\n    targetNetns: lab.netns.target,\n    lost,\n  };\n};\n\n/** This function divides a count of runs across environments, as equally as possible. */\nexport const spread = (runs: number, environments: number): number[] =>\n  Array.from(\n    { length: environments },\n    (_, at) => Math.floor(runs / environments) + (at < runs % environments ? 1 : 0),\n  ).filter((count) => count > 0);\n\n/** This function does the part of the runs of a cell in one environment, and records its manifest. */\nconst inEnvironment = async (\n  cell: Cell,\n  environment: number,\n  count: number,\n  say: (line: string) => void,\n): Promise<{ manifest: Manifest; runs: Run[] }> => {\n  const lab = await openLab(cell.target, cell.probe.driver?.name ?? \"tcp\");\n  try {\n    // the settings of the target first: Redis applies tcp-keepalive to a client when it accepts it\n    const settings = await lab.configure(cell.targetSettings ?? {});\n    const targetSysctls = await lab.targetSysctl(cell.targetSysctls ?? {});\n    const manifest = await lab.manifest(await lab.sysctl(cell.sysctls), settings, targetSysctls);\n    const runs: Run[] = [];\n    for (let index = 0; index < count; index += 1) {\n      const run = await once(lab, cell, environment, index);\n      runs.push(run);\n      const read = measure(cell, run);\n      say(\n        `${cell.name}: env ${environment} run ${index} ${read.outcome.kind}` +\n          `${read.elapsed === null ? \"\" : ` ${read.elapsed.value} s`}` +\n          `${read.void === null ? \"\" : ` (void: ${read.void})`}`,\n      );\n    }\n    return { manifest, runs };\n  } finally {\n    await lab.close();\n  }\n};\n\n/**\n * This function does the runs of a cell. They go across new environments, and each environment records\n * its manifest. Then the verdict judges the runs. A median across the edge of the band extends\n * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a\n * concurrency of more than one, that many environments operate at the same time, each with its\n * own prober. The runs then go across at least as many environments.\n */\nexport const runCell = async (\n  cell: Cell,\n  say: (line: string) => void = () => {},\n): Promise<CellResult> => {\n  const started = new Date().toISOString();\n  const manifests: Manifest[] = [];\n  const runs: Run[] = [];\n  const concurrency = Math.max(1, cell.concurrency ?? 1);\n  let target: number = cell.runs;\n  let next = 0;\n  for (;;) {\n    const counts = spread(target - runs.length, Math.max(cell.environments, concurrency));\n    for (let at = 0; at < counts.length; at += concurrency) {\n      const batch = counts.slice(at, at + concurrency).map((count) => ({ count, env: next++ }));\n      const done = await Promise.all(\n        batch.map(({ count, env }) => inEnvironment(cell, env, count, say)),\n      );\n      for (const each of done) {\n        manifests.push(each.manifest);\n        runs.push(...each.runs);\n      }\n    }\n    const { measured, verdict } = judge(cell, runs);\n    const larger = STEPS.find((step) => step > runs.length);\n    if (verdict.decision !== \"extend\" || larger === undefined) {\n      const at = { started, ended: new Date().toISOString() };\n      return { cell, manifests, runs, measured, verdict, at };\n    }\n    target = larger;\n  }\n};\n"],"mappings":";;;;;;;;;AA8CA,MAAM,QAA0C;CAC9C,IAAI;CACJ,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;AACP;;AAGA,MAAM,WAAW,MAAc,OAC7B,SAAS,KAAK,OAAO,KAAK,QAAQ,GAAG,IAAI,KAAK,GAAG;AAEnD,MAAM,SAAS,OAA+B,UAC5C,UAAU,KAAA,IAAY,IAAI,OAAO,MAAM,UAAU,CAAC;;AAGpD,MAAM,iBACJ,MACA,QACA,OACA,YACoB;CACpB,IAAI,CAAC,KAAK,WAAW,SAAS,OAAO;CACrC,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,WAAW,MAAM,OAAO,KAAK;CAC/E,MAAM,YACJ,KAAK,WAAW,SAAS,kBACrB,QACA,KAAK,WAAW,SAAS,iBACvB,UACA,OAAO,MACJ,UACC,MAAM,OAAO,YAAY,MAAM,KAAK,QAAQ,KAAK,MAAM,OAAO,aAAa,IAAI,CACnF;CACR,OAAO,UAAU,KAAA,KAAa,cAAc,KAAA,IAAY,OAAO,QAAQ,UAAU,GAAG,MAAM,CAAC;AAC7F;;;;;;;;;AAUA,MAAa,WAAW,MAAY,QAAuB;CACzD,MAAM,EAAE,QAAQ,WAAW;CAC3B,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,OAAO;CACzD,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,WAAW,CAAC,GAAG;CACjE,MAAM,UACJ,KAAK,MAAM,SAAS,aAAa,OAAO,QAAQ,UAAU,MAAM,OAAO,MAAM,CAAC,CAAC,KAAK;CAGtF,MAAM,SAAS,KAAK,aAAa;CACjC,MAAM,QAAQ,SAAS,IAAI,cAAc,IAAI;CAC7C,MAAM,SAAS,OAAO,MACnB,WAAW,OAAO,UAAU,UAAU,SAAS,OAAO,MAAM,OAAO,SAAS,KAC/E,CAAC,EAAE;CACH,MAAM,SAAS,OAAO,QAAQ,WAAW,OAAO,UAAU,SAAS,OAAO,WAAW,MAAM;CAC3F,MAAM,aAAa,OAAO,QACvB,WAAW,OAAO,OAAO,WAAW,OAAO,SAAS,WACvD;CACA,MAAM,gBAAgB,OAAO,QAAQ,WAAW,OAAO,OAAO,SAAS;;;;;;;;;;CAUvE,MAAM,WAAW,MAAc,IAAY,UAAqD;EAC9F,MAAM,UAAU,OAAO,QACpB,WACC,OAAO,OAAO,WAAW,OAAO,SAAS,SAAS,OAAO,QAAQ,QAAQ,OAAO,QAAQ,EAC5F;EACA,MAAM,OAAO,QAAQ,MAAM,YAAY,OAAO,QAAQ,SAAS,MAAM,CAAC;EACtE,IAAI,SAAS,KAAA,GACX,OAAO,OAAO,MAAM,aAAa,KAAK,OAAQ,MAAM;EAKtD,OADa,QAAQ,MAAM,WAAW,UAAU,eAAe,OAAO,UAAU,IACtE,MAAM,KAAA,IACZ,OACA;CACN;CACA,MAAM,OACJ,YAAY,KAAA,IACR,IACA,OAAO,QAAQ,WAAW,OAAO,OAAO,QAAQ,OAAO,OAAO,QAAQ,CAAC,CAAC,CAAC;CAC/E,MAAM,OAAO;EACX,aAAa,IAAI;EACjB,OAAO,IAAI;EACX,QAAQ,WAAW;EACnB,iBAAiB,cAAc;EAC/B;CACF;CACA,MAAM,UAAU;EACd,SAAS,EAAE,MAAM,SAAS;EAC1B,SAAS;EACT,QAAQ;EACR,SAAS;EACT,OAAO,CAAC;EACR,SAAS;CACX;CAEA,IAAI,UAAU,KAAA,GAAW,OAAO;EAAE,GAAG;EAAM,GAAG;EAAS,MAAM;CAAiC;CAC9F,IAAI,YAAY,KAAA,GACd,OAAO;EAAE,GAAG;EAAM,GAAG;EAAS,MAAM;CAA4C;CAClF,IAAI,IAAI,SAAS,MACf,OAAO;EACL,GAAG;EACH,GAAG;EACH,MAAM;CACR;CACF,IAAI,IAAI,OAAO,GAAG,OAAO;EAAE,GAAG;EAAM,GAAG;EAAS,MAAM,sBAAsB,IAAI,KAAK;CAAU;CAC/F,IAAI,WAAW,KAAA,GACb,OAAO;EAAE,GAAG;EAAM,GAAG;EAAS,MAAM;CAA4C;CAClF,IAAI,SAAwB;CAC5B,IAAI,IAAI,UAAU,MAAM;EAGtB,MAAM,UAAU,OAAO,MACpB,UAAU,MAAM,OAAO,UAAU,MAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,IAAI,MAAO,EAChF;EACA,IAAI,KAAK,MAAM,SAAS,cAAc,CAAC,UAAU,IAAI,MAAM,KAAK,QAAQ,GACtE,SAAS;OACN,IAAI,KAAK,MAAM,SAAS,cAAc,UAAU,IAAI,MAAM,KAAK,QAAQ,GAC1E,SAAS;OACN,IAAI,KAAK,MAAM,SAAS,cAAc,UAAU,SACnD,SAAS;OACN,IAAI,WAAW,OAAO,KAAA,KAAa,WAAW,EAAE,CAAC,OAAO,IAAI,MAAM,IACrE,SAAS;CACb;CAIA,MAAM,WAAW,OAAO,MAAM,WAAW,OAAO,OAAO,YAAY,OAAO,OAAO,OAAO,CAAC;CACzF,MAAM,OAAO,SACT,KAAA,IACA,OAAO,MAAM,UAAU,MAAM,OAAO,WAAW,MAAM,OAAO,KAAK;CACrE,MAAM,UACJ,aAAa,KAAA,IACT;EAAE,MAAM;EAAS,OAAO,MAAM,SAAS,QAAS,IAAI,SAAS;CAAM,IACnE,MAAM,OAAO,QACX,EAAE,MAAM,MAAM,IACd,SAAS,KAAA,IACP;EAAE,MAAM;EAAS,OAAO,OAAO,KAAK,QAAQ;CAAE,IAC9C,EAAE,MAAM,SAAS;CAC3B,MAAM,QAAQ,UAAU,QAAQ,MAAM;CACtC,MAAM,UACJ,aAAa,KAAA,KAAa,SAAS,KAAA,IAAY,QAAQ,SAAS,MAAM,KAAK,CAAC,IAAI;CAClF,MAAM,EAAE,eAAe;CAIvB,IAAI,CAAC,WAAW,WAAW,WAAW,aAAa,KAAA,GAAW;EAC5D,MAAM,EAAE,aAAa;EACrB,MAAM,OAAO,IAAI,QAAQ;EACzB,IAAI,WAAW,QAAQ,SAAS,MAAM,SAAS;OAC1C,IAAI,WAAW,QAAQ,SAAS,MAAM;GACzC,MAAM,OAAO,QAAQ,QAAQ,GAAG,KAAK,EAAE;GACvC,MAAM,KAAK,QAAQ,QAAQ,GAAG,KAAK,EAAE;GACrC,IACE,QAAQ,MAAM,GAAG,KAAK,QAAQ,SAAS,KAAK,KAAK,GAAG,KACpD,QAAQ,IAAI,GAAG,KAAK,QAAQ,SAAS,KAAK,MAAM,GAAG,GAEnD,SAAS,mBAAmB,KAAK,MAAM,GAAG,GAAG,MAAM,iCAAiC,SAAS,KAAK,IAAI,MAAM,GAAG,SAAS,KAAK,KAAK,MAAM;EAC5I;EACA,MAAM,QAAQ,cACX,QAAQ,WAAW,OAAO,OAAO,QAAQ,CAAC,CAAC,CAC3C,KAAK,WAAW,QAAQ,QAAQ,GAAG,OAAO,IAAI,CAAC;EAClD,IAAI,UAAU,KAAA,GACZ,OAAO;GACL,GAAG;GACH,GAAG;GACH;GACA;GACA,MAAM;GACN,SAAS,kBAAkB,QAAQ,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM;EAC7D;EAGF,MAAM,SACJ,SAAS,OACL,KAAA,IACA,OAAO,MAAM,WAAW,OAAO,OAAO,QAAQ,OAAO,OAAO,KAAK,EAAE;EACzE,IAAI,WAAW,KAAA,GACb,OAAO;GACL,GAAG;GACH,GAAG;GACH;GACA,MAAM;GACN,SAAS;EACX;EACF,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,UAAU,MAAM,IAAI,QAAQ,CAAC;EAC/E,WAAW,QAAQ,QAAQ,GAAG,OAAO,MAAM,YAAY;EACvD,OAAO;GACL,GAAG;GACH,SAAS,EAAE,MAAM,WAAW;GAC5B,SAAS,QAAQ,QAAQ,GAAG,OAAO,IAAI;GACvC,QAAQ,UAAU,KAAA,IAAY,OAAO,QAAQ,QAAQ,GAAG,MAAM,CAAC;GAC/D,SAAS,UAAU,KAAA,IAAY,OAAO,QAAQ,OAAO,MAAM,MAAM,CAAC;GAClE;GACA,MAAM;GACN,SAAS;EACX;CACF;CAEA,IAAI,CAAC,WAAW,SAAS;EAEvB,MAAM,SAAS,QAAQ,WAAW,QAAQ,GAAG,IAAI;EACjD,IAAI,UAAU,KAAA,KAAa,QAAQ,QAAQ,IAAI,QAC7C,OAAO;GAAE,GAAG;GAAM,GAAG;GAAS,MAAM;EAAO;EAC7C,OAAO;GACL,GAAG;GACH,GAAG;GACH;GACA,SAAS,QAAQ,QAAQ,GAAG,KAAK;GACjC,MAAM;GACN,SAAS,kBAAkB,QAAQ,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM;EAC7D;CACF;CAEA,IAAI,UAAU,KAAA,GACZ,OAAO;EACL,GAAG;EACH,GAAG;EACH,MAAM;EACN,SAAS,wBAAwB,WAAW,MAAM;CACpD;CAKF,MAAM,SAAS,IAAI,OAAO,MAAM,MAAM;CACtC,MAAM,kBACJ,WAAW,SAAS,kBAChB,OAAO,QAAQ,WAAW,OAAO,OAAO,QAAQ,OAAO,QAAQ,MAAM,CAAC,CAAC,GAAG,EAAE,IAC5E,WAAW,SAAS,yBAClB,cAAc,MAAM,WAAW,OAAO,QAAQ,KAAK,IACnD,KAAA;CACR,MAAM,YACJ,WAAW,SAAS,iBAAiB,QAAQ,IAAI,iBAAiB;CACpE,IAAI,cAAc,KAAA,GAChB,OAAO;EACL,GAAG;EACH,GAAG;EACH;EACA;EACA,MAAM;EACN,SAAS,yCAAyC,WAAW,KAAK;CACpE;CAEF,MAAM,SACJ,WAAW,SAAS,kBAChB,aACA,WAAW,SAAS,yBAClB,cAAc,QAAQ,WAAW,OAAO,QAAQ,SAAS,OAAO,OAAO,SAAS,IAChF,CAAC,EAAA,CACP,KAAK,WAAW,QAAQ,WAAW,OAAO,IAAI,CAAC;CACjD,MAAM,UACJ,QAAQ,SAAS,WAAW,QAAQ,UAAU,WAAW,QACrD,OACA,iBAAiB,QAAQ,SAAS,UAAU,QAAQ,QAAQ,cAAc,QAAQ,WAAW;CAGnG,IAAI,WAAW,SAAS,gBACtB,WAAW,QACT,WACA,OACA,WAAW,SAAS,kBAAkB,cAAc,YACtD;CACF,OAAO;EACL,GAAG;EACH;EACA,SAAS,QAAQ,WAAW,KAAK;EACjC,QAAQ,SAAS,OAAO,cAAc,MAAM,QAAQ,OAAO,OAAO;EAClE;EACA;EACA,MAAM;EACN;CACF;AACF;;;;;;;;;;AChUA,MAAa,QAAkB,SAAS,GAAG,IAAI;;AAE/C,MAAa,WAAW;;AAExB,MAAa,QAAQ;CAAC;CAAI;CAAI;AAAE;AA0BhC,MAAM,KAAK,UAA4B,SAAS,KAAK,MAAM,QAAQ,GAAG,IAAI,KAAK,GAAG;AAClF,MAAM,QAAQ,QAA0B,OAAO,IAAI,YAAY,OAAO,IAAI;;AAG1E,MAAM,SAAS,GAAW,MAAsB;CAC9C,IAAI,OAAO;CACX,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,KAAK,GAAG;EAC9B,OAAO;EACP,OAAQ,QAAQ,IAAI,MAAO,IAAI;CACjC;CACA,OAAO,MAAM,KAAK;AACpB;;;;;;AAOA,MAAa,kBAAkB,WAAgD;CAC7E,MAAM,IAAI,OAAO;CACjB,IAAI,IAAI;CACR,OAAO,IAAI,KAAK,IAAI,KAAK,MAAM,GAAG,CAAC,KAAK,MAAO,KAAK;CACpD,OAAO,MAAM,IAAI,CAAC,OAAO,IAAK,OAAO,IAAI,EAAG,IAAI,CAAC,OAAO,IAAI,IAAK,OAAO,IAAI,EAAG;AACjF;AAEA,MAAM,UAAU,WAAsC;CACpD,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,CAAC;CACzC,OAAO,OAAO,SAAS,MAAM,IAAI,OAAO,SAAU,OAAO,OAAO,KAAM,OAAO,SAAU;AACzF;;;;;;;;;;;;;AAcA,MAAa,aAAa,YAAwB,SAAuC;CACvF,MAAM,UAAU,KAAK,QAAQ,QAAQ,IAAI,SAAS,IAAI;CACtD,MAAM,SAAS,KAAK,SAAS,QAAQ;CACrC,MAAM,UAAU,QACb,QAAQ,QAAQ,IAAI,YAAY,IAAI,CAAC,CACrC,KAAK,SAAS;EAAE,KAAK,KAAK,GAAG;EAAG,KAAK,IAAI;CAAS,EAAE;CACvD,MAAM,QAAQ;EAAE,SAAS,QAAQ;EAAQ;EAAQ,QAAQ;EAAM,gBAAgB;CAAK;CACpF,IAAI,QAAQ,WAAW,GACrB,OAAO;EAAE,GAAG;EAAO,UAAU;EAAgB,MAAM;EAAM;EAAS,KAAK;CAAiB;CAE1F,IAAI,CAAC,WAAW,WAAW,WAAW,aAAa,KAAA,GAAW;EAC5D,MAAM,WAAW,QAAQ,WAAW,IAAI,UAAU;EAClD,MAAM,MACJ,aAAa,UACT,0BAA0B,QAAQ,OAAO,WAAW,QAAQ,WAAW,QAAQ,GAAG,EAAE,MACpF,GAAG,QAAQ,OAAO,MAAM,QAAQ,OAAO;EAC7C,OAAO;GAAE,GAAG;GAAO;GAAU,MAAM;GAAM;GAAS;EAAI;CACxD;CAGA,MAAM,QAAQ,WAAW,UAAU,aAAa,WAAW;CAC3D,MAAM,QAAQ,QAAQ,OAAO,GAAG;CAChC,MAAM,MAAM,QAAQ,MAAM,KAAK,GAAG;CAClC,MAAM,OAAO,QAAQ,MAAM,MAAM,GAAG;CACpC,MAAM,OAAO;EAAE,KAAK,EAAE,MAAM,KAAK;EAAG,MAAM,EAAE,OAAO,KAAK;CAAE;CAC1D,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,OAAO,SAAS;EACzB,IAAI,IAAI,YAAY,MAAM;EAC1B,MAAM,QAAQ,QAAQ,IAAI,SAAS,GAAG;EACtC,OAAO,KAAK,KAAK;EACjB,IAAI,QAAQ,MAAM,SAAS,QAAQ,OAAO,OACxC,QAAQ,KAAK;GACX,KAAK,KAAK,GAAG;GACb,KAAK,GAAG,MAAM,iBAAiB,KAAK,IAAI,MAAM,IAAI,KAAK,KAAK,MAAM;EACpE,CAAC;CACL;CACA,OAAO,MAAM,GAAG,MAAM,IAAI,CAAC;CAC3B,IAAI,OAAO,WAAW,GACpB,OAAO;EAAE,GAAG;EAAO,UAAU;EAAS;EAAM;EAAS,KAAK;CAA2B;CACvF,MAAM,CAAC,MAAM,MAAM,eAAe,MAAM;CACxC,MAAM,SAAS,EAAE,OAAO,MAAM,CAAC;CAC/B,MAAM,WAAW;EAAE,KAAK,EAAE,IAAI;EAAG,MAAM,EAAE,EAAE;CAAE;CAC7C,MAAM,OAAO;EAAE,KAAK,MAAO;EAAe,MAAM,OAAQ;CAAc;CACtE,MAAM,SAAS,QAAQ,KAAK,OAAO,MAAM,KAAK;CAC9C,MAAM,WAAW,KAAK,KAAK,OAAO,OAAO,KAAK;CAC9C,MAAM,SAAS,GAAG,QAAQ,SAAS,QAAQ,OAAO,MAAM,QAAQ,OAAO,gBAAgB,KAAK,IAAI,MAAM,IAAI,KAAK,KAAK,MAAM;CAC1H,MAAM,SAAS,UAAU,OAAO,MAAM,oBAAoB,SAAS,IAAI,MAAM,IAAI,SAAS,KAAK,MAAM;CACrG,MAAM,SAAS;EACb,SAAS,QAAQ;EACjB;EACA;EACA;EACA;EACA,gBAAgB;CAClB;CACA,IAAI,QAAQ,SAAS,KAAK,UACxB,OAAO;EAAE,GAAG;EAAQ,UAAU;EAAS,KAAK,GAAG,OAAO,IAAI;CAAS;CACrE,IAAI,QAAQ,OAAO;EAAE,GAAG;EAAQ,UAAU;EAAS,KAAK,GAAG,OAAO,IAAI;CAAS;CAC/E,MAAM,WAAW,QAAQ,SAAS,MAAM,KAAK,WAAW;CACxD,OAAO;EAAE,GAAG;EAAQ;EAAU,KAAK,GAAG,OAAO,IAAI,OAAO;CAAqB;AAC/E;;;;;;;;;ACxHA,MAAM,QAA0C;CAC9C,SAAS,cAAc,IAAI,IAAI,MAAM,YAAY,GAAG,CAAC;CACrD,QAAQ,QACN,aAAa,cAAc,YAAY,GAAG,CAAC,CAAC,QAAQ,oCAAoC,CAAC,CAC3F;AACF;;AAGA,MAAM,UAAU,SAAyB;CACvC,MAAM,CAAC,MAAM,GAAG,QAAQ,KAAK,MAAM,GAAG;CACtC,MAAM,OAAO,MAAM;CACnB,IAAI,SAAS,KAAA,GAAW,MAAM,IAAI,MAAM,GAAG,KAAK,uCAAuC;CACvF,OAAO,KAAK,MAAM,GAAG,IAAI;AAC3B;;AAGA,MAAM,mBAAmD,OAAO;;AAUhE,MAAM,UAAiB;CACrB,MAAM;CACN,YAAY;CACZ,MAAM;EACJ;EACA;EACA;EACA;EACA;CACF;AACF;;AAEA,MAAM,OAAc;CAClB,MAAM;CACN,YAAY;CACZ,MAAM,CAAC,oBAAoB;AAC7B;;AAGA,MAAM,WAAW,SAA2B;CAC1C,IAAI,SAAS,IAAI,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI;CAEzC,OADgB,YAAY,MAAM,EAAE,eAAe,KAAK,CAC3C,CAAC,CACX,QAAQ,UAAU,MAAM,SAAS,YAAY,MAAM,SAAS,cAAc,CAAC,CAC3E,SAAS,UAAU,QAAQ,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,CAAC,CACnD,KAAK;AACV;;AAGA,MAAM,WAAW,UACf,MAAM,KAAK,SAAS,SAAS;CAC3B,MAAM,OAAO,OAAO,IAAI;CACxB,OAAO,QAAQ,IAAI,CAAC,CAAC,KAAK,UAAU;EAClC;EACA,IAAI,KAAK,MAAM,SAAS,MAAM,IAAI,CAAC,CAAC,CAAC,WAAW,MAAM,GAAG;CAC3D,EAAE;AACJ,CAAC;;AAGH,MAAM,YAAY,UAAyB;CACzC,MAAM,OAAO,WAAW,QAAQ;CAChC,KAAK,MAAM,EAAE,MAAM,QAAQ,QAAQ,KAAK,GAAG;EACzC,KAAK,OAAO,EAAE;EACd,KAAK,OAAO,aAAa,IAAI,CAAC;CAChC;CACA,OAAO,GAAG,MAAM,KAAK,GAAG,KAAK,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE;AACxD;;AAGA,MAAM,SAAS,UAAyB;CACtC,MAAM,UAAU,YAAY,KAAK,OAAO,GAAG,GAAG,MAAM,KAAK,EAAE,CAAC;CAC5D,KAAK,MAAM,EAAE,MAAM,QAAQ,QAAQ,KAAK,GAAG;EACzC,UAAU,QAAQ,KAAK,SAAS,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,CAAC;EACzD,aAAa,MAAM,KAAK,SAAS,EAAE,CAAC;CACtC;CACA,OAAO;AACT;AAEA,MAAM,wBAAQ,IAAI,IAA6B;;AAE/C,MAAM,SAAS,UAAkC;CAC/C,IAAI,MAAM,MAAM,IAAI,MAAM,IAAI;CAC9B,IAAI,QAAQ,KAAA,GAAW;EACrB,OAAO,YAAY;GACjB,MAAM,OAAO,SAAS,KAAK;GAC3B,MAAM,EAAE,qBAAqB,MAAM,WAAW;GAC9C,MAAM,UAAU,MAAM,KAAK;GAC3B,IAAI;IACF,MAAM,iBAAiB,eAAe,SAAS,MAAM,UAAU,CAAC,CAAC,MAAM,MAAM,EAC3E,cAAc,MAChB,CAAC;GACH,UAAU;IACR,OAAO,SAAS;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC;GAClD;GACA,OAAO;EACT,EAAA,CAAG;EACH,MAAM,IAAI,MAAM,MAAM,GAAG;CAC3B;CACA,OAAO;AACT;;AAGA,MAAa,qBAAsC,MAAM,OAAO;;AAEhE,MAAa,wBAAyC,MAAM,IAAI;AAkDhE,MAAM,cAA2D;CAC/D,MAAM,OAAY,CAAC;CACnB,MAAM,QAAkB,CAAC;CACzB,MAAM,UAA2E,CAAC;CAClF,IAAI,UAAU;CACd,MAAM,QAAQ,SAAiB;EAC7B,IAAI,CAAC,KAAK,WAAW,GAAG,GAAG;GACzB,IAAI,KAAK,SAAS,GAAG,MAAM,KAAK,IAAI;GACpC;EACF;EACA,MAAM,OAAO,KAAK,MAAM,IAAI;EAC5B,KAAK,KAAK,IAAI;EACd,KAAK,MAAM,QAAQ,QAAQ,OAAO,CAAC,GACjC,IAAI,KAAK,MAAM,SAAS,KAAK,EAAE,GAAG,KAAK,QAAQ,IAAI;OAC9C,QAAQ,KAAK,IAAI;CAE1B;CACA,OAAO;EACL;EACA;EACA,UAAU,WACR,OAAO,GAAG,SAAS,UAA2B;GAC5C,WAAW,MAAM,SAAS;GAC1B,MAAM,QAAQ,QAAQ,MAAM,IAAI;GAChC,UAAU,MAAM,IAAI,KAAK;GACzB,KAAK,MAAM,QAAQ,OAAO,KAAK,KAAK,KAAK,CAAC;EAC5C,CAAC;EACH,MAAM,OAAO,QAAQ,KAAK;GACxB,MAAM,QAAQ,KAAK,MAAM,SAAS,MAAM,SAAS,KAAK,EAAE,CAAC;GACzD,IAAI,UAAU,KAAA,GAAW,OAAO,QAAQ,QAAQ,KAAK;GACrD,OAAO,IAAI,SAAS,SAAS,WAAW;IACtC,MAAM,QAAQ,iBAAiB;KAC7B,MAAM,OAAO,MAAM,WAAW,IAAI,KAAK,eAAe,MAAM,MAAM,EAAE,CAAC,CAAC,KAAK,KAAK;KAChF,uBAAO,IAAI,MAAM,MAAM,MAAM,KAAK,MAAM,EAAE,YAAY,IAAI,MAAM,OAAO,KAAK,MAAM,CAAC;IACrF,GAAG,MAAM;IACT,QAAQ,KAAK;KACX;KACA,UAAU,SAAS;MACjB,aAAa,KAAK;MAClB,QAAQ,IAAI;KACd;IACF,CAAC;GACH,CAAC;EACH;CACF;AACF;AAmEA,MAAM,OAAO,EAAE,SAAS,EAAE;AAE1B,MAAM,SAAS,OAAO,WAAiC,SAAoC;CACzF,MAAM,SAAS,MAAM,UAAU,KAAK,IAAI;CACxC,IAAI,OAAO,aAAa,GACtB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK,GAAG,EAAE,UAAU,OAAO,SAAS,IAAI,OAAO,OAAO,KAAK,GAAG;CACxF,OAAO,OAAO,OAAO,KAAK;AAC5B;;AAGA,MAAM,UAAU,OAAO,cAAqD;CAC1E,MAAM,OAAO,MAAM,OAAO,WAAW,CAAC,YAAY,mBAAmB,CAAC;CACtE,MAAM,QAAQ,kBAAkB,KAAK,IAAI,CAAC,GAAG;CAC7C,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,4BAA4B,MAAM;CAC3E,OAAO,OAAO,KAAK;AACrB;;;;;;;;;;;;;;AAeA,MAAa,UAAU,OAAO,QAAgB,OAAe,UAAwB;CACnF,MAAM,QAAQ,MAAM,aAAa;CACjC,MAAM,cAAc,SAAS,YAAY,MAAM,gBAAgB,IAAI;CACnE,MAAM,UACJ,SAAS,YAAY,CAAC,QAAQ,sCAAsC,IAAI,CAAC,qBAAqB;CAChG,MAAM,EAAE,kBAAkB,SAAS,MAAM,8BAA8B,MAAM,WAAW;CACxF,MAAM,SAAiC,CAAC;CACxC,IAAI;CACJ,MAAM,QAAQ,YAAY;EACxB,KAAK,MAAM,aAAa,OAAO,QAAQ,GAAG,MAAM,UAAU,KAAK,IAAI,CAAC,CAAC,YAAY,KAAA,CAAS;EAC1F,MAAM,SAAS,KAAK,CAAC,CAAC,YAAY,KAAA,CAAS;CAC7C;CACA,IAAI;EACF,UAAU,MAAM,IAAI,QAAQ,CAAC,CAAC,MAAM;EACpC,MAAM,SAAS,MAAM,IAAI,iBAAiB,OAAO,KAAK,CAAC,CACpD,YAAY,OAAO,CAAC,CACpB,iBAAiB,KAAK,cAAc,OAAO,KAAK,CAAC,CAAC,CAClD,MAAM;EACT,OAAO,KAAK,MAAM;EAClB,MAAM,SAAS,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAAC,YAAY,OAAO,CAAC,CAAC,MAAM;EAC5E,OAAO,KAAK,MAAM;EAClB,MAAM,QAAQ,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAC5C,gBAAgB,aAAa,OAAO,MAAM,GAAG,CAAC,CAC9C,mBAAmB,CAAC,CACpB,MAAM;EACT,OAAO,KAAK,KAAK;EACjB,MAAM,WAAW,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAC/C,gBAAgB,aAAa,OAAO,MAAM,GAAG,CAAC,CAC9C,sBAAsB,aAAa,SAAS,CAAC,CAC7C,MAAM;EACT,OAAO,KAAK,QAAQ;EACpB,MAAM,QAAQ;GAAE,QAAQ,MAAM,QAAQ,KAAK;GAAG,QAAQ,MAAM,QAAQ,QAAQ;EAAE;EAC9E,MAAM,UAAU,MAAoB;EACpC,MAAM,SAAS,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAC7C,mBAAmB,CAAC,CACpB,gBAAgB,MAAM,CAAC,CACvB,YAAY;GAAC;GAAuB,OAAO,MAAM,MAAM;GAAG,OAAO,MAAM,MAAM;EAAC,CAAC,CAAC,CAChF,gBAAgB,QAAQ,OAAO,CAAC,CAChC,MAAM;EACT,OAAO,KAAK,MAAM;EAClB,MAAM,QAAQ,MAAM,QAAQ,MAAM,CAAC,SAAS,MAAM,GAAG,KAAQ,QAAQ;EACrE,IAAI,MAAM,OAAO,QAAQ,MAAM,IAAI,MAAM,sBAAsB,OAAO,MAAM,MAAM,GAAG;EACrF,MAAM,WAAW,OAAO,aAAa,QAAQ,QAAQ,CAAC;EACtD,MAAM,WAAW,OAAO,aAAa,QAAQ,QAAQ,CAAC;EACtD,MAAM,UAAU,MAAM,0BAA0B;EAChD,MAAM,SAAS,QAAQ,UAAU,QAAQ,OAAO,MAAM,CAAC;EACvD,IAAI;EAEJ,OAAO;GACL;GACA;GACA;GACA,QAAQ,QAAQ;GAChB,MAAM,OAAO,QAAQ;IACnB,MAAM,WAAW,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,GAAG,IAAI,GAAG,OAAO;IAC/E,IAAI,SAAS,SAAS,GAAG,MAAM,OAAO,OAAO;KAAC;KAAU;KAAM;KAAM,GAAG;IAAQ,CAAC;IAChF,MAAM,OAA+B,CAAC;IACtC,KAAK,MAAM,OAAO,OAAO,KAAK,MAAM,GAClC,KAAK,OAAO,MAAM,OAAO,OAAO;KAAC;KAAU;KAAM;IAAG,CAAC;IACvD,OAAO;GACT;GACA,MAAM,aAAa,QAAQ;IACzB,MAAM,OAA+B,CAAC;IACtC,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC,WAAW,GAAG,OAAO;IAE7C,gBAAgB,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAC9C,gBAAgB,aAAa,OAAO,MAAM,GAAG,CAAC,CAC9C,mBAAmB,CAAC,CACpB,MAAM;IACT,OAAO,KAAK,WAAW;IACvB,MAAM,WAAW,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,GAAG,IAAI,GAAG,OAAO;IAC/E,MAAM,OAAO,aAAa;KAAC;KAAU;KAAM;KAAM,GAAG;IAAQ,CAAC;IAC7D,KAAK,MAAM,OAAO,OAAO,KAAK,MAAM,GAClC,KAAK,OAAO,MAAM,OAAO,aAAa;KAAC;KAAU;KAAM;IAAG,CAAC;IAC7D,OAAO;GACT;GACA,MAAM,UAAU,QAAQ;IACtB,MAAM,OAA+B,CAAC;IACtC,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC,WAAW,GAAG,OAAO;IAC7C,MAAM,MAAM,OAAO;IACnB,IAAI,QAAQ,KAAA,GAAW,MAAM,IAAI,MAAM,GAAG,OAAO,MAAM,kCAAkC;IACzF,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO,QAAQ,MAAM,GAAG;KAClD,MAAM,OAAO,QAAQ;MAAC,GAAG,IAAI;MAAK;MAAM;KAAK,CAAC;KAC9C,KAAK,SAAS,MAAM,OAAO,QAAQ,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,EAAA,CAAG,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;IAChF;IACA,OAAO;GACT;GACA,MAAM,MAAM,MAAM,KAAK,UAAU;IAC/B,MAAM,CAAC,OAAO,QAAQ,OAAO,WAAW,CAAC,OAAO,QAAQ,IAAI,CAAC,UAAU,QAAQ;IAC/E,MAAM,QAAQ,KAAK,MAAM,MAAM,OAAO,OAAO;KAAC;KAAuB;KAAM;IAAI,CAAC,CAAC;IAIjF,OAAO;KAAE,IAAI,MAAM;KAAI,IAAI,MAAM;IAAG;GACtC;GACA,MAAM,KAAK,MAAM,KAAK,UAAU;IAC9B,MAAM,CAAC,OAAO,QAAQ,OAAO,WAAW,CAAC,OAAO,QAAQ,IAAI,CAAC,UAAU,QAAQ;IAE/E,MAAM,OAAO,OAAO;KAAC;KADN,SAAS,cAAc,UAAU;KACI;IAAI,CAAC;GAC3D;GACA,MAAM,KAAK,IAAI,MAAM;IACnB,MAAM,CAAC,OAAO,QAAQ,OAAO,WAAW,CAAC,OAAO,QAAQ,IAAI,CAAC,UAAU,QAAQ;IAE/E,MAAM,QAAQ,KAAK,MAAM,MAAM,OAAO,OAAO;KAD/B;KAAuB;KAAS;KAAM;KAAQ,OAAO,KAAK,MAAM,IAAI,CAAC;IACnC,CAAC,CAAC;IAClD,OAAO;KAAE,IAAI,MAAM;KAAI,IAAI,MAAM;IAAG;GACtC;GACA,MAAM,SAAS;IACb,MAAM,OAAO,OAAO;KAAC;KAAM;KAAe;KAAS;IAAK,CAAC;IACzD,MAAM,OAAO,UAAU;KAAC;KAAM;KAAe;KAAS;IAAK,CAAC;GAC9D;GACA,cAAc,OAAO,MAAM,CAAC,CAAC,WAAW,KAAA,CAAS;GACjD,YAAY,OAAO,QAAQ,CAAC,CAAC,WAAW,KAAA,CAAS;GACjD,MAAM,MAAM,MAAM;IAChB,MAAM,SAAS,MAAkB;IACjC,MAAM,YAAY,MAAM,IAAI,iBAAiB,WAAW,CAAC,CACtD,gBAAgB,aAAa,OAAO,MAAM,GAAG,CAAC,CAC9C,YAAY,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAClC,gBAAgB,OAAO,OAAO,CAAC,CAC/B,MAAM;IACT,OAAO;KACL,QAAQ,OAAO;KACf,QAAQ,OAAO,WAAW,OAAO,MAAM,OAAO,QAAQ,QAAQ;KAC9D,UAAU,OAAO,WAAW,CAAC,SAAS,SAAS,CAAC,CAAC,CAAC,WAAW,KAAA,CAAS;KACtE,cACE,QAAQ,UACL,QAAQ,UAAU,MAAM,CAAC,CAAC,CAC1B,MAAM,CAAC,CACP,WAAW,KAAA,CAAS;KACzB,YACE,QAAQ,UACL,QAAQ,UAAU,MAAM,CAAC,CAAC,CAC1B,QAAQ,CAAC,CACT,WAAW,KAAA,CAAS;KACzB,YAAY,UAAU,KAAK,IAAI,CAAC,CAAC,WAAW,KAAA,CAAS;IACvD;GACF;GACA,MAAM,SAAS;IAEb,MAAM,QAAQ,QAAQ,KAAK;IAC3B,MAAM,QAAQ,MAAM,IAAI,SAA8B,YAAY;KAChE,MAAM,UAAU,KAAK,IAAI;KACzB,MAAM,aAAa;MACjB,MAAM,OAAO,QAAQ,KAAK,MAAM,KAAK,CAAC,CAAC,MAAM,WAAW,OAAO,OAAO,OAAO;MAC7E,IAAI,SAAS,KAAA,GAAW,QAAQ,IAAI;WAC/B,IAAI,KAAK,IAAI,IAAI,UAAU,KAAO,QAAQ,IAAI;WAC9C,WAAW,MAAM,EAAE;KAC1B;KACA,KAAK;IACP,CAAC;IACD,OAAO,UAAU,OAAO,OAAO,OAAO,MAAM,WAAW,CAAC;GAC1D;GACA,MAAM,SAAS,SAAS,WAAW,CAAC,GAAG,gBAAgB,CAAC,GAAG;IACzD,MAAM,SAAS,MAAM,OAAO,OAAO,CAAC,QAAQ,iBAAiB,CAAC,CAAC,CAAC,YAAY,EAAE;IAC9E,MAAM,UAAU,OAAO,cACrB,QAAQ,UAAU,QAAQ,QAAQ,UAAU,QAAQ,UAAU,MAAM,CAAC,CAAC;IACxE,OAAO;KACL,QAAQ,MAAM,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC;KAC5C,cAAc,MAAM,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC;KAClD,IAAI,qBAAqB,KAAK,MAAM,CAAC,GAAG,MAAM;KAC9C,SAAS,MAAM,OAAO,QAAQ,CAAC,OAAO,iCAAiC,CAAC;KACxE,QAAQ,GAAG,QAAQ,KAAK,iBAAiB,cAAc,MAAM,QAAQ,KAAK,iBAAiB;KAC3F,QAAQ;MACN,OAAO,OAAO;MACd,KAAK,MAAM,QAAQ,MAAM,EAAA,CAAG;MAC5B;MACA,SAAS;KACX;KACA,SAAS;MACP;MACA,KAAK,MAAM,QAAQ,MAAM,EAAA,CAAG;MAC5B,WAAW,MAAM,OAAO,QAAQ;OAAC;OAAO;OAAQ;MAAI,CAAC,EAAA,CAAG,MAAM,IAAI,CAAC,CAAC,KAAK;KAC3E;KACA,QAAQ;MAAE;MAAM,OAAO;KAAY;KACnC;KACA;IACF;GACF;GACA;EACF;CACF,SAAS,OAAO;EACd,MAAM,MAAM;EACZ,MAAM;CACR;AACF;;;;;;;;ACreA,MAAa,SACX,MACA,SAC+C;CAC/C,MAAM,WAAW,KAAK,KAAK,QAAQ,QAAQ,MAAM,GAAG,CAAC;CACrD,OAAO;EAAE;EAAU,SAAS,UAAU,KAAK,YAAY,QAAQ;CAAE;AACnE;;AAGA,MAAM,gBAAgB,YAA6C;CACjE,KAAK,IAAI,KAAK,QAAQ,SAAS,GAAG,MAAM,GAAG,MAAM,GAC/C,IAAI,QAAQ,GAAG,CAAE,OAAO,SAAS,OAAO,OAAO,QAAQ,GAAG,CAAE,WAAW,CAAC;CAC1E,OAAO;AACT;;AAGA,MAAM,aAAa,MAAY,QAAuB;CACpD,MAAM,EAAE,YAAY,UAAU;CAC9B,MAAM,UAAU,WAAW,UACvB,QAAQ,WAAW,MAAM,GAAG,IAC5B,QAAQ,WAAW,QAAQ,GAAG;CAClC,MAAM,OAAO;EAAC,IAAI;EAAU,OAAO,KAAK,OAAO,IAAI;EAAG;EAAU,MAAM;CAAI;CAC1E,IAAI,MAAM,YAAY,KAAA,GAAW,KAAK,KAAK,aAAa,MAAM,OAAO;CACrE,IAAI,MAAM,WAAW,KAAA,GAAW;EAE9B,KAAK,KAAK,kBAAkB,OAAO,QAAQ,MAAM,OAAO,WAAW,IAAI,CAAC,CAAC;EACzE,KAAK,KAAK,WAAW,OAAO,KAAK,KAAK,UAAU,IAAI,EAAE,CAAC,CAAC;EACxD,OAAO;CACT;CACA,IAAI,MAAM,WAAW,KAAK,KAAK,aAAa;CAC5C,IAAI,MAAM,gBAAgB,KAAA,GACxB,KAAK,KAAK,qBAAqB,OAAO,QAAQ,MAAM,aAAa,IAAI,CAAC,CAAC;CACzE,KAAK,KAAK,WAAW,OAAO,KAAK,KAAK,UAAU,IAAI,EAAE,CAAC,CAAC;CACxD,OAAO;AACT;;;;;;AAOA,MAAM,eAAe,KAAU,OAAgB,WAC7C,IAAI,SAAS,YAAY;CACvB,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,aAAa;EAQjB,IAPa,IAAI,OAAO,MACrB,WACC,OAAO,OAAO,YACb,OAAO,OAAO,OAAO,KACtB,OAAO,UAAU,IAAI,MAAM,UAC3B,OAAO,QAAQ,KAEZ,KAAK,KAAK,IAAI,IAAI,UAAU,QAAQ,QAAQ;OAC9C,WAAW,MAAM,EAAE;CAC1B;CACA,KAAK;AACP,CAAC;;;;;AAMH,MAAM,eAAe,OAAc,WACjC,IAAI,SAAS,SAAS,WAAW;CAC/B,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,aAAa;EACjB,MAAM,OAAO,MAAM,OAAO,QAAQ,UAAU,MAAM,OAAO,MAAM,CAAC,CAAC;EACjE,IAAI,SAAS,KAAA,GAAW,QAAQ,IAAI;OAC/B,IAAI,KAAK,IAAI,IAAI,UAAU,QAC9B,uBAAO,IAAI,MAAM,qCAAqC,OAAO,qBAAqB,CAAC;OAChF,WAAW,MAAM,CAAC;CACzB;CACA,KAAK;AACP,CAAC;;;;;;;;;;;;AAaH,MAAM,OAAO,OAAO,KAAU,MAAY,aAAqB,UAAgC;CAE7F,MAAM,IAAI,OAAO;CACjB,MAAM,OAAO,IAAI,OAAO;CACxB,MAAM,aAAa,aAAa,IAAI,MAAM;CAC1C,MAAM,QAAQ,MAAM,IAAI,MAAM,UAAU,MAAM,GAAG,CAAC;CAClD,MAAM,SAAS,KAAK,aAAa;CACjC,MAAM,OAAO,SACT;EAAE,cAAc,MAAM,OAAO;EAAG,YAAY,MAAM,KAAK;CAAE,IACzD;EAAE,cAAc,IAAI,OAAO;EAAG,YAAY,IAAI,KAAK;CAAE;CAEzD,MAAM,WACJ,MAAM,GAAG,CAAC,CAAC,OAAO,UAAmB;EACnC,MAAM,OAAO,MAAM,OAAO,MAAM,EAAE,CAAC,CAAC,KAAK,UAAU,KAAK,UAAU,KAAK,CAAC;EACxE,MAAM,IAAI,MAAM,GAAI,MAAgB,QAAQ,6BAA6B,KAAK,KAAK,GAAG,GAAG;CAC3F,CAAC;CACH,IAAI;CACJ,IAAI,OAA0C;CAC9C,IAAI;EACF,MAAM,MAAM,MAAM,CAAC,OAAO,GAAG,IAAM;EAGnC,MAAM,gBAAgB,UAAU,KAAK,MAAM,SAAS;EACpD,IAAI,eAAe;GAGjB,MAAM,GAAG;GACT,MAAM,IAAI,SAAS,YAAY,WAAW,SAAS,GAAG,CAAC;EACzD;EACA,MAAM,UAAU,SAAS,WAAW;EAEpC,MAAM,UAAU,KAAK,UAAU,UAAU,cAAc,KAAK;EAC5D,QAAQ,YAAY,mBAAmB,OAAO,MAAM,IAAI,MAAM,SAAS,OAAO;EAC9E,IAAI,YAAY,kBAAkB,MAAM,KAAK,OAAO;EACpD,IAAI;GACF,IAAI,KAAK,MAAM,SAAS,cAAc,CAAC,eAAe,MAAM,GAAG;GAC/D,MAAM,EAAE,eAAe;GACvB,IAAI,KAAK,UAAU,SAAS;IAC1B,IAAI,KAAK,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK,wBAAwB;IAEnF,MAAM,OAAO,MAAM,YAAY,OAAO,GAAK;IAC3C,OAAO,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,QAAQ,KAAK,OAAO,GAAG,IAAI,GAAG;IACtE,MAAM,SAAS,QAAQ,WAAW,UAAU,WAAW,OAAO,WAAW,QAAQ,GAAG;IACpF,MAAM,MAAM,MAAM;KAAC;KAAQ;KAAS;IAAK,GAAG,SAAS,GAAK,CAAC,CAAC,YAAY,KAAA,CAAS;GACnF,OAAO,IAAI,WAAW,SAAS;IAC7B,MAAM,SAAS,QAAQ,WAAW,MAAM,GAAG,IAAI,MAAQ;IACvD,IAAI,QAAQ;KACV,MAAM,QAAQ,MAAM,OAAO,MAAM,UAAU,MAAM,OAAO,WAAW,CAAC,GAAG;KACvE,MAAM,YAAY,KAAK,OAAO,MAAM;IACtC,OACE,MAAM,MAAM,MAAM;KAAC;KAAS;KAAO;IAAU,GAAG,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GAEjF,OACE,MAAM,IAAI,SAAS,YACjB,WAAW,SAAS,QAAQ,WAAW,QAAQ,GAAG,IAAI,GAAK,CAC7D;EAEJ,UAAU;GACR,IAAI,YAAY,kBAAkB,MAAM,KAAK,KAAK;QAC7C,IAAI,SAAS,MAAM,MAAM,IAAI,KAAK,SAAS,OAAO;EACzD;EAEA,IAAI,KAAK,UAAU,oBAAoB,CAAC,QACtC,MAAM,MAAM,MAAM,CAAC,MAAM,GAAG,GAAK,CAAC,CAAC,YAAY,KAAA,CAAS;CAC5D,UAAU;EACR,MAAM,MAAM,KAAK;CACnB;CAEA,MAAM,UAAU,MAAM,IAAI,OAAO;CACjC,MAAM,OAAO,YAAY,OAAO,OAAO,UAAU;CACjD,OAAO;EACL;EACA;EACA,QAAQ,CAAC,GAAG,MAAM,MAAM;EACxB;EACA,GAAI,KAAK,UAAU,UAAU,EAAE,KAAK,IAAI,CAAC;EACzC,QAAQ,IAAI,OAAO,MAAM,IAAI;EAC7B,OAAO,IAAI,MAAM;EACjB,aAAa,IAAI,MAAM;EACvB;CACF;AACF;;AAGA,MAAa,UAAU,MAAc,iBACnC,MAAM,KACJ,EAAE,QAAQ,aAAa,IACtB,GAAG,OAAO,KAAK,MAAM,OAAO,YAAY,KAAK,KAAK,OAAO,eAAe,IAAI,EAC/E,CAAC,CAAC,QAAQ,UAAU,QAAQ,CAAC;;AAG/B,MAAM,gBAAgB,OACpB,MACA,aACA,OACA,QACiD;CACjD,MAAM,MAAM,MAAM,QAAQ,KAAK,QAAQ,KAAK,MAAM,QAAQ,QAAQ,KAAK;CACvE,IAAI;EAEF,MAAM,WAAW,MAAM,IAAI,UAAU,KAAK,kBAAkB,CAAC,CAAC;EAC9D,MAAM,gBAAgB,MAAM,IAAI,aAAa,KAAK,iBAAiB,CAAC,CAAC;EACrE,MAAM,WAAW,MAAM,IAAI,SAAS,MAAM,IAAI,OAAO,KAAK,OAAO,GAAG,UAAU,aAAa;EAC3F,MAAM,OAAc,CAAC;EACrB,KAAK,IAAI,QAAQ,GAAG,QAAQ,OAAO,SAAS,GAAG;GAC7C,MAAM,MAAM,MAAM,KAAK,KAAK,MAAM,aAAa,KAAK;GACpD,KAAK,KAAK,GAAG;GACb,MAAM,OAAO,QAAQ,MAAM,GAAG;GAC9B,IACE,GAAG,KAAK,KAAK,QAAQ,YAAY,OAAO,MAAM,GAAG,KAAK,QAAQ,OACzD,KAAK,YAAY,OAAO,KAAK,IAAI,KAAK,QAAQ,MAAM,MACpD,KAAK,SAAS,OAAO,KAAK,WAAW,KAAK,KAAK,IACtD;EACF;EACA,OAAO;GAAE;GAAU;EAAK;CAC1B,UAAU;EACR,MAAM,IAAI,MAAM;CAClB;AACF;;;;;;;;AASA,MAAa,UAAU,OACrB,MACA,YAAoC,CAAC,MACb;CACxB,MAAM,2BAAU,IAAI,KAAK,EAAA,CAAE,YAAY;CACvC,MAAM,YAAwB,CAAC;CAC/B,MAAM,OAAc,CAAC;CACrB,MAAM,cAAc,KAAK,IAAI,GAAG,KAAK,eAAe,CAAC;CACrD,IAAI,SAAiB,KAAK;CAC1B,IAAI,OAAO;CACX,SAAS;EACP,MAAM,SAAS,OAAO,SAAS,KAAK,QAAQ,KAAK,IAAI,KAAK,cAAc,WAAW,CAAC;EACpF,KAAK,IAAI,KAAK,GAAG,KAAK,OAAO,QAAQ,MAAM,aAAa;GACtD,MAAM,QAAQ,OAAO,MAAM,IAAI,KAAK,WAAW,CAAC,CAAC,KAAK,WAAW;IAAE;IAAO,KAAK;GAAO,EAAE;GACxF,MAAM,OAAO,MAAM,QAAQ,IACzB,MAAM,KAAK,EAAE,OAAO,UAAU,cAAc,MAAM,KAAK,OAAO,GAAG,CAAC,CACpE;GACA,KAAK,MAAM,QAAQ,MAAM;IACvB,UAAU,KAAK,KAAK,QAAQ;IAC5B,KAAK,KAAK,GAAG,KAAK,IAAI;GACxB;EACF;EACA,MAAM,EAAE,UAAU,YAAY,MAAM,MAAM,IAAI;EAC9C,MAAM,SAAS,MAAM,MAAM,SAAS,OAAO,KAAK,MAAM;EACtD,IAAI,QAAQ,aAAa,YAAY,WAAW,KAAA,GAE9C,OAAO;GAAE;GAAM;GAAW;GAAM;GAAU;GAAS,IAAA;IADtC;IAAS,wBAAO,IAAI,KAAK,EAAA,CAAE,YAAY;GACA;EAAE;EAExD,SAAS;CACX;AACF"}

package/image/Dockerfile

@@ -0,0 +1,30 @@
+# The harness's own image: the prober, the cell client, the injector and the namespace admin all
+# run from it. A target's image stays stock. Both stages are pinned by digest,
+# the crates by Cargo.lock, and the packages apk installs are recorded in each run's manifest.
+# lab.ts stages the build context: the prober's and the harness's native sources, and the prober's
+# identity fixture, each under its package's name.
+
+# --- build: the prober's kernel program in C (CO-RE), the user side in Rust --------------------
+FROM alpine:3.23@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62 AS build
+RUN apk add --no-cache clang llvm bpftool libbpf-dev linux-headers \
+      rust cargo build-base elfutils-dev zlib-dev zstd-dev pkgconf
+WORKDIR /src
+COPY prober/bpf/ prober/bpf/
+# the kernel's own types, from the kernel the build runs on: the program relocates against them
+RUN bpftool btf dump file /sys/kernel/btf/vmlinux format c > prober/bpf/vmlinux.h \
+ && clang -O2 -g -target bpf -D__TARGET_ARCH_x86 -Iprober/bpf -c prober/bpf/prober.bpf.c -o prober.bpf.o
+COPY prober/native/ prober/native/
+# the record tests read the prober's identity fixture
+COPY prober/fixtures/ prober/fixtures/
+RUN cd prober/native && cargo build --release --locked --features collector \
+ && cargo test --release --locked --features collector
+COPY harness/image/native/ harness/image/native/
+RUN cd harness/image/native && cargo build --release --locked && cargo test --release --locked
+
+# --- run ------------------------------------------------------------------------------------------
+FROM alpine:3.23@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62
+RUN apk add --no-cache iproute2 iptables libelf zlib zstd-libs libgcc
+COPY --from=build /src/prober.bpf.o /src/prober/native/target/release/prober /opt/harness/
+COPY --from=build /src/harness/image/native/target/release/client /src/harness/image/native/target/release/inject /opt/harness/
+WORKDIR /opt/harness
+CMD ["sleep", "infinity"]

package/image/native/Cargo.lock

@@ -0,0 +1,123 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "harness-native"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "prober",
+ "serde_json",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "libc"
+version = "0.2.190"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "prober"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

package/image/native/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "harness-native"
+version = "0.1.0"
+edition = "2021"
+publish = false
+description = "The harness's native side: the cell client and the injector"
+
+[dependencies]
+# the clock, the quantities and the errno names, without the collector
+prober = { path = "../../../prober/native" }
+libc = "0.2"
+serde_json = "1"
+
+[profile.release]
+lto = true
+codegen-units = 1

package/image/native/src/bin/client.rs

@@ -0,0 +1,224 @@
+//! client HOST PORT --mode idle|inflight [--keepalive] [--user-timeout-ms N] [--go PATH]
+//!        [--request TEXT] [--sample-ms N] [--max-s N]
+//!
+//! A cell's TCP client: one connection, every event a JSON line stamped on CLOCK_MONOTONIC. It
+//! sets no per-socket TCP timer unless the cell asks, so the namespace's sysctls are what gets
+//! measured: Rust's std::net turns nothing on by itself. While it waits, it samples the kernel's
+//! view of its own socket (TCP_INFO) every --sample-ms and logs each change, a second vantage
+//! beside the prober's records. It waits in poll(), never holding its socket's lock, so it never
+//! puts off the socket's own keepalive timer.
+//!
+//! Both modes send one request and read its reply first; that reply is the last segment received.
+//! idle: then hold the connection idle until it fails.
+//! inflight: then wait for the file --go to appear, send the request again, and wait. --request
+//! names that second request, a Redis inline command: PING unless it says otherwise. BLPOP with a
+//! timeout makes the server answer only when the timeout runs out, so its reply can be the one in
+//! flight when a fault lands.
+
+use prober::{errno_name, monotonic_ns, quantity};
+use serde_json::{json, Value};
+use std::error::Error;
+use std::io::{ErrorKind, Read, Write};
+use std::net::TcpStream;
+use std::os::fd::AsRawFd;
+use std::path::Path;
+use std::time::Duration;
+
+/// a Redis inline command; any line-based server echoes or errors on it, which is a reply too
+const REQUEST: &[u8] = b"PING\r\n";
+
+fn emit(event: &str, fields: Value) {
+    emit_at(event, monotonic_ns(), fields);
+}
+
+fn emit_at(event: &str, t: u64, fields: Value) {
+    let mut line = json!({ "ev": event, "t": t });
+    if let (Some(line), Value::Object(fields)) = (line.as_object_mut(), fields) {
+        line.extend(fields);
+    }
+    println!("{line}");
+}
+
+/// Sends the request, stamped before the write: on a bridge, the kernel can take a reset to the
+/// request before write() returns, and a stamp after it would come later than the reset's own.
+fn send(stream: &mut TcpStream, request: &[u8]) -> std::io::Result<()> {
+    let t = monotonic_ns();
+    stream.write_all(request)?;
+    emit_at("sent", t, json!({}));
+    Ok(())
+}
+
+fn set_int(fd: i32, level: i32, name: i32, value: i32) -> std::io::Result<()> {
+    // SAFETY: the option value is an int on this stack, passed with its exact size
+    let status = unsafe {
+        libc::setsockopt(
+            fd,
+            level,
+            name,
+            (&value as *const i32).cast(),
+            std::mem::size_of::<i32>() as libc::socklen_t,
+        )
+    };
+    if status == 0 { Ok(()) } else { Err(std::io::Error::last_os_error()) }
+}
+
+fn tcp_info(fd: i32) -> std::io::Result<libc::tcp_info> {
+    // SAFETY: tcp_info is plain old data, and the kernel writes at most the length it is given
+    let mut info: libc::tcp_info = unsafe { std::mem::zeroed() };
+    let mut len = std::mem::size_of::<libc::tcp_info>() as libc::socklen_t;
+    let status = unsafe {
+        libc::getsockopt(fd, libc::IPPROTO_TCP, libc::TCP_INFO, (&mut info as *mut libc::tcp_info).cast(), &mut len)
+    };
+    if status == 0 { Ok(info) } else { Err(std::io::Error::last_os_error()) }
+}
+
+/// Whether the socket has something to read, an error or an end, within `ms`: poll() asks the
+/// socket without taking its lock.
+fn readable(fd: i32, ms: i32) -> std::io::Result<bool> {
+    let mut wanted = libc::pollfd { fd, events: libc::POLLIN | libc::POLLRDHUP, revents: 0 };
+    // SAFETY: one pollfd on this stack, and its count
+    match unsafe { libc::poll(&mut wanted, 1, ms) } {
+        -1 if std::io::Error::last_os_error().kind() == ErrorKind::Interrupted => Ok(false),
+        -1 => Err(std::io::Error::last_os_error()),
+        _ => Ok(wanted.revents != 0),
+    }
+}
+
+/// what in TCP_INFO is worth a line when it changes: the timers' own state, never the ages
+fn watched(info: &libc::tcp_info) -> [u32; 7] {
+    [
+        info.tcpi_state as u32,
+        info.tcpi_retransmits as u32,
+        info.tcpi_probes as u32,
+        info.tcpi_backoff as u32,
+        info.tcpi_rto,
+        info.tcpi_unacked,
+        info.tcpi_total_retrans,
+    ]
+}
+
+struct Args {
+    host: String,
+    port: u16,
+    inflight: bool,
+    keepalive: bool,
+    user_timeout_ms: Option<i32>,
+    go: String,
+    sample: Duration,
+    max: Duration,
+    /// the request sent at --go, with its line ending
+    request: Vec<u8>,
+}
+
+fn args() -> Result<Args, Box<dyn Error>> {
+    let raw: Vec<String> = std::env::args().skip(1).collect();
+    let value = |flag: &str| raw.iter().position(|arg| arg == flag).and_then(|at| raw.get(at + 1)).cloned();
+    let mode = value("--mode").ok_or("--mode idle|inflight is required")?;
+    if mode != "idle" && mode != "inflight" {
+        return Err(format!("--mode is idle or inflight, not {mode}").into());
+    }
+    Ok(Args {
+        host: raw.first().ok_or("the target's host comes first")?.clone(),
+        port: raw.get(1).ok_or("then its port")?.parse()?,
+        inflight: mode == "inflight",
+        keepalive: raw.iter().any(|arg| arg == "--keepalive"),
+        user_timeout_ms: value("--user-timeout-ms").map(|v| v.parse()).transpose()?,
+        go: value("--go").unwrap_or_else(|| "/tmp/go".into()),
+        sample: Duration::from_micros((value("--sample-ms").map(|v| v.parse::<f64>()).transpose()?.unwrap_or(2.0) * 1000.0) as u64),
+        max: Duration::from_secs_f64(value("--max-s").map(|v| v.parse::<f64>()).transpose()?.unwrap_or(60.0)),
+        request: value("--request").map_or_else(|| REQUEST.to_vec(), |r| format!("{r}\r\n").into_bytes()),
+    })
+}
+
+fn run() -> Result<(), Box<dyn Error>> {
+    let args = args()?;
+    let boot_id = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?.trim().to_string();
+    emit("start", json!({ "boot_id": boot_id, "mode": if args.inflight { "inflight" } else { "idle" } }));
+    let deadline = monotonic_ns() + args.max.as_nanos() as u64;
+
+    let mut stream = TcpStream::connect((args.host.as_str(), args.port))?;
+    stream.set_nodelay(true)?;
+    let fd = stream.as_raw_fd();
+    if args.keepalive {
+        set_int(fd, libc::SOL_SOCKET, libc::SO_KEEPALIVE, 1)?;
+    }
+    if let Some(ms) = args.user_timeout_ms {
+        set_int(fd, libc::IPPROTO_TCP, libc::TCP_USER_TIMEOUT, ms)?;
+    }
+    emit("connected", json!({ "local": stream.local_addr()?.to_string() }));
+    send(&mut stream, REQUEST)?;
+    let mut buf = [0u8; 65536];
+    let n = stream.read(&mut buf)?;
+    // the last segment received: a keepalive's idle time counts from here
+    let info = tcp_info(fd)?;
+    emit(
+        "ready",
+        json!({
+            "reply": String::from_utf8_lossy(&buf[..n.min(40)]),
+            "rtt": quantity(info.tcpi_rtt, "us"),
+            "rto": quantity(info.tcpi_rto, "us"),
+        }),
+    );
+
+    // the loop waits in poll(), which takes no lock on the socket, and reads only what is there. A
+    // read that waits takes the socket's lock each time it wakes, and a keepalive timer that finds
+    // the socket held puts itself off by HZ/20: the client would move the timer
+    // it measures
+    stream.set_nonblocking(true)?;
+    let tick = args.sample.as_millis().clamp(1, i32::MAX as u128) as i32;
+    let mut waiting = args.inflight;
+    let mut seen: Option<[u32; 7]> = None;
+    let mut age = info.tcpi_last_ack_recv;
+    loop {
+        if readable(fd, tick)? {
+            match stream.read(&mut buf) {
+                Ok(0) => {
+                    emit("eof", json!({}));
+                    return Ok(());
+                }
+                Ok(n) => emit("data", json!({ "n": n, "text": String::from_utf8_lossy(&buf[..n.min(40)]) })),
+                // nothing after all; ETIMEDOUT is the connection's own end
+                Err(error) if error.kind() == ErrorKind::WouldBlock || error.kind() == ErrorKind::Interrupted => {}
+                Err(error) => {
+                    let code = error.raw_os_error().unwrap_or(0);
+                    emit("error", json!({ "errno": errno_name(code) }));
+                    return Ok(());
+                }
+            }
+        }
+        let info = tcp_info(fd)?;
+        let now = watched(&info);
+        if seen != Some(now) {
+            emit(
+                "kernel",
+                json!({
+                    "state": now[0], "retransmits": now[1], "probes": now[2], "backoff": now[3],
+                    "rto": quantity(now[4], "us"), "unacked": now[5], "total_retrans": now[6],
+                    "last_ack_recv": quantity(info.tcpi_last_ack_recv, "ms"),
+                    "last_data_recv": quantity(info.tcpi_last_data_recv, "ms"),
+                }),
+            );
+            seen = Some(now);
+        }
+        // an ACK arrived since the last sample: what a frozen process's kernel keeps sending
+        if info.tcpi_last_ack_recv < age {
+            emit("ack", json!({ "last_ack_recv": quantity(info.tcpi_last_ack_recv, "ms") }));
+        }
+        age = info.tcpi_last_ack_recv;
+        if waiting && Path::new(&args.go).exists() {
+            send(&mut stream, &args.request)?;
+            waiting = false;
+        }
+        if monotonic_ns() > deadline {
+            emit("deadline", json!({}));
+            return Ok(());
+        }
+    }
+}
+
+fn main() {
+    if let Err(error) = run() {
+        emit("fail", json!({ "why": error.to_string() }));
+        std::process::exit(1);
+    }
+}