@mark1russell7/harness 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +111 -2
- package/build/index.d.ts +546 -0
- package/build/index.js +1126 -0
- package/build/index.js.map +1 -0
- package/image/Dockerfile +31 -0
- package/image/native/Cargo.lock +123 -0
- package/image/native/Cargo.toml +16 -0
- package/image/native/src/bin/client.rs +263 -0
- package/image/native/src/bin/inject.rs +268 -0
- package/image/node/Dockerfile +11 -0
- package/image/node/client-ioredis.mjs +105 -0
- package/image/node/npm-lock.json +96 -0
- package/image/node/package.json +8 -0
- package/package.json +76 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../src/observe.ts","../src/verdict.ts","../src/lab.ts","../src/run.ts"],"sourcesContent":["import type { Quantity } from \"@mark1russell7/quantity\";\nimport { quantity, valueIn } from \"@mark1russell7/quantity\";\nimport type { Cell, KernelRecord, ProbeEvent, Run } from \"./cell.ts\";\n\n/**\n * An outcome tells how the wait of a run ended. The socket failed with an errno, or it read the\n * end of the file. Or nothing came, or the reply came back after a stall.\n */\nexport type Outcome =\n | { readonly kind: \"error\"; readonly errno: string }\n | { readonly kind: \"eof\" }\n | { readonly kind: \"silent\" }\n | { readonly kind: \"answered\" };\n\n/** A measurement is what one run measured, against the prediction of its cell. */\nexport interface Measured {\n readonly environment: number;\n readonly index: number;\n readonly outcome: Outcome;\n /**\n * This time is from the reference event of the prediction to the detection, both as the kernel\n * stamped them in the records of the prober. It is null when nothing detected the fault.\n */\n readonly elapsed: Quantity | null;\n /** This time is the same interval from the TCP_INFO samples of the client, the second vantage. */\n readonly socket: Quantity | null;\n /** This time is how long the client took to see the error after the kernel reported it. */\n readonly noticed: Quantity | null;\n /**\n * These are the steps of the kernel after the reference: each keepalive probe, or each firing of\n * the RTO timer. After a stall, they are each retransmission from the send, the loss probe first.\n */\n readonly steps: readonly Quantity[];\n /**\n * These are the reply times of a measurement-only cell. For each request after the first, a\n * sample is the time from its send to the first data after it.\n */\n readonly samples?: readonly Quantity[];\n /** This count is the number of keepalive probes that the kernel sent. */\n readonly probes: number;\n /** This count is the number of segments that the kernel sent again. */\n readonly retransmissions: number;\n /** This count is the number of segments that the socket received after the request. */\n readonly acks: number;\n /** This text is the reason that the run cannot count, or null. A void run is not a pass and not a finding. */\n readonly void: string | null;\n /** This text tells what in the run disagrees with the prediction, other than its timing, or null. */\n readonly finding: string | null;\n}\n\n/** These are the errno numbers that the kernel reports to a socket, by the names that predictions use. */\nconst ERRNO: Readonly<Record<number, string>> = {\n 32: \"EPIPE\",\n 101: \"ENETUNREACH\",\n 103: \"ECONNABORTED\",\n 104: \"ECONNRESET\",\n 110: \"ETIMEDOUT\",\n 111: \"ECONNREFUSED\",\n 113: \"EHOSTUNREACH\",\n};\n\n/** This function gives the interval between two instants on the one clock, in seconds to the microsecond. */\nconst between = (from: number, to: number): Quantity =>\n quantity(Math.round((to - from) / 1e3) / 1e6, \"s\");\n\n/** These functions give the states of a state record: before the change, and after it. TCP_ESTABLISHED is 1, TCP_CLOSE_WAIT 8. */\nconst oldState = (record: KernelRecord): number | undefined => (record as { readonly old?: number }).old;\nconst newState = (record: KernelRecord): number | undefined => (record as { readonly new?: number }).new;\n\nconst count = (event: ProbeEvent | undefined, field: string): number =>\n event === undefined ? 0 : Number(event[field] ?? 0);\n\n/** This function gives the view of the client on the same interval: TCP_INFO, with a sample each 2 ms. */\nconst socketVantage = (\n cell: Cell,\n events: readonly ProbeEvent[],\n ready: ProbeEvent,\n request: ProbeEvent,\n): Quantity | null => {\n if (!cell.prediction.detects || cell.prediction.from === \"peer-closed\") return null;\n const ended = events.find((event) => event.ev === \"error\" || event.ev === \"eof\");\n const reference =\n cell.prediction.from === \"last-received\"\n ? ready\n : cell.prediction.from === \"request-sent\"\n ? request\n : events.find(\n (event) =>\n event.ev === \"kernel\" && event.t >= request.t && count(event, \"retransmits\") > 0,\n );\n return ended === undefined || reference === undefined ? null : between(reference.t, ended.t);\n};\n\n/**\n * This function reads one run against its cell, from the records of the kernel first.\n * - The socket of the run is the socket that the client connected from, in its namespace.\n * - The fault must hold before the event that it must meet. That is before the request goes into\n * it, and before the first keepalive probe. A live peer answers that probe, and the answer\n * starts the idle time again. A run where the fault was late is void. A run whose records the\n * prober lost is also void.\n */\nexport const measure = (cell: Cell, run: Run): Measured => {\n const { events, kernel } = run;\n const ready = events.find((event) => event.ev === \"ready\");\n const local = events.find((event) => event.ev === \"connected\")?.[\"local\"];\n const inflight = cell.probe.mode !== \"idle\";\n const request = inflight ? events.filter((event) => event.ev === \"sent\")[1] : ready;\n // the socket of the observer: the client socket, by the address that it connected from, or the\n // socket of the target for the same connection, in its namespace, whose peer is that address\n const server = cell.observer === \"server\";\n const netns = server ? run.targetNetns : run.netns;\n const cookie = kernel.find(\n (record) => record.netns === netns && (server ? record.dst : record.src) === local,\n )?.cookie;\n const socket = kernel.filter((record) => record.netns === netns && record.cookie === cookie);\n // the socket of the peer for the same connection, in the other namespace: a FIN counts from the\n // change of state that its close makes\n const peerNetns = server ? run.netns : run.targetNetns;\n const peerCookie = kernel.find(\n (record) => record.netns === peerNetns && (server ? record.src : record.dst) === local,\n )?.cookie;\n const keepalives = socket.filter(\n (record) => record.ev === \"probe\" && record.what === \"keepalive\",\n );\n const retransmitted = socket.filter((record) => record.ev === \"retrans\");\n /**\n * This function examines the timer that drives the measurement: keepalive or retransmit. It\n * finds the first firing in an interval that was two jiffies or more behind its wheel slot. The\n * wheel does a slot in the tick when it is due, thus such a delay comes from the CPU: the host\n * stopped a vCPU. The prediction is about the timing of the kernel, and that firing measured\n * the timing of the host. One jiffy is a tick that occurred while the wheel worked, and δ_abs\n * includes it. The function ignores the other timer: the retransmit timer fires one time, idle,\n * after a reply, and it does not move a keepalive.\n */\n const delayed = (from: number, to: number, timer: \"keepalive\" | \"retransmit\"): string | null => {\n const firings = socket.filter(\n (record) =>\n record.ev === \"timer\" && record.what === timer && record.mono >= from && record.mono <= to,\n );\n const late = firings.find((record) => (record.behind?.value ?? 0) >= 2);\n if (late !== undefined)\n return `its ${timer} timer ran ${late.behind!.value} jiffies behind its wheel slot: the VM delayed it, not the kernel`;\n // a keepalive timer that finds its socket held by its owner sends nothing, and it tries again\n // HZ/20 later (tcp_keepalive_timer). The retransmit timer only gives its work to the release\n // by the owner, some microseconds later\n const held = firings.find((record) => timer === \"keepalive\" && record.owned === true);\n return held === undefined\n ? null\n : \"its keepalive timer found the socket held by its owner and put itself off by HZ/20: the run measured the owner, not the timer\";\n };\n const acks =\n request === undefined\n ? 0\n : socket.filter((record) => record.ev === \"rx\" && record.mono > request.t).length;\n const base = {\n environment: run.environment,\n index: run.index,\n probes: keepalives.length,\n retransmissions: retransmitted.length,\n acks,\n };\n const nothing = {\n outcome: { kind: \"silent\" } as const,\n elapsed: null,\n socket: null,\n noticed: null,\n steps: [],\n finding: null,\n };\n\n if (ready === undefined) return { ...base, ...nothing, void: \"the client never had its reply\" };\n if (request === undefined)\n return { ...base, ...nothing, void: \"the request was never sent into the fault\" };\n if (run.lost === null)\n return {\n ...base,\n ...nothing,\n void: \"the prober wrote no clock line after the run, so nothing says what it dropped\",\n };\n if (run.lost > 0) return { ...base, ...nothing, void: `the prober dropped ${run.lost} records` };\n if (cookie === undefined)\n return { ...base, ...nothing, void: \"the prober saw nothing of this connection\" };\n let voided: string | null = null;\n if (run.fault !== null) {\n // the request of a client must go into the fault. The request of a cell on the server side must\n // reach the server first, and its reply must not reach the client before the fault holds. An\n // mtu-blackhole drops only the long reply, thus its request goes after the fault, as a client's does\n const replyFirst = server && inflight && cell.fault !== \"mtu-blackhole\";\n const replied = events.some(\n (event) => event.ev === \"data\" && event.t > request.t && event.t < run.fault!.t1,\n );\n if (inflight && !replyFirst && run.fault.t1 > request.t)\n voided = \"the request went out before the fault held\";\n else if (replyFirst && run.fault.t1 < request.t)\n voided = \"the request went out into the fault, so the server never had it\";\n else if (replyFirst && replied)\n voided = \"the server's reply arrived before the fault held\";\n else if (keepalives[0] !== undefined && keepalives[0].mono < run.fault.t1)\n voided = \"a keepalive probe went out before the fault held\";\n }\n\n // the detection as the kernel reported it to the owner of the socket, and as the client saw it.\n // The process of a server tells nothing that the harness reads, thus its kernel tells all\n const reported = socket.find((record) => record.ev === \"error\" && (record.err ?? 0) !== 0);\n const seen = server\n ? undefined\n : events.find((event) => event.ev === \"error\" || event.ev === \"eof\");\n // an end of file without an error is the change of state that the FIN of the peer makes, from\n // ESTABLISHED to CLOSE_WAIT, as the kernel stamps it\n const closeWait = socket.find((record) => record.ev === \"state\" && newState(record) === 8);\n const outcome: Outcome =\n reported !== undefined\n ? { kind: \"error\", errno: ERRNO[reported.err!] ?? `E${reported.err}` }\n : closeWait !== undefined || seen?.ev === \"eof\"\n ? { kind: \"eof\" }\n : seen !== undefined\n ? { kind: \"error\", errno: String(seen[\"errno\"]) }\n : { kind: \"silent\" };\n const ended = reported?.mono ?? closeWait?.mono ?? seen?.t;\n const noticed =\n reported !== undefined && seen !== undefined ? between(reported.mono, seen.t) : null;\n const { prediction } = cell;\n\n // a cell whose outcome is a distribution: each request of the run, and the time to its reply\n if (prediction.detects === null) {\n const sends = events.filter((event) => event.ev === \"sent\").slice(1);\n const samples = sends.flatMap((send, n) => {\n const next = sends[n + 1]?.t ?? Infinity;\n const reply = events.find((event) => event.ev === \"data\" && event.t > send.t && event.t < next);\n return reply === undefined ? [] : [between(send.t, reply.t)];\n });\n return {\n ...base,\n ...nothing,\n outcome: ended !== undefined ? outcome : { kind: \"answered\" },\n steps: retransmitted\n .filter((record) => record.mono > request.t)\n .map((record) => between(request.t, record.mono)),\n samples,\n void: voided,\n };\n }\n\n // a stall: nothing must detect it, and the connection must move again at the first\n // retransmission after it stops, timed from the send of the request\n if (!prediction.detects && prediction.recovers !== undefined) {\n const { recovers } = prediction;\n const heal = run.heal ?? null;\n if (voided === null && heal === null) voided = \"the stall never ended\";\n else if (voided === null && heal !== null) {\n const from = between(request.t, heal.t0);\n const to = between(request.t, heal.t1);\n if (\n valueIn(from, \"s\") <= valueIn(recovers.heal.low, \"s\") ||\n valueIn(to, \"s\") >= valueIn(recovers.heal.high, \"s\")\n )\n voided = `the stall ended ${from.value}–${to.value} s after the send, outside the ${recovers.heal.low.value}–${recovers.heal.high.value} s its prediction assumed`;\n }\n const steps = retransmitted\n .filter((record) => record.mono > request.t)\n .map((record) => between(request.t, record.mono));\n if (ended !== undefined)\n return {\n ...base,\n ...nothing,\n outcome,\n steps,\n void: voided,\n finding: `detected after ${between(request.t, ended).value} s, where it should recover`,\n };\n // the first segment back after the stall stopped: the answer of the server to the\n // retransmission that got through, and the read of the reply by the client\n const answer =\n heal === null\n ? undefined\n : socket.find((record) => record.ev === \"rx\" && record.mono > heal.t0);\n if (answer === undefined)\n return {\n ...base,\n ...nothing,\n steps,\n void: voided,\n finding: \"it never moved again: nothing came back after the stall ended\",\n };\n const reply = events.find((event) => event.ev === \"data\" && event.t > request.t);\n voided ??= delayed(request.t, answer.mono, \"retransmit\");\n return {\n ...base,\n outcome: { kind: \"answered\" },\n elapsed: between(request.t, answer.mono),\n socket: reply === undefined ? null : between(request.t, reply.t),\n noticed: reply === undefined ? null : between(answer.mono, reply.t),\n steps,\n void: voided,\n finding: null,\n };\n }\n\n if (!prediction.detects) {\n // the test of silence covers the window from the send of the request\n const window = valueIn(prediction.window, \"s\") * 1e9;\n if (ended === undefined || ended - request.t > window)\n return { ...base, ...nothing, void: voided };\n return {\n ...base,\n ...nothing,\n outcome,\n elapsed: between(request.t, ended),\n void: voided,\n finding: `detected after ${between(request.t, ended).value} s, where nothing should`,\n };\n }\n\n if (ended === undefined)\n return {\n ...base,\n ...nothing,\n void: voided,\n finding: `nothing detected it: ${prediction.timer} never fired`,\n };\n // keepalive counts from the last segment received before the fault. The budget of the give-up\n // counts from the first retransmission of the RTO timer, not from the tail loss probe before it.\n // A reset counts from the send of the client. The kernel has no record of that send, thus the\n // stamp is the stamp of the client\n const before = run.fault?.t1 ?? ready.t;\n const kernelReference: KernelRecord | undefined =\n prediction.from === \"last-received\"\n ? socket.filter((record) => record.ev === \"rx\" && record.mono <= before).at(-1)\n : prediction.from === \"first-retransmission\"\n ? retransmitted.find((record) => record.why === \"rto\")\n : prediction.from === \"peer-closed\"\n ? kernel.find(\n (record) =>\n record.netns === peerNetns &&\n record.cookie === peerCookie &&\n record.ev === \"state\" &&\n oldState(record) === 1,\n )\n : undefined;\n const reference: number | undefined =\n prediction.from === \"request-sent\" ? request.t : kernelReference?.mono;\n if (reference === undefined) {\n return {\n ...base,\n ...nothing,\n outcome,\n noticed,\n void: voided,\n finding: `it ended without its reference event (${prediction.from})`,\n };\n }\n const steps = (\n prediction.from === \"last-received\"\n ? keepalives\n : prediction.from === \"first-retransmission\"\n ? retransmitted.filter((record) => record.why === \"rto\" && record.mono > reference)\n : []\n ).map((record) => between(reference, record.mono));\n const matches =\n prediction.errno === \"EOF\"\n ? outcome.kind === \"eof\"\n : outcome.kind === \"error\" && outcome.errno === prediction.errno;\n const finding = matches\n ? null\n : `it ended with ${outcome.kind === \"error\" ? outcome.errno : \"end of file\"}, not ${prediction.errno}`;\n // a firing at the reference itself only moves the start of the interval. A reset meets the send\n // with no timer, and a FIN arrives with none\n if (prediction.from !== \"request-sent\" && prediction.from !== \"peer-closed\")\n voided ??= delayed(\n reference,\n ended,\n prediction.from === \"last-received\" ? \"keepalive\" : \"retransmit\",\n );\n return {\n ...base,\n outcome,\n elapsed: between(reference, ended),\n socket: server ? null : socketVantage(cell, events, ready, request),\n noticed,\n steps,\n void: voided,\n finding,\n };\n};\n","import type { Quantity } from \"@mark1russell7/quantity\";\nimport { quantity, valueIn } from \"@mark1russell7/quantity\";\nimport type { Prediction } from \"./cell.ts\";\nimport type { Measured } from \"./observe.ts\";\n\n/**\n * This amount is the noise of the pipeline itself, δ_abs, as the calibration measured it. It is\n * the 99.9th percentile of the whole latency of a reset, at the concurrency cap or less. Its\n * minimum is one jiffy. On 2026-10-12 the worst of 60 resets at 1 and 4 environments took\n * 1.141 ms, thus the minimum sets it to 4 ms. The calibration occurs again after each change to\n * the harness, the kernel or Docker.\n */\nexport const NOISE: Quantity = quantity(4, \"ms\");\n/** This fraction, δ_rel, is how far the median of a cell can be outside the prediction and stay equivalent. */\nexport const RELATIVE = 0.05;\n/** These are the run counts of a cell: 11, then 22 and 44 while its median crosses the edge of the band. */\nexport const STEPS = [11, 22, 44] as const;\n\n/**\n * A decision is the result of a verdict. A cell whose outcome is a distribution has no band, and\n * its decision is `measured`: a record, not a test.\n */\nexport type Decision = \"holds\" | \"fails\" | \"extend\" | \"inconclusive\" | \"measured\";\n\n/** An interval is a range of times. */\nexport interface Interval {\n readonly low: Quantity;\n readonly high: Quantity;\n}\n\n/** A verdict is the judgement of a cell on its runs. */\nexport interface Verdict {\n readonly decision: Decision;\n readonly counted: number;\n readonly voided: number;\n /** This band is where each run must land: [T_lo − δ_abs, T_hi + δ_abs]. It is null for a silent prediction. */\n readonly band: Interval | null;\n /** These are the runs outside the band, and the reason for each. Each is a finding. */\n readonly outside: ReadonlyArray<{ readonly run: string; readonly why: string }>;\n readonly median: Quantity | null;\n /** This interval is the order-statistic interval of the median at 95 %, or the whole range for fewer than six runs. */\n readonly medianInterval: Interval | null;\n readonly why: string;\n}\n\nconst s = (value: number): Quantity => quantity(Math.round(value * 1e6) / 1e6, \"s\");\nconst name = (run: Measured): string => `env ${run.environment} run ${run.index}`;\n\n/** This function gives P(X ≤ j) for X ~ Binomial(n, 1/2), exactly. */\nconst below = (n: number, j: number): number => {\n let term = 1;\n let sum = 0;\n for (let i = 0; i <= j; i += 1) {\n sum += term;\n term = (term * (n - i)) / (i + 1);\n }\n return sum / 2 ** n;\n};\n\n/**\n * This function gives the distribution-free interval of a median: [x_(k), x_(n+1−k)], for the\n * largest k whose two tails each hold 2.5 % or less. For eleven runs, it gives [x_(2), x_(10)],\n * at 98.8 %.\n */\nexport const medianInterval = (sorted: readonly number[]): [number, number] => {\n const n = sorted.length;\n let k = 0;\n while (k + 1 <= n / 2 && below(n, k) <= 0.025) k += 1;\n return k === 0 ? [sorted[0]!, sorted[n - 1]!] : [sorted[k - 1]!, sorted[n - k]!];\n};\n\nconst middle = (sorted: readonly number[]): number => {\n const half = Math.floor(sorted.length / 2);\n return sorted.length % 2 === 1 ? sorted[half]! : (sorted[half - 1]! + sorted[half]!) / 2;\n};\n\n/**\n * This function gives the verdict of a cell, from two tests:\n * - Containment: each run lands in [T_lo − δ_abs, T_hi + δ_abs]. A run outside is a finding.\n * - Median equivalence: the interval of the median is in the band, made wider by δ_rel. An\n * interval outside the band fails. An interval across its edge extends the cell to 22 runs, then\n * 44, and then the verdict is inconclusive.\n *\n * A silent prediction holds only if no run detected the fault in its window. After a stall, the\n * return of the reply gets the same test as a detection, by its interval, and a detection is a\n * finding. The distribution test (a DKW band) needs a predicted distribution from a simulator,\n * which does not exist at this time.\n */\nexport const verdictOf = (prediction: Prediction, runs: readonly Measured[]): Verdict => {\n const counted = runs.filter((run) => run.void === null);\n const voided = runs.length - counted.length;\n const outside = counted\n .filter((run) => run.finding !== null)\n .map((run) => ({ run: name(run), why: run.finding! }));\n const empty = { counted: counted.length, voided, median: null, medianInterval: null };\n if (counted.length === 0)\n return { ...empty, decision: \"inconclusive\", band: null, outside, why: \"no run counted\" };\n\n if (prediction.detects === null) {\n const samples = counted\n .flatMap((run) => (run.samples ?? []).map((sample) => valueIn(sample, \"s\")))\n .sort((a, b) => a - b);\n const retransmissions = counted.reduce((sum, run) => sum + run.retransmissions, 0);\n const ended = counted.filter((run) => run.outcome.kind === \"error\" || run.outcome.kind === \"eof\");\n const why =\n samples.length === 0\n ? `${counted.length} runs measured, with no reply: ${prediction.why}`\n : `${counted.length} runs, ${samples.length} replies: median ${s(middle(samples)).value} s, slowest ${s(samples.at(-1)!).value} s, ${retransmissions} retransmissions, ${ended.length} connections ended. ${prediction.why}`;\n return {\n ...empty,\n decision: \"measured\",\n band: null,\n outside: [],\n median: samples.length === 0 ? null : s(middle(samples)),\n why,\n };\n }\n\n if (!prediction.detects && prediction.recovers === undefined) {\n const decision = outside.length === 0 ? \"holds\" : \"fails\";\n const why =\n decision === \"holds\"\n ? `nothing detected it in ${counted.length} runs of ${valueIn(prediction.window, \"s\")} s`\n : `${outside.length} of ${counted.length} runs detected it`;\n return { ...empty, decision, band: null, outside, why };\n }\n\n // the interval of a detection, or of the return of the reply after a stall\n const timed = prediction.detects ? prediction : prediction.recovers!;\n const noise = valueIn(NOISE, \"s\");\n const low = valueIn(timed.low, \"s\");\n const high = valueIn(timed.high, \"s\");\n const band = { low: s(low - noise), high: s(high + noise) };\n const values: number[] = [];\n for (const run of counted) {\n if (run.elapsed === null) continue;\n const value = valueIn(run.elapsed, \"s\");\n values.push(value);\n if (value < low - noise || value > high + noise)\n outside.push({\n run: name(run),\n why: `${value} s is outside [${band.low.value}, ${band.high.value}] s`,\n });\n }\n values.sort((a, b) => a - b);\n if (values.length === 0)\n return { ...empty, decision: \"fails\", band, outside, why: \"no run measured its time\" };\n const [from, to] = medianInterval(values);\n const median = s(middle(values));\n const interval = { low: s(from), high: s(to) };\n const wide = { low: low * (1 - RELATIVE), high: high * (1 + RELATIVE) };\n const inside = from >= wide.low && to <= wide.high;\n const disjoint = to < wide.low || from > wide.high;\n const counts = `${counted.length - outside.length} of ${counted.length} runs inside [${band.low.value}, ${band.high.value}] s`;\n const centre = `median ${median.value} s, its interval [${interval.low.value}, ${interval.high.value}] s`;\n const result = {\n counted: counted.length,\n voided,\n band,\n outside,\n median,\n medianInterval: interval,\n };\n if (outside.length > 0 || disjoint)\n return { ...result, decision: \"fails\", why: `${counts}; ${centre}` };\n if (inside) return { ...result, decision: \"holds\", why: `${counts}; ${centre}` };\n const decision = counted.length < STEPS[2] ? \"extend\" : \"inconclusive\";\n return { ...result, decision, why: `${counts}; ${centre} straddles the band` };\n};\n","import { createHash } from \"node:crypto\";\nimport {\n copyFileSync,\n mkdirSync,\n mkdtempSync,\n readFileSync,\n readdirSync,\n realpathSync,\n rmSync,\n statSync,\n} from \"node:fs\";\nimport { createRequire } from \"node:module\";\nimport { tmpdir } from \"node:os\";\nimport { dirname, join, relative } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport type { StartedNetwork, StartedTestContainer } from \"testcontainers\";\nimport type * as Testcontainers from \"testcontainers\";\nimport type { Client, KernelRecord, ProbeEvent, Target } from \"./cell.ts\";\n\n/**\n * These are the roots of the two packages that the images read: this package, and the prober\n * that Node resolves for it. In a workspace, they are two folders side by side. From npm, the\n * prober is a link, and a Docker build context does not follow a link. Thus each image builds\n * from a context that the harness copies from the two roots.\n */\nconst ROOTS: Readonly<Record<string, string>> = {\n harness: fileURLToPath(new URL(\"..\", import.meta.url)),\n prober: dirname(\n realpathSync(createRequire(import.meta.url).resolve(\"@mark1russell7/prober/package.json\")),\n ),\n};\n\n/** This function gives the path of a part of an image, which is `<package>/<path>` in the context. */\nconst pathOf = (part: string): string => {\n const [root, ...rest] = part.split(\"/\");\n const base = ROOTS[root!];\n if (base === undefined) throw new Error(`${part} is in no package that the images read`);\n return join(base, ...rest);\n};\n\n/** This function loads Testcontainers when a lab first opens. Code that only measures and judges does not load it. */\nconst containers = (): Promise<typeof Testcontainers> => import(\"testcontainers\");\n\n/** An image is an image that the harness builds: its Dockerfile, and each file or folder that the Dockerfile reads. */\ninterface Image {\n readonly name: string;\n readonly dockerfile: string;\n readonly from: readonly string[];\n}\n\n/** This image holds the prober, the TCP client, the injector and the namespace admin. */\nconst HARNESS: Image = {\n name: \"loom-harness\",\n dockerfile: \"harness/image/Dockerfile\",\n from: [\n \"harness/image/Dockerfile\",\n \"harness/image/native\",\n \"prober/bpf\",\n \"prober/native\",\n \"prober/fixtures\",\n ],\n};\n/** This image holds the Node client: ioredis 6.0.0 on Node 24.21.0. */\nconst NODE: Image = {\n name: \"loom-harness-node\",\n dockerfile: \"harness/image/node/Dockerfile\",\n from: [\"harness/image/node\"],\n};\n\n/** This function gives each file under a path, in a stable sequence, but not the output of a local build. */\nconst sources = (path: string): string[] => {\n if (statSync(path).isFile()) return [path];\n const entries = readdirSync(path, { withFileTypes: true });\n return entries\n .filter((entry) => entry.name !== \"target\" && entry.name !== \"node_modules\")\n .flatMap((entry) => sources(join(path, entry.name)))\n .sort();\n};\n\n/** This function gives each file of an image, with its path in the build context. */\nconst filesOf = (image: Image): Array<{ readonly file: string; readonly at: string }> =>\n image.from.flatMap((part) => {\n const base = pathOf(part);\n return sources(base).map((file) => ({\n file,\n at: join(part, relative(base, file)).replaceAll(\"\\\\\", \"/\"),\n }));\n });\n\n/** This function gives the tag of an image from its content. Thus a changed probe or client cannot operate as an old one. */\nconst imageTag = (image: Image): string => {\n const hash = createHash(\"sha256\");\n for (const { file, at } of filesOf(image)) {\n hash.update(at);\n hash.update(readFileSync(file));\n }\n return `${image.name}:${hash.digest(\"hex\").slice(0, 12)}`;\n};\n\n/** This function copies the files of an image into a new build context, and gives its path. */\nconst stage = (image: Image): string => {\n const context = mkdtempSync(join(tmpdir(), `${image.name}-`));\n for (const { file, at } of filesOf(image)) {\n mkdirSync(dirname(join(context, at)), { recursive: true });\n copyFileSync(file, join(context, at));\n }\n return context;\n};\n\nconst built = new Map<string, Promise<string>>();\n/** This function builds an image one time for each process. Docker keeps its layers between runs. */\nconst build = (image: Image): Promise<string> => {\n let tag = built.get(image.name);\n if (tag === undefined) {\n tag = (async () => {\n const name = imageTag(image);\n const { GenericContainer } = await containers();\n const context = stage(image);\n try {\n await GenericContainer.fromDockerfile(context, image.dockerfile).build(name, {\n deleteOnExit: false,\n });\n } finally {\n rmSync(context, { recursive: true, force: true });\n }\n return name;\n })();\n built.set(image.name, tag);\n }\n return tag;\n};\n\n/** This function gives the image of the harness. */\nexport const harnessImage = (): Promise<string> => build(HARNESS);\n/** This function gives the image of the Node client. */\nexport const nodeClientImage = (): Promise<string> => build(NODE);\n\n/**\n * A network fault is a fault that the injector applies as a filter rule. It is a black hole (both\n * directions, or one), or a reset on the next segment. It can also be an ICMP unreachable for each\n * segment, or a drop of each segment that is longer than a length.\n */\nexport type NetworkFault = \"blackhole\" | \"rst\" | \"reject\" | \"mtu\";\n\n/** These are the options of a network fault: the direction of a black hole, the code of a reject, the length of an MTU drop. */\nexport interface FaultOptions {\n /** This direction is relative to the namespace of the rule: `in` drops what comes from the peer, `out` what goes to it. */\n readonly direction?: \"in\" | \"out\";\n readonly code?: \"host\" | \"net\" | \"port\" | \"admin\";\n /** This length is the longest segment in bytes that an MTU drop lets through. */\n readonly above?: number;\n}\n\n/** This function gives the arguments of the injector for the options of a fault. */\nconst optionArgs = (options: FaultOptions): string[] => [\n ...(options.direction === undefined ? [] : [\"--direction\", options.direction]),\n ...(options.code === undefined ? [] : [\"--code\", options.code]),\n ...(options.above === undefined ? [] : [\"--above\", String(options.above)]),\n];\n\n/** These are the actions of the injector that lift each network fault. */\nconst LIFT: Readonly<Record<NetworkFault, string>> = {\n blackhole: \"clear\",\n rst: \"clear-rst\",\n reject: \"clear-reject\",\n mtu: \"clear-mtu\",\n};\n\n/**\n * A fault side is the namespace that gets the rules of a fault: the namespace of the peer of the\n * observer. For the client, it is the namespace of the target (the injector). For the server, it\n * is the namespace of the client (its privileged admin). Rules in the namespace of the observer\n * make its sends fail locally, with EPERM, and the network does not lose them. Thus its\n * retransmissions do not go out. The retransmission tracepoint of the kernel then records none,\n * because it fires only for a retransmission that went out.\n */\nexport type FaultSide = \"target\" | \"client\";\n\n/** A manifest is what a lab records about its machine, its images and its settings. */\nexport interface Manifest {\n readonly kernel: string;\n /** This text is the architecture of the machine, as uname -m gives it: x86_64. */\n readonly architecture: string;\n readonly hz: string | null;\n readonly boot_id: string;\n readonly docker: string;\n /** This record is the target, and its settings as it read them back. */\n readonly target: {\n readonly image: string;\n readonly id: string;\n readonly settings: Readonly<Record<string, string>>;\n /** These are the sysctls of its namespace, as read back after the harness set them. */\n readonly sysctls: Readonly<Record<string, string>>;\n };\n readonly harness: { readonly image: string; readonly id: string; readonly packages: string[] };\n /** This record is the client of each run: the TCP client of the harness, or a driver from its own image. */\n readonly client: { readonly kind: Client; readonly image: string };\n /** These are the namespaces that the prober watched, by inode. */\n readonly netns: { readonly client: number; readonly target: number };\n /** These are the sysctls of the client namespace, as read back after the harness set them. */\n readonly sysctls: Readonly<Record<string, string>>;\n}\n\n/** These are the JSON lines of a program, as they arrive. */\ninterface Lines<T extends { readonly ev: string }> {\n readonly seen: T[];\n /** These are the other lines that it wrote, for example an error. They explain a program that stopped its output. */\n readonly other: string[];\n consume: (stream: NodeJS.ReadableStream) => void;\n /** This method gives the first line of one of these kinds, or it rejects after `within` ms. */\n until(kinds: readonly string[], within: number, who: string): Promise<T>;\n}\n\nconst lines = <T extends { readonly ev: string }>(): Lines<T> => {\n const seen: T[] = [];\n const other: string[] = [];\n const waiting: Array<{ kinds: readonly string[]; resolve: (line: T) => void }> = [];\n let partial = \"\";\n const take = (text: string) => {\n if (!text.startsWith(\"{\")) {\n if (text.length > 0) other.push(text);\n return;\n }\n const line = JSON.parse(text) as T;\n seen.push(line);\n for (const wait of waiting.splice(0)) {\n if (wait.kinds.includes(line.ev)) wait.resolve(line);\n else waiting.push(wait);\n }\n };\n return {\n seen,\n other,\n consume: (stream) =>\n stream.on(\"data\", (chunk: Buffer | string) => {\n partial += chunk.toString();\n const split = partial.split(\"\\n\");\n partial = split.pop() ?? \"\";\n for (const text of split) take(text.trim());\n }),\n until(kinds, within, who) {\n const found = seen.find((line) => kinds.includes(line.ev));\n if (found !== undefined) return Promise.resolve(found);\n return new Promise((resolve, reject) => {\n const timer = setTimeout(() => {\n const said = other.length === 0 ? \"\" : `; it wrote: ${other.slice(-3).join(\" | \")}`;\n reject(new Error(`no ${kinds.join(\" or \")} from the ${who} in ${within} ms${said}`));\n }, within);\n waiting.push({\n kinds,\n resolve: (line) => {\n clearTimeout(timer);\n resolve(line);\n },\n });\n });\n },\n };\n};\n\n/** A probe is the client container of one run. */\nexport interface Probe {\n /** This method gives the first event of one of these kinds, or it rejects after `within` ms. */\n until(kinds: readonly string[], within: number): Promise<ProbeEvent>;\n /** This method lets an in-flight probe send its request. */\n go(): Promise<void>;\n /** This method freezes the client with the cgroup freezer: its process stops, and its kernel continues to answer. */\n freeze(): Promise<void>;\n thaw(): Promise<void>;\n /** This method stops the process of the client with SIGKILL. Its kernel then closes its socket with a FIN, or a reset. */\n kill(): Promise<void>;\n readonly events: readonly ProbeEvent[];\n stop(): Promise<void>;\n}\n\n/** A lab is one environment: the target, the client, the injector and the prober. */\nexport interface Lab {\n readonly targetIp: string;\n readonly clientIp: string;\n readonly netns: { readonly client: number; readonly target: number };\n /** These are all the records that the prober wrote, in sequence. */\n readonly kernel: readonly KernelRecord[];\n /** This method sets the sysctls of the client namespace through its privileged sidecar, and reads them back. */\n sysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;\n /** This method sets the settings of the target in it before a client connects, and reads them back. */\n configure(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;\n /** This method sets the sysctls of the target namespace through a privileged sidecar there, and reads them back. */\n targetSysctl(values: Readonly<Record<string, string>>): Promise<Record<string, string>>;\n /**\n * This method applies a fault between the client and the target, from the namespace of the\n * target. The fault is a black hole, or a reset that answers the next segment of the client. The stamps of\n * the injector bracket it.\n */\n fault(\n kind: NetworkFault,\n at?: FaultSide,\n options?: FaultOptions,\n ): Promise<{ t0: number; t1: number }>;\n /** This method removes a fault. */\n lift(kind: NetworkFault, at?: FaultSide, options?: FaultOptions): Promise<void>;\n /**\n * This method puts tc netem on the egress of a namespace, for example \"delay 20ms\" or \"loss 5%\",\n * or removes it with null. The stamps of the injector bracket it.\n */\n netem(spec: string | null, at?: FaultSide): Promise<{ t0: number; t1: number }>;\n /**\n * This method sets the TSO, GSO and GRO of a namespace. With them off, netem and a length match\n * act on segments, not on GSO super-packets of up to 64 KB.\n */\n offload(on: boolean, at?: FaultSide): Promise<void>;\n /** This method stops the server process in the target with SIGKILL, with the command of the target. */\n kill(): Promise<void>;\n /**\n * This method resolves the neighbors of the path in the two directions, with a ping from each\n * namespace. Without it, the first connection of an environment waits for ARP. Under a netem\n * delay, that wait is one more delayed round, and its round trip and RTO are longer.\n */\n warm(): Promise<void>;\n /** This method runs commands in the target before a client connects, each as argv. */\n setup(commands: ReadonlyArray<readonly string[]>): Promise<void>;\n /**\n * This method stops a black hole at an instant on the monotonic clock of the VM. The injector\n * waits for that instant in the VM, thus the length of a stall does not include the latency of\n * a docker exec. Stamps bracket it, as for a fault.\n */\n heal(at: FaultSide, when: number): Promise<{ t0: number; t1: number }>;\n /**\n * This method removes what the two kernels learned from their earlier connections. These are\n * the TCP metrics that each kernel keeps for a peer address when a connection closes. These metrics set the\n * first round trip of the next connection, and thus its RTO. A run with these metrics is not\n * independent of the run before it.\n */\n forget(): Promise<void>;\n /** This method freezes the target with the cgroup freezer: its process stops, and its kernel continues to answer. */\n freeze(): Promise<void>;\n thaw(): Promise<void>;\n probe(args: readonly string[]): Promise<Probe>;\n /**\n * This method waits for the next clock line of the prober, which gives the count of lost\n * records. It gives null when no clock line comes in 5 s, because then nothing tells the count.\n */\n settle(): Promise<number | null>;\n manifest(\n sysctls: Readonly<Record<string, string>>,\n settings?: Readonly<Record<string, string>>,\n targetSysctls?: Readonly<Record<string, string>>,\n ): Promise<Manifest>;\n close(): Promise<void>;\n}\n\n// each container here is a holder, a probe or the prober: nothing drains, thus the stop is immediate\nconst STOP = { timeout: 0 };\n\nconst output = async (container: StartedTestContainer, argv: string[]): Promise<string> => {\n const result = await container.exec(argv);\n if (result.exitCode !== 0)\n throw new Error(`${argv.join(\" \")} exited ${result.exitCode}: ${result.output.trim()}`);\n return result.stdout.trim();\n};\n\n/** This function gives the inode of the network namespace of a container: net:[4026532583]. */\nconst netnsOf = async (container: StartedTestContainer): Promise<number> => {\n const link = await output(container, [\"readlink\", \"/proc/self/ns/net\"]);\n const inode = /^net:\\[(\\d+)\\]$/.exec(link)?.[1];\n if (inode === undefined) throw new Error(`not a network namespace: ${link}`);\n return Number(inode);\n};\n\n/**\n * This function opens one environment:\n * - a new network and the stock target.\n * - a client that holds its own network namespace, with a privileged sidecar in it for its\n * sysctls. Docker Desktop keeps /proc/sys read-only in a container that operates.\n * - an injector sidecar in the namespace of the target.\n * - the prober, a privileged container that loads the kernel program and watches the two\n * namespaces.\n *\n * The client of each run is its own container in the namespace of the client. It is the TCP\n * client of the harness, or a driver from its own image (ioredis on Node). The two print the same\n * lines.\n */\nexport const openLab = async (target: Target, kind: Client = \"tcp\"): Promise<Lab> => {\n const image = await harnessImage();\n const clientImage = kind === \"ioredis\" ? await nodeClientImage() : image;\n const command =\n kind === \"ioredis\" ? [\"node\", \"/opt/harness/node/client-ioredis.mjs\"] : [\"/opt/harness/client\"];\n const { GenericContainer, Network, Wait, getContainerRuntimeClient } = await containers();\n const opened: StartedTestContainer[] = [];\n let network: StartedNetwork | undefined;\n const close = async () => {\n for (const container of opened.reverse()) await container.stop(STOP).catch(() => undefined);\n await network?.stop().catch(() => undefined);\n };\n try {\n network = await new Network().start();\n const stock = new GenericContainer(target.image)\n .withNetwork(network)\n .withWaitStrategy(Wait.forLogMessage(target.ready));\n // a holder command keeps the container up when a fin fault kills the server process\n const server = await (\n target.command === undefined ? stock : stock.withCommand([...target.command])\n ).start();\n opened.push(server);\n const client = await new GenericContainer(image).withNetwork(network).start();\n opened.push(client);\n const admin = await new GenericContainer(image)\n .withNetworkMode(`container:${client.getId()}`)\n .withPrivilegedMode()\n .start();\n opened.push(admin);\n const injector = await new GenericContainer(image)\n .withNetworkMode(`container:${server.getId()}`)\n .withAddedCapabilities(\"NET_ADMIN\", \"NET_RAW\")\n .start();\n opened.push(injector);\n const netns = { client: await netnsOf(admin), target: await netnsOf(injector) };\n const records = lines<KernelRecord>();\n const prober = await new GenericContainer(image)\n .withPrivilegedMode()\n .withNetworkMode(\"none\")\n .withCommand([\"/opt/harness/prober\", String(netns.client), String(netns.target)])\n .withLogConsumer(records.consume)\n .start();\n opened.push(prober);\n const begun = await records.until([\"begin\", \"fail\"], 30_000, \"prober\");\n if (begun.ev === \"fail\") throw new Error(`the prober failed: ${String(begun[\"why\"])}`);\n const targetIp = server.getIpAddress(network.getName());\n const clientIp = client.getIpAddress(network.getName());\n const runtime = await getContainerRuntimeClient();\n const paused = runtime.container.getById(server.getId());\n let targetAdmin: StartedTestContainer | undefined;\n\n return {\n targetIp,\n clientIp,\n netns,\n kernel: records.seen,\n async sysctl(values) {\n const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);\n if (settings.length > 0) await output(admin, [\"sysctl\", \"-q\", \"-w\", ...settings]);\n const read: Record<string, string> = {};\n for (const key of Object.keys(values))\n read[key] = await output(admin, [\"sysctl\", \"-n\", key]);\n return read;\n },\n async targetSysctl(values) {\n const read: Record<string, string> = {};\n if (Object.keys(values).length === 0) return read;\n // privileged, and only when a cell asks for it: otherwise the namespace of the target stays as it is\n targetAdmin ??= await new GenericContainer(image)\n .withNetworkMode(`container:${server.getId()}`)\n .withPrivilegedMode()\n .start();\n opened.push(targetAdmin);\n const settings = Object.entries(values).map(([key, value]) => `${key}=${value}`);\n await output(targetAdmin, [\"sysctl\", \"-q\", \"-w\", ...settings]);\n for (const key of Object.keys(values))\n read[key] = await output(targetAdmin, [\"sysctl\", \"-n\", key]);\n return read;\n },\n async configure(values) {\n const read: Record<string, string> = {};\n if (Object.keys(values).length === 0) return read;\n const how = target.settings;\n if (how === undefined) throw new Error(`${target.image} names no way to set its settings`);\n for (const [name, value] of Object.entries(values)) {\n await output(server, [...how.set, name, value]);\n read[name] = (await output(server, [...how.get, name])).split(\"\\n\").at(-1) ?? \"\";\n }\n return read;\n },\n async fault(kind, at = \"target\", options = {}) {\n const [where, peer] = at === \"client\" ? [admin, targetIp] : [injector, clientIp];\n const args = [\"/opt/harness/inject\", kind, peer, ...optionArgs(options)];\n const stamp = JSON.parse(await output(where, args)) as { t0: number; t1: number };\n return { t0: stamp.t0, t1: stamp.t1 };\n },\n async lift(kind, at = \"target\", options = {}) {\n const [where, peer] = at === \"client\" ? [admin, targetIp] : [injector, clientIp];\n await output(where, [\"/opt/harness/inject\", LIFT[kind], peer, ...optionArgs(options)]);\n },\n async netem(spec, at = \"target\") {\n const where = at === \"client\" ? admin : injector;\n const args =\n spec === null\n ? [\"/opt/harness/inject\", \"clear-netem\", \"eth0\"]\n : [\"/opt/harness/inject\", \"netem\", \"eth0\", ...spec.split(/\\s+/)];\n const stamp = JSON.parse(await output(where, args)) as { t0: number; t1: number };\n return { t0: stamp.t0, t1: stamp.t1 };\n },\n async offload(on, at = \"target\") {\n const where = at === \"client\" ? admin : injector;\n await output(where, [\"/opt/harness/inject\", \"offload\", \"eth0\", on ? \"on\" : \"off\"]);\n },\n async kill() {\n if (target.kill === undefined) throw new Error(`${target.image} names no way to kill its server`);\n await output(server, [...target.kill]);\n },\n async warm() {\n await output(admin, [\"ping\", \"-c\", \"2\", \"-W\", \"2\", targetIp]);\n await output(injector, [\"ping\", \"-c\", \"2\", \"-W\", \"2\", clientIp]);\n },\n async setup(commands) {\n for (const argv of commands) await output(server, [...argv]);\n },\n async heal(at, when) {\n const [where, peer] = at === \"client\" ? [admin, targetIp] : [injector, clientIp];\n const args = [\"/opt/harness/inject\", \"clear\", peer, \"--at\", String(Math.round(when))];\n const stamp = JSON.parse(await output(where, args)) as { t0: number; t1: number };\n return { t0: stamp.t0, t1: stamp.t1 };\n },\n async forget() {\n await output(admin, [\"ip\", \"tcp_metrics\", \"flush\", \"all\"]);\n await output(injector, [\"ip\", \"tcp_metrics\", \"flush\", \"all\"]);\n },\n freeze: () => paused.pause().then(() => undefined),\n thaw: () => paused.unpause().then(() => undefined),\n async probe(args) {\n const events = lines<ProbeEvent>();\n const container = await new GenericContainer(clientImage)\n .withNetworkMode(`container:${client.getId()}`)\n .withCommand([...command, ...args])\n .withLogConsumer(events.consume)\n .start();\n return {\n events: events.seen,\n until: (kinds, within) => events.until(kinds, within, \"client\"),\n go: () => output(container, [\"touch\", \"/tmp/go\"]).then(() => undefined),\n freeze: () =>\n runtime.container\n .getById(container.getId())\n .pause()\n .then(() => undefined),\n thaw: () =>\n runtime.container\n .getById(container.getId())\n .unpause()\n .then(() => undefined),\n kill: () =>\n runtime.container\n .getById(container.getId())\n .kill({ signal: \"SIGKILL\" })\n .then(() => undefined),\n // a killed client has no container to stop\n stop: () =>\n container\n .stop(STOP)\n .then(() => undefined)\n .catch(() => undefined),\n };\n },\n async settle() {\n // the prober drains its ring each 20 ms, and it writes a clock line each second\n const after = records.seen.length;\n const clock = await new Promise<KernelRecord | null>((resolve) => {\n const started = Date.now();\n const look = () => {\n const line = records.seen.slice(after).find((record) => record.ev === \"clock\");\n if (line !== undefined) resolve(line);\n else if (Date.now() - started > 5_000) resolve(null);\n else setTimeout(look, 50);\n };\n look();\n });\n return clock === null ? null : Number(clock[\"lost\"] ?? 0);\n },\n async manifest(sysctls, settings = {}, targetSysctls = {}) {\n const config = await output(admin, [\"zcat\", \"/proc/config.gz\"]).catch(() => \"\");\n const inspect = async (container: StartedTestContainer) =>\n runtime.container.inspect(runtime.container.getById(container.getId()));\n return {\n kernel: await output(client, [\"uname\", \"-r\"]),\n architecture: await output(client, [\"uname\", \"-m\"]),\n hz: /^CONFIG_HZ=(\\d+)$/m.exec(config)?.[1] ?? null,\n boot_id: await output(client, [\"cat\", \"/proc/sys/kernel/random/boot_id\"]),\n docker: `${runtime.info.containerRuntime.serverVersion} on ${runtime.info.containerRuntime.operatingSystem}`,\n target: {\n image: target.image,\n id: (await inspect(server)).Image,\n settings,\n sysctls: targetSysctls,\n },\n harness: {\n image,\n id: (await inspect(client)).Image,\n packages: (await output(client, [\"apk\", \"info\", \"-v\"])).split(\"\\n\").sort(),\n },\n client: { kind, image: clientImage },\n netns,\n sysctls,\n };\n },\n close,\n };\n } catch (error) {\n await close();\n throw error;\n }\n};\n","import { valueIn } from \"@mark1russell7/quantity\";\nimport type { Cell, KernelRecord, ProbeEvent, Run } from \"./cell.ts\";\nimport type { FaultOptions, Lab, Manifest, Probe } from \"./lab.ts\";\nimport { openLab } from \"./lab.ts\";\nimport type { Measured } from \"./observe.ts\";\nimport { measure } from \"./observe.ts\";\nimport type { Verdict } from \"./verdict.ts\";\nimport { STEPS, verdictOf } from \"./verdict.ts\";\n\n/** A cell result is a cell with its manifests, its runs, their measurements and its verdict. */\nexport interface CellResult {\n readonly cell: Cell;\n readonly manifests: readonly Manifest[];\n readonly runs: readonly Run[];\n readonly measured: readonly Measured[];\n readonly verdict: Verdict;\n /** These times are when its first environment opened and its last environment closed, on the wall clock. */\n readonly at: { readonly started: string; readonly ended: string };\n}\n\n/**\n * This function measures and judges the raw records of a cell, with the code of the harness at\n * this time. `runCell` uses it during a cell. A holder of the raw records of a cell can use it\n * again later. Thus the records get a new analysis after a change to observe.ts or to the verdict.\n */\nexport const judge = (\n cell: Cell,\n runs: readonly Run[],\n): { measured: Measured[]; verdict: Verdict } => {\n const measured = runs.map((run) => measure(cell, run));\n return { measured, verdict: verdictOf(cell.prediction, measured) };\n};\n\n/** This function gives the count of lost records in the latest clock line of the prober, since it began. */\nconst droppedSoFar = (records: readonly KernelRecord[]): number => {\n for (let at = records.length - 1; at >= 0; at -= 1)\n if (records[at]!.ev === \"clock\") return Number(records[at]![\"lost\"] ?? 0);\n return 0;\n};\n\n/** This function gives the argv of the client for a cell, with a deadline far after all that it waits for. */\nconst probeArgs = (cell: Cell, lab: Lab): string[] => {\n const { prediction, probe } = cell;\n const longest = prediction.detects\n ? valueIn(prediction.high, \"s\")\n : valueIn(prediction.window, \"s\");\n const args = [lab.targetIp, String(cell.target.port), \"--mode\", probe.mode];\n if (probe.request !== undefined) args.push(\"--request\", probe.request);\n if (probe.count !== undefined) args.push(\"--count\", String(probe.count));\n if (probe.interval !== undefined) args.push(\"--interval-ms\", String(valueIn(probe.interval, \"ms\")));\n if (probe.driver !== undefined) {\n // a driver sets its socket options from its own options: the keepAlive of ioredis\n args.push(\"--keepalive-ms\", String(valueIn(probe.driver.keepAlive, \"ms\")));\n args.push(\"--max-s\", String(Math.ceil(longest * 2 + 10)));\n return args;\n }\n if (probe.keepalive) args.push(\"--keepalive\");\n if (probe.userTimeout !== undefined)\n args.push(\"--user-timeout-ms\", String(valueIn(probe.userTimeout, \"ms\")));\n args.push(\"--max-s\", String(Math.ceil(longest * 2 + 10)));\n return args;\n};\n\n/**\n * This function waits for an error on the socket of the target for the connection of the\n * client, or for its end of file (CLOSE_WAIT, state 8). The prober records both in the namespace\n * of the target. A cell on the server side waits for that. The promise resolves after `within` ms\n * at the latest.\n */\nconst serverError = (lab: Lab, local: unknown, within: number): Promise<void> =>\n new Promise((resolve) => {\n const started = Date.now();\n const look = () => {\n const seen = lab.kernel.some(\n (record) =>\n record.netns === lab.netns.target &&\n record.dst === local &&\n ((record.ev === \"error\" && (record.err ?? 0) !== 0) ||\n (record.ev === \"state\" && (record as { readonly new?: number }).new === 8)),\n );\n if (seen || Date.now() - started > within) resolve();\n else setTimeout(look, 50);\n };\n look();\n });\n\n/**\n * This function waits for the request that an in-flight client sends on the signal: its second\n * send, after the PING that made it ready. Its stamp is when the request first met the fault.\n */\nconst requestSent = (probe: Probe, within: number): Promise<ProbeEvent> =>\n new Promise((resolve, reject) => {\n const started = Date.now();\n const look = () => {\n const sent = probe.events.filter((event) => event.ev === \"sent\")[1];\n if (sent !== undefined) resolve(sent);\n else if (Date.now() - started > within)\n reject(new Error(`the client sent no request within ${within} ms of being told to`));\n else setTimeout(look, 1);\n };\n look();\n });\n\n/**\n * This function does one run. The client connects and gets its reply, the fault occurs, and an\n * in-flight client sends into it. The harness then waits for the observer to see the fault, or for\n * the silent window to end. The observer is the client, or the socket of the target for a cell on\n * the server side.\n *\n * A stall stops by itself, at its length after the send of the request, and the\n * run waits for the reply. Then the harness removes the fault, thus the environment can serve the\n * next run. The run keeps the records of the prober from its start to its end. The frozen process\n * is the peer of the observer: the target, or the client for the server.\n */\n/** This function gives the netem specification of a cell: its latency, then the netem of its fault when it has one. */\nconst netemSpec = (cell: Cell, fault: string | undefined): string | null => {\n const parts = [\n ...(cell.latency === undefined ? [] : [`delay ${valueIn(cell.latency, \"ms\")}ms`]),\n ...(fault === undefined ? [] : [fault]),\n ];\n return parts.length === 0 ? null : parts.join(\" \");\n};\n\nconst once = async (lab: Lab, cell: Cell, environment: number, index: number): Promise<Run> => {\n // each run starts from kernels that keep nothing from the runs before it\n await lab.forget();\n const from = lab.kernel.length;\n const lostBefore = droppedSoFar(lab.kernel);\n const probe = await lab.probe(probeArgs(cell, lab));\n const server = cell.observer === \"server\";\n const peer = server\n ? { freeze: () => probe.freeze(), thaw: () => probe.thaw() }\n : { freeze: () => lab.freeze(), thaw: () => lab.thaw() };\n // a client that stopped before the signal to send tells why in its last lines\n const go = () =>\n probe.go().catch((error: unknown) => {\n const last = probe.events.slice(-3).map((event) => JSON.stringify(event));\n throw new Error(`${(error as Error).message}; the client's last lines: ${last.join(\" \")}`);\n });\n let fault: Run[\"fault\"];\n let heal: { t0: number; t1: number } | null = null;\n try {\n await probe.until([\"ready\"], 15_000);\n // the in-flight data of a run on the server side is the reply of the server. The request goes\n // out first, and the fault occurs while the server holds it (the timeout of BLPOP). A fin with\n // a request in flight kills the server while it holds the request, so it goes first too\n const replyInFlight =\n cell.probe.mode === \"inflight\" &&\n ((server && cell.fault !== \"mtu-blackhole\") || cell.fault === \"fin\");\n if (replyInFlight) {\n // the client sends one sample after the file appears. The request reaches the server well\n // in the pause, and the reply waits for the timeout of the request\n await go();\n await new Promise((resolve) => setTimeout(resolve, 200));\n }\n const faultAt = server ? \"client\" : \"target\";\n // the direction of a simplex drop, relative to the namespace of the rule: in the target, the\n // segments to the server come in; in the client, they go out\n const direction: FaultOptions[\"direction\"] =\n cell.drops === undefined\n ? undefined\n : (cell.drops === \"to-server\") === (faultAt === \"target\")\n ? \"in\"\n : \"out\";\n const options: FaultOptions = {\n ...(direction === undefined ? {} : { direction }),\n ...(cell.icmp === undefined ? {} : { code: cell.icmp }),\n ...(cell.mtu === undefined ? {} : { above: cell.mtu }),\n };\n // an MTU drop goes where the long reply arrives: the namespace of the client\n const apply = async (): Promise<Run[\"fault\"]> => {\n switch (cell.fault) {\n case \"freeze-process\":\n await peer.freeze();\n return null;\n case \"fin\":\n await (server ? probe.kill() : lab.kill());\n return null;\n case \"unreachable\":\n return lab.fault(\"reject\", faultAt, { code: cell.icmp ?? \"host\" });\n case \"simplex\":\n return lab.fault(\"blackhole\", faultAt, options);\n case \"mtu-blackhole\":\n return lab.fault(\"mtu\", \"client\", { above: cell.mtu ?? 1200 });\n case \"netem\":\n return lab.netem(netemSpec(cell, cell.netem), \"target\");\n case \"rst\":\n return lab.fault(\"rst\", faultAt);\n default:\n // a stall is a black hole that stops by itself\n return lab.fault(\"blackhole\", faultAt);\n }\n };\n const lift = async (): Promise<void> => {\n switch (cell.fault) {\n case \"freeze-process\":\n return peer.thaw();\n case \"fin\":\n return undefined;\n case \"unreachable\":\n return lab.lift(\"reject\", faultAt, { code: cell.icmp ?? \"host\" });\n case \"simplex\":\n return lab.lift(\"blackhole\", faultAt, options);\n case \"mtu-blackhole\":\n return lab.lift(\"mtu\", \"client\", { above: cell.mtu ?? 1200 });\n case \"netem\":\n await lab.netem(netemSpec(cell, undefined), \"target\");\n return undefined;\n case \"rst\":\n return lab.lift(\"rst\", faultAt);\n default:\n return heal === null ? lab.lift(\"blackhole\", faultAt) : undefined;\n }\n };\n fault = await apply();\n try {\n if (cell.probe.mode !== \"idle\" && !replyInFlight) await go();\n const { prediction } = cell;\n if (cell.fault === \"stall\") {\n if (cell.stall === undefined) throw new Error(`${cell.name} says no stall's length`);\n // the injector holds the end in the VM until the length of the stall after the send\n const sent = await requestSent(probe, 5_000);\n heal = await lab.heal(faultAt, sent.t + valueIn(cell.stall, \"s\") * 1e9);\n const window = valueIn(prediction.detects ? prediction.high : prediction.window, \"s\");\n await probe.until([\"data\", \"error\", \"eof\"], window * 1_000).catch(() => undefined);\n } else if (prediction.detects === null) {\n // a distribution: the stream ends by itself, or the connection does\n const window = valueIn(prediction.window, \"s\") * 1_000;\n await probe.until([\"done\", \"error\", \"eof\", \"deadline\"], window).catch(() => undefined);\n } else if (prediction.detects) {\n const within = valueIn(prediction.high, \"s\") * 2_000 + 10_000;\n if (server) {\n const local = probe.events.find((event) => event.ev === \"connected\")?.[\"local\"];\n await serverError(lab, local, within);\n } else {\n await probe.until([\"error\", \"eof\", \"deadline\"], within).catch(() => undefined);\n }\n } else {\n await new Promise((resolve) =>\n setTimeout(resolve, valueIn(prediction.window, \"s\") * 1_000),\n );\n }\n } finally {\n await lift();\n }\n // a thawed target answers the request that it held: the client sees the late reply\n if (cell.fault === \"freeze-process\" && !server)\n await probe.until([\"data\"], 5_000).catch(() => undefined);\n } finally {\n await probe.stop();\n }\n // the prober drains its ring each 20 ms. Its next clock line closes the records of the run\n const settled = await lab.settle();\n const lost = settled === null ? null : settled - lostBefore;\n return {\n environment,\n index,\n events: [...probe.events],\n fault,\n ...(cell.fault === \"stall\" ? { heal } : {}),\n kernel: lab.kernel.slice(from),\n netns: lab.netns.client,\n targetNetns: lab.netns.target,\n lost,\n };\n};\n\n/** This function divides a count of runs across environments, as equally as possible. */\nexport const spread = (runs: number, environments: number): number[] =>\n Array.from(\n { length: environments },\n (_, at) => Math.floor(runs / environments) + (at < runs % environments ? 1 : 0),\n ).filter((count) => count > 0);\n\n/** This function does the part of the runs of a cell in one environment, and records its manifest. */\nconst inEnvironment = async (\n cell: Cell,\n environment: number,\n count: number,\n say: (line: string) => void,\n): Promise<{ manifest: Manifest; runs: Run[] }> => {\n const lab = await openLab(cell.target, cell.probe.driver?.name ?? \"tcp\");\n try {\n // the settings of the target first: Redis applies tcp-keepalive to a client when it accepts it\n const settings = await lab.configure(cell.targetSettings ?? {});\n const targetSysctls = await lab.targetSysctl(cell.targetSysctls ?? {});\n if (cell.targetSetup !== undefined) await lab.setup(cell.targetSetup);\n // netem and a length match act on segments only when TSO, GSO and GRO are off on the two ends\n if (cell.fault === \"mtu-blackhole\" || cell.fault === \"netem\") {\n await lab.offload(false, \"target\");\n await lab.offload(false, \"client\");\n }\n // a latency is on the path before the client connects, thus its round trip and RTO include it\n const latency = netemSpec(cell, undefined);\n if (latency !== null) await lab.netem(latency, \"target\");\n // the first connection must not wait for ARP: under a delay, that is a longer first round trip\n await lab.warm();\n const manifest = await lab.manifest(await lab.sysctl(cell.sysctls), settings, targetSysctls);\n const runs: Run[] = [];\n for (let index = 0; index < count; index += 1) {\n const run = await once(lab, cell, environment, index);\n runs.push(run);\n const read = measure(cell, run);\n say(\n `${cell.name}: env ${environment} run ${index} ${read.outcome.kind}` +\n `${read.elapsed === null ? \"\" : ` ${read.elapsed.value} s`}` +\n `${read.void === null ? \"\" : ` (void: ${read.void})`}`,\n );\n }\n return { manifest, runs };\n } finally {\n await lab.close();\n }\n};\n\n/**\n * This function does the runs of a cell. They go across new environments, and each environment records\n * its manifest. Then the verdict judges the runs. A median across the edge of the band extends\n * the cell from 11 runs to 22, and then to 44, each extension in new environments. With a\n * concurrency of more than one, that many environments operate at the same time, each with its\n * own prober. The runs then go across at least as many environments.\n */\nexport const runCell = async (\n cell: Cell,\n say: (line: string) => void = () => {},\n): Promise<CellResult> => {\n const started = new Date().toISOString();\n const manifests: Manifest[] = [];\n const runs: Run[] = [];\n const concurrency = Math.max(1, cell.concurrency ?? 1);\n let target: number = cell.runs;\n let next = 0;\n for (;;) {\n const counts = spread(target - runs.length, Math.max(cell.environments, concurrency));\n for (let at = 0; at < counts.length; at += concurrency) {\n const batch = counts.slice(at, at + concurrency).map((count) => ({ count, env: next++ }));\n const done = await Promise.all(\n batch.map(({ count, env }) => inEnvironment(cell, env, count, say)),\n );\n for (const each of done) {\n manifests.push(each.manifest);\n runs.push(...each.runs);\n }\n }\n const { measured, verdict } = judge(cell, runs);\n const larger = STEPS.find((step) => step > runs.length);\n if (verdict.decision !== \"extend\" || larger === undefined) {\n const at = { started, ended: new Date().toISOString() };\n return { cell, manifests, runs, measured, verdict, at };\n }\n target = larger;\n }\n};\n"],"mappings":";;;;;;;;;AAmDA,MAAM,QAA0C;CAC9C,IAAI;CACJ,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;AACP;;AAGA,MAAM,WAAW,MAAc,OAC7B,SAAS,KAAK,OAAO,KAAK,QAAQ,GAAG,IAAI,KAAK,GAAG;;AAGnD,MAAM,YAAY,WAA8C,OAAqC;AACrG,MAAM,YAAY,WAA8C,OAAqC;AAErG,MAAM,SAAS,OAA+B,UAC5C,UAAU,KAAA,IAAY,IAAI,OAAO,MAAM,UAAU,CAAC;;AAGpD,MAAM,iBACJ,MACA,QACA,OACA,YACoB;CACpB,IAAI,CAAC,KAAK,WAAW,WAAW,KAAK,WAAW,SAAS,eAAe,OAAO;CAC/E,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,WAAW,MAAM,OAAO,KAAK;CAC/E,MAAM,YACJ,KAAK,WAAW,SAAS,kBACrB,QACA,KAAK,WAAW,SAAS,iBACvB,UACA,OAAO,MACJ,UACC,MAAM,OAAO,YAAY,MAAM,KAAK,QAAQ,KAAK,MAAM,OAAO,aAAa,IAAI,CACnF;CACR,OAAO,UAAU,KAAA,KAAa,cAAc,KAAA,IAAY,OAAO,QAAQ,UAAU,GAAG,MAAM,CAAC;AAC7F;;;;;;;;;AAUA,MAAa,WAAW,MAAY,QAAuB;CACzD,MAAM,EAAE,QAAQ,WAAW;CAC3B,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,OAAO;CACzD,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,WAAW,CAAC,GAAG;CACjE,MAAM,WAAW,KAAK,MAAM,SAAS;CACrC,MAAM,UAAU,WAAW,OAAO,QAAQ,UAAU,MAAM,OAAO,MAAM,CAAC,CAAC,KAAK;CAG9E,MAAM,SAAS,KAAK,aAAa;CACjC,MAAM,QAAQ,SAAS,IAAI,cAAc,IAAI;CAC7C,MAAM,SAAS,OAAO,MACnB,WAAW,OAAO,UAAU,UAAU,SAAS,OAAO,MAAM,OAAO,SAAS,KAC/E,CAAC,EAAE;CACH,MAAM,SAAS,OAAO,QAAQ,WAAW,OAAO,UAAU,SAAS,OAAO,WAAW,MAAM;CAG3F,MAAM,YAAY,SAAS,IAAI,QAAQ,IAAI;CAC3C,MAAM,aAAa,OAAO,MACvB,WAAW,OAAO,UAAU,cAAc,SAAS,OAAO,MAAM,OAAO,SAAS,KACnF,CAAC,EAAE;CACH,MAAM,aAAa,OAAO,QACvB,WAAW,OAAO,OAAO,WAAW,OAAO,SAAS,WACvD;CACA,MAAM,gBAAgB,OAAO,QAAQ,WAAW,OAAO,OAAO,SAAS;;;;;;;;;;CAUvE,MAAM,WAAW,MAAc,IAAY,UAAqD;EAC9F,MAAM,UAAU,OAAO,QACpB,WACC,OAAO,OAAO,WAAW,OAAO,SAAS,SAAS,OAAO,QAAQ,QAAQ,OAAO,QAAQ,EAC5F;EACA,MAAM,OAAO,QAAQ,MAAM,YAAY,OAAO,QAAQ,SAAS,MAAM,CAAC;EACtE,IAAI,SAAS,KAAA,GACX,OAAO,OAAO,MAAM,aAAa,KAAK,OAAQ,MAAM;EAKtD,OADa,QAAQ,MAAM,WAAW,UAAU,eAAe,OAAO,UAAU,IACtE,MAAM,KAAA,IACZ,OACA;CACN;CACA,MAAM,OACJ,YAAY,KAAA,IACR,IACA,OAAO,QAAQ,WAAW,OAAO,OAAO,QAAQ,OAAO,OAAO,QAAQ,CAAC,CAAC,CAAC;CAC/E,MAAM,OAAO;EACX,aAAa,IAAI;EACjB,OAAO,IAAI;EACX,QAAQ,WAAW;EACnB,iBAAiB,cAAc;EAC/B;CACF;CACA,MAAM,UAAU;EACd,SAAS,EAAE,MAAM,SAAS;EAC1B,SAAS;EACT,QAAQ;EACR,SAAS;EACT,OAAO,CAAC;EACR,SAAS;CACX;CAEA,IAAI,UAAU,KAAA,GAAW,OAAO;EAAE,GAAG;EAAM,GAAG;EAAS,MAAM;CAAiC;CAC9F,IAAI,YAAY,KAAA,GACd,OAAO;EAAE,GAAG;EAAM,GAAG;EAAS,MAAM;CAA4C;CAClF,IAAI,IAAI,SAAS,MACf,OAAO;EACL,GAAG;EACH,GAAG;EACH,MAAM;CACR;CACF,IAAI,IAAI,OAAO,GAAG,OAAO;EAAE,GAAG;EAAM,GAAG;EAAS,MAAM,sBAAsB,IAAI,KAAK;CAAU;CAC/F,IAAI,WAAW,KAAA,GACb,OAAO;EAAE,GAAG;EAAM,GAAG;EAAS,MAAM;CAA4C;CAClF,IAAI,SAAwB;CAC5B,IAAI,IAAI,UAAU,MAAM;EAItB,MAAM,aAAa,UAAU,YAAY,KAAK,UAAU;EACxD,MAAM,UAAU,OAAO,MACpB,UAAU,MAAM,OAAO,UAAU,MAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,IAAI,MAAO,EAChF;EACA,IAAI,YAAY,CAAC,cAAc,IAAI,MAAM,KAAK,QAAQ,GACpD,SAAS;OACN,IAAI,cAAc,IAAI,MAAM,KAAK,QAAQ,GAC5C,SAAS;OACN,IAAI,cAAc,SACrB,SAAS;OACN,IAAI,WAAW,OAAO,KAAA,KAAa,WAAW,EAAE,CAAC,OAAO,IAAI,MAAM,IACrE,SAAS;CACb;CAIA,MAAM,WAAW,OAAO,MAAM,WAAW,OAAO,OAAO,YAAY,OAAO,OAAO,OAAO,CAAC;CACzF,MAAM,OAAO,SACT,KAAA,IACA,OAAO,MAAM,UAAU,MAAM,OAAO,WAAW,MAAM,OAAO,KAAK;CAGrE,MAAM,YAAY,OAAO,MAAM,WAAW,OAAO,OAAO,WAAW,SAAS,MAAM,MAAM,CAAC;CACzF,MAAM,UACJ,aAAa,KAAA,IACT;EAAE,MAAM;EAAS,OAAO,MAAM,SAAS,QAAS,IAAI,SAAS;CAAM,IACnE,cAAc,KAAA,KAAa,MAAM,OAAO,QACtC,EAAE,MAAM,MAAM,IACd,SAAS,KAAA,IACP;EAAE,MAAM;EAAS,OAAO,OAAO,KAAK,QAAQ;CAAE,IAC9C,EAAE,MAAM,SAAS;CAC3B,MAAM,QAAQ,UAAU,QAAQ,WAAW,QAAQ,MAAM;CACzD,MAAM,UACJ,aAAa,KAAA,KAAa,SAAS,KAAA,IAAY,QAAQ,SAAS,MAAM,KAAK,CAAC,IAAI;CAClF,MAAM,EAAE,eAAe;CAGvB,IAAI,WAAW,YAAY,MAAM;EAC/B,MAAM,QAAQ,OAAO,QAAQ,UAAU,MAAM,OAAO,MAAM,CAAC,CAAC,MAAM,CAAC;EACnE,MAAM,UAAU,MAAM,SAAS,MAAM,MAAM;GACzC,MAAM,OAAO,MAAM,IAAI,EAAE,EAAE,KAAK;GAChC,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,UAAU,MAAM,IAAI,KAAK,KAAK,MAAM,IAAI,IAAI;GAC9F,OAAO,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,QAAQ,KAAK,GAAG,MAAM,CAAC,CAAC;EAC7D,CAAC;EACD,OAAO;GACL,GAAG;GACH,GAAG;GACH,SAAS,UAAU,KAAA,IAAY,UAAU,EAAE,MAAM,WAAW;GAC5D,OAAO,cACJ,QAAQ,WAAW,OAAO,OAAO,QAAQ,CAAC,CAAC,CAC3C,KAAK,WAAW,QAAQ,QAAQ,GAAG,OAAO,IAAI,CAAC;GAClD;GACA,MAAM;EACR;CACF;CAIA,IAAI,CAAC,WAAW,WAAW,WAAW,aAAa,KAAA,GAAW;EAC5D,MAAM,EAAE,aAAa;EACrB,MAAM,OAAO,IAAI,QAAQ;EACzB,IAAI,WAAW,QAAQ,SAAS,MAAM,SAAS;OAC1C,IAAI,WAAW,QAAQ,SAAS,MAAM;GACzC,MAAM,OAAO,QAAQ,QAAQ,GAAG,KAAK,EAAE;GACvC,MAAM,KAAK,QAAQ,QAAQ,GAAG,KAAK,EAAE;GACrC,IACE,QAAQ,MAAM,GAAG,KAAK,QAAQ,SAAS,KAAK,KAAK,GAAG,KACpD,QAAQ,IAAI,GAAG,KAAK,QAAQ,SAAS,KAAK,MAAM,GAAG,GAEnD,SAAS,mBAAmB,KAAK,MAAM,GAAG,GAAG,MAAM,iCAAiC,SAAS,KAAK,IAAI,MAAM,GAAG,SAAS,KAAK,KAAK,MAAM;EAC5I;EACA,MAAM,QAAQ,cACX,QAAQ,WAAW,OAAO,OAAO,QAAQ,CAAC,CAAC,CAC3C,KAAK,WAAW,QAAQ,QAAQ,GAAG,OAAO,IAAI,CAAC;EAClD,IAAI,UAAU,KAAA,GACZ,OAAO;GACL,GAAG;GACH,GAAG;GACH;GACA;GACA,MAAM;GACN,SAAS,kBAAkB,QAAQ,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM;EAC7D;EAGF,MAAM,SACJ,SAAS,OACL,KAAA,IACA,OAAO,MAAM,WAAW,OAAO,OAAO,QAAQ,OAAO,OAAO,KAAK,EAAE;EACzE,IAAI,WAAW,KAAA,GACb,OAAO;GACL,GAAG;GACH,GAAG;GACH;GACA,MAAM;GACN,SAAS;EACX;EACF,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAM,OAAO,UAAU,MAAM,IAAI,QAAQ,CAAC;EAC/E,WAAW,QAAQ,QAAQ,GAAG,OAAO,MAAM,YAAY;EACvD,OAAO;GACL,GAAG;GACH,SAAS,EAAE,MAAM,WAAW;GAC5B,SAAS,QAAQ,QAAQ,GAAG,OAAO,IAAI;GACvC,QAAQ,UAAU,KAAA,IAAY,OAAO,QAAQ,QAAQ,GAAG,MAAM,CAAC;GAC/D,SAAS,UAAU,KAAA,IAAY,OAAO,QAAQ,OAAO,MAAM,MAAM,CAAC;GAClE;GACA,MAAM;GACN,SAAS;EACX;CACF;CAEA,IAAI,CAAC,WAAW,SAAS;EAEvB,MAAM,SAAS,QAAQ,WAAW,QAAQ,GAAG,IAAI;EACjD,IAAI,UAAU,KAAA,KAAa,QAAQ,QAAQ,IAAI,QAC7C,OAAO;GAAE,GAAG;GAAM,GAAG;GAAS,MAAM;EAAO;EAC7C,OAAO;GACL,GAAG;GACH,GAAG;GACH;GACA,SAAS,QAAQ,QAAQ,GAAG,KAAK;GACjC,MAAM;GACN,SAAS,kBAAkB,QAAQ,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM;EAC7D;CACF;CAEA,IAAI,UAAU,KAAA,GACZ,OAAO;EACL,GAAG;EACH,GAAG;EACH,MAAM;EACN,SAAS,wBAAwB,WAAW,MAAM;CACpD;CAKF,MAAM,SAAS,IAAI,OAAO,MAAM,MAAM;CACtC,MAAM,kBACJ,WAAW,SAAS,kBAChB,OAAO,QAAQ,WAAW,OAAO,OAAO,QAAQ,OAAO,QAAQ,MAAM,CAAC,CAAC,GAAG,EAAE,IAC5E,WAAW,SAAS,yBAClB,cAAc,MAAM,WAAW,OAAO,QAAQ,KAAK,IACnD,WAAW,SAAS,gBAClB,OAAO,MACJ,WACC,OAAO,UAAU,aACjB,OAAO,WAAW,cAClB,OAAO,OAAO,WACd,SAAS,MAAM,MAAM,CACzB,IACA,KAAA;CACV,MAAM,YACJ,WAAW,SAAS,iBAAiB,QAAQ,IAAI,iBAAiB;CACpE,IAAI,cAAc,KAAA,GAChB,OAAO;EACL,GAAG;EACH,GAAG;EACH;EACA;EACA,MAAM;EACN,SAAS,yCAAyC,WAAW,KAAK;CACpE;CAEF,MAAM,SACJ,WAAW,SAAS,kBAChB,aACA,WAAW,SAAS,yBAClB,cAAc,QAAQ,WAAW,OAAO,QAAQ,SAAS,OAAO,OAAO,SAAS,IAChF,CAAC,EAAA,CACP,KAAK,WAAW,QAAQ,WAAW,OAAO,IAAI,CAAC;CAKjD,MAAM,WAHJ,WAAW,UAAU,QACjB,QAAQ,SAAS,QACjB,QAAQ,SAAS,WAAW,QAAQ,UAAU,WAAW,SAE3D,OACA,iBAAiB,QAAQ,SAAS,UAAU,QAAQ,QAAQ,cAAc,QAAQ,WAAW;CAGjG,IAAI,WAAW,SAAS,kBAAkB,WAAW,SAAS,eAC5D,WAAW,QACT,WACA,OACA,WAAW,SAAS,kBAAkB,cAAc,YACtD;CACF,OAAO;EACL,GAAG;EACH;EACA,SAAS,QAAQ,WAAW,KAAK;EACjC,QAAQ,SAAS,OAAO,cAAc,MAAM,QAAQ,OAAO,OAAO;EAClE;EACA;EACA,MAAM;EACN;CACF;AACF;;;;;;;;;;ACnXA,MAAa,QAAkB,SAAS,GAAG,IAAI;;AAE/C,MAAa,WAAW;;AAExB,MAAa,QAAQ;CAAC;CAAI;CAAI;AAAE;AA6BhC,MAAM,KAAK,UAA4B,SAAS,KAAK,MAAM,QAAQ,GAAG,IAAI,KAAK,GAAG;AAClF,MAAM,QAAQ,QAA0B,OAAO,IAAI,YAAY,OAAO,IAAI;;AAG1E,MAAM,SAAS,GAAW,MAAsB;CAC9C,IAAI,OAAO;CACX,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,KAAK,GAAG;EAC9B,OAAO;EACP,OAAQ,QAAQ,IAAI,MAAO,IAAI;CACjC;CACA,OAAO,MAAM,KAAK;AACpB;;;;;;AAOA,MAAa,kBAAkB,WAAgD;CAC7E,MAAM,IAAI,OAAO;CACjB,IAAI,IAAI;CACR,OAAO,IAAI,KAAK,IAAI,KAAK,MAAM,GAAG,CAAC,KAAK,MAAO,KAAK;CACpD,OAAO,MAAM,IAAI,CAAC,OAAO,IAAK,OAAO,IAAI,EAAG,IAAI,CAAC,OAAO,IAAI,IAAK,OAAO,IAAI,EAAG;AACjF;AAEA,MAAM,UAAU,WAAsC;CACpD,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,CAAC;CACzC,OAAO,OAAO,SAAS,MAAM,IAAI,OAAO,SAAU,OAAO,OAAO,KAAM,OAAO,SAAU;AACzF;;;;;;;;;;;;;AAcA,MAAa,aAAa,YAAwB,SAAuC;CACvF,MAAM,UAAU,KAAK,QAAQ,QAAQ,IAAI,SAAS,IAAI;CACtD,MAAM,SAAS,KAAK,SAAS,QAAQ;CACrC,MAAM,UAAU,QACb,QAAQ,QAAQ,IAAI,YAAY,IAAI,CAAC,CACrC,KAAK,SAAS;EAAE,KAAK,KAAK,GAAG;EAAG,KAAK,IAAI;CAAS,EAAE;CACvD,MAAM,QAAQ;EAAE,SAAS,QAAQ;EAAQ;EAAQ,QAAQ;EAAM,gBAAgB;CAAK;CACpF,IAAI,QAAQ,WAAW,GACrB,OAAO;EAAE,GAAG;EAAO,UAAU;EAAgB,MAAM;EAAM;EAAS,KAAK;CAAiB;CAE1F,IAAI,WAAW,YAAY,MAAM;EAC/B,MAAM,UAAU,QACb,SAAS,SAAS,IAAI,WAAW,CAAC,EAAA,CAAG,KAAK,WAAW,QAAQ,QAAQ,GAAG,CAAC,CAAC,CAAC,CAC3E,MAAM,GAAG,MAAM,IAAI,CAAC;EACvB,MAAM,kBAAkB,QAAQ,QAAQ,KAAK,QAAQ,MAAM,IAAI,iBAAiB,CAAC;EACjF,MAAM,QAAQ,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,WAAW,IAAI,QAAQ,SAAS,KAAK;EAChG,MAAM,MACJ,QAAQ,WAAW,IACf,GAAG,QAAQ,OAAO,iCAAiC,WAAW,QAC9D,GAAG,QAAQ,OAAO,SAAS,QAAQ,OAAO,mBAAmB,EAAE,OAAO,OAAO,CAAC,CAAC,CAAC,MAAM,cAAc,EAAE,QAAQ,GAAG,EAAE,CAAE,CAAC,CAAC,MAAM,MAAM,gBAAgB,oBAAoB,MAAM,OAAO,sBAAsB,WAAW;EAC3N,OAAO;GACL,GAAG;GACH,UAAU;GACV,MAAM;GACN,SAAS,CAAC;GACV,QAAQ,QAAQ,WAAW,IAAI,OAAO,EAAE,OAAO,OAAO,CAAC;GACvD;EACF;CACF;CAEA,IAAI,CAAC,WAAW,WAAW,WAAW,aAAa,KAAA,GAAW;EAC5D,MAAM,WAAW,QAAQ,WAAW,IAAI,UAAU;EAClD,MAAM,MACJ,aAAa,UACT,0BAA0B,QAAQ,OAAO,WAAW,QAAQ,WAAW,QAAQ,GAAG,EAAE,MACpF,GAAG,QAAQ,OAAO,MAAM,QAAQ,OAAO;EAC7C,OAAO;GAAE,GAAG;GAAO;GAAU,MAAM;GAAM;GAAS;EAAI;CACxD;CAGA,MAAM,QAAQ,WAAW,UAAU,aAAa,WAAW;CAC3D,MAAM,QAAQ,QAAQ,OAAO,GAAG;CAChC,MAAM,MAAM,QAAQ,MAAM,KAAK,GAAG;CAClC,MAAM,OAAO,QAAQ,MAAM,MAAM,GAAG;CACpC,MAAM,OAAO;EAAE,KAAK,EAAE,MAAM,KAAK;EAAG,MAAM,EAAE,OAAO,KAAK;CAAE;CAC1D,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,OAAO,SAAS;EACzB,IAAI,IAAI,YAAY,MAAM;EAC1B,MAAM,QAAQ,QAAQ,IAAI,SAAS,GAAG;EACtC,OAAO,KAAK,KAAK;EACjB,IAAI,QAAQ,MAAM,SAAS,QAAQ,OAAO,OACxC,QAAQ,KAAK;GACX,KAAK,KAAK,GAAG;GACb,KAAK,GAAG,MAAM,iBAAiB,KAAK,IAAI,MAAM,IAAI,KAAK,KAAK,MAAM;EACpE,CAAC;CACL;CACA,OAAO,MAAM,GAAG,MAAM,IAAI,CAAC;CAC3B,IAAI,OAAO,WAAW,GACpB,OAAO;EAAE,GAAG;EAAO,UAAU;EAAS;EAAM;EAAS,KAAK;CAA2B;CACvF,MAAM,CAAC,MAAM,MAAM,eAAe,MAAM;CACxC,MAAM,SAAS,EAAE,OAAO,MAAM,CAAC;CAC/B,MAAM,WAAW;EAAE,KAAK,EAAE,IAAI;EAAG,MAAM,EAAE,EAAE;CAAE;CAC7C,MAAM,OAAO;EAAE,KAAK,MAAO;EAAe,MAAM,OAAQ;CAAc;CACtE,MAAM,SAAS,QAAQ,KAAK,OAAO,MAAM,KAAK;CAC9C,MAAM,WAAW,KAAK,KAAK,OAAO,OAAO,KAAK;CAC9C,MAAM,SAAS,GAAG,QAAQ,SAAS,QAAQ,OAAO,MAAM,QAAQ,OAAO,gBAAgB,KAAK,IAAI,MAAM,IAAI,KAAK,KAAK,MAAM;CAC1H,MAAM,SAAS,UAAU,OAAO,MAAM,oBAAoB,SAAS,IAAI,MAAM,IAAI,SAAS,KAAK,MAAM;CACrG,MAAM,SAAS;EACb,SAAS,QAAQ;EACjB;EACA;EACA;EACA;EACA,gBAAgB;CAClB;CACA,IAAI,QAAQ,SAAS,KAAK,UACxB,OAAO;EAAE,GAAG;EAAQ,UAAU;EAAS,KAAK,GAAG,OAAO,IAAI;CAAS;CACrE,IAAI,QAAQ,OAAO;EAAE,GAAG;EAAQ,UAAU;EAAS,KAAK,GAAG,OAAO,IAAI;CAAS;CAC/E,MAAM,WAAW,QAAQ,SAAS,MAAM,KAAK,WAAW;CACxD,OAAO;EAAE,GAAG;EAAQ;EAAU,KAAK,GAAG,OAAO,IAAI,OAAO;CAAqB;AAC/E;;;;;;;;;AC/IA,MAAM,QAA0C;CAC9C,SAAS,cAAc,IAAI,IAAI,MAAM,YAAY,GAAG,CAAC;CACrD,QAAQ,QACN,aAAa,cAAc,YAAY,GAAG,CAAC,CAAC,QAAQ,oCAAoC,CAAC,CAC3F;AACF;;AAGA,MAAM,UAAU,SAAyB;CACvC,MAAM,CAAC,MAAM,GAAG,QAAQ,KAAK,MAAM,GAAG;CACtC,MAAM,OAAO,MAAM;CACnB,IAAI,SAAS,KAAA,GAAW,MAAM,IAAI,MAAM,GAAG,KAAK,uCAAuC;CACvF,OAAO,KAAK,MAAM,GAAG,IAAI;AAC3B;;AAGA,MAAM,mBAAmD,OAAO;;AAUhE,MAAM,UAAiB;CACrB,MAAM;CACN,YAAY;CACZ,MAAM;EACJ;EACA;EACA;EACA;EACA;CACF;AACF;;AAEA,MAAM,OAAc;CAClB,MAAM;CACN,YAAY;CACZ,MAAM,CAAC,oBAAoB;AAC7B;;AAGA,MAAM,WAAW,SAA2B;CAC1C,IAAI,SAAS,IAAI,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI;CAEzC,OADgB,YAAY,MAAM,EAAE,eAAe,KAAK,CAC3C,CAAC,CACX,QAAQ,UAAU,MAAM,SAAS,YAAY,MAAM,SAAS,cAAc,CAAC,CAC3E,SAAS,UAAU,QAAQ,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,CAAC,CACnD,KAAK;AACV;;AAGA,MAAM,WAAW,UACf,MAAM,KAAK,SAAS,SAAS;CAC3B,MAAM,OAAO,OAAO,IAAI;CACxB,OAAO,QAAQ,IAAI,CAAC,CAAC,KAAK,UAAU;EAClC;EACA,IAAI,KAAK,MAAM,SAAS,MAAM,IAAI,CAAC,CAAC,CAAC,WAAW,MAAM,GAAG;CAC3D,EAAE;AACJ,CAAC;;AAGH,MAAM,YAAY,UAAyB;CACzC,MAAM,OAAO,WAAW,QAAQ;CAChC,KAAK,MAAM,EAAE,MAAM,QAAQ,QAAQ,KAAK,GAAG;EACzC,KAAK,OAAO,EAAE;EACd,KAAK,OAAO,aAAa,IAAI,CAAC;CAChC;CACA,OAAO,GAAG,MAAM,KAAK,GAAG,KAAK,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE;AACxD;;AAGA,MAAM,SAAS,UAAyB;CACtC,MAAM,UAAU,YAAY,KAAK,OAAO,GAAG,GAAG,MAAM,KAAK,EAAE,CAAC;CAC5D,KAAK,MAAM,EAAE,MAAM,QAAQ,QAAQ,KAAK,GAAG;EACzC,UAAU,QAAQ,KAAK,SAAS,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,CAAC;EACzD,aAAa,MAAM,KAAK,SAAS,EAAE,CAAC;CACtC;CACA,OAAO;AACT;AAEA,MAAM,wBAAQ,IAAI,IAA6B;;AAE/C,MAAM,SAAS,UAAkC;CAC/C,IAAI,MAAM,MAAM,IAAI,MAAM,IAAI;CAC9B,IAAI,QAAQ,KAAA,GAAW;EACrB,OAAO,YAAY;GACjB,MAAM,OAAO,SAAS,KAAK;GAC3B,MAAM,EAAE,qBAAqB,MAAM,WAAW;GAC9C,MAAM,UAAU,MAAM,KAAK;GAC3B,IAAI;IACF,MAAM,iBAAiB,eAAe,SAAS,MAAM,UAAU,CAAC,CAAC,MAAM,MAAM,EAC3E,cAAc,MAChB,CAAC;GACH,UAAU;IACR,OAAO,SAAS;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC;GAClD;GACA,OAAO;EACT,EAAA,CAAG;EACH,MAAM,IAAI,MAAM,MAAM,GAAG;CAC3B;CACA,OAAO;AACT;;AAGA,MAAa,qBAAsC,MAAM,OAAO;;AAEhE,MAAa,wBAAyC,MAAM,IAAI;;AAmBhE,MAAM,cAAc,YAAoC;CACtD,GAAI,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,CAAC,eAAe,QAAQ,SAAS;CAC5E,GAAI,QAAQ,SAAS,KAAA,IAAY,CAAC,IAAI,CAAC,UAAU,QAAQ,IAAI;CAC7D,GAAI,QAAQ,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,WAAW,OAAO,QAAQ,KAAK,CAAC;AAC1E;;AAGA,MAAM,OAA+C;CACnD,WAAW;CACX,KAAK;CACL,QAAQ;CACR,KAAK;AACP;AA+CA,MAAM,cAA2D;CAC/D,MAAM,OAAY,CAAC;CACnB,MAAM,QAAkB,CAAC;CACzB,MAAM,UAA2E,CAAC;CAClF,IAAI,UAAU;CACd,MAAM,QAAQ,SAAiB;EAC7B,IAAI,CAAC,KAAK,WAAW,GAAG,GAAG;GACzB,IAAI,KAAK,SAAS,GAAG,MAAM,KAAK,IAAI;GACpC;EACF;EACA,MAAM,OAAO,KAAK,MAAM,IAAI;EAC5B,KAAK,KAAK,IAAI;EACd,KAAK,MAAM,QAAQ,QAAQ,OAAO,CAAC,GACjC,IAAI,KAAK,MAAM,SAAS,KAAK,EAAE,GAAG,KAAK,QAAQ,IAAI;OAC9C,QAAQ,KAAK,IAAI;CAE1B;CACA,OAAO;EACL;EACA;EACA,UAAU,WACR,OAAO,GAAG,SAAS,UAA2B;GAC5C,WAAW,MAAM,SAAS;GAC1B,MAAM,QAAQ,QAAQ,MAAM,IAAI;GAChC,UAAU,MAAM,IAAI,KAAK;GACzB,KAAK,MAAM,QAAQ,OAAO,KAAK,KAAK,KAAK,CAAC;EAC5C,CAAC;EACH,MAAM,OAAO,QAAQ,KAAK;GACxB,MAAM,QAAQ,KAAK,MAAM,SAAS,MAAM,SAAS,KAAK,EAAE,CAAC;GACzD,IAAI,UAAU,KAAA,GAAW,OAAO,QAAQ,QAAQ,KAAK;GACrD,OAAO,IAAI,SAAS,SAAS,WAAW;IACtC,MAAM,QAAQ,iBAAiB;KAC7B,MAAM,OAAO,MAAM,WAAW,IAAI,KAAK,eAAe,MAAM,MAAM,EAAE,CAAC,CAAC,KAAK,KAAK;KAChF,uBAAO,IAAI,MAAM,MAAM,MAAM,KAAK,MAAM,EAAE,YAAY,IAAI,MAAM,OAAO,KAAK,MAAM,CAAC;IACrF,GAAG,MAAM;IACT,QAAQ,KAAK;KACX;KACA,UAAU,SAAS;MACjB,aAAa,KAAK;MAClB,QAAQ,IAAI;KACd;IACF,CAAC;GACH,CAAC;EACH;CACF;AACF;AA6FA,MAAM,OAAO,EAAE,SAAS,EAAE;AAE1B,MAAM,SAAS,OAAO,WAAiC,SAAoC;CACzF,MAAM,SAAS,MAAM,UAAU,KAAK,IAAI;CACxC,IAAI,OAAO,aAAa,GACtB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK,GAAG,EAAE,UAAU,OAAO,SAAS,IAAI,OAAO,OAAO,KAAK,GAAG;CACxF,OAAO,OAAO,OAAO,KAAK;AAC5B;;AAGA,MAAM,UAAU,OAAO,cAAqD;CAC1E,MAAM,OAAO,MAAM,OAAO,WAAW,CAAC,YAAY,mBAAmB,CAAC;CACtE,MAAM,QAAQ,kBAAkB,KAAK,IAAI,CAAC,GAAG;CAC7C,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,4BAA4B,MAAM;CAC3E,OAAO,OAAO,KAAK;AACrB;;;;;;;;;;;;;;AAeA,MAAa,UAAU,OAAO,QAAgB,OAAe,UAAwB;CACnF,MAAM,QAAQ,MAAM,aAAa;CACjC,MAAM,cAAc,SAAS,YAAY,MAAM,gBAAgB,IAAI;CACnE,MAAM,UACJ,SAAS,YAAY,CAAC,QAAQ,sCAAsC,IAAI,CAAC,qBAAqB;CAChG,MAAM,EAAE,kBAAkB,SAAS,MAAM,8BAA8B,MAAM,WAAW;CACxF,MAAM,SAAiC,CAAC;CACxC,IAAI;CACJ,MAAM,QAAQ,YAAY;EACxB,KAAK,MAAM,aAAa,OAAO,QAAQ,GAAG,MAAM,UAAU,KAAK,IAAI,CAAC,CAAC,YAAY,KAAA,CAAS;EAC1F,MAAM,SAAS,KAAK,CAAC,CAAC,YAAY,KAAA,CAAS;CAC7C;CACA,IAAI;EACF,UAAU,MAAM,IAAI,QAAQ,CAAC,CAAC,MAAM;EACpC,MAAM,QAAQ,IAAI,iBAAiB,OAAO,KAAK,CAAC,CAC7C,YAAY,OAAO,CAAC,CACpB,iBAAiB,KAAK,cAAc,OAAO,KAAK,CAAC;EAEpD,MAAM,SAAS,OACb,OAAO,YAAY,KAAA,IAAY,QAAQ,MAAM,YAAY,CAAC,GAAG,OAAO,OAAO,CAAC,EAAA,CAC5E,MAAM;EACR,OAAO,KAAK,MAAM;EAClB,MAAM,SAAS,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAAC,YAAY,OAAO,CAAC,CAAC,MAAM;EAC5E,OAAO,KAAK,MAAM;EAClB,MAAM,QAAQ,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAC5C,gBAAgB,aAAa,OAAO,MAAM,GAAG,CAAC,CAC9C,mBAAmB,CAAC,CACpB,MAAM;EACT,OAAO,KAAK,KAAK;EACjB,MAAM,WAAW,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAC/C,gBAAgB,aAAa,OAAO,MAAM,GAAG,CAAC,CAC9C,sBAAsB,aAAa,SAAS,CAAC,CAC7C,MAAM;EACT,OAAO,KAAK,QAAQ;EACpB,MAAM,QAAQ;GAAE,QAAQ,MAAM,QAAQ,KAAK;GAAG,QAAQ,MAAM,QAAQ,QAAQ;EAAE;EAC9E,MAAM,UAAU,MAAoB;EACpC,MAAM,SAAS,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAC7C,mBAAmB,CAAC,CACpB,gBAAgB,MAAM,CAAC,CACvB,YAAY;GAAC;GAAuB,OAAO,MAAM,MAAM;GAAG,OAAO,MAAM,MAAM;EAAC,CAAC,CAAC,CAChF,gBAAgB,QAAQ,OAAO,CAAC,CAChC,MAAM;EACT,OAAO,KAAK,MAAM;EAClB,MAAM,QAAQ,MAAM,QAAQ,MAAM,CAAC,SAAS,MAAM,GAAG,KAAQ,QAAQ;EACrE,IAAI,MAAM,OAAO,QAAQ,MAAM,IAAI,MAAM,sBAAsB,OAAO,MAAM,MAAM,GAAG;EACrF,MAAM,WAAW,OAAO,aAAa,QAAQ,QAAQ,CAAC;EACtD,MAAM,WAAW,OAAO,aAAa,QAAQ,QAAQ,CAAC;EACtD,MAAM,UAAU,MAAM,0BAA0B;EAChD,MAAM,SAAS,QAAQ,UAAU,QAAQ,OAAO,MAAM,CAAC;EACvD,IAAI;EAEJ,OAAO;GACL;GACA;GACA;GACA,QAAQ,QAAQ;GAChB,MAAM,OAAO,QAAQ;IACnB,MAAM,WAAW,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,GAAG,IAAI,GAAG,OAAO;IAC/E,IAAI,SAAS,SAAS,GAAG,MAAM,OAAO,OAAO;KAAC;KAAU;KAAM;KAAM,GAAG;IAAQ,CAAC;IAChF,MAAM,OAA+B,CAAC;IACtC,KAAK,MAAM,OAAO,OAAO,KAAK,MAAM,GAClC,KAAK,OAAO,MAAM,OAAO,OAAO;KAAC;KAAU;KAAM;IAAG,CAAC;IACvD,OAAO;GACT;GACA,MAAM,aAAa,QAAQ;IACzB,MAAM,OAA+B,CAAC;IACtC,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC,WAAW,GAAG,OAAO;IAE7C,gBAAgB,MAAM,IAAI,iBAAiB,KAAK,CAAC,CAC9C,gBAAgB,aAAa,OAAO,MAAM,GAAG,CAAC,CAC9C,mBAAmB,CAAC,CACpB,MAAM;IACT,OAAO,KAAK,WAAW;IACvB,MAAM,WAAW,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,GAAG,IAAI,GAAG,OAAO;IAC/E,MAAM,OAAO,aAAa;KAAC;KAAU;KAAM;KAAM,GAAG;IAAQ,CAAC;IAC7D,KAAK,MAAM,OAAO,OAAO,KAAK,MAAM,GAClC,KAAK,OAAO,MAAM,OAAO,aAAa;KAAC;KAAU;KAAM;IAAG,CAAC;IAC7D,OAAO;GACT;GACA,MAAM,UAAU,QAAQ;IACtB,MAAM,OAA+B,CAAC;IACtC,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC,WAAW,GAAG,OAAO;IAC7C,MAAM,MAAM,OAAO;IACnB,IAAI,QAAQ,KAAA,GAAW,MAAM,IAAI,MAAM,GAAG,OAAO,MAAM,kCAAkC;IACzF,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO,QAAQ,MAAM,GAAG;KAClD,MAAM,OAAO,QAAQ;MAAC,GAAG,IAAI;MAAK;MAAM;KAAK,CAAC;KAC9C,KAAK,SAAS,MAAM,OAAO,QAAQ,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,EAAA,CAAG,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;IAChF;IACA,OAAO;GACT;GACA,MAAM,MAAM,MAAM,KAAK,UAAU,UAAU,CAAC,GAAG;IAC7C,MAAM,CAAC,OAAO,QAAQ,OAAO,WAAW,CAAC,OAAO,QAAQ,IAAI,CAAC,UAAU,QAAQ;IAC/E,MAAM,OAAO;KAAC;KAAuB;KAAM;KAAM,GAAG,WAAW,OAAO;IAAC;IACvE,MAAM,QAAQ,KAAK,MAAM,MAAM,OAAO,OAAO,IAAI,CAAC;IAClD,OAAO;KAAE,IAAI,MAAM;KAAI,IAAI,MAAM;IAAG;GACtC;GACA,MAAM,KAAK,MAAM,KAAK,UAAU,UAAU,CAAC,GAAG;IAC5C,MAAM,CAAC,OAAO,QAAQ,OAAO,WAAW,CAAC,OAAO,QAAQ,IAAI,CAAC,UAAU,QAAQ;IAC/E,MAAM,OAAO,OAAO;KAAC;KAAuB,KAAK;KAAO;KAAM,GAAG,WAAW,OAAO;IAAC,CAAC;GACvF;GACA,MAAM,MAAM,MAAM,KAAK,UAAU;IAC/B,MAAM,QAAQ,OAAO,WAAW,QAAQ;IACxC,MAAM,OACJ,SAAS,OACL;KAAC;KAAuB;KAAe;IAAM,IAC7C;KAAC;KAAuB;KAAS;KAAQ,GAAG,KAAK,MAAM,KAAK;IAAC;IACnE,MAAM,QAAQ,KAAK,MAAM,MAAM,OAAO,OAAO,IAAI,CAAC;IAClD,OAAO;KAAE,IAAI,MAAM;KAAI,IAAI,MAAM;IAAG;GACtC;GACA,MAAM,QAAQ,IAAI,KAAK,UAAU;IAE/B,MAAM,OADQ,OAAO,WAAW,QAAQ,UACpB;KAAC;KAAuB;KAAW;KAAQ,KAAK,OAAO;IAAK,CAAC;GACnF;GACA,MAAM,OAAO;IACX,IAAI,OAAO,SAAS,KAAA,GAAW,MAAM,IAAI,MAAM,GAAG,OAAO,MAAM,iCAAiC;IAChG,MAAM,OAAO,QAAQ,CAAC,GAAG,OAAO,IAAI,CAAC;GACvC;GACA,MAAM,OAAO;IACX,MAAM,OAAO,OAAO;KAAC;KAAQ;KAAM;KAAK;KAAM;KAAK;IAAQ,CAAC;IAC5D,MAAM,OAAO,UAAU;KAAC;KAAQ;KAAM;KAAK;KAAM;KAAK;IAAQ,CAAC;GACjE;GACA,MAAM,MAAM,UAAU;IACpB,KAAK,MAAM,QAAQ,UAAU,MAAM,OAAO,QAAQ,CAAC,GAAG,IAAI,CAAC;GAC7D;GACA,MAAM,KAAK,IAAI,MAAM;IACnB,MAAM,CAAC,OAAO,QAAQ,OAAO,WAAW,CAAC,OAAO,QAAQ,IAAI,CAAC,UAAU,QAAQ;IAE/E,MAAM,QAAQ,KAAK,MAAM,MAAM,OAAO,OAAO;KAD/B;KAAuB;KAAS;KAAM;KAAQ,OAAO,KAAK,MAAM,IAAI,CAAC;IACnC,CAAC,CAAC;IAClD,OAAO;KAAE,IAAI,MAAM;KAAI,IAAI,MAAM;IAAG;GACtC;GACA,MAAM,SAAS;IACb,MAAM,OAAO,OAAO;KAAC;KAAM;KAAe;KAAS;IAAK,CAAC;IACzD,MAAM,OAAO,UAAU;KAAC;KAAM;KAAe;KAAS;IAAK,CAAC;GAC9D;GACA,cAAc,OAAO,MAAM,CAAC,CAAC,WAAW,KAAA,CAAS;GACjD,YAAY,OAAO,QAAQ,CAAC,CAAC,WAAW,KAAA,CAAS;GACjD,MAAM,MAAM,MAAM;IAChB,MAAM,SAAS,MAAkB;IACjC,MAAM,YAAY,MAAM,IAAI,iBAAiB,WAAW,CAAC,CACtD,gBAAgB,aAAa,OAAO,MAAM,GAAG,CAAC,CAC9C,YAAY,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAClC,gBAAgB,OAAO,OAAO,CAAC,CAC/B,MAAM;IACT,OAAO;KACL,QAAQ,OAAO;KACf,QAAQ,OAAO,WAAW,OAAO,MAAM,OAAO,QAAQ,QAAQ;KAC9D,UAAU,OAAO,WAAW,CAAC,SAAS,SAAS,CAAC,CAAC,CAAC,WAAW,KAAA,CAAS;KACtE,cACE,QAAQ,UACL,QAAQ,UAAU,MAAM,CAAC,CAAC,CAC1B,MAAM,CAAC,CACP,WAAW,KAAA,CAAS;KACzB,YACE,QAAQ,UACL,QAAQ,UAAU,MAAM,CAAC,CAAC,CAC1B,QAAQ,CAAC,CACT,WAAW,KAAA,CAAS;KACzB,YACE,QAAQ,UACL,QAAQ,UAAU,MAAM,CAAC,CAAC,CAC1B,KAAK,EAAE,QAAQ,UAAU,CAAC,CAAC,CAC3B,WAAW,KAAA,CAAS;KAEzB,YACE,UACG,KAAK,IAAI,CAAC,CACV,WAAW,KAAA,CAAS,CAAC,CACrB,YAAY,KAAA,CAAS;IAC5B;GACF;GACA,MAAM,SAAS;IAEb,MAAM,QAAQ,QAAQ,KAAK;IAC3B,MAAM,QAAQ,MAAM,IAAI,SAA8B,YAAY;KAChE,MAAM,UAAU,KAAK,IAAI;KACzB,MAAM,aAAa;MACjB,MAAM,OAAO,QAAQ,KAAK,MAAM,KAAK,CAAC,CAAC,MAAM,WAAW,OAAO,OAAO,OAAO;MAC7E,IAAI,SAAS,KAAA,GAAW,QAAQ,IAAI;WAC/B,IAAI,KAAK,IAAI,IAAI,UAAU,KAAO,QAAQ,IAAI;WAC9C,WAAW,MAAM,EAAE;KAC1B;KACA,KAAK;IACP,CAAC;IACD,OAAO,UAAU,OAAO,OAAO,OAAO,MAAM,WAAW,CAAC;GAC1D;GACA,MAAM,SAAS,SAAS,WAAW,CAAC,GAAG,gBAAgB,CAAC,GAAG;IACzD,MAAM,SAAS,MAAM,OAAO,OAAO,CAAC,QAAQ,iBAAiB,CAAC,CAAC,CAAC,YAAY,EAAE;IAC9E,MAAM,UAAU,OAAO,cACrB,QAAQ,UAAU,QAAQ,QAAQ,UAAU,QAAQ,UAAU,MAAM,CAAC,CAAC;IACxE,OAAO;KACL,QAAQ,MAAM,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC;KAC5C,cAAc,MAAM,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC;KAClD,IAAI,qBAAqB,KAAK,MAAM,CAAC,GAAG,MAAM;KAC9C,SAAS,MAAM,OAAO,QAAQ,CAAC,OAAO,iCAAiC,CAAC;KACxE,QAAQ,GAAG,QAAQ,KAAK,iBAAiB,cAAc,MAAM,QAAQ,KAAK,iBAAiB;KAC3F,QAAQ;MACN,OAAO,OAAO;MACd,KAAK,MAAM,QAAQ,MAAM,EAAA,CAAG;MAC5B;MACA,SAAS;KACX;KACA,SAAS;MACP;MACA,KAAK,MAAM,QAAQ,MAAM,EAAA,CAAG;MAC5B,WAAW,MAAM,OAAO,QAAQ;OAAC;OAAO;OAAQ;MAAI,CAAC,EAAA,CAAG,MAAM,IAAI,CAAC,CAAC,KAAK;KAC3E;KACA,QAAQ;MAAE;MAAM,OAAO;KAAY;KACnC;KACA;IACF;GACF;GACA;EACF;CACF,SAAS,OAAO;EACd,MAAM,MAAM;EACZ,MAAM;CACR;AACF;;;;;;;;AC7jBA,MAAa,SACX,MACA,SAC+C;CAC/C,MAAM,WAAW,KAAK,KAAK,QAAQ,QAAQ,MAAM,GAAG,CAAC;CACrD,OAAO;EAAE;EAAU,SAAS,UAAU,KAAK,YAAY,QAAQ;CAAE;AACnE;;AAGA,MAAM,gBAAgB,YAA6C;CACjE,KAAK,IAAI,KAAK,QAAQ,SAAS,GAAG,MAAM,GAAG,MAAM,GAC/C,IAAI,QAAQ,GAAG,CAAE,OAAO,SAAS,OAAO,OAAO,QAAQ,GAAG,CAAE,WAAW,CAAC;CAC1E,OAAO;AACT;;AAGA,MAAM,aAAa,MAAY,QAAuB;CACpD,MAAM,EAAE,YAAY,UAAU;CAC9B,MAAM,UAAU,WAAW,UACvB,QAAQ,WAAW,MAAM,GAAG,IAC5B,QAAQ,WAAW,QAAQ,GAAG;CAClC,MAAM,OAAO;EAAC,IAAI;EAAU,OAAO,KAAK,OAAO,IAAI;EAAG;EAAU,MAAM;CAAI;CAC1E,IAAI,MAAM,YAAY,KAAA,GAAW,KAAK,KAAK,aAAa,MAAM,OAAO;CACrE,IAAI,MAAM,UAAU,KAAA,GAAW,KAAK,KAAK,WAAW,OAAO,MAAM,KAAK,CAAC;CACvE,IAAI,MAAM,aAAa,KAAA,GAAW,KAAK,KAAK,iBAAiB,OAAO,QAAQ,MAAM,UAAU,IAAI,CAAC,CAAC;CAClG,IAAI,MAAM,WAAW,KAAA,GAAW;EAE9B,KAAK,KAAK,kBAAkB,OAAO,QAAQ,MAAM,OAAO,WAAW,IAAI,CAAC,CAAC;EACzE,KAAK,KAAK,WAAW,OAAO,KAAK,KAAK,UAAU,IAAI,EAAE,CAAC,CAAC;EACxD,OAAO;CACT;CACA,IAAI,MAAM,WAAW,KAAK,KAAK,aAAa;CAC5C,IAAI,MAAM,gBAAgB,KAAA,GACxB,KAAK,KAAK,qBAAqB,OAAO,QAAQ,MAAM,aAAa,IAAI,CAAC,CAAC;CACzE,KAAK,KAAK,WAAW,OAAO,KAAK,KAAK,UAAU,IAAI,EAAE,CAAC,CAAC;CACxD,OAAO;AACT;;;;;;;AAQA,MAAM,eAAe,KAAU,OAAgB,WAC7C,IAAI,SAAS,YAAY;CACvB,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,aAAa;EAQjB,IAPa,IAAI,OAAO,MACrB,WACC,OAAO,UAAU,IAAI,MAAM,UAC3B,OAAO,QAAQ,UACb,OAAO,OAAO,YAAY,OAAO,OAAO,OAAO,KAC9C,OAAO,OAAO,WAAY,OAAqC,QAAQ,EAEvE,KAAK,KAAK,IAAI,IAAI,UAAU,QAAQ,QAAQ;OAC9C,WAAW,MAAM,EAAE;CAC1B;CACA,KAAK;AACP,CAAC;;;;;AAMH,MAAM,eAAe,OAAc,WACjC,IAAI,SAAS,SAAS,WAAW;CAC/B,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,aAAa;EACjB,MAAM,OAAO,MAAM,OAAO,QAAQ,UAAU,MAAM,OAAO,MAAM,CAAC,CAAC;EACjE,IAAI,SAAS,KAAA,GAAW,QAAQ,IAAI;OAC/B,IAAI,KAAK,IAAI,IAAI,UAAU,QAC9B,uBAAO,IAAI,MAAM,qCAAqC,OAAO,qBAAqB,CAAC;OAChF,WAAW,MAAM,CAAC;CACzB;CACA,KAAK;AACP,CAAC;;;;;;;;;;;;;AAcH,MAAM,aAAa,MAAY,UAA6C;CAC1E,MAAM,QAAQ,CACZ,GAAI,KAAK,YAAY,KAAA,IAAY,CAAC,IAAI,CAAC,SAAS,QAAQ,KAAK,SAAS,IAAI,EAAE,GAAG,GAC/E,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,KAAK,CACvC;CACA,OAAO,MAAM,WAAW,IAAI,OAAO,MAAM,KAAK,GAAG;AACnD;AAEA,MAAM,OAAO,OAAO,KAAU,MAAY,aAAqB,UAAgC;CAE7F,MAAM,IAAI,OAAO;CACjB,MAAM,OAAO,IAAI,OAAO;CACxB,MAAM,aAAa,aAAa,IAAI,MAAM;CAC1C,MAAM,QAAQ,MAAM,IAAI,MAAM,UAAU,MAAM,GAAG,CAAC;CAClD,MAAM,SAAS,KAAK,aAAa;CACjC,MAAM,OAAO,SACT;EAAE,cAAc,MAAM,OAAO;EAAG,YAAY,MAAM,KAAK;CAAE,IACzD;EAAE,cAAc,IAAI,OAAO;EAAG,YAAY,IAAI,KAAK;CAAE;CAEzD,MAAM,WACJ,MAAM,GAAG,CAAC,CAAC,OAAO,UAAmB;EACnC,MAAM,OAAO,MAAM,OAAO,MAAM,EAAE,CAAC,CAAC,KAAK,UAAU,KAAK,UAAU,KAAK,CAAC;EACxE,MAAM,IAAI,MAAM,GAAI,MAAgB,QAAQ,6BAA6B,KAAK,KAAK,GAAG,GAAG;CAC3F,CAAC;CACH,IAAI;CACJ,IAAI,OAA0C;CAC9C,IAAI;EACF,MAAM,MAAM,MAAM,CAAC,OAAO,GAAG,IAAM;EAInC,MAAM,gBACJ,KAAK,MAAM,SAAS,eAClB,UAAU,KAAK,UAAU,mBAAoB,KAAK,UAAU;EAChE,IAAI,eAAe;GAGjB,MAAM,GAAG;GACT,MAAM,IAAI,SAAS,YAAY,WAAW,SAAS,GAAG,CAAC;EACzD;EACA,MAAM,UAAU,SAAS,WAAW;EAGpC,MAAM,YACJ,KAAK,UAAU,KAAA,IACX,KAAA,IACC,KAAK,UAAU,iBAAkB,YAAY,YAC5C,OACA;EACR,MAAM,UAAwB;GAC5B,GAAI,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU;GAC/C,GAAI,KAAK,SAAS,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM,KAAK,KAAK;GACrD,GAAI,KAAK,QAAQ,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,IAAI;EACtD;EAEA,MAAM,QAAQ,YAAmC;GAC/C,QAAQ,KAAK,OAAb;IACE,KAAK;KACH,MAAM,KAAK,OAAO;KAClB,OAAO;IACT,KAAK;KACH,OAAO,SAAS,MAAM,KAAK,IAAI,IAAI,KAAK;KACxC,OAAO;IACT,KAAK,eACH,OAAO,IAAI,MAAM,UAAU,SAAS,EAAE,MAAM,KAAK,QAAQ,OAAO,CAAC;IACnE,KAAK,WACH,OAAO,IAAI,MAAM,aAAa,SAAS,OAAO;IAChD,KAAK,iBACH,OAAO,IAAI,MAAM,OAAO,UAAU,EAAE,OAAO,KAAK,OAAO,KAAK,CAAC;IAC/D,KAAK,SACH,OAAO,IAAI,MAAM,UAAU,MAAM,KAAK,KAAK,GAAG,QAAQ;IACxD,KAAK,OACH,OAAO,IAAI,MAAM,OAAO,OAAO;IACjC,SAEE,OAAO,IAAI,MAAM,aAAa,OAAO;GACzC;EACF;EACA,MAAM,OAAO,YAA2B;GACtC,QAAQ,KAAK,OAAb;IACE,KAAK,kBACH,OAAO,KAAK,KAAK;IACnB,KAAK,OACH;IACF,KAAK,eACH,OAAO,IAAI,KAAK,UAAU,SAAS,EAAE,MAAM,KAAK,QAAQ,OAAO,CAAC;IAClE,KAAK,WACH,OAAO,IAAI,KAAK,aAAa,SAAS,OAAO;IAC/C,KAAK,iBACH,OAAO,IAAI,KAAK,OAAO,UAAU,EAAE,OAAO,KAAK,OAAO,KAAK,CAAC;IAC9D,KAAK;KACH,MAAM,IAAI,MAAM,UAAU,MAAM,KAAA,CAAS,GAAG,QAAQ;KACpD;IACF,KAAK,OACH,OAAO,IAAI,KAAK,OAAO,OAAO;IAChC,SACE,OAAO,SAAS,OAAO,IAAI,KAAK,aAAa,OAAO,IAAI,KAAA;GAC5D;EACF;EACA,QAAQ,MAAM,MAAM;EACpB,IAAI;GACF,IAAI,KAAK,MAAM,SAAS,UAAU,CAAC,eAAe,MAAM,GAAG;GAC3D,MAAM,EAAE,eAAe;GACvB,IAAI,KAAK,UAAU,SAAS;IAC1B,IAAI,KAAK,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK,wBAAwB;IAEnF,MAAM,OAAO,MAAM,YAAY,OAAO,GAAK;IAC3C,OAAO,MAAM,IAAI,KAAK,SAAS,KAAK,IAAI,QAAQ,KAAK,OAAO,GAAG,IAAI,GAAG;IACtE,MAAM,SAAS,QAAQ,WAAW,UAAU,WAAW,OAAO,WAAW,QAAQ,GAAG;IACpF,MAAM,MAAM,MAAM;KAAC;KAAQ;KAAS;IAAK,GAAG,SAAS,GAAK,CAAC,CAAC,YAAY,KAAA,CAAS;GACnF,OAAO,IAAI,WAAW,YAAY,MAAM;IAEtC,MAAM,SAAS,QAAQ,WAAW,QAAQ,GAAG,IAAI;IACjD,MAAM,MAAM,MAAM;KAAC;KAAQ;KAAS;KAAO;IAAU,GAAG,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GACvF,OAAO,IAAI,WAAW,SAAS;IAC7B,MAAM,SAAS,QAAQ,WAAW,MAAM,GAAG,IAAI,MAAQ;IACvD,IAAI,QAAQ;KACV,MAAM,QAAQ,MAAM,OAAO,MAAM,UAAU,MAAM,OAAO,WAAW,CAAC,GAAG;KACvE,MAAM,YAAY,KAAK,OAAO,MAAM;IACtC,OACE,MAAM,MAAM,MAAM;KAAC;KAAS;KAAO;IAAU,GAAG,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GAEjF,OACE,MAAM,IAAI,SAAS,YACjB,WAAW,SAAS,QAAQ,WAAW,QAAQ,GAAG,IAAI,GAAK,CAC7D;EAEJ,UAAU;GACR,MAAM,KAAK;EACb;EAEA,IAAI,KAAK,UAAU,oBAAoB,CAAC,QACtC,MAAM,MAAM,MAAM,CAAC,MAAM,GAAG,GAAK,CAAC,CAAC,YAAY,KAAA,CAAS;CAC5D,UAAU;EACR,MAAM,MAAM,KAAK;CACnB;CAEA,MAAM,UAAU,MAAM,IAAI,OAAO;CACjC,MAAM,OAAO,YAAY,OAAO,OAAO,UAAU;CACjD,OAAO;EACL;EACA;EACA,QAAQ,CAAC,GAAG,MAAM,MAAM;EACxB;EACA,GAAI,KAAK,UAAU,UAAU,EAAE,KAAK,IAAI,CAAC;EACzC,QAAQ,IAAI,OAAO,MAAM,IAAI;EAC7B,OAAO,IAAI,MAAM;EACjB,aAAa,IAAI,MAAM;EACvB;CACF;AACF;;AAGA,MAAa,UAAU,MAAc,iBACnC,MAAM,KACJ,EAAE,QAAQ,aAAa,IACtB,GAAG,OAAO,KAAK,MAAM,OAAO,YAAY,KAAK,KAAK,OAAO,eAAe,IAAI,EAC/E,CAAC,CAAC,QAAQ,UAAU,QAAQ,CAAC;;AAG/B,MAAM,gBAAgB,OACpB,MACA,aACA,OACA,QACiD;CACjD,MAAM,MAAM,MAAM,QAAQ,KAAK,QAAQ,KAAK,MAAM,QAAQ,QAAQ,KAAK;CACvE,IAAI;EAEF,MAAM,WAAW,MAAM,IAAI,UAAU,KAAK,kBAAkB,CAAC,CAAC;EAC9D,MAAM,gBAAgB,MAAM,IAAI,aAAa,KAAK,iBAAiB,CAAC,CAAC;EACrE,IAAI,KAAK,gBAAgB,KAAA,GAAW,MAAM,IAAI,MAAM,KAAK,WAAW;EAEpE,IAAI,KAAK,UAAU,mBAAmB,KAAK,UAAU,SAAS;GAC5D,MAAM,IAAI,QAAQ,OAAO,QAAQ;GACjC,MAAM,IAAI,QAAQ,OAAO,QAAQ;EACnC;EAEA,MAAM,UAAU,UAAU,MAAM,KAAA,CAAS;EACzC,IAAI,YAAY,MAAM,MAAM,IAAI,MAAM,SAAS,QAAQ;EAEvD,MAAM,IAAI,KAAK;EACf,MAAM,WAAW,MAAM,IAAI,SAAS,MAAM,IAAI,OAAO,KAAK,OAAO,GAAG,UAAU,aAAa;EAC3F,MAAM,OAAc,CAAC;EACrB,KAAK,IAAI,QAAQ,GAAG,QAAQ,OAAO,SAAS,GAAG;GAC7C,MAAM,MAAM,MAAM,KAAK,KAAK,MAAM,aAAa,KAAK;GACpD,KAAK,KAAK,GAAG;GACb,MAAM,OAAO,QAAQ,MAAM,GAAG;GAC9B,IACE,GAAG,KAAK,KAAK,QAAQ,YAAY,OAAO,MAAM,GAAG,KAAK,QAAQ,OACzD,KAAK,YAAY,OAAO,KAAK,IAAI,KAAK,QAAQ,MAAM,MACpD,KAAK,SAAS,OAAO,KAAK,WAAW,KAAK,KAAK,IACtD;EACF;EACA,OAAO;GAAE;GAAU;EAAK;CAC1B,UAAU;EACR,MAAM,IAAI,MAAM;CAClB;AACF;;;;;;;;AASA,MAAa,UAAU,OACrB,MACA,YAAoC,CAAC,MACb;CACxB,MAAM,2BAAU,IAAI,KAAK,EAAA,CAAE,YAAY;CACvC,MAAM,YAAwB,CAAC;CAC/B,MAAM,OAAc,CAAC;CACrB,MAAM,cAAc,KAAK,IAAI,GAAG,KAAK,eAAe,CAAC;CACrD,IAAI,SAAiB,KAAK;CAC1B,IAAI,OAAO;CACX,SAAS;EACP,MAAM,SAAS,OAAO,SAAS,KAAK,QAAQ,KAAK,IAAI,KAAK,cAAc,WAAW,CAAC;EACpF,KAAK,IAAI,KAAK,GAAG,KAAK,OAAO,QAAQ,MAAM,aAAa;GACtD,MAAM,QAAQ,OAAO,MAAM,IAAI,KAAK,WAAW,CAAC,CAAC,KAAK,WAAW;IAAE;IAAO,KAAK;GAAO,EAAE;GACxF,MAAM,OAAO,MAAM,QAAQ,IACzB,MAAM,KAAK,EAAE,OAAO,UAAU,cAAc,MAAM,KAAK,OAAO,GAAG,CAAC,CACpE;GACA,KAAK,MAAM,QAAQ,MAAM;IACvB,UAAU,KAAK,KAAK,QAAQ;IAC5B,KAAK,KAAK,GAAG,KAAK,IAAI;GACxB;EACF;EACA,MAAM,EAAE,UAAU,YAAY,MAAM,MAAM,IAAI;EAC9C,MAAM,SAAS,MAAM,MAAM,SAAS,OAAO,KAAK,MAAM;EACtD,IAAI,QAAQ,aAAa,YAAY,WAAW,KAAA,GAE9C,OAAO;GAAE;GAAM;GAAW;GAAM;GAAU;GAAS,IAAA;IADtC;IAAS,wBAAO,IAAI,KAAK,EAAA,CAAE,YAAY;GACA;EAAE;EAExD,SAAS;CACX;AACF"}
|
package/image/Dockerfile
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# The harness's own image: the prober, the cell client, the injector and the namespace admin all
|
|
2
|
+
# run from it. A target's image stays stock. Both stages are pinned by digest,
|
|
3
|
+
# the crates by Cargo.lock, and the packages apk installs are recorded in each run's manifest.
|
|
4
|
+
# lab.ts stages the build context: the prober's and the harness's native sources, and the prober's
|
|
5
|
+
# identity fixture, each under its package's name.
|
|
6
|
+
|
|
7
|
+
# --- build: the prober's kernel program in C (CO-RE), the user side in Rust --------------------
|
|
8
|
+
FROM alpine:3.23@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62 AS build
|
|
9
|
+
RUN apk add --no-cache clang llvm bpftool libbpf-dev linux-headers \
|
|
10
|
+
rust cargo build-base elfutils-dev zlib-dev zstd-dev pkgconf
|
|
11
|
+
WORKDIR /src
|
|
12
|
+
COPY prober/bpf/ prober/bpf/
|
|
13
|
+
# the kernel's own types, from the kernel the build runs on: the program relocates against them
|
|
14
|
+
RUN bpftool btf dump file /sys/kernel/btf/vmlinux format c > prober/bpf/vmlinux.h \
|
|
15
|
+
&& clang -O2 -g -target bpf -D__TARGET_ARCH_x86 -Iprober/bpf -c prober/bpf/prober.bpf.c -o prober.bpf.o
|
|
16
|
+
COPY prober/native/ prober/native/
|
|
17
|
+
# the record tests read the prober's identity fixture
|
|
18
|
+
COPY prober/fixtures/ prober/fixtures/
|
|
19
|
+
RUN cd prober/native && cargo build --release --locked --features collector \
|
|
20
|
+
&& cargo test --release --locked --features collector
|
|
21
|
+
COPY harness/image/native/ harness/image/native/
|
|
22
|
+
RUN cd harness/image/native && cargo build --release --locked && cargo test --release --locked
|
|
23
|
+
|
|
24
|
+
# --- run ------------------------------------------------------------------------------------------
|
|
25
|
+
FROM alpine:3.23@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62
|
|
26
|
+
# tc for netem, ethtool for the offloads that decide whether netem acts on segments
|
|
27
|
+
RUN apk add --no-cache iproute2 iproute2-tc iptables ethtool libelf zlib zstd-libs libgcc
|
|
28
|
+
COPY --from=build /src/prober.bpf.o /src/prober/native/target/release/prober /opt/harness/
|
|
29
|
+
COPY --from=build /src/harness/image/native/target/release/client /src/harness/image/native/target/release/inject /opt/harness/
|
|
30
|
+
WORKDIR /opt/harness
|
|
31
|
+
CMD ["sleep", "infinity"]
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "harness-native"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
dependencies = [
|
|
9
|
+
"libc",
|
|
10
|
+
"prober",
|
|
11
|
+
"serde_json",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[[package]]
|
|
15
|
+
name = "itoa"
|
|
16
|
+
version = "1.0.18"
|
|
17
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
+
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
19
|
+
|
|
20
|
+
[[package]]
|
|
21
|
+
name = "libc"
|
|
22
|
+
version = "0.2.190"
|
|
23
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
24
|
+
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
|
|
25
|
+
|
|
26
|
+
[[package]]
|
|
27
|
+
name = "memchr"
|
|
28
|
+
version = "2.8.3"
|
|
29
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
30
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
31
|
+
|
|
32
|
+
[[package]]
|
|
33
|
+
name = "prober"
|
|
34
|
+
version = "0.1.0"
|
|
35
|
+
dependencies = [
|
|
36
|
+
"libc",
|
|
37
|
+
"serde",
|
|
38
|
+
"serde_json",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[[package]]
|
|
42
|
+
name = "proc-macro2"
|
|
43
|
+
version = "1.0.107"
|
|
44
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
45
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
46
|
+
dependencies = [
|
|
47
|
+
"unicode-ident",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[[package]]
|
|
51
|
+
name = "quote"
|
|
52
|
+
version = "1.0.47"
|
|
53
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
54
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
55
|
+
dependencies = [
|
|
56
|
+
"proc-macro2",
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
[[package]]
|
|
60
|
+
name = "serde"
|
|
61
|
+
version = "1.0.229"
|
|
62
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
63
|
+
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
64
|
+
dependencies = [
|
|
65
|
+
"serde_core",
|
|
66
|
+
"serde_derive",
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
[[package]]
|
|
70
|
+
name = "serde_core"
|
|
71
|
+
version = "1.0.229"
|
|
72
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
73
|
+
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
74
|
+
dependencies = [
|
|
75
|
+
"serde_derive",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
[[package]]
|
|
79
|
+
name = "serde_derive"
|
|
80
|
+
version = "1.0.229"
|
|
81
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
82
|
+
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
83
|
+
dependencies = [
|
|
84
|
+
"proc-macro2",
|
|
85
|
+
"quote",
|
|
86
|
+
"syn",
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
[[package]]
|
|
90
|
+
name = "serde_json"
|
|
91
|
+
version = "1.0.151"
|
|
92
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
93
|
+
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
|
94
|
+
dependencies = [
|
|
95
|
+
"itoa",
|
|
96
|
+
"memchr",
|
|
97
|
+
"serde",
|
|
98
|
+
"serde_core",
|
|
99
|
+
"zmij",
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
[[package]]
|
|
103
|
+
name = "syn"
|
|
104
|
+
version = "3.0.6"
|
|
105
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
106
|
+
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
|
107
|
+
dependencies = [
|
|
108
|
+
"proc-macro2",
|
|
109
|
+
"quote",
|
|
110
|
+
"unicode-ident",
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
[[package]]
|
|
114
|
+
name = "unicode-ident"
|
|
115
|
+
version = "1.0.26"
|
|
116
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
117
|
+
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
|
118
|
+
|
|
119
|
+
[[package]]
|
|
120
|
+
name = "zmij"
|
|
121
|
+
version = "1.0.23"
|
|
122
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
123
|
+
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "harness-native"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
publish = false
|
|
6
|
+
description = "The harness's native side: the cell client and the injector"
|
|
7
|
+
|
|
8
|
+
[dependencies]
|
|
9
|
+
# the clock, the quantities and the errno names, without the collector
|
|
10
|
+
prober = { path = "../../../prober/native" }
|
|
11
|
+
libc = "0.2"
|
|
12
|
+
serde_json = "1"
|
|
13
|
+
|
|
14
|
+
[profile.release]
|
|
15
|
+
lto = true
|
|
16
|
+
codegen-units = 1
|