@deliciousmonster/datadog-agent-binary 7.82.1-next.0

package/runtime/delivery.js

@@ -0,0 +1,137 @@
+// Whether anything this node collected reached Datadog. Every counter here is a one-minute window the agent
+// resets, so nothing is cumulative and nothing diffs.
+
+import { parseJson, pollEndpoint } from "./probe.js";
+
+// The trace-agent's own expvar, and https because it serves it under the self-signed IPC certificate.
+// Stated once so the URL that is read and the `source` that is reported can never be different ports or schemes.
+export const debugVarsUrl = (port) => `https://127.0.0.1:${port}/debug/vars`;
+
+// Carried in the payload and bumped when the shape or the verdict set changes, so a consumer written against
+// an older one can tell rather than guess.
+const DELIVERY_SIGNAL_VERSION = 1;
+
+const unavailable = (source, detail) => ({
+	source,
+	signalVersion: DELIVERY_SIGNAL_VERSION,
+	verdict: "unavailable",
+	detail,
+});
+
+/**
+ * Three hops read apart, because proving one proves nothing about the next: spans into the local receiver,
+ * APM stats into Datadog, trace payloads into Datadog.
+ */
+export function deliveryVerdict(vars, source) {
+	const clients = Array.isArray(vars?.receiver) ? vars.receiver : [];
+	const sum = (field) =>
+		clients.reduce((total, entry) => total + (Number(entry?.[field]) || 0), 0);
+	const count = (writer, field) => Number(vars?.[writer]?.[field]) || 0;
+
+	const receiver = {
+		tracesReceived: sum("TracesReceived"),
+		spansReceived: sum("SpansReceived"),
+		clients: clients.map(
+			(entry) => `${entry?.Lang ?? "?"} ${entry?.TracerVersion ?? "?"}`
+		),
+	};
+	const statsWriter = {
+		payloads: count("stats_writer", "Payloads"),
+		errors: count("stats_writer", "Errors"),
+		retries: count("stats_writer", "Retries"),
+		buckets: count("stats_writer", "StatsBuckets"),
+		clientPayloads: count("stats_writer", "ClientPayloads"),
+	};
+	const traceWriter = {
+		payloads: count("trace_writer", "Payloads"),
+		bytes: count("trace_writer", "Bytes"),
+		errors: count("trace_writer", "Errors"),
+		retries: count("trace_writer", "Retries"),
+	};
+
+	const arriving = receiver.tracesReceived > 0 || receiver.spansReceived > 0;
+	// The receiver snapshot is refreshed only when a payload arrives, so on a quiet node it is the last busy
+	// minute. The concentrator builds buckets from received spans before anything is sent, so these date it.
+	const thisMinute = statsWriter.buckets > 0 || statsWriter.clientPayloads > 0;
+	const statsRefused = statsWriter.errors > 0 || statsWriter.retries > 0;
+	// Read asymmetrically: two writers register into the one trace_writer expvar slot and the last one wins,
+	// so a non-zero came from whichever did the work and is evidence, while a zero carries nothing at all.
+	const tracesRefused = traceWriter.errors > 0 || traceWriter.retries > 0;
+
+	const signal = {
+		source,
+		signalVersion: DELIVERY_SIGNAL_VERSION,
+		agentVersion: vars?.version?.Version,
+		receiver,
+		statsWriter,
+		traceWriter,
+		// true is evidence the hop works, false evidence it does not, null no evidence either way.
+		proven: {
+			tracesAtReceiver: arriving ? true : null,
+			statsAtDatadog:
+				statsWriter.payloads > 0 ? true : statsRefused ? false : null,
+			tracesAtDatadog:
+				traceWriter.payloads > 0 ? true : tracesRefused ? false : null,
+		},
+	};
+
+	if (traceWriter.payloads > 0) {
+		signal.verdict = "delivering";
+		signal.detail = `the intake took ${traceWriter.payloads} trace payload(s), ${traceWriter.bytes} bytes, in the last minute.`;
+	} else if (tracesRefused) {
+		signal.verdict = "rejected";
+		signal.detail = `every trace payload sent in the last minute came back refused (${traceWriter.retries} retries, ${traceWriter.errors} errors) and none were accepted. Check DD_API_KEY and DD_SITE.`;
+	} else if (statsWriter.payloads > 0) {
+		signal.verdict = "traces-unconfirmed";
+		signal.detail =
+			`the intake accepted ${statsWriter.payloads} APM stats payload(s) in the last minute, so the key, ` +
+			`the site and the route out are good. Trace payloads are a separate hop on that route and nothing ` +
+			`here proves one landed: trace_writer is zero, and on this agent build a zero is the shared expvar ` +
+			`slot rather than a measurement. Confirm in the Datadog trace explorer.`;
+	} else if (statsRefused) {
+		signal.verdict = "rejected";
+		signal.detail = `the intake refused every APM stats payload in the last minute (${statsWriter.retries} retries, ${statsWriter.errors} errors) and accepted none. Check DD_API_KEY and DD_SITE.`;
+	} else if (arriving && thisMinute) {
+		signal.verdict = "not-delivering";
+		signal.detail = `${receiver.spansReceived} spans reached the receiver this minute and neither the stats hop nor the traces hop has had anything accepted. Both windows reset every minute; read this again before believing it.`;
+	} else if (arriving) {
+		signal.verdict = "idle";
+		signal.detail = `the receiver still reports ${receiver.spansReceived} spans while the stats writer saw no work at all, so that snapshot is left over from an earlier minute and nothing arrived in this one.`;
+	} else {
+		signal.verdict = "idle";
+		signal.detail =
+			"no spans reached the trace-agent in the last minute. Send the application some traffic and read this again; nothing here separates a quiet node from a dead tracer.";
+	}
+	return signal;
+}
+
+/** The trace-agent's own delivery counters. Never rejects: an endpoint that does not answer is itself a verdict. */
+export async function readDeliverySignal(port) {
+	const source = debugVarsUrl(port);
+	if (port === 0) {
+		return unavailable(
+			source,
+			"apm_config.debug.port is 0 (DD_APM_DEBUG_PORT), so the trace-agent publishes no expvar to read."
+		);
+	}
+	// One attempt, because a status endpoint answers now or reports that nothing did; waiting out a bind is
+	// what the start-up verifies already do.
+	const body = await pollEndpoint({
+		url: source,
+		timeoutMs: 1000,
+		giveUp: () => true,
+	});
+	if (body === null) {
+		return unavailable(
+			source,
+			`nothing answered ${source}, so no trace-agent is running on this node, or the one that is does not serve apm_config.debug.port.`
+		);
+	}
+	const vars = parseJson(body);
+	return vars === null
+		? unavailable(
+				source,
+				`${source} answered with something that is not expvar JSON.`
+			)
+		: deliveryVerdict(vars, source);
+}

package/runtime/probe.js

@@ -0,0 +1,115 @@
+// Every request this module makes is invisible to APM: the plugin polls the agents before they bind, when
+// polls fail, and on the line this replaces those failures became errored client spans on the customer's own service.
+
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { createRequire } from "node:module";
+
+const PROBE_TIMEOUT_MS = 1000;
+const MAX_INTERVAL_MS = 5_000;
+
+// The store dd-trace keeps its OWN agent traffic out of the customer's APM with, applied to these probes for
+// the same reason. Private path, so a miss falls back to untraceAgentProbes, which is the public half of this.
+const untraced = (() => {
+	try {
+		const core = createRequire(import.meta.url)(
+			"dd-trace/packages/datadog-core"
+		);
+		const legacy = core.storage("legacy");
+		return (run) => legacy.run({ noop: true }, run);
+	} catch {
+		return (run) => run();
+	}
+})();
+
+// A process-global setting, so the next `tracer.use('http', ...)` from anywhere replaces it: dd-trace's
+// configurePlugin overwrites a plugin's config rather than merging into it. Kept only as a fallback for releases where the private store above has moved; it cannot stand alone.
+export function untraceAgentProbes(tracer, blocklist) {
+	tracer.use("http", { client: { blocklist } }); // under `client`, or the server half drops inbound traces too
+}
+
+/** A probe body as JSON, or null. Never throws: these bodies come off a socket and pollEndpoint's own contract is the same. */
+export function parseJson(body) {
+	try {
+		return body === null ? null : JSON.parse(body);
+	} catch {
+		return null;
+	}
+}
+
+/** GET over http or https, accepting the trace-agent's self-signed IPC certificate. Loopback only: never point this off 127.0.0.1. Global fetch cannot stand in for it, because Node exposes no public dispatcher for that certificate. */
+function get(url, timeoutMs) {
+	return new Promise((resolve) => {
+		let deadline;
+		const settle = (body) => {
+			clearTimeout(deadline);
+			resolve(body);
+		};
+		// node:http ignores rejectUnauthorized, so the scheme is the only difference between the two probes.
+		const send = url.startsWith("https:") ? httpsRequest : httpRequest;
+		const call = send(url, { rejectUnauthorized: false }, (response) => {
+			const status = response.statusCode;
+			if (status === undefined || status < 200 || status >= 300) {
+				response.resume();
+				settle(null);
+				return;
+			}
+			let body = "";
+			response.setEncoding("utf-8");
+			response.on("data", (chunk) => (body += chunk));
+			response.on("end", () => settle(body));
+			// The reset a destroy() lands on an open response arrives here, not on the request, and an
+			// unheard one leaves this promise pending for the life of the process.
+			response.on("error", () => settle(null));
+		});
+		call.on("error", () => settle(null));
+		// One deadline over the whole exchange rather than the socket's own inactivity timeout: a response
+		// that starts and then stalls, or drips a byte at a time, never trips that one.
+		deadline = setTimeout(() => {
+			call.destroy();
+			settle(null);
+		}, timeoutMs);
+		call.end();
+	});
+}
+
+// Run under `untraced`: the span is created where the request is made, so that is the only place suppression
+// cannot be undone by other code in the process.
+async function probe(url, timeoutMs) {
+	try {
+		// Awaited inside the try rather than returned: `untraced` reaches into a dd-trace private path, and
+		// pollEndpoint's never-throws contract has to hold if that path moves.
+		return await untraced(() => get(url, timeoutMs));
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * GET until something answers: the body text, or null on deadline or giveUp(). Never throws. `intervalMs`
+ * doubles every retry up to MAX_INTERVAL_MS: a flat 250ms wait costs ~120 probes over a ~6-7s expvar bind.
+ */
+export async function pollEndpoint({
+	url,
+	timeoutMs = 30_000,
+	intervalMs = 250,
+	giveUp,
+}) {
+	const deadline = Date.now() + timeoutMs;
+	let interval = intervalMs;
+	for (;;) {
+		const budget = Math.min(
+			PROBE_TIMEOUT_MS,
+			Math.max(deadline - Date.now(), 1)
+		);
+		const body = await probe(url, budget);
+		if (body !== null) return body;
+		// Asked between probes, and only after one has failed, so a target that answered then died still counts.
+		if (giveUp?.() || Date.now() >= deadline) return null;
+		// Clamped to what is left: backing off must not spend the caller's budget asleep past the deadline.
+		await new Promise((resolve) =>
+			setTimeout(resolve, Math.min(interval, deadline - Date.now()))
+		);
+		interval = Math.min(interval * 2, MAX_INTERVAL_MS);
+	}
+}

package/runtime/supervisor.js

@@ -0,0 +1,172 @@
+// Who holds the agents up. Released Harper has no sidecar API, so the bundled guard is the other half; one
+// agent per node comes from a PID lock either way and only the holder changes.
+
+import { join } from "node:path";
+
+// Pinned to one exact version, never a range: Harper runs `npm install` when it installs a component, so a
+// caret here would let a customer's node resolve a guard no test in this repo has run against.
+import { fingerprint, guard } from "@deliciousmonster/harper-process-guard";
+import { describeSpawnFailure } from "./agent-exit.js";
+import { writeConfigFiles } from "./config.js";
+
+// No released Harper has `scope.processes` - harper@5.2.9 is latest and its Scope carries no such member - so
+// this answers false on every node a customer can run today, and the guard below is the only shipping path.
+const supervisesNatively = (scope) =>
+	typeof scope?.processes?.start === "function";
+
+// The state a supervisor never reached, in the shape both of them report. `started` is what answers "is it
+// running": `exited: false` here means it never ran, not that it still does.
+export const unstarted = (agent, error) => ({
+	name: agent.name,
+	title: agent.title,
+	kind: agent.kind,
+	started: false,
+	adopted: false,
+	exited: false,
+	restarts: 0,
+	error,
+});
+
+// What the NATIVE path publishes about the reaper, where the object comes from a Harper this package does not
+// ship and may carry anything. The guard's own reaper state is a documented shape and is published whole.
+const REAPER_FIELDS = ["name", "started", "adopted", "error"];
+const knownReaperFields = (reaper) =>
+	reaper &&
+	Object.fromEntries(
+		REAPER_FIELDS.filter((field) => reaper[field] !== undefined).map(
+			(field) => [field, reaper[field]]
+		)
+	);
+
+// Mutated, never copied: the guard writes this same object for the life of the node - a death, a restart,
+// a give-up - and a copy taken here freezes the status endpoint on what was true at boot.
+const identify = (state, agent) =>
+	Object.assign(state, {
+		name: agent.name,
+		title: agent.title,
+		kind: agent.kind,
+	});
+
+// Symmetric with the guard path's own notes: an operator reading the boot log sees this line, and a
+// caller reading status.supervisionReport sees the same words, not just the absence of `kind: "guard"`.
+const GUARD_UNUSED_NOTE =
+	"the bundled process guard is present but unused: this Harper supervises the agents natively, so the guard never runs.";
+
+// Harper's own sidecar, one call per process; it writes the config files behind its own sweep. Also the seam
+// test/support/component.js fakes, so most of the component's supervision tests run through this branch.
+const harperSupervisor = (scope, log) => ({
+	kind: "harper",
+	async start(agents, { configFiles, fingerprintParts }) {
+		log.warn(`Datadog supervisor: ${GUARD_UNUSED_NOTE}`);
+		const processes = await Promise.all(
+			agents.map((agent) =>
+				scope.processes
+					.start({
+						name: agent.name,
+						title: agent.title,
+						command: agent.command,
+						args: agent.args,
+						// On BOTH: start() writes after its own sweep, so naming them on one alone lets the
+						// other spawn before the files exist.
+						configFiles,
+						fingerprint: fingerprintParts,
+						exitHint: agent.exitHint,
+						verify: agent.verify,
+					})
+					.then((state) => {
+						// Only here: the guard reports its own verdicts through the log it was handed.
+						if (state.verified !== true) {
+							log.error(
+								`Datadog supervisor: the ${agent.title} started but did not verify: ${state.verifyDetail ?? "no detail"}`
+							);
+						}
+						return identify(state, agent);
+					})
+					.catch((error) =>
+						unstarted(agent, describeSpawnFailure(error, agent.command))
+					)
+			)
+		);
+		return {
+			processes,
+			reaper: knownReaperFields(scope.processes.reaper),
+			report: [GUARD_UNUSED_NOTE],
+		};
+	},
+});
+
+// The reaper takes its own lock beside the agents', so its name is what a second component sharing the
+// directory would collide on; this one names the package rather than taking the guard's generic default.
+const REAPER_NAME = "datadog-agent-reaper";
+
+/** The bundled guard, one call for both agents. `spawn` is the entry module's own, which is the one Harper constrains. */
+const guardSupervisor = (log, spawn) => ({
+	kind: "guard",
+	async start(agents, { runtime, configFiles, fingerprintParts }) {
+		// Harper's start() writes these itself; on this path nothing else will, and both agents read them.
+		writeConfigFiles(configFiles, log);
+		let result;
+		try {
+			result = await guard({
+				pidDir: runtime.paths.pidDir,
+				spawn,
+				log,
+				version: fingerprint(...fingerprintParts),
+				// What makes the fingerprint a replacement rather than a second lock holder: without it a rotated
+				// key leaves the old agent running under no lock, so not even the reaper below can stop it again.
+				stopOrphans: true,
+				processes: agents.map((agent) => ({
+					name: agent.name,
+					title: agent.title,
+					binaryPath: agent.command,
+					args: agent.args,
+					exitHint: agent.exitHint,
+					verify: agent.verify,
+				})),
+				reaper: {
+					name: REAPER_NAME,
+					logFile: runtime.paths.reaperLog,
+					// Harper records its own pid here, so a restart inside the grace window keeps the agents
+					// running for the replacement node to adopt.
+					...(runtime.root
+						? { replacementPidFile: join(runtime.root, "hdb.pid") }
+						: {}),
+				},
+			});
+		} catch (error) {
+			// Anything guard() does not catch itself reaches here, and no enumeration of those stays true: the
+			// last one written missed ctx.log.info. It starts the agents in order and rejects out of the one
+			// it was on, so an agent ahead of it is running under a committed lock with nothing watching it.
+			const message =
+				`${error instanceof Error ? error.message : String(error)}. Neither agent is reported ` +
+				`started because the call threw before it reported either; an agent it had already spawned ` +
+				`is still running unsupervised, under a lock in ${runtime.paths.pidDir}`;
+			log.error(
+				`Datadog supervisor: the guard call for both agents threw: ${error.stack ?? message}`
+			);
+			return {
+				// Not describeSpawnFailure: its ENOEXEC/EACCES/ENOENT translations are harperSupervisor's
+				// per-agent contract, where the error IS that one agent's own spawn rejection. Here the cause
+				// is unproven to be about either binary, so both agents get the same raw message.
+				processes: agents.map((agent) => unstarted(agent, message)),
+				report: [message],
+			};
+		}
+		return {
+			processes: result.processes.map((state, index) =>
+				identify(state, agents[index])
+			),
+			// Whole: the guard's index.js built this and its ReaperState typedef is the shape. Filtering it here
+			// dropped the reaper's own pid, which is the one field an operator needs to find the process.
+			reaper: result.reaper,
+			report: result.report,
+		};
+	},
+});
+
+// The one place either supervisor is chosen. Everything downstream takes what this returns and never reads
+// `scope.processes` again, so a second reading cannot disagree with the first.
+export const supervisorFor = (scope, { log, spawn }) =>
+	supervisesNatively(scope)
+		? harperSupervisor(scope, log)
+		: guardSupervisor(log, spawn);

package/runtime/verify.js

@@ -0,0 +1,190 @@
+// What separates "the process is up" from "the process is the agent this node needs". A live process of the
+// wrong kind, or a stray socket on the port, passes every cheaper check and is reported healthy.
+
+import { describeExit } from "./agent-exit.js";
+import { debugVarsUrl } from "./delivery.js";
+import { parseJson, pollEndpoint } from "./probe.js";
+
+/** The path dd-trace posts spans to. A receiver that does not advertise it is not one this node can use. */
+const TRACE_ENDPOINT = "/v0.4/traces";
+
+// Stated once so resources.js's probe blocklist can never name a different receiver URL than the one this
+// file verifies against; a mismatch there is exactly the traffic the blocklist exists to keep out of APM.
+export const receiverInfoUrl = (port) => `http://127.0.0.1:${port}/info`;
+
+/** The core agent's expvar endpoint, built the same way for the same reason. */
+export const expvarUrl = (port) => `http://127.0.0.1:${port}/debug/vars`;
+
+// Both verifiers poll the same way and differ only in url; state.exited is the one giveUp condition either
+// agent has, since a dead process cannot bind the port it is being polled for.
+const pollAgent = (url, state) =>
+	pollEndpoint({ url, giveUp: () => state.exited === true });
+
+/** The pid this node's supervisor started, or null. A guard attempt that never reached a spawn leaves it undefined (the guard's src/supervise.js:156), and comparing against that reads a healthy agent as stale. */
+const heldPid = (state) => (typeof state?.pid === "number" ? state.pid : null);
+
+/** The pid an expvar body reports, or null when it publishes none. The trace-agent publishes it as a string, so a strict number test reads a real pid as no pid at all. */
+function expvarPid(vars) {
+	const pid = Number(vars?.pid);
+	return Number.isInteger(pid) && pid > 0 ? pid : null;
+}
+
+/** Both agents get the same verdict for the same reason, so the operator reads one sentence either way. */
+const foreignPid = (answering, held, url, title) =>
+	`a ${title} answered ${url} as pid ${answering}, not the pid ${held} this node started, so the port ` +
+	`belongs to a process nothing here supervises; stop it, or remove the stale .pid file under the node's ` +
+	`pids/ directory, and restart`;
+
+/** What the process did, when it did anything. A signalled exit reports no code, so `code || 0` reads it as a clean stop. */
+function exitDetail(state) {
+	if (state?.exited !== true) return "";
+	if (typeof state.signal !== "string" && typeof state.code !== "number") {
+		return " The process this node started is gone.";
+	}
+	const { detail } = describeExit(state.code ?? null, state.signal ?? null);
+	return ` The process this node started ${detail}.`;
+}
+
+/** Prove the trace-agent serves the endpoint dd-trace posts to and is the process this node started; a bare TCP connect is satisfied by any stray socket, and dd-trace reports a successful flush either way. */
+async function verifyTraceAgent(state, { paths, ports }) {
+	if (ports.receiver === 0) {
+		return {
+			ok: false,
+			detail:
+				"apm_config.receiver_port is 0 (DD_APM_RECEIVER_PORT): the HTTP receiver is off, and " +
+				"dd-trace drops every span unless it is pointed at a Unix socket instead",
+		};
+	}
+	if (ports.debug === 0) {
+		return {
+			ok: false,
+			detail:
+				"apm_config.debug.port is 0 (DD_APM_DEBUG_PORT), so the trace-agent publishes no expvar and " +
+				"nothing can tie whatever holds the receiver port to the process this node started",
+		};
+	}
+	const url = receiverInfoUrl(ports.receiver);
+	const body = await pollAgent(url, state);
+	const endpoints = parseJson(body)?.endpoints;
+	const serving =
+		Array.isArray(endpoints) &&
+		endpoints.some(
+			(entry) => typeof entry === "string" && entry.includes(TRACE_ENDPOINT)
+		);
+	if (!serving) {
+		return {
+			ok: false,
+			detail:
+				body === null
+					? `nothing answered ${url}, so dd-trace has nowhere to send spans.${exitDetail(state)} ` +
+						`Check apm_config.enabled in ${paths.configFile} and DD_APM_ENABLED, then read ${paths.traceLog}`
+					: `whatever answered ${url} does not advertise ${TRACE_ENDPOINT}, so it is not a trace-agent this node can rely on`,
+		};
+	}
+	// /info identifies nobody: an agent left from an earlier boot answers it exactly like this node's own,
+	// and it is the one holding the port this node's agent could not bind. The expvar names the pid.
+	const identity = debugVarsUrl(ports.debug);
+	const vars = parseJson(await pollAgent(identity, state));
+	const answering = expvarPid(vars);
+	if (answering === null) {
+		return {
+			ok: false,
+			detail:
+				`something serves ${TRACE_ENDPOINT} on 127.0.0.1:${ports.receiver}, but nothing answering ` +
+				`${identity} named a pid, so it cannot be shown to be the trace-agent this node ` +
+				`started.${exitDetail(state)} Read ${paths.traceLog}`,
+		};
+	}
+	const held = heldPid(state);
+	if (held !== null && answering !== held) {
+		return {
+			ok: false,
+			detail: foreignPid(answering, held, identity, "trace-agent"),
+		};
+	}
+	return {
+		ok: true,
+		detail: `the APM receiver serves ${TRACE_ENDPOINT} on 127.0.0.1:${ports.receiver} as pid ${answering}; dd-trace has somewhere to send spans`,
+	};
+}
+
+/** Prove the process behind the lock is a core agent: only it publishes aggregator and forwarder, and a live process of the wrong kind passes every cheaper check. */
+async function verifyCoreAgent(state, { paths, ports }) {
+	if (ports.expvar === 0) {
+		return {
+			ok: false,
+			detail:
+				"expvar_port is 0 (DD_EXPVAR_PORT), so nothing can confirm the core agent is the process " +
+				"holding its PID lock, and no host metric can be shown to be collected",
+		};
+	}
+	const url = expvarUrl(ports.expvar);
+	const vars = parseJson(await pollAgent(url, state));
+	if (!vars || !("aggregator" in vars) || !("forwarder" in vars)) {
+		return {
+			ok: false,
+			detail:
+				`nothing answering ${url} identified itself as a core agent, so host metrics and tags are ` +
+				`going nowhere while traces may still flow.${exitDetail(state)} A stale PID lock adopted by ` +
+				`the wrong process produces exactly this; read ${paths.coreLog} and check ${paths.configFile}`,
+		};
+	}
+	// Measured on 7.82.1: the core agent's expvar publishes no pid at all, so this engages only against a
+	// build that grows one. What it cannot do is refuse a healthy agent for not publishing it.
+	const held = heldPid(state);
+	const answering = expvarPid(vars);
+	if (answering !== null && held !== null && answering !== held) {
+		return {
+			ok: false,
+			detail: foreignPid(answering, held, url, "core agent"),
+		};
+	}
+	return {
+		ok: true,
+		detail:
+			`the core agent serves expvar on 127.0.0.1:${ports.expvar}` +
+			(held === null ? "" : ` as pid ${held}`),
+	};
+}
+
+// A supervisor that never started it has nothing to verify: both verifiers would poll on, and read whatever
+// else holds the port. Strictly false, because a caller that reports no `started` field does have a process.
+const notStarted = (state) =>
+	state?.started === false
+		? {
+				ok: false,
+				detail:
+					`this node never started it${state.error ? `: ${state.error}` : ""}, so nothing was ` +
+					`polled and anything answering its port belongs to another process`,
+			}
+		: null;
+
+/** The verdict for one launched agent. Both supervisors call this, so neither can reach a verdict the other cannot. */
+export const verifyLaunch = (agent, state, context) => {
+	// Each supervisor verifies once, after the first spawn, and then rewrites `pid` and `restarts` on this
+	// same object without retaking the verdict. The pid it was taken against is the only record of that.
+	state.verifiedPid = state.pid ?? null;
+	return (
+		notStarted(state) ??
+		(agent.kind === "trace"
+			? verifyTraceAgent(state, context)
+			: verifyCoreAgent(state, context))
+	);
+};
+
+/** True once the process the verdict describes has been replaced. A verdict taken against no pid at all cannot go stale, because it never named one. */
+const stale = (state) =>
+	typeof state?.verifiedPid === "number" && state.verifiedPid !== state.pid;
+
+/** The verdict as it stands now. Read at the endpoint rather than stamped at boot, because the supervisor keeps writing pid and restarts to the same object for the life of the node. */
+export const currentVerdict = (state) =>
+	stale(state)
+		? {
+				...state,
+				verified: null,
+				verifyDetail:
+					`the last verdict was taken against pid ${state.verifiedPid}, which this node has since ` +
+					`restarted ${state.restarts} time(s) as pid ${state.pid}. Nothing has verified the process ` +
+					`now running; what the dead one proved was: ${state.verifyDetail}`,
+			}
+		: state;