@mark1russell7/service 0.5.13 → 0.5.15

package/build/index.js

@@ -136,6 +136,114 @@ const latch = (timing, clock, tripped, changed, trace) => {
 	};
 };
 //#endregion
+//#region src/metrics.ts
+/** These fractions of the timeout of a poll are the bounds of its histogram. */
+const DURATION_FRACTIONS = [
+	.05,
+	.1,
+	.25,
+	.5,
+	.75,
+	1
+];
+/** The runtime keeps this many episodes of each check. */
+const EPISODES_KEPT = 8;
+/**
+* This function removes the secrets that a cause can hold. These are the user and the password of a
+* URL, and the value of a key, for example `password=`. A driver error can hold a connection string.
+*/
+const redact = (cause) => cause.replace(/(\b[a-z][a-z0-9+.-]*:\/\/)[^\s/@]*@/gi, "$1***@").replace(/\b(password|passwd|pwd|secret|token|api[_-]?key|auth)(\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s,;&]+)/gi, "$1$2***");
+/** This function makes a meter. */
+const meter = () => {
+	const checks = /* @__PURE__ */ new Map();
+	const transitions = /* @__PURE__ */ new Map();
+	const add = (check) => {
+		if (checks.has(check.name)) return;
+		const timeout = valueIn(check.poll.timeout, "ms");
+		checks.set(check.name, {
+			ok: 0,
+			error: 0,
+			indeterminate: 0,
+			thrown: 0,
+			forced: 0,
+			le: DURATION_FRACTIONS.map((fraction) => fraction * timeout),
+			counts: DURATION_FRACTIONS.map(() => 0),
+			timedOut: 0,
+			episodes: [],
+			open: null
+		});
+	};
+	const close = (counted, end) => {
+		if (counted.open === null) return null;
+		return {
+			start: counted.open.start,
+			end,
+			failures: counted.open.failures,
+			cause: counted.open.cause
+		};
+	};
+	const poll = (check, result) => {
+		add(check);
+		const counted = checks.get(check.name);
+		counted[result.outcome] += 1;
+		if (result.thrown) counted.thrown += 1;
+		if (result.forced) counted.forced += 1;
+		if (result.timedOut) counted.timedOut += 1;
+		else {
+			const at = counted.le.findIndex((bound) => result.took <= bound);
+			const bucket = at === -1 ? counted.counts.length - 1 : at;
+			counted.counts[bucket] = (counted.counts[bucket] ?? 0) + 1;
+		}
+		if (result.outcome === "error") {
+			if (counted.open === null) counted.open = {
+				start: result.at,
+				failures: 0,
+				cause: result.cause === null ? null : redact(result.cause)
+			};
+			counted.open.failures += 1;
+		} else if (result.outcome === "ok" && counted.open !== null) {
+			counted.episodes.push(close(counted, result.at));
+			counted.open = null;
+			if (counted.episodes.length > 8) counted.episodes.shift();
+		}
+	};
+	const event = (event) => {
+		const key = `${event.machine} ${event.instance} ${event.from} -> ${event.to}`;
+		transitions.set(key, (transitions.get(key) ?? 0) + 1);
+	};
+	const metrics = () => {
+		const out = {};
+		for (const [name, counted] of checks) {
+			const open = close(counted, null);
+			out[name] = {
+				outcomes: {
+					ok: counted.ok,
+					error: counted.error,
+					indeterminate: counted.indeterminate
+				},
+				thrown: counted.thrown,
+				forced: counted.forced,
+				durations: {
+					le: [...counted.le],
+					counts: [...counted.counts],
+					timedOut: counted.timedOut
+				},
+				episodes: open === null ? [...counted.episodes] : [...counted.episodes, open].slice(-8)
+			};
+		}
+		return {
+			checks: out,
+			transitions: Object.fromEntries(transitions)
+		};
+	};
+	return {
+		add,
+		poll,
+		event,
+		metrics
+	};
+};
+//#endregion
 //#region src/poller.ts
 const message$1 = (error) => error instanceof Error ? error.message : String(error);
 /**
@@ -304,7 +412,12 @@ const healthRuntime = (description, runs, options = {}) => {
 	let draining = false;
 	let stopped = false;
 	let gate = "closed";
-	const tracer = (machine, instance) => (from, to, on) => options.record?.({
+	const counts = meter();
+	const record = (event) => {
+		counts.event(event);
+		options.record?.(event);
+	};
+	const tracer = (machine, instance) => (from, to, on) => record({
 		at: clock.now(),
 		machine,
 		instance,
@@ -316,7 +429,7 @@ const healthRuntime = (description, runs, options = {}) => {
 	const moveGate = (to, on, detail) => {
 		const from = gate;
 		gate = to;
-		options.record?.({
+		record({
 			at: clock.now(),
 			machine: "gate",
 			instance: description.name,
@@ -350,6 +463,7 @@ const healthRuntime = (description, runs, options = {}) => {
 				random,
 				forced: () => forced.has(check.name) || check.dependency !== null && forced.has(check.dependency),
 				record: (result) => {
+					counts.poll(check, result);
 					made?.record(result);
 					if (result.outcome === "ok" && check.dependency !== null) {
 						if (lateDependencies.delete(check.dependency)) changed();
@@ -359,6 +473,7 @@ const healthRuntime = (description, runs, options = {}) => {
 			})
 		};
 		entries.set(check.name, entry);
+		counts.add(check);
 		if (late && !stopped) {
 			entry.poller.pollNow();
 			entry.poller.start(entry.poller.jittered());
@@ -425,6 +540,7 @@ const healthRuntime = (description, runs, options = {}) => {
 	});
 	return {
 		verdict,
+		metrics: counts.metrics,
 		probe(kind, controller) {
 			const now = verdict();
 			if (kind === "startup") return answer(kind, now.started);
@@ -2202,8 +2318,74 @@ const verdictOf = (rule, bind, evaluated) => {
 	const all = inequalities(rule);
 	return applicable.some((index) => readsAbsent(all[index], bind)) ? "unbounded" : "holds";
 };
+/** This function gives the mark of a symbol: its mark at its hop, else the mark of the bare symbol. */
+const markOf = (marks, key) => marks[key] ?? marks[key.split("@")[0] ?? key];
+const STEP_EPSILON = 1e-9;
+const scaled = (seconds, k, mark) => {
+	switch (mark.kind) {
+		case "scalable": return seconds * k;
+		case "fixed": return seconds;
+		case "stepped": return Math.max(Math.ceil(seconds * k / mark.step - STEP_EPSILON), 0) * mark.step;
+		case "floored": return Math.max(seconds * k, mark.floor);
+	}
+};
+/**
+* This function scales each timer of a binding by k, by its mark (E10). An absent timer ("inf" or
+* "off") stays absent. A chain scales each value. The margins of the binding and the constants of
+* the rules do not scale: they are tolerances, not timers. A symbol with no mark keeps its value.
+*/
+const compress = (bind, k, marks) => {
+	if (!Number.isFinite(k) || k <= 0) throw new RangeError(`a compression scales time by a finite factor above 0, not ${k}`);
+	const out = {};
+	const unmarked = [];
+	for (const [key, bound] of Object.entries(bind)) {
+		if (key === "margin" || key === "margins") continue;
+		const mark = markOf(marks, key);
+		if (mark === void 0) {
+			if (bound !== void 0) unmarked.push(key);
+			out[key] = bound;
+		} else if (typeof bound === "number") out[key] = scaled(bound, k, mark);
+		else if (Array.isArray(bound)) out[key] = bound.map((seconds) => scaled(seconds, k, mark));
+		else out[key] = bound;
+	}
+	const margins = {
+		...bind.margin !== void 0 ? { margin: bind.margin } : {},
+		...bind.margins !== void 0 ? { margins: bind.margins } : {}
+	};
+	return {
+		bind: {
+			...out,
+			...margins
+		},
+		unmarked
+	};
+};
+/**
+* This function tells which rules a compression by k changes. A compression keeps each ordering
+* when no verdict changes. A change in either direction counts: a violation that the compression
+* hides is as wrong as one that it makes. The unmarked symbols are the symbols that the rules read.
+*/
+const compression = (rules, bind, k, marks) => {
+	const compressed = compress(bind, k, marks);
+	const changed = rules.flatMap((rule) => {
+		const before = verdictOf(rule, bind, evaluate(rule, bind));
+		const after = verdictOf(rule, compressed.bind, evaluate(rule, compressed.bind));
+		return before === after ? [] : [{
+			rule: rule.id,
+			before,
+			after
+		}];
+	});
+	const read = new Set(rules.flatMap(symbolsOf));
+	return {
+		changed,
+		unmarked: compressed.unmarked.filter((symbol) => read.has(symbol))
+	};
+};
 //#endregion
 //#region src/lint.ts
+/** The law binds this statistic of a distribution. */
+const LAW_STATISTIC = "p99";
 /** These are the identifiers of the rules that each declaration gets. */
 const SERVICE_RULES = [
 	"O04",
@@ -2294,6 +2476,35 @@ const LIMITER_RULES = [{
 const secondsOf = (amount) => rationals.toNumber(convertExact(rationals.fromNumber(amount.value), amount.unit, "s"));
 /** This function gives an amount, or no bound, as the law binds it. */
 const boundOf = (amount) => amount.value === "infinite" ? "inf" : secondsOf(amount);
+/**
+* This function gives a value of the scenario as the law binds it, with its epoch. A distribution
+* binds its p99. A distribution without a p99 binds nothing, thus each rule that reads it depends.
+*/
+const claimOf = (value) => {
+	if (!("epoch" in value)) return {
+		bound: boundOf(value),
+		epoch: null
+	};
+	if ("value" in value) return {
+		bound: boundOf(value.value),
+		epoch: value.epoch
+	};
+	const statistic = value.distribution["p99"];
+	return {
+		bound: statistic === void 0 ? void 0 : secondsOf(statistic),
+		epoch: value.epoch
+	};
+};
+/** This function binds each value of a part of the scenario, at the symbol that `at` gives, and records its epoch. */
+const bindScenario = (values, at, bind, epochs) => {
+	for (const [symbol, value] of Object.entries(values)) {
+		const claim = claimOf(value);
+		const key = at(symbol);
+		epochs[key] = claim.epoch;
+		if (claim.bound !== void 0) bind[key] = claim.bound;
+		else delete bind[key];
+	}
+};
 /** This function tells if a unit is a time. A unit that the package does not know is not a time. */
 const isTime = (unit) => {
 	try {
@@ -2308,6 +2519,7 @@ const most = (values) => values.length === 0 ? void 0 : Math.max(...values);
 const serviceBindings = (svc, scenario) => {
 	const bind = {};
 	const from = {};
+	const epochs = {};
 	const set = (symbol, bound, said) => {
 		if (bound === void 0) return;
 		bind[symbol] = bound;
@@ -2330,18 +2542,25 @@ const serviceBindings = (svc, scenario) => {
 	set("cooperative_window@process", boundOf(svc.shutdown.cooperativeWindow), "the declared cooperative window");
 	const hold = svc.shutdown.phases.find((phase) => phase.name === "fail readiness");
 	set("readiness_hold@process", hold === void 0 ? 0 : secondsOf(hold.cap), hold === void 0 ? "no fail readiness phase: the intake stops at once after the signal" : "the cap of the fail readiness phase, which holds while the endpoints update");
-	for (const [symbol, amount] of Object.entries(scenario.platform)) set(symbol, boundOf(amount), "the scenario");
+	bindScenario(scenario.platform, (symbol) => symbol, bind, epochs);
+	for (const [symbol, epoch] of Object.entries(epochs)) if (bind[symbol] === void 0) delete from[symbol];
+	else from[symbol] = epoch === null ? "the scenario" : `the scenario, for ${epoch}`;
 	return {
 		bind,
-		from
+		from,
+		epochs
 	};
 };
 /** This function gives the client knobs of a dependency at @client, and the values of the scenario at its own hops. */
 const dependencyBindings = (dependency, scenario) => {
 	const bind = {};
+	const epochs = {};
 	for (const [knob, amount] of Object.entries(dependency.knobs)) bind[`${knob}@client`] = boundOf(amount);
-	for (const [symbol, amount] of Object.entries(scenario.dependencies?.[dependency.name] ?? {})) bind[symbol.includes("@") ? symbol : `${symbol}@dependency`] = boundOf(amount);
-	return bind;
+	bindScenario(scenario.dependencies?.[dependency.name] ?? {}, (symbol) => symbol.includes("@") ? symbol : `${symbol}@dependency`, bind, epochs);
+	return {
+		bind,
+		epochs
+	};
 };
 /**
 * These are the hops of the law at which the knobs of a transport bind, by its kind. A worker does
@@ -2358,12 +2577,16 @@ const TRANSPORT_HOPS = {
 */
 const transportBindings = (transport, hop, scenario) => {
 	const bind = {};
+	const epochs = {};
 	for (const [knob, amount] of Object.entries(transport.knobs)) {
 		if (!isTime(amount.unit)) continue;
 		bind[knob.includes("@") ? knob : `${knob}@${hop}`] = boundOf(amount);
 	}
-	for (const [symbol, amount] of Object.entries(scenario.transports?.[transport.name] ?? {})) bind[symbol] = boundOf(amount);
-	return bind;
+	bindScenario(scenario.transports?.[transport.name] ?? {}, (symbol) => symbol, bind, epochs);
+	return {
+		bind,
+		epochs
+	};
 };
 /**
 * This function gives the symbols of a limiter, and the values of the scenario for its callers.
@@ -2377,8 +2600,12 @@ const limiterBindings = (limiter, scenario) => {
 		"retry_after@limiter": retryAfterSeconds(longestRetryAfter(limiter.policy)),
 		"retry_after_at_cap@limiter": retryAfterSeconds(idleSpan(limiter.policy))
 	};
-	for (const [symbol, amount] of Object.entries(scenario.limiters?.[limiter.name] ?? {})) bind[symbol.includes("@") ? symbol : `${symbol}@caller`] = boundOf(amount);
-	return bind;
+	const epochs = {};
+	bindScenario(scenario.limiters?.[limiter.name] ?? {}, (symbol) => symbol.includes("@") ? symbol : `${symbol}@caller`, bind, epochs);
+	return {
+		bind,
+		epochs
+	};
 };
 const SCOPED = /@(client|dependency|server)$/;
 const TRANSPORT_SCOPED = /@consumer$/;
@@ -2528,6 +2755,79 @@ const progressFindings = (svc) => {
 	}
 	return findings;
 };
+/** This function gives the scopes of a rule that reads these symbols: the service, each client, each consumer and worker, or each limiter. */
+const scopesOf = (svc, scenario, service, symbols) => {
+	const within = (name, scoped) => {
+		const bind = {
+			...service.bind,
+			...scoped.bind
+		};
+		for (const symbol of Object.keys(scoped.epochs)) if (!(symbol in scoped.bind)) delete bind[symbol];
+		return [
+			name,
+			bind,
+			{
+				...service.epochs,
+				...scoped.epochs
+			}
+		];
+	};
+	return symbols.some((symbol) => LIMITER_SCOPED.test(symbol)) ? (svc.limiters ?? []).map((limiter) => within(limiter.name, limiterBindings(limiter, scenario))) : symbols.some((symbol) => TRANSPORT_SCOPED.test(symbol)) ? svc.transports.flatMap((transport) => {
+		const hop = TRANSPORT_HOPS[transport.kind];
+		return hop === null ? [] : [within(transport.name, transportBindings(transport, hop, scenario))];
+	}) : symbols.some((symbol) => SCOPED.test(symbol)) ? svc.dependencies.map((dependency) => within(dependency.name, dependencyBindings(dependency, scenario))) : [[
+		svc.name,
+		service.bind,
+		service.epochs
+	]];
+};
+/** This function gives the epochs of the symbols of a rule that the scenario binds. */
+const epochsOf = (symbols, epochs) => Object.fromEntries(symbols.flatMap((symbol) => symbol in epochs ? [[symbol, epochs[symbol] ?? null]] : []));
+/**
+* These are the marks of the timers of the platform (E10). Kubernetes takes the probe period, the
+* preStop sleep and the termination grace in whole seconds, thus each is stepped by 1 s. The
+* cluster sets the propagation of the endpoints, thus a compression of the service cannot scale it.
+*/
+const PLATFORM_MARKS = {
+	"probe_period@pod": {
+		kind: "stepped",
+		step: 1
+	},
+	"prestop_sleep@pod": {
+		kind: "stepped",
+		step: 1
+	},
+	"termination_grace@pod": {
+		kind: "stepped",
+		step: 1
+	},
+	"propagation_bound@cluster": { kind: "fixed" }
+};
+/**
+* This function tells which findings of `lint` a compression by k changes (E10). It scales the
+* bindings of each scope by the marks. Thus a harness that runs the service k times faster learns
+* the orderings that its run breaks. The unmarked symbols are the symbols of the rules with no mark.
+*/
+const compressionOf = (svc, scenario, rules, k, marks) => {
+	const service = serviceBindings(svc, scenario);
+	const changed = [];
+	const unmarked = /* @__PURE__ */ new Set();
+	for (const rule of rules) {
+		if (!SERVICE_RULES.includes(rule.id)) continue;
+		for (const [scope, bind] of scopesOf(svc, scenario, service, symbolsOf(rule))) {
+			const result = compression([rule], bind, k, marks);
+			for (const change of result.changed) changed.push({
+				...change,
+				scope
+			});
+			for (const symbol of result.unmarked) unmarked.add(symbol);
+		}
+	}
+	return {
+		changed,
+		unmarked: [...unmarked]
+	};
+};
 /**
 * This function applies each service rule to a declaration and a scenario. A rule applies one time
 * for the service, for each client, for each consumer and worker, or for each limiter. The function
@@ -2535,26 +2835,14 @@ const progressFindings = (svc) => {
 * Each progress check that names its loops also gets its findings (`PROGRESS_RULE`).
 */
 const lint = (svc, scenario, rules) => {
-	const service = serviceBindings(svc, scenario).bind;
+	const service = serviceBindings(svc, scenario);
 	const findings = [...declarationFindings(svc), ...progressFindings(svc)];
 	for (const rule of rules) {
 		if (!SERVICE_RULES.includes(rule.id)) continue;
 		const symbols = symbolsOf(rule);
-		const scopes = symbols.some((symbol) => LIMITER_SCOPED.test(symbol)) ? (svc.limiters ?? []).map((limiter) => [limiter.name, {
-			...service,
-			...limiterBindings(limiter, scenario)
-		}]) : symbols.some((symbol) => TRANSPORT_SCOPED.test(symbol)) ? svc.transports.flatMap((transport) => {
-			const hop = TRANSPORT_HOPS[transport.kind];
-			return hop === null ? [] : [[transport.name, {
-				...service,
-				...transportBindings(transport, hop, scenario)
-			}]];
-		}) : symbols.some((symbol) => SCOPED.test(symbol)) ? svc.dependencies.map((dependency) => [dependency.name, {
-			...service,
-			...dependencyBindings(dependency, scenario)
-		}]) : [[svc.name, service]];
-		for (const [scope, bind] of scopes) {
+		for (const [scope, bind, all] of scopesOf(svc, scenario, service, symbols)) {
 			const evaluated = evaluate(rule, bind);
+			const epochs = epochsOf(symbols, all);
 			findings.push({
 				rule: rule.id,
 				scope,
@@ -2562,7 +2850,8 @@ const lint = (svc, scenario, rules) => {
 				statement: rule.rule,
 				...rule.window !== void 0 ? { window: rule.window } : {},
 				evaluated,
-				unbound: symbols.filter((symbol) => bind[symbol] === void 0)
+				unbound: symbols.filter((symbol) => bind[symbol] === void 0),
+				...Object.values(epochs).some((epoch) => epoch !== null) ? { epochs } : {}
 			});
 		}
 	}
@@ -3555,6 +3844,6 @@ const MACHINES = {
 	watchdog: WATCHDOG_MACHINE
 };
 //#endregion
-export { DECLARATION_RULE, EXIT_MACHINE, EXIT_SIGNALS, FORMAT, GATE_MACHINE, LATCH_MACHINE, LIMITER_RULES, MACHINES, POLLER_MACHINE, PROCESS_TRIGGERS, PROGRESS_RULE, SERVICE_RULES, SLIDING_WINDOW_MACHINE, STAGES, TOKEN_BUCKET_MACHINE, WATCHDOG_MACHINE, WINDOW_MACHINE, asDebounce, asWindow, backoffBand, backoffDelay, backoffProblems, capKnob, cooperativeAbortAt, daemon, decide, detectionTime, drainMachine, embeddedProcess, ephemeral, evaluate, failSpan, freeAt, heartbeat, heartbeatDetection, idleSpan, inequalities, isExitCode, isExitSignal, isMessageShape, keyedLimiter, kubernetes, limitRule, limiterMachine, limiterProblems, limiterStateOf, lint, localDev, longestRetryAfter, matchesMessage, mayReissue, memoryStore, nodeProcess, platformClock, processPortOf, queueDrain, realClock, recoverSpan, redeliverOnce, replay, retryAfterSeconds, service, serviceBindings, stageRank, symbolsOf, verdictOf, watchdog, workerLoop };
+export { DECLARATION_RULE, DURATION_FRACTIONS, EPISODES_KEPT, EXIT_MACHINE, EXIT_SIGNALS, FORMAT, GATE_MACHINE, LATCH_MACHINE, LAW_STATISTIC, LIMITER_RULES, MACHINES, PLATFORM_MARKS, POLLER_MACHINE, PROCESS_TRIGGERS, PROGRESS_RULE, SERVICE_RULES, SLIDING_WINDOW_MACHINE, STAGES, TOKEN_BUCKET_MACHINE, WATCHDOG_MACHINE, WINDOW_MACHINE, asDebounce, asWindow, backoffBand, backoffDelay, backoffProblems, capKnob, compress, compression, compressionOf, cooperativeAbortAt, daemon, decide, detectionTime, drainMachine, embeddedProcess, ephemeral, evaluate, failSpan, freeAt, heartbeat, heartbeatDetection, idleSpan, inequalities, isExitCode, isExitSignal, isMessageShape, keyedLimiter, kubernetes, limitRule, limiterMachine, limiterProblems, limiterStateOf, lint, localDev, longestRetryAfter, matchesMessage, mayReissue, memoryStore, nodeProcess, platformClock, processPortOf, queueDrain, realClock, recoverSpan, redact, redeliverOnce, replay, retryAfterSeconds, service, serviceBindings, stageRank, symbolsOf, verdictOf, watchdog, workerLoop };
 
 //# sourceMappingURL=index.js.map