@mark1russell7/highways 0.3.0 → 0.4.0

package/src/bench/screen/index.ts

@@ -0,0 +1,6 @@
+export * from "./hash.ts";
+export * from "./forms.ts";
+export * from "./measures.ts";
+export * from "./set.ts";
+export * from "./screen.ts";
+export * from "./generators.ts";

package/src/bench/screen/measures.ts

@@ -0,0 +1,114 @@
+import type { Objective } from "../measures/pareto.ts";
+
+/**
+ * One measure of a record. `key` is the key of the measure in the values of `runSuite` and `compareStep`.
+ * `name` is the name of the measure in a record of the form `layout-record/1`.
+ */
+export interface MeasureInfo {
+  /** The key in the values of `runSuite` and `compareStep`, for example `R5.median`. */
+  readonly key: string;
+  /** The id of the measure in `docs/research/MEASURES.md` (MEAS0), for example `R5`. */
+  readonly id: string;
+  /** The name in a record: lowercase, with a dot after the stage, for example `route.detour.median`. */
+  readonly name: string;
+  readonly unit: string;
+  readonly better: "lower" | "higher";
+  /**
+   * This value is `true` when the positions of the boxes alone can move the value, with the same router.
+   * Then two values compare only on the same input. The value is `false` when the measure scores the routes or the curves for a fixed placement.
+   */
+  readonly placementDependent: boolean;
+}
+
+/** The text of a row of `MEASURES`: the key, the id, the name, the unit, the better side and the placement dependence. */
+type Row = readonly [key: string, id: string, name: string, unit: string, better: "lower" | "higher", placement: boolean];
+
+const ROWS: readonly Row[] = [
+  // The routes. A placement makes crossings, plugs on short sides and boxes that touch, so these measures depend on it.
+  ["R1.crossings", "R1", "route.crossings", "count", "lower", true],
+  ["R1.perEdge", "R1", "route.crossings.per-edge", "count for each edge", "lower", true],
+  ["R2.angle", "R2", "route.crossing-angle.mean", "0 to 1", "higher", true],
+  ["R2.min", "R2", "route.crossing-angle.min", "degrees", "higher", true],
+  ["R3.bends", "R3", "route.bends", "count for each route", "lower", false],
+  ["R4.turn", "R4", "route.turn.mean", "half turns for each route", "lower", false],
+  ["R4.max", "R4", "route.turn.max", "degrees", "lower", false],
+  ["R5.median", "R5", "route.detour.median", "ratio", "lower", false],
+  ["R5.q95", "R5", "route.detour.q95", "ratio", "lower", false],
+  ["R5.mean", "R5", "route.detour.mean", "ratio", "lower", false],
+  ["R6.worstDetour", "R6", "route.detour.max", "ratio", "lower", false],
+  ["R7.backtrack", "R7", "route.backtrack", "0 to 1", "lower", false],
+  ["R8.clearance", "R8", "route.clearance.mean", "units", "higher", true],
+  ["R8.tight", "R8", "route.clearance.tight", "0 to 1", "lower", true],
+  ["R9.through", "R9", "route.through", "count", "lower", true],
+  ["R10.plugAngle", "R10", "route.plug-angle", "degrees", "lower", false],
+  ["R11.min", "R11", "route.plug-spacing.min", "0 to 1", "higher", true],
+  ["R11.q05", "R11", "route.plug-spacing.q05", "0 to 1", "higher", true],
+  ["R13.outside", "R13", "route.outside", "count", "lower", false],
+  ["R15.inflections", "R15", "route.inflections", "count for each route", "lower", false],
+  ["ink", "R1", "route.ink", "units", "lower", true],
+  // The bundles.
+  ["B1.inkSaved", "B1", "route.bundle.ink-saved", "0 to 1", "higher", false],
+  ["B4.ambiguity", "B4", "route.bundle.ambiguity", "0 to 1", "lower", false],
+  // The stability after the small change of `changeGraph`, and against the run on the base input of a variant.
+  ["Z10.mean", "Z10", "route.stability.mean", "units of the board", "lower", false],
+  ["Z10.max", "Z10", "route.stability.max", "units of the board", "lower", false],
+  ["Z10.moved", "Z10", "route.stability.moved", "0 to 1", "lower", false],
+  ["Z10.base.mean", "Z10", "route.stability.base.mean", "units of the board", "lower", false],
+  ["Z10.base.max", "Z10", "route.stability.base.max", "units of the board", "lower", false],
+  ["Z10.base.moved", "Z10", "route.stability.base.moved", "0 to 1", "lower", false],
+  // The zoom levels and the groups. The default grouping rule reads the positions, so most of these measures depend on the placement.
+  ["level-nesting", "Z6", "route.level.nesting", "count", "lower", false],
+  ["Z3.through", "Z3", "route.level.through", "count", "lower", true],
+  ["Z3.length", "Z3", "route.level.link-length", "units", "lower", true],
+  ["Z4.farItems", "Z4", "route.level.far-items", "count", "lower", true],
+  ["Z7.position", "Z7", "route.level.position", "no unit", "lower", true],
+  ["Z8.order", "Z8", "route.level.order", "-1 to 1", "higher", true],
+  ["G1.modularity", "G1", "route.group.modularity", "-0.5 to 1", "higher", true],
+  ["G2.cut", "G2", "route.group.cut", "0 to 1", "lower", true],
+  ["G6.balance", "G6", "route.group.balance", "0 to 1", "higher", true],
+  ["G8.treeCost", "G8", "route.group.tree-cost", "0 to 1", "lower", true],
+  ["G9.ari", "G9", "route.group.stability", "-1 to 1", "higher", true],
+  // The curves. A curve draws a fixed polyline, so no measure of a curve depends on the placement.
+  ["C1.max", "C1", "route.curve.c1.max", "degrees", "lower", false],
+  ["C1.share", "C1", "route.curve.c1.share", "0 to 1", "lower", false],
+  ["C2.median", "C2", "route.curve.c2.median", "no unit", "lower", false],
+  ["C2.max", "C2", "route.curve.c2.max", "no unit", "lower", false],
+  ["C3.median", "C3", "route.curve.c3.median", "no unit", "lower", false],
+  ["C3.q95", "C3", "route.curve.c3.q95", "no unit", "lower", false],
+  ["C5.peak", "C5", "route.curve.c5.peak", "curvature times the unit of the board", "lower", false],
+  ["C6.length", "C6", "route.curve.c6.length", "ratio", "lower", false],
+  ["curve-clear", "C7", "route.curve.rule.clear", "count", "lower", false],
+  ["curve-ends", "C10", "route.curve.rule.ends", "count", "lower", false],
+  ["curve-g1", "C1", "route.curve.rule.g1", "count", "lower", false],
+  ["curve-simple", "C9", "route.curve.rule.simple", "count", "lower", false],
+];
+
+/**
+ * The measures of the records of the stage `route`, with the placement dependence of each one.
+ * A joint harness that moves the boxes and routes the edges reads `placementDependent`.
+ * It compares a measure with the value `true` only between runs on the same input.
+ */
+export const MEASURES: readonly MeasureInfo[] = Object.freeze(ROWS.map(([key, id, name, unit, better, placementDependent]) => Object.freeze({ key, id, name, unit, better, placementDependent })));
+
+const BY_KEY: ReadonlyMap<string, MeasureInfo> = new Map(MEASURES.map((m) => [m.key, m]));
+const BY_NAME: ReadonlyMap<string, MeasureInfo> = new Map(MEASURES.map((m) => [m.name, m]));
+
+/** This function gives the measure of a key of `runSuite` or `compareStep`, or `undefined` for a key that is not in `MEASURES`. */
+export function measureByKey(key: string): MeasureInfo | undefined {
+  return BY_KEY.get(key);
+}
+
+/** This function gives the measure of a name of a record, or `undefined` for a name that is not in `MEASURES`. */
+export function measureByName(name: string): MeasureInfo | undefined {
+  return BY_NAME.get(name);
+}
+
+/** This function gives the name of a key in a record. A key that is not in `MEASURES` gets the name `route.other.<key>`. */
+export function measureName(key: string): string {
+  return BY_KEY.get(key)?.name ?? `route.other.${key.toLowerCase()}`;
+}
+
+/** This function gives the objectives of a Pareto front by the names of a record, from objectives by the keys of `runSuite`. */
+export function namedObjectives(objectives: readonly Objective[]): Objective[] {
+  return objectives.map((o) => ({ key: measureName(o.key), better: o.better }));
+}

package/src/bench/screen/screen.ts

@@ -0,0 +1,390 @@
+import { highwayRouterStrategy } from "../../highways/router.ts";
+import type { Graph, Result } from "../../model.ts";
+import { STRATEGIES } from "../../strategies/builtins.ts";
+import { createRegistry, getStrategy, listStrategies, STEPS, type AnyStrategy, type Registry, type Strategy } from "../../strategies/model.ts";
+import { compareStep, type CompareRow, type CompareStep, type StepComparison } from "../compare.ts";
+import { paretoFront, type Objective } from "../measures/pareto.ts";
+import { routeStability } from "../measures/stability.ts";
+import { quantile } from "../measures/stats.ts";
+import { checkLevels, checkProperties, TOLERANCE } from "../properties.ts";
+import { measureQuality } from "../quality.ts";
+import { sameData } from "../run.ts";
+import { FRONT_OBJECTIVES } from "../suite.ts";
+import { inputGraph, jsonSafe, type LayoutInput, type LayoutPrediction, type LayoutRecord, type PredictionOp } from "./forms.ts";
+import { measureByName, measureName, namedObjectives } from "./measures.ts";
+import { SCREEN_V1, screenInputs, type ScreenSet } from "./set.ts";
+
+/**
+ * The steps of the screen: the steps of `compareStep` and the step `highway`.
+ * The screen of the step `highway` routes the plan of each strategy with `follow`, and measures the routes as the routes of a router.
+ */
+export type ScreenStep = CompareStep | "highway";
+
+/** The steps of the screen, in the order of the pipeline. */
+export const SCREEN_STEPS: readonly ScreenStep[] = ["group", "router", "highway", "curve"];
+
+/** The routers that the screen of the step `highway` also runs by default. They are the baselines of the note on highways. */
+export const HIGHWAY_BASE_ROUTERS: readonly string[] = ["corners", "shared", "trunks"];
+
+/** The hard rules of the routes of a router, or of a plan after `follow`. */
+const ROUTE_RULES: readonly string[] = ["complete", "ends-at-plugs", "fixed-plugs", "no-pass-through", "counts-sum", "link-ends", "bundle-trunk", "run-shared", "same-twice"];
+
+/** The hard rules of each step of the screen, in the order of the records. A good run has 0 for each rule. */
+export const SCREEN_RULES: Readonly<Record<ScreenStep, readonly string[]>> = {
+  router: ROUTE_RULES,
+  highway: ROUTE_RULES,
+  group: ["sections", "counts-sum", "link-ends", "same-twice"],
+  curve: ["same-twice"],
+};
+
+/** The options of `screen`. */
+export interface ScreenOptions {
+  /** The screen set. The default is `SCREEN_V1`. */
+  readonly set?: ScreenSet;
+  /** The input files. The default is the files of `set`. */
+  readonly inputs?: readonly LayoutInput[];
+  /** The families of the inputs that the screen runs. The default is each family. */
+  readonly families?: readonly string[];
+  /**
+   * With `true`, the screen also runs each built-in strategy of the step, and the front compares the new strategies with them.
+   * With `false`, it runs only the strategies of the call. The default is `true`.
+   */
+  readonly baseline?: boolean;
+  /** The options of each strategy, by its name. A strategy without an entry gets its defaults. */
+  readonly options?: Readonly<Record<string, object>>;
+  /** The router whose routes the step `curve` draws. The default is `corners`. */
+  readonly curveRouter?: string;
+  /**
+   * The routers that the screen of the step `highway` also runs. Their records have the step `router`.
+   * The default is `HIGHWAY_BASE_ROUTERS`. The other steps do not read it.
+   */
+  readonly baseRouters?: readonly string[];
+  /** The seed of the small change of the stability measures. The default is 1. */
+  readonly seed?: number;
+  /** The commit of the code, for the records. */
+  readonly commit?: string;
+  /** The predictions of the run. Each record names the first prediction with its strategy, its set and its input. */
+  readonly predictions?: readonly LayoutPrediction[];
+  /** This function gets one line of progress for each input. */
+  readonly log?: (line: string) => void;
+}
+
+/** The summary of one strategy over the inputs of a screen. */
+export interface ScreenStrategySummary {
+  readonly strategy: string;
+  /** This value is `true` for a strategy of the call, and `false` for a built-in strategy of the baseline. */
+  readonly candidate: boolean;
+  /** The number of inputs where the strategy ran. */
+  readonly runs: number;
+  /** The number of inputs that the strategy skipped. */
+  readonly skipped: number;
+  /** This value is `true` when each run gave the same output two times. */
+  readonly deterministic: boolean;
+  /** The sum of the breaks of each hard rule over the runs. */
+  readonly hard: Readonly<Record<string, number>>;
+  /** The sum of `hard`. */
+  readonly hardTotal: number;
+  /**
+   * The breaks over the floor of each input. The floor is the counts of the built-in strategy with the fewest breaks on that input.
+   * Each built-in breaks a rule on some boards, for example on a board where the boxes touch. Without built-ins, the floor is 0.
+   */
+  readonly hardOverFloor: number;
+  /** The median of each measure over the runs, by the names of a record. */
+  readonly medians: Readonly<Record<string, number>>;
+  /** The median time of a run, in milliseconds. */
+  readonly ms: number;
+  /** The number of inputs where the strategy is on the Pareto front of its input. */
+  readonly onFront: number;
+}
+
+/** A prediction with the measured value. */
+export interface PredictionCheck extends LayoutPrediction {
+  readonly op: PredictionOp;
+  /** The measured value: the value of the record of the input, or the median over the set. It is absent when no record has the measure. */
+  readonly measured?: number;
+  /** This value is `true` when the measured value meets the prediction. It is absent when there is no measured value. */
+  readonly met?: boolean;
+}
+
+/** The summary of a screen. */
+export interface ScreenSummary {
+  readonly set: string;
+  readonly step: ScreenStep;
+  /** The number of inputs. */
+  readonly inputs: number;
+  /** The axes of the Pareto front, by the names of a record. */
+  readonly objectives: readonly Objective[];
+  /** One summary for each strategy, in the order of the registry. */
+  readonly strategies: readonly ScreenStrategySummary[];
+  /** The strategies on the Pareto front of the medians over the set. */
+  readonly front: readonly string[];
+  /** The strategies with a break over the floor, without a run or with two different outputs of one input. They do not enter the front. */
+  readonly excluded: readonly string[];
+  readonly predictions: readonly PredictionCheck[];
+}
+
+/** The result of `screen`: one record for each strategy and input, and the summary. */
+export interface ScreenReport {
+  readonly records: readonly LayoutRecord[];
+  readonly summary: ScreenSummary;
+}
+
+/** This function counts the violations of each rule, with 0 for each rule of `rules`. */
+function countRules(rules: readonly string[], violations: readonly { readonly rule: string }[]): Record<string, number> {
+  const out: Record<string, number> = Object.fromEntries(rules.map((r) => [r, 0]));
+  for (const v of violations) out[v.rule] = (out[v.rule] ?? 0) + 1;
+  return out;
+}
+
+/** This function gives the median of a list of numbers, or `NaN` for an empty list. */
+function median(values: readonly number[]): number {
+  return quantile(values, 0.5);
+}
+
+/** This function tells if a value meets a comparison with an expected value. */
+function meets(value: number, op: PredictionOp, expected: number): boolean {
+  if (op === "<") return value < expected;
+  if (op === "<=") return value <= expected;
+  if (op === ">") return value > expected;
+  if (op === ">=") return value >= expected;
+  return value === expected;
+}
+
+/** This function makes the registry of a screen. It refuses a strategy of another step, and a new strategy with the name of a built-in strategy. */
+function screenRegistry(step: ScreenStep, strategies: readonly (Strategy<ScreenStep> | string)[], baseline: boolean): { registry: Registry; candidates: Set<string> } {
+  const builtins = listStrategies(STRATEGIES, step) as readonly Strategy<ScreenStep>[];
+  const chosen: Strategy<ScreenStep>[] = [];
+  for (const s of strategies) {
+    if (typeof s === "string") {
+      const known = builtins.find((b) => b.name === s);
+      if (!known) throw new Error(`There is no ${step} "${s}". The registry has ${builtins.map((b) => b.name).join(", ")}.`);
+      chosen.push(known);
+      continue;
+    }
+    if (s.step !== step) throw new Error(`The strategy "${s.name}" is of the step "${s.step}", not "${step}".`);
+    const known = builtins.find((b) => b.name === s.name);
+    if (known && known !== s) throw new Error(`The new ${step} "${s.name}" has the name of a built-in strategy. Give it another name.`);
+    chosen.push(s);
+  }
+  const candidates = new Set(chosen.map((s) => s.name));
+  const others = STEPS.filter((x) => x !== step).flatMap((x) => listStrategies(STRATEGIES, x) as readonly AnyStrategy[]);
+  const own = baseline ? [...builtins, ...chosen.filter((s) => !builtins.includes(s))] : chosen.filter((s, i) => chosen.indexOf(s) === i);
+  return { registry: createRegistry([...others, ...(own as readonly AnyStrategy[])]), candidates };
+}
+
+/** This function tells if a step of the screen gives routes. The step `router` gives routes, and the step `highway` gives routes after `follow`. */
+const givesRoutes = (step: ScreenStep): step is "router" | "highway" => step === "router" || step === "highway";
+
+/** This function gives the values of a row of `compareStep`, with the measures of `measureQuality` for a router. */
+function rowValues(step: ScreenStep, graph: Graph, row: CompareRow): Record<string, number> {
+  const values: Record<string, number> = { ...row.values };
+  delete values["ms"];
+  if (givesRoutes(step) && row.result) {
+    const q = measureQuality(graph, row.result);
+    values["R3.bends"] = q.bends;
+    values["R8.clearance"] = q.clearance;
+    values["R8.tight"] = q.tight;
+    values["R10.plugAngle"] = q.plugAngle;
+    values["R15.inflections"] = q.inflections;
+  }
+  return values;
+}
+
+/** This function gives the hard rules of a row of `compareStep`, and tells if the row gave the same output two times. */
+function rowRules(step: ScreenStep, graph: Graph, row: CompareRow, again: CompareRow | undefined): { hard: Record<string, number>; deterministic: boolean } {
+  const rules = SCREEN_RULES[step];
+  if (givesRoutes(step)) {
+    const all = row.result ? checkProperties(graph, row.result, 1_000_000) : [];
+    // `compareStep` adds one violation when two routes of the input differ.
+    const deterministic = row.violations === all.length;
+    return { hard: { ...countRules(rules, all), "same-twice": deterministic ? 0 : 1 }, deterministic };
+  }
+  if (step === "group") {
+    const all = row.result ? checkLevels(graph, row.result.levels ?? [], 1_000_000) : [{ rule: "sections" }];
+    const deterministic = again !== undefined && sameData(row.tree, again.tree) && sameData(row.result, again.result);
+    return { hard: { ...countRules(rules, all), "same-twice": deterministic ? 0 : 1 }, deterministic };
+  }
+  const deterministic = again !== undefined && sameData(row.curves, again.curves);
+  return { hard: { ...countRules(rules, []), "same-twice": deterministic ? 0 : 1 }, deterministic };
+}
+
+/** This function keeps the finite values and gives them the names of a record. */
+function named(values: Readonly<Record<string, number>>): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const [k, v] of Object.entries(values)) if (Number.isFinite(v)) out[measureName(k)] = v;
+  return out;
+}
+
+/**
+ * This function screens strategies of one step on the inputs of a screen set. It is the cheap test of a new idea before a full build.
+ * Each strategy runs on each input with `compareStep`, and each run gives one record of the form `layout-record/1`.
+ * A strategy can be a name of the registry or a strategy object, for example from a prototype file. The registry does not change.
+ *
+ * The step `router` routes each input two times for the rule "same twice". The steps `group` and `curve` run each input two times for the same rule.
+ *
+ * The step `highway` makes the plan of each strategy and routes it with `follow`, two times. It also runs the routers of `baseRouters`.
+ * An input with a `base` in the inputs also gets the measures `route.stability.base.*`: the routes on the variant against the routes on its base.
+ * The same inputs and the same options give the same records. Only the times change.
+ */
+export function screen(step: ScreenStep, strategies: readonly (Strategy<ScreenStep> | string)[] = [], options: ScreenOptions = {}): ScreenReport {
+  const set = options.set ?? SCREEN_V1;
+  const screened = screenRegistry(step, strategies, options.baseline ?? true);
+  const candidates = screened.candidates;
+  // The step `highway` runs as the step `router`. Each highway strategy with `follow` is a router, after the base routers.
+  const base = step === "highway" ? (options.baseRouters ?? HIGHWAY_BASE_ROUTERS).map((name) => getStrategy(STRATEGIES, "router", name)) : [];
+  const registry =
+    step === "highway"
+      ? createRegistry([...base, ...listStrategies(screened.registry, "highway").map((s) => highwayRouterStrategy(s))] as readonly AnyStrategy[])
+      : screened.registry;
+  const compareAs: CompareStep = step === "highway" ? "router" : step;
+  const baseNames = new Set(base.map((s) => s.name));
+  const stepOf = (name: string): ScreenStep => (baseNames.has(name) ? "router" : step);
+  const own = options.options ?? {};
+  const seed = options.seed ?? 1;
+  const log = options.log ?? ((): void => {});
+  const families = options.families;
+  const inputs = (options.inputs ?? screenInputs(set)).filter((i) => families?.includes(i.family) ?? true);
+  const bases = new Set(inputs.flatMap((i) => (i.base !== undefined ? [i.base] : [])));
+  const byName = new Map(listStrategies(registry, compareAs).map((s) => [s.name, s as Strategy<CompareStep>]));
+  const records: (LayoutRecord & { measures: Record<string, number> })[] = [];
+  const comparisons: StepComparison[] = [];
+  const kept = new Map<string, { graph: Graph; results: Map<string, Result> }>();
+  const variants: { record: (typeof records)[number]; base: string; result: Result }[] = [];
+
+  for (const input of inputs) {
+    const graph = inputGraph(input);
+    const compare = { registry, options: own, seed, name: input.id, ...(options.curveRouter ? { curveRouter: options.curveRouter } : {}) };
+    const cmp = compareStep(compareAs, graph, compare);
+    const again = givesRoutes(step) ? undefined : compareStep(compareAs, graph, compare);
+    comparisons.push(cmp);
+    log(`${input.id}: ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${cmp.rows.map((r) => `${r.strategy} ${r.ms.toFixed(0)}`).join(", ")}`);
+    if (bases.has(input.id)) kept.set(input.id, { graph, results: new Map(cmp.rows.flatMap((r) => (r.result ? [[r.strategy, r.result] as const] : []))) });
+    cmp.rows.forEach((row, i) => {
+      const strategy = byName.get(row.strategy)!;
+      const head = {
+        schema: "layout-record/1" as const,
+        stage: "route" as const,
+        step: stepOf(row.strategy),
+        strategy: row.strategy,
+        options: jsonSafe({ ...strategy.defaults, ...own[row.strategy] }) as Record<string, unknown>,
+        corpus: input.set,
+        input: input.id,
+        inputHash: input.hash,
+        set: input.set,
+        seed: input.seed,
+        nodes: graph.nodes.length,
+        edges: graph.edges.length,
+      };
+      const prediction = options.predictions?.find((p) => p.strategy === row.strategy && p.set === input.set && p.input === input.id)?.id;
+      const tail = { ...(prediction !== undefined ? { prediction } : {}), ...(options.commit !== undefined ? { commit: options.commit } : {}) };
+      if (row.skipped !== undefined) {
+        records.push({ ...head, measures: {}, hard: {}, tolerance: { through: TOLERANCE }, deterministic: true, ms: 0, ...tail, skipped: row.skipped });
+        return;
+      }
+      const { hard, deterministic } = rowRules(step, graph, row, again?.rows[i]);
+      const record = { ...head, measures: named(rowValues(step, graph, row)), hard, tolerance: { through: TOLERANCE }, deterministic, ms: row.ms, ...tail };
+      records.push(record);
+      if (givesRoutes(step) && input.base !== undefined && row.result) variants.push({ record, base: input.base, result: row.result });
+    });
+  }
+
+  // The stability against the base: the same strategy on the base input and on its variant.
+  for (const v of variants) {
+    const base = kept.get(v.base);
+    const before = base?.results.get(v.record.strategy);
+    if (!base || !before) continue;
+    const s = routeStability(base.graph, before, v.result, { seed });
+    Object.assign(v.record.measures, named({ "Z10.base.mean": s.mean, "Z10.base.max": s.max, "Z10.base.moved": s.moved }));
+  }
+
+  return { records, summary: summarizeScreen(records, comparisons, { set: set.id, step, candidates, ...(options.predictions ? { predictions: options.predictions } : {}) }) };
+}
+
+/** This function gives the summary of the records of a screen. */
+function summarizeScreen(
+  records: readonly LayoutRecord[],
+  comparisons: readonly StepComparison[],
+  at: { set: string; step: ScreenStep; candidates: ReadonlySet<string>; predictions?: readonly LayoutPrediction[] },
+): ScreenSummary {
+  const objectives = namedObjectives(FRONT_OBJECTIVES[at.step]);
+  const names = [...new Set(records.map((r) => r.strategy))];
+  const total = (r: LayoutRecord): number => Object.values(r.hard).reduce((n, v) => n + v, 0);
+  const floor = new Map<string, Readonly<Record<string, number>>>();
+  const best = new Map<string, number>();
+  for (const r of records) {
+    if (r.skipped !== undefined || at.candidates.has(r.strategy)) continue;
+    if (total(r) >= (best.get(r.input) ?? Number.POSITIVE_INFINITY)) continue;
+    best.set(r.input, total(r));
+    floor.set(r.input, r.hard);
+  }
+  const overFloor = (r: LayoutRecord): number => Object.entries(r.hard).reduce((n, [k, v]) => n + Math.max(0, v - (floor.get(r.input)?.[k] ?? 0)), 0);
+  const strategies = names.map((strategy): ScreenStrategySummary => {
+    const all = records.filter((r) => r.strategy === strategy);
+    const ran = all.filter((r) => r.skipped === undefined);
+    const hard: Record<string, number> = {};
+    for (const r of ran) for (const [k, v] of Object.entries(r.hard)) hard[k] = (hard[k] ?? 0) + v;
+    const keys = [...new Set(ran.flatMap((r) => Object.keys(r.measures)))];
+    const medians = Object.fromEntries(keys.map((k) => [k, median(ran.flatMap((r) => (r.measures[k] === undefined ? [] : [r.measures[k]])))]));
+    return {
+      strategy,
+      candidate: at.candidates.has(strategy),
+      runs: ran.length,
+      skipped: all.length - ran.length,
+      deterministic: ran.every((r) => r.deterministic),
+      hard,
+      hardTotal: Object.values(hard).reduce((s, v) => s + v, 0),
+      hardOverFloor: ran.reduce((n, r) => n + overFloor(r), 0),
+      medians,
+      ms: median(ran.map((r) => r.ms)),
+      onFront: comparisons.filter((c) => c.front.includes(strategy)).length,
+    };
+  });
+  const fair = strategies.filter((s) => s.runs > 0 && s.hardOverFloor === 0 && s.deterministic);
+  const front = paretoFront(fair, (s) => Object.fromEntries(objectives.map((o) => [o.key, s.medians[o.key] ?? Number.NaN])), objectives).map((s) => s.strategy);
+  const predictions = (at.predictions ?? []).map((p): PredictionCheck => {
+    const op = p.op ?? (measureByName(p.measure)?.better === "higher" ? ">=" : "<=");
+    const mine = records.filter((r) => r.strategy === p.strategy && r.set === p.set && r.skipped === undefined && (p.input === undefined || r.input === p.input));
+    const values = mine.flatMap((r) => (r.measures[p.measure] === undefined ? [] : [r.measures[p.measure]!]));
+    const measured = values.length === 0 ? undefined : p.input !== undefined ? values[0]! : median(values);
+    return { ...p, op, ...(measured === undefined ? {} : { measured, met: meets(measured, op, p.expected) }) };
+  });
+  return {
+    set: at.set,
+    step: at.step,
+    inputs: comparisons.length,
+    objectives,
+    strategies,
+    front,
+    excluded: strategies.filter((s) => !fair.includes(s)).map((s) => s.strategy),
+    predictions,
+  };
+}
+
+const fmt = (v: number | undefined): string => (v === undefined || Number.isNaN(v) ? "-" : Number.isInteger(v) ? String(v) : v.toFixed(3));
+
+/** This function writes the summary of a screen as Markdown: one row for each strategy, the front and the predictions. */
+export function formatScreen(summary: ScreenSummary): string {
+  const axes = summary.objectives.map((o) => `${o.key} ${o.better === "min" ? "↓" : "↑"}`);
+  const lines = [
+    `## Screen of the step \`${summary.step}\` on \`${summary.set}\``,
+    "",
+    `${summary.inputs} inputs, ${summary.strategies.length} strategies. Each cell of a measure is the median over the inputs.`,
+    "The floor of an input is the counts of the hard rules of the built-in strategy with the fewest breaks on that input.",
+    "",
+    `| strategy | new | runs | skipped | hard | over the floor | same twice | on the front of an input | ms | ${axes.join(" | ")} |`,
+    `|---|---|---:|---:|---:|---:|---|---:|---:|${axes.map(() => "---:").join("|")}|`,
+    ...summary.strategies.map(
+      (s) =>
+        `| ${s.strategy} | ${s.candidate ? "yes" : ""} | ${s.runs} | ${s.skipped} | ${s.hardTotal} | ${s.hardOverFloor} | ${s.deterministic ? "yes" : "no"} | ${s.onFront} | ${fmt(s.ms)} | ${summary.objectives.map((o) => fmt(s.medians[o.key])).join(" | ")} |`,
+    ),
+    "",
+    `Pareto front over the medians: ${summary.front.join(", ") || "-"}.`,
+    `Not on the front because of a break over the floor, no run or two different outputs: ${summary.excluded.join(", ") || "-"}.`,
+  ];
+  if (summary.predictions.length > 0) {
+    lines.push("", "| prediction | strategy | input | measure | expected | measured | met | reason |", "|---|---|---|---|---:|---:|---|---|");
+    for (const p of summary.predictions)
+      lines.push(`| ${p.id} | ${p.strategy} | ${p.input ?? "median"} | ${p.measure} | ${p.op} ${fmt(p.expected)} | ${fmt(p.measured)} | ${p.met === undefined ? "-" : p.met ? "yes" : "no"} | ${p.reason} |`);
+  }
+  return lines.join("\n") + "\n";
+}

package/src/bench/screen/set.ts

@@ -0,0 +1,136 @@
+import { hubChainBoard, manyGroupsBoard, packedGridBoard } from "../../cases.ts";
+import { FIXTURES, makeFixture, type FixtureSpec } from "../../fixtures.ts";
+import type { Graph } from "../../model.ts";
+import { familySeed } from "../suite.ts";
+import { graphInput, type LayoutCell, type LayoutGenerator, type LayoutInput } from "./forms.ts";
+
+/** A family of boards of a screen set. The member `k` of the family has the seed `k`. */
+export interface ScreenFamily {
+  readonly name: string;
+  /** The known hard case that the family shows, in a few words. */
+  readonly shows: string;
+  /** The number of members. */
+  readonly seeds: number;
+  /** This function makes the member `k`. The same `k` always gives the same graph. */
+  readonly make: (k: number) => Graph;
+  /** This function gives the generator of the member `k`: its name, version, parameters and seed. */
+  readonly generator: (k: number) => LayoutGenerator;
+  /** The cell of each member in the taxonomy of the corpus. */
+  readonly cell: LayoutCell;
+}
+
+/** A screen set: a fixed list of small boards with an id. A new board or a changed board gets a new id. */
+export interface ScreenSet {
+  /** The id of the set, for example `screen-v1`. The records name it in `set` and `corpus`. */
+  readonly id: string;
+  readonly families: readonly ScreenFamily[];
+}
+
+const fixture = (name: string): FixtureSpec => FIXTURES.find((f) => f.name === name)!;
+const DENSE: FixtureSpec = { name: "dense-groups", kind: "clustered", nodes: 150, edgesPerNode: 2, seed: 3 };
+const PACKED = { cols: 12, rows: 8, long: 48 } as const;
+
+/** This function gives a family of `makeFixture`. The member `k` has the seed `familySeed(spec, k)`, as in `runSuite`. */
+function fixtureFamily(spec: FixtureSpec, shows: string, seeds: number, cell: LayoutCell): ScreenFamily {
+  const { name, seed: _seed, ...params } = spec;
+  return {
+    name,
+    shows,
+    seeds,
+    cell,
+    make: (k) => makeFixture({ ...spec, seed: familySeed(spec, k) }),
+    generator: (k) => ({ name: "fixture", version: 1, params, seed: familySeed(spec, k) }),
+  };
+}
+
+/** The cell dimensions that each family of `screen-v1` shares. */
+const BASE = { containerSize: "freeAspect", containerShape: "rect", obstacles: "none", mode: "offline", groups: "none" } as const;
+
+/**
+ * The screen set `screen-v1`: 14 boards of made-up data from 10 to 150 nodes. Each family shows a known hard case.
+ * The seed 0 of each family is the fixture or the case board of the benchmark. The seed `k` of a fixture family is `familySeed(spec, k)`, as in `runSuite`.
+ * The generators wrap the fixtures and the case boards without a change to their output.
+ * This set does not change. The tests compare it with the files in `corpus/screen-v1`.
+ */
+export const SCREEN_V1: ScreenSet = {
+  id: "screen-v1",
+  families: [
+    fixtureFamily(fixture("tiny-uniform"), "A tiny board: the cost of a strategy at its start.", 3, { ...BASE, count: "10", edgeDensity: "medium", edgeShape: "random", ports: "free" }),
+    fixtureFamily(fixture("small-clustered"), "Clusters, and fixed plugs on 20% of the nodes.", 3, { ...BASE, count: "100", edgeDensity: "medium", edgeShape: "clustered", ports: "fixed" }),
+    fixtureFamily(DENSE, "Dense groups whose boxes overlap at the far view.", 3, { ...BASE, count: "100", edgeDensity: "medium", edgeShape: "clustered", ports: "free" }),
+    {
+      name: "hub-chain",
+      shows: "A chain of hubs: routes climb away from their targets.",
+      seeds: 1,
+      cell: { ...BASE, count: "100", edgeDensity: "medium", edgeShape: "hubChain", ports: "free" },
+      make: () => hubChainBoard(),
+      generator: () => ({ name: "case.hub-chain", version: 1, params: {}, seed: 3 }),
+    },
+    {
+      name: "many-groups",
+      shows: "Many groups: level links go around the board.",
+      seeds: 1,
+      cell: { ...BASE, count: "100", edgeDensity: "medium", edgeShape: "clustered", ports: "free" },
+      make: () => manyGroupsBoard(),
+      generator: () => ({ name: "case.many-groups", version: 1, params: {}, seed: 7 }),
+    },
+    {
+      name: "packed-grid",
+      shows: "A packed grid: boxes touch, with no channel for the routes.",
+      seeds: 3,
+      cell: { ...BASE, count: "100", edgeDensity: "medium", edgeShape: "longRange", ports: "free" },
+      make: (k) => packedGridBoard({ ...PACKED, seed: 11 + 1009 * k }),
+      generator: (k) => ({ name: "case.packed-grid", version: 1, params: PACKED, seed: 11 + 1009 * k }),
+    },
+  ],
+};
+
+/** The screen sets by id. */
+export const SCREEN_SETS: Readonly<Record<string, ScreenSet>> = { [SCREEN_V1.id]: SCREEN_V1 };
+
+/** This function gives the id of the member `seed` of a family of a set: `<set>/<family>/s<seed>`. */
+export function inputId(set: string, family: string, seed: number): string {
+  return `${set}/${family}/s${seed}`;
+}
+
+/** This function gives the input files of a screen set, in the order of its families and seeds. Each file has its hash. */
+export function screenInputs(set: ScreenSet = SCREEN_V1): LayoutInput[] {
+  return set.families.flatMap((f) =>
+    Array.from({ length: f.seeds }, (_, k) => graphInput(f.make(k), { id: inputId(set.id, f.name, k), set: set.id, family: f.name, seed: k, generator: f.generator(k), cell: f.cell })),
+  );
+}
+
+/** A spec of a manifest: one generator with its version and parameters, and the seeds that the corpus uses. */
+export interface CorpusSpec {
+  readonly generator: string;
+  readonly version: number;
+  readonly params: Readonly<Record<string, unknown>>;
+  readonly seeds: readonly number[];
+}
+
+/** The manifest `corpus.json` of a corpus, of the form `layout-corpus/1`. */
+export interface LayoutCorpus {
+  readonly schema: "layout-corpus/1";
+  readonly corpus: string;
+  readonly specs: readonly CorpusSpec[];
+  readonly inputs: readonly { readonly id: string; readonly hash: string; readonly cell: LayoutCell }[];
+}
+
+/** This function gives the manifest of a list of inputs. The inputs of one generator, version and parameters share one spec. */
+export function corpusManifest(corpus: string, inputs: readonly LayoutInput[]): LayoutCorpus {
+  const specs = new Map<string, { generator: string; version: number; params: Readonly<Record<string, unknown>>; seeds: number[] }>();
+  for (const input of inputs) {
+    const g = input.generator;
+    if (!g) continue;
+    const key = JSON.stringify([g.name, g.version, g.params]);
+    const spec = specs.get(key) ?? { generator: g.name, version: g.version, params: g.params, seeds: [] };
+    if (!spec.seeds.includes(g.seed)) spec.seeds.push(g.seed);
+    specs.set(key, spec);
+  }
+  return {
+    schema: "layout-corpus/1",
+    corpus,
+    specs: [...specs.values()],
+    inputs: inputs.map((i) => ({ id: i.id, hash: i.hash, cell: i.cell ?? {} })),
+  };
+}