@openmixer/plugin-qualify 0.16.0 → 0.17.0

package/src/probe-runner.ts

@@ -25,6 +25,9 @@ import type { PluginTarget } from "./loader.js";
 import { SPAWNED_ADMISSIONS, admitPlugin, admittedHostOf, type AdmissionRuns, type HostAdmission } from "./host-admission.js";
 import { judgeProbeOutput, judgmentVerdicts, type Judgment } from "./probe-judge.js";
 import { readProbeOutput, type ProbeOutput } from "./probe-output.js";
+import { budgetCells, gridCells, runnerInvocations, type GridCell, type GridSpec, type RunnerInvocation } from "./measurement-grid.js";
+import type { MemoryBound } from "./memory-bound.js";
+import { mergeProbeRuns, ProbeMergeError } from "./probe-merge.js";
 import type { QualifyVerdict } from "./qualify-cli.js";
 import {
   CLAP_QUALIFY_TIMEOUT_S,
@@ -33,8 +36,10 @@ import {
   LV2_HOSTING_BLOCK_FRAMES,
   LV2_HOSTING_TIMEOUT_S,
   LV2_MEASURE_TOOL_NAME,
+  MEMORY_BOUND_EXIT_ENV,
   PROBE_RUNNER_DEFAULT,
   PROBE_RUNNER_ENV,
+  QUALIFY_QUANTA,
 } from "./qualify-declarations.js";
 
 export class ProbeRunnerError extends Error {
@@ -67,8 +72,14 @@ const spawnRunner: ProbeRun = (tool, args, env) => {
 };
 
 export interface ProbeTargetOptions {
-  /** The rates the plugin is measured at. */
+  /** The rates the plugin is measured at; with no `cells`, each at `CLAP_SWEEP_FRAMES`. */
   readonly rates: readonly number[];
+  /** The cells of the grid this plugin is measured at (`measurement-grid.ts`); each format's runs are planned from them. */
+  readonly cells?: readonly GridCell[];
+  /** Blocks of the CLAP sweep a cell; else `CLAP_SWEEP_BLOCKS`. The LV2 pass has a fixed shape. */
+  readonly blocks?: number;
+  /** The memory bound every run is held to; none (the runner's own default) when absent. */
+  readonly memory?: MemoryBound;
   /** Host extensions to offer beyond the prober's own (a CLAP run). */
   readonly offer?: readonly string[];
   /** The wall clock the runner gives one run, in seconds; else the format's declared one. */
@@ -80,7 +91,17 @@ export interface ProbeTargetOptions {
 
 /** The runner's arguments for a CLAP file; `child` is the prober the runner starts. */
 export function clapRunnerArgs(
-  options: { rates: readonly number[]; offer?: readonly string[]; wallS?: number; out: string; log: string; child: string },
+  options: {
+    rates: readonly number[];
+    quantum?: number;
+    blocks?: number;
+    dataBytes?: number;
+    offer?: readonly string[];
+    wallS?: number;
+    out: string;
+    log: string;
+    child: string;
+  },
   binaryPath: string,
   id: string | undefined,
 ): string[] {
@@ -91,12 +112,13 @@ export function clapRunnerArgs(
     "--rates",
     options.rates.join(","),
     "--quantum",
-    String(CLAP_SWEEP_FRAMES),
+    String(options.quantum ?? CLAP_SWEEP_FRAMES),
     "--blocks",
-    String(CLAP_SWEEP_BLOCKS),
+    String(options.blocks ?? CLAP_SWEEP_BLOCKS),
     ...(options.offer === undefined || options.offer.length === 0 ? [] : ["--offer", options.offer.join(",")]),
     "--wall-s",
     String(options.wallS ?? CLAP_QUALIFY_TIMEOUT_S),
+    ...(options.dataBytes === undefined ? [] : ["--data-bytes", String(options.dataBytes)]),
     "--out",
     options.out,
     "--log",
@@ -110,16 +132,20 @@ export function clapRunnerArgs(
  * The runner's arguments for an LV2 plugin, named as `lv2:<uri>` and measured by `lv2_measure`: the one this run
  * located when `child` names it, else the one the runner looks for where the packages install it.
  */
-export function lv2RunnerArgs(options: { rates: readonly number[]; wallS?: number; out: string; log: string; child?: string }, uri: string): string[] {
+export function lv2RunnerArgs(
+  options: { rates: readonly number[]; quantum?: number; dataBytes?: number; wallS?: number; out: string; log: string; child?: string },
+  uri: string,
+): string[] {
   return [
     "run",
     ...(options.child === undefined ? [] : ["--child", options.child]),
     "--rates",
     options.rates.join(","),
     "--quantum",
-    String(LV2_HOSTING_BLOCK_FRAMES),
+    String(options.quantum ?? LV2_HOSTING_BLOCK_FRAMES),
     "--wall-s",
     String(options.wallS ?? LV2_HOSTING_TIMEOUT_S),
+    ...(options.dataBytes === undefined ? [] : ["--data-bytes", String(options.dataBytes)]),
     "--out",
     options.out,
     "--log",
@@ -134,7 +160,7 @@ export interface ProbedTarget {
   readonly output: ProbeOutput;
 }
 
-function runOne(target: PluginTarget, options: ProbeTargetOptions): ProbedTarget {
+function runOne(target: PluginTarget, options: ProbeTargetOptions, invocation: RunnerInvocation): ProbedTarget {
   const env = options.env ?? process.env;
   const tool = probeRunnerTool(env);
   const work = mkdtempSync(join(tmpdir(), "plugin-qualify-probe-"));
@@ -145,20 +171,28 @@ function runOne(target: PluginTarget, options: ProbeTargetOptions): ProbedTarget
     let runEnv: Record<string, string | undefined> = { ...env };
     let expectedId: string;
     const wall = options.wallS === undefined ? {} : { wallS: options.wallS };
+    const bound = options.memory === undefined ? {} : { dataBytes: options.memory.bytes };
+    const at = { rates: invocation.rates, quantum: invocation.quantum, ...bound };
     if (target.format === "clap") {
       if (target.binaryPath === undefined) throw new ProbeRunnerError(`${target.uri}: no .clap to load; name the file or a directory holding it`);
       expectedId = target.uri.startsWith("urn:clap:") ? target.uri.slice("urn:clap:".length) : target.uri;
       const { tool: child, interposer } = clapQualifyTools(env, options.exists ?? existsSync);
-      args = clapRunnerArgs({ rates: options.rates, ...(options.offer === undefined ? {} : { offer: options.offer }), ...wall, out, log, child }, target.binaryPath, expectedId);
+      args = clapRunnerArgs(
+        { ...at, ...(options.blocks === undefined ? {} : { blocks: options.blocks }), ...(options.offer === undefined ? {} : { offer: options.offer }), ...wall, out, log, child },
+        target.binaryPath,
+        expectedId,
+      );
       runEnv = { ...env, LD_PRELOAD: interposer };
     } else {
       expectedId = target.uri;
       const child = locateNativeTool(LV2_MEASURE_TOOL_NAME, env, options.exists ?? existsSync).path;
       args = lv2RunnerArgs(
-        { rates: options.rates, ...wall, out, log, ...(child === undefined ? {} : { child }) },
+        { ...at, ...wall, out, log, ...(child === undefined ? {} : { child }) },
         target.uri,
       );
     }
+    // under a bound, the interposer ends the child the moment an allocation is refused, with the status that names it
+    if (options.memory !== undefined) runEnv = { ...runEnv, [MEMORY_BOUND_EXIT_ENV]: "1" };
     const r = (options.run ?? spawnRunner)(tool, args, runEnv);
     if (r.error !== undefined) {
       throw new ProbeRunnerError(`cannot run the probe runner "${tool}" (${r.error.message}): install plugin-hostd or set ${PROBE_RUNNER_ENV} to plugin-hostd-probe`);
@@ -186,9 +220,23 @@ function runOne(target: PluginTarget, options: ProbeTargetOptions): ProbedTarget
   }
 }
 
-/** The probe output of one target, through the runner. Throws {@link ProbeRunnerError} when there is none to judge. */
+/** The cells a target is measured at: the options' cells, else every rate at `CLAP_SWEEP_FRAMES`. */
+export function targetCells(options: Pick<ProbeTargetOptions, "rates" | "cells">): readonly GridCell[] {
+  return options.cells ?? options.rates.map((rate) => ({ rate, quantum: CLAP_SWEEP_FRAMES }));
+}
+
+/**
+ * The probe output of one target over its grid, through the runner: one run per quantum (CLAP) or per cell (LV2),
+ * merged into one document (`probe-merge.ts`). Throws {@link ProbeRunnerError} when a run leaves nothing to judge.
+ */
 export function probeTarget(target: PluginTarget, options: ProbeTargetOptions): ProbedTarget {
-  return runOne(target, options);
+  const parts = runnerInvocations(target.format, targetCells(options)).map((invocation) => ({ ...runOne(target, options, invocation), invocation }));
+  try {
+    return mergeProbeRuns(parts, options.memory);
+  } catch (e) {
+    if (e instanceof ProbeMergeError) throw new ProbeRunnerError(e.message);
+    throw e;
+  }
 }
 
 /** What the pass over a run's targets produced. */
@@ -210,7 +258,16 @@ export interface ProbePass {
  */
 export function qualifyTargets(
   targets: readonly PluginTarget[],
-  options: ProbeTargetOptions & { readonly profiles?: readonly HostProfile[]; readonly admissions?: AdmissionRuns },
+  options: ProbeTargetOptions & {
+    readonly profiles?: readonly HostProfile[];
+    readonly admissions?: AdmissionRuns;
+    /** How the grid is chosen; with it, each target's cells come from the hosts it is judged under. */
+    readonly grid?: GridSpec;
+    /** The quanta of a `full` grid; else `QUALIFY_QUANTA`. */
+    readonly quanta?: readonly number[];
+    /** At most this many cells a plugin, the first of the grid's order; 0 or absent is every cell. */
+    readonly cellsPerPluginBudget?: number;
+  },
 ): ProbePass {
   const profiles = options.profiles ?? [OPENMIXER_CONSOLE_PROFILE];
   const admissionRuns = options.admissions ?? SPAWNED_ADMISSIONS;
@@ -225,6 +282,11 @@ export function qualifyTargets(
       .flatMap((id) => (id === undefined ? [] : [shippedHostProfile(id)]))
       .filter((p): p is HostProfile => p !== undefined);
     const judgedProfiles = [...profiles, ...tierProfiles.filter((t) => !profiles.some((p) => p.id === t.id))];
+    const cells =
+      options.grid === undefined
+        ? options.cells
+        : gridCells(options.grid, { rates: options.rates, quanta: options.quanta ?? QUALIFY_QUANTA, hosts: judgedProfiles.map((p) => ({ rate: p.cost.rate, quantum: p.cost.quantum })) });
+    const measured = cells === undefined ? {} : { cells: budgetCells(cells, options.cellsPerPluginBudget) };
     // an LV2 plugin is run under each measured host once, whichever probe output it is judged with
     const admissions: Record<string, HostAdmission> = {};
     if (target.format === "lv2") {
@@ -246,7 +308,7 @@ export function qualifyTargets(
     const targetRuns: { bytes: Buffer; judgment: Judgment }[] = [];
     try {
       for (const { offer, profiles: forOffer } of byOffer.values()) {
-        const probed = probeTarget(target, { ...options, offer });
+        const probed = probeTarget(target, { ...options, ...measured, offer });
         const judgment = judgeProbeOutput(probed.output, forOffer, { admissions });
         targetRuns.push({ bytes: probed.bytes, judgment });
         for (const v of judgmentVerdicts(judgment)) judged.set(v.host, v);

package/src/qualification-record.ts

@@ -21,6 +21,8 @@ import type { Judgment, RecordReason } from "./probe-judge.js";
 import { sha256Hex, PROBE_OK, type ProbeGridPoint, type ProbeOutput } from "./probe-output.js";
 import type { VoicesHeld } from "./measurement/format.js";
 import { writeFileWhole } from "./whole-file.js";
+import { HOSTING_CODE } from "./hosting-codes.js";
+import { MEASUREMENT_UNMEASURED_REASONS } from "@openmixer/omx-contract";
 
 export const QUALIFICATION_RECORD_SCHEMA = "plugin-qualify.qualification-record/1";
 
@@ -89,6 +91,44 @@ export function unmeasuredFigures(output: ProbeOutput): readonly UnmeasuredFigur
     }));
 }
 
+/** A cell's latency when the run took no figure: the contract's unmeasured word and the hosting code a reader classifies by. */
+export interface UnmeasuredLatency {
+  readonly unmeasuredReason: string;
+  readonly code: string;
+}
+
+/** A grid point as the record writes it: every cell carries `latency`, the figures or why there are none (#201). */
+export type RecordGridPoint = Omit<ProbeGridPoint, "latency"> & {
+  readonly latency: NonNullable<ProbeGridPoint["latency"]> | UnmeasuredLatency;
+};
+
+const UNMEASURED_WORDS: ReadonlySet<string> = new Set(Object.values(MEASUREMENT_UNMEASURED_REASONS).map((r) => r.word));
+const word = (key: keyof typeof MEASUREMENT_UNMEASURED_REASONS): string => MEASUREMENT_UNMEASURED_REASONS[key].word;
+
+/** The unmeasured word of a cell whose run did not end well, by its outcome code. */
+const OUTCOME_WORD: Readonly<Record<string, string>> = {
+  "probe.crashed": word("probeCrashed"),
+  "probe.timeout": word("probeTimeout"),
+  "probe.refused-instantiation": word("refusedInstantiation"),
+  "probe.non-finite-output": word("nonFiniteOutput"),
+};
+
+/**
+ * Why a cell has no latency figure. A cell whose run failed: its outcome's word. A latency step at the cell that did not
+ * end well: the child's own reason, which the runner writes first in the step's detail, when it is a contract word,
+ * else `probe-error`. No latency step at all: the run did not take the figure (`not-attempted`), as a runner or a child
+ * older than the latency arm writes it.
+ */
+export function latencyOfCell(output: ProbeOutput, point: ProbeGridPoint): RecordGridPoint["latency"] {
+  if (point.latency !== undefined) return point.latency;
+  const code = HOSTING_CODE.latencyUnmeasured;
+  if (point.outcome !== PROBE_OK) return { unmeasuredReason: OUTCOME_WORD[point.outcome] ?? word("probeError"), code };
+  const step = output.steps.find((st) => st.step === "latency" && st.rate === point.rate && (st.quantum === undefined || st.quantum === point.quantum));
+  if (step === undefined) return { unmeasuredReason: word("notAttempted"), code };
+  const said = step.detail?.split(":")[0]?.trim() ?? "";
+  return { unmeasuredReason: UNMEASURED_WORDS.has(said) ? said : word("probeError"), code: step.hostingCode ?? code };
+}
+
 /**
  * What the running history made of the measured verdicts, for one host class: the second verdict beside the measured one.
  * Written by `plugin-qualify adjust --history`; the measured `verdicts` above are never changed by it. A record without the
@@ -136,7 +176,7 @@ export interface QualificationRecord {
    * {declaredFrames, measuredFrames}, `cost`, `rtSafety`, `noteSweep`), and the figures the run could not take, named.
    */
   readonly measurements: {
-    readonly grid: readonly ProbeGridPoint[];
+    readonly grid: readonly RecordGridPoint[];
     readonly behaviour?: Readonly<Record<string, unknown>>;
     readonly unmeasured?: readonly UnmeasuredFigure[];
   };
@@ -150,6 +190,8 @@ export interface QualificationRecord {
   readonly measuredAt: string;
   /** The second verdict, per host class of the history that was read; absent until `adjust --history` has run. */
   readonly history?: readonly HistoryBlock[];
+  /** Where the record came from when it was not measured on this device: the distributed set's provenance, as the set's package wrote it. */
+  readonly provenance?: Readonly<Record<string, unknown>>;
   /** The probe output this record judged. */
   readonly probe: {
     readonly file: string;
@@ -227,7 +269,7 @@ export function buildQualificationRecord(
     tiers: tiersOf(judgment),
     cost: judgment.hosts.map((h) => ({ host: h.verdict.host, ...h.cost })),
     measurements: {
-      grid: o.grid,
+      grid: o.grid.map((p) => ({ ...p, latency: latencyOfCell(o, p) })),
       ...(o.behaviour === undefined ? {} : { behaviour: o.behaviour }),
       ...(unmeasuredFigures(o).length === 0 ? {} : { unmeasured: unmeasuredFigures(o) }),
     },

package/src/qualify-cli.ts

@@ -20,12 +20,22 @@ import { classifyPluginHosting, type QualifiedPlugin } from "./hosting-suitabili
 import type { SuitabilityRating } from "./rating.js";
 import type { PluginLoader, PluginTarget } from "./loader.js";
 import { EXIT_CODE, OUTPUT_FORMATS, QUALIFY_OPTIONS, type OutputFormat } from "./qualify-options.js";
+import { GRID_HOSTS, parseGridSpec, type GridSpec } from "./measurement-grid.js";
+import { parseMemoryLimit } from "./memory-bound.js";
 
 export interface QualifyArgs {
   readonly all: boolean;
   readonly targets: readonly string[];
   readonly rates: readonly number[];
   readonly quanta: readonly number[];
+  /** How the grid is chosen (`--grid`, `--grid-hosts`). */
+  readonly grid: GridSpec;
+  /** `--cells-per-plugin-budget`: at most this many cells a plugin; 0 is every cell. */
+  readonly cellsPerPluginBudget: number;
+  /** `--blocks`: blocks of the CLAP sweep a cell, when given. */
+  readonly blocks?: number;
+  /** `--memory-limit`: bytes, or `off`; absent is the default, half of the RAM. */
+  readonly memoryLimit?: number | "off";
   readonly out?: string;
   /** `--pins`: where the pins file is written. */
   readonly pins?: string;
@@ -79,6 +89,23 @@ export function parseQualifyArgs(argv: readonly string[]): ParseResult {
   if (rates === undefined) return { ok: false, error: "--rates must be positive integers, comma-separated" };
   const quanta = numberList((seen.get("--quanta") as string | undefined) ?? defaultOf("--quanta"));
   if (quanta === undefined) return { ok: false, error: "--quanta must be positive integers, comma-separated" };
+  const gridText = seen.get("--grid") as string | undefined;
+  if (gridText !== undefined && seen.has("--grid-hosts") && gridText.trim() !== "hosts") return { ok: false, error: "--grid-hosts and --grid name two grids; give one" };
+  const grid = gridText === undefined ? { ok: true as const, spec: GRID_HOSTS } : parseGridSpec(gridText);
+  if (!grid.ok) return { ok: false, error: `--grid: ${grid.error}` };
+  const count = (flag: string, min: number): number | undefined => {
+    const v = seen.get(flag) as string | undefined;
+    if (v === undefined) return undefined;
+    const n = Number(v);
+    return /^\d+$/.test(v.trim()) && Number.isSafeInteger(n) && n >= min ? n : NaN;
+  };
+  const cellsPerPluginBudget = count("--cells-per-plugin-budget", 0) ?? 0;
+  if (Number.isNaN(cellsPerPluginBudget)) return { ok: false, error: "--cells-per-plugin-budget must be a whole number, 0 or more" };
+  const blocks = count("--blocks", 1);
+  if (blocks !== undefined && Number.isNaN(blocks)) return { ok: false, error: "--blocks must be a whole number, 1 or more" };
+  const memoryText = seen.get("--memory-limit") as string | undefined;
+  const memory = memoryText === undefined ? undefined : parseMemoryLimit(memoryText);
+  if (memory !== undefined && !memory.ok) return { ok: false, error: `--memory-limit: ${memory.error}` };
   const format = (seen.get("--format") as string | undefined) ?? defaultOf("--format");
   if (!(OUTPUT_FORMATS as readonly string[]).includes(format)) {
     return { ok: false, error: `--format must be one of ${OUTPUT_FORMATS.join(", ")}` };
@@ -94,6 +121,10 @@ export function parseQualifyArgs(argv: readonly string[]): ParseResult {
       targets,
       rates,
       quanta,
+      grid: grid.spec,
+      cellsPerPluginBudget,
+      ...(blocks === undefined ? {} : { blocks }),
+      ...(memory === undefined ? {} : { memoryLimit: memory.bytes }),
       ...(out === undefined ? {} : { out }),
       ...(pins === undefined ? {} : { pins }),
       format: format as OutputFormat,
@@ -108,7 +139,7 @@ export function helpText(): string {
   const rows = QUALIFY_OPTIONS.map((o) => {
     const left = o.value === undefined ? o.flag : `${o.flag} <${o.value}>`;
     const def = o.default === undefined ? "" : ` (default: ${o.default})`;
-    return `  ${left.padEnd(28)}${o.describe}${def}`;
+    return `  ${left.padEnd(36)}${o.describe}${def}`;
   });
   return [
     "plugin-qualify — qualify audio plugins for real-time hosting",

package/src/qualify-declarations.ts

@@ -133,6 +133,23 @@ export const NATIVE_TOOL_DIRS: readonly string[] = ["/usr/libexec/plugin-qualify
 export const CLAP_SWEEP_BLOCKS = 512;
 export const CLAP_SWEEP_FRAMES = 512;
 
+/**
+ * The measurement grid (#193). Every record written before the grid was measured at `GRID_CONTINUITY_QUANTUM`, so
+ * the hosts' grid also takes that quantum at every rate: a new record stays comparable with an old one.
+ */
+export const GRID_CONTINUITY_QUANTUM = CLAP_SWEEP_FRAMES;
+
+/**
+ * The memory bound of a run (#193). The runner's data limit (`--data-bytes`, `RLIMIT_DATA` on the measuring child) is
+ * `MEMORY_LIMIT_RAM_FRACTION` of the machine's RAM unless `--memory-limit` says otherwise. With `MEMORY_BOUND_EXIT_ENV`
+ * set, the interposer ends the child with `MEMORY_BOUND_EXIT_STATUS` the moment an allocation is refused, so a run
+ * that outgrew the bound is told from a crash by an exit status, never by words; `native/rt-interpose.h` holds the
+ * same two (src/memory-bound.test.ts holds them together).
+ */
+export const MEMORY_LIMIT_RAM_FRACTION = 0.5;
+export const MEMORY_BOUND_EXIT_ENV = "OMX_RT_OOM_EXIT";
+export const MEMORY_BOUND_EXIT_STATUS = 121;
+
 /** Seconds one `.clap` may take to qualify at every rate before the prober is killed. */
 export const CLAP_QUALIFY_TIMEOUT_S = 120;
 

package/src/qualify-options.ts

@@ -4,7 +4,7 @@
  * The `plugin-qualify` CLI's OPTIONS, declared once. The parser, `--help` and the tests all read this table; every default is a declaration
  * from `declarations.ts`, never a literal here.
  */
-import { QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
+import { CLAP_SWEEP_BLOCKS, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
 import { HOST_PROFILES, OPENMIXER_CONSOLE_PROFILE } from "./host-profiles.js";
 
 /** How the result is printed. `text` = plugin, verdict, the reason that decided it. */
@@ -35,6 +35,16 @@ export const QUALIFY_OPTIONS: readonly QualifyOption[] = [
   { flag: "--all", describe: "every LV2 plugin on the LV2 search path and every CLAP plugin on the CLAP search path; with --out, one record each" },
   { flag: "--rates", value: "list", describe: "sample rates to measure at, comma-separated", default: QUALIFY_RATES.join(",") },
   { flag: "--quanta", value: "list", describe: "quanta (frames) to rate cost at, comma-separated", default: QUALIFY_QUANTA.join(",") },
+  {
+    flag: "--grid",
+    value: "hosts|full|cells",
+    describe: "the cells measured: hosts = each judged host's rate/quantum, then 512 at every rate; full = every rate at every quantum; or <rate>/<quantum>,...",
+    default: "hosts",
+  },
+  { flag: "--grid-hosts", describe: "the same as --grid hosts" },
+  { flag: "--cells-per-plugin-budget", value: "n", describe: "measure at most the first n cells of the grid a plugin (0: every cell)", default: "0" },
+  { flag: "--blocks", value: "n", describe: "blocks of the CLAP sweep a cell (the LV2 pass has a fixed shape)", default: String(CLAP_SWEEP_BLOCKS) },
+  { flag: "--memory-limit", value: "size|off", describe: "the memory bound of every run (bytes, or with K, M, G); past it a cell is probe.oom", default: "half of the RAM" },
   { flag: "--out", value: "dir", describe: "write one interchange JSON document per run into this directory" },
   { flag: "--format", value: OUTPUT_FORMATS.join("|"), describe: "json = the verdicts as JSON; text = plugin, verdict, deciding reason", default: "text" },
   { flag: "--isolate", describe: "a fresh worker per plugin, so a crash or hang takes only that plugin" },