@openmixer/plugin-qualify 0.16.0 → 0.17.0

package/src/qualify-sync.ts

@@ -24,6 +24,7 @@ import { homedir, tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "node:path";
 
 import { clapQualifyTools } from "./clap-qualify.js";
+import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, matchDistributed, readBoardModel, readDistributedLayer, type DistributedLayer } from "./distributed-records.js";
 import { clapSearchPath, parseClapScan } from "./clap-loader.js";
 import { HOSTING_CODE } from "./hosting-codes.js";
 import { shippedHostProfile, type HostProfile } from "./host-profiles.js";
@@ -31,8 +32,10 @@ import { lv2BundlesOnPath, manifestPlugins } from "./lv2-loader.js";
 import { judgmentPinEntries, pinsFile, type PinEntry } from "./pins-file.js";
 import type { Judgment } from "./probe-judge.js";
 import { qualifyTargets } from "./probe-runner.js";
+import { GRID_HOSTS, parseGridSpec, type GridSpec } from "./measurement-grid.js";
+import { memoryBoundOf, parseMemoryLimit, type MemoryBound } from "./memory-bound.js";
 import { writeQualification, type NotMeasured } from "./qualification-record.js";
-import { DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM } from "./qualify-declarations.js";
+import { DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
 import { toolsDir } from "./tool-paths.js";
 import { writeFileWhole } from "./whole-file.js";
 
@@ -64,11 +67,16 @@ export const SYNC_SETTINGS: readonly SyncSetting[] = [
   { key: "qualification_dir", meaning: "where the qualification records go", default: "/var/lib/omx/qualification (system), else $XDG_STATE_HOME/omx/qualification" },
   { key: "descriptor_dir", meaning: "where the scanned descriptors go", default: `${DESCRIPTOR_DIR_SYSTEM} (system), else $XDG_STATE_HOME/${DESCRIPTOR_DIR_STATE_SUBDIR}` },
   { key: "state_file", meaning: "what was qualified, from which bytes, and how it ended", default: "/var/lib/omx/qualify-sync.json (system), else $XDG_STATE_HOME/omx/qualify-sync.json" },
-  { key: "budget_s", meaning: "the wall clock one plugin gets, in seconds; past it the plugin is recorded as hung", default: "60" },
+  { key: "budget_s", meaning: "the wall clock one run gets (one cell of an LV2 plugin, one quantum of a CLAP one), in seconds; past it the plugin is recorded as hung", default: "60" },
   { key: "host_profile", meaning: "the host the verdict is for (a shipped profile id)", default: "omx-clap-host" },
+  { key: "grid", meaning: "the cells measured: hosts (the host's and the console tiers' rate/quantum, then 512 at every rate), full, or <rate>/<quantum>,...", default: "hosts" },
+  { key: "cells_per_plugin_budget", meaning: "measure at most the first n cells of the grid a plugin; 0 is every cell", default: "0" },
+  { key: "memory_limit", meaning: "the memory bound of every run: bytes, or with K, M, G; off for none; past it a cell is probe.oom", default: "half of the RAM" },
   { key: "hostd_port", meaning: "plugin-hostd's command port the new pins are sent to; 0 sends none", default: "5555" },
   { key: "hostd_feedback_port", meaning: "plugin-hostd's feedback port, connected before the command port; 0 is the command port's next", default: "0" },
   { key: "settle_s", meaning: "wait until the plugin directories have been quiet this long before a run", default: "0" },
+  { key: "host_class", meaning: "the machine class whose distributed records are read (a directory name)", default: "the board's own from the device tree (rpi5 on a Raspberry Pi 5), else none" },
+  { key: "distributed_dir", meaning: "the directories of distributed records, colon-separated, each holding <host class>/<set>/", default: `${DISTRIBUTED_DIR_SYSTEM} (system), else $XDG_DATA_HOME/omx/qualification and each $XDG_DATA_DIRS/omx/qualification` },
 ];
 
 export interface SyncSettings {
@@ -80,9 +88,19 @@ export interface SyncSettings {
   readonly stateFile: string;
   readonly budgetS: number;
   readonly hostProfile: HostProfile;
+  /** The cells measured, and at most how many a plugin (0: every cell). */
+  readonly grid: GridSpec;
+  readonly cellsPerPluginBudget: number;
+  /** The memory bound of every run; `undefined` is `off`. */
+  readonly memory?: MemoryBound;
   readonly hostdPort: number;
   readonly hostdFeedbackPort: number;
   readonly settleS: number;
+  /** The machine class the distributed records are read for; none, and the layer is not read. */
+  readonly hostClass?: string;
+  readonly distributedDirs: readonly string[];
+  /** False under `--no-distributed`. */
+  readonly distributed: boolean;
 }
 
 /** A settings file read: one `key value` per line, `#` starts a comment; a key alone is the empty value. An unknown key is refused by line. */
@@ -105,6 +123,7 @@ export interface SyncArgs {
   readonly config?: string;
   readonly system: boolean;
   readonly force: boolean;
+  readonly noDistributed: boolean;
   readonly help: boolean;
   /** The settings given as flags, by key. */
   readonly values: ReadonlyMap<string, string>;
@@ -118,11 +137,13 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
   let config: string | undefined;
   let system = false;
   let force = false;
+  let noDistributed = false;
   let help = false;
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i]!;
     if (a === "--system") system = true;
     else if (a === "--force") force = true;
+    else if (a === "--no-distributed") noDistributed = true;
     else if (a === "--help") help = true;
     else if (a === "--config" || byFlag.has(a)) {
       const v = argv[++i];
@@ -131,18 +152,19 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
       else values.set(byFlag.get(a)!, v);
     } else return { ok: false, error: `unknown argument ${a}` };
   }
-  return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, help, values } };
+  return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, help, values } };
 }
 
 export function syncHelpText(): string {
-  const rows = SYNC_SETTINGS.map((s) => `  ${`${flagOf(s.key)} <value>`.padEnd(28)}${s.meaning}${s.default === "" ? "" : ` (default: ${s.default})`}`);
+  const rows = SYNC_SETTINGS.map((s) => `  ${`${flagOf(s.key)} <value>`.padEnd(36)}${s.meaning}${s.default === "" ? "" : ` (default: ${s.default})`}`);
   return [
     "plugin-qualify-sync: qualify every plugin on the search paths once, and again when its files change",
     "",
-    `  --config <file>             the settings file (default: ${SYNC_CONF}; a missing file is the defaults)`,
-    "  --system                    a system service's directories, under /var/lib/omx",
-    "  --force                     qualify every plugin again, recorded or not",
-    "  --help                      this text",
+    `  ${"--config <file>".padEnd(36)}the settings file (default: ${SYNC_CONF}; a missing file is the defaults)`,
+    "  --system                            a system service's directories, under /var/lib/omx",
+    "  --force                             qualify every plugin again, recorded or not; no distributed record is read",
+    "  --no-distributed                    read no distributed record: every plugin not recorded is measured",
+    "  --help                              this text",
     ...rows,
     "",
     "every setting is also a line of the settings file, its key as the first word",
@@ -159,6 +181,7 @@ export function resolveSyncSettings(
   args: SyncArgs,
   env: Readonly<Record<string, string | undefined>>,
   home: string = homedir(),
+  boardModel: () => string | undefined = readBoardModel,
 ): { ok: true; settings: SyncSettings } | { ok: false; error: string } {
   const get = (key: string): string | undefined => args.values.get(key) ?? file.get(key);
   const state = env["XDG_STATE_HOME"] !== undefined && env["XDG_STATE_HOME"] !== "" ? env["XDG_STATE_HOME"] : join(home, ".local", "state");
@@ -182,10 +205,31 @@ export function resolveSyncSettings(
   if (hostdPort === undefined || !Number.isInteger(hostdPort) || hostdPort > 65535) return { ok: false, error: "hostd_port must be a port number, or 0" };
   if (feedbackPort === undefined || !Number.isInteger(feedbackPort) || feedbackPort > 65535) return { ok: false, error: "hostd_feedback_port must be a port number, or 0" };
   if (settleS === undefined) return { ok: false, error: "settle_s must be a number of seconds" };
+  const gridText = get("grid");
+  const grid = gridText === undefined || gridText === "" ? { ok: true as const, spec: GRID_HOSTS } : parseGridSpec(gridText);
+  if (!grid.ok) return { ok: false, error: `grid: ${grid.error}` };
+  const cellsPerPluginBudget = number("cells_per_plugin_budget", 0);
+  if (cellsPerPluginBudget === undefined || !Number.isInteger(cellsPerPluginBudget)) return { ok: false, error: "cells_per_plugin_budget must be a whole number, 0 or more" };
+  const memoryText = get("memory_limit");
+  const memoryLimit = memoryText === undefined || memoryText === "" ? undefined : parseMemoryLimit(memoryText);
+  if (memoryLimit !== undefined && !memoryLimit.ok) return { ok: false, error: `memory_limit: ${memoryLimit.error}` };
+  const memory = memoryBoundOf(memoryLimit?.bytes);
   const profileId = get("host_profile") || "omx-clap-host";
   const hostProfile = shippedHostProfile(profileId);
   if (hostProfile === undefined) return { ok: false, error: `host_profile ${profileId} is not a shipped host profile` };
   const once = (dirs: readonly string[]): string[] => [...new Set(dirs.map((d) => resolve(d)))];
+  const dataHome = env["XDG_DATA_HOME"] !== undefined && env["XDG_DATA_HOME"] !== "" ? env["XDG_DATA_HOME"] : join(home, ".local", "share");
+  const dataDirs = dirList(env["XDG_DATA_DIRS"] ?? "").length > 0 ? dirList(env["XDG_DATA_DIRS"]!) : ["/usr/local/share", "/usr/share"];
+  const distributedNamed = get("distributed_dir");
+  const distributedDirs =
+    distributedNamed !== undefined && distributedNamed !== ""
+      ? dirList(distributedNamed)
+      : args.system
+        ? [DISTRIBUTED_DIR_SYSTEM]
+        : [dataHome, ...dataDirs].map((d) => join(d, "omx", "qualification"));
+  const classNamed = get("host_class");
+  const hostClass = classNamed !== undefined && classNamed !== "" ? classNamed : hostClassOfBoard(boardModel());
+  if (hostClass !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._+-]*$/.test(hostClass)) return { ok: false, error: `host_class ${hostClass} is not a directory name` };
   return {
     ok: true,
     settings: {
@@ -197,9 +241,15 @@ export function resolveSyncSettings(
       stateFile: place("state_file", "/var/lib/omx/qualify-sync.json", join("omx", "qualify-sync.json")),
       budgetS,
       hostProfile,
+      grid: grid.spec,
+      cellsPerPluginBudget,
+      ...(memory === undefined ? {} : { memory }),
       hostdPort,
       hostdFeedbackPort: feedbackPort === 0 && hostdPort !== 0 ? hostdPort + 1 : feedbackPort,
       settleS,
+      ...(hostClass === undefined ? {} : { hostClass }),
+      distributedDirs: once(distributedDirs),
+      distributed: !args.noDistributed,
     },
   };
 }
@@ -390,6 +440,8 @@ export interface SyncEntry {
   readonly plugins: readonly SyncPlugin[];
   readonly qualifiedAt: string;
   readonly seconds: number;
+  /** The distributed set the plugins were taken from, when they were not measured here. */
+  readonly distributed?: string;
 }
 
 export interface SyncState {
@@ -484,7 +536,17 @@ function judgedPlugins(ctx: SyncContext, runs: readonly { bytes: Buffer; judgmen
 }
 
 function runPass(ctx: SyncContext, target: Parameters<typeof qualifyTargets>[0][number], env: Readonly<Record<string, string | undefined>>) {
-  return qualifyTargets([target], { rates: [ctx.settings.hostProfile.cost.rate], profiles: [ctx.settings.hostProfile], wallS: ctx.settings.budgetS, env });
+  const s = ctx.settings;
+  return qualifyTargets([target], {
+    rates: QUALIFY_RATES,
+    quanta: QUALIFY_QUANTA,
+    grid: s.grid,
+    cellsPerPluginBudget: s.cellsPerPluginBudget,
+    ...(s.memory === undefined ? {} : { memory: s.memory }),
+    profiles: [s.hostProfile],
+    wallS: s.budgetS,
+    env,
+  });
 }
 
 /** The descriptors of a scanned `.clap`, through the one descriptor writer; a failure is named and the run goes on. */
@@ -566,6 +628,66 @@ export function qualifyItem(ctx: SyncContext, item: SyncItem, files: readonly Fi
   return { format: item.format, root: item.root, files: all, plugins, qualifiedAt: new Date(started).toISOString(), seconds: (Date.now() - started) / 1000 };
 }
 
+// --- the distributed records ----------------------------------------------------------------------------------------
+
+/** The layer a run reads: none under `--force` or `--no-distributed`, or without a host class. */
+function distributedLayerOf(settings: SyncSettings, force: boolean, log: (line: string) => void): DistributedLayer {
+  if (force || !settings.distributed) return EMPTY_LAYER;
+  if (settings.hostClass === undefined) {
+    log("distributed: no host class (host_class unset, and the board names none); no distributed record is read");
+    return EMPTY_LAYER;
+  }
+  const layer = readDistributedLayer(settings.distributedDirs, settings.hostClass, log);
+  if (layer.records > 0) log(`distributed: ${String(layer.records)} records of ${layer.sets.join(", ")} for ${settings.hostClass}`);
+  return layer;
+}
+
+/**
+ * The item's entry taken from the distributed records, or `undefined` when it is to be measured: no record finds it,
+ * or the ones that do were not taken, each reason said. Its descriptors are written as a measured item's are: a
+ * `.clap` is scanned for them (the scan lists the plugins and measures nothing), an LV2 plugin joins the run's
+ * descriptor pass.
+ */
+function takeDistributed(ctx: SyncContext, layer: DistributedLayer, item: SyncItem, files: readonly FileDigest[]): SyncEntry | undefined {
+  if (layer.records === 0) return undefined;
+  const started = Date.now();
+  const host = ctx.settings.hostProfile.id;
+  const match = matchDistributed(layer, item, host, (paths) => digestFiles(item.root, paths, files));
+  if (match.status === "none") return undefined;
+  if (match.status === "ignored") {
+    for (const line of match.lines) ctx.log(line);
+    return undefined;
+  }
+  let plugins: SyncPlugin[];
+  try {
+    plugins = applyDistributed(ctx.settings.qualificationDir, item, match, host);
+  } catch (e) {
+    ctx.log(`distributed: ${distributedUri(item, match.records[0]!)}: ${(e as Error).message}; measured here`);
+    return undefined;
+  }
+  if (item.format === "clap") scanClapDescriptors(ctx, item);
+  const all = digestFiles(item.root, [...item.files, ...match.files], files) ?? [...files];
+  return { format: item.format, root: item.root, files: all, plugins, qualifiedAt: new Date(started).toISOString(), seconds: (Date.now() - started) / 1000, distributed: match.set };
+}
+
+/** The descriptors of a `.clap` taken from the layer: the scanner lists its plugins, the descriptor writer writes them. */
+function scanClapDescriptors(ctx: SyncContext, item: SyncItem): void {
+  const file = join(item.root, item.files[0]!);
+  let tool: string;
+  try {
+    tool = clapQualifyTools(ctx.env, existsSync).tool;
+  } catch (e) {
+    ctx.log(`${file}: descriptors not written: ${(e as Error).message}`);
+    return;
+  }
+  const r = spawnSync(tool, ["--json", file], { encoding: "utf8", timeout: Math.max(ctx.settings.budgetS, 35) * 1000, killSignal: "SIGKILL", maxBuffer: 64 << 20 });
+  if (r.error !== undefined || r.status !== 0) {
+    ctx.log(`${file}: descriptors not written: the scan ended ${r.error?.message ?? `with exit ${String(r.status)}`}`);
+    return;
+  }
+  writeClapDescriptors(ctx, r.stdout);
+}
+
 // --- the outputs ------------------------------------------------------------------------------------------------------
 
 /** The pin file's text: every whole pin of the state, the worse verdict kept for a uri pinned twice. */
@@ -598,17 +720,32 @@ export function recordStem(recordPath: string): string | undefined {
   return /^(.*-[0-9a-f]{12})\.(?:[^.]+\.(?:qualification|probe)|unmeasured)\.json$/.exec(basename(recordPath))?.[1];
 }
 
+/**
+ * A stem's id and binary hash, its version segment dropped: what a record and its descriptor share even when their
+ * version segments differ (a CLAP record is keyed `@unversioned` while its descriptor carries the plugin's declared
+ * version). Undefined for a name that is not of the `<id>@<version>-<hash>` shape.
+ */
+function idAndHash(stem: string): string | undefined {
+  const m = /^(.+)@.+-([0-9a-f]{12})$/.exec(stem);
+  return m === null ? undefined : `${m[1]}\0${m[2]}`;
+}
+
 /**
  * What a plugin the sync no longer finds, or finds as other bytes, leaves behind: its qualification record, the probe
  * output copied beside it and its descriptor. A stem the state before held and the state after does not hold is
- * stale; every file of that stem in the two directories is removed. A record another plugin of the state still names
- * (two copies of one binary) is kept. Returns the files removed.
+ * stale; every record file of that stem is removed, and so is a descriptor of its id and binary hash whichever
+ * version suffix it carries — unless that id and hash is still kept under another stem (two copies of one binary, or
+ * a record and a descriptor the sync wrote at different version suffixes for the same plugin). A record another
+ * plugin of the state still names (two copies of one binary) is kept, as is a descriptor the sync did not write.
+ * Returns the files removed.
  */
 export function pruneStale(settings: Pick<SyncSettings, "qualificationDir" | "descriptorDir">, before: SyncState, after: SyncState): string[] {
   const stems = (state: SyncState): Set<string> =>
     new Set(Object.values(state.entries).flatMap((e) => e.plugins.flatMap((p) => (p.record === undefined ? [] : [recordStem(p.record) ?? ""]))).filter((x) => x !== ""));
   const kept = stems(after);
   const stale = [...stems(before)].filter((stem) => !kept.has(stem));
+  const keptIdHash = new Set([...kept].map(idAndHash).filter((x): x is string => x !== undefined));
+  const staleIdHash = new Set(stale.map(idAndHash).filter((x): x is string => x !== undefined));
   const removed: string[] = [];
   const drop = (dir: string, file: string): void => {
     try {
@@ -624,9 +761,17 @@ export function pruneStale(settings: Pick<SyncSettings, "qualificationDir" | "de
   } catch {
     // no directory, no record
   }
-  for (const stem of stale) {
-    for (const f of records) if (f.startsWith(`${stem}.`) && recordStem(f) === stem) drop(settings.qualificationDir, f);
-    drop(settings.descriptorDir, `${stem}.json`);
+  for (const stem of stale) for (const f of records) if (f.startsWith(`${stem}.`) && recordStem(f) === stem) drop(settings.qualificationDir, f);
+  let descriptors: string[] = [];
+  try {
+    descriptors = readdirSync(settings.descriptorDir);
+  } catch {
+    // no directory, no descriptor
+  }
+  for (const f of descriptors) {
+    if (!f.endsWith(".json")) continue;
+    const key = idAndHash(f.slice(0, -".json".length));
+    if (key !== undefined && staleIdHash.has(key) && !keptIdHash.has(key)) drop(settings.descriptorDir, f);
   }
   return removed;
 }
@@ -731,13 +876,15 @@ export interface SyncSummary {
   readonly crashed: number;
   readonly hung: number;
   readonly unmeasured: number;
+  /** The plugins taken from the distributed records, not measured. */
+  readonly distributed: number;
   readonly seconds: number;
   /** The pins a running daemon took, when one answered. */
   readonly sent?: number;
 }
 
 export function summaryLine(s: SyncSummary): string {
-  return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, ${s.seconds.toFixed(1)} s`;
+  return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}, ${s.seconds.toFixed(1)} s`;
 }
 
 /** The lock of one state file: a second sync on it while one runs returns at once. A lock whose process is gone is taken over. */
@@ -791,7 +938,8 @@ export async function runSync(settings: SyncSettings, options: SyncRunOptions =
     const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
     await waitQuiet(dirs, settings.settleS, log);
     const ctx: SyncContext = { settings, env, log };
-    const count = { qualified: 0, skipped: 0, refused: 0, crashed: 0, hung: 0, unmeasured: 0 };
+    const count = { qualified: 0, skipped: 0, refused: 0, crashed: 0, hung: 0, unmeasured: 0, distributed: 0 };
+    const layer = distributedLayerOf(settings, options.force === true, log);
     let before: string;
     try {
       before = readFileSync(settings.pinFile, "utf8");
@@ -823,6 +971,17 @@ export async function runSync(settings: SyncSettings, options: SyncRunOptions =
           log(`${item.key}: a file of it cannot be read; skipped this run`);
           continue;
         }
+        const taken = takeDistributed(ctx, layer, item, files);
+        if (taken !== undefined) {
+          entries[item.key] = taken;
+          if (item.format === "lv2") lv2Pinned.push(...taken.plugins.filter((p) => p.pin !== undefined).map((p) => p.uri));
+          count.distributed += taken.plugins.length;
+          for (const p of taken.plugins) log(`distributed: ${p.uri}: applied (${taken.distributed!})`);
+          state = { schema: SYNC_STATE_SCHEMA, entries };
+          writeFileWhole(settings.stateFile, `${JSON.stringify(state, null, 1)}\n`);
+          writeFileWhole(settings.pinFile, syncPinsText(state));
+          continue;
+        }
         const entry = qualifyItem(ctx, item, files);
         entries[item.key] = entry;
         if (item.format === "lv2") lv2Pinned.push(...entry.plugins.filter((p) => p.pin !== undefined).map((p) => p.uri));