@openmixer/plugin-qualify 0.17.5 → 0.18.1

package/dist/qualify-sync.js

@@ -18,10 +18,10 @@
  */
 import { spawn, spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
-import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs";
+import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
 import { createConnection } from "node:net";
 import { homedir, tmpdir } from "node:os";
-import { basename, dirname, join, resolve } from "node:path";
+import { basename, dirname, join, resolve, sep } from "node:path";
 import { clapQualifyTools } from "./clap-qualify.js";
 import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, matchDistributed, readBoardModel, readDistributedLayer } from "./distributed-records.js";
 import { clapSearchPath, parseClapScan } from "./clap-loader.js";
@@ -32,9 +32,11 @@ import { judgmentPinEntries, pinsFile } from "./pins-file.js";
 import { hostdPinFiles } from "./hostd-pin-files.js";
 import { qualifyTargets } from "./probe-runner.js";
 import { GRID_FULL, parseGridSpec } from "./measurement-grid.js";
+import { adapterPass } from "./qualify-adapter-pass.js";
+import { lv2World, rejudgeState, memoryPass } from "./qualify-rejudge.js";
 import { memoryBoundOf, parseMemoryLimit } from "./memory-bound.js";
 import { writeQualification } from "./qualification-record.js";
-import { DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
+import { DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, MEMORY_PASS_PROBE_NOTES, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
 import { toolsDir } from "./tool-paths.js";
 import { writeFileWhole } from "./whole-file.js";
 // --- the settings -----------------------------------------------------------------------------------------------------
@@ -59,6 +61,7 @@ export const SYNC_SETTINGS = [
     { key: "grid", meaning: "the cells measured: full (every rate at every block size, 16 cells), hosts (the fast option: the host's and the console tiers' rate/quantum, then 512 at every rate; also --grid-hosts), or <rate>/<quantum>,...", default: "full" },
     { key: "cells_per_plugin_budget", meaning: "measure at most the first n cells of the grid a plugin; 0 is every cell", default: "0" },
     { key: "memory_limit", meaning: "the memory bound of every run: bytes, or with K, M, G; off for none; past it a cell is probe.oom", default: "half of the RAM" },
+    { key: "memory_notes", meaning: "the notes an instrument that declares no voice capacity holds in a --memory-only run, 1 to 128 (basis probe)", default: String(MEMORY_PASS_PROBE_NOTES) },
     { key: "hostd_port", meaning: "plugin-hostd's command port the new pins are sent to; 0 sends none", default: "5555" },
     { key: "hostd_feedback_port", meaning: "plugin-hostd's feedback port, connected before the command port; 0 is the command port's next", default: "0" },
     { key: "settle_s", meaning: "wait until the plugin directories have been quiet this long before a run", default: "0" },
@@ -92,6 +95,9 @@ export function parseSyncArgs(argv) {
     let noDistributed = false;
     let noWait = false;
     let refreshPinsOnly = false;
+    let rejudge = false;
+    let memoryOnly = false;
+    let adapterOnly = false;
     let help = false;
     for (let i = 0; i < argv.length; i++) {
         const a = argv[i];
@@ -105,6 +111,12 @@ export function parseSyncArgs(argv) {
             noWait = true;
         else if (a === "--refresh-pins")
             refreshPinsOnly = true;
+        else if (a === "--rejudge")
+            rejudge = true;
+        else if (a === "--memory-only")
+            memoryOnly = true;
+        else if (a === "--adapter-only")
+            adapterOnly = true;
         else if (a === "--help")
             help = true;
         else if (a === "--grid-hosts")
@@ -121,7 +133,11 @@ export function parseSyncArgs(argv) {
         else
             return { ok: false, error: `unknown argument ${a}` };
     }
-    return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, help, values } };
+    if (rejudge && (memoryOnly || adapterOnly || force || refreshPinsOnly))
+        return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --force and --refresh-pins" };
+    if ([memoryOnly, adapterOnly, refreshPinsOnly].filter(Boolean).length > 1)
+        return { ok: false, error: "--memory-only, --adapter-only and --refresh-pins are different runs: one at a time" };
+    return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, help, values } };
 }
 export function syncHelpText() {
     const rows = SYNC_SETTINGS.map((s) => `  ${`${flagOf(s.key)} <value>`.padEnd(36)}${s.meaning}${s.default === "" ? "" : ` (default: ${s.default})`}`);
@@ -134,6 +150,9 @@ export function syncHelpText() {
         "  --no-wait                           a run in progress is not waited for: exit 0, and the holder runs once more at its end",
         "  --no-distributed                    read no distributed record: every plugin not recorded is measured",
         "  --refresh-pins                      only bring every pin's files up to the ones plugin-hostd names, measuring nothing",
+        "  --rejudge                           derive every record's verdicts, tiers and pin again from the figures it holds; measures nothing",
+        "  --memory-only                       measure only the footprint of each recorded plugin at the Pi's cells and merge it; with --force, those already merged too",
+        "  --adapter-only                      run each LV2 plugin only under omx-clap-host's LV2 adapter at the Pi's cells, on private JACK servers, and merge it; with --force, those already done too",
         "  --grid-hosts                        the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
         "  --help                              this text",
         ...rows,
@@ -184,6 +203,9 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
     if (memoryLimit !== undefined && !memoryLimit.ok)
         return { ok: false, error: `memory_limit: ${memoryLimit.error}` };
     const memory = memoryBoundOf(memoryLimit?.bytes);
+    const memoryNotes = number("memory_notes", 1);
+    if (memoryNotes === undefined || !Number.isInteger(memoryNotes) || memoryNotes > 128)
+        return { ok: false, error: "memory_notes must be a whole number of notes, 1 to 128" };
     const profileId = get("host_profile") || "omx-clap-host";
     const hostProfile = shippedHostProfile(profileId);
     if (hostProfile === undefined)
@@ -215,6 +237,7 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
             grid: grid.spec,
             cellsPerPluginBudget,
             ...(memory === undefined ? {} : { memory }),
+            memoryNotes,
             hostdPort,
             hostdFeedbackPort: feedbackPort === 0 && hostdPort !== 0 ? hostdPort + 1 : feedbackPort,
             settleS,
@@ -224,6 +247,29 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
         },
     };
 }
+/** lilv itself, one world over the whole search path (tools/lv2_resolve.py), as every host built on it resolves (#199). */
+export const LILV_RESOLVE = (dirs) => {
+    const r = spawnSync("python3", [join(toolsDir(), "lv2_resolve.py")], {
+        encoding: "utf8", timeout: 120_000, killSignal: "SIGKILL", maxBuffer: 64 << 20, env: { ...process.env, LV2_PATH: dirs.join(":") },
+    });
+    if (r.error !== undefined || r.status !== 0)
+        return { problem: (r.stderr ?? "").trim().split("\n").pop() || (r.error?.message ?? `exit ${String(r.status)}`) };
+    try {
+        return JSON.parse(r.stdout).plugins;
+    }
+    catch (e) {
+        return { problem: `lv2_resolve.py printed no JSON: ${e.message}` };
+    }
+};
+/** Whether `path` exists and is, through its links, the file `target` is. */
+function linksTo(path, target) {
+    try {
+        return realpathSync(path) === realpathSync(target);
+    }
+    catch {
+        return false;
+    }
+}
 const CLAP_SUFFIX = ".clap";
 /** Every `.clap` under a directory, its subdirectories included, in name order; a directory reached twice is walked once. */
 function clapFiles(dir, seen, problems) {
@@ -270,9 +316,10 @@ function clapFiles(dir, seen, problems) {
  * LV2 directories that declares it, as plugin-hostd finds it. A directory that cannot be read is a problem named, and
  * the others are still walked.
  */
-export function discoverSyncItems(settings) {
+export function discoverSyncItems(settings, lv2Resolve) {
     const items = [];
     const problems = [];
+    const notes = [];
     const seen = new Set();
     const clap = new Set();
     for (const dir of settings.clapDirs) {
@@ -284,6 +331,14 @@ export function discoverSyncItems(settings) {
             items.push({ key: `clap:${path}`, format: "clap", root: dirname(path), files: [basename(path)] });
         }
     }
+    // the bundle lilv takes for each plugin uri; without lilv, the first bundle found (and the log says so)
+    const resolved = lv2Resolve === undefined || settings.lv2Dirs.length === 0 ? undefined : lv2Resolve(settings.lv2Dirs);
+    const taken = new Map();
+    if (resolved !== undefined && "problem" in resolved)
+        notes.push(`lilv could not resolve the LV2 plugins (${resolved.problem}); each is keyed by the first bundle found`);
+    else if (resolved !== undefined)
+        for (const p of resolved)
+            taken.set(p.uri, p);
     const lv2 = new Set();
     for (const dir of settings.lv2Dirs) {
         let bundles;
@@ -304,14 +359,23 @@ export function discoverSyncItems(settings) {
                 continue;
             }
             for (const p of manifestPlugins(ttl)) {
+                const lilv = taken.get(p.uri);
+                // lilv takes another bundle for this uri: this one is not the plugin a host loads
+                if (lilv !== undefined && resolve(lilv.bundle) !== resolve(bundle))
+                    continue;
                 if (lv2.has(p.uri))
                     continue;
                 lv2.add(p.uri);
-                items.push({ key: `lv2:${p.uri}`, format: "lv2", root: bundle, files: ["manifest.ttl", ...p.files.filter((f) => f !== "manifest.ttl")], uri: p.uri });
+                const binary = lilv?.binary ?? (p.binary === undefined ? undefined : join(bundle, p.binary));
+                const files = ["manifest.ttl", ...p.files.filter((f) => f !== "manifest.ttl")];
+                // a binary named by an absolute IRI (a complement's) is tracked through the bundle's link to it, when it has one
+                if (binary !== undefined && !files.includes(basename(binary)) && linksTo(join(bundle, basename(binary)), binary))
+                    files.push(basename(binary));
+                items.push({ key: `lv2:${p.uri}`, format: "lv2", root: bundle, files, uri: p.uri, ...(binary === undefined ? {} : { binary }) });
             }
         }
     }
-    return { items, problems };
+    return { items, problems, notes };
 }
 /** The SHA-256 of a file's bytes, read in pieces over one descriptor. */
 export function fileSha256(path) {
@@ -365,6 +429,20 @@ export const SYNC_CODE = {
     /** The probe runner wrote no document to judge. */
     notMeasured: "qualify.sync.not-measured",
 };
+/**
+ * Why a plugin is unmeasured because of the machine it ran on rather than what it is (#192): an EACCES, EPERM or
+ * ENOSPC the run hit, or a measurement that could not run or wrote no document. Such a plugin is measured again by
+ * the next run although its files are unchanged; a reason of the plugin's own (probe.limit.not-implemented, no such
+ * plugin, a crash) is not. `undefined` for every other outcome.
+ */
+export function machineReason(p) {
+    if (p.outcome !== "unmeasured" || p.code !== SYNC_CODE.notMeasured)
+        return undefined;
+    const errno = /\b(EACCES|EPERM|ENOSPC)\b/.exec(p.detail ?? "")?.[1];
+    if (errno !== undefined)
+        return errno;
+    return /wrote no document|cannot run the probe runner/.test(p.detail ?? "") ? "the measurement wrote no document" : undefined;
+}
 /** What the quick qualification never measures, stated in every record it writes. */
 export const QUICK_NOT_MEASURED = [{ dimension: "stability", code: HOSTING_CODE.stabilityUnsoaked }];
 const EMPTY_STATE = { schema: SYNC_STATE_SCHEMA, entries: {} };
@@ -402,6 +480,32 @@ export function recordedAsIs(item, entry) {
     const same = now.length === entry.files.length && now.every((f) => entry.files.some((e) => e.path === f.path && e.sha256 === f.sha256));
     return { same, files: now };
 }
+/**
+ * The entry of an LV2 plugin a host now loads from another bundle than it was qualified in, when its binary is the
+ * same bytes (#199): the entry with the new bundle and the new bundle's file digests, its verdicts and records kept.
+ * `undefined` when the item is not that case: not LV2, no entry, the same bundle, a binary that cannot be read, or one
+ * whose bytes differ from the recorded binary's (then it is measured again).
+ */
+export function sameBinaryElsewhere(item, entry) {
+    if (item.format !== "lv2" || entry === undefined || item.binary === undefined || resolve(entry.root) === resolve(item.root))
+        return undefined;
+    const was = entry.files.find((f) => basename(f.path) === basename(item.binary));
+    if (was === undefined)
+        return undefined;
+    let now;
+    try {
+        now = fileSha256(item.binary);
+    }
+    catch {
+        return undefined;
+    }
+    if (now !== was.sha256)
+        return undefined;
+    const files = digestFiles(item.root, item.files);
+    if (files === undefined)
+        return undefined;
+    return { ...entry, root: item.root, files };
+}
 /**
  * The pin of a judged run with the files plugin-hostd collects for its plugin (plugin-qualify#228): the probe's list is
  * the probe's view, and an add checks the daemon's, so a pin holds the daemon's. No pin when plugin-hostd cannot name
@@ -595,15 +699,10 @@ function qualifyClap(ctx, item) {
 }
 /** An LV2 plugin, measured in a world of its own bundle only, as plugin-hostd runs a pinned LV2 plugin. */
 function qualifyLv2(ctx, item) {
-    const world = mkdtempSync(join(tmpdir(), "plugin-qualify-sync-world-"));
-    try {
-        symlinkSync(item.root, join(world, basename(item.root)));
+    return lv2World(item.root, (world) => {
         const pass = runPass(ctx, { format: "lv2", uri: item.uri, bundle: item.root }, { ...ctx.env, LV2_PATH: world });
         return [...pass.failures.map((f) => ({ uri: item.uri, outcome: "unmeasured", code: SYNC_CODE.notMeasured, detail: f.problem })), ...judgedPlugins(ctx, pass.runs)];
-    }
-    finally {
-        rmSync(world, { recursive: true, force: true });
-    }
+    });
 }
 /** One item qualified, whatever happens to it: an error of the sync's own while measuring it is its outcome, not the run's end. */
 export function qualifyItem(ctx, item, files) {
@@ -913,7 +1012,7 @@ export function descriptorAccessProblems(dir) {
     return problems;
 }
 export function summaryLine(s) {
-    return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}, ${s.seconds.toFixed(1)} s`;
+    return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
 }
 /** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
 const LOCK_HELD = 99;
@@ -995,8 +1094,19 @@ function addSummaries(a, b) {
         seconds: a.seconds + b.seconds,
         ...(sent === 0 ? {} : { sent }),
         ...(refreshed === 0 ? {} : { refreshed }),
+        ...(b.rejudge === undefined ? (a.rejudge === undefined ? {} : { rejudge: a.rejudge }) : { rejudge: b.rejudge }),
+        ...(b.memoryOnly === undefined ? (a.memoryOnly === undefined ? {} : { memoryOnly: a.memoryOnly }) : { memoryOnly: b.memoryOnly }),
+        ...(b.adapterOnly === undefined ? (a.adapterOnly === undefined ? {} : { adapterOnly: a.adapterOnly }) : { adapterOnly: b.adapterOnly }),
     };
 }
+/** Whether this run's scan covers an entry: its root lies under one of the run's directories of its own format. */
+function scannedBy(settings, entry) {
+    const root = resolve(entry.root);
+    return (entry.format === "clap" ? settings.clapDirs : settings.lv2Dirs).some((d) => {
+        const dir = resolve(d);
+        return root === dir || root.startsWith(dir.endsWith(sep) ? dir : dir + sep);
+    });
+}
 /**
  * One run: wait for quiet, qualify what is new or changed, drop what is gone, write the state, the records, the
  * descriptors and the pin file, and send the changed pins to a running daemon. The state and the pin file are written
@@ -1005,7 +1115,7 @@ function addSummaries(a, b) {
 async function runLocked(settings, options) {
     const log = options.log ?? (() => { });
     const env = options.env ?? process.env;
-    log(options.refreshOnly === true ? "pin refresh begins" : "sync run begins");
+    log(options.rejudge === true ? "re-judge begins: nothing is measured" : options.memoryOnly === true ? "memory-only pass begins" : options.adapterOnly === true ? "adapter-only pass begins" : options.refreshOnly === true ? "pin refresh begins" : "sync run begins");
     {
         const started = Date.now();
         const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
@@ -1018,6 +1128,24 @@ async function runLocked(settings, options) {
         }
         if (options.refreshOnly === true)
             return sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
+        if (options.rejudge === true) {
+            const r = rejudgeState(ctx, fresh.state);
+            if (JSON.stringify(r.state) !== JSON.stringify(fresh.state))
+                writeFileWhole(settings.stateFile, `${JSON.stringify(r.state, null, 1)}\n`);
+            const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
+            return { ...summary, rejudge: { changed: r.changed, unchanged: r.unchanged, owing: r.owing.length } };
+        }
+        if (options.adapterOnly === true) {
+            const r = adapterPass(ctx, fresh.state, { ...options.adapterPass, force: options.force === true });
+            log(`adapter runs under ${Object.entries(r.versions).map(([k, v]) => `${k} ${v}`).join(", ")}`);
+            const summary = await sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
+            return { ...summary, adapterOnly: { measured: r.measured, already: r.already, failed: r.failed } };
+        }
+        if (options.memoryOnly === true) {
+            const r = memoryPass(ctx, fresh.state, { force: options.force === true, ...(options.probeRun === undefined ? {} : { run: options.probeRun }) });
+            const summary = await sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
+            return { ...summary, memoryOnly: { measured: r.measured, already: r.already, failed: r.failed } };
+        }
         await waitQuiet(dirs, settings.settleS, log);
         const count = { qualified: 0, skipped: 0, refused: 0, crashed: 0, hung: 0, unmeasured: 0, distributed: 0 };
         const layer = distributedLayerOf(settings, options.force === true, log);
@@ -1032,28 +1160,58 @@ async function runLocked(settings, options) {
         const stateBefore = state;
         let unreadable = false;
         const lv2Pinned = [];
+        // the plugins measured this run, and how many of them the machine left unmeasured, by reason (#192)
+        let measuredHere = 0;
+        const byMachine = new Map();
         // a change made while a pass ran is caught by one more pass, a few at most
         for (let pass = 0; pass < 3; pass++) {
             const signature = directoriesSignature(dirs);
-            const { items, problems } = discoverSyncItems(settings);
+            const { items, problems, notes } = discoverSyncItems(settings, options.lv2Resolve ?? LILV_RESOLVE);
             for (const p of problems)
                 log(p);
+            if (pass === 0)
+                for (const n of notes)
+                    log(n);
             unreadable = problems.length > 0;
             const found = new Set(items.map((i) => i.key));
             const entries = {};
-            for (const [key, entry] of Object.entries(state.entries))
+            for (const [key, entry] of Object.entries(state.entries)) {
                 if (found.has(key))
                     entries[key] = entry;
-            for (const key of Object.keys(state.entries))
-                if (!found.has(key))
+                // a run scoped to some directories keeps what it did not scan: gone means not found where this run looked
+                else if (!scannedBy(settings, entry)) {
+                    entries[key] = entry;
+                    if (pass === 0)
+                        log(`${key}: not under a directory this run scans; its pin is kept`);
+                }
+                else
                     log(`${key}: gone, its pin is dropped`);
+            }
             for (const item of items) {
                 const recorded = recordedAsIs(item, entries[item.key]);
-                if (recorded.same && options.force !== true) {
+                const retry = recorded.same ? entries[item.key].plugins.map(machineReason).find((r) => r !== undefined) : undefined;
+                if (recorded.same && options.force !== true && retry === undefined) {
                     if (pass === 0)
                         count.skipped += entries[item.key].plugins.length;
                     continue;
                 }
+                // an LV2 plugin a host now loads from another bundle with the same binary (a complement that restates its TTL,
+                // #199): the record stands, keyed by its uri and its binary; only the bundle, its files and the pins' files move
+                const moved = options.force === true ? undefined : sameBinaryElsewhere(item, entries[item.key]);
+                if (moved !== undefined) {
+                    entries[item.key] = { ...moved, plugins: moved.plugins.map((p) => ({ ...p, ...(p.pin === undefined ? {} : { pin: withHostdFiles(ctx, p.pin) }) })) };
+                    if (item.format === "lv2")
+                        lv2Pinned.push(...entries[item.key].plugins.filter((p) => p.pin !== undefined).map((p) => p.uri));
+                    if (pass === 0)
+                        count.skipped += moved.plugins.length;
+                    log(`${item.key}: loaded from ${item.root} now, with the binary it was qualified with; its record stands and its pin's files follow the bundle`);
+                    state = { schema: SYNC_STATE_SCHEMA, entries };
+                    writeFileWhole(settings.stateFile, `${JSON.stringify(state, null, 1)}\n`);
+                    writeFileWhole(settings.pinFile, syncPinsText(state));
+                    continue;
+                }
+                if (recorded.same && retry !== undefined)
+                    log(`${item.key}: unmeasured last run by the machine (${retry}); measured again`);
                 const files = recorded.files ?? digestFiles(item.root, item.files);
                 if (files === undefined) {
                     log(`${item.key}: a file of it cannot be read; skipped this run`);
@@ -1078,6 +1236,13 @@ async function runLocked(settings, options) {
                     lv2Pinned.push(...entry.plugins.filter((p) => p.pin !== undefined).map((p) => p.uri));
                 for (const p of entry.plugins) {
                     count[p.outcome]++;
+                    measuredHere++;
+                    // a reason of the machine is said once: the plugins after the first are counted, not listed
+                    const why = machineReason(p);
+                    if (why !== undefined)
+                        byMachine.set(why, (byMachine.get(why) ?? 0) + 1);
+                    if (why !== undefined && byMachine.get(why) > 1)
+                        continue;
                     log(`${p.uri}: ${p.outcome}${p.code === undefined ? "" : ` (${p.code})`}${p.detail === undefined ? "" : `: ${p.detail}`}`);
                 }
                 state = { schema: SYNC_STATE_SCHEMA, entries };
@@ -1119,8 +1284,16 @@ async function runLocked(settings, options) {
             if (replies[i] !== "resp 0")
                 log(`plugin-hostd answered ${replies[i]} to ${changed[i].split(" ")[1]}`);
         const taken = replies.filter((r) => r === "resp 0").length;
+        for (const [why, n] of byMachine)
+            if (n > 1)
+                log(`${String(n)} plugins unmeasured by the machine (${why}); the first is listed above`);
+        const [only] = byMachine.size === 1 ? [...byMachine] : [];
+        const machineFailure = only !== undefined && only[1] === measuredHere ? only[0] : undefined;
+        if (machineFailure !== undefined)
+            log(`every plugin this run measured was left unmeasured by the machine (${machineFailure}): the run failed`);
         return {
             ...count,
+            ...(machineFailure === undefined ? {} : { machineFailure }),
             seconds: (Date.now() - started) / 1000,
             ...(taken === 0 ? {} : { sent: taken }),
             ...(fresh.refreshed.length === 0 ? {} : { refreshed: fresh.refreshed.length }),