@openmixer/plugin-qualify 0.20.1 → 0.20.4

package/dist/qualify-sync.js

@@ -22,38 +22,35 @@ import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, rea
 import { createConnection } from "node:net";
 import { homedir, tmpdir } from "node:os";
 import { basename, dirname, join, resolve, sep } from "node:path";
+import { withRunHome } from "./run-home.js";
 import { clapQualifyTools } from "./clap-qualify.js";
 import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, hostClassOfMachine, outcomeOfVerdict, matchDistributed, readBoardModel, readMachine, readDistributedLayer } from "./distributed-records.js";
-import { clapSearchPath, parseClapScan } from "./clap-loader.js";
+import { parseClapScan } from "./clap-loader.js";
+import { SPAWNED_SEARCH_PATH, hostdSearchPath, searchPathWarnings } from "./hostd-search-path.js";
 import { HOSTING_CODE } from "./hosting-codes.js";
 import { shippedHostProfile } from "./host-profiles.js";
-import { lv2BundlesOnPath, manifestPlugins } from "./lv2-loader.js";
+import { NO_JACK_SERVER } from "./host-admission.js";
+import { iriPath, lv2BundlesOnPath, manifestPlugins } from "./lv2-loader.js";
 import { judgmentPinEntries, pinClearLine, pinLineUri, pinsFile } from "./pins-file.js";
 import { POOL_RESULT, completePool, judgePoolRun, pinAfterPool, pinFromRecordWithPool, poolEnv, spawnedPoolRunner } from "./qualify-pool.js";
 import { hostdPinFiles } from "./hostd-pin-files.js";
+import { localeOf, reasonText } from "./text-catalog.js";
 import { qualifyTargets } from "./probe-runner.js";
 import { GRID_FULL, hostCells, parseGridSpec } from "./measurement-grid.js";
 import { adapterPass } from "./qualify-adapter-pass.js";
 import { ISOLATED_RESULT, baseSteps, completeIsolated, isolatedCell, missingSteps, judgeIsolatedRun, owesIsolated, pinAfterStep, spawnedIsolatedRunner } from "./qualify-isolated.js";
 import { lv2World, rejudgeRecord, rejudgeState, rejudgeToolName, memoryPass } from "./qualify-rejudge.js";
 import { memoryBoundOf, parseMemoryLimit } from "./memory-bound.js";
-import { QUALIFICATION_RECORD_SCHEMA, latencyNeverTaken, rewriteRecord, writeQualification } from "./qualification-record.js";
+import { QUALIFICATION_RECORD_SCHEMA, descriptorNamePart, latencyNeverTaken, rewriteRecord, writeQualification } from "./qualification-record.js";
 import { ADMIT_STEP_MS_ENV, DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, MEMORY_PASS_PROBE_NOTES, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
 import { toolsDir } from "./tool-paths.js";
 import { writeFileWhole } from "./whole-file.js";
 // --- the settings -----------------------------------------------------------------------------------------------------
 /** The settings file a system service reads, `key value` per line. */
 export const SYNC_CONF = "/etc/omx/qualify.conf";
-/** The LV2 directories plugin-hostd searches when `LV2_PATH` is unset (its `lv2_path` default). */
-export const HOSTD_LV2_PATH = ["/usr/lib64/lv2", "/usr/lib/lv2", "/usr/local/lib/lv2"];
-/** Zynthian's own LV2 directories: the image's plugins and the ones its user installs. */
-export const ZYNTHIAN_LV2_DIRS = ["/zynthian/zynthian-plugins/lv2", "/zynthian/zynthian-my-data/plugins/lv2"];
 /** Every setting, declared once: the settings file, the flags (`--pin-file` for `pin_file`) and `--help` read this. */
 export const SYNC_SETTINGS = [
-    { key: "lv2_path", meaning: "the LV2 directories, colon-separated, in the order plugin-hostd searches them", default: "$LV2_PATH, else " + HOSTD_LV2_PATH.join(":") },
-    { key: "lv2_extra", meaning: "more LV2 directories, searched after lv2_path", default: ZYNTHIAN_LV2_DIRS.join(":") },
-    { key: "clap_path", meaning: "the CLAP directories, colon-separated", default: "$CLAP_PATH, then ~/.clap, then /usr/lib/clap" },
-    { key: "clap_extra", meaning: "more CLAP directories, searched after clap_path", default: "" },
+    { key: "hostd_conf", meaning: "plugin-hostd's settings file: the LV2 and CLAP directories scanned are the ones plugin-hostd --search-path -c <file> prints", default: "plugin-hostd's own (its XDG_CONFIG_HOME/plugin-hostd.conf, else ~/.config/plugin-hostd.conf)" },
     { key: "pin_file", meaning: "the pin file plugin-hostd loads as pin_file", default: "/var/lib/omx/qualified.pins (system), else $XDG_STATE_HOME/omx/qualified.pins" },
     { key: "qualification_dir", meaning: "where the qualification records go", default: "/var/lib/omx/qualification (system), else $XDG_STATE_HOME/omx/qualification" },
     { key: "descriptor_dir", meaning: "where the scanned descriptors go", default: `${DESCRIPTOR_DIR_SYSTEM} (system), else $XDG_STATE_HOME/${DESCRIPTOR_DIR_STATE_SUBDIR}` },
@@ -70,6 +67,9 @@ export const SYNC_SETTINGS = [
     { key: "host_class", meaning: "the machine class whose distributed records are read (a directory name)", default: "the board's own from the device tree (rpi5 on a Raspberry Pi 5), else none" },
     { key: "distributed_dir", meaning: "the directories of distributed records, colon-separated, each holding <host class>/<set>/", default: `${DISTRIBUTED_DIR_SYSTEM} (system), else $XDG_DATA_HOME/omx/qualification and each $XDG_DATA_DIRS/omx/qualification` },
 ];
+/** The search path settings that moved to plugin-hostd's settings file (plugin-qualify#9): refused here, with where they went. */
+export const MOVED_TO_HOSTD = ["lv2_path", "lv2_extra", "clap_path", "clap_extra"];
+const movedText = (key) => `${key} is plugin-hostd's setting now: write it in plugin-hostd's settings file (plugin-hostd.conf, named here by hostd_conf); the directories scanned are the ones plugin-hostd --search-path prints${key === "clap_extra" ? " (plugin-hostd has no clap_extra: add the directories to clap_path)" : ""}`;
 /** A settings file read: one `key value` per line, `#` starts a comment; a key alone is the empty value. An unknown key is refused by line. */
 export function parseSyncConf(text, source = SYNC_CONF) {
     const known = new Set(SYNC_SETTINGS.map((s) => s.key));
@@ -81,6 +81,8 @@ export function parseSyncConf(text, source = SYNC_CONF) {
             continue;
         const m = /^(\S+)(?:\s+(.*))?$/.exec(line);
         const key = m[1];
+        if (MOVED_TO_HOSTD.includes(key))
+            return { ok: false, error: `${source}:${String(n + 1)}: ${movedText(key)}` };
         if (!known.has(key))
             return { ok: false, error: `${source}:${String(n + 1)}: ${key} is not a setting` };
         values.set(key, (m[2] ?? "").trim());
@@ -138,6 +140,8 @@ export function parseSyncArgs(argv) {
             else
                 values.set(byFlag.get(a), v);
         }
+        else if (MOVED_TO_HOSTD.some((k) => flagOf(k) === a))
+            return { ok: false, error: `${a}: ${movedText(a.slice(2).replace(/-/g, "_"))}` };
         else
             return { ok: false, error: `unknown argument ${a}` };
     }
@@ -174,17 +178,18 @@ export function syncHelpText() {
 }
 const dirList = (text) => text.split(":").filter((d) => d !== "");
 /** The settings of a run: a flag over the settings file over the default. */
-export function resolveSyncSettings(file, args, env, home = homedir(), boardModel = readBoardModel, machineArch = readMachine) {
+export function resolveSyncSettings(file, args, env, home = homedir(), boardModel = readBoardModel, machineArch = readMachine, searchPathRun = SPAWNED_SEARCH_PATH) {
     const get = (key) => args.values.get(key) ?? file.get(key);
     const state = env["XDG_STATE_HOME"] !== undefined && env["XDG_STATE_HOME"] !== "" ? env["XDG_STATE_HOME"] : join(home, ".local", "state");
     const place = (key, system, user) => {
         const v = get(key);
         return v !== undefined && v !== "" ? v : args.system ? system : join(state, user);
     };
-    const lv2Named = get("lv2_path");
-    const lv2Base = lv2Named !== undefined && lv2Named !== "" ? dirList(lv2Named) : dirList(env["LV2_PATH"] ?? "").length > 0 ? dirList(env["LV2_PATH"]) : HOSTD_LV2_PATH;
-    const clapNamed = get("clap_path");
-    const clapBase = clapNamed !== undefined && clapNamed !== "" ? dirList(clapNamed) : clapSearchPath(env, home);
+    const hostdConfNamed = get("hostd_conf");
+    const hostdConf = hostdConfNamed !== undefined && hostdConfNamed !== "" ? hostdConfNamed : undefined;
+    const search = hostdSearchPath(hostdConf, env, searchPathRun);
+    if (!search.ok)
+        return { ok: false, error: `the directories to scan: ${search.error}` };
     const number = (key, min) => {
         const v = Number(get(key) ?? SYNC_SETTINGS.find((s) => s.key === key).default);
         return Number.isFinite(v) && v >= min ? v : undefined;
@@ -235,9 +240,10 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
         return { ok: false, error: `host_class ${hostClass} is not a directory name` };
     return {
         ok: true,
+        warnings: searchPathWarnings(search.path, hostdConf),
         settings: {
-            lv2Dirs: once([...lv2Base, ...dirList(get("lv2_extra") ?? ZYNTHIAN_LV2_DIRS.join(":"))]),
-            clapDirs: once([...clapBase, ...dirList(get("clap_extra") ?? "")]),
+            lv2Dirs: once(search.path.lv2),
+            clapDirs: once(search.path.clap),
             pinFile: place("pin_file", "/var/lib/omx/qualified.pins", join("omx", "qualified.pins")),
             qualificationDir: place("qualification_dir", "/var/lib/omx/qualification", join("omx", "qualification")),
             descriptorDir: place("descriptor_dir", DESCRIPTOR_DIR_SYSTEM, DESCRIPTOR_DIR_STATE_SUBDIR),
@@ -379,7 +385,7 @@ export function discoverSyncItems(settings, lv2Resolve) {
                 const binary = lilv?.binary ?? (p.binary === undefined ? undefined : join(bundle, p.binary));
                 const files = ["manifest.ttl", ...p.files.filter((f) => f !== "manifest.ttl")];
                 // a binary named by an absolute IRI (a complement's) is tracked through the bundle's link to it, when it has one
-                if (binary !== undefined && !files.includes(basename(binary)) && linksTo(join(bundle, basename(binary)), binary))
+                if (binary !== undefined && !files.some((f) => iriPath(f) === basename(binary)) && linksTo(join(bundle, basename(binary)), binary))
                     files.push(basename(binary));
                 items.push({ key: `lv2:${p.uri}`, format: "lv2", root: bundle, files, uri: p.uri, ...(binary === undefined ? {} : { binary }) });
             }
@@ -409,7 +415,10 @@ export function fileSha256(path) {
 export function digestFiles(root, files, known = []) {
     const out = [];
     for (const path of [...new Set(files)].sort()) {
-        const full = join(root, path);
+        // a name may be an IRI's word (the pin keeps `libSurge%20XT.so`); the file is the path it decodes to, and a name
+        // that is a file as written (a .clap with a % in it) is read as written when the decoded one is not there
+        const decoded = join(root, iriPath(path));
+        const full = existsSync(decoded) ? decoded : join(root, path);
         try {
             const st = statSync(full);
             if (!st.isFile())
@@ -451,6 +460,8 @@ export function machineReason(p) {
     const errno = /\b(EACCES|EPERM|ENOSPC)\b/.exec(p.detail ?? "")?.[1];
     if (errno !== undefined)
         return errno;
+    if ((p.detail ?? "").startsWith(NO_JACK_SERVER))
+        return NO_JACK_SERVER;
     return /wrote no document|cannot run the probe runner/.test(p.detail ?? "") ? "the measurement wrote no document" : undefined;
 }
 /**
@@ -520,7 +531,7 @@ export function recordedAsIs(item, entry) {
 export function sameBinaryElsewhere(item, entry) {
     if (item.format !== "lv2" || entry === undefined || item.binary === undefined || resolve(entry.root) === resolve(item.root))
         return undefined;
-    const was = entry.files.find((f) => basename(f.path) === basename(item.binary));
+    const was = entry.files.find((f) => basename(iriPath(f.path)) === basename(item.binary));
     if (was === undefined)
         return undefined;
     let now;
@@ -547,7 +558,7 @@ function withHostdFiles(ctx, pin) {
         return pin;
     const r = hostdPinFiles(pin.uri, ctx.settings.lv2Dirs, ctx.env, ...(ctx.pinFilesRun === undefined ? [] : [ctx.pinFilesRun]));
     if (!r.ok) {
-        ctx.log(`${pin.uri}: not pinned: ${r.detail}`);
+        ctx.log(`${pin.uri}: not pinned: ${reasonText(r.detail, localeOf(ctx.env))}`);
         return undefined;
     }
     return { ...pin, integrity: { ...pin.integrity, files: r.files } };
@@ -570,7 +581,7 @@ export function refreshPins(ctx, state) {
                 return plugin;
             const r = hostdPinFiles(pin.uri, ctx.settings.lv2Dirs, ctx.env, ...(ctx.pinFilesRun === undefined ? [] : [ctx.pinFilesRun]));
             if (!r.ok) {
-                ctx.log(`${pin.uri}: pin files not refreshed: ${r.detail}`);
+                ctx.log(`${pin.uri}: pin files not refreshed: ${reasonText(r.detail, localeOf(ctx.env))}`);
                 return plugin;
             }
             const old = new Map(pin.integrity.files.map((f) => [f.path, f.sha256]));
@@ -604,7 +615,7 @@ function outcomeOf(judgment, host) {
         return { outcome: "qualified", ...(code === undefined ? {} : { code }) };
     return { outcome: v.rating === "unsuitable" ? "refused" : "unmeasured", ...(code === undefined ? {} : { code }) };
 }
-const safe = (text) => text.replace(/[^A-Za-z0-9._-]/g, "_");
+const safe = descriptorNamePart;
 /** The record of a plugin no probe output exists for: what it is, the bytes it was, and why nothing was measured. */
 function writeUnmeasuredRecord(ctx, item, files, plugin) {
     const main = files.find((f) => f.path === item.files[item.format === "clap" ? 0 : 1]) ?? files[0];
@@ -622,8 +633,18 @@ function writeUnmeasuredRecord(ctx, item, files, plugin) {
     writeFileWhole(path, `${JSON.stringify(record, null, 2)}\n`);
     return path;
 }
-/** The step's environment: each of its commands has the sync's budget. */
-const isolatedEnv = (ctx) => ({ ...ctx.env, [ADMIT_STEP_MS_ENV]: String(ctx.settings.budgetS * 1000) });
+/** The directories the sync scanned, as the plugin-hostd a step starts reads them: with neither variable set it searches
+ * only its own defaults, and every plugin of `lv2_extra` or of a directory the settings add is refused at add. */
+export const hostdSearchEnv = (settings) => ({
+    LV2_PATH: settings.lv2Dirs.join(":"),
+    CLAP_PATH: settings.clapDirs.join(":"),
+});
+/** The step's environment: the sync's directories, and each of its commands has the sync's budget. */
+export const isolatedEnv = (ctx) => ({
+    ...ctx.env,
+    ...hostdSearchEnv(ctx.settings),
+    [ADMIT_STEP_MS_ENV]: String(ctx.settings.budgetS * 1000),
+});
 /** Each judged run of a pass as the plugins it qualified: the record written, the pin taken, the outcome. */
 function judgedPlugins(ctx, runs, describeProblem) {
     const host = ctx.settings.hostProfile.id;
@@ -660,7 +681,7 @@ function judgedPlugins(ctx, runs, describeProblem) {
         // the pool is for a CLAP plugin that runs: the thread pool is granted by measuring it, here and nowhere else
         if (judgment.output.plugin.format === "clap" && pin !== undefined && pin.verdict !== "refused" && outcome === "qualified") {
             const runner = ctx.poolRunner ?? spawnedPoolRunner();
-            const step = judgePoolRun(runner(pin.uri, isolatedCell(record, host), poolEnv(ctx.env, ctx.settings.budgetS)));
+            const step = judgePoolRun(runner(pin.uri, isolatedCell(record, host), poolEnv({ ...ctx.env, ...hostdSearchEnv(ctx.settings) }, ctx.settings.budgetS)));
             steps.push(step);
             pin = pinAfterPool(pin, step);
         }
@@ -733,7 +754,7 @@ function qualifyClap(ctx, item) {
         return [{ uri: fileUri, outcome: "unmeasured", code: SYNC_CODE.notMeasured, detail: e.message }];
     }
     // the scanner lists the plugins of the file in a child of its own, and says how that child ended
-    const r = spawnSync(tool, ["--json", file], { encoding: "utf8", timeout: Math.max(ctx.settings.budgetS, 35) * 1000, killSignal: "SIGKILL", maxBuffer: 64 << 20 });
+    const r = spawnSync(tool, ["--json", file], { encoding: "utf8", timeout: Math.max(ctx.settings.budgetS, 35) * 1000, killSignal: "SIGKILL", maxBuffer: 64 << 20, env: withRunHome(process.env) });
     let scanned;
     try {
         if (r.error !== undefined)
@@ -873,7 +894,7 @@ export function poolPass(ctx, state) {
                     ctx.log(`${p.uri}: its record does not read; no pool step`);
                     return p;
                 }
-                const done = completePool(path, record, p.pin.uri, isolatedCell(record, host), runner, poolEnv(ctx.env, ctx.settings.budgetS));
+                const done = completePool(path, record, p.pin.uri, isolatedCell(record, host), runner, poolEnv({ ...ctx.env, ...hostdSearchEnv(ctx.settings) }, ctx.settings.budgetS));
                 if (done === undefined)
                     return p;
                 ctx.log(`${p.uri}: pool step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
@@ -987,7 +1008,7 @@ function scanClapDescriptors(ctx, item) {
         ctx.log(`${file}: descriptors not written: ${e.message}`);
         return;
     }
-    const r = spawnSync(tool, ["--json", file], { encoding: "utf8", timeout: Math.max(ctx.settings.budgetS, 35) * 1000, killSignal: "SIGKILL", maxBuffer: 64 << 20 });
+    const r = spawnSync(tool, ["--json", file], { encoding: "utf8", timeout: Math.max(ctx.settings.budgetS, 35) * 1000, killSignal: "SIGKILL", maxBuffer: 64 << 20, env: withRunHome(process.env) });
     if (r.error !== undefined || r.status !== 0) {
         ctx.log(`${file}: descriptors not written: the scan ended ${r.error?.message ?? `with exit ${String(r.status)}`}`);
         return;