@openmixer/plugin-qualify 0.17.4 → 0.18.0

package/src/qualify-sync.ts

@@ -21,7 +21,7 @@ import { createHash } from "node:crypto";
 import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs";
 import { createConnection } from "node:net";
 import { homedir, tmpdir } from "node:os";
-import { basename, dirname, join, resolve } from "node:path";
+import { basename, dirname, join, resolve, sep } from "node:path";
 
 import { clapQualifyTools } from "./clap-qualify.js";
 import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, matchDistributed, readBoardModel, readDistributedLayer, type DistributedLayer } from "./distributed-records.js";
@@ -126,6 +126,8 @@ export interface SyncArgs {
   readonly force: boolean;
   readonly noDistributed: boolean;
   readonly noWait: boolean;
+  /** `--refresh-pins`: refresh the pins' files from plugin-hostd and measure nothing (the package's upgrade runs it). */
+  readonly refreshPins: boolean;
   readonly help: boolean;
   /** The settings given as flags, by key. */
   readonly values: ReadonlyMap<string, string>;
@@ -141,6 +143,7 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
   let force = false;
   let noDistributed = false;
   let noWait = false;
+  let refreshPinsOnly = false;
   let help = false;
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i]!;
@@ -148,6 +151,7 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
     else if (a === "--force") force = true;
     else if (a === "--no-distributed") noDistributed = true;
     else if (a === "--no-wait") noWait = true;
+    else if (a === "--refresh-pins") refreshPinsOnly = true;
     else if (a === "--help") help = true;
     else if (a === "--grid-hosts") values.set("grid", "hosts");
     else if (a === "--config" || byFlag.has(a)) {
@@ -157,7 +161,7 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
       else values.set(byFlag.get(a)!, v);
     } else return { ok: false, error: `unknown argument ${a}` };
   }
-  return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, help, values } };
+  return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, help, values } };
 }
 
 export function syncHelpText(): string {
@@ -170,6 +174,7 @@ export function syncHelpText(): string {
     "  --force                             qualify every plugin again, recorded or not; no distributed record is read",
     "  --no-wait                           a run in progress is not waited for: exit 0, and the holder runs once more at its end",
     "  --no-distributed                    read no distributed record: every plugin not recorded is measured",
+    "  --refresh-pins                      only bring every pin's files up to the ones plugin-hostd names, measuring nothing",
     "  --grid-hosts                        the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
     "  --help                              this text",
     ...rows,
@@ -515,6 +520,42 @@ function withHostdFiles(ctx: SyncContext, pin: PinEntry | undefined): PinEntry |
   return { ...pin, integrity: { ...pin.integrity, files: r.files } };
 }
 
+/**
+ * Every pin of the state with the files plugin-hostd collects for its plugin now, measured again by nothing
+ * (plugin-qualify#239). A pin written before plugin-qualify 0.17.3 holds the probe's view of an LV2 (binary, plugin
+ * TTL, manifest) and not the preset files plugin-hostd 0.15.0 also checks, so its add answers -509 until the plugin
+ * is qualified again: hours on a device with hundreds. The new list replaces the old one when no file both lists
+ * hold changed its hash, so the verdict stands for the same bytes; a changed file is the sync's to measure again,
+ * and that pin is left as it is. A plugin plugin-hostd cannot name the files of keeps its pin.
+ */
+export function refreshPins(ctx: SyncContext, state: SyncState): { state: SyncState; refreshed: string[] } {
+  const refreshed: string[] = [];
+  const entries: Record<string, SyncEntry> = {};
+  for (const [key, entry] of Object.entries(state.entries)) {
+    const plugins = entry.plugins.map((plugin) => {
+      const pin = plugin.pin;
+      if (pin === undefined || pin.integrity.files === undefined) return plugin;
+      const r = hostdPinFiles(pin.uri, ctx.settings.lv2Dirs, ctx.env, ...(ctx.pinFilesRun === undefined ? [] : [ctx.pinFilesRun]));
+      if (!r.ok) return plugin;
+      const old = new Map(pin.integrity.files.map((f) => [f.path, f.sha256]));
+      const changed = r.files.filter((f) => old.has(f.path) && old.get(f.path) !== f.sha256).map((f) => f.path);
+      const same = r.files.length === old.size && r.files.every((f) => old.get(f.path) === f.sha256);
+      if (same) return plugin;
+      if (changed.length > 0) {
+        ctx.log(`${pin.uri}: ${changed.join(", ")} changed since it was qualified; its pin waits for the sync`);
+        return plugin;
+      }
+      const added = r.files.filter((f) => !old.has(f.path)).map((f) => f.path);
+      const gone = [...old.keys()].filter((path) => !r.files.some((f) => f.path === path));
+      ctx.log(`${pin.uri}: pin files refreshed${added.length ? `, ${added.join(", ")} added` : ""}${gone.length ? `, ${gone.join(", ")} gone` : ""}`);
+      refreshed.push(pin.uri);
+      return { ...plugin, pin: { ...pin, integrity: { ...pin.integrity, files: r.files } } };
+    });
+    entries[key] = refreshed.length === 0 ? entry : { ...entry, plugins };
+  }
+  return { state: refreshed.length === 0 ? state : { schema: SYNC_STATE_SCHEMA, entries }, refreshed };
+}
+
 /** A judged run's outcome for the host: qualified, refused, or why it was not measured. */
 function outcomeOf(judgment: Judgment, host: string): { outcome: SyncOutcome; code?: string } {
   const v = judgment.hosts.find((h) => h.verdict.host === host)?.verdict;
@@ -941,6 +982,8 @@ export interface SyncSummary {
   readonly seconds: number;
   /** The pins a running daemon took, when one answered. */
   readonly sent?: number;
+  /** The pins whose files were refreshed from plugin-hostd without a measurement (plugin-qualify#239). */
+  readonly refreshed?: number;
 }
 
 /**
@@ -977,7 +1020,7 @@ export function descriptorAccessProblems(dir: string): string[] {
 }
 
 export function summaryLine(s: SyncSummary): string {
-  return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}, ${s.seconds.toFixed(1)} s`;
+  return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}, ${s.seconds.toFixed(1)} s`;
 }
 
 /** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
@@ -1021,6 +1064,10 @@ export const pendingMarker = (stateFile: string): string => join(dirname(stateFi
 
 export interface SyncRunOptions {
   readonly force?: boolean;
+  /** `--refresh-pins`: only refresh the pins' files from plugin-hostd and send the changed ones; measure nothing. */
+  readonly refreshOnly?: boolean;
+  /** The run of `plugin-hostd --pin-files`, a stand-in in tests. */
+  readonly pinFilesRun?: PinFilesRun;
   /** False under `--no-wait`: a run in progress is not waited for; the holder runs once more at its end instead. */
   readonly wait?: boolean;
   readonly env?: Readonly<Record<string, string | undefined>>;
@@ -1064,6 +1111,7 @@ export async function runSync(settings: SyncSettings, options: SyncRunOptions =
 
 function addSummaries(a: SyncSummary, b: SyncSummary): SyncSummary {
   const sent = (a.sent ?? 0) + (b.sent ?? 0);
+  const refreshed = (a.refreshed ?? 0) + (b.refreshed ?? 0);
   return {
     qualified: a.qualified + b.qualified,
     skipped: a.skipped + b.skipped,
@@ -1074,9 +1122,19 @@ function addSummaries(a: SyncSummary, b: SyncSummary): SyncSummary {
     distributed: a.distributed + b.distributed,
     seconds: a.seconds + b.seconds,
     ...(sent === 0 ? {} : { sent }),
+    ...(refreshed === 0 ? {} : { refreshed }),
   };
 }
 
+/** Whether this run's scan covers an entry: its root lies under one of the run's directories of its own format. */
+function scannedBy(settings: Pick<SyncSettings, "lv2Dirs" | "clapDirs">, entry: SyncEntry): boolean {
+  const root = resolve(entry.root);
+  return (entry.format === "clap" ? settings.clapDirs : settings.lv2Dirs).some((d) => {
+    const dir = resolve(d);
+    return root === dir || root.startsWith(dir.endsWith(sep) ? dir : dir + sep);
+  });
+}
+
 /**
  * One run: wait for quiet, qualify what is new or changed, drop what is gone, write the state, the records, the
  * descriptors and the pin file, and send the changed pins to a running daemon. The state and the pin file are written
@@ -1085,12 +1143,19 @@ function addSummaries(a: SyncSummary, b: SyncSummary): SyncSummary {
 async function runLocked(settings: SyncSettings, options: SyncRunOptions): Promise<SyncSummary> {
   const log = options.log ?? (() => {});
   const env = options.env ?? process.env;
-  log("sync run begins");
+  log(options.refreshOnly === true ? "pin refresh begins" : "sync run begins");
   {
     const started = Date.now();
     const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
+    const ctx: SyncContext = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }) };
+    // the pins first, before the quiet wait and any measurement: an upgrade of plugin-hostd that collects more files
+    // than the pins on disk hold refuses those plugins until their pins name them (plugin-qualify#239)
+    const fresh = refreshPins(ctx, readSyncState(settings.stateFile, log));
+    if (fresh.refreshed.length > 0) {
+      writeFileWhole(settings.stateFile, `${JSON.stringify(fresh.state, null, 1)}\n`);
+    }
+    if (options.refreshOnly === true) return sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
     await waitQuiet(dirs, settings.settleS, log);
-    const ctx: SyncContext = { settings, env, log };
     const count = { qualified: 0, skipped: 0, refused: 0, crashed: 0, hung: 0, unmeasured: 0, distributed: 0 };
     const layer = distributedLayerOf(settings, options.force === true, log);
     let before: string;
@@ -1099,7 +1164,7 @@ async function runLocked(settings: SyncSettings, options: SyncRunOptions): Promi
     } catch {
       before = "";
     }
-    let state = readSyncState(settings.stateFile, log);
+    let state = fresh.state;
     const stateBefore = state;
     let unreadable = false;
     const lv2Pinned: string[] = [];
@@ -1111,8 +1176,14 @@ async function runLocked(settings: SyncSettings, options: SyncRunOptions): Promi
       unreadable = problems.length > 0;
       const found = new Set(items.map((i) => i.key));
       const entries: Record<string, SyncEntry> = {};
-      for (const [key, entry] of Object.entries(state.entries)) if (found.has(key)) entries[key] = entry;
-      for (const key of Object.keys(state.entries)) if (!found.has(key)) log(`${key}: gone, its pin is dropped`);
+      for (const [key, entry] of Object.entries(state.entries)) {
+        if (found.has(key)) entries[key] = entry;
+        // a run scoped to some directories keeps what it did not scan: gone means not found where this run looked
+        else if (!scannedBy(settings, entry)) {
+          entries[key] = entry;
+          if (pass === 0) log(`${key}: not under a directory this run scans; its pin is kept`);
+        } else log(`${key}: gone, its pin is dropped`);
+      }
       for (const item of items) {
         const recorded = recordedAsIs(item, entries[item.key]);
         if (recorded.same && options.force !== true) {
@@ -1173,8 +1244,31 @@ async function runLocked(settings: SyncSettings, options: SyncRunOptions): Promi
     const replies = "replies" in sent ? sent.replies : [];
     for (let i = 0; i < replies.length; i++) if (replies[i] !== "resp 0") log(`plugin-hostd answered ${replies[i]!} to ${changed[i]!.split(" ")[1]!}`);
     const taken = replies.filter((r) => r === "resp 0").length;
-    return { ...count, seconds: (Date.now() - started) / 1000, ...(taken === 0 ? {} : { sent: taken }) };
+    return {
+      ...count,
+      seconds: (Date.now() - started) / 1000,
+      ...(taken === 0 ? {} : { sent: taken }),
+      ...(fresh.refreshed.length === 0 ? {} : { refreshed: fresh.refreshed.length }),
+    };
+  }
+}
+
+/** The end of a `--refresh-pins` run: the pin file from the refreshed state, and its changed pins to a running daemon. */
+async function sendRefreshed(settings: SyncSettings, state: SyncState, refreshed: number, started: number, log: (line: string) => void): Promise<SyncSummary> {
+  let before = "";
+  try {
+    before = readFileSync(settings.pinFile, "utf8");
+  } catch {
+    before = "";
   }
+  const after = syncPinsText(state);
+  if (after !== before) writeFileWhole(settings.pinFile, after);
+  const changed = changedPinLines(before, after);
+  const sent = await sendPins(settings.hostdPort, settings.hostdFeedbackPort, changed);
+  const taken = "replies" in sent ? sent.replies.filter((r) => r === "resp 0").length : 0;
+  if ("unsent" in sent && changed.length > 0) log(`the ${String(changed.length)} changed pins apply at plugin-hostd's next start, from the pin file`);
+  const zero = { qualified: 0, skipped: 0, refused: 0, crashed: 0, hung: 0, unmeasured: 0, distributed: 0 };
+  return { ...zero, seconds: (Date.now() - started) / 1000, ...(taken === 0 ? {} : { sent: taken }), ...(refreshed === 0 ? {} : { refreshed }) };
 }
 
 /** What a failed descriptor pass says: plainly, once, when the lilv binding is what is missing (a suite without python3-lilv). */

package/tools/_declarations.py

@@ -17,6 +17,7 @@ OPERATIONAL_RATE_FLOOR = 48000
 QUALIFY_RATES = (44100, 48000, 96000, 192000)
 QUALIFY_QUANTA = (128, 256, 512, 1024)
 PLUGIN_TIMEOUT_S = 15
+ADDRESS_SPACE_LIMIT_RAM_FRACTION = 1
 BOOT_TIMEOUT_S = 120
 COST_TIMEOUT_S = 60
 DESCRIPTOR_DIR_KEY = "descriptor_dir"

package/tools/benchmark.py

@@ -42,6 +42,7 @@ Usage:
 import argparse
 import json
 import os
+import resource
 import select
 import shlex
 import subprocess
@@ -53,10 +54,13 @@ from collections import namedtuple
 # declarations (with the rate list and timeouts), generated from `src/qualify-declarations.ts` because this script cannot import
 # a TS module (2026-09-17 declare-derive audit, F8) — see `_declarations.py`'s own header for the
 # regenerate/verify commands.
-from _declarations import (BOOT_TIMEOUT_S, COST_REFERENCE_RATE, COST_TIMEOUT_S,
+from _declarations import (ADDRESS_SPACE_LIMIT_RAM_FRACTION, BOOT_TIMEOUT_S, COST_REFERENCE_RATE, COST_TIMEOUT_S,
                            MEASURE_BASE_RATE, OPERATIONAL_RATE_FLOOR, PLUGIN_TIMEOUT_S,
                            QUALIFY_RATES, QUANTUM_FRAMES)
 
+# the outcome of a worker past its address space, the same word as a direct spawn of plugin-qualify's (src/direct-limit.ts)
+MEMORY_LIMIT_EXCEEDED = "memory-limit-exceeded"
+
 try:  # numpy is only needed by the worker (and as a fast path in the pure helpers)
     import numpy as _np
 except ImportError:  # pragma: no cover - CI runs the pure tests without numpy
@@ -172,7 +176,7 @@ COST_TONE_AMP = 0.5
 # whose tail is genuinely fat — the one that xruns mid-show — while being a stable
 # statistic at this sample count.
 COST_PERCENTILE = 95.0
-# Cost is anchored at the rig's canonical rate: the headline figure the operator asked for is
+# Cost is anchored at the rig's canonical rate: the headline figure asked for is
 # "how many of these fit in one core at the rig's rate". `COST_REFERENCE_RATE` is imported above
 # from `_declarations.py` — the same fact `cpu-cost.ts` declares, not a second 96000 literal.
 # A cost probe runs many more blocks than a latency probe; give it room.
@@ -1908,6 +1912,8 @@ def worker_main(rate, block, measure=False, cost=False, cpu=None,
                         host, uri, declared, res["measuredFrames"], job["sweep"])
                     if swept:
                         res["sweep"] = swept
+            except MemoryError:
+                res = {"error": MEMORY_LIMIT_EXCEEDED}
             except Exception as e:  # noqa: BLE001 - report, don't die
                 res = {"error": f"probe error: {e}"}
             if "measuredFrames" in res:
@@ -1924,6 +1930,8 @@ def worker_main(rate, block, measure=False, cost=False, cpu=None,
         symbol = arg
         try:
             frames, reason = probe_one(host, uri, symbol)
+        except MemoryError:
+            frames, reason = None, MEMORY_LIMIT_EXCEEDED
         except Exception as e:  # noqa: BLE001 - report, don't die, on python-level errors
             frames, reason = None, f"probe error: {e}"
         if frames is None:
@@ -1935,6 +1943,20 @@ def worker_main(rate, block, measure=False, cost=False, cpu=None,
 # --- the parent: batch loop with crash containment -----------------------------
 
 
+def address_space_limit(ram_bytes=None):
+    """What the worker runs under: RLIMIT_AS at ADDRESS_SPACE_LIMIT_RAM_FRACTION of the machine's RAM, since a direct
+    spawn has no runner to give it a cgroup; set between fork and exec. A plugin past it fails its allocation, which the
+    worker reports as `MEMORY_LIMIT_EXCEEDED` when Python sees it and which crash containment takes when the plugin dies."""
+    if ram_bytes is None:
+        ram_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    limit = int(ram_bytes * ADDRESS_SPACE_LIMIT_RAM_FRACTION)
+
+    def apply():
+        resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
+
+    return apply
+
+
 class _Worker:
     """Handle on one worker subprocess: spawn, job/reply exchange, kill."""
 
@@ -1942,6 +1964,7 @@ class _Worker:
         self.proc = subprocess.Popen(
             cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.DEVNULL, text=True, bufsize=1,
+            preexec_fn=address_space_limit(),
         )
         ready = self._read_line(boot_timeout)
         parsed = parse_worker_line(ready)

package/tools/gen-python-declarations.mjs

@@ -33,6 +33,7 @@ const PY_NAMES = [
   'QUALIFY_RATES',
   'QUALIFY_QUANTA',
   'PLUGIN_TIMEOUT_S',
+  'ADDRESS_SPACE_LIMIT_RAM_FRACTION',
   'BOOT_TIMEOUT_S',
   'COST_TIMEOUT_S',
   'DESCRIPTOR_DIR_KEY',

package/tools/lv2-measure.mjs

@@ -60,6 +60,7 @@ import {
   urisFromScan,
 } from "../dist/measurement/index.js";
 import { LV2_HOSTING_TIMEOUT_S, LV2_MEASURE_TOOL_NAME, RT_INTERPOSE_NAME } from "../dist/qualify-declarations.js";
+import { addressSpaceLimited, directOutcome } from "../dist/direct-limit.js";
 import { halfPopulatedRefusal, locateNativeTools } from "../dist/tool-paths.js";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
@@ -135,9 +136,10 @@ function runHosting(step, uris, onProgress) {
   const lines = [];
   onProgress(step, { kind: "phase", plugins: uris.length, rate: Number(step.args[0]) });
   for (const uri of uris) {
-    const r = spawnSync(step.command, [uri, ...step.args], {
+    const limited = addressSpaceLimited(step.command, [uri, ...step.args]);
+    const r = spawnSync(limited.command, [...limited.args], {
       encoding: "utf8",
-      env: { ...process.env, ...step.env },
+      env: { ...process.env, ...step.env, ...limited.env },
       stdio: ["ignore", "pipe", "inherit"],
       timeout: LV2_HOSTING_TIMEOUT_S * 1000,
       maxBuffer: 16 << 20,
@@ -149,9 +151,10 @@ function runHosting(step, uris, onProgress) {
       ...(r.error?.code === "ETIMEDOUT" ? { timedOut: true } : {}),
       stdout: r.stdout ?? "",
     });
-    process.stderr.write(`  ${uri}: exit ${line.exit}\n`);
+    const limitHit = directOutcome(r.status, limited);
+    process.stderr.write(`  ${uri}: exit ${line.exit}${limitHit === undefined ? "" : ` (${limitHit.outcome})`}\n`);
     onProgress(step, { kind: "plugin" });
-    lines.push(line);
+    lines.push(limitHit === undefined ? line : { ...line, ...limitHit });
   }
   writeFileSync(step.produces, lines.map((l) => `${JSON.stringify(l)}\n`).join(""));
   return lines;

package/tools/lv2_presets.py

@@ -16,7 +16,7 @@ body, and judges it against the plugin's own descriptor params:
   listed — refuse or report, never drop.
 
 Only the bundles lilv loads from LV2_PATH are read, which is what the scan reads: the presets a
-plugin ships, never one another host saved in a user's home (ruling 2). The output is sorted by
+plugin ships, never one another host saved in a user's home (decision 2). The output is sorted by
 preset URI and the values by symbol, so the same install yields the same bytes.
 """
 

package/tools/mod-host-admit.mjs

@@ -40,7 +40,8 @@ async function connect() {
 async function main() {
   if ((await Promise.race([spawnFailed, sleep(200).then(() => undefined)])) === "ENOENT") return { installed: false, steps: [] };
   const sock = await connect();
-  if (sock === undefined) return { installed: true, steps: [{ cmd: "add", died: hostEnd ?? "closed" }] };
+  // the host never listened (omx-clap-host with no JACK server): the plugin was never loaded
+  if (sock === undefined) return { installed: true, started: false, steps: [] };
   let buffer = "";
   let closed = false;
   sock.on("data", (d) => (buffer += d.toString()));