@openmixer/plugin-qualify 0.17.2 → 0.17.3

package/src/qualification-record.ts

@@ -93,13 +93,15 @@ export function unmeasuredFigures(output: ProbeOutput): readonly UnmeasuredFigur
 
 /** A cell's latency when the run took no figure: the contract's unmeasured word and the hosting code a reader classifies by. */
 export interface UnmeasuredLatency {
+  /** The figure the plugin declared (LV2 latency port, CLAP latency extension) when it loaded but the probe did not measure. */
+  readonly declaredFrames?: number;
   readonly unmeasuredReason: string;
   readonly code: string;
 }
 
 /** A grid point as the record writes it: every cell carries `latency`, the figures or why there are none (#201). */
 export type RecordGridPoint = Omit<ProbeGridPoint, "latency"> & {
-  readonly latency: NonNullable<ProbeGridPoint["latency"]> | UnmeasuredLatency;
+  readonly latency: (Omit<NonNullable<ProbeGridPoint["latency"]>, "measuredFrames"> & { readonly measuredFrames: number }) | UnmeasuredLatency;
 };
 
 const UNMEASURED_WORDS: ReadonlySet<string> = new Set(Object.values(MEASUREMENT_UNMEASURED_REASONS).map((r) => r.word));
@@ -120,7 +122,13 @@ const OUTCOME_WORD: Readonly<Record<string, string>> = {
  * older than the latency arm writes it.
  */
 export function latencyOfCell(output: ProbeOutput, point: ProbeGridPoint): RecordGridPoint["latency"] {
-  if (point.latency !== undefined) return point.latency;
+  const had = point.latency;
+  if (had?.measuredFrames !== undefined) return { ...had, measuredFrames: had.measuredFrames };
+  const why = unmeasuredOfCell(output, point);
+  return had === undefined ? why : { declaredFrames: had.declaredFrames, ...why };
+}
+
+function unmeasuredOfCell(output: ProbeOutput, point: ProbeGridPoint): UnmeasuredLatency {
   const code = HOSTING_CODE.latencyUnmeasured;
   if (point.outcome !== PROBE_OK) return { unmeasuredReason: OUTCOME_WORD[point.outcome] ?? word("probeError"), code };
   const step = output.steps.find((st) => st.step === "latency" && st.rate === point.rate && (st.quantum === undefined || st.quantum === point.quantum));
@@ -162,6 +170,11 @@ export interface QualificationRecord {
   readonly testedVersion?: string;
   readonly binary: ProbeOutput["binary"];
   readonly verdicts: readonly RecordVerdict[];
+  /**
+   * An LV2 plugin run under plugin-hostd's LV2 adapter (omx-clap-host), as under mod-host: admitted, or refused with
+   * its code, or `measured: false` when it was not tried. Beside the verdicts, which it does not change (#223).
+   */
+  readonly adapter?: { readonly host: string; readonly measured: boolean; readonly admitted: boolean; readonly code?: string; readonly params: Readonly<Record<string, string | number>> };
   readonly tiers: RecordTiers;
   /** The cost reasons at each host's rate and quantum, beside the verdicts; an instrument's with the notes it held. */
   readonly cost: readonly {
@@ -266,6 +279,7 @@ export function buildQualificationRecord(
       ...(h.verdict.deciding === undefined ? {} : { deciding: h.verdict.deciding }),
       reasons: h.reasons,
     })),
+    ...(judgment.adapter === undefined ? {} : { adapter: judgment.adapter }),
     tiers: tiersOf(judgment),
     cost: judgment.hosts.map((h) => ({ host: h.verdict.host, ...h.cost })),
     measurements: {

package/src/qualify-sync.ts

@@ -16,9 +16,9 @@
  * When a daemon answers on the command port, the pins that changed are also sent to it with `pin_set`, and the ones
  * of a plugin that is gone with `pin_clear`. What a gone plugin left in the records and the descriptors is removed.
  */
-import { spawnSync } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
-import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync, writeSync } from "node:fs";
+import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs";
 import { createConnection } from "node:net";
 import { homedir, tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "node:path";
@@ -30,6 +30,7 @@ import { HOSTING_CODE } from "./hosting-codes.js";
 import { shippedHostProfile, type HostProfile } from "./host-profiles.js";
 import { lv2BundlesOnPath, manifestPlugins } from "./lv2-loader.js";
 import { judgmentPinEntries, pinsFile, type PinEntry } from "./pins-file.js";
+import { hostdPinFiles, type PinFilesRun } from "./hostd-pin-files.js";
 import type { Judgment } from "./probe-judge.js";
 import { qualifyTargets } from "./probe-runner.js";
 import { GRID_FULL, parseGridSpec, type GridSpec } from "./measurement-grid.js";
@@ -124,6 +125,7 @@ export interface SyncArgs {
   readonly system: boolean;
   readonly force: boolean;
   readonly noDistributed: boolean;
+  readonly noWait: boolean;
   readonly help: boolean;
   /** The settings given as flags, by key. */
   readonly values: ReadonlyMap<string, string>;
@@ -138,12 +140,14 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
   let system = false;
   let force = false;
   let noDistributed = false;
+  let noWait = false;
   let help = false;
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i]!;
     if (a === "--system") system = true;
     else if (a === "--force") force = true;
     else if (a === "--no-distributed") noDistributed = true;
+    else if (a === "--no-wait") noWait = true;
     else if (a === "--help") help = true;
     else if (a === "--grid-hosts") values.set("grid", "hosts");
     else if (a === "--config" || byFlag.has(a)) {
@@ -153,7 +157,7 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
       else values.set(byFlag.get(a)!, v);
     } else return { ok: false, error: `unknown argument ${a}` };
   }
-  return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, help, values } };
+  return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, help, values } };
 }
 
 export function syncHelpText(): string {
@@ -164,6 +168,7 @@ export function syncHelpText(): string {
     `  ${"--config <file>".padEnd(36)}the settings file (default: ${SYNC_CONF}; a missing file is the defaults)`,
     "  --system                            a system service's directories, under /var/lib/omx",
     "  --force                             qualify every plugin again, recorded or not; no distributed record is read",
+    "  --no-wait                           a run in progress is not waited for: exit 0, and the holder runs once more at its end",
     "  --no-distributed                    read no distributed record: every plugin not recorded is measured",
     "  --grid-hosts                        the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
     "  --help                              this text",
@@ -491,6 +496,23 @@ export interface SyncContext {
   readonly settings: SyncSettings;
   readonly env: Readonly<Record<string, string | undefined>>;
   readonly log: (line: string) => void;
+  /** The run of `plugin-hostd --pin-files`, a stand-in in tests. */
+  readonly pinFilesRun?: PinFilesRun;
+}
+
+/**
+ * The pin of a judged run with the files plugin-hostd collects for its plugin (plugin-qualify#228): the probe's list is
+ * the probe's view, and an add checks the daemon's, so a pin holds the daemon's. No pin when plugin-hostd cannot name
+ * them; the plugin is then pinned on its first add (pin_mode first-use), and the log says why.
+ */
+function withHostdFiles(ctx: SyncContext, pin: PinEntry | undefined): PinEntry | undefined {
+  if (pin === undefined || pin.integrity.files === undefined) return pin;
+  const r = hostdPinFiles(pin.uri, ctx.settings.lv2Dirs, ctx.env, ...(ctx.pinFilesRun === undefined ? [] : [ctx.pinFilesRun]));
+  if (!r.ok) {
+    ctx.log(`${pin.uri}: not pinned: ${r.detail}`);
+    return undefined;
+  }
+  return { ...pin, integrity: { ...pin.integrity, files: r.files } };
 }
 
 /** A judged run's outcome for the host: qualified, refused, or why it was not measured. */
@@ -528,7 +550,7 @@ function judgedPlugins(ctx: SyncContext, runs: readonly { bytes: Buffer; judgmen
   const host = ctx.settings.hostProfile.id;
   const entries = runs.map(({ bytes, judgment }) => {
     const { recordPath } = writeQualification(ctx.settings.qualificationDir, bytes, judgment, QUICK_NOT_MEASURED);
-    const pin = judgmentPinEntries(judgment, host)[0];
+    const pin = withHostdFiles(ctx, judgmentPinEntries(judgment, host)[0]);
     const { outcome, code } = outcomeOf(judgment, host);
     const plugin: SyncPlugin = { uri: pin?.uri ?? judgment.output.plugin.id, outcome, ...(code === undefined ? {} : { code }), ...(pin === undefined ? {} : { pin }), record: recordPath };
     return { plugin, judgedHere: judgment.hosts.some((h) => h.verdict.host === host) };
@@ -550,6 +572,8 @@ function runPass(ctx: SyncContext, target: Parameters<typeof qualifyTargets>[0][
     profiles: [s.hostProfile],
     wallS: s.budgetS,
     env,
+    // mod-host and jalv are admitted on the device's LV2 directories, not on the one-bundle world the probe measures in
+    admissionLv2Path: s.lv2Dirs,
   });
 }
 
@@ -919,57 +943,143 @@ export interface SyncSummary {
   readonly sent?: number;
 }
 
+/**
+ * Why a reader that is not the sync's user (the console, #221) cannot read the descriptors: the directory or one of its
+ * parent without search permission for others, the directory or a descriptor without read permission for others. One
+ * line each; none when the directory does not exist yet or everything is open.
+ */
+export function descriptorAccessProblems(dir: string): string[] {
+  const problems: string[] = [];
+  const mode = (path: string): number | undefined => {
+    try {
+      return statSync(path).mode & 0o777;
+    } catch {
+      return undefined;
+    }
+  };
+  const octal = (m: number): string => `0${m.toString(8)}`;
+  const top = mode(dir);
+  if (top === undefined) return problems;
+  // the parent too (StateDirectory's /var/lib/omx); not further up, where a private home is a user's own choice
+  const parent = dirname(resolve(dir));
+  const parentMode = mode(parent);
+  if (parentMode !== undefined && (parentMode & 0o001) === 0) problems.push(`${parent} is mode ${octal(parentMode)}: other users, the console among them, cannot enter it to read the descriptors`);
+  if ((top & 0o004) === 0) problems.push(`${dir} is mode ${octal(top)}: other users, the console among them, cannot list the descriptors`);
+  let files: string[] = [];
+  try {
+    files = readdirSync(dir).filter((f) => f.endsWith(".json"));
+  } catch {
+    return problems;
+  }
+  const closed = files.filter((f) => ((mode(join(dir, f)) ?? 0o004) & 0o004) === 0);
+  if (closed.length > 0) problems.push(`${String(closed.length)} descriptor(s) in ${dir} cannot be read by other users, the console among them (first: ${closed[0]!})`);
+  return problems;
+}
+
 export function summaryLine(s: SyncSummary): string {
   return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}, ${s.seconds.toFixed(1)} s`;
 }
 
-/** The lock of one state file: a second sync on it while one runs returns at once. A lock whose process is gone is taken over. */
-function takeLock(stateFile: string): (() => void) | undefined {
+/** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
+const LOCK_HELD = 99;
+
+export interface SyncLock {
+  release(): Promise<void>;
+}
+
+/**
+ * The lock of one state file: flock(1) on `<state file>.lock`, held by a child process for as long as this one lives or
+ * until `release`, so a crashed run frees it. Waits for the holder, or with `wait` false returns `undefined` at once.
+ */
+export function takeLock(stateFile: string, wait: boolean): Promise<SyncLock | undefined> {
   const lock = `${stateFile}.lock`;
   mkdirSync(dirname(lock), { recursive: true });
-  for (let attempt = 0; attempt < 2; attempt++) {
-    try {
-      const fd = openSync(lock, "wx");
-      writeSync(fd, String(process.pid));
-      closeSync(fd);
-      return () => rmSync(lock, { force: true });
-    } catch (e) {
-      if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
-      const pid = Number(readFileSync(lock, "utf8"));
-      try {
-        if (Number.isInteger(pid) && pid > 0) {
-          process.kill(pid, 0);
-          return undefined;
-        }
-      } catch {
-        // its process is gone
-      }
-      unlinkSync(lock);
-    }
-  }
-  return undefined;
+  const child = spawn("flock", ["-x", "-E", String(LOCK_HELD), ...(wait ? [] : ["-n"]), lock, "sh", "-c", "echo locked; exec cat"], { stdio: ["pipe", "pipe", "inherit"] });
+  return new Promise((resolvePromise, reject) => {
+    const exited = new Promise<void>((done) => child.once("close", () => done()));
+    child.once("error", reject);
+    child.stdout.once("data", () =>
+      resolvePromise({
+        release: () => {
+          child.stdin.end();
+          return exited;
+        },
+      }),
+    );
+    child.once("close", (code) => {
+      if (code === LOCK_HELD) resolvePromise(undefined);
+      else reject(new Error(`flock ${lock} failed (exit ${String(code)})`));
+    });
+  });
 }
 
+/** What a second run leaves for the holder: one more run at its end, over what changed meanwhile. */
+const againMarker = (stateFile: string): string => `${stateFile}.again`;
+
 export interface SyncRunOptions {
   readonly force?: boolean;
+  /** False under `--no-wait`: a run in progress is not waited for; the holder runs once more at its end instead. */
+  readonly wait?: boolean;
   readonly env?: Readonly<Record<string, string | undefined>>;
   readonly log?: (line: string) => void;
 }
 
 /**
- * One run: wait for quiet, qualify what is new or changed, drop what is gone, write the state, the records, the
- * descriptors and the pin file, and send the changed pins to a running daemon. The state and the pin file are written
- * whole after every item, so a run cut short keeps all it finished. `undefined` when another run holds the lock.
+ * One run under the lock: waits for a run in progress, or with `wait` false leaves a marker for it and returns
+ * `undefined`. The holder runs once more at its end for every marker left meanwhile.
  */
 export async function runSync(settings: SyncSettings, options: SyncRunOptions = {}): Promise<SyncSummary | undefined> {
   const log = options.log ?? (() => {});
-  const env = options.env ?? process.env;
-  const unlock = takeLock(settings.stateFile);
-  if (unlock === undefined) {
-    log(`another sync holds ${settings.stateFile}.lock; this one does nothing`);
-    return undefined;
+  const marker = againMarker(settings.stateFile);
+  let lock = await takeLock(settings.stateFile, options.wait !== false);
+  if (lock === undefined) {
+    // the marker first, then the lock once more: a holder that finished in between is not left without a successor
+    writeFileSync(marker, "");
+    lock = await takeLock(settings.stateFile, false);
+    if (lock === undefined) {
+      log(`a run is in progress (${settings.stateFile}.lock); it runs once more at its end over what changed`);
+      return undefined;
+    }
   }
   try {
+    let total: SyncSummary | undefined;
+    for (;;) {
+      rmSync(marker, { force: true });
+      const one = await runLocked(settings, options);
+      total = total === undefined ? one : addSummaries(total, one);
+      if (!existsSync(marker)) return total;
+      log("a run was asked for while this one ran; running once more");
+    }
+  } finally {
+    await lock.release();
+  }
+}
+
+function addSummaries(a: SyncSummary, b: SyncSummary): SyncSummary {
+  const sent = (a.sent ?? 0) + (b.sent ?? 0);
+  return {
+    qualified: a.qualified + b.qualified,
+    skipped: a.skipped + b.skipped,
+    refused: a.refused + b.refused,
+    crashed: a.crashed + b.crashed,
+    hung: a.hung + b.hung,
+    unmeasured: a.unmeasured + b.unmeasured,
+    distributed: a.distributed + b.distributed,
+    seconds: a.seconds + b.seconds,
+    ...(sent === 0 ? {} : { sent }),
+  };
+}
+
+/**
+ * One run: wait for quiet, qualify what is new or changed, drop what is gone, write the state, the records, the
+ * descriptors and the pin file, and send the changed pins to a running daemon. The state and the pin file are written
+ * whole after every item, so a run cut short keeps all it finished.
+ */
+async function runLocked(settings: SyncSettings, options: SyncRunOptions): Promise<SyncSummary> {
+  const log = options.log ?? (() => {});
+  const env = options.env ?? process.env;
+  log("sync run begins");
+  {
     const started = Date.now();
     const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
     await waitQuiet(dirs, settings.settleS, log);
@@ -1037,6 +1147,7 @@ export async function runSync(settings: SyncSettings, options: SyncRunOptions =
     const after = syncPinsText(state);
     if (after !== before) writeFileWhole(settings.pinFile, after);
     writeLv2Descriptors(ctx, lv2Pinned);
+    for (const line of descriptorAccessProblems(settings.descriptorDir)) log(line);
     // the files of a plugin that is gone, or was replaced by other bytes: only when every directory could be read, so
     // an unreadable one is never taken for an uninstall
     if (unreadable) log("a plugin directory could not be read; no record or descriptor is removed this run");
@@ -1056,8 +1167,6 @@ export async function runSync(settings: SyncSettings, options: SyncRunOptions =
     for (let i = 0; i < replies.length; i++) if (replies[i] !== "resp 0") log(`plugin-hostd answered ${replies[i]!} to ${changed[i]!.split(" ")[1]!}`);
     const taken = replies.filter((r) => r === "resp 0").length;
     return { ...count, seconds: (Date.now() - started) / 1000, ...(taken === 0 ? {} : { sent: taken }) };
-  } finally {
-    unlock();
   }
 }
 

package/tools/descriptors.py

@@ -138,16 +138,23 @@ class SchemaValidator:
             raise DescriptorRefused(DESCRIPTOR_CODE_SCHEMA, f"{where}: {e.message}")
 
 
+DESCRIPTOR_DIR_MODE = 0o755
+DESCRIPTOR_FILE_MODE = 0o644
+
+
 def write_descriptor(directory, doc, sha256, validator):
     """Hold `doc` to the schema and write it whole into `directory`; the path written, or DescriptorRefused."""
     validator.check(doc)
     name = descriptor_file_name(doc["plugin"]["id"], doc["plugin"].get("version"), sha256)
     try:
-        os.makedirs(directory, mode=0o700, exist_ok=True)
+        # the console reads this directory as another user (#221): published world-readable whatever the umask
+        os.makedirs(directory, mode=DESCRIPTOR_DIR_MODE, exist_ok=True)
+        os.chmod(directory, DESCRIPTOR_DIR_MODE)
         path = os.path.join(directory, name)
         pending = path + ".new"
         with open(pending, "w", encoding="utf-8") as f:
             f.write(json.dumps(doc, indent=2, ensure_ascii=False) + "\n")
+        os.chmod(pending, DESCRIPTOR_FILE_MODE)
         os.replace(pending, path)
     except OSError as e:
         raise DescriptorRefused(DESCRIPTOR_CODE_DIR_UNWRITABLE, f"{directory}: {e.strerror}") from e

package/tools/mod-host-admit.mjs

@@ -8,6 +8,8 @@ import { spawn } from "node:child_process";
 import { createConnection } from "node:net";
 
 const uri = process.argv[2];
+// the host to admit under: mod-host, or omx-clap-host for plugin-hostd's LV2 adapter, which answers the same protocol
+const bin = process.argv[3] ?? "mod-host";
 const port = 20000 + (process.pid % 20000);
 const STEP_MS = 10000;
 const INSTANCE = 0;
@@ -16,7 +18,7 @@ const COMMANDS = [
   ["remove", `remove ${INSTANCE}`],
 ];
 
-const host = spawn("mod-host", ["-n", "-p", String(port)], { stdio: "ignore" });
+const host = spawn(bin, ["-n", "-p", String(port)], { stdio: "ignore" });
 let hostEnd; // how the host went, once it has
 host.on("exit", (code, signal) => (hostEnd = signal ?? "exit"));
 const spawnFailed = new Promise((resolve) => host.on("error", (e) => resolve(e.code)));