@openmixer/plugin-qualify 0.18.0 → 0.18.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +93 -3
  2. package/bin/plugin-qualify-sync.mjs +3 -1
  3. package/dist/clap-qualify.d.ts +3 -1
  4. package/dist/clap-qualify.js +3 -1
  5. package/dist/clap-qualify.js.map +1 -1
  6. package/dist/distributed-records.d.ts +4 -2
  7. package/dist/distributed-records.js +1 -1
  8. package/dist/distributed-records.js.map +1 -1
  9. package/dist/host-admission.d.ts +21 -0
  10. package/dist/host-admission.js +33 -3
  11. package/dist/host-admission.js.map +1 -1
  12. package/dist/hostd-pin-files.js +3 -1
  13. package/dist/hostd-pin-files.js.map +1 -1
  14. package/dist/index.d.ts +2 -0
  15. package/dist/index.js +2 -0
  16. package/dist/index.js.map +1 -1
  17. package/dist/measurement/format.d.ts +10 -0
  18. package/dist/measurement/format.js +8 -0
  19. package/dist/measurement/format.js.map +1 -1
  20. package/dist/measurement/probe-json.js +2 -1
  21. package/dist/measurement/probe-json.js.map +1 -1
  22. package/dist/probe-judge.d.ts +11 -0
  23. package/dist/probe-judge.js +6 -3
  24. package/dist/probe-judge.js.map +1 -1
  25. package/dist/probe-runner.js +3 -2
  26. package/dist/probe-runner.js.map +1 -1
  27. package/dist/qualification-record.d.ts +113 -8
  28. package/dist/qualification-record.js +18 -13
  29. package/dist/qualification-record.js.map +1 -1
  30. package/dist/qualify-adapter-pass.d.ts +67 -0
  31. package/dist/qualify-adapter-pass.js +255 -0
  32. package/dist/qualify-adapter-pass.js.map +1 -0
  33. package/dist/qualify-declarations.d.ts +30 -0
  34. package/dist/qualify-declarations.js +24 -1
  35. package/dist/qualify-declarations.js.map +1 -1
  36. package/dist/qualify-isolated.d.ts +70 -0
  37. package/dist/qualify-isolated.js +184 -0
  38. package/dist/qualify-isolated.js.map +1 -0
  39. package/dist/qualify-rejudge.d.ts +82 -0
  40. package/dist/qualify-rejudge.js +434 -0
  41. package/dist/qualify-rejudge.js.map +1 -0
  42. package/dist/qualify-sync.d.ts +97 -1
  43. package/dist/qualify-sync.js +276 -24
  44. package/dist/qualify-sync.js.map +1 -1
  45. package/fixtures/records-0.16.1/http___example.org_raffo@unversioned-2060c09ff626.default.probe.json +46 -0
  46. package/fixtures/records-0.16.1/http___example.org_raffo@unversioned-2060c09ff626.default.qualification.json +185 -0
  47. package/fixtures/records-0.16.1/http___lv2plug.in_plugins_eg-amp@unversioned-f046a60efd45.default.probe.json +51 -0
  48. package/fixtures/records-0.16.1/http___lv2plug.in_plugins_eg-amp@unversioned-f046a60efd45.default.qualification.json +232 -0
  49. package/fixtures/records-0.16.1/https___github.com_lucianodato_noise-repellent@unversioned-c3b5e21dd9a7.default.probe.json +51 -0
  50. package/fixtures/records-0.16.1/https___github.com_lucianodato_noise-repellent@unversioned-c3b5e21dd9a7.default.qualification.json +245 -0
  51. package/fixtures/records-0.16.1/in.lsp-plug.impulse_responses_mono@unversioned-d5235d2f5f53.custom.probe.json +52 -0
  52. package/fixtures/records-0.16.1/in.lsp-plug.impulse_responses_mono@unversioned-d5235d2f5f53.custom.qualification.json +145 -0
  53. package/fixtures/records-0.16.1/in.lsp-plug.trigger_midi_mono@unversioned-d5235d2f5f53.custom.probe.json +52 -0
  54. package/fixtures/records-0.16.1/in.lsp-plug.trigger_midi_mono@unversioned-d5235d2f5f53.custom.qualification.json +155 -0
  55. package/package.json +1 -1
  56. package/src/clap-qualify.ts +5 -1
  57. package/src/distributed-records.ts +1 -1
  58. package/src/host-admission.ts +40 -3
  59. package/src/hostd-pin-files.ts +3 -1
  60. package/src/index.ts +2 -0
  61. package/src/measurement/format.ts +17 -0
  62. package/src/measurement/probe-json.ts +2 -1
  63. package/src/probe-judge.ts +10 -3
  64. package/src/probe-runner.ts +2 -2
  65. package/src/qualification-record.ts +126 -15
  66. package/src/qualify-adapter-pass.ts +278 -0
  67. package/src/qualify-declarations.ts +26 -1
  68. package/src/qualify-isolated.ts +201 -0
  69. package/src/qualify-rejudge.ts +466 -0
  70. package/src/qualify-sync.ts +295 -24
  71. package/tools/isolated-step.mjs +122 -0
  72. package/tools/lv2_resolve.py +49 -0
  73. package/tools/mod-host-admit.mjs +99 -7
@@ -18,7 +18,7 @@
18
18
  */
19
19
  import { spawn, spawnSync } from "node:child_process";
20
20
  import { createHash } from "node:crypto";
21
- import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs";
21
+ import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
22
22
  import { createConnection } from "node:net";
23
23
  import { homedir, tmpdir } from "node:os";
24
24
  import { basename, dirname, join, resolve, sep } from "node:path";
@@ -32,9 +32,12 @@ import { judgmentPinEntries, pinsFile } from "./pins-file.js";
32
32
  import { hostdPinFiles } from "./hostd-pin-files.js";
33
33
  import { qualifyTargets } from "./probe-runner.js";
34
34
  import { GRID_FULL, parseGridSpec } from "./measurement-grid.js";
35
+ import { adapterPass } from "./qualify-adapter-pass.js";
36
+ import { ISOLATED_RESULT, baseSteps, completeIsolated, isolatedCell, missingSteps, judgeIsolatedRun, owesIsolated, pinAfterStep, spawnedIsolatedRunner } from "./qualify-isolated.js";
37
+ import { lv2World, rejudgeState, memoryPass } from "./qualify-rejudge.js";
35
38
  import { memoryBoundOf, parseMemoryLimit } from "./memory-bound.js";
36
- import { writeQualification } from "./qualification-record.js";
37
- import { DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
39
+ import { rewriteRecord, writeQualification } from "./qualification-record.js";
40
+ import { ADMIT_STEP_MS_ENV, DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, MEMORY_PASS_PROBE_NOTES, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
38
41
  import { toolsDir } from "./tool-paths.js";
39
42
  import { writeFileWhole } from "./whole-file.js";
40
43
  // --- the settings -----------------------------------------------------------------------------------------------------
@@ -59,6 +62,7 @@ export const SYNC_SETTINGS = [
59
62
  { key: "grid", meaning: "the cells measured: full (every rate at every block size, 16 cells), hosts (the fast option: the host's and the console tiers' rate/quantum, then 512 at every rate; also --grid-hosts), or <rate>/<quantum>,...", default: "full" },
60
63
  { key: "cells_per_plugin_budget", meaning: "measure at most the first n cells of the grid a plugin; 0 is every cell", default: "0" },
61
64
  { key: "memory_limit", meaning: "the memory bound of every run: bytes, or with K, M, G; off for none; past it a cell is probe.oom", default: "half of the RAM" },
65
+ { key: "memory_notes", meaning: "the notes an instrument that declares no voice capacity holds in a --memory-only run, 1 to 128 (basis probe)", default: String(MEMORY_PASS_PROBE_NOTES) },
62
66
  { key: "hostd_port", meaning: "plugin-hostd's command port the new pins are sent to; 0 sends none", default: "5555" },
63
67
  { key: "hostd_feedback_port", meaning: "plugin-hostd's feedback port, connected before the command port; 0 is the command port's next", default: "0" },
64
68
  { key: "settle_s", meaning: "wait until the plugin directories have been quiet this long before a run", default: "0" },
@@ -92,6 +96,10 @@ export function parseSyncArgs(argv) {
92
96
  let noDistributed = false;
93
97
  let noWait = false;
94
98
  let refreshPinsOnly = false;
99
+ let rejudge = false;
100
+ let memoryOnly = false;
101
+ let adapterOnly = false;
102
+ let isolatedOnly = false;
95
103
  let help = false;
96
104
  for (let i = 0; i < argv.length; i++) {
97
105
  const a = argv[i];
@@ -105,6 +113,14 @@ export function parseSyncArgs(argv) {
105
113
  noWait = true;
106
114
  else if (a === "--refresh-pins")
107
115
  refreshPinsOnly = true;
116
+ else if (a === "--rejudge")
117
+ rejudge = true;
118
+ else if (a === "--memory-only")
119
+ memoryOnly = true;
120
+ else if (a === "--adapter-only")
121
+ adapterOnly = true;
122
+ else if (a === "--isolated-only")
123
+ isolatedOnly = true;
108
124
  else if (a === "--help")
109
125
  help = true;
110
126
  else if (a === "--grid-hosts")
@@ -121,7 +137,11 @@ export function parseSyncArgs(argv) {
121
137
  else
122
138
  return { ok: false, error: `unknown argument ${a}` };
123
139
  }
124
- return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, help, values } };
140
+ if (rejudge && (memoryOnly || adapterOnly || isolatedOnly || force || refreshPinsOnly))
141
+ return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --force and --refresh-pins" };
142
+ if ([memoryOnly, adapterOnly, isolatedOnly, refreshPinsOnly].filter(Boolean).length > 1)
143
+ return { ok: false, error: "--memory-only, --adapter-only and --refresh-pins are different runs: one at a time" };
144
+ return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, isolatedOnly, help, values } };
125
145
  }
126
146
  export function syncHelpText() {
127
147
  const rows = SYNC_SETTINGS.map((s) => ` ${`${flagOf(s.key)} <value>`.padEnd(36)}${s.meaning}${s.default === "" ? "" : ` (default: ${s.default})`}`);
@@ -134,6 +154,10 @@ export function syncHelpText() {
134
154
  " --no-wait a run in progress is not waited for: exit 0, and the holder runs once more at its end",
135
155
  " --no-distributed read no distributed record: every plugin not recorded is measured",
136
156
  " --refresh-pins only bring every pin's files up to the ones plugin-hostd names, measuring nothing",
157
+ " --rejudge derive every record's verdicts, tiers and pin again from the figures it holds; measures nothing",
158
+ " --memory-only measure only the footprint of each recorded plugin at the Pi's cells and merge it; with --force, those already merged too",
159
+ " --isolated-only run only the isolated step each record still owes, adding it to the record's run",
160
+ " --adapter-only run each LV2 plugin only under omx-clap-host's LV2 adapter at the Pi's cells, on private JACK servers, and merge it; with --force, those already done too",
137
161
  " --grid-hosts the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
138
162
  " --help this text",
139
163
  ...rows,
@@ -184,6 +208,9 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
184
208
  if (memoryLimit !== undefined && !memoryLimit.ok)
185
209
  return { ok: false, error: `memory_limit: ${memoryLimit.error}` };
186
210
  const memory = memoryBoundOf(memoryLimit?.bytes);
211
+ const memoryNotes = number("memory_notes", 1);
212
+ if (memoryNotes === undefined || !Number.isInteger(memoryNotes) || memoryNotes > 128)
213
+ return { ok: false, error: "memory_notes must be a whole number of notes, 1 to 128" };
187
214
  const profileId = get("host_profile") || "omx-clap-host";
188
215
  const hostProfile = shippedHostProfile(profileId);
189
216
  if (hostProfile === undefined)
@@ -215,6 +242,7 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
215
242
  grid: grid.spec,
216
243
  cellsPerPluginBudget,
217
244
  ...(memory === undefined ? {} : { memory }),
245
+ memoryNotes,
218
246
  hostdPort,
219
247
  hostdFeedbackPort: feedbackPort === 0 && hostdPort !== 0 ? hostdPort + 1 : feedbackPort,
220
248
  settleS,
@@ -224,6 +252,29 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
224
252
  },
225
253
  };
226
254
  }
255
+ /** lilv itself, one world over the whole search path (tools/lv2_resolve.py), as every host built on it resolves (#199). */
256
+ export const LILV_RESOLVE = (dirs) => {
257
+ const r = spawnSync("python3", [join(toolsDir(), "lv2_resolve.py")], {
258
+ encoding: "utf8", timeout: 120_000, killSignal: "SIGKILL", maxBuffer: 64 << 20, env: { ...process.env, LV2_PATH: dirs.join(":") },
259
+ });
260
+ if (r.error !== undefined || r.status !== 0)
261
+ return { problem: (r.stderr ?? "").trim().split("\n").pop() || (r.error?.message ?? `exit ${String(r.status)}`) };
262
+ try {
263
+ return JSON.parse(r.stdout).plugins;
264
+ }
265
+ catch (e) {
266
+ return { problem: `lv2_resolve.py printed no JSON: ${e.message}` };
267
+ }
268
+ };
269
+ /** Whether `path` exists and is, through its links, the file `target` is. */
270
+ function linksTo(path, target) {
271
+ try {
272
+ return realpathSync(path) === realpathSync(target);
273
+ }
274
+ catch {
275
+ return false;
276
+ }
277
+ }
227
278
  const CLAP_SUFFIX = ".clap";
228
279
  /** Every `.clap` under a directory, its subdirectories included, in name order; a directory reached twice is walked once. */
229
280
  function clapFiles(dir, seen, problems) {
@@ -270,9 +321,10 @@ function clapFiles(dir, seen, problems) {
270
321
  * LV2 directories that declares it, as plugin-hostd finds it. A directory that cannot be read is a problem named, and
271
322
  * the others are still walked.
272
323
  */
273
- export function discoverSyncItems(settings) {
324
+ export function discoverSyncItems(settings, lv2Resolve) {
274
325
  const items = [];
275
326
  const problems = [];
327
+ const notes = [];
276
328
  const seen = new Set();
277
329
  const clap = new Set();
278
330
  for (const dir of settings.clapDirs) {
@@ -284,6 +336,14 @@ export function discoverSyncItems(settings) {
284
336
  items.push({ key: `clap:${path}`, format: "clap", root: dirname(path), files: [basename(path)] });
285
337
  }
286
338
  }
339
+ // the bundle lilv takes for each plugin uri; without lilv, the first bundle found (and the log says so)
340
+ const resolved = lv2Resolve === undefined || settings.lv2Dirs.length === 0 ? undefined : lv2Resolve(settings.lv2Dirs);
341
+ const taken = new Map();
342
+ if (resolved !== undefined && "problem" in resolved)
343
+ notes.push(`lilv could not resolve the LV2 plugins (${resolved.problem}); each is keyed by the first bundle found`);
344
+ else if (resolved !== undefined)
345
+ for (const p of resolved)
346
+ taken.set(p.uri, p);
287
347
  const lv2 = new Set();
288
348
  for (const dir of settings.lv2Dirs) {
289
349
  let bundles;
@@ -304,14 +364,23 @@ export function discoverSyncItems(settings) {
304
364
  continue;
305
365
  }
306
366
  for (const p of manifestPlugins(ttl)) {
367
+ const lilv = taken.get(p.uri);
368
+ // lilv takes another bundle for this uri: this one is not the plugin a host loads
369
+ if (lilv !== undefined && resolve(lilv.bundle) !== resolve(bundle))
370
+ continue;
307
371
  if (lv2.has(p.uri))
308
372
  continue;
309
373
  lv2.add(p.uri);
310
- items.push({ key: `lv2:${p.uri}`, format: "lv2", root: bundle, files: ["manifest.ttl", ...p.files.filter((f) => f !== "manifest.ttl")], uri: p.uri });
374
+ const binary = lilv?.binary ?? (p.binary === undefined ? undefined : join(bundle, p.binary));
375
+ const files = ["manifest.ttl", ...p.files.filter((f) => f !== "manifest.ttl")];
376
+ // a binary named by an absolute IRI (a complement's) is tracked through the bundle's link to it, when it has one
377
+ if (binary !== undefined && !files.includes(basename(binary)) && linksTo(join(bundle, basename(binary)), binary))
378
+ files.push(basename(binary));
379
+ items.push({ key: `lv2:${p.uri}`, format: "lv2", root: bundle, files, uri: p.uri, ...(binary === undefined ? {} : { binary }) });
311
380
  }
312
381
  }
313
382
  }
314
- return { items, problems };
383
+ return { items, problems, notes };
315
384
  }
316
385
  /** The SHA-256 of a file's bytes, read in pieces over one descriptor. */
317
386
  export function fileSha256(path) {
@@ -365,6 +434,20 @@ export const SYNC_CODE = {
365
434
  /** The probe runner wrote no document to judge. */
366
435
  notMeasured: "qualify.sync.not-measured",
367
436
  };
437
+ /**
438
+ * Why a plugin is unmeasured because of the machine it ran on rather than what it is (#192): an EACCES, EPERM or
439
+ * ENOSPC the run hit, or a measurement that could not run or wrote no document. Such a plugin is measured again by
440
+ * the next run although its files are unchanged; a reason of the plugin's own (probe.limit.not-implemented, no such
441
+ * plugin, a crash) is not. `undefined` for every other outcome.
442
+ */
443
+ export function machineReason(p) {
444
+ if (p.outcome !== "unmeasured" || p.code !== SYNC_CODE.notMeasured)
445
+ return undefined;
446
+ const errno = /\b(EACCES|EPERM|ENOSPC)\b/.exec(p.detail ?? "")?.[1];
447
+ if (errno !== undefined)
448
+ return errno;
449
+ return /wrote no document|cannot run the probe runner/.test(p.detail ?? "") ? "the measurement wrote no document" : undefined;
450
+ }
368
451
  /** What the quick qualification never measures, stated in every record it writes. */
369
452
  export const QUICK_NOT_MEASURED = [{ dimension: "stability", code: HOSTING_CODE.stabilityUnsoaked }];
370
453
  const EMPTY_STATE = { schema: SYNC_STATE_SCHEMA, entries: {} };
@@ -402,6 +485,32 @@ export function recordedAsIs(item, entry) {
402
485
  const same = now.length === entry.files.length && now.every((f) => entry.files.some((e) => e.path === f.path && e.sha256 === f.sha256));
403
486
  return { same, files: now };
404
487
  }
488
+ /**
489
+ * The entry of an LV2 plugin a host now loads from another bundle than it was qualified in, when its binary is the
490
+ * same bytes (#199): the entry with the new bundle and the new bundle's file digests, its verdicts and records kept.
491
+ * `undefined` when the item is not that case: not LV2, no entry, the same bundle, a binary that cannot be read, or one
492
+ * whose bytes differ from the recorded binary's (then it is measured again).
493
+ */
494
+ export function sameBinaryElsewhere(item, entry) {
495
+ if (item.format !== "lv2" || entry === undefined || item.binary === undefined || resolve(entry.root) === resolve(item.root))
496
+ return undefined;
497
+ const was = entry.files.find((f) => basename(f.path) === basename(item.binary));
498
+ if (was === undefined)
499
+ return undefined;
500
+ let now;
501
+ try {
502
+ now = fileSha256(item.binary);
503
+ }
504
+ catch {
505
+ return undefined;
506
+ }
507
+ if (now !== was.sha256)
508
+ return undefined;
509
+ const files = digestFiles(item.root, item.files);
510
+ if (files === undefined)
511
+ return undefined;
512
+ return { ...entry, root: item.root, files };
513
+ }
405
514
  /**
406
515
  * The pin of a judged run with the files plugin-hostd collects for its plugin (plugin-qualify#228): the probe's list is
407
516
  * the probe's view, and an add checks the daemon's, so a pin holds the daemon's. No pin when plugin-hostd cannot name
@@ -485,13 +594,42 @@ function writeUnmeasuredRecord(ctx, item, files, plugin) {
485
594
  writeFileWhole(path, `${JSON.stringify(record, null, 2)}\n`);
486
595
  return path;
487
596
  }
597
+ /** The step's environment: each of its commands has the sync's budget. */
598
+ const isolatedEnv = (ctx) => ({ ...ctx.env, [ADMIT_STEP_MS_ENV]: String(ctx.settings.budgetS * 1000) });
488
599
  /** Each judged run of a pass as the plugins it qualified: the record written, the pin taken, the outcome. */
489
- function judgedPlugins(ctx, runs) {
600
+ function judgedPlugins(ctx, runs, describeProblem) {
490
601
  const host = ctx.settings.hostProfile.id;
491
602
  const entries = runs.map(({ bytes, judgment }) => {
492
- const { recordPath } = writeQualification(ctx.settings.qualificationDir, bytes, judgment, QUICK_NOT_MEASURED);
493
- const pin = withHostdFiles(ctx, judgmentPinEntries(judgment, host)[0]);
494
- const { outcome, code } = outcomeOf(judgment, host);
603
+ const { recordPath, record } = writeQualification(ctx.settings.qualificationDir, bytes, judgment, QUICK_NOT_MEASURED);
604
+ const steps = baseSteps(record, describeProblem);
605
+ let pin = withHostdFiles(ctx, judgmentPinEntries(judgment, host)[0]);
606
+ let { outcome, code } = outcomeOf(judgment, host);
607
+ if (owesIsolated(record)) {
608
+ // in this same pass: a record's verdict=isolated is a run seen alone, never a rating alone
609
+ const uri = pin?.uri ?? judgment.output.plugin.id;
610
+ const runner = ctx.isolatedRunner ?? spawnedIsolatedRunner();
611
+ const step = judgeIsolatedRun(runner(uri, isolatedCell(record, host), isolatedEnv(ctx)));
612
+ steps.push(step);
613
+ pin = pinAfterStep(pin, step);
614
+ // a crash or a hang in the probe stays what it was unless the step saw the plugin run: only a rating waited on the step
615
+ if (step.result === ISOLATED_RESULT.refused) {
616
+ if (outcome === "qualified") {
617
+ outcome = "refused";
618
+ code = step.code;
619
+ }
620
+ }
621
+ else if (step.result === ISOLATED_RESULT.notRun) {
622
+ if (outcome === "qualified") {
623
+ outcome = "unmeasured";
624
+ code = step.code;
625
+ }
626
+ }
627
+ else if (outcome !== "qualified") {
628
+ outcome = "qualified";
629
+ code = undefined;
630
+ }
631
+ }
632
+ rewriteRecord(recordPath, { ...record, run: { id: judgment.output.run.id, steps } });
495
633
  const plugin = { uri: pin?.uri ?? judgment.output.plugin.id, outcome, ...(code === undefined ? {} : { code }), ...(pin === undefined ? {} : { pin }), record: recordPath };
496
634
  return { plugin, judgedHere: judgment.hosts.some((h) => h.verdict.host === host) };
497
635
  });
@@ -589,21 +727,16 @@ function qualifyClap(ctx, item) {
589
727
  for (const f of pass.failures)
590
728
  out.push({ uri, outcome: "unmeasured", code: SYNC_CODE.notMeasured, detail: f.problem });
591
729
  const problem = descriptorFailed.get(p.id);
592
- out.push(...judgedPlugins(ctx, pass.runs).map((j) => (problem === undefined ? j : { ...j, descriptorProblem: problem })));
730
+ out.push(...judgedPlugins(ctx, pass.runs, problem).map((j) => (problem === undefined ? j : { ...j, descriptorProblem: problem })));
593
731
  }
594
732
  return out;
595
733
  }
596
734
  /** An LV2 plugin, measured in a world of its own bundle only, as plugin-hostd runs a pinned LV2 plugin. */
597
735
  function qualifyLv2(ctx, item) {
598
- const world = mkdtempSync(join(tmpdir(), "plugin-qualify-sync-world-"));
599
- try {
600
- symlinkSync(item.root, join(world, basename(item.root)));
736
+ return lv2World(item.root, (world) => {
601
737
  const pass = runPass(ctx, { format: "lv2", uri: item.uri, bundle: item.root }, { ...ctx.env, LV2_PATH: world });
602
738
  return [...pass.failures.map((f) => ({ uri: item.uri, outcome: "unmeasured", code: SYNC_CODE.notMeasured, detail: f.problem })), ...judgedPlugins(ctx, pass.runs)];
603
- }
604
- finally {
605
- rmSync(world, { recursive: true, force: true });
606
- }
739
+ });
607
740
  }
608
741
  /** One item qualified, whatever happens to it: an error of the sync's own while measuring it is its outcome, not the run's end. */
609
742
  export function qualifyItem(ctx, item, files) {
@@ -623,6 +756,50 @@ export function qualifyItem(ctx, item, files) {
623
756
  const all = digestFiles(item.root, [...item.files, ...pinned], files) ?? [...files];
624
757
  return { format: item.format, root: item.root, files: all, plugins, qualifiedAt: new Date(started).toISOString(), seconds: (Date.now() - started) / 1000 };
625
758
  }
759
+ /**
760
+ * The isolated steps the records of a state still owe, each extending its record's run. A plugin's pin follows its step, and
761
+ * a step that could not run stays owed: the next pass tries it again.
762
+ */
763
+ export function isolatedPass(ctx, state) {
764
+ const count = { isolated: 0, refused: 0, notRun: 0 };
765
+ const runner = ctx.isolatedRunner ?? spawnedIsolatedRunner();
766
+ const host = ctx.settings.hostProfile.id;
767
+ const entries = {};
768
+ for (const [key, entry] of Object.entries(state.entries)) {
769
+ entries[key] = {
770
+ ...entry,
771
+ plugins: entry.plugins.map((p) => {
772
+ if (p.record === undefined || !p.record.endsWith(".qualification.json"))
773
+ return p;
774
+ const path = join(ctx.settings.qualificationDir, basename(p.record));
775
+ let record;
776
+ try {
777
+ record = JSON.parse(readFileSync(path, "utf8"));
778
+ }
779
+ catch {
780
+ ctx.log(`${p.uri}: its record does not read; no isolated step`);
781
+ return p;
782
+ }
783
+ const done = completeIsolated(path, record, p.pin?.uri ?? record.plugin.id, isolatedCell(record, host), runner, isolatedEnv(ctx));
784
+ if (done === undefined)
785
+ return p;
786
+ ctx.log(`${p.uri}: isolated step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
787
+ const pin = pinAfterStep(p.pin, done.step);
788
+ const { pin: _pin, code: _code, ...rest } = p;
789
+ if (done.step.result === ISOLATED_RESULT.isolated) {
790
+ count.isolated++;
791
+ return { ...rest, outcome: "qualified", ...(pin === undefined ? {} : { pin }) };
792
+ }
793
+ if (done.step.result === ISOLATED_RESULT.refused)
794
+ count.refused++;
795
+ else
796
+ count.notRun++;
797
+ return { ...rest, outcome: done.step.result === ISOLATED_RESULT.refused ? "refused" : "unmeasured", ...(done.step.code === undefined ? {} : { code: done.step.code }), ...(pin === undefined ? {} : { pin }) };
798
+ }),
799
+ };
800
+ }
801
+ return { ...count, state: count.isolated + count.refused + count.notRun === 0 ? state : { ...state, entries } };
802
+ }
626
803
  // --- the distributed records ----------------------------------------------------------------------------------------
627
804
  /** The layer a run reads: none under `--force` or `--no-distributed`, or without a host class. */
628
805
  function distributedLayerOf(settings, force, log) {
@@ -656,6 +833,13 @@ function takeDistributed(ctx, layer, item, files) {
656
833
  ctx.log(line);
657
834
  return undefined;
658
835
  }
836
+ for (const r of match.records) {
837
+ const missing = missingSteps(r.record);
838
+ if (missing.length > 0) {
839
+ ctx.log(`distributed: ${distributedUri(item, r)}: its record lacks the step${missing.length === 1 ? "" : "s"} ${missing.join(", ")}; measured here`);
840
+ return undefined;
841
+ }
842
+ }
659
843
  let plugins;
660
844
  try {
661
845
  plugins = applyDistributed(ctx.settings.qualificationDir, item, match, host);
@@ -913,7 +1097,7 @@ export function descriptorAccessProblems(dir) {
913
1097
  return problems;
914
1098
  }
915
1099
  export function summaryLine(s) {
916
- return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}, ${s.seconds.toFixed(1)} s`;
1100
+ return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
917
1101
  }
918
1102
  /** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
919
1103
  const LOCK_HELD = 99;
@@ -995,6 +1179,10 @@ function addSummaries(a, b) {
995
1179
  seconds: a.seconds + b.seconds,
996
1180
  ...(sent === 0 ? {} : { sent }),
997
1181
  ...(refreshed === 0 ? {} : { refreshed }),
1182
+ ...(b.rejudge === undefined ? (a.rejudge === undefined ? {} : { rejudge: a.rejudge }) : { rejudge: b.rejudge }),
1183
+ ...(b.memoryOnly === undefined ? (a.memoryOnly === undefined ? {} : { memoryOnly: a.memoryOnly }) : { memoryOnly: b.memoryOnly }),
1184
+ ...(b.isolatedOnly === undefined ? (a.isolatedOnly === undefined ? {} : { isolatedOnly: a.isolatedOnly }) : { isolatedOnly: b.isolatedOnly }),
1185
+ ...(b.adapterOnly === undefined ? (a.adapterOnly === undefined ? {} : { adapterOnly: a.adapterOnly }) : { adapterOnly: b.adapterOnly }),
998
1186
  };
999
1187
  }
1000
1188
  /** Whether this run's scan covers an entry: its root lies under one of the run's directories of its own format. */
@@ -1013,11 +1201,11 @@ function scannedBy(settings, entry) {
1013
1201
  async function runLocked(settings, options) {
1014
1202
  const log = options.log ?? (() => { });
1015
1203
  const env = options.env ?? process.env;
1016
- log(options.refreshOnly === true ? "pin refresh begins" : "sync run begins");
1204
+ log(options.rejudge === true ? "re-judge begins: nothing is measured" : options.memoryOnly === true ? "memory-only pass begins" : options.adapterOnly === true ? "adapter-only pass begins" : options.refreshOnly === true ? "pin refresh begins" : "sync run begins");
1017
1205
  {
1018
1206
  const started = Date.now();
1019
1207
  const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
1020
- const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }) };
1208
+ const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }), ...(options.isolatedRunner === undefined ? {} : { isolatedRunner: options.isolatedRunner }) };
1021
1209
  // the pins first, before the quiet wait and any measurement: an upgrade of plugin-hostd that collects more files
1022
1210
  // than the pins on disk hold refuses those plugins until their pins name them (plugin-qualify#239)
1023
1211
  const fresh = refreshPins(ctx, readSyncState(settings.stateFile, log));
@@ -1026,6 +1214,31 @@ async function runLocked(settings, options) {
1026
1214
  }
1027
1215
  if (options.refreshOnly === true)
1028
1216
  return sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
1217
+ if (options.rejudge === true) {
1218
+ const r = rejudgeState(ctx, fresh.state);
1219
+ if (JSON.stringify(r.state) !== JSON.stringify(fresh.state))
1220
+ writeFileWhole(settings.stateFile, `${JSON.stringify(r.state, null, 1)}\n`);
1221
+ const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
1222
+ return { ...summary, rejudge: { changed: r.changed, unchanged: r.unchanged, owing: r.owing.length } };
1223
+ }
1224
+ if (options.isolatedOnly === true) {
1225
+ const r = isolatedPass(ctx, fresh.state);
1226
+ if (r.state !== fresh.state)
1227
+ writeFileWhole(settings.stateFile, `${JSON.stringify(r.state, null, 1)}\n`);
1228
+ const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
1229
+ return { ...summary, isolatedOnly: { isolated: r.isolated, refused: r.refused, notRun: r.notRun } };
1230
+ }
1231
+ if (options.adapterOnly === true) {
1232
+ const r = adapterPass(ctx, fresh.state, { ...options.adapterPass, force: options.force === true });
1233
+ log(`adapter runs under ${Object.entries(r.versions).map(([k, v]) => `${k} ${v}`).join(", ")}`);
1234
+ const summary = await sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
1235
+ return { ...summary, adapterOnly: { measured: r.measured, already: r.already, failed: r.failed } };
1236
+ }
1237
+ if (options.memoryOnly === true) {
1238
+ const r = memoryPass(ctx, fresh.state, { force: options.force === true, ...(options.probeRun === undefined ? {} : { run: options.probeRun }) });
1239
+ const summary = await sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
1240
+ return { ...summary, memoryOnly: { measured: r.measured, already: r.already, failed: r.failed } };
1241
+ }
1029
1242
  await waitQuiet(dirs, settings.settleS, log);
1030
1243
  const count = { qualified: 0, skipped: 0, refused: 0, crashed: 0, hung: 0, unmeasured: 0, distributed: 0 };
1031
1244
  const layer = distributedLayerOf(settings, options.force === true, log);
@@ -1040,12 +1253,18 @@ async function runLocked(settings, options) {
1040
1253
  const stateBefore = state;
1041
1254
  let unreadable = false;
1042
1255
  const lv2Pinned = [];
1256
+ // the plugins measured this run, and how many of them the machine left unmeasured, by reason (#192)
1257
+ let measuredHere = 0;
1258
+ const byMachine = new Map();
1043
1259
  // a change made while a pass ran is caught by one more pass, a few at most
1044
1260
  for (let pass = 0; pass < 3; pass++) {
1045
1261
  const signature = directoriesSignature(dirs);
1046
- const { items, problems } = discoverSyncItems(settings);
1262
+ const { items, problems, notes } = discoverSyncItems(settings, options.lv2Resolve ?? LILV_RESOLVE);
1047
1263
  for (const p of problems)
1048
1264
  log(p);
1265
+ if (pass === 0)
1266
+ for (const n of notes)
1267
+ log(n);
1049
1268
  unreadable = problems.length > 0;
1050
1269
  const found = new Set(items.map((i) => i.key));
1051
1270
  const entries = {};
@@ -1063,11 +1282,29 @@ async function runLocked(settings, options) {
1063
1282
  }
1064
1283
  for (const item of items) {
1065
1284
  const recorded = recordedAsIs(item, entries[item.key]);
1066
- if (recorded.same && options.force !== true) {
1285
+ const retry = recorded.same ? entries[item.key].plugins.map(machineReason).find((r) => r !== undefined) : undefined;
1286
+ if (recorded.same && options.force !== true && retry === undefined) {
1067
1287
  if (pass === 0)
1068
1288
  count.skipped += entries[item.key].plugins.length;
1069
1289
  continue;
1070
1290
  }
1291
+ // an LV2 plugin a host now loads from another bundle with the same binary (a complement that restates its TTL,
1292
+ // #199): the record stands, keyed by its uri and its binary; only the bundle, its files and the pins' files move
1293
+ const moved = options.force === true ? undefined : sameBinaryElsewhere(item, entries[item.key]);
1294
+ if (moved !== undefined) {
1295
+ entries[item.key] = { ...moved, plugins: moved.plugins.map((p) => ({ ...p, ...(p.pin === undefined ? {} : { pin: withHostdFiles(ctx, p.pin) }) })) };
1296
+ if (item.format === "lv2")
1297
+ lv2Pinned.push(...entries[item.key].plugins.filter((p) => p.pin !== undefined).map((p) => p.uri));
1298
+ if (pass === 0)
1299
+ count.skipped += moved.plugins.length;
1300
+ log(`${item.key}: loaded from ${item.root} now, with the binary it was qualified with; its record stands and its pin's files follow the bundle`);
1301
+ state = { schema: SYNC_STATE_SCHEMA, entries };
1302
+ writeFileWhole(settings.stateFile, `${JSON.stringify(state, null, 1)}\n`);
1303
+ writeFileWhole(settings.pinFile, syncPinsText(state));
1304
+ continue;
1305
+ }
1306
+ if (recorded.same && retry !== undefined)
1307
+ log(`${item.key}: unmeasured last run by the machine (${retry}); measured again`);
1071
1308
  const files = recorded.files ?? digestFiles(item.root, item.files);
1072
1309
  if (files === undefined) {
1073
1310
  log(`${item.key}: a file of it cannot be read; skipped this run`);
@@ -1092,6 +1329,13 @@ async function runLocked(settings, options) {
1092
1329
  lv2Pinned.push(...entry.plugins.filter((p) => p.pin !== undefined).map((p) => p.uri));
1093
1330
  for (const p of entry.plugins) {
1094
1331
  count[p.outcome]++;
1332
+ measuredHere++;
1333
+ // a reason of the machine is said once: the plugins after the first are counted, not listed
1334
+ const why = machineReason(p);
1335
+ if (why !== undefined)
1336
+ byMachine.set(why, (byMachine.get(why) ?? 0) + 1);
1337
+ if (why !== undefined && byMachine.get(why) > 1)
1338
+ continue;
1095
1339
  log(`${p.uri}: ${p.outcome}${p.code === undefined ? "" : ` (${p.code})`}${p.detail === undefined ? "" : `: ${p.detail}`}`);
1096
1340
  }
1097
1341
  state = { schema: SYNC_STATE_SCHEMA, entries };
@@ -1133,8 +1377,16 @@ async function runLocked(settings, options) {
1133
1377
  if (replies[i] !== "resp 0")
1134
1378
  log(`plugin-hostd answered ${replies[i]} to ${changed[i].split(" ")[1]}`);
1135
1379
  const taken = replies.filter((r) => r === "resp 0").length;
1380
+ for (const [why, n] of byMachine)
1381
+ if (n > 1)
1382
+ log(`${String(n)} plugins unmeasured by the machine (${why}); the first is listed above`);
1383
+ const [only] = byMachine.size === 1 ? [...byMachine] : [];
1384
+ const machineFailure = only !== undefined && only[1] === measuredHere ? only[0] : undefined;
1385
+ if (machineFailure !== undefined)
1386
+ log(`every plugin this run measured was left unmeasured by the machine (${machineFailure}): the run failed`);
1136
1387
  return {
1137
1388
  ...count,
1389
+ ...(machineFailure === undefined ? {} : { machineFailure }),
1138
1390
  seconds: (Date.now() - started) / 1000,
1139
1391
  ...(taken === 0 ? {} : { sent: taken }),
1140
1392
  ...(fresh.refreshed.length === 0 ? {} : { refreshed: fresh.refreshed.length }),