@openmixer/plugin-qualify 0.18.0 → 0.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -3
- package/bin/plugin-qualify-sync.mjs +3 -1
- package/dist/clap-qualify.d.ts +3 -1
- package/dist/clap-qualify.js +3 -1
- package/dist/clap-qualify.js.map +1 -1
- package/dist/distributed-records.d.ts +4 -2
- package/dist/distributed-records.js +1 -1
- package/dist/distributed-records.js.map +1 -1
- package/dist/host-admission.d.ts +21 -0
- package/dist/host-admission.js +33 -3
- package/dist/host-admission.js.map +1 -1
- package/dist/hostd-pin-files.js +3 -1
- package/dist/hostd-pin-files.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/measurement/format.d.ts +10 -0
- package/dist/measurement/format.js +8 -0
- package/dist/measurement/format.js.map +1 -1
- package/dist/measurement/probe-json.js +2 -1
- package/dist/measurement/probe-json.js.map +1 -1
- package/dist/probe-judge.d.ts +11 -0
- package/dist/probe-judge.js +6 -3
- package/dist/probe-judge.js.map +1 -1
- package/dist/probe-runner.js +3 -2
- package/dist/probe-runner.js.map +1 -1
- package/dist/qualification-record.d.ts +113 -8
- package/dist/qualification-record.js +18 -13
- package/dist/qualification-record.js.map +1 -1
- package/dist/qualify-adapter-pass.d.ts +67 -0
- package/dist/qualify-adapter-pass.js +255 -0
- package/dist/qualify-adapter-pass.js.map +1 -0
- package/dist/qualify-declarations.d.ts +30 -0
- package/dist/qualify-declarations.js +24 -1
- package/dist/qualify-declarations.js.map +1 -1
- package/dist/qualify-isolated.d.ts +70 -0
- package/dist/qualify-isolated.js +184 -0
- package/dist/qualify-isolated.js.map +1 -0
- package/dist/qualify-rejudge.d.ts +82 -0
- package/dist/qualify-rejudge.js +434 -0
- package/dist/qualify-rejudge.js.map +1 -0
- package/dist/qualify-sync.d.ts +97 -1
- package/dist/qualify-sync.js +276 -24
- package/dist/qualify-sync.js.map +1 -1
- package/fixtures/records-0.16.1/http___example.org_raffo@unversioned-2060c09ff626.default.probe.json +46 -0
- package/fixtures/records-0.16.1/http___example.org_raffo@unversioned-2060c09ff626.default.qualification.json +185 -0
- package/fixtures/records-0.16.1/http___lv2plug.in_plugins_eg-amp@unversioned-f046a60efd45.default.probe.json +51 -0
- package/fixtures/records-0.16.1/http___lv2plug.in_plugins_eg-amp@unversioned-f046a60efd45.default.qualification.json +232 -0
- package/fixtures/records-0.16.1/https___github.com_lucianodato_noise-repellent@unversioned-c3b5e21dd9a7.default.probe.json +51 -0
- package/fixtures/records-0.16.1/https___github.com_lucianodato_noise-repellent@unversioned-c3b5e21dd9a7.default.qualification.json +245 -0
- package/fixtures/records-0.16.1/in.lsp-plug.impulse_responses_mono@unversioned-d5235d2f5f53.custom.probe.json +52 -0
- package/fixtures/records-0.16.1/in.lsp-plug.impulse_responses_mono@unversioned-d5235d2f5f53.custom.qualification.json +145 -0
- package/fixtures/records-0.16.1/in.lsp-plug.trigger_midi_mono@unversioned-d5235d2f5f53.custom.probe.json +52 -0
- package/fixtures/records-0.16.1/in.lsp-plug.trigger_midi_mono@unversioned-d5235d2f5f53.custom.qualification.json +155 -0
- package/package.json +1 -1
- package/src/clap-qualify.ts +5 -1
- package/src/distributed-records.ts +1 -1
- package/src/host-admission.ts +40 -3
- package/src/hostd-pin-files.ts +3 -1
- package/src/index.ts +2 -0
- package/src/measurement/format.ts +17 -0
- package/src/measurement/probe-json.ts +2 -1
- package/src/probe-judge.ts +10 -3
- package/src/probe-runner.ts +2 -2
- package/src/qualification-record.ts +126 -15
- package/src/qualify-adapter-pass.ts +278 -0
- package/src/qualify-declarations.ts +26 -1
- package/src/qualify-isolated.ts +201 -0
- package/src/qualify-rejudge.ts +466 -0
- package/src/qualify-sync.ts +295 -24
- package/tools/isolated-step.mjs +122 -0
- package/tools/lv2_resolve.py +49 -0
- package/tools/mod-host-admit.mjs +99 -7
package/dist/qualify-sync.js
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import { spawn, spawnSync } from "node:child_process";
|
|
20
20
|
import { createHash } from "node:crypto";
|
|
21
|
-
import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync,
|
|
21
|
+
import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
22
22
|
import { createConnection } from "node:net";
|
|
23
23
|
import { homedir, tmpdir } from "node:os";
|
|
24
24
|
import { basename, dirname, join, resolve, sep } from "node:path";
|
|
@@ -32,9 +32,12 @@ import { judgmentPinEntries, pinsFile } from "./pins-file.js";
|
|
|
32
32
|
import { hostdPinFiles } from "./hostd-pin-files.js";
|
|
33
33
|
import { qualifyTargets } from "./probe-runner.js";
|
|
34
34
|
import { GRID_FULL, parseGridSpec } from "./measurement-grid.js";
|
|
35
|
+
import { adapterPass } from "./qualify-adapter-pass.js";
|
|
36
|
+
import { ISOLATED_RESULT, baseSteps, completeIsolated, isolatedCell, missingSteps, judgeIsolatedRun, owesIsolated, pinAfterStep, spawnedIsolatedRunner } from "./qualify-isolated.js";
|
|
37
|
+
import { lv2World, rejudgeState, memoryPass } from "./qualify-rejudge.js";
|
|
35
38
|
import { memoryBoundOf, parseMemoryLimit } from "./memory-bound.js";
|
|
36
|
-
import { writeQualification } from "./qualification-record.js";
|
|
37
|
-
import { DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
|
|
39
|
+
import { rewriteRecord, writeQualification } from "./qualification-record.js";
|
|
40
|
+
import { ADMIT_STEP_MS_ENV, DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, MEMORY_PASS_PROBE_NOTES, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
|
|
38
41
|
import { toolsDir } from "./tool-paths.js";
|
|
39
42
|
import { writeFileWhole } from "./whole-file.js";
|
|
40
43
|
// --- the settings -----------------------------------------------------------------------------------------------------
|
|
@@ -59,6 +62,7 @@ export const SYNC_SETTINGS = [
|
|
|
59
62
|
{ key: "grid", meaning: "the cells measured: full (every rate at every block size, 16 cells), hosts (the fast option: the host's and the console tiers' rate/quantum, then 512 at every rate; also --grid-hosts), or <rate>/<quantum>,...", default: "full" },
|
|
60
63
|
{ key: "cells_per_plugin_budget", meaning: "measure at most the first n cells of the grid a plugin; 0 is every cell", default: "0" },
|
|
61
64
|
{ key: "memory_limit", meaning: "the memory bound of every run: bytes, or with K, M, G; off for none; past it a cell is probe.oom", default: "half of the RAM" },
|
|
65
|
+
{ key: "memory_notes", meaning: "the notes an instrument that declares no voice capacity holds in a --memory-only run, 1 to 128 (basis probe)", default: String(MEMORY_PASS_PROBE_NOTES) },
|
|
62
66
|
{ key: "hostd_port", meaning: "plugin-hostd's command port the new pins are sent to; 0 sends none", default: "5555" },
|
|
63
67
|
{ key: "hostd_feedback_port", meaning: "plugin-hostd's feedback port, connected before the command port; 0 is the command port's next", default: "0" },
|
|
64
68
|
{ key: "settle_s", meaning: "wait until the plugin directories have been quiet this long before a run", default: "0" },
|
|
@@ -92,6 +96,10 @@ export function parseSyncArgs(argv) {
|
|
|
92
96
|
let noDistributed = false;
|
|
93
97
|
let noWait = false;
|
|
94
98
|
let refreshPinsOnly = false;
|
|
99
|
+
let rejudge = false;
|
|
100
|
+
let memoryOnly = false;
|
|
101
|
+
let adapterOnly = false;
|
|
102
|
+
let isolatedOnly = false;
|
|
95
103
|
let help = false;
|
|
96
104
|
for (let i = 0; i < argv.length; i++) {
|
|
97
105
|
const a = argv[i];
|
|
@@ -105,6 +113,14 @@ export function parseSyncArgs(argv) {
|
|
|
105
113
|
noWait = true;
|
|
106
114
|
else if (a === "--refresh-pins")
|
|
107
115
|
refreshPinsOnly = true;
|
|
116
|
+
else if (a === "--rejudge")
|
|
117
|
+
rejudge = true;
|
|
118
|
+
else if (a === "--memory-only")
|
|
119
|
+
memoryOnly = true;
|
|
120
|
+
else if (a === "--adapter-only")
|
|
121
|
+
adapterOnly = true;
|
|
122
|
+
else if (a === "--isolated-only")
|
|
123
|
+
isolatedOnly = true;
|
|
108
124
|
else if (a === "--help")
|
|
109
125
|
help = true;
|
|
110
126
|
else if (a === "--grid-hosts")
|
|
@@ -121,7 +137,11 @@ export function parseSyncArgs(argv) {
|
|
|
121
137
|
else
|
|
122
138
|
return { ok: false, error: `unknown argument ${a}` };
|
|
123
139
|
}
|
|
124
|
-
|
|
140
|
+
if (rejudge && (memoryOnly || adapterOnly || isolatedOnly || force || refreshPinsOnly))
|
|
141
|
+
return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --force and --refresh-pins" };
|
|
142
|
+
if ([memoryOnly, adapterOnly, isolatedOnly, refreshPinsOnly].filter(Boolean).length > 1)
|
|
143
|
+
return { ok: false, error: "--memory-only, --adapter-only and --refresh-pins are different runs: one at a time" };
|
|
144
|
+
return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, isolatedOnly, help, values } };
|
|
125
145
|
}
|
|
126
146
|
export function syncHelpText() {
|
|
127
147
|
const rows = SYNC_SETTINGS.map((s) => ` ${`${flagOf(s.key)} <value>`.padEnd(36)}${s.meaning}${s.default === "" ? "" : ` (default: ${s.default})`}`);
|
|
@@ -134,6 +154,10 @@ export function syncHelpText() {
|
|
|
134
154
|
" --no-wait a run in progress is not waited for: exit 0, and the holder runs once more at its end",
|
|
135
155
|
" --no-distributed read no distributed record: every plugin not recorded is measured",
|
|
136
156
|
" --refresh-pins only bring every pin's files up to the ones plugin-hostd names, measuring nothing",
|
|
157
|
+
" --rejudge derive every record's verdicts, tiers and pin again from the figures it holds; measures nothing",
|
|
158
|
+
" --memory-only measure only the footprint of each recorded plugin at the Pi's cells and merge it; with --force, those already merged too",
|
|
159
|
+
" --isolated-only run only the isolated step each record still owes, adding it to the record's run",
|
|
160
|
+
" --adapter-only run each LV2 plugin only under omx-clap-host's LV2 adapter at the Pi's cells, on private JACK servers, and merge it; with --force, those already done too",
|
|
137
161
|
" --grid-hosts the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
|
|
138
162
|
" --help this text",
|
|
139
163
|
...rows,
|
|
@@ -184,6 +208,9 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
|
|
|
184
208
|
if (memoryLimit !== undefined && !memoryLimit.ok)
|
|
185
209
|
return { ok: false, error: `memory_limit: ${memoryLimit.error}` };
|
|
186
210
|
const memory = memoryBoundOf(memoryLimit?.bytes);
|
|
211
|
+
const memoryNotes = number("memory_notes", 1);
|
|
212
|
+
if (memoryNotes === undefined || !Number.isInteger(memoryNotes) || memoryNotes > 128)
|
|
213
|
+
return { ok: false, error: "memory_notes must be a whole number of notes, 1 to 128" };
|
|
187
214
|
const profileId = get("host_profile") || "omx-clap-host";
|
|
188
215
|
const hostProfile = shippedHostProfile(profileId);
|
|
189
216
|
if (hostProfile === undefined)
|
|
@@ -215,6 +242,7 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
|
|
|
215
242
|
grid: grid.spec,
|
|
216
243
|
cellsPerPluginBudget,
|
|
217
244
|
...(memory === undefined ? {} : { memory }),
|
|
245
|
+
memoryNotes,
|
|
218
246
|
hostdPort,
|
|
219
247
|
hostdFeedbackPort: feedbackPort === 0 && hostdPort !== 0 ? hostdPort + 1 : feedbackPort,
|
|
220
248
|
settleS,
|
|
@@ -224,6 +252,29 @@ export function resolveSyncSettings(file, args, env, home = homedir(), boardMode
|
|
|
224
252
|
},
|
|
225
253
|
};
|
|
226
254
|
}
|
|
255
|
+
/** lilv itself, one world over the whole search path (tools/lv2_resolve.py), as every host built on it resolves (#199). */
|
|
256
|
+
export const LILV_RESOLVE = (dirs) => {
|
|
257
|
+
const r = spawnSync("python3", [join(toolsDir(), "lv2_resolve.py")], {
|
|
258
|
+
encoding: "utf8", timeout: 120_000, killSignal: "SIGKILL", maxBuffer: 64 << 20, env: { ...process.env, LV2_PATH: dirs.join(":") },
|
|
259
|
+
});
|
|
260
|
+
if (r.error !== undefined || r.status !== 0)
|
|
261
|
+
return { problem: (r.stderr ?? "").trim().split("\n").pop() || (r.error?.message ?? `exit ${String(r.status)}`) };
|
|
262
|
+
try {
|
|
263
|
+
return JSON.parse(r.stdout).plugins;
|
|
264
|
+
}
|
|
265
|
+
catch (e) {
|
|
266
|
+
return { problem: `lv2_resolve.py printed no JSON: ${e.message}` };
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
/** Whether `path` exists and is, through its links, the file `target` is. */
|
|
270
|
+
function linksTo(path, target) {
|
|
271
|
+
try {
|
|
272
|
+
return realpathSync(path) === realpathSync(target);
|
|
273
|
+
}
|
|
274
|
+
catch {
|
|
275
|
+
return false;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
227
278
|
const CLAP_SUFFIX = ".clap";
|
|
228
279
|
/** Every `.clap` under a directory, its subdirectories included, in name order; a directory reached twice is walked once. */
|
|
229
280
|
function clapFiles(dir, seen, problems) {
|
|
@@ -270,9 +321,10 @@ function clapFiles(dir, seen, problems) {
|
|
|
270
321
|
* LV2 directories that declares it, as plugin-hostd finds it. A directory that cannot be read is a problem named, and
|
|
271
322
|
* the others are still walked.
|
|
272
323
|
*/
|
|
273
|
-
export function discoverSyncItems(settings) {
|
|
324
|
+
export function discoverSyncItems(settings, lv2Resolve) {
|
|
274
325
|
const items = [];
|
|
275
326
|
const problems = [];
|
|
327
|
+
const notes = [];
|
|
276
328
|
const seen = new Set();
|
|
277
329
|
const clap = new Set();
|
|
278
330
|
for (const dir of settings.clapDirs) {
|
|
@@ -284,6 +336,14 @@ export function discoverSyncItems(settings) {
|
|
|
284
336
|
items.push({ key: `clap:${path}`, format: "clap", root: dirname(path), files: [basename(path)] });
|
|
285
337
|
}
|
|
286
338
|
}
|
|
339
|
+
// the bundle lilv takes for each plugin uri; without lilv, the first bundle found (and the log says so)
|
|
340
|
+
const resolved = lv2Resolve === undefined || settings.lv2Dirs.length === 0 ? undefined : lv2Resolve(settings.lv2Dirs);
|
|
341
|
+
const taken = new Map();
|
|
342
|
+
if (resolved !== undefined && "problem" in resolved)
|
|
343
|
+
notes.push(`lilv could not resolve the LV2 plugins (${resolved.problem}); each is keyed by the first bundle found`);
|
|
344
|
+
else if (resolved !== undefined)
|
|
345
|
+
for (const p of resolved)
|
|
346
|
+
taken.set(p.uri, p);
|
|
287
347
|
const lv2 = new Set();
|
|
288
348
|
for (const dir of settings.lv2Dirs) {
|
|
289
349
|
let bundles;
|
|
@@ -304,14 +364,23 @@ export function discoverSyncItems(settings) {
|
|
|
304
364
|
continue;
|
|
305
365
|
}
|
|
306
366
|
for (const p of manifestPlugins(ttl)) {
|
|
367
|
+
const lilv = taken.get(p.uri);
|
|
368
|
+
// lilv takes another bundle for this uri: this one is not the plugin a host loads
|
|
369
|
+
if (lilv !== undefined && resolve(lilv.bundle) !== resolve(bundle))
|
|
370
|
+
continue;
|
|
307
371
|
if (lv2.has(p.uri))
|
|
308
372
|
continue;
|
|
309
373
|
lv2.add(p.uri);
|
|
310
|
-
|
|
374
|
+
const binary = lilv?.binary ?? (p.binary === undefined ? undefined : join(bundle, p.binary));
|
|
375
|
+
const files = ["manifest.ttl", ...p.files.filter((f) => f !== "manifest.ttl")];
|
|
376
|
+
// a binary named by an absolute IRI (a complement's) is tracked through the bundle's link to it, when it has one
|
|
377
|
+
if (binary !== undefined && !files.includes(basename(binary)) && linksTo(join(bundle, basename(binary)), binary))
|
|
378
|
+
files.push(basename(binary));
|
|
379
|
+
items.push({ key: `lv2:${p.uri}`, format: "lv2", root: bundle, files, uri: p.uri, ...(binary === undefined ? {} : { binary }) });
|
|
311
380
|
}
|
|
312
381
|
}
|
|
313
382
|
}
|
|
314
|
-
return { items, problems };
|
|
383
|
+
return { items, problems, notes };
|
|
315
384
|
}
|
|
316
385
|
/** The SHA-256 of a file's bytes, read in pieces over one descriptor. */
|
|
317
386
|
export function fileSha256(path) {
|
|
@@ -365,6 +434,20 @@ export const SYNC_CODE = {
|
|
|
365
434
|
/** The probe runner wrote no document to judge. */
|
|
366
435
|
notMeasured: "qualify.sync.not-measured",
|
|
367
436
|
};
|
|
437
|
+
/**
|
|
438
|
+
* Why a plugin is unmeasured because of the machine it ran on rather than what it is (#192): an EACCES, EPERM or
|
|
439
|
+
* ENOSPC the run hit, or a measurement that could not run or wrote no document. Such a plugin is measured again by
|
|
440
|
+
* the next run although its files are unchanged; a reason of the plugin's own (probe.limit.not-implemented, no such
|
|
441
|
+
* plugin, a crash) is not. `undefined` for every other outcome.
|
|
442
|
+
*/
|
|
443
|
+
export function machineReason(p) {
|
|
444
|
+
if (p.outcome !== "unmeasured" || p.code !== SYNC_CODE.notMeasured)
|
|
445
|
+
return undefined;
|
|
446
|
+
const errno = /\b(EACCES|EPERM|ENOSPC)\b/.exec(p.detail ?? "")?.[1];
|
|
447
|
+
if (errno !== undefined)
|
|
448
|
+
return errno;
|
|
449
|
+
return /wrote no document|cannot run the probe runner/.test(p.detail ?? "") ? "the measurement wrote no document" : undefined;
|
|
450
|
+
}
|
|
368
451
|
/** What the quick qualification never measures, stated in every record it writes. */
|
|
369
452
|
export const QUICK_NOT_MEASURED = [{ dimension: "stability", code: HOSTING_CODE.stabilityUnsoaked }];
|
|
370
453
|
const EMPTY_STATE = { schema: SYNC_STATE_SCHEMA, entries: {} };
|
|
@@ -402,6 +485,32 @@ export function recordedAsIs(item, entry) {
|
|
|
402
485
|
const same = now.length === entry.files.length && now.every((f) => entry.files.some((e) => e.path === f.path && e.sha256 === f.sha256));
|
|
403
486
|
return { same, files: now };
|
|
404
487
|
}
|
|
488
|
+
/**
|
|
489
|
+
* The entry of an LV2 plugin a host now loads from another bundle than it was qualified in, when its binary is the
|
|
490
|
+
* same bytes (#199): the entry with the new bundle and the new bundle's file digests, its verdicts and records kept.
|
|
491
|
+
* `undefined` when the item is not that case: not LV2, no entry, the same bundle, a binary that cannot be read, or one
|
|
492
|
+
* whose bytes differ from the recorded binary's (then it is measured again).
|
|
493
|
+
*/
|
|
494
|
+
export function sameBinaryElsewhere(item, entry) {
|
|
495
|
+
if (item.format !== "lv2" || entry === undefined || item.binary === undefined || resolve(entry.root) === resolve(item.root))
|
|
496
|
+
return undefined;
|
|
497
|
+
const was = entry.files.find((f) => basename(f.path) === basename(item.binary));
|
|
498
|
+
if (was === undefined)
|
|
499
|
+
return undefined;
|
|
500
|
+
let now;
|
|
501
|
+
try {
|
|
502
|
+
now = fileSha256(item.binary);
|
|
503
|
+
}
|
|
504
|
+
catch {
|
|
505
|
+
return undefined;
|
|
506
|
+
}
|
|
507
|
+
if (now !== was.sha256)
|
|
508
|
+
return undefined;
|
|
509
|
+
const files = digestFiles(item.root, item.files);
|
|
510
|
+
if (files === undefined)
|
|
511
|
+
return undefined;
|
|
512
|
+
return { ...entry, root: item.root, files };
|
|
513
|
+
}
|
|
405
514
|
/**
|
|
406
515
|
* The pin of a judged run with the files plugin-hostd collects for its plugin (plugin-qualify#228): the probe's list is
|
|
407
516
|
* the probe's view, and an add checks the daemon's, so a pin holds the daemon's. No pin when plugin-hostd cannot name
|
|
@@ -485,13 +594,42 @@ function writeUnmeasuredRecord(ctx, item, files, plugin) {
|
|
|
485
594
|
writeFileWhole(path, `${JSON.stringify(record, null, 2)}\n`);
|
|
486
595
|
return path;
|
|
487
596
|
}
|
|
597
|
+
/** The step's environment: each of its commands has the sync's budget. */
|
|
598
|
+
const isolatedEnv = (ctx) => ({ ...ctx.env, [ADMIT_STEP_MS_ENV]: String(ctx.settings.budgetS * 1000) });
|
|
488
599
|
/** Each judged run of a pass as the plugins it qualified: the record written, the pin taken, the outcome. */
|
|
489
|
-
function judgedPlugins(ctx, runs) {
|
|
600
|
+
function judgedPlugins(ctx, runs, describeProblem) {
|
|
490
601
|
const host = ctx.settings.hostProfile.id;
|
|
491
602
|
const entries = runs.map(({ bytes, judgment }) => {
|
|
492
|
-
const { recordPath } = writeQualification(ctx.settings.qualificationDir, bytes, judgment, QUICK_NOT_MEASURED);
|
|
493
|
-
const
|
|
494
|
-
|
|
603
|
+
const { recordPath, record } = writeQualification(ctx.settings.qualificationDir, bytes, judgment, QUICK_NOT_MEASURED);
|
|
604
|
+
const steps = baseSteps(record, describeProblem);
|
|
605
|
+
let pin = withHostdFiles(ctx, judgmentPinEntries(judgment, host)[0]);
|
|
606
|
+
let { outcome, code } = outcomeOf(judgment, host);
|
|
607
|
+
if (owesIsolated(record)) {
|
|
608
|
+
// in this same pass: a record's verdict=isolated is a run seen alone, never a rating alone
|
|
609
|
+
const uri = pin?.uri ?? judgment.output.plugin.id;
|
|
610
|
+
const runner = ctx.isolatedRunner ?? spawnedIsolatedRunner();
|
|
611
|
+
const step = judgeIsolatedRun(runner(uri, isolatedCell(record, host), isolatedEnv(ctx)));
|
|
612
|
+
steps.push(step);
|
|
613
|
+
pin = pinAfterStep(pin, step);
|
|
614
|
+
// a crash or a hang in the probe stays what it was unless the step saw the plugin run: only a rating waited on the step
|
|
615
|
+
if (step.result === ISOLATED_RESULT.refused) {
|
|
616
|
+
if (outcome === "qualified") {
|
|
617
|
+
outcome = "refused";
|
|
618
|
+
code = step.code;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
else if (step.result === ISOLATED_RESULT.notRun) {
|
|
622
|
+
if (outcome === "qualified") {
|
|
623
|
+
outcome = "unmeasured";
|
|
624
|
+
code = step.code;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
else if (outcome !== "qualified") {
|
|
628
|
+
outcome = "qualified";
|
|
629
|
+
code = undefined;
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
rewriteRecord(recordPath, { ...record, run: { id: judgment.output.run.id, steps } });
|
|
495
633
|
const plugin = { uri: pin?.uri ?? judgment.output.plugin.id, outcome, ...(code === undefined ? {} : { code }), ...(pin === undefined ? {} : { pin }), record: recordPath };
|
|
496
634
|
return { plugin, judgedHere: judgment.hosts.some((h) => h.verdict.host === host) };
|
|
497
635
|
});
|
|
@@ -589,21 +727,16 @@ function qualifyClap(ctx, item) {
|
|
|
589
727
|
for (const f of pass.failures)
|
|
590
728
|
out.push({ uri, outcome: "unmeasured", code: SYNC_CODE.notMeasured, detail: f.problem });
|
|
591
729
|
const problem = descriptorFailed.get(p.id);
|
|
592
|
-
out.push(...judgedPlugins(ctx, pass.runs).map((j) => (problem === undefined ? j : { ...j, descriptorProblem: problem })));
|
|
730
|
+
out.push(...judgedPlugins(ctx, pass.runs, problem).map((j) => (problem === undefined ? j : { ...j, descriptorProblem: problem })));
|
|
593
731
|
}
|
|
594
732
|
return out;
|
|
595
733
|
}
|
|
596
734
|
/** An LV2 plugin, measured in a world of its own bundle only, as plugin-hostd runs a pinned LV2 plugin. */
|
|
597
735
|
function qualifyLv2(ctx, item) {
|
|
598
|
-
|
|
599
|
-
try {
|
|
600
|
-
symlinkSync(item.root, join(world, basename(item.root)));
|
|
736
|
+
return lv2World(item.root, (world) => {
|
|
601
737
|
const pass = runPass(ctx, { format: "lv2", uri: item.uri, bundle: item.root }, { ...ctx.env, LV2_PATH: world });
|
|
602
738
|
return [...pass.failures.map((f) => ({ uri: item.uri, outcome: "unmeasured", code: SYNC_CODE.notMeasured, detail: f.problem })), ...judgedPlugins(ctx, pass.runs)];
|
|
603
|
-
}
|
|
604
|
-
finally {
|
|
605
|
-
rmSync(world, { recursive: true, force: true });
|
|
606
|
-
}
|
|
739
|
+
});
|
|
607
740
|
}
|
|
608
741
|
/** One item qualified, whatever happens to it: an error of the sync's own while measuring it is its outcome, not the run's end. */
|
|
609
742
|
export function qualifyItem(ctx, item, files) {
|
|
@@ -623,6 +756,50 @@ export function qualifyItem(ctx, item, files) {
|
|
|
623
756
|
const all = digestFiles(item.root, [...item.files, ...pinned], files) ?? [...files];
|
|
624
757
|
return { format: item.format, root: item.root, files: all, plugins, qualifiedAt: new Date(started).toISOString(), seconds: (Date.now() - started) / 1000 };
|
|
625
758
|
}
|
|
759
|
+
/**
|
|
760
|
+
* The isolated steps the records of a state still owe, each extending its record's run. A plugin's pin follows its step, and
|
|
761
|
+
* a step that could not run stays owed: the next pass tries it again.
|
|
762
|
+
*/
|
|
763
|
+
export function isolatedPass(ctx, state) {
|
|
764
|
+
const count = { isolated: 0, refused: 0, notRun: 0 };
|
|
765
|
+
const runner = ctx.isolatedRunner ?? spawnedIsolatedRunner();
|
|
766
|
+
const host = ctx.settings.hostProfile.id;
|
|
767
|
+
const entries = {};
|
|
768
|
+
for (const [key, entry] of Object.entries(state.entries)) {
|
|
769
|
+
entries[key] = {
|
|
770
|
+
...entry,
|
|
771
|
+
plugins: entry.plugins.map((p) => {
|
|
772
|
+
if (p.record === undefined || !p.record.endsWith(".qualification.json"))
|
|
773
|
+
return p;
|
|
774
|
+
const path = join(ctx.settings.qualificationDir, basename(p.record));
|
|
775
|
+
let record;
|
|
776
|
+
try {
|
|
777
|
+
record = JSON.parse(readFileSync(path, "utf8"));
|
|
778
|
+
}
|
|
779
|
+
catch {
|
|
780
|
+
ctx.log(`${p.uri}: its record does not read; no isolated step`);
|
|
781
|
+
return p;
|
|
782
|
+
}
|
|
783
|
+
const done = completeIsolated(path, record, p.pin?.uri ?? record.plugin.id, isolatedCell(record, host), runner, isolatedEnv(ctx));
|
|
784
|
+
if (done === undefined)
|
|
785
|
+
return p;
|
|
786
|
+
ctx.log(`${p.uri}: isolated step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
|
|
787
|
+
const pin = pinAfterStep(p.pin, done.step);
|
|
788
|
+
const { pin: _pin, code: _code, ...rest } = p;
|
|
789
|
+
if (done.step.result === ISOLATED_RESULT.isolated) {
|
|
790
|
+
count.isolated++;
|
|
791
|
+
return { ...rest, outcome: "qualified", ...(pin === undefined ? {} : { pin }) };
|
|
792
|
+
}
|
|
793
|
+
if (done.step.result === ISOLATED_RESULT.refused)
|
|
794
|
+
count.refused++;
|
|
795
|
+
else
|
|
796
|
+
count.notRun++;
|
|
797
|
+
return { ...rest, outcome: done.step.result === ISOLATED_RESULT.refused ? "refused" : "unmeasured", ...(done.step.code === undefined ? {} : { code: done.step.code }), ...(pin === undefined ? {} : { pin }) };
|
|
798
|
+
}),
|
|
799
|
+
};
|
|
800
|
+
}
|
|
801
|
+
return { ...count, state: count.isolated + count.refused + count.notRun === 0 ? state : { ...state, entries } };
|
|
802
|
+
}
|
|
626
803
|
// --- the distributed records ----------------------------------------------------------------------------------------
|
|
627
804
|
/** The layer a run reads: none under `--force` or `--no-distributed`, or without a host class. */
|
|
628
805
|
function distributedLayerOf(settings, force, log) {
|
|
@@ -656,6 +833,13 @@ function takeDistributed(ctx, layer, item, files) {
|
|
|
656
833
|
ctx.log(line);
|
|
657
834
|
return undefined;
|
|
658
835
|
}
|
|
836
|
+
for (const r of match.records) {
|
|
837
|
+
const missing = missingSteps(r.record);
|
|
838
|
+
if (missing.length > 0) {
|
|
839
|
+
ctx.log(`distributed: ${distributedUri(item, r)}: its record lacks the step${missing.length === 1 ? "" : "s"} ${missing.join(", ")}; measured here`);
|
|
840
|
+
return undefined;
|
|
841
|
+
}
|
|
842
|
+
}
|
|
659
843
|
let plugins;
|
|
660
844
|
try {
|
|
661
845
|
plugins = applyDistributed(ctx.settings.qualificationDir, item, match, host);
|
|
@@ -913,7 +1097,7 @@ export function descriptorAccessProblems(dir) {
|
|
|
913
1097
|
return problems;
|
|
914
1098
|
}
|
|
915
1099
|
export function summaryLine(s) {
|
|
916
|
-
return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}, ${s.seconds.toFixed(1)} s`;
|
|
1100
|
+
return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
|
|
917
1101
|
}
|
|
918
1102
|
/** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
|
|
919
1103
|
const LOCK_HELD = 99;
|
|
@@ -995,6 +1179,10 @@ function addSummaries(a, b) {
|
|
|
995
1179
|
seconds: a.seconds + b.seconds,
|
|
996
1180
|
...(sent === 0 ? {} : { sent }),
|
|
997
1181
|
...(refreshed === 0 ? {} : { refreshed }),
|
|
1182
|
+
...(b.rejudge === undefined ? (a.rejudge === undefined ? {} : { rejudge: a.rejudge }) : { rejudge: b.rejudge }),
|
|
1183
|
+
...(b.memoryOnly === undefined ? (a.memoryOnly === undefined ? {} : { memoryOnly: a.memoryOnly }) : { memoryOnly: b.memoryOnly }),
|
|
1184
|
+
...(b.isolatedOnly === undefined ? (a.isolatedOnly === undefined ? {} : { isolatedOnly: a.isolatedOnly }) : { isolatedOnly: b.isolatedOnly }),
|
|
1185
|
+
...(b.adapterOnly === undefined ? (a.adapterOnly === undefined ? {} : { adapterOnly: a.adapterOnly }) : { adapterOnly: b.adapterOnly }),
|
|
998
1186
|
};
|
|
999
1187
|
}
|
|
1000
1188
|
/** Whether this run's scan covers an entry: its root lies under one of the run's directories of its own format. */
|
|
@@ -1013,11 +1201,11 @@ function scannedBy(settings, entry) {
|
|
|
1013
1201
|
async function runLocked(settings, options) {
|
|
1014
1202
|
const log = options.log ?? (() => { });
|
|
1015
1203
|
const env = options.env ?? process.env;
|
|
1016
|
-
log(options.refreshOnly === true ? "pin refresh begins" : "sync run begins");
|
|
1204
|
+
log(options.rejudge === true ? "re-judge begins: nothing is measured" : options.memoryOnly === true ? "memory-only pass begins" : options.adapterOnly === true ? "adapter-only pass begins" : options.refreshOnly === true ? "pin refresh begins" : "sync run begins");
|
|
1017
1205
|
{
|
|
1018
1206
|
const started = Date.now();
|
|
1019
1207
|
const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
|
|
1020
|
-
const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }) };
|
|
1208
|
+
const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }), ...(options.isolatedRunner === undefined ? {} : { isolatedRunner: options.isolatedRunner }) };
|
|
1021
1209
|
// the pins first, before the quiet wait and any measurement: an upgrade of plugin-hostd that collects more files
|
|
1022
1210
|
// than the pins on disk hold refuses those plugins until their pins name them (plugin-qualify#239)
|
|
1023
1211
|
const fresh = refreshPins(ctx, readSyncState(settings.stateFile, log));
|
|
@@ -1026,6 +1214,31 @@ async function runLocked(settings, options) {
|
|
|
1026
1214
|
}
|
|
1027
1215
|
if (options.refreshOnly === true)
|
|
1028
1216
|
return sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
|
|
1217
|
+
if (options.rejudge === true) {
|
|
1218
|
+
const r = rejudgeState(ctx, fresh.state);
|
|
1219
|
+
if (JSON.stringify(r.state) !== JSON.stringify(fresh.state))
|
|
1220
|
+
writeFileWhole(settings.stateFile, `${JSON.stringify(r.state, null, 1)}\n`);
|
|
1221
|
+
const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
|
|
1222
|
+
return { ...summary, rejudge: { changed: r.changed, unchanged: r.unchanged, owing: r.owing.length } };
|
|
1223
|
+
}
|
|
1224
|
+
if (options.isolatedOnly === true) {
|
|
1225
|
+
const r = isolatedPass(ctx, fresh.state);
|
|
1226
|
+
if (r.state !== fresh.state)
|
|
1227
|
+
writeFileWhole(settings.stateFile, `${JSON.stringify(r.state, null, 1)}\n`);
|
|
1228
|
+
const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
|
|
1229
|
+
return { ...summary, isolatedOnly: { isolated: r.isolated, refused: r.refused, notRun: r.notRun } };
|
|
1230
|
+
}
|
|
1231
|
+
if (options.adapterOnly === true) {
|
|
1232
|
+
const r = adapterPass(ctx, fresh.state, { ...options.adapterPass, force: options.force === true });
|
|
1233
|
+
log(`adapter runs under ${Object.entries(r.versions).map(([k, v]) => `${k} ${v}`).join(", ")}`);
|
|
1234
|
+
const summary = await sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
|
|
1235
|
+
return { ...summary, adapterOnly: { measured: r.measured, already: r.already, failed: r.failed } };
|
|
1236
|
+
}
|
|
1237
|
+
if (options.memoryOnly === true) {
|
|
1238
|
+
const r = memoryPass(ctx, fresh.state, { force: options.force === true, ...(options.probeRun === undefined ? {} : { run: options.probeRun }) });
|
|
1239
|
+
const summary = await sendRefreshed(settings, fresh.state, fresh.refreshed.length, started, log);
|
|
1240
|
+
return { ...summary, memoryOnly: { measured: r.measured, already: r.already, failed: r.failed } };
|
|
1241
|
+
}
|
|
1029
1242
|
await waitQuiet(dirs, settings.settleS, log);
|
|
1030
1243
|
const count = { qualified: 0, skipped: 0, refused: 0, crashed: 0, hung: 0, unmeasured: 0, distributed: 0 };
|
|
1031
1244
|
const layer = distributedLayerOf(settings, options.force === true, log);
|
|
@@ -1040,12 +1253,18 @@ async function runLocked(settings, options) {
|
|
|
1040
1253
|
const stateBefore = state;
|
|
1041
1254
|
let unreadable = false;
|
|
1042
1255
|
const lv2Pinned = [];
|
|
1256
|
+
// the plugins measured this run, and how many of them the machine left unmeasured, by reason (#192)
|
|
1257
|
+
let measuredHere = 0;
|
|
1258
|
+
const byMachine = new Map();
|
|
1043
1259
|
// a change made while a pass ran is caught by one more pass, a few at most
|
|
1044
1260
|
for (let pass = 0; pass < 3; pass++) {
|
|
1045
1261
|
const signature = directoriesSignature(dirs);
|
|
1046
|
-
const { items, problems } = discoverSyncItems(settings);
|
|
1262
|
+
const { items, problems, notes } = discoverSyncItems(settings, options.lv2Resolve ?? LILV_RESOLVE);
|
|
1047
1263
|
for (const p of problems)
|
|
1048
1264
|
log(p);
|
|
1265
|
+
if (pass === 0)
|
|
1266
|
+
for (const n of notes)
|
|
1267
|
+
log(n);
|
|
1049
1268
|
unreadable = problems.length > 0;
|
|
1050
1269
|
const found = new Set(items.map((i) => i.key));
|
|
1051
1270
|
const entries = {};
|
|
@@ -1063,11 +1282,29 @@ async function runLocked(settings, options) {
|
|
|
1063
1282
|
}
|
|
1064
1283
|
for (const item of items) {
|
|
1065
1284
|
const recorded = recordedAsIs(item, entries[item.key]);
|
|
1066
|
-
|
|
1285
|
+
const retry = recorded.same ? entries[item.key].plugins.map(machineReason).find((r) => r !== undefined) : undefined;
|
|
1286
|
+
if (recorded.same && options.force !== true && retry === undefined) {
|
|
1067
1287
|
if (pass === 0)
|
|
1068
1288
|
count.skipped += entries[item.key].plugins.length;
|
|
1069
1289
|
continue;
|
|
1070
1290
|
}
|
|
1291
|
+
// an LV2 plugin a host now loads from another bundle with the same binary (a complement that restates its TTL,
|
|
1292
|
+
// #199): the record stands, keyed by its uri and its binary; only the bundle, its files and the pins' files move
|
|
1293
|
+
const moved = options.force === true ? undefined : sameBinaryElsewhere(item, entries[item.key]);
|
|
1294
|
+
if (moved !== undefined) {
|
|
1295
|
+
entries[item.key] = { ...moved, plugins: moved.plugins.map((p) => ({ ...p, ...(p.pin === undefined ? {} : { pin: withHostdFiles(ctx, p.pin) }) })) };
|
|
1296
|
+
if (item.format === "lv2")
|
|
1297
|
+
lv2Pinned.push(...entries[item.key].plugins.filter((p) => p.pin !== undefined).map((p) => p.uri));
|
|
1298
|
+
if (pass === 0)
|
|
1299
|
+
count.skipped += moved.plugins.length;
|
|
1300
|
+
log(`${item.key}: loaded from ${item.root} now, with the binary it was qualified with; its record stands and its pin's files follow the bundle`);
|
|
1301
|
+
state = { schema: SYNC_STATE_SCHEMA, entries };
|
|
1302
|
+
writeFileWhole(settings.stateFile, `${JSON.stringify(state, null, 1)}\n`);
|
|
1303
|
+
writeFileWhole(settings.pinFile, syncPinsText(state));
|
|
1304
|
+
continue;
|
|
1305
|
+
}
|
|
1306
|
+
if (recorded.same && retry !== undefined)
|
|
1307
|
+
log(`${item.key}: unmeasured last run by the machine (${retry}); measured again`);
|
|
1071
1308
|
const files = recorded.files ?? digestFiles(item.root, item.files);
|
|
1072
1309
|
if (files === undefined) {
|
|
1073
1310
|
log(`${item.key}: a file of it cannot be read; skipped this run`);
|
|
@@ -1092,6 +1329,13 @@ async function runLocked(settings, options) {
|
|
|
1092
1329
|
lv2Pinned.push(...entry.plugins.filter((p) => p.pin !== undefined).map((p) => p.uri));
|
|
1093
1330
|
for (const p of entry.plugins) {
|
|
1094
1331
|
count[p.outcome]++;
|
|
1332
|
+
measuredHere++;
|
|
1333
|
+
// a reason of the machine is said once: the plugins after the first are counted, not listed
|
|
1334
|
+
const why = machineReason(p);
|
|
1335
|
+
if (why !== undefined)
|
|
1336
|
+
byMachine.set(why, (byMachine.get(why) ?? 0) + 1);
|
|
1337
|
+
if (why !== undefined && byMachine.get(why) > 1)
|
|
1338
|
+
continue;
|
|
1095
1339
|
log(`${p.uri}: ${p.outcome}${p.code === undefined ? "" : ` (${p.code})`}${p.detail === undefined ? "" : `: ${p.detail}`}`);
|
|
1096
1340
|
}
|
|
1097
1341
|
state = { schema: SYNC_STATE_SCHEMA, entries };
|
|
@@ -1133,8 +1377,16 @@ async function runLocked(settings, options) {
|
|
|
1133
1377
|
if (replies[i] !== "resp 0")
|
|
1134
1378
|
log(`plugin-hostd answered ${replies[i]} to ${changed[i].split(" ")[1]}`);
|
|
1135
1379
|
const taken = replies.filter((r) => r === "resp 0").length;
|
|
1380
|
+
for (const [why, n] of byMachine)
|
|
1381
|
+
if (n > 1)
|
|
1382
|
+
log(`${String(n)} plugins unmeasured by the machine (${why}); the first is listed above`);
|
|
1383
|
+
const [only] = byMachine.size === 1 ? [...byMachine] : [];
|
|
1384
|
+
const machineFailure = only !== undefined && only[1] === measuredHere ? only[0] : undefined;
|
|
1385
|
+
if (machineFailure !== undefined)
|
|
1386
|
+
log(`every plugin this run measured was left unmeasured by the machine (${machineFailure}): the run failed`);
|
|
1136
1387
|
return {
|
|
1137
1388
|
...count,
|
|
1389
|
+
...(machineFailure === undefined ? {} : { machineFailure }),
|
|
1138
1390
|
seconds: (Date.now() - started) / 1000,
|
|
1139
1391
|
...(taken === 0 ? {} : { sent: taken }),
|
|
1140
1392
|
...(fresh.refreshed.length === 0 ? {} : { refreshed: fresh.refreshed.length }),
|