@openmixer/plugin-qualify 0.18.5 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -730
- package/bin/plugin-qualify-sync.mjs +1 -1
- package/bin/plugin-qualify.mjs +9 -1
- package/dist/clap-qualify.d.ts +1 -1
- package/dist/compare.d.ts +311 -0
- package/dist/compare.js +502 -0
- package/dist/compare.js.map +1 -0
- package/dist/distributed-records.d.ts +0 -2
- package/dist/distributed-records.js +0 -2
- package/dist/distributed-records.js.map +1 -1
- package/dist/host-admission.d.ts +2 -2
- package/dist/host-admission.js +7 -0
- package/dist/host-admission.js.map +1 -1
- package/dist/hosting-codes.generated.d.ts +1 -0
- package/dist/hosting-codes.generated.js +1 -0
- package/dist/hosting-codes.generated.js.map +1 -1
- package/dist/hosting-suitability.d.ts +3 -1
- package/dist/hosting-suitability.js +9 -0
- package/dist/hosting-suitability.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/lv2-loader.js +6 -5
- package/dist/lv2-loader.js.map +1 -1
- package/dist/measurement/format.d.ts +15 -0
- package/dist/measurement/format.js +19 -0
- package/dist/measurement/format.js.map +1 -1
- package/dist/measurement/sweep-jsonl.js +5 -1
- package/dist/measurement/sweep-jsonl.js.map +1 -1
- package/dist/measurement-grid.js +1 -1
- package/dist/pins-file.d.ts +22 -2
- package/dist/pins-file.js +118 -17
- package/dist/pins-file.js.map +1 -1
- package/dist/probe-merge.js +6 -0
- package/dist/probe-merge.js.map +1 -1
- package/dist/probe-runner.d.ts +4 -0
- package/dist/probe-runner.js +12 -3
- package/dist/probe-runner.js.map +1 -1
- package/dist/qualification-record.d.ts +33 -1
- package/dist/qualification-record.js +36 -3
- package/dist/qualification-record.js.map +1 -1
- package/dist/qualify-declarations.d.ts +6 -0
- package/dist/qualify-declarations.js +8 -0
- package/dist/qualify-declarations.js.map +1 -1
- package/dist/qualify-isolated.d.ts +16 -3
- package/dist/qualify-isolated.js +31 -3
- package/dist/qualify-isolated.js.map +1 -1
- package/dist/qualify-pool.d.ts +129 -0
- package/dist/qualify-pool.js +290 -0
- package/dist/qualify-pool.js.map +1 -0
- package/dist/qualify-rejudge.js +7 -0
- package/dist/qualify-rejudge.js.map +1 -1
- package/dist/qualify-sync.d.ts +37 -2
- package/dist/qualify-sync.js +148 -24
- package/dist/qualify-sync.js.map +1 -1
- package/dist/records-package.d.ts +5 -0
- package/dist/records-package.js +25 -8
- package/dist/records-package.js.map +1 -1
- package/fixtures/clap-scan/fake.json +8 -2
- package/fixtures/probe-output/success-msi.json +2 -2
- package/package.json +3 -3
- package/src/clap-qualify.ts +1 -1
- package/src/compare.ts +699 -0
- package/src/distributed-records.ts +0 -3
- package/src/host-admission.ts +9 -2
- package/src/hosting-codes.generated.ts +1 -0
- package/src/hosting-suitability.ts +14 -0
- package/src/index.ts +1 -0
- package/src/lv2-loader.ts +5 -4
- package/src/measurement/format.ts +32 -0
- package/src/measurement/sweep-jsonl.ts +5 -1
- package/src/measurement-grid.ts +1 -1
- package/src/pins-file.ts +109 -16
- package/src/probe-merge.ts +4 -0
- package/src/probe-runner.ts +16 -6
- package/src/qualification-record.ts +59 -4
- package/src/qualify-declarations.ts +8 -0
- package/src/qualify-isolated.ts +31 -5
- package/src/qualify-pool.ts +318 -0
- package/src/qualify-rejudge.ts +8 -0
- package/src/qualify-sync.ts +145 -24
- package/src/records-package.ts +25 -8
- package/tools/benchmark.py +9 -3
- package/tools/compare-host.mjs +432 -0
- package/tools/compare-run.mjs +355 -0
- package/tools/descriptors.py +27 -0
- package/tools/isolated-step.mjs +3 -1
- package/tools/lilv_load.py +25 -0
- package/tools/lv2_resolve.py +3 -1
- package/tools/pool-step.mjs +186 -0
- package/tools/scan.py +16 -20
- package/tools/audit-overlap.sh +0 -250
- package/tools/commit-message-check.sh +0 -128
- package/tools/commit-message-patterns.json +0 -41
- package/tools/private-refs-check.sh +0 -90
package/dist/qualify-sync.js
CHANGED
|
@@ -23,20 +23,21 @@ import { createConnection } from "node:net";
|
|
|
23
23
|
import { homedir, tmpdir } from "node:os";
|
|
24
24
|
import { basename, dirname, join, resolve, sep } from "node:path";
|
|
25
25
|
import { clapQualifyTools } from "./clap-qualify.js";
|
|
26
|
-
import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, matchDistributed, readBoardModel, readDistributedLayer } from "./distributed-records.js";
|
|
26
|
+
import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, outcomeOfVerdict, matchDistributed, readBoardModel, readDistributedLayer } from "./distributed-records.js";
|
|
27
27
|
import { clapSearchPath, parseClapScan } from "./clap-loader.js";
|
|
28
28
|
import { HOSTING_CODE } from "./hosting-codes.js";
|
|
29
29
|
import { shippedHostProfile } from "./host-profiles.js";
|
|
30
30
|
import { lv2BundlesOnPath, manifestPlugins } from "./lv2-loader.js";
|
|
31
|
-
import { judgmentPinEntries, pinsFile } from "./pins-file.js";
|
|
31
|
+
import { judgmentPinEntries, pinClearLine, pinLineUri, pinsFile } from "./pins-file.js";
|
|
32
|
+
import { POOL_RESULT, completePool, judgePoolRun, pinAfterPool, pinFromRecordWithPool, poolEnv, spawnedPoolRunner } from "./qualify-pool.js";
|
|
32
33
|
import { hostdPinFiles } from "./hostd-pin-files.js";
|
|
33
34
|
import { qualifyTargets } from "./probe-runner.js";
|
|
34
35
|
import { GRID_FULL, parseGridSpec } from "./measurement-grid.js";
|
|
35
36
|
import { adapterPass } from "./qualify-adapter-pass.js";
|
|
36
37
|
import { ISOLATED_RESULT, baseSteps, completeIsolated, isolatedCell, missingSteps, judgeIsolatedRun, owesIsolated, pinAfterStep, spawnedIsolatedRunner } from "./qualify-isolated.js";
|
|
37
|
-
import { lv2World, rejudgeState, memoryPass } from "./qualify-rejudge.js";
|
|
38
|
+
import { lv2World, rejudgeRecord, rejudgeState, rejudgeToolName, memoryPass } from "./qualify-rejudge.js";
|
|
38
39
|
import { memoryBoundOf, parseMemoryLimit } from "./memory-bound.js";
|
|
39
|
-
import { rewriteRecord, writeQualification } from "./qualification-record.js";
|
|
40
|
+
import { QUALIFICATION_RECORD_SCHEMA, latencyNeverTaken, rewriteRecord, writeQualification } from "./qualification-record.js";
|
|
40
41
|
import { ADMIT_STEP_MS_ENV, DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, MEMORY_PASS_PROBE_NOTES, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
|
|
41
42
|
import { toolsDir } from "./tool-paths.js";
|
|
42
43
|
import { writeFileWhole } from "./whole-file.js";
|
|
@@ -100,6 +101,7 @@ export function parseSyncArgs(argv) {
|
|
|
100
101
|
let memoryOnly = false;
|
|
101
102
|
let adapterOnly = false;
|
|
102
103
|
let isolatedOnly = false;
|
|
104
|
+
let poolOnly = false;
|
|
103
105
|
let help = false;
|
|
104
106
|
for (let i = 0; i < argv.length; i++) {
|
|
105
107
|
const a = argv[i];
|
|
@@ -121,6 +123,8 @@ export function parseSyncArgs(argv) {
|
|
|
121
123
|
adapterOnly = true;
|
|
122
124
|
else if (a === "--isolated-only")
|
|
123
125
|
isolatedOnly = true;
|
|
126
|
+
else if (a === "--pool-only")
|
|
127
|
+
poolOnly = true;
|
|
124
128
|
else if (a === "--help")
|
|
125
129
|
help = true;
|
|
126
130
|
else if (a === "--grid-hosts")
|
|
@@ -137,11 +141,11 @@ export function parseSyncArgs(argv) {
|
|
|
137
141
|
else
|
|
138
142
|
return { ok: false, error: `unknown argument ${a}` };
|
|
139
143
|
}
|
|
140
|
-
if (rejudge && (memoryOnly || adapterOnly || isolatedOnly || force || refreshPinsOnly))
|
|
141
|
-
return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --force and --refresh-pins" };
|
|
142
|
-
if ([memoryOnly, adapterOnly, isolatedOnly, refreshPinsOnly].filter(Boolean).length > 1)
|
|
143
|
-
return { ok: false, error: "--memory-only, --adapter-only and --refresh-pins are different runs: one at a time" };
|
|
144
|
-
return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, isolatedOnly, help, values } };
|
|
144
|
+
if (rejudge && (memoryOnly || adapterOnly || isolatedOnly || poolOnly || force || refreshPinsOnly))
|
|
145
|
+
return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --isolated-only, --pool-only, --force and --refresh-pins" };
|
|
146
|
+
if ([memoryOnly, adapterOnly, isolatedOnly, poolOnly, refreshPinsOnly].filter(Boolean).length > 1)
|
|
147
|
+
return { ok: false, error: "--memory-only, --adapter-only, --isolated-only, --pool-only and --refresh-pins are different runs: one at a time" };
|
|
148
|
+
return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, isolatedOnly, poolOnly, help, values } };
|
|
145
149
|
}
|
|
146
150
|
export function syncHelpText() {
|
|
147
151
|
const rows = SYNC_SETTINGS.map((s) => ` ${`${flagOf(s.key)} <value>`.padEnd(36)}${s.meaning}${s.default === "" ? "" : ` (default: ${s.default})`}`);
|
|
@@ -157,6 +161,7 @@ export function syncHelpText() {
|
|
|
157
161
|
" --rejudge derive every record's verdicts, tiers and pin again from the figures it holds; measures nothing",
|
|
158
162
|
" --memory-only measure only the footprint of each recorded plugin at the Pi's cells and merge it; with --force, those already merged too",
|
|
159
163
|
" --isolated-only run only the isolated step each record still owes, adding it to the record's run",
|
|
164
|
+
" --pool-only run only the pool step each CLAP record still owes: a plugin that asks for the thread pool is run with it off and on, and its pin gets pool=cores or pool=off",
|
|
160
165
|
" --adapter-only run each LV2 plugin only under omx-clap-host's LV2 adapter at the Pi's cells, on private JACK servers, and merge it; with --force, those already done too",
|
|
161
166
|
" --grid-hosts the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
|
|
162
167
|
" --help this text",
|
|
@@ -448,6 +453,27 @@ export function machineReason(p) {
|
|
|
448
453
|
return errno;
|
|
449
454
|
return /wrote no document|cannot run the probe runner/.test(p.detail ?? "") ? "the measurement wrote no document" : undefined;
|
|
450
455
|
}
|
|
456
|
+
/**
|
|
457
|
+
* The figure the records of a recorded entry never took, or `undefined`: a record whose probe ran before that figure's
|
|
458
|
+
* pass ({@link latencyNeverTaken}) owes a measurement, and the sync measures the entry again although its bytes are the
|
|
459
|
+
* ones it recorded. A record that cannot be read owes nothing here: the sync's other checks deal with it.
|
|
460
|
+
*/
|
|
461
|
+
export function figureOwed(entry) {
|
|
462
|
+
for (const p of entry.plugins) {
|
|
463
|
+
if (p.record === undefined)
|
|
464
|
+
continue;
|
|
465
|
+
let r;
|
|
466
|
+
try {
|
|
467
|
+
r = JSON.parse(readFileSync(p.record, "utf8"));
|
|
468
|
+
}
|
|
469
|
+
catch {
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
if (r.schema === QUALIFICATION_RECORD_SCHEMA && latencyNeverTaken(r))
|
|
473
|
+
return "latency";
|
|
474
|
+
}
|
|
475
|
+
return undefined;
|
|
476
|
+
}
|
|
451
477
|
/** What the quick qualification never measures, stated in every record it writes. */
|
|
452
478
|
export const QUICK_NOT_MEASURED = [{ dimension: "stability", code: HOSTING_CODE.stabilityUnsoaked }];
|
|
453
479
|
const EMPTY_STATE = { schema: SYNC_STATE_SCHEMA, entries: {} };
|
|
@@ -599,8 +625,8 @@ const isolatedEnv = (ctx) => ({ ...ctx.env, [ADMIT_STEP_MS_ENV]: String(ctx.sett
|
|
|
599
625
|
/** Each judged run of a pass as the plugins it qualified: the record written, the pin taken, the outcome. */
|
|
600
626
|
function judgedPlugins(ctx, runs, describeProblem) {
|
|
601
627
|
const host = ctx.settings.hostProfile.id;
|
|
602
|
-
const entries = runs.map(({ bytes, judgment }) => {
|
|
603
|
-
const { recordPath, record } = writeQualification(ctx.settings.qualificationDir, bytes, judgment, QUICK_NOT_MEASURED);
|
|
628
|
+
const entries = runs.map(({ bytes, judgment, stderr }) => {
|
|
629
|
+
const { recordPath, record } = writeQualification(ctx.settings.qualificationDir, bytes, judgment, QUICK_NOT_MEASURED, stderr);
|
|
604
630
|
const steps = baseSteps(record, describeProblem);
|
|
605
631
|
let pin = withHostdFiles(ctx, judgmentPinEntries(judgment, host)[0]);
|
|
606
632
|
let { outcome, code } = outcomeOf(judgment, host);
|
|
@@ -629,6 +655,13 @@ function judgedPlugins(ctx, runs, describeProblem) {
|
|
|
629
655
|
code = undefined;
|
|
630
656
|
}
|
|
631
657
|
}
|
|
658
|
+
// the pool is for a CLAP plugin that runs: the thread pool is granted by measuring it, here and nowhere else
|
|
659
|
+
if (judgment.output.plugin.format === "clap" && pin !== undefined && pin.verdict !== "refused" && outcome === "qualified") {
|
|
660
|
+
const runner = ctx.poolRunner ?? spawnedPoolRunner();
|
|
661
|
+
const step = judgePoolRun(runner(pin.uri, isolatedCell(record, host), poolEnv(ctx.env, ctx.settings.budgetS)));
|
|
662
|
+
steps.push(step);
|
|
663
|
+
pin = pinAfterPool(pin, step);
|
|
664
|
+
}
|
|
632
665
|
rewriteRecord(recordPath, { ...record, run: { id: judgment.output.run.id, steps } });
|
|
633
666
|
const plugin = { uri: pin?.uri ?? judgment.output.plugin.id, outcome, ...(code === undefined ? {} : { code }), ...(pin === undefined ? {} : { pin }), record: recordPath };
|
|
634
667
|
return { plugin, judgedHere: judgment.hosts.some((h) => h.verdict.host === host) };
|
|
@@ -761,7 +794,7 @@ export function qualifyItem(ctx, item, files) {
|
|
|
761
794
|
* a step that could not run stays owed: the next pass tries it again.
|
|
762
795
|
*/
|
|
763
796
|
export function isolatedPass(ctx, state) {
|
|
764
|
-
const count = { isolated: 0, refused: 0, notRun: 0 };
|
|
797
|
+
const count = { isolated: 0, refused: 0, notRun: 0, repinned: 0 };
|
|
765
798
|
const runner = ctx.isolatedRunner ?? spawnedIsolatedRunner();
|
|
766
799
|
const host = ctx.settings.hostProfile.id;
|
|
767
800
|
const entries = {};
|
|
@@ -781,8 +814,19 @@ export function isolatedPass(ctx, state) {
|
|
|
781
814
|
return p;
|
|
782
815
|
}
|
|
783
816
|
const done = completeIsolated(path, record, p.pin?.uri ?? record.plugin.id, isolatedCell(record, host), runner, isolatedEnv(ctx));
|
|
784
|
-
if (done === undefined)
|
|
785
|
-
|
|
817
|
+
if (done === undefined) {
|
|
818
|
+
// a plugin the isolated step already decided but the state holds no pin for: the record keeps the bytes and the
|
|
819
|
+
// step, and the pin follows them (hostd names the files, as at every other pin)
|
|
820
|
+
if (p.pin !== undefined)
|
|
821
|
+
return p;
|
|
822
|
+
const rebuilt = withHostdFiles(ctx, pinFromRecordWithPool(record));
|
|
823
|
+
if (rebuilt === undefined)
|
|
824
|
+
return p;
|
|
825
|
+
count.repinned++;
|
|
826
|
+
ctx.log(`${p.uri}: pinned again from its record (isolated step ${rebuilt.verdict ?? "undecided"})`);
|
|
827
|
+
const { code: _c, ...rest } = p;
|
|
828
|
+
return rest.outcome === "qualified" && rebuilt.verdict === "refused" ? { ...rest, outcome: "refused", pin: rebuilt } : { ...rest, pin: rebuilt };
|
|
829
|
+
}
|
|
786
830
|
ctx.log(`${p.uri}: isolated step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
|
|
787
831
|
const pin = pinAfterStep(p.pin, done.step);
|
|
788
832
|
const { pin: _pin, code: _code, ...rest } = p;
|
|
@@ -798,7 +842,50 @@ export function isolatedPass(ctx, state) {
|
|
|
798
842
|
}),
|
|
799
843
|
};
|
|
800
844
|
}
|
|
801
|
-
return { ...count, state: count.isolated + count.refused + count.notRun === 0 ? state : { ...state, entries } };
|
|
845
|
+
return { ...count, state: count.isolated + count.refused + count.notRun + count.repinned === 0 ? state : { ...state, entries } };
|
|
846
|
+
}
|
|
847
|
+
/**
|
|
848
|
+
* The pool steps the CLAP records of a state still owe, each extending its record's run. A plugin's pin takes the pool word
|
|
849
|
+
* of its step (`cores`, or `off` for a plugin that was measured and gains nothing) and none for a plugin that does not ask
|
|
850
|
+
* for the pool. A step that could not run stays owed: the next pass tries it again.
|
|
851
|
+
*/
|
|
852
|
+
export function poolPass(ctx, state) {
|
|
853
|
+
const count = { cores: 0, off: 0, notAsked: 0, notRun: 0 };
|
|
854
|
+
const runner = ctx.poolRunner ?? spawnedPoolRunner();
|
|
855
|
+
const host = ctx.settings.hostProfile.id;
|
|
856
|
+
const entries = {};
|
|
857
|
+
for (const [key, entry] of Object.entries(state.entries)) {
|
|
858
|
+
entries[key] = {
|
|
859
|
+
...entry,
|
|
860
|
+
plugins: entry.plugins.map((p) => {
|
|
861
|
+
if (p.record === undefined || !p.record.endsWith(".qualification.json") || p.pin === undefined || p.pin.verdict === "refused")
|
|
862
|
+
return p;
|
|
863
|
+
const path = join(ctx.settings.qualificationDir, basename(p.record));
|
|
864
|
+
let record;
|
|
865
|
+
try {
|
|
866
|
+
record = JSON.parse(readFileSync(path, "utf8"));
|
|
867
|
+
}
|
|
868
|
+
catch {
|
|
869
|
+
ctx.log(`${p.uri}: its record does not read; no pool step`);
|
|
870
|
+
return p;
|
|
871
|
+
}
|
|
872
|
+
const done = completePool(path, record, p.pin.uri, isolatedCell(record, host), runner, poolEnv(ctx.env, ctx.settings.budgetS));
|
|
873
|
+
if (done === undefined)
|
|
874
|
+
return p;
|
|
875
|
+
ctx.log(`${p.uri}: pool step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
|
|
876
|
+
if (done.step.result === POOL_RESULT.cores)
|
|
877
|
+
count.cores++;
|
|
878
|
+
else if (done.step.result === POOL_RESULT.off)
|
|
879
|
+
count.off++;
|
|
880
|
+
else if (done.step.result === POOL_RESULT.notAsked)
|
|
881
|
+
count.notAsked++;
|
|
882
|
+
else
|
|
883
|
+
count.notRun++;
|
|
884
|
+
return { ...p, pin: pinAfterPool(p.pin, done.step) };
|
|
885
|
+
}),
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
return { ...count, state: count.cores + count.off + count.notAsked + count.notRun === 0 ? state : { ...state, entries } };
|
|
802
889
|
}
|
|
803
890
|
// --- the distributed records ----------------------------------------------------------------------------------------
|
|
804
891
|
/** The layer a run reads: none under `--force` or `--no-distributed`, or without a host class. */
|
|
@@ -839,10 +926,36 @@ function takeDistributed(ctx, layer, item, files) {
|
|
|
839
926
|
ctx.log(`distributed: ${distributedUri(item, r)}: its record lacks the step${missing.length === 1 ? "" : "s"} ${missing.join(", ")}; measured here`);
|
|
840
927
|
return undefined;
|
|
841
928
|
}
|
|
929
|
+
if (latencyNeverTaken(r.record)) {
|
|
930
|
+
ctx.log(`distributed: ${distributedUri(item, r)}: its record never took the latency figure (a probe from before that pass); measured here`);
|
|
931
|
+
return undefined;
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
// The set was judged by the plugin-qualify that measured it. Its figures are carried across and judged again under
|
|
935
|
+
// today's rules, as an upgrade's are (`--rejudge`): a verdict of an older judge (RT-unsafe read as refused, before
|
|
936
|
+
// #196) is never taken as it stands. A record today's judge cannot decide for this host is measured here.
|
|
937
|
+
const by = rejudgeToolName();
|
|
938
|
+
const at = new Date(started).toISOString();
|
|
939
|
+
const records = [];
|
|
940
|
+
for (const r of match.records) {
|
|
941
|
+
let record;
|
|
942
|
+
try {
|
|
943
|
+
record = rejudgeRecord(r.record, readFileSync(join(r.path, "..", r.record.probe.file)), { host, by, at }).record;
|
|
944
|
+
}
|
|
945
|
+
catch (e) {
|
|
946
|
+
ctx.log(`distributed: ${distributedUri(item, r)}: ${e.message}; measured here`);
|
|
947
|
+
return undefined;
|
|
948
|
+
}
|
|
949
|
+
const v = record.verdicts.find((x) => x.host === host);
|
|
950
|
+
if (v !== undefined && outcomeOfVerdict(v) === "unmeasured") {
|
|
951
|
+
ctx.log(`distributed: ${distributedUri(item, r)}: judged again, ${v.deciding ?? "unknown"} for ${host}; measured here`);
|
|
952
|
+
return undefined;
|
|
953
|
+
}
|
|
954
|
+
records.push({ ...r, record });
|
|
842
955
|
}
|
|
843
956
|
let plugins;
|
|
844
957
|
try {
|
|
845
|
-
plugins = applyDistributed(ctx.settings.qualificationDir, item, match, host);
|
|
958
|
+
plugins = applyDistributed(ctx.settings.qualificationDir, item, { ...match, records }, host);
|
|
846
959
|
}
|
|
847
960
|
catch (e) {
|
|
848
961
|
ctx.log(`distributed: ${distributedUri(item, match.records[0])}: ${e.message}; measured here`);
|
|
@@ -882,7 +995,7 @@ function pinLinesOf(text) {
|
|
|
882
995
|
const out = new Map();
|
|
883
996
|
for (const line of text.split("\n"))
|
|
884
997
|
if (line !== "" && !line.startsWith("#"))
|
|
885
|
-
out.set(line
|
|
998
|
+
out.set(pinLineUri(line), line);
|
|
886
999
|
return out;
|
|
887
1000
|
}
|
|
888
1001
|
/** The pin lines of `after` that `before` does not hold as they are. */
|
|
@@ -895,9 +1008,9 @@ export function removedPinUris(before, after) {
|
|
|
895
1008
|
const now = pinLinesOf(after);
|
|
896
1009
|
return [...pinLinesOf(before).keys()].filter((uri) => !now.has(uri));
|
|
897
1010
|
}
|
|
898
|
-
/** `<id>@<version>-<binary sha first 12>`, the stem a qualification record, its probe copy and its descriptor share. */
|
|
1011
|
+
/** `<id>@<version>-<binary sha first 12>`, the stem a qualification record, its probe copy, its children's stderr and its descriptor share. */
|
|
899
1012
|
export function recordStem(recordPath) {
|
|
900
|
-
return /^(.*-[0-9a-f]{12})\.(?:[^.]+\.(?:qualification|probe)|unmeasured)\.json$/.exec(basename(recordPath))?.[1];
|
|
1013
|
+
return /^(.*-[0-9a-f]{12})\.(?:(?:[^.]+\.(?:qualification|probe)|unmeasured)\.json|[^.]+\.stderr\.[0-9a-f]{12}\.log)$/.exec(basename(recordPath))?.[1];
|
|
901
1014
|
}
|
|
902
1015
|
/**
|
|
903
1016
|
* A stem's id and binary hash, its version segment dropped: what a record and its descriptor share even when their
|
|
@@ -1137,7 +1250,7 @@ export function pluginCountsLine(c) {
|
|
|
1137
1250
|
return `plugins ${part(c.plugins)}, offered ${part(c.offered)}, not offered ${part(c.notOffered)}`;
|
|
1138
1251
|
}
|
|
1139
1252
|
export function summaryLine(s) {
|
|
1140
|
-
return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
|
|
1253
|
+
return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.poolOnly === undefined ? "" : `, pool granted ${String(s.poolOnly.cores)}, off ${String(s.poolOnly.off)}, not asked ${String(s.poolOnly.notAsked)}, not run ${String(s.poolOnly.notRun)}`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
|
|
1141
1254
|
}
|
|
1142
1255
|
/** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
|
|
1143
1256
|
const LOCK_HELD = 99;
|
|
@@ -1222,6 +1335,7 @@ function addSummaries(a, b) {
|
|
|
1222
1335
|
...(b.rejudge === undefined ? (a.rejudge === undefined ? {} : { rejudge: a.rejudge }) : { rejudge: b.rejudge }),
|
|
1223
1336
|
...(b.memoryOnly === undefined ? (a.memoryOnly === undefined ? {} : { memoryOnly: a.memoryOnly }) : { memoryOnly: b.memoryOnly }),
|
|
1224
1337
|
...(b.isolatedOnly === undefined ? (a.isolatedOnly === undefined ? {} : { isolatedOnly: a.isolatedOnly }) : { isolatedOnly: b.isolatedOnly }),
|
|
1338
|
+
...(b.poolOnly === undefined ? (a.poolOnly === undefined ? {} : { poolOnly: a.poolOnly }) : { poolOnly: b.poolOnly }),
|
|
1225
1339
|
...(b.adapterOnly === undefined ? (a.adapterOnly === undefined ? {} : { adapterOnly: a.adapterOnly }) : { adapterOnly: b.adapterOnly }),
|
|
1226
1340
|
};
|
|
1227
1341
|
}
|
|
@@ -1245,7 +1359,7 @@ async function runLocked(settings, options) {
|
|
|
1245
1359
|
{
|
|
1246
1360
|
const started = Date.now();
|
|
1247
1361
|
const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
|
|
1248
|
-
const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }), ...(options.isolatedRunner === undefined ? {} : { isolatedRunner: options.isolatedRunner }) };
|
|
1362
|
+
const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }), ...(options.isolatedRunner === undefined ? {} : { isolatedRunner: options.isolatedRunner }), ...(options.poolRunner === undefined ? {} : { poolRunner: options.poolRunner }) };
|
|
1249
1363
|
// the pins first, before the quiet wait and any measurement: an upgrade of plugin-hostd that collects more files
|
|
1250
1364
|
// than the pins on disk hold refuses those plugins until their pins name them (plugin-qualify#239)
|
|
1251
1365
|
const fresh = refreshPins(ctx, readSyncState(settings.stateFile, log));
|
|
@@ -1268,6 +1382,13 @@ async function runLocked(settings, options) {
|
|
|
1268
1382
|
const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
|
|
1269
1383
|
return { ...summary, isolatedOnly: { isolated: r.isolated, refused: r.refused, notRun: r.notRun } };
|
|
1270
1384
|
}
|
|
1385
|
+
if (options.poolOnly === true) {
|
|
1386
|
+
const r = poolPass(ctx, fresh.state);
|
|
1387
|
+
if (r.state !== fresh.state)
|
|
1388
|
+
writeFileWhole(settings.stateFile, `${JSON.stringify(r.state, null, 1)}\n`);
|
|
1389
|
+
const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
|
|
1390
|
+
return { ...summary, poolOnly: { cores: r.cores, off: r.off, notAsked: r.notAsked, notRun: r.notRun } };
|
|
1391
|
+
}
|
|
1271
1392
|
if (options.adapterOnly === true) {
|
|
1272
1393
|
const r = adapterPass(ctx, fresh.state, { ...options.adapterPass, force: options.force === true });
|
|
1273
1394
|
log(`adapter runs under ${Object.entries(r.versions).map(([k, v]) => `${k} ${v}`).join(", ")}`);
|
|
@@ -1323,7 +1444,8 @@ async function runLocked(settings, options) {
|
|
|
1323
1444
|
for (const item of items) {
|
|
1324
1445
|
const recorded = recordedAsIs(item, entries[item.key]);
|
|
1325
1446
|
const retry = recorded.same ? entries[item.key].plugins.map(machineReason).find((r) => r !== undefined) : undefined;
|
|
1326
|
-
|
|
1447
|
+
const owed = recorded.same && options.force !== true && retry === undefined ? figureOwed(entries[item.key]) : undefined;
|
|
1448
|
+
if (recorded.same && options.force !== true && retry === undefined && owed === undefined) {
|
|
1327
1449
|
if (pass === 0)
|
|
1328
1450
|
count.skipped += entries[item.key].plugins.length;
|
|
1329
1451
|
continue;
|
|
@@ -1345,6 +1467,8 @@ async function runLocked(settings, options) {
|
|
|
1345
1467
|
}
|
|
1346
1468
|
if (recorded.same && retry !== undefined)
|
|
1347
1469
|
log(`${item.key}: unmeasured last run by the machine (${retry}); measured again`);
|
|
1470
|
+
if (owed !== undefined)
|
|
1471
|
+
log(`${item.key}: its record never took the ${owed} figure (a probe from before that pass); measured again`);
|
|
1348
1472
|
const files = recorded.files ?? digestFiles(item.root, item.files);
|
|
1349
1473
|
if (files === undefined) {
|
|
1350
1474
|
log(`${item.key}: a file of it cannot be read; skipped this run`);
|
|
@@ -1402,7 +1526,7 @@ async function runLocked(settings, options) {
|
|
|
1402
1526
|
for (const f of pruneStale(settings, stateBefore, state))
|
|
1403
1527
|
log(`${f}: removed, its plugin is gone or changed`);
|
|
1404
1528
|
// a plugin that was pinned and is not any more is also forgotten by a running daemon
|
|
1405
|
-
const changed = [...changedPinLines(before, after), ...removedPinUris(before, after).map(
|
|
1529
|
+
const changed = [...changedPinLines(before, after), ...removedPinUris(before, after).map(pinClearLine)];
|
|
1406
1530
|
const sent = await sendPins(settings.hostdPort, settings.hostdFeedbackPort, changed);
|
|
1407
1531
|
if ("unsent" in sent) {
|
|
1408
1532
|
const why = {
|
|
@@ -1415,7 +1539,7 @@ async function runLocked(settings, options) {
|
|
|
1415
1539
|
const replies = "replies" in sent ? sent.replies : [];
|
|
1416
1540
|
for (let i = 0; i < replies.length; i++)
|
|
1417
1541
|
if (replies[i] !== "resp 0")
|
|
1418
|
-
log(`plugin-hostd answered ${replies[i]} to ${changed[i]
|
|
1542
|
+
log(`plugin-hostd answered ${replies[i]} to ${pinLineUri(changed[i])}`);
|
|
1419
1543
|
const taken = replies.filter((r) => r === "resp 0").length;
|
|
1420
1544
|
for (const [why, n] of byMachine)
|
|
1421
1545
|
if (n > 1)
|