@openmixer/plugin-qualify 0.18.4 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -730
- package/bin/plugin-qualify-sync.mjs +4 -2
- package/bin/plugin-qualify.mjs +7 -0
- package/dist/clap-qualify.d.ts +1 -1
- package/dist/compare.d.ts +311 -0
- package/dist/compare.js +502 -0
- package/dist/compare.js.map +1 -0
- package/dist/distributed-records.d.ts +0 -2
- package/dist/distributed-records.js +0 -2
- package/dist/distributed-records.js.map +1 -1
- package/dist/host-admission.d.ts +2 -2
- package/dist/host-admission.js +7 -0
- package/dist/host-admission.js.map +1 -1
- package/dist/hosting-codes.generated.d.ts +1 -0
- package/dist/hosting-codes.generated.js +1 -0
- package/dist/hosting-codes.generated.js.map +1 -1
- package/dist/hosting-suitability.d.ts +3 -1
- package/dist/hosting-suitability.js +9 -0
- package/dist/hosting-suitability.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/jack-registry.d.ts +6 -0
- package/dist/jack-registry.js +52 -0
- package/dist/jack-registry.js.map +1 -0
- package/dist/lv2-loader.js +6 -5
- package/dist/lv2-loader.js.map +1 -1
- package/dist/measurement/format.d.ts +15 -0
- package/dist/measurement/format.js +19 -0
- package/dist/measurement/format.js.map +1 -1
- package/dist/measurement/sweep-jsonl.js +5 -1
- package/dist/measurement/sweep-jsonl.js.map +1 -1
- package/dist/measurement-grid.js +1 -1
- package/dist/pins-file.d.ts +22 -2
- package/dist/pins-file.js +118 -17
- package/dist/pins-file.js.map +1 -1
- package/dist/probe-judge.d.ts +5 -0
- package/dist/probe-judge.js.map +1 -1
- package/dist/probe-runner.d.ts +8 -0
- package/dist/probe-runner.js +8 -4
- package/dist/probe-runner.js.map +1 -1
- package/dist/qualification-record.d.ts +45 -0
- package/dist/qualification-record.js +36 -0
- package/dist/qualification-record.js.map +1 -1
- package/dist/qualify-adapter-pass.js +20 -5
- package/dist/qualify-adapter-pass.js.map +1 -1
- package/dist/qualify-declarations.d.ts +6 -0
- package/dist/qualify-declarations.js +8 -0
- package/dist/qualify-declarations.js.map +1 -1
- package/dist/qualify-isolated.d.ts +16 -3
- package/dist/qualify-isolated.js +31 -3
- package/dist/qualify-isolated.js.map +1 -1
- package/dist/qualify-pool.d.ts +129 -0
- package/dist/qualify-pool.js +290 -0
- package/dist/qualify-pool.js.map +1 -0
- package/dist/qualify-rejudge.js +7 -0
- package/dist/qualify-rejudge.js.map +1 -1
- package/dist/qualify-sync.d.ts +46 -1
- package/dist/qualify-sync.js +184 -20
- package/dist/qualify-sync.js.map +1 -1
- package/dist/records-package.d.ts +5 -0
- package/dist/records-package.js +25 -8
- package/dist/records-package.js.map +1 -1
- package/fixtures/clap-scan/fake.json +8 -2
- package/fixtures/probe-output/success-msi.json +2 -2
- package/package.json +3 -3
- package/src/clap-qualify.ts +1 -1
- package/src/compare.ts +699 -0
- package/src/distributed-records.ts +0 -3
- package/src/host-admission.ts +9 -2
- package/src/hosting-codes.generated.ts +1 -0
- package/src/hosting-suitability.ts +14 -0
- package/src/index.ts +1 -0
- package/src/jack-registry.ts +49 -0
- package/src/lv2-loader.ts +5 -4
- package/src/measurement/format.ts +32 -0
- package/src/measurement/sweep-jsonl.ts +5 -1
- package/src/measurement-grid.ts +1 -1
- package/src/pins-file.ts +109 -16
- package/src/probe-judge.ts +2 -0
- package/src/probe-runner.ts +18 -4
- package/src/qualification-record.ts +69 -0
- package/src/qualify-adapter-pass.ts +18 -5
- package/src/qualify-declarations.ts +8 -0
- package/src/qualify-isolated.ts +31 -5
- package/src/qualify-pool.ts +318 -0
- package/src/qualify-rejudge.ts +8 -0
- package/src/qualify-sync.ts +187 -19
- package/src/records-package.ts +25 -8
- package/tools/benchmark.py +9 -3
- package/tools/compare-host.mjs +432 -0
- package/tools/compare-run.mjs +355 -0
- package/tools/descriptors.py +27 -0
- package/tools/isolated-step.mjs +3 -1
- package/tools/lilv_load.py +25 -0
- package/tools/lv2_resolve.py +3 -1
- package/tools/pool-step.mjs +186 -0
- package/tools/scan.py +16 -20
- package/tools/audit-overlap.sh +0 -250
- package/tools/commit-message-check.sh +0 -128
- package/tools/commit-message-patterns.json +0 -41
- package/tools/private-refs-check.sh +0 -90
package/dist/qualify-sync.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type HostProfile } from "./host-profiles.js";
|
|
2
2
|
import { type PinEntry } from "./pins-file.js";
|
|
3
|
+
import { type PoolRunner } from "./qualify-pool.js";
|
|
3
4
|
import { type PinFilesRun } from "./hostd-pin-files.js";
|
|
4
5
|
import { type ProbeRun } from "./probe-runner.js";
|
|
5
6
|
import { type GridSpec } from "./measurement-grid.js";
|
|
@@ -16,7 +17,7 @@ export declare const ZYNTHIAN_LV2_DIRS: string[];
|
|
|
16
17
|
export interface SyncSetting {
|
|
17
18
|
readonly key: string;
|
|
18
19
|
readonly meaning: string;
|
|
19
|
-
/** The default as `--help` and
|
|
20
|
+
/** The default as `--help` and docs/overview.md print it. */
|
|
20
21
|
readonly default: string;
|
|
21
22
|
}
|
|
22
23
|
/** Every setting, declared once: the settings file, the flags (`--pin-file` for `pin_file`) and `--help` read this. */
|
|
@@ -70,6 +71,8 @@ export interface SyncArgs {
|
|
|
70
71
|
readonly adapterOnly: boolean;
|
|
71
72
|
/** `--isolated-only`: the missing isolated step of each record already judged, extending its run. */
|
|
72
73
|
readonly isolatedOnly: boolean;
|
|
74
|
+
/** `--pool-only`: the pool step of each CLAP record already judged that owes it, extending its run and giving its pin the word. */
|
|
75
|
+
readonly poolOnly: boolean;
|
|
73
76
|
readonly help: boolean;
|
|
74
77
|
/** The settings given as flags, by key. */
|
|
75
78
|
readonly values: ReadonlyMap<string, string>;
|
|
@@ -165,6 +168,12 @@ export declare const SYNC_CODE: {
|
|
|
165
168
|
* plugin, a crash) is not. `undefined` for every other outcome.
|
|
166
169
|
*/
|
|
167
170
|
export declare function machineReason(p: Pick<SyncPlugin, "outcome" | "code" | "detail">): string | undefined;
|
|
171
|
+
/**
|
|
172
|
+
* The figure the records of a recorded entry never took, or `undefined`: a record whose probe ran before that figure's
|
|
173
|
+
* pass ({@link latencyNeverTaken}) owes a measurement, and the sync measures the entry again although its bytes are the
|
|
174
|
+
* ones it recorded. A record that cannot be read owes nothing here: the sync's other checks deal with it.
|
|
175
|
+
*/
|
|
176
|
+
export declare function figureOwed(entry: Pick<SyncEntry, "plugins">): "latency" | undefined;
|
|
168
177
|
/** What the quick qualification never measures, stated in every record it writes. */
|
|
169
178
|
export declare const QUICK_NOT_MEASURED: readonly NotMeasured[];
|
|
170
179
|
export interface SyncPlugin {
|
|
@@ -218,6 +227,8 @@ export interface SyncContext {
|
|
|
218
227
|
readonly pinFilesRun?: PinFilesRun;
|
|
219
228
|
/** The isolated step, a stand-in in tests. */
|
|
220
229
|
readonly isolatedRunner?: IsolatedRunner;
|
|
230
|
+
/** The pool step, a stand-in in tests. */
|
|
231
|
+
readonly poolRunner?: PoolRunner;
|
|
221
232
|
}
|
|
222
233
|
/**
|
|
223
234
|
* Every pin of the state with the files plugin-hostd collects for its plugin now, measured again by nothing
|
|
@@ -242,6 +253,19 @@ export declare function isolatedPass(ctx: SyncContext, state: SyncState): {
|
|
|
242
253
|
readonly isolated: number;
|
|
243
254
|
readonly refused: number;
|
|
244
255
|
readonly notRun: number;
|
|
256
|
+
readonly repinned: number;
|
|
257
|
+
};
|
|
258
|
+
/**
|
|
259
|
+
* The pool steps the CLAP records of a state still owe, each extending its record's run. A plugin's pin takes the pool word
|
|
260
|
+
* of its step (`cores`, or `off` for a plugin that was measured and gains nothing) and none for a plugin that does not ask
|
|
261
|
+
* for the pool. A step that could not run stays owed: the next pass tries it again.
|
|
262
|
+
*/
|
|
263
|
+
export declare function poolPass(ctx: SyncContext, state: SyncState): {
|
|
264
|
+
readonly state: SyncState;
|
|
265
|
+
readonly cores: number;
|
|
266
|
+
readonly off: number;
|
|
267
|
+
readonly notAsked: number;
|
|
268
|
+
readonly notRun: number;
|
|
245
269
|
};
|
|
246
270
|
/** The pin file's text: every whole pin of the state, the worse verdict kept for a uri pinned twice. */
|
|
247
271
|
export declare function syncPinsText(state: SyncState): string;
|
|
@@ -315,6 +339,13 @@ export interface SyncSummary {
|
|
|
315
339
|
readonly refused: number;
|
|
316
340
|
readonly notRun: number;
|
|
317
341
|
};
|
|
342
|
+
/** `--pool-only`: the plugins granted the thread pool, those measured and not granted it, those that do not ask for it, and those the step could not run for. */
|
|
343
|
+
readonly poolOnly?: {
|
|
344
|
+
readonly cores: number;
|
|
345
|
+
readonly off: number;
|
|
346
|
+
readonly notAsked: number;
|
|
347
|
+
readonly notRun: number;
|
|
348
|
+
};
|
|
318
349
|
/** `--adapter-only`: the plugins run, those done already at these host versions, and those whose host could not be run. */
|
|
319
350
|
readonly adapterOnly?: {
|
|
320
351
|
readonly measured: number;
|
|
@@ -328,6 +359,16 @@ export interface SyncSummary {
|
|
|
328
359
|
* line each; none when the directory does not exist yet or everything is open.
|
|
329
360
|
*/
|
|
330
361
|
export declare function descriptorAccessProblems(dir: string): string[];
|
|
362
|
+
/** What the records say per plugin, not per record: a CLAP plugin has a record per offer (.default, the console's
|
|
363
|
+
* in-process tier, and .custom, omx-clap-host's isolated tier), and is offered when any of its records is. */
|
|
364
|
+
export interface PluginCounts {
|
|
365
|
+
readonly plugins: Readonly<Record<string, number>>;
|
|
366
|
+
readonly offered: Readonly<Record<string, number>>;
|
|
367
|
+
readonly notOffered: Readonly<Record<string, number>>;
|
|
368
|
+
}
|
|
369
|
+
export declare function pluginCounts(qualificationDir: string): PluginCounts;
|
|
370
|
+
/** The per-plugin counts as one line: `plugins 1577 (clap 248, lv2 1329), offered 1399 (...), not offered 178 (...)`. */
|
|
371
|
+
export declare function pluginCountsLine(c: PluginCounts): string;
|
|
331
372
|
export declare function summaryLine(s: SyncSummary): string;
|
|
332
373
|
export interface SyncLock {
|
|
333
374
|
release(): Promise<void>;
|
|
@@ -351,6 +392,10 @@ export interface SyncRunOptions {
|
|
|
351
392
|
readonly isolatedRunner?: IsolatedRunner;
|
|
352
393
|
/** `--isolated-only`: take only the isolated steps records still owe. */
|
|
353
394
|
readonly isolatedOnly?: boolean;
|
|
395
|
+
/** The pool step, a stand-in in tests. */
|
|
396
|
+
readonly poolRunner?: PoolRunner;
|
|
397
|
+
/** `--pool-only`: take only the pool steps records still owe. */
|
|
398
|
+
readonly poolOnly?: boolean;
|
|
354
399
|
/** `--adapter-only`: take only the adapter runs; `force` takes those already done again. */
|
|
355
400
|
readonly adapterOnly?: boolean;
|
|
356
401
|
/** Stand-ins for the pass's hosts, in tests. */
|
package/dist/qualify-sync.js
CHANGED
|
@@ -23,20 +23,21 @@ import { createConnection } from "node:net";
|
|
|
23
23
|
import { homedir, tmpdir } from "node:os";
|
|
24
24
|
import { basename, dirname, join, resolve, sep } from "node:path";
|
|
25
25
|
import { clapQualifyTools } from "./clap-qualify.js";
|
|
26
|
-
import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, matchDistributed, readBoardModel, readDistributedLayer } from "./distributed-records.js";
|
|
26
|
+
import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, outcomeOfVerdict, matchDistributed, readBoardModel, readDistributedLayer } from "./distributed-records.js";
|
|
27
27
|
import { clapSearchPath, parseClapScan } from "./clap-loader.js";
|
|
28
28
|
import { HOSTING_CODE } from "./hosting-codes.js";
|
|
29
29
|
import { shippedHostProfile } from "./host-profiles.js";
|
|
30
30
|
import { lv2BundlesOnPath, manifestPlugins } from "./lv2-loader.js";
|
|
31
|
-
import { judgmentPinEntries, pinsFile } from "./pins-file.js";
|
|
31
|
+
import { judgmentPinEntries, pinClearLine, pinLineUri, pinsFile } from "./pins-file.js";
|
|
32
|
+
import { POOL_RESULT, completePool, judgePoolRun, pinAfterPool, pinFromRecordWithPool, poolEnv, spawnedPoolRunner } from "./qualify-pool.js";
|
|
32
33
|
import { hostdPinFiles } from "./hostd-pin-files.js";
|
|
33
34
|
import { qualifyTargets } from "./probe-runner.js";
|
|
34
35
|
import { GRID_FULL, parseGridSpec } from "./measurement-grid.js";
|
|
35
36
|
import { adapterPass } from "./qualify-adapter-pass.js";
|
|
36
37
|
import { ISOLATED_RESULT, baseSteps, completeIsolated, isolatedCell, missingSteps, judgeIsolatedRun, owesIsolated, pinAfterStep, spawnedIsolatedRunner } from "./qualify-isolated.js";
|
|
37
|
-
import { lv2World, rejudgeState, memoryPass } from "./qualify-rejudge.js";
|
|
38
|
+
import { lv2World, rejudgeRecord, rejudgeState, rejudgeToolName, memoryPass } from "./qualify-rejudge.js";
|
|
38
39
|
import { memoryBoundOf, parseMemoryLimit } from "./memory-bound.js";
|
|
39
|
-
import { rewriteRecord, writeQualification } from "./qualification-record.js";
|
|
40
|
+
import { QUALIFICATION_RECORD_SCHEMA, latencyNeverTaken, rewriteRecord, writeQualification } from "./qualification-record.js";
|
|
40
41
|
import { ADMIT_STEP_MS_ENV, DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, MEMORY_PASS_PROBE_NOTES, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
|
|
41
42
|
import { toolsDir } from "./tool-paths.js";
|
|
42
43
|
import { writeFileWhole } from "./whole-file.js";
|
|
@@ -100,6 +101,7 @@ export function parseSyncArgs(argv) {
|
|
|
100
101
|
let memoryOnly = false;
|
|
101
102
|
let adapterOnly = false;
|
|
102
103
|
let isolatedOnly = false;
|
|
104
|
+
let poolOnly = false;
|
|
103
105
|
let help = false;
|
|
104
106
|
for (let i = 0; i < argv.length; i++) {
|
|
105
107
|
const a = argv[i];
|
|
@@ -121,6 +123,8 @@ export function parseSyncArgs(argv) {
|
|
|
121
123
|
adapterOnly = true;
|
|
122
124
|
else if (a === "--isolated-only")
|
|
123
125
|
isolatedOnly = true;
|
|
126
|
+
else if (a === "--pool-only")
|
|
127
|
+
poolOnly = true;
|
|
124
128
|
else if (a === "--help")
|
|
125
129
|
help = true;
|
|
126
130
|
else if (a === "--grid-hosts")
|
|
@@ -137,11 +141,11 @@ export function parseSyncArgs(argv) {
|
|
|
137
141
|
else
|
|
138
142
|
return { ok: false, error: `unknown argument ${a}` };
|
|
139
143
|
}
|
|
140
|
-
if (rejudge && (memoryOnly || adapterOnly || isolatedOnly || force || refreshPinsOnly))
|
|
141
|
-
return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --force and --refresh-pins" };
|
|
142
|
-
if ([memoryOnly, adapterOnly, isolatedOnly, refreshPinsOnly].filter(Boolean).length > 1)
|
|
143
|
-
return { ok: false, error: "--memory-only, --adapter-only and --refresh-pins are different runs: one at a time" };
|
|
144
|
-
return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, isolatedOnly, help, values } };
|
|
144
|
+
if (rejudge && (memoryOnly || adapterOnly || isolatedOnly || poolOnly || force || refreshPinsOnly))
|
|
145
|
+
return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --isolated-only, --pool-only, --force and --refresh-pins" };
|
|
146
|
+
if ([memoryOnly, adapterOnly, isolatedOnly, poolOnly, refreshPinsOnly].filter(Boolean).length > 1)
|
|
147
|
+
return { ok: false, error: "--memory-only, --adapter-only, --isolated-only, --pool-only and --refresh-pins are different runs: one at a time" };
|
|
148
|
+
return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, isolatedOnly, poolOnly, help, values } };
|
|
145
149
|
}
|
|
146
150
|
export function syncHelpText() {
|
|
147
151
|
const rows = SYNC_SETTINGS.map((s) => ` ${`${flagOf(s.key)} <value>`.padEnd(36)}${s.meaning}${s.default === "" ? "" : ` (default: ${s.default})`}`);
|
|
@@ -157,6 +161,7 @@ export function syncHelpText() {
|
|
|
157
161
|
" --rejudge derive every record's verdicts, tiers and pin again from the figures it holds; measures nothing",
|
|
158
162
|
" --memory-only measure only the footprint of each recorded plugin at the Pi's cells and merge it; with --force, those already merged too",
|
|
159
163
|
" --isolated-only run only the isolated step each record still owes, adding it to the record's run",
|
|
164
|
+
" --pool-only run only the pool step each CLAP record still owes: a plugin that asks for the thread pool is run with it off and on, and its pin gets pool=cores or pool=off",
|
|
160
165
|
" --adapter-only run each LV2 plugin only under omx-clap-host's LV2 adapter at the Pi's cells, on private JACK servers, and merge it; with --force, those already done too",
|
|
161
166
|
" --grid-hosts the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
|
|
162
167
|
" --help this text",
|
|
@@ -448,6 +453,27 @@ export function machineReason(p) {
|
|
|
448
453
|
return errno;
|
|
449
454
|
return /wrote no document|cannot run the probe runner/.test(p.detail ?? "") ? "the measurement wrote no document" : undefined;
|
|
450
455
|
}
|
|
456
|
+
/**
|
|
457
|
+
* The figure the records of a recorded entry never took, or `undefined`: a record whose probe ran before that figure's
|
|
458
|
+
* pass ({@link latencyNeverTaken}) owes a measurement, and the sync measures the entry again although its bytes are the
|
|
459
|
+
* ones it recorded. A record that cannot be read owes nothing here: the sync's other checks deal with it.
|
|
460
|
+
*/
|
|
461
|
+
export function figureOwed(entry) {
|
|
462
|
+
for (const p of entry.plugins) {
|
|
463
|
+
if (p.record === undefined)
|
|
464
|
+
continue;
|
|
465
|
+
let r;
|
|
466
|
+
try {
|
|
467
|
+
r = JSON.parse(readFileSync(p.record, "utf8"));
|
|
468
|
+
}
|
|
469
|
+
catch {
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
if (r.schema === QUALIFICATION_RECORD_SCHEMA && latencyNeverTaken(r))
|
|
473
|
+
return "latency";
|
|
474
|
+
}
|
|
475
|
+
return undefined;
|
|
476
|
+
}
|
|
451
477
|
/** What the quick qualification never measures, stated in every record it writes. */
|
|
452
478
|
export const QUICK_NOT_MEASURED = [{ dimension: "stability", code: HOSTING_CODE.stabilityUnsoaked }];
|
|
453
479
|
const EMPTY_STATE = { schema: SYNC_STATE_SCHEMA, entries: {} };
|
|
@@ -629,6 +655,13 @@ function judgedPlugins(ctx, runs, describeProblem) {
|
|
|
629
655
|
code = undefined;
|
|
630
656
|
}
|
|
631
657
|
}
|
|
658
|
+
// the pool is for a CLAP plugin that runs: the thread pool is granted by measuring it, here and nowhere else
|
|
659
|
+
if (judgment.output.plugin.format === "clap" && pin !== undefined && pin.verdict !== "refused" && outcome === "qualified") {
|
|
660
|
+
const runner = ctx.poolRunner ?? spawnedPoolRunner();
|
|
661
|
+
const step = judgePoolRun(runner(pin.uri, isolatedCell(record, host), poolEnv(ctx.env, ctx.settings.budgetS)));
|
|
662
|
+
steps.push(step);
|
|
663
|
+
pin = pinAfterPool(pin, step);
|
|
664
|
+
}
|
|
632
665
|
rewriteRecord(recordPath, { ...record, run: { id: judgment.output.run.id, steps } });
|
|
633
666
|
const plugin = { uri: pin?.uri ?? judgment.output.plugin.id, outcome, ...(code === undefined ? {} : { code }), ...(pin === undefined ? {} : { pin }), record: recordPath };
|
|
634
667
|
return { plugin, judgedHere: judgment.hosts.some((h) => h.verdict.host === host) };
|
|
@@ -761,7 +794,7 @@ export function qualifyItem(ctx, item, files) {
|
|
|
761
794
|
* a step that could not run stays owed: the next pass tries it again.
|
|
762
795
|
*/
|
|
763
796
|
export function isolatedPass(ctx, state) {
|
|
764
|
-
const count = { isolated: 0, refused: 0, notRun: 0 };
|
|
797
|
+
const count = { isolated: 0, refused: 0, notRun: 0, repinned: 0 };
|
|
765
798
|
const runner = ctx.isolatedRunner ?? spawnedIsolatedRunner();
|
|
766
799
|
const host = ctx.settings.hostProfile.id;
|
|
767
800
|
const entries = {};
|
|
@@ -781,8 +814,19 @@ export function isolatedPass(ctx, state) {
|
|
|
781
814
|
return p;
|
|
782
815
|
}
|
|
783
816
|
const done = completeIsolated(path, record, p.pin?.uri ?? record.plugin.id, isolatedCell(record, host), runner, isolatedEnv(ctx));
|
|
784
|
-
if (done === undefined)
|
|
785
|
-
|
|
817
|
+
if (done === undefined) {
|
|
818
|
+
// a plugin the isolated step already decided but the state holds no pin for: the record keeps the bytes and the
|
|
819
|
+
// step, and the pin follows them (hostd names the files, as at every other pin)
|
|
820
|
+
if (p.pin !== undefined)
|
|
821
|
+
return p;
|
|
822
|
+
const rebuilt = withHostdFiles(ctx, pinFromRecordWithPool(record));
|
|
823
|
+
if (rebuilt === undefined)
|
|
824
|
+
return p;
|
|
825
|
+
count.repinned++;
|
|
826
|
+
ctx.log(`${p.uri}: pinned again from its record (isolated step ${rebuilt.verdict ?? "undecided"})`);
|
|
827
|
+
const { code: _c, ...rest } = p;
|
|
828
|
+
return rest.outcome === "qualified" && rebuilt.verdict === "refused" ? { ...rest, outcome: "refused", pin: rebuilt } : { ...rest, pin: rebuilt };
|
|
829
|
+
}
|
|
786
830
|
ctx.log(`${p.uri}: isolated step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
|
|
787
831
|
const pin = pinAfterStep(p.pin, done.step);
|
|
788
832
|
const { pin: _pin, code: _code, ...rest } = p;
|
|
@@ -798,7 +842,50 @@ export function isolatedPass(ctx, state) {
|
|
|
798
842
|
}),
|
|
799
843
|
};
|
|
800
844
|
}
|
|
801
|
-
return { ...count, state: count.isolated + count.refused + count.notRun === 0 ? state : { ...state, entries } };
|
|
845
|
+
return { ...count, state: count.isolated + count.refused + count.notRun + count.repinned === 0 ? state : { ...state, entries } };
|
|
846
|
+
}
|
|
847
|
+
/**
|
|
848
|
+
* The pool steps the CLAP records of a state still owe, each extending its record's run. A plugin's pin takes the pool word
|
|
849
|
+
* of its step (`cores`, or `off` for a plugin that was measured and gains nothing) and none for a plugin that does not ask
|
|
850
|
+
* for the pool. A step that could not run stays owed: the next pass tries it again.
|
|
851
|
+
*/
|
|
852
|
+
export function poolPass(ctx, state) {
|
|
853
|
+
const count = { cores: 0, off: 0, notAsked: 0, notRun: 0 };
|
|
854
|
+
const runner = ctx.poolRunner ?? spawnedPoolRunner();
|
|
855
|
+
const host = ctx.settings.hostProfile.id;
|
|
856
|
+
const entries = {};
|
|
857
|
+
for (const [key, entry] of Object.entries(state.entries)) {
|
|
858
|
+
entries[key] = {
|
|
859
|
+
...entry,
|
|
860
|
+
plugins: entry.plugins.map((p) => {
|
|
861
|
+
if (p.record === undefined || !p.record.endsWith(".qualification.json") || p.pin === undefined || p.pin.verdict === "refused")
|
|
862
|
+
return p;
|
|
863
|
+
const path = join(ctx.settings.qualificationDir, basename(p.record));
|
|
864
|
+
let record;
|
|
865
|
+
try {
|
|
866
|
+
record = JSON.parse(readFileSync(path, "utf8"));
|
|
867
|
+
}
|
|
868
|
+
catch {
|
|
869
|
+
ctx.log(`${p.uri}: its record does not read; no pool step`);
|
|
870
|
+
return p;
|
|
871
|
+
}
|
|
872
|
+
const done = completePool(path, record, p.pin.uri, isolatedCell(record, host), runner, poolEnv(ctx.env, ctx.settings.budgetS));
|
|
873
|
+
if (done === undefined)
|
|
874
|
+
return p;
|
|
875
|
+
ctx.log(`${p.uri}: pool step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
|
|
876
|
+
if (done.step.result === POOL_RESULT.cores)
|
|
877
|
+
count.cores++;
|
|
878
|
+
else if (done.step.result === POOL_RESULT.off)
|
|
879
|
+
count.off++;
|
|
880
|
+
else if (done.step.result === POOL_RESULT.notAsked)
|
|
881
|
+
count.notAsked++;
|
|
882
|
+
else
|
|
883
|
+
count.notRun++;
|
|
884
|
+
return { ...p, pin: pinAfterPool(p.pin, done.step) };
|
|
885
|
+
}),
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
return { ...count, state: count.cores + count.off + count.notAsked + count.notRun === 0 ? state : { ...state, entries } };
|
|
802
889
|
}
|
|
803
890
|
// --- the distributed records ----------------------------------------------------------------------------------------
|
|
804
891
|
/** The layer a run reads: none under `--force` or `--no-distributed`, or without a host class. */
|
|
@@ -839,10 +926,36 @@ function takeDistributed(ctx, layer, item, files) {
|
|
|
839
926
|
ctx.log(`distributed: ${distributedUri(item, r)}: its record lacks the step${missing.length === 1 ? "" : "s"} ${missing.join(", ")}; measured here`);
|
|
840
927
|
return undefined;
|
|
841
928
|
}
|
|
929
|
+
if (latencyNeverTaken(r.record)) {
|
|
930
|
+
ctx.log(`distributed: ${distributedUri(item, r)}: its record never took the latency figure (a probe from before that pass); measured here`);
|
|
931
|
+
return undefined;
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
// The set was judged by the plugin-qualify that measured it. Its figures are carried across and judged again under
|
|
935
|
+
// today's rules, as an upgrade's are (`--rejudge`): a verdict of an older judge (RT-unsafe read as refused, before
|
|
936
|
+
// #196) is never taken as it stands. A record today's judge cannot decide for this host is measured here.
|
|
937
|
+
const by = rejudgeToolName();
|
|
938
|
+
const at = new Date(started).toISOString();
|
|
939
|
+
const records = [];
|
|
940
|
+
for (const r of match.records) {
|
|
941
|
+
let record;
|
|
942
|
+
try {
|
|
943
|
+
record = rejudgeRecord(r.record, readFileSync(join(r.path, "..", r.record.probe.file)), { host, by, at }).record;
|
|
944
|
+
}
|
|
945
|
+
catch (e) {
|
|
946
|
+
ctx.log(`distributed: ${distributedUri(item, r)}: ${e.message}; measured here`);
|
|
947
|
+
return undefined;
|
|
948
|
+
}
|
|
949
|
+
const v = record.verdicts.find((x) => x.host === host);
|
|
950
|
+
if (v !== undefined && outcomeOfVerdict(v) === "unmeasured") {
|
|
951
|
+
ctx.log(`distributed: ${distributedUri(item, r)}: judged again, ${v.deciding ?? "unknown"} for ${host}; measured here`);
|
|
952
|
+
return undefined;
|
|
953
|
+
}
|
|
954
|
+
records.push({ ...r, record });
|
|
842
955
|
}
|
|
843
956
|
let plugins;
|
|
844
957
|
try {
|
|
845
|
-
plugins = applyDistributed(ctx.settings.qualificationDir, item, match, host);
|
|
958
|
+
plugins = applyDistributed(ctx.settings.qualificationDir, item, { ...match, records }, host);
|
|
846
959
|
}
|
|
847
960
|
catch (e) {
|
|
848
961
|
ctx.log(`distributed: ${distributedUri(item, match.records[0])}: ${e.message}; measured here`);
|
|
@@ -882,7 +995,7 @@ function pinLinesOf(text) {
|
|
|
882
995
|
const out = new Map();
|
|
883
996
|
for (const line of text.split("\n"))
|
|
884
997
|
if (line !== "" && !line.startsWith("#"))
|
|
885
|
-
out.set(line
|
|
998
|
+
out.set(pinLineUri(line), line);
|
|
886
999
|
return out;
|
|
887
1000
|
}
|
|
888
1001
|
/** The pin lines of `after` that `before` does not hold as they are. */
|
|
@@ -1096,8 +1209,48 @@ export function descriptorAccessProblems(dir) {
|
|
|
1096
1209
|
problems.push(`${String(closed.length)} descriptor(s) in ${dir} cannot be read by other users, the console among them (first: ${closed[0]})`);
|
|
1097
1210
|
return problems;
|
|
1098
1211
|
}
|
|
1212
|
+
export function pluginCounts(qualificationDir) {
|
|
1213
|
+
const offeredBy = new Map();
|
|
1214
|
+
let files = [];
|
|
1215
|
+
try {
|
|
1216
|
+
files = readdirSync(qualificationDir).filter((f) => f.endsWith(".qualification.json"));
|
|
1217
|
+
}
|
|
1218
|
+
catch {
|
|
1219
|
+
// no directory, no record
|
|
1220
|
+
}
|
|
1221
|
+
for (const f of files) {
|
|
1222
|
+
let r;
|
|
1223
|
+
try {
|
|
1224
|
+
r = JSON.parse(readFileSync(join(qualificationDir, f), "utf8"));
|
|
1225
|
+
}
|
|
1226
|
+
catch {
|
|
1227
|
+
continue;
|
|
1228
|
+
}
|
|
1229
|
+
if (!r.plugin?.format || !r.plugin.id)
|
|
1230
|
+
continue;
|
|
1231
|
+
const key = `${r.plugin.format}\0${r.plugin.id}`;
|
|
1232
|
+
offeredBy.set(key, (offeredBy.get(key) ?? false) || (r.tiers?.offeredOn?.length ?? 0) > 0);
|
|
1233
|
+
}
|
|
1234
|
+
const plugins = {}, offered = {}, notOffered = {};
|
|
1235
|
+
for (const [key, isOffered] of offeredBy) {
|
|
1236
|
+
const format = key.split("\0")[0];
|
|
1237
|
+
plugins[format] = (plugins[format] ?? 0) + 1;
|
|
1238
|
+
const into = isOffered ? offered : notOffered;
|
|
1239
|
+
into[format] = (into[format] ?? 0) + 1;
|
|
1240
|
+
}
|
|
1241
|
+
return { plugins, offered, notOffered };
|
|
1242
|
+
}
|
|
1243
|
+
/** The per-plugin counts as one line: `plugins 1577 (clap 248, lv2 1329), offered 1399 (...), not offered 178 (...)`. */
|
|
1244
|
+
export function pluginCountsLine(c) {
|
|
1245
|
+
const part = (m) => {
|
|
1246
|
+
const keys = Object.keys(m).sort();
|
|
1247
|
+
const total = keys.reduce((n, k) => n + m[k], 0);
|
|
1248
|
+
return `${String(total)}${keys.length === 0 ? "" : ` (${keys.map((k) => `${k} ${String(m[k])}`).join(", ")})`}`;
|
|
1249
|
+
};
|
|
1250
|
+
return `plugins ${part(c.plugins)}, offered ${part(c.offered)}, not offered ${part(c.notOffered)}`;
|
|
1251
|
+
}
|
|
1099
1252
|
export function summaryLine(s) {
|
|
1100
|
-
return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
|
|
1253
|
+
return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.poolOnly === undefined ? "" : `, pool granted ${String(s.poolOnly.cores)}, off ${String(s.poolOnly.off)}, not asked ${String(s.poolOnly.notAsked)}, not run ${String(s.poolOnly.notRun)}`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
|
|
1101
1254
|
}
|
|
1102
1255
|
/** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
|
|
1103
1256
|
const LOCK_HELD = 99;
|
|
@@ -1182,6 +1335,7 @@ function addSummaries(a, b) {
|
|
|
1182
1335
|
...(b.rejudge === undefined ? (a.rejudge === undefined ? {} : { rejudge: a.rejudge }) : { rejudge: b.rejudge }),
|
|
1183
1336
|
...(b.memoryOnly === undefined ? (a.memoryOnly === undefined ? {} : { memoryOnly: a.memoryOnly }) : { memoryOnly: b.memoryOnly }),
|
|
1184
1337
|
...(b.isolatedOnly === undefined ? (a.isolatedOnly === undefined ? {} : { isolatedOnly: a.isolatedOnly }) : { isolatedOnly: b.isolatedOnly }),
|
|
1338
|
+
...(b.poolOnly === undefined ? (a.poolOnly === undefined ? {} : { poolOnly: a.poolOnly }) : { poolOnly: b.poolOnly }),
|
|
1185
1339
|
...(b.adapterOnly === undefined ? (a.adapterOnly === undefined ? {} : { adapterOnly: a.adapterOnly }) : { adapterOnly: b.adapterOnly }),
|
|
1186
1340
|
};
|
|
1187
1341
|
}
|
|
@@ -1205,7 +1359,7 @@ async function runLocked(settings, options) {
|
|
|
1205
1359
|
{
|
|
1206
1360
|
const started = Date.now();
|
|
1207
1361
|
const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
|
|
1208
|
-
const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }), ...(options.isolatedRunner === undefined ? {} : { isolatedRunner: options.isolatedRunner }) };
|
|
1362
|
+
const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }), ...(options.isolatedRunner === undefined ? {} : { isolatedRunner: options.isolatedRunner }), ...(options.poolRunner === undefined ? {} : { poolRunner: options.poolRunner }) };
|
|
1209
1363
|
// the pins first, before the quiet wait and any measurement: an upgrade of plugin-hostd that collects more files
|
|
1210
1364
|
// than the pins on disk hold refuses those plugins until their pins name them (plugin-qualify#239)
|
|
1211
1365
|
const fresh = refreshPins(ctx, readSyncState(settings.stateFile, log));
|
|
@@ -1228,6 +1382,13 @@ async function runLocked(settings, options) {
|
|
|
1228
1382
|
const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
|
|
1229
1383
|
return { ...summary, isolatedOnly: { isolated: r.isolated, refused: r.refused, notRun: r.notRun } };
|
|
1230
1384
|
}
|
|
1385
|
+
if (options.poolOnly === true) {
|
|
1386
|
+
const r = poolPass(ctx, fresh.state);
|
|
1387
|
+
if (r.state !== fresh.state)
|
|
1388
|
+
writeFileWhole(settings.stateFile, `${JSON.stringify(r.state, null, 1)}\n`);
|
|
1389
|
+
const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
|
|
1390
|
+
return { ...summary, poolOnly: { cores: r.cores, off: r.off, notAsked: r.notAsked, notRun: r.notRun } };
|
|
1391
|
+
}
|
|
1231
1392
|
if (options.adapterOnly === true) {
|
|
1232
1393
|
const r = adapterPass(ctx, fresh.state, { ...options.adapterPass, force: options.force === true });
|
|
1233
1394
|
log(`adapter runs under ${Object.entries(r.versions).map(([k, v]) => `${k} ${v}`).join(", ")}`);
|
|
@@ -1283,7 +1444,8 @@ async function runLocked(settings, options) {
|
|
|
1283
1444
|
for (const item of items) {
|
|
1284
1445
|
const recorded = recordedAsIs(item, entries[item.key]);
|
|
1285
1446
|
const retry = recorded.same ? entries[item.key].plugins.map(machineReason).find((r) => r !== undefined) : undefined;
|
|
1286
|
-
|
|
1447
|
+
const owed = recorded.same && options.force !== true && retry === undefined ? figureOwed(entries[item.key]) : undefined;
|
|
1448
|
+
if (recorded.same && options.force !== true && retry === undefined && owed === undefined) {
|
|
1287
1449
|
if (pass === 0)
|
|
1288
1450
|
count.skipped += entries[item.key].plugins.length;
|
|
1289
1451
|
continue;
|
|
@@ -1305,6 +1467,8 @@ async function runLocked(settings, options) {
|
|
|
1305
1467
|
}
|
|
1306
1468
|
if (recorded.same && retry !== undefined)
|
|
1307
1469
|
log(`${item.key}: unmeasured last run by the machine (${retry}); measured again`);
|
|
1470
|
+
if (owed !== undefined)
|
|
1471
|
+
log(`${item.key}: its record never took the ${owed} figure (a probe from before that pass); measured again`);
|
|
1308
1472
|
const files = recorded.files ?? digestFiles(item.root, item.files);
|
|
1309
1473
|
if (files === undefined) {
|
|
1310
1474
|
log(`${item.key}: a file of it cannot be read; skipped this run`);
|
|
@@ -1362,7 +1526,7 @@ async function runLocked(settings, options) {
|
|
|
1362
1526
|
for (const f of pruneStale(settings, stateBefore, state))
|
|
1363
1527
|
log(`${f}: removed, its plugin is gone or changed`);
|
|
1364
1528
|
// a plugin that was pinned and is not any more is also forgotten by a running daemon
|
|
1365
|
-
const changed = [...changedPinLines(before, after), ...removedPinUris(before, after).map(
|
|
1529
|
+
const changed = [...changedPinLines(before, after), ...removedPinUris(before, after).map(pinClearLine)];
|
|
1366
1530
|
const sent = await sendPins(settings.hostdPort, settings.hostdFeedbackPort, changed);
|
|
1367
1531
|
if ("unsent" in sent) {
|
|
1368
1532
|
const why = {
|
|
@@ -1375,7 +1539,7 @@ async function runLocked(settings, options) {
|
|
|
1375
1539
|
const replies = "replies" in sent ? sent.replies : [];
|
|
1376
1540
|
for (let i = 0; i < replies.length; i++)
|
|
1377
1541
|
if (replies[i] !== "resp 0")
|
|
1378
|
-
log(`plugin-hostd answered ${replies[i]} to ${changed[i]
|
|
1542
|
+
log(`plugin-hostd answered ${replies[i]} to ${pinLineUri(changed[i])}`);
|
|
1379
1543
|
const taken = replies.filter((r) => r === "resp 0").length;
|
|
1380
1544
|
for (const [why, n] of byMachine)
|
|
1381
1545
|
if (n > 1)
|