@openmixer/plugin-qualify 0.18.4 → 0.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +19 -730
  2. package/bin/plugin-qualify-sync.mjs +4 -2
  3. package/bin/plugin-qualify.mjs +7 -0
  4. package/dist/clap-qualify.d.ts +1 -1
  5. package/dist/compare.d.ts +311 -0
  6. package/dist/compare.js +502 -0
  7. package/dist/compare.js.map +1 -0
  8. package/dist/distributed-records.d.ts +0 -2
  9. package/dist/distributed-records.js +0 -2
  10. package/dist/distributed-records.js.map +1 -1
  11. package/dist/host-admission.d.ts +2 -2
  12. package/dist/host-admission.js +7 -0
  13. package/dist/host-admission.js.map +1 -1
  14. package/dist/hosting-codes.generated.d.ts +1 -0
  15. package/dist/hosting-codes.generated.js +1 -0
  16. package/dist/hosting-codes.generated.js.map +1 -1
  17. package/dist/hosting-suitability.d.ts +3 -1
  18. package/dist/hosting-suitability.js +9 -0
  19. package/dist/hosting-suitability.js.map +1 -1
  20. package/dist/index.d.ts +1 -0
  21. package/dist/index.js +1 -0
  22. package/dist/index.js.map +1 -1
  23. package/dist/jack-registry.d.ts +6 -0
  24. package/dist/jack-registry.js +52 -0
  25. package/dist/jack-registry.js.map +1 -0
  26. package/dist/lv2-loader.js +6 -5
  27. package/dist/lv2-loader.js.map +1 -1
  28. package/dist/measurement/format.d.ts +15 -0
  29. package/dist/measurement/format.js +19 -0
  30. package/dist/measurement/format.js.map +1 -1
  31. package/dist/measurement/sweep-jsonl.js +5 -1
  32. package/dist/measurement/sweep-jsonl.js.map +1 -1
  33. package/dist/measurement-grid.js +1 -1
  34. package/dist/pins-file.d.ts +22 -2
  35. package/dist/pins-file.js +118 -17
  36. package/dist/pins-file.js.map +1 -1
  37. package/dist/probe-judge.d.ts +5 -0
  38. package/dist/probe-judge.js.map +1 -1
  39. package/dist/probe-runner.d.ts +8 -0
  40. package/dist/probe-runner.js +8 -4
  41. package/dist/probe-runner.js.map +1 -1
  42. package/dist/qualification-record.d.ts +45 -0
  43. package/dist/qualification-record.js +36 -0
  44. package/dist/qualification-record.js.map +1 -1
  45. package/dist/qualify-adapter-pass.js +20 -5
  46. package/dist/qualify-adapter-pass.js.map +1 -1
  47. package/dist/qualify-declarations.d.ts +6 -0
  48. package/dist/qualify-declarations.js +8 -0
  49. package/dist/qualify-declarations.js.map +1 -1
  50. package/dist/qualify-isolated.d.ts +16 -3
  51. package/dist/qualify-isolated.js +31 -3
  52. package/dist/qualify-isolated.js.map +1 -1
  53. package/dist/qualify-pool.d.ts +129 -0
  54. package/dist/qualify-pool.js +290 -0
  55. package/dist/qualify-pool.js.map +1 -0
  56. package/dist/qualify-rejudge.js +7 -0
  57. package/dist/qualify-rejudge.js.map +1 -1
  58. package/dist/qualify-sync.d.ts +46 -1
  59. package/dist/qualify-sync.js +184 -20
  60. package/dist/qualify-sync.js.map +1 -1
  61. package/dist/records-package.d.ts +5 -0
  62. package/dist/records-package.js +25 -8
  63. package/dist/records-package.js.map +1 -1
  64. package/fixtures/clap-scan/fake.json +8 -2
  65. package/fixtures/probe-output/success-msi.json +2 -2
  66. package/package.json +3 -3
  67. package/src/clap-qualify.ts +1 -1
  68. package/src/compare.ts +699 -0
  69. package/src/distributed-records.ts +0 -3
  70. package/src/host-admission.ts +9 -2
  71. package/src/hosting-codes.generated.ts +1 -0
  72. package/src/hosting-suitability.ts +14 -0
  73. package/src/index.ts +1 -0
  74. package/src/jack-registry.ts +49 -0
  75. package/src/lv2-loader.ts +5 -4
  76. package/src/measurement/format.ts +32 -0
  77. package/src/measurement/sweep-jsonl.ts +5 -1
  78. package/src/measurement-grid.ts +1 -1
  79. package/src/pins-file.ts +109 -16
  80. package/src/probe-judge.ts +2 -0
  81. package/src/probe-runner.ts +18 -4
  82. package/src/qualification-record.ts +69 -0
  83. package/src/qualify-adapter-pass.ts +18 -5
  84. package/src/qualify-declarations.ts +8 -0
  85. package/src/qualify-isolated.ts +31 -5
  86. package/src/qualify-pool.ts +318 -0
  87. package/src/qualify-rejudge.ts +8 -0
  88. package/src/qualify-sync.ts +187 -19
  89. package/src/records-package.ts +25 -8
  90. package/tools/benchmark.py +9 -3
  91. package/tools/compare-host.mjs +432 -0
  92. package/tools/compare-run.mjs +355 -0
  93. package/tools/descriptors.py +27 -0
  94. package/tools/isolated-step.mjs +3 -1
  95. package/tools/lilv_load.py +25 -0
  96. package/tools/lv2_resolve.py +3 -1
  97. package/tools/pool-step.mjs +186 -0
  98. package/tools/scan.py +16 -20
  99. package/tools/audit-overlap.sh +0 -250
  100. package/tools/commit-message-check.sh +0 -128
  101. package/tools/commit-message-patterns.json +0 -41
  102. package/tools/private-refs-check.sh +0 -90
@@ -1,5 +1,6 @@
1
1
  import { type HostProfile } from "./host-profiles.js";
2
2
  import { type PinEntry } from "./pins-file.js";
3
+ import { type PoolRunner } from "./qualify-pool.js";
3
4
  import { type PinFilesRun } from "./hostd-pin-files.js";
4
5
  import { type ProbeRun } from "./probe-runner.js";
5
6
  import { type GridSpec } from "./measurement-grid.js";
@@ -16,7 +17,7 @@ export declare const ZYNTHIAN_LV2_DIRS: string[];
16
17
  export interface SyncSetting {
17
18
  readonly key: string;
18
19
  readonly meaning: string;
19
- /** The default as `--help` and the README print it. */
20
+ /** The default as `--help` and docs/overview.md print it. */
20
21
  readonly default: string;
21
22
  }
22
23
  /** Every setting, declared once: the settings file, the flags (`--pin-file` for `pin_file`) and `--help` read this. */
@@ -70,6 +71,8 @@ export interface SyncArgs {
70
71
  readonly adapterOnly: boolean;
71
72
  /** `--isolated-only`: the missing isolated step of each record already judged, extending its run. */
72
73
  readonly isolatedOnly: boolean;
74
+ /** `--pool-only`: the pool step of each CLAP record already judged that owes it, extending its run and giving its pin the word. */
75
+ readonly poolOnly: boolean;
73
76
  readonly help: boolean;
74
77
  /** The settings given as flags, by key. */
75
78
  readonly values: ReadonlyMap<string, string>;
@@ -165,6 +168,12 @@ export declare const SYNC_CODE: {
165
168
  * plugin, a crash) is not. `undefined` for every other outcome.
166
169
  */
167
170
  export declare function machineReason(p: Pick<SyncPlugin, "outcome" | "code" | "detail">): string | undefined;
171
+ /**
172
+ * The figure the records of a recorded entry never took, or `undefined`: a record whose probe ran before that figure's
173
+ * pass ({@link latencyNeverTaken}) owes a measurement, and the sync measures the entry again although its bytes are the
174
+ * ones it recorded. A record that cannot be read owes nothing here: the sync's other checks deal with it.
175
+ */
176
+ export declare function figureOwed(entry: Pick<SyncEntry, "plugins">): "latency" | undefined;
168
177
  /** What the quick qualification never measures, stated in every record it writes. */
169
178
  export declare const QUICK_NOT_MEASURED: readonly NotMeasured[];
170
179
  export interface SyncPlugin {
@@ -218,6 +227,8 @@ export interface SyncContext {
218
227
  readonly pinFilesRun?: PinFilesRun;
219
228
  /** The isolated step, a stand-in in tests. */
220
229
  readonly isolatedRunner?: IsolatedRunner;
230
+ /** The pool step, a stand-in in tests. */
231
+ readonly poolRunner?: PoolRunner;
221
232
  }
222
233
  /**
223
234
  * Every pin of the state with the files plugin-hostd collects for its plugin now, measured again by nothing
@@ -242,6 +253,19 @@ export declare function isolatedPass(ctx: SyncContext, state: SyncState): {
242
253
  readonly isolated: number;
243
254
  readonly refused: number;
244
255
  readonly notRun: number;
256
+ readonly repinned: number;
257
+ };
258
+ /**
259
+ * The pool steps the CLAP records of a state still owe, each extending its record's run. A plugin's pin takes the pool word
260
+ * of its step (`cores`, or `off` for a plugin that was measured and gains nothing) and none for a plugin that does not ask
261
+ * for the pool. A step that could not run stays owed: the next pass tries it again.
262
+ */
263
+ export declare function poolPass(ctx: SyncContext, state: SyncState): {
264
+ readonly state: SyncState;
265
+ readonly cores: number;
266
+ readonly off: number;
267
+ readonly notAsked: number;
268
+ readonly notRun: number;
245
269
  };
246
270
  /** The pin file's text: every whole pin of the state, the worse verdict kept for a uri pinned twice. */
247
271
  export declare function syncPinsText(state: SyncState): string;
@@ -315,6 +339,13 @@ export interface SyncSummary {
315
339
  readonly refused: number;
316
340
  readonly notRun: number;
317
341
  };
342
+ /** `--pool-only`: the plugins granted the thread pool, those measured and not granted it, those that do not ask for it, and those the step could not run for. */
343
+ readonly poolOnly?: {
344
+ readonly cores: number;
345
+ readonly off: number;
346
+ readonly notAsked: number;
347
+ readonly notRun: number;
348
+ };
318
349
  /** `--adapter-only`: the plugins run, those done already at these host versions, and those whose host could not be run. */
319
350
  readonly adapterOnly?: {
320
351
  readonly measured: number;
@@ -328,6 +359,16 @@ export interface SyncSummary {
328
359
  * line each; none when the directory does not exist yet or everything is open.
329
360
  */
330
361
  export declare function descriptorAccessProblems(dir: string): string[];
362
+ /** What the records say per plugin, not per record: a CLAP plugin has a record per offer (.default, the console's
363
+ * in-process tier, and .custom, omx-clap-host's isolated tier), and is offered when any of its records is. */
364
+ export interface PluginCounts {
365
+ readonly plugins: Readonly<Record<string, number>>;
366
+ readonly offered: Readonly<Record<string, number>>;
367
+ readonly notOffered: Readonly<Record<string, number>>;
368
+ }
369
+ export declare function pluginCounts(qualificationDir: string): PluginCounts;
370
+ /** The per-plugin counts as one line: `plugins 1577 (clap 248, lv2 1329), offered 1399 (...), not offered 178 (...)`. */
371
+ export declare function pluginCountsLine(c: PluginCounts): string;
331
372
  export declare function summaryLine(s: SyncSummary): string;
332
373
  export interface SyncLock {
333
374
  release(): Promise<void>;
@@ -351,6 +392,10 @@ export interface SyncRunOptions {
351
392
  readonly isolatedRunner?: IsolatedRunner;
352
393
  /** `--isolated-only`: take only the isolated steps records still owe. */
353
394
  readonly isolatedOnly?: boolean;
395
+ /** The pool step, a stand-in in tests. */
396
+ readonly poolRunner?: PoolRunner;
397
+ /** `--pool-only`: take only the pool steps records still owe. */
398
+ readonly poolOnly?: boolean;
354
399
  /** `--adapter-only`: take only the adapter runs; `force` takes those already done again. */
355
400
  readonly adapterOnly?: boolean;
356
401
  /** Stand-ins for the pass's hosts, in tests. */
@@ -23,20 +23,21 @@ import { createConnection } from "node:net";
23
23
  import { homedir, tmpdir } from "node:os";
24
24
  import { basename, dirname, join, resolve, sep } from "node:path";
25
25
  import { clapQualifyTools } from "./clap-qualify.js";
26
- import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, matchDistributed, readBoardModel, readDistributedLayer } from "./distributed-records.js";
26
+ import { DISTRIBUTED_DIR_SYSTEM, EMPTY_LAYER, applyDistributed, distributedUri, hostClassOfBoard, outcomeOfVerdict, matchDistributed, readBoardModel, readDistributedLayer } from "./distributed-records.js";
27
27
  import { clapSearchPath, parseClapScan } from "./clap-loader.js";
28
28
  import { HOSTING_CODE } from "./hosting-codes.js";
29
29
  import { shippedHostProfile } from "./host-profiles.js";
30
30
  import { lv2BundlesOnPath, manifestPlugins } from "./lv2-loader.js";
31
- import { judgmentPinEntries, pinsFile } from "./pins-file.js";
31
+ import { judgmentPinEntries, pinClearLine, pinLineUri, pinsFile } from "./pins-file.js";
32
+ import { POOL_RESULT, completePool, judgePoolRun, pinAfterPool, pinFromRecordWithPool, poolEnv, spawnedPoolRunner } from "./qualify-pool.js";
32
33
  import { hostdPinFiles } from "./hostd-pin-files.js";
33
34
  import { qualifyTargets } from "./probe-runner.js";
34
35
  import { GRID_FULL, parseGridSpec } from "./measurement-grid.js";
35
36
  import { adapterPass } from "./qualify-adapter-pass.js";
36
37
  import { ISOLATED_RESULT, baseSteps, completeIsolated, isolatedCell, missingSteps, judgeIsolatedRun, owesIsolated, pinAfterStep, spawnedIsolatedRunner } from "./qualify-isolated.js";
37
- import { lv2World, rejudgeState, memoryPass } from "./qualify-rejudge.js";
38
+ import { lv2World, rejudgeRecord, rejudgeState, rejudgeToolName, memoryPass } from "./qualify-rejudge.js";
38
39
  import { memoryBoundOf, parseMemoryLimit } from "./memory-bound.js";
39
- import { rewriteRecord, writeQualification } from "./qualification-record.js";
40
+ import { QUALIFICATION_RECORD_SCHEMA, latencyNeverTaken, rewriteRecord, writeQualification } from "./qualification-record.js";
40
41
  import { ADMIT_STEP_MS_ENV, DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, MEMORY_PASS_PROBE_NOTES, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
41
42
  import { toolsDir } from "./tool-paths.js";
42
43
  import { writeFileWhole } from "./whole-file.js";
@@ -100,6 +101,7 @@ export function parseSyncArgs(argv) {
100
101
  let memoryOnly = false;
101
102
  let adapterOnly = false;
102
103
  let isolatedOnly = false;
104
+ let poolOnly = false;
103
105
  let help = false;
104
106
  for (let i = 0; i < argv.length; i++) {
105
107
  const a = argv[i];
@@ -121,6 +123,8 @@ export function parseSyncArgs(argv) {
121
123
  adapterOnly = true;
122
124
  else if (a === "--isolated-only")
123
125
  isolatedOnly = true;
126
+ else if (a === "--pool-only")
127
+ poolOnly = true;
124
128
  else if (a === "--help")
125
129
  help = true;
126
130
  else if (a === "--grid-hosts")
@@ -137,11 +141,11 @@ export function parseSyncArgs(argv) {
137
141
  else
138
142
  return { ok: false, error: `unknown argument ${a}` };
139
143
  }
140
- if (rejudge && (memoryOnly || adapterOnly || isolatedOnly || force || refreshPinsOnly))
141
- return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --force and --refresh-pins" };
142
- if ([memoryOnly, adapterOnly, isolatedOnly, refreshPinsOnly].filter(Boolean).length > 1)
143
- return { ok: false, error: "--memory-only, --adapter-only and --refresh-pins are different runs: one at a time" };
144
- return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, isolatedOnly, help, values } };
144
+ if (rejudge && (memoryOnly || adapterOnly || isolatedOnly || poolOnly || force || refreshPinsOnly))
145
+ return { ok: false, error: "--rejudge measures nothing: it goes with none of --memory-only, --adapter-only, --isolated-only, --pool-only, --force and --refresh-pins" };
146
+ if ([memoryOnly, adapterOnly, isolatedOnly, poolOnly, refreshPinsOnly].filter(Boolean).length > 1)
147
+ return { ok: false, error: "--memory-only, --adapter-only, --isolated-only, --pool-only and --refresh-pins are different runs: one at a time" };
148
+ return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, refreshPins: refreshPinsOnly, rejudge, memoryOnly, adapterOnly, isolatedOnly, poolOnly, help, values } };
145
149
  }
146
150
  export function syncHelpText() {
147
151
  const rows = SYNC_SETTINGS.map((s) => ` ${`${flagOf(s.key)} <value>`.padEnd(36)}${s.meaning}${s.default === "" ? "" : ` (default: ${s.default})`}`);
@@ -157,6 +161,7 @@ export function syncHelpText() {
157
161
  " --rejudge derive every record's verdicts, tiers and pin again from the figures it holds; measures nothing",
158
162
  " --memory-only measure only the footprint of each recorded plugin at the Pi's cells and merge it; with --force, those already merged too",
159
163
  " --isolated-only run only the isolated step each record still owes, adding it to the record's run",
164
+ " --pool-only run only the pool step each CLAP record still owes: a plugin that asks for the thread pool is run with it off and on, and its pin gets pool=cores or pool=off",
160
165
  " --adapter-only run each LV2 plugin only under omx-clap-host's LV2 adapter at the Pi's cells, on private JACK servers, and merge it; with --force, those already done too",
161
166
  " --grid-hosts the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
162
167
  " --help this text",
@@ -448,6 +453,27 @@ export function machineReason(p) {
448
453
  return errno;
449
454
  return /wrote no document|cannot run the probe runner/.test(p.detail ?? "") ? "the measurement wrote no document" : undefined;
450
455
  }
456
+ /**
457
+ * The figure the records of a recorded entry never took, or `undefined`: a record whose probe ran before that figure's
458
+ * pass ({@link latencyNeverTaken}) owes a measurement, and the sync measures the entry again although its bytes are the
459
+ * ones it recorded. A record that cannot be read owes nothing here: the sync's other checks deal with it.
460
+ */
461
+ export function figureOwed(entry) {
462
+ for (const p of entry.plugins) {
463
+ if (p.record === undefined)
464
+ continue;
465
+ let r;
466
+ try {
467
+ r = JSON.parse(readFileSync(p.record, "utf8"));
468
+ }
469
+ catch {
470
+ continue;
471
+ }
472
+ if (r.schema === QUALIFICATION_RECORD_SCHEMA && latencyNeverTaken(r))
473
+ return "latency";
474
+ }
475
+ return undefined;
476
+ }
451
477
  /** What the quick qualification never measures, stated in every record it writes. */
452
478
  export const QUICK_NOT_MEASURED = [{ dimension: "stability", code: HOSTING_CODE.stabilityUnsoaked }];
453
479
  const EMPTY_STATE = { schema: SYNC_STATE_SCHEMA, entries: {} };
@@ -629,6 +655,13 @@ function judgedPlugins(ctx, runs, describeProblem) {
629
655
  code = undefined;
630
656
  }
631
657
  }
658
+ // the pool is for a CLAP plugin that runs: the thread pool is granted by measuring it, here and nowhere else
659
+ if (judgment.output.plugin.format === "clap" && pin !== undefined && pin.verdict !== "refused" && outcome === "qualified") {
660
+ const runner = ctx.poolRunner ?? spawnedPoolRunner();
661
+ const step = judgePoolRun(runner(pin.uri, isolatedCell(record, host), poolEnv(ctx.env, ctx.settings.budgetS)));
662
+ steps.push(step);
663
+ pin = pinAfterPool(pin, step);
664
+ }
632
665
  rewriteRecord(recordPath, { ...record, run: { id: judgment.output.run.id, steps } });
633
666
  const plugin = { uri: pin?.uri ?? judgment.output.plugin.id, outcome, ...(code === undefined ? {} : { code }), ...(pin === undefined ? {} : { pin }), record: recordPath };
634
667
  return { plugin, judgedHere: judgment.hosts.some((h) => h.verdict.host === host) };
@@ -761,7 +794,7 @@ export function qualifyItem(ctx, item, files) {
761
794
  * a step that could not run stays owed: the next pass tries it again.
762
795
  */
763
796
  export function isolatedPass(ctx, state) {
764
- const count = { isolated: 0, refused: 0, notRun: 0 };
797
+ const count = { isolated: 0, refused: 0, notRun: 0, repinned: 0 };
765
798
  const runner = ctx.isolatedRunner ?? spawnedIsolatedRunner();
766
799
  const host = ctx.settings.hostProfile.id;
767
800
  const entries = {};
@@ -781,8 +814,19 @@ export function isolatedPass(ctx, state) {
781
814
  return p;
782
815
  }
783
816
  const done = completeIsolated(path, record, p.pin?.uri ?? record.plugin.id, isolatedCell(record, host), runner, isolatedEnv(ctx));
784
- if (done === undefined)
785
- return p;
817
+ if (done === undefined) {
818
+ // a plugin the isolated step already decided but the state holds no pin for: the record keeps the bytes and the
819
+ // step, and the pin follows them (hostd names the files, as at every other pin)
820
+ if (p.pin !== undefined)
821
+ return p;
822
+ const rebuilt = withHostdFiles(ctx, pinFromRecordWithPool(record));
823
+ if (rebuilt === undefined)
824
+ return p;
825
+ count.repinned++;
826
+ ctx.log(`${p.uri}: pinned again from its record (isolated step ${rebuilt.verdict ?? "undecided"})`);
827
+ const { code: _c, ...rest } = p;
828
+ return rest.outcome === "qualified" && rebuilt.verdict === "refused" ? { ...rest, outcome: "refused", pin: rebuilt } : { ...rest, pin: rebuilt };
829
+ }
786
830
  ctx.log(`${p.uri}: isolated step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
787
831
  const pin = pinAfterStep(p.pin, done.step);
788
832
  const { pin: _pin, code: _code, ...rest } = p;
@@ -798,7 +842,50 @@ export function isolatedPass(ctx, state) {
798
842
  }),
799
843
  };
800
844
  }
801
- return { ...count, state: count.isolated + count.refused + count.notRun === 0 ? state : { ...state, entries } };
845
+ return { ...count, state: count.isolated + count.refused + count.notRun + count.repinned === 0 ? state : { ...state, entries } };
846
+ }
847
+ /**
848
+ * The pool steps the CLAP records of a state still owe, each extending its record's run. A plugin's pin takes the pool word
849
+ * of its step (`cores`, or `off` for a plugin that was measured and gains nothing) and none for a plugin that does not ask
850
+ * for the pool. A step that could not run stays owed: the next pass tries it again.
851
+ */
852
+ export function poolPass(ctx, state) {
853
+ const count = { cores: 0, off: 0, notAsked: 0, notRun: 0 };
854
+ const runner = ctx.poolRunner ?? spawnedPoolRunner();
855
+ const host = ctx.settings.hostProfile.id;
856
+ const entries = {};
857
+ for (const [key, entry] of Object.entries(state.entries)) {
858
+ entries[key] = {
859
+ ...entry,
860
+ plugins: entry.plugins.map((p) => {
861
+ if (p.record === undefined || !p.record.endsWith(".qualification.json") || p.pin === undefined || p.pin.verdict === "refused")
862
+ return p;
863
+ const path = join(ctx.settings.qualificationDir, basename(p.record));
864
+ let record;
865
+ try {
866
+ record = JSON.parse(readFileSync(path, "utf8"));
867
+ }
868
+ catch {
869
+ ctx.log(`${p.uri}: its record does not read; no pool step`);
870
+ return p;
871
+ }
872
+ const done = completePool(path, record, p.pin.uri, isolatedCell(record, host), runner, poolEnv(ctx.env, ctx.settings.budgetS));
873
+ if (done === undefined)
874
+ return p;
875
+ ctx.log(`${p.uri}: pool step ${done.step.result}${done.step.code === undefined ? "" : ` (${done.step.code})`}`);
876
+ if (done.step.result === POOL_RESULT.cores)
877
+ count.cores++;
878
+ else if (done.step.result === POOL_RESULT.off)
879
+ count.off++;
880
+ else if (done.step.result === POOL_RESULT.notAsked)
881
+ count.notAsked++;
882
+ else
883
+ count.notRun++;
884
+ return { ...p, pin: pinAfterPool(p.pin, done.step) };
885
+ }),
886
+ };
887
+ }
888
+ return { ...count, state: count.cores + count.off + count.notAsked + count.notRun === 0 ? state : { ...state, entries } };
802
889
  }
803
890
  // --- the distributed records ----------------------------------------------------------------------------------------
804
891
  /** The layer a run reads: none under `--force` or `--no-distributed`, or without a host class. */
@@ -839,10 +926,36 @@ function takeDistributed(ctx, layer, item, files) {
839
926
  ctx.log(`distributed: ${distributedUri(item, r)}: its record lacks the step${missing.length === 1 ? "" : "s"} ${missing.join(", ")}; measured here`);
840
927
  return undefined;
841
928
  }
929
+ if (latencyNeverTaken(r.record)) {
930
+ ctx.log(`distributed: ${distributedUri(item, r)}: its record never took the latency figure (a probe from before that pass); measured here`);
931
+ return undefined;
932
+ }
933
+ }
934
+ // The set was judged by the plugin-qualify that measured it. Its figures are carried across and judged again under
935
+ // today's rules, as an upgrade's are (`--rejudge`): a verdict of an older judge (RT-unsafe read as refused, before
936
+ // #196) is never taken as it stands. A record today's judge cannot decide for this host is measured here.
937
+ const by = rejudgeToolName();
938
+ const at = new Date(started).toISOString();
939
+ const records = [];
940
+ for (const r of match.records) {
941
+ let record;
942
+ try {
943
+ record = rejudgeRecord(r.record, readFileSync(join(r.path, "..", r.record.probe.file)), { host, by, at }).record;
944
+ }
945
+ catch (e) {
946
+ ctx.log(`distributed: ${distributedUri(item, r)}: ${e.message}; measured here`);
947
+ return undefined;
948
+ }
949
+ const v = record.verdicts.find((x) => x.host === host);
950
+ if (v !== undefined && outcomeOfVerdict(v) === "unmeasured") {
951
+ ctx.log(`distributed: ${distributedUri(item, r)}: judged again, ${v.deciding ?? "unknown"} for ${host}; measured here`);
952
+ return undefined;
953
+ }
954
+ records.push({ ...r, record });
842
955
  }
843
956
  let plugins;
844
957
  try {
845
- plugins = applyDistributed(ctx.settings.qualificationDir, item, match, host);
958
+ plugins = applyDistributed(ctx.settings.qualificationDir, item, { ...match, records }, host);
846
959
  }
847
960
  catch (e) {
848
961
  ctx.log(`distributed: ${distributedUri(item, match.records[0])}: ${e.message}; measured here`);
@@ -882,7 +995,7 @@ function pinLinesOf(text) {
882
995
  const out = new Map();
883
996
  for (const line of text.split("\n"))
884
997
  if (line !== "" && !line.startsWith("#"))
885
- out.set(line.split(" ")[1], line);
998
+ out.set(pinLineUri(line), line);
886
999
  return out;
887
1000
  }
888
1001
  /** The pin lines of `after` that `before` does not hold as they are. */
@@ -1096,8 +1209,48 @@ export function descriptorAccessProblems(dir) {
1096
1209
  problems.push(`${String(closed.length)} descriptor(s) in ${dir} cannot be read by other users, the console among them (first: ${closed[0]})`);
1097
1210
  return problems;
1098
1211
  }
1212
+ export function pluginCounts(qualificationDir) {
1213
+ const offeredBy = new Map();
1214
+ let files = [];
1215
+ try {
1216
+ files = readdirSync(qualificationDir).filter((f) => f.endsWith(".qualification.json"));
1217
+ }
1218
+ catch {
1219
+ // no directory, no record
1220
+ }
1221
+ for (const f of files) {
1222
+ let r;
1223
+ try {
1224
+ r = JSON.parse(readFileSync(join(qualificationDir, f), "utf8"));
1225
+ }
1226
+ catch {
1227
+ continue;
1228
+ }
1229
+ if (!r.plugin?.format || !r.plugin.id)
1230
+ continue;
1231
+ const key = `${r.plugin.format}\0${r.plugin.id}`;
1232
+ offeredBy.set(key, (offeredBy.get(key) ?? false) || (r.tiers?.offeredOn?.length ?? 0) > 0);
1233
+ }
1234
+ const plugins = {}, offered = {}, notOffered = {};
1235
+ for (const [key, isOffered] of offeredBy) {
1236
+ const format = key.split("\0")[0];
1237
+ plugins[format] = (plugins[format] ?? 0) + 1;
1238
+ const into = isOffered ? offered : notOffered;
1239
+ into[format] = (into[format] ?? 0) + 1;
1240
+ }
1241
+ return { plugins, offered, notOffered };
1242
+ }
1243
+ /** The per-plugin counts as one line: `plugins 1577 (clap 248, lv2 1329), offered 1399 (...), not offered 178 (...)`. */
1244
+ export function pluginCountsLine(c) {
1245
+ const part = (m) => {
1246
+ const keys = Object.keys(m).sort();
1247
+ const total = keys.reduce((n, k) => n + m[k], 0);
1248
+ return `${String(total)}${keys.length === 0 ? "" : ` (${keys.map((k) => `${k} ${String(m[k])}`).join(", ")})`}`;
1249
+ };
1250
+ return `plugins ${part(c.plugins)}, offered ${part(c.offered)}, not offered ${part(c.notOffered)}`;
1251
+ }
1099
1252
  export function summaryLine(s) {
1100
- return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
1253
+ return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}${s.refreshed ? `, pins refreshed ${String(s.refreshed)}` : ""}${s.rejudge === undefined ? "" : `, rejudged ${String(s.rejudge.changed)}, unchanged ${String(s.rejudge.unchanged)}, owing ${String(s.rejudge.owing)}`}${s.memoryOnly === undefined ? "" : `, memory measured ${String(s.memoryOnly.measured)}, already ${String(s.memoryOnly.already)}, failed ${String(s.memoryOnly.failed)}, leaks not measured`}${s.poolOnly === undefined ? "" : `, pool granted ${String(s.poolOnly.cores)}, off ${String(s.poolOnly.off)}, not asked ${String(s.poolOnly.notAsked)}, not run ${String(s.poolOnly.notRun)}`}${s.adapterOnly === undefined ? "" : `, adapter run ${String(s.adapterOnly.measured)}, already ${String(s.adapterOnly.already)}, failed ${String(s.adapterOnly.failed)}`}, ${s.seconds.toFixed(1)} s`;
1101
1254
  }
1102
1255
  /** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
1103
1256
  const LOCK_HELD = 99;
@@ -1182,6 +1335,7 @@ function addSummaries(a, b) {
1182
1335
  ...(b.rejudge === undefined ? (a.rejudge === undefined ? {} : { rejudge: a.rejudge }) : { rejudge: b.rejudge }),
1183
1336
  ...(b.memoryOnly === undefined ? (a.memoryOnly === undefined ? {} : { memoryOnly: a.memoryOnly }) : { memoryOnly: b.memoryOnly }),
1184
1337
  ...(b.isolatedOnly === undefined ? (a.isolatedOnly === undefined ? {} : { isolatedOnly: a.isolatedOnly }) : { isolatedOnly: b.isolatedOnly }),
1338
+ ...(b.poolOnly === undefined ? (a.poolOnly === undefined ? {} : { poolOnly: a.poolOnly }) : { poolOnly: b.poolOnly }),
1185
1339
  ...(b.adapterOnly === undefined ? (a.adapterOnly === undefined ? {} : { adapterOnly: a.adapterOnly }) : { adapterOnly: b.adapterOnly }),
1186
1340
  };
1187
1341
  }
@@ -1205,7 +1359,7 @@ async function runLocked(settings, options) {
1205
1359
  {
1206
1360
  const started = Date.now();
1207
1361
  const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
1208
- const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }), ...(options.isolatedRunner === undefined ? {} : { isolatedRunner: options.isolatedRunner }) };
1362
+ const ctx = { settings, env, log, ...(options.pinFilesRun === undefined ? {} : { pinFilesRun: options.pinFilesRun }), ...(options.isolatedRunner === undefined ? {} : { isolatedRunner: options.isolatedRunner }), ...(options.poolRunner === undefined ? {} : { poolRunner: options.poolRunner }) };
1209
1363
  // the pins first, before the quiet wait and any measurement: an upgrade of plugin-hostd that collects more files
1210
1364
  // than the pins on disk hold refuses those plugins until their pins name them (plugin-qualify#239)
1211
1365
  const fresh = refreshPins(ctx, readSyncState(settings.stateFile, log));
@@ -1228,6 +1382,13 @@ async function runLocked(settings, options) {
1228
1382
  const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
1229
1383
  return { ...summary, isolatedOnly: { isolated: r.isolated, refused: r.refused, notRun: r.notRun } };
1230
1384
  }
1385
+ if (options.poolOnly === true) {
1386
+ const r = poolPass(ctx, fresh.state);
1387
+ if (r.state !== fresh.state)
1388
+ writeFileWhole(settings.stateFile, `${JSON.stringify(r.state, null, 1)}\n`);
1389
+ const summary = await sendRefreshed(settings, r.state, fresh.refreshed.length, started, log);
1390
+ return { ...summary, poolOnly: { cores: r.cores, off: r.off, notAsked: r.notAsked, notRun: r.notRun } };
1391
+ }
1231
1392
  if (options.adapterOnly === true) {
1232
1393
  const r = adapterPass(ctx, fresh.state, { ...options.adapterPass, force: options.force === true });
1233
1394
  log(`adapter runs under ${Object.entries(r.versions).map(([k, v]) => `${k} ${v}`).join(", ")}`);
@@ -1283,7 +1444,8 @@ async function runLocked(settings, options) {
1283
1444
  for (const item of items) {
1284
1445
  const recorded = recordedAsIs(item, entries[item.key]);
1285
1446
  const retry = recorded.same ? entries[item.key].plugins.map(machineReason).find((r) => r !== undefined) : undefined;
1286
- if (recorded.same && options.force !== true && retry === undefined) {
1447
+ const owed = recorded.same && options.force !== true && retry === undefined ? figureOwed(entries[item.key]) : undefined;
1448
+ if (recorded.same && options.force !== true && retry === undefined && owed === undefined) {
1287
1449
  if (pass === 0)
1288
1450
  count.skipped += entries[item.key].plugins.length;
1289
1451
  continue;
@@ -1305,6 +1467,8 @@ async function runLocked(settings, options) {
1305
1467
  }
1306
1468
  if (recorded.same && retry !== undefined)
1307
1469
  log(`${item.key}: unmeasured last run by the machine (${retry}); measured again`);
1470
+ if (owed !== undefined)
1471
+ log(`${item.key}: its record never took the ${owed} figure (a probe from before that pass); measured again`);
1308
1472
  const files = recorded.files ?? digestFiles(item.root, item.files);
1309
1473
  if (files === undefined) {
1310
1474
  log(`${item.key}: a file of it cannot be read; skipped this run`);
@@ -1362,7 +1526,7 @@ async function runLocked(settings, options) {
1362
1526
  for (const f of pruneStale(settings, stateBefore, state))
1363
1527
  log(`${f}: removed, its plugin is gone or changed`);
1364
1528
  // a plugin that was pinned and is not any more is also forgotten by a running daemon
1365
- const changed = [...changedPinLines(before, after), ...removedPinUris(before, after).map((uri) => `pin_clear ${uri}`)];
1529
+ const changed = [...changedPinLines(before, after), ...removedPinUris(before, after).map(pinClearLine)];
1366
1530
  const sent = await sendPins(settings.hostdPort, settings.hostdFeedbackPort, changed);
1367
1531
  if ("unsent" in sent) {
1368
1532
  const why = {
@@ -1375,7 +1539,7 @@ async function runLocked(settings, options) {
1375
1539
  const replies = "replies" in sent ? sent.replies : [];
1376
1540
  for (let i = 0; i < replies.length; i++)
1377
1541
  if (replies[i] !== "resp 0")
1378
- log(`plugin-hostd answered ${replies[i]} to ${changed[i].split(" ")[1]}`);
1542
+ log(`plugin-hostd answered ${replies[i]} to ${pinLineUri(changed[i])}`);
1379
1543
  const taken = replies.filter((r) => r === "resp 0").length;
1380
1544
  for (const [why, n] of byMachine)
1381
1545
  if (n > 1)