@openmixer/plugin-qualify 0.17.1 → 0.17.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -6
- package/bin/plugin-qualify-sync.mjs +1 -1
- package/dist/host-admission.d.ts +25 -6
- package/dist/host-admission.js +45 -17
- package/dist/host-admission.js.map +1 -1
- package/dist/hostd-pin-files.d.ts +25 -0
- package/dist/hostd-pin-files.js +58 -0
- package/dist/hostd-pin-files.js.map +1 -0
- package/dist/probe-judge.d.ts +2 -0
- package/dist/probe-judge.js +3 -2
- package/dist/probe-judge.js.map +1 -1
- package/dist/probe-output-fixtures.js +4 -1
- package/dist/probe-output-fixtures.js.map +1 -1
- package/dist/probe-output.d.ts +2 -1
- package/dist/probe-output.js +4 -1
- package/dist/probe-output.js.map +1 -1
- package/dist/probe-runner.d.ts +5 -0
- package/dist/probe-runner.js +8 -3
- package/dist/probe-runner.js.map +1 -1
- package/dist/qualification-record.d.ts +16 -1
- package/dist/qualification-record.js +8 -2
- package/dist/qualification-record.js.map +1 -1
- package/dist/qualify-cli.js +10 -4
- package/dist/qualify-cli.js.map +1 -1
- package/dist/qualify-options.js +3 -3
- package/dist/qualify-options.js.map +1 -1
- package/dist/qualify-sync.d.ts +26 -4
- package/dist/qualify-sync.js +191 -53
- package/dist/qualify-sync.js.map +1 -1
- package/package.json +1 -1
- package/src/host-admission.ts +69 -23
- package/src/hostd-pin-files.ts +75 -0
- package/src/probe-judge.ts +5 -1
- package/src/probe-output-fixtures.ts +4 -1
- package/src/probe-output.ts +6 -2
- package/src/probe-runner.ts +13 -3
- package/src/qualification-record.ts +16 -2
- package/src/qualify-cli.ts +8 -4
- package/src/qualify-options.ts +3 -3
- package/src/qualify-sync.ts +197 -52
- package/tools/descriptors.py +8 -1
- package/tools/mod-host-admit.mjs +3 -1
package/src/qualify-sync.ts
CHANGED
|
@@ -16,9 +16,9 @@
|
|
|
16
16
|
* When a daemon answers on the command port, the pins that changed are also sent to it with `pin_set`, and the ones
|
|
17
17
|
* of a plugin that is gone with `pin_clear`. What a gone plugin left in the records and the descriptors is removed.
|
|
18
18
|
*/
|
|
19
|
-
import { spawnSync } from "node:child_process";
|
|
19
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
20
20
|
import { createHash } from "node:crypto";
|
|
21
|
-
import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync,
|
|
21
|
+
import { closeSync, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs";
|
|
22
22
|
import { createConnection } from "node:net";
|
|
23
23
|
import { homedir, tmpdir } from "node:os";
|
|
24
24
|
import { basename, dirname, join, resolve } from "node:path";
|
|
@@ -30,9 +30,10 @@ import { HOSTING_CODE } from "./hosting-codes.js";
|
|
|
30
30
|
import { shippedHostProfile, type HostProfile } from "./host-profiles.js";
|
|
31
31
|
import { lv2BundlesOnPath, manifestPlugins } from "./lv2-loader.js";
|
|
32
32
|
import { judgmentPinEntries, pinsFile, type PinEntry } from "./pins-file.js";
|
|
33
|
+
import { hostdPinFiles, type PinFilesRun } from "./hostd-pin-files.js";
|
|
33
34
|
import type { Judgment } from "./probe-judge.js";
|
|
34
35
|
import { qualifyTargets } from "./probe-runner.js";
|
|
35
|
-
import {
|
|
36
|
+
import { GRID_FULL, parseGridSpec, type GridSpec } from "./measurement-grid.js";
|
|
36
37
|
import { memoryBoundOf, parseMemoryLimit, type MemoryBound } from "./memory-bound.js";
|
|
37
38
|
import { writeQualification, type NotMeasured } from "./qualification-record.js";
|
|
38
39
|
import { DESCRIPTOR_DIR_STATE_SUBDIR, DESCRIPTOR_DIR_SYSTEM, QUALIFY_QUANTA, QUALIFY_RATES } from "./qualify-declarations.js";
|
|
@@ -69,7 +70,7 @@ export const SYNC_SETTINGS: readonly SyncSetting[] = [
|
|
|
69
70
|
{ key: "state_file", meaning: "what was qualified, from which bytes, and how it ended", default: "/var/lib/omx/qualify-sync.json (system), else $XDG_STATE_HOME/omx/qualify-sync.json" },
|
|
70
71
|
{ key: "budget_s", meaning: "the wall clock one run gets (one cell of an LV2 plugin, one quantum of a CLAP one), in seconds; past it the plugin is recorded as hung", default: "60" },
|
|
71
72
|
{ key: "host_profile", meaning: "the host the verdict is for (a shipped profile id)", default: "omx-clap-host" },
|
|
72
|
-
{ key: "grid", meaning: "the cells measured: hosts (the host's and the console tiers' rate/quantum, then 512 at every rate),
|
|
73
|
+
{ key: "grid", meaning: "the cells measured: full (every rate at every block size, 16 cells), hosts (the fast option: the host's and the console tiers' rate/quantum, then 512 at every rate; also --grid-hosts), or <rate>/<quantum>,...", default: "full" },
|
|
73
74
|
{ key: "cells_per_plugin_budget", meaning: "measure at most the first n cells of the grid a plugin; 0 is every cell", default: "0" },
|
|
74
75
|
{ key: "memory_limit", meaning: "the memory bound of every run: bytes, or with K, M, G; off for none; past it a cell is probe.oom", default: "half of the RAM" },
|
|
75
76
|
{ key: "hostd_port", meaning: "plugin-hostd's command port the new pins are sent to; 0 sends none", default: "5555" },
|
|
@@ -124,6 +125,7 @@ export interface SyncArgs {
|
|
|
124
125
|
readonly system: boolean;
|
|
125
126
|
readonly force: boolean;
|
|
126
127
|
readonly noDistributed: boolean;
|
|
128
|
+
readonly noWait: boolean;
|
|
127
129
|
readonly help: boolean;
|
|
128
130
|
/** The settings given as flags, by key. */
|
|
129
131
|
readonly values: ReadonlyMap<string, string>;
|
|
@@ -138,13 +140,16 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
|
|
|
138
140
|
let system = false;
|
|
139
141
|
let force = false;
|
|
140
142
|
let noDistributed = false;
|
|
143
|
+
let noWait = false;
|
|
141
144
|
let help = false;
|
|
142
145
|
for (let i = 0; i < argv.length; i++) {
|
|
143
146
|
const a = argv[i]!;
|
|
144
147
|
if (a === "--system") system = true;
|
|
145
148
|
else if (a === "--force") force = true;
|
|
146
149
|
else if (a === "--no-distributed") noDistributed = true;
|
|
150
|
+
else if (a === "--no-wait") noWait = true;
|
|
147
151
|
else if (a === "--help") help = true;
|
|
152
|
+
else if (a === "--grid-hosts") values.set("grid", "hosts");
|
|
148
153
|
else if (a === "--config" || byFlag.has(a)) {
|
|
149
154
|
const v = argv[++i];
|
|
150
155
|
if (v === undefined) return { ok: false, error: `${a} needs a value` };
|
|
@@ -152,7 +157,7 @@ export function parseSyncArgs(argv: readonly string[]): { ok: true; args: SyncAr
|
|
|
152
157
|
else values.set(byFlag.get(a)!, v);
|
|
153
158
|
} else return { ok: false, error: `unknown argument ${a}` };
|
|
154
159
|
}
|
|
155
|
-
return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, help, values } };
|
|
160
|
+
return { ok: true, args: { ...(config === undefined ? {} : { config }), system, force, noDistributed, noWait, help, values } };
|
|
156
161
|
}
|
|
157
162
|
|
|
158
163
|
export function syncHelpText(): string {
|
|
@@ -163,7 +168,9 @@ export function syncHelpText(): string {
|
|
|
163
168
|
` ${"--config <file>".padEnd(36)}the settings file (default: ${SYNC_CONF}; a missing file is the defaults)`,
|
|
164
169
|
" --system a system service's directories, under /var/lib/omx",
|
|
165
170
|
" --force qualify every plugin again, recorded or not; no distributed record is read",
|
|
171
|
+
" --no-wait a run in progress is not waited for: exit 0, and the holder runs once more at its end",
|
|
166
172
|
" --no-distributed read no distributed record: every plugin not recorded is measured",
|
|
173
|
+
" --grid-hosts the fast grid, the same as --grid hosts (7 cells with the shipped hosts)",
|
|
167
174
|
" --help this text",
|
|
168
175
|
...rows,
|
|
169
176
|
"",
|
|
@@ -206,7 +213,7 @@ export function resolveSyncSettings(
|
|
|
206
213
|
if (feedbackPort === undefined || !Number.isInteger(feedbackPort) || feedbackPort > 65535) return { ok: false, error: "hostd_feedback_port must be a port number, or 0" };
|
|
207
214
|
if (settleS === undefined) return { ok: false, error: "settle_s must be a number of seconds" };
|
|
208
215
|
const gridText = get("grid");
|
|
209
|
-
const grid = gridText === undefined || gridText === "" ? { ok: true as const, spec:
|
|
216
|
+
const grid = gridText === undefined || gridText === "" ? { ok: true as const, spec: GRID_FULL } : parseGridSpec(gridText);
|
|
210
217
|
if (!grid.ok) return { ok: false, error: `grid: ${grid.error}` };
|
|
211
218
|
const cellsPerPluginBudget = number("cells_per_plugin_budget", 0);
|
|
212
219
|
if (cellsPerPluginBudget === undefined || !Number.isInteger(cellsPerPluginBudget)) return { ok: false, error: "cells_per_plugin_budget must be a whole number, 0 or more" };
|
|
@@ -426,6 +433,8 @@ export interface SyncPlugin {
|
|
|
426
433
|
/** The code that decided a plugin that is not qualified. */
|
|
427
434
|
readonly code?: string;
|
|
428
435
|
readonly detail?: string;
|
|
436
|
+
/** Why its descriptor was not written, when the descriptor writer failed or timed out on it. */
|
|
437
|
+
readonly descriptorProblem?: string;
|
|
429
438
|
/** Its pin, when the run took a whole one. */
|
|
430
439
|
readonly pin?: PinEntry;
|
|
431
440
|
/** The record written for it. */
|
|
@@ -487,6 +496,23 @@ export interface SyncContext {
|
|
|
487
496
|
readonly settings: SyncSettings;
|
|
488
497
|
readonly env: Readonly<Record<string, string | undefined>>;
|
|
489
498
|
readonly log: (line: string) => void;
|
|
499
|
+
/** The run of `plugin-hostd --pin-files`, a stand-in in tests. */
|
|
500
|
+
readonly pinFilesRun?: PinFilesRun;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* The pin of a judged run with the files plugin-hostd collects for its plugin (plugin-qualify#228): the probe's list is
|
|
505
|
+
* the probe's view, and an add checks the daemon's, so a pin holds the daemon's. No pin when plugin-hostd cannot name
|
|
506
|
+
* them; the plugin is then pinned on its first add (pin_mode first-use), and the log says why.
|
|
507
|
+
*/
|
|
508
|
+
function withHostdFiles(ctx: SyncContext, pin: PinEntry | undefined): PinEntry | undefined {
|
|
509
|
+
if (pin === undefined || pin.integrity.files === undefined) return pin;
|
|
510
|
+
const r = hostdPinFiles(pin.uri, ctx.settings.lv2Dirs, ctx.env, ...(ctx.pinFilesRun === undefined ? [] : [ctx.pinFilesRun]));
|
|
511
|
+
if (!r.ok) {
|
|
512
|
+
ctx.log(`${pin.uri}: not pinned: ${r.detail}`);
|
|
513
|
+
return undefined;
|
|
514
|
+
}
|
|
515
|
+
return { ...pin, integrity: { ...pin.integrity, files: r.files } };
|
|
490
516
|
}
|
|
491
517
|
|
|
492
518
|
/** A judged run's outcome for the host: qualified, refused, or why it was not measured. */
|
|
@@ -524,7 +550,7 @@ function judgedPlugins(ctx: SyncContext, runs: readonly { bytes: Buffer; judgmen
|
|
|
524
550
|
const host = ctx.settings.hostProfile.id;
|
|
525
551
|
const entries = runs.map(({ bytes, judgment }) => {
|
|
526
552
|
const { recordPath } = writeQualification(ctx.settings.qualificationDir, bytes, judgment, QUICK_NOT_MEASURED);
|
|
527
|
-
const pin = judgmentPinEntries(judgment, host)[0];
|
|
553
|
+
const pin = withHostdFiles(ctx, judgmentPinEntries(judgment, host)[0]);
|
|
528
554
|
const { outcome, code } = outcomeOf(judgment, host);
|
|
529
555
|
const plugin: SyncPlugin = { uri: pin?.uri ?? judgment.output.plugin.id, outcome, ...(code === undefined ? {} : { code }), ...(pin === undefined ? {} : { pin }), record: recordPath };
|
|
530
556
|
return { plugin, judgedHere: judgment.hosts.some((h) => h.verdict.host === host) };
|
|
@@ -546,18 +572,42 @@ function runPass(ctx: SyncContext, target: Parameters<typeof qualifyTargets>[0][
|
|
|
546
572
|
profiles: [s.hostProfile],
|
|
547
573
|
wallS: s.budgetS,
|
|
548
574
|
env,
|
|
575
|
+
// mod-host and jalv are admitted on the device's LV2 directories, not on the one-bundle world the probe measures in
|
|
576
|
+
admissionLv2Path: s.lv2Dirs,
|
|
549
577
|
});
|
|
550
578
|
}
|
|
551
579
|
|
|
552
|
-
/**
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
}
|
|
560
|
-
|
|
580
|
+
/**
|
|
581
|
+
* The descriptors of a scanned `.clap`, through the one descriptor writer, one plugin to a run so that each has the
|
|
582
|
+
* budget to itself and a plugin that hangs the writer costs only its own descriptor. What failed is returned by plugin
|
|
583
|
+
* id, so the outcome of that plugin records it; the run goes on.
|
|
584
|
+
*/
|
|
585
|
+
function writeClapDescriptors(ctx: SyncContext, scan: string): Map<string, string> {
|
|
586
|
+
const failed = new Map<string, string>();
|
|
587
|
+
let doc: { files?: { plugins?: { id?: string }[] }[] };
|
|
588
|
+
try {
|
|
589
|
+
doc = JSON.parse(scan);
|
|
590
|
+
} catch (e) {
|
|
591
|
+
ctx.log(`descriptors of a .clap not written (the scan is not JSON): ${(e as Error).message}`);
|
|
592
|
+
return failed;
|
|
593
|
+
}
|
|
594
|
+
for (const file of doc.files ?? []) {
|
|
595
|
+
for (const p of file.plugins ?? []) {
|
|
596
|
+
const id = p.id ?? "";
|
|
597
|
+
const r = spawnSync("python3", [join(toolsDir(), "descriptors.py"), "clap", "--descriptor-dir", ctx.settings.descriptorDir], {
|
|
598
|
+
input: JSON.stringify({ ...doc, files: [{ ...file, plugins: [p] }] }),
|
|
599
|
+
encoding: "utf8",
|
|
600
|
+
timeout: ctx.settings.budgetS * 1000,
|
|
601
|
+
killSignal: "SIGKILL",
|
|
602
|
+
env: { ...ctx.env },
|
|
603
|
+
});
|
|
604
|
+
if (r.status === 0) continue;
|
|
605
|
+
const problem = `descriptor not written (${r.error?.message ?? `exit ${String(r.status)}`}): ${(r.stderr ?? "").trim()}`;
|
|
606
|
+
failed.set(id, problem);
|
|
607
|
+
ctx.log(`${id}: ${problem}`);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
return failed;
|
|
561
611
|
}
|
|
562
612
|
|
|
563
613
|
function qualifyClap(ctx: SyncContext, item: SyncItem): SyncPlugin[] {
|
|
@@ -584,7 +634,7 @@ function qualifyClap(ctx: SyncContext, item: SyncItem): SyncPlugin[] {
|
|
|
584
634
|
const hung = /timed out/.test(scanned.error);
|
|
585
635
|
return [{ uri: fileUri, outcome: hung ? "hung" : "crashed", code: hung ? SYNC_CODE.scanHung : SYNC_CODE.scanCrashed, detail: scanned.error }];
|
|
586
636
|
}
|
|
587
|
-
writeClapDescriptors(ctx, r.stdout);
|
|
637
|
+
const descriptorFailed = writeClapDescriptors(ctx, r.stdout);
|
|
588
638
|
const out: SyncPlugin[] = [];
|
|
589
639
|
for (const p of scanned.plugins) {
|
|
590
640
|
const uri = `${fileUri}#${p.id}`;
|
|
@@ -594,7 +644,8 @@ function qualifyClap(ctx: SyncContext, item: SyncItem): SyncPlugin[] {
|
|
|
594
644
|
}
|
|
595
645
|
const pass = runPass(ctx, { format: "clap", uri: `urn:clap:${p.id}`, bundle: item.root, binary: item.files[0]!, binaryPath: file }, ctx.env);
|
|
596
646
|
for (const f of pass.failures) out.push({ uri, outcome: "unmeasured", code: SYNC_CODE.notMeasured, detail: f.problem });
|
|
597
|
-
|
|
647
|
+
const problem = descriptorFailed.get(p.id);
|
|
648
|
+
out.push(...judgedPlugins(ctx, pass.runs).map((j) => (problem === undefined ? j : { ...j, descriptorProblem: problem })));
|
|
598
649
|
}
|
|
599
650
|
return out;
|
|
600
651
|
}
|
|
@@ -735,7 +786,8 @@ function idAndHash(stem: string): string | undefined {
|
|
|
735
786
|
* output copied beside it and its descriptor. A stem the state before held and the state after does not hold is
|
|
736
787
|
* stale; every record file of that stem is removed, and so is a descriptor of its id and binary hash whichever
|
|
737
788
|
* version suffix it carries — unless that id and hash is still kept under another stem (two copies of one binary, or
|
|
738
|
-
* a record and a descriptor the sync wrote at different version suffixes for the same plugin). A record
|
|
789
|
+
* a record and a descriptor the sync wrote at different version suffixes for the same plugin). A record of an id the
|
|
790
|
+
* state after holds, made from other bytes than it holds, goes too, though no state ever named it (#198). A record another
|
|
739
791
|
* plugin of the state still names (two copies of one binary) is kept, as is a descriptor the sync did not write.
|
|
740
792
|
* Returns the files removed.
|
|
741
793
|
*/
|
|
@@ -762,6 +814,14 @@ export function pruneStale(settings: Pick<SyncSettings, "qualificationDir" | "de
|
|
|
762
814
|
// no directory, no record
|
|
763
815
|
}
|
|
764
816
|
for (const stem of stale) for (const f of records) if (f.startsWith(`${stem}.`) && recordStem(f) === stem) drop(settings.qualificationDir, f);
|
|
817
|
+
// a record of an installed plugin's id made from other bytes, whether or not any state ever named it (the other
|
|
818
|
+
// tier's record, a lost state entry): the id is kept, the binary hash is not
|
|
819
|
+
const keptIds = new Set([...keptIdHash].map((k) => k.split("\0")[0]!));
|
|
820
|
+
for (const f of records) {
|
|
821
|
+
const s = recordStem(f);
|
|
822
|
+
const key = s === undefined ? undefined : idAndHash(s);
|
|
823
|
+
if (key !== undefined && keptIds.has(key.split("\0")[0]!) && !keptIdHash.has(key)) drop(settings.qualificationDir, f);
|
|
824
|
+
}
|
|
765
825
|
let descriptors: string[] = [];
|
|
766
826
|
try {
|
|
767
827
|
descriptors = readdirSync(settings.descriptorDir);
|
|
@@ -883,57 +943,143 @@ export interface SyncSummary {
|
|
|
883
943
|
readonly sent?: number;
|
|
884
944
|
}
|
|
885
945
|
|
|
946
|
+
/**
|
|
947
|
+
* Why a reader that is not the sync's user (the console, #221) cannot read the descriptors: the directory or one of its
|
|
948
|
+
* parent without search permission for others, the directory or a descriptor without read permission for others. One
|
|
949
|
+
* line each; none when the directory does not exist yet or everything is open.
|
|
950
|
+
*/
|
|
951
|
+
export function descriptorAccessProblems(dir: string): string[] {
|
|
952
|
+
const problems: string[] = [];
|
|
953
|
+
const mode = (path: string): number | undefined => {
|
|
954
|
+
try {
|
|
955
|
+
return statSync(path).mode & 0o777;
|
|
956
|
+
} catch {
|
|
957
|
+
return undefined;
|
|
958
|
+
}
|
|
959
|
+
};
|
|
960
|
+
const octal = (m: number): string => `0${m.toString(8)}`;
|
|
961
|
+
const top = mode(dir);
|
|
962
|
+
if (top === undefined) return problems;
|
|
963
|
+
// the parent too (StateDirectory's /var/lib/omx); not further up, where a private home is a user's own choice
|
|
964
|
+
const parent = dirname(resolve(dir));
|
|
965
|
+
const parentMode = mode(parent);
|
|
966
|
+
if (parentMode !== undefined && (parentMode & 0o001) === 0) problems.push(`${parent} is mode ${octal(parentMode)}: other users, the console among them, cannot enter it to read the descriptors`);
|
|
967
|
+
if ((top & 0o004) === 0) problems.push(`${dir} is mode ${octal(top)}: other users, the console among them, cannot list the descriptors`);
|
|
968
|
+
let files: string[] = [];
|
|
969
|
+
try {
|
|
970
|
+
files = readdirSync(dir).filter((f) => f.endsWith(".json"));
|
|
971
|
+
} catch {
|
|
972
|
+
return problems;
|
|
973
|
+
}
|
|
974
|
+
const closed = files.filter((f) => ((mode(join(dir, f)) ?? 0o004) & 0o004) === 0);
|
|
975
|
+
if (closed.length > 0) problems.push(`${String(closed.length)} descriptor(s) in ${dir} cannot be read by other users, the console among them (first: ${closed[0]!})`);
|
|
976
|
+
return problems;
|
|
977
|
+
}
|
|
978
|
+
|
|
886
979
|
export function summaryLine(s: SyncSummary): string {
|
|
887
980
|
return `qualified ${String(s.qualified)}, skipped ${String(s.skipped)}, refused ${String(s.refused)}, crashed ${String(s.crashed)}, hung ${String(s.hung)}, unmeasured ${String(s.unmeasured)}, distributed ${String(s.distributed)}, ${s.seconds.toFixed(1)} s`;
|
|
888
981
|
}
|
|
889
982
|
|
|
890
|
-
/** The
|
|
891
|
-
|
|
983
|
+
/** The exit status `flock -n` returns when the lock is held (-E), told apart from a failure of flock itself. */
|
|
984
|
+
const LOCK_HELD = 99;
|
|
985
|
+
|
|
986
|
+
export interface SyncLock {
|
|
987
|
+
release(): Promise<void>;
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/**
|
|
991
|
+
* The lock of one state file: flock(1) on `<state file>.lock`, held by a child process for as long as this one lives or
|
|
992
|
+
* until `release`, so a crashed run frees it. Waits for the holder, or with `wait` false returns `undefined` at once.
|
|
993
|
+
*/
|
|
994
|
+
export function takeLock(stateFile: string, wait: boolean): Promise<SyncLock | undefined> {
|
|
892
995
|
const lock = `${stateFile}.lock`;
|
|
893
996
|
mkdirSync(dirname(lock), { recursive: true });
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
unlinkSync(lock);
|
|
912
|
-
}
|
|
913
|
-
}
|
|
914
|
-
return undefined;
|
|
997
|
+
const child = spawn("flock", ["-x", "-E", String(LOCK_HELD), ...(wait ? [] : ["-n"]), lock, "sh", "-c", "echo locked; exec cat"], { stdio: ["pipe", "pipe", "inherit"] });
|
|
998
|
+
return new Promise((resolvePromise, reject) => {
|
|
999
|
+
const exited = new Promise<void>((done) => child.once("close", () => done()));
|
|
1000
|
+
child.once("error", reject);
|
|
1001
|
+
child.stdout.once("data", () =>
|
|
1002
|
+
resolvePromise({
|
|
1003
|
+
release: () => {
|
|
1004
|
+
child.stdin.end();
|
|
1005
|
+
return exited;
|
|
1006
|
+
},
|
|
1007
|
+
}),
|
|
1008
|
+
);
|
|
1009
|
+
child.once("close", (code) => {
|
|
1010
|
+
if (code === LOCK_HELD) resolvePromise(undefined);
|
|
1011
|
+
else reject(new Error(`flock ${lock} failed (exit ${String(code)})`));
|
|
1012
|
+
});
|
|
1013
|
+
});
|
|
915
1014
|
}
|
|
916
1015
|
|
|
1016
|
+
/** What a second run leaves for the holder: one more run at its end, over what changed meanwhile. */
|
|
1017
|
+
const againMarker = (stateFile: string): string => `${stateFile}.again`;
|
|
1018
|
+
|
|
917
1019
|
export interface SyncRunOptions {
|
|
918
1020
|
readonly force?: boolean;
|
|
1021
|
+
/** False under `--no-wait`: a run in progress is not waited for; the holder runs once more at its end instead. */
|
|
1022
|
+
readonly wait?: boolean;
|
|
919
1023
|
readonly env?: Readonly<Record<string, string | undefined>>;
|
|
920
1024
|
readonly log?: (line: string) => void;
|
|
921
1025
|
}
|
|
922
1026
|
|
|
923
1027
|
/**
|
|
924
|
-
* One run:
|
|
925
|
-
*
|
|
926
|
-
* whole after every item, so a run cut short keeps all it finished. `undefined` when another run holds the lock.
|
|
1028
|
+
* One run under the lock: waits for a run in progress, or with `wait` false leaves a marker for it and returns
|
|
1029
|
+
* `undefined`. The holder runs once more at its end for every marker left meanwhile.
|
|
927
1030
|
*/
|
|
928
1031
|
export async function runSync(settings: SyncSettings, options: SyncRunOptions = {}): Promise<SyncSummary | undefined> {
|
|
929
1032
|
const log = options.log ?? (() => {});
|
|
930
|
-
const
|
|
931
|
-
|
|
932
|
-
if (
|
|
933
|
-
|
|
934
|
-
|
|
1033
|
+
const marker = againMarker(settings.stateFile);
|
|
1034
|
+
let lock = await takeLock(settings.stateFile, options.wait !== false);
|
|
1035
|
+
if (lock === undefined) {
|
|
1036
|
+
// the marker first, then the lock once more: a holder that finished in between is not left without a successor
|
|
1037
|
+
writeFileSync(marker, "");
|
|
1038
|
+
lock = await takeLock(settings.stateFile, false);
|
|
1039
|
+
if (lock === undefined) {
|
|
1040
|
+
log(`a run is in progress (${settings.stateFile}.lock); it runs once more at its end over what changed`);
|
|
1041
|
+
return undefined;
|
|
1042
|
+
}
|
|
935
1043
|
}
|
|
936
1044
|
try {
|
|
1045
|
+
let total: SyncSummary | undefined;
|
|
1046
|
+
for (;;) {
|
|
1047
|
+
rmSync(marker, { force: true });
|
|
1048
|
+
const one = await runLocked(settings, options);
|
|
1049
|
+
total = total === undefined ? one : addSummaries(total, one);
|
|
1050
|
+
if (!existsSync(marker)) return total;
|
|
1051
|
+
log("a run was asked for while this one ran; running once more");
|
|
1052
|
+
}
|
|
1053
|
+
} finally {
|
|
1054
|
+
await lock.release();
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
function addSummaries(a: SyncSummary, b: SyncSummary): SyncSummary {
|
|
1059
|
+
const sent = (a.sent ?? 0) + (b.sent ?? 0);
|
|
1060
|
+
return {
|
|
1061
|
+
qualified: a.qualified + b.qualified,
|
|
1062
|
+
skipped: a.skipped + b.skipped,
|
|
1063
|
+
refused: a.refused + b.refused,
|
|
1064
|
+
crashed: a.crashed + b.crashed,
|
|
1065
|
+
hung: a.hung + b.hung,
|
|
1066
|
+
unmeasured: a.unmeasured + b.unmeasured,
|
|
1067
|
+
distributed: a.distributed + b.distributed,
|
|
1068
|
+
seconds: a.seconds + b.seconds,
|
|
1069
|
+
...(sent === 0 ? {} : { sent }),
|
|
1070
|
+
};
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
/**
|
|
1074
|
+
* One run: wait for quiet, qualify what is new or changed, drop what is gone, write the state, the records, the
|
|
1075
|
+
* descriptors and the pin file, and send the changed pins to a running daemon. The state and the pin file are written
|
|
1076
|
+
* whole after every item, so a run cut short keeps all it finished.
|
|
1077
|
+
*/
|
|
1078
|
+
async function runLocked(settings: SyncSettings, options: SyncRunOptions): Promise<SyncSummary> {
|
|
1079
|
+
const log = options.log ?? (() => {});
|
|
1080
|
+
const env = options.env ?? process.env;
|
|
1081
|
+
log("sync run begins");
|
|
1082
|
+
{
|
|
937
1083
|
const started = Date.now();
|
|
938
1084
|
const dirs = [...settings.clapDirs, ...settings.lv2Dirs];
|
|
939
1085
|
await waitQuiet(dirs, settings.settleS, log);
|
|
@@ -1001,6 +1147,7 @@ export async function runSync(settings: SyncSettings, options: SyncRunOptions =
|
|
|
1001
1147
|
const after = syncPinsText(state);
|
|
1002
1148
|
if (after !== before) writeFileWhole(settings.pinFile, after);
|
|
1003
1149
|
writeLv2Descriptors(ctx, lv2Pinned);
|
|
1150
|
+
for (const line of descriptorAccessProblems(settings.descriptorDir)) log(line);
|
|
1004
1151
|
// the files of a plugin that is gone, or was replaced by other bytes: only when every directory could be read, so
|
|
1005
1152
|
// an unreadable one is never taken for an uninstall
|
|
1006
1153
|
if (unreadable) log("a plugin directory could not be read; no record or descriptor is removed this run");
|
|
@@ -1020,8 +1167,6 @@ export async function runSync(settings: SyncSettings, options: SyncRunOptions =
|
|
|
1020
1167
|
for (let i = 0; i < replies.length; i++) if (replies[i] !== "resp 0") log(`plugin-hostd answered ${replies[i]!} to ${changed[i]!.split(" ")[1]!}`);
|
|
1021
1168
|
const taken = replies.filter((r) => r === "resp 0").length;
|
|
1022
1169
|
return { ...count, seconds: (Date.now() - started) / 1000, ...(taken === 0 ? {} : { sent: taken }) };
|
|
1023
|
-
} finally {
|
|
1024
|
-
unlock();
|
|
1025
1170
|
}
|
|
1026
1171
|
}
|
|
1027
1172
|
|
package/tools/descriptors.py
CHANGED
|
@@ -138,16 +138,23 @@ class SchemaValidator:
|
|
|
138
138
|
raise DescriptorRefused(DESCRIPTOR_CODE_SCHEMA, f"{where}: {e.message}")
|
|
139
139
|
|
|
140
140
|
|
|
141
|
+
DESCRIPTOR_DIR_MODE = 0o755
|
|
142
|
+
DESCRIPTOR_FILE_MODE = 0o644
|
|
143
|
+
|
|
144
|
+
|
|
141
145
|
def write_descriptor(directory, doc, sha256, validator):
|
|
142
146
|
"""Hold `doc` to the schema and write it whole into `directory`; the path written, or DescriptorRefused."""
|
|
143
147
|
validator.check(doc)
|
|
144
148
|
name = descriptor_file_name(doc["plugin"]["id"], doc["plugin"].get("version"), sha256)
|
|
145
149
|
try:
|
|
146
|
-
|
|
150
|
+
# the console reads this directory as another user (#221): published world-readable whatever the umask
|
|
151
|
+
os.makedirs(directory, mode=DESCRIPTOR_DIR_MODE, exist_ok=True)
|
|
152
|
+
os.chmod(directory, DESCRIPTOR_DIR_MODE)
|
|
147
153
|
path = os.path.join(directory, name)
|
|
148
154
|
pending = path + ".new"
|
|
149
155
|
with open(pending, "w", encoding="utf-8") as f:
|
|
150
156
|
f.write(json.dumps(doc, indent=2, ensure_ascii=False) + "\n")
|
|
157
|
+
os.chmod(pending, DESCRIPTOR_FILE_MODE)
|
|
151
158
|
os.replace(pending, path)
|
|
152
159
|
except OSError as e:
|
|
153
160
|
raise DescriptorRefused(DESCRIPTOR_CODE_DIR_UNWRITABLE, f"{directory}: {e.strerror}") from e
|
package/tools/mod-host-admit.mjs
CHANGED
|
@@ -8,6 +8,8 @@ import { spawn } from "node:child_process";
|
|
|
8
8
|
import { createConnection } from "node:net";
|
|
9
9
|
|
|
10
10
|
const uri = process.argv[2];
|
|
11
|
+
// the host to admit under: mod-host, or omx-clap-host for plugin-hostd's LV2 adapter, which answers the same protocol
|
|
12
|
+
const bin = process.argv[3] ?? "mod-host";
|
|
11
13
|
const port = 20000 + (process.pid % 20000);
|
|
12
14
|
const STEP_MS = 10000;
|
|
13
15
|
const INSTANCE = 0;
|
|
@@ -16,7 +18,7 @@ const COMMANDS = [
|
|
|
16
18
|
["remove", `remove ${INSTANCE}`],
|
|
17
19
|
];
|
|
18
20
|
|
|
19
|
-
const host = spawn(
|
|
21
|
+
const host = spawn(bin, ["-n", "-p", String(port)], { stdio: "ignore" });
|
|
20
22
|
let hostEnd; // how the host went, once it has
|
|
21
23
|
host.on("exit", (code, signal) => (hostEnd = signal ?? "exit"));
|
|
22
24
|
const spawnFailed = new Promise((resolve) => host.on("error", (e) => resolve(e.code)));
|