@smoothbricks/nx-plugin 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -9
- package/dist/cargo-source-hash.d.ts +2 -2
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +106 -46
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +13 -0
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +9 -9
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +20 -6
- package/dist/executors/bounded-exec/progress.d.ts +8 -3
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +100 -10
- package/dist/executors/bounded-exec/verdict.d.ts +5 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +1 -0
- package/dist/testing.js +1 -1
- package/managed/raw/patches/nx@23.2.1.patch +208 -45
- package/package.json +2 -2
- package/src/cargo-source-hash.test.ts +143 -2
- package/src/cargo-source-hash.ts +108 -44
- package/src/executors/bounded-exec/cpu-tokens.ts +15 -9
- package/src/executors/bounded-exec/executor.test.ts +26 -23
- package/src/executors/bounded-exec/executor.ts +19 -4
- package/src/executors/bounded-exec/go-progress-host.test.ts +171 -0
- package/src/executors/bounded-exec/progress.test.ts +73 -0
- package/src/executors/bounded-exec/progress.ts +95 -12
- package/src/executors/bounded-exec/verdict.ts +5 -4
- package/src/nx-daemon-start-never.test.ts +155 -0
- package/src/nx-task-output-lease.test.ts +136 -0
- package/src/testing.ts +1 -1
- package/test-timings.json +92 -91
|
@@ -1,11 +1,25 @@
|
|
|
1
1
|
import { expect, it } from 'bun:test';
|
|
2
2
|
import { execFileSync } from 'node:child_process';
|
|
3
|
-
import {
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { statSync } from 'node:fs';
|
|
5
|
+
import {
|
|
6
|
+
chmod,
|
|
7
|
+
mkdir,
|
|
8
|
+
mkdtemp,
|
|
9
|
+
readFile,
|
|
10
|
+
realpath,
|
|
11
|
+
rename,
|
|
12
|
+
rm,
|
|
13
|
+
stat,
|
|
14
|
+
symlink,
|
|
15
|
+
utimes,
|
|
16
|
+
writeFile,
|
|
17
|
+
} from 'node:fs/promises';
|
|
4
18
|
import { homedir, tmpdir } from 'node:os';
|
|
5
19
|
import { dirname, join, relative } from 'node:path';
|
|
6
20
|
import { useFixtureCargoHome } from './__tests__/fixture-cargo-home.js';
|
|
7
21
|
import { runtimeInputValue } from './__tests__/runtime-input.js';
|
|
8
|
-
import { hashCargoPathInputs, readCargoResolve } from './cargo-source-hash.js';
|
|
22
|
+
import { CARGO_ANCESTOR_INPUTS, compiledInFiles, hashCargoPathInputs, readCargoResolve } from './cargo-source-hash.js';
|
|
9
23
|
|
|
10
24
|
useFixtureCargoHome();
|
|
11
25
|
|
|
@@ -564,3 +578,130 @@ it.each(['helper', 'cli'])(
|
|
|
564
578
|
}
|
|
565
579
|
},
|
|
566
580
|
);
|
|
581
|
+
|
|
582
|
+
it('keeps the v2 transcript across forward source includes and an ignored source included through an alias', async () => {
|
|
583
|
+
const root = await realpath(await mkdtemp(join(tmpdir(), 'cargo-hash-fused-')));
|
|
584
|
+
const manifest = join(root, 'Cargo.toml');
|
|
585
|
+
const contents = new Map([
|
|
586
|
+
['Cargo.toml', '[workspace]\nmembers=["a","z"]\nresolver="2"\n'],
|
|
587
|
+
['a/Cargo.toml', '[package]\nname="a"\nversion="0.1.0"\nedition="2021"\n'],
|
|
588
|
+
['z/Cargo.toml', '[package]\nname="z"\nversion="0.1.0"\nedition="2021"\n'],
|
|
589
|
+
['a/src/lib.rs', 'const Z: &str = include_str!("../../z/src/lib.rs");\n'],
|
|
590
|
+
['z/src/lib.rs', 'const H: &str = include_str!("../hidden.txt");\nconst P: &str = include_str!("payload.bin");\n'],
|
|
591
|
+
['z/src/hidden.rs', 'const S: &str = include_str!("secret.bin");\n'],
|
|
592
|
+
['z/src/payload.bin', 'payload'],
|
|
593
|
+
['z/src/secret.bin', 'secret'],
|
|
594
|
+
['.gitignore', 'z/src/hidden.rs\n'],
|
|
595
|
+
]);
|
|
596
|
+
try {
|
|
597
|
+
for (const [path, text] of contents) {
|
|
598
|
+
await mkdir(dirname(join(root, path)), { recursive: true });
|
|
599
|
+
await writeFile(join(root, path), text);
|
|
600
|
+
}
|
|
601
|
+
await symlink('src/hidden.rs', join(root, 'z/hidden.txt'));
|
|
602
|
+
execFileSync('git', ['init', '--quiet', root], { stdio: 'pipe' });
|
|
603
|
+
execFileSync('cargo', ['generate-lockfile', '--offline', '--manifest-path', manifest], { stdio: 'pipe' });
|
|
604
|
+
// Independent explicit v2 transcript. In particular, z's source is named by a,
|
|
605
|
+
// while hidden.rs is discovery-only until the unignored alias names its bytes.
|
|
606
|
+
const inputs: { file: string; key: string }[] = [
|
|
607
|
+
{ file: manifest, key: JSON.stringify(['a', '0.1.0', 'ancestor', '1/Cargo.toml']) },
|
|
608
|
+
{ file: join(root, 'a/Cargo.toml'), key: JSON.stringify(['a', '0.1.0', 'file', 'Cargo.toml']) },
|
|
609
|
+
{ file: join(root, 'a/src/lib.rs'), key: JSON.stringify(['a', '0.1.0', 'file', 'src/lib.rs']) },
|
|
610
|
+
{
|
|
611
|
+
file: join(root, 'z/src/lib.rs'),
|
|
612
|
+
key: JSON.stringify(['a', '0.1.0', 'include', 'src/lib.rs\0../../z/src/lib.rs']),
|
|
613
|
+
},
|
|
614
|
+
{ file: join(root, 'z/Cargo.toml'), key: JSON.stringify(['z', '0.1.0', 'file', 'Cargo.toml']) },
|
|
615
|
+
{
|
|
616
|
+
file: join(root, 'z/src/secret.bin'),
|
|
617
|
+
key: JSON.stringify(['z', '0.1.0', 'include', 'src/hidden.rs\0secret.bin']),
|
|
618
|
+
},
|
|
619
|
+
{
|
|
620
|
+
file: join(root, 'z/src/hidden.rs'),
|
|
621
|
+
key: JSON.stringify(['z', '0.1.0', 'include', 'src/lib.rs\0../hidden.txt']),
|
|
622
|
+
},
|
|
623
|
+
{
|
|
624
|
+
file: join(root, 'z/src/payload.bin'),
|
|
625
|
+
key: JSON.stringify(['z', '0.1.0', 'include', 'src/lib.rs\0payload.bin']),
|
|
626
|
+
},
|
|
627
|
+
];
|
|
628
|
+
for (let ancestor = dirname(root), depth = 2; ; ancestor = dirname(ancestor), depth += 1) {
|
|
629
|
+
for (const name of CARGO_ANCESTOR_INPUTS) {
|
|
630
|
+
const file = join(ancestor, name);
|
|
631
|
+
if (statSync(file, { throwIfNoEntry: false })?.isFile()) {
|
|
632
|
+
inputs.push({ file, key: JSON.stringify(['a', '0.1.0', 'ancestor', depth + '/' + name]) });
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
if (dirname(ancestor) === ancestor) break;
|
|
636
|
+
}
|
|
637
|
+
const expected = createHash('sha256');
|
|
638
|
+
expected.update('cargo-path-inputs-v2\0');
|
|
639
|
+
expected.update(JSON.stringify(['z', '0.1.0', 'link', 'hidden.txt']));
|
|
640
|
+
expected.update('\0symlink\0src/hidden.rs\0');
|
|
641
|
+
for (const { file, key } of inputs.sort((left, right) =>
|
|
642
|
+
left.key < right.key ? -1 : left.key > right.key ? 1 : 0,
|
|
643
|
+
)) {
|
|
644
|
+
const bytes = await readFile(file);
|
|
645
|
+
expected.update(key);
|
|
646
|
+
expected.update('\0' + bytes.length + '\0');
|
|
647
|
+
expected.update(bytes);
|
|
648
|
+
}
|
|
649
|
+
const before = await hashCargoPathInputs(manifest, root, { includeWorkspace: true });
|
|
650
|
+
expect(before).toBe(expected.digest('hex'));
|
|
651
|
+
await writeFile(join(root, 'z/src/secret.bin'), 'changed');
|
|
652
|
+
expect(await hashCargoPathInputs(manifest, root, { includeWorkspace: true })).not.toBe(before);
|
|
653
|
+
} finally {
|
|
654
|
+
await rm(root, { recursive: true, force: true });
|
|
655
|
+
}
|
|
656
|
+
});
|
|
657
|
+
|
|
658
|
+
it('keeps literal discovery exact and refuses unreadable sources', async () => {
|
|
659
|
+
const root = await realpath(await mkdtemp(join(tmpdir(), 'cargo-hash-literals-')));
|
|
660
|
+
const source = join(root, 'src/lib.rs');
|
|
661
|
+
const unicode = 'spelling-e\u0301-\u{10000}-payload';
|
|
662
|
+
try {
|
|
663
|
+
await mkdir(dirname(source), { recursive: true });
|
|
664
|
+
await writeFile(
|
|
665
|
+
join(root, 'Cargo.toml'),
|
|
666
|
+
'[package]\nname="literals"\nversion="0.1.0"\nedition="2021"\n[workspace]\n',
|
|
667
|
+
);
|
|
668
|
+
await writeFile(join(root, 'src/data'), 'data');
|
|
669
|
+
await writeFile(join(root, 'src', unicode), 'unicode spelling');
|
|
670
|
+
await writeFile(
|
|
671
|
+
source,
|
|
672
|
+
[
|
|
673
|
+
'const A: &str = include_str!(r#"data"#,);',
|
|
674
|
+
'const B: &[u8] = include_bytes!["data"];',
|
|
675
|
+
'const C: &str = include_str!{"data"};',
|
|
676
|
+
`const U: &str = include_str!(${JSON.stringify(unicode)});`,
|
|
677
|
+
'// include_str!("missing")',
|
|
678
|
+
'/* outer /* include_bytes!("missing") */ comment */',
|
|
679
|
+
'const D: &str = r###"include_str!("missing")"###;',
|
|
680
|
+
'include!("missing");',
|
|
681
|
+
'include_str!("");',
|
|
682
|
+
'include_str!("da\\\\ta");',
|
|
683
|
+
'include_str!(concat!("da", "ta"));',
|
|
684
|
+
'include_str!("missing");',
|
|
685
|
+
'include_str!("data"];',
|
|
686
|
+
'include_str!("unterminated',
|
|
687
|
+
].join('\n'),
|
|
688
|
+
);
|
|
689
|
+
expect(compiledInFiles(root, new Map())).toEqual([
|
|
690
|
+
{ file: join(root, 'src/data'), source, literal: 'data' },
|
|
691
|
+
{ file: join(root, 'src', unicode), source, literal: unicode },
|
|
692
|
+
]);
|
|
693
|
+
execFileSync('cargo', ['generate-lockfile', '--offline', '--manifest-path', join(root, 'Cargo.toml')], {
|
|
694
|
+
stdio: 'pipe',
|
|
695
|
+
});
|
|
696
|
+
await chmod(source, 0);
|
|
697
|
+
expect(() => compiledInFiles(root, new Map())).toThrow();
|
|
698
|
+
// A cold native Cargo census can report the denial before the source read;
|
|
699
|
+
// both paths must preserve the operational refusal rather than skip the file.
|
|
700
|
+
await expect(hashCargoPathInputs(join(root, 'Cargo.toml'), root, { includeWorkspace: true })).rejects.toThrow(
|
|
701
|
+
'EACCES',
|
|
702
|
+
);
|
|
703
|
+
} finally {
|
|
704
|
+
await chmod(source, 0o600);
|
|
705
|
+
await rm(root, { recursive: true, force: true });
|
|
706
|
+
}
|
|
707
|
+
});
|
package/src/cargo-source-hash.ts
CHANGED
|
@@ -812,15 +812,26 @@ export async function hashCargoPathInputs(
|
|
|
812
812
|
const inputKey = (pkg: LocalCargoPackage, role: InputRole, path: string) =>
|
|
813
813
|
JSON.stringify([pkg.name, pkg.version, role, path.split(sep).join('/')]);
|
|
814
814
|
const directories = new Map<string, LocalCargoPackage>();
|
|
815
|
-
|
|
815
|
+
interface FileInput {
|
|
816
|
+
readonly file: string;
|
|
817
|
+
readonly key: string;
|
|
818
|
+
readonly order: number;
|
|
819
|
+
}
|
|
820
|
+
const files = new Map<string, FileInput>();
|
|
821
|
+
const compareInputs = (left: FileInput, right: FileInput) =>
|
|
822
|
+
left.key < right.key ? -1 : left.key > right.key ? 1 : left.order - right.order;
|
|
816
823
|
const ancestors = new Set<string>();
|
|
817
824
|
const links = new Map<string, { key: string; target: string }>();
|
|
818
825
|
const packageSources: { file: string; key: string }[] = [];
|
|
819
826
|
// Read shared bytes once. The smallest declared identity makes multiple owners
|
|
820
827
|
// deterministic without using a consuming checkout's mount path as an input.
|
|
821
|
-
const includeFile = (file: string, key: string) => {
|
|
828
|
+
const includeFile = (file: string, key: string): FileInput | undefined => {
|
|
822
829
|
const prior = files.get(file);
|
|
823
|
-
if (prior
|
|
830
|
+
if (prior !== undefined && key >= prior.key) return undefined;
|
|
831
|
+
// Rekeying must retain the old Map insertion order for equal v2 identities.
|
|
832
|
+
const input = { file, key, order: prior?.order ?? files.size };
|
|
833
|
+
files.set(file, input);
|
|
834
|
+
return input;
|
|
824
835
|
};
|
|
825
836
|
const packages = [...cargo.local].sort((left, right) =>
|
|
826
837
|
left.name < right.name
|
|
@@ -899,17 +910,24 @@ export async function hashCargoPathInputs(
|
|
|
899
910
|
}
|
|
900
911
|
}
|
|
901
912
|
for (const [directory, owner] of directories) await walk(directory, owner, '');
|
|
902
|
-
|
|
913
|
+
interface SourceScan {
|
|
914
|
+
readonly directory: string;
|
|
915
|
+
readonly owner: LocalCargoPackage;
|
|
916
|
+
readonly sources: readonly string[];
|
|
917
|
+
readonly found: Map<string, CompiledCargoInput>;
|
|
918
|
+
}
|
|
919
|
+
const scans = new Map<string, SourceScan>();
|
|
920
|
+
const groups = new Map<string, SourceScan[]>();
|
|
903
921
|
for (const [directory, owner] of directories) {
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
}
|
|
922
|
+
const scan: SourceScan = { directory, owner, sources: [...compiledSourceFiles(directory)], found: new Map() };
|
|
923
|
+
for (const source of scan.sources) scans.set(source, scan);
|
|
924
|
+
// Equal package identities can interleave, so their discovery shares one boundary.
|
|
925
|
+
const prefix = JSON.stringify([owner.name, owner.version]).slice(0, -1) + ',';
|
|
926
|
+
const group = groups.get(prefix);
|
|
927
|
+
if (group === undefined) groups.set(prefix, [scan]);
|
|
928
|
+
else group.push(scan);
|
|
912
929
|
}
|
|
930
|
+
const scheduled = [...files.values()].sort(compareInputs);
|
|
913
931
|
const hash = createHash('sha256');
|
|
914
932
|
hash.update('cargo-path-inputs-v2\0');
|
|
915
933
|
for (const { key, target } of [...links.values()].sort((left, right) =>
|
|
@@ -920,14 +938,51 @@ export async function hashCargoPathInputs(
|
|
|
920
938
|
hash.update(target);
|
|
921
939
|
hash.update('\0');
|
|
922
940
|
}
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
941
|
+
async function consume(input: FileInput): Promise<void> {
|
|
942
|
+
if (files.get(input.file) !== input) return;
|
|
943
|
+
const bytes = await readFile(input.file);
|
|
944
|
+
const scan = scans.get(input.file);
|
|
945
|
+
if (scan !== undefined) {
|
|
946
|
+
scans.delete(input.file);
|
|
947
|
+
collectCompiledInputs(input.file, bytes.toString('utf8'), scan.found);
|
|
948
|
+
}
|
|
949
|
+
hash.update(input.key);
|
|
926
950
|
hash.update('\0');
|
|
927
951
|
hash.update(String(bytes.length));
|
|
928
952
|
hash.update('\0');
|
|
929
953
|
hash.update(bytes);
|
|
930
954
|
}
|
|
955
|
+
let next = 0;
|
|
956
|
+
for (const [prefix, group] of [...groups].sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))) {
|
|
957
|
+
// All ordinary sources have a file identity before this package's include identities.
|
|
958
|
+
// Earlier includes may already have consumed a later package's source; only its
|
|
959
|
+
// discovered references survive that boundary, never the source buffer.
|
|
960
|
+
const boundary = prefix + '"include",';
|
|
961
|
+
while (next < scheduled.length && scheduled[next].key < boundary) await consume(scheduled[next++]);
|
|
962
|
+
const included: FileInput[] = [];
|
|
963
|
+
for (const scan of group) {
|
|
964
|
+
for (const source of scan.sources) {
|
|
965
|
+
if (!scans.delete(source)) continue;
|
|
966
|
+
// Discovery also visits ignored sources. They have no scheduled identity;
|
|
967
|
+
// if an include alias later names one, preserve the old read-then-hash path.
|
|
968
|
+
collectCompiledInputs(source, readFileSync(source, 'utf8'), scan.found);
|
|
969
|
+
}
|
|
970
|
+
for (const input of [...scan.found.values()].sort((left, right) =>
|
|
971
|
+
left.file < right.file ? -1 : left.file > right.file ? 1 : 0,
|
|
972
|
+
)) {
|
|
973
|
+
if (ignored(input.file)) continue;
|
|
974
|
+
const accepted = includeFile(
|
|
975
|
+
await realpath(input.file),
|
|
976
|
+
inputKey(scan.owner, 'include', relative(scan.directory, input.source) + '\0' + input.literal),
|
|
977
|
+
);
|
|
978
|
+
if (accepted !== undefined) included.push(accepted);
|
|
979
|
+
}
|
|
980
|
+
scan.found.clear();
|
|
981
|
+
}
|
|
982
|
+
for (const input of included.sort(compareInputs)) await consume(input);
|
|
983
|
+
while (next < scheduled.length && scheduled[next].key.startsWith(prefix)) await consume(scheduled[next++]);
|
|
984
|
+
}
|
|
985
|
+
while (next < scheduled.length) await consume(scheduled[next++]);
|
|
931
986
|
return hash.digest('hex');
|
|
932
987
|
}
|
|
933
988
|
|
|
@@ -1062,15 +1117,7 @@ const INCLUDE_CLOSERS: Readonly<Record<string, string>> = { '(': ')', '[': ']',
|
|
|
1062
1117
|
const INCLUDE_STRING = /^"([^"\\\r\n]*)"$/u;
|
|
1063
1118
|
const INCLUDE_RAW_STRING = /^r(#*)"([\s\S]*)"\1$/u;
|
|
1064
1119
|
|
|
1065
|
-
|
|
1066
|
-
* Existing literal include_str!/include_bytes! files read by this package's Rust sources.
|
|
1067
|
-
* One graph owns the memo; runtime hashes use a fresh memo. Nested packages own their
|
|
1068
|
-
* sources separately. Computed paths and arbitrary build-script reads remain explicit inputs.
|
|
1069
|
-
*/
|
|
1070
|
-
export function compiledInFiles(root: string, cache: Map<string, CompiledCargoInput[]>): CompiledCargoInput[] {
|
|
1071
|
-
const cached = cache.get(root);
|
|
1072
|
-
if (cached !== undefined) return cached;
|
|
1073
|
-
const found = new Map<string, CompiledCargoInput>();
|
|
1120
|
+
function* compiledSourceFiles(root: string): Generator<string> {
|
|
1074
1121
|
const skipped: readonly string[] = HASH_SKIPPED_DIRECTORIES;
|
|
1075
1122
|
const scan = [root];
|
|
1076
1123
|
while (scan.length > 0) {
|
|
@@ -1081,29 +1128,46 @@ export function compiledInFiles(root: string, cache: Map<string, CompiledCargoIn
|
|
|
1081
1128
|
if (entry.isDirectory()) {
|
|
1082
1129
|
if (!skipped.includes(entry.name) && !existsSync(join(child, 'Cargo.toml'))) scan.push(child);
|
|
1083
1130
|
} else if (entry.isFile() && entry.name.endsWith('.rs')) {
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1131
|
+
yield child;
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
function collectCompiledInputs(source: string, text: string, found: Map<string, CompiledCargoInput>): void {
|
|
1138
|
+
if (!text.includes('include_')) return;
|
|
1139
|
+
const tokens = rustTokens(text);
|
|
1140
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
1141
|
+
if (tokens[index]?.text !== 'include_str' && tokens[index]?.text !== 'include_bytes') continue;
|
|
1142
|
+
if (tokens[index + 1]?.text !== '!') continue;
|
|
1143
|
+
const closing = INCLUDE_CLOSERS[tokens[index + 2]?.text ?? ''];
|
|
1144
|
+
const end = tokens[index + 4]?.text === ',' ? index + 5 : index + 4;
|
|
1145
|
+
if (closing === undefined || tokens[end]?.text !== closing) continue;
|
|
1146
|
+
// Own the spelling before matching: captures (and RegExp's last-match state)
|
|
1147
|
+
// must reference path-sized storage, not a view into the decoded source.
|
|
1148
|
+
const literal = Buffer.from(tokens[index + 3]?.text ?? '', 'utf8').toString('utf8');
|
|
1149
|
+
const path = INCLUDE_STRING.exec(literal)?.[1] ?? INCLUDE_RAW_STRING.exec(literal)?.[2];
|
|
1150
|
+
if (path === undefined || path === '') continue;
|
|
1151
|
+
const file = resolve(dirname(source), path);
|
|
1152
|
+
if (statSync(file, { throwIfNoEntry: false })?.isFile()) {
|
|
1153
|
+
const prior = found.get(file);
|
|
1154
|
+
if (prior === undefined || source < prior.source || (source === prior.source && path < prior.literal)) {
|
|
1155
|
+
found.set(file, { file, source, literal: path });
|
|
1104
1156
|
}
|
|
1105
1157
|
}
|
|
1106
1158
|
}
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
/**
|
|
1162
|
+
* Existing literal include_str!/include_bytes! files read by this package's Rust sources.
|
|
1163
|
+
* One graph owns the memo; runtime hashing consumes the same discovery alongside source
|
|
1164
|
+
* bytes. Nested packages own their sources separately. Computed paths remain explicit inputs.
|
|
1165
|
+
*/
|
|
1166
|
+
export function compiledInFiles(root: string, cache: Map<string, CompiledCargoInput[]>): CompiledCargoInput[] {
|
|
1167
|
+
const cached = cache.get(root);
|
|
1168
|
+
if (cached !== undefined) return cached;
|
|
1169
|
+
const found = new Map<string, CompiledCargoInput>();
|
|
1170
|
+
for (const source of compiledSourceFiles(root)) collectCompiledInputs(source, readFileSync(source, 'utf8'), found);
|
|
1107
1171
|
const files = [...found.values()].sort((left, right) =>
|
|
1108
1172
|
left.file < right.file ? -1 : left.file > right.file ? 1 : 0,
|
|
1109
1173
|
);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { GATEWAY_SOCKET, holdLease, type UnleasedCause } from './gateway-lease.js';
|
|
2
|
-
import { BUN_TEST, NEXTEST_RUN } from './verdict.js';
|
|
2
|
+
import { BUN_TEST, GO_TEST, NEXTEST_RUN } from './verdict.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* The client half of cowshed's host CPU budget (specs/cowshed/05_gateway.md, "Host CPU budget").
|
|
@@ -105,7 +105,7 @@ const WORKER_FLAGS: Readonly<Record<WorkerFlagKind, WorkerFlag>> = {
|
|
|
105
105
|
},
|
|
106
106
|
/** `go test -p`: packages built and tested at once, `GOMAXPROCS` by default, normally one per core. */
|
|
107
107
|
'go-test': {
|
|
108
|
-
invocation:
|
|
108
|
+
invocation: GO_TEST,
|
|
109
109
|
flag: new RegExp(String.raw`(?<![\w-])-p(?:=|\s+)${WORD}`, 'g'),
|
|
110
110
|
sized: (count) => `-p=${count}`,
|
|
111
111
|
ownDefault: (cores, env) => parseInheritedTokens(env.GOMAXPROCS) ?? cores,
|
|
@@ -236,7 +236,7 @@ function namedCounts(text: string, flag: RegExp): number[] {
|
|
|
236
236
|
* Where `invocation` ends in `command`, and where its own arguments end: the next `;`, `&`, `|`, `(` or `)` outside
|
|
237
237
|
* quotes and `$(…)`.
|
|
238
238
|
*/
|
|
239
|
-
function invocationSpan(command: string, invocation: RegExp): { end: number; argsEnd: number } {
|
|
239
|
+
export function invocationSpan(command: string, invocation: RegExp): { end: number; argsEnd: number } {
|
|
240
240
|
const found = invocation.exec(command);
|
|
241
241
|
const end = found === null ? command.length : found.index + found[0].length;
|
|
242
242
|
let quote: string | null = null;
|
|
@@ -265,13 +265,19 @@ function invocationArgs(command: string, invocation: RegExp): string {
|
|
|
265
265
|
}
|
|
266
266
|
|
|
267
267
|
/**
|
|
268
|
-
* `command` with every `flag` in `invocation`'s own arguments replaced by `
|
|
268
|
+
* `command` with every `flag` in `invocation`'s own arguments replaced by `value`; when it names none, `value` is
|
|
269
269
|
* added right after the invocation if `add` says so.
|
|
270
270
|
*/
|
|
271
|
-
function
|
|
271
|
+
export function withInvocationFlag(
|
|
272
|
+
command: string,
|
|
273
|
+
invocation: RegExp,
|
|
274
|
+
flag: RegExp,
|
|
275
|
+
value: string,
|
|
276
|
+
add: boolean,
|
|
277
|
+
): string {
|
|
272
278
|
const { end, argsEnd } = invocationSpan(command, invocation);
|
|
273
279
|
const args = command.slice(end, argsEnd);
|
|
274
|
-
const sizedArgs = args.search(flag) === -1 && add ? ` ${
|
|
280
|
+
const sizedArgs = args.search(flag) === -1 && add ? ` ${value}${args}` : args.replace(flag, value);
|
|
275
281
|
return `${command.slice(0, end)}${sizedArgs}${command.slice(argsEnd)}`;
|
|
276
282
|
}
|
|
277
283
|
|
|
@@ -317,7 +323,7 @@ export function sizeRunner(
|
|
|
317
323
|
case 'go-test': {
|
|
318
324
|
const runner = WORKER_FLAGS[demand.kind];
|
|
319
325
|
return {
|
|
320
|
-
command:
|
|
326
|
+
command: withInvocationFlag(command, runner.invocation, runner.flag, runner.sized(runnerCount), true),
|
|
321
327
|
env: { [CPU_TOKENS_ENV]: held },
|
|
322
328
|
};
|
|
323
329
|
}
|
|
@@ -326,9 +332,9 @@ export function sizeRunner(
|
|
|
326
332
|
// `--jobs` is lowered only where it was set: cargo-mutants refuses it beside `--in-place`.
|
|
327
333
|
const jobs = Math.min(demand.jobs, runnerCount);
|
|
328
334
|
const share = String(Math.max(1, Math.floor(runnerCount / jobs)));
|
|
329
|
-
const tasks =
|
|
335
|
+
const tasks = withInvocationFlag(command, CARGO_MUTANTS, JOBSERVER_TASKS, `--jobserver-tasks=${count}`, true);
|
|
330
336
|
return {
|
|
331
|
-
command:
|
|
337
|
+
command: withInvocationFlag(tasks, CARGO_MUTANTS, MUTANTS_JOBS, `--jobs=${jobs}`, false),
|
|
332
338
|
env: {
|
|
333
339
|
[CPU_TOKENS_ENV]: held,
|
|
334
340
|
...(demand.jobs > 1 ? { CARGO_MUTANTS_JOBS: String(jobs) } : {}),
|
|
@@ -81,9 +81,7 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
81
81
|
process.kill(-pid, 'SIGKILL');
|
|
82
82
|
} catch (cause) {
|
|
83
83
|
const code = cause instanceof Error && 'code' in cause ? cause.code : undefined;
|
|
84
|
-
|
|
85
|
-
// descendant waiting for launchd to reap it): such a group has nothing left to kill.
|
|
86
|
-
if (code !== 'ESRCH' && !(code === 'EPERM' && onlyZombiesIn(pid))) {
|
|
84
|
+
if (code !== 'ESRCH') {
|
|
87
85
|
throw new Error(`cleanup owned readiness process group pgid=${pid} failed`, { cause });
|
|
88
86
|
}
|
|
89
87
|
} finally {
|
|
@@ -574,6 +572,8 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
574
572
|
);
|
|
575
573
|
|
|
576
574
|
await termination.armed();
|
|
575
|
+
// This fixture has no descendants; runBoundedExec has observed its leader's exit.
|
|
576
|
+
termination.release();
|
|
577
577
|
expect(result.success).toBe(false);
|
|
578
578
|
expect(await readFile(handled, 'utf8'), `cooperative handler pid=${handlerPid} must handle SIGTERM`).toBe(
|
|
579
579
|
`handled pid=${handlerPid}\n`,
|
|
@@ -736,7 +736,9 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
736
736
|
);
|
|
737
737
|
|
|
738
738
|
await termination.armed();
|
|
739
|
-
|
|
739
|
+
const descendantOutput = await closed;
|
|
740
|
+
termination.release();
|
|
741
|
+
expect(descendantOutput, `ready descendant pid=${descendantPid} must close its lifetime FIFO on exit`).toBe('');
|
|
740
742
|
expect(result.success).toBe(false);
|
|
741
743
|
});
|
|
742
744
|
|
|
@@ -750,19 +752,20 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
750
752
|
const ready = join(workspace.root, 'straggler-ready.fifo');
|
|
751
753
|
expect(spawnSync('mkfifo', [fifo, ready]).status).toBe(0);
|
|
752
754
|
// The leader exits on SIGTERM. The subshell beside it ignores SIGTERM, a disposition its
|
|
753
|
-
// `tail` inherits, and announces itself only once
|
|
754
|
-
//
|
|
755
|
-
//
|
|
755
|
+
// `tail` inherits, and announces itself only once that disposition is installed. Every
|
|
756
|
+
// script descendant inherits the lifetime FIFO, so EOF proves all of them ended, not
|
|
757
|
+
// just the ignoring child. A fork during signal delivery inherits that witness too.
|
|
756
758
|
await workspace.write(
|
|
757
759
|
'straggle.sh',
|
|
758
760
|
[
|
|
759
|
-
'
|
|
761
|
+
'exec 3>"$1"',
|
|
762
|
+
'( trap "" TERM; echo ready; echo armed >"$2"; exec tail -f /dev/null ) | {',
|
|
760
763
|
' read -r line; echo "$line"; exec tail -f /dev/null',
|
|
761
764
|
'}',
|
|
762
765
|
'',
|
|
763
766
|
].join('\n'),
|
|
764
767
|
);
|
|
765
|
-
//
|
|
768
|
+
// All script descendants own a writer; the read ends only when the last one exits.
|
|
766
769
|
const straggler = readToEnd(fifo);
|
|
767
770
|
const termination = armedKiller('TERM-ignoring FIFO straggler', async (pid) => {
|
|
768
771
|
expect(await readToEnd(ready), `straggler readiness pgid=${pid} must follow trap and FIFO open`).toBe('armed\n');
|
|
@@ -777,10 +780,16 @@ describe('@smoothbricks/nx-plugin:bounded-exec', () => {
|
|
|
777
780
|
);
|
|
778
781
|
|
|
779
782
|
await termination.armed();
|
|
783
|
+
const stragglerOutput = await straggler;
|
|
784
|
+
termination.release();
|
|
780
785
|
expect(result.terminalOutput).toContain('ready');
|
|
781
|
-
expect(
|
|
786
|
+
expect(stragglerOutput).toBe('');
|
|
782
787
|
expect(result.success).toBe(false);
|
|
783
788
|
expect(result.terminalOutput).toContain('Force-killing what the timed-out command left in its process group');
|
|
789
|
+
expect(
|
|
790
|
+
[...ownedProcessGroups],
|
|
791
|
+
'leader exit and straggler FIFO EOF retire cleanup authority; afterEach must not signal their stale PGID',
|
|
792
|
+
).toEqual([]);
|
|
784
793
|
});
|
|
785
794
|
|
|
786
795
|
it('runs the command with its RAM lease as TMPDIR, names held dead leases, and ends the lease after it exits', async () => {
|
|
@@ -1573,10 +1582,12 @@ function armedKiller(subject: string, ready: (pid: number) => Promise<void>) {
|
|
|
1573
1582
|
const real = createProcessTreeKiller();
|
|
1574
1583
|
const calls: NodeJS.Signals[] = [];
|
|
1575
1584
|
let armed: Promise<void> | undefined;
|
|
1585
|
+
let group: number | undefined;
|
|
1576
1586
|
return {
|
|
1577
1587
|
calls,
|
|
1578
1588
|
killer: {
|
|
1579
1589
|
async kill(pid: number, signal: NodeJS.Signals) {
|
|
1590
|
+
group = pid;
|
|
1580
1591
|
ownedProcessGroups.add(pid);
|
|
1581
1592
|
armed ??= ready(pid);
|
|
1582
1593
|
await armed;
|
|
@@ -1589,6 +1600,11 @@ function armedKiller(subject: string, ready: (pid: number) => Promise<void>) {
|
|
|
1589
1600
|
// The executor tolerates killer errors; re-await here so a failed readiness assertion cannot be swallowed.
|
|
1590
1601
|
await armed;
|
|
1591
1602
|
},
|
|
1603
|
+
/** Called only after leader exit and EOF from every descendant's inherited lifetime writer. */
|
|
1604
|
+
release() {
|
|
1605
|
+
if (group === undefined) throw new Error(`${subject}: no process group was armed`);
|
|
1606
|
+
ownedProcessGroups.delete(group);
|
|
1607
|
+
},
|
|
1592
1608
|
};
|
|
1593
1609
|
}
|
|
1594
1610
|
|
|
@@ -1754,16 +1770,3 @@ function inheritedFrom(pool: CpuPool): InheritedCpuPool {
|
|
|
1754
1770
|
function lentCount(want: number, tokens: number): number {
|
|
1755
1771
|
return Math.min(want, tokens);
|
|
1756
1772
|
}
|
|
1757
|
-
|
|
1758
|
-
/** Every process left in group `pgid` is a zombie: `ps` lists no member in any other state. */
|
|
1759
|
-
function onlyZombiesIn(pgid: number): boolean {
|
|
1760
|
-
const listed = spawnSync('ps', ['-A', '-o', 'pgid=,stat='], { encoding: 'utf8' });
|
|
1761
|
-
if (listed.status !== 0) {
|
|
1762
|
-
return false;
|
|
1763
|
-
}
|
|
1764
|
-
return listed.stdout
|
|
1765
|
-
.split('\n')
|
|
1766
|
-
.map((line) => line.trim().split(/\s+/))
|
|
1767
|
-
.filter(([group]) => Number(group) === pgid)
|
|
1768
|
-
.every(([, state]) => state?.startsWith('Z') === true);
|
|
1769
|
-
}
|
|
@@ -474,7 +474,7 @@ export async function runBoundedExec(
|
|
|
474
474
|
}
|
|
475
475
|
|
|
476
476
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
477
|
-
const progress = createRunProgress();
|
|
477
|
+
const progress = createRunProgress(command);
|
|
478
478
|
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
479
479
|
// since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
|
|
480
480
|
const noSpace = createNoSpaceWatch();
|
|
@@ -542,7 +542,7 @@ export async function runBoundedExec(
|
|
|
542
542
|
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
|
|
543
543
|
};
|
|
544
544
|
const armIdleTimer = (): void => {
|
|
545
|
-
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
|
|
545
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || !progress.isTesting()) {
|
|
546
546
|
return;
|
|
547
547
|
}
|
|
548
548
|
idleLeftMs = idleTimeoutMs;
|
|
@@ -577,6 +577,10 @@ export async function runBoundedExec(
|
|
|
577
577
|
const waitedMs = Date.now() - queuedSinceMs;
|
|
578
578
|
queuedMs += waitedMs;
|
|
579
579
|
queuedSinceMs = null;
|
|
580
|
+
if (!progress.isTesting()) {
|
|
581
|
+
appendStderr(`bounded-exec: host CPU wait ended after ${waitedMs}ms; no test runner is active\n`);
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
580
584
|
appendStderr(
|
|
581
585
|
`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
|
|
582
586
|
);
|
|
@@ -587,16 +591,27 @@ export async function runBoundedExec(
|
|
|
587
591
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
588
592
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
589
593
|
// message extend the very bound that just fired.
|
|
594
|
+
const scanProgress = (stream: 'stdout' | 'stderr', text: string): void => {
|
|
595
|
+
const wasTesting = progress.isTesting();
|
|
596
|
+
const finished = progress.scan(stream, text);
|
|
597
|
+
if (!progress.isTesting()) {
|
|
598
|
+
clearTimeout(idleTimer);
|
|
599
|
+
cancelCapture();
|
|
600
|
+
idleCapture = undefined;
|
|
601
|
+
} else if (finished || !wasTesting) {
|
|
602
|
+
armIdleTimer();
|
|
603
|
+
}
|
|
604
|
+
};
|
|
590
605
|
child.stdout?.on('data', (chunk: Buffer | string) => {
|
|
591
606
|
const text = chunk.toString();
|
|
592
|
-
|
|
607
|
+
scanProgress('stdout', text);
|
|
593
608
|
const said = noSpace.scan('stdout', text);
|
|
594
609
|
if (said !== null) state.noSpace = sight(said);
|
|
595
610
|
appendStdout(text);
|
|
596
611
|
});
|
|
597
612
|
child.stderr?.on('data', (chunk: Buffer | string) => {
|
|
598
613
|
const text = chunk.toString();
|
|
599
|
-
|
|
614
|
+
scanProgress('stderr', text);
|
|
600
615
|
const said = noSpace.scan('stderr', text);
|
|
601
616
|
if (said !== null) state.noSpace = sight(said);
|
|
602
617
|
appendStderr(text);
|