@smoothbricks/nx-plugin 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -2
- package/dist/cargo-source-hash.d.ts +2 -2
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +106 -46
- package/managed/raw/patches/nx@23.2.1.patch +182 -45
- package/package.json +1 -1
- package/src/cargo-source-hash.test.ts +143 -2
- package/src/cargo-source-hash.ts +108 -44
- package/src/nx-daemon-start-never.test.ts +126 -0
- package/src/nx-task-output-lease.test.ts +136 -0
- package/test-timings.json +92 -91
|
@@ -1,11 +1,25 @@
|
|
|
1
1
|
import { expect, it } from 'bun:test';
|
|
2
2
|
import { execFileSync } from 'node:child_process';
|
|
3
|
-
import {
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { statSync } from 'node:fs';
|
|
5
|
+
import {
|
|
6
|
+
chmod,
|
|
7
|
+
mkdir,
|
|
8
|
+
mkdtemp,
|
|
9
|
+
readFile,
|
|
10
|
+
realpath,
|
|
11
|
+
rename,
|
|
12
|
+
rm,
|
|
13
|
+
stat,
|
|
14
|
+
symlink,
|
|
15
|
+
utimes,
|
|
16
|
+
writeFile,
|
|
17
|
+
} from 'node:fs/promises';
|
|
4
18
|
import { homedir, tmpdir } from 'node:os';
|
|
5
19
|
import { dirname, join, relative } from 'node:path';
|
|
6
20
|
import { useFixtureCargoHome } from './__tests__/fixture-cargo-home.js';
|
|
7
21
|
import { runtimeInputValue } from './__tests__/runtime-input.js';
|
|
8
|
-
import { hashCargoPathInputs, readCargoResolve } from './cargo-source-hash.js';
|
|
22
|
+
import { CARGO_ANCESTOR_INPUTS, compiledInFiles, hashCargoPathInputs, readCargoResolve } from './cargo-source-hash.js';
|
|
9
23
|
|
|
10
24
|
useFixtureCargoHome();
|
|
11
25
|
|
|
@@ -564,3 +578,130 @@ it.each(['helper', 'cli'])(
|
|
|
564
578
|
}
|
|
565
579
|
},
|
|
566
580
|
);
|
|
581
|
+
|
|
582
|
+
it('keeps the v2 transcript across forward source includes and an ignored source included through an alias', async () => {
|
|
583
|
+
const root = await realpath(await mkdtemp(join(tmpdir(), 'cargo-hash-fused-')));
|
|
584
|
+
const manifest = join(root, 'Cargo.toml');
|
|
585
|
+
const contents = new Map([
|
|
586
|
+
['Cargo.toml', '[workspace]\nmembers=["a","z"]\nresolver="2"\n'],
|
|
587
|
+
['a/Cargo.toml', '[package]\nname="a"\nversion="0.1.0"\nedition="2021"\n'],
|
|
588
|
+
['z/Cargo.toml', '[package]\nname="z"\nversion="0.1.0"\nedition="2021"\n'],
|
|
589
|
+
['a/src/lib.rs', 'const Z: &str = include_str!("../../z/src/lib.rs");\n'],
|
|
590
|
+
['z/src/lib.rs', 'const H: &str = include_str!("../hidden.txt");\nconst P: &str = include_str!("payload.bin");\n'],
|
|
591
|
+
['z/src/hidden.rs', 'const S: &str = include_str!("secret.bin");\n'],
|
|
592
|
+
['z/src/payload.bin', 'payload'],
|
|
593
|
+
['z/src/secret.bin', 'secret'],
|
|
594
|
+
['.gitignore', 'z/src/hidden.rs\n'],
|
|
595
|
+
]);
|
|
596
|
+
try {
|
|
597
|
+
for (const [path, text] of contents) {
|
|
598
|
+
await mkdir(dirname(join(root, path)), { recursive: true });
|
|
599
|
+
await writeFile(join(root, path), text);
|
|
600
|
+
}
|
|
601
|
+
await symlink('src/hidden.rs', join(root, 'z/hidden.txt'));
|
|
602
|
+
execFileSync('git', ['init', '--quiet', root], { stdio: 'pipe' });
|
|
603
|
+
execFileSync('cargo', ['generate-lockfile', '--offline', '--manifest-path', manifest], { stdio: 'pipe' });
|
|
604
|
+
// Independent explicit v2 transcript. In particular, z's source is named by a,
|
|
605
|
+
// while hidden.rs is discovery-only until the unignored alias names its bytes.
|
|
606
|
+
const inputs: { file: string; key: string }[] = [
|
|
607
|
+
{ file: manifest, key: JSON.stringify(['a', '0.1.0', 'ancestor', '1/Cargo.toml']) },
|
|
608
|
+
{ file: join(root, 'a/Cargo.toml'), key: JSON.stringify(['a', '0.1.0', 'file', 'Cargo.toml']) },
|
|
609
|
+
{ file: join(root, 'a/src/lib.rs'), key: JSON.stringify(['a', '0.1.0', 'file', 'src/lib.rs']) },
|
|
610
|
+
{
|
|
611
|
+
file: join(root, 'z/src/lib.rs'),
|
|
612
|
+
key: JSON.stringify(['a', '0.1.0', 'include', 'src/lib.rs\0../../z/src/lib.rs']),
|
|
613
|
+
},
|
|
614
|
+
{ file: join(root, 'z/Cargo.toml'), key: JSON.stringify(['z', '0.1.0', 'file', 'Cargo.toml']) },
|
|
615
|
+
{
|
|
616
|
+
file: join(root, 'z/src/secret.bin'),
|
|
617
|
+
key: JSON.stringify(['z', '0.1.0', 'include', 'src/hidden.rs\0secret.bin']),
|
|
618
|
+
},
|
|
619
|
+
{
|
|
620
|
+
file: join(root, 'z/src/hidden.rs'),
|
|
621
|
+
key: JSON.stringify(['z', '0.1.0', 'include', 'src/lib.rs\0../hidden.txt']),
|
|
622
|
+
},
|
|
623
|
+
{
|
|
624
|
+
file: join(root, 'z/src/payload.bin'),
|
|
625
|
+
key: JSON.stringify(['z', '0.1.0', 'include', 'src/lib.rs\0payload.bin']),
|
|
626
|
+
},
|
|
627
|
+
];
|
|
628
|
+
for (let ancestor = dirname(root), depth = 2; ; ancestor = dirname(ancestor), depth += 1) {
|
|
629
|
+
for (const name of CARGO_ANCESTOR_INPUTS) {
|
|
630
|
+
const file = join(ancestor, name);
|
|
631
|
+
if (statSync(file, { throwIfNoEntry: false })?.isFile()) {
|
|
632
|
+
inputs.push({ file, key: JSON.stringify(['a', '0.1.0', 'ancestor', depth + '/' + name]) });
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
if (dirname(ancestor) === ancestor) break;
|
|
636
|
+
}
|
|
637
|
+
const expected = createHash('sha256');
|
|
638
|
+
expected.update('cargo-path-inputs-v2\0');
|
|
639
|
+
expected.update(JSON.stringify(['z', '0.1.0', 'link', 'hidden.txt']));
|
|
640
|
+
expected.update('\0symlink\0src/hidden.rs\0');
|
|
641
|
+
for (const { file, key } of inputs.sort((left, right) =>
|
|
642
|
+
left.key < right.key ? -1 : left.key > right.key ? 1 : 0,
|
|
643
|
+
)) {
|
|
644
|
+
const bytes = await readFile(file);
|
|
645
|
+
expected.update(key);
|
|
646
|
+
expected.update('\0' + bytes.length + '\0');
|
|
647
|
+
expected.update(bytes);
|
|
648
|
+
}
|
|
649
|
+
const before = await hashCargoPathInputs(manifest, root, { includeWorkspace: true });
|
|
650
|
+
expect(before).toBe(expected.digest('hex'));
|
|
651
|
+
await writeFile(join(root, 'z/src/secret.bin'), 'changed');
|
|
652
|
+
expect(await hashCargoPathInputs(manifest, root, { includeWorkspace: true })).not.toBe(before);
|
|
653
|
+
} finally {
|
|
654
|
+
await rm(root, { recursive: true, force: true });
|
|
655
|
+
}
|
|
656
|
+
});
|
|
657
|
+
|
|
658
|
+
it('keeps literal discovery exact and refuses unreadable sources', async () => {
|
|
659
|
+
const root = await realpath(await mkdtemp(join(tmpdir(), 'cargo-hash-literals-')));
|
|
660
|
+
const source = join(root, 'src/lib.rs');
|
|
661
|
+
const unicode = 'spelling-e\u0301-\u{10000}-payload';
|
|
662
|
+
try {
|
|
663
|
+
await mkdir(dirname(source), { recursive: true });
|
|
664
|
+
await writeFile(
|
|
665
|
+
join(root, 'Cargo.toml'),
|
|
666
|
+
'[package]\nname="literals"\nversion="0.1.0"\nedition="2021"\n[workspace]\n',
|
|
667
|
+
);
|
|
668
|
+
await writeFile(join(root, 'src/data'), 'data');
|
|
669
|
+
await writeFile(join(root, 'src', unicode), 'unicode spelling');
|
|
670
|
+
await writeFile(
|
|
671
|
+
source,
|
|
672
|
+
[
|
|
673
|
+
'const A: &str = include_str!(r#"data"#,);',
|
|
674
|
+
'const B: &[u8] = include_bytes!["data"];',
|
|
675
|
+
'const C: &str = include_str!{"data"};',
|
|
676
|
+
`const U: &str = include_str!(${JSON.stringify(unicode)});`,
|
|
677
|
+
'// include_str!("missing")',
|
|
678
|
+
'/* outer /* include_bytes!("missing") */ comment */',
|
|
679
|
+
'const D: &str = r###"include_str!("missing")"###;',
|
|
680
|
+
'include!("missing");',
|
|
681
|
+
'include_str!("");',
|
|
682
|
+
'include_str!("da\\\\ta");',
|
|
683
|
+
'include_str!(concat!("da", "ta"));',
|
|
684
|
+
'include_str!("missing");',
|
|
685
|
+
'include_str!("data"];',
|
|
686
|
+
'include_str!("unterminated',
|
|
687
|
+
].join('\n'),
|
|
688
|
+
);
|
|
689
|
+
expect(compiledInFiles(root, new Map())).toEqual([
|
|
690
|
+
{ file: join(root, 'src/data'), source, literal: 'data' },
|
|
691
|
+
{ file: join(root, 'src', unicode), source, literal: unicode },
|
|
692
|
+
]);
|
|
693
|
+
execFileSync('cargo', ['generate-lockfile', '--offline', '--manifest-path', join(root, 'Cargo.toml')], {
|
|
694
|
+
stdio: 'pipe',
|
|
695
|
+
});
|
|
696
|
+
await chmod(source, 0);
|
|
697
|
+
expect(() => compiledInFiles(root, new Map())).toThrow();
|
|
698
|
+
// A cold native Cargo census can report the denial before the source read;
|
|
699
|
+
// both paths must preserve the operational refusal rather than skip the file.
|
|
700
|
+
await expect(hashCargoPathInputs(join(root, 'Cargo.toml'), root, { includeWorkspace: true })).rejects.toThrow(
|
|
701
|
+
'EACCES',
|
|
702
|
+
);
|
|
703
|
+
} finally {
|
|
704
|
+
await chmod(source, 0o600);
|
|
705
|
+
await rm(root, { recursive: true, force: true });
|
|
706
|
+
}
|
|
707
|
+
});
|
package/src/cargo-source-hash.ts
CHANGED
|
@@ -812,15 +812,26 @@ export async function hashCargoPathInputs(
|
|
|
812
812
|
const inputKey = (pkg: LocalCargoPackage, role: InputRole, path: string) =>
|
|
813
813
|
JSON.stringify([pkg.name, pkg.version, role, path.split(sep).join('/')]);
|
|
814
814
|
const directories = new Map<string, LocalCargoPackage>();
|
|
815
|
-
|
|
815
|
+
interface FileInput {
|
|
816
|
+
readonly file: string;
|
|
817
|
+
readonly key: string;
|
|
818
|
+
readonly order: number;
|
|
819
|
+
}
|
|
820
|
+
const files = new Map<string, FileInput>();
|
|
821
|
+
const compareInputs = (left: FileInput, right: FileInput) =>
|
|
822
|
+
left.key < right.key ? -1 : left.key > right.key ? 1 : left.order - right.order;
|
|
816
823
|
const ancestors = new Set<string>();
|
|
817
824
|
const links = new Map<string, { key: string; target: string }>();
|
|
818
825
|
const packageSources: { file: string; key: string }[] = [];
|
|
819
826
|
// Read shared bytes once. The smallest declared identity makes multiple owners
|
|
820
827
|
// deterministic without using a consuming checkout's mount path as an input.
|
|
821
|
-
const includeFile = (file: string, key: string) => {
|
|
828
|
+
const includeFile = (file: string, key: string): FileInput | undefined => {
|
|
822
829
|
const prior = files.get(file);
|
|
823
|
-
if (prior
|
|
830
|
+
if (prior !== undefined && key >= prior.key) return undefined;
|
|
831
|
+
// Rekeying must retain the old Map insertion order for equal v2 identities.
|
|
832
|
+
const input = { file, key, order: prior?.order ?? files.size };
|
|
833
|
+
files.set(file, input);
|
|
834
|
+
return input;
|
|
824
835
|
};
|
|
825
836
|
const packages = [...cargo.local].sort((left, right) =>
|
|
826
837
|
left.name < right.name
|
|
@@ -899,17 +910,24 @@ export async function hashCargoPathInputs(
|
|
|
899
910
|
}
|
|
900
911
|
}
|
|
901
912
|
for (const [directory, owner] of directories) await walk(directory, owner, '');
|
|
902
|
-
|
|
913
|
+
interface SourceScan {
|
|
914
|
+
readonly directory: string;
|
|
915
|
+
readonly owner: LocalCargoPackage;
|
|
916
|
+
readonly sources: readonly string[];
|
|
917
|
+
readonly found: Map<string, CompiledCargoInput>;
|
|
918
|
+
}
|
|
919
|
+
const scans = new Map<string, SourceScan>();
|
|
920
|
+
const groups = new Map<string, SourceScan[]>();
|
|
903
921
|
for (const [directory, owner] of directories) {
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
}
|
|
922
|
+
const scan: SourceScan = { directory, owner, sources: [...compiledSourceFiles(directory)], found: new Map() };
|
|
923
|
+
for (const source of scan.sources) scans.set(source, scan);
|
|
924
|
+
// Equal package identities can interleave, so their discovery shares one boundary.
|
|
925
|
+
const prefix = JSON.stringify([owner.name, owner.version]).slice(0, -1) + ',';
|
|
926
|
+
const group = groups.get(prefix);
|
|
927
|
+
if (group === undefined) groups.set(prefix, [scan]);
|
|
928
|
+
else group.push(scan);
|
|
912
929
|
}
|
|
930
|
+
const scheduled = [...files.values()].sort(compareInputs);
|
|
913
931
|
const hash = createHash('sha256');
|
|
914
932
|
hash.update('cargo-path-inputs-v2\0');
|
|
915
933
|
for (const { key, target } of [...links.values()].sort((left, right) =>
|
|
@@ -920,14 +938,51 @@ export async function hashCargoPathInputs(
|
|
|
920
938
|
hash.update(target);
|
|
921
939
|
hash.update('\0');
|
|
922
940
|
}
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
941
|
+
async function consume(input: FileInput): Promise<void> {
|
|
942
|
+
if (files.get(input.file) !== input) return;
|
|
943
|
+
const bytes = await readFile(input.file);
|
|
944
|
+
const scan = scans.get(input.file);
|
|
945
|
+
if (scan !== undefined) {
|
|
946
|
+
scans.delete(input.file);
|
|
947
|
+
collectCompiledInputs(input.file, bytes.toString('utf8'), scan.found);
|
|
948
|
+
}
|
|
949
|
+
hash.update(input.key);
|
|
926
950
|
hash.update('\0');
|
|
927
951
|
hash.update(String(bytes.length));
|
|
928
952
|
hash.update('\0');
|
|
929
953
|
hash.update(bytes);
|
|
930
954
|
}
|
|
955
|
+
let next = 0;
|
|
956
|
+
for (const [prefix, group] of [...groups].sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))) {
|
|
957
|
+
// All ordinary sources have a file identity before this package's include identities.
|
|
958
|
+
// Earlier includes may already have consumed a later package's source; only its
|
|
959
|
+
// discovered references survive that boundary, never the source buffer.
|
|
960
|
+
const boundary = prefix + '"include",';
|
|
961
|
+
while (next < scheduled.length && scheduled[next].key < boundary) await consume(scheduled[next++]);
|
|
962
|
+
const included: FileInput[] = [];
|
|
963
|
+
for (const scan of group) {
|
|
964
|
+
for (const source of scan.sources) {
|
|
965
|
+
if (!scans.delete(source)) continue;
|
|
966
|
+
// Discovery also visits ignored sources. They have no scheduled identity;
|
|
967
|
+
// if an include alias later names one, preserve the old read-then-hash path.
|
|
968
|
+
collectCompiledInputs(source, readFileSync(source, 'utf8'), scan.found);
|
|
969
|
+
}
|
|
970
|
+
for (const input of [...scan.found.values()].sort((left, right) =>
|
|
971
|
+
left.file < right.file ? -1 : left.file > right.file ? 1 : 0,
|
|
972
|
+
)) {
|
|
973
|
+
if (ignored(input.file)) continue;
|
|
974
|
+
const accepted = includeFile(
|
|
975
|
+
await realpath(input.file),
|
|
976
|
+
inputKey(scan.owner, 'include', relative(scan.directory, input.source) + '\0' + input.literal),
|
|
977
|
+
);
|
|
978
|
+
if (accepted !== undefined) included.push(accepted);
|
|
979
|
+
}
|
|
980
|
+
scan.found.clear();
|
|
981
|
+
}
|
|
982
|
+
for (const input of included.sort(compareInputs)) await consume(input);
|
|
983
|
+
while (next < scheduled.length && scheduled[next].key.startsWith(prefix)) await consume(scheduled[next++]);
|
|
984
|
+
}
|
|
985
|
+
while (next < scheduled.length) await consume(scheduled[next++]);
|
|
931
986
|
return hash.digest('hex');
|
|
932
987
|
}
|
|
933
988
|
|
|
@@ -1062,15 +1117,7 @@ const INCLUDE_CLOSERS: Readonly<Record<string, string>> = { '(': ')', '[': ']',
|
|
|
1062
1117
|
const INCLUDE_STRING = /^"([^"\\\r\n]*)"$/u;
|
|
1063
1118
|
const INCLUDE_RAW_STRING = /^r(#*)"([\s\S]*)"\1$/u;
|
|
1064
1119
|
|
|
1065
|
-
|
|
1066
|
-
* Existing literal include_str!/include_bytes! files read by this package's Rust sources.
|
|
1067
|
-
* One graph owns the memo; runtime hashes use a fresh memo. Nested packages own their
|
|
1068
|
-
* sources separately. Computed paths and arbitrary build-script reads remain explicit inputs.
|
|
1069
|
-
*/
|
|
1070
|
-
export function compiledInFiles(root: string, cache: Map<string, CompiledCargoInput[]>): CompiledCargoInput[] {
|
|
1071
|
-
const cached = cache.get(root);
|
|
1072
|
-
if (cached !== undefined) return cached;
|
|
1073
|
-
const found = new Map<string, CompiledCargoInput>();
|
|
1120
|
+
function* compiledSourceFiles(root: string): Generator<string> {
|
|
1074
1121
|
const skipped: readonly string[] = HASH_SKIPPED_DIRECTORIES;
|
|
1075
1122
|
const scan = [root];
|
|
1076
1123
|
while (scan.length > 0) {
|
|
@@ -1081,29 +1128,46 @@ export function compiledInFiles(root: string, cache: Map<string, CompiledCargoIn
|
|
|
1081
1128
|
if (entry.isDirectory()) {
|
|
1082
1129
|
if (!skipped.includes(entry.name) && !existsSync(join(child, 'Cargo.toml'))) scan.push(child);
|
|
1083
1130
|
} else if (entry.isFile() && entry.name.endsWith('.rs')) {
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1131
|
+
yield child;
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
function collectCompiledInputs(source: string, text: string, found: Map<string, CompiledCargoInput>): void {
|
|
1138
|
+
if (!text.includes('include_')) return;
|
|
1139
|
+
const tokens = rustTokens(text);
|
|
1140
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
1141
|
+
if (tokens[index]?.text !== 'include_str' && tokens[index]?.text !== 'include_bytes') continue;
|
|
1142
|
+
if (tokens[index + 1]?.text !== '!') continue;
|
|
1143
|
+
const closing = INCLUDE_CLOSERS[tokens[index + 2]?.text ?? ''];
|
|
1144
|
+
const end = tokens[index + 4]?.text === ',' ? index + 5 : index + 4;
|
|
1145
|
+
if (closing === undefined || tokens[end]?.text !== closing) continue;
|
|
1146
|
+
// Own the spelling before matching: captures (and RegExp's last-match state)
|
|
1147
|
+
// must reference path-sized storage, not a view into the decoded source.
|
|
1148
|
+
const literal = Buffer.from(tokens[index + 3]?.text ?? '', 'utf8').toString('utf8');
|
|
1149
|
+
const path = INCLUDE_STRING.exec(literal)?.[1] ?? INCLUDE_RAW_STRING.exec(literal)?.[2];
|
|
1150
|
+
if (path === undefined || path === '') continue;
|
|
1151
|
+
const file = resolve(dirname(source), path);
|
|
1152
|
+
if (statSync(file, { throwIfNoEntry: false })?.isFile()) {
|
|
1153
|
+
const prior = found.get(file);
|
|
1154
|
+
if (prior === undefined || source < prior.source || (source === prior.source && path < prior.literal)) {
|
|
1155
|
+
found.set(file, { file, source, literal: path });
|
|
1104
1156
|
}
|
|
1105
1157
|
}
|
|
1106
1158
|
}
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
/**
|
|
1162
|
+
* Existing literal include_str!/include_bytes! files read by this package's Rust sources.
|
|
1163
|
+
* One graph owns the memo; runtime hashing consumes the same discovery alongside source
|
|
1164
|
+
* bytes. Nested packages own their sources separately. Computed paths remain explicit inputs.
|
|
1165
|
+
*/
|
|
1166
|
+
export function compiledInFiles(root: string, cache: Map<string, CompiledCargoInput[]>): CompiledCargoInput[] {
|
|
1167
|
+
const cached = cache.get(root);
|
|
1168
|
+
if (cached !== undefined) return cached;
|
|
1169
|
+
const found = new Map<string, CompiledCargoInput>();
|
|
1170
|
+
for (const source of compiledSourceFiles(root)) collectCompiledInputs(source, readFileSync(source, 'utf8'), found);
|
|
1107
1171
|
const files = [...found.values()].sort((left, right) =>
|
|
1108
1172
|
left.file < right.file ? -1 : left.file > right.file ? 1 : 0,
|
|
1109
1173
|
);
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { expect, it } from 'bun:test';
|
|
2
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
3
|
+
import { mkdir, readFile, symlink, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { createRequire } from 'node:module';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { fixtureNxEnv, withNxFixture } from './__tests__/fixture-nx-env.js';
|
|
7
|
+
import { awaitExit, isRunning, processTable, terminate } from './testing.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Guards the `NX_DAEMON_START=never` hunk of `patches/nx@23.2.1.patch`. A stock client that finds no live daemon
|
|
11
|
+
* starts one, detached into a session of its own, and the record shares it with every client of the checkout. Cowshed
|
|
12
|
+
* ends a job's whole process tree when the job ends (`specs/cowshed/11_shell.md`), so a daemon one job's client
|
|
13
|
+
* started would end with that job while another job's `nx run` still used it. Every cowshed job therefore runs with
|
|
14
|
+
* `NX_DAEMON_START=never` (`specs/cowshed/04_sandbox.md`): its client uses the daemon the record names while that
|
|
15
|
+
* runs, otherwise computes the command without one, and never starts a daemon. A daemon that stops under such a
|
|
16
|
+
* client turns the daemon off for that process alone, and the command continues without it: its successor is its
|
|
17
|
+
* starter's to start, and the stop is no reason to disable the checkout's daemon for anyone else.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const repositoryRoot = join(import.meta.dir, '../../..');
|
|
21
|
+
const nxEntry = join(repositoryRoot, 'node_modules', '.bin', 'nx');
|
|
22
|
+
const nxVersion: string = createRequire(join(repositoryRoot, 'package.json'))('nx/package.json').version;
|
|
23
|
+
|
|
24
|
+
const record = (workspace: string) => join(workspace, '.nx/workspace-data/d/server-process.json');
|
|
25
|
+
|
|
26
|
+
async function recordedPid(workspace: string): Promise<number | null> {
|
|
27
|
+
const text = await readFile(record(workspace), 'utf8').catch((error: unknown) => {
|
|
28
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null;
|
|
29
|
+
throw error;
|
|
30
|
+
});
|
|
31
|
+
if (text === null) return null;
|
|
32
|
+
const parsed: unknown = JSON.parse(text);
|
|
33
|
+
if (
|
|
34
|
+
typeof parsed !== 'object' ||
|
|
35
|
+
parsed === null ||
|
|
36
|
+
!('processId' in parsed) ||
|
|
37
|
+
typeof parsed.processId !== 'number'
|
|
38
|
+
) {
|
|
39
|
+
throw new Error(`the daemon record names no process: ${text}`);
|
|
40
|
+
}
|
|
41
|
+
return parsed.processId;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function fixtureWorkspace(workspace: string): Promise<void> {
|
|
45
|
+
await symlink(join(repositoryRoot, 'node_modules'), join(workspace, 'node_modules'), 'dir');
|
|
46
|
+
await writeFile(join(workspace, 'nx.json'), '{}\n');
|
|
47
|
+
await writeFile(join(workspace, 'package.json'), JSON.stringify({ name: 'daemon-start-never', private: true }));
|
|
48
|
+
await mkdir(join(workspace, 'p'));
|
|
49
|
+
await writeFile(join(workspace, 'p', 'project.json'), JSON.stringify({ name: 'p' }));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Run `nx show projects --json`, require it to list the fixture's one project, and return everything it printed. The
|
|
54
|
+
* format is asked for, never inferred: unasked, Nx prints JSON only when it detects an AI agent in the environment
|
|
55
|
+
* (`show/command-object.js`, `isAiAgent`) and a plain name per line otherwise. The list is stdout's last line: a note
|
|
56
|
+
* Nx prints about its daemon goes to stdout before it.
|
|
57
|
+
*/
|
|
58
|
+
function showProjects(workspace: string, env: Record<string, string>): string {
|
|
59
|
+
const ran = spawnSync('bun', [nxEntry, 'show', 'projects', '--json'], { cwd: workspace, env, encoding: 'utf8' });
|
|
60
|
+
const output = `${ran.stdout}${ran.stderr}`;
|
|
61
|
+
expect(ran.status, output).toBe(0);
|
|
62
|
+
expect(JSON.parse(ran.stdout.trimEnd().split('\n').at(-1) ?? ''), output).toEqual(['p']);
|
|
63
|
+
return output;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
it('a client that may never start a daemon uses a running one and otherwise runs without', async () => {
|
|
67
|
+
await withNxFixture('nx-daemon-start-never', async ({ workspace, socketDirectory }) => {
|
|
68
|
+
await fixtureWorkspace(workspace);
|
|
69
|
+
const env = { ...fixtureNxEnv(workspace, socketDirectory), NX_DAEMON: 'true' };
|
|
70
|
+
const never = { ...env, NX_DAEMON_START: 'never' };
|
|
71
|
+
|
|
72
|
+
// No daemon runs: the command is computed in the client, and no daemon is started.
|
|
73
|
+
showProjects(workspace, never);
|
|
74
|
+
expect(await recordedPid(workspace)).toBeNull();
|
|
75
|
+
|
|
76
|
+
// Control: the same client without the variable starts one, so the absence above is the variable's doing.
|
|
77
|
+
showProjects(workspace, env);
|
|
78
|
+
const daemon = await recordedPid(workspace);
|
|
79
|
+
if (daemon === null) throw new Error('a client allowed to start a daemon left no record');
|
|
80
|
+
expect(isRunning(daemon)).toBe(true);
|
|
81
|
+
|
|
82
|
+
// A daemon runs: the client uses it, and it is still the one the record names.
|
|
83
|
+
const log = join(workspace, '.nx/workspace-data/d/daemon.log');
|
|
84
|
+
const before = (await readFile(log, 'utf8')).length;
|
|
85
|
+
showProjects(workspace, never);
|
|
86
|
+
expect(await recordedPid(workspace)).toBe(daemon);
|
|
87
|
+
expect((await readFile(log, 'utf8')).length).toBeGreaterThan(before);
|
|
88
|
+
|
|
89
|
+
// A record whose daemon has gone (killed, so it never cleared its record) is no daemon to use and no reason to
|
|
90
|
+
// start one.
|
|
91
|
+
const killed = (await processTable()).filter((entry) => entry.pid === daemon);
|
|
92
|
+
terminate(daemon, 'SIGKILL');
|
|
93
|
+
await awaitExit(killed, 'the killed fixture daemon');
|
|
94
|
+
showProjects(workspace, never);
|
|
95
|
+
expect(await recordedPid(workspace)).toBe(daemon);
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it('a client that may never start a daemon continues without one that stopped under it, and disables nothing', async () => {
|
|
100
|
+
await withNxFixture('nx-daemon-start-gone', async ({ workspace, socketDirectory }) => {
|
|
101
|
+
await fixtureWorkspace(workspace);
|
|
102
|
+
const never = { ...fixtureNxEnv(workspace, socketDirectory), NX_DAEMON: 'true', NX_DAEMON_START: 'never' };
|
|
103
|
+
|
|
104
|
+
// The record names a running process that serves no socket: a daemon that stopped answering after the client
|
|
105
|
+
// chose to use it.
|
|
106
|
+
const holder = spawn('sleep', ['300'], { stdio: 'ignore' });
|
|
107
|
+
try {
|
|
108
|
+
const pid = holder.pid;
|
|
109
|
+
if (pid === undefined) throw new Error('sleep did not start');
|
|
110
|
+
await mkdir(join(workspace, '.nx/workspace-data/d'), { recursive: true });
|
|
111
|
+
await writeFile(
|
|
112
|
+
record(workspace),
|
|
113
|
+
JSON.stringify({ processId: pid, socketPath: join(socketDirectory, 'gone.sock'), nxVersion }),
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
expect(showProjects(workspace, never)).toContain('NX_DAEMON_START=never: this client never starts a daemon');
|
|
117
|
+
expect(await recordedPid(workspace)).toBe(pid);
|
|
118
|
+
// Not disabled for the checkout: the marker only `nx reset` clears would follow it into every terminal.
|
|
119
|
+
await expect(readFile(join(workspace, '.nx/workspace-data/d/disabled'), 'utf8')).rejects.toMatchObject({
|
|
120
|
+
code: 'ENOENT',
|
|
121
|
+
});
|
|
122
|
+
} finally {
|
|
123
|
+
holder.kill('SIGKILL');
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
});
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { expect, it } from 'bun:test';
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
2
3
|
import { channel } from 'node:diagnostics_channel';
|
|
4
|
+
import { existsSync, mkdirSync, readdirSync, readlinkSync, realpathSync } from 'node:fs';
|
|
3
5
|
import { mkdir, readdir, readFile, symlink, writeFile } from 'node:fs/promises';
|
|
4
6
|
import { join } from 'node:path';
|
|
5
7
|
import type { Subprocess } from 'bun';
|
|
@@ -247,6 +249,140 @@ it('shares ownership across configurations without blocking independent targets,
|
|
|
247
249
|
}
|
|
248
250
|
});
|
|
249
251
|
|
|
252
|
+
/** One output target of its own, so persistent lock files from other runs never contend. */
|
|
253
|
+
function leaseTask(target: string) {
|
|
254
|
+
return {
|
|
255
|
+
id: `${target}:build`,
|
|
256
|
+
target: { project: `${target}-${crypto.randomUUID()}`, target: 'build' },
|
|
257
|
+
overrides: {},
|
|
258
|
+
outputs: ['app/dist'],
|
|
259
|
+
cache: true,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** The helper pid a held lease's lock file names; the file is the helper's own record. */
|
|
264
|
+
async function leaseHolder(path: string): Promise<number> {
|
|
265
|
+
const record: unknown = JSON.parse(await readFile(path, 'utf8'));
|
|
266
|
+
if (typeof record !== 'object' || record === null || !('pid' in record) || typeof record.pid !== 'number') {
|
|
267
|
+
throw new Error(`lease file ${path} names no helper pid`);
|
|
268
|
+
}
|
|
269
|
+
return record.pid;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** The lock path of the next lease `acquire` asks for, read from the diagnostics channel it publishes before waiting. */
|
|
273
|
+
function nextLeasePath(): Promise<string> {
|
|
274
|
+
const path = Promise.withResolvers<string>();
|
|
275
|
+
const events = channel('nx:task-output-lease');
|
|
276
|
+
const onEvent = (message: unknown) => {
|
|
277
|
+
if (typeof message !== 'object' || message === null || !('phase' in message) || message.phase !== 'waiting') return;
|
|
278
|
+
events.unsubscribe(onEvent);
|
|
279
|
+
if ('paths' in message && Array.isArray(message.paths) && typeof message.paths[0] === 'string')
|
|
280
|
+
path.resolve(message.paths[0]);
|
|
281
|
+
else path.reject(new Error(`waiting event names no lock path: ${JSON.stringify(message)}`));
|
|
282
|
+
};
|
|
283
|
+
events.subscribe(onEvent);
|
|
284
|
+
return path.promise;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/** Whether process `pid` is reaped: the parent's exit handler has then run in the same libuv callback. */
|
|
288
|
+
async function reaped(pid: number): Promise<void> {
|
|
289
|
+
for (;;) {
|
|
290
|
+
try {
|
|
291
|
+
process.kill(pid, 0);
|
|
292
|
+
} catch {
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
const turn = Promise.withResolvers<void>();
|
|
296
|
+
setImmediate(turn.resolve);
|
|
297
|
+
await turn.promise;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
it('wakes a waiter with the slot of a helper that died holding a lease, failing only the holder', async () => {
|
|
302
|
+
const leases = new TaskOutputLeases(1, null, 'output');
|
|
303
|
+
try {
|
|
304
|
+
const heldPath = nextLeasePath();
|
|
305
|
+
const release = await leases.acquire([leaseTask('held')]);
|
|
306
|
+
const helper = await leaseHolder(await heldPath);
|
|
307
|
+
// The one slot is taken, so this waits for it.
|
|
308
|
+
const waiting = leases.acquire([leaseTask('queued')]);
|
|
309
|
+
process.kill(helper, 'SIGKILL');
|
|
310
|
+
await expect(release()).rejects.toThrow('worker exited (code null, signal SIGKILL)');
|
|
311
|
+
// The helper's exit freed its slot: the waiter forks a successor rather than waiting for a release no one can send.
|
|
312
|
+
const releaseQueued = await guardEvent(waiting, "a waiter for a dead helper's slot", () => 'no successor helper');
|
|
313
|
+
await releaseQueued();
|
|
314
|
+
} finally {
|
|
315
|
+
await leases.close();
|
|
316
|
+
}
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
it('retires a helper that replied with an error, so the next acquire takes its slot', async () => {
|
|
320
|
+
const leases = new TaskOutputLeases(1, null, 'output');
|
|
321
|
+
try {
|
|
322
|
+
// The helper cannot open a lock path that is a directory, so it replies with an error and serves no more.
|
|
323
|
+
const blockedPath = nextLeasePath().then((path) => mkdirSync(path, { recursive: true }));
|
|
324
|
+
const refused = leases.acquire([leaseTask('unopenable')]);
|
|
325
|
+
await blockedPath;
|
|
326
|
+
await expect(refused).rejects.toThrow('failed:');
|
|
327
|
+
const release = await guardEvent(
|
|
328
|
+
leases.acquire([leaseTask('after-error')]),
|
|
329
|
+
'an acquire after a helper replied with an error',
|
|
330
|
+
() => 'the failed helper still holds the only slot',
|
|
331
|
+
);
|
|
332
|
+
await release();
|
|
333
|
+
} finally {
|
|
334
|
+
await leases.close();
|
|
335
|
+
}
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
it('does not hand an idle helper that died to the next acquire', async () => {
|
|
339
|
+
const leases = new TaskOutputLeases(1, null, 'output');
|
|
340
|
+
try {
|
|
341
|
+
const idlePath = nextLeasePath();
|
|
342
|
+
const release = await leases.acquire([leaseTask('idle')]);
|
|
343
|
+
const helper = await leaseHolder(await idlePath);
|
|
344
|
+
await release();
|
|
345
|
+
process.kill(helper, 'SIGKILL');
|
|
346
|
+
await reaped(helper);
|
|
347
|
+
const releaseNext = await guardEvent(
|
|
348
|
+
leases.acquire([leaseTask('after-idle-death')]),
|
|
349
|
+
'an acquire after the idle helper died',
|
|
350
|
+
() => 'no successor helper',
|
|
351
|
+
);
|
|
352
|
+
await releaseNext();
|
|
353
|
+
} finally {
|
|
354
|
+
await leases.close();
|
|
355
|
+
}
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
/** The descriptors process `pid` holds open on `file`: /proc on Linux, lsof elsewhere, both naming its real path. */
|
|
359
|
+
function openDescriptors(pid: number, file: string): number {
|
|
360
|
+
const path = realpathSync(file);
|
|
361
|
+
const proc = `/proc/${pid}/fd`;
|
|
362
|
+
if (existsSync(proc)) return readdirSync(proc).filter((fd) => readlinkSync(join(proc, fd)) === path).length;
|
|
363
|
+
const listed = spawnSync('lsof', ['-nP', '-p', String(pid), '-Fn'], { encoding: 'utf8' });
|
|
364
|
+
if (listed.status !== 0 && listed.stdout === '') throw new Error(`lsof -p ${pid} failed: ${listed.stderr}`);
|
|
365
|
+
return listed.stdout.split('\n').filter((line) => line === `n${path}`).length;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
it('keeps one lock descriptor per lock file in a helper across acquires', async () => {
|
|
369
|
+
const leases = new TaskOutputLeases(1, null, 'output');
|
|
370
|
+
const task = leaseTask('repeated');
|
|
371
|
+
try {
|
|
372
|
+
const counts: number[] = [];
|
|
373
|
+
for (let round = 0; round < 5; round++) {
|
|
374
|
+
const pathEvent = nextLeasePath();
|
|
375
|
+
const release = await leases.acquire([task]);
|
|
376
|
+
const path = await pathEvent;
|
|
377
|
+
counts.push(openDescriptors(await leaseHolder(path), path));
|
|
378
|
+
await release();
|
|
379
|
+
}
|
|
380
|
+
expect(counts).toEqual([1, 1, 1, 1, 1]);
|
|
381
|
+
} finally {
|
|
382
|
+
await leases.close();
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
|
|
250
386
|
it('refuses an ancestor-owned target before waiting for its output lease', async () => {
|
|
251
387
|
await withNxFixture('nx-task-output-loop-', async ({ workspace, socketDirectory }) => {
|
|
252
388
|
await mkdir(join(workspace, 'app'));
|