@smoothbricks/nx-plugin 0.4.22 → 0.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -33
- package/dist/bin/smoo-nx-exec.js +35 -11
- package/dist/bin/smoo-nx-nextest-extract.d.ts +3 -0
- package/dist/bin/smoo-nx-nextest-extract.d.ts.map +1 -0
- package/dist/bin/smoo-nx-nextest-extract.js +41 -0
- package/dist/cargo-closure-input.d.ts +63 -0
- package/dist/cargo-closure-input.d.ts.map +1 -0
- package/dist/cargo-closure-input.js +197 -0
- package/dist/cargo-source-hash.d.ts +37 -0
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +108 -190
- package/dist/cross-check-policy.d.ts +14 -0
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +15 -1
- package/dist/ensure-built.d.ts +3 -2
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +33 -56
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +152 -91
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +12 -15
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +10 -0
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +12 -6
- package/dist/manifest-hash.d.ts.map +1 -1
- package/dist/manifest-hash.js +24 -30
- package/dist/nextest-extraction.d.ts +101 -0
- package/dist/nextest-extraction.d.ts.map +1 -0
- package/dist/nextest-extraction.js +268 -0
- package/dist/package-target-policy.js +1 -1
- package/dist/platform-targets.d.ts +7 -0
- package/dist/platform-targets.d.ts.map +1 -1
- package/dist/platform-targets.js +7 -0
- package/dist/workspace-config-policy.d.ts +1 -1
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +1 -1
- package/managed/raw/envrc +38 -1
- package/managed/raw/tooling/direnv/developer-links.ts +201 -0
- package/managed/raw/tooling/direnv/devenv.smoo.nix +75 -51
- package/managed/raw/tooling/direnv/secret-references.ts +3 -1
- package/managed/raw/tooling/direnv/setup-environment.ts +562 -78
- package/managed/raw/tooling/direnv/shared-caches.sh +53 -0
- package/managed/raw/tooling/git-hooks/pre-push.sh +22 -1
- package/managed/templates/github/workflows/managed-files.yml +4 -3
- package/nextest.toml +5 -5
- package/package.json +4 -3
- package/src/__tests__/fixture-nx-env.ts +31 -0
- package/src/__tests__/in-process-nx-index.ts +9 -0
- package/src/bin/smoo-nx-exec.ts +40 -11
- package/src/bin/smoo-nx-nextest-extract.ts +43 -0
- package/src/bounded-test-policy.test.ts +4 -1
- package/src/cargo-closure-input.test.ts +184 -0
- package/src/cargo-closure-input.ts +247 -0
- package/src/cargo-source-hash.test.ts +8 -9
- package/src/cargo-source-hash.ts +146 -47
- package/src/cross-check-policy.ts +15 -3
- package/src/ensure-built.test.ts +65 -5
- package/src/ensure-built.ts +40 -85
- package/src/index.test.ts +222 -75
- package/src/index.ts +213 -101
- package/src/managed-files/ci-workflow.ts +12 -15
- package/src/managed-files/files.ts +10 -0
- package/src/managed-files/generator.test.ts +29 -0
- package/src/managed-files/publish-workflow.ts +17 -6
- package/src/manifest-hash.test.ts +21 -1
- package/src/manifest-hash.ts +26 -29
- package/src/nextest-extraction.test.ts +74 -0
- package/src/nextest-extraction.ts +329 -0
- package/src/package-target-policy.ts +1 -1
- package/src/platform-targets.ts +8 -0
- package/src/workspace-config-policy.ts +6 -1
|
@@ -42,6 +42,19 @@ async function generate(tree: Tree, graph: WorkspaceProjects = projects): Promis
|
|
|
42
42
|
assertNoManagedConflicts(await generateManagedFiles(tree, graph, new Map()));
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
/** Every step of every job in a workflow, as its fields. */
|
|
46
|
+
function workflowSteps(yaml: string): Array<Map<string, unknown>> {
|
|
47
|
+
const workflow: unknown = Bun.YAML.parse(yaml);
|
|
48
|
+
if (typeof workflow !== 'object' || workflow === null || !('jobs' in workflow)) return [];
|
|
49
|
+
const jobs = workflow.jobs;
|
|
50
|
+
if (typeof jobs !== 'object' || jobs === null) return [];
|
|
51
|
+
return Object.values(jobs).flatMap((job: unknown) =>
|
|
52
|
+
typeof job === 'object' && job !== null && 'steps' in job && Array.isArray(job.steps)
|
|
53
|
+
? job.steps.map((step: unknown) => new Map(typeof step === 'object' && step !== null ? Object.entries(step) : []))
|
|
54
|
+
: [],
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
45
58
|
describe('managed-files generator', () => {
|
|
46
59
|
it('generates workflows and packaged bootstrap assets in a virtual workspace', async () => {
|
|
47
60
|
const tree = workspace();
|
|
@@ -65,6 +78,22 @@ describe('managed-files generator', () => {
|
|
|
65
78
|
expect(ci).not.toContain('nixos-latest');
|
|
66
79
|
});
|
|
67
80
|
|
|
81
|
+
// The package policies fail `smoo monorepo check --warn`, so the step must run where a change is
|
|
82
|
+
// judged — pull requests included — while drift healing stays a default-branch concern.
|
|
83
|
+
it('runs the policy check on every CI event and heals drift only from the default branch', async () => {
|
|
84
|
+
const tree = workspace();
|
|
85
|
+
await generate(tree);
|
|
86
|
+
const steps = workflowSteps(tree.read('.github/workflows/ci.yml', 'utf8') ?? '');
|
|
87
|
+
const check = steps.find((step) => step.get('id') === 'managed-drift');
|
|
88
|
+
expect(check?.get('run')).toBe('smoo monorepo check --warn');
|
|
89
|
+
expect(check?.has('if')).toBe(false);
|
|
90
|
+
const dispatch = steps.find((step) => String(step.get('run')).includes('--workflow managed-files.yml'));
|
|
91
|
+
const condition = String(dispatch?.get('if'));
|
|
92
|
+
expect(condition).toContain("github.event_name == 'push'");
|
|
93
|
+
expect(condition).toContain('github.event.repository.default_branch');
|
|
94
|
+
expect(condition).toContain('steps.managed-drift.outputs.drifted');
|
|
95
|
+
});
|
|
96
|
+
|
|
68
97
|
it('discovers publication from workspace packages and does not publish another repository', async () => {
|
|
69
98
|
const tree = workspace();
|
|
70
99
|
updateJson<PackageJson>(tree, 'packages/library/package.json', (pkg) => ({
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
/* biome-ignore-all lint/suspicious/noTemplateCurlyInString: GitHub Actions expressions are emitted literally. */
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
LINUX_PLATFORM_TARGET_GLOBS,
|
|
5
|
+
MACOS_PLATFORM_TARGET_GLOBS,
|
|
6
|
+
RELEASE_CONFIGURATION,
|
|
7
|
+
} from '../platform-targets.js';
|
|
4
8
|
import type {
|
|
5
9
|
PackageCargoCredentialsConfig,
|
|
6
10
|
PackagePrivateNpmConfig,
|
|
@@ -21,6 +25,13 @@ import {
|
|
|
21
25
|
} from './ci-workflow.js';
|
|
22
26
|
import { GITHUB_HOSTED_LINUX_RUNNER, renderRunsOnLine, type WorkflowRunsOn } from './github-runs-on.js';
|
|
23
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Carried by every step that builds what ships: the build aggregate and each
|
|
30
|
+
* platform leg. Lint and unit tests keep the default configuration, cargo's
|
|
31
|
+
* dev profile, like every local and CI run.
|
|
32
|
+
*/
|
|
33
|
+
const RELEASE_BUILD_ARGUMENT = `--configuration ${RELEASE_CONFIGURATION}`;
|
|
34
|
+
|
|
24
35
|
export type PublishWorkflowBump = 'auto' | 'patch' | 'minor' | 'major' | 'prerelease';
|
|
25
36
|
export type PublishWorkflowCondition =
|
|
26
37
|
| 'plan-mode-not-none'
|
|
@@ -610,7 +621,7 @@ function yamlLinesForStep(step: PublishWorkflowStep, options: PublishWorkflowDef
|
|
|
610
621
|
case PublishWorkflowStepKind.Build:
|
|
611
622
|
return conditionalRunStep(
|
|
612
623
|
step,
|
|
613
|
-
`smoo github-ci nx-run-many --targets build --projects "${githubExpression('steps.version.outputs.projects')}"`,
|
|
624
|
+
`smoo github-ci nx-run-many --targets build --projects "${githubExpression('steps.version.outputs.projects')}" ${RELEASE_BUILD_ARGUMENT}`,
|
|
614
625
|
);
|
|
615
626
|
case PublishWorkflowStepKind.Lint:
|
|
616
627
|
return conditionalRunStep(
|
|
@@ -800,7 +811,7 @@ function renderSingleJobPublishWorkflowSteps(
|
|
|
800
811
|
" if: steps.version.outputs.mode != 'none'",
|
|
801
812
|
` run: smoo github-ci nx-run-many --targets "${LINUX_PLATFORM_TARGET_GLOBS.join(',')}" --projects "${githubExpression(
|
|
802
813
|
'steps.version.outputs.projects',
|
|
803
|
-
)}"`,
|
|
814
|
+
)}" ${RELEASE_BUILD_ARGUMENT}`,
|
|
804
815
|
'',
|
|
805
816
|
);
|
|
806
817
|
if (crossOnThisRunner) {
|
|
@@ -809,7 +820,7 @@ function renderSingleJobPublishWorkflowSteps(
|
|
|
809
820
|
" if: steps.version.outputs.mode != 'none'",
|
|
810
821
|
` run: smoo github-ci nx-run-many --targets "${macosPlatformFamilies(options).join(',')}" --projects "${githubExpression(
|
|
811
822
|
'steps.version.outputs.projects',
|
|
812
|
-
)}"`,
|
|
823
|
+
)}" ${RELEASE_BUILD_ARGUMENT}`,
|
|
813
824
|
'',
|
|
814
825
|
);
|
|
815
826
|
}
|
|
@@ -947,7 +958,7 @@ function renderLinuxReleaseCandidateSteps(
|
|
|
947
958
|
' run:',
|
|
948
959
|
` smoo github-ci nx-run-many --targets build --projects "${githubExpression(
|
|
949
960
|
'steps.version.outputs.projects',
|
|
950
|
-
)}" --collect-outputs "${githubExpression('runner.temp')}/release-build-outputs"`,
|
|
961
|
+
)}" ${RELEASE_BUILD_ARGUMENT} --collect-outputs "${githubExpression('runner.temp')}/release-build-outputs"`,
|
|
951
962
|
);
|
|
952
963
|
} else {
|
|
953
964
|
lines.push(...yamlLinesForStep(step, options));
|
|
@@ -974,7 +985,7 @@ function renderLinuxReleaseCandidateSteps(
|
|
|
974
985
|
',',
|
|
975
986
|
)}" --projects "${githubExpression(
|
|
976
987
|
'steps.version.outputs.projects',
|
|
977
|
-
)}" --collect-outputs "${githubExpression('runner.temp')}/linux-platform-outputs"`,
|
|
988
|
+
)}" ${RELEASE_BUILD_ARGUMENT} --collect-outputs "${githubExpression('runner.temp')}/linux-platform-outputs"`,
|
|
978
989
|
);
|
|
979
990
|
stepNumber += 1;
|
|
980
991
|
}
|
|
@@ -2,8 +2,9 @@ import { describe, expect, it } from 'bun:test';
|
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
import { mkdir, mkdtemp, rename, rm, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
|
-
import { dirname, join } from 'node:path';
|
|
5
|
+
import { dirname, join, posix } from 'node:path';
|
|
6
6
|
|
|
7
|
+
import { resetWorkspaceContext, updateFilesInContext } from 'nx/src/utils/workspace-context.js';
|
|
7
8
|
import { hashVersionlessCrateManifests, updateVersionlessCrateManifest } from './manifest-hash.js';
|
|
8
9
|
|
|
9
10
|
const CRATE_MANIFEST = `[package]
|
|
@@ -141,6 +142,20 @@ describe('versionless crate manifest hashing', () => {
|
|
|
141
142
|
});
|
|
142
143
|
});
|
|
143
144
|
|
|
145
|
+
it('hashes only the manifests Nx indexes, never ones under ignored build output', async () => {
|
|
146
|
+
await withProject(async (project) => {
|
|
147
|
+
const baseline = await hashVersionlessCrateManifests(project.root, project.workspaceRoot);
|
|
148
|
+
// A build that copies crates into ignored output (a vendored dist, a
|
|
149
|
+
// cache restore) must neither move the digest nor be read at all: a
|
|
150
|
+
// restore may delete that tree while the digest is computed.
|
|
151
|
+
await project.write('.gitignore', 'generated/\n');
|
|
152
|
+
await project.write('generated/crate/Cargo.toml', CRATE_MANIFEST.replace('1.0.200', '9.9.9'));
|
|
153
|
+
project.refreshIndex();
|
|
154
|
+
|
|
155
|
+
expect(await hashVersionlessCrateManifests(project.root, project.workspaceRoot)).toBe(baseline);
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
|
|
144
159
|
it('hashes a manifest it cannot read instead of dropping it', async () => {
|
|
145
160
|
await withProject(async (project) => {
|
|
146
161
|
await project.write('crates/core/Cargo.toml', 'this is not toml [[[');
|
|
@@ -169,6 +184,8 @@ interface ProjectFixture {
|
|
|
169
184
|
root: string;
|
|
170
185
|
write(path: string, contents: string): Promise<void>;
|
|
171
186
|
move(from: string, to: string): Promise<void>;
|
|
187
|
+
/** Rebuild Nx's file index for the fixture, as a fresh Nx process would. */
|
|
188
|
+
refreshIndex(): void;
|
|
172
189
|
}
|
|
173
190
|
|
|
174
191
|
async function withProject(body: (project: ProjectFixture) => Promise<void>): Promise<void> {
|
|
@@ -191,7 +208,10 @@ async function withProject(body: (project: ProjectFixture) => Promise<void>): Pr
|
|
|
191
208
|
const target = join(workspaceRoot, root, to);
|
|
192
209
|
await mkdir(dirname(target), { recursive: true });
|
|
193
210
|
await rename(join(workspaceRoot, root, from), target);
|
|
211
|
+
// What Nx's file watcher reports to the index for a move.
|
|
212
|
+
updateFilesInContext(workspaceRoot, [posix.join(root, to)], [posix.join(root, from)]);
|
|
194
213
|
},
|
|
214
|
+
refreshIndex: resetWorkspaceContext,
|
|
195
215
|
});
|
|
196
216
|
} finally {
|
|
197
217
|
await rm(workspaceRoot, { recursive: true, force: true });
|
package/src/manifest-hash.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createHash, type Hash } from 'node:crypto';
|
|
2
|
-
import {
|
|
3
|
-
import { join,
|
|
2
|
+
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { join, posix } from 'node:path';
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { globWithWorkspaceContext } from 'nx/src/utils/workspace-context.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* A release rewrites `version` in every manifest it publishes. Those manifests
|
|
@@ -150,41 +150,38 @@ export function updateVersionlessCrateManifest(hash: Hash, manifest: Buffer): vo
|
|
|
150
150
|
* answer: the dependency the digest describes really did change.
|
|
151
151
|
*/
|
|
152
152
|
export async function hashVersionlessCrateManifests(projectRoot: string, workspaceRoot: string): Promise<string> {
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
153
|
+
const manifests = await Promise.all(
|
|
154
|
+
(await listCrateManifests(projectRoot, workspaceRoot)).map(async (path) => ({
|
|
155
|
+
path,
|
|
156
|
+
bytes: await readFile(join(workspaceRoot, projectRoot, path)),
|
|
157
|
+
})),
|
|
158
|
+
);
|
|
158
159
|
const hash = createHash('sha256');
|
|
159
160
|
hash.update(DIGEST_TAG);
|
|
160
161
|
hash.update(SEPARATOR);
|
|
161
|
-
for (
|
|
162
|
-
hash.update(
|
|
162
|
+
for (const { path, bytes } of manifests) {
|
|
163
|
+
hash.update(path);
|
|
163
164
|
hash.update(SEPARATOR);
|
|
164
|
-
updateVersionlessCrateManifest(hash,
|
|
165
|
+
updateVersionlessCrateManifest(hash, bytes);
|
|
165
166
|
hash.update(SEPARATOR);
|
|
166
167
|
}
|
|
167
168
|
return hash.digest('hex');
|
|
168
169
|
}
|
|
169
170
|
|
|
170
|
-
/**
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
} else if (entry.name === CARGO_MANIFEST) {
|
|
185
|
-
found.push(`${prefix}${entry.name}`);
|
|
186
|
-
}
|
|
187
|
-
}
|
|
171
|
+
/**
|
|
172
|
+
* Crate manifest paths relative to the project root, `/`-separated and sorted,
|
|
173
|
+
* so the digest is platform-independent. They come from Nx's own workspace
|
|
174
|
+
* file index — exactly the files the project's `Cargo.toml` exclusion removes
|
|
175
|
+
* from its filesets, with `.gitignore` applied — rather than from a walk of
|
|
176
|
+
* the tree: a walk reads ignored build output, which an Nx cache restore can
|
|
177
|
+
* delete while it is being read. No `catch`: a listing or read that fails
|
|
178
|
+
* would otherwise hash as a smaller manifest set, the silent staleness this
|
|
179
|
+
* module exists to avoid; the caller degrades to the raw manifests instead.
|
|
180
|
+
*/
|
|
181
|
+
async function listCrateManifests(projectRoot: string, workspaceRoot: string): Promise<string[]> {
|
|
182
|
+
const prefix = projectRoot === '.' ? '' : `${posix.normalize(projectRoot)}/`;
|
|
183
|
+
const files = await globWithWorkspaceContext(workspaceRoot, [`${prefix}**/${CARGO_MANIFEST}`]);
|
|
184
|
+
return files.map((file) => file.slice(prefix.length)).sort();
|
|
188
185
|
}
|
|
189
186
|
|
|
190
187
|
/** `from` is just past the `[`. */
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { describe, expect, it } from 'bun:test';
|
|
2
|
+
import { mkdir, mkdtemp, readdir, rm, stat, utimes, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import {
|
|
6
|
+
EXTRACTED_BINARIES_METADATA,
|
|
7
|
+
EXTRACTED_CARGO_METADATA,
|
|
8
|
+
ensureNextestArchiveExtracted,
|
|
9
|
+
type NextestArchiveExtractor,
|
|
10
|
+
type NextestExtractionEvent,
|
|
11
|
+
nextestArchiveKey,
|
|
12
|
+
nextestExtractionPaths,
|
|
13
|
+
} from './nextest-extraction.js';
|
|
14
|
+
|
|
15
|
+
/** What nextest leaves behind: the two metadata files a reuse run reads. */
|
|
16
|
+
const unpack: NextestArchiveExtractor = async (_archive, destination) => {
|
|
17
|
+
await mkdir(join(destination, 'target/nextest'), { recursive: true });
|
|
18
|
+
await writeFile(join(destination, EXTRACTED_BINARIES_METADATA), '{}');
|
|
19
|
+
await writeFile(join(destination, EXTRACTED_CARGO_METADATA), '{}');
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
const FAST = { heartbeatMs: 20, staleAfterMs: 400, pollMs: 10 };
|
|
23
|
+
|
|
24
|
+
describe('ensureNextestArchiveExtracted', () => {
|
|
25
|
+
it('takes over a lock its holder stopped refreshing instead of waiting on it forever', async () => {
|
|
26
|
+
const root = await mkdtemp(join(tmpdir(), 'nextest-extraction-'));
|
|
27
|
+
try {
|
|
28
|
+
const archive = join(root, 'archive.tar.zst');
|
|
29
|
+
await writeFile(archive, 'archive bytes');
|
|
30
|
+
const paths = nextestExtractionPaths(archive, await nextestArchiveKey(archive));
|
|
31
|
+
// A holder killed mid-extraction (a bounded run's SIGKILL) leaves its
|
|
32
|
+
// lock behind, and a pid in it that may since belong to anyone.
|
|
33
|
+
await mkdir(paths.root, { recursive: true });
|
|
34
|
+
await writeFile(paths.lock, `${process.pid} abandoned\n`);
|
|
35
|
+
const minuteAgo = new Date(Date.now() - 60_000);
|
|
36
|
+
await utimes(paths.lock, minuteAgo, minuteAgo);
|
|
37
|
+
|
|
38
|
+
const events: NextestExtractionEvent[] = [];
|
|
39
|
+
const extraction = await ensureNextestArchiveExtracted(archive, unpack, (event) => events.push(event), FAST);
|
|
40
|
+
|
|
41
|
+
expect(extraction).toMatchObject({ kind: 'extracted', directory: paths.directory });
|
|
42
|
+
expect(events).toEqual([{ kind: 'stale-lock', holder: `pid ${process.pid}`, ageMs: expect.any(Number) }]);
|
|
43
|
+
expect(await readdir(paths.root)).toEqual([paths.directory.slice(paths.root.length + 1)]);
|
|
44
|
+
} finally {
|
|
45
|
+
await rm(root, { recursive: true, force: true });
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('publishes nothing when the archive changes while it is being extracted', async () => {
|
|
50
|
+
const root = await mkdtemp(join(tmpdir(), 'nextest-extraction-'));
|
|
51
|
+
try {
|
|
52
|
+
const archive = join(root, 'archive.tar.zst');
|
|
53
|
+
await writeFile(archive, 'archive bytes');
|
|
54
|
+
const paths = nextestExtractionPaths(archive, await nextestArchiveKey(archive));
|
|
55
|
+
// Unpacked from the old bytes, but the directory would be named for
|
|
56
|
+
// them while the path now holds others: a later run hashing the new
|
|
57
|
+
// bytes would never find it, and one hashing the old bytes again — the
|
|
58
|
+
// archive restored from cache — must not find a mixture.
|
|
59
|
+
const rebuiltMidway: NextestArchiveExtractor = async (source, destination) => {
|
|
60
|
+
await unpack(source, destination);
|
|
61
|
+
await writeFile(archive, 'rebuilt archive bytes');
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
await expect(ensureNextestArchiveExtracted(archive, rebuiltMidway, () => {}, FAST)).rejects.toThrow(
|
|
65
|
+
/changed while it was being extracted/,
|
|
66
|
+
);
|
|
67
|
+
// No directory under the old key, no staging left over, no lock held.
|
|
68
|
+
expect(await readdir(paths.root)).toEqual([]);
|
|
69
|
+
await expect(stat(paths.directory)).rejects.toMatchObject({ code: 'ENOENT' });
|
|
70
|
+
} finally {
|
|
71
|
+
await rm(root, { recursive: true, force: true });
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
});
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { once } from 'node:events';
|
|
4
|
+
import { createReadStream } from 'node:fs';
|
|
5
|
+
import { mkdir, mkdtemp, open, readdir, readFile, rename, rm, stat, utimes, writeFile } from 'node:fs/promises';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The helper a per-crate runner calls before `nextest run`, as the root
|
|
11
|
+
* `node_modules` links it. Spelled through the package link rather than this
|
|
12
|
+
* file's real location: under an isolated install that location is a store
|
|
13
|
+
* path outside the workspace, which would differ per machine in a command Nx
|
|
14
|
+
* hashes.
|
|
15
|
+
*/
|
|
16
|
+
export const NEXTEST_EXTRACT_BIN = 'node_modules/@smoothbricks/nx-plugin/dist/bin/smoo-nx-nextest-extract.js';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Where nextest puts things inside an extracted archive, relative to the
|
|
20
|
+
* extraction directory (nextest-runner's `reuse_build` constants). A run given
|
|
21
|
+
* these three paths reuses that build exactly as `--archive-file` would after
|
|
22
|
+
* unpacking it.
|
|
23
|
+
*/
|
|
24
|
+
export const EXTRACTED_TARGET_DIR = 'target';
|
|
25
|
+
export const EXTRACTED_BINARIES_METADATA = 'target/nextest/binaries-metadata.json';
|
|
26
|
+
export const EXTRACTED_CARGO_METADATA = 'target/nextest/cargo-metadata.json';
|
|
27
|
+
|
|
28
|
+
/** Unpacks `archive` into the empty, existing directory `destination`. */
|
|
29
|
+
export type NextestArchiveExtractor = (archive: string, destination: string) => Promise<void>;
|
|
30
|
+
|
|
31
|
+
export type NextestExtraction =
|
|
32
|
+
| { kind: 'reused'; key: string; directory: string }
|
|
33
|
+
| { kind: 'extracted'; key: string; directory: string; elapsedMs: number }
|
|
34
|
+
| { kind: 'awaited'; key: string; directory: string; elapsedMs: number; holder: string };
|
|
35
|
+
|
|
36
|
+
/** What a caller is told while it does not hold the extraction itself. */
|
|
37
|
+
export type NextestExtractionEvent =
|
|
38
|
+
| { kind: 'waiting'; holder: string }
|
|
39
|
+
| { kind: 'stale-lock'; holder: string; ageMs: number };
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* How a lock proves its holder is alive: it rewrites the lock's mtime every
|
|
43
|
+
* `heartbeatMs`, and a lock older than `staleAfterMs` belongs to nobody. A pid
|
|
44
|
+
* cannot answer that — a holder SIGKILLed at its bound leaves a pid the system
|
|
45
|
+
* may already have handed to something else.
|
|
46
|
+
*/
|
|
47
|
+
export interface NextestExtractionTiming {
|
|
48
|
+
heartbeatMs: number;
|
|
49
|
+
staleAfterMs: number;
|
|
50
|
+
pollMs: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const DEFAULT_TIMING: NextestExtractionTiming = { heartbeatMs: 1_000, staleAfterMs: 10_000, pollMs: 100 };
|
|
54
|
+
|
|
55
|
+
/** The SHA-256 of the archive's bytes: the only identity an extraction is ever looked up by. */
|
|
56
|
+
export async function nextestArchiveKey(archive: string): Promise<string> {
|
|
57
|
+
const hash = createHash('sha256');
|
|
58
|
+
for await (const chunk of createReadStream(archive, { highWaterMark: 1 << 20 })) {
|
|
59
|
+
hash.update(chunk);
|
|
60
|
+
}
|
|
61
|
+
return hash.digest('hex');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* One archive path's extractions live beside it, each under the hash of the
|
|
66
|
+
* bytes it came from; the lock and staging directories of a key carry that
|
|
67
|
+
* key as their prefix, so cleanup can tell them from another key's.
|
|
68
|
+
*/
|
|
69
|
+
export interface NextestExtractionPaths {
|
|
70
|
+
root: string;
|
|
71
|
+
directory: string;
|
|
72
|
+
lock: string;
|
|
73
|
+
stagingPrefix: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function nextestExtractionPaths(archive: string, key: string): NextestExtractionPaths {
|
|
77
|
+
const root = `${archive}.extracted`;
|
|
78
|
+
return {
|
|
79
|
+
root,
|
|
80
|
+
directory: join(root, key),
|
|
81
|
+
lock: join(root, `${key}.lock`),
|
|
82
|
+
stagingPrefix: join(root, `${key}.staging-`),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The directory holding `archive`'s extraction, extracting it first if no
|
|
88
|
+
* caller has yet.
|
|
89
|
+
*
|
|
90
|
+
* Correctness does not rest on the lock. An extraction lands under the hash of
|
|
91
|
+
* the bytes it came from by renaming a private staging directory into place,
|
|
92
|
+
* so a directory under a key is a complete extraction of exactly those bytes
|
|
93
|
+
* or it does not exist: concurrent extractors cannot corrupt it, the loser of
|
|
94
|
+
* a race discards its own copy, and an archive rewritten mid-extraction is
|
|
95
|
+
* caught by hashing it again before publishing. The lock only stops every
|
|
96
|
+
* runner of one Nx run from unpacking the same bytes side by side, which is the
|
|
97
|
+
* cost it exists to remove.
|
|
98
|
+
*
|
|
99
|
+
* Publishing a key removes every other key's extraction of this archive path,
|
|
100
|
+
* including stale locks and staging left by a holder that died. Leftovers of
|
|
101
|
+
* the CURRENT key are not touched: a holder that took over a stale lock may
|
|
102
|
+
* still be extracting into one.
|
|
103
|
+
*/
|
|
104
|
+
export async function ensureNextestArchiveExtracted(
|
|
105
|
+
archive: string,
|
|
106
|
+
extract: NextestArchiveExtractor,
|
|
107
|
+
onEvent: (event: NextestExtractionEvent) => void,
|
|
108
|
+
timing: NextestExtractionTiming = DEFAULT_TIMING,
|
|
109
|
+
): Promise<NextestExtraction> {
|
|
110
|
+
const started = performance.now();
|
|
111
|
+
const key = await nextestArchiveKey(archive);
|
|
112
|
+
const paths = nextestExtractionPaths(archive, key);
|
|
113
|
+
if (await isDirectory(paths.directory)) {
|
|
114
|
+
return { kind: 'reused', key, directory: paths.directory };
|
|
115
|
+
}
|
|
116
|
+
await mkdir(paths.root, { recursive: true });
|
|
117
|
+
let holder: string | undefined;
|
|
118
|
+
const settled = (): NextestExtraction =>
|
|
119
|
+
holder === undefined
|
|
120
|
+
? { kind: 'reused', key, directory: paths.directory }
|
|
121
|
+
: { kind: 'awaited', key, directory: paths.directory, elapsedMs: performance.now() - started, holder };
|
|
122
|
+
for (;;) {
|
|
123
|
+
const lock = await acquireLock(paths.lock, timing.heartbeatMs);
|
|
124
|
+
if (lock !== undefined) {
|
|
125
|
+
try {
|
|
126
|
+
if (await isDirectory(paths.directory)) {
|
|
127
|
+
return settled();
|
|
128
|
+
}
|
|
129
|
+
const published = await extractAndPublish(archive, key, paths, extract);
|
|
130
|
+
if (published) {
|
|
131
|
+
await removeOtherKeys(paths.root, key);
|
|
132
|
+
return { kind: 'extracted', key, directory: paths.directory, elapsedMs: performance.now() - started };
|
|
133
|
+
}
|
|
134
|
+
return settled();
|
|
135
|
+
} finally {
|
|
136
|
+
await lock.release();
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (await isDirectory(paths.directory)) {
|
|
140
|
+
return settled();
|
|
141
|
+
}
|
|
142
|
+
const current = await readLock(paths.lock);
|
|
143
|
+
if (current === undefined) {
|
|
144
|
+
// Released between our attempt and this read: try again at once.
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
if (current.ageMs > timing.staleAfterMs) {
|
|
148
|
+
onEvent({ kind: 'stale-lock', holder: current.holder, ageMs: current.ageMs });
|
|
149
|
+
await rm(paths.lock, { force: true });
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (holder === undefined) {
|
|
153
|
+
holder = current.holder;
|
|
154
|
+
onEvent({ kind: 'waiting', holder });
|
|
155
|
+
}
|
|
156
|
+
await sleep(timing.pollMs);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Extract into a private staging directory and rename it to `paths.directory`.
|
|
162
|
+
* False when another extractor published the same key first, which leaves a
|
|
163
|
+
* complete extraction of the same bytes in place.
|
|
164
|
+
*/
|
|
165
|
+
async function extractAndPublish(
|
|
166
|
+
archive: string,
|
|
167
|
+
key: string,
|
|
168
|
+
paths: NextestExtractionPaths,
|
|
169
|
+
extract: NextestArchiveExtractor,
|
|
170
|
+
): Promise<boolean> {
|
|
171
|
+
const staging = await mkdtemp(paths.stagingPrefix);
|
|
172
|
+
try {
|
|
173
|
+
await extract(archive, staging);
|
|
174
|
+
for (const file of [EXTRACTED_BINARIES_METADATA, EXTRACTED_CARGO_METADATA]) {
|
|
175
|
+
if (!(await statOrUndefined(join(staging, file)))?.isFile()) {
|
|
176
|
+
throw new Error(`extracting ${archive} produced no ${file}; a reuse run cannot read this layout`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const after = await nextestArchiveKey(archive);
|
|
180
|
+
if (after !== key) {
|
|
181
|
+
throw new Error(
|
|
182
|
+
`${archive} changed while it was being extracted (sha256 ${key} before, ${after} after); ` +
|
|
183
|
+
'rerun once whatever is rewriting it has finished',
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
await rename(staging, paths.directory);
|
|
187
|
+
return true;
|
|
188
|
+
} catch (error) {
|
|
189
|
+
await rm(staging, { recursive: true, force: true });
|
|
190
|
+
const code = errorCode(error);
|
|
191
|
+
if ((code === 'ENOTEMPTY' || code === 'EEXIST') && (await isDirectory(paths.directory))) {
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
throw error;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
async function removeOtherKeys(root: string, key: string): Promise<void> {
|
|
199
|
+
for (const entry of await readdir(root)) {
|
|
200
|
+
if (entry !== key && !entry.startsWith(`${key}.`)) {
|
|
201
|
+
await rm(join(root, entry), { recursive: true, force: true });
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
interface HeldLock {
|
|
207
|
+
release(): Promise<void>;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Create the lock exclusively, or report it held. A held lock is refreshed until released. */
|
|
211
|
+
async function acquireLock(path: string, heartbeatMs: number): Promise<HeldLock | undefined> {
|
|
212
|
+
const token = `${process.pid} ${randomUUID()}`;
|
|
213
|
+
try {
|
|
214
|
+
await writeFile(path, `${token}\n`, { flag: 'wx' });
|
|
215
|
+
} catch (error) {
|
|
216
|
+
if (errorCode(error) === 'EEXIST') {
|
|
217
|
+
return undefined;
|
|
218
|
+
}
|
|
219
|
+
throw error;
|
|
220
|
+
}
|
|
221
|
+
const heartbeat = setInterval(() => {
|
|
222
|
+
const now = new Date();
|
|
223
|
+
// ENOENT: a waiter judged this lock stale and removed it. The extraction
|
|
224
|
+
// carries on; publishing is atomic whoever else extracts beside it.
|
|
225
|
+
utimes(path, now, now).catch((error: unknown) => {
|
|
226
|
+
if (errorCode(error) !== 'ENOENT') throw error;
|
|
227
|
+
});
|
|
228
|
+
}, heartbeatMs);
|
|
229
|
+
return {
|
|
230
|
+
async release() {
|
|
231
|
+
clearInterval(heartbeat);
|
|
232
|
+
// Remove it only while it is still ours: a waiter that took it over owns it now.
|
|
233
|
+
const content = await readFile(path, 'utf8').catch((error: unknown) => {
|
|
234
|
+
if (errorCode(error) === 'ENOENT') return undefined;
|
|
235
|
+
throw error;
|
|
236
|
+
});
|
|
237
|
+
if (content?.trim() === token) {
|
|
238
|
+
await rm(path, { force: true });
|
|
239
|
+
}
|
|
240
|
+
},
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async function readLock(path: string): Promise<{ holder: string; ageMs: number } | undefined> {
|
|
245
|
+
try {
|
|
246
|
+
const handle = await open(path, 'r');
|
|
247
|
+
try {
|
|
248
|
+
const [content, metadata] = await Promise.all([handle.readFile('utf8'), handle.stat()]);
|
|
249
|
+
const pid = content.trim().split(' ')[0];
|
|
250
|
+
return {
|
|
251
|
+
// Empty while its creator is between creating and writing it.
|
|
252
|
+
holder: pid === undefined || pid.length === 0 ? 'a process that has not yet named itself' : `pid ${pid}`,
|
|
253
|
+
ageMs: Date.now() - metadata.mtimeMs,
|
|
254
|
+
};
|
|
255
|
+
} finally {
|
|
256
|
+
await handle.close();
|
|
257
|
+
}
|
|
258
|
+
} catch (error) {
|
|
259
|
+
if (errorCode(error) === 'ENOENT') {
|
|
260
|
+
return undefined;
|
|
261
|
+
}
|
|
262
|
+
throw error;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* nextest's own extractor: `nextest list --list-type binaries-only` unpacks the
|
|
268
|
+
* archive and executes nothing from it.
|
|
269
|
+
*
|
|
270
|
+
* `list` loads nextest configuration after extracting, and a repository's
|
|
271
|
+
* `.config/nextest.toml` may name test groups only the plugin's tool config
|
|
272
|
+
* defines, so it is given an empty config of its own instead: unpacking bytes
|
|
273
|
+
* does not depend on how tests are scheduled. `--workspace-remap .` because
|
|
274
|
+
* nextest insists the workspace root exist, and the archive's own is the
|
|
275
|
+
* producing tree's path.
|
|
276
|
+
*/
|
|
277
|
+
export const extractWithNextest: NextestArchiveExtractor = async (archive, destination) => {
|
|
278
|
+
const config = join(destination, 'nextest-extract.toml');
|
|
279
|
+
await writeFile(config, '');
|
|
280
|
+
try {
|
|
281
|
+
const args = [
|
|
282
|
+
'--frozen',
|
|
283
|
+
'nextest',
|
|
284
|
+
'list',
|
|
285
|
+
'--archive-file',
|
|
286
|
+
archive,
|
|
287
|
+
'--extract-to',
|
|
288
|
+
destination,
|
|
289
|
+
'--workspace-remap',
|
|
290
|
+
'.',
|
|
291
|
+
'--list-type',
|
|
292
|
+
'binaries-only',
|
|
293
|
+
'--user-config-file',
|
|
294
|
+
'none',
|
|
295
|
+
'--config-file',
|
|
296
|
+
config,
|
|
297
|
+
];
|
|
298
|
+
// The binary list on stdout is not wanted; nextest's progress and errors are on stderr.
|
|
299
|
+
const child = spawn('cargo', args, { stdio: ['ignore', 'ignore', 'inherit'] });
|
|
300
|
+
// Rejects with the spawn error when cargo cannot be started at all.
|
|
301
|
+
const [code, signal]: unknown[] = await once(child, 'exit');
|
|
302
|
+
if (code !== 0) {
|
|
303
|
+
throw new Error(
|
|
304
|
+
`cargo ${args.join(' ')} ${code === null ? `died of ${String(signal)}` : `exited with status ${String(code)}`}`,
|
|
305
|
+
);
|
|
306
|
+
}
|
|
307
|
+
} finally {
|
|
308
|
+
await rm(config, { force: true });
|
|
309
|
+
}
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
async function isDirectory(path: string): Promise<boolean> {
|
|
313
|
+
return (await statOrUndefined(path))?.isDirectory() ?? false;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
async function statOrUndefined(path: string) {
|
|
317
|
+
try {
|
|
318
|
+
return await stat(path);
|
|
319
|
+
} catch (error) {
|
|
320
|
+
if (errorCode(error) === 'ENOENT') {
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
throw error;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function errorCode(error: unknown): unknown {
|
|
328
|
+
return error instanceof Error && 'code' in error ? error.code : undefined;
|
|
329
|
+
}
|
|
@@ -327,7 +327,7 @@ function isRedundantBuildDeclaration(build: Record<string, unknown>, resolvedTar
|
|
|
327
327
|
* Deliberately NOT decided against the resolved project graph. Nx replaces a
|
|
328
328
|
* named property of an inferred target with the declared one, so the resolved
|
|
329
329
|
* `build.dependsOn` IS the declaration: comparing them answered "redundant"
|
|
330
|
-
* for every noop aggregate anyone ever wrote.
|
|
330
|
+
* for every noop aggregate anyone ever wrote. A downstream runtime host lost seven
|
|
331
331
|
* edges that way — four of them targets no inference can reach — and a clean
|
|
332
332
|
* checkout then built it without its producers. The question this has to
|
|
333
333
|
* answer is what inference yields WITHOUT the declaration, so the test is
|
package/src/platform-targets.ts
CHANGED
|
@@ -2,3 +2,11 @@
|
|
|
2
2
|
export const MACOS_PLATFORM_TARGET_GLOBS = ['*-macos', '*-ios'] as const;
|
|
3
3
|
export const LINUX_PLATFORM_TARGET_GLOBS = ['*-linux'] as const;
|
|
4
4
|
export const PLATFORM_TARGET_GLOBS = [...MACOS_PLATFORM_TARGET_GLOBS, ...LINUX_PLATFORM_TARGET_GLOBS] as const;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The Nx configuration that builds what ships. A cargo build target's default
|
|
8
|
+
* options compile cargo's `dev` profile, which every local build, test and
|
|
9
|
+
* gate shares; this configuration adds `--release`. Only release tooling
|
|
10
|
+
* selects it, so a shipped artifact is the only thing compiled twice.
|
|
11
|
+
*/
|
|
12
|
+
export const RELEASE_CONFIGURATION = 'production';
|