@smoothbricks/nx-plugin 0.4.30 → 0.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -10
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +22 -5
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +189 -18
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +3 -2
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/run-cpu-pool.js +63 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -3
- package/managed/raw/tooling/direnv/setup-environment.ts +13 -0
- package/package.json +1 -1
- package/src/cargo-closure-key.test.ts +3 -2
- package/src/executors/bounded-exec/cpu-tokens.test.ts +205 -0
- package/src/executors/bounded-exec/cpu-tokens.ts +229 -22
- package/src/executors/bounded-exec/executor.ts +3 -2
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +76 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +75 -0
- package/src/index.test.ts +12 -2
- package/src/index.ts +13 -3
- package/test-timings.json +1 -0
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { lstat } from 'node:fs/promises';
|
|
2
|
+
import { availableParallelism } from 'node:os';
|
|
3
|
+
import { logger } from 'nx/src/devkit-exports.js';
|
|
4
|
+
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
5
|
+
import { type CpuPool, createCpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
6
|
+
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The CPU budget of an Nx run on a host without a gateway (specs/cowshed/05_gateway.md, "Host CPU budget", "A run
|
|
10
|
+
* without a gateway").
|
|
11
|
+
*
|
|
12
|
+
* Every runner sizes itself to the whole host unless something budgets it, and without a gateway nothing did: under
|
|
13
|
+
* `NX_PARALLEL=100%` Nx starts every target side by side and each one's runner takes the whole machine. The run
|
|
14
|
+
* therefore serves its own pool of one token per core, as a gateway-granted executor serves its children, and every
|
|
15
|
+
* task inherits it: `bounded-exec` and the `cargo` launcher take from it as from any parent pool, FIFO, the head
|
|
16
|
+
* taking `min(want, free)` once one token is free, so the runners of one run together stay within the cores.
|
|
17
|
+
*
|
|
18
|
+
* The run serves it only when nothing else budgets it: it inherits no pool, and the gateway's control socket does
|
|
19
|
+
* not exist. Only `ENOENT` proves the gateway absent, as for the launcher; a socket that exists but does not answer
|
|
20
|
+
* may be a live, saturated gateway, and a run budgeting itself beside it would count the host twice.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** The pools this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
24
|
+
const pools = new Map<string, CpuPool>();
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Serve `context`'s run a pool of `cores` tokens and publish it in `env`, which every task inherits, when the run
|
|
28
|
+
* inherits no pool and no gateway exists at `gateway`. A pool that cannot be made is said and leaves the run
|
|
29
|
+
* unbudgeted, as a run without a gateway was before.
|
|
30
|
+
*/
|
|
31
|
+
export async function serveRunCpuPool(
|
|
32
|
+
context: Pick<PreTasksExecutionContext, 'id'>,
|
|
33
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
34
|
+
gateway: string = GATEWAY_SOCKET,
|
|
35
|
+
cores: number = availableParallelism(),
|
|
36
|
+
): Promise<void> {
|
|
37
|
+
if (readInheritedCpuPool(env) !== null || !(await absent(gateway))) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
// The pool lives until the run's `postTasksExecution` closes it, or with this process.
|
|
41
|
+
const made = await createCpuPool(cores, new Promise<void>(() => {}), { begin() {}, end() {} });
|
|
42
|
+
if (!made.ok) {
|
|
43
|
+
logger.warn(
|
|
44
|
+
`bounded-exec: no gateway at ${gateway}, and this run's CPU pool failed (${made.error}); its tasks run unbudgeted`,
|
|
45
|
+
);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
pools.set(context.id, made.pool);
|
|
49
|
+
for (const [name, value] of Object.entries(made.pool.environment)) {
|
|
50
|
+
env[name] = value;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Close the pool `context`'s run was served, if it was served one. */
|
|
55
|
+
export async function closeRunCpuPool(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void> {
|
|
56
|
+
const pool = pools.get(context.id);
|
|
57
|
+
if (pool === undefined) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
pools.delete(context.id);
|
|
61
|
+
const closed = await pool.close();
|
|
62
|
+
if (!closed.ok) {
|
|
63
|
+
logger.warn(`bounded-exec: this run's CPU pool did not close cleanly: ${closed.error}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function absent(path: string): Promise<boolean> {
|
|
68
|
+
try {
|
|
69
|
+
await lstat(path);
|
|
70
|
+
return false;
|
|
71
|
+
} catch (error) {
|
|
72
|
+
const code = error instanceof Error && 'code' in error ? error.code : undefined;
|
|
73
|
+
return code === 'ENOENT' || code === 'ENOTDIR';
|
|
74
|
+
}
|
|
75
|
+
}
|
package/src/index.test.ts
CHANGED
|
@@ -980,7 +980,12 @@ describe('@smoothbricks/nx-plugin inferred targets', () => {
|
|
|
980
980
|
expect(targets.test?.executor).toBe('nx:noop');
|
|
981
981
|
expect(targets.test?.dependsOn).toEqual(['cargo-test']);
|
|
982
982
|
expect(targets.mutation?.cache).toBe(false);
|
|
983
|
-
|
|
983
|
+
// On bounded-exec, so cargo-mutants' own per-core jobserver is sized to a CPU grant.
|
|
984
|
+
expect(targets.mutation?.executor).toBe('@smoothbricks/nx-plugin:bounded-exec');
|
|
985
|
+
expect(targets.mutation?.options).toMatchObject({
|
|
986
|
+
command: 'cargo --frozen mutants --workspace',
|
|
987
|
+
timeoutMs: 6 * 60 * 60 * 1000,
|
|
988
|
+
});
|
|
984
989
|
expect(targets.bench?.options).toMatchObject({ command: 'cargo --frozen bench --workspace' });
|
|
985
990
|
expectNoSchedulerThrottle(targets);
|
|
986
991
|
} finally {
|
|
@@ -2698,7 +2703,12 @@ describe('@smoothbricks/nx-plugin inferred targets', () => {
|
|
|
2698
2703
|
|
|
2699
2704
|
// `options` merges key by key, so a declared command overrides only itself.
|
|
2700
2705
|
const mutation = resolveDeclaredOverInferred(targets, declared, 'mutation');
|
|
2701
|
-
expect(mutation?.options).toEqual({
|
|
2706
|
+
expect(mutation?.options).toEqual({
|
|
2707
|
+
command: 'cargo mutants --in-diff',
|
|
2708
|
+
cwd: 'packages/ferris',
|
|
2709
|
+
timeoutMs: 6 * 60 * 60 * 1000,
|
|
2710
|
+
});
|
|
2711
|
+
expect(mutation?.executor).toBe('@smoothbricks/nx-plugin:bounded-exec');
|
|
2702
2712
|
expect(mutation?.cache).toBe(false);
|
|
2703
2713
|
|
|
2704
2714
|
const bench = resolveDeclaredOverInferred(targets, declared, 'bench');
|
package/src/index.ts
CHANGED
|
@@ -66,6 +66,7 @@ import {
|
|
|
66
66
|
cargoFrozen,
|
|
67
67
|
} from './cross-check-policy.js';
|
|
68
68
|
import type { EslintFileSet } from './eslint-file-set.js';
|
|
69
|
+
import { closeRunCpuPool, serveRunCpuPool } from './executors/bounded-exec/run-cpu-pool.js';
|
|
69
70
|
import { hashVersionlessCrateManifests } from './manifest-hash.js';
|
|
70
71
|
import {
|
|
71
72
|
EXTRACTED_BINARIES_METADATA,
|
|
@@ -522,6 +523,8 @@ const CARGO_INPUTS = [
|
|
|
522
523
|
// build really reads is the package's `napi` block (binary name, targets) and
|
|
523
524
|
// the `@napi-rs/cli` it runs; those are hashed as such by napiInputs().
|
|
524
525
|
const CARGO_OUTPUT_INPUTS = CARGO_INPUTS;
|
|
526
|
+
/** A full cargo-mutants sweep's wall-clock backstop: a sweep takes hours, and a hung one must still end. */
|
|
527
|
+
const MUTATION_TIMEOUT_MS = 6 * 60 * 60 * 1000;
|
|
525
528
|
// The toolchain half of a cargo task's identity lives in its own module
|
|
526
529
|
// because `smoo monorepo validate` reads the same two constants to refuse a
|
|
527
530
|
// declaration that drops it. See cargo-toolchain-policy.ts for why it is the
|
|
@@ -1087,15 +1090,19 @@ function isBuildOutputPackageJson(packageJsonPath: string): boolean {
|
|
|
1087
1090
|
/**
|
|
1088
1091
|
* Nx accepts one pre/post task-hook pair per plugin. Two independent observers share it: the controller's task-results
|
|
1089
1092
|
* hint and the opt-in producer completion receipts. Each sees the same options and run context; neither reads the
|
|
1090
|
-
* other's output. The hint is recorded first so a producer-receipt bug cannot cost the controller its evidence.
|
|
1093
|
+
* other's output. The hint is recorded first so a producer-receipt bug cannot cost the controller its evidence. The
|
|
1094
|
+
* pair also serves a run on a host without a gateway its own CPU pool, which every task inherits
|
|
1095
|
+
* (`run-cpu-pool.ts`), and closes it when the run ends.
|
|
1091
1096
|
*/
|
|
1092
1097
|
export const preTasksExecution: PreTasksExecution<ProducerReceiptsOptions> = async (options, context) => {
|
|
1093
1098
|
preTaskResults(options, context);
|
|
1094
1099
|
await preProducerReceipts(options, context);
|
|
1100
|
+
await serveRunCpuPool(context);
|
|
1095
1101
|
};
|
|
1096
1102
|
export const postTasksExecution: PostTasksExecution<ProducerReceiptsOptions> = async (options, context) => {
|
|
1097
1103
|
await postTaskResults(options, context);
|
|
1098
1104
|
await postProducerReceipts(options, context);
|
|
1105
|
+
await closeRunCpuPool(context);
|
|
1099
1106
|
};
|
|
1100
1107
|
|
|
1101
1108
|
export default { createNodesV2, postTasksExecution, preTasksExecution };
|
|
@@ -1543,10 +1550,13 @@ async function createProjectTargets(
|
|
|
1543
1550
|
};
|
|
1544
1551
|
// Mutation runs are minutes-to-hours: never cached, never part of build/lint.
|
|
1545
1552
|
// CI runs these per-PR via `cargo mutants --in-diff` (see mutants.toml docs).
|
|
1553
|
+
// bounded-exec sizes cargo-mutants' own jobserver to its CPU grant (its `cargo-mutants` runner kind), which
|
|
1554
|
+
// otherwise starts one task per core whatever else runs on the host. It reports no test bounded-exec can read as
|
|
1555
|
+
// progress, so its bound is a wall-clock backstop, not a performance budget.
|
|
1546
1556
|
targets.mutation = {
|
|
1547
|
-
executor: 'nx:
|
|
1557
|
+
executor: '@smoothbricks/nx-plugin:bounded-exec',
|
|
1548
1558
|
cache: false,
|
|
1549
|
-
options: { command: cargoFrozen('mutants --workspace'), cwd: cargoWorkspaceRoot },
|
|
1559
|
+
options: { command: cargoFrozen('mutants --workspace'), cwd: cargoWorkspaceRoot, timeoutMs: MUTATION_TIMEOUT_MS },
|
|
1550
1560
|
};
|
|
1551
1561
|
targets.bench = {
|
|
1552
1562
|
executor: 'nx:run-commands',
|
package/test-timings.json
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
"src/cargo-closure-key.test.ts": 22410,
|
|
8
8
|
"src/ensure-built.test.ts": 18391,
|
|
9
9
|
"src/graph-input-contract.test.ts": 16550,
|
|
10
|
+
"src/executors/bounded-exec/run-cpu-pool.test.ts": 15709,
|
|
10
11
|
"src/executors/bounded-exec/executor.test.ts": 12975,
|
|
11
12
|
"src/cargo-resolve-cache.test.ts": 11421,
|
|
12
13
|
"src/cargo-closure-input.property.test.ts": 10902,
|