@smoothbricks/nx-plugin 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -35
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +14 -9
- package/dist/executors/bounded-exec/cpu-pool.d.ts +14 -7
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +297 -38
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +19 -7
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-tokens.js +21 -7
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +12 -6
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +5 -4
- package/dist/producer-receipts.d.ts +10 -0
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +30 -5
- package/managed/raw/tooling/direnv/devenv.smoo.nix +1 -3
- package/managed/raw/tooling/direnv/linked-snapshots.ts +18 -8
- package/managed/raw/tooling/direnv/repo-path +1 -1
- package/package.json +1 -1
- package/src/__tests__/counted-cargo.ts +1 -1
- package/src/cargo-closure-input.test.ts +1 -1
- package/src/ensure-built.test.ts +39 -0
- package/src/ensure-built.ts +23 -9
- package/src/executors/bounded-exec/cpu-pool.test.ts +132 -4
- package/src/executors/bounded-exec/cpu-pool.ts +300 -46
- package/src/executors/bounded-exec/cpu-tokens.test.ts +60 -15
- package/src/executors/bounded-exec/cpu-tokens.ts +34 -11
- package/src/executors/bounded-exec/executor.test.ts +72 -4
- package/src/executors/bounded-exec/executor.ts +12 -6
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +3 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +5 -4
- package/src/producer-receipts.test.ts +37 -15
- package/src/producer-receipts.ts +29 -5
|
@@ -10,12 +10,13 @@ import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
|
10
10
|
* Every runner sizes itself to the whole host unless something budgets it, and without a gateway nothing did: under
|
|
11
11
|
* `NX_PARALLEL=100%` Nx starts every target side by side and each one's runner takes the whole machine. The run
|
|
12
12
|
* therefore serves its own pool of one token per core, as a gateway-granted executor serves its children, and every
|
|
13
|
-
* task inherits it: `bounded-exec`
|
|
14
|
-
*
|
|
13
|
+
* task inherits it: `bounded-exec` takes from it as from any parent pool, FIFO, the head taking `min(want, free)` once
|
|
14
|
+
* one token is free, and a Cargo that Nx's own `nx:run-commands` starts reads its jobserver fifo through the
|
|
15
|
+
* `CARGO_MAKEFLAGS` the pool exports, so the runners of one run together stay within the cores.
|
|
15
16
|
*
|
|
16
17
|
* The run serves it only when nothing else budgets it: it inherits no pool, and the gateway's control socket does
|
|
17
|
-
* not exist. Only `ENOENT` proves the gateway absent
|
|
18
|
-
*
|
|
18
|
+
* not exist. Only `ENOENT` proves the gateway absent; a socket that exists but does not answer may be a live,
|
|
19
|
+
* saturated gateway, and a run budgeting itself beside it would count the host twice.
|
|
19
20
|
*/
|
|
20
21
|
/** The pools this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
21
22
|
const pools = new Map();
|
|
@@ -7,6 +7,16 @@ export interface ProducerReceiptsOptions {
|
|
|
7
7
|
readonly targets: readonly string[];
|
|
8
8
|
};
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* Where a checkout keeps its producer records, relative to its root: in its own source tree, never in Nx's
|
|
12
|
+
* workspace-data. Under cowshed that directory is build state on the checkout's build volume
|
|
13
|
+
* (specs/cowshed/16_build_volumes.md), and a consumer's read grant of the provider checkout does not reach a volume
|
|
14
|
+
* the checkout only links to (specs/cowshed/04_sandbox.md, "Read grants have one narrow meaning"), so a consumer
|
|
15
|
+
* could never read the receipt it is asked to validate. The directory ignores itself (a `.gitignore` of `*` written
|
|
16
|
+
* with its first record), so in any checkout git never tracks a record and Nx's file walk never hashes one: a record
|
|
17
|
+
* is no source or input of the run that publishes it.
|
|
18
|
+
*/
|
|
19
|
+
export declare const PRODUCER_RECORDS: string;
|
|
10
20
|
export declare function completedProducerTasks(graph: TaskGraph, results: PostTasksExecutionContext['taskResults']): ReceiptEvidence<readonly Task[]>;
|
|
11
21
|
export declare function producerEnvironmentIdentity(inputs: EnsureBuiltProducerReceipt['inputs']): string;
|
|
12
22
|
export declare function preTasksExecution(options: ProducerReceiptsOptions | undefined, context: PreTasksExecutionContext): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"producer-receipts.d.ts","sourceRoot":"","sources":["../src/producer-receipts.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,MAAM,yCAAyC,CAAC;AAEnH,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,kBAAkB,EAIxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAKL,KAAK,eAAe,EAErB,MAAM,2BAA2B,CAAC;AAUnC,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,gBAAgB,CAAC,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAC;CACrE;
|
|
1
|
+
{"version":3,"file":"producer-receipts.d.ts","sourceRoot":"","sources":["../src/producer-receipts.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,MAAM,yCAAyC,CAAC;AAEnH,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,kBAAkB,EAIxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAKL,KAAK,eAAe,EAErB,MAAM,2BAA2B,CAAC;AAUnC,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,gBAAgB,CAAC,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAC;CACrE;AACD;;;;;;;;GAQG;AACH,eAAO,MAAM,gBAAgB,QAAsC,CAAC;AA2QpE,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,SAAS,EAChB,OAAO,EAAE,yBAAyB,CAAC,aAAa,CAAC,GAChD,eAAe,CAAC,SAAS,IAAI,EAAE,CAAC,CAqBlC;AACD,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,0BAA0B,CAAC,QAAQ,CAAC,GAAG,MAAM,CAShG;AACD,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,uBAAuB,GAAG,SAAS,EAC5C,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC,CAyCf;AACD,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,uBAAuB,GAAG,SAAS,EAC5C,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CAoIf;AACD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC,kBAAkB,CAAC,CAOrG"}
|
|
@@ -7,6 +7,16 @@ import { createWorkspaceNxRequire, parseTargetSelector, providerInputProof, sele
|
|
|
7
7
|
import { captureProducerSource, producerOutputPaths, providerReceipt, receiptNode, } from './ensure-built-receipt.js';
|
|
8
8
|
import { censusStart, graphInstallationIdentity, movedNodes, workspaceSnapshot, } from './ensure-built-witness.js';
|
|
9
9
|
import { graphInputContractIdentity } from './graph-input-contract.js';
|
|
10
|
+
/**
|
|
11
|
+
* Where a checkout keeps its producer records, relative to its root: in its own source tree, never in Nx's
|
|
12
|
+
* workspace-data. Under cowshed that directory is build state on the checkout's build volume
|
|
13
|
+
* (specs/cowshed/16_build_volumes.md), and a consumer's read grant of the provider checkout does not reach a volume
|
|
14
|
+
* the checkout only links to (specs/cowshed/04_sandbox.md, "Read grants have one narrow meaning"), so a consumer
|
|
15
|
+
* could never read the receipt it is asked to validate. The directory ignores itself (a `.gitignore` of `*` written
|
|
16
|
+
* with its first record), so in any checkout git never tracks a record and Nx's file walk never hashes one: a record
|
|
17
|
+
* is no source or input of the run that publishes it.
|
|
18
|
+
*/
|
|
19
|
+
export const PRODUCER_RECORDS = join('.cache', 'producer-receipts');
|
|
10
20
|
const runs = new Map();
|
|
11
21
|
function sha(value) {
|
|
12
22
|
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
@@ -19,13 +29,15 @@ function configured(options) {
|
|
|
19
29
|
* its products never replaces the receipt an earlier run published: readers validate that receipt on its own evidence.
|
|
20
30
|
*/
|
|
21
31
|
function recordPath(root, target, disposition) {
|
|
22
|
-
|
|
23
|
-
return join(load('nx/src/utils/cache-directory').workspaceDataDirectoryForWorkspace(root), `producer-${disposition === 'published' ? 'receipt' : 'refusal'}-${sha(target)}.json`);
|
|
32
|
+
return join(root, PRODUCER_RECORDS, `producer-${disposition === 'published' ? 'receipt' : 'refusal'}-${sha(target)}.json`);
|
|
24
33
|
}
|
|
25
34
|
function publish(root, target, record) {
|
|
26
35
|
try {
|
|
27
36
|
const path = recordPath(root, target, record.disposition);
|
|
28
|
-
|
|
37
|
+
const records = dirname(path);
|
|
38
|
+
mkdirSync(records, { recursive: true });
|
|
39
|
+
if (!existsSync(join(records, '.gitignore')))
|
|
40
|
+
writeFileSync(join(records, '.gitignore'), '*\n');
|
|
29
41
|
const temporary = `${path}.${randomUUID()}`;
|
|
30
42
|
try {
|
|
31
43
|
writeFileSync(temporary, JSON.stringify(record), { flag: 'wx', mode: 0o600 });
|
|
@@ -443,6 +455,19 @@ function latestRefusal(root, target) {
|
|
|
443
455
|
}
|
|
444
456
|
}
|
|
445
457
|
function validatedReceipt(root, target) {
|
|
458
|
+
const path = recordPath(root, target, 'published');
|
|
459
|
+
let text;
|
|
460
|
+
try {
|
|
461
|
+
text = readFileSync(path, 'utf8');
|
|
462
|
+
}
|
|
463
|
+
catch (error) {
|
|
464
|
+
// No receipt is a producer that never published one; any other failure is a receipt this reader cannot reach.
|
|
465
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
466
|
+
return { ok: false, detail: `No producer completion receipt at ${path}` };
|
|
467
|
+
if (error instanceof Error)
|
|
468
|
+
return { ok: false, detail: `Cannot read producer completion receipt: ${error.message}` };
|
|
469
|
+
throw error;
|
|
470
|
+
}
|
|
446
471
|
try {
|
|
447
472
|
const record = (() => {
|
|
448
473
|
const _ip0 = (input, _vctx = {}) => 1 === input["version"];
|
|
@@ -863,9 +888,9 @@ function validatedReceipt(root, target) {
|
|
|
863
888
|
})();
|
|
864
889
|
const __is = (input, _vctx = {}) => "object" === typeof input && null !== input && _io0(input, _vctx);
|
|
865
890
|
return input => { input = JSON.parse(input); return __is(input) ? input : null; };
|
|
866
|
-
})()(
|
|
891
|
+
})()(text);
|
|
867
892
|
if (record === null || record.version !== 1)
|
|
868
|
-
return { ok: false, detail:
|
|
893
|
+
return { ok: false, detail: `Corrupt producer completion contract at ${path}` };
|
|
869
894
|
const receipt = record.receipt;
|
|
870
895
|
if (receipt.workspaceRoot !== realpathSync(root) ||
|
|
871
896
|
receipt.producer.version !== 1 ||
|
|
@@ -537,9 +537,7 @@ in {
|
|
|
537
537
|
# use, so a hook and a shell resolve one binary the same way. The Rust
|
|
538
538
|
# toolchain's store bin then goes ahead of all of it (rustToolchainBin
|
|
539
539
|
# above): the same files the profile links, named by a path no checkout
|
|
540
|
-
# owns.
|
|
541
|
-
# `cargo` link is the jobserver launcher, so the budget reaches builds
|
|
542
|
-
# no Nx executor started.
|
|
540
|
+
# owns.
|
|
543
541
|
# 3. ttsc drives the native TypeScript 7 binary while Nx imports the TypeScript
|
|
544
542
|
# 6 API, so the two must be named separately.
|
|
545
543
|
# 4. ttsc caches in the checkout's .cache/ttsc (the path the CI ttsc-plugin
|
|
@@ -23,7 +23,8 @@
|
|
|
23
23
|
* names: the checkout's own build, never a consumer override;
|
|
24
24
|
* - each such package's `node_modules` links. A link into the provider's own
|
|
25
25
|
* `node_modules/.bun` store copies that store entry whole; a link into Bun's
|
|
26
|
-
* global cache
|
|
26
|
+
* global cache, as `bun pm cache` names it in the consumer checkout, stays an
|
|
27
|
+
* absolute link to it. Any other target refuses.
|
|
27
28
|
*
|
|
28
29
|
* Provenance is the provider's own Nx producer receipt. Every declared target
|
|
29
30
|
* must be a probe hit (the read-only `probeBuilt`, which runs and
|
|
@@ -156,6 +157,8 @@ type PlanNode =
|
|
|
156
157
|
interface SnapshotPlan {
|
|
157
158
|
readonly name: string;
|
|
158
159
|
readonly root: string;
|
|
160
|
+
/** Bun's immutable install cache the plan's links were classified against (`bunCacheRoot`). */
|
|
161
|
+
readonly cache: string;
|
|
159
162
|
readonly packagePath: string;
|
|
160
163
|
readonly nodes: ReadonlyMap<string, PlanNode>;
|
|
161
164
|
}
|
|
@@ -184,7 +187,7 @@ export async function snapshotBinding(
|
|
|
184
187
|
): Promise<SnapshotBinding> {
|
|
185
188
|
try {
|
|
186
189
|
const store = snapshotStore(root);
|
|
187
|
-
const plan = planSnapshot(source, name, declaration);
|
|
190
|
+
const plan = planSnapshot(source, name, declaration, bunCacheRoot(root));
|
|
188
191
|
const id = await capture(store, plan, declaration, probe, fingerprintOf(plan));
|
|
189
192
|
return { kind: 'snapshot', id, directory: path.join(store, id, plan.packagePath) };
|
|
190
193
|
} catch (error) {
|
|
@@ -236,11 +239,10 @@ export function pluginProbe(from: string): ProducerProbe {
|
|
|
236
239
|
};
|
|
237
240
|
}
|
|
238
241
|
|
|
239
|
-
/** What a snapshot of `source` would hold, read from the provider as it is now. */
|
|
240
|
-
function planSnapshot(source: string, name: string, declaration: SnapshotDeclaration): SnapshotPlan {
|
|
242
|
+
/** What a snapshot of `source` would hold, read from the provider as it is now, with `cache` as Bun's cache. */
|
|
243
|
+
function planSnapshot(source: string, name: string, declaration: SnapshotDeclaration, cache: string): SnapshotPlan {
|
|
241
244
|
const directory = realpathSync(source);
|
|
242
245
|
const root = providerRoot(directory);
|
|
243
|
-
const cache = bunCacheRoot(root);
|
|
244
246
|
const nodes = new Map<string, PlanNode>();
|
|
245
247
|
const pending = [directory];
|
|
246
248
|
const packages = new Map<string, Record<string, unknown>>();
|
|
@@ -307,7 +309,7 @@ function planSnapshot(source: string, name: string, declaration: SnapshotDeclara
|
|
|
307
309
|
addFile(path.join(root, 'package.json'), 'provider');
|
|
308
310
|
for (const specifier of declaration.artifactModules) addFile(nativeArtifact(specifier, directory), 'provider');
|
|
309
311
|
for (const [at, manifest] of packages) assertExportsPublished(at, manifest, root, nodes);
|
|
310
|
-
return { name, root, packagePath: path.relative(root, directory), nodes };
|
|
312
|
+
return { name, root, cache, packagePath: path.relative(root, directory), nodes };
|
|
311
313
|
}
|
|
312
314
|
|
|
313
315
|
/**
|
|
@@ -356,7 +358,12 @@ function providerRoot(directory: string): string {
|
|
|
356
358
|
}
|
|
357
359
|
}
|
|
358
360
|
|
|
359
|
-
/**
|
|
361
|
+
/**
|
|
362
|
+
* Bun's immutable install cache as Bun names it in the consumer checkout `root`. Never in the provider: `bun pm`
|
|
363
|
+
* opens its root manifest for writing just to print a path, and a sandboxed consumer may only read its provider (a
|
|
364
|
+
* cowshed workspace's sibling grant). Every checkout reaches the cache through one literal path (smoothbricks
|
|
365
|
+
* specs/cowshed/03_caches.md, "Link-target caches"), so a provider link into another cache refuses as outside it.
|
|
366
|
+
*/
|
|
360
367
|
function bunCacheRoot(root: string): string {
|
|
361
368
|
const run = spawnSync('bun', ['pm', 'cache'], { cwd: root, encoding: 'utf8' });
|
|
362
369
|
if (run.error !== undefined || run.status !== 0) {
|
|
@@ -560,7 +567,10 @@ async function capture(
|
|
|
560
567
|
attest(relative, bytes, digest, executable, tracked, outputs, plan, provenance);
|
|
561
568
|
census.push({ path: relative, type: 'file', executable, sha256: digest });
|
|
562
569
|
}
|
|
563
|
-
if (
|
|
570
|
+
if (
|
|
571
|
+
fingerprintOf(planSnapshot(path.join(plan.root, plan.packagePath), plan.name, declaration, plan.cache)) !==
|
|
572
|
+
fingerprint
|
|
573
|
+
) {
|
|
564
574
|
throw new SnapshotRefusal(
|
|
565
575
|
`${plan.root} changed while it was copied; nothing was published, so retry once it is quiet`,
|
|
566
576
|
);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
|
-
# Single source of truth for repo-local PATH and installed
|
|
2
|
+
# Single source of truth for repo-local PATH and the installed cowshed's place in it.
|
|
3
3
|
# All tooling that sets up PATH must call this instead of hardcoding paths.
|
|
4
4
|
#
|
|
5
5
|
# Usage:
|
package/package.json
CHANGED
|
@@ -558,7 +558,7 @@ async function shim(
|
|
|
558
558
|
join(bin, 'cargo'),
|
|
559
559
|
[
|
|
560
560
|
'#!/bin/sh',
|
|
561
|
-
// Delegate with the caller's original search path:
|
|
561
|
+
// Delegate with the caller's original search path: nothing the real Cargo starts may resolve this shim again.
|
|
562
562
|
`export PATH=${quote(path)}`,
|
|
563
563
|
`real=${quote(real)}`,
|
|
564
564
|
`if [ "$1" != metadata ]; then ${forward}; fi`,
|
|
@@ -254,7 +254,7 @@ function leafControls(workspace: string, edit: (path: string, text: string) => P
|
|
|
254
254
|
};
|
|
255
255
|
}
|
|
256
256
|
|
|
257
|
-
it(
|
|
257
|
+
it("delegates counted Cargo to the caller's own cargo without re-entering its broker", async () => {
|
|
258
258
|
const root = await realpath(await mkdtemp(join(tmpdir(), 'cargo-counted-delegate-')));
|
|
259
259
|
let fixtureCargo: CountedCargo | undefined;
|
|
260
260
|
try {
|
package/src/ensure-built.test.ts
CHANGED
|
@@ -1191,6 +1191,45 @@ await new Promise<void>((resolve, reject) => process.stdout.write(JSON.stringify
|
|
|
1191
1191
|
}
|
|
1192
1192
|
});
|
|
1193
1193
|
|
|
1194
|
+
// A sandboxed consumer is granted its provider read-only, and starting a daemon writes the provider's workspace
|
|
1195
|
+
// data (`.nx/workspace-data/d`): the read-only probe connects to a serving daemon or reports that none serves.
|
|
1196
|
+
it('reports no-daemon and writes nothing when no daemon serves a provider without a receipt', async () => {
|
|
1197
|
+
await withNxFixture(
|
|
1198
|
+
'ensure-built-probe-no-daemon-',
|
|
1199
|
+
async ({ root, workspace, socketDirectory }) => {
|
|
1200
|
+
await writeTwoProjectWorkspace(workspace, join(root, 'lib-builds.log'), true);
|
|
1201
|
+
const listing = async () =>
|
|
1202
|
+
(await readdir(workspace, { recursive: true })).filter((entry) => !entry.startsWith('node_modules')).sort();
|
|
1203
|
+
const before = await listing();
|
|
1204
|
+
const probeEntry = join(root, 'probe-no-daemon.ts');
|
|
1205
|
+
await writeFile(
|
|
1206
|
+
probeEntry,
|
|
1207
|
+
`
|
|
1208
|
+
import { probeBuilt } from ${JSON.stringify(join(packageRoot, 'dist', 'ensure-built.js'))};
|
|
1209
|
+
const result = await probeBuilt({ target: 'app:build', cwd: ${JSON.stringify(workspace)} });
|
|
1210
|
+
await new Promise<void>((resolve, reject) => process.stdout.write(JSON.stringify(result) + "\\n", error => error ? reject(error) : resolve()));
|
|
1211
|
+
`,
|
|
1212
|
+
);
|
|
1213
|
+
// Run from the fixture root, outside the provider workspace, as a consumer calls it.
|
|
1214
|
+
const probed = await run('bun', probeEntry, root, [], { NX_DAEMON: 'true', NX_SOCKET_DIR: socketDirectory });
|
|
1215
|
+
expect(probed.code, probed.stdout + probed.stderr).toBe(0);
|
|
1216
|
+
const result = typia.json.assertParse<EnsureBuiltProbeResult>(probed.stdout);
|
|
1217
|
+
expect(result).toMatchObject({
|
|
1218
|
+
disposition: 'failed',
|
|
1219
|
+
reason: { kind: 'provider-proof', taskId: 'app:build' },
|
|
1220
|
+
});
|
|
1221
|
+
if (result.disposition !== 'failed' || result.reason.kind !== 'provider-proof') {
|
|
1222
|
+
throw new Error(`the probe must refuse with its provider proof, not ${probed.stdout}`);
|
|
1223
|
+
}
|
|
1224
|
+
expect(result.reason.detail).toEndWith(
|
|
1225
|
+
`authoritative graph unavailable: ${describeMiss({ kind: 'no-daemon', taskId: 'app:build' })}`,
|
|
1226
|
+
);
|
|
1227
|
+
expect(await listing()).toEqual(before);
|
|
1228
|
+
},
|
|
1229
|
+
'workspace',
|
|
1230
|
+
);
|
|
1231
|
+
});
|
|
1232
|
+
|
|
1194
1233
|
// The graph/exec case runs real Nx because its subject is the daemon:
|
|
1195
1234
|
// the daemon a probe starts holds no output records, so every output must be
|
|
1196
1235
|
// vouched for by comparing the working tree with Nx's cache artifact. The
|
package/src/ensure-built.ts
CHANGED
|
@@ -302,7 +302,7 @@ export function describeMiss(reason: MissReason): string {
|
|
|
302
302
|
case 'cache-disabled':
|
|
303
303
|
return `the Nx cache is disabled, so ${reason.taskId} must run`;
|
|
304
304
|
case 'no-daemon':
|
|
305
|
-
return `
|
|
305
|
+
return `no Nx daemon serves this workspace, so ${reason.taskId} cannot be checked without running it`;
|
|
306
306
|
}
|
|
307
307
|
}
|
|
308
308
|
|
|
@@ -735,23 +735,36 @@ type PreparedWorkspaceProbe =
|
|
|
735
735
|
| ReadyWorkspaceProbe
|
|
736
736
|
| { readonly kind: 'no-daemon'; readonly reason: Extract<MissReason, { kind: 'no-daemon' }> };
|
|
737
737
|
|
|
738
|
+
/**
|
|
739
|
+
* The two operations a probe is prepared for. `build` may start the workspace's Nx daemon, as `nx` itself does.
|
|
740
|
+
* `read-only` only connects to a daemon already serving the workspace, and asks it for the authoritative graph:
|
|
741
|
+
* starting one writes the workspace's data directory (`.nx/workspace-data/d`), and a read-only caller may not
|
|
742
|
+
* write it at all — a sandboxed consumer is granted its provider read-only. A daemon that exits between that
|
|
743
|
+
* connection check and the first request is the one window in which Nx's client still starts a successor.
|
|
744
|
+
*/
|
|
745
|
+
type ProbeOperation = 'build' | 'read-only';
|
|
746
|
+
|
|
738
747
|
async function prepareWorkspaceProbe(
|
|
739
748
|
workspaceRoot: string,
|
|
740
749
|
selector: TargetSelector,
|
|
741
|
-
|
|
750
|
+
operation: ProbeOperation,
|
|
742
751
|
): Promise<PreparedWorkspaceProbe> {
|
|
743
752
|
const taskId = selectorTaskId(selector);
|
|
753
|
+
const noDaemon: PreparedWorkspaceProbe = { kind: 'no-daemon', reason: { kind: 'no-daemon', taskId } };
|
|
744
754
|
// This explicit fallback path loads no Nx. The build operation may run the
|
|
745
755
|
// checkout-local CLI; the read-only operation can only report the miss.
|
|
746
756
|
if (process.env.NX_DAEMON === 'false') {
|
|
747
|
-
return
|
|
757
|
+
return noDaemon;
|
|
748
758
|
}
|
|
749
759
|
// Runtime modules always belong to the provider's installed Nx version.
|
|
750
760
|
const requireNx = createWorkspaceNxRequire(workspaceRoot);
|
|
751
761
|
bindWorkspaceRoot(workspaceRoot, requireNx);
|
|
752
762
|
const { daemonClient } = requireNx('nx/src/daemon/client/client');
|
|
753
763
|
if (!daemonClient.enabled()) {
|
|
754
|
-
return
|
|
764
|
+
return noDaemon;
|
|
765
|
+
}
|
|
766
|
+
if (operation === 'read-only' && !(await daemonClient.isServerAvailable())) {
|
|
767
|
+
return noDaemon;
|
|
755
768
|
}
|
|
756
769
|
assignEnvironment(process.env, withoutOuterTaskCacheBypass(process.env));
|
|
757
770
|
|
|
@@ -771,7 +784,7 @@ async function prepareWorkspaceProbe(
|
|
|
771
784
|
setEnvVarsBasedOnArgs(nxArgs, loadDotEnvFiles);
|
|
772
785
|
|
|
773
786
|
performance.mark('ensureBuilt:graph:start');
|
|
774
|
-
const projectGraph = await currentProjectGraph(workspaceRoot, nxJson, requireNx,
|
|
787
|
+
const projectGraph = await currentProjectGraph(workspaceRoot, nxJson, requireNx, operation === 'read-only');
|
|
775
788
|
nxJson = readNxJson();
|
|
776
789
|
requireProject(projectGraph, selector.project);
|
|
777
790
|
const taskGraph = createTaskGraph(
|
|
@@ -819,7 +832,7 @@ async function withTaskHooks<Result>(
|
|
|
819
832
|
}
|
|
820
833
|
|
|
821
834
|
async function ensureBuiltInWorkspace(workspaceRoot: string, selector: TargetSelector): Promise<EnsureBuiltResult> {
|
|
822
|
-
const prepared = await prepareWorkspaceProbe(workspaceRoot, selector);
|
|
835
|
+
const prepared = await prepareWorkspaceProbe(workspaceRoot, selector, 'build');
|
|
823
836
|
if (prepared.kind === 'no-daemon') return runViaCli(workspaceRoot, selector, prepared.reason);
|
|
824
837
|
return withTaskHooks(workspaceRoot, selector, prepared, buildPreparedTarget);
|
|
825
838
|
}
|
|
@@ -842,9 +855,10 @@ async function probeBuiltInWorkspace(workspaceRoot: string, selector: TargetSele
|
|
|
842
855
|
const taskId = selectorTaskId(selector);
|
|
843
856
|
const receipt = readProducerReceipt(workspaceRoot, taskId);
|
|
844
857
|
if (receipt.ok) return { disposition: 'hit', receipt: receipt.value };
|
|
845
|
-
// A failed publication proof asks Nx for the authoritative
|
|
846
|
-
// never hashes against the consumer environment
|
|
847
|
-
|
|
858
|
+
// A failed publication proof asks a serving Nx daemon for the authoritative
|
|
859
|
+
// generation, but never hashes against the consumer environment, starts a
|
|
860
|
+
// provider runner or starts the provider's daemon.
|
|
861
|
+
const prepared = await prepareWorkspaceProbe(workspaceRoot, selector, 'read-only');
|
|
848
862
|
const detail =
|
|
849
863
|
prepared.kind === 'no-daemon'
|
|
850
864
|
? `${receipt.detail}; authoritative graph unavailable: ${describeMiss(prepared.reason)}`
|
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
import { afterEach, describe, expect, it } from 'bun:test';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
chmodSync,
|
|
4
|
+
closeSync,
|
|
5
|
+
constants,
|
|
6
|
+
existsSync,
|
|
7
|
+
mkdirSync,
|
|
8
|
+
openSync,
|
|
9
|
+
readdirSync,
|
|
10
|
+
readSync,
|
|
11
|
+
rmSync,
|
|
12
|
+
statSync,
|
|
13
|
+
writeSync,
|
|
14
|
+
} from 'node:fs';
|
|
3
15
|
import { writeFile } from 'node:fs/promises';
|
|
4
16
|
import { connect, type Socket } from 'node:net';
|
|
5
17
|
import { dirname, join } from 'node:path';
|
|
@@ -82,11 +94,16 @@ const QUEUED: PoolReply = { ok: true, lease: 'queued' };
|
|
|
82
94
|
const pools: CpuPool[] = [];
|
|
83
95
|
const sockets: Socket[] = [];
|
|
84
96
|
const directories: string[] = [];
|
|
97
|
+
// Descriptors a test opened on a pool's jobserver fifo, as a make-protocol client does.
|
|
98
|
+
const descriptors: number[] = [];
|
|
85
99
|
|
|
86
100
|
afterEach(async () => {
|
|
87
101
|
for (const socket of sockets.splice(0)) {
|
|
88
102
|
socket.destroy();
|
|
89
103
|
}
|
|
104
|
+
for (const fd of descriptors.splice(0)) {
|
|
105
|
+
closeSync(fd);
|
|
106
|
+
}
|
|
90
107
|
for (const pool of pools.splice(0)) {
|
|
91
108
|
// Every pool a test leaves open is closed here, and its outcome is asserted, not dropped.
|
|
92
109
|
expect(await pool.close()).toEqual({ ok: true });
|
|
@@ -220,8 +237,59 @@ async function untilLine(stream: ReadableStream<Uint8Array>, marker: string): Pr
|
|
|
220
237
|
reader.releaseLock();
|
|
221
238
|
}
|
|
222
239
|
|
|
240
|
+
/** The fifo a pool's environment names for Cargo and make, which name the same one. */
|
|
241
|
+
function jobserverFifo(pool: CpuPool): string {
|
|
242
|
+
const flags = pool.environment.CARGO_MAKEFLAGS ?? '';
|
|
243
|
+
expect(pool.environment.MAKEFLAGS).toBe(flags);
|
|
244
|
+
const named = /^-j --jobserver-auth=fifo:(\/\S+)$/.exec(flags)?.[1];
|
|
245
|
+
if (named === undefined) {
|
|
246
|
+
throw new Error(`the pool names no jobserver fifo: ${JSON.stringify(pool.environment)}`);
|
|
247
|
+
}
|
|
248
|
+
return named;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* A make-protocol client of `fifo`, opened read-write as Cargo opens it, and non-blocking here so that `take` reads
|
|
253
|
+
* what the fifo holds now and an empty fifo reads 0.
|
|
254
|
+
*/
|
|
255
|
+
function jobserverClient(fifo: string): { take(most: number): number; give(count: number): void } {
|
|
256
|
+
const fd = openSync(fifo, constants.O_RDWR | constants.O_NONBLOCK);
|
|
257
|
+
descriptors.push(fd);
|
|
258
|
+
return {
|
|
259
|
+
take(most) {
|
|
260
|
+
try {
|
|
261
|
+
return readSync(fd, Buffer.alloc(most), 0, most, null);
|
|
262
|
+
} catch (error) {
|
|
263
|
+
if (error instanceof Error && 'code' in error && error.code === 'EAGAIN') return 0;
|
|
264
|
+
throw error;
|
|
265
|
+
}
|
|
266
|
+
},
|
|
267
|
+
give(count) {
|
|
268
|
+
expect(writeSync(fd, Buffer.alloc(count, '+'))).toBe(count);
|
|
269
|
+
},
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** Wait for `count` token bytes in `fifo` and take them, as a blocked Cargo does; resolves to the count it took. */
|
|
274
|
+
async function awaitTokens(fifo: string, count: number): Promise<number> {
|
|
275
|
+
const fd = openSync(fifo, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
276
|
+
const reader = Bun.file(fd).stream().getReader();
|
|
277
|
+
let taken = 0;
|
|
278
|
+
try {
|
|
279
|
+
while (taken < count) {
|
|
280
|
+
const chunk = await reader.read();
|
|
281
|
+
if (chunk.done) break;
|
|
282
|
+
taken += chunk.value.byteLength;
|
|
283
|
+
}
|
|
284
|
+
} finally {
|
|
285
|
+
await reader.cancel();
|
|
286
|
+
closeSync(fd);
|
|
287
|
+
}
|
|
288
|
+
return taken;
|
|
289
|
+
}
|
|
290
|
+
|
|
223
291
|
describe('the parent-held CPU pool', () => {
|
|
224
|
-
it('serves
|
|
292
|
+
it('serves an allocation as its own live child pool without competing with the occupied ancestor', async () => {
|
|
225
293
|
const { parent, pool } = await openPool(4);
|
|
226
294
|
const cargo = childOf(parent.socket, JSON.stringify({ op: 'cpu-pool', want: 4, children: true }));
|
|
227
295
|
expect(await cargo.next()).toEqual(QUEUED);
|
|
@@ -382,7 +450,7 @@ describe('the parent-held CPU pool', () => {
|
|
|
382
450
|
}
|
|
383
451
|
});
|
|
384
452
|
|
|
385
|
-
it('names
|
|
453
|
+
it('names its endpoint, count and jobserver fifo in private owned paths, and removes them on close', async () => {
|
|
386
454
|
const made = await createCpuPool(3, NEVER, countedWaits());
|
|
387
455
|
if (!made.ok) {
|
|
388
456
|
throw new Error(made.error);
|
|
@@ -396,8 +464,17 @@ describe('the parent-held CPU pool', () => {
|
|
|
396
464
|
expect(made.pool.environment[CPU_TOKENS_ENV]).toBe('3');
|
|
397
465
|
expect(statSync(dirname(inherited.socket)).mode & 0o777).toBe(0o700);
|
|
398
466
|
expect(statSync(inherited.socket).mode & 0o777).toBe(0o600);
|
|
467
|
+
// The jobserver fifo lies beside the socket, as private, and holds the pool's tokens but the implicit first.
|
|
468
|
+
const fifo = jobserverFifo(made.pool);
|
|
469
|
+
expect(dirname(fifo)).toBe(dirname(inherited.socket));
|
|
470
|
+
expect(statSync(fifo).isFIFO()).toBe(true);
|
|
471
|
+
expect(statSync(fifo).mode & 0o777).toBe(0o600);
|
|
472
|
+
const cargo = jobserverClient(fifo);
|
|
473
|
+
expect(cargo.take(3)).toBe(2);
|
|
474
|
+
cargo.give(2);
|
|
399
475
|
expect(await made.pool.close()).toEqual({ ok: true });
|
|
400
476
|
expect(await made.pool.close()).toEqual({ ok: true });
|
|
477
|
+
expect(existsSync(fifo)).toBe(false);
|
|
401
478
|
expect(existsSync(dirname(inherited.socket))).toBe(false);
|
|
402
479
|
});
|
|
403
480
|
|
|
@@ -408,6 +485,8 @@ describe('the parent-held CPU pool', () => {
|
|
|
408
485
|
'{"op":"other","want":1}',
|
|
409
486
|
'{"op":"cpu-pool","want":0}',
|
|
410
487
|
'{"op":"cpu-pool","want":1.5}',
|
|
488
|
+
// A grant never grows, so `grow` is a field like any other this pool does not read.
|
|
489
|
+
'{"op":"cpu-pool","want":1,"grow":true}',
|
|
411
490
|
'x'.repeat(5000),
|
|
412
491
|
'x'.repeat(1024 * 1024),
|
|
413
492
|
];
|
|
@@ -428,7 +507,7 @@ describe('the parent-held CPU pool', () => {
|
|
|
428
507
|
// A wait holds no token: the whole pool is still there to grant.
|
|
429
508
|
expect(await grantedFor(parent, 4)).toBe(4);
|
|
430
509
|
|
|
431
|
-
//
|
|
510
|
+
// A child is served a pool of its own; the test it runs reports its waits there, to the same run.
|
|
432
511
|
const cargo = childOf(parent.socket, JSON.stringify({ op: 'cpu-pool', want: 2, children: true }));
|
|
433
512
|
expect(await cargo.next()).toEqual(QUEUED);
|
|
434
513
|
const granted = await cargo.next();
|
|
@@ -553,4 +632,53 @@ describe('the parent-held CPU pool', () => {
|
|
|
553
632
|
expect(cpuPoolDirectories()).toEqual(before);
|
|
554
633
|
await expect(createCpuPool(0, NEVER, countedWaits())).rejects.toBeInstanceOf(RangeError);
|
|
555
634
|
});
|
|
635
|
+
|
|
636
|
+
it("lends a 1-token pool's implicit slot to a socket child, since its fifo holds no byte", async () => {
|
|
637
|
+
const { pool, parent } = await openPool(1);
|
|
638
|
+
expect(jobserverClient(jobserverFifo(pool)).take(1)).toBe(0);
|
|
639
|
+
// The holder is not using its own token while it waits on this child: lending it is what avoids the deadlock.
|
|
640
|
+
const first = childOf(parent.socket, JSON.stringify({ op: 'cpu-pool', want: 2 }));
|
|
641
|
+
expect(await first.next()).toEqual(QUEUED);
|
|
642
|
+
expect(await first.next()).toEqual({ ok: true, lease: 'granted', tokens: 1 });
|
|
643
|
+
const second = childOf(parent.socket, JSON.stringify({ op: 'cpu-pool', want: 1 }));
|
|
644
|
+
expect(await second.next()).toEqual(QUEUED);
|
|
645
|
+
first.socket.destroy();
|
|
646
|
+
expect(await second.next()).toEqual({ ok: true, lease: 'granted', tokens: 1 });
|
|
647
|
+
});
|
|
648
|
+
|
|
649
|
+
it('grants no byte a fifo reader holds, and wakes its queued head when the reader writes it back', async () => {
|
|
650
|
+
const { pool, parent } = await openPool(2);
|
|
651
|
+
const cargo = jobserverClient(jobserverFifo(pool));
|
|
652
|
+
// Cargo takes the fifo's one byte: only the implicit slot is left to lend.
|
|
653
|
+
expect(cargo.take(1)).toBe(1);
|
|
654
|
+
const first = childOf(parent.socket, JSON.stringify({ op: 'cpu-pool', want: 2 }));
|
|
655
|
+
expect(await first.next()).toEqual(QUEUED);
|
|
656
|
+
expect(await first.next()).toEqual({ ok: true, lease: 'granted', tokens: 1 });
|
|
657
|
+
const second = childOf(parent.socket, JSON.stringify({ op: 'cpu-pool', want: 1 }));
|
|
658
|
+
expect(await second.next()).toEqual(QUEUED);
|
|
659
|
+
const third = childOf(parent.socket, JSON.stringify({ op: 'cpu-pool', want: 1 }));
|
|
660
|
+
expect(await third.next()).toEqual(QUEUED);
|
|
661
|
+
// No socket child releases anything: only the fifo turning readable can wake the head, and it takes the byte.
|
|
662
|
+
cargo.give(1);
|
|
663
|
+
expect(await second.next()).toEqual({ ok: true, lease: 'granted', tokens: 1 });
|
|
664
|
+
// The implicit slot goes back with the first child, to the next in line.
|
|
665
|
+
first.socket.destroy();
|
|
666
|
+
expect(await third.next()).toEqual({ ok: true, lease: 'granted', tokens: 1 });
|
|
667
|
+
});
|
|
668
|
+
|
|
669
|
+
it("writes a released child's bytes back to the fifo, where Cargo and the next child both find them", async () => {
|
|
670
|
+
const { pool, parent } = await openPool(3);
|
|
671
|
+
const fifo = jobserverFifo(pool);
|
|
672
|
+
const cargo = jobserverClient(fifo);
|
|
673
|
+
const holder = await takeOrThrow(parent, 3);
|
|
674
|
+
expect(holder.tokens).toBe(3);
|
|
675
|
+
// The grant took both bytes and the implicit slot: the fifo is empty while it holds them.
|
|
676
|
+
expect(cargo.take(2)).toBe(0);
|
|
677
|
+
holder.release();
|
|
678
|
+
await holder.closed;
|
|
679
|
+
// The pool hears the close after the child does; a waiting reader takes the two bytes it writes back.
|
|
680
|
+
expect(await awaitTokens(fifo, 2)).toBe(2);
|
|
681
|
+
cargo.give(2);
|
|
682
|
+
expect(await grantedFor(parent, 3)).toBe(3);
|
|
683
|
+
});
|
|
556
684
|
});
|