@smoothbricks/nx-plugin 0.4.20 → 0.4.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -18
- package/dist/bin/smoo-nx-cargo-hash.js +32 -4
- package/dist/cargo-source-hash.d.ts +10 -1
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +251 -41
- package/dist/cargo-workspace.d.ts +0 -1
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +9 -2
- package/dist/ensure-built.d.ts +6 -5
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +227 -30
- package/dist/index.js +18 -10
- package/dist/managed-files/ci-workflow.d.ts +20 -0
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +42 -13
- package/dist/managed-files/files.d.ts +1 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +8 -1
- package/dist/managed-files/pr-preview-cleanup-workflow.d.ts +37 -0
- package/dist/managed-files/pr-preview-cleanup-workflow.d.ts.map +1 -1
- package/dist/managed-files/pr-preview-cleanup-workflow.js +34 -6
- package/dist/workspace-manifest.d.ts +3 -1
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/managed/raw/tooling/direnv/setup-environment.ts +23 -91
- package/managed/raw/tooling/git-hooks/commit-msg.sh +6 -0
- package/managed/raw/tooling/git-hooks/pre-commit.sh +6 -0
- package/managed/raw/tooling/git-hooks/pre-push.sh +6 -0
- package/managed/templates/github/actions/setup-devenv/action.yml +19 -7
- package/package.json +1 -1
- package/src/bin/smoo-nx-cargo-hash.ts +29 -5
- package/src/cargo-source-hash.test.ts +291 -1
- package/src/cargo-source-hash.ts +190 -28
- package/src/cargo-workspace.test.ts +17 -10
- package/src/cargo-workspace.ts +9 -3
- package/src/ensure-built.test.ts +87 -29
- package/src/ensure-built.ts +274 -29
- package/src/index.test.ts +117 -20
- package/src/index.ts +18 -11
- package/src/managed-files/ci-workflow.ts +45 -14
- package/src/managed-files/files.ts +9 -2
- package/src/managed-files/generator.test.ts +5 -1
- package/src/managed-files/github-host-cache-isolation.test.ts +32 -6
- package/src/managed-files/pr-preview-cleanup-workflow.ts +87 -6
- package/src/workspace-manifest.ts +3 -1
package/src/cargo-workspace.ts
CHANGED
|
@@ -221,7 +221,6 @@ export interface CargoPackageTestInputsOptions {
|
|
|
221
221
|
/** The cargo workspace root holding the top-level `Cargo.toml`. */
|
|
222
222
|
absoluteProjectRoot: string;
|
|
223
223
|
memberDir: string;
|
|
224
|
-
inputRoot?: string;
|
|
225
224
|
/** Shared across every call of one graph computation; see {@link createCargoInputsCache}. */
|
|
226
225
|
cache?: CargoInputsCache;
|
|
227
226
|
}
|
|
@@ -247,7 +246,7 @@ export function createCargoInputsCache(): CargoInputsCache {
|
|
|
247
246
|
|
|
248
247
|
export function cargoPackageTestInputs(options: CargoPackageTestInputsOptions): Promise<string[]> {
|
|
249
248
|
const cache = options.cache ?? createCargoInputsCache();
|
|
250
|
-
const key = `${options.absoluteProjectRoot}\0${options.memberDir}
|
|
249
|
+
const key = `${options.absoluteProjectRoot}\0${options.memberDir}`;
|
|
251
250
|
const cached = cache.results.get(key);
|
|
252
251
|
if (cached !== undefined) return cached;
|
|
253
252
|
const derived = deriveCargoPackageTestInputs(options, cache);
|
|
@@ -256,9 +255,16 @@ export function cargoPackageTestInputs(options: CargoPackageTestInputsOptions):
|
|
|
256
255
|
}
|
|
257
256
|
|
|
258
257
|
async function deriveCargoPackageTestInputs(
|
|
259
|
-
{ workspaceRoot, absoluteProjectRoot, memberDir
|
|
258
|
+
{ workspaceRoot, absoluteProjectRoot, memberDir }: CargoPackageTestInputsOptions,
|
|
260
259
|
cache: CargoInputsCache,
|
|
261
260
|
): Promise<string[]> {
|
|
261
|
+
// Every fileset is anchored at `{workspaceRoot}`, never `{projectRoot}`: Nx
|
|
262
|
+
// hashes a `{projectRoot}` glob against only the files its project OWNS, and
|
|
263
|
+
// a nested Nx project owns the crates under it. A repository-root cargo
|
|
264
|
+
// workspace whose members live in `packages/*` would then hash none of its
|
|
265
|
+
// members' sources, and its archive would cache-hit against edited crates
|
|
266
|
+
// while every per-crate runner reran on the stale binaries.
|
|
267
|
+
const inputRoot = posix.join('{workspaceRoot}', relative(workspaceRoot, absoluteProjectRoot).split(sep).join('/'));
|
|
262
268
|
const loadManifest = (path: string): Promise<TomlTable> => {
|
|
263
269
|
const normalized = resolve(path);
|
|
264
270
|
const cached = cache.manifests.get(normalized);
|
package/src/ensure-built.test.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
|
|
2
2
|
import { spawn } from 'node:child_process';
|
|
3
|
-
import { existsSync } from 'node:fs';
|
|
4
3
|
import { chmod, mkdir, mkdtemp, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises';
|
|
5
4
|
import { tmpdir } from 'node:os';
|
|
6
5
|
import { dirname, join } from 'node:path';
|
|
@@ -10,13 +9,7 @@ import type { Task } from 'nx/src/config/task-graph';
|
|
|
10
9
|
|
|
11
10
|
import { signalToCode } from 'nx/src/utils/exit-codes';
|
|
12
11
|
|
|
13
|
-
import {
|
|
14
|
-
cliExitOutcome,
|
|
15
|
-
describeMiss,
|
|
16
|
-
firstCacheMiss,
|
|
17
|
-
firstStaleOutputs,
|
|
18
|
-
parseTargetSelector,
|
|
19
|
-
} from './ensure-built.js';
|
|
12
|
+
import { cliExitOutcome, describeMiss, firstCacheMiss, parseTargetSelector, unvouched } from './ensure-built.js';
|
|
20
13
|
|
|
21
14
|
function task(overrides: Partial<Task> & Pick<Task, 'id'>): Task {
|
|
22
15
|
const [project, target] = overrides.id.split(':');
|
|
@@ -98,20 +91,20 @@ describe('firstCacheMiss', () => {
|
|
|
98
91
|
});
|
|
99
92
|
});
|
|
100
93
|
|
|
101
|
-
describe('
|
|
94
|
+
describe('unvouched', () => {
|
|
102
95
|
const tasks = [task({ id: 'lib:build', outputs: ['lib/dist'] }), task({ id: 'app:build', outputs: ['app/dist'] })];
|
|
103
96
|
|
|
104
97
|
it('clears outputs the daemon vouches for', () => {
|
|
105
|
-
expect(
|
|
98
|
+
expect(unvouched(tasks, [true, true])).toEqual([]);
|
|
106
99
|
});
|
|
107
100
|
|
|
108
|
-
it('
|
|
109
|
-
expect(
|
|
101
|
+
it('keeps the tasks whose outputs the daemon does not vouch for', () => {
|
|
102
|
+
expect(unvouched(tasks, [true, false]).map(({ id }) => id)).toEqual(['app:build']);
|
|
110
103
|
});
|
|
111
104
|
|
|
112
|
-
it('treats a missing verdict as
|
|
113
|
-
expect(
|
|
114
|
-
expect(
|
|
105
|
+
it('treats a missing verdict as unvouched rather than as a hit', () => {
|
|
106
|
+
expect(unvouched(tasks, [true]).map(({ id }) => id)).toEqual(['app:build']);
|
|
107
|
+
expect(unvouched(tasks, []).map(({ id }) => id)).toEqual(['lib:build', 'app:build']);
|
|
115
108
|
});
|
|
116
109
|
});
|
|
117
110
|
|
|
@@ -260,15 +253,9 @@ function runBuiltBin(workspace: string, args: readonly string[]): Promise<BinRun
|
|
|
260
253
|
return runBinWith('node', builtBinEntry, workspace, args, {});
|
|
261
254
|
}
|
|
262
255
|
|
|
263
|
-
function nx(workspace: string, args: readonly string[]): Promise<
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
env: fixtureNxEnvironment({ NX_WORKSPACE_ROOT_PATH: workspace }),
|
|
267
|
-
stdio: 'ignore',
|
|
268
|
-
});
|
|
269
|
-
return new Promise((settle, reject) => {
|
|
270
|
-
child.once('error', reject);
|
|
271
|
-
child.once('close', (code) => settle(code));
|
|
256
|
+
function nx(workspace: string, args: readonly string[]): Promise<BinRun> {
|
|
257
|
+
return runBinWith('node', join(repoRoot, 'node_modules', '.bin', 'nx'), workspace, args, {
|
|
258
|
+
NX_WORKSPACE_ROOT_PATH: workspace,
|
|
272
259
|
});
|
|
273
260
|
}
|
|
274
261
|
|
|
@@ -279,6 +266,7 @@ describe('smoo-nx-exec', () => {
|
|
|
279
266
|
// Outside the workspace: a log inside it would be an undeclared output the
|
|
280
267
|
// snapshot diff rightly reports as a changed input.
|
|
281
268
|
const builds = () => `${workspace}-lib-builds.log`;
|
|
269
|
+
const ships = () => `${workspace}-app-ships.log`;
|
|
282
270
|
|
|
283
271
|
beforeAll(async () => {
|
|
284
272
|
// Realpath because macOS puts the temp directory behind a /private
|
|
@@ -343,6 +331,20 @@ describe('smoo-nx-exec', () => {
|
|
|
343
331
|
cache: true,
|
|
344
332
|
options: { command: 'exit 3', cwd: '{projectRoot}' },
|
|
345
333
|
},
|
|
334
|
+
// A command-less aggregate, as Nx normalizes any target that only
|
|
335
|
+
// names dependencies. It declares no `cache`, so it is uncacheable.
|
|
336
|
+
stage: {
|
|
337
|
+
executor: 'nx:noop',
|
|
338
|
+
dependsOn: ['build'],
|
|
339
|
+
},
|
|
340
|
+
// A cacheable leaf that reaches its build only through the aggregate.
|
|
341
|
+
ship: {
|
|
342
|
+
executor: 'nx:run-commands',
|
|
343
|
+
cache: true,
|
|
344
|
+
inputs: ['{projectRoot}/source.txt'],
|
|
345
|
+
dependsOn: ['stage'],
|
|
346
|
+
options: { command: `cat dist/marker.txt >> ${ships()}`, cwd: '{projectRoot}' },
|
|
347
|
+
},
|
|
346
348
|
},
|
|
347
349
|
implicitDependencies: ['lib'],
|
|
348
350
|
}),
|
|
@@ -363,15 +365,22 @@ describe('smoo-nx-exec', () => {
|
|
|
363
365
|
await nx(workspace, ['daemon', '--stop']);
|
|
364
366
|
await rm(workspace, { recursive: true, force: true });
|
|
365
367
|
await rm(builds(), { force: true });
|
|
368
|
+
await rm(ships(), { force: true });
|
|
366
369
|
}
|
|
367
370
|
});
|
|
368
371
|
|
|
369
|
-
it('
|
|
372
|
+
it('reuses the normal Nx CLI cache before executing the binary', async () => {
|
|
373
|
+
const built = await nx(workspace, ['run-many', '-t', 'build', '-p', 'app', '--outputStyle=static-failures-only']);
|
|
374
|
+
expect(built.code, built.stdout + built.stderr).toBe(0);
|
|
375
|
+
expect(await readFile(marker(), 'utf-8')).toBe('built\nlib\n');
|
|
376
|
+
expect(await readFile(builds(), 'utf-8')).toBe('lib\n');
|
|
377
|
+
|
|
370
378
|
const run = await runBin(workspace, ['app:build', '--', './report', 'one']);
|
|
371
379
|
expect(run.code).toBe(0);
|
|
372
|
-
expect(run.
|
|
373
|
-
expect(
|
|
374
|
-
expect(
|
|
380
|
+
expect(run.stderr).toBe('');
|
|
381
|
+
expect(run.stdout.split('\n')[0]).toBe(`${MARKER} one`);
|
|
382
|
+
expect(await readFile(marker(), 'utf-8')).toBe('built\nlib\n');
|
|
383
|
+
expect(await readFile(builds(), 'utf-8')).toBe('lib\n');
|
|
375
384
|
});
|
|
376
385
|
|
|
377
386
|
it('stays completely silent on a full cache hit', async () => {
|
|
@@ -381,6 +390,53 @@ describe('smoo-nx-exec', () => {
|
|
|
381
390
|
// Nothing but the exec'd binary's own output: no Nx banner, no task log.
|
|
382
391
|
expect(run.stdout.split('\n')[0]).toBe(MARKER);
|
|
383
392
|
expect(run.stdout).not.toContain('nx run');
|
|
393
|
+
expect(await readFile(marker(), 'utf-8')).toBe('built\nlib\n');
|
|
394
|
+
expect(await readFile(builds(), 'utf-8')).toBe('lib\n');
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
it('stays silent through an uncacheable nx:noop aggregate, which has no command to run', async () => {
|
|
398
|
+
const first = await runBin(workspace, ['app:ship', '--', './report']);
|
|
399
|
+
expect(first.code, first.stdout + first.stderr).toBe(0);
|
|
400
|
+
expect(await readFile(ships(), 'utf-8')).toBe('built\nlib\n');
|
|
401
|
+
|
|
402
|
+
// Every task but the aggregate is now a recorded success. Treating the
|
|
403
|
+
// aggregate like an uncacheable command hands the whole graph back to Nx,
|
|
404
|
+
// which replays each dependency's cached log on every invocation.
|
|
405
|
+
const again = await runBin(workspace, ['app:ship', '--', './report']);
|
|
406
|
+
expect(again.code).toBe(0);
|
|
407
|
+
expect(again.stderr).toBe('');
|
|
408
|
+
expect(again.stdout.split('\n')[0], again.stdout).toBe(MARKER);
|
|
409
|
+
expect(again.stdout).not.toContain('nx run');
|
|
410
|
+
expect(await readFile(ships(), 'utf-8')).toBe('built\nlib\n');
|
|
411
|
+
expect(await readFile(builds(), 'utf-8')).toBe('lib\n');
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
it('stays silent when the daemon has lost its record of outputs that still match the cache', async () => {
|
|
415
|
+
// The daemon's output records live in memory and are lossy: a restart
|
|
416
|
+
// drops every one, and in a busy workspace a restore's own write events can
|
|
417
|
+
// be processed after Nx's 2 s grace and erase the record just made. Either
|
|
418
|
+
// way the working tree still holds exactly the cached bytes.
|
|
419
|
+
const stopped = await nx(workspace, ['daemon', '--stop']);
|
|
420
|
+
expect(stopped.code, stopped.stdout + stopped.stderr).toBe(0);
|
|
421
|
+
|
|
422
|
+
const run = await runBin(workspace, ['app:build', '--', './report']);
|
|
423
|
+
expect(run.code).toBe(0);
|
|
424
|
+
expect(run.stderr).toBe('');
|
|
425
|
+
expect(run.stdout.split('\n')[0], run.stdout).toBe(MARKER);
|
|
426
|
+
expect(await readFile(builds(), 'utf-8')).toBe('lib\n');
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
it('never vouches for rewritten output bytes the daemon holds no record of', async () => {
|
|
430
|
+
const stopped = await nx(workspace, ['daemon', '--stop']);
|
|
431
|
+
expect(stopped.code, stopped.stdout + stopped.stderr).toBe(0);
|
|
432
|
+
// Same length, different bytes: only a content comparison can tell.
|
|
433
|
+
await writeFile(marker(), 'BUILT\nLIB\n');
|
|
434
|
+
|
|
435
|
+
const run = await runBin(workspace, ['app:build', '--', './report'], { NX_VERBOSE_LOGGING: 'true' });
|
|
436
|
+
expect(run.code).toBe(0);
|
|
437
|
+
expect(run.stderr).toContain('app:build outputs are missing or modified on disk');
|
|
438
|
+
expect(await readFile(marker(), 'utf-8')).toBe('built\nlib\n');
|
|
439
|
+
expect(await readFile(builds(), 'utf-8')).toBe('lib\n');
|
|
384
440
|
});
|
|
385
441
|
|
|
386
442
|
it('runs the emitted wrapper under Node without losing the hot path', async () => {
|
|
@@ -442,12 +498,14 @@ describe('smoo-nx-exec', () => {
|
|
|
442
498
|
});
|
|
443
499
|
|
|
444
500
|
it('runs again once a recorded output is gone from the working tree', async () => {
|
|
501
|
+
const before = await readFile(builds(), 'utf-8');
|
|
445
502
|
await rm(marker());
|
|
446
503
|
const run = await runBin(workspace, ['app:build', '--', './report'], { NX_VERBOSE_LOGGING: 'true' });
|
|
447
504
|
expect(run.code).toBe(0);
|
|
448
505
|
expect(run.stderr).toContain('outputs are missing or modified on disk');
|
|
449
506
|
expect(run.stdout).toContain(MARKER);
|
|
450
|
-
expect(
|
|
507
|
+
expect(await readFile(marker(), 'utf-8')).toBe('built\nlib\n');
|
|
508
|
+
expect(await readFile(builds(), 'utf-8')).toBe(before);
|
|
451
509
|
});
|
|
452
510
|
|
|
453
511
|
it('runs again when a dependency\u0027s own inputs change', async () => {
|
package/src/ensure-built.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
|
-
import {
|
|
3
|
+
import type { BigIntStats } from 'node:fs';
|
|
4
|
+
import { closeSync, existsSync, lstatSync, openSync, readdirSync, readlinkSync, readSync } from 'node:fs';
|
|
4
5
|
import { createRequire } from 'node:module';
|
|
5
|
-
import { join, resolve } from 'node:path';
|
|
6
|
+
import { isAbsolute, join, resolve } from 'node:path';
|
|
6
7
|
|
|
7
8
|
import type * as NxConfiguration from 'nx/src/config/nx-json';
|
|
8
9
|
import type { NxJsonConfiguration } from 'nx/src/config/nx-json';
|
|
@@ -19,6 +20,7 @@ import type * as NxCreateTaskGraph from 'nx/src/tasks-runner/create-task-graph';
|
|
|
19
20
|
import type { TaskResults } from 'nx/src/tasks-runner/life-cycle';
|
|
20
21
|
import type * as NxRunCommand from 'nx/src/tasks-runner/run-command';
|
|
21
22
|
import type * as NxTaskEnv from 'nx/src/tasks-runner/task-env';
|
|
23
|
+
import type * as NxTasksRunnerUtils from 'nx/src/tasks-runner/utils';
|
|
22
24
|
import type * as NxCacheDirectory from 'nx/src/utils/cache-directory';
|
|
23
25
|
import type * as NxCommandLineUtils from 'nx/src/utils/command-line-utils';
|
|
24
26
|
import type { NxArgs } from 'nx/src/utils/command-line-utils';
|
|
@@ -38,6 +40,7 @@ interface NxRuntimeModules {
|
|
|
38
40
|
readonly 'nx/src/tasks-runner/create-task-graph': typeof NxCreateTaskGraph;
|
|
39
41
|
readonly 'nx/src/tasks-runner/run-command': typeof NxRunCommand;
|
|
40
42
|
readonly 'nx/src/tasks-runner/task-env': typeof NxTaskEnv;
|
|
43
|
+
readonly 'nx/src/tasks-runner/utils': typeof NxTasksRunnerUtils;
|
|
41
44
|
readonly 'nx/src/utils/cache-directory': typeof NxCacheDirectory;
|
|
42
45
|
readonly 'nx/src/utils/command-line-utils': typeof NxCommandLineUtils;
|
|
43
46
|
readonly 'nx/src/utils/exit-codes': typeof NxExitCodes;
|
|
@@ -203,19 +206,15 @@ export function firstCacheMiss(
|
|
|
203
206
|
}
|
|
204
207
|
|
|
205
208
|
/**
|
|
206
|
-
*
|
|
209
|
+
* The entries the daemon does not vouch for.
|
|
207
210
|
*
|
|
208
|
-
* `
|
|
209
|
-
* ragged array
|
|
210
|
-
* positive one, and being wrong here means exec'ing a binary
|
|
211
|
+
* `verdicts[index]` is the daemon's verdict for `entries[index]`. A short or
|
|
212
|
+
* ragged array leaves the remainder unvouched rather than cleared: a missing
|
|
213
|
+
* verdict is not a positive one, and being wrong here means exec'ing a binary
|
|
214
|
+
* that was deleted.
|
|
211
215
|
*/
|
|
212
|
-
export function
|
|
213
|
-
|
|
214
|
-
if (matches[index] !== true) {
|
|
215
|
-
return { kind: 'stale-outputs', taskId: tasks[index].id };
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
return null;
|
|
216
|
+
export function unvouched<Entry>(entries: readonly Entry[], verdicts: readonly boolean[]): Entry[] {
|
|
217
|
+
return entries.filter((_, index) => verdicts[index] !== true);
|
|
219
218
|
}
|
|
220
219
|
|
|
221
220
|
/**
|
|
@@ -366,7 +365,7 @@ async function ensureBuiltInWorkspace(
|
|
|
366
365
|
} else if (inputsChanged) {
|
|
367
366
|
reason = { kind: 'stale-inputs', taskId: selectorTaskId };
|
|
368
367
|
} else {
|
|
369
|
-
reason = await probe(nxJson, nxArgs, projectGraph, taskGraph, tasks, requireNx);
|
|
368
|
+
reason = await probe(workspaceRoot, nxJson, nxArgs, projectGraph, taskGraph, tasks, requireNx);
|
|
370
369
|
}
|
|
371
370
|
performance.measure('ensureBuilt:probe', 'ensureBuilt:probe:start');
|
|
372
371
|
const outcome =
|
|
@@ -472,6 +471,7 @@ async function refreshWorkspaceContext(
|
|
|
472
471
|
|
|
473
472
|
/** `null` when the target is already built; otherwise the first reason it is not. */
|
|
474
473
|
async function probe(
|
|
474
|
+
workspaceRoot: string,
|
|
475
475
|
nxJson: NxJsonConfiguration,
|
|
476
476
|
nxArgs: NxArgs,
|
|
477
477
|
projectGraph: ProjectGraph,
|
|
@@ -485,6 +485,7 @@ async function probe(
|
|
|
485
485
|
const { getTaskSpecificEnv } = requireNx('nx/src/tasks-runner/task-env');
|
|
486
486
|
const { getCache } = requireNx('nx/src/tasks-runner/cache');
|
|
487
487
|
const { getRunnerOptions } = requireNx('nx/src/tasks-runner/run-command');
|
|
488
|
+
const { getExecutorNameForTask } = requireNx('nx/src/tasks-runner/utils');
|
|
488
489
|
|
|
489
490
|
// `isCloudDefault: false` because these options are only read here by the
|
|
490
491
|
// hasher, which looks at `selectivelyHashTsConfig`; the cloud credentials the
|
|
@@ -512,20 +513,29 @@ async function probe(
|
|
|
512
513
|
await hashTasks(hasher, projectGraph, taskGraph, perTaskEnvs, getTaskDetails(), tasks);
|
|
513
514
|
performance.measure('ensureBuilt:hash', 'ensureBuilt:hash:start');
|
|
514
515
|
|
|
516
|
+
// An `nx:noop` task has no command: Nx completes it without spawning
|
|
517
|
+
// anything, and normalizes a command-less target that only names dependencies
|
|
518
|
+
// to one. A cache record would vouch for no work — the tasks it aggregates are
|
|
519
|
+
// in this graph and answer for themselves — so none is required. Demanding one
|
|
520
|
+
// made every call through an uncacheable aggregate hand the whole graph to Nx,
|
|
521
|
+
// which replays each dependency's cached log. Outputs a noop declares are
|
|
522
|
+
// still verified below, with every other task's.
|
|
523
|
+
const recordedTasks = tasks.filter((task) => getExecutorNameForTask(task, projectGraph) !== 'nx:noop');
|
|
524
|
+
|
|
515
525
|
// Nx's own factory, but deliberately never `init()`ed: that is what makes
|
|
516
526
|
// this a local-only, side-effect-free question. `init()` attaches the remote
|
|
517
527
|
// cache — a network round-trip, and a download is precisely the work this
|
|
518
528
|
// probe exists to detect — and asserts that the cache directory matches the
|
|
519
|
-
// database. Neither matters here:
|
|
520
|
-
//
|
|
529
|
+
// database. Neither matters here: the directory is read only to compare
|
|
530
|
+
// outputs against, and an artifact missing from it is a miss.
|
|
521
531
|
performance.mark('ensureBuilt:cache:start');
|
|
522
532
|
const cache = getCache(runnerOptions);
|
|
523
|
-
const cachedResults = await cache.getBatch(
|
|
533
|
+
const cachedResults = await cache.getBatch(recordedTasks.filter((task) => task.cache && task.hash));
|
|
524
534
|
const cachedCodeByHash = new Map<string, number>();
|
|
525
535
|
for (const [hash, result] of cachedResults) {
|
|
526
536
|
cachedCodeByHash.set(hash, result.code);
|
|
527
537
|
}
|
|
528
|
-
const cacheMiss = firstCacheMiss(
|
|
538
|
+
const cacheMiss = firstCacheMiss(recordedTasks, cachedCodeByHash);
|
|
529
539
|
performance.measure('ensureBuilt:cache', 'ensureBuilt:cache:start');
|
|
530
540
|
if (cacheMiss) {
|
|
531
541
|
return cacheMiss;
|
|
@@ -534,22 +544,257 @@ async function probe(
|
|
|
534
544
|
// A cache record only proves the task once succeeded with these inputs. The
|
|
535
545
|
// artifact this facility is about — the binary that is about to be exec'd —
|
|
536
546
|
// lives in the working tree, where anything may have deleted or rewritten it
|
|
537
|
-
// since.
|
|
538
|
-
//
|
|
539
|
-
const
|
|
540
|
-
const outputEntries: { outputs: string[]; hash: string }[] = [];
|
|
547
|
+
// since. The daemon answers that cheaply from the output hashes it recorded
|
|
548
|
+
// when Nx last wrote them.
|
|
549
|
+
const claims: OutputClaim[] = [];
|
|
541
550
|
for (const task of tasks) {
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
continue;
|
|
551
|
+
if (task.outputs.length > 0 && task.hash !== undefined) {
|
|
552
|
+
claims.push({ taskId: task.id, outputs: task.outputs, hash: task.hash });
|
|
545
553
|
}
|
|
546
|
-
withOutputs.push(task);
|
|
547
|
-
outputEntries.push({ outputs: task.outputs, hash });
|
|
548
554
|
}
|
|
549
555
|
performance.mark('ensureBuilt:outputs:start');
|
|
550
|
-
const
|
|
556
|
+
const verdicts = await daemonClient.outputsHashesMatchBatch(claims.map(({ outputs, hash }) => ({ outputs, hash })));
|
|
551
557
|
performance.measure('ensureBuilt:outputs', 'ensureBuilt:outputs:start');
|
|
552
|
-
|
|
558
|
+
const doubted = unvouched(claims, verdicts);
|
|
559
|
+
if (doubted.length === 0) {
|
|
560
|
+
return null;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// Its records are lossy, though, and a lost record is not a changed output.
|
|
564
|
+
// They live in memory, so a restart drops every one. They are kept per
|
|
565
|
+
// collapsed directory (Nx tracks at most three paths per level), so a write
|
|
566
|
+
// anywhere under `packages/` voids a task whose outputs span four projects.
|
|
567
|
+
// And a write the daemon processes more than 2 s after a record erases it,
|
|
568
|
+
// which a daemon busy hashing does to a restore's own writes — so each
|
|
569
|
+
// restore guarantees the next one. What settles the question is what a hit
|
|
570
|
+
// would restore: Nx's own local artifact for this exact hash. A working tree
|
|
571
|
+
// that already holds every entry of it is what the restore would leave.
|
|
572
|
+
// Anything else — no artifact, a difference, an error while reading — stays
|
|
573
|
+
// a miss.
|
|
574
|
+
performance.mark('ensureBuilt:artifacts:start');
|
|
575
|
+
const { expandOutputs } = requireNx('nx/src/native');
|
|
576
|
+
const comparison: ArtifactComparison = {
|
|
577
|
+
expandOutputs,
|
|
578
|
+
left: Buffer.allocUnsafe(COMPARE_CHUNK_BYTES),
|
|
579
|
+
right: Buffer.allocUnsafe(COMPARE_CHUNK_BYTES),
|
|
580
|
+
observed: [],
|
|
581
|
+
realDirectories: new Set(),
|
|
582
|
+
};
|
|
583
|
+
for (const claim of doubted) {
|
|
584
|
+
// `recordedTasks` excludes `nx:noop`, so a noop's declared outputs have no
|
|
585
|
+
// artifact here and stay a miss.
|
|
586
|
+
const artifact = cachedResults.get(claim.hash);
|
|
587
|
+
if (artifact === undefined || !outputsMatchArtifact(comparison, workspaceRoot, artifact.outputsPath, claim)) {
|
|
588
|
+
return { kind: 'stale-outputs', taskId: claim.taskId };
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
// A write that landed while the bytes were being compared would otherwise be
|
|
592
|
+
// recorded as current.
|
|
593
|
+
const movedBeforeRecord = firstMovedClaim(comparison.observed);
|
|
594
|
+
if (movedBeforeRecord !== null) {
|
|
595
|
+
return { kind: 'stale-outputs', taskId: movedBeforeRecord };
|
|
596
|
+
}
|
|
597
|
+
// Re-arm the daemon the way Nx's runner does after restoring outputs, so the
|
|
598
|
+
// next call is back on the cheap path.
|
|
599
|
+
await daemonClient.recordOutputsHashBatch(doubted.map(({ outputs, hash }) => ({ outputs, hash })));
|
|
600
|
+
const movedDuringRecord = firstMovedClaim(comparison.observed);
|
|
601
|
+
performance.measure('ensureBuilt:artifacts', 'ensureBuilt:artifacts:start');
|
|
602
|
+
return movedDuringRecord === null ? null : { kind: 'stale-outputs', taskId: movedDuringRecord };
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/** A task's declared outputs, resolved to workspace-relative paths, and the hash they were cached under. */
|
|
606
|
+
interface OutputClaim {
|
|
607
|
+
readonly taskId: string;
|
|
608
|
+
readonly outputs: string[];
|
|
609
|
+
readonly hash: string;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/** A working-tree node as it was when compared, so a later write to it can be noticed. */
|
|
613
|
+
interface ObservedNode {
|
|
614
|
+
readonly taskId: string;
|
|
615
|
+
readonly path: string;
|
|
616
|
+
readonly stats: BigIntStats;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
interface ArtifactComparison {
|
|
620
|
+
readonly expandOutputs: typeof NxNative.expandOutputs;
|
|
621
|
+
readonly left: Buffer;
|
|
622
|
+
readonly right: Buffer;
|
|
623
|
+
readonly observed: ObservedNode[];
|
|
624
|
+
/** Workspace directories already proven real (not symlinks) on the way to an entry. */
|
|
625
|
+
readonly realDirectories: Set<string>;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
/** Read size for byte comparison: large enough to stream a binary quickly, small enough never to load one whole. */
|
|
629
|
+
const COMPARE_CHUNK_BYTES = 1 << 20;
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Whether restoring `claim` from the artifact would leave the working tree as
|
|
633
|
+
* it is.
|
|
634
|
+
*
|
|
635
|
+
* The entries are the ones Nx's restore copies: `expandOutputs` over the
|
|
636
|
+
* artifact root with the task's outputs, globs and negations included. That
|
|
637
|
+
* walk is bounded by the artifact, which holds nothing but outputs, and it
|
|
638
|
+
* needs no reading of the pattern here. A restore replaces each entry whole
|
|
639
|
+
* and touches nothing else, so a working-tree file that matches a glob but is
|
|
640
|
+
* absent from the artifact survives it and is no difference. An output path
|
|
641
|
+
* Nx would refuse to restore (absolute, or climbing out with `..`), an
|
|
642
|
+
* artifact holding no entry at all, and system errors while reading are all
|
|
643
|
+
* misses.
|
|
644
|
+
*/
|
|
645
|
+
function outputsMatchArtifact(
|
|
646
|
+
comparison: ArtifactComparison,
|
|
647
|
+
workspaceRoot: string,
|
|
648
|
+
artifactRoot: string,
|
|
649
|
+
claim: OutputClaim,
|
|
650
|
+
): boolean {
|
|
651
|
+
if (claim.outputs.some((output) => isAbsolute(output) || output.split('/').includes('..'))) {
|
|
652
|
+
return false;
|
|
653
|
+
}
|
|
654
|
+
try {
|
|
655
|
+
const entries = comparison.expandOutputs(artifactRoot, claim.outputs);
|
|
656
|
+
return (
|
|
657
|
+
entries.length > 0 &&
|
|
658
|
+
entries.every(
|
|
659
|
+
(entry) =>
|
|
660
|
+
throughRealDirectories(comparison.realDirectories, workspaceRoot, entry) &&
|
|
661
|
+
sameTree(comparison, claim.taskId, join(workspaceRoot, entry), join(artifactRoot, entry)),
|
|
662
|
+
)
|
|
663
|
+
);
|
|
664
|
+
} catch (error) {
|
|
665
|
+
if (isSystemError(error)) {
|
|
666
|
+
return false;
|
|
667
|
+
}
|
|
668
|
+
throw error;
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
/**
|
|
673
|
+
* Whether every directory between the workspace root and `entry` is a real
|
|
674
|
+
* directory. A restore realizes a symlinked parent as a directory, so reading
|
|
675
|
+
* an entry through one would compare bytes the restore does not leave there —
|
|
676
|
+
* and would read the filesystem behind a link.
|
|
677
|
+
*/
|
|
678
|
+
function throughRealDirectories(realDirectories: Set<string>, workspaceRoot: string, entry: string): boolean {
|
|
679
|
+
let directory = workspaceRoot;
|
|
680
|
+
const segments = entry.split('/');
|
|
681
|
+
for (const segment of segments.slice(0, -1)) {
|
|
682
|
+
directory = join(directory, segment);
|
|
683
|
+
if (realDirectories.has(directory)) {
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
if (lstatSync(directory, { throwIfNoEntry: false })?.isDirectory() !== true) {
|
|
687
|
+
return false;
|
|
688
|
+
}
|
|
689
|
+
realDirectories.add(directory);
|
|
690
|
+
}
|
|
691
|
+
return true;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
/**
|
|
695
|
+
* Whether `actual` is `expected` as a restore would recreate it: same node
|
|
696
|
+
* type; a symlink by its text, never followed, because the link is what Nx
|
|
697
|
+
* stores and restores; a file by permission bits and bytes; a directory by its
|
|
698
|
+
* entry names and each entry in turn. Every working-tree node compared is
|
|
699
|
+
* observed, so a write during the comparison can be caught afterwards.
|
|
700
|
+
*/
|
|
701
|
+
function sameTree(comparison: ArtifactComparison, taskId: string, actual: string, expected: string): boolean {
|
|
702
|
+
const actualStats = lstatSync(actual, { bigint: true, throwIfNoEntry: false });
|
|
703
|
+
const expectedStats = lstatSync(expected, { bigint: true, throwIfNoEntry: false });
|
|
704
|
+
if (actualStats === undefined || expectedStats === undefined) {
|
|
705
|
+
return false;
|
|
706
|
+
}
|
|
707
|
+
comparison.observed.push({ taskId, path: actual, stats: actualStats });
|
|
708
|
+
if (expectedStats.isSymbolicLink()) {
|
|
709
|
+
return actualStats.isSymbolicLink() && readlinkSync(actual) === readlinkSync(expected);
|
|
710
|
+
}
|
|
711
|
+
if (expectedStats.isFile()) {
|
|
712
|
+
return (
|
|
713
|
+
actualStats.isFile() &&
|
|
714
|
+
actualStats.size === expectedStats.size &&
|
|
715
|
+
(actualStats.mode & 0o7777n) === (expectedStats.mode & 0o7777n) &&
|
|
716
|
+
sameBytes(comparison, actual, expected, Number(expectedStats.size))
|
|
717
|
+
);
|
|
718
|
+
}
|
|
719
|
+
if (expectedStats.isDirectory()) {
|
|
720
|
+
if (!actualStats.isDirectory()) {
|
|
721
|
+
return false;
|
|
722
|
+
}
|
|
723
|
+
const names = readdirSync(actual).sort();
|
|
724
|
+
const expectedNames = readdirSync(expected).sort();
|
|
725
|
+
if (names.length !== expectedNames.length) {
|
|
726
|
+
return false;
|
|
727
|
+
}
|
|
728
|
+
for (let index = 0; index < names.length; index += 1) {
|
|
729
|
+
if (names[index] !== expectedNames[index]) {
|
|
730
|
+
return false;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
return names.every((name) => sameTree(comparison, taskId, join(actual, name), join(expected, name)));
|
|
734
|
+
}
|
|
735
|
+
return false;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/** Byte equality of two files already known to be `size` bytes, streamed through the comparison's buffers. */
|
|
739
|
+
function sameBytes(comparison: ArtifactComparison, actual: string, expected: string, size: number): boolean {
|
|
740
|
+
const actualFd = openSync(actual, 'r');
|
|
741
|
+
try {
|
|
742
|
+
const expectedFd = openSync(expected, 'r');
|
|
743
|
+
try {
|
|
744
|
+
for (let offset = 0; offset < size; offset += COMPARE_CHUNK_BYTES) {
|
|
745
|
+
const length = Math.min(COMPARE_CHUNK_BYTES, size - offset);
|
|
746
|
+
// A short read means the file changed length since it was sized.
|
|
747
|
+
if (
|
|
748
|
+
readSync(actualFd, comparison.left, 0, length, offset) !== length ||
|
|
749
|
+
readSync(expectedFd, comparison.right, 0, length, offset) !== length ||
|
|
750
|
+
comparison.left.compare(comparison.right, 0, length, 0, length) !== 0
|
|
751
|
+
) {
|
|
752
|
+
return false;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return true;
|
|
756
|
+
} finally {
|
|
757
|
+
closeSync(expectedFd);
|
|
758
|
+
}
|
|
759
|
+
} finally {
|
|
760
|
+
closeSync(actualFd);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
/**
|
|
765
|
+
* The task owning the first observed node that is no longer the node it was
|
|
766
|
+
* compared as: replaced (device, inode), rewritten (size, mtime) or otherwise
|
|
767
|
+
* touched (mode, ctime).
|
|
768
|
+
*/
|
|
769
|
+
function firstMovedClaim(observed: readonly ObservedNode[]): string | null {
|
|
770
|
+
for (const { taskId, path, stats } of observed) {
|
|
771
|
+
let current: BigIntStats | undefined;
|
|
772
|
+
try {
|
|
773
|
+
current = lstatSync(path, { bigint: true, throwIfNoEntry: false });
|
|
774
|
+
} catch (error) {
|
|
775
|
+
if (isSystemError(error)) {
|
|
776
|
+
return taskId;
|
|
777
|
+
}
|
|
778
|
+
throw error;
|
|
779
|
+
}
|
|
780
|
+
if (
|
|
781
|
+
current === undefined ||
|
|
782
|
+
current.dev !== stats.dev ||
|
|
783
|
+
current.ino !== stats.ino ||
|
|
784
|
+
current.mode !== stats.mode ||
|
|
785
|
+
current.size !== stats.size ||
|
|
786
|
+
current.mtimeNs !== stats.mtimeNs ||
|
|
787
|
+
current.ctimeNs !== stats.ctimeNs
|
|
788
|
+
) {
|
|
789
|
+
return taskId;
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
return null;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/** An operating-system or native-binding failure, as opposed to a programming error. */
|
|
796
|
+
function isSystemError(error: unknown): boolean {
|
|
797
|
+
return error instanceof Error && 'code' in error && typeof error.code === 'string';
|
|
553
798
|
}
|
|
554
799
|
|
|
555
800
|
interface CompletedRun {
|