@vzn/vx 0.0.594 → 0.0.596
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +5 -5
- package/src/bin.ts +26 -11
- package/src/cache/archive.ts +2 -2
- package/src/cache/cache.ts +235 -174
- package/src/cache/chained-cache.ts +25 -7
- package/src/cache/config-evals.ts +19 -4
- package/src/cache/git-inputs.ts +19 -8
- package/src/cache/index.ts +1 -0
- package/src/cache/inputs.ts +122 -49
- package/src/cache/layer.ts +7 -5
- package/src/cache/layered-cache.ts +25 -2
- package/src/cache/policy.ts +5 -0
- package/src/cache/tar-stream.ts +1 -1
- package/src/cache/zstd.ts +10 -4
- package/src/cli/cache.ts +11 -15
- package/src/cli/completions.ts +42 -6
- package/src/cli/foreign-flags.ts +26 -5
- package/src/cli/help.ts +5 -3
- package/src/cli/init.ts +1 -1
- package/src/cli/last.ts +4 -1
- package/src/cli/plan-format.ts +9 -0
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +60 -42
- package/src/cli/select.ts +30 -3
- package/src/cli/show.ts +42 -3
- package/src/cli/watch-filter.ts +77 -13
- package/src/cli/watch-fs.ts +15 -7
- package/src/cli/watch-judge.ts +16 -9
- package/src/cli/watch.ts +22 -15
- package/src/config.ts +26 -26
- package/src/exec/env.ts +5 -0
- package/src/exec/index.ts +1 -0
- package/src/exec/proc-sample.ts +6 -2
- package/src/exec/runner.ts +98 -40
- package/src/exec/sandbox-paths.ts +12 -6
- package/src/exec/sandbox-runtime.ts +136 -44
- package/src/exec/sandbox-violations.ts +58 -11
- package/src/graph/index.ts +1 -0
- package/src/graph/scheduler.ts +29 -13
- package/src/graph/task-graph.ts +134 -25
- package/src/index.ts +6 -0
- package/src/orchestrator/admission.ts +12 -3
- package/src/orchestrator/events.ts +10 -1
- package/src/orchestrator/execute-task.ts +65 -20
- package/src/orchestrator/failure-mode.ts +8 -2
- package/src/orchestrator/framed-output.ts +3 -3
- package/src/orchestrator/hit-restore.ts +6 -3
- package/src/orchestrator/local-shortcircuit.ts +4 -1
- package/src/orchestrator/lockfile-claim.ts +43 -46
- package/src/orchestrator/logger.ts +11 -6
- package/src/orchestrator/metrics.ts +19 -8
- package/src/orchestrator/miss-save.ts +2 -2
- package/src/orchestrator/options.ts +7 -0
- package/src/orchestrator/plan.ts +28 -4
- package/src/orchestrator/plugin-host.ts +25 -3
- package/src/orchestrator/plugin.ts +39 -17
- package/src/orchestrator/prepare.ts +13 -4
- package/src/orchestrator/run-artifacts.ts +2 -1
- package/src/orchestrator/run.ts +72 -12
- package/src/orchestrator/sandbox-request.ts +65 -9
- package/src/orchestrator/save-lane.ts +5 -5
- package/src/orchestrator/signals.ts +5 -2
- package/src/orchestrator/summary.ts +16 -5
- package/src/orchestrator/telemetry.ts +16 -7
- package/src/util/index.ts +7 -1
- package/src/util/paths.ts +23 -3
- package/src/util/secret-mask.ts +36 -0
- package/src/util/settle.ts +15 -0
- package/src/workspace/affected.ts +30 -8
- package/src/workspace/config-cache.ts +25 -4
- package/src/workspace/config-imports.ts +8 -6
- package/src/workspace/config-schema.ts +74 -9
- package/src/workspace/filter.ts +19 -0
- package/src/workspace/migrate-scripts.ts +5 -3
- package/src/workspace/migration.ts +1 -1
- package/src/workspace/package-graph.ts +15 -3
- package/src/workspace/project-loader.ts +76 -2
- package/src/workspace/repo-id.ts +42 -4
- package/src/workspace/workspace.ts +98 -20
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
// once. Restore goes to the layer that produced the hit — remembered per
|
|
5
5
|
// hash — because an entry's artifact lives wherever it was found.
|
|
6
6
|
//
|
|
7
|
-
// A layer that throws in a lookup
|
|
7
|
+
// A layer that throws in a lookup (a batch row load or a has-many included)
|
|
8
|
+
// is a miss there and the walk goes on; a throwing drain is passed; one
|
|
8
9
|
// that throws in a save is skipped and the rest still save. Unisolated, a
|
|
9
10
|
// plugin layer's throw ended the walk before the local floor under it: its
|
|
10
11
|
// `get` failed the task, its `save` kept the entry from every later layer
|
|
@@ -122,14 +123,22 @@ export class ChainedCache implements CacheLayer {
|
|
|
122
123
|
const found = await this.ask(
|
|
123
124
|
layer,
|
|
124
125
|
'remoteHasMany',
|
|
125
|
-
|
|
126
|
+
// The shape check and the mark sit inside the ask: a layer that
|
|
127
|
+
// answers a non-Set or throws marking gave no answer.
|
|
128
|
+
async () => {
|
|
129
|
+
const got: unknown = (await layer.remoteHasMany?.(hashes)) ?? null
|
|
130
|
+
if (got === null) return null
|
|
131
|
+
if (!(got instanceof Set)) throw new TypeError('remoteHasMany answered a non-Set')
|
|
132
|
+
const set = got as Set<string>
|
|
133
|
+
layer.markRemoteAbsent?.(hashes.filter((h) => !set.has(h)))
|
|
134
|
+
return set
|
|
135
|
+
},
|
|
126
136
|
null,
|
|
127
137
|
)
|
|
128
138
|
if (found === null) {
|
|
129
139
|
complete = false
|
|
130
140
|
continue
|
|
131
141
|
}
|
|
132
|
-
layer.markRemoteAbsent?.(hashes.filter((h) => !found.has(h)))
|
|
133
142
|
out ??= new Set()
|
|
134
143
|
for (const h of found) out.add(h)
|
|
135
144
|
}
|
|
@@ -142,15 +151,24 @@ export class ChainedCache implements CacheLayer {
|
|
|
142
151
|
}
|
|
143
152
|
|
|
144
153
|
async drainUploads(): Promise<void> {
|
|
145
|
-
await Promise.all(
|
|
154
|
+
await Promise.all(
|
|
155
|
+
this.layers.map((l) =>
|
|
156
|
+
this.ask(l, 'drainUploads', async () => l.drainUploads?.(), undefined),
|
|
157
|
+
),
|
|
158
|
+
)
|
|
146
159
|
}
|
|
147
160
|
|
|
148
161
|
loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
|
|
149
162
|
const out = new Map<string, OutputFileRow[]>()
|
|
150
|
-
for (const layer of this.layers) {
|
|
151
|
-
|
|
152
|
-
|
|
163
|
+
for (const [i, layer] of this.layers.entries()) {
|
|
164
|
+
let rows: Map<string, OutputFileRow[]>
|
|
165
|
+
try {
|
|
166
|
+
rows = layer.loadOutputFilesBatch(hashes)
|
|
167
|
+
} catch (err) {
|
|
168
|
+
this.onLayerError(i, 'loadOutputFilesBatch', err)
|
|
169
|
+
continue
|
|
153
170
|
}
|
|
171
|
+
for (const [h, r] of rows) if (!out.has(h)) out.set(h, r)
|
|
154
172
|
}
|
|
155
173
|
return out
|
|
156
174
|
}
|
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
import type { Database, SQLQueryBindings } from 'bun:sqlite'
|
|
7
7
|
import { lazyStatement } from './schema.js'
|
|
8
8
|
|
|
9
|
+
const CONFIG_EVALS_SWEPT_AT = 'config_evals_swept_at'
|
|
10
|
+
const DAY_MS = 24 * 60 * 60 * 1000
|
|
11
|
+
|
|
9
12
|
export class ConfigEvalTable {
|
|
10
13
|
private readonly selectConfigEval: ReturnType<Database['prepare']>
|
|
11
14
|
private readonly insertConfigEval: ReturnType<Database['prepare']>
|
|
@@ -106,10 +109,22 @@ export class ConfigEvalTable {
|
|
|
106
109
|
/**
|
|
107
110
|
* Retention: a row not WRITTEN since `cutoff`. A hit does not refresh it
|
|
108
111
|
* (that would be a write per config on every warm run), so a config that
|
|
109
|
-
* hit for thirty days is evaluated once more and stored again.
|
|
112
|
+
* hit for thirty days is evaluated once more and stored again. No index
|
|
113
|
+
* covers `created_at`, so each sweep reads every row with its JSON (1–6 ms
|
|
114
|
+
* at 8,000 rows, 2 MB); it runs once a day, on its own clock in
|
|
115
|
+
* `schema_meta`, as the file-hash sweep does.
|
|
110
116
|
*/
|
|
111
|
-
pruneOlderThan(cutoff: number): void {
|
|
112
|
-
|
|
113
|
-
|
|
117
|
+
pruneOlderThan(cutoff: number, now: number = Date.now()): void {
|
|
118
|
+
const last = this.db
|
|
119
|
+
.prepare('SELECT value FROM schema_meta WHERE key = ?')
|
|
120
|
+
.get(CONFIG_EVALS_SWEPT_AT) as { value: string } | null
|
|
121
|
+
if (last !== null && now - Number(last.value) < DAY_MS) return
|
|
122
|
+
this.db.transaction(() => {
|
|
123
|
+
this.db.prepare('DELETE FROM config_evals WHERE created_at < ?').run(cutoff)
|
|
124
|
+
this.db.prepare('DELETE FROM config_closures WHERE created_at < ?').run(cutoff)
|
|
125
|
+
this.db
|
|
126
|
+
.prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
|
|
127
|
+
.run(CONFIG_EVALS_SWEPT_AT, String(now))
|
|
128
|
+
})()
|
|
114
129
|
}
|
|
115
130
|
}
|
package/src/cache/git-inputs.ts
CHANGED
|
@@ -1304,8 +1304,11 @@ export async function startGitEnumeration(
|
|
|
1304
1304
|
const facts = repoFacts(workspaceRoot)
|
|
1305
1305
|
// The blob-size check's verdict is a function of the index's entries, so
|
|
1306
1306
|
// the index file's bytes key it: one read, where `ls-files --debug` and a
|
|
1307
|
-
// lookup per entry cost 550 ms at 100,000 files (A-60).
|
|
1308
|
-
|
|
1307
|
+
// lookup per entry cost 550 ms at 100,000 files (A-60). The verdict's
|
|
1308
|
+
// paths are workspace-relative, so the repo→workspace prefix keys it too:
|
|
1309
|
+
// a nested workspace sharing the cache read the outer one's verdict and
|
|
1310
|
+
// trusted a resized blob.
|
|
1311
|
+
const pathspecKey = xxh3hex([facts?.prefix ?? '', ...pathspecs].join('\0'))
|
|
1309
1312
|
const indexFile =
|
|
1310
1313
|
facts === null || facts.indexFile === ''
|
|
1311
1314
|
? undefined
|
|
@@ -1521,9 +1524,10 @@ export async function applyGitEnumeration(
|
|
|
1521
1524
|
// Sort once, then each project's files are a contiguous range found
|
|
1522
1525
|
// by binary search on its `dir/` prefix — O((F+P) log F) instead of
|
|
1523
1526
|
// the O(P·F) per-project startsWith scan (54 ms at 1090 projects ×
|
|
1524
|
-
// ~9k files; ~5 ms this way).
|
|
1525
|
-
//
|
|
1526
|
-
//
|
|
1527
|
+
// ~9k files; ~5 ms this way). Every path under `dir/` sorts in
|
|
1528
|
+
// [`dir/`, `dir0`): '0' is the code unit after '/'. An upper bound of
|
|
1529
|
+
// `dir/` + U+FFFF left out a file whose name starts with U+FFFF, and
|
|
1530
|
+
// it never entered the key.
|
|
1527
1531
|
// Git lists in order; a list that already is skips the sort.
|
|
1528
1532
|
let inOrder = true
|
|
1529
1533
|
for (let i = 1; i < all.length; i++) {
|
|
@@ -1557,10 +1561,15 @@ export async function applyGitEnumeration(
|
|
|
1557
1561
|
return lo
|
|
1558
1562
|
}
|
|
1559
1563
|
for (const projectDir of projectDirs) {
|
|
1560
|
-
const
|
|
1564
|
+
const spelled =
|
|
1561
1565
|
base !== undefined && projectDir.startsWith(base) && path.normalize(projectDir) === projectDir
|
|
1562
1566
|
? projectDir.slice(base.length).replace(/\/$/, '')
|
|
1563
1567
|
: relPosix(workspaceRoot, projectDir)
|
|
1568
|
+
// macOS git reports paths NFC (core.precomposeunicode) while a dir
|
|
1569
|
+
// discovered by readdir keeps the spelling it was created with: match
|
|
1570
|
+
// NFC, and key the OIDs by the project's spelling, which is how
|
|
1571
|
+
// `resolveFiles` looks them up.
|
|
1572
|
+
const relPrefix = spelled.normalize('NFC')
|
|
1564
1573
|
if (relPrefix === '' || relPrefix === '.') {
|
|
1565
1574
|
cache.set(projectDir, all)
|
|
1566
1575
|
const rootOids = new Map<string, string>()
|
|
@@ -1570,14 +1579,16 @@ export async function applyGitEnumeration(
|
|
|
1570
1579
|
}
|
|
1571
1580
|
const prefix = `${relPrefix}/`
|
|
1572
1581
|
const start = lowerBound(prefix)
|
|
1573
|
-
const end = lowerBound(`${
|
|
1582
|
+
const end = lowerBound(`${relPrefix}0`)
|
|
1574
1583
|
const matches: string[] = []
|
|
1575
1584
|
const projOids = new Map<string, string>()
|
|
1585
|
+
const projAbs =
|
|
1586
|
+
relPrefix === spelled ? abs : (rel: string) => path.join(projectDir, rel.slice(prefix.length))
|
|
1576
1587
|
for (let i = start; i < end; i++) {
|
|
1577
1588
|
const rel = sorted[i]!
|
|
1578
1589
|
matches.push(rel.slice(prefix.length))
|
|
1579
1590
|
const oid = trusted.get(rel)
|
|
1580
|
-
if (oid !== undefined) projOids.set(
|
|
1591
|
+
if (oid !== undefined) projOids.set(projAbs(rel), oid)
|
|
1581
1592
|
}
|
|
1582
1593
|
// An empty slice is a directory git did not see, not an empty project:
|
|
1583
1594
|
// a project has at least its package.json, tracked or untracked. A
|
package/src/cache/index.ts
CHANGED
package/src/cache/inputs.ts
CHANGED
|
@@ -109,10 +109,10 @@ export interface ResolveInputsArgs {
|
|
|
109
109
|
envSource: NodeJS.ProcessEnv
|
|
110
110
|
inputs: CacheInputs | undefined
|
|
111
111
|
/** Project-relative output globs to exclude from inputs. */
|
|
112
|
-
ownOutputs: string[]
|
|
112
|
+
ownOutputs: readonly string[]
|
|
113
113
|
/** Root-relative `outputs.workspaceFiles` globs to exclude from
|
|
114
114
|
* `inputs.workspaceFiles` (a task cannot invalidate itself). */
|
|
115
|
-
ownWorkspaceOutputs?: string[]
|
|
115
|
+
ownWorkspaceOutputs?: readonly string[]
|
|
116
116
|
/** Absolute dirs of nested projects (cross-boundary isolation). */
|
|
117
117
|
nestedProjectDirs: string[]
|
|
118
118
|
/**
|
|
@@ -182,11 +182,14 @@ export type ProjectFilesCache = Map<
|
|
|
182
182
|
>
|
|
183
183
|
|
|
184
184
|
export async function resolveInputs(args: ResolveInputsArgs): Promise<ResolvedInputs> {
|
|
185
|
+
const projectRel = path.relative(args.workspaceRoot, args.projectDir).split(path.sep).join('/')
|
|
185
186
|
const { files: projectFiles, listing } = await resolveFiles({
|
|
186
187
|
projectDir: args.projectDir,
|
|
187
188
|
workspaceRoot: args.workspaceRoot,
|
|
188
189
|
files: args.inputs?.files,
|
|
189
190
|
ownOutputs: args.ownOutputs,
|
|
191
|
+
ownWorkspaceOutputs: args.ownWorkspaceOutputs ?? [],
|
|
192
|
+
projectRel,
|
|
190
193
|
nestedProjectDirs: args.nestedProjectDirs,
|
|
191
194
|
...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
|
|
192
195
|
...(args.projectFilesCache !== undefined ? { projectFilesCache: args.projectFilesCache } : {}),
|
|
@@ -199,6 +202,8 @@ export async function resolveInputs(args: ResolveInputsArgs): Promise<ResolvedIn
|
|
|
199
202
|
workspaceRoot: args.workspaceRoot,
|
|
200
203
|
workspaceFiles: wsDecl,
|
|
201
204
|
ownWorkspaceOutputs: args.ownWorkspaceOutputs ?? [],
|
|
205
|
+
ownOutputs: args.ownOutputs,
|
|
206
|
+
projectRel,
|
|
202
207
|
...(args.gitFilesCache !== undefined ? { gitFilesCache: args.gitFilesCache } : {}),
|
|
203
208
|
...(args.workspaceFilesCache !== undefined ? { memo: args.workspaceFilesCache } : {}),
|
|
204
209
|
})
|
|
@@ -258,6 +263,9 @@ function resolveWorkspaceFiles(args: {
|
|
|
258
263
|
workspaceRoot: string
|
|
259
264
|
workspaceFiles: readonly string[]
|
|
260
265
|
ownWorkspaceOutputs: readonly string[]
|
|
266
|
+
/** The task's project-relative `outputs.files`, and its project's root-relative directory. */
|
|
267
|
+
ownOutputs: readonly string[]
|
|
268
|
+
projectRel: string
|
|
261
269
|
gitFilesCache?: GitFilesCache
|
|
262
270
|
memo?: WorkspaceFilesCache
|
|
263
271
|
}): { files: Promise<string[]>; listing: InputListing } | undefined {
|
|
@@ -274,6 +282,11 @@ function resolveWorkspaceFiles(args: {
|
|
|
274
282
|
// A path the task's own outputs take back with `!` is no output, so it
|
|
275
283
|
// stays an input (A-44).
|
|
276
284
|
const ownOutput = outputMatcher(args.ownWorkspaceOutputs)
|
|
285
|
+
// Its project outputs too, which a root-anchored glob can reach as well:
|
|
286
|
+
// left in, the task's own build moved its key, and no run was ever saved.
|
|
287
|
+
let ownProjectOutput: ((rel: string) => boolean) | undefined
|
|
288
|
+
const ownProject = (rel: string): boolean =>
|
|
289
|
+
(ownProjectOutput ??= underProject(args.projectRel, outputMatcher(args.ownOutputs)))(rel)
|
|
277
290
|
const positiveGlobs = asTrees(positive).map(globFor)
|
|
278
291
|
// Workspace-wide partition, keyed by the workspace root. Populated
|
|
279
292
|
// up-front by `populateGitFilesCache(..., workspaceWide: true)` when
|
|
@@ -293,9 +306,15 @@ function resolveWorkspaceFiles(args: {
|
|
|
293
306
|
const memoKey =
|
|
294
307
|
args.memo === undefined
|
|
295
308
|
? undefined
|
|
296
|
-
: JSON.stringify([
|
|
309
|
+
: JSON.stringify([
|
|
310
|
+
positive,
|
|
311
|
+
negative,
|
|
312
|
+
args.ownWorkspaceOutputs,
|
|
313
|
+
args.ownOutputs,
|
|
314
|
+
args.projectRel,
|
|
315
|
+
])
|
|
297
316
|
let isPositive: ((rel: string) => boolean) | undefined
|
|
298
|
-
const excluded = (rel: string): boolean => isExcluded(rel) || ownOutput(rel)
|
|
317
|
+
const excluded = (rel: string): boolean => isExcluded(rel) || ownOutput(rel) || ownProject(rel)
|
|
299
318
|
const listing: InputListing = {
|
|
300
319
|
root: args.workspaceRoot,
|
|
301
320
|
listed: gitFiles,
|
|
@@ -352,6 +371,24 @@ async function resolveWorkspaceFilesOver(
|
|
|
352
371
|
return candidates.filter((abs) => oids?.has(abs) === true || isInputOnDisk(abs)).sort()
|
|
353
372
|
}
|
|
354
373
|
|
|
374
|
+
/** A project-relative matcher asked of root-relative paths: false outside the project. */
|
|
375
|
+
function underProject(
|
|
376
|
+
projectRel: string,
|
|
377
|
+
matches: (rel: string) => boolean,
|
|
378
|
+
): (rel: string) => boolean {
|
|
379
|
+
if (projectRel === '') return matches
|
|
380
|
+
const prefix = `${projectRel}/`
|
|
381
|
+
return (rel) => rel.startsWith(prefix) && matches(rel.slice(prefix.length))
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** A root-relative matcher asked of project-relative paths. */
|
|
385
|
+
function inProject(
|
|
386
|
+
projectRel: string,
|
|
387
|
+
matches: (rel: string) => boolean,
|
|
388
|
+
): (rel: string) => boolean {
|
|
389
|
+
return projectRel === '' ? matches : (rel) => matches(`${projectRel}/${rel}`)
|
|
390
|
+
}
|
|
391
|
+
|
|
355
392
|
/** Anything at the path — file, directory, symlink to anything or to nothing. */
|
|
356
393
|
function existsOnDisk(abs: string): boolean {
|
|
357
394
|
try {
|
|
@@ -609,7 +646,7 @@ function outputExcludes(outputs: readonly string[]): Bun.Glob[] {
|
|
|
609
646
|
/** Resolve declared output globs (project-relative) to actual produced files. */
|
|
610
647
|
export async function resolveOutputs(args: {
|
|
611
648
|
projectDir: string
|
|
612
|
-
outputs: string[]
|
|
649
|
+
outputs: readonly string[]
|
|
613
650
|
nestedProjectDirs: string[]
|
|
614
651
|
}): Promise<string[]> {
|
|
615
652
|
const { positive, negative } = splitNegations(args.outputs)
|
|
@@ -710,27 +747,15 @@ function isInside(dir: string, abs: string): boolean {
|
|
|
710
747
|
*/
|
|
711
748
|
export async function cleanOutputs(args: {
|
|
712
749
|
projectDir: string
|
|
713
|
-
outputs: string[]
|
|
750
|
+
outputs: readonly string[]
|
|
714
751
|
nestedProjectDirs: string[]
|
|
715
|
-
/**
|
|
716
|
-
* Before a miss: keep the directory each wildcard glob is rooted at
|
|
717
|
-
* (`dist` for `dist/**`). The task writes its matches under it, so a
|
|
718
|
-
* remove there bought only an rmdir and the task's mkdir: 0.4 ms of a
|
|
719
|
-
* 4.9 ms one-file miss (B-49). A restore prunes it, since the entry's
|
|
720
|
-
* shape decides there.
|
|
721
|
-
*/
|
|
722
|
-
keepGlobRoots?: boolean
|
|
723
752
|
}): Promise<string[]> {
|
|
724
753
|
const files = await resolveOutputs(args)
|
|
725
754
|
// `force: true` makes rm tolerate ENOENT (e.g. when two output
|
|
726
755
|
// globs overlap and a sibling already deleted a path mid-iteration).
|
|
727
756
|
// A symlink is unlinked, never followed.
|
|
728
757
|
const removed = await removeAll(files, args.projectDir)
|
|
729
|
-
await pruneEmptiedDirs(
|
|
730
|
-
args.projectDir,
|
|
731
|
-
removed,
|
|
732
|
-
args.keepGlobRoots === true ? globRoots(args.projectDir, args.outputs) : undefined,
|
|
733
|
-
)
|
|
758
|
+
await pruneEmptiedDirs(args.projectDir, removed, pruneScope(args.projectDir, args.outputs))
|
|
734
759
|
// Project-relative posix paths of what was removed — the caller
|
|
735
760
|
// feeds these to GitFilesCache.markOutputsChanged after a restore.
|
|
736
761
|
return files.map((f) => relPosix(args.projectDir, f))
|
|
@@ -740,15 +765,20 @@ export async function cleanOutputs(args: {
|
|
|
740
765
|
* Remove exactly these project-relative paths — the recorded rows of an
|
|
741
766
|
* ADDITIVE task's own artifact (item 588), never a glob: the glob would
|
|
742
767
|
* take the upstream's files the task adds beside. Emptied directories are
|
|
743
|
-
* pruned as `cleanOutputs` prunes them,
|
|
744
|
-
* fills.
|
|
768
|
+
* pruned as `cleanOutputs` prunes them, only inside the declared `outputs`
|
|
769
|
+
* (`pruneScope`), and stop at one the upstream still fills.
|
|
745
770
|
*/
|
|
746
771
|
export async function cleanOutputPaths(args: {
|
|
747
772
|
projectDir: string
|
|
748
773
|
rels: readonly string[]
|
|
774
|
+
outputs: readonly string[]
|
|
749
775
|
}): Promise<void> {
|
|
750
776
|
const files = args.rels.map((r) => path.resolve(args.projectDir, r))
|
|
751
|
-
await pruneEmptiedDirs(
|
|
777
|
+
await pruneEmptiedDirs(
|
|
778
|
+
args.projectDir,
|
|
779
|
+
await removeAll(files, args.projectDir),
|
|
780
|
+
pruneScope(args.projectDir, args.outputs),
|
|
781
|
+
)
|
|
752
782
|
}
|
|
753
783
|
|
|
754
784
|
/**
|
|
@@ -771,7 +801,7 @@ export interface OutputStamp {
|
|
|
771
801
|
*/
|
|
772
802
|
export async function stampOutputs(args: {
|
|
773
803
|
projectDir: string
|
|
774
|
-
outputs: string[]
|
|
804
|
+
outputs: readonly string[]
|
|
775
805
|
nestedProjectDirs: string[]
|
|
776
806
|
}): Promise<Map<string, OutputStamp>> {
|
|
777
807
|
return stampFiles(await resolveOutputs(args))
|
|
@@ -780,7 +810,7 @@ export async function stampOutputs(args: {
|
|
|
780
810
|
/** `stampOutputs` for root-anchored `cache.outputs.workspaceFiles` (A-43). */
|
|
781
811
|
export async function stampWorkspaceOutputs(args: {
|
|
782
812
|
workspaceRoot: string
|
|
783
|
-
outputs: string[]
|
|
813
|
+
outputs: readonly string[]
|
|
784
814
|
}): Promise<Map<string, OutputStamp>> {
|
|
785
815
|
return stampFiles(await resolveWorkspaceOutputs(args))
|
|
786
816
|
}
|
|
@@ -811,7 +841,7 @@ function stampFiles(files: readonly string[]): Map<string, OutputStamp> {
|
|
|
811
841
|
* file back, so no entry reproduces that run and none is saved.
|
|
812
842
|
*/
|
|
813
843
|
export async function ownOutputsSince(
|
|
814
|
-
args: { projectDir: string; outputs: string[]; nestedProjectDirs: string[] },
|
|
844
|
+
args: { projectDir: string; outputs: readonly string[]; nestedProjectDirs: string[] },
|
|
815
845
|
before: ReadonlyMap<string, OutputStamp>,
|
|
816
846
|
): Promise<string[] | undefined> {
|
|
817
847
|
return changedSince(await resolveOutputs(args), before)
|
|
@@ -819,7 +849,7 @@ export async function ownOutputsSince(
|
|
|
819
849
|
|
|
820
850
|
/** `ownOutputsSince` for root-anchored `cache.outputs.workspaceFiles` (A-43). */
|
|
821
851
|
export async function ownWorkspaceOutputsSince(
|
|
822
|
-
args: { workspaceRoot: string; outputs: string[] },
|
|
852
|
+
args: { workspaceRoot: string; outputs: readonly string[] },
|
|
823
853
|
before: ReadonlyMap<string, OutputStamp>,
|
|
824
854
|
): Promise<string[] | undefined> {
|
|
825
855
|
return changedSince(await resolveWorkspaceOutputs(args), before)
|
|
@@ -914,7 +944,10 @@ function notThroughLink(files: readonly string[], root: string): string[] {
|
|
|
914
944
|
return null
|
|
915
945
|
}
|
|
916
946
|
}
|
|
917
|
-
|
|
947
|
+
// Resolved only once a directory exists to compare: after a clean pruned
|
|
948
|
+
// the outputs (the common restore) nothing does, and the call was most
|
|
949
|
+
// of an empty clean.
|
|
950
|
+
let realRoot: string | undefined
|
|
918
951
|
const own = new Map<string, boolean>()
|
|
919
952
|
return files.filter((f) => {
|
|
920
953
|
const dir = path.dirname(f)
|
|
@@ -923,7 +956,8 @@ function notThroughLink(files: readonly string[], root: string): string[] {
|
|
|
923
956
|
// A directory already gone has nothing to delete through; its path
|
|
924
957
|
// stays so the prune still reaches the parents it emptied.
|
|
925
958
|
const r = real(dir)
|
|
926
|
-
ok =
|
|
959
|
+
ok =
|
|
960
|
+
r === null || r === path.join((realRoot ??= real(root) ?? root), path.relative(root, dir))
|
|
927
961
|
own.set(dir, ok)
|
|
928
962
|
}
|
|
929
963
|
return ok
|
|
@@ -937,12 +971,13 @@ function notThroughLink(files: readonly string[], root: string): string[] {
|
|
|
937
971
|
* wrote `dist/out/…` and now writes `dist/out`) blocks the rename, and an
|
|
938
972
|
* empty directory is not an output anyone declared. A directory that still
|
|
939
973
|
* holds something — a stray the globs do not cover — stays, and the restore
|
|
940
|
-
* says so if it is in the way.
|
|
974
|
+
* says so if it is in the way. With a `scope`, only a directory inside it
|
|
975
|
+
* goes (`pruneScope`).
|
|
941
976
|
*/
|
|
942
977
|
async function pruneEmptiedDirs(
|
|
943
978
|
root: string,
|
|
944
979
|
removed: readonly string[],
|
|
945
|
-
|
|
980
|
+
scope?: PruneScope,
|
|
946
981
|
): Promise<void> {
|
|
947
982
|
const rootResolved = path.resolve(root)
|
|
948
983
|
// LEVEL ORDER, not a walk-up per directory. A parent is attempted only
|
|
@@ -960,7 +995,9 @@ async function pruneEmptiedDirs(
|
|
|
960
995
|
const parents = new Set<string>()
|
|
961
996
|
const dirs = [...level].filter(
|
|
962
997
|
(dir) =>
|
|
963
|
-
dir !== rootResolved &&
|
|
998
|
+
dir !== rootResolved &&
|
|
999
|
+
dir.startsWith(rootResolved + path.sep) &&
|
|
1000
|
+
(scope === undefined || inScope(dir, scope)),
|
|
964
1001
|
)
|
|
965
1002
|
const gone = (err: NodeJS.ErrnoException): boolean => err.code === 'ENOENT'
|
|
966
1003
|
if (dirs.length <= SYNC_CLEAN_MAX) {
|
|
@@ -983,19 +1020,42 @@ async function pruneEmptiedDirs(
|
|
|
983
1020
|
}
|
|
984
1021
|
}
|
|
985
1022
|
|
|
1023
|
+
interface PruneScope {
|
|
1024
|
+
/** Wildcard globs' roots (`dist` for `dist/**`): prune strictly below. */
|
|
1025
|
+
below: string[]
|
|
1026
|
+
/** Literal outputs (`out` as a tree): prune the path itself and below. */
|
|
1027
|
+
atOrBelow: string[]
|
|
1028
|
+
}
|
|
1029
|
+
|
|
986
1030
|
/**
|
|
987
|
-
*
|
|
988
|
-
*
|
|
989
|
-
*
|
|
1031
|
+
* Where a clean, before a miss or a restore, may prune: only the trees the
|
|
1032
|
+
* task declared. A directory above a glob's root or holding a literal
|
|
1033
|
+
* output (`out` for `out/a.txt`) stays: a sibling task running beside this
|
|
1034
|
+
* one may have just made it and not yet written into it, and pruning it
|
|
1035
|
+
* failed that task "Directory nonexistent". A glob's root stays: the task
|
|
1036
|
+
* writes under it, and a remove there bought only an rmdir and the task's
|
|
1037
|
+
* mkdir (B-49). A literal's own path goes: the entry or the task may need
|
|
1038
|
+
* a file there. A glob rooted at the project prunes nothing.
|
|
990
1039
|
*/
|
|
991
|
-
function
|
|
992
|
-
const
|
|
1040
|
+
function pruneScope(base: string, outputs: readonly string[]): PruneScope {
|
|
1041
|
+
const scope: PruneScope = { below: [], atOrBelow: [] }
|
|
993
1042
|
for (const g of outputs) {
|
|
994
|
-
if (g.startsWith('!')
|
|
1043
|
+
if (g.startsWith('!')) continue
|
|
1044
|
+
if (isLiteralPattern(g)) {
|
|
1045
|
+
scope.atOrBelow.push(path.resolve(base, stripTrailingSlash(normalizeGlob(g))))
|
|
1046
|
+
continue
|
|
1047
|
+
}
|
|
995
1048
|
const prefix = staticPrefix(g)
|
|
996
|
-
if (prefix !== '.')
|
|
1049
|
+
if (prefix !== '.') scope.below.push(path.resolve(base, prefix))
|
|
997
1050
|
}
|
|
998
|
-
return
|
|
1051
|
+
return scope
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
function inScope(dir: string, scope: PruneScope): boolean {
|
|
1055
|
+
return (
|
|
1056
|
+
scope.below.some((r) => dir.startsWith(r + path.sep)) ||
|
|
1057
|
+
scope.atOrBelow.some((r) => dir === r || dir.startsWith(r + path.sep))
|
|
1058
|
+
)
|
|
999
1059
|
}
|
|
1000
1060
|
|
|
1001
1061
|
/**
|
|
@@ -1007,7 +1067,7 @@ function globRoots(projectDir: string, outputs: readonly string[]): Set<string>
|
|
|
1007
1067
|
*/
|
|
1008
1068
|
export async function resolveWorkspaceOutputs(args: {
|
|
1009
1069
|
workspaceRoot: string
|
|
1010
|
-
outputs: string[]
|
|
1070
|
+
outputs: readonly string[]
|
|
1011
1071
|
}): Promise<string[]> {
|
|
1012
1072
|
const { positive, negative } = splitNegations(args.outputs)
|
|
1013
1073
|
if (positive.length === 0) return []
|
|
@@ -1030,10 +1090,14 @@ export async function resolveWorkspaceOutputs(args: {
|
|
|
1030
1090
|
*/
|
|
1031
1091
|
export async function cleanWorkspaceOutputs(args: {
|
|
1032
1092
|
workspaceRoot: string
|
|
1033
|
-
outputs: string[]
|
|
1093
|
+
outputs: readonly string[]
|
|
1034
1094
|
}): Promise<string[]> {
|
|
1035
1095
|
const files = await resolveWorkspaceOutputs(args)
|
|
1036
|
-
await pruneEmptiedDirs(
|
|
1096
|
+
await pruneEmptiedDirs(
|
|
1097
|
+
args.workspaceRoot,
|
|
1098
|
+
await removeAll(files, args.workspaceRoot),
|
|
1099
|
+
pruneScope(args.workspaceRoot, args.outputs),
|
|
1100
|
+
)
|
|
1037
1101
|
return files.map((f) => relPosix(args.workspaceRoot, f))
|
|
1038
1102
|
}
|
|
1039
1103
|
|
|
@@ -1131,8 +1195,11 @@ async function assertNoInvisibleLiteralInputs(
|
|
|
1131
1195
|
interface ResolveFilesArgs {
|
|
1132
1196
|
projectDir: string
|
|
1133
1197
|
workspaceRoot: string
|
|
1134
|
-
files: string[] | undefined
|
|
1135
|
-
ownOutputs: string[]
|
|
1198
|
+
files: readonly string[] | undefined
|
|
1199
|
+
ownOutputs: readonly string[]
|
|
1200
|
+
/** Root-relative `outputs.workspaceFiles`, and the project's root-relative directory. */
|
|
1201
|
+
ownWorkspaceOutputs: readonly string[]
|
|
1202
|
+
projectRel: string
|
|
1136
1203
|
nestedProjectDirs: string[]
|
|
1137
1204
|
gitFilesCache?: GitFilesCache
|
|
1138
1205
|
projectFilesCache?: ProjectFilesCache
|
|
@@ -1324,6 +1391,8 @@ async function resolveFiles(
|
|
|
1324
1391
|
const { positive, negative, isExcluded, ownOutput, positiveGlobs, isPositive } = plan
|
|
1325
1392
|
|
|
1326
1393
|
const nested = inNestedProject(args.projectDir, args.nestedProjectDirs)
|
|
1394
|
+
// Its root-anchored outputs that land in the project, as `files` sees them.
|
|
1395
|
+
const ownWsOutput = inProject(args.projectRel, outputMatcher(args.ownWorkspaceOutputs))
|
|
1327
1396
|
|
|
1328
1397
|
// Defer to git for the file set (Turbo / Nx parity). Nested .gitignore
|
|
1329
1398
|
// files, .git/info/exclude, and global excludes all participate
|
|
@@ -1335,7 +1404,7 @@ async function resolveFiles(
|
|
|
1335
1404
|
// duration of one orchestrator run.
|
|
1336
1405
|
// Everything below the snapshot that decides the result: the project, what
|
|
1337
1406
|
// it declares, what it excludes as its own outputs, and the boundaries.
|
|
1338
|
-
const memoKey = `${args.projectDir}\0${positive.join('\u0001')}\0${negative.join('\u0001')}\0${args.ownOutputs.join('\u0001')}\0${args.nestedProjectDirs.join('\u0001')}`
|
|
1407
|
+
const memoKey = `${args.projectDir}\0${positive.join('\u0001')}\0${negative.join('\u0001')}\0${args.ownOutputs.join('\u0001')}\0${args.ownWorkspaceOutputs.join('\u0001')}\0${args.nestedProjectDirs.join('\u0001')}`
|
|
1339
1408
|
let gitFiles = args.gitFilesCache?.snapshotFor(args.projectDir, positiveGlobs)
|
|
1340
1409
|
let undecodable = args.gitFilesCache?.undecodableNames
|
|
1341
1410
|
if (gitFiles !== undefined) {
|
|
@@ -1346,7 +1415,7 @@ async function resolveFiles(
|
|
|
1346
1415
|
if (memo !== undefined && memo.snapshot === gitFiles) {
|
|
1347
1416
|
return {
|
|
1348
1417
|
files: [...memo.result],
|
|
1349
|
-
listing: listingFor(args.projectDir, gitFiles, plan, nested),
|
|
1418
|
+
listing: listingFor(args.projectDir, gitFiles, plan, nested, ownWsOutput),
|
|
1350
1419
|
}
|
|
1351
1420
|
}
|
|
1352
1421
|
}
|
|
@@ -1397,7 +1466,7 @@ async function resolveFiles(
|
|
|
1397
1466
|
input = isPositive(rel) && !isExcluded(rel) && !ownOutput(rel)
|
|
1398
1467
|
verdicts.set(rel, input)
|
|
1399
1468
|
}
|
|
1400
|
-
if (!input || nested(rel)) continue
|
|
1469
|
+
if (!input || nested(rel) || ownWsOutput(rel)) continue
|
|
1401
1470
|
candidates.push(base === undefined ? path.resolve(args.projectDir, rel) : base + rel)
|
|
1402
1471
|
}
|
|
1403
1472
|
const unmatchedLiterals = unanswered(plan.literals, gitFiles)
|
|
@@ -1424,7 +1493,10 @@ async function resolveFiles(
|
|
|
1424
1493
|
// Stored only on the way out: a declaration whose literal named an
|
|
1425
1494
|
// invisible file threw above, and every task sharing it must throw too.
|
|
1426
1495
|
args.projectFilesCache?.set(memoKey, { snapshot: gitFiles, result: resolved })
|
|
1427
|
-
return {
|
|
1496
|
+
return {
|
|
1497
|
+
files: [...resolved],
|
|
1498
|
+
listing: listingFor(args.projectDir, gitFiles, plan, nested, ownWsOutput),
|
|
1499
|
+
}
|
|
1428
1500
|
}
|
|
1429
1501
|
|
|
1430
1502
|
function listingFor(
|
|
@@ -1432,11 +1504,12 @@ function listingFor(
|
|
|
1432
1504
|
listed: readonly string[],
|
|
1433
1505
|
plan: FilesPlan,
|
|
1434
1506
|
nested: (rel: string) => boolean,
|
|
1507
|
+
ownWsOutput: (rel: string) => boolean,
|
|
1435
1508
|
): InputListing {
|
|
1436
1509
|
return {
|
|
1437
1510
|
root,
|
|
1438
1511
|
listed,
|
|
1439
|
-
isInput: (rel) => inPlan(plan, rel),
|
|
1512
|
+
isInput: (rel) => inPlan(plan, rel) && !ownWsOutput(rel),
|
|
1440
1513
|
nested,
|
|
1441
1514
|
prefixes: plan.prefixes,
|
|
1442
1515
|
literals: plan.literals,
|
package/src/cache/layer.ts
CHANGED
|
@@ -296,7 +296,8 @@ export interface PruneOptions {
|
|
|
296
296
|
olderThanMs?: number
|
|
297
297
|
/**
|
|
298
298
|
* After applying olderThanMs, if the cache still exceeds this size in
|
|
299
|
-
* bytes, evict LRU (smallest `accessed_at` first
|
|
299
|
+
* bytes, evict LRU (smallest `accessed_at` first, a row-less artifact
|
|
300
|
+
* by its file time) until under it.
|
|
300
301
|
*/
|
|
301
302
|
maxBytes?: number
|
|
302
303
|
/**
|
|
@@ -312,10 +313,11 @@ export interface PruneResult {
|
|
|
312
313
|
/** Bytes the evicted entries occupied, per the index. */
|
|
313
314
|
bytesFreed: number
|
|
314
315
|
/**
|
|
315
|
-
* Artifacts and temp files with no index row, unlinked by the
|
|
316
|
-
*
|
|
317
|
-
* `cache.db`, a save that crashed before its rename).
|
|
318
|
-
*
|
|
316
|
+
* Artifacts and temp files with no index row, unlinked by the same
|
|
317
|
+
* policy (another vx version's open dropped the row, a deleted
|
|
318
|
+
* `cache.db`, a save that crashed before its rename). An artifact's file
|
|
319
|
+
* time plus an hour stands for its last use; a temp goes once an hour
|
|
320
|
+
* old. Files younger than one hour are never counted.
|
|
319
321
|
*/
|
|
320
322
|
orphans: number
|
|
321
323
|
/** Bytes the reaped orphans occupied on disk. */
|
|
@@ -185,7 +185,27 @@ class ScopedRemote implements RemoteCacheLayer {
|
|
|
185
185
|
}
|
|
186
186
|
|
|
187
187
|
async has(hash: string): Promise<boolean> {
|
|
188
|
-
return
|
|
188
|
+
return this.fallThrough(
|
|
189
|
+
() => this.remote.has(this.scoped(hash)),
|
|
190
|
+
() => this.remote.has(hash),
|
|
191
|
+
)
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* The scope's answer, else the trusted one. A scope lookup that fails
|
|
196
|
+
* still asks the trusted key, so a hit there is not lost; its error is
|
|
197
|
+
* rethrown (and warned) when the trusted key fails or has nothing too.
|
|
198
|
+
*/
|
|
199
|
+
private async fallThrough<T>(own: () => Promise<T>, trusted: () => Promise<T>): Promise<T> {
|
|
200
|
+
let found: T
|
|
201
|
+
try {
|
|
202
|
+
found = await own()
|
|
203
|
+
} catch (err) {
|
|
204
|
+
const fallback = await trusted().catch(() => undefined)
|
|
205
|
+
if (!fallback) throw err
|
|
206
|
+
return fallback
|
|
207
|
+
}
|
|
208
|
+
return found || trusted()
|
|
189
209
|
}
|
|
190
210
|
|
|
191
211
|
private async scopedHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
@@ -207,7 +227,10 @@ class ScopedRemote implements RemoteCacheLayer {
|
|
|
207
227
|
hash: string,
|
|
208
228
|
): Promise<{ body: Blob | Response; durationMs: number | undefined } | null> {
|
|
209
229
|
if (this.trustedOnly.has(hash)) return this.remote.get(hash)
|
|
210
|
-
return
|
|
230
|
+
return this.fallThrough(
|
|
231
|
+
() => this.remote.get(this.scoped(hash)),
|
|
232
|
+
() => this.remote.get(hash),
|
|
233
|
+
)
|
|
211
234
|
}
|
|
212
235
|
|
|
213
236
|
put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void> {
|
package/src/cache/policy.ts
CHANGED
|
@@ -36,6 +36,11 @@ export const FULL_CACHE_POLICY: CachePolicy = {
|
|
|
36
36
|
remoteWrite: true,
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
/** No axis on (`--no-cache`, `--cache=local:`): no cache answers for any task. */
|
|
40
|
+
export function cachesNothing(p: CachePolicy): boolean {
|
|
41
|
+
return !(p.localRead || p.localWrite || p.remoteRead || p.remoteWrite)
|
|
42
|
+
}
|
|
43
|
+
|
|
39
44
|
/**
|
|
40
45
|
* Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
|
|
41
46
|
* base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
|
package/src/cache/tar-stream.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// short entry. Non-regular entries are reported with their type and their
|
|
12
12
|
// bodies skipped; the caller decides what to materialise (nothing but `0`).
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
interface TarEntry {
|
|
15
15
|
name: string
|
|
16
16
|
size: number
|
|
17
17
|
/** POSIX typeflag: '0' regular, '5' directory, '2' symlink, … */
|