@vzn/vx 0.0.651 → 0.0.653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -4
- package/launcher.sh +2 -1
- package/package.json +5 -5
- package/schemas/error.json +1 -1
- package/schemas/show.json +15 -20
- package/skills/vx/SKILL.md +3 -3
- package/src/bin.ts +37 -3
- package/src/cache/archive.ts +197 -32
- package/src/cache/cache.ts +882 -230
- package/src/cache/checkpointer.ts +102 -0
- package/src/cache/file-hashes.ts +2 -1
- package/src/cache/fold-memo.ts +24 -4
- package/src/cache/fs-clock.ts +26 -0
- package/src/cache/git-inputs.ts +188 -22
- package/src/cache/history-stream.ts +151 -0
- package/src/cache/history-writer.ts +76 -0
- package/src/cache/index.ts +12 -1
- package/src/cache/inputs.ts +616 -135
- package/src/cache/key-fold.ts +1 -1
- package/src/cache/layer.ts +29 -16
- package/src/cache/layered-cache.ts +28 -7
- package/src/cache/lockfile-memo.ts +2 -1
- package/src/cache/output-index.ts +106 -44
- package/src/cache/plugin-cache.ts +2 -2
- package/src/cache/policy.ts +6 -2
- package/src/cache/remote-lane.ts +3 -1
- package/src/cache/run-history.ts +94 -25
- package/src/cache/schema.ts +46 -10
- package/src/cli/cache.ts +149 -36
- package/src/cli/completions.ts +2 -2
- package/src/cli/core-alias.ts +9 -2
- package/src/cli/docs.ts +4 -2
- package/src/cli/help.ts +27 -4
- package/src/cli/index.ts +62 -7
- package/src/cli/info.ts +5 -4
- package/src/cli/init.ts +55 -20
- package/src/cli/last.ts +218 -43
- package/src/cli/lock.ts +187 -24
- package/src/cli/plan-format.ts +24 -21
- package/src/cli/plugin-commands.ts +6 -1
- package/src/cli/plugin-templates.ts +36 -45
- package/src/cli/run.ts +68 -34
- package/src/cli/select.ts +160 -64
- package/src/cli/show.ts +70 -28
- package/src/cli/task-verb.ts +12 -7
- package/src/cli/upgrade.ts +210 -76
- package/src/cli/watch-filter.ts +88 -15
- package/src/cli/watch-fs.ts +34 -47
- package/src/cli/watch-judge.ts +73 -9
- package/src/cli/watch-set.ts +10 -8
- package/src/cli/watch.ts +190 -33
- package/src/cli/why.ts +107 -33
- package/src/cli/workspace-config.ts +11 -1
- package/src/config.ts +56 -30
- package/src/exec/env.ts +3 -1
- package/src/exec/executor.ts +116 -16
- package/src/exec/index.ts +1 -0
- package/src/exec/kill-tree.ts +7 -7
- package/src/exec/runner.ts +270 -108
- package/src/exec/sandbox-binds.ts +109 -18
- package/src/exec/sandbox-paths.ts +3 -1
- package/src/exec/sandbox-runtime.ts +446 -64
- package/src/exec/sandbox-violations.ts +250 -46
- package/src/exec/spawn-off.ts +10 -5
- package/src/exec/task-proxy.ts +93 -0
- package/src/exec/warm-shell.ts +29 -7
- package/src/graph/definition.ts +16 -3
- package/src/graph/folds.ts +89 -16
- package/src/graph/index.ts +8 -0
- package/src/graph/priorities.ts +8 -5
- package/src/graph/scheduler.ts +176 -39
- package/src/graph/stand-ins.ts +36 -0
- package/src/graph/task-graph.ts +347 -160
- package/src/graph/task-status.ts +5 -1
- package/src/handoff.ts +45 -10
- package/src/history/failure-mode.ts +39 -6
- package/src/history/history.ts +103 -30
- package/src/history/index.ts +2 -0
- package/src/history/metrics.ts +193 -73
- package/src/index.ts +6 -3
- package/src/orchestrator/admission.ts +14 -10
- package/src/orchestrator/affected-tasks.ts +35 -13
- package/src/orchestrator/command-windows.ts +8 -5
- package/src/orchestrator/deferred-outputs.ts +42 -13
- package/src/orchestrator/doctor.ts +21 -9
- package/src/orchestrator/download-policy.ts +8 -10
- package/src/orchestrator/excluded-keys.ts +15 -3
- package/src/orchestrator/execute-args.ts +9 -2
- package/src/orchestrator/execute-task.ts +612 -138
- package/src/orchestrator/graph-memo.ts +26 -4
- package/src/orchestrator/graph-stage.ts +27 -4
- package/src/orchestrator/held-servers.ts +36 -14
- package/src/orchestrator/hit-restore.ts +141 -93
- package/src/orchestrator/index.ts +4 -1
- package/src/orchestrator/keyed-projects.ts +53 -0
- package/src/orchestrator/local-floor.ts +21 -3
- package/src/orchestrator/local-shortcircuit.ts +2 -2
- package/src/orchestrator/lockfile-claim.ts +30 -4
- package/src/orchestrator/miss-save.ts +60 -61
- package/src/orchestrator/options.ts +13 -0
- package/src/orchestrator/output-host.ts +16 -4
- package/src/orchestrator/output-log.ts +1 -54
- package/src/orchestrator/persistent-task.ts +32 -16
- package/src/orchestrator/persistent.ts +152 -15
- package/src/orchestrator/placement.ts +82 -16
- package/src/orchestrator/plan.ts +39 -17
- package/src/orchestrator/plugin-host.ts +238 -110
- package/src/orchestrator/plugin.ts +59 -27
- package/src/orchestrator/prepare.ts +194 -38
- package/src/orchestrator/projects.ts +89 -10
- package/src/orchestrator/read-only.ts +115 -0
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-context.ts +6 -2
- package/src/orchestrator/run-failures.ts +101 -23
- package/src/orchestrator/run-id.ts +13 -2
- package/src/orchestrator/run-keys.ts +87 -28
- package/src/orchestrator/run-live.ts +16 -13
- package/src/orchestrator/run-lock.ts +27 -4
- package/src/orchestrator/run-records.ts +95 -74
- package/src/orchestrator/run-report.ts +3 -2
- package/src/orchestrator/run-stop.ts +17 -9
- package/src/orchestrator/run.ts +531 -122
- package/src/orchestrator/sandbox-fix.ts +40 -12
- package/src/orchestrator/sandbox-request.ts +314 -44
- package/src/orchestrator/save-lane.ts +24 -13
- package/src/orchestrator/shell-verdict.ts +29 -1
- package/src/orchestrator/signals.ts +103 -8
- package/src/orchestrator/stable-keys.ts +34 -27
- package/src/orchestrator/task-artifacts.ts +16 -3
- package/src/orchestrator/task-hash.ts +191 -34
- package/src/orchestrator/telemetry-host.ts +25 -6
- package/src/orchestrator/telemetry.ts +85 -13
- package/src/orchestrator/upstream.ts +3 -1
- package/src/report/events.ts +23 -6
- package/src/report/forecast.ts +103 -37
- package/src/report/framed-output.ts +179 -162
- package/src/report/index.ts +2 -1
- package/src/report/logger.ts +108 -31
- package/src/report/plain-output.ts +15 -9
- package/src/report/status-line.ts +28 -55
- package/src/report/summary.ts +61 -18
- package/src/settings.ts +50 -14
- package/src/ui/components.ts +36 -42
- package/src/ui/index.ts +3 -1
- package/src/ui/live-region.ts +67 -8
- package/src/ui/terminal.ts +12 -0
- package/src/ui/theme.ts +16 -1
- package/src/util/config-frame.ts +142 -6
- package/src/util/edit-distance.ts +17 -1
- package/src/util/errors.ts +23 -7
- package/src/util/foreign-cache-dir.ts +60 -0
- package/src/util/git-ignored.ts +59 -5
- package/src/util/index.ts +15 -1
- package/src/util/paths.ts +51 -7
- package/src/util/plain-text.ts +19 -0
- package/src/util/secret-mask.ts +57 -1
- package/src/util/settle.ts +18 -1
- package/src/util/size.ts +12 -2
- package/src/util/tail.ts +4 -1
- package/src/util/write-file.ts +86 -9
- package/src/workspace/affected.ts +225 -27
- package/src/workspace/bracket-literal.ts +2 -2
- package/src/workspace/config-bytecode.ts +23 -4
- package/src/workspace/config-eval.ts +90 -16
- package/src/workspace/config-imports.ts +96 -41
- package/src/workspace/config-schema.ts +192 -86
- package/src/workspace/dependencies-lock.ts +7 -14
- package/src/workspace/filter.ts +18 -2
- package/src/workspace/index.ts +3 -1
- package/src/workspace/json-data.ts +34 -8
- package/src/workspace/lockfile.ts +86 -37
- package/src/workspace/migration.ts +58 -31
- package/src/workspace/module-state.ts +62 -0
- package/src/workspace/nested-dirs.ts +8 -0
- package/src/workspace/project-loader.ts +276 -32
- package/src/workspace/repo-id.ts +29 -6
- package/src/workspace/workspace.ts +190 -30
package/src/history/metrics.ts
CHANGED
|
@@ -36,10 +36,11 @@ export interface RunSummaryRow {
|
|
|
36
36
|
durationMs: number
|
|
37
37
|
startedAt: number
|
|
38
38
|
endedAt: number
|
|
39
|
+
/** Null: no cache outcome, a cached task whose hit failed to restore. */
|
|
39
40
|
cacheHit: boolean | null
|
|
40
41
|
/** On a hit: true when outputs were restored, false when up-to-date (v31); null otherwise. */
|
|
41
42
|
restored: boolean | null
|
|
42
|
-
/** Whether the task declared
|
|
43
|
+
/** Whether the task declared `cache: true`; null on rows older than the column. */
|
|
43
44
|
cached: boolean | null
|
|
44
45
|
hash: string
|
|
45
46
|
cpuMs: number | null
|
|
@@ -382,7 +383,7 @@ export interface WhyDidThisRerun {
|
|
|
382
383
|
cacheHit: boolean | null
|
|
383
384
|
/** On a hit: outputs restored (true) or up-to-date (false); null otherwise. */
|
|
384
385
|
restored: boolean | null
|
|
385
|
-
/** Whether the task declared
|
|
386
|
+
/** Whether the task declared `cache: true`; null on rows older than the column. */
|
|
386
387
|
cached: boolean | null
|
|
387
388
|
startedAt: number
|
|
388
389
|
}
|
|
@@ -409,6 +410,17 @@ function failedWithWrites(db: Database, id: string | null): number {
|
|
|
409
410
|
return row !== null && writesCache(row.p) ? row.f : 0
|
|
410
411
|
}
|
|
411
412
|
|
|
413
|
+
const NO_WRITE = '--no-cache, or a --cache-policy without write'
|
|
414
|
+
|
|
415
|
+
/** False only when the run's recorded policy writes no cache; an unrecorded one may have. */
|
|
416
|
+
function runWritesCache(db: Database, id: string | null): boolean {
|
|
417
|
+
if (id === null) return true
|
|
418
|
+
const row = db.query('SELECT cache_policy AS p FROM invocations WHERE run_id = ?').get(id) as {
|
|
419
|
+
p: string
|
|
420
|
+
} | null
|
|
421
|
+
return row === null || writesCache(row.p)
|
|
422
|
+
}
|
|
423
|
+
|
|
412
424
|
/**
|
|
413
425
|
* The endings an UNCHANGED cache key can have. A re-execution is told apart
|
|
414
426
|
* by the evidence the index holds: every one used to blame `--no-cache` /
|
|
@@ -468,6 +480,22 @@ function unchangedKeyNote(
|
|
|
468
480
|
return `cache key unchanged — the previous run on this key ran beside a failed task (${before} failed in it) and was not saved: a task run past a failed dependency (--continue) is never cached, so there was nothing to hit`
|
|
469
481
|
}
|
|
470
482
|
}
|
|
483
|
+
// A task that passed behind a dependency its run never recorded: the
|
|
484
|
+
// one `--exclude-dependencies` left out, keyed but not run, so nothing
|
|
485
|
+
// built on it was saved (excluded-keys.ts). Before the prune: a key this
|
|
486
|
+
// run saved named an eviction that never happened.
|
|
487
|
+
if (
|
|
488
|
+
(createdAt === undefined || createdAt >= this_.startedAt) &&
|
|
489
|
+
prev.status === 'success' &&
|
|
490
|
+
prev.cacheHit === 0 &&
|
|
491
|
+
prev.blockedBy !== null &&
|
|
492
|
+
prev.runId !== null &&
|
|
493
|
+
db
|
|
494
|
+
.query('SELECT 1 FROM runs WHERE run_id = ? AND project = ? AND task = ?')
|
|
495
|
+
.get(prev.runId, ...splitTaskId(prev.blockedBy)) === null
|
|
496
|
+
) {
|
|
497
|
+
return `cache key unchanged — the previous run on this key built on a dependency it left out (${prev.blockedBy}, --exclude-dependencies), so it saved nothing and there was nothing to hit`
|
|
498
|
+
}
|
|
471
499
|
if (createdAt !== undefined && createdAt >= this_.startedAt) {
|
|
472
500
|
return 'cache key unchanged — no entry for this key was in the cache when it ran (pruned or evicted), so it executed and saved one'
|
|
473
501
|
}
|
|
@@ -493,18 +521,6 @@ function unchangedKeyNote(
|
|
|
493
521
|
if (prevPolicy !== undefined && !writesCache(prevPolicy)) {
|
|
494
522
|
return 'cache key unchanged — the previous run on this key did not write the cache (--no-cache, or a --cache-policy without write), so there was nothing to hit'
|
|
495
523
|
}
|
|
496
|
-
// A task that passed behind a dependency its run never recorded: the
|
|
497
|
-
// one `--exclude-dependencies` left out, keyed but not run, so nothing
|
|
498
|
-
// built on it was saved (excluded-keys.ts).
|
|
499
|
-
if (
|
|
500
|
-
prev.blockedBy !== null &&
|
|
501
|
-
prev.runId !== null &&
|
|
502
|
-
db
|
|
503
|
-
.query('SELECT 1 FROM runs WHERE run_id = ? AND project = ? AND task = ?')
|
|
504
|
-
.get(prev.runId, ...splitTaskId(prev.blockedBy)) === null
|
|
505
|
-
) {
|
|
506
|
-
return `cache key unchanged — the previous run on this key built on a dependency it left out (${prev.blockedBy}, --exclude-dependencies), so it saved nothing and there was nothing to hit`
|
|
507
|
-
}
|
|
508
524
|
return 'cache key unchanged — the previous run on this key executed but no entry for it is in the cache (its save failed, or it was pruned since), so there was nothing to hit'
|
|
509
525
|
}
|
|
510
526
|
return 'cache key unchanged — re-executed on the same key though this run read the cache; vx cannot name the cause'
|
|
@@ -614,10 +630,10 @@ export function whyDidThisRerun(db: Database, runId: string, taskId: string): Wh
|
|
|
614
630
|
// one's is new each run, so every dependent misses: "folded by
|
|
615
631
|
// dependents" read as if they could hit.
|
|
616
632
|
this_.keylessBy === null
|
|
617
|
-
? 'this task declares no `cache`
|
|
633
|
+
? 'this task declares no `cache: true` — it runs on every invocation; its key is folded by dependents only'
|
|
618
634
|
: this_.keylessBy === taskId
|
|
619
|
-
? 'this task declares no `cache`
|
|
620
|
-
: `this task declares no \`cache\`
|
|
635
|
+
? 'this task declares no `cache: true` — it runs on every invocation; it declares no `inputs` either, so each run derives a new key and every task that depends on it misses; declare its `inputs` (and `outputs` with `cache: true` to cache it too)'
|
|
636
|
+
: `this task declares no \`cache: true\` — it runs on every invocation; its key includes ${this_.keylessBy}, which declares no \`inputs\`, so each run derives a new key and every task that depends on it misses; declare inputs on ${this_.keylessBy}`
|
|
621
637
|
: this_.keylessBy !== null
|
|
622
638
|
? keylessNote(taskId, this_.keylessBy)
|
|
623
639
|
: prev && prev.hash !== this_.hash
|
|
@@ -837,7 +853,13 @@ export function cacheKeyDiff(db: Database, runId: string, taskId: string): Cache
|
|
|
837
853
|
// the failure, or the one that broke it).
|
|
838
854
|
// So did one that passed beside a failed task (--continue).
|
|
839
855
|
const tainted = isPassStatus(prev.status) && failedWithWrites(db, prev.runId) > 0
|
|
840
|
-
|
|
856
|
+
// So did one whose run wrote no cache (--no-cache, a read-only policy).
|
|
857
|
+
const prevNoWrite = isPassStatus(prev.status) && !runWritesCache(db, prev.runId)
|
|
858
|
+
if (
|
|
859
|
+
cur.length > 0 &&
|
|
860
|
+
old.length === 0 &&
|
|
861
|
+
(!isPassStatus(prev.status) || tainted || prevNoWrite)
|
|
862
|
+
) {
|
|
841
863
|
const saved = db
|
|
842
864
|
.query(
|
|
843
865
|
`SELECT run_id AS runId, hash FROM runs
|
|
@@ -848,7 +870,12 @@ export function cacheKeyDiff(db: Database, runId: string, taskId: string): Cache
|
|
|
848
870
|
.get(project, task, this_.id) as { runId: string | null; hash: string } | undefined
|
|
849
871
|
if (saved) {
|
|
850
872
|
const { entries, unchangedCount } = diffKeyComponents(loadEntryInputs(db, saved.hash), cur)
|
|
851
|
-
const
|
|
873
|
+
const why = tainted
|
|
874
|
+
? 'the previous run ran beside a failed task, so it was not saved'
|
|
875
|
+
: prevNoWrite
|
|
876
|
+
? `the previous run did not write the cache: ${NO_WRITE}`
|
|
877
|
+
: `the previous run ended ${prev.status}`
|
|
878
|
+
const since = `run ${saved.runId}, the last that saved an entry (${why})`
|
|
852
879
|
return {
|
|
853
880
|
runId,
|
|
854
881
|
taskId,
|
|
@@ -873,11 +900,15 @@ export function cacheKeyDiff(db: Database, runId: string, taskId: string): Cache
|
|
|
873
900
|
const unsaved =
|
|
874
901
|
cur.length === 0 && !isPassStatus(this_.status)
|
|
875
902
|
? `this run ended ${this_.status}`
|
|
876
|
-
:
|
|
877
|
-
? `
|
|
878
|
-
: old.length === 0 &&
|
|
879
|
-
?
|
|
880
|
-
:
|
|
903
|
+
: cur.length === 0 && !runWritesCache(db, runId)
|
|
904
|
+
? `this run did not write the cache (${NO_WRITE})`
|
|
905
|
+
: old.length === 0 && !isPassStatus(prev.status)
|
|
906
|
+
? `the previous run ended ${prev.status}`
|
|
907
|
+
: old.length === 0 && tainted
|
|
908
|
+
? 'the previous run ran beside a failed task'
|
|
909
|
+
: old.length === 0 && prevNoWrite
|
|
910
|
+
? `the previous run did not write the cache (${NO_WRITE})`
|
|
911
|
+
: null
|
|
881
912
|
return {
|
|
882
913
|
runId,
|
|
883
914
|
taskId,
|
|
@@ -889,7 +920,7 @@ export function cacheKeyDiff(db: Database, runId: string, taskId: string): Cache
|
|
|
889
920
|
this_.cached === 0
|
|
890
921
|
? // The verdict already says the task is uncached; this line says
|
|
891
922
|
// why no input can be named.
|
|
892
|
-
'
|
|
923
|
+
'without `cache: true` it saves no entry, so no input fingerprints are kept to say which of its `inputs` moved its key; with no `inputs`, every run gives it a new key'
|
|
893
924
|
: unsaved !== null
|
|
894
925
|
? `cache key changed but input fingerprints are unavailable — ${unsaved} and saved no entry; only the key change is known`
|
|
895
926
|
: 'cache key changed but input fingerprints are unavailable — the entry was pruned (or the row predates the `cached` column); only the key change is known',
|
|
@@ -924,18 +955,67 @@ export function upstreamTask(name: string): string {
|
|
|
924
955
|
return i < 0 ? name : name.slice(i + GROUP_VIA.length)
|
|
925
956
|
}
|
|
926
957
|
|
|
958
|
+
/**
|
|
959
|
+
* The tasks `runId` recorded a row for, as `project#task`. One read: runs
|
|
960
|
+
* has no (project, task) index (schema.ts), so asking per task scans the
|
|
961
|
+
* run's rows each time, and a 10,000-task fan-in asked 10,000 times.
|
|
962
|
+
*/
|
|
963
|
+
export function recordedTasks(db: Database, runId: string): Set<string> {
|
|
964
|
+
const rows = db.query('SELECT project, task FROM runs WHERE run_id = ?').all(runId) as {
|
|
965
|
+
project: string
|
|
966
|
+
task: string
|
|
967
|
+
}[]
|
|
968
|
+
return new Set(rows.map((r) => `${r.project}#${r.task}`))
|
|
969
|
+
}
|
|
970
|
+
|
|
927
971
|
export interface RootCause {
|
|
928
|
-
/**
|
|
929
|
-
|
|
930
|
-
|
|
972
|
+
/**
|
|
973
|
+
* From the asked task down to the one whose own inputs moved. Built on
|
|
974
|
+
* each read, as long as the chain: a deep graph has as many roots as
|
|
975
|
+
* tasks, so read it only for a root that is shown.
|
|
976
|
+
*/
|
|
977
|
+
readonly chain: string[]
|
|
978
|
+
/** The task whose own inputs moved: the chain's last id. */
|
|
979
|
+
readonly task: string
|
|
980
|
+
readonly entries: CacheKeyDiff['entries']
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
/** A visited task: the step that reached it (`group ← task` through a group) and the visit it came from. */
|
|
984
|
+
interface Reach {
|
|
985
|
+
readonly up: Reach | null
|
|
986
|
+
readonly step: readonly string[]
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
class Cause implements RootCause {
|
|
990
|
+
constructor(
|
|
991
|
+
private readonly reach: Reach,
|
|
992
|
+
readonly task: string,
|
|
993
|
+
readonly entries: CacheKeyDiff['entries'],
|
|
994
|
+
) {}
|
|
995
|
+
|
|
996
|
+
get chain(): string[] {
|
|
997
|
+
const steps: (readonly string[])[] = []
|
|
998
|
+
for (let r: Reach | null = this.reach; r !== null; r = r.up) steps.push(r.step)
|
|
999
|
+
return steps.reverse().flat()
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/** `--format json`'s root: its chain and entries, as before the back links. */
|
|
1003
|
+
toJSON(): { chain: string[]; entries: CacheKeyDiff['entries'] } {
|
|
1004
|
+
return { chain: this.chain, entries: this.entries }
|
|
1005
|
+
}
|
|
931
1006
|
}
|
|
932
1007
|
|
|
933
1008
|
/**
|
|
934
1009
|
* The tasks under `taskId`'s moved upstreams whose OWN key components
|
|
935
|
-
* moved
|
|
936
|
-
*
|
|
937
|
-
*
|
|
938
|
-
*
|
|
1010
|
+
* moved, each with the chain that carried it up. An upstream is diffed
|
|
1011
|
+
* between the two keys its dependant folded (the upstream row's before
|
|
1012
|
+
* and after hashes), not against its own previous run, which names what
|
|
1013
|
+
* moved since a run of it alone in between. A task visited once is not
|
|
1014
|
+
* walked again (a diamond names its root once). A group records no run,
|
|
1015
|
+
* so `groups` (group id → the tasks it gathers) carries the walk through
|
|
1016
|
+
* it. Depth first, on a stack of its own: a 20,000-task chain is deeper
|
|
1017
|
+
* than recursion goes, and each visit keeps only a link to the one that
|
|
1018
|
+
* reached it, so no chain is copied per level.
|
|
939
1019
|
*/
|
|
940
1020
|
export function rootCauses(
|
|
941
1021
|
db: Database,
|
|
@@ -944,32 +1024,50 @@ export function rootCauses(
|
|
|
944
1024
|
entries: CacheKeyDiff['entries'],
|
|
945
1025
|
groups: ReadonlyMap<string, readonly string[]> = new Map(),
|
|
946
1026
|
): RootCause[] {
|
|
1027
|
+
const recorded = recordedTasks(db, runId)
|
|
947
1028
|
const roots: RootCause[] = []
|
|
948
1029
|
const seen = new Set([taskId])
|
|
949
|
-
const
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
const
|
|
968
|
-
if (
|
|
969
|
-
|
|
1030
|
+
const stack = [{ from: { up: null, step: [taskId] } as Reach, moved: entries, i: 0 }]
|
|
1031
|
+
while (stack.length > 0) {
|
|
1032
|
+
const frame = stack.at(-1)!
|
|
1033
|
+
const { moved } = frame
|
|
1034
|
+
if (frame.i === moved.length) {
|
|
1035
|
+
stack.pop()
|
|
1036
|
+
continue
|
|
1037
|
+
}
|
|
1038
|
+
const e = moved[frame.i++]!
|
|
1039
|
+
if (e.kind !== 'upstream') continue
|
|
1040
|
+
const task = upstreamTask(e.name)
|
|
1041
|
+
if (seen.has(task)) continue
|
|
1042
|
+
seen.add(task)
|
|
1043
|
+
const reach: Reach = { up: frame.from, step: e.name.split(GROUP_VIA) }
|
|
1044
|
+
if (!recorded.has(task)) {
|
|
1045
|
+
// A group: its members carry the walk, by their own rows named
|
|
1046
|
+
// through it (`group ← task`) when the key has them, else by the
|
|
1047
|
+
// graph, whose members carry none of the dependant's keys.
|
|
1048
|
+
const via = `${e.name}${GROUP_VIA}`
|
|
1049
|
+
if (moved.some((x) => x.kind === 'upstream' && x.name.startsWith(via))) continue
|
|
1050
|
+
const members = groups.get(task) ?? []
|
|
1051
|
+
stack.push({
|
|
1052
|
+
from: reach,
|
|
1053
|
+
moved: members.map((name) => ({ ...e, name, before: null, after: null })),
|
|
1054
|
+
i: 0,
|
|
1055
|
+
})
|
|
1056
|
+
continue
|
|
970
1057
|
}
|
|
1058
|
+
let d: CacheKeyDiff['entries']
|
|
1059
|
+
if (e.before !== null && e.after !== null) {
|
|
1060
|
+
const before = loadEntryInputs(db, e.before)
|
|
1061
|
+
const after = loadEntryInputs(db, e.after)
|
|
1062
|
+
// A pruned or unsaved side kept no fingerprints: nothing to name.
|
|
1063
|
+
d = before.length > 0 && after.length > 0 ? diffKeyComponents(before, after).entries : []
|
|
1064
|
+
} else {
|
|
1065
|
+
d = cacheKeyDiff(db, runId, task).entries
|
|
1066
|
+
}
|
|
1067
|
+
const own = d.filter((x) => x.kind !== 'upstream')
|
|
1068
|
+
if (own.length > 0) roots.push(new Cause(reach, task, own))
|
|
1069
|
+
stack.push({ from: reach, moved: d, i: 0 })
|
|
971
1070
|
}
|
|
972
|
-
walk([taskId], entries)
|
|
973
1071
|
return roots
|
|
974
1072
|
}
|
|
975
1073
|
|
|
@@ -994,43 +1092,65 @@ const WRITE_SLACK_MS = 5
|
|
|
994
1092
|
* that writes a file another key reads re-keys that task on every run.
|
|
995
1093
|
* The file's mtime against the recorded windows, so
|
|
996
1094
|
* a writer that runs each time is named from its latest run, and an
|
|
997
|
-
* edit made outside a run names none.
|
|
1095
|
+
* edit made outside a run, or after run `runId` ended, names none.
|
|
998
1096
|
*/
|
|
999
1097
|
export function inputWriters(
|
|
1000
1098
|
db: Database,
|
|
1099
|
+
runId: string,
|
|
1001
1100
|
workspaceRoot: string,
|
|
1002
1101
|
taskId: string,
|
|
1003
1102
|
entries: CacheKeyDiff['entries'],
|
|
1004
1103
|
roots: readonly RootCause[],
|
|
1005
1104
|
): InputWriter[] {
|
|
1105
|
+
// Bounded on both sides by the longest recorded window, so the query
|
|
1106
|
+
// walks `runs_started_at` over that span, not every older row.
|
|
1006
1107
|
const during = db.query(
|
|
1007
|
-
`SELECT project, task FROM runs
|
|
1008
|
-
WHERE started_at
|
|
1009
|
-
AND status != 'skipped'
|
|
1108
|
+
`SELECT project, task FROM main.runs
|
|
1109
|
+
WHERE started_at BETWEEN ?1 - ?2 - ${WRITE_SLACK_MS} AND ?1 + ${WRITE_SLACK_MS}
|
|
1110
|
+
AND ended_at >= ?1 - ${WRITE_SLACK_MS} AND status != 'skipped'
|
|
1010
1111
|
GROUP BY project, task ORDER BY MIN(started_at) LIMIT 3`,
|
|
1011
1112
|
)
|
|
1113
|
+
let bounds: { longest: number; runEnd: number } | undefined
|
|
1114
|
+
// A deep graph has as many roots as tasks, most reading the same files:
|
|
1115
|
+
// each file is stat'd and asked about once.
|
|
1116
|
+
const byFile = new Map<string, string[] | null>()
|
|
1117
|
+
const writersOf = (file: string): string[] | null => {
|
|
1118
|
+
let w = byFile.get(file)
|
|
1119
|
+
if (w !== undefined) return w
|
|
1120
|
+
w = null
|
|
1121
|
+
let mtime: number | undefined
|
|
1122
|
+
try {
|
|
1123
|
+
mtime = statSync(path.join(workspaceRoot, file)).mtimeMs
|
|
1124
|
+
} catch {}
|
|
1125
|
+
if (mtime !== undefined && bounds === undefined) {
|
|
1126
|
+
const b = db
|
|
1127
|
+
.query(
|
|
1128
|
+
`SELECT (SELECT MAX(ended_at - started_at) FROM main.runs WHERE status != 'skipped') AS ms,
|
|
1129
|
+
(SELECT MAX(r.ended_at) FROM main.runs r JOIN main.run_ids k ON k.id = r.run
|
|
1130
|
+
WHERE k.run_id = ?1) AS runEnd`,
|
|
1131
|
+
)
|
|
1132
|
+
.get(runId) as { ms: number | null; runEnd: number | null }
|
|
1133
|
+
bounds = { longest: b.ms ?? 0, runEnd: b.runEnd ?? 0 }
|
|
1134
|
+
}
|
|
1135
|
+
// A change after the asked run ended came after its tasks read the
|
|
1136
|
+
// file: it says nothing about who wrote what they saw.
|
|
1137
|
+
if (mtime !== undefined && mtime <= bounds!.runEnd + WRITE_SLACK_MS) {
|
|
1138
|
+
const rows = during.all(mtime, bounds!.longest) as { project: string; task: string }[]
|
|
1139
|
+
if (rows.length > 0) w = rows.map((r) => `${r.project}#${r.task}`)
|
|
1140
|
+
}
|
|
1141
|
+
byFile.set(file, w)
|
|
1142
|
+
return w
|
|
1143
|
+
}
|
|
1012
1144
|
const out: InputWriter[] = []
|
|
1013
1145
|
const owners = [
|
|
1014
1146
|
{ taskId, entries },
|
|
1015
|
-
...roots.map((r) => ({ taskId: r.
|
|
1147
|
+
...roots.map((r) => ({ taskId: r.task, entries: r.entries })),
|
|
1016
1148
|
]
|
|
1017
1149
|
for (const owner of owners) {
|
|
1018
1150
|
for (const e of owner.entries) {
|
|
1019
1151
|
if (e.kind !== 'file' || e.change === 'removed') continue
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
mtime = statSync(path.join(workspaceRoot, e.name)).mtimeMs
|
|
1023
|
-
} catch {
|
|
1024
|
-
continue
|
|
1025
|
-
}
|
|
1026
|
-
const rows = during.all(mtime) as { project: string; task: string }[]
|
|
1027
|
-
if (rows.length > 0) {
|
|
1028
|
-
out.push({
|
|
1029
|
-
taskId: owner.taskId,
|
|
1030
|
-
file: e.name,
|
|
1031
|
-
writers: rows.map((r) => `${r.project}#${r.task}`),
|
|
1032
|
-
})
|
|
1033
|
-
}
|
|
1152
|
+
const writers = writersOf(e.name)
|
|
1153
|
+
if (writers !== null) out.push({ taskId: owner.taskId, file: e.name, writers })
|
|
1034
1154
|
}
|
|
1035
1155
|
}
|
|
1036
1156
|
return out
|
package/src/index.ts
CHANGED
|
@@ -41,9 +41,10 @@ export type {
|
|
|
41
41
|
TaskConfig,
|
|
42
42
|
ExecConfig,
|
|
43
43
|
ExecEnv,
|
|
44
|
-
CacheConfig,
|
|
45
44
|
InputsConfig,
|
|
46
|
-
|
|
45
|
+
TaskOutputs,
|
|
46
|
+
WorkspaceInputs,
|
|
47
|
+
WorkspaceOutputs,
|
|
47
48
|
InputStability,
|
|
48
49
|
SandboxConfig,
|
|
49
50
|
SandboxGrants,
|
|
@@ -94,7 +95,8 @@ export { criticalPath, outputsOverlap, splitTaskId, taskDefinitions } from './gr
|
|
|
94
95
|
export type { TaskNode, TaskOutcome, TaskStatus } from './graph/index.js'
|
|
95
96
|
|
|
96
97
|
// Cache classes + the layer interface (the `cache` capability's currency).
|
|
97
|
-
|
|
98
|
+
// A remote layer errors a body it refuses with `CorruptArtifactError`.
|
|
99
|
+
export { Cache, CorruptArtifactError } from './cache/index.js'
|
|
98
100
|
export type {
|
|
99
101
|
CacheLayer,
|
|
100
102
|
PluginCache,
|
|
@@ -245,6 +247,7 @@ export type {
|
|
|
245
247
|
ServerSpec,
|
|
246
248
|
TaskExecutor,
|
|
247
249
|
TaskPlacement,
|
|
250
|
+
WorkerAdvice,
|
|
248
251
|
} from './exec/index.js'
|
|
249
252
|
|
|
250
253
|
// Telemetry — THE canonical, versioned data-export contract every exporter
|
|
@@ -23,7 +23,7 @@ import type { CachePolicy } from '../cache/index.js'
|
|
|
23
23
|
import { isGroupTask, RestoreDemoted, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
24
24
|
import { executeTask, type ExecuteArgs } from './execute-task.js'
|
|
25
25
|
import type { ShortCircuit } from './local-shortcircuit.js'
|
|
26
|
-
import { computeTaskHash, type ComputeHashArgs } from './task-hash.js'
|
|
26
|
+
import { computeTaskHash, type ComputeHashArgs, keyOf } from './task-hash.js'
|
|
27
27
|
|
|
28
28
|
/** What `taintTracker` answers per dispatch, and what it learns per finish. */
|
|
29
29
|
export interface TaintTracker {
|
|
@@ -52,7 +52,8 @@ const UNTAINTED: TaintTracker = {
|
|
|
52
52
|
* Records which tasks ran behind a failure, or on the key of a dependency
|
|
53
53
|
* that did not run (`seeds`), and answers, per task, whether this one does.
|
|
54
54
|
* Disabled (always `false`, nothing recorded) unless the run's
|
|
55
|
-
* `continueMode` is `'always'
|
|
55
|
+
* `continueMode` is `'always'`, there is a seed, or one may be added while
|
|
56
|
+
* the run goes on (`seedsLater`: a server that dies under a running task).
|
|
56
57
|
*
|
|
57
58
|
* A task's taint is read from its deps' SETTLED outcomes, not from what it
|
|
58
59
|
* saw at dispatch: a restore-tier hit dispatches before its deps settle,
|
|
@@ -67,8 +68,9 @@ export function taintTracker(
|
|
|
67
68
|
continueAlways: boolean,
|
|
68
69
|
seeds: ReadonlySet<string>,
|
|
69
70
|
nodes: ReadonlyMap<string, TaskNode>,
|
|
71
|
+
seedsLater = false,
|
|
70
72
|
): TaintTracker {
|
|
71
|
-
if (!continueAlways && seeds.size === 0) return UNTAINTED
|
|
73
|
+
if (!continueAlways && seeds.size === 0 && !seedsLater) return UNTAINTED
|
|
72
74
|
const outcomes = new Map<string, TaskOutcome>()
|
|
73
75
|
const memo = new Map<string, boolean>()
|
|
74
76
|
const causes = new Map<string, string>()
|
|
@@ -186,7 +188,7 @@ export function admitTasks(
|
|
|
186
188
|
const cacheable =
|
|
187
189
|
!isGroupTask(node) &&
|
|
188
190
|
node.config.exec?.persistent === undefined &&
|
|
189
|
-
node.config.cache
|
|
191
|
+
node.config.cache === true &&
|
|
190
192
|
canRead &&
|
|
191
193
|
canWrite
|
|
192
194
|
// A restore-tier task (confirmed local hit, may run before its deps)
|
|
@@ -206,12 +208,14 @@ export function admitTasks(
|
|
|
206
208
|
return executeTask(buildExecuteArgs(node, upstream)).catch(dropProbe)
|
|
207
209
|
}
|
|
208
210
|
const { nestedDirsByProject, ...rest } = hashArgs
|
|
209
|
-
const hash = await
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
211
|
+
const hash = await keyOf(
|
|
212
|
+
computeTaskHash({
|
|
213
|
+
...rest,
|
|
214
|
+
node,
|
|
215
|
+
upstream,
|
|
216
|
+
nestedProjectDirs: nestedDirsByProject.get(node.projectName) ?? [],
|
|
217
|
+
}),
|
|
218
|
+
)
|
|
215
219
|
// Join a sibling already computing this exact task: wait, then
|
|
216
220
|
// executeTask cache-hits on the artifact it just saved. A sibling that
|
|
217
221
|
// saved nothing (it failed) leaves this task to run, so the joiners
|
|
@@ -27,6 +27,7 @@ import { applyGraphStage } from './graph-stage.js'
|
|
|
27
27
|
import { hasHook } from './plugin-host.js'
|
|
28
28
|
import type { VxPlugin } from './plugin.js'
|
|
29
29
|
import { isDefaultBuild } from '../graph/index.js'
|
|
30
|
+
import { relPosix } from '../util/index.js'
|
|
30
31
|
|
|
31
32
|
/** Every file in this set re-keys each of its project's tasks, whatever their inputs. */
|
|
32
33
|
const KEYED_BY_EVERY_TASK: ReadonlySet<string> = new Set(PROJECT_CONFIG_FILENAMES)
|
|
@@ -51,9 +52,10 @@ function wholeProjects(
|
|
|
51
52
|
(t) => t.exec !== undefined && t.inputs !== undefined,
|
|
52
53
|
)
|
|
53
54
|
const links = changes.dirLinksIn?.get(name)
|
|
55
|
+
const loose = changes.untrackedIn?.get(name)
|
|
54
56
|
const unclaimed = (rel: string): boolean =>
|
|
55
57
|
KEYED_BY_EVERY_TASK.has(rel) ||
|
|
56
|
-
(!byKeys && !keyed.some((t) => declaresInput(t, rel, null, links?.has(rel))))
|
|
58
|
+
(!byKeys && !keyed.some((t) => declaresInput(t, rel, null, links?.has(rel), loose?.has(rel))))
|
|
57
59
|
const rel = rels.find(unclaimed)
|
|
58
60
|
if (rel !== undefined) whole.set(name, rel)
|
|
59
61
|
}
|
|
@@ -78,11 +80,12 @@ export interface AffectedReason {
|
|
|
78
80
|
file?: string
|
|
79
81
|
project?: string
|
|
80
82
|
via?: string[]
|
|
83
|
+
/** `workspace`: the file reached every project. The text label's, never on the JSON wire. */
|
|
84
|
+
wide?: true
|
|
81
85
|
}
|
|
82
86
|
|
|
83
87
|
/** Collects each kept task's `AffectedReason` when given; a run without one pays nothing. */
|
|
84
88
|
export interface AffectedExplain {
|
|
85
|
-
workspaceRoot: string
|
|
86
89
|
reasons: Map<string, AffectedReason>
|
|
87
90
|
}
|
|
88
91
|
|
|
@@ -101,6 +104,7 @@ export function affectedRoots(
|
|
|
101
104
|
changes: AffectedChanges,
|
|
102
105
|
projects: ReadonlyMap<string, ProjectEntry>,
|
|
103
106
|
packageGraph: PackageGraph,
|
|
107
|
+
workspaceRoot: string,
|
|
104
108
|
explain?: AffectedExplain,
|
|
105
109
|
/** Only what moves a key: a changed path no task declares reaches no task. */
|
|
106
110
|
byKeys = false,
|
|
@@ -111,9 +115,12 @@ export function affectedRoots(
|
|
|
111
115
|
const inProject = (name: string, rel: string | undefined): AffectedReason => {
|
|
112
116
|
if (rel === undefined) {
|
|
113
117
|
const file = changes.wholeBy?.get(name)
|
|
114
|
-
|
|
118
|
+
if (file === undefined) return { kind: 'project' }
|
|
119
|
+
return changes.wide === true
|
|
120
|
+
? { kind: 'workspace', file, wide: true }
|
|
121
|
+
: { kind: 'workspace', file }
|
|
115
122
|
}
|
|
116
|
-
const dir = path.relative(
|
|
123
|
+
const dir = path.relative(workspaceRoot, projects.get(name)!.dir)
|
|
117
124
|
return { kind: 'project', file: dir === '' ? rel : path.posix.join(dir, rel) }
|
|
118
125
|
}
|
|
119
126
|
// The builder's `^name` walk (task-graph.ts), keeping only the packages
|
|
@@ -134,7 +141,9 @@ export function affectedRoots(
|
|
|
134
141
|
if (visited.has(target)) continue
|
|
135
142
|
visited.add(target)
|
|
136
143
|
if (holds(target)) continue
|
|
137
|
-
|
|
144
|
+
// A loaded project without the task folds nothing into the key; the
|
|
145
|
+
// holders past it seed as nodes of their own.
|
|
146
|
+
if (!projects.has(target) && changes.projects.has(target)) {
|
|
138
147
|
seeds?.set(n.id, { kind: 'package', project: target })
|
|
139
148
|
return true
|
|
140
149
|
}
|
|
@@ -149,9 +158,12 @@ export function affectedRoots(
|
|
|
149
158
|
// Asked of every node, not only the changed projects' (the
|
|
150
159
|
// `workspaceFiles` owners): a `graph` hook may have given the glob.
|
|
151
160
|
if (keyed) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
161
|
+
let projectDir: string | undefined
|
|
162
|
+
const file = changes.changed.find((rel) => {
|
|
163
|
+
const untracked = changes.untracked?.has(rel) === true
|
|
164
|
+
if (untracked) projectDir ??= relPosix(workspaceRoot, n.projectDir)
|
|
165
|
+
return declaresInput(task, null, rel, changes.dirLinks?.has(rel), untracked, projectDir)
|
|
166
|
+
})
|
|
155
167
|
const dir =
|
|
156
168
|
file === undefined
|
|
157
169
|
? (changes.nested ?? []).find((d) => workspaceFilesReachInto(task, d))
|
|
@@ -176,7 +188,8 @@ export function affectedRoots(
|
|
|
176
188
|
return true
|
|
177
189
|
}
|
|
178
190
|
const links = changes.dirLinksIn?.get(n.projectName)
|
|
179
|
-
const
|
|
191
|
+
const loose = changes.untrackedIn?.get(n.projectName)
|
|
192
|
+
const rel = rels.find((r) => declaresInput(task, r, null, links?.has(r), loose?.has(r)))
|
|
180
193
|
if (rel === undefined) return false
|
|
181
194
|
if (seeds !== null) seeds.set(n.id, { ...inProject(n.projectName, rel), kind: 'input' })
|
|
182
195
|
return true
|
|
@@ -245,11 +258,14 @@ export function keptByAffected<R extends { project: string; task: string }>(
|
|
|
245
258
|
changes: AffectedChanges,
|
|
246
259
|
projects: ReadonlyMap<string, ProjectEntry>,
|
|
247
260
|
packageGraph: PackageGraph,
|
|
261
|
+
workspaceRoot: string,
|
|
248
262
|
keep: { named?: ReadonlySet<string>; outright?: ReadonlySet<string> } = {},
|
|
249
263
|
explain?: AffectedExplain,
|
|
250
264
|
): R[] {
|
|
251
265
|
const ids = requested.map((r) => `${r.project}#${r.task}`)
|
|
252
|
-
const reached = new Set(
|
|
266
|
+
const reached = new Set(
|
|
267
|
+
affectedRoots(nodes, ids, changes, projects, packageGraph, workspaceRoot, explain),
|
|
268
|
+
)
|
|
253
269
|
return requested.filter((r, i) => {
|
|
254
270
|
const id = ids[i]!
|
|
255
271
|
const why: AffectedReason['kind'] | null =
|
|
@@ -291,8 +307,14 @@ export async function affectedTaskProjects(args: {
|
|
|
291
307
|
rules: args.rules,
|
|
292
308
|
})
|
|
293
309
|
if (hasHook(args.plugins, 'graph')) await applyGraphStage(args.plugins, nodes, args)
|
|
294
|
-
const kept = keptByAffected(
|
|
295
|
-
|
|
296
|
-
|
|
310
|
+
const kept = keptByAffected(
|
|
311
|
+
nodes,
|
|
312
|
+
requested,
|
|
313
|
+
args.changes,
|
|
314
|
+
args.projects,
|
|
315
|
+
args.packageGraph,
|
|
316
|
+
args.workspaceRoot,
|
|
317
|
+
{ outright: new Set(args.outright) },
|
|
318
|
+
)
|
|
297
319
|
return kept.map((r) => r.project)
|
|
298
320
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
// When each command of the run ran and
|
|
2
|
-
// `
|
|
3
|
-
// rewrite from another task's write and restore (STATUS item
|
|
1
|
+
// When each command of the run ran, and each restore or deferred fetch, and where it
|
|
2
|
+
// may write, so `inputs.stability.strategy: 'mutable'` can tell a task's
|
|
3
|
+
// own identical rewrite from another task's write and restore (STATUS item
|
|
4
|
+
// 28), and `vx watch` a user's write from the run's own: a file
|
|
4
5
|
// whose bytes match the key after a write is the task's own only if no
|
|
5
6
|
// other command that may write it ran between the describe and that write.
|
|
6
7
|
|
|
@@ -16,7 +17,7 @@ interface Window {
|
|
|
16
17
|
}
|
|
17
18
|
|
|
18
19
|
export interface CommandWindows {
|
|
19
|
-
/** A command starts; the returned function marks its end. A server never calls it. */
|
|
20
|
+
/** A command, a hit's restore or a deferred fetch starts; the returned function marks its end. A server never calls it. */
|
|
20
21
|
open(id: string, projectDir: string, reach: WriteReach): () => void
|
|
21
22
|
/** Another command that may write `file` and ran at some point in [from, to]. */
|
|
22
23
|
writerDuring(self: string, file: string, from: number, to: number): string | undefined
|
|
@@ -30,7 +31,9 @@ export function commandWindows(): CommandWindows {
|
|
|
30
31
|
const w: Window = { id, projectDir, reach, from: Date.now(), to: Infinity }
|
|
31
32
|
windows.push(w)
|
|
32
33
|
return () => {
|
|
33
|
-
|
|
34
|
+
// `Date.now()` floors to the ms and a ctime does not: the command's
|
|
35
|
+
// last write can be stamped up to a millisecond past it.
|
|
36
|
+
w.to = Date.now() + 1
|
|
34
37
|
}
|
|
35
38
|
},
|
|
36
39
|
writerDuring(self, file, from, to) {
|