@vzn/vx 0.0.595 → 0.0.597
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/launcher.cjs +7 -3
- package/package.json +5 -5
- package/src/cache/archive.ts +35 -6
- package/src/cache/cache.ts +170 -99
- package/src/cache/chained-cache.ts +25 -7
- package/src/cache/git-inputs.ts +23 -0
- package/src/cache/index.ts +3 -0
- package/src/cache/inputs.ts +99 -44
- package/src/cache/key-fold.ts +1 -1
- package/src/cache/layer.ts +9 -6
- package/src/cache/policy.ts +5 -0
- package/src/cache/zstd.ts +10 -4
- package/src/cli/cache.ts +12 -6
- package/src/cli/foreign-flags.ts +57 -3
- package/src/cli/info.ts +1 -1
- package/src/cli/init.ts +2 -2
- package/src/cli/last.ts +10 -4
- package/src/cli/lock.ts +5 -3
- package/src/cli/plan-format.ts +4 -4
- package/src/cli/run.ts +73 -30
- package/src/cli/select.ts +6 -3
- package/src/cli/show.ts +8 -2
- package/src/cli/upgrade.ts +1 -1
- package/src/cli/watch-filter.ts +63 -12
- package/src/cli/watch-fs.ts +11 -3
- package/src/cli/watch-judge.ts +15 -8
- package/src/cli/watch.ts +18 -11
- package/src/cli/workspace-config.ts +3 -1
- package/src/config.ts +42 -21
- package/src/exec/executor.ts +1 -1
- package/src/exec/kill-tree.ts +89 -18
- package/src/exec/proc-sample.ts +1 -1
- package/src/exec/runner.ts +74 -36
- package/src/exec/sandbox-paths.ts +12 -1
- package/src/exec/sandbox-runtime.ts +283 -88
- package/src/exec/sandbox-violations.ts +115 -15
- package/src/graph/scheduler.ts +30 -14
- package/src/graph/task-graph.ts +265 -44
- package/src/orchestrator/admission.ts +12 -3
- package/src/orchestrator/affected-tasks.ts +6 -2
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/doctor.ts +2 -2
- package/src/orchestrator/events.ts +10 -1
- package/src/orchestrator/excluded-keys.ts +2 -13
- package/src/orchestrator/execute-task.ts +85 -39
- package/src/orchestrator/failure-mode.ts +43 -28
- package/src/orchestrator/framed-output.ts +8 -8
- package/src/orchestrator/index.ts +0 -1
- package/src/orchestrator/keyed-projects.ts +6 -3
- package/src/orchestrator/logger.ts +21 -9
- package/src/orchestrator/metrics.ts +19 -8
- package/src/orchestrator/miss-save.ts +9 -13
- package/src/orchestrator/options.ts +7 -0
- package/src/orchestrator/plan.ts +7 -23
- package/src/orchestrator/plugin.ts +37 -16
- package/src/orchestrator/prepare.ts +13 -4
- package/src/orchestrator/run-artifacts.ts +4 -3
- package/src/orchestrator/run-lock.ts +25 -24
- package/src/orchestrator/run.ts +22 -5
- package/src/orchestrator/sandbox-request.ts +96 -18
- package/src/orchestrator/save-lane.ts +5 -5
- package/src/orchestrator/signals.ts +22 -4
- package/src/orchestrator/stable-keys.ts +3 -11
- package/src/orchestrator/summary.ts +7 -13
- package/src/orchestrator/task-hash.ts +13 -3
- package/src/orchestrator/telemetry.ts +16 -7
- package/src/orchestrator/upstream.ts +11 -0
- package/src/util/hangup.ts +42 -0
- package/src/util/index.ts +2 -0
- package/src/util/paths.ts +24 -4
- package/src/workspace/affected.ts +79 -42
- package/src/workspace/config-eval.ts +27 -6
- package/src/workspace/config-imports.ts +8 -53
- package/src/workspace/config-schema.ts +56 -9
- package/src/workspace/filter.ts +19 -0
- package/src/workspace/lockfile.ts +2 -1
- package/src/workspace/migrate-scripts.ts +5 -3
- package/src/workspace/migration.ts +1 -1
- package/src/workspace/project-loader.ts +19 -4
- package/src/workspace/repo-id.ts +42 -4
- package/src/workspace/workspace.ts +53 -24
|
@@ -362,12 +362,17 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
362
362
|
args.cacheDir,
|
|
363
363
|
)
|
|
364
364
|
placeholders = sb.placeholders
|
|
365
|
+
// A wrap that refuses (a port the host holds) leaves the server unspawned
|
|
366
|
+
// and the placeholders vx made for it in the project for every later run.
|
|
365
367
|
const wrapped = await wrapSandboxedCommand({
|
|
366
368
|
command: plainCommand,
|
|
367
369
|
cwd: node.projectDir,
|
|
368
370
|
env,
|
|
369
371
|
...sb.sandbox,
|
|
370
372
|
server: true,
|
|
373
|
+
}).catch(async (err: unknown) => {
|
|
374
|
+
await sweepPlaceholders(placeholders)
|
|
375
|
+
throw err
|
|
371
376
|
})
|
|
372
377
|
command = wrapped.wrapped
|
|
373
378
|
bridgeTag = wrapped.tag
|
|
@@ -519,11 +524,10 @@ async function executePersistentTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
519
524
|
// exited before ready keeps its own exit code rather than a made-up 1.
|
|
520
525
|
// One the readiness timeout is killing reports the signal's, as an
|
|
521
526
|
// ordinary timeout does (X-24).
|
|
527
|
+
// A readiness timeout rejects once the group is gone (runner.ts), so a
|
|
528
|
+
// server that ignored the TERM holds no port into the next `vx watch`
|
|
529
|
+
// cycle, and a second signal here would cut a one-shot handler short.
|
|
522
530
|
const ready = err instanceof PersistentReadyError ? err : undefined
|
|
523
|
-
// The timer's SIGKILL is a grace away and the shell may die on the
|
|
524
|
-
// TERM first: a server that ignores it held its port past run() into
|
|
525
|
-
// the next `vx watch` cycle. Return once the group is gone.
|
|
526
|
-
if (ready?.reason === 'timeout') await terminateChildren(() => [spawn.child])
|
|
527
531
|
return {
|
|
528
532
|
node,
|
|
529
533
|
status: 'failed',
|
|
@@ -632,7 +636,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
632
636
|
// `signal` stops a plugin executor on the run's stop AND on
|
|
633
637
|
// `exec.timeout`: core cannot kill a process an executor spawned, so a
|
|
634
638
|
// declared timeout meant nothing on one that kept its own clock (H-12).
|
|
635
|
-
// The local executor keeps its own timer (it signals the
|
|
639
|
+
// The local executor keeps its own timer from the spawn (it signals the
|
|
640
|
+
// process group), so its request carries no clock: armed here, it also
|
|
641
|
+
// timed the sandbox's wrap, and a wrap slower than the timeout aborted
|
|
642
|
+
// the request before the spawn — a task reported killed, never run.
|
|
636
643
|
let timeoutTimer: ReturnType<typeof setTimeout> | undefined
|
|
637
644
|
let timeoutFired = false
|
|
638
645
|
// The attempt's listener on the run's stop signal, taken off once the
|
|
@@ -640,7 +647,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
640
647
|
// grew to the task count, and each add scans it for a duplicate, ~100 ms
|
|
641
648
|
// of a 1,000-task cold run.
|
|
642
649
|
let unlistenStop: (() => void) | undefined
|
|
643
|
-
function requestSignal(): AbortSignal {
|
|
650
|
+
function requestSignal(local: boolean): AbortSignal {
|
|
644
651
|
const stop = new AbortController()
|
|
645
652
|
const abort = (): void => stop.abort(args.stopSignal?.reason)
|
|
646
653
|
unlistenStop?.()
|
|
@@ -651,9 +658,9 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
651
658
|
run.addEventListener('abort', abort, { once: true })
|
|
652
659
|
unlistenStop = () => run.removeEventListener('abort', abort)
|
|
653
660
|
}
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
661
|
+
clearTimeout(timeoutTimer)
|
|
662
|
+
timeoutFired = false
|
|
663
|
+
if (effectiveTimeout !== undefined && !local) {
|
|
657
664
|
timeoutTimer = setTimeout(() => {
|
|
658
665
|
timeoutFired = true
|
|
659
666
|
stop.abort(new Error(`timed out after ${effectiveTimeout}ms`))
|
|
@@ -803,7 +810,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
803
810
|
}
|
|
804
811
|
if (preProbed !== undefined) {
|
|
805
812
|
if (preProbed.hit !== null) {
|
|
806
|
-
|
|
813
|
+
// A key taken before a task rewrote the lockfile names an install
|
|
814
|
+
// the tree no longer holds, and the lazy path below never probes
|
|
815
|
+
// one (X-124): demoted like a vanished hit, it runs dep-gated.
|
|
816
|
+
const restored = fingerprintMoved() ? null : await restoreOrMiss(preProbed.hit)
|
|
807
817
|
if (restored !== null) return restored
|
|
808
818
|
// The up-front probe's hit may be restoring AHEAD of this task's
|
|
809
819
|
// deps, and a command run now would build from outputs they have
|
|
@@ -912,6 +922,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
912
922
|
async function runAttempt(): Promise<{
|
|
913
923
|
result: ExecuteResult
|
|
914
924
|
exitCode: number
|
|
925
|
+
withdrawn?: true
|
|
915
926
|
}> {
|
|
916
927
|
// A deferred task's outputs are deliberately NOT coming, so wiping the
|
|
917
928
|
// tree would replace a stale build with nothing at all. The eligibility
|
|
@@ -947,17 +958,33 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
947
958
|
}
|
|
948
959
|
violations = []
|
|
949
960
|
const endReq = span('miss: build request')
|
|
950
|
-
const req = await buildRequest()
|
|
961
|
+
const req = await buildRequest(isLocalExecutor(args.executor))
|
|
951
962
|
endReq()
|
|
952
|
-
//
|
|
953
|
-
//
|
|
954
|
-
//
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
963
|
+
// A sibling's failure under --continue=never can land while a retry
|
|
964
|
+
// cleans or builds its request, after the loop's own check: the retry
|
|
965
|
+
// is withdrawn and the attempt before it stands.
|
|
966
|
+
if (attempt > 1 && args.failFast?.aborted === true) {
|
|
967
|
+
clearTimeout(timeoutTimer)
|
|
968
|
+
unlistenStop?.()
|
|
969
|
+
unlistenStop = undefined
|
|
970
|
+
await sweepPlaceholders(placeholders)
|
|
971
|
+
return { result, exitCode: effectiveExitCode, withdrawn: true }
|
|
972
|
+
}
|
|
973
|
+
// An executor that THROWS produces no captured output. Rethrown: the
|
|
974
|
+
// scheduler classifies it and prints its one line into the task's own
|
|
975
|
+
// stream (run()'s onError), where the frame reads it; a copy written
|
|
976
|
+
// here too printed the reason twice.
|
|
959
977
|
const endExec = span('miss: execute')
|
|
960
|
-
|
|
978
|
+
// A plugin's execute may throw before returning or return a bare result:
|
|
979
|
+
// the throw is its rejection, the value its resolution, so both reach
|
|
980
|
+
// the naming catch and the cleanup in finally below.
|
|
981
|
+
let running: Promise<unknown>
|
|
982
|
+
try {
|
|
983
|
+
running = Promise.resolve(args.executor.execute(req))
|
|
984
|
+
} catch (thrown) {
|
|
985
|
+
running = Promise.reject(thrown)
|
|
986
|
+
}
|
|
987
|
+
let res = await boundAfterAbort(running, req.signal)
|
|
961
988
|
.then((r: unknown) => {
|
|
962
989
|
assertExecuteResult(args.executor.name, node.id, r)
|
|
963
990
|
return r
|
|
@@ -973,23 +1000,19 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
973
1000
|
raw = new UserError(`${why}, and remote: 'only' keeps it off this machine`)
|
|
974
1001
|
} else {
|
|
975
1002
|
log.status(`[vx] ${node.id}: ${why} — running it here`)
|
|
976
|
-
|
|
977
|
-
const local = await localExecutor().execute(await buildRequest())
|
|
1003
|
+
const local = await localExecutor().execute(await buildRequest(true))
|
|
978
1004
|
assertExecuteResult('local', node.id, local)
|
|
979
1005
|
return local
|
|
980
1006
|
}
|
|
981
1007
|
}
|
|
982
1008
|
const err = nameExecutorFailure(args.executor, raw)
|
|
983
1009
|
// A remote executor's message carries the server's own text, which
|
|
984
|
-
// may echo the env it was sent: masked
|
|
985
|
-
//
|
|
1010
|
+
// may echo the env it was sent: masked on the error the scheduler
|
|
1011
|
+
// prints with its cause (L-39).
|
|
986
1012
|
if (secrets !== null) {
|
|
987
1013
|
for (const e of [err, err instanceof Error ? err.cause : undefined])
|
|
988
1014
|
if (e instanceof Error) e.message = secrets.mask(e.message)
|
|
989
1015
|
}
|
|
990
|
-
const message =
|
|
991
|
-
err instanceof Error ? err.message : (secrets?.mask(String(err)) ?? String(err))
|
|
992
|
-
log.taskStderr(node, `${message}\n`)
|
|
993
1016
|
await sweepPlaceholders(placeholders)
|
|
994
1017
|
throw err
|
|
995
1018
|
})
|
|
@@ -1008,11 +1031,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1008
1031
|
stdout: maskCaptured(res.stdout, secrets),
|
|
1009
1032
|
stderr: maskCaptured(res.stderr, secrets),
|
|
1010
1033
|
}
|
|
1011
|
-
// An executor that stopped on the timeout's abort
|
|
1012
|
-
//
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
}
|
|
1034
|
+
// An executor that stopped on the timeout's abort timed out, whatever
|
|
1035
|
+
// its exit: a child that traps the TERM and exits 0 left partial outputs
|
|
1036
|
+
// a pass would save (X-125), as the local runner's own `timedOut` says.
|
|
1037
|
+
// A local request carries no timer, so the runner alone decides there.
|
|
1038
|
+
if (timeoutFired && res.timedOut !== true) res = { ...res, timedOut: true }
|
|
1016
1039
|
violations = [...res.violations]
|
|
1017
1040
|
// A denial under a dependency the key does not answer for says only
|
|
1018
1041
|
// ENOENT; the line beside it names the package and how to key it. Only
|
|
@@ -1141,13 +1164,33 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1141
1164
|
args.executor.remote === true ? 'none' : undeclaredWriteReach(node, args.workspaceRoot)
|
|
1142
1165
|
const writesFingerprint =
|
|
1143
1166
|
args.executor.remote !== true && mayWriteFingerprint(node, args.workspaceRoot)
|
|
1167
|
+
// A stop that landed during the awaits above (the key, the probe, the
|
|
1168
|
+
// input description): the first attempt would wipe the last build and
|
|
1169
|
+
// hand the executor a request after the run had stopped. Through a call,
|
|
1170
|
+
// so the loop's own reads of the stop are not narrowed to false.
|
|
1171
|
+
if (isAborted(args.stopSignal)) {
|
|
1172
|
+
return {
|
|
1173
|
+
node,
|
|
1174
|
+
status: 'aborted',
|
|
1175
|
+
exitCode: signalExitCode(forwardedSignal(args.stopSignal?.reason)),
|
|
1176
|
+
durationMs: 0,
|
|
1177
|
+
hash,
|
|
1178
|
+
wallclockStartNs,
|
|
1179
|
+
wallclockEndNs: process.hrtime.bigint() - args.runStartHrTimeNs,
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1144
1182
|
for (;;) {
|
|
1145
1183
|
attempt++
|
|
1146
1184
|
const a = await runAttempt()
|
|
1147
|
-
forgetUndeclaredWrites(args, writeReach)
|
|
1148
|
-
if (writesFingerprint) args.fingerprintWatch?.wrote()
|
|
1149
1185
|
result = a.result
|
|
1150
1186
|
effectiveExitCode = a.exitCode
|
|
1187
|
+
if (a.withdrawn) {
|
|
1188
|
+
attempt--
|
|
1189
|
+
failedAttempts.pop()
|
|
1190
|
+
break
|
|
1191
|
+
}
|
|
1192
|
+
forgetUndeclaredWrites(args, writeReach)
|
|
1193
|
+
if (writesFingerprint) args.fingerprintWatch?.wrote()
|
|
1151
1194
|
spentMs += result.durationMs
|
|
1152
1195
|
|
|
1153
1196
|
// An attempt that ended in a shutdown never finished on its own terms —
|
|
@@ -1196,7 +1239,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1196
1239
|
)
|
|
1197
1240
|
}
|
|
1198
1241
|
|
|
1199
|
-
async function buildRequest(): Promise<ExecuteRequest> {
|
|
1242
|
+
async function buildRequest(local: boolean): Promise<ExecuteRequest> {
|
|
1200
1243
|
const out = secrets && maskedEmitter(secrets, (t) => log.taskStdout(node, t))
|
|
1201
1244
|
const err = secrets && maskedEmitter(secrets, (t) => log.taskStderr(node, t))
|
|
1202
1245
|
flushMasked = () => {
|
|
@@ -1234,10 +1277,10 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1234
1277
|
outputs: { files: outputs, workspaceFiles: wsOutputs },
|
|
1235
1278
|
...(args.terminal === true ? { terminal: true as const } : {}),
|
|
1236
1279
|
}
|
|
1237
|
-
// The signal last: it arms `exec.timeout`, and the sandbox's arming
|
|
1280
|
+
// The signal last: it arms a plugin's `exec.timeout`, and the sandbox's arming
|
|
1238
1281
|
// below is vx's work, not the task's. Armed first, a 60 ms timeout
|
|
1239
1282
|
// expired before the spawn and failed `echo` as timed out, unrun.
|
|
1240
|
-
if (!userSandbox) return { ...base, signal: requestSignal() }
|
|
1283
|
+
if (!userSandbox) return { ...base, signal: requestSignal(local) }
|
|
1241
1284
|
await args.armSandbox?.()
|
|
1242
1285
|
const sb = await sandboxRequestFor(
|
|
1243
1286
|
node,
|
|
@@ -1249,7 +1292,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1249
1292
|
)
|
|
1250
1293
|
placeholders = sb.placeholders
|
|
1251
1294
|
withheld = sb.withheld
|
|
1252
|
-
return { ...base, sandbox: sb.sandbox, signal: requestSignal() }
|
|
1295
|
+
return { ...base, sandbox: sb.sandbox, signal: requestSignal(local) }
|
|
1253
1296
|
}
|
|
1254
1297
|
|
|
1255
1298
|
const wallclockEndNs = process.hrtime.bigint() - args.runStartHrTimeNs
|
|
@@ -1260,12 +1303,15 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1260
1303
|
// record (a row with no artifact) is exactly the corrupt-entry shape
|
|
1261
1304
|
// `restoreOutputs` refuses, so writing none is the only clean answer.
|
|
1262
1305
|
// The closure is what a later local consumer — or a later eager run —
|
|
1263
|
-
// pulls the bytes with.
|
|
1306
|
+
// pulls the bytes with. Its fetch saves them under `hash` only past the
|
|
1307
|
+
// eager save's check: an input edited before the describe filed the
|
|
1308
|
+
// edit's bytes under the old key (X-123).
|
|
1264
1309
|
const materialize = result.outputs?.kind === 'deferred' ? result.outputs.materialize : undefined
|
|
1265
1310
|
if (materialize !== undefined) {
|
|
1311
|
+
const keyed = await keyStillTrue()
|
|
1266
1312
|
args.deferred?.register(node.id, {
|
|
1267
1313
|
materialize,
|
|
1268
|
-
hash,
|
|
1314
|
+
...(keyed ? { hash } : {}),
|
|
1269
1315
|
entry: {
|
|
1270
1316
|
taskId: node.id,
|
|
1271
1317
|
command: storedCommand,
|
|
@@ -21,32 +21,40 @@ export type FailureMode = 'stable' | 'flaky-recoverable' | 'flaky-fatal'
|
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* Per-key outcome projection: every keyed (project, task, hash) under
|
|
24
|
-
* `source` (a relation with `project`, `task`, `hash`,
|
|
25
|
-
* columns) with how often it failed
|
|
26
|
-
* an executed success or a cache hit, which replays a success
|
|
24
|
+
* `source` (`runs`, or a relation with its `id`, `project`, `task`, `hash`,
|
|
25
|
+
* `status`, `cache_hit` columns) with how often it failed, how often it
|
|
26
|
+
* passed — an executed success or a cache hit, which replays a success —
|
|
27
|
+
* and how often it failed AFTER its first pass (`relapses`). Every
|
|
27
28
|
* flakiness reader is a filter over this one projection.
|
|
28
29
|
*/
|
|
29
30
|
function keyOutcomesSql(source: string, where = ''): string {
|
|
30
|
-
return `SELECT project, task, hash,
|
|
31
|
-
SUM(
|
|
32
|
-
SUM(CASE WHEN
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
31
|
+
return `SELECT project, task, hash, SUM(failed) AS failures, SUM(passed) AS passes,
|
|
32
|
+
SUM(retried) AS retried_failures,
|
|
33
|
+
SUM(CASE WHEN failed = 1 AND id > first_pass THEN 1 ELSE 0 END) AS relapses
|
|
34
|
+
FROM (SELECT project, task, hash, id, failed, passed, retried,
|
|
35
|
+
MIN(CASE WHEN passed = 1 THEN id END) OVER (PARTITION BY project, task, hash) AS first_pass
|
|
36
|
+
FROM (SELECT project, task, hash, id,
|
|
37
|
+
CASE WHEN status = 'failed' THEN 1 ELSE 0 END AS failed,
|
|
38
|
+
CASE WHEN status = 'success' AND attempts > 1 THEN attempts - 1 ELSE 0 END AS retried,
|
|
39
|
+
CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
|
|
40
|
+
THEN 1 ELSE 0 END AS passed
|
|
41
|
+
FROM ${source}
|
|
42
|
+
WHERE ${KEYED_RUNS_SQL}${where}))
|
|
36
43
|
GROUP BY project, task, hash`
|
|
37
44
|
}
|
|
38
45
|
|
|
39
46
|
/**
|
|
40
47
|
* Per-key outcome subquery: the distinct cache keys under `source` that
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* legitimate break (a changed input that fails),
|
|
44
|
-
*
|
|
48
|
+
* FAILED AFTER THEY HAD PASSED — the definitional flake: identical inputs,
|
|
49
|
+
* a green result, then a red one. A failure whose key never succeeded is a
|
|
50
|
+
* legitimate break (a changed input that fails), and failures that end in
|
|
51
|
+
* a first pass are a recovery (a missing tool installed, a service brought
|
|
52
|
+
* up): neither is flakiness. One projection of the rule serves the
|
|
45
53
|
* all-time query below and the windowed one in `history.ts`.
|
|
46
54
|
*/
|
|
47
55
|
export function mixedOutcomeKeysSql(source: string, where = ''): string {
|
|
48
56
|
return `SELECT project, task FROM (${keyOutcomesSql(source, where)})
|
|
49
|
-
WHERE
|
|
57
|
+
WHERE relapses > 0`
|
|
50
58
|
}
|
|
51
59
|
|
|
52
60
|
/** Mixed-outcome keys of one (project, task) over its WHOLE recorded history. */
|
|
@@ -61,7 +69,7 @@ export function mixedOutcomeKeyCount(db: Database, project: string, task: string
|
|
|
61
69
|
|
|
62
70
|
/**
|
|
63
71
|
* The verdict. Flaky requires a NONDETERMINISM signal: a within-run retry,
|
|
64
|
-
* or a cache key that
|
|
72
|
+
* or a cache key that failed after it had passed (`mixedKeys`). Failures
|
|
65
73
|
* alone — each on its own key — are legitimate breaks, however many there
|
|
66
74
|
* are. `mixedKeys` is a thunk so a task that never failed is stable without
|
|
67
75
|
* the key count being computed at all.
|
|
@@ -118,9 +126,9 @@ const keyOf = (c: { project: string; task: string; hash: string }): string =>
|
|
|
118
126
|
|
|
119
127
|
/**
|
|
120
128
|
* Which of this run's executed tasks are flaky, judged BEFORE the run's own
|
|
121
|
-
* rows are recorded: a
|
|
122
|
-
*
|
|
123
|
-
* run in.
|
|
129
|
+
* rows are recorded: a failure on a key that has passed before, a pass on a
|
|
130
|
+
* key that already did so, or a within-run retry. A first pass after only
|
|
131
|
+
* failures is a recovery, not a flake. The counts fold this run in.
|
|
124
132
|
*
|
|
125
133
|
* The cost follows the run's colour. No candidate (every task a hit or a
|
|
126
134
|
* skip): no query. A green miss: one probe of the `runs_failed` partial
|
|
@@ -146,7 +154,7 @@ export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[])
|
|
|
146
154
|
(c) => c.status === 'failed' || c.attempts > 1 || everFailed.has(keyOf(c)),
|
|
147
155
|
)
|
|
148
156
|
if (suspects.length === 0) return []
|
|
149
|
-
const counts = new Map<string, { passes: number; failures: number }>()
|
|
157
|
+
const counts = new Map<string, { passes: number; failures: number; relapses: number }>()
|
|
150
158
|
chunked(suspects, (chunk) => {
|
|
151
159
|
const rows = db
|
|
152
160
|
.query(keyOutcomesSql('runs', ` AND hash IN (${chunk.map(() => '?').join(', ')})`))
|
|
@@ -156,27 +164,30 @@ export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[])
|
|
|
156
164
|
hash: string
|
|
157
165
|
passes: number
|
|
158
166
|
failures: number
|
|
167
|
+
relapses: number
|
|
159
168
|
}[]
|
|
160
|
-
for (const r of rows)
|
|
169
|
+
for (const r of rows)
|
|
170
|
+
counts.set(keyOf(r), { passes: r.passes, failures: r.failures, relapses: r.relapses })
|
|
161
171
|
})
|
|
162
172
|
const out: FlakyFinding[] = []
|
|
163
173
|
for (const c of suspects) {
|
|
164
|
-
const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0 }
|
|
174
|
+
const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0, relapses: 0 }
|
|
175
|
+
const relapsed = c.status === 'failed' ? before.passes > 0 : before.relapses > 0
|
|
176
|
+
if (!relapsed && c.attempts <= 1) continue
|
|
165
177
|
const passes = before.passes + (c.status === 'success' ? 1 : 0)
|
|
166
178
|
const failures = before.failures + (c.status === 'failed' ? 1 : 0)
|
|
167
|
-
if (!(passes > 0 && failures > 0) && c.attempts <= 1) continue
|
|
168
179
|
out.push({ ...c, taskId: `${c.project}#${c.task}`, passes, failures })
|
|
169
180
|
}
|
|
170
181
|
return out
|
|
171
182
|
}
|
|
172
183
|
|
|
173
|
-
/** One task the recorded history shows
|
|
184
|
+
/** One task the recorded history shows failing on a key that had passed. */
|
|
174
185
|
export interface FlakyTask {
|
|
175
186
|
/** `project#task`. */
|
|
176
187
|
taskId: string
|
|
177
188
|
project: string
|
|
178
189
|
task: string
|
|
179
|
-
/** Distinct cache keys that
|
|
190
|
+
/** Distinct cache keys that failed after they had passed. */
|
|
180
191
|
keys: number
|
|
181
192
|
/** Outcomes over those keys. */
|
|
182
193
|
passes: number
|
|
@@ -184,16 +195,20 @@ export interface FlakyTask {
|
|
|
184
195
|
}
|
|
185
196
|
|
|
186
197
|
/**
|
|
187
|
-
* Every task with a
|
|
198
|
+
* Every task with a key that failed after it had passed anywhere in the retained history
|
|
188
199
|
* (30 days, `RunHistory.pruneOlderThan`), most failures first: the
|
|
189
|
-
* doctor's list. A
|
|
200
|
+
* doctor's list. A pass that took a retry mixes its key on its own: it is
|
|
201
|
+
* one `success` row whose failed attempts count as failures (`vx info`
|
|
202
|
+
* said `none` over one). A full scan of `runs`, which an inspection verb
|
|
203
|
+
* affords.
|
|
190
204
|
*/
|
|
191
205
|
export function flakyTasks(db: Database): FlakyTask[] {
|
|
192
206
|
const rows = db
|
|
193
207
|
.query(
|
|
194
208
|
`SELECT project, task, COUNT(*) AS keys, SUM(passes) AS passes, SUM(failures) AS failures
|
|
195
|
-
FROM (
|
|
196
|
-
|
|
209
|
+
FROM (SELECT project, task, passes, failures + retried_failures AS failures
|
|
210
|
+
FROM (${keyOutcomesSql('runs')})
|
|
211
|
+
WHERE relapses > 0 OR retried_failures > 0)
|
|
197
212
|
GROUP BY project, task
|
|
198
213
|
ORDER BY failures DESC, passes DESC, project, task`,
|
|
199
214
|
)
|
|
@@ -27,10 +27,10 @@
|
|
|
27
27
|
|
|
28
28
|
import { withForwardArgs } from '../exec/index.js'
|
|
29
29
|
import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
30
|
-
import { maskedCommand } from '../util/index.js'
|
|
30
|
+
import { formatElapsed, maskedCommand } from '../util/index.js'
|
|
31
31
|
import { paint, type ColorSupport } from './colors.js'
|
|
32
|
-
import {
|
|
33
|
-
import { outcomeLabel, skippedReason } from './events.js'
|
|
32
|
+
import { neverStarted } from './summary.js'
|
|
33
|
+
import { outcomeLabel, ranNoCache, skippedReason } from './events.js'
|
|
34
34
|
|
|
35
35
|
const NO_COLOR: ColorSupport = { enabled: false }
|
|
36
36
|
|
|
@@ -220,7 +220,7 @@ const CACHE_COL = 8 // "no-cache" / "remote" / "local" / "fresh" / "miss"
|
|
|
220
220
|
|
|
221
221
|
/** Right-align the duration in a TIME_COL cell (pad on the left). */
|
|
222
222
|
function timeCell(ms: number | null, colors: ColorSupport): string {
|
|
223
|
-
const raw = ms === null ? '' :
|
|
223
|
+
const raw = ms === null ? '' : formatElapsed(ms)
|
|
224
224
|
return ' '.repeat(Math.max(0, TIME_COL - raw.length)) + paint('', raw, colors, { dim: true })
|
|
225
225
|
}
|
|
226
226
|
|
|
@@ -274,8 +274,8 @@ function cacheOf(o: TaskOutcome): { word: string; color: string } {
|
|
|
274
274
|
switch (o.status) {
|
|
275
275
|
case 'success':
|
|
276
276
|
case 'failed':
|
|
277
|
-
// A task
|
|
278
|
-
return { word: o
|
|
277
|
+
// A task no cache answered for never consulted one: not a miss.
|
|
278
|
+
return { word: ranNoCache(o) ? 'no-cache' : 'miss', color: '' } // dim
|
|
279
279
|
case 'cache-hit':
|
|
280
280
|
return o.restored === false
|
|
281
281
|
? { word: 'fresh', color: SUCCESS }
|
|
@@ -487,7 +487,7 @@ export function formatFrameClose(
|
|
|
487
487
|
// both ends of the frame as a long-lived task.
|
|
488
488
|
const tail =
|
|
489
489
|
persistent && outcome.status === 'success'
|
|
490
|
-
? ` ${paint('', `(${
|
|
490
|
+
? ` ${paint('', `(${formatElapsed(outcome.durationMs)})`, colors, { dim: true })} ${paint(ACCENT, 'running', colors)}`
|
|
491
491
|
: formatBlockFooter(outcome, colors)
|
|
492
492
|
const close = `${corner('└─')} ${mark}${paintTaskId(node, colors, { bold: true })} ${corner('──')}${tail}`
|
|
493
493
|
const violations = violationSection(outcome, colors)
|
|
@@ -594,7 +594,7 @@ function formatBlockFooter(o: TaskOutcome, colors: ColorSupport): string {
|
|
|
594
594
|
// saved" on the strength of it. The stored time lives on
|
|
595
595
|
// `TaskOutcome.storedDurationMs`.) Status differs by outcome — see
|
|
596
596
|
// formatStatusTag.
|
|
597
|
-
const dur = paint('', `(${
|
|
597
|
+
const dur = paint('', `(${formatElapsed(o.durationMs)})`, colors, { dim: true })
|
|
598
598
|
const tag = formatStatusTag(o, colors)
|
|
599
599
|
return ` ${dur} ${tag}${flakyNote(o, colors)}`
|
|
600
600
|
}
|
|
@@ -30,7 +30,6 @@ export { defaultLogger, resolveOutputView } from './logger.js'
|
|
|
30
30
|
export { detectColors, paint, type ColorSupport } from './colors.js'
|
|
31
31
|
export type { Logger, OutputView } from './logger.js'
|
|
32
32
|
export type { RunPlan, PlannedTask, PlanPrediction, CacheStatus } from './plan.js'
|
|
33
|
-
export { formatDuration } from './summary.js'
|
|
34
33
|
export {
|
|
35
34
|
createEventBus,
|
|
36
35
|
wireForwarder,
|
|
@@ -93,9 +93,12 @@ export function foldedDeps(node: TaskNode, nodeOf: (id: string) => TaskNode): Fo
|
|
|
93
93
|
* the hashes are. An exec task's key folds its own id (`task:<id>`), so its
|
|
94
94
|
* id is enough; a group's hash is a roll-up of its members' ids and hashes
|
|
95
95
|
* with no id of its own, so two groups over the same members hash alike.
|
|
96
|
+
* The members are `keyUpstream`'s, which `computeGroupHash` is handed: under
|
|
97
|
+
* `--exclude-dependencies` `node.deps` holds order-only edges and lacks the
|
|
98
|
+
* dropped ones, and two groups the key told apart shared a unit.
|
|
96
99
|
*/
|
|
97
100
|
function foldUnit(node: TaskNode): string {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
if (!isGroupTask(node) || node.config.cache !== undefined) return `task|${node.id}`
|
|
102
|
+
const members = [...keyedDeps(node), ...(node.excludedUpstream ?? []).map((u) => u.node.id)]
|
|
103
|
+
return `group|${members.sort().join('|')}`
|
|
101
104
|
}
|
|
@@ -19,11 +19,18 @@ import {
|
|
|
19
19
|
type StatusStream,
|
|
20
20
|
type WorkerSlot,
|
|
21
21
|
} from './status-line.js'
|
|
22
|
-
import {
|
|
22
|
+
import { formatSummarySection, neverStarted, type RunContext } from './summary.js'
|
|
23
23
|
import { isGroupTask } from '../graph/index.js'
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
appendTail,
|
|
26
|
+
createTail,
|
|
27
|
+
formatElapsed,
|
|
28
|
+
resetTail,
|
|
29
|
+
tailText,
|
|
30
|
+
type Tail,
|
|
31
|
+
} from '../util/index.js'
|
|
25
32
|
import { isCacheHit } from './telemetry.js'
|
|
26
|
-
import { failedLabel, outcomeWord } from './events.js'
|
|
33
|
+
import { failedLabel, outcomeWord, ranNoCache } from './events.js'
|
|
27
34
|
|
|
28
35
|
export interface Logger {
|
|
29
36
|
/** Header / footer / status text. Written verbatim, one trailing \n added. */
|
|
@@ -139,8 +146,7 @@ function ghaFence(body: string, token: string): string {
|
|
|
139
146
|
return `::stop-commands::${token}\n${body}::${token}::\n`
|
|
140
147
|
}
|
|
141
148
|
|
|
142
|
-
const cacheWordOf = (
|
|
143
|
-
n.config.cache === undefined ? 'no-cache' : 'miss'
|
|
149
|
+
const cacheWordOf = (o: TaskOutcome): 'miss' | 'no-cache' => (ranNoCache(o) ? 'no-cache' : 'miss')
|
|
144
150
|
|
|
145
151
|
export function resolveOutputView(
|
|
146
152
|
options: {
|
|
@@ -440,7 +446,7 @@ export function defaultLogger(
|
|
|
440
446
|
(view.mode === 'focused' && isPrimary(node) && !isGroupTask(node) && requestedCount <= 1)
|
|
441
447
|
|
|
442
448
|
const failureRow = (node: TaskNode, outcome: TaskOutcome): string =>
|
|
443
|
-
formatFailureLine(node.id, outcome.durationMs, colors, cacheWordOf(
|
|
449
|
+
formatFailureLine(node.id, outcome.durationMs, colors, cacheWordOf(outcome)) +
|
|
444
450
|
flakyNote(outcome, colors)
|
|
445
451
|
|
|
446
452
|
return {
|
|
@@ -453,6 +459,12 @@ export function defaultLogger(
|
|
|
453
459
|
// A kept server's held partial line is older than this line: held
|
|
454
460
|
// past it, a server's last words printed below its own exit notice.
|
|
455
461
|
flushKept()
|
|
462
|
+
// A live-streamed server keeps writing after its frame closes, so a
|
|
463
|
+
// summary row or "exited with code" can follow its partial line.
|
|
464
|
+
if (streamMidLine) {
|
|
465
|
+
writer.write('\n')
|
|
466
|
+
streamMidLine = false
|
|
467
|
+
}
|
|
456
468
|
writer.write(`${line}\n`)
|
|
457
469
|
},
|
|
458
470
|
runStart(info) {
|
|
@@ -654,7 +666,7 @@ export function defaultLogger(
|
|
|
654
666
|
done++
|
|
655
667
|
if (outcome.status === 'failed') {
|
|
656
668
|
failed++
|
|
657
|
-
if (
|
|
669
|
+
if (ranNoCache(outcome)) noCache++
|
|
658
670
|
} else if (node.config.exec?.persistent !== undefined && outcome.status === 'success') {
|
|
659
671
|
// A persistent task's outcome arrives at READY; the child
|
|
660
672
|
// keeps running until the orchestrator SIGTERMs it at run
|
|
@@ -678,7 +690,7 @@ export function defaultLogger(
|
|
|
678
690
|
switch (outcome.status) {
|
|
679
691
|
case 'success':
|
|
680
692
|
succeeded++
|
|
681
|
-
if (
|
|
693
|
+
if (ranNoCache(outcome)) noCache++
|
|
682
694
|
break
|
|
683
695
|
case 'cache-hit':
|
|
684
696
|
if (outcome.restored === false) upToDate++
|
|
@@ -875,7 +887,7 @@ export function defaultLogger(
|
|
|
875
887
|
} else {
|
|
876
888
|
emitBlock(
|
|
877
889
|
`::group::${ghaData(
|
|
878
|
-
`${node.id} (${outcomeWord(outcome)} ${
|
|
890
|
+
`${node.id} (${outcomeWord(outcome)} ${formatElapsed(outcome.durationMs)})`,
|
|
879
891
|
)}\n${body}::endgroup::\n`,
|
|
880
892
|
)
|
|
881
893
|
}
|
|
@@ -407,11 +407,18 @@ function unchangedKeyNote(
|
|
|
407
407
|
at: number
|
|
408
408
|
} | null
|
|
409
409
|
)?.at
|
|
410
|
-
//
|
|
411
|
-
//
|
|
412
|
-
// (admission.ts), which runs a task downstream of
|
|
413
|
-
// saves it. Blaming `--no-cache` / `--force` named a
|
|
414
|
-
|
|
410
|
+
// The previous run executed the task and succeeded with the cache
|
|
411
|
+
// writable beside a failure, and no entry older than this run holds the
|
|
412
|
+
// key: the continue-taint (admission.ts), which runs a task downstream of
|
|
413
|
+
// a failure and never saves it. Blaming `--no-cache` / `--force` named a
|
|
414
|
+
// flag nobody passed, and a prune when this run saved the key named an
|
|
415
|
+
// eviction that never happened.
|
|
416
|
+
if (
|
|
417
|
+
(createdAt === undefined || createdAt >= this_.startedAt) &&
|
|
418
|
+
this_.status === 'success' &&
|
|
419
|
+
prev.status === 'success' &&
|
|
420
|
+
prev.cacheHit === 0
|
|
421
|
+
) {
|
|
415
422
|
const failedWithWrites = (id: string | null): number => {
|
|
416
423
|
if (id === null) return 0
|
|
417
424
|
const row = db
|
|
@@ -420,9 +427,13 @@ function unchangedKeyNote(
|
|
|
420
427
|
const writes = row?.p.split(',').some((axis) => axis === 'lW' || axis === 'rW') === true
|
|
421
428
|
return writes ? (row?.f ?? 0) : 0
|
|
422
429
|
}
|
|
423
|
-
const
|
|
424
|
-
if (
|
|
425
|
-
|
|
430
|
+
const before = failedWithWrites(prev.runId)
|
|
431
|
+
if (before > 0) {
|
|
432
|
+
const now = failedWithWrites(runId)
|
|
433
|
+
if (createdAt === undefined && now > 0) {
|
|
434
|
+
return `cache key unchanged — neither run saved it: each ran beside a failed task (${now} failed in this one), and a task run past a failed dependency (--continue) is never cached`
|
|
435
|
+
}
|
|
436
|
+
return `cache key unchanged — the previous run on this key ran beside a failed task (${before} failed in it) and was not saved: a task run past a failed dependency (--continue) is never cached, so there was nothing to hit`
|
|
426
437
|
}
|
|
427
438
|
}
|
|
428
439
|
if (createdAt !== undefined && createdAt >= this_.startedAt) {
|