@vzn/vx 0.0.596 → 0.0.597
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/launcher.cjs +7 -3
- package/package.json +5 -5
- package/src/cache/archive.ts +35 -6
- package/src/cache/cache.ts +31 -15
- package/src/cache/git-inputs.ts +23 -0
- package/src/cache/index.ts +2 -0
- package/src/cache/inputs.ts +87 -32
- package/src/cache/key-fold.ts +1 -1
- package/src/cache/layer.ts +2 -1
- package/src/cli/cache.ts +2 -2
- package/src/cli/foreign-flags.ts +56 -2
- package/src/cli/info.ts +1 -1
- package/src/cli/init.ts +2 -2
- package/src/cli/last.ts +6 -3
- package/src/cli/lock.ts +5 -3
- package/src/cli/plan-format.ts +4 -4
- package/src/cli/run.ts +42 -15
- package/src/cli/select.ts +6 -3
- package/src/cli/show.ts +8 -2
- package/src/cli/upgrade.ts +1 -1
- package/src/cli/watch-fs.ts +11 -3
- package/src/cli/watch.ts +14 -7
- package/src/cli/workspace-config.ts +3 -1
- package/src/config.ts +21 -0
- package/src/exec/executor.ts +1 -1
- package/src/exec/kill-tree.ts +89 -18
- package/src/exec/runner.ts +74 -36
- package/src/exec/sandbox-paths.ts +11 -0
- package/src/exec/sandbox-runtime.ts +168 -49
- package/src/exec/sandbox-violations.ts +59 -6
- package/src/graph/scheduler.ts +2 -2
- package/src/graph/task-graph.ts +145 -33
- package/src/orchestrator/affected-tasks.ts +6 -2
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/doctor.ts +2 -2
- package/src/orchestrator/excluded-keys.ts +2 -13
- package/src/orchestrator/execute-task.ts +49 -26
- package/src/orchestrator/failure-mode.ts +38 -29
- package/src/orchestrator/framed-output.ts +5 -5
- package/src/orchestrator/index.ts +0 -1
- package/src/orchestrator/keyed-projects.ts +6 -3
- package/src/orchestrator/logger.ts +10 -3
- package/src/orchestrator/miss-save.ts +7 -11
- package/src/orchestrator/plan.ts +4 -20
- package/src/orchestrator/run-artifacts.ts +2 -2
- package/src/orchestrator/run-lock.ts +25 -24
- package/src/orchestrator/run.ts +2 -1
- package/src/orchestrator/sandbox-request.ts +31 -9
- package/src/orchestrator/signals.ts +22 -4
- package/src/orchestrator/stable-keys.ts +3 -11
- package/src/orchestrator/summary.ts +5 -9
- package/src/orchestrator/task-hash.ts +13 -3
- package/src/orchestrator/upstream.ts +11 -0
- package/src/util/hangup.ts +42 -0
- package/src/util/index.ts +2 -0
- package/src/util/paths.ts +1 -1
- package/src/workspace/affected.ts +79 -42
- package/src/workspace/config-eval.ts +27 -6
- package/src/workspace/config-imports.ts +8 -55
- package/src/workspace/config-schema.ts +30 -6
- package/src/workspace/lockfile.ts +2 -1
- package/src/workspace/migrate-scripts.ts +1 -1
- package/src/workspace/project-loader.ts +15 -2
- package/src/workspace/workspace.ts +36 -14
|
@@ -21,34 +21,40 @@ export type FailureMode = 'stable' | 'flaky-recoverable' | 'flaky-fatal'
|
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* Per-key outcome projection: every keyed (project, task, hash) under
|
|
24
|
-
* `source` (a relation with `project`, `task`, `hash`,
|
|
25
|
-
* columns) with how often it failed
|
|
26
|
-
* an executed success or a cache hit, which replays a success
|
|
24
|
+
* `source` (`runs`, or a relation with its `id`, `project`, `task`, `hash`,
|
|
25
|
+
* `status`, `cache_hit` columns) with how often it failed, how often it
|
|
26
|
+
* passed — an executed success or a cache hit, which replays a success —
|
|
27
|
+
* and how often it failed AFTER its first pass (`relapses`). Every
|
|
27
28
|
* flakiness reader is a filter over this one projection.
|
|
28
29
|
*/
|
|
29
30
|
function keyOutcomesSql(source: string, where = ''): string {
|
|
30
|
-
return `SELECT project, task, hash,
|
|
31
|
-
SUM(
|
|
32
|
-
SUM(CASE WHEN
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
31
|
+
return `SELECT project, task, hash, SUM(failed) AS failures, SUM(passed) AS passes,
|
|
32
|
+
SUM(retried) AS retried_failures,
|
|
33
|
+
SUM(CASE WHEN failed = 1 AND id > first_pass THEN 1 ELSE 0 END) AS relapses
|
|
34
|
+
FROM (SELECT project, task, hash, id, failed, passed, retried,
|
|
35
|
+
MIN(CASE WHEN passed = 1 THEN id END) OVER (PARTITION BY project, task, hash) AS first_pass
|
|
36
|
+
FROM (SELECT project, task, hash, id,
|
|
37
|
+
CASE WHEN status = 'failed' THEN 1 ELSE 0 END AS failed,
|
|
38
|
+
CASE WHEN status = 'success' AND attempts > 1 THEN attempts - 1 ELSE 0 END AS retried,
|
|
39
|
+
CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
|
|
40
|
+
THEN 1 ELSE 0 END AS passed
|
|
41
|
+
FROM ${source}
|
|
42
|
+
WHERE ${KEYED_RUNS_SQL}${where}))
|
|
38
43
|
GROUP BY project, task, hash`
|
|
39
44
|
}
|
|
40
45
|
|
|
41
46
|
/**
|
|
42
47
|
* Per-key outcome subquery: the distinct cache keys under `source` that
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* legitimate break (a changed input that fails),
|
|
46
|
-
*
|
|
48
|
+
* FAILED AFTER THEY HAD PASSED — the definitional flake: identical inputs,
|
|
49
|
+
* a green result, then a red one. A failure whose key never succeeded is a
|
|
50
|
+
* legitimate break (a changed input that fails), and failures that end in
|
|
51
|
+
* a first pass are a recovery (a missing tool installed, a service brought
|
|
52
|
+
* up): neither is flakiness. One projection of the rule serves the
|
|
47
53
|
* all-time query below and the windowed one in `history.ts`.
|
|
48
54
|
*/
|
|
49
55
|
export function mixedOutcomeKeysSql(source: string, where = ''): string {
|
|
50
56
|
return `SELECT project, task FROM (${keyOutcomesSql(source, where)})
|
|
51
|
-
WHERE
|
|
57
|
+
WHERE relapses > 0`
|
|
52
58
|
}
|
|
53
59
|
|
|
54
60
|
/** Mixed-outcome keys of one (project, task) over its WHOLE recorded history. */
|
|
@@ -63,7 +69,7 @@ export function mixedOutcomeKeyCount(db: Database, project: string, task: string
|
|
|
63
69
|
|
|
64
70
|
/**
|
|
65
71
|
* The verdict. Flaky requires a NONDETERMINISM signal: a within-run retry,
|
|
66
|
-
* or a cache key that
|
|
72
|
+
* or a cache key that failed after it had passed (`mixedKeys`). Failures
|
|
67
73
|
* alone — each on its own key — are legitimate breaks, however many there
|
|
68
74
|
* are. `mixedKeys` is a thunk so a task that never failed is stable without
|
|
69
75
|
* the key count being computed at all.
|
|
@@ -120,9 +126,9 @@ const keyOf = (c: { project: string; task: string; hash: string }): string =>
|
|
|
120
126
|
|
|
121
127
|
/**
|
|
122
128
|
* Which of this run's executed tasks are flaky, judged BEFORE the run's own
|
|
123
|
-
* rows are recorded: a
|
|
124
|
-
*
|
|
125
|
-
* run in.
|
|
129
|
+
* rows are recorded: a failure on a key that has passed before, a pass on a
|
|
130
|
+
* key that already did so, or a within-run retry. A first pass after only
|
|
131
|
+
* failures is a recovery, not a flake. The counts fold this run in.
|
|
126
132
|
*
|
|
127
133
|
* The cost follows the run's colour. No candidate (every task a hit or a
|
|
128
134
|
* skip): no query. A green miss: one probe of the `runs_failed` partial
|
|
@@ -148,7 +154,7 @@ export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[])
|
|
|
148
154
|
(c) => c.status === 'failed' || c.attempts > 1 || everFailed.has(keyOf(c)),
|
|
149
155
|
)
|
|
150
156
|
if (suspects.length === 0) return []
|
|
151
|
-
const counts = new Map<string, { passes: number; failures: number }>()
|
|
157
|
+
const counts = new Map<string, { passes: number; failures: number; relapses: number }>()
|
|
152
158
|
chunked(suspects, (chunk) => {
|
|
153
159
|
const rows = db
|
|
154
160
|
.query(keyOutcomesSql('runs', ` AND hash IN (${chunk.map(() => '?').join(', ')})`))
|
|
@@ -158,27 +164,30 @@ export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[])
|
|
|
158
164
|
hash: string
|
|
159
165
|
passes: number
|
|
160
166
|
failures: number
|
|
167
|
+
relapses: number
|
|
161
168
|
}[]
|
|
162
|
-
for (const r of rows)
|
|
169
|
+
for (const r of rows)
|
|
170
|
+
counts.set(keyOf(r), { passes: r.passes, failures: r.failures, relapses: r.relapses })
|
|
163
171
|
})
|
|
164
172
|
const out: FlakyFinding[] = []
|
|
165
173
|
for (const c of suspects) {
|
|
166
|
-
const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0 }
|
|
174
|
+
const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0, relapses: 0 }
|
|
175
|
+
const relapsed = c.status === 'failed' ? before.passes > 0 : before.relapses > 0
|
|
176
|
+
if (!relapsed && c.attempts <= 1) continue
|
|
167
177
|
const passes = before.passes + (c.status === 'success' ? 1 : 0)
|
|
168
178
|
const failures = before.failures + (c.status === 'failed' ? 1 : 0)
|
|
169
|
-
if (!(passes > 0 && failures > 0) && c.attempts <= 1) continue
|
|
170
179
|
out.push({ ...c, taskId: `${c.project}#${c.task}`, passes, failures })
|
|
171
180
|
}
|
|
172
181
|
return out
|
|
173
182
|
}
|
|
174
183
|
|
|
175
|
-
/** One task the recorded history shows
|
|
184
|
+
/** One task the recorded history shows failing on a key that had passed. */
|
|
176
185
|
export interface FlakyTask {
|
|
177
186
|
/** `project#task`. */
|
|
178
187
|
taskId: string
|
|
179
188
|
project: string
|
|
180
189
|
task: string
|
|
181
|
-
/** Distinct cache keys that
|
|
190
|
+
/** Distinct cache keys that failed after they had passed. */
|
|
182
191
|
keys: number
|
|
183
192
|
/** Outcomes over those keys. */
|
|
184
193
|
passes: number
|
|
@@ -186,7 +195,7 @@ export interface FlakyTask {
|
|
|
186
195
|
}
|
|
187
196
|
|
|
188
197
|
/**
|
|
189
|
-
* Every task with a
|
|
198
|
+
* Every task with a key that failed after it had passed anywhere in the retained history
|
|
190
199
|
* (30 days, `RunHistory.pruneOlderThan`), most failures first: the
|
|
191
200
|
* doctor's list. A pass that took a retry mixes its key on its own: it is
|
|
192
201
|
* one `success` row whose failed attempts count as failures (`vx info`
|
|
@@ -198,8 +207,8 @@ export function flakyTasks(db: Database): FlakyTask[] {
|
|
|
198
207
|
.query(
|
|
199
208
|
`SELECT project, task, COUNT(*) AS keys, SUM(passes) AS passes, SUM(failures) AS failures
|
|
200
209
|
FROM (SELECT project, task, passes, failures + retried_failures AS failures
|
|
201
|
-
FROM (${keyOutcomesSql('runs')})
|
|
202
|
-
|
|
210
|
+
FROM (${keyOutcomesSql('runs')})
|
|
211
|
+
WHERE relapses > 0 OR retried_failures > 0)
|
|
203
212
|
GROUP BY project, task
|
|
204
213
|
ORDER BY failures DESC, passes DESC, project, task`,
|
|
205
214
|
)
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
|
|
28
28
|
import { withForwardArgs } from '../exec/index.js'
|
|
29
29
|
import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
30
|
-
import { maskedCommand } from '../util/index.js'
|
|
30
|
+
import { formatElapsed, maskedCommand } from '../util/index.js'
|
|
31
31
|
import { paint, type ColorSupport } from './colors.js'
|
|
32
|
-
import {
|
|
32
|
+
import { neverStarted } from './summary.js'
|
|
33
33
|
import { outcomeLabel, ranNoCache, skippedReason } from './events.js'
|
|
34
34
|
|
|
35
35
|
const NO_COLOR: ColorSupport = { enabled: false }
|
|
@@ -220,7 +220,7 @@ const CACHE_COL = 8 // "no-cache" / "remote" / "local" / "fresh" / "miss"
|
|
|
220
220
|
|
|
221
221
|
/** Right-align the duration in a TIME_COL cell (pad on the left). */
|
|
222
222
|
function timeCell(ms: number | null, colors: ColorSupport): string {
|
|
223
|
-
const raw = ms === null ? '' :
|
|
223
|
+
const raw = ms === null ? '' : formatElapsed(ms)
|
|
224
224
|
return ' '.repeat(Math.max(0, TIME_COL - raw.length)) + paint('', raw, colors, { dim: true })
|
|
225
225
|
}
|
|
226
226
|
|
|
@@ -487,7 +487,7 @@ export function formatFrameClose(
|
|
|
487
487
|
// both ends of the frame as a long-lived task.
|
|
488
488
|
const tail =
|
|
489
489
|
persistent && outcome.status === 'success'
|
|
490
|
-
? ` ${paint('', `(${
|
|
490
|
+
? ` ${paint('', `(${formatElapsed(outcome.durationMs)})`, colors, { dim: true })} ${paint(ACCENT, 'running', colors)}`
|
|
491
491
|
: formatBlockFooter(outcome, colors)
|
|
492
492
|
const close = `${corner('└─')} ${mark}${paintTaskId(node, colors, { bold: true })} ${corner('──')}${tail}`
|
|
493
493
|
const violations = violationSection(outcome, colors)
|
|
@@ -594,7 +594,7 @@ function formatBlockFooter(o: TaskOutcome, colors: ColorSupport): string {
|
|
|
594
594
|
// saved" on the strength of it. The stored time lives on
|
|
595
595
|
// `TaskOutcome.storedDurationMs`.) Status differs by outcome — see
|
|
596
596
|
// formatStatusTag.
|
|
597
|
-
const dur = paint('', `(${
|
|
597
|
+
const dur = paint('', `(${formatElapsed(o.durationMs)})`, colors, { dim: true })
|
|
598
598
|
const tag = formatStatusTag(o, colors)
|
|
599
599
|
return ` ${dur} ${tag}${flakyNote(o, colors)}`
|
|
600
600
|
}
|
|
@@ -30,7 +30,6 @@ export { defaultLogger, resolveOutputView } from './logger.js'
|
|
|
30
30
|
export { detectColors, paint, type ColorSupport } from './colors.js'
|
|
31
31
|
export type { Logger, OutputView } from './logger.js'
|
|
32
32
|
export type { RunPlan, PlannedTask, PlanPrediction, CacheStatus } from './plan.js'
|
|
33
|
-
export { formatDuration } from './summary.js'
|
|
34
33
|
export {
|
|
35
34
|
createEventBus,
|
|
36
35
|
wireForwarder,
|
|
@@ -93,9 +93,12 @@ export function foldedDeps(node: TaskNode, nodeOf: (id: string) => TaskNode): Fo
|
|
|
93
93
|
* the hashes are. An exec task's key folds its own id (`task:<id>`), so its
|
|
94
94
|
* id is enough; a group's hash is a roll-up of its members' ids and hashes
|
|
95
95
|
* with no id of its own, so two groups over the same members hash alike.
|
|
96
|
+
* The members are `keyUpstream`'s, which `computeGroupHash` is handed: under
|
|
97
|
+
* `--exclude-dependencies` `node.deps` holds order-only edges and lacks the
|
|
98
|
+
* dropped ones, and two groups the key told apart shared a unit.
|
|
96
99
|
*/
|
|
97
100
|
function foldUnit(node: TaskNode): string {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
if (!isGroupTask(node) || node.config.cache !== undefined) return `task|${node.id}`
|
|
102
|
+
const members = [...keyedDeps(node), ...(node.excludedUpstream ?? []).map((u) => u.node.id)]
|
|
103
|
+
return `group|${members.sort().join('|')}`
|
|
101
104
|
}
|
|
@@ -19,9 +19,16 @@ import {
|
|
|
19
19
|
type StatusStream,
|
|
20
20
|
type WorkerSlot,
|
|
21
21
|
} from './status-line.js'
|
|
22
|
-
import {
|
|
22
|
+
import { formatSummarySection, neverStarted, type RunContext } from './summary.js'
|
|
23
23
|
import { isGroupTask } from '../graph/index.js'
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
appendTail,
|
|
26
|
+
createTail,
|
|
27
|
+
formatElapsed,
|
|
28
|
+
resetTail,
|
|
29
|
+
tailText,
|
|
30
|
+
type Tail,
|
|
31
|
+
} from '../util/index.js'
|
|
25
32
|
import { isCacheHit } from './telemetry.js'
|
|
26
33
|
import { failedLabel, outcomeWord, ranNoCache } from './events.js'
|
|
27
34
|
|
|
@@ -880,7 +887,7 @@ export function defaultLogger(
|
|
|
880
887
|
} else {
|
|
881
888
|
emitBlock(
|
|
882
889
|
`::group::${ghaData(
|
|
883
|
-
`${node.id} (${outcomeWord(outcome)} ${
|
|
890
|
+
`${node.id} (${outcomeWord(outcome)} ${formatElapsed(outcome.durationMs)})`,
|
|
884
891
|
)}\n${body}::endgroup::\n`,
|
|
885
892
|
)
|
|
886
893
|
}
|
|
@@ -148,11 +148,9 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
148
148
|
// `cache.outputs` is not a write grant; `exec.sandbox.allow.write` is.
|
|
149
149
|
const sandboxed = node.config.exec?.sandbox !== undefined
|
|
150
150
|
const grantsWrite = (node.config.exec?.sandbox?.allow?.write?.length ?? 0) > 0
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
const linkedOut =
|
|
154
|
-
outputDirLinkedOut(node.projectDir, a.outputs, 'the project') ??
|
|
155
|
-
outputDirLinkedOut(a.workspaceRoot, a.wsOutputs, 'the workspace')
|
|
151
|
+
// A `workspaceFiles` directory linked out of the workspace drops every
|
|
152
|
+
// file (M-65); a project one is refused by the resolver (X-88).
|
|
153
|
+
const linkedOut = outputDirLinkedOut(a.workspaceRoot, a.wsOutputs)
|
|
156
154
|
log.status(
|
|
157
155
|
`[vx] ${node.id}: cache.outputs matched no files (${[...a.outputs, ...a.wsOutputs].join(', ')}) — ` +
|
|
158
156
|
`an empty artifact is saved; a later hit restores nothing` +
|
|
@@ -300,15 +298,13 @@ function markWritten(
|
|
|
300
298
|
}
|
|
301
299
|
|
|
302
300
|
/**
|
|
303
|
-
* The first
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
* empty-artifact warning blamed the glob (M-61, M-65).
|
|
301
|
+
* The first `workspaceFiles` directory that is a symlink resolving outside
|
|
302
|
+
* the workspace root: every file under it is dropped as outside, and the
|
|
303
|
+
* empty-artifact warning blamed the glob (M-65).
|
|
307
304
|
*/
|
|
308
305
|
function outputDirLinkedOut(
|
|
309
306
|
baseDir: string,
|
|
310
307
|
outputs: readonly string[],
|
|
311
|
-
base: string,
|
|
312
308
|
): { dir: string; target: string; base: string } | undefined {
|
|
313
309
|
const realProject = realpathOrNull(baseDir) ?? baseDir
|
|
314
310
|
// `dist` names the tree `dist/**`: the same reading the resolver gives.
|
|
@@ -326,7 +322,7 @@ function outputDirLinkedOut(
|
|
|
326
322
|
}
|
|
327
323
|
const target = realpathOrNull(abs)
|
|
328
324
|
if (target !== null && target !== realProject && !target.startsWith(realProject + path.sep)) {
|
|
329
|
-
return { dir, target, base }
|
|
325
|
+
return { dir, target, base: 'the workspace' }
|
|
330
326
|
}
|
|
331
327
|
}
|
|
332
328
|
}
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -14,16 +14,10 @@
|
|
|
14
14
|
import type { CacheLayer, CachePolicy, GitFilesCache } from '../cache/index.js'
|
|
15
15
|
import { cachesNothing, FULL_CACHE_POLICY } from '../cache/index.js'
|
|
16
16
|
import { ranNoCache } from './events.js'
|
|
17
|
-
import {
|
|
18
|
-
idleServers,
|
|
19
|
-
isGroupTask,
|
|
20
|
-
runGraph,
|
|
21
|
-
type TaskNode,
|
|
22
|
-
type TaskOutcome,
|
|
23
|
-
} from '../graph/index.js'
|
|
17
|
+
import { idleServers, isGroupTask, runGraph, type TaskNode } from '../graph/index.js'
|
|
24
18
|
import type { HistoryProvider } from './history.js'
|
|
25
19
|
import { computeGroupKey, computeTaskHash } from './task-hash.js'
|
|
26
|
-
import { keyUpstream } from './upstream.js'
|
|
20
|
+
import { keyedOutcome, keyUpstream } from './upstream.js'
|
|
27
21
|
|
|
28
22
|
export type CacheStatus =
|
|
29
23
|
| 'hit-local' // entry exists in local cache
|
|
@@ -157,7 +151,7 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
157
151
|
}
|
|
158
152
|
if (isGroupTask(node)) {
|
|
159
153
|
cacheStatusById.set(node.id, 'group')
|
|
160
|
-
return
|
|
154
|
+
return keyedOutcome(node, await computeGroupKey(hashArgs))
|
|
161
155
|
}
|
|
162
156
|
|
|
163
157
|
const hash = await computeTaskHash(hashArgs)
|
|
@@ -180,7 +174,7 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
180
174
|
status = where === null ? 'miss' : where === 'remote' ? 'hit-remote' : 'hit-local'
|
|
181
175
|
}
|
|
182
176
|
cacheStatusById.set(node.id, status)
|
|
183
|
-
return
|
|
177
|
+
return keyedOutcome(node, hash)
|
|
184
178
|
},
|
|
185
179
|
})
|
|
186
180
|
|
|
@@ -303,13 +297,3 @@ function predictPlan(tasks: PlannedTask[]): PlanPrediction {
|
|
|
303
297
|
}
|
|
304
298
|
return { wallMs, workMs, unknownCount }
|
|
305
299
|
}
|
|
306
|
-
|
|
307
|
-
function planOutcome(node: TaskNode, hash: string | undefined): TaskOutcome {
|
|
308
|
-
return {
|
|
309
|
-
node,
|
|
310
|
-
status: 'success',
|
|
311
|
-
exitCode: 0,
|
|
312
|
-
durationMs: 0,
|
|
313
|
-
...(hash !== undefined ? { hash } : {}),
|
|
314
|
-
}
|
|
315
|
-
}
|
|
@@ -94,8 +94,8 @@ function taskEntry(o: TaskOutcome, flaky?: FlakyFinding): SummaryTaskJson {
|
|
|
94
94
|
// task that could never hit (its `hash` is still set — dependents fold
|
|
95
95
|
// it). Present only when true, so every other row is byte-identical.
|
|
96
96
|
...(ranNoCache(o) ? { noCache: true } : {}),
|
|
97
|
-
// Present only when the run proved the task flaky: this exact key
|
|
98
|
-
//
|
|
97
|
+
// Present only when the run proved the task flaky: this exact key
|
|
98
|
+
// failed after it had passed (this run counted), or the task
|
|
99
99
|
// needed a retry. A consumer gating on `failed` can tell a break from a
|
|
100
100
|
// flake without the history; every other row is byte-identical.
|
|
101
101
|
...(flaky !== undefined
|
|
@@ -43,7 +43,8 @@
|
|
|
43
43
|
// waited for itself forever (nx#36473 reproduced on vx, 2026-09-24). So a
|
|
44
44
|
// lock naming OUR pid is stale unless this process placed that entry and
|
|
45
45
|
// has not left it (`placed`) — and on Linux the entry also carries the holder's start
|
|
46
|
-
// time, so a pid another process now wears is stale too
|
|
46
|
+
// time, so a pid another process now wears is stale too, and so is a
|
|
47
|
+
// holder procfs shows as a zombie (killed, not yet reaped). Where the lock
|
|
47
48
|
// cannot be made or read for any reason but "held" (a temp directory this
|
|
48
49
|
// user cannot write), the run says so once and proceeds unlocked: the
|
|
49
50
|
// lock is a courtesy between cooperating runs, and refusing to run would
|
|
@@ -183,7 +184,7 @@ function ownLockRoot(lockDir: string): void {
|
|
|
183
184
|
|
|
184
185
|
interface Holder {
|
|
185
186
|
pid: number
|
|
186
|
-
/** The holder's start time as `
|
|
187
|
+
/** The holder's start time as `procStat` read it; null where the lock names none. */
|
|
187
188
|
start: string | null
|
|
188
189
|
/** The entry in the lock directory that names it. */
|
|
189
190
|
entry: string
|
|
@@ -223,17 +224,18 @@ async function holder(lockDir: string): Promise<Holder | 'free' | null> {
|
|
|
223
224
|
}
|
|
224
225
|
|
|
225
226
|
/**
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
* off Linux, where the answer costs a `ps`
|
|
229
|
-
* mounted for another pid namespace, and for a pid
|
|
227
|
+
* `/proc/<pid>/stat` from field 3 on: the state first, and at [19] field
|
|
228
|
+
* 22, the start time in clock ticks since boot, where two processes that
|
|
229
|
+
* wore one pid differ. Null off Linux, where the answer costs a `ps`
|
|
230
|
+
* spawn, under a procfs mounted for another pid namespace, and for a pid
|
|
231
|
+
* procfs does not show.
|
|
230
232
|
*/
|
|
231
|
-
function
|
|
233
|
+
function procStat(pid: number | 'self'): string[] | null {
|
|
232
234
|
if (!procfsIsOwn()) return null
|
|
233
235
|
try {
|
|
234
236
|
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8')
|
|
235
237
|
// Fields after the LAST ')' start at field 3; comm may hold spaces.
|
|
236
|
-
return stat.slice(stat.lastIndexOf(')') + 2).split(' ')
|
|
238
|
+
return stat.slice(stat.lastIndexOf(')') + 2).split(' ')
|
|
237
239
|
} catch {
|
|
238
240
|
return null
|
|
239
241
|
}
|
|
@@ -244,21 +246,24 @@ let ownId: string | undefined
|
|
|
244
246
|
/** Tells this process's takings of a lock apart, and their staging names. */
|
|
245
247
|
let takings = 0
|
|
246
248
|
function nextEntry(): string {
|
|
247
|
-
ownId ??= `h-${process.pid}-${
|
|
249
|
+
ownId ??= `h-${process.pid}-${procStat('self')?.[19] ?? 'x'}`
|
|
248
250
|
return `${ownId}-${++takings}`
|
|
249
251
|
}
|
|
250
252
|
|
|
251
253
|
/**
|
|
252
|
-
* Is the lock's holder a live run of another process?
|
|
253
|
-
* holder
|
|
254
|
-
*
|
|
254
|
+
* Is the lock's holder a live run of another process? Read on every poll:
|
|
255
|
+
* a holder killed whose parent does not reap it (a container's pid 1 that
|
|
256
|
+
* never waits) is a zombie, which signal 0 still reaches and whose start
|
|
257
|
+
* time still matches, and a wait that trusted either waited for it until
|
|
258
|
+
* the reaper came, or for good.
|
|
255
259
|
*/
|
|
256
|
-
function holderLive(h: Holder
|
|
260
|
+
function holderLive(h: Holder): boolean {
|
|
257
261
|
if (h.pid === process.pid) return placed.has(h.entry)
|
|
258
262
|
if (!alive(h.pid)) return false
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
263
|
+
const stat = procStat(h.pid)
|
|
264
|
+
if (stat === null) return true
|
|
265
|
+
if (stat[0] === 'Z' || stat[0] === 'X') return false
|
|
266
|
+
return h.start === null || stat[19] === h.start
|
|
262
267
|
}
|
|
263
268
|
|
|
264
269
|
function alive(pid: number): boolean {
|
|
@@ -326,7 +331,6 @@ export async function acquireRunLock(
|
|
|
326
331
|
const lockDir = runLockPath(workspaceRoot, opts.dir)
|
|
327
332
|
const started = Date.now()
|
|
328
333
|
let said = false
|
|
329
|
-
let lastLive: Holder | undefined
|
|
330
334
|
const release = async (): Promise<void> => {
|
|
331
335
|
const left = (heldHere.get(lockDir) ?? 1) - 1
|
|
332
336
|
if (left > 0) {
|
|
@@ -377,7 +381,7 @@ export async function acquireRunLock(
|
|
|
377
381
|
await rm(lockDir, { recursive: true, force: true })
|
|
378
382
|
continue
|
|
379
383
|
}
|
|
380
|
-
} else if (!holderLive(h
|
|
384
|
+
} else if (!holderLive(h)) {
|
|
381
385
|
if (h.entry === LEGACY_PID) await rm(lockDir, { recursive: true, force: true })
|
|
382
386
|
else await leave(lockDir, h.entry)
|
|
383
387
|
continue
|
|
@@ -391,12 +395,9 @@ export async function acquireRunLock(
|
|
|
391
395
|
)
|
|
392
396
|
return async () => {}
|
|
393
397
|
}
|
|
394
|
-
if (h !== null) {
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
said = true
|
|
398
|
-
opts.log(`[vx] waiting for another vx run (pid ${h.pid}) on this workspace to finish…`)
|
|
399
|
-
}
|
|
398
|
+
if (h !== null && !said && Date.now() - started >= SAY_AFTER_MS) {
|
|
399
|
+
said = true
|
|
400
|
+
opts.log(`[vx] waiting for another vx run (pid ${h.pid}) on this workspace to finish…`)
|
|
400
401
|
}
|
|
401
402
|
await Bun.sleep(POLL_MS)
|
|
402
403
|
}
|
package/src/orchestrator/run.ts
CHANGED
|
@@ -631,6 +631,7 @@ async function runOnBus(
|
|
|
631
631
|
stopRun.abort(signal)
|
|
632
632
|
},
|
|
633
633
|
done: runLeft,
|
|
634
|
+
handsTerminal: holders.size > 0,
|
|
634
635
|
// The grace, then one flush and each teardown at their own bound, and
|
|
635
636
|
// slack for the summary and the cache close.
|
|
636
637
|
boundMs:
|
|
@@ -1762,7 +1763,7 @@ function flakyCandidates(outcomes: readonly TaskOutcome[]): FlakyCandidate[] {
|
|
|
1762
1763
|
const out: FlakyCandidate[] = []
|
|
1763
1764
|
for (const o of outcomes) {
|
|
1764
1765
|
if (o.status !== 'success' && o.status !== 'failed') continue
|
|
1765
|
-
// "Same inputs,
|
|
1766
|
+
// "Same inputs, failed after passing" is a claim only a task with declared
|
|
1766
1767
|
// inputs can make: a task with no `cache` block keys on its config alone
|
|
1767
1768
|
// and runs every time, so one bad network day would read as a flake for
|
|
1768
1769
|
// thirty days. Groups do no work.
|
|
@@ -326,8 +326,11 @@ export interface WithheldLink {
|
|
|
326
326
|
* task is granted and what it is not.
|
|
327
327
|
*
|
|
328
328
|
* One level deep, plus one level inside a `@scope/` directory — the shape
|
|
329
|
-
* a package manager writes
|
|
330
|
-
*
|
|
329
|
+
* a package manager writes — and then the same in the `node_modules` of
|
|
330
|
+
* each granted target, so a dependency's own workspace dependencies are
|
|
331
|
+
* reached as the runtime reaches them. A target inside the granted
|
|
332
|
+
* `node_modules` or inside the package holding the link is dropped; what
|
|
333
|
+
* is left is a sibling project's real path.
|
|
331
334
|
*
|
|
332
335
|
* A link to the task's OWN project, or to a directory holding it, is
|
|
333
336
|
* dropped too: npm and Yarn classic link every workspace package at the
|
|
@@ -371,20 +374,39 @@ async function linkedDeps(
|
|
|
371
374
|
}
|
|
372
375
|
return found
|
|
373
376
|
}
|
|
377
|
+
const keyedDirs =
|
|
378
|
+
keyed === undefined ? undefined : new Set(await Promise.all([...keyed].map((d) => realpath(d))))
|
|
379
|
+
const grants = (target: string): boolean =>
|
|
380
|
+
keyedDirs === undefined || !atOrUnder(target, root) || keyedDirs.has(target)
|
|
374
381
|
// Merged in `dirs` order, so the link a hint names is the project's own
|
|
375
|
-
// when both have one.
|
|
382
|
+
// when both have one. A granted package is scanned in turn: pnpm and
|
|
383
|
+
// Bun link a dependency's own workspace dependencies only under ITS
|
|
384
|
+
// `node_modules`, so `app -> ui -> core` left core hidden and the import
|
|
385
|
+
// of ui failed with ENOENT, unreported.
|
|
376
386
|
const targets = new Map<string, { link: string; name: string }>()
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
387
|
+
let frontier: Array<[nodeModules: string, owner: string | undefined]> = dirs.map((d) => [
|
|
388
|
+
d,
|
|
389
|
+
undefined,
|
|
390
|
+
])
|
|
391
|
+
while (frontier.length > 0) {
|
|
392
|
+
const next: typeof frontier = []
|
|
393
|
+
const scans = await Promise.all(frontier.map(([d]) => scan(d, '')))
|
|
394
|
+
for (const [i, found] of scans.entries()) {
|
|
395
|
+
const owner = frontier[i]![1]
|
|
396
|
+
for (const [target, link, name] of found) {
|
|
397
|
+
if (targets.has(target) || (owner !== undefined && atOrUnder(target, owner))) continue
|
|
398
|
+
targets.set(target, { link, name })
|
|
399
|
+
if (grants(target)) {
|
|
400
|
+
next.push([path.join(target, 'node_modules'), target])
|
|
401
|
+
}
|
|
402
|
+
}
|
|
380
403
|
}
|
|
404
|
+
frontier = next
|
|
381
405
|
}
|
|
382
|
-
if (keyed === undefined) return { granted: [...targets.keys()], withheld: [] }
|
|
383
|
-
const keyedDirs = new Set(await Promise.all([...keyed].map((d) => realpath(d))))
|
|
384
406
|
const granted: string[] = []
|
|
385
407
|
const withheld: WithheldLink[] = []
|
|
386
408
|
for (const [target, { link, name }] of targets) {
|
|
387
|
-
if (
|
|
409
|
+
if (grants(target)) granted.push(target)
|
|
388
410
|
else
|
|
389
411
|
withheld.push({
|
|
390
412
|
dir: target,
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// — the scheduler dispatches nothing more, and the signal is forwarded to
|
|
3
3
|
// everything live, SIGKILLed after a bounded grace — then waits, bounded,
|
|
4
4
|
// for run() to leave its own end-of-run path (telemetry flush, each
|
|
5
|
-
// plugin's teardown, the cache close), and
|
|
6
|
-
// second signal
|
|
5
|
+
// plugin's teardown, the cache close), and dies of the signal (130/143 to
|
|
6
|
+
// its parent). A second signal does so at once. Without the forward, a
|
|
7
7
|
// programmatic signal to the vx process alone (CI cancellation,
|
|
8
8
|
// `kill <pid>`) orphans every running child — terminal Ctrl-C only worked
|
|
9
9
|
// via process-group propagation. Without the escalation (added
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// them to the runner around every spawn.
|
|
15
15
|
|
|
16
16
|
import { signalExitCode } from '../exec/index.js'
|
|
17
|
-
import { claimExitForSignal, killGraceMs, settleWithin } from '../util/index.js'
|
|
17
|
+
import { claimExitForSignal, hangupIgnored, killGraceMs, settleWithin } from '../util/index.js'
|
|
18
18
|
import { holdGroups, killTree, untilGroupsGone } from '../exec/index.js'
|
|
19
19
|
import type { Logger } from './logger.js'
|
|
20
20
|
|
|
@@ -111,6 +111,13 @@ export function forwardSignals(args: {
|
|
|
111
111
|
done: Promise<void>
|
|
112
112
|
/** How long a signal waits for `done` before it exits anyway. */
|
|
113
113
|
boundMs: number
|
|
114
|
+
/**
|
|
115
|
+
* A task may be handed the terminal (`exec.interactive` on a TTY) and may
|
|
116
|
+
* leave it raw or without echo. Bun puts back the terminal it started
|
|
117
|
+
* with when it exits, not when a signal kills it, so this run exits with
|
|
118
|
+
* the signal's code rather than dying of the signal.
|
|
119
|
+
*/
|
|
120
|
+
handsTerminal?: boolean
|
|
114
121
|
/** In-flight children; the runner adds and removes each around its spawn. */
|
|
115
122
|
liveChildren: ReadonlySet<Child>
|
|
116
123
|
/** Ready persistent tasks the orchestrator owns until the graph finishes. */
|
|
@@ -124,6 +131,16 @@ export function forwardSignals(args: {
|
|
|
124
131
|
} catch {
|
|
125
132
|
// double-close race with the normal path; we're exiting anyway
|
|
126
133
|
}
|
|
134
|
+
// Die of the signal once the exit hooks have run: a shell running vx
|
|
135
|
+
// in a script stops it on Ctrl-C only when its child died of SIGINT,
|
|
136
|
+
// and a 130 exit ran the script's next line. The code stays the
|
|
137
|
+
// fallback should the signal not land.
|
|
138
|
+
if (args.handsTerminal !== true) {
|
|
139
|
+
process.once('exit', () => {
|
|
140
|
+
process.removeAllListeners(signal)
|
|
141
|
+
process.kill(process.pid, signal)
|
|
142
|
+
})
|
|
143
|
+
}
|
|
127
144
|
process.exit(signalExitCode(signal))
|
|
128
145
|
}
|
|
129
146
|
let stopping: StopSignal | undefined
|
|
@@ -158,7 +175,8 @@ export function forwardSignals(args: {
|
|
|
158
175
|
if (args.enabled) {
|
|
159
176
|
process.on('SIGINT', onSigint)
|
|
160
177
|
process.on('SIGTERM', onSigterm)
|
|
161
|
-
|
|
178
|
+
// A hang-up vx was started deaf to (nohup) stays ignored.
|
|
179
|
+
if (!hangupIgnored()) process.on('SIGHUP', onSighup)
|
|
162
180
|
}
|
|
163
181
|
return {
|
|
164
182
|
remove: () => {
|
|
@@ -17,7 +17,7 @@ import { isLiteralPattern, normalizeGlob, relPosix } from '../util/index.js'
|
|
|
17
17
|
import { commandWriteReach, mayWriteFingerprint, undeclaredWriteReach } from './sandbox-request.js'
|
|
18
18
|
import { foldedDeps } from './keyed-projects.js'
|
|
19
19
|
import { computeGroupKey, computeTaskHash, type HashCache } from './task-hash.js'
|
|
20
|
-
import { filterUpstreamHashes, keyUpstream } from './upstream.js'
|
|
20
|
+
import { filterUpstreamHashes, keyedOutcome, keyUpstream } from './upstream.js'
|
|
21
21
|
|
|
22
22
|
export interface DeriveStableKeysArgs {
|
|
23
23
|
nodes: Map<string, TaskNode>
|
|
@@ -382,9 +382,7 @@ function foldedBy(
|
|
|
382
382
|
}
|
|
383
383
|
|
|
384
384
|
/**
|
|
385
|
-
* Synthetic upstream outcomes for key derivation
|
|
386
|
-
* are read by filterUpstreamHashes / computeGroupHash; the rest of
|
|
387
|
-
* TaskOutcome is irrelevant here, so we cast a minimal shape.
|
|
385
|
+
* Synthetic upstream outcomes for key derivation (`keyedOutcome`).
|
|
388
386
|
*/
|
|
389
387
|
function synthUpstream(
|
|
390
388
|
node: TaskNode,
|
|
@@ -395,13 +393,7 @@ function synthUpstream(
|
|
|
395
393
|
for (const dep of node.deps) {
|
|
396
394
|
const depNode = nodes.get(dep)
|
|
397
395
|
if (!depNode) continue
|
|
398
|
-
out.push(
|
|
399
|
-
node: depNode,
|
|
400
|
-
status: 'success',
|
|
401
|
-
exitCode: 0,
|
|
402
|
-
durationMs: 0,
|
|
403
|
-
hash: keyById.get(dep),
|
|
404
|
-
} as TaskOutcome)
|
|
396
|
+
out.push(keyedOutcome(depNode, keyById.get(dep)))
|
|
405
397
|
}
|
|
406
398
|
return keyUpstream(node, out)
|
|
407
399
|
}
|