@vzn/vx 0.0.0 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +332 -1
- package/launcher.mjs +84 -0
- package/package.json +44 -3
- package/src/bin.ts +25 -0
- package/src/cache/cache.ts +1832 -0
- package/src/cache/cas-backend.ts +97 -0
- package/src/cache/digest.ts +47 -0
- package/src/cache/index.ts +29 -0
- package/src/cache/inputs.ts +833 -0
- package/src/cache/layered-cache.ts +335 -0
- package/src/cache/remote-cache.ts +251 -0
- package/src/cache/tar.ts +343 -0
- package/src/cli/backend.ts +23 -0
- package/src/cli/cache.ts +101 -0
- package/src/cli/format.ts +15 -0
- package/src/cli/help.ts +123 -0
- package/src/cli/index.ts +85 -0
- package/src/cli/info.ts +83 -0
- package/src/cli/lock.ts +135 -0
- package/src/cli/mcp-rpc.ts +292 -0
- package/src/cli/mcp.ts +92 -0
- package/src/cli/migrate-nx.ts +450 -0
- package/src/cli/migrate-turbo.ts +364 -0
- package/src/cli/migrate.ts +219 -0
- package/src/cli/plan-format.ts +148 -0
- package/src/cli/run.ts +661 -0
- package/src/cli/show.ts +201 -0
- package/src/cli/upgrade.ts +103 -0
- package/src/cli/watch.ts +287 -0
- package/src/config.ts +421 -0
- package/src/exec/env.ts +74 -0
- package/src/exec/index.ts +13 -0
- package/src/exec/runner.ts +470 -0
- package/src/exec/sandbox-runtime.ts +578 -0
- package/src/graph/dependency-spec.ts +69 -0
- package/src/graph/index.ts +18 -0
- package/src/graph/scheduler.ts +482 -0
- package/src/graph/task-graph.ts +325 -0
- package/src/index.ts +197 -0
- package/src/orchestrator/colors.ts +63 -0
- package/src/orchestrator/devframe-surface.ts +74 -0
- package/src/orchestrator/events.ts +277 -0
- package/src/orchestrator/execute-task.ts +883 -0
- package/src/orchestrator/framed-output.ts +461 -0
- package/src/orchestrator/history.ts +148 -0
- package/src/orchestrator/index.ts +163 -0
- package/src/orchestrator/local-shortcircuit.ts +147 -0
- package/src/orchestrator/logger.ts +526 -0
- package/src/orchestrator/metrics.ts +1576 -0
- package/src/orchestrator/options.ts +176 -0
- package/src/orchestrator/plan.ts +129 -0
- package/src/orchestrator/plugin-host.ts +190 -0
- package/src/orchestrator/plugin.ts +283 -0
- package/src/orchestrator/predict.ts +106 -0
- package/src/orchestrator/prepare.ts +297 -0
- package/src/orchestrator/protocol.ts +142 -0
- package/src/orchestrator/remote-cache-setup.ts +60 -0
- package/src/orchestrator/remote-prefetch.ts +81 -0
- package/src/orchestrator/run-artifacts.ts +111 -0
- package/src/orchestrator/run-context.ts +205 -0
- package/src/orchestrator/run-report.ts +132 -0
- package/src/orchestrator/run-state.ts +145 -0
- package/src/orchestrator/run.ts +842 -0
- package/src/orchestrator/stable-keys.ts +177 -0
- package/src/orchestrator/status-line.ts +287 -0
- package/src/orchestrator/summary.ts +291 -0
- package/src/orchestrator/tally.ts +69 -0
- package/src/orchestrator/task-hash.ts +238 -0
- package/src/orchestrator/telemetry-host.ts +80 -0
- package/src/orchestrator/telemetry.ts +350 -0
- package/src/orchestrator/upstream.ts +93 -0
- package/src/orchestrator/verify.ts +141 -0
- package/src/orchestrator/wire-render.ts +92 -0
- package/src/orchestrator/wire.ts +212 -0
- package/src/util/errors.ts +11 -0
- package/src/util/hash.ts +25 -0
- package/src/util/index.ts +7 -0
- package/src/util/paths.ts +16 -0
- package/src/util/ulid.ts +15 -0
- package/src/version.ts +10 -0
- package/src/workspace/affected.ts +118 -0
- package/src/workspace/filter.ts +155 -0
- package/src/workspace/fingerprint.ts +41 -0
- package/src/workspace/index.ts +27 -0
- package/src/workspace/lockfile.ts +118 -0
- package/src/workspace/nested-dirs.ts +34 -0
- package/src/workspace/package-graph.ts +148 -0
- package/src/workspace/project-loader.ts +470 -0
- package/src/workspace/workspace.ts +176 -0
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
// The telemetry contract — THE canonical, versioned data-export shape.
|
|
2
|
+
//
|
|
3
|
+
// This is the one neutral boundary the observability/integration design
|
|
4
|
+
// (docs/design/observability-architecture-2026-06.md) is built around.
|
|
5
|
+
// Core projects its live `RunEvent` stream into these records ONCE, in
|
|
6
|
+
// one place (`createTelemetrySource`), and hands them to every registered
|
|
7
|
+
// `TelemetrySink`. Exporters (OTel, a manual HTTP POST, any third-party sink) all read
|
|
8
|
+
// the SAME records — with the analytics fields + git/CI context already
|
|
9
|
+
// folded in — instead of each re-deriving an ad-hoc shape from the raw,
|
|
10
|
+
// rendering-oriented `WireEvent` stream.
|
|
11
|
+
//
|
|
12
|
+
// A sink is observe-only BY CONSTRUCTION: its only input is an immutable
|
|
13
|
+
// record; its `TelemetryContext` carries read-only metadata and NO mutable
|
|
14
|
+
// run handle (no bus, no Cache, no RunRequest). There is no API path from a
|
|
15
|
+
// sink back into scheduling/caching/exec — telemetry provably cannot change
|
|
16
|
+
// what or how tasks run. Contrast `backend`/`cache`, which return objects
|
|
17
|
+
// core calls INTO; those are the behavior capabilities, kept separate.
|
|
18
|
+
|
|
19
|
+
import type { TaskStatus, VerifyVerdict } from '../graph/index.js'
|
|
20
|
+
import type { RunEvent, RunEventSubscriber } from './events.js'
|
|
21
|
+
|
|
22
|
+
/** Bumped when the record shape changes. Readers MUST check `v`. */
|
|
23
|
+
export const TELEMETRY_SCHEMA_VERSION = 2
|
|
24
|
+
|
|
25
|
+
/** Where a task's result came from, derived ONCE in core from the status. */
|
|
26
|
+
export type CacheSource = 'miss' | 'local' | 'remote' | 'none'
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Map a task outcome status to its cache source. `success`/`failed` ran
|
|
30
|
+
* (a miss as far as the cache read is concerned); the two cache-hit
|
|
31
|
+
* statuses restored from local/remote; `skipped`/`aborted` never engaged
|
|
32
|
+
* the cache. Pure function — the single place this mapping is decided, so
|
|
33
|
+
* no consumer re-implements `status LIKE 'cache-hit%'`.
|
|
34
|
+
*/
|
|
35
|
+
export function deriveCacheSource(status: TaskStatus): CacheSource {
|
|
36
|
+
switch (status) {
|
|
37
|
+
case 'cache-hit':
|
|
38
|
+
return 'local'
|
|
39
|
+
case 'cache-hit-remote':
|
|
40
|
+
return 'remote'
|
|
41
|
+
case 'success':
|
|
42
|
+
case 'failed':
|
|
43
|
+
return 'miss'
|
|
44
|
+
case 'skipped':
|
|
45
|
+
case 'aborted':
|
|
46
|
+
return 'none'
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Identifies which run a record belongs to + its captured context. Maps
|
|
51
|
+
* cleanly onto OTel CI/CD + VCS resource attributes. */
|
|
52
|
+
export interface RunContextRecord {
|
|
53
|
+
/** ULID, shared by every record in one `vx run`. */
|
|
54
|
+
runId: string
|
|
55
|
+
vxVersion: string
|
|
56
|
+
/** The invocation command line (process.argv-derived). */
|
|
57
|
+
command: string
|
|
58
|
+
requestedTasks: readonly string[]
|
|
59
|
+
/** Compact cache-policy flags, e.g. `'lR,lW,rR,rW'`. */
|
|
60
|
+
cachePolicy: string
|
|
61
|
+
concurrency: number
|
|
62
|
+
flow: 'focused' | 'broad' | null
|
|
63
|
+
/**
|
|
64
|
+
* Stable workspace identity (v2) — the multi-workspace server key.
|
|
65
|
+
* Derived from the normalized git remote (any checkout of the same
|
|
66
|
+
* repo → same id); see run-context.ts captureWorkspaceIdentity. A v1
|
|
67
|
+
* consumer synthesizes 'default' for pushes that predate it.
|
|
68
|
+
*/
|
|
69
|
+
workspaceId: string
|
|
70
|
+
workspaceName: string
|
|
71
|
+
// git / CI / host — straight from run-context.ts.
|
|
72
|
+
commitSha: string | null
|
|
73
|
+
branch: string | null
|
|
74
|
+
dirty: boolean | null
|
|
75
|
+
ci: boolean
|
|
76
|
+
ciProvider: string | null
|
|
77
|
+
host: string | null
|
|
78
|
+
os: string
|
|
79
|
+
arch: string
|
|
80
|
+
/** `--tag k=v` pairs. */
|
|
81
|
+
tags: Readonly<Record<string, string>>
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Denormalized per-task analytics — shared by the streaming `task.end`
|
|
85
|
+
* record and the per-run summary's `tasks[]`. */
|
|
86
|
+
export interface TaskTelemetry {
|
|
87
|
+
taskId: string
|
|
88
|
+
project: string
|
|
89
|
+
task: string
|
|
90
|
+
status: TaskStatus
|
|
91
|
+
cacheSource: CacheSource
|
|
92
|
+
exitCode: number
|
|
93
|
+
durationMs: number
|
|
94
|
+
hash?: string
|
|
95
|
+
cpuMs?: number
|
|
96
|
+
peakRssBytes?: number
|
|
97
|
+
/** Total attempts when the task RETRIED (>1) — set only when `exec.retries`
|
|
98
|
+
* / `--retry` produced more than one attempt. A retried-then-passed task is
|
|
99
|
+
* flaky by definition; this is the telemetry-side flaky signal. */
|
|
100
|
+
attempts?: number
|
|
101
|
+
/** Cache-correctness verdict — set ONLY on a `--verify` run (absent
|
|
102
|
+
* otherwise). A `nondeterministic` verdict means the task's cache entry is
|
|
103
|
+
* unsound (its outputs aren't a pure function of its declared inputs); this
|
|
104
|
+
* is the telemetry-side hermeticity signal a dashboard surfaces. */
|
|
105
|
+
verify?: VerifyVerdict
|
|
106
|
+
/** bigint hrtime ns relative to run t=0, encoded as a decimal string. */
|
|
107
|
+
wallclockStartNs?: string
|
|
108
|
+
wallclockEndNs?: string
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* A streaming telemetry record — one per lifecycle event. A superset of the
|
|
113
|
+
* rendering-oriented `WireEvent`: it carries the run context + the per-task
|
|
114
|
+
* analytics fields a consumer needs WITHOUT re-deriving from the stream.
|
|
115
|
+
* `task.log` records are large and OPT-IN (see `TelemetrySink.wants`).
|
|
116
|
+
*/
|
|
117
|
+
export type TelemetryRecord =
|
|
118
|
+
| { v: number; kind: 'run.start'; run: RunContextRecord; total: number; ts: number }
|
|
119
|
+
| {
|
|
120
|
+
v: number
|
|
121
|
+
kind: 'task.start'
|
|
122
|
+
runId: string
|
|
123
|
+
taskId: string
|
|
124
|
+
project: string
|
|
125
|
+
task: string
|
|
126
|
+
command?: string
|
|
127
|
+
ts: number
|
|
128
|
+
}
|
|
129
|
+
| {
|
|
130
|
+
v: number
|
|
131
|
+
kind: 'task.log'
|
|
132
|
+
runId: string
|
|
133
|
+
taskId: string
|
|
134
|
+
stream: 'stdout' | 'stderr'
|
|
135
|
+
chunk: string
|
|
136
|
+
ts: number
|
|
137
|
+
}
|
|
138
|
+
| ({ v: number; kind: 'task.end'; runId: string; ts: number } & TaskTelemetry)
|
|
139
|
+
| { v: number; kind: 'run.end'; runId: string; ts: number }
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* A per-run SUMMARY record — the denormalized invocation header plus the
|
|
143
|
+
* per-task outcome list, emitted once at run:end. An ingesting store can
|
|
144
|
+
* persist a whole run in one write without replaying the stream. The
|
|
145
|
+
* manual-API exporter + a service's ingest endpoint primarily speak this shape.
|
|
146
|
+
*/
|
|
147
|
+
export interface RunSummaryRecord {
|
|
148
|
+
v: number
|
|
149
|
+
run: RunContextRecord
|
|
150
|
+
startedAt: number
|
|
151
|
+
endedAt: number
|
|
152
|
+
totalDurationMs: number
|
|
153
|
+
taskCount: number
|
|
154
|
+
failedCount: number
|
|
155
|
+
hitCount: number
|
|
156
|
+
hitLocalCount: number
|
|
157
|
+
hitRemoteCount: number
|
|
158
|
+
exitOk: boolean
|
|
159
|
+
tasks: readonly TaskTelemetry[]
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** A telemetry consumer. Observe-only: receives records, holds no run handle. */
|
|
163
|
+
export interface TelemetrySink {
|
|
164
|
+
readonly name?: string
|
|
165
|
+
/**
|
|
166
|
+
* Which streaming record kinds to receive. Default (undefined): all
|
|
167
|
+
* EXCEPT `task.log` (large; most sinks don't want build-log chunks).
|
|
168
|
+
* The source checks this before projecting/cloning, so a sink pays
|
|
169
|
+
* nothing for kinds it declines.
|
|
170
|
+
*/
|
|
171
|
+
readonly wants?: ReadonlyArray<TelemetryRecord['kind']>
|
|
172
|
+
/** A streaming record. MUST return promptly — buffer; do NOT await I/O. */
|
|
173
|
+
onRecord?(record: TelemetryRecord): void
|
|
174
|
+
/** The per-run summary, at run:end. MUST return promptly. */
|
|
175
|
+
onRunSummary?(summary: RunSummaryRecord): void
|
|
176
|
+
/** Drain buffered data. Awaited (time-bounded by the sink) at run:end. */
|
|
177
|
+
flush?(): Promise<void>
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Read-only context a sink is created with. No mutable run handle — the
|
|
181
|
+
* isolation guarantee is structural. */
|
|
182
|
+
export interface TelemetryContext {
|
|
183
|
+
readonly workspaceRoot: string
|
|
184
|
+
/** A STRING, not a Cache handle — a sink cannot reach the cache. */
|
|
185
|
+
readonly cacheDir: string
|
|
186
|
+
warn(message: string): void
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** A live telemetry source: a bus subscriber + the run-summary emit + flush. */
|
|
190
|
+
export interface TelemetrySource {
|
|
191
|
+
/** Subscribe this to the run event bus to stream records to the sinks. */
|
|
192
|
+
readonly subscriber: RunEventSubscriber
|
|
193
|
+
/** Fan the per-run summary to every sink's `onRunSummary` (crash-isolated). */
|
|
194
|
+
emitSummary(summary: RunSummaryRecord): void
|
|
195
|
+
/** Await every sink's `flush()` (each crash-isolated). */
|
|
196
|
+
flush(): Promise<void>
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const DEFAULT_KINDS: ReadonlyArray<TelemetryRecord['kind']> = [
|
|
200
|
+
'run.start',
|
|
201
|
+
'task.start',
|
|
202
|
+
'task.end',
|
|
203
|
+
'run.end',
|
|
204
|
+
]
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Build a telemetry source over a fixed set of sinks. The returned
|
|
208
|
+
* `subscriber` projects each `RunEvent` into a `TelemetryRecord` and fans
|
|
209
|
+
* it to the sinks that want its kind, under crash isolation (a throwing
|
|
210
|
+
* sink is disabled for the rest of the run, never propagating into the
|
|
211
|
+
* orchestrator). `task.log` is projected ONLY if some sink opted in — so
|
|
212
|
+
* the large-payload path costs nothing by default.
|
|
213
|
+
*
|
|
214
|
+
* The `run` context (captured once by run.ts) is stamped onto `run.start`
|
|
215
|
+
* and supplies the `runId` every other record carries.
|
|
216
|
+
*/
|
|
217
|
+
export function createTelemetrySource(args: {
|
|
218
|
+
sinks: readonly TelemetrySink[]
|
|
219
|
+
run: RunContextRecord
|
|
220
|
+
}): TelemetrySource {
|
|
221
|
+
const { sinks, run } = args
|
|
222
|
+
const runId = run.runId
|
|
223
|
+
// A sink is disabled the first time it throws — its name (or index) goes
|
|
224
|
+
// here and it's skipped for the rest of the run.
|
|
225
|
+
const disabled = new Set<TelemetrySink>()
|
|
226
|
+
// Precompute which sinks want each kind, so per-event fan-out is a plain
|
|
227
|
+
// array walk with no per-record `wants` scanning.
|
|
228
|
+
const wantsLog = sinks.some((s) => (s.wants ?? DEFAULT_KINDS).includes('task.log'))
|
|
229
|
+
|
|
230
|
+
function deliver(record: TelemetryRecord): void {
|
|
231
|
+
for (const sink of sinks) {
|
|
232
|
+
if (disabled.has(sink) || sink.onRecord === undefined) continue
|
|
233
|
+
const kinds = sink.wants ?? DEFAULT_KINDS
|
|
234
|
+
if (!kinds.includes(record.kind)) continue
|
|
235
|
+
try {
|
|
236
|
+
sink.onRecord(record)
|
|
237
|
+
} catch {
|
|
238
|
+
disabled.add(sink)
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
let endEmitted = false
|
|
244
|
+
const subscriber: RunEventSubscriber = (event: RunEvent) => {
|
|
245
|
+
const ts = Date.now()
|
|
246
|
+
switch (event.kind) {
|
|
247
|
+
case 'run:start':
|
|
248
|
+
deliver({
|
|
249
|
+
v: TELEMETRY_SCHEMA_VERSION,
|
|
250
|
+
kind: 'run.start',
|
|
251
|
+
run,
|
|
252
|
+
total: event.info.total,
|
|
253
|
+
ts,
|
|
254
|
+
})
|
|
255
|
+
return
|
|
256
|
+
case 'task:start': {
|
|
257
|
+
const node = event.node
|
|
258
|
+
if (node.config.exec === undefined) return // group task — no command, skip
|
|
259
|
+
const rec: TelemetryRecord = {
|
|
260
|
+
v: TELEMETRY_SCHEMA_VERSION,
|
|
261
|
+
kind: 'task.start',
|
|
262
|
+
runId,
|
|
263
|
+
taskId: node.id,
|
|
264
|
+
project: node.projectName,
|
|
265
|
+
task: node.taskName,
|
|
266
|
+
ts,
|
|
267
|
+
}
|
|
268
|
+
if (node.config.exec.command !== undefined) rec.command = node.config.exec.command
|
|
269
|
+
deliver(rec)
|
|
270
|
+
return
|
|
271
|
+
}
|
|
272
|
+
case 'task:stdout':
|
|
273
|
+
case 'task:stderr': {
|
|
274
|
+
if (!wantsLog) return // nobody wants logs — pay nothing
|
|
275
|
+
deliver({
|
|
276
|
+
v: TELEMETRY_SCHEMA_VERSION,
|
|
277
|
+
kind: 'task.log',
|
|
278
|
+
runId,
|
|
279
|
+
taskId: event.node.id,
|
|
280
|
+
stream: event.kind === 'task:stdout' ? 'stdout' : 'stderr',
|
|
281
|
+
chunk: event.chunk,
|
|
282
|
+
ts,
|
|
283
|
+
})
|
|
284
|
+
return
|
|
285
|
+
}
|
|
286
|
+
case 'task:complete': {
|
|
287
|
+
const { node, outcome } = event
|
|
288
|
+
if (node.config.exec === undefined) return // group task
|
|
289
|
+
const rec: TelemetryRecord = {
|
|
290
|
+
v: TELEMETRY_SCHEMA_VERSION,
|
|
291
|
+
kind: 'task.end',
|
|
292
|
+
runId,
|
|
293
|
+
ts,
|
|
294
|
+
taskId: node.id,
|
|
295
|
+
project: node.projectName,
|
|
296
|
+
task: node.taskName,
|
|
297
|
+
status: outcome.status,
|
|
298
|
+
cacheSource: deriveCacheSource(outcome.status),
|
|
299
|
+
exitCode: outcome.exitCode,
|
|
300
|
+
durationMs: outcome.durationMs,
|
|
301
|
+
}
|
|
302
|
+
if (outcome.hash !== undefined) rec.hash = outcome.hash
|
|
303
|
+
if (outcome.cpuMs !== undefined) rec.cpuMs = outcome.cpuMs
|
|
304
|
+
if (outcome.peakRssBytes !== undefined) rec.peakRssBytes = outcome.peakRssBytes
|
|
305
|
+
if (outcome.attempts !== undefined) rec.attempts = outcome.attempts
|
|
306
|
+
if (outcome.verify !== undefined) rec.verify = outcome.verify
|
|
307
|
+
if (outcome.wallclockStartNs !== undefined)
|
|
308
|
+
rec.wallclockStartNs = outcome.wallclockStartNs.toString()
|
|
309
|
+
if (outcome.wallclockEndNs !== undefined)
|
|
310
|
+
rec.wallclockEndNs = outcome.wallclockEndNs.toString()
|
|
311
|
+
deliver(rec)
|
|
312
|
+
return
|
|
313
|
+
}
|
|
314
|
+
case 'run:status':
|
|
315
|
+
return // status lines are terminal-rendering noise, not telemetry
|
|
316
|
+
case 'run:end':
|
|
317
|
+
// run() emits run:end twice (normal + finally); emit one record.
|
|
318
|
+
if (endEmitted) return
|
|
319
|
+
endEmitted = true
|
|
320
|
+
deliver({ v: TELEMETRY_SCHEMA_VERSION, kind: 'run.end', runId, ts })
|
|
321
|
+
return
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
return {
|
|
326
|
+
subscriber,
|
|
327
|
+
emitSummary(summary: RunSummaryRecord): void {
|
|
328
|
+
for (const sink of sinks) {
|
|
329
|
+
if (disabled.has(sink) || sink.onRunSummary === undefined) continue
|
|
330
|
+
try {
|
|
331
|
+
sink.onRunSummary(summary)
|
|
332
|
+
} catch {
|
|
333
|
+
disabled.add(sink)
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
},
|
|
337
|
+
async flush(): Promise<void> {
|
|
338
|
+
await Promise.all(
|
|
339
|
+
sinks.map(async (sink) => {
|
|
340
|
+
if (sink.flush === undefined) return
|
|
341
|
+
try {
|
|
342
|
+
await sink.flush()
|
|
343
|
+
} catch {
|
|
344
|
+
// a sink's flush failure can never break the run
|
|
345
|
+
}
|
|
346
|
+
}),
|
|
347
|
+
)
|
|
348
|
+
},
|
|
349
|
+
}
|
|
350
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DependencySpecError,
|
|
3
|
+
parseDependencySpec,
|
|
4
|
+
type DependencySpec,
|
|
5
|
+
type TaskOutcome,
|
|
6
|
+
} from '../graph/index.js'
|
|
7
|
+
import { UserError } from '../util/index.js'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Pick which upstream task hashes participate in the current task's
|
|
11
|
+
* cache key, filtered by `cache.inputs.tasks`. The folded value is the
|
|
12
|
+
* upstream's own cache key (its input-based task hash) — pure-input
|
|
13
|
+
* transitive hashing, like Turbo/Nx. An upstream change propagates
|
|
14
|
+
* through its key into every dependent's key. There is deliberately NO
|
|
15
|
+
* output-content folding: an upstream that re-runs but emits identical
|
|
16
|
+
* output still re-runs its dependents (early cutoff was removed —
|
|
17
|
+
* rare in practice, not worth the cascade complexity).
|
|
18
|
+
*
|
|
19
|
+
* Patterns (Turbo/Nx micro-syntax + filter extensions):
|
|
20
|
+
* '*' all same-project upstream
|
|
21
|
+
* '^*' all dep-workspace upstream
|
|
22
|
+
* 'name' same-project task `name`
|
|
23
|
+
* '^name' `name` task in every dep workspace
|
|
24
|
+
* 'pkg#name' specific package's `name` task
|
|
25
|
+
* '!<form>' exclude — any of the above with a leading `!`
|
|
26
|
+
*
|
|
27
|
+
* Patterns are applied in order; last write wins, so
|
|
28
|
+
* `['*', '^*', '!^noisy']` reads as "all minus deps' noisy".
|
|
29
|
+
*
|
|
30
|
+
* Defaults:
|
|
31
|
+
* - `filter === undefined` → all upstream contribute.
|
|
32
|
+
* - `filter === []` → none contribute (fully decoupled).
|
|
33
|
+
*
|
|
34
|
+
* Returns `[upstreamTaskId, hash]` pairs. The hash is the only thing
|
|
35
|
+
* folded into the cache key (the fold sorts by hash, so ordering here
|
|
36
|
+
* doesn't affect derivation); the task id rides along so Tier-3's
|
|
37
|
+
* `entry_inputs` rows can NAME which upstream a hash came from. The
|
|
38
|
+
* filter dedups by hash, as before — two upstream tasks with identical
|
|
39
|
+
* hashes contribute one pair (first id wins; their key contribution is
|
|
40
|
+
* identical anyway).
|
|
41
|
+
*/
|
|
42
|
+
export function filterUpstreamHashes(
|
|
43
|
+
upstream: TaskOutcome[],
|
|
44
|
+
filter: readonly string[] | undefined,
|
|
45
|
+
selfProjectName: string,
|
|
46
|
+
selfTaskId: string,
|
|
47
|
+
): Array<[upstreamTaskId: string, hash: string]> {
|
|
48
|
+
if (filter === undefined) {
|
|
49
|
+
const out: Array<[string, string]> = []
|
|
50
|
+
for (const u of upstream) if (u.hash) out.push([u.node.id, u.hash])
|
|
51
|
+
return out
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const specs: DependencySpec[] = filter.map((raw) => {
|
|
55
|
+
try {
|
|
56
|
+
return parseDependencySpec(raw)
|
|
57
|
+
} catch (err) {
|
|
58
|
+
if (err instanceof DependencySpecError) {
|
|
59
|
+
throw new UserError(`${selfTaskId}: cache.inputs.tasks: ${err.message}`)
|
|
60
|
+
}
|
|
61
|
+
throw err
|
|
62
|
+
}
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
// Dedup by hash (the key fold's unit), but remember the first task id
|
|
66
|
+
// seen for each hash so the diff row can name the upstream.
|
|
67
|
+
const selected = new Map<string, string>()
|
|
68
|
+
for (const spec of specs) {
|
|
69
|
+
for (const u of upstream) {
|
|
70
|
+
if (!u.hash) continue
|
|
71
|
+
const isSelf = u.node.projectName === selfProjectName
|
|
72
|
+
if (!matches(spec, u, isSelf)) continue
|
|
73
|
+
if (spec.negated) selected.delete(u.hash)
|
|
74
|
+
else if (!selected.has(u.hash)) selected.set(u.hash, u.node.id)
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return [...selected].map(([hash, id]) => [id, hash])
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function matches(spec: DependencySpec, u: TaskOutcome, isSelf: boolean): boolean {
|
|
81
|
+
switch (spec.kind) {
|
|
82
|
+
case 'wildcardSelf':
|
|
83
|
+
return isSelf
|
|
84
|
+
case 'wildcardDeps':
|
|
85
|
+
return !isSelf
|
|
86
|
+
case 'self':
|
|
87
|
+
return isSelf && u.node.taskName === spec.task
|
|
88
|
+
case 'deps':
|
|
89
|
+
return !isSelf && u.node.taskName === spec.task
|
|
90
|
+
case 'cross':
|
|
91
|
+
return u.node.projectName === spec.project && u.node.taskName === spec.task
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// Cache-correctness verification (Phase 1: determinism). Pure helpers for the
|
|
2
|
+
// `vx run --verify` hook in execute-task.ts: content-fingerprint a task's
|
|
3
|
+
// output tree (mtime-independent) and diff two runs' trees, so a task whose
|
|
4
|
+
// outputs are NOT a pure function of its declared inputs — a non-hermetic task
|
|
5
|
+
// whose cache entry would replay arbitrary past bytes — is caught and named.
|
|
6
|
+
// See docs/design/cache-correctness-2026-07.md.
|
|
7
|
+
|
|
8
|
+
import path from 'node:path'
|
|
9
|
+
import type { CacheLayer } from '../cache/index.js'
|
|
10
|
+
import type { TaskOutcome, VerifyVerdict } from '../graph/index.js'
|
|
11
|
+
|
|
12
|
+
/** An output file to fingerprint: absolute path + a stable, run-independent
|
|
13
|
+
* key (project-relative, or `workspace-outputs/<rel-to-root>` for workspace
|
|
14
|
+
* outputs — matching the cache artifact's namespace so the two never collide). */
|
|
15
|
+
export interface OutputRef {
|
|
16
|
+
abs: string
|
|
17
|
+
key: string
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Build the fingerprint key set from a task's resolved output paths. */
|
|
21
|
+
export function outputRefs(
|
|
22
|
+
projectDir: string,
|
|
23
|
+
outputFiles: readonly string[],
|
|
24
|
+
workspaceRoot: string,
|
|
25
|
+
workspaceOutputFiles: readonly string[],
|
|
26
|
+
): OutputRef[] {
|
|
27
|
+
const rel = (from: string, abs: string): string =>
|
|
28
|
+
path.relative(from, abs).split(path.sep).join('/')
|
|
29
|
+
return [
|
|
30
|
+
...outputFiles.map((abs) => ({ abs, key: rel(projectDir, abs) })),
|
|
31
|
+
...workspaceOutputFiles.map((abs) => ({
|
|
32
|
+
abs,
|
|
33
|
+
key: `workspace-outputs/${rel(workspaceRoot, abs)}`,
|
|
34
|
+
})),
|
|
35
|
+
]
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Content-fingerprint an output tree: key → git-blob OID. Reuses the same
|
|
39
|
+
* `Cache.hashFile` primitive the input hashing folds, so it's byte-true and
|
|
40
|
+
* mtime-independent — two byte-identical trees produce equal fingerprints
|
|
41
|
+
* even though their tar.zst artifacts (which embed mtimes) would differ. */
|
|
42
|
+
export async function hashOutputTree(
|
|
43
|
+
cache: CacheLayer,
|
|
44
|
+
refs: readonly OutputRef[],
|
|
45
|
+
): Promise<Map<string, string>> {
|
|
46
|
+
const out = new Map<string, string>()
|
|
47
|
+
for (const r of refs) out.set(r.key, await cache.hashFile(r.abs))
|
|
48
|
+
return out
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** The output keys whose content OID differs between two fingerprints
|
|
52
|
+
* (added, removed, or changed), sorted. */
|
|
53
|
+
export function diffOutputTrees(a: Map<string, string>, b: Map<string, string>): string[] {
|
|
54
|
+
const changed: string[] = []
|
|
55
|
+
for (const k of new Set([...a.keys(), ...b.keys()])) {
|
|
56
|
+
if (a.get(k) !== b.get(k)) changed.push(k)
|
|
57
|
+
}
|
|
58
|
+
return changed.sort()
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Turn a determinism diff into a verdict, honoring the `--verify-allow`
|
|
62
|
+
* escape hatch (an allowed divergence is reported but never fails the run). */
|
|
63
|
+
export function classifyDeterminism(changed: readonly string[], allowed: boolean): VerifyVerdict {
|
|
64
|
+
if (changed.length === 0) return { kind: 'proven-deterministic' }
|
|
65
|
+
return allowed
|
|
66
|
+
? { kind: 'allowed-nondeterministic', changed }
|
|
67
|
+
: { kind: 'nondeterministic', changed }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Extract the workspace-relative paths a task read outside its declared inputs
|
|
71
|
+
* (Phase 2, `--verify=inputs`). A strace-derived violation line carries the
|
|
72
|
+
* offending absolute path in trailing brackets (`… [/abs/path]`); pull it out,
|
|
73
|
+
* make it workspace-relative, dedup, and sort. Lines without a bracketed path
|
|
74
|
+
* (e.g. macOS syscall lines) fall back to the raw line so nothing is lost. */
|
|
75
|
+
export function undeclaredInputPaths(
|
|
76
|
+
violations: readonly { line: string }[],
|
|
77
|
+
workspaceRoot: string,
|
|
78
|
+
): string[] {
|
|
79
|
+
const rels = new Set<string>()
|
|
80
|
+
for (const v of violations) {
|
|
81
|
+
const m = /\[([^\]]+)\]\s*$/.exec(v.line)
|
|
82
|
+
const raw = m ? m[1]! : v.line
|
|
83
|
+
rels.add(
|
|
84
|
+
path.isAbsolute(raw) ? path.relative(workspaceRoot, raw).split(path.sep).join('/') : raw,
|
|
85
|
+
)
|
|
86
|
+
}
|
|
87
|
+
return [...rels].sort()
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** The end-of-run `--verify` summary section: a counts line plus a
|
|
91
|
+
* failures block naming each non-hermetic task and its diverging outputs.
|
|
92
|
+
* Empty when no task carried a verdict. */
|
|
93
|
+
export function formatVerifySection(outcomes: readonly TaskOutcome[]): string[] {
|
|
94
|
+
const verdicts = outcomes.filter((o) => o.verify !== undefined)
|
|
95
|
+
if (verdicts.length === 0) return []
|
|
96
|
+
let proven = 0
|
|
97
|
+
let bad = 0
|
|
98
|
+
let na = 0
|
|
99
|
+
let notVerified = 0
|
|
100
|
+
for (const o of verdicts) {
|
|
101
|
+
switch (o.verify!.kind) {
|
|
102
|
+
case 'proven-deterministic':
|
|
103
|
+
case 'proven-complete':
|
|
104
|
+
proven++
|
|
105
|
+
break
|
|
106
|
+
case 'nondeterministic':
|
|
107
|
+
case 'rerun-failed':
|
|
108
|
+
case 'undeclared-inputs':
|
|
109
|
+
bad++
|
|
110
|
+
break
|
|
111
|
+
case 'allowed-nondeterministic':
|
|
112
|
+
case 'no-outputs':
|
|
113
|
+
na++
|
|
114
|
+
break
|
|
115
|
+
case 'not-verified':
|
|
116
|
+
notVerified++
|
|
117
|
+
break
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const lines = [
|
|
121
|
+
'',
|
|
122
|
+
` Verify: ${proven} proven · ${bad} unsafe · ${na} n/a · ${notVerified} not-verified`,
|
|
123
|
+
]
|
|
124
|
+
for (const o of outcomes) {
|
|
125
|
+
const v = o.verify
|
|
126
|
+
if (v === undefined) continue
|
|
127
|
+
if (v.kind === 'nondeterministic') {
|
|
128
|
+
lines.push(` ✗ ${o.node.id} — nondeterministic`)
|
|
129
|
+
lines.push(` changed: ${v.changed.join(', ')}`)
|
|
130
|
+
} else if (v.kind === 'undeclared-inputs') {
|
|
131
|
+
lines.push(` ✗ ${o.node.id} — read undeclared inputs`)
|
|
132
|
+
if (v.paths.length > 0) lines.push(` ${v.paths.join(', ')}`)
|
|
133
|
+
lines.push(' add them to cache.inputs.files / workspaceFiles')
|
|
134
|
+
} else if (v.kind === 'rerun-failed') {
|
|
135
|
+
lines.push(` ✗ ${o.node.id} — verify re-run failed (exit ${v.exitCode})`)
|
|
136
|
+
} else if (v.kind === 'allowed-nondeterministic') {
|
|
137
|
+
lines.push(` ⚠ ${o.node.id} — nondeterministic (allowed)`)
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return lines
|
|
141
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// The inverse of `wireForwarder`: turn a stream of `WireEvent`s back into
|
|
2
|
+
// `Logger` calls on a concrete renderer. A `vx serve` client feeds the
|
|
3
|
+
// service's event stream through this into a normal `defaultLogger`, so a
|
|
4
|
+
// DELEGATED run renders identically to a local one — the terminal
|
|
5
|
+
// renderer stays completely untouched (it just receives reconstructed
|
|
6
|
+
// node-shaped objects instead of live graph nodes). This file is the
|
|
7
|
+
// whole adapter; it can be rewritten or removed without touching the
|
|
8
|
+
// renderer or the producer side.
|
|
9
|
+
|
|
10
|
+
import type { TaskNode, TaskOutcome } from '../graph/index.js'
|
|
11
|
+
import type { Logger } from './logger.js'
|
|
12
|
+
import type { OutcomeView, TaskView, WireEvent } from './events.js'
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Rebuild a node-shaped object from a `TaskView`. The formatters only ever
|
|
16
|
+
* read `id` / `projectName` / `taskName` / `requested` / `surfaced` and
|
|
17
|
+
* `config.exec` (presence ⇒ not a group; `.command`; `.persistent`), so a
|
|
18
|
+
* structural stand-in is indistinguishable from a real `TaskNode` to them.
|
|
19
|
+
*/
|
|
20
|
+
function reconstructNode(view: TaskView): TaskNode {
|
|
21
|
+
const exec = view.isGroup
|
|
22
|
+
? undefined
|
|
23
|
+
: {
|
|
24
|
+
command: view.command ?? '',
|
|
25
|
+
...(view.persistent ? { persistent: { readyWhen: '' } } : {}),
|
|
26
|
+
}
|
|
27
|
+
return {
|
|
28
|
+
id: view.id,
|
|
29
|
+
projectName: view.project,
|
|
30
|
+
taskName: view.task,
|
|
31
|
+
requested: view.requested,
|
|
32
|
+
surfaced: view.surfaced,
|
|
33
|
+
config: exec === undefined ? {} : { exec },
|
|
34
|
+
} as unknown as TaskNode
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function reconstructOutcome(view: OutcomeView, node: TaskNode): TaskOutcome {
|
|
38
|
+
return {
|
|
39
|
+
node,
|
|
40
|
+
status: view.status,
|
|
41
|
+
exitCode: view.exitCode,
|
|
42
|
+
durationMs: view.durationMs,
|
|
43
|
+
...(view.restored !== undefined ? { restored: view.restored } : {}),
|
|
44
|
+
...(view.sandboxViolations !== undefined ? { sandboxViolations: view.sandboxViolations } : {}),
|
|
45
|
+
...(view.sandboxViolationLines !== undefined
|
|
46
|
+
? { sandboxViolationLines: view.sandboxViolationLines }
|
|
47
|
+
: {}),
|
|
48
|
+
} as TaskOutcome
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Build a `WireEvent` consumer that drives `sink` (typically a
|
|
53
|
+
* `defaultLogger`). Stateful: it accumulates the reconstructed nodes from
|
|
54
|
+
* `task:start` so later `task:*` events can resolve their node by id.
|
|
55
|
+
*/
|
|
56
|
+
export function createWireRenderer(sink: Logger): (event: WireEvent) => void {
|
|
57
|
+
const nodes = new Map<string, TaskNode>()
|
|
58
|
+
return (event) => {
|
|
59
|
+
switch (event.kind) {
|
|
60
|
+
case 'run:start':
|
|
61
|
+
sink.runStart?.(event.info)
|
|
62
|
+
return
|
|
63
|
+
case 'task:start': {
|
|
64
|
+
const node = reconstructNode(event.task)
|
|
65
|
+
nodes.set(node.id, node)
|
|
66
|
+
sink.taskStart?.(node)
|
|
67
|
+
return
|
|
68
|
+
}
|
|
69
|
+
case 'task:stdout': {
|
|
70
|
+
const node = nodes.get(event.taskId)
|
|
71
|
+
if (node) sink.taskStdout(node, event.chunk)
|
|
72
|
+
return
|
|
73
|
+
}
|
|
74
|
+
case 'task:stderr': {
|
|
75
|
+
const node = nodes.get(event.taskId)
|
|
76
|
+
if (node) sink.taskStderr(node, event.chunk)
|
|
77
|
+
return
|
|
78
|
+
}
|
|
79
|
+
case 'task:complete': {
|
|
80
|
+
const node = nodes.get(event.outcome.taskId)
|
|
81
|
+
if (node) sink.taskComplete(node, reconstructOutcome(event.outcome, node))
|
|
82
|
+
return
|
|
83
|
+
}
|
|
84
|
+
case 'run:status':
|
|
85
|
+
sink.status(event.line)
|
|
86
|
+
return
|
|
87
|
+
case 'run:end':
|
|
88
|
+
sink.runEnd?.()
|
|
89
|
+
return
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|