@vzn/vx 0.0.651 → 0.0.653

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/README.md +3 -4
  2. package/launcher.sh +2 -1
  3. package/package.json +5 -5
  4. package/schemas/error.json +1 -1
  5. package/schemas/show.json +15 -20
  6. package/skills/vx/SKILL.md +3 -3
  7. package/src/bin.ts +37 -3
  8. package/src/cache/archive.ts +197 -32
  9. package/src/cache/cache.ts +882 -230
  10. package/src/cache/checkpointer.ts +102 -0
  11. package/src/cache/file-hashes.ts +2 -1
  12. package/src/cache/fold-memo.ts +24 -4
  13. package/src/cache/fs-clock.ts +26 -0
  14. package/src/cache/git-inputs.ts +188 -22
  15. package/src/cache/history-stream.ts +151 -0
  16. package/src/cache/history-writer.ts +76 -0
  17. package/src/cache/index.ts +12 -1
  18. package/src/cache/inputs.ts +616 -135
  19. package/src/cache/key-fold.ts +1 -1
  20. package/src/cache/layer.ts +29 -16
  21. package/src/cache/layered-cache.ts +28 -7
  22. package/src/cache/lockfile-memo.ts +2 -1
  23. package/src/cache/output-index.ts +106 -44
  24. package/src/cache/plugin-cache.ts +2 -2
  25. package/src/cache/policy.ts +6 -2
  26. package/src/cache/remote-lane.ts +3 -1
  27. package/src/cache/run-history.ts +94 -25
  28. package/src/cache/schema.ts +46 -10
  29. package/src/cli/cache.ts +149 -36
  30. package/src/cli/completions.ts +2 -2
  31. package/src/cli/core-alias.ts +9 -2
  32. package/src/cli/docs.ts +4 -2
  33. package/src/cli/help.ts +27 -4
  34. package/src/cli/index.ts +62 -7
  35. package/src/cli/info.ts +5 -4
  36. package/src/cli/init.ts +55 -20
  37. package/src/cli/last.ts +218 -43
  38. package/src/cli/lock.ts +187 -24
  39. package/src/cli/plan-format.ts +24 -21
  40. package/src/cli/plugin-commands.ts +6 -1
  41. package/src/cli/plugin-templates.ts +36 -45
  42. package/src/cli/run.ts +68 -34
  43. package/src/cli/select.ts +160 -64
  44. package/src/cli/show.ts +70 -28
  45. package/src/cli/task-verb.ts +12 -7
  46. package/src/cli/upgrade.ts +210 -76
  47. package/src/cli/watch-filter.ts +88 -15
  48. package/src/cli/watch-fs.ts +34 -47
  49. package/src/cli/watch-judge.ts +73 -9
  50. package/src/cli/watch-set.ts +10 -8
  51. package/src/cli/watch.ts +190 -33
  52. package/src/cli/why.ts +107 -33
  53. package/src/cli/workspace-config.ts +11 -1
  54. package/src/config.ts +56 -30
  55. package/src/exec/env.ts +3 -1
  56. package/src/exec/executor.ts +116 -16
  57. package/src/exec/index.ts +1 -0
  58. package/src/exec/kill-tree.ts +7 -7
  59. package/src/exec/runner.ts +270 -108
  60. package/src/exec/sandbox-binds.ts +109 -18
  61. package/src/exec/sandbox-paths.ts +3 -1
  62. package/src/exec/sandbox-runtime.ts +446 -64
  63. package/src/exec/sandbox-violations.ts +250 -46
  64. package/src/exec/spawn-off.ts +10 -5
  65. package/src/exec/task-proxy.ts +93 -0
  66. package/src/exec/warm-shell.ts +29 -7
  67. package/src/graph/definition.ts +16 -3
  68. package/src/graph/folds.ts +89 -16
  69. package/src/graph/index.ts +8 -0
  70. package/src/graph/priorities.ts +8 -5
  71. package/src/graph/scheduler.ts +176 -39
  72. package/src/graph/stand-ins.ts +36 -0
  73. package/src/graph/task-graph.ts +347 -160
  74. package/src/graph/task-status.ts +5 -1
  75. package/src/handoff.ts +45 -10
  76. package/src/history/failure-mode.ts +39 -6
  77. package/src/history/history.ts +103 -30
  78. package/src/history/index.ts +2 -0
  79. package/src/history/metrics.ts +193 -73
  80. package/src/index.ts +6 -3
  81. package/src/orchestrator/admission.ts +14 -10
  82. package/src/orchestrator/affected-tasks.ts +35 -13
  83. package/src/orchestrator/command-windows.ts +8 -5
  84. package/src/orchestrator/deferred-outputs.ts +42 -13
  85. package/src/orchestrator/doctor.ts +21 -9
  86. package/src/orchestrator/download-policy.ts +8 -10
  87. package/src/orchestrator/excluded-keys.ts +15 -3
  88. package/src/orchestrator/execute-args.ts +9 -2
  89. package/src/orchestrator/execute-task.ts +612 -138
  90. package/src/orchestrator/graph-memo.ts +26 -4
  91. package/src/orchestrator/graph-stage.ts +27 -4
  92. package/src/orchestrator/held-servers.ts +36 -14
  93. package/src/orchestrator/hit-restore.ts +141 -93
  94. package/src/orchestrator/index.ts +4 -1
  95. package/src/orchestrator/keyed-projects.ts +53 -0
  96. package/src/orchestrator/local-floor.ts +21 -3
  97. package/src/orchestrator/local-shortcircuit.ts +2 -2
  98. package/src/orchestrator/lockfile-claim.ts +30 -4
  99. package/src/orchestrator/miss-save.ts +60 -61
  100. package/src/orchestrator/options.ts +13 -0
  101. package/src/orchestrator/output-host.ts +16 -4
  102. package/src/orchestrator/output-log.ts +1 -54
  103. package/src/orchestrator/persistent-task.ts +32 -16
  104. package/src/orchestrator/persistent.ts +152 -15
  105. package/src/orchestrator/placement.ts +82 -16
  106. package/src/orchestrator/plan.ts +39 -17
  107. package/src/orchestrator/plugin-host.ts +238 -110
  108. package/src/orchestrator/plugin.ts +59 -27
  109. package/src/orchestrator/prepare.ts +194 -38
  110. package/src/orchestrator/projects.ts +89 -10
  111. package/src/orchestrator/read-only.ts +115 -0
  112. package/src/orchestrator/remote-prefetch.ts +1 -1
  113. package/src/orchestrator/run-context.ts +6 -2
  114. package/src/orchestrator/run-failures.ts +101 -23
  115. package/src/orchestrator/run-id.ts +13 -2
  116. package/src/orchestrator/run-keys.ts +87 -28
  117. package/src/orchestrator/run-live.ts +16 -13
  118. package/src/orchestrator/run-lock.ts +27 -4
  119. package/src/orchestrator/run-records.ts +95 -74
  120. package/src/orchestrator/run-report.ts +3 -2
  121. package/src/orchestrator/run-stop.ts +17 -9
  122. package/src/orchestrator/run.ts +531 -122
  123. package/src/orchestrator/sandbox-fix.ts +40 -12
  124. package/src/orchestrator/sandbox-request.ts +314 -44
  125. package/src/orchestrator/save-lane.ts +24 -13
  126. package/src/orchestrator/shell-verdict.ts +29 -1
  127. package/src/orchestrator/signals.ts +103 -8
  128. package/src/orchestrator/stable-keys.ts +34 -27
  129. package/src/orchestrator/task-artifacts.ts +16 -3
  130. package/src/orchestrator/task-hash.ts +191 -34
  131. package/src/orchestrator/telemetry-host.ts +25 -6
  132. package/src/orchestrator/telemetry.ts +85 -13
  133. package/src/orchestrator/upstream.ts +3 -1
  134. package/src/report/events.ts +23 -6
  135. package/src/report/forecast.ts +103 -37
  136. package/src/report/framed-output.ts +179 -162
  137. package/src/report/index.ts +2 -1
  138. package/src/report/logger.ts +108 -31
  139. package/src/report/plain-output.ts +15 -9
  140. package/src/report/status-line.ts +28 -55
  141. package/src/report/summary.ts +61 -18
  142. package/src/settings.ts +50 -14
  143. package/src/ui/components.ts +36 -42
  144. package/src/ui/index.ts +3 -1
  145. package/src/ui/live-region.ts +67 -8
  146. package/src/ui/terminal.ts +12 -0
  147. package/src/ui/theme.ts +16 -1
  148. package/src/util/config-frame.ts +142 -6
  149. package/src/util/edit-distance.ts +17 -1
  150. package/src/util/errors.ts +23 -7
  151. package/src/util/foreign-cache-dir.ts +60 -0
  152. package/src/util/git-ignored.ts +59 -5
  153. package/src/util/index.ts +15 -1
  154. package/src/util/paths.ts +51 -7
  155. package/src/util/plain-text.ts +19 -0
  156. package/src/util/secret-mask.ts +57 -1
  157. package/src/util/settle.ts +18 -1
  158. package/src/util/size.ts +12 -2
  159. package/src/util/tail.ts +4 -1
  160. package/src/util/write-file.ts +86 -9
  161. package/src/workspace/affected.ts +225 -27
  162. package/src/workspace/bracket-literal.ts +2 -2
  163. package/src/workspace/config-bytecode.ts +23 -4
  164. package/src/workspace/config-eval.ts +90 -16
  165. package/src/workspace/config-imports.ts +96 -41
  166. package/src/workspace/config-schema.ts +192 -86
  167. package/src/workspace/dependencies-lock.ts +7 -14
  168. package/src/workspace/filter.ts +18 -2
  169. package/src/workspace/index.ts +3 -1
  170. package/src/workspace/json-data.ts +34 -8
  171. package/src/workspace/lockfile.ts +86 -37
  172. package/src/workspace/migration.ts +58 -31
  173. package/src/workspace/module-state.ts +62 -0
  174. package/src/workspace/nested-dirs.ts +8 -0
  175. package/src/workspace/project-loader.ts +276 -32
  176. package/src/workspace/repo-id.ts +29 -6
  177. package/src/workspace/workspace.ts +190 -30
package/README.md CHANGED
@@ -47,9 +47,8 @@ export default defineProject({
47
47
  exec: { command: 'tsc -b' },
48
48
  dependsOn: ['^build'], // dependencies build first
49
49
  inputs: { files: ['src/**', 'tsconfig.json'] },
50
- cache: {
51
- outputs: { files: ['dist/**'] },
52
- },
50
+ outputs: { files: ['dist/**', '*.tsbuildinfo'] }, // tsc's incremental state too
51
+ cache: true,
53
52
  },
54
53
  },
55
54
  })
@@ -127,7 +126,7 @@ Beside `turbo.json` or `nx.json`, `bunx @vzn/vx init` runs `@vzn/vx-migrate`:
127
126
  it writes a `vx.config.ts` per package from `turbo.json` or the Nx graph,
128
127
  and you keep those and delete the old config. `vx init --keep` writes only
129
128
  a `.vx/workspace.ts` that maps the old config instead, and its lock: a temporary start,
130
- not a way to run the repo. The benchmark below uses native vx config.
129
+ not a way to run the workspace. The benchmark below uses native vx config.
131
130
  [Migration guide](https://vznjs.github.io/vx/guides/migrate/).
132
131
 
133
132
  ## How it compares
package/launcher.sh CHANGED
@@ -48,7 +48,8 @@ if [ -z "${VX_HANDED_OFF-}" ] && [ -z "$stay" ]; then
48
48
  while :; do
49
49
  root=
50
50
  for ext in ts mts js mjs cts cjs; do
51
- if [ -f "$d/.vx/workspace.$ext" ]; then
51
+ # Any entry, a broken link or a directory too: vx refuses it there.
52
+ if [ -e "$d/.vx/workspace.$ext" ] || [ -L "$d/.vx/workspace.$ext" ]; then
52
53
  root=${d:-/}
53
54
  break
54
55
  fi
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vzn/vx",
3
- "version": "0.0.651",
3
+ "version": "0.0.653",
4
4
  "description": "Task orchestration you own: correct, sandboxed, extensible, with a content-addressed build cache.",
5
5
  "type": "module",
6
6
  "exports": {
@@ -21,10 +21,10 @@
21
21
  "node": ">=18"
22
22
  },
23
23
  "optionalDependencies": {
24
- "@vzn/vx-linux-x64": "0.0.651",
25
- "@vzn/vx-linux-arm64": "0.0.651",
26
- "@vzn/vx-darwin-x64": "0.0.651",
27
- "@vzn/vx-darwin-arm64": "0.0.651"
24
+ "@vzn/vx-linux-x64": "0.0.653",
25
+ "@vzn/vx-linux-arm64": "0.0.653",
26
+ "@vzn/vx-darwin-x64": "0.0.653",
27
+ "@vzn/vx-darwin-arm64": "0.0.653"
28
28
  },
29
29
  "dependencies": {
30
30
  "@anthropic-ai/sandbox-runtime": "^0.0.76"
@@ -14,7 +14,7 @@
14
14
  "properties": {
15
15
  "code": {
16
16
  "type": "string",
17
- "description": "Stable; branch on it. VX_E_USAGE, VX_E_UNKNOWN_COMMAND, VX_E_UNKNOWN_TASK, VX_E_UNKNOWN_PROJECT, VX_E_NO_HISTORY, VX_E_AMBIGUOUS, VX_E_CONFIG, VX_E_CYCLE, VX_E_LOCK, VX_E_NO_WORKSPACE, VX_E_CACHE_VERSION, VX_E_REFUSED, VX_E_FS, VX_E_FDS, VX_E_AFFECTED_BASE, VX_E_NO_GIT, VX_E_GIT, VX_E_NETWORK, VX_E_DEPENDS_ON, VX_E_OUTPUT_OVERLAP, VX_E_INPUT_IS_OUTPUT, VX_E_DEPENDENCY_RULE, VX_E_NESTED_INPUT, VX_E_INTERNAL. A plugin may name its own."
17
+ "description": "Stable; branch on it. VX_E_USAGE, VX_E_UNKNOWN_COMMAND, VX_E_UNKNOWN_TASK, VX_E_UNKNOWN_PROJECT, VX_E_NO_HISTORY, VX_E_AMBIGUOUS, VX_E_CONFIG, VX_E_CYCLE, VX_E_LOCK, VX_E_NO_WORKSPACE, VX_E_CACHE_VERSION, VX_E_REFUSED, VX_E_FS, VX_E_FDS, VX_E_AFFECTED_BASE, VX_E_NO_GIT, VX_E_GIT, VX_E_NETWORK, VX_E_DEPENDS_ON, VX_E_OUTPUT_OVERLAP, VX_E_INPUT_IS_OUTPUT, VX_E_DEPENDENCY_RULE, VX_E_NESTED_INPUT, VX_E_NESTED_OUTPUT, VX_E_INTERNAL. A plugin may name its own."
18
18
  },
19
19
  "message": { "type": "string", "description": "The refusal in prose; may change." },
20
20
  "docs": {
package/schemas/show.json CHANGED
@@ -93,6 +93,7 @@
93
93
  "type": "array",
94
94
  "items": { "enum": ["linux-x64", "linux-arm64", "darwin-x64", "darwin-arm64"] }
95
95
  },
96
+ "worker": { "type": "string" },
96
97
  "sandbox": {
97
98
  "type": "object",
98
99
  "description": "as declared; its fields are the `exec.sandbox` section of the config schema"
@@ -109,25 +110,8 @@
109
110
  "env": { "$ref": "#/$defs/stringList" },
110
111
  "tasks": { "$ref": "#/$defs/stringList" },
111
112
  "runtime": { "$ref": "#/$defs/stringList" },
112
- "workspaceRuntime": { "$ref": "#/$defs/stringList" }
113
- }
114
- },
115
- "cache": {
116
- "type": "object",
117
- "additionalProperties": false,
118
- "required": ["outputs"],
119
- "properties": {
120
- "outputs": {
121
- "type": "object",
122
- "additionalProperties": false,
123
- "required": ["files"],
124
- "properties": {
125
- "files": { "$ref": "#/$defs/stringList" },
126
- "workspaceFiles": { "$ref": "#/$defs/stringList" },
127
- "clean": { "type": "boolean" }
128
- }
129
- },
130
- "inputStability": {
113
+ "workspaceRuntime": { "$ref": "#/$defs/stringList" },
114
+ "stability": {
131
115
  "type": "object",
132
116
  "additionalProperties": false,
133
117
  "properties": {
@@ -136,7 +120,18 @@
136
120
  }
137
121
  }
138
122
  }
139
- }
123
+ },
124
+ "outputs": {
125
+ "type": "object",
126
+ "additionalProperties": false,
127
+ "required": ["files"],
128
+ "properties": {
129
+ "files": { "$ref": "#/$defs/stringList" },
130
+ "workspaceFiles": { "$ref": "#/$defs/stringList" },
131
+ "clean": { "type": "boolean" }
132
+ }
133
+ },
134
+ "cache": { "type": "boolean" }
140
135
  }
141
136
  }
142
137
  }
@@ -116,9 +116,9 @@ vx lock --check --format json # a vx.config that changed since the lock, or no
116
116
  print whole.
117
117
  - Docs as markdown: https://vznjs.github.io/vx/llms.txt (its index).
118
118
  When that host is unreachable, the same pages are
119
- `https://raw.githubusercontent.com/vznjs/vx/main/private/vx-docs/src/content/docs/<page>.md`:
120
- `quickstart`, `guides/migrate`, `guides/configure`, `guides/agents`,
121
- `guides/troubleshooting`, `guides/ci`, `guides/plugins`.
119
+ `https://raw.githubusercontent.com/vznjs/vx/main/private/vx-docs/src/content/docs/<page>.mdx`:
120
+ `quickstart`, `guides/migrate`, `docs/workspace`, `docs/agents`,
121
+ `guides/troubleshooting`, `docs/ci`, `docs/plugins`.
122
122
 
123
123
  ## Install
124
124
 
package/src/bin.ts CHANGED
@@ -7,6 +7,21 @@ let settled = false
7
7
 
8
8
  async function main(): Promise<void> {
9
9
  const argv = process.argv.slice(2)
10
+ // A rejection nothing awaited (a plugin verb's floating promise) is a
11
+ // failure: the exit after the streams end came before Bun reported it,
12
+ // and the run said nothing and exited 0.
13
+ process.on('unhandledRejection', (reason) => {
14
+ process.exitCode = 1
15
+ void Promise.all([import('./ui/index.js'), import('./util/index.js')]).then(
16
+ ([ui, { maskedLine }]) =>
17
+ ui.say(
18
+ 'error',
19
+ maskedLine(
20
+ `vx: unhandled rejection: ${reason instanceof Error ? reason.message : String(reason)}\n`,
21
+ ),
22
+ ),
23
+ )
24
+ })
10
25
  try {
11
26
  // Inside the try: `process.cwd()` throws EMFILE when out of descriptors,
12
27
  // and that is a one-line UserError below, not a stack.
@@ -38,7 +53,10 @@ async function main(): Promise<void> {
38
53
  // package, a workspace or project config — resolves to THIS core, not to
39
54
  // a second copy from node_modules (core-alias.ts says why and what it
40
55
  // costs). Registered before any verb runs; the façade loads on first use.
41
- registerCoreAlias(() => require('./index.js') as Record<string, unknown>)
56
+ registerCoreAlias(
57
+ () => require('./index.js') as Record<string, unknown>,
58
+ () => require('./config.js') as Record<string, unknown>,
59
+ )
42
60
  const code = await run(argv)
43
61
  // Not `process.exit` here: Bun drops what a pipe has not yet taken
44
62
  // when it follows a large write (300 KB written, 64 KiB delivered;
@@ -71,14 +89,19 @@ async function main(): Promise<void> {
71
89
  // A message that names its verb (`vx why: …`, a plugin's `vx dte
72
90
  // worker: …`) is that verb's; any other is the verb typed. A config
73
91
  // refusal shows its line.
74
- const named = /^(vx [a-z][\w-]*(?: [a-z][\w-]*)?): /.exec(err.message)
75
- const typed = /^[a-z][\w-]*$/.test(argv[0] ?? '') ? `vx ${argv[0]}` : 'vx'
92
+ // A plugin verb may hold a capital, `.` or `:` (`my.verb`).
93
+ const named = /^(vx [A-Za-z0-9][\w:.-]*(?: [a-z][\w-]*)?): /.exec(err.message)
94
+ const typed = /^[A-Za-z0-9][\w:.-]*$/.test(argv[0] ?? '') ? `vx ${argv[0]}` : 'vx'
76
95
  const block = refusalNotice(
77
96
  named?.[1] ?? typed,
78
97
  named === null ? err.message : err.message.slice(named[0].length),
79
98
  err.parts,
80
99
  )
81
100
  ui.data('err', maskedLine(ui.render([block], ui.colorsOf('err'))))
101
+ } else if (cwdGone(err)) {
102
+ // Removed under the shell (a checkout of another branch, `rm -rf` in
103
+ // another): every verb starts from the cwd.
104
+ say('vx: the current directory no longer exists — cd into the workspace again and re-run\n')
82
105
  } else if (isOutOfFds(err)) {
83
106
  say(`vx: ${err.message} — ${OUT_OF_FDS_HINT}\n`)
84
107
  } else if (isFsRefusal(err)) {
@@ -116,6 +139,9 @@ async function main(): Promise<void> {
116
139
  */
117
140
  async function exitOnceFlushed(): Promise<void> {
118
141
  if ((await import('./util/index.js')).exitClaimedBySignal()) return
142
+ // One macrotask: a floating rejection is reported after the microtasks
143
+ // drain, and its handler above sets the code and writes its line.
144
+ await new Promise((resolve) => setImmediate(resolve))
119
145
  let open = 2
120
146
  const ended = (): void => {
121
147
  if (--open === 0) process.exit()
@@ -159,6 +185,14 @@ process.on('beforeExit', () => {
159
185
  process.exitCode = 1
160
186
  })
161
187
 
188
+ /** `process.cwd()`'s refusal of a directory removed under the process. */
189
+ function cwdGone(err: unknown): boolean {
190
+ return (
191
+ (err as NodeJS.ErrnoException | null)?.code === 'ENOENT' &&
192
+ /\buv_cwd\b/.test((err as Error).message)
193
+ )
194
+ }
195
+
162
196
  // A reader that leaves is not the run's failure. `vx run build | head -1`
163
197
  // closes the pipe after one line; every later write gets EPIPE, which Bun
164
198
  // raises as an `error` event on the stream, and an `error` nobody listens
@@ -11,8 +11,10 @@
11
11
  // lost executable bit builds cold and breaks warm, and `isOutputsCurrent`
12
12
  // skips a restore when size+mode+mtime match at millisecond precision —
13
13
  // so the pack side stats each output once and writes both into
14
- // `.vx-meta.json`, and the restore side applies them. That also makes a
15
- // REMOTE-ingested entry's index rows millisecond-accurate.
14
+ // `.vx-meta.json`. A restore applies the mode; it dates a file it writes
15
+ // now, as the task's write would be, and leaves the mtime of one whose
16
+ // bytes already stood. The sidecar mtime is the index rows' (a
17
+ // REMOTE-ingested entry's too) and the cache's own tree form's.
16
18
  //
17
19
  // Both directions stream. `packArtifactStream` reads each output as it is
18
20
  // written; `scanArtifact` lists what an artifact holds without
@@ -341,16 +343,23 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
341
343
  `${shown} is a symlink to an absolute path (${to}): vx restores a link as written, and an absolute one names this machine's checkout — make it relative, or take it back with a '!' entry`,
342
344
  )
343
345
  }
344
- if (args.root === undefined || !inside(path.resolve(path.dirname(abs), to), args.root)) {
346
+ const refused =
347
+ args.root === undefined ? 'out' : linkRefused(path.dirname(abs), to, args.root)
348
+ if (refused === 'out') {
345
349
  throw new UserError(
346
350
  `${shown} is a symlink to ${to}, outside the workspace: vx restores a link as written, and a project's outputs stay in its workspace — emit a file there, or take it back with a '!' entry`,
347
351
  )
348
352
  }
353
+ if (refused === '.git') {
354
+ throw new UserError(
355
+ `${shown} is a symlink to ${to}, inside .git: vx never restores into Git's metadata — emit a file there, or take it back with a '!' entry`,
356
+ )
357
+ }
349
358
  return { name, link: to, abs, shown }
350
359
  }
351
360
  if (!st.isFile()) {
352
361
  throw new UserError(
353
- `${shown} is not a regular file: vx stores regular files and symlinks only — emit a file there, or narrow cache.outputs.files to the files the task produces, or take it back with a '!' entry`,
362
+ `${shown} is not a regular file: vx stores regular files and symlinks only — emit a file there, or narrow outputs.files to the files the task produces, or take it back with a '!' entry`,
354
363
  )
355
364
  }
356
365
  const mode = st.mode & 0o777
@@ -380,6 +389,23 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
380
389
  )
381
390
  }
382
391
  }
392
+ if (symlinks.length > 0 && args.root !== undefined) {
393
+ const realRoot = await realOf(args.root)
394
+ const realDirs = new Map<string, Promise<string>>()
395
+ for (const f of symlinks) {
396
+ const refused = await linkLandsRefused(path.dirname(f.abs), f.link, realRoot, realDirs)
397
+ if (refused === 'out') {
398
+ throw new UserError(
399
+ `${f.shown} is a symlink to ${f.link}, outside the workspace: vx restores a link as written, and a project's outputs stay in its workspace — emit a file there, or take it back with a '!' entry`,
400
+ )
401
+ }
402
+ if (refused === '.git') {
403
+ throw new UserError(
404
+ `${f.shown} is a symlink to ${f.link}, inside .git: vx never restores into Git's metadata — emit a file there, or take it back with a '!' entry`,
405
+ )
406
+ }
407
+ }
408
+ }
383
409
  // In the outputs' order, not as each stat settled: one tree, one artifact.
384
410
  const regular: Array<Exclude<(typeof files)[number], { link: string }>> = []
385
411
  for (const f of files) {
@@ -567,7 +593,9 @@ export async function writeTree(plan: ArtifactPlan, dir: string): Promise<Artifa
567
593
  }
568
594
  // A stdout-only entry writes no file to make the directory.
569
595
  await mkdir(dir, { recursive: true })
570
- await Promise.all(
596
+ // Every write settles before a failure is thrown: the caller removes
597
+ // `dir`, and a write still in flight would make its parents again.
598
+ const settled = await Promise.allSettled(
571
599
  plan.inputs.map(async (i) => {
572
600
  if (!(i.body instanceof Blob)) return
573
601
  const to = path.join(dir, i.name)
@@ -579,6 +607,8 @@ export async function writeTree(plan: ArtifactPlan, dir: string): Promise<Artifa
579
607
  await utimes(to, at, at)
580
608
  }),
581
609
  )
610
+ const failed = settled.find((s) => s.status === 'rejected')
611
+ if (failed !== undefined) throw failed.reason
582
612
  await writeFile(path.join(dir, META_ENTRY), meta)
583
613
  return fold.result()
584
614
  }
@@ -912,6 +942,81 @@ const inside = (p: string, root: string): boolean => {
912
942
  return p === r || p.startsWith(r + path.sep)
913
943
  }
914
944
 
945
+ /**
946
+ * Where a link in `dir` reading `to` lands against `bound`, if a restore may
947
+ * not write it: out of it, or into its `.git` (lexical: a write through a
948
+ * link is judged where it resolves, so an alias chain into .git gets no
949
+ * further than this link would). Out is any step above `bound`, even one
950
+ * that comes back: `../../../ws/app/x` resolves inside only a root named
951
+ * `ws`, so judged by where it ends it saved here and was refused on every
952
+ * checkout under another name. The save asks it too, or it stored a link
953
+ * every restore refused, an entry that never hit on any machine.
954
+ */
955
+ function linkRefused(dir: string, to: string, bound: string): 'out' | '.git' | undefined {
956
+ const root = path.resolve(bound)
957
+ const from = path.resolve(dir)
958
+ if (!inside(from, root)) return 'out'
959
+ let depth = from === root ? 0 : path.relative(root, from).split(path.sep).length
960
+ for (const seg of to.split(path.sep === '/' ? '/' : /[\\/]/)) {
961
+ if (seg === '..') depth--
962
+ else if (seg !== '' && seg !== '.') depth++
963
+ if (depth < 0) return 'out'
964
+ }
965
+ return writableUnder(path.resolve(from, to), root) ? undefined : '.git'
966
+ }
967
+
968
+ /**
969
+ * `linkRefused` on disk: the kernel resolves a `..` from where the links
970
+ * before it lead, so with a committed `app/up -> ..` the text
971
+ * `../up/../x` read as inside and landed beside the workspace. Walked from
972
+ * `dir`'s real path, each link on disk before the last `..` followed,
973
+ * every step held to `realBound`. A target with no `..` touches no disk;
974
+ * the links the artifact itself restores are `throughLink`'s.
975
+ */
976
+ async function linkLandsRefused(
977
+ dir: string,
978
+ to: string,
979
+ realBound: string,
980
+ realDirs: Map<string, Promise<string>>,
981
+ ): Promise<'out' | '.git' | undefined> {
982
+ const segs = to.split(path.sep === '/' ? '/' : /[\\/]/)
983
+ const lastUp = segs.lastIndexOf('..')
984
+ if (lastUp < 0) return undefined
985
+ let realDir = realDirs.get(dir)
986
+ if (realDir === undefined) {
987
+ // At a restore the link's directory may not exist yet.
988
+ realDir = realOf(dir).catch(() => resolveThrough(dir))
989
+ realDirs.set(dir, realDir)
990
+ }
991
+ let at = await realDir
992
+ let onDisk = true
993
+ for (let i = 0; i <= lastUp; i++) {
994
+ const seg = segs[i]!
995
+ if (seg === '' || seg === '.') continue
996
+ if (seg === '..') at = path.dirname(at)
997
+ else {
998
+ at = path.join(at, seg)
999
+ if (onDisk) {
1000
+ const st = statOrAbsent(at)
1001
+ if (st === undefined) onDisk = false
1002
+ else if (st.isSymbolicLink()) at = await resolveThrough(at)
1003
+ }
1004
+ }
1005
+ if (!inside(at, realBound)) return 'out'
1006
+ }
1007
+ return writableUnder(path.join(at, ...segs.slice(lastUp + 1)), realBound) ? undefined : '.git'
1008
+ }
1009
+
1010
+ /** `lstat`, or undefined for a name that is not there (under a file too). */
1011
+ function statOrAbsent(p: string): Stats | undefined {
1012
+ try {
1013
+ return lstatSync(p, { throwIfNoEntry: false })
1014
+ } catch (err) {
1015
+ if ((err as NodeJS.ErrnoException).code === 'ENOTDIR') return undefined
1016
+ throw err
1017
+ }
1018
+ }
1019
+
915
1020
  /** The usage worth carrying: each axis a finite non-negative number, or absent; nothing → undefined. */
916
1021
  function usageOf(raw: unknown): ExecUsage | undefined {
917
1022
  if (typeof raw !== 'object' || raw === null) return undefined
@@ -944,6 +1049,13 @@ export async function extractArtifactStream(
944
1049
  stageRoot?: string,
945
1050
  /** Handed what `scanArtifact` would read of the same bytes, once their sum has checked. */
946
1051
  onScan?: (scan: ArtifactScan) => void,
1052
+ /**
1053
+ * Given, a restore into a workspace: each file written is dated now, and
1054
+ * one whose bytes already stood keeps its own mtime; either is set here
1055
+ * by entry name. Absent, an unpack into the cache's tree form: the saved
1056
+ * mtimes apply.
1057
+ */
1058
+ dated?: Map<string, number>,
947
1059
  ): Promise<Set<string>> {
948
1060
  // No `mkdir -p` of the destination here: `stage` creates each entry's
949
1061
  // directory chain, the destination included, and the cache restores
@@ -1023,6 +1135,8 @@ export async function extractArtifactStream(
1023
1135
  const dest = x.destFor(name)
1024
1136
  if (dest !== null && dest.rel.length > 0) linked.set(path.join(dest.base, dest.rel), name)
1025
1137
  }
1138
+ let realBound: Promise<string> | undefined
1139
+ const realDirs = new Map<string, Promise<string>>()
1026
1140
  for (const [name, to] of links) {
1027
1141
  // Renamed over an entry just written, the link would stand where the
1028
1142
  // sidecar says a file is, and a re-pack of the tree reads through it.
@@ -1038,8 +1152,8 @@ export async function extractArtifactStream(
1038
1152
  if (dest === null || dest.rel.length === 0) continue
1039
1153
  const target = path.join(dest.base, dest.rel)
1040
1154
  await x.assertContained(dest.base, target, name)
1041
- const at = path.resolve(path.dirname(target), to)
1042
- if (!inside(at, bound)) {
1155
+ let refused = linkRefused(path.dirname(target), to, bound)
1156
+ if (refused === 'out') {
1043
1157
  throw new ArchiveSecurityError(`archive link ${name} points out of the workspace: ${to}`)
1044
1158
  }
1045
1159
  const via = throughLink(path.dirname(target), to, linked)
@@ -1048,9 +1162,14 @@ export async function extractArtifactStream(
1048
1162
  `archive link ${name} points through ${linked.get(via)}, another link it restores: ${to}`,
1049
1163
  )
1050
1164
  }
1051
- // Lexical: a write through a link is judged where it resolves, so an
1052
- // alias chain into .git gets no further than this link would.
1053
- if (!writableUnder(at, path.resolve(bound))) {
1165
+ if (refused === undefined) {
1166
+ realBound ??= realOf(bound)
1167
+ refused = await linkLandsRefused(path.dirname(target), to, await realBound, realDirs)
1168
+ }
1169
+ if (refused === 'out') {
1170
+ throw new ArchiveSecurityError(`archive link ${name} points out of the workspace: ${to}`)
1171
+ }
1172
+ if (refused === '.git') {
1054
1173
  throw new ArchiveSecurityError(`archive link ${name} points into .git: ${to}`)
1055
1174
  }
1056
1175
  await x.stageLink(name, dest.base, target, to)
@@ -1071,7 +1190,7 @@ export async function extractArtifactStream(
1071
1190
  if (m !== undefined) return [m[0], m[1]]
1072
1191
  const header = headerMtime.get(name)
1073
1192
  return [0o644, header !== undefined && header > 0 ? header : undefined]
1074
- })
1193
+ }, dated)
1075
1194
  } catch (err) {
1076
1195
  await x.abort()
1077
1196
  throw err
@@ -1113,11 +1232,12 @@ const textOf = async (body: AsyncIterable<Uint8Array>): Promise<string> =>
1113
1232
  utf8.decode(await bytesOf(body))
1114
1233
 
1115
1234
  /**
1116
- * Bring a target that already held the entry's bytes to its mode and
1117
- * mtime, through a descriptor opened without following a link: one
1118
- * planted since the compare is refused, never written through.
1235
+ * Bring a target that already held the entry's bytes to its mode and, when
1236
+ * given, `mtimeMs`, through a descriptor opened without following a link:
1237
+ * one planted since the compare is refused, never written through.
1238
+ * Answers the mtime it leaves, in whole milliseconds.
1119
1239
  */
1120
- function keep(s: Staged, ino: number, mode: number, mtimeMs: number | undefined): void {
1240
+ function keep(s: Staged, ino: number, mode: number, mtimeMs: number | undefined): number {
1121
1241
  const fd = openSync(s.target, constants.O_RDONLY | constants.O_NOFOLLOW)
1122
1242
  try {
1123
1243
  const st = fstatSync(fd)
@@ -1129,7 +1249,9 @@ function keep(s: Staged, ino: number, mode: number, mtimeMs: number | undefined)
1129
1249
  if (mtimeMs !== undefined && Math.floor(st.mtimeMs) !== mtimeMs) {
1130
1250
  const t = new Date(mtimeMs)
1131
1251
  futimesSync(fd, t, t)
1252
+ return mtimeMs
1132
1253
  }
1254
+ return Math.floor(st.mtimeMs)
1133
1255
  } finally {
1134
1256
  closeSync(fd)
1135
1257
  }
@@ -1367,17 +1489,7 @@ class Extractor {
1367
1489
  }
1368
1490
  if (top === undefined) return undefined
1369
1491
  const staging = await stagingDir(this.stageRoot)
1370
- const st = statSync(staging)
1371
- const parent = statSync(probe, { throwIfNoEntry: false })
1372
- if (
1373
- parent === undefined ||
1374
- !parent.isDirectory() ||
1375
- parent.dev !== st.dev ||
1376
- ((parent.mode | st.mode) & 0o2000) !== 0 ||
1377
- (INHERITS_GROUP && parent.gid !== st.gid)
1378
- ) {
1379
- return undefined
1380
- }
1492
+ if (!stagesInto(statSync(staging), statSync(probe, { throwIfNoEntry: false }))) return undefined
1381
1493
  const at = path.join(staging, (tmpSeq++).toString(36))
1382
1494
  this.trees.set(top, { at, base, dirs: [] })
1383
1495
  return at + abs.slice(top.length)
@@ -1457,7 +1569,10 @@ class Extractor {
1457
1569
  // A target that already holds these bytes stays: renaming over a live
1458
1570
  // file costs ~0.4 ms on ext4 (auto_da_alloc), and a hit that changed
1459
1571
  // one of 2,000 outputs paid it 2,000 times. Bytes, not a stat stamp: a
1460
- // reproducible build gives every output one mtime (item 886).
1572
+ // reproducible build gives every output one mtime (item 886). Never an
1573
+ // empty file: one left in place reads as an untouched write-grant
1574
+ // placeholder, and the sweep after a remote executor's task removed
1575
+ // the empty output its restore had just landed.
1461
1576
  const st = created === undefined ? lstatSync(target, { throwIfNoEntry: false }) : undefined
1462
1577
  // A directory there fails its rename at commit, after the renames
1463
1578
  // before it landed: refused now, no output is replaced.
@@ -1466,7 +1581,7 @@ class Extractor {
1466
1581
  new Error(`EISDIR: illegal operation on a directory, rename '${target}'`),
1467
1582
  { code: 'EISDIR', path: target },
1468
1583
  )
1469
- if (body instanceof Uint8Array && created === undefined) {
1584
+ if (body instanceof Uint8Array && body.byteLength > 0 && created === undefined) {
1470
1585
  if (
1471
1586
  st !== undefined &&
1472
1587
  st.isFile() &&
@@ -1607,9 +1722,10 @@ class Extractor {
1607
1722
  */
1608
1723
  async commit(
1609
1724
  metaFor: (name: string) => [mode: number, mtimeMs: number | undefined],
1725
+ dated?: Map<string, number>,
1610
1726
  ): Promise<void> {
1611
1727
  try {
1612
- await this.commitAll(metaFor)
1728
+ await this.commitAll(metaFor, dated)
1613
1729
  } finally {
1614
1730
  this.closeReopened()
1615
1731
  }
@@ -1617,8 +1733,15 @@ class Extractor {
1617
1733
 
1618
1734
  private async commitAll(
1619
1735
  metaFor: (name: string) => [mode: number, mtimeMs: number | undefined],
1736
+ dated: Map<string, number> | undefined,
1620
1737
  ): Promise<void> {
1621
1738
  await this.drain()
1739
+ // A restore dates each file it writes now, as the task's own write
1740
+ // would, and moves no file's mtime back: the saved mtime predates
1741
+ // outputs written since, and a make-style dependant took a changed
1742
+ // upstream as older than its own output and kept bytes built from
1743
+ // another one. Read once: one clock for the whole restore.
1744
+ const now = dated === undefined ? undefined : Date.now()
1622
1745
  // One stat per directory, shared by the filesystem check and the group.
1623
1746
  const dirStats = new Map<string, Stats>()
1624
1747
  const statDir = (dir: string): Stats => {
@@ -1642,11 +1765,14 @@ class Extractor {
1642
1765
  for (const s of this.staged) {
1643
1766
  if (s.tree === undefined)
1644
1767
  await this.assertStillContained(s.base, path.dirname(s.target), checked)
1645
- const [mode, mtimeMs] = metaFor(s.name)
1768
+ const [mode, savedMtimeMs] = metaFor(s.name)
1646
1769
  if (s.same !== undefined) {
1647
- keep(s, s.same, mode, mtimeMs)
1770
+ const own = keep(s, s.same, mode, now === undefined ? savedMtimeMs : undefined)
1771
+ dated?.set(s.name, own)
1648
1772
  continue
1649
1773
  }
1774
+ const mtimeMs = now ?? savedMtimeMs
1775
+ if (now !== undefined && s.link === undefined) dated!.set(s.name, now)
1650
1776
  if (s.link !== undefined) {
1651
1777
  if (s.tree !== undefined) continue
1652
1778
  renames.push(handled(this.place(s, mode, undefined)))
@@ -1988,7 +2114,7 @@ const stagingMade = new Set<string>()
1988
2114
  */
1989
2115
  async function stagingDir(root: string): Promise<string> {
1990
2116
  const top = path.join(root, '.vx', STAGING)
1991
- const own = path.join(top, `${process.pid.toString(36)}-0`)
2117
+ const own = ownStaging(root)
1992
2118
  if (stagingMade.has(own) && existsSync(own)) return own
1993
2119
  await mkdir(own, { recursive: true })
1994
2120
  // Ignored by git whatever `.vx/.gitignore` an older init committed.
@@ -2001,6 +2127,45 @@ async function stagingDir(root: string): Promise<string> {
2001
2127
  return own
2002
2128
  }
2003
2129
 
2130
+ function ownStaging(root: string): string {
2131
+ return path.join(root, '.vx', STAGING, `${process.pid.toString(36)}-0`)
2132
+ }
2133
+
2134
+ /**
2135
+ * Where a tree restore's copy-out stages: this process's directory under
2136
+ * `root`'s `.vx/tmp/`, its stats, and a fresh name in it per call of
2137
+ * `temp`. With no root each temp sits beside its target.
2138
+ */
2139
+ export async function copyStaging(
2140
+ root: string | undefined,
2141
+ ): Promise<{ stats: Stats | undefined; temp: (target: string) => string }> {
2142
+ if (root === undefined) return { stats: undefined, temp: siblingTemp }
2143
+ // Made already: its stat is the check that it still stands, one call
2144
+ // where `stagingDir`'s check and a stat were two on every restore.
2145
+ let dir = ownStaging(root)
2146
+ let stats = stagingMade.has(dir) ? statSync(dir, { throwIfNoEntry: false }) : undefined
2147
+ if (stats === undefined) {
2148
+ dir = await stagingDir(root)
2149
+ stats = statSync(dir)
2150
+ }
2151
+ return { stats, temp: () => path.join(dir, (tmpSeq++).toString(36)) }
2152
+ }
2153
+
2154
+ /**
2155
+ * Whether what is made in the directory `staging` names keeps, renamed
2156
+ * into `dir`, what a write in place would give it: the same filesystem
2157
+ * (a rename across fails), and the same group.
2158
+ */
2159
+ export function stagesInto(staging: Stats, dir: Stats | undefined): boolean {
2160
+ return (
2161
+ dir !== undefined &&
2162
+ dir.isDirectory() &&
2163
+ dir.dev === staging.dev &&
2164
+ ((dir.mode | staging.mode) & 0o2000) === 0 &&
2165
+ (!INHERITS_GROUP || dir.gid === staging.gid)
2166
+ )
2167
+ }
2168
+
2004
2169
  /**
2005
2170
  * A temp beside `target`, with its own short name, never `<target>.suffix`:
2006
2171
  * a suffix pushed a legal name of 242–255 bytes past NAME_MAX, and the