@vzn/vx 0.0.642 → 0.0.644

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx",
-  "version": "0.0.642",
+  "version": "0.0.644",
   "description": "Task orchestration you own: correct, sandboxed, extensible, with a content-addressed build cache.",
   "type": "module",
   "exports": {
@@ -21,10 +21,10 @@
     "node": ">=18"
   },
   "optionalDependencies": {
-    "@vzn/vx-linux-x64": "0.0.642",
-    "@vzn/vx-linux-arm64": "0.0.642",
-    "@vzn/vx-darwin-x64": "0.0.642",
-    "@vzn/vx-darwin-arm64": "0.0.642"
+    "@vzn/vx-linux-x64": "0.0.644",
+    "@vzn/vx-linux-arm64": "0.0.644",
+    "@vzn/vx-darwin-x64": "0.0.644",
+    "@vzn/vx-darwin-arm64": "0.0.644"
   },
   "dependencies": {
     "@anthropic-ai/sandbox-runtime": "^0.0.76"

package/schemas/info.json

@@ -26,7 +26,6 @@
     "cacheEntries",
     "cacheBytes",
     "settings",
-    "cacheRetention",
     "orphans",
     "runs24h",
     "hits24h",
@@ -105,16 +104,6 @@
         }
       }
     },
-    "cacheRetention": {
-      "type": ["object", "null"],
-      "description": "cache.retention from its variables, else the workspace, maxSize also in bytes; null when none is declared",
-      "additionalProperties": false,
-      "properties": {
-        "olderThan": { "type": "string" },
-        "maxSize": { "type": "string" },
-        "maxBytes": { "type": "integer" }
-      }
-    },
     "orphans": {
       "type": "object",
       "additionalProperties": false,

package/skills/vx/SKILL.md

@@ -110,4 +110,4 @@ vx info --format json   # vx and Bun versions, projects, config errors, plugins,
 has installed, copy it from there instead:
 `cp node_modules/@vzn/vx/skills/vx/SKILL.md .claude/skills/vx/`.
 `vx upgrade` inside a workspace moves the project's `@vzn/vx` and the
-global vx that ran it to the same version.
+global vx that ran it to the same version, then runs `vx lock`.

package/src/cache/archive.ts

@@ -70,6 +70,7 @@ import {
   stat,
   symlink,
   unlink,
+  utimes,
   writeFile,
 } from 'node:fs/promises'
 import path from 'node:path'
@@ -162,6 +163,8 @@ class SumCheck {
   add(bytes: Uint8Array): void {
     this.sum.add(bytes)
   }
+  /** The checked sum's text, once `verify` passed. */
+  checked = ''
   verify(got: string): void {
     if (got !== this.sum.hex()) {
       throw new TarFormatError(
@@ -169,6 +172,7 @@ class SumCheck {
       )
     }
     this.seen = true
+    this.checked = got
   }
   done(): void {
     if (!this.seen) throw new TarFormatError('artifact carries no checksum')
@@ -471,6 +475,132 @@ function streamOf(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
   })
 }
 
+/**
+ * The tree form (`cache.store.decompress`, v47): an artifact's members
+ * as plain files under one directory, `outputs/<rel>` and
+ * `workspace-outputs/<rel>` as named in the tar, beside its sidecar and a
+ * `.vx-sum` over them. stdout lives in the index. A restore copies (or
+ * clones) each file out, so a hit decodes nothing; an upload packs the
+ * tree back into the artifact a remote expects.
+ */
+const TREE_SUM = SUM_ENTRY
+
+/**
+ * Write `plan`'s outputs into the empty `dir` as the tree form, each copied
+ * by the kernel (`copy_file_range`, or a clone where the filesystem has one)
+ * and given its sidecar mtime, and return what a scan of the plan's tar
+ * would: the save indexes from it without packing a tar.
+ */
+export async function writeTree(plan: ArtifactPlan, dir: string): Promise<ArtifactScan> {
+  const fold = new ScanFold()
+  const meta = plan.inputs.find((i) => i.name === META_ENTRY)!.body as Uint8Array
+  const metaText = utf8.decode(meta)
+  fold.sidecar(metaText)
+  const times = (JSON.parse(metaText) as MetaFile).files
+  for (const i of plan.inputs) {
+    if (i.name === META_ENTRY) continue
+    assertSafeName(i.name)
+    if (i.name === STDOUT_ENTRY) fold.stdout = utf8.decode(i.body as Uint8Array)
+    fold.entry(i.name, i.size, times[i.name]?.[1] ?? 0)
+  }
+  // A stdout-only entry writes no file to make the directory.
+  await mkdir(dir, { recursive: true })
+  await Promise.all(
+    plan.inputs.map(async (i) => {
+      if (!(i.body instanceof Blob)) return
+      const to = path.join(dir, i.name)
+      await Bun.write(to, i.body)
+      const at = new Date(times[i.name]![1])
+      await utimes(to, at, at)
+    }),
+  )
+  await writeFile(path.join(dir, META_ENTRY), meta)
+  await writeFile(path.join(dir, TREE_SUM), await treeSum(dir, stdoutOf(plan)))
+  return fold.result()
+}
+
+function stdoutOf(plan: ArtifactPlan): string {
+  return utf8.decode(plan.inputs.find((i) => i.name === STDOUT_ENTRY)!.body as Uint8Array)
+}
+
+/**
+ * Write a scanned artifact's sidecar and sum into `dir`, whose outputs an
+ * extract just wrote: an ingested remote hit kept as a tree.
+ */
+export async function sealTree(
+  dir: string,
+  scan: ArtifactScan,
+  /** The archive's own sidecar text and checked sum (`extractArtifactStream`'s `onScan`). */
+  raw?: { meta: string; sum: string },
+): Promise<void> {
+  // An artifact with no outputs extracts nothing, so nothing made `dir`.
+  await mkdir(dir, { recursive: true })
+  const meta: MetaFile = { version: 1, files: {} }
+  if (scan.key !== undefined) meta.key = scan.key
+  for (const e of scan.entries) {
+    if (e.name !== STDOUT_ENTRY) meta.files[e.name] = [e.mode, e.mtimeMs]
+  }
+  if (scan.dirs.length > 0) meta.dirs = scan.dirs
+  if (scan.links.length > 0) meta.links = Object.fromEntries(scan.links)
+  if (scan.exec !== undefined) meta.exec = scan.exec
+  const text = JSON.stringify(meta)
+  await writeFile(path.join(dir, META_ENTRY), text)
+  // The tree packs back to the archive it came from when the sidecar is
+  // the same bytes and the entries came in its order (stdout first, the
+  // sidecar last): the sum the extract checked is then the tree's, and the
+  // tree is not read back to sum it.
+  const names = Object.keys(meta.files)
+  const same =
+    raw !== undefined &&
+    raw.meta === text &&
+    scan.entries[0]?.name === STDOUT_ENTRY &&
+    scan.entries.length === names.length + 1 &&
+    names.every((n, i) => scan.entries[i + 1]!.name === n)
+  await writeFile(path.join(dir, TREE_SUM), same ? raw.sum : await treeSum(dir, scan.stdout ?? ''))
+}
+
+/** The plan that packs the tree in `dir` back into the artifact it holds. */
+export function planOfTree(dir: string, stdout: string): ArtifactPlan {
+  const metaBytes = readFileSync(path.join(dir, META_ENTRY))
+  const meta = JSON.parse(utf8.decode(metaBytes)) as MetaFile
+  const out = new TextEncoder().encode(stdout)
+  const inputs: TarInput[] = [{ name: STDOUT_ENTRY, size: out.byteLength, body: out }]
+  for (const [name, [mode, mtimeMs]] of Object.entries(meta.files)) {
+    const file = Bun.file(path.join(dir, name))
+    inputs.push({
+      name,
+      size: statSync(path.join(dir, name)).size,
+      mode,
+      mtime: Math.min(USTAR_MTIME_MAX, Math.max(0, Math.floor(mtimeMs / 1000))),
+      body: file,
+    })
+  }
+  inputs.push({ name: META_ENTRY, size: metaBytes.byteLength, body: new Uint8Array(metaBytes) })
+  return { inputs, size: tarSize([...inputs, sumInput('0'.repeat(SUM_SIZE))]) }
+}
+
+/** The sum over the tree's members, as `SUM_ENTRY` defines it for the tar it packs to. */
+async function treeSum(dir: string, stdout: string): Promise<string> {
+  const sum = new EntrySum()
+  for (const i of planOfTree(dir, stdout).inputs) {
+    sum.entry(i.name)
+    if (i.body instanceof Blob && i.size > 1 << 20)
+      for await (const c of i.body.stream()) sum.add(c)
+    else if (i.body instanceof Blob) sum.add(await i.body.bytes())
+    else sum.add(i.body as Uint8Array)
+  }
+  return sum.hex()
+}
+
+/** Whether the tree's bytes still sum to what was stored (`cache.store.verify: 'hash'`). */
+export async function treeIntact(dir: string, stdout: string): Promise<boolean> {
+  try {
+    return (await treeSum(dir, stdout)) === readFileSync(path.join(dir, TREE_SUM), 'utf8')
+  } catch {
+    return false
+  }
+}
+
 /** What a scan of an artifact reads out of it. */
 interface ArtifactScan {
   entries: ArchiveEntry[]
@@ -664,6 +794,12 @@ export async function extractArtifactStream(
   accept?: (name: string) => boolean,
   /** The workspace root whose `.vx/tmp/` stages the entries; absent, each stages beside its target. */
   stageRoot?: string,
+  /**
+   * Handed what `scanArtifact` would read of the same bytes, once their sum
+   * has checked, with the sidecar's text and the sum when the sidecar came
+   * last: what `sealTree` may reuse rather than read the tree back.
+   */
+  onScan?: (scan: ArtifactScan, raw: { meta: string; sum: string } | undefined) => void,
 ): Promise<Set<string>> {
   // No `mkdir -p` of the destination here: `stage` creates each entry's
   // directory chain, the destination included, and the cache restores
@@ -676,14 +812,22 @@ export async function extractArtifactStream(
   let meta: MetaFile['files'] = {}
   let dirs: string[] = []
   let links: Array<[string, string]> = []
+  const fold = onScan === undefined ? undefined : new ScanFold()
+  let metaText: string | undefined
+  let metaLast = false
   try {
     const check = new SumCheck()
     for await (const e of tarEntries(tar)) {
       if (e.type !== '0') continue
       const body = await check.body(e.name, e.body)
       if (body === null) continue
+      metaLast = false
       if (e.name === META_ENTRY) {
-        const parsed = JSON.parse(await textOf(body)) as MetaFile
+        const text = await textOf(body)
+        fold?.sidecar(text)
+        metaText = text
+        metaLast = true
+        const parsed = JSON.parse(text) as MetaFile
         meta = parsed.files ?? {}
         dirs = dirsOf(parsed)
         links = linksOf(parsed)
@@ -697,9 +841,11 @@ export async function extractArtifactStream(
           `archive entry is not one of the task's declared outputs: ${e.name}`,
         )
       }
+      fold?.entry(e.name, e.size, e.mtimeMs)
       const dest = x.destFor(e.name)
       if (dest === null || dest.rel.length === 0) {
-        await drain(body)
+        if (fold !== undefined && e.name === STDOUT_ENTRY) fold.stdout = await textOf(body)
+        else await drain(body)
         continue
       }
       const target = path.join(dest.base, dest.rel)
@@ -709,6 +855,11 @@ export async function extractArtifactStream(
       await x.stage(e.name, dest.base, target, e.size <= SMALL_ENTRY ? await bytesOf(body) : body)
     }
     check.done()
+    if (fold !== undefined)
+      onScan!(
+        fold.result(),
+        metaLast && metaText !== undefined ? { meta: metaText, sum: check.checked } : undefined,
+      )
     for (const name of dirs) {
       if (accept !== undefined && !accept(name)) {
         throw new ArchiveSecurityError(