@vzn/vx-dte 0.0.652 → 0.0.653

package/src/local.ts

@@ -3,7 +3,7 @@
 // loopback. For tests, demos
 // and trying a distributed run before a CI starts the workers.
 
-import { mkdtemp, writeFile } from 'node:fs/promises'
+import { mkdtemp, readFile, writeFile } from 'node:fs/promises'
 import os from 'node:os'
 import path from 'node:path'
 import { refuseUnknownOptions, type PluginOptionKinds } from '@vzn/vx'
@@ -24,7 +24,8 @@ export function local(options: LocalOptions = {}): DteAdapter {
   const procs = new Set<Bun.Subprocess>()
   /** A worktree add or a prepare: what close stops at once. */
   const helpers = new Set<Bun.Subprocess>()
-  const trees: Array<{ root: string; dir: string }> = []
+  /** The repositories this run adds worktrees to, whose records close prunes. */
+  const roots = new Set<string>()
   const pending = new Set<Promise<unknown>>()
 
   // Async throughout: a run starts its workers while it shows its first rows.
@@ -83,7 +84,14 @@ export function local(options: LocalOptions = {}): DteAdapter {
 
     async spawn(a) {
       const work = (async () => {
-        base ??= mkdtemp(path.join(os.tmpdir(), 'vx-dte-'))
+        base ??= (async () => {
+          const d = await mkdtemp(path.join(os.tmpdir(), BASE_PREFIX))
+          // Whose they are, for a later run to reclaim once this vx is gone.
+          await writeFile(path.join(d, OWNER), String(process.pid))
+          void reclaim(a.workspaceRoot)
+          return d
+        })()
+        roots.add(a.workspaceRoot)
         const dir = path.join(await base, String(a.index))
         if (closed) return
         // The run's own snapshot: one taken here, after a hit restored a
@@ -95,7 +103,6 @@ export function local(options: LocalOptions = {}): DteAdapter {
           { ...process.env, GIT_INDEX_FILE: undefined },
         )
         if (add.code !== 0) throw new Error(`git worktree add: ${errorLine(add.err)}`)
-        trees.push({ root: a.workspaceRoot, dir })
         // The workspace's own place in the checkout: from the worktree's
         // root, a workspace in a subdirectory of its repository had no
         // `.vx/workspace` above, and every worker exited (2026-10-25).
@@ -137,38 +144,115 @@ export function local(options: LocalOptions = {}): DteAdapter {
       relay?.stop()
       relay = undefined
       for (const p of helpers) group(p, 'SIGTERM')
-      const grace = setTimeout(() => {
-        for (const p of procs) {
-          if (helpers.has(p)) group(p, 'SIGKILL')
-          else p.kill('SIGKILL')
-        }
-      }, 5_000)
-      await Promise.allSettled([...pending])
-      await Promise.all([...procs].map((p) => p.exited))
-      clearTimeout(grace)
-      // A detached shell removes the worktrees after vx exits: removing ones
-      // an install filled took seconds after a Ctrl-C (2026-10-21), and a
-      // 300k-file tree 2.7 s. `prune` then drops git's record of each.
-      const roots = [...new Set(trees.splice(0).map((t) => t.root))]
+      // A detached shell waits for what this run started, kills what is
+      // left after 5 s, then removes the worktrees, after vx exits:
+      // removing ones an install filled took seconds after a Ctrl-C
+      // (2026-10-21), and a 300k-file tree 2.7 s. Spawned first: core stops
+      // waiting on a teardown after 3 s, and a worker slow to leave kept
+      // close from spawning it before vx exited, so every worktree stayed
+      // (2026-11-08). `prune` then drops git's record of each.
       if (base !== undefined) {
         const dir = await base
         base = undefined
-        Bun.spawn(
-          [
-            'sh',
-            '-c',
-            'rm -rf "$1"; shift; for r; do git -C "$r" worktree prune; done',
-            'sh',
-            dir,
-            ...roots,
-          ],
-          { stdio: ['ignore', 'ignore', 'ignore'], detached: true },
-        ).unref()
+        const helping = [...helpers].map((p) => p.pid).join(' ')
+        const working = [...procs]
+          .filter((p) => !helpers.has(p))
+          .map((p) => p.pid)
+          .join(' ')
+        Bun.spawn(['sh', '-c', CLEANUP, 'sh', dir, helping, working, ...roots], {
+          stdio: ['ignore', 'ignore', 'ignore'],
+          detached: true,
+        }).unref()
       }
+      roots.clear()
+      // Bounded under core's deadline: what still runs after it is the shell's.
+      await Promise.race([
+        Promise.allSettled([...pending, ...[...procs].map((p) => p.exited)]),
+        Bun.sleep(2_000),
+      ])
     },
   }
 }
 
+/**
+ * `sh -c CLEANUP sh <base> <helper pids> <worker pids> <roots…>`: waits up
+ * to 5 s for the pids, kills each left (a helper's whole group), removes
+ * the base and prunes each root's worktree records.
+ */
+const CLEANUP = `d=$1 h=$2 w=$3; shift 3
+i=0
+while [ $i -lt 50 ]; do
+  up=; for p in $h $w; do kill -0 $p 2>/dev/null && up=1; done
+  [ -z "$up" ] && break
+  sleep 0.1; i=$((i+1))
+done
+for p in $h; do kill -9 -$p 2>/dev/null; done
+for p in $w; do kill -9 $p 2>/dev/null; done
+rm -rf "$d"; for r; do git -C "$r" worktree prune; done`
+
+const BASE_PREFIX = 'vx-dte-'
+const OWNER = 'owner'
+/** `mkdtemp`'s name: the prefix and six characters. */
+const OWN_BASE = new RegExp(`^${BASE_PREFIX}[A-Za-z0-9]{6}$`)
+
+/**
+ * The worktrees a run whose vx was killed (SIGKILL, a lost machine) left in
+ * `root`'s repository, removed: each under a base whose owner process is
+ * gone. A killed run's close never ran, and they stayed for good
+ * (2026-11-06). One `git worktree list`, off the run's path.
+ */
+async function reclaim(root: string): Promise<void> {
+  try {
+    const env: Record<string, string | undefined> = { ...process.env }
+    for (const name of GIT_REPO_ENV) delete env[name]
+    const list = Bun.spawn(['git', 'worktree', 'list', '--porcelain'], {
+      cwd: root,
+      env,
+      stdin: 'ignore',
+      stdout: 'pipe',
+      stderr: 'ignore',
+    })
+    const bases = new Set<string>()
+    for (const line of (await new Response(list.stdout).text()).split('\n')) {
+      if (!line.startsWith('worktree ')) continue
+      const dir = line.slice('worktree '.length)
+      const base = path.dirname(dir)
+      if (/^\d+$/.test(path.basename(dir)) && OWN_BASE.test(path.basename(base))) bases.add(base)
+    }
+    const gone: string[] = []
+    for (const base of bases) {
+      const pid = Number(
+        await readFile(path.join(base, OWNER), 'utf8').catch(
+          // quiet: a base with no owner is not known to be gone
+          () => '',
+        ),
+      )
+      if (Number.isInteger(pid) && pid > 0 && !alive(pid)) gone.push(base)
+    }
+    if (gone.length === 0) return
+    Bun.spawn(
+      ['sh', '-c', 'r=$1; shift; rm -rf "$@"; git -C "$r" worktree prune', 'sh', root, ...gone],
+      {
+        env,
+        stdio: ['ignore', 'ignore', 'ignore'],
+        detached: true,
+      },
+    ).unref()
+  } catch {
+    // quiet: what is left is reclaimed by the next run
+  }
+}
+
+function alive(pid: number): boolean {
+  try {
+    process.kill(pid, 0)
+    return true
+  } catch (err) {
+    // Another user's process is alive too.
+    return (err as NodeJS.ErrnoException).code === 'EPERM'
+  }
+}
+
 /** Signal `p`'s process group: a helper leads its own (`detached`). */
 function group(p: Bun.Subprocess, signal: NodeJS.Signals): void {
   try {

package/src/messages.ts

@@ -54,6 +54,8 @@ export interface RunOrder {
   cwd: string
   project: string
   env: Record<string, string>
+  /** `HOST_ENV` names its key folds as unset: the worker leaves them unset, not its own. */
+  unset?: string[]
   /** Its `exec.env.secret`: the names in `env` whose values its saved output masks. */
   secret?: string[]
   timeoutMs?: number
@@ -65,9 +67,10 @@ export interface RunOrder {
    * Its `exec.sandbox`, and for a cached task the projects its key answers
    * for (workspace-relative): the worker builds the request on its own
    * paths and OS (`sandboxRequestFor`), as its orchestrator may run another;
-   * `reads`, the dependencies' outputs it may read (workspace-relative).
+   * `reads`, the dependencies' outputs it may read, and `reported`, the
+   * heads a refused read under is reported (workspace-relative).
    */
-  sandbox?: { config: SandboxConfig; keyed?: string[]; reads?: string[] }
+  sandbox?: { config: SandboxConfig; keyed?: string[]; reads?: string[]; reported?: string[] }
   /** Its persistent dependencies (`ServerSpec`), `cwd` workspace-relative: the worker starts each once. */
   servers?: ServerOrder[]
   /** The runtime probes its key includes, as the orchestrator answered them: the worker's must answer the same. */
@@ -119,6 +122,8 @@ export type ToWorker =
   /** Part of an order's `deps`, ahead of the batch whose order was too long for one frame. */
   | { type: 'deps'; task: string; deps: WiredDep[] }
   | { type: 'cancel'; task: string }
+  /** No task left in the run uses persistent task `task`: it stops here. */
+  | { type: 'unserve'; task: string }
   /**
    * It was taken into the run, under the name the run's output gives it,
    * and reads the cache as the run does: its refs, and the access the
@@ -152,8 +157,12 @@ export interface Load {
    * `cpu` (one that sent 3, each as a task started, read 0.6 of 2.0 used).
    */
   readonly avg?: number
+  /** Cores the worker's own process used on average since it started: what the run's bookkeeping costs it. */
+  readonly self?: number
   /** When it was read, by the worker's clock: its task times are read against it. */
   readonly at?: number
+  /** Each server it runs, its base as fitted (`server-fit.ts`): cores it uses with no dependant's work. */
+  readonly servers?: Record<string, number>
 }
 
 /** A task's times on its worker, epoch ms by the worker's clock: begun, command ended. */
@@ -219,7 +228,9 @@ export type WiredResult = Omit<ExecuteResult, 'violations' | 'outputs' | 'stdout
 
 /**
  * Names whose value is the worker's own: the orchestrator sends one only
- * when a task's `env.define` sets it, a value its key includes.
+ * when a task's `env.define` sets it or its `inputs.env` keys it, a value
+ * its key includes; a keyed one unset there is unset on the worker too
+ * (`RunOrder.unset`).
  */
 export const HOST_ENV = [
   'HOME',