@vzn/vx-dte 0.0.652 → 0.0.653

package/src/plugin.ts

@@ -22,7 +22,13 @@ import type { Orchestrator } from './orchestrator.js'
 import type { Proof } from './seal.js'
 import type { Forecast, Size } from './planner.js'
 
-/** One kind of worker the adapter can start. */
+/** One worker type: the machines of it the adapter can start, by name. */
+export interface WorkerType {
+  /** Its machines by name: each a size the adapter may start, named `<type>.<machine>`. */
+  readonly machines: Readonly<Record<string, WorkerSize>>
+}
+
+/** One kind of machine the adapter can start. */
 export interface WorkerSize {
   /** How many of it one run may start; absent, as many as the work needs. */
   readonly max?: number
@@ -32,6 +38,8 @@ export interface WorkerSize {
   readonly memory: number | string
   /** Its machines' platform, which a joining worker must report; absent, planned as this machine's and not checked. A task naming `exec.platforms` runs only on a worker of one of them. */
   readonly platform?: Platform
+  /** The CI runner label its workers run on, which `vx dte init` writes (`github()`); absent, its platform's GitHub-hosted runner. */
+  readonly runsOn?: string
 }
 
 /** What an adapter is asked to start: one worker of `size`. */
@@ -103,8 +111,13 @@ export interface DteAdapter {
 }
 
 export interface DteOptions {
-  /** What the adapter may start, by size name. */
-  readonly workers?: Readonly<Record<string, WorkerSize>>
+  /**
+   * The worker types, in order of preference: a task naming `exec.worker`
+   * runs only on that type's machines, any other on the first type with
+   * a machine of one of its platforms. Each machine is a size the adapter
+   * may start, named `<type>.<machine>` (`ubuntu.small`).
+   */
+  readonly workers?: Readonly<Record<string, WorkerType>>
   /** Starts workers; absent, the run uses the workers that join it, which needs `relay`. */
   readonly adapter?: DteAdapter
   /** The relay's URL; absent, the adapter's own. */
@@ -139,12 +152,14 @@ const KEYS: PluginOptionKinds<DteOptions> = {
   headers: 'object',
 }
 
+const TYPE_KEYS: PluginOptionKinds<WorkerType> = { machines: 'object' }
 // A typo in a size or in `unmeasured` (`maxx`) was dropped, and the size ran uncapped.
 const SIZE_KEYS: PluginOptionKinds<WorkerSize> = {
   max: 'number',
   cpu: 'number',
   memory: 'any',
   platform: 'string',
+  runsOn: 'string',
 }
 const UNMEASURED_KEYS: PluginOptionKinds<NonNullable<DteOptions['unmeasured']>> = {
   cpu: 'number',
@@ -189,8 +204,19 @@ export function dte(options: DteOptions = {}) {
       refuse(`scheduler.${layer}`, `one of ${choices.join(', ')}`, choice)
     ;(how as unknown as Record<string, unknown>)[layer] = choice
   }
-  const sizes = Object.entries(options.workers ?? {}).map(([name, s]) => {
-    const at = `workers.${name}`
+  const declared = Object.entries(options.workers ?? {}).flatMap(([type, t]) => {
+    refuseUnknownOptions(`dte() option "workers.${type}"`, t, TYPE_KEYS)
+    const machines = Object.entries(t?.machines ?? {})
+    if (machines.length === 0)
+      refuse(
+        `workers.${type}.machines`,
+        "machines by name, such as small: { cpu: 4, memory: '8GB' }",
+        t?.machines,
+      )
+    return machines.map(([machine, s]) => ({ type, name: `${type}.${machine}`, s }))
+  })
+  const sized = declared.map(({ type, name, s }) => {
+    const at = `workers.${type}.machines.${name.slice(type.length + 1)}`
     refuseUnknownOptions(`dte() option "${at}"`, s, SIZE_KEYS)
     if (!(Number.isInteger(s?.cpu) && s.cpu >= 1)) refuse(`${at}.cpu`, 'a whole number ≥ 1', s?.cpu)
     const memory = bytes(s.memory)
@@ -198,23 +224,62 @@ export function dte(options: DteOptions = {}) {
       refuse(`${at}.memory`, "bytes above 0 or a size such as '8GB'", s.memory)
     if (!(s.max === undefined || (Number.isInteger(s.max) && s.max >= 0)))
       refuse(`${at}.max`, 'a whole number ≥ 0', s.max)
+    if (s.runsOn !== undefined && s.runsOn.trim() === '')
+      refuse(`${at}.runsOn`, 'a runner label', s.runsOn)
     if (s.platform !== undefined && !PLATFORMS.includes(s.platform))
       refuse(`${at}.platform`, `one of ${PLATFORMS.join(', ')}`, s.platform)
     return {
+      type,
       name,
       max: s.max ?? Infinity,
       cpu: s.cpu,
       memory: memory!,
       platform: s.platform ?? hostPlatform(),
+      named: s.platform,
+      runsOn: s.runsOn,
     }
   })
+  const types = [...new Set(sized.map((s) => s.type))]
+  const typeOf = new Map(sized.map((s) => [s.name, s.type]))
+  // With one type every worker is of it; with more, a worker's type rides
+  // its platform as `<type>:<platform>`, which every match compares whole.
+  const multi = types.length > 1
+  const sizes: Size[] = sized.map((s) => ({
+    name: s.name,
+    max: s.max,
+    cpu: s.cpu,
+    memory: s.memory,
+    platform: multi ? `${s.type}:${s.platform}` : s.platform,
+  }))
+  const host = hostPlatform()
+  /** The platform a task of `type` naming none runs on: this machine's if the type has it, else its first. */
+  const homeOfType = (type: string): string => {
+    const of = sized.filter((s) => s.type === type).map((s) => s.platform)
+    return of.includes(host) ? host : of[0]!
+  }
+  /** Where a task runs, as workers' platforms read: its type's place, or as one type's run reads it. */
+  const placeOf = (worker: string | undefined, platform: string | undefined): string | undefined =>
+    !multi || worker === undefined
+      ? (platform ?? home)
+      : `${worker}:${platform ?? homeOfType(worker)}`
+  /** A worker's place: its size's type, else (one that joined on its own) the first type of its platform. */
+  const joinedPlace = (
+    size: string | undefined,
+    platform: string | undefined,
+  ): string | undefined => {
+    if (!multi || platform === undefined) return platform
+    const type =
+      (size === undefined ? undefined : typeOf.get(size)) ??
+      sized.find((s) => s.platform === platform)?.type
+    return type === undefined ? platform : `${type}:${platform}`
+  }
   // Only a platform the size names refuses a worker reporting another: an
   // unnamed one is planned as this machine's, and takes whatever joins.
   const named = (size: string): { platform?: Platform } => {
-    const platform = options.workers?.[size]?.platform
+    const platform = sized.find((s) => s.name === size)?.named
     return platform === undefined ? {} : { platform }
   }
-  const home = homeOf(sizes, hostPlatform())
+  const home = multi ? undefined : homeOf(sizes, host)
   const unmeasured = unmeasuredOf(options.unmeasured)
   const adapter = options.adapter
   // An adapter's run in its own CI with a relay names itself: the CI file
@@ -234,7 +299,7 @@ export function dte(options: DteOptions = {}) {
   // With none, no worker was ever asked for and the run waited until killed.
   if (adapter !== undefined && !sizes.some((s) => s.max > 0)) {
     throw new UserError(
-      "dte(): workers names no size the adapter may start (one with max above 0), such as small: { cpu: 2, memory: '8GB' }",
+      "dte(): workers names no machine the adapter may start (one with max above 0), such as linux: { machines: { small: { cpu: 2, memory: '8GB' } } }",
     )
   }
   if (adapter === undefined && options.relay === undefined) {
@@ -267,6 +332,10 @@ export function dte(options: DteOptions = {}) {
   let peaks: ReadonlyMap<string, TaskResources> = new Map()
   /** Each size's warm-up an earlier run learned (`warm-ups.ts`), and the read of it. */
   let learned: ReadonlyMap<string, number> = new Map()
+  /** Each server's base an earlier run learned, then this run's workers' over it (`server-fit.ts`). */
+  let bases = new Map<string, number>()
+  /** The bases as read, before this run's: kept with this run's over them. */
+  let basesRead: ReadonlyMap<string, number> = new Map()
   let warming: Promise<void> = Promise.resolve()
   /** On CI, where the run keeps the warm-ups it learned: `setup`'s cache, root and warn. */
   let keep: { cache: PluginCache; root: string; warn: (m: string) => void } | undefined
@@ -281,6 +350,11 @@ export function dte(options: DteOptions = {}) {
   // Each finished task's cache key, a hit's too: what a worker restores a
   // dependency by.
   const keys = new Map<string, string>()
+  // Tasks that run a command and have not ended, by the platform they name
+  // ('' for none): a worker the run cannot start again stays while its
+  // platform has one (`keepLast`).
+  const leftOf = new Map<string, number>()
+  const ended = new Set<string>()
   let orchestrator: Orchestrator | undefined
   let closing: Promise<void> | undefined
 
@@ -298,7 +372,18 @@ export function dte(options: DteOptions = {}) {
         k.warn(`[vx] dte: warm-ups not kept: ${err instanceof Error ? err.message : String(err)}`),
       )
     }
+    if (k !== undefined && o !== undefined) {
+      const fitted = new Map([...bases].filter(([id, c]) => basesRead.get(id) !== c))
+      const { writeServerBases } = await import('./warm-ups.js')
+      await writeServerBases(k.cache, k.root, basesRead, fitted).catch((err: unknown) =>
+        k.warn(
+          `[vx] dte: server bases not kept: ${err instanceof Error ? err.message : String(err)}`,
+        ),
+      )
+    }
     learned = new Map()
+    bases = new Map()
+    basesRead = new Map()
     warming = Promise.resolve()
     peaks = new Map()
     tails = undefined
@@ -343,10 +428,11 @@ export function dte(options: DteOptions = {}) {
       workspaceRoot: root,
       dir: git('--show-prefix').replace(/\/$/, ''),
       ...(options.relay !== undefined ? { relay: options.relay } : {}),
-      sizes: sizes.map((s) => ({
+      sizes: sized.map((s) => ({
         name: s.name,
         ...(s.max === Infinity ? {} : { max: s.max }),
-        ...named(s.name),
+        ...(s.named === undefined ? {} : { platform: s.named }),
+        ...(s.runsOn === undefined ? {} : { runsOn: s.runsOn }),
       })),
       manager: managerOf(root),
       tasks: askTasks(),
@@ -432,6 +518,10 @@ export function dte(options: DteOptions = {}) {
       if (ctx.ci) keep = { cache: ctx.cache, root: ctx.workspaceRoot, warn: (m) => ctx.warn(m) }
       ctx.on('onTaskComplete', (node, outcome) => {
         if (outcome.hash !== undefined) keys.set(node.id, outcome.hash)
+        if (node.config.exec?.command === undefined || ended.has(node.id)) return
+        ended.add(node.id)
+        const p = placeOf(node.worker, node.platform) ?? ''
+        leftOf.set(p, (leftOf.get(p) ?? 1) - 1)
       })
     },
 
@@ -441,9 +531,32 @@ export function dte(options: DteOptions = {}) {
       tails = undefined
       // A run that is not distributed reads nothing.
       if (orchestrator === undefined) return undefined
+      // An uncached upstream runs again where its dependant runs, on a
+      // worker type or platform it may name otherwise: refused here, not
+      // run there (2026-11-08).
+      const { apart } = await import('./orchestrator.js')
+      const where = (n: TaskNode): string => placeOf(n.worker, n.platform) ?? ''
+      const platformOf = (n: TaskNode): string => where(n).slice(where(n).indexOf(':') + 1)
+      const bound = apart(
+        graph,
+        (u, d) =>
+          (multi && u.config.exec?.worker !== undefined && u.worker !== d.worker) ||
+          (u.config.exec?.platforms !== undefined && platformOf(u) !== platformOf(d)),
+      )
+      if (bound !== undefined)
+        throw new UserError(
+          `dte(): ${bound.dependant} runs on ${where(graph.get(bound.dependant)!)} and reads what uncached ${bound.upstream} leaves where it runs, which cannot be there: its exec.worker or exec.platforms say ${where(graph.get(bound.upstream)!)}. Cache ${bound.upstream}, or run both alike`,
+        )
       const ids = [...graph.values()]
         .filter((n) => n.config.exec?.command !== undefined)
         .map((n) => n.id)
+      leftOf.clear()
+      ended.clear()
+      for (const id of ids) {
+        const n = graph.get(id)!
+        const p = placeOf(n.worker, n.platform) ?? ''
+        leftOf.set(p, (leftOf.get(p) ?? 0) + 1)
+      }
       const read = async (): Promise<void> => {
         if (orchestrator === undefined) return
         try {
@@ -452,6 +565,7 @@ export function dte(options: DteOptions = {}) {
             defs: taskDefinitions(graph.values()),
           }).resourcesFor(ids)
           tails = undefined
+          orchestrator?.rerank()
         } catch (err) {
           ctx.warn(
             `[vx] dte: tasks are packed one core each: ${err instanceof Error ? err.message : String(err)}`,
@@ -459,6 +573,18 @@ export function dte(options: DteOptions = {}) {
         }
       }
       let reading = read()
+      const basesIn = import('./warm-ups.js')
+        .then((w) => w.readServerBases(ctx.cache, ctx.workspaceRoot))
+        .then(
+          (m) => {
+            basesRead = m
+            for (const [id, cores] of m) if (!bases.has(id)) bases.set(id, cores)
+          },
+          (err: unknown) =>
+            ctx.warn(
+              `[vx] dte: no server bases learned: ${err instanceof Error ? err.message : String(err)}`,
+            ),
+        )
       warming = import('./warm-ups.js')
         .then((w) => w.readWarmUps(ctx.cache, ctx.workspaceRoot))
         .then(
@@ -472,7 +598,7 @@ export function dte(options: DteOptions = {}) {
         const { likelyMisses } = await import('./forecast.js')
         likely = likelyMisses(graph, changed, ctx.workspaceRoot)
       })
-      await Promise.all([reading, guess])
+      await Promise.all([reading, guess, basesIn])
       // Needs another plugin fetched (the default branch's timings) land
       // in history after this: the tasks not yet placed pack by them, and
       // the first ask for workers waits for them.
@@ -497,8 +623,8 @@ export function dte(options: DteOptions = {}) {
       const [
         { Orchestrator },
         { joinOffLoop },
-        { decide, knownMisses, slowdown, tailsMs, WARM_UP_MS },
-        { snapshot },
+        { decide, keepLast, knownMisses, slowdown, tailsMs, WARM_UP_MS },
+        { snapshot, untrackedWorkspace },
         seal,
       ] = await Promise.all([
         import('./orchestrator.js'),
@@ -527,7 +653,23 @@ export function dte(options: DteOptions = {}) {
       proof ??= seal.secretProof(secret!)
       const { commit, tree } = await snapshot(ctx.workspaceRoot)
       if (tree === '') throw new UserError('dte(): a DTE run needs a git commit')
-      const need = (id: string): Need => booked(needOf(peaks.get(id), unmeasured), how.book)
+      const untracked = await untrackedWorkspace(ctx.workspaceRoot, tree)
+      if (untracked !== undefined)
+        throw new UserError(
+          `dte(): ${untracked} is untracked, so the commit workers check out lacks it: commit it`,
+        )
+      const serves = new Map<string, readonly string[]>()
+      const need = (id: string): Need => {
+        const n = booked(needOf(peaks.get(id), unmeasured), how.book)
+        let ids = serves.get(id)
+        if (ids === undefined) {
+          ids = serversOf(nodes, id)
+          // Read again once the graph is: none before it.
+          if (nodes !== undefined) serves.set(id, ids)
+        }
+        if (ids.length === 0) return n
+        return { ...n, servers: ids.map((sv) => ({ id: sv, cores: bases.get(sv) ?? 0 })) }
+      }
       // The run plans again on it, now when `now`; set once the plan is.
       let onChange = (_now?: boolean): void => {}
       const o = new Orchestrator({
@@ -543,6 +685,7 @@ export function dte(options: DteOptions = {}) {
         tree,
         nodes: () => nodes,
         need,
+        bases,
         artifacts: ctx.artifacts,
         cacheFacts: () => ctx.cacheFacts(),
         run,
@@ -552,6 +695,7 @@ export function dte(options: DteOptions = {}) {
           (tails ??= tailsMs(nodes ?? new Map(), (x) => peaks.get(x)?.p50DurationMs))(id),
         how,
         home,
+        joinedPlace,
         lost: (id) => {
           const s = sizeOf.get(id)
           if (s === undefined) return
@@ -592,6 +736,7 @@ export function dte(options: DteOptions = {}) {
           cpu: s.cpu,
           memory: s.memory,
           ...named(s.name),
+          ...(multi && s.platform !== undefined ? { place: s.platform } : {}),
         })
         sizeOf.set(id, s)
         return {
@@ -657,7 +802,14 @@ export function dte(options: DteOptions = {}) {
           const m = p50(id)
           return m === undefined ? undefined : m * slow
         }
-        const { workers, running } = o.planState(ms, warmUp)
+        const state = o.planState(ms, warmUp)
+        const running = state.running
+        const workers = keepLast(
+          state.workers,
+          sizes,
+          started,
+          (p) => (leftOf.get(p) ?? 0) > 0 || (p === home && (leftOf.get('') ?? 0) > 0),
+        )
         const final = adapter === undefined
         const p = decide({
           remaining,
@@ -676,7 +828,7 @@ export function dte(options: DteOptions = {}) {
           limitMs: deadline === undefined ? undefined : deadline - performance.now(),
           known,
           how,
-          home,
+          home: multi ? (n: TaskNode) => placeOf(n.worker, n.platform) : home,
           queued: o.queued(warmUp),
           last,
           now: performance.now(),
@@ -815,23 +967,30 @@ export function dte(options: DteOptions = {}) {
         // A worker answers a task's runtime probes before it runs it.
         runtime: true,
         // Its sizes' platforms; workers that join on their own, this machine's.
-        platforms:
-          sizes.length === 0 ? [hostPlatform()] : [...new Set(sizes.map((s) => s.platform))],
-        // Room for the workers' cores and their prefetch; the orchestrator
-        // queues the rest. Workers that join on their own are as many as join.
-        capacity:
-          adapter === undefined || sizes.some((s) => s.max === Infinity)
-            ? JOINED_CAPACITY
-            : Math.max(
-                1,
-                sizes.reduce((n, s) => n + 2 * s.max * s.cpu, 0),
+        platforms: sized.length === 0 ? [host] : [...new Set(sized.map((s) => s.platform))],
+        // Its worker types, in declared order, each with its machines' platforms.
+        ...(types.length === 0
+          ? {}
+          : {
+              workers: types.map(
+                (t) =>
+                  [
+                    t,
+                    [...new Set(sized.filter((s) => s.type === t).map((s) => s.platform))],
+                  ] as const,
               ),
+            }),
+        // Every ready task: the orchestrator's queue knows which platform's
+        // worker is free, core's order does not. A pool of twice the cores,
+        // shared by all platforms, held a 2 s arm task 79 s behind x64 work
+        // with its worker idle (main a1950a2, 2026-11-07).
+        capacity: Number.MAX_SAFE_INTEGER,
         execute: (req) => {
           // A miss: the run's known work grows.
           missed = true
           executed.add(req.taskId)
           replan()
-          return o.execute(req)
+          return o.execute(multi ? { ...req, platform: placeOf(req.worker, req.platform)! } : req)
         },
         footer: () =>
           o.footer(
@@ -843,6 +1002,7 @@ export function dte(options: DteOptions = {}) {
         advice: () => o.advice(depsOf, sizes),
         profile: () => o.profile(),
         running: (id) => o.where(id),
+        unneeded: (id) => o.unneeded(id),
         demand(remaining, misses) {
           if (pending === undefined) o.probed(remaining.size, misses.size)
           o.missed = misses
@@ -948,6 +1108,28 @@ function refuse(option: string, want: string, got: unknown): never {
   throw new UserError(`dte() option "${option}" must be ${want}, got ${said}`)
 }
 
+/**
+ * The persistent tasks `id` needs where it runs, as core hands them to a
+ * `servers` executor: its dependencies that are servers, through groups,
+ * and the servers they depend on.
+ */
+export function serversOf(nodes: ReadonlyMap<string, TaskNode> | undefined, id: string): string[] {
+  const out: string[] = []
+  const seen = new Set<string>()
+  const stack = [...(nodes?.get(id)?.deps ?? [])]
+  while (stack.length > 0) {
+    const d = stack.pop()!
+    if (seen.has(d)) continue
+    seen.add(d)
+    const n = nodes?.get(d)
+    if (n === undefined) continue
+    if (n.config.exec?.persistent !== undefined) out.push(d)
+    else if (n.config.exec !== undefined) continue
+    stack.push(...n.deps)
+  }
+  return out
+}
+
 /** What a task asks of a worker, from its history. */
 export function needOf(r: TaskResources | undefined, ask: Need = FRESH): Need {
   // Unmeasured (new, or over before the Linux sandbox's first look, which
@@ -969,9 +1151,6 @@ export function needOf(r: TaskResources | undefined, ask: Need = FRESH): Need {
   return { cores: Math.max(0.25, r.maxCpuParallelism), memory: r.maxPeakRssBytes ?? 0 }
 }
 
-/** The executor's pool when workers are uncapped: more than any run starts at once. */
-const JOINED_CAPACITY = 1024
-
 const UNITS: Record<string, number> = {
   B: 1,
   KB: 1e3,