@vzn/vx 0.0.644 → 0.0.646

package/src/orchestrator/plugin.ts

@@ -24,6 +24,7 @@ import { maskedLine, nearest, type PluginSecrets, SecretBroker, UserError } from
 import type { ProjectMeta } from '../workspace/index.js'
 import type { EventBus, Logger, RunStartInfo } from '../report/index.js'
 import type { CiContext } from './run-context.js'
+import type { CacheFacts, TaskArtifacts } from './task-artifacts.js'
 import type { TelemetryContext, TelemetrySink } from './telemetry.js'
 
 /**
@@ -334,6 +335,20 @@ export interface CommandContext extends BaseContext {
    * workspace's cache plugins supply, under the run's scope. Opened on first use.
    */
   readonly cache: PluginCache
+  /**
+   * Task entries by key in the workspace's cache, under a run's policy and
+   * scope (a laptop writes no remote unless the workspace's policy names
+   * it): what a verb that runs a run's tasks on this machine saves and
+   * restores through (`TaskArtifacts`). Opened on first use.
+   */
+  readonly artifacts: TaskArtifacts
+  /**
+   * `artifacts` read as another machine's run would read them: under its
+   * refs, with the access its remotes lent (`CacheFacts`, asked on each
+   * request, so a token it lends again replaces the last). Opened on first
+   * use; until `facts` answers, under this machine's own.
+   */
+  artifactsFor(facts: () => CacheFacts | undefined): TaskArtifacts
 }
 
 /** `config` runs before the cache dir is known — it may be what the hook changes. */
@@ -346,10 +361,9 @@ export interface WorkspaceHookContext {
  * What a CI knows of the run's branch (owner, 2026-10-17): `scope`, the
  * cache scope (`'vznjs/vx:feat-x'`; `VX_CACHE_SCOPE` beats it, and it
  * beats the one git names), and `base`, the commit the branch is compared
- * with (a pull request's base), for a bare `--affected` and for which
- * tasks read the scope. `trusted`: the run has no scope, whatever git
- * names, and reads and writes only the default keys (a merge queue's run,
- * owner 2026-10-18). A later plugin's field wins.
+ * with (a pull request's base), for a bare `--affected`. `trusted`: the
+ * run has no scope, whatever git names (a merge queue's run, owner
+ * 2026-10-18). The scope keys plugin entries only. A later plugin's field wins.
  */
 export interface BranchFacts {
   scope?: string
@@ -501,16 +515,35 @@ export interface ScheduleHookContext extends BaseContext {
    * from a remote). Ignored in a plan and once the run's tasks are done.
    */
   refine(weights: ReadonlyMap<string, number>): void
+  /**
+   * Calls `listener` each time another plugin `refine`s: the late data it
+   * waited for (timings fetched from a remote) has landed in history, so a
+   * plugin that read history in `schedule` may read it again. Never called
+   * in a plan.
+   */
+  onRefine(listener: () => void): void
 }
 
 /** A `cache` hook's context: the workspace and the run's scope, never the local store or the policy, which core owns. */
 export interface CacheContext extends BaseContext {
   /**
-   * The run's cache scope (`'vznjs/vx:feat-x'`), `undefined` when none. Core already keys a scoped run's
-   * writes apart; a remote reads it to pick a credential or check a token's
-   * claims (an OIDC branch) for every call it makes.
+   * The run's cache scope (`'vznjs/vx:feat-x'`), `undefined` when none: what
+   * plugins' `ctx.cache` entries and stored runs are keyed apart by.
    */
   readonly scope: string | undefined
+  /**
+   * The refs the run reads, in order (`['refs/heads/feat-x', 'refs/heads/main']`):
+   * its own branch, the branch it merges into, the default branch. A remote
+   * keys or scopes its entries by them; core keys task entries by none.
+   */
+  readonly refs: readonly string[]
+  /**
+   * The token a run lent this plugin's remote (`RemoteCacheLayer.lend`),
+   * on a machine that runs that run's tasks (`CommandContext.artifactsFor`);
+   * undefined anywhere else. Read it per request: a run lends again before
+   * a token ends. A plugin uses it only when it has no access of its own.
+   */
+  readonly lent?: () => string | undefined
 }
 
 export interface OutputContext extends BaseContext {
@@ -523,6 +556,14 @@ export interface OutputContext extends BaseContext {
 export interface ExecutorContext extends BaseContext {
   /** The run's worker count — an executor that paces itself reads it here. */
   readonly concurrency: number
+  /** Task entries by key in the run's cache, for a task another machine ran (`TaskArtifacts`). */
+  readonly artifacts: TaskArtifacts
+  /**
+   * What a machine that runs this run's tasks reads the cache under
+   * (`CacheFacts`): the run's refs, and each remote's lent access, asked
+   * now. Ask again before `expiresAt`.
+   */
+  cacheFacts(): Promise<CacheFacts>
 }
 
 export interface PluginContext extends PluginSetupContext {

package/src/orchestrator/prepare.ts

@@ -26,7 +26,6 @@ import {
   writesRunStore,
   type PluginCache,
   type ResolvedRemote,
-  BranchScope,
   applyGitEnumeration,
   gitPathspecs,
   MAX_SCOPED_PATHSPECS,
@@ -34,14 +33,11 @@ import {
   lazyGitEnumeration,
   measureFsClockLag,
   DEFAULT_STORE,
-  type StoreOptions,
 } from '../cache/index.js'
 import {
   buildPackageGraph,
   computeNestedProjectDirs,
   findWorkspaceRoot,
-  type AffectedChanges,
-  branchChanges,
   configuredBaseBranch,
   type LoadReads,
   PROJECT_CONFIG_FILENAMES,
@@ -81,13 +77,13 @@ import {
   lockReader,
   type LoadedProjects,
 } from './projects.js'
-import { keptByAffected, keyedByChange } from './affected-tasks.js'
+import { keptByAffected } from './affected-tasks.js'
 import { applyGraphStage } from './graph-stage.js'
 import { keyExcludedDependencies } from './excluded-keys.js'
 import { FingerprintWatch } from './fingerprint-watch.js'
 import type { VxPlugin } from './plugin.js'
 import { askIgnoredOutputs, gatedOutputs } from './stable-keys.js'
-import { currentBranch, resolveCacheScope } from './run-context.js'
+import { cacheRefs, currentBranch, resolveCacheScope } from './run-context.js'
 import { createHashCache, type HashCache } from './task-hash.js'
 import { type Logger, warnOf } from '../report/index.js'
 import type { RunOptions } from './options.js'
@@ -120,6 +116,8 @@ export interface PreparedRun {
   cacheScope: string | undefined
   /** The run's remotes in declared order, as the policy gates each; `openRemotes` turns one off. */
   remotes: ResolvedRemote[]
+  /** The refs the run reads, in order (`CacheContext.refs`). */
+  cacheRefs: readonly string[]
   /**
    * Scheduling priorities from plugins' `schedule` stage (task id → weight,
    * merged over the structural baseline by the scheduler). Empty when no
@@ -291,33 +289,6 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
     reused !== undefined
       ? Promise.resolve(reused.projects)
       : discoverProjects(workspaceRoot, readLock, plugins, cacheDir, warnOf(log), git)
-  // What the branch changed decides which tasks read the scope (owner,
-  // 2026-10-17). Only a scoped run's first remote call asks git: a run the
-  // local cache answers makes no remote call, and diffing beside discovery
-  // cost it 20-60 ms at 1k-5k projects.
-  const scopeChanges: (() => Promise<AffectedChanges>) | undefined =
-    cacheScope === undefined
-      ? undefined
-      : options.scopeChanges !== undefined
-        ? async () => options.scopeChanges!
-        : () =>
-            branchChanges(
-              workspaceRoot,
-              options.scopeBase ?? branchFacts.base,
-              baseBranch,
-              discovering,
-              {
-                untracked: async () => (await git.start()).untracked,
-                // A scoped enumeration lists only its projects: null asks git.
-                links: async () => {
-                  const g = await git.start()
-                  if (!g.scopes.includes('.')) return null
-                  const out: string[] = []
-                  for (const [rel, oid] of g.trusted) if (oid.startsWith('120000:')) out.push(rel)
-                  return out
-                },
-              },
-            )
   const projectMetas = await discovering
   mark('discover projects')
 
@@ -383,8 +354,6 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
         DEFAULT_STORE.embeddings) as number,
       decompress: (resolveSetting('cache.store.decompress', workspaceConfig).value ??
         DEFAULT_STORE.decompress) as boolean,
-      verify: (resolveSetting('cache.store.verify', workspaceConfig).value ??
-        DEFAULT_STORE.verify) as StoreOptions['verify'],
     },
   )
   const unwritable = localCache.readOnlyNotice()
@@ -519,9 +488,6 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
   // cache plugin.
   let cache: CacheLayer
   const remotes: ResolvedRemote[] = []
-  // A scope only picks remote keys (owner, 2026-10-17): the local store
-  // keeps every branch's entries under the task keys.
-  const branchScope = cacheScope === undefined ? undefined : new BranchScope(cacheScope)
   if (options.remoteCache) {
     remotes.push({
       remote: options.remoteCache,
@@ -530,18 +496,20 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
       onError: (err) => warnOf(log)(`[vx] remote cache: ${err.message}`),
     })
   }
+  // An injected remote is no plugin's: nothing reads refs for it.
+  const refs = options.remoteCache ? [] : cacheRefs(workspaceRoot, baseBranch)
   try {
     cache = options.remoteCache
-      ? new LayeredCache(localCache, remotes, branchScope, options.uploadConcurrency)
+      ? new LayeredCache(localCache, remotes, options.uploadConcurrency)
       : await resolveCache(plugins, {
           workspaceRoot,
           cacheDir,
           warn: warnOf(log),
           scope: cacheScope,
+          refs,
           localCache,
           policy,
           remotes,
-          ...(branchScope !== undefined ? { branch: branchScope } : {}),
           ...(options.uploadConcurrency !== undefined
             ? { uploadConcurrency: options.uploadConcurrency }
             : {}),
@@ -590,6 +558,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
       cachePolicy: policy,
       cacheScope,
       remotes,
+      cacheRefs: refs,
       keepRunEntry: undefined,
       priorities: () => Promise.resolve(new Map()),
       branch,
@@ -776,13 +745,6 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
     if (hasHook(plugins, 'key')) {
       await applyKeyHooks(plugins, nodes, { workspaceRoot, cacheDir, warn: warnOf(log) })
     }
-    // The final inputs decide which tasks the branch changed: they read
-    // the scope, every other task the default key.
-    if (branchScope !== undefined && scopeChanges !== undefined) {
-      branchScope.routeTasks(async () =>
-        keyedByChange(nodes, await scopeChanges(), projects, packageGraph),
-      )
-    }
     refuseFiltersThroughGroups(nodes)
     const exclude = options.excludeDependencies
     let keyOnly: ReadonlyMap<string, TaskNode> = new Map()
@@ -847,6 +809,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
       cachePolicy: policy,
       cacheScope,
       remotes,
+      cacheRefs: refs,
       priorities,
       branch,
       cacheFor,
@@ -958,7 +921,7 @@ function pruneToClosure(nodes: Map<string, TaskNode>, roots: ReadonlySet<string>
   }
 }
 
-const namesRemote = (spec: string | undefined): boolean =>
+export const namesRemote = (spec: string | undefined): boolean =>
   spec !== undefined && /remote:[rw]/.test(spec)
 
 /**
@@ -966,7 +929,9 @@ const namesRemote = (spec: string | undefined): boolean =>
  * since the grammar lives in the cache module the workspace module may not
  * import. Undefined when it names none.
  */
-function workspaceCachePolicy(config: WorkspaceConfig | null | undefined): CachePolicy | undefined {
+export function workspaceCachePolicy(
+  config: WorkspaceConfig | null | undefined,
+): CachePolicy | undefined {
   const spec = config?.cache?.policy
   if (spec === undefined) return undefined
   try {

package/src/orchestrator/run-context.ts

@@ -553,6 +553,44 @@ export function localBranchScope(
   return branchCacheScope(repo, name)
 }
 
+/**
+ * The refs a remote reads, in order (2026-10-20): the run's own branch,
+ * then the branch it merges into, then the default branch, each once. On
+ * CI from its env (a GitHub pull request's head branch, a merge queue's
+ * queue ref, a GitLab pipeline's ref), else from git: the checked-out
+ * branch, then the default (`origin/HEAD`). A remote plugin keys or scopes
+ * its entries by them (`scope` on turboCache/nxCache); core keys nothing
+ * by them.
+ */
+export function cacheRefs(
+  workspaceRoot: string,
+  baseBranch: string | undefined,
+  env: NodeJS.ProcessEnv = process.env,
+): string[] {
+  const set = (v: string | undefined): string | undefined =>
+    v !== undefined && v.trim() !== '' ? v.trim() : undefined
+  const head = (b: string | undefined) => (b === undefined ? undefined : `refs/heads/${b}`)
+  let own: string | undefined
+  let base: string | undefined
+  if (env['GITHUB_ACTIONS'] === 'true') {
+    own = head(set(env['GITHUB_HEAD_REF'])) ?? set(env['GITHUB_REF'])
+    base = head(set(env['GITHUB_BASE_REF']))
+    const queue = /^refs\/heads\/gh-readonly-queue\/(.+)\/pr-\d+-[0-9a-f]+$/.exec(own ?? '')
+    if (queue !== null) base = head(queue[1])
+  } else if (env['GITLAB_CI'] === 'true') {
+    const tag = set(env['CI_COMMIT_TAG'])
+    own = tag !== undefined ? `refs/tags/${tag}` : head(set(env['CI_COMMIT_REF_NAME']))
+    base = head(set(env['CI_MERGE_REQUEST_TARGET_BRANCH_NAME']))
+  } else {
+    own = head(
+      (readHeadDirect(workspaceRoot) ?? captureGitContext(workspaceRoot, null, env)).branch ??
+        undefined,
+    )
+  }
+  const trunk = head(baseBranch ?? captureDefaultBranch(env, workspaceRoot) ?? undefined)
+  return [...new Set([own, base, trunk].filter((r): r is string => r !== undefined))]
+}
+
 /**
  * The repository's path on its host (`owner/repo`; a GitLab subgroup's
  * `group/sub/repo` whole, as `gitlab()` names it); undefined for a local path.
@@ -591,7 +629,7 @@ function gitConfigSections(workspaceRoot: string): Map<string, Map<string, strin
  * The run's cache scope (owner, 2026-10-17): `VX_CACHE_SCOPE` (empty: none),
  * else a plugin's (`setBranch`, `github()` on Actions; null: the plugin
  * named the run trusted), else, with a cache plugin to share it, the
- * branch's from git. Undefined: the default keys.
+ * branch's from git. Undefined: no scope. It keys plugin entries only.
  */
 export function resolveCacheScope(
   workspaceRoot: string,

package/src/orchestrator/run-keys.ts

@@ -0,0 +1,164 @@
+// The run key memo: every up-front key a warm run derived, read back whole
+// when nothing any of them folds has moved. Deriving them again cost 250 ms
+// of a 1,070 ms warm run at 9,600 tasks even with every fold memo row
+// answering (2026-10-22): the per-task walk itself, not the files.
+//
+// The token is everything `deriveStableKeys` reads: each node's id, edges,
+// resolved config, plugin key parts and files token, the declared env
+// values, the forward args and the code that derives. A run the token cannot
+// vouch for (a keyless or runtime-probed task, a declaration no files token
+// covers) derives as before.
+
+import { readdirSync, statSync } from 'node:fs'
+import path from 'node:path'
+import type { GitFilesCache } from '../cache/index.js'
+import type { TaskNode } from '../graph/index.js'
+import { xxh3hex } from '../util/index.js'
+import type { StableKey } from './stable-keys.js'
+import { hashTaskConfig, isKeyless, taskFilesToken, type HashCache } from './task-hash.js'
+
+/** Moves every remembered run: bump it when what a token stands for changes. */
+const RUN_MEMO_VERSION = 'run-memo-v1'
+
+export interface RunKeysArgs {
+  nodes: Map<string, TaskNode>
+  workspaceRoot: string
+  claimedFiles: readonly string[]
+  forwardArgs?: readonly string[] | undefined
+  nestedDirsByProject: Map<string, string[]>
+  gitFilesCache: GitFilesCache
+  hashCache: HashCache
+}
+
+/** What a run's up-front pass yields, as the memo holds it. */
+export interface RunKeys {
+  stableKeys: StableKey[]
+  uncachedKeys: Map<string, string>
+}
+
+/**
+ * The memo row's name (the selection) and the token its keys stand for, or
+ * undefined when a node's key reads something the token cannot name.
+ */
+export function runKeysToken(args: RunKeysArgs): { id: string; token: string } | undefined {
+  if (args.hashCache.folds === undefined) return undefined
+  const parts: string[] = [
+    RUN_MEMO_VERSION,
+    codeIdentity(),
+    args.workspaceRoot,
+    JSON.stringify(args.forwardArgs ?? []),
+    JSON.stringify([...args.claimedFiles].sort()),
+    JSON.stringify([...args.nestedDirsByProject]),
+    // Which outputs git ignores decides which a reader may see (`readableOutputs`).
+    [...args.gitFilesCache.ignoredOutputs].sort().join('\0'),
+  ]
+  const ids: string[] = []
+  const envNames = new Set<string>()
+  for (const node of args.nodes.values()) {
+    const cfg = node.config
+    if (isKeyless(cfg) || node.excludedUpstream !== undefined) return undefined
+    const inputs = cfg.inputs
+    if ((inputs?.runtime?.length ?? 0) > 0 || (inputs?.workspaceRuntime?.length ?? 0) > 0) {
+      return undefined
+    }
+    let files = ''
+    if (inputs !== undefined) {
+      if ((inputs.workspaceFiles?.length ?? 0) > 0) return undefined
+      // No positive entry resolves no files (`filesPlan`).
+      if (inputs.files === undefined || inputs.files.some((e) => !e.startsWith('!'))) {
+        const token = taskFilesToken({
+          node,
+          workspaceRoot: args.workspaceRoot,
+          nestedProjectDirs: args.nestedDirsByProject.get(node.projectName) ?? [],
+          gitFilesCache: args.gitFilesCache,
+          hashCache: args.hashCache,
+        })
+        if (token === undefined) return undefined
+        files = token
+      }
+      for (const name of inputs.env ?? []) envNames.add(name)
+    }
+    ids.push(node.id)
+    parts.push(
+      node.id,
+      node.projectName,
+      node.projectDir,
+      node.deps.join('\u0001'),
+      (node.orderOnly ?? []).join('\u0001'),
+      node.requested ? '1' : '0',
+      hashTaskConfig(cfg, args.hashCache),
+      JSON.stringify(node.keyParts ?? []),
+      JSON.stringify(node.addsToOutputsOf ?? []),
+      JSON.stringify(node.outputsAddedToBy ?? []),
+      files,
+    )
+  }
+  for (const name of [...envNames].sort()) {
+    const value = process.env[name]
+    parts.push(value === undefined ? name : `${name}=${value}`)
+  }
+  return { id: `\0run\0${xxh3hex(ids.join('\0'))}`, token: xxh3hex(parts.join('\0')) }
+}
+
+/** The keys remembered under `token`, mapped back onto this run's nodes. */
+export function readRunKeys(
+  args: RunKeysArgs,
+  at: { id: string; token: string },
+): RunKeys | undefined {
+  const row = args.hashCache.folds?.get(at.id, at.token)
+  if (row === undefined) return undefined
+  let parsed: { s: [string, string][]; u: [string, string][] }
+  try {
+    parsed = JSON.parse(row)
+  } catch {
+    return undefined
+  }
+  const stableKeys: StableKey[] = []
+  for (const [id, hash] of parsed.s) {
+    const node = args.nodes.get(id)
+    if (node === undefined) return undefined
+    stableKeys.push({ hash, node })
+  }
+  return { stableKeys, uncachedKeys: new Map(parsed.u) }
+}
+
+export function writeRunKeys(
+  args: RunKeysArgs,
+  at: { id: string; token: string },
+  keys: RunKeys,
+): void {
+  args.hashCache.folds?.set(
+    at.id,
+    at.token,
+    JSON.stringify({
+      s: keys.stableKeys.map((k) => [k.node.id, k.hash]),
+      u: [...keys.uncachedKeys],
+    }),
+  )
+}
+
+let identity: string | undefined
+
+/**
+ * The code that derives the keys: a compiled binary is its file; a source
+ * checkout is every module under `src/`, so an edit to the derivation from
+ * source never reads back a key the old code made.
+ */
+function codeIdentity(): string {
+  if (identity !== undefined) return identity
+  // `Bun.main`, not `import.meta.dir`: `--minify` inlines the build
+  // machine's source path, so an installed binary walked a `src/` that is
+  // not there, threw, and lost every warm run's short circuit (2026-10-22).
+  if (Bun.main.startsWith('/$bunfs/')) {
+    const st = statSync(process.execPath)
+    return (identity = `${process.execPath}:${st.size}:${st.mtimeMs}:${st.ino}`)
+  }
+  const src = path.resolve(import.meta.dir, '..')
+  const stamps: string[] = []
+  for (const rel of readdirSync(src, { recursive: true, encoding: 'utf8' }).sort()) {
+    if (!rel.endsWith('.ts')) continue
+    const st = statSync(path.join(src, rel))
+    stamps.push(`${rel}:${st.size}:${st.mtimeMs}`)
+  }
+  return (identity = xxh3hex(stamps.join('\0')))
+}

package/src/orchestrator/run-records.ts

@@ -113,6 +113,11 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
       runId,
       ...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
       ...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
+      ...(o.peakCores !== undefined ? { peakCores: o.peakCores } : {}),
+      ...(o.avgRssBytes !== undefined ? { avgRssBytes: o.avgRssBytes } : {}),
+      ...(o.where !== undefined ? { placement: o.where } : {}),
+      ...(o.whereCores !== undefined ? { placementCores: o.whereCores } : {}),
+      ...(o.whereMemoryBytes !== undefined ? { placementMemoryBytes: o.whereMemoryBytes } : {}),
       ...(o.wallclockStartNs !== undefined ? { wallclockStartNs: o.wallclockStartNs } : {}),
       ...(o.wallclockEndNs !== undefined ? { wallclockEndNs: o.wallclockEndNs } : {}),
       cacheHit: isCacheHit(o.status),
@@ -126,6 +131,7 @@ export function assembleRunRecords(input: RunRecordsInput): RunRecords {
       ...(o.sandboxViolations !== undefined ? { sandboxViolations: o.sandboxViolations } : {}),
       ...(o.notReady !== undefined ? { notReady: o.notReady } : {}),
       ...(remoteError !== undefined ? { remoteError } : {}),
+      ...(o.keylessBy !== undefined ? { keylessBy: o.keylessBy } : {}),
     })
     if (o.status === 'failed') failedCount++
     if (o.status === 'cache-hit') hitLocalCount++