@vzn/vx-migrate 0.0.595 → 0.0.597

package/src/paths.ts

@@ -33,6 +33,9 @@ export function relPosix(from: string, to: string): string {
 export function packageScripts(meta: ProjectMeta): Record<string, unknown> {
   const raw = (meta.packageJson as unknown as { scripts?: unknown }).scripts
   const scripts: Record<string, unknown> = Object.create(null)
-  if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return scripts
+  if (!isRecord(raw)) return scripts
   return Object.assign(scripts, raw)
 }
+
+export const isRecord = (v: unknown): v is Record<string, unknown> =>
+  typeof v === 'object' && v !== null && !Array.isArray(v)

package/src/remote-breaker.ts

@@ -0,0 +1,60 @@
+/**
+ * Turbo's outage breaker (`turborepo-cache/src/outage_breaker.rs`): three
+ * outages in a row — no connection, no answer in time, or a 5xx once the
+ * retries are spent — open it for 30 s, and while open no request is sent.
+ * After the cooldown one request probes; its success closes the breaker,
+ * its outage opens it again. Without it a server that hung cost every
+ * task its full deadline. A result that lands after the breaker moved on
+ * (a request already in flight when it opened) changes nothing. Shared by
+ * `turboCache()` and `nxCache()`.
+ */
+export class OutageBreaker {
+  private failures = 0
+  private openUntil: number | undefined
+  private probing = false
+  private generation = 0
+  constructor(private readonly now: () => number = Date.now) {}
+
+  async send(request: () => Promise<Response>): Promise<Response> {
+    let probe = false
+    if (this.openUntil !== undefined) {
+      if (this.now() < this.openUntil || this.probing)
+        throw new Error(
+          `unavailable after ${FAILURE_THRESHOLD} failures in a row; not asked again for ${COOLDOWN_MS / 1000} s`,
+        )
+      this.probing = probe = true
+    }
+    const generation = this.generation
+    let res: Response
+    try {
+      res = await request()
+    } catch (err) {
+      this.finish(generation, probe, true)
+      throw err
+    }
+    this.finish(generation, probe, res.status >= 500)
+    return res
+  }
+
+  private finish(generation: number, probe: boolean, outage: boolean): void {
+    if (generation !== this.generation) return
+    if (outage) {
+      this.failures++
+      if (probe || this.failures >= FAILURE_THRESHOLD) {
+        this.openUntil = this.now() + COOLDOWN_MS
+        this.probing = false
+        this.generation++
+      }
+      return
+    }
+    this.failures = 0
+    if (probe) {
+      this.openUntil = undefined
+      this.probing = false
+      this.generation++
+    }
+  }
+}
+
+const FAILURE_THRESHOLD = 3
+const COOLDOWN_MS = 30_000

package/src/remote-deadline.ts

@@ -2,10 +2,18 @@
  * A request's deadline, named. The runtime's abort says "The operation
  * timed out." and nothing else; core's degrade line already names the
  * request, the artifact and the server, so the cause names the deadline
- * that was spent. Shared by `turboCache()` and `nxCache()`.
+ * that was spent. The name rides the signal's reason, not a catch around
+ * `fetch`: the deadline covers the body too, and a body read past it
+ * (core's ingest, the signed download's temp) rejects with the reason
+ * (measured on Bun 1.4.2), where a catch around `fetch` never saw it.
+ * Unref'd, as `AbortSignal.timeout`'s timer is. Shared by `turboCache()`
+ * and `nxCache()`.
  */
-export function deadlineNamed(err: unknown, timeoutMs: number): unknown {
-  return (err as { name?: unknown } | null)?.name === 'TimeoutError'
-    ? new Error(`no answer within ${timeoutMs} ms`)
-    : err
+export function deadline(timeoutMs: number): AbortSignal {
+  const controller = new AbortController()
+  setTimeout(
+    () => controller.abort(new Error(`no answer within ${timeoutMs} ms`)),
+    timeoutMs,
+  ).unref()
+  return controller.signal
 }

package/src/shared-outputs.ts

@@ -413,6 +413,8 @@ export function excludeWorkspaceOutputs(
     readonly dir: string
     readonly tasks: GeneratedTask[]
   }[],
+  /** The globs an opaque entry stands for (`vx-migrate`'s `...globalInputs` spread). */
+  expand: (entry: unknown) => readonly string[] = () => [],
 ): void {
   const positive = (v: unknown): string[] =>
     Array.isArray(v)
@@ -444,7 +446,7 @@ export function excludeWorkspaceOutputs(
         ?.inputs
       if (inputs === undefined || !Array.isArray(inputs.workspaceFiles)) continue
       const files = inputs.workspaceFiles as string[]
-      const reads = positive(files)
+      const reads = positive(files.flatMap((f) => (typeof f === 'string' ? [f] : expand(f))))
       if (reads.length === 0) continue
       for (const i of candidates(reads)) {
         const w = writers[i]!

package/src/spelled-env.ts

@@ -0,0 +1,36 @@
+// The env names a package's tracked source spells: what stands in, in a
+// written config, for a `*` env entry Turbo or Vite Task matched against
+// the environment a run sees.
+
+import path from 'node:path'
+import { relPosix } from './paths.js'
+
+/** Source and env-example files a framework build reads its variables from. */
+const SPELLS_ENV =
+  /\.(c|m)?(j|t)sx?$|\.(vue|svelte|astro|html)$|(^|\/)\.env\.(example|sample|template)$/
+
+/**
+ * The upper-case names the tracked source under `dirs` spells (`NEXT_PUBLIC_API`
+ * in `process.env.NEXT_PUBLIC_API` or an `.env.example`), sorted. Without
+ * git, none: the note still names the framework's prefix.
+ */
+export async function spelledNames(
+  root: string,
+  dirs: readonly string[],
+  tracked: readonly string[] | null,
+): Promise<string[]> {
+  if (tracked === null) return []
+  const prefixes = dirs.map((dir) => {
+    const rel = relPosix(root, dir)
+    return rel === '' || rel === '.' ? '' : `${rel}/`
+  })
+  const names = new Set<string>()
+  for (const f of tracked) {
+    if (!SPELLS_ENV.test(f) || !prefixes.some((p) => f.startsWith(p))) continue
+    const text = await Bun.file(path.join(root, f))
+      .text()
+      .catch(() => '')
+    for (const m of text.matchAll(/\b[A-Z][A-Z0-9_]*_[A-Z0-9_]+\b/g)) names.add(m[0])
+  }
+  return [...names].sort()
+}

package/src/tracked-outputs.ts

@@ -10,6 +10,7 @@
 import { lstat, readFile } from 'node:fs/promises'
 import path from 'node:path'
 import { isLiteralPattern, outputsOverlap } from '@vzn/vx'
+import { relPosix } from './paths.js'
 
 /** The files git tracks under `root`, root-relative; null outside a repo or without git. */
 export async function trackedFiles(root: string): Promise<string[] | null> {
@@ -220,7 +221,7 @@ export function spareTrackedOutputs(
   /** Drop the `!` entries readers carry for a committed file no output names any more. */
   const unhide = (owner: (typeof projects)[number] | null, rooted: string) => {
     for (const q of projects) {
-      const qrel = path.relative(root, q.dir).split(path.sep).join('/')
+      const qrel = relPosix(root, q.dir)
       for (const r of q.tasks) {
         const inputs = (r.task?.['cache'] as { inputs?: Outputs } | undefined)?.inputs
         if (inputs === undefined) continue
@@ -238,7 +239,7 @@ export function spareTrackedOutputs(
     }
   }
   for (const p of projects) {
-    const rel = path.relative(root, p.dir).split(path.sep).join('/')
+    const rel = relPosix(root, p.dir)
     const own = rel === '' ? tracked : filesUnder(tracked, rel).map((f) => f.slice(rel.length + 1))
     for (const t of p.tasks) {
       const outputs = (t.task?.['cache'] as { outputs?: Outputs } | undefined)?.outputs

package/src/turbo/turbo-map.ts

@@ -28,7 +28,7 @@ import {
   wildcardOutput,
   wildcardTodo,
 } from '../shared-outputs.js'
-import { packageScripts, relPosix } from '../paths.js'
+import { isRecord, packageScripts, relPosix } from '../paths.js'
 import { MAX_SPARED, type TrackedKinds } from '../tracked-outputs.js'
 import {
   DOTENV_PROBE,
@@ -65,6 +65,8 @@ interface TurboJson {
   envMode?: unknown
 }
 
+const SHELL_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/
+
 /** A name core takes in `cache.inputs.env` and `exec.env.passThrough`. */
 const keyable = (name: string): boolean =>
   name.length > 0 && !name.startsWith('!') && !/[*?[\]{}=\0]/.test(name)
@@ -437,9 +439,6 @@ async function readTurboJson(file: string, root: string): Promise<TurboJson> {
   return parsed as TurboJson
 }
 
-const isRecord = (v: unknown): v is Record<string, unknown> =>
-  typeof v === 'object' && v !== null && !Array.isArray(v)
-
 const TOP_LISTS = [
   'globalDependencies',
   'globalEnv',
@@ -1164,14 +1163,19 @@ export async function mapTurboWorkspace(
   // One with edges of its own (with-shell-commands' `tooling-config#build`
   // → `prebuild`) was a group, which keys nothing: Turbo's node still
   // hashes the package's files, so it is key-only with its edges. A node
-  // that starts persistent sidecars stays a group.
+  // that starts persistent sidecars stays a group. One a `^name` reaches
+  // whose own edges lack `^name` (opencode's `build: { dependsOn: [] }`)
+  // is key-only too: Turbo's `^build` stops at it, and core's walked past
+  // it to the builds below, running and keying ones Turbo never waits on.
   const keyOnly = new Map<string, Set<string>>()
   for (const meta of metas) {
     const scripts = packageScripts(meta)
     const { defined, defFor } = definitions(meta)
+    const depended = metas.some((m) => m !== meta && declares(m, meta.name))
     for (const name of defined) {
       const def = defFor(name)
-      if (!caretSelf.has(name) || !withScript.has(name) || transit.has(name)) continue
+      const stops = depended && caretNames.has(name) && !(def?.dependsOn ?? []).includes(`^${name}`)
+      if (!(caretSelf.has(name) || stops) || !withScript.has(name) || transit.has(name)) continue
       if (scripts[name] !== undefined || commandOverride(def) !== undefined) continue
       if (sidecarGroups.has(`${meta.name}#${name}`)) continue
       if (def?.cache === false || def?.persistent === true) continue
@@ -1291,7 +1295,13 @@ export async function mapTurboWorkspace(
       // Core gives a project with no `build` this very node (a group behind
       // `^build`, keyed on the project's files), so writing it is noise:
       // solid's three script-less packages each got a `build` running `true`.
-      if (noop && name === 'build' && (defFor(name)!.dependsOn ?? []).every((d) => d === '^build'))
+      const noopEdges = defFor(name)!.dependsOn ?? []
+      if (
+        noop &&
+        name === 'build' &&
+        noopEdges.includes('^build') &&
+        noopEdges.every((d) => d === '^build')
+      )
         continue
       if (
         override === undefined &&
@@ -1488,7 +1498,14 @@ export async function mapTurboWorkspace(
     )
   literalEnvGapsOnce(projects, notes)
   resolveSharedWorkspaceOutputs(root, projects)
-  excludeWorkspaceOutputs(root, projects)
+  // A written config spreads the preset (`...globalInputs`), which the
+  // take-back pass must read as its globs: unread, opencode's configs
+  // read `packages/plugin/**` with no `!packages/plugin/dist/**` and
+  // core refused to load them.
+  const opaque = new Map<string, readonly string[]>()
+  for (const e of opts.splice('inputs', globals.inputs))
+    if (typeof e !== 'string') opaque.set(JSON.stringify(e), globals.inputs)
+  excludeWorkspaceOutputs(root, projects, (e) => opaque.get(JSON.stringify(e)) ?? [])
   pruneUnreachedPersistentNotes(projects, metas, opts.persistentTodo)
   pruneOrphanPersistentNotes(projects, opts.persistentTodo)
   return { projects, notes, globals }
@@ -1882,10 +1899,12 @@ function buildTask(
     spelled,
   )
 
+  // A name that is no shell identifier never reached a task (`sh` drops it)
+  // and core refuses it in passThrough.
   const passThrough = uniq(
     [...global('env'), ...global('pass'), ...envNames, ...passNames],
     hidden('env', 'pass'),
-  )
+  ).filter((n) => typeof n !== 'string' || SHELL_NAME.test(n))
 
   const exec: Record<string, unknown> = { command }
   if (passThrough.length > 0) exec.env = { passThrough }

package/src/turbo-cache/index.ts

@@ -24,7 +24,8 @@ import {
   refuseUnknownOptions,
   type PluginOptionKinds,
 } from '@vzn/vx'
-import { deadlineNamed } from '../remote-deadline.js'
+import { deadline } from '../remote-deadline.js'
+import { OutageBreaker } from '../remote-breaker.js'
 import { withRetry } from '../remote-retry.js'
 import { headerValueFault } from '../remote-token.js'
 
@@ -55,6 +56,12 @@ export interface TurboCacheOptions {
   uploadTimeoutMs?: number
   /** Resends of a request answered 429 / 5xx or never connected (default 1, Turbo's); 0 turns them off. */
   retries?: number
+  /**
+   * Send Turbo's preflight (`OPTIONS`) before each artifact request and
+   * follow its `Location`, or `TURBO_PREFLIGHT` / turbo.json's
+   * `remoteCache.preflight`; default off.
+   */
+  preflight?: boolean
 }
 
 /** turbo.json's `remoteCache` block: the source below Turbo's environment, as in Turbo. */
@@ -66,6 +73,7 @@ export interface TurboJsonRemoteCache {
   enabled?: unknown
   timeout?: unknown
   uploadTimeout?: unknown
+  preflight?: unknown
 }
 
 /** `.turbo/config.json`'s fields, under any of the spellings Turbo accepts. */
@@ -85,6 +93,7 @@ export interface TurboCacheConfig {
   timeoutMs: number
   uploadTimeoutMs: number
   retries: number
+  preflight: boolean
 }
 
 /** Turbo's signature message prefix (`crates/turborepo-cache/src/signature_authentication.rs`). */
@@ -250,18 +259,14 @@ export function resolveTurboCacheConfig(
     fromFile(file.teamId)
   const teamSlug =
     options.teamSlug ?? env['TURBO_TEAM'] ?? fromFile(local.teamSlug) ?? fromFile(file.teamSlug)
+  const preflight =
+    options.preflight ??
+    truthy(env['TURBO_PREFLIGHT'], 'TURBO_PREFLIGHT') ??
+    file.preflight === true
   // Turbo signs only under `remoteCache.signature: true`; the env key alone
   // signs nothing there, and a short one here refused the whole cache.
   // `TURBO_SIGNATURE` (1/true, 0/false) sits above turbo.json, as in Turbo.
-  const signEnv = env['TURBO_SIGNATURE']
-  let signing = file.signature === true
-  if (signEnv !== undefined && signEnv !== '') {
-    if (!/^(1|0|true|false)$/.test(signEnv))
-      throw new Error(
-        `vx/turbo-cache: TURBO_SIGNATURE should be 1 or 0, got ${JSON.stringify(signEnv)}`,
-      )
-    signing = signEnv === '1' || signEnv === 'true'
-  }
+  const signing = truthy(env['TURBO_SIGNATURE'], 'TURBO_SIGNATURE') ?? file.signature === true
   const signatureKey =
     options.signatureKey ?? (signing ? env['TURBO_REMOTE_CACHE_SIGNATURE_KEY'] : undefined)
   if (signatureKey !== undefined) {
@@ -303,9 +308,18 @@ export function resolveTurboCacheConfig(
       seconds(file.uploadTimeout, 'remoteCache.uploadTimeout') ??
       60_000,
     retries,
+    preflight,
   }
 }
 
+/** Turbo's `truth_env_var`: 1/true, 0/false; unset or empty is `undefined`. */
+function truthy(v: string | undefined, name: string): boolean | undefined {
+  if (v === undefined || v === '') return undefined
+  if (!/^(1|0|true|false)$/.test(v))
+    throw new Error(`vx/turbo-cache: ${name} should be 1 or 0, got ${JSON.stringify(v)}`)
+  return v === '1' || v === 'true'
+}
+
 /**
  * Turbo's timeout, whole seconds, as ms; `undefined` when unset. A value
  * Turbo would refuse is refused here rather than read as some other wait.
@@ -365,6 +379,7 @@ const MAX_SIGNED_BODY = 2 * 1024 ** 3 + ((2 * 1024 ** 3) >> 8) + 64 * 1024
  */
 export class TurboRemoteCache implements RemoteCacheLayer {
   private disabled = false
+  private readonly breaker = new OutageBreaker()
   /**
    * A `403` on an upload is a read-only token or server
    * (turborepo-remote-cache's `READ_ONLY` and its JWTs without the write
@@ -396,14 +411,50 @@ export class TurboRemoteCache implements RemoteCacheLayer {
     return u.toString()
   }
 
-  private headers(extra: Record<string, string> = {}): Record<string, string> {
+  private headers(extra: Record<string, string> = {}, auth = true): Record<string, string> {
     return {
-      Authorization: `Bearer ${this.config.token}`,
+      ...(auth ? { Authorization: `Bearer ${this.config.token}` } : {}),
       'x-artifact-client-interactive': process.stdout.isTTY ? '1' : '0',
       ...extra,
     }
   }
 
+  /**
+   * Turbo's preflight: an `OPTIONS` to the request's URL names the method
+   * and headers to come; the answer's `Location` (relative to `apiUrl`) is
+   * where the request goes, as given — a signed storage URL signs its own
+   * query, so nothing is appended — and the token goes along only when
+   * `Access-Control-Allow-Headers` admits `Authorization` (or `*`), so it
+   * never reaches a third-party store. The answer's status is not read, as
+   * in Turbo.
+   */
+  private async preflight(
+    url: string,
+    method: string,
+    headers: Record<string, string>,
+    signal: () => AbortSignal | undefined,
+  ): Promise<{ url: string; auth: boolean }> {
+    const s = signal()
+    const res = await this.fetchImpl(url, {
+      method: 'OPTIONS',
+      headers: {
+        Authorization: `Bearer ${this.config.token}`,
+        'Access-Control-Request-Method': method === 'HEAD' ? 'GET' : method,
+        'Access-Control-Request-Headers': ['Authorization', ...Object.keys(headers)].join(', '),
+      },
+      ...(s === undefined ? {} : { signal: s }),
+    })
+    await res.body?.cancel()
+    const location = res.headers.get('location')
+    const allowed = res.headers.get('access-control-allow-headers') ?? ''
+    return {
+      url: location === null ? res.url || url : new URL(location, this.config.apiUrl).toString(),
+      auth:
+        allowed.trim() === '*' ||
+        allowed.split(',').some((h) => h.trim().toLowerCase() === 'authorization'),
+    }
+  }
+
   /**
    * `undefined` = the token was refused and the refusal is ALREADY
    * reported, so the caller degrades to its miss value in silence. Only the
@@ -416,20 +467,29 @@ export class TurboRemoteCache implements RemoteCacheLayer {
     init: { body?: Blob | string; headers?: Record<string, string>; timeoutMs?: number } = {},
   ): Promise<Response | undefined> {
     const timeoutMs = init.timeoutMs ?? this.config.timeoutMs
-    const res = await withRetry(
-      () =>
-        this.fetchImpl(this.url(pathname), {
-          method,
-          headers: this.headers(init.headers),
-          ...(init.body === undefined ? {} : { body: init.body }),
-          // 0 is Turbo's "no deadline".
-          ...(timeoutMs === 0 ? {} : { signal: AbortSignal.timeout(timeoutMs) }),
-        }),
-      this.config.retries,
-      this.wait,
-    ).catch((err: unknown) => {
-      throw deadlineNamed(err, timeoutMs)
-    })
+    // 0 is Turbo's "no deadline".
+    const signal = () => (timeoutMs === 0 ? undefined : deadline(timeoutMs))
+    // The batch query is never preflighted, as in Turbo.
+    const preflight = this.config.preflight && method !== 'POST'
+    const res = await this.breaker.send(() =>
+      withRetry(
+        async () => {
+          const extra = init.headers ?? {}
+          const to = preflight
+            ? await this.preflight(this.url(pathname), method, extra, signal)
+            : { url: this.url(pathname), auth: true }
+          const s = signal()
+          return this.fetchImpl(to.url, {
+            method,
+            headers: this.headers(extra, to.auth),
+            ...(init.body === undefined ? {} : { body: init.body }),
+            ...(s === undefined ? {} : { signal: s }),
+          })
+        },
+        this.config.retries,
+        this.wait,
+      ),
+    )
     if (method === 'PUT' && res.status === 403) {
       const first = !this.writesRefused
       this.writesRefused = true
@@ -551,7 +611,9 @@ export class TurboRemoteCache implements RemoteCacheLayer {
       timeoutMs: this.config.uploadTimeoutMs,
     })
     if (res === undefined) return
-    if (res.status !== 200 && res.status !== 202) throw new Error(`HTTP ${res.status}`)
+    // Turbo's client takes any 2xx (`error_for_status`); a server's 201 or
+    // 204 stored the artifact and was counted a failed upload.
+    if (!res.ok) throw new Error(`HTTP ${res.status}`)
   }
 }
 
@@ -636,6 +698,7 @@ const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
   timeoutMs: 'number',
   uploadTimeoutMs: 'number',
   retries: 'number',
+  preflight: 'boolean',
 }
 
 /**