@vzn/vx-migrate 0.0.0 → 0.0.484

package/src/dotenv-probe.ts

@@ -0,0 +1,86 @@
+import { shellQuote } from './nx-command.js'
+
+/**
+ * Every `.env`-shaped file under the probe's directory, name and bytes, in
+ * a stable order. Git ignores them, so a glob over git's files keys none of
+ * them; this is a superset of what a `.env` glob names, so an edit to one
+ * misses and nothing else is lost (item 1032). node_modules and .git are
+ * pruned. Each file ends on a `.`: core trims a probe's output, so a
+ * newline added to the last file keyed nothing.
+ */
+export const DOTENV_PROBE =
+  'find . \\( -name node_modules -o -name .git \\) -prune -o -type f \\( -name \'.env*\' -o -name \'*.env\' \\) -print | LC_ALL=C sort | while IFS= read -r f; do echo "$f"; cat -- "$f"; echo .; done'
+
+/**
+ * The same for a package whose `.env` globs all sit at its root (`.env*`,
+ * create-turbo's default): the files named `.env…` or `…env` there, by the
+ * shell's own globbing. `DOTENV_PROBE` is three processes and walks the
+ * package; this is one shell, 1.2 ms against 3.3 (kitchen-sink's warm run
+ * spent ~45 ms of ~145 in probes). `LC_ALL=C` keeps the glob order, and so
+ * the key, the same on every machine.
+ */
+export const DOTENV_PROBE_TOP =
+  'LC_ALL=C; export LC_ALL; for f in .env* .*.env *.env; do [ -f "$f" ] && { printf \'%s\\n\' "$f"; cat -- "$f"; echo .; }; done; :'
+
+/**
+ * The named root-relative files, name and bytes, run at the workspace root:
+ * the inputs a turbo.json names by path that git ignores
+ * (`config.local.json`), which Turbo hashes and a glob over git's files
+ * keys nothing of. Core trims a probe's output, so each file ends on a `.`:
+ * a trailing newline added to the last one is still an edit.
+ */
+export function ignoredFilesProbe(rels: readonly string[]): string {
+  return `for f in ${rels.map(shellQuote).join(' ')}; do [ -f "$f" ] && { printf '%s\\n' "$f"; cat -- "$f"; echo .; }; done; :`
+}
+
+/** How every `dotenvGlobsProbe` line opens, and no other probe's. */
+export const DOTENV_GLOBS_HEAD = 'LC_ALL=C; export LC_ALL; { '
+
+/**
+ * The files the root-relative `.env` globs name, name and bytes, run at the
+ * workspace root: Turbo hashes exactly these, and `DOTENV_PROBE` over the
+ * whole tree also keyed `hosting/docker/.env.example`, which trigger.dev's
+ * `apps/*\/.env`-shaped globs never reach. Turbo's `*` matches a leading
+ * dot and the shell's does not, so a segment that opens on `*` is spelled
+ * again for hidden names. `<dir>/**\/<name>` (create-turbo's
+ * `**\/.env.*local`) is a `find -name`, which matches a leading dot and
+ * skips `node_modules` as Turbo does. Null for a glob the shell would read
+ * otherwise (`**` elsewhere, a class, a brace, a quote): the caller keeps
+ * the walk.
+ */
+export function dotenvGlobsProbe(globs: readonly string[]): string | null {
+  const safe = (seg: string): boolean =>
+    /^[A-Za-z0-9._@+*-]+$/.test(seg) && !seg.includes('**') && seg !== '.' && seg !== '..'
+  const words = new Set<string>()
+  const finds: string[] = []
+  for (const glob of globs) {
+    const segs = glob.split('/')
+    const deep = segs.indexOf('**')
+    if (deep !== -1) {
+      const dir = segs.slice(0, deep)
+      const name = segs.slice(deep + 1)
+      if (name.length !== 1 || !safe(name[0]!) || dir.some((d) => !safe(d) || d.includes('*')))
+        return null
+      finds.push(
+        `find ${dir.length === 0 ? '.' : dir.join('/')} \\( -name node_modules -o -name .git \\) -prune -o -type f -name '${name[0]}' -print`,
+      )
+      continue
+    }
+    let alts = ['']
+    for (const seg of segs) {
+      if (!safe(seg)) return null
+      const forms =
+        seg === '*' ? ['*', '.[!.]*', '..?*'] : seg.startsWith('*') ? [seg, `.${seg}`] : [seg]
+      alts = alts.flatMap((a) => forms.map((f) => (a === '' ? f : `${a}/${f}`)))
+      if (alts.length > 64) return null
+    }
+    for (const a of alts) words.add(a)
+  }
+  const list = [
+    ...(words.size === 0
+      ? []
+      : [`for f in ${[...words].join(' ')}; do [ -f "$f" ] && printf '%s\\n' "$f"; done`]),
+    ...finds.map((f) => `${f} 2>/dev/null | sed 's|^\\./||'`),
+  ]
+  return `${DOTENV_GLOBS_HEAD}${list.join('; ')}; } | sort -u | while IFS= read -r f; do printf '%s\\n' "$f"; cat -- "$f"; echo .; done`
+}

package/src/glob-grammar.ts

@@ -0,0 +1,63 @@
+// The glob grammar Nx and Turbo share and vx's own differ in brackets and
+// extglobs; both adapters translate through here (items 914, 1031).
+
+const EXTGLOB = /([?@+*!])\(([^()]*)\)/g
+
+/**
+ * A minimatch-family glob (Nx's minimatch, Turbo's wax) in vx's grammar,
+ * or null when it has no safe form. vx has no character classes (a bracket is a literal, for route
+ * dirs, item 667) and no extglobs, so `src/**\/*.[jt]s` matched nothing
+ * and Nx's own default `production` negation,
+ * `?(*.)+(spec|test).[jt]s?(x)`, excluded nothing (item 914):
+ * - `[jt]` is the brace set `{j,t}`, plus the literal `[jt]` itself in a
+ *   positive glob (a superset of inputs is safe; the route dir may be
+ *   meant); a range within digits or one case of letters (`[a-c]`,
+ *   `[0-9]`) is its members;
+ * - a negated class (`[!a]`) is `?` in a positive glob, a superset;
+ * - `?(a|b)` is `{a,b,}` and `@(a|b)` is `{a,b}`;
+ * - `+(a|b)` and `*(a|b)` narrow to one repetition, which only a
+ *   NEGATION may do — it then excludes fewer files, never more;
+ * - any other range, a negated class in a negation, `!(…)`, or nesting is
+ *   null.
+ */
+export function minimatchToVx(glob: string, negated: boolean): string | null {
+  if (!/[[(]/.test(glob)) return glob
+  let out = glob.replace(EXTGLOB, (whole, kind: string, body: string) => {
+    const alts = body.split('|')
+    if (alts.some((a) => /[{},]/.test(a))) return '\0'
+    if (kind === '?') return `{${alts.join(',')},}`
+    if (kind === '@') return `{${alts.join(',')}}`
+    if (!negated || kind === '!') return '\0'
+    return kind === '+' ? `{${alts.join(',')}}` : `{${alts.join(',')},}`
+  })
+  if (out.includes('\0') || /[?@+*!]\(/.test(out)) return null
+  out = out.replace(/\[([^\]]*)\]/g, (whole, body: string) => {
+    if (body === '' || /[{},/]/.test(body)) return '\0'
+    if (/^[!^]/.test(body)) return negated || body.length === 1 ? '\0' : '?'
+    const chars = classMembers(body)
+    if (chars === null) return '\0'
+    return negated ? `{${chars.join(',')}}` : `{${whole},${chars.join(',')}}`
+  })
+  return out.includes('\0') ? null : out
+}
+
+const RANGES = ['0123456789', 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ']
+
+/** A class body's characters, ranges spelled out; null for a range across kinds or backwards. */
+function classMembers(body: string): string[] | null {
+  const out = new Set<string>()
+  for (let i = 0; i < body.length; i++) {
+    const c = body[i]!
+    // A `-` first or last is itself, as in minimatch.
+    if (body[i + 1] === '-' && i + 2 < body.length) {
+      const end = body[i + 2]!
+      const kind = RANGES.find((r) => r.includes(c))
+      const from = kind?.indexOf(c) ?? -1
+      const to = kind?.indexOf(end) ?? -1
+      if (kind === undefined || to < from) return null
+      for (const m of kind.slice(from, to + 1)) out.add(m)
+      i += 2
+    } else out.add(c)
+  }
+  return [...out]
+}

package/src/index.ts

@@ -0,0 +1,15 @@
+// @vzn/vx-migrate — adoption: the Turbo and Nx project stages (a repo
+// runs unchanged) and the two remote caches speaking Turbo's and Nx's
+// wire. The CLI (`bunx @vzn/vx-migrate`) is `migrate.ts`; mappers and
+// cache clients are internal (1.0 freezes what this file exports).
+export { turbo, type TurboPluginOptions } from './turbo/index.js'
+export {
+  nx,
+  type NxExecutors,
+  type NxExecutorTarget,
+  type NxExecutorTranslation,
+  type NxExecutorTranslator,
+  type NxPluginOptions,
+} from './nx/index.js'
+export { turboCache, type TurboCacheOptions } from './turbo-cache/index.js'
+export { nxCache, type NxCacheOptions } from './nx-cache/index.js'

package/src/mapping-cache.ts

@@ -0,0 +1,101 @@
+// A plugin's mapping, kept under the cache dir and keyed on everything it
+// reads. Both adoption plugins mapped every package of the workspace on
+// every run, when a run asks for a few: nx() ~55 ms cold on refine (206
+// projects), turbo() ~43 ms on astro (553). A hit costs the key's reads
+// and one JSON parse.
+//
+// The key is the plugin's own list of what its mapping read, plus this
+// package's sources and the core helpers the mappers call: a new
+// `@vzn/vx-migrate` or `@vzn/vx` maps afresh.
+
+import { mkdir, rename } from 'node:fs/promises'
+import path from 'node:path'
+import {
+  buildPackageGraph,
+  foldScriptHooks,
+  isLiteralPattern,
+  outputsOverlap,
+  VERSION,
+} from '@vzn/vx'
+import type { Gaps } from './plugin-gaps.js'
+
+/** What the stage uses of a mapping: tasks per package name, and the gaps to report once. */
+export interface AdoptionMapping {
+  readonly byName: ReadonlyMap<
+    string,
+    {
+      readonly tasks: readonly {
+        readonly name: string
+        readonly task: Record<string, unknown> | null
+      }[]
+    }
+  >
+  readonly gaps: Gaps
+}
+
+let code: Promise<string> | undefined
+
+/** This package's sources and the core helpers the mappers call, hashed once per process. */
+function codeId(): Promise<string> {
+  code ??= (async () => {
+    const parts: string[] = [
+      VERSION,
+      ...[buildPackageGraph, foldScriptHooks, isLiteralPattern, outputsOverlap].map(String),
+    ]
+    const files: string[] = []
+    for await (const f of new Bun.Glob('**/*.{ts,cjs}').scan(import.meta.dir)) files.push(f)
+    files.sort()
+    for (const f of files) parts.push(f, await Bun.file(path.join(import.meta.dir, f)).text())
+    return hex(parts.join('\0'))
+  })()
+  return code
+}
+
+function hex(s: string): string {
+  return Bun.hash.xxHash3(s).toString(16).padStart(16, '0')
+}
+
+interface Held {
+  readonly key: string
+  readonly byName: [string, { tasks: { name: string; task: Record<string, unknown> | null }[] }][]
+  readonly notes: string[]
+  readonly todos: [string, string[]][]
+}
+
+/** `map()`, or the mapping it returned last under the same `parts`. */
+export async function cachedMapping(
+  cacheDir: string,
+  name: string,
+  parts: readonly string[],
+  map: () => Promise<AdoptionMapping>,
+): Promise<AdoptionMapping> {
+  const key = hex([await codeId(), ...parts].join('\0'))
+  const file = path.join(cacheDir, `vx-migrate-${name}-mapping.json`)
+  const held = (await Bun.file(file)
+    .json()
+    .catch(() => null)) as Held | null
+  if (held?.key === key) {
+    return {
+      byName: new Map(held.byName),
+      gaps: { notes: held.notes, todos: new Map(held.todos) },
+    }
+  }
+  const mapping = await map()
+  const out: Held = {
+    key,
+    byName: [...mapping.byName].map(([name, p]) => [
+      name,
+      { tasks: p.tasks.map((t) => ({ name: t.name, task: t.task })) },
+    ]),
+    notes: [...mapping.gaps.notes],
+    todos: [...mapping.gaps.todos].map(([todo, ids]) => [todo, [...ids]]),
+  }
+  // Best-effort, and whole or absent: a reader never sees half a file, and
+  // a cache dir it cannot write costs the next run a mapping, not the run.
+  const tmp = `${file}.${process.pid}.tmp`
+  await mkdir(cacheDir, { recursive: true })
+    .then(() => Bun.write(tmp, JSON.stringify(out)))
+    .then(() => rename(tmp, file))
+    .catch(() => {})
+  return mapping
+}

package/src/migrate-nx.ts

@@ -0,0 +1,154 @@
+// `bunx @vzn/vx-migrate --from nx`: the resolved graph snapshot on disk,
+// through the mapper the `nx()` plugin runs live, as a migration plan.
+
+import { stat } from 'node:fs/promises'
+import path from 'node:path'
+import { type MigrationFormat, type MigrationPlan, type ProjectMeta } from '@vzn/vx'
+import { mapNxWorkspace, nxSizeText, parseNxGraph, readNxJson } from './nx/nx-map.js'
+import { spareTrackedOutputs, trackedFiles, trackedKinds } from './tracked-outputs.js'
+import { adoptedToolNotes } from './workspace-notes.js'
+
+/** What a task's `npm_package_*` read: the manifest, so a bump reaches them. */
+const MANIFEST_IMPORT = "import pkg from './package.json' with { type: 'json' }"
+
+export const NX_GRAPH_REL = '.nx/workspace-data/project-graph.json'
+
+/** `snapshot`: the graph file to read, absolute; the checked-in one by default. */
+export async function migrateNx(
+  root: string,
+  metas: readonly ProjectMeta[],
+  format: MigrationFormat = 'ts',
+  snapshot = path.join(root, NX_GRAPH_REL),
+): Promise<MigrationPlan> {
+  const graph = parseNxGraph(await Bun.file(snapshot).text(), path.relative(root, snapshot))
+  const tracked = await trackedFiles(root)
+  const mapped = await mapNxWorkspace(root, metas, graph, {
+    cacheable: new Set(),
+    ...(tracked === null ? {} : { tracked: trackedKinds(tracked) }),
+    // The file this writes is each task's config.
+    ownConfig: () => `vx.config.${format}`,
+    manifestField: (key) => ({ raw: `pkg.${key}` }),
+    migration: true,
+  })
+  // nx() takes each committed file under an output back every run; a
+  // written config must carry the same `!` entries, or its first run's
+  // clean deletes them.
+  if (tracked !== null)
+    for (const [id, todo] of spareTrackedOutputs(root, mapped.projects, tracked)) {
+      const at = id.lastIndexOf('#')
+      const p = mapped.projects.find((x) => x.name === id.slice(0, at))
+      p?.tasks.find((t) => t.name === id.slice(at + 1))?.todos.push(todo)
+    }
+  const adopted = await adoptedToolNotes(root, {
+    plugin: 'nx',
+    config: 'nx.json',
+    runner: 'nx',
+    keys: (lock) => `Nx keys each project on the npm packages it depends on in ${lock}`,
+  })
+  const unlisted = await unlistedProjects(root, metas, mapped.projects)
+  return {
+    headerNotes: [
+      'migrating from the resolved project-graph snapshot — plugin-inferred targets ' +
+        'are frozen as static config; `nx:run-commands` targets are their shell lines, and ' +
+        'every other executor runs as itself through `nx-exec` (keep Nx and @vzn/vx-migrate ' +
+        'installed until those targets are rewritten as commands); targets with `.env` files ' +
+        'run through `nx-env`',
+      ...adopted,
+    ],
+    projects: mapped.projects.map((p) =>
+      p.tasks.some((t) => JSON.stringify(t.task ?? {}).includes('"pkg.'))
+        ? { ...p, importLines: [MANIFEST_IMPORT, ...(p.importLines ?? [])] }
+        : p,
+    ),
+    extraFiles: unlisted.manifests,
+    notes: [
+      ...mapped.notes,
+      ...unlisted.notes,
+      ...workspaceNotes((await readNxJson(root).catch(() => null))?.json),
+    ],
+  }
+}
+
+/**
+ * Nx projects no workspace glob lists (an integrated repo's `project.json`
+ * libraries): core finds a project only by a listed package.json, so a
+ * config written there would never run. Each gets a package.json when it
+ * has none, and one note names the globs to add — the root manifest is
+ * the user's to edit, and `applyMigration` never overwrites a file.
+ */
+async function unlistedProjects(
+  root: string,
+  metas: readonly ProjectMeta[],
+  projects: readonly { name: string; dir: string }[],
+): Promise<{ manifests: { relPath: string; contents: string }[]; notes: string[] }> {
+  const listed = new Set(metas.map((m) => path.resolve(m.dir)))
+  const unlisted = projects.filter(
+    (p) => path.resolve(p.dir) !== path.resolve(root) && !listed.has(path.resolve(p.dir)),
+  )
+  if (unlisted.length === 0) return { manifests: [], notes: [] }
+  const rels = unlisted.map((p) => path.relative(root, p.dir).split(path.sep).join('/')).sort()
+  const manifests: { relPath: string; contents: string }[] = []
+  for (const p of unlisted) {
+    const file = path.join(p.dir, 'package.json')
+    if (
+      await stat(file).then(
+        () => true,
+        () => false,
+      )
+    )
+      continue
+    manifests.push({
+      relPath: path.relative(root, file).split(path.sep).join('/'),
+      contents: `${JSON.stringify({ name: p.name, private: true }, null, 2)}\n`,
+    })
+  }
+  const pnpm = await stat(path.join(root, 'pnpm-workspace.yaml')).then(
+    () => true,
+    () => false,
+  )
+  const where = pnpm ? "pnpm-workspace.yaml's `packages`" : 'package.json `workspaces`'
+  const wrote = manifests.length > 0 ? ' (a package.json is written where there was none)' : ''
+  return {
+    manifests,
+    notes: [
+      `${unlisted.length} Nx project${unlisted.length === 1 ? ' is' : 's are'} in no workspace glob${wrote} — ` +
+        `vx finds a project by a package.json the workspace lists: add ${rels.map((r) => JSON.stringify(r)).join(', ')} to ${where}`,
+    ],
+  }
+}
+
+/**
+ * nx.json's run-wide settings `nx()` applies live and the written
+ * `vx.workspace.ts` cannot hold (core writes it): each is a line naming
+ * the field to add, or a migrated repo that ran one task at a time for a
+ * shared database ran on every core. nx.json only: the files are written
+ * for every machine, so `NX_PARALLEL` and friends are not read.
+ */
+function workspaceNotes(json: Record<string, unknown> | undefined): string[] {
+  const nx = json as
+    | {
+        parallel?: unknown
+        tasksRunnerOptions?: { default?: { options?: { parallel?: unknown } } }
+        defaultBase?: unknown
+        affected?: { defaultBase?: unknown }
+        maxCacheSize?: unknown
+      }
+    | undefined
+  const out: string[] = []
+  const parallel = nx?.parallel ?? nx?.tasksRunnerOptions?.default?.options?.parallel
+  if (typeof parallel === 'number' && Number.isInteger(parallel) && parallel > 0)
+    out.push(
+      `nx.json \`parallel: ${parallel}\`: add \`concurrency: ${parallel}\` to vx.workspace.ts`,
+    )
+  const base = nx?.defaultBase ?? nx?.affected?.defaultBase
+  if (typeof base === 'string' && base.trim() !== '')
+    out.push(
+      `nx.json \`defaultBase\` ${JSON.stringify(base.trim())}: add \`affectedBase: ${JSON.stringify(base.trim())}\` to vx.workspace.ts`,
+    )
+  const size = nxSizeText(nx?.maxCacheSize)
+  if (size !== undefined)
+    out.push(
+      `nx.json \`maxCacheSize\`: add \`cacheRetention: { maxSize: ${JSON.stringify(size)} }\` to vx.workspace.ts`,
+    )
+  return out
+}