@vzn/vx 0.0.650 → 0.0.652

package/src/workspace/dependencies-lock.ts

@@ -0,0 +1,191 @@
+// .vx/dependencies.lock.json — what the workspace's dependencies resolve
+// to, written by `vx lock` beside the config lock and committed.
+//
+// Each project's edges (the `discover` stage's `deps` and `reach`), by
+// directory, and each claimed file's per-project digests
+// (`FingerprintClaim.digests`) with the hash of the bytes they came from
+// and of each other file the digests read. A run keys from these digests
+// and never parses the file: one whose bytes moved since refuses
+// (orchestrator/locked-digests.ts). `--affected` reads the base commit's
+// copy with git: a deleted project's dependants are in its edges, and a
+// changed lockfile's base digests are there, so the base side is never
+// parsed. `vx lock --check` holds it to a fresh derivation.
+
+import { rename, rm } from 'node:fs/promises'
+import path from 'node:path'
+import type { ProjectMeta } from './workspace.js'
+import { lockKey } from './lockfile.js'
+import { type LoadReads, readOnce } from './load-reads.js'
+
+export const DEPENDENCIES_LOCK_NAME = '.vx/dependencies.lock.json'
+const DEPENDENCIES_LOCK_VERSION = 3
+
+// Flat: each directory once, everything else an index into `projects`, so
+// a parse builds few strings and a lookup is an array read.
+export interface DependenciesLock {
+  version: number
+  /** Project directories (root-relative POSIX, `.` for the root), sorted. */
+  projects: string[]
+  /** Per project, the indexes of the projects it builds after. */
+  deps: number[][]
+  /** Per project, the indexes of the projects it reaches for selection only. */
+  reach: number[][]
+  /** Claimed file → its bytes' xxh3, each project's digest by index ('' when none), and each other file read → its hash. */
+  files: Record<string, LockedFile>
+}
+
+interface LockedFile {
+  hash: string
+  digests: string[]
+  /** Root-relative files besides this one whose bytes the digests read → their hash (`hashExtra`). */
+  extras?: Record<string, string>
+}
+
+/** The lock for `metas`, with `files` keyed by project name as the claims answer. */
+export function dependenciesLock(
+  root: string,
+  metas: readonly ProjectMeta[],
+  files: ReadonlyMap<
+    string,
+    {
+      hash: string
+      projects: Readonly<Record<string, string>>
+      extras?: Readonly<Record<string, string>>
+    }
+  >,
+): DependenciesLock {
+  const dirOf = new Map(metas.map((m) => [m.name, lockKey(root, m.dir)]))
+  const projects = [...new Set(dirOf.values())].sort(cmp)
+  const at = new Map(projects.map((d, i) => [d, i]))
+  const byDir = new Map(metas.map((m) => [dirOf.get(m.name)!, m]))
+  const indexes = (names: readonly string[] | undefined): number[] =>
+    [...new Set((names ?? []).flatMap((n) => at.get(dirOf.get(n) ?? '') ?? []))].sort(
+      (a, b) => a - b,
+    )
+  const out: DependenciesLock['files'] = {}
+  for (const file of [...files.keys()].sort(cmp)) {
+    const { hash, projects: byName, extras } = files.get(file)!
+    const digests = projects.map(() => '')
+    for (const [name, digest] of Object.entries(byName)) {
+      const i = at.get(dirOf.get(name) ?? '')
+      if (i !== undefined) digests[i] = digest
+    }
+    const rels = Object.keys(extras ?? {}).sort(cmp)
+    out[file] =
+      rels.length === 0
+        ? { hash, digests }
+        : { hash, digests, extras: Object.fromEntries(rels.map((r) => [r, extras![r]!])) }
+  }
+  return {
+    version: DEPENDENCIES_LOCK_VERSION,
+    projects,
+    deps: projects.map((d) => indexes(byDir.get(d)!.deps)),
+    reach: projects.map((d) => indexes(byDir.get(d)!.reach)),
+    files: out,
+  }
+}
+
+/** Minified, as the config lock: only vx reads it, and minified JSON parses faster. */
+function dependenciesLockText(lock: DependenciesLock): string {
+  return `${JSON.stringify(lock)}\n`
+}
+
+/** Written beside its name and renamed over it, as `writeLockfile`: a reader never sees half a file. */
+export async function writeDependenciesLock(root: string, lock: DependenciesLock): Promise<void> {
+  const file = path.join(root, DEPENDENCIES_LOCK_NAME)
+  const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2)}`
+  try {
+    await Bun.write(tmp, dependenciesLockText(lock))
+    await rename(tmp, file)
+  } catch (err) {
+    await rm(tmp, { force: true }).catch(() => {})
+    const e = err as NodeJS.ErrnoException
+    if (typeof e.message === 'string') e.message = e.message.split(tmp).join(file)
+    if (e.path === tmp) e.path = file
+    throw err
+  }
+}
+
+/** The lock's bytes at `root`, read once per `reads`; null when there is none. */
+export function readDependenciesLock(root: string, reads?: LoadReads): Promise<Uint8Array | null> {
+  return readOnce(reads, path.join(root, DEPENDENCIES_LOCK_NAME))
+}
+
+/** The lock in `bytes`; null when absent, another version, or not this shape. */
+export function parseDependenciesLock(bytes: Uint8Array | null): DependenciesLock | null {
+  if (bytes === null) return null
+  try {
+    const lock = JSON.parse(new TextDecoder().decode(bytes)) as Partial<DependenciesLock>
+    return lock.version === DEPENDENCIES_LOCK_VERSION && wellFormed(lock)
+      ? (lock as DependenciesLock)
+      : null
+  } catch {
+    return null
+  }
+}
+
+// A committed file a merge or a hand edit can break inside, so every
+// entry is checked: one bad entry read on would throw a raw TypeError at
+// its first lookup instead of the "invalid, run `vx lock`" refusal.
+function wellFormed(lock: Partial<DependenciesLock>): boolean {
+  const { projects, deps, reach, files } = lock
+  if (!Array.isArray(projects) || !projects.every((d) => typeof d === 'string')) return false
+  const n = projects.length
+  const edges = (lists: unknown): boolean =>
+    Array.isArray(lists) &&
+    lists.length === n &&
+    lists.every((l) => Array.isArray(l) && l.every((i) => Number.isInteger(i) && i >= 0 && i < n))
+  if (!edges(deps) || !edges(reach) || !isRecord(files)) return false
+  return Object.values(files).every(
+    (rec: unknown) =>
+      isRecord(rec) &&
+      typeof rec['hash'] === 'string' &&
+      Array.isArray(rec['digests']) &&
+      rec['digests'].length === n &&
+      rec['digests'].every((d) => typeof d === 'string') &&
+      (rec['extras'] === undefined ||
+        (isRecord(rec['extras']) &&
+          Object.values(rec['extras']).every((h) => typeof h === 'string'))),
+  )
+}
+
+function isRecord(v: unknown): v is Record<string, unknown> {
+  return typeof v === 'object' && v !== null && !Array.isArray(v)
+}
+
+/** The project directories whose edges reach `dir`. */
+export function dependantsOf(lock: DependenciesLock, dir: string): string[] {
+  const i = lock.projects.indexOf(dir)
+  if (i < 0) return []
+  return lock.projects.filter((_, j) => lock.deps[j]!.includes(i) || lock.reach[j]!.includes(i))
+}
+
+/** `file`'s recorded hashes and each project directory's digest (undefined when none). */
+export function recordedDigests(
+  lock: DependenciesLock,
+  file: string,
+):
+  | {
+      hash: string
+      extras: Readonly<Record<string, string>>
+      has: (dir: string) => boolean
+      of: (dir: string) => string | undefined
+    }
+  | undefined {
+  const rec = Object.hasOwn(lock.files, file) ? lock.files[file] : undefined
+  if (rec === undefined) return undefined
+  const at = new Map(lock.projects.map((d, i) => [d, i]))
+  return {
+    hash: rec.hash,
+    extras: rec.extras ?? {},
+    has: (dir) => at.has(dir),
+    of: (dir) => {
+      const i = at.get(dir)
+      return i === undefined || rec.digests[i] === '' ? undefined : rec.digests[i]
+    },
+  }
+}
+
+function cmp(a: string, b: string): number {
+  return a < b ? -1 : a > b ? 1 : 0
+}

package/src/workspace/filter.ts

@@ -22,7 +22,7 @@
 // given, the base set is "all projects" and excluded packages are removed.
 
 import path from 'node:path'
-import { BUN_GLOB_WILDCARDS, relPosix, UserError } from '../util/index.js'
+import { BUN_GLOB_WILDCARDS, hasControl, printable, relPosix, UserError } from '../util/index.js'
 import type { PackageGraph } from './package-graph.js'
 import { canonicalDir } from './canonical-dir.js'
 import type { ProjectMeta } from './workspace.js'
@@ -66,6 +66,8 @@ export interface ParsedFilter {
    * `./apps/*` when it names no project, as Nx's `--projects` reads it.
    */
   dirFallback?: ParsedFilter
+  /** `<pattern>{<dir>}`: the `{<dir>}` path form the name matches are narrowed to. */
+  within?: ParsedFilter
 }
 
 export function parseFilter(
@@ -74,6 +76,13 @@ export function parseFilter(
   /** The filter as typed, when `raw` is the `./` reading of a name (`dirFallback`). */
   typed = raw,
 ): ParsedFilter {
+  // No project name, path or git ref holds one; echoed raw, a newline split
+  // the refusal in two and an escape painted the terminal.
+  if (hasControl(raw)) {
+    throw new UserError(
+      `filter "${printable(raw)}" holds a control character: no project name, path or git ref does`,
+    )
+  }
   let s = raw
   const negate = s.startsWith('!')
   if (negate) s = s.slice(1)
@@ -175,6 +184,27 @@ export function parseFilter(
     }
   }
 
+  // `<pattern>{<dir>}`: the packages the name selects in the directory, as
+  // pnpm and Turbo read it (turborepo#887, `pattern{foo}[master]`). Read as
+  // one name glob, it matched nothing.
+  const named = /^([^{}]+)\{([^{}]*)\}$/.exec(s)
+  if (named !== null && !named[1]!.startsWith('./') && named[1] !== '.') {
+    if (named[2]!.trim() === '') throw new UserError(`filter "${raw}" names no project`)
+    return {
+      raw,
+      negate,
+      withDeps,
+      withDependents,
+      onlyDeps,
+      onlyDependents,
+      isPath: false,
+      matcher: named[1]!,
+      within: parseFilter(`{${named[2]}}`, workspaceRoot, raw),
+      ...(gitSince !== undefined ? { gitSince } : {}),
+      ...(sinceViaDeps ? { sinceViaDeps: true as const } : {}),
+    }
+  }
+
   let isPath = false
   let matcher = s
   let pathGlob: Bun.Glob | undefined
@@ -266,6 +296,11 @@ function matchSelector(
   projects: ProjectMeta[],
   tags: ProjectTags | undefined,
 ): string[] {
+  if (filter.within !== undefined) {
+    const { within, ...name } = filter
+    const inDir = new Set(matchSelector(within, projects, tags))
+    return matchSelector(name, projects, tags).filter((n) => inDir.has(n))
+  }
   const out: string[] = []
   if (filter.tag === true) {
     const re = compileNameGlob(filter.matcher)

package/src/workspace/index.ts

@@ -32,6 +32,15 @@ export {
   type Lockfile,
   type LockedProject,
 } from './lockfile.js'
+export {
+  DEPENDENCIES_LOCK_NAME,
+  dependenciesLock,
+  parseDependenciesLock,
+  readDependenciesLock,
+  recordedDigests,
+  writeDependenciesLock,
+  type DependenciesLock,
+} from './dependencies-lock.js'
 export {
   ConfigsRefused,
   loadProjectConfig,

package/src/workspace/lockfile.ts

@@ -21,7 +21,7 @@ import { validateProjectConfig } from './config-schema.js'
 import { PROJECT_CONFIG_FILENAMES } from './workspace.js'
 
 export const LOCKFILE_NAME = '.vx/projects.lock.json'
-export const LOCKFILE_VERSION = 4
+export const LOCKFILE_VERSION = 5
 
 /** One project as the lock holds it. */
 export interface LockedProject {
@@ -55,14 +55,19 @@ export const LOCK_INVALID = `${LOCKFILE_NAME} is invalid, run \`vx lock\``
  * raw TypeError; asked only on that crash's path, so a run pays nothing.
  */
 export function lockEntryBroken(root: string): boolean {
-  let projects: unknown
+  let lock: Pooled
   try {
-    projects = (JSON.parse(readFileSync(lockfilePath(root), 'utf8')) as { projects?: unknown })
-      .projects
+    lock = JSON.parse(readFileSync(lockfilePath(root), 'utf8')) as Pooled
   } catch {
     return false
   }
-  if (typeof projects !== 'object' || projects === null) return false
+  if (typeof lock?.projects !== 'object' || lock.projects === null) return false
+  try {
+    pooledIn(lock)
+  } catch {
+    return true
+  }
+  const projects = lock.projects
   for (const [dir, entry] of Object.entries(projects)) {
     try {
       validateProjectConfig((entry as LockedProject).config, `${LOCKFILE_NAME}: "${dir}"`, null)
@@ -158,10 +163,157 @@ export async function readLockfile(root: string): Promise<Lockfile | null> {
       )
     }
   }
+  pooledIn(parsed as Pooled)
   digests.set(parsed as Lockfile, xxh3hex(text))
   return parsed as Lockfile
 }
 
+/**
+ * The project entries of a lock's bytes with its pool put back, or null
+ * for bytes that do not parse as a lock: an entry compares across two
+ * locks only once both are read the same way.
+ */
+export function lockEntries(bytes: Uint8Array | null): Record<string, unknown> | null {
+  if (bytes === null) return null
+  try {
+    const lock = JSON.parse(new TextDecoder().decode(bytes)) as Pooled
+    if (typeof lock?.projects !== 'object' || lock.projects === null) return null
+    pooledIn(lock)
+    return lock.projects
+  } catch {
+    return null
+  }
+}
+
+/**
+ * A task or task part that two places in the lock hold alike is written once,
+ * in `pool`, and each place holds its index. Each slot is one the schema
+ * types as an object, so a number there can only be an index. Big
+ * workspaces repeat the same tasks: 2,000 projects of five tasks read in
+ * 0.71 ms against 2.62 ms, a fifth of the bytes. A run reads them shared,
+ * once; a stage that edits a config in place copies it first.
+ */
+const POOLS = ['env', 'sandbox', 'exec', 'inputs', 'cache', 'task'] as const
+type PoolName = (typeof POOLS)[number]
+type Part = Exclude<PoolName, 'task'>
+type Pools = Partial<Record<PoolName, unknown[]>>
+type Pooled = { pool?: Pools; projects: Record<string, LockedProject> }
+type Obj = Record<string, unknown>
+const isObj = (v: unknown): v is Obj => typeof v === 'object' && v !== null && !Array.isArray(v)
+const TASK_PARTS = ['exec', 'inputs', 'cache'] as const
+const EXEC_PARTS = ['env', 'sandbox'] as const
+
+/** `lock.projects` with every index replaced by the value it names, in place. */
+function pooledIn(lock: Pooled): void {
+  const pool = lock.pool
+  delete lock.pool
+  if (pool === undefined) return
+  if (!isObj(pool) || Object.values(pool).some((list) => !Array.isArray(list))) {
+    throw new UserError(
+      `${LOCKFILE_NAME}: \`pool\` must be an object of lists — re-run \`vx lock\``,
+      'VX_E_LOCK',
+    )
+  }
+  const at = (name: PoolName, i: number): unknown => {
+    const hit = pool[name]?.[i]
+    if (hit === undefined) {
+      throw new UserError(
+        `${LOCKFILE_NAME}: no ${name} ${i} in its pool — re-run \`vx lock\``,
+        'VX_E_LOCK',
+      )
+    }
+    return hit
+  }
+  const parts = (o: unknown, names: readonly Part[]): void => {
+    if (!isObj(o)) return
+    for (const n of names) if (typeof o[n] === 'number') o[n] = at(n, o[n])
+  }
+  const task = (t: unknown): void => {
+    parts(t, TASK_PARTS)
+    if (isObj(t)) parts(t['exec'], EXEC_PARTS)
+  }
+  // Inner pools first: a pooled task's exec may itself be pooled.
+  for (const e of pool.exec ?? []) parts(e, EXEC_PARTS)
+  for (const t of pool.task ?? []) task(t)
+  for (const entry of Object.values(lock.projects)) {
+    const tasks = (entry as { config?: { tasks?: unknown } } | null)?.config?.tasks
+    if (!isObj(tasks)) continue
+    for (const name of Object.keys(tasks)) {
+      const t = tasks[name]
+      if (typeof t === 'number') tasks[name] = at('task', t)
+      else task(t)
+    }
+  }
+}
+
+/** `projects` as written: each value two slots hold alike, once in `pool`. */
+function pooledOut(projects: Record<string, LockedProject>): Pooled {
+  const seen = new Map<string, number>()
+  const keyOf = (name: PoolName, v: Obj): string => `${name}\0${JSON.stringify(v)}`
+  const count = (name: PoolName, v: unknown): void => {
+    if (!isObj(v)) return
+    const k = keyOf(name, v)
+    seen.set(k, (seen.get(k) ?? 0) + 1)
+  }
+  const tasksOf = (p: LockedProject): Obj | undefined =>
+    isObj(p.config.tasks) ? p.config.tasks : undefined
+  for (const p of Object.values(projects)) {
+    for (const t of Object.values(tasksOf(p) ?? {})) {
+      if (!isObj(t)) continue
+      count('task', t)
+      for (const n of TASK_PARTS) count(n, t[n])
+      if (isObj(t['exec'])) for (const n of EXEC_PARTS) count(n, t['exec'][n])
+    }
+  }
+  const pool: Pools = {}
+  const index = new Map<string, number>()
+  const out = (name: PoolName, v: unknown, inner: (v: Obj) => Obj): unknown => {
+    if (!isObj(v)) return v
+    const k = keyOf(name, v)
+    if ((seen.get(k) ?? 0) < 2) return inner(v)
+    let i = index.get(k)
+    if (i === undefined) {
+      const list = (pool[name] ??= [])
+      index.set(k, (i = list.length))
+      list.push(inner(v))
+    }
+    return i
+  }
+  const same = (v: Obj): Obj => v
+  const withParts = (o: Obj, names: readonly Part[]): Obj => {
+    const r = { ...o }
+    for (const n of names) if (n in o) r[n] = out(n, o[n], n === 'exec' ? exec : same)
+    return r
+  }
+  const exec = (e: Obj): Obj => withParts(e, EXEC_PARTS)
+  const task = (t: Obj): Obj => withParts(t, TASK_PARTS)
+  // Entries, not assignment: a directory named `__proto__` set the prototype (D-152).
+  const written = Object.fromEntries(
+    Object.entries(projects).map(([dir, p]) => {
+      const tasks = tasksOf(p)
+      return [
+        dir,
+        tasks === undefined
+          ? p
+          : {
+              ...p,
+              config: {
+                ...p.config,
+                tasks: Object.fromEntries(
+                  Object.entries(tasks).map(([name, t]) => [name, out('task', t, task)]),
+                ),
+              } as ProjectConfig,
+            },
+      ]
+    }),
+  )
+  const ordered: Pools = {}
+  for (const n of POOLS) if (pool[n] !== undefined) ordered[n] = pool[n]
+  return POOLS.some((n) => pool[n] !== undefined)
+    ? { pool: ordered, projects: written }
+    : { projects: written }
+}
+
 /**
  * `vx lock` evaluates and validates every config, then writes a digest of
  * the bytes it wrote, under vx's version, as the last field: the lock's
@@ -194,7 +346,8 @@ export async function writeLockfile(root: string, lock: Lockfile): Promise<void>
     // Minified: the file only vx reads, and it loads faster than pretty
     // JSON (measured, 10k projects: 15.6 ms against 22.5 ms). `vx.lock.json`
     // beside each config is the copy for people.
-    const body = JSON.stringify({ version: lock.version, projects: lock.projects }, taskKeysSorted)
+    const sorted = JSON.parse(JSON.stringify(lock.projects, taskKeysSorted)) as Lockfile['projects']
+    const body = JSON.stringify({ version: lock.version, ...pooledOut(sorted) })
     await Bun.write(tmp, `${body.slice(0, -1)},"validated":"${validatedDigest(body)}"}\n`)
     await rename(tmp, file)
   } catch (err) {

package/src/workspace/migration.ts

@@ -16,9 +16,9 @@ import {
   refuseSymlink,
   relPosix,
   UserError,
-  ui,
   writeNoFollow,
 } from '../util/index.js'
+import * as ui from '../ui/index.js'
 import { VERSION } from '../version.js'
 import { WORKSPACE_CONFIG_FILENAMES } from './project-loader.js'
 
@@ -42,6 +42,10 @@ export function isStarterConfig(file: string): boolean {
   }
 }
 
+/** A C0 or C1 control, or DEL. */
+// oxlint-disable-next-line no-control-regex
+const CONTROL = /[\x00-\x1f\x7f-\x9f]/g
+
 /**
  * Escape an arbitrary string into a single-quoted TS literal. Escapes
  * backslash + quote AND raw newlines/CR — a value with an embedded newline
@@ -55,7 +59,10 @@ export function quoteTsLiteral(s: string): string {
     .replaceAll('\\', '\\\\')
     .replaceAll("'", "\\'")
     .replaceAll('\n', '\\n')
-    .replaceAll('\r', '\\r')}'`
+    .replaceAll('\r', '\\r')
+    // Any other control as its escape: a script's raw ESC landed in the
+    // file as the byte, and `cat vx.config.ts` retitled the terminal.
+    .replace(CONTROL, (c) => `\\x${c.charCodeAt(0).toString(16).padStart(2, '0')}`)}'`
 }
 
 /**
@@ -451,7 +458,8 @@ export async function applyMigration(args: ApplyMigrationArgs): Promise<number>
     ...plan.notes,
   ]
   if (args.json === true) {
-    process.stdout.write(
+    ui.data(
+      'out',
       `${JSON.stringify({
         dry,
         source,
@@ -472,7 +480,7 @@ export async function applyMigration(args: ApplyMigrationArgs): Promise<number>
 
   // The lock's own words become a row; a refusal prints above the report.
   const lock = dry || args.lock === undefined ? undefined : lockAnswer(await args.lock(root))
-  if (lock !== undefined && lock.code !== 0) process.stderr.write(lock.said)
+  if (lock !== undefined && lock.code !== 0) ui.data('err', lock.said)
 
   const rows: ui.FieldRow[] = [['from', source]]
   if (dry) rows.push(['dry run', 'nothing written'])
@@ -657,7 +665,7 @@ const EXAMPLE_BODY = `export default {
     build: {
       exec: { command: 'tsc -b' },
       dependsOn: ['^build'],
-      inputs: { files: ['src/**'] }, cache: { outputs: { files: ['dist/**'] } },
+      inputs: { files: ['src/**'] }, outputs: { files: ['dist/**'] }, cache: true,
     },
   },
 }`
@@ -746,7 +754,7 @@ function renderConfigFile(p: GeneratedProject, format: MigrationFormat): string
     // The suggested block sits in the task, commented: deleting `// `
     // after setting the real globs caches it, no copying from the TODO.
     const block =
-      t.task['cache'] === undefined
+      t.task['cache'] !== true
         ? t.todos.find(isCacheTodo)?.match(/^cache: add `(.+?)`/)?.[1]
         : undefined
     lines.push(
@@ -789,11 +797,10 @@ function ownConfigNamed(
   return {
     ...p,
     tasks: p.tasks.map((t) => {
-      const outputs = (t.task?.['cache'] as { outputs?: { files?: unknown } } | undefined)?.outputs
+      const outputs = t.task?.['outputs'] as { files?: unknown } | undefined
       if (!Array.isArray(outputs?.files) || !outputs.files.includes(planned)) return t
-      const cache = t.task!['cache'] as Record<string, unknown>
       const files = outputs.files.map((f: unknown) => (f === planned ? `!${name}` : f))
-      return { ...t, task: { ...t.task, cache: { ...cache, outputs: { ...outputs, files } } } }
+      return { ...t, task: { ...t.task, outputs: { ...outputs, files } } }
     }),
   }
 }

package/src/workspace/workspace.ts

@@ -159,6 +159,9 @@ export async function resolveStoreRoot(
   let id = repoIds.get(root)
   if (id === undefined) repoIds.set(root, (id = repoIdOf(root)))
   const resolved = await id
+  // No identity yet (no commit, no remote) is asked again: the first
+  // commit gives one, and `vx watch` runs on in the same process.
+  if (resolved === null) repoIds.delete(root)
   return resolved === null ? null : path.join(home, '.vx', resolved, 'cache')
 }