@getarbiter/cli 0.5.0

package/dist/templates/scripts/lib/gate-mutex.mjs.ejs

@@ -0,0 +1,340 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: Apache-2.0
+// scripts/lib/gate-mutex.mjs (#2427) — the per-repo gate mutex, reachable from
+// `.mjs` land (the pre-push hook and `scripts/check-all.mjs`).
+//
+// The incident this exists for: a `git push` was killed while its pre-push L2
+// ran. The orphaned gate kept going; the branch took another commit; a second
+// push started a SECOND L2 in the same worktree. The two interfered (`docs:build`
+// tripped over a half-deleted vitepress temp file, a subprocess-heavy unit test
+// flaked under the doubled load) and the orphan finished green and stamped a
+// marker for a tree it had never fully tested.
+//
+// CANON-16 — this is NOT a second mutex. `src/commands/gate-exec.ts` remains the
+// owner of the contract; this file reproduces its KEY DERIVATION exactly (sha256
+// of the resolved `git rev-parse --git-common-dir`, first 16 hex chars, under
+// `$XDG_RUNTIME_DIR/arbiter`) and delegates acquisition to the same `flock(1)`
+// primitive against the same lock file. The derivation moved here OUT of
+// `scripts/capacity-probe.mjs`, which carried it as a documented duplicate; that
+// script now imports it, so the copy count does not grow. Reuse-by-import is
+// impossible in both directions: `gate-exec.ts` compiles into `dist/`, which
+// ships WITHOUT `scripts/` (package.json "files"), and this file must run
+// pre-build in a consumer checkout that has no arbiter `dist/` at all — the same
+// constraint that already forced `scripts/lib/waiter-count.mjs` to exist and
+// forced `gate-exec.ts` to shell OUT to it. `__tests__/scripts/gate-mutex.test.ts`
+// pins the two derivations byte-for-byte so they cannot drift.
+//
+// Lock ordering (ADR-103 §4) is unchanged: the gate lock is a LEAF — it is never
+// taken while `.arbiter/.lock` is held. Total order: gate-lock ≺ worktree-lock ≺
+// wave-claim.
+//
+// AC-3 shape: the gate runs in the wrapper's OWN process group — no `setsid`, no
+// `detached`, no background `&`. A terminal Ctrl-C therefore reaches the gate the
+// same way it reaches `git push`. A pid-targeted signal is handled by forwarding
+// it down the whole descendant tree, and a SIGKILL (which cannot be trapped) is
+// covered by the independent orphan guard in `scripts/lib/run-helpers.mjs`, armed
+// through `ARBITER_GATE_PARENT_PID`.
+//
+// INV-12 exception: direct child_process use is the documented carve-out for
+// `.mjs` gate-utility libraries that run pre-build and cannot pull from `src/`
+// (same carve-out as `scripts/lib/waiter-count.mjs` and `scripts/lib/loud-bypass.mjs`).
+import { execFileSync, spawn, spawnSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { mkdirSync, readdirSync, readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { isMainModule, isProcessAlive } from './run-helpers.mjs'
+
+export { isProcessAlive }
+
+/** Set on the gate's environment once the mutex is held, so it is never re-taken. */
+export const GATE_MUTEX_HELD_ENV = 'ARBITER_GATE_MUTEX_HELD'
+/** The pid the gate exists to serve — the orphan guard watches it (AC-3). */
+export const GATE_MUTEX_PARENT_ENV = 'ARBITER_GATE_PARENT_PID'
+/** `flock`'s exit code when the mutex could not be taken (busy / wait expired). */
+export const GATE_MUTEX_BUSY_EXIT = 111
+/** How long a waiting gate blocks before failing closed. 30 min > one L2 run. */
+export const GATE_MUTEX_DEFAULT_WAIT_SEC = 1800
+/** Grace between the polite teardown signal and the SIGKILL sweep. */
+const TEARDOWN_GRACE_MS = 5_000
+/** Signal numbers for the shell's 128+n exit convention. */
+const SIGNAL_NUMBERS = Object.freeze({ SIGHUP: 1, SIGINT: 2, SIGKILL: 9, SIGTERM: 15 })
+
+/**
+ * The per-repo lock path — byte-identical to `gateLockPath(deriveGateKey(dir))`
+ * in `src/commands/gate-exec.ts`. Every worktree of a repo shares the main
+ * repo's common dir, so they converge on ONE lock. The file lives OUTSIDE the
+ * repo: a per-worktree lock would be a null mutex, an in-repo one would dirty
+ * every tree it guards.
+ */
+export function gateLockPathFor(dir = process.cwd(), env = process.env) {
+  const commonDir = execFileSync('git', ['rev-parse', '--git-common-dir'], {
+    cwd: dir,
+    encoding: 'utf-8',
+  }).trim()
+  // --git-common-dir may be relative (e.g. ".git" from the repo root).
+  const absolute = resolve(dir, commonDir)
+  const key = createHash('sha256').update(absolute).digest('hex').slice(0, 16)
+  const base = env.XDG_RUNTIME_DIR || tmpdir()
+  const lockDir = join(base, 'arbiter')
+  mkdirSync(lockDir, { recursive: true })
+  return join(lockDir, `${key}-gate.lock`)
+}
+
+/** True when `flock(1)` can actually be run here. */
+export function flockAvailable(env = process.env) {
+  const probe = spawnSync('flock', ['--version'], { env, stdio: 'ignore', timeout: 10_000 })
+  return probe.error === undefined && probe.status === 0
+}
+
+/**
+ * Direct children of `pid` via Linux procfs. Returns [] wherever
+ * `/proc/<pid>/task/<tid>/children` is unavailable — the teardown then degrades
+ * to signalling the direct child only, and the orphan guard remains the backstop.
+ */
+function childPids(pid) {
+  const out = []
+  try {
+    const taskDir = `/proc/${pid}/task`
+    for (const tid of readdirSync(taskDir)) {
+      const raw = readFileSync(join(taskDir, tid, 'children'), 'utf-8').trim()
+      if (raw === '') continue
+      for (const part of raw.split(/\s+/)) {
+        const child = Number(part)
+        if (Number.isInteger(child) && child > 0) out.push(child)
+      }
+    }
+    // An unreadable procfs entry means "no discoverable children", which only
+    // narrows the teardown — the orphan guard is the backstop for the residual.
+    // FAIL-OPEN-INTENT: procfs is Linux-only and best-effort; the teardown degrades to the direct child.
+  } catch {
+    return out
+  }
+  return out
+}
+
+/**
+ * Signal `pid` and every descendant, deepest-first — the tree is read BEFORE the
+ * parents are signalled, so nothing is lost to a vanishing intermediate process.
+ */
+export function killTree(pid, signal) {
+  if (!Number.isInteger(pid) || pid <= 0) return
+  for (const child of childPids(pid)) killTree(child, signal)
+  try {
+    process.kill(pid, signal)
+    // FAIL-OPEN-INTENT: ESRCH means the process is already gone — the very outcome this function exists to produce.
+  } catch {
+    void 0
+  }
+}
+
+/**
+ * `flock` argv for the requested mode. `-o` (--close) drops the lock fd before
+ * the gate is exec'd so no gate DESCENDANT retains the mutex; `flock` itself
+ * forks and keeps the fd, so the lock is held for the gate's whole life.
+ * `--` ends flock's option parsing so the gate's own flags are never eaten.
+ */
+export function gateMutexArgv(lockPath, cmdArgs, { mode = 'wait', waitSec } = {}) {
+  const acquire = mode === 'fail' ? ['-n'] : ['-w', String(waitSec ?? GATE_MUTEX_DEFAULT_WAIT_SEC)]
+  return ['flock', ...acquire, '-E', String(GATE_MUTEX_BUSY_EXIT), '-o', '--', lockPath, ...cmdArgs]
+}
+
+/** Non-destructive probe: is somebody else holding the mutex right now? */
+function mutexIsBusy(lockPath, env) {
+  const probe = spawnSync(
+    'flock',
+    ['-n', '-E', String(GATE_MUTEX_BUSY_EXIT), '--', lockPath, 'true'],
+    {
+      env,
+      stdio: 'ignore',
+      timeout: 10_000,
+    },
+  )
+  return probe.status === GATE_MUTEX_BUSY_EXIT
+}
+
+/** Resolve the wait budget, ignoring a non-positive or unparseable override. */
+function waitBudgetSec(env) {
+  const raw = Number(env.ARBITER_GATE_MUTEX_WAIT_SEC)
+  return Number.isFinite(raw) && raw > 0 ? raw : GATE_MUTEX_DEFAULT_WAIT_SEC
+}
+
+/** Run `cmdArgs` as a plain foreground child, forwarding signals and the exit code. */
+function runForeground(argv, { env, cwd }) {
+  return new Promise((resolvePromise) => {
+    // detached is deliberately ABSENT (AC-3): the gate stays in this process
+    // group, so a terminal signal reaches it exactly as it reaches `git push`.
+    const child = spawn(argv[0], argv.slice(1), { env, cwd, stdio: 'inherit' })
+    let sweep = null
+
+    const teardown = (signal) => {
+      killTree(child.pid, signal)
+      if (sweep === null) {
+        sweep = setTimeout(() => killTree(child.pid, 'SIGKILL'), TEARDOWN_GRACE_MS)
+        sweep.unref()
+      }
+    }
+    const onSignal = (signal) => teardown(signal)
+    const signals = ['SIGINT', 'SIGTERM', 'SIGHUP']
+    for (const signal of signals) process.on(signal, onSignal)
+    // A wrapper that dies for any other reason must not leave the gate behind.
+    const onExit = () => killTree(child.pid, 'SIGKILL')
+    process.on('exit', onExit)
+
+    child.on('error', (err) => {
+      process.stderr.write(`gate-mutex: could not start the gate: ${err.message}\n`)
+      resolvePromise(127)
+    })
+    child.on('close', (code, signal) => {
+      for (const s of signals) process.off(s, onSignal)
+      process.off('exit', onExit)
+      if (sweep !== null) clearTimeout(sweep)
+      if (signal) resolvePromise(128 + (SIGNAL_NUMBERS[signal] ?? 15))
+      else resolvePromise(code ?? 1)
+    })
+  })
+}
+
+/** The gate's environment: the mutex now held, and the pid the gate serves. */
+function gateChildEnv(env, lockPath) {
+  return {
+    ...env,
+    [GATE_MUTEX_HELD_ENV]: lockPath,
+    // The process the gate exists to SERVE — this wrapper's own parent (the git
+    // hook shell, the terminal, the agent), not the wrapper. An outer layer that
+    // already declared one wins, so the watched pid is always the outermost
+    // launcher rather than an intermediate relay.
+    [GATE_MUTEX_PARENT_ENV]: env[GATE_MUTEX_PARENT_ENV] || String(process.ppid),
+  }
+}
+
+/**
+ * The mode actually in force. `off` when asked for, and also where `flock(1)`
+ * does not exist (the macOS base system) — announced loudly, because a gate
+ * running unserialised is a fact the operator must be able to see.
+ */
+function effectiveMode(env) {
+  const requested = env.ARBITER_GATE_MUTEX_MODE ?? 'wait'
+  if (requested === 'off') {
+    // #2427: an operator-requested `off` is as unserialised as a missing
+    // flock(1), so it is announced the same way. Without this the two
+    // degradations are asymmetric — the platform one loud, the requested one
+    // silent — and `.arbiter/gate-pass.json` is stamped identically either way,
+    // making the transcript of an unserialised gate indistinguishable from a
+    // serialised one. The doc above says running unserialised is a fact the
+    // operator must be able to see; this makes that true on both paths.
+    process.stderr.write(
+      'gate-mutex: ARBITER_GATE_MUTEX_MODE=off — the gate is running UNSERIALISED ' +
+        'by request. Two gates in this repo can now interfere; unset the variable ' +
+        'to restore serialisation.\n',
+    )
+    return 'off'
+  }
+  if (flockAvailable(env)) return requested
+  process.stderr.write(
+    'gate-mutex: flock(1) is not available on this platform — the gate is running ' +
+      'UNSERIALISED. Two gates in this repo can now interfere; run them one at a time, ' +
+      'or use `arbiter gate-exec` on a platform that has flock.\n',
+  )
+  return 'off'
+}
+
+/**
+ * Say so the moment the gate starts queueing. A push that stalls silently for
+ * twenty minutes is how operators learn to reach for --no-verify.
+ */
+function announceWait(mode, lockPath, waitSec, env) {
+  if (mode === 'fail' || !mutexIsBusy(lockPath, env)) return
+  process.stderr.write(
+    `gate-mutex: another gate is already running in this repo — waiting up to ${waitSec}s ` +
+      `for ${lockPath}. Set ARBITER_GATE_MUTEX_MODE=fail to refuse instead of queueing.\n`,
+  )
+}
+
+/** The refusal line, so a caller never has to infer "the gate did not run". */
+function announceRefusal(mode, lockPath, waitSec) {
+  process.stderr.write(
+    `gate-mutex: refusing to run — another gate holds ${lockPath}` +
+      `${mode === 'fail' ? '' : ` after waiting ${waitSec}s`}. The gate did NOT run.\n`,
+  )
+}
+
+/**
+ * Run the gate under the per-repo mutex.
+ *
+ * Modes (`ARBITER_GATE_MUTEX_MODE`): `wait` (default — block, having ANNOUNCED
+ * the wait on stderr the moment it starts, then fail closed when the budget
+ * expires), `fail` (never queue: refuse immediately if another gate holds the
+ * mutex), `off` (run unserialised — also the automatic fallback where `flock(1)`
+ * does not exist).
+ *
+ * Returns the gate's own exit code verbatim, or GATE_MUTEX_BUSY_EXIT when the
+ * mutex could not be taken. That code is deliberately rare but not reserved: a
+ * gate that itself exits 111 is indistinguishable, which is why the refusal is
+ * always accompanied by its own stderr line.
+ */
+export async function runUnderGateLock(opts = {}) {
+  const { dir = process.cwd(), cmdArgs = [], env = process.env } = opts
+  if (cmdArgs.length === 0) {
+    process.stderr.write('gate-mutex: nothing to run — expected a command after `--`\n')
+    return 2
+  }
+  const lockPath = gateLockPathFor(dir, env)
+  const childEnv = gateChildEnv(env, lockPath)
+
+  // Already held by an ancestor (`arbiter gate-exec`, or an outer gate-mutex):
+  // re-acquiring the same flock from a second process would DEADLOCK.
+  const mode = env[GATE_MUTEX_HELD_ENV] === lockPath ? 'off' : effectiveMode(env)
+  if (mode === 'off') return runForeground(cmdArgs, { env: childEnv })
+
+  const waitSec = waitBudgetSec(env)
+  announceWait(mode, lockPath, waitSec, env)
+  const exitCode = await runForeground(gateMutexArgv(lockPath, cmdArgs, { mode, waitSec }), {
+    env: childEnv,
+  })
+  if (exitCode === GATE_MUTEX_BUSY_EXIT) announceRefusal(mode, lockPath, waitSec)
+  return exitCode
+}
+
+// ── CLI: `node scripts/lib/gate-mutex.mjs path|run [--dir d] [-- cmd...]` ─────
+function flagValue(argv, name, fallback) {
+  const index = argv.indexOf(`--${name}`)
+  if (index === -1 || index + 1 >= argv.length) return fallback
+  return argv[index + 1]
+}
+
+async function main(argv) {
+  const subcommand = argv[0]
+  const dir = flagValue(argv, 'dir', process.cwd())
+  if (subcommand === 'path') {
+    process.stdout.write(`${gateLockPathFor(dir)}\n`)
+    return 0
+  }
+  if (subcommand === 'run') {
+    const sep = argv.indexOf('--')
+    const cmdArgs = sep === -1 ? [] : argv.slice(sep + 1)
+    return runUnderGateLock({ dir, cmdArgs })
+  }
+  process.stderr.write('usage: gate-mutex.mjs path [--dir d] | run [--dir d] -- <cmd...>\n')
+  return 2
+}
+
+// Top-level guard: a synchronous throw (bad argv) and a rejected run must both
+// end the process non-zero rather than fall through to the shell's default 0,
+// which a caller would read as "the gate ran and passed".
+if (isMainModule(import.meta.url)) {
+  try {
+    main(process.argv.slice(2)).then(
+      (code) => {
+        process.exitCode = code
+      },
+      (err) => {
+        process.stderr.write(`gate-mutex: unexpected error: ${err?.stack ?? err}\n`)
+        process.exit(1)
+      },
+    )
+  } catch (err) {
+    process.stderr.write(`gate-mutex: unexpected error: ${err?.stack ?? err}\n`)
+    process.exit(1)
+  }
+}

package/dist/templates/scripts/lib/glob-walk.mjs.ejs

@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: Apache-2.0
+// CATALOG: Shared glob + tree-walk helpers for presence-gate scripts (#1366).
+// CATALOG:   Used by check-render-smoke.mjs (INV-127). A minimal, dependency-free
+// CATALOG:   matcher: `**` crosses directories, `*` stays within one path component.
+// CATALOG:   walkRepo returns repo-relative POSIX paths; SKIP_DIRS prunes vendor trees.
+// Pure module — no process exit, no I/O side effects beyond readdir/stat.
+import { readdirSync, lstatSync, statSync } from 'node:fs'
+import { join, isAbsolute } from 'node:path'
+
+// #1840 F4 tranche-3: '.venv'/'venv'/'__pycache__' added — a Python fixture's
+// dependency-closure venv lives INSIDE the project tree (mirrors post-edit-
+// dispatch.mjs.ejs's own SKIP_DIRS regex), and third-party packages sometimes
+// ship their OWN test suites inside site-packages (e.g. greenlet, a transitive
+// sqlalchemy/uvicorn dep) — those tripped check-muted-test.mjs on a plain
+// `@unittest.skipIf` in a dependency's bundled tests, a false positive with
+// zero relationship to the project's own code.
+export const SKIP_DIRS = new Set([
+  'node_modules',
+  '.git',
+  'dist',
+  'build',
+  'coverage',
+  '.coverage',
+  '.venv',
+  'venv',
+  '__pycache__',
+])
+
+/**
+ * Compile a restricted glob (`**`, `*`) into a RegExp ONCE. Compiling is the per-check cost; a
+ * caller filtering many paths against one glob compiles a single RegExp here and `.test()`s it per
+ * file instead of `new RegExp` per file (#1522/#1600). Output-invariant ⇒ engine-parity preserved.
+ * `**\/` crosses directory boundaries; `**` (not followed by /) matches any run; `*` stays within
+ * a single path component. Mirrors src/conformance/shared.ts globToRegExp byte-for-byte.
+ */
+export function globToRegExp(pattern) {
+  let reStr = '^'
+  let i = 0
+  while (i < pattern.length) {
+    const ch = pattern[i]
+    if (ch === '*' && pattern[i + 1] === '*') {
+      if (pattern[i + 2] === '/') {
+        reStr += '(?:[^/]*/)*'
+        i += 3
+      } else {
+        reStr += '[\\s\\S]*'
+        i += 2
+      }
+    } else if (ch === '*') {
+      reStr += '[^/]*'
+      i++
+    } else if ('\\.+?^${}()|[]'.includes(ch)) {
+      reStr += '\\' + ch
+      i++
+    } else {
+      reStr += ch
+      i++
+    }
+  }
+  reStr += '$'
+  return new RegExp(reStr)
+}
+
+/**
+ * Translate a restricted glob (`**`, `*`) into a RegExp and test a POSIX path. Convenience wrapper
+ * over {@link globToRegExp} for one-off single-path tests (callers filtering a LIST should compile
+ * once via globToRegExp and reuse the RegExp — see gold-audit-lib expandGlob).
+ */
+export function globMatch(pattern, filepath) {
+  return globToRegExp(pattern).test(filepath)
+}
+
+/** Reject absolute or `..`-traversal globs (path-traversal guard). */
+export function validateGlob(pattern) {
+  if (isAbsolute(pattern)) return false
+  return !pattern.split('/').includes('..')
+}
+
+/**
+ * True when `dir` is a nested checkout — a git worktree, submodule or vendored clone — rather than
+ * part of THIS repo's tree. Such a directory carries its own `.git` entry (a FILE for a worktree, a
+ * DIRECTORY for a clone/submodule) and its files belong to a different commit, so folding them into
+ * the walk makes every walker lie: it makes a debt ratchet count another branch's TODO and go red on
+ * main, and a secret scan re-scan each worktree's copy of the whole tree (62,974 files vs 20,528 on
+ * one measured repo). `.git` itself is in SKIP_DIRS, so only the SIBLING content leaks — prune at the
+ * checkout boundary instead. `base` is exempt: the repo root has `.git` too. Takes the already-read
+ * `entries` (no extra stat per subdirectory).
+ */
+function isNestedCheckout(dir, base, entries) {
+  return dir !== base && entries.includes('.git')
+}
+
+/**
+ * Walk `root` collecting repo-relative POSIX file paths, pruning SKIP_DIRS.
+ * Unreadable dirs/entries are skipped silently (never throws).
+ */
+export function walkRepo(root) {
+  // Normalize a trailing slash once: a `dir/` root would make `slice(root.length+1)` drop an extra
+  // leading char and mangle every relative path (a determinism foot-gun for the glob check types).
+  const base = root.replace(/[/\\]+$/, '') || root
+  const files = []
+  // Track visited real directories by device:inode so a directory that resolves back to an
+  // ancestor (symlink OR hardlink/bind-mount cycle) terminates instead of looping forever / OOMing.
+  // Strictly stronger than the skip-symlink guard below — that prevents following a symlinked dir;
+  // this also catches a cycle reachable through real directory entries. #1521.
+  const visited = new Set()
+  const visit = (dir) => {
+    let dirStat
+    try {
+      dirStat = statSync(dir)
+    } catch {
+      return
+    }
+    const key = `${dirStat.dev}:${dirStat.ino}`
+    if (visited.has(key)) return
+    visited.add(key)
+    let entries
+    try {
+      entries = readdirSync(dir)
+    } catch {
+      return
+    }
+    if (isNestedCheckout(dir, base, entries)) return
+    for (const entry of entries) {
+      if (SKIP_DIRS.has(entry)) continue
+      const full = join(dir, entry)
+      let stat
+      try {
+        stat = lstatSync(full)
+      } catch {
+        continue
+      }
+      if (stat.isSymbolicLink()) {
+        // Record the symlink path (file/stat checks evaluate it via their own lstat) but NEVER
+        // recurse into it — a symlinked directory could cycle and stack-overflow the walk. #1471.
+        files.push(full.slice(base.length + 1).replace(/\\/g, '/'))
+        continue
+      }
+      if (stat.isDirectory()) {
+        visit(full)
+      } else {
+        files.push(full.slice(base.length + 1).replace(/\\/g, '/'))
+      }
+    }
+  }
+  visit(base)
+  return files
+}