@autor3search/javascript 0.2.0

package/src/doctor.js

@@ -0,0 +1,282 @@
+/**
+ * Checks whether this machine can measure benchmarks reliably.
+ *
+ * Every check is INFORMATIONAL. Numbers are only as good as the machine
+ * producing them, and a thermally throttled laptop on battery produces noise
+ * dressed as data — but whether that is acceptable is the human's call, not
+ * the harness's, so doctor reports and never blocks.
+ */
+import { execFile } from 'node:child_process'
+import { readFile, stat, statfs } from 'node:fs/promises'
+import { cpus, loadavg, platform } from 'node:os'
+import { resolve as resolvePath } from 'node:path'
+import { promisify } from 'node:util'
+import * as gitx from './gitx.js'
+import { vitestBin } from './adapters/bench/vitest.js'
+
+const exec = promisify(execFile)
+
+export const SEVERITY = { OK: 0, WARN: 1, FAIL: 2, NA: -1 }
+
+/** The Node floor: parseArgs, module.register and --heap-prof all need it. */
+const MIN_NODE_MAJOR = 20
+
+/**
+ * Runs every check. Each one is individually guarded, so an unfamiliar
+ * platform or a missing tool degrades one line rather than the whole report.
+ *
+ * @param {string} dir the repository root, or a directory inside it
+ * @returns {Promise<{name: string, detail: string, severity: number}[]>}
+ */
+export async function check(dir) {
+  // `-C` defaults to '.', so every check is given an ABSOLUTE directory once,
+  // up front: the git checks shell out with it as cwd, the disk check statfs's
+  // it, and the vitest check reports the path it looked in — a relative one
+  // would make that message meaningless. Resolving here keeps every check
+  // consistent regardless of what the caller passed.
+  const absDir = resolvePath(dir)
+
+  const platformCheck =
+    platform() === 'darwin'
+      ? ['power', checkDarwin]
+      : platform() === 'linux'
+        ? ['cpu governor', checkLinux]
+        : platform() === 'win32'
+          ? ['platform', checkWindows]
+          : null
+
+  const checks = [
+    ['node', checkNode],
+    ['git', checkGit],
+    ['git repo', () => checkGitRepo(absDir)],
+    ['cpu', checkCpu],
+    ['load', checkLoad],
+    ['vitest', () => checkVitest(absDir)],
+    ['heap hint', checkHeapHint],
+    platformCheck,
+    ['disk', () => checkDisk(absDir)],
+  ].filter(Boolean)
+
+  const findings = []
+  for (const [name, fn] of checks) {
+    try {
+      findings.push(await fn())
+    } catch (err) {
+      findings.push({ name, detail: `check failed: ${err.message}`, severity: SEVERITY.NA })
+    }
+  }
+  return findings
+}
+
+function checkNode() {
+  const major = Number(process.versions.node.split('.')[0])
+  return major >= MIN_NODE_MAJOR
+    ? { name: 'node', detail: `node ${process.versions.node}`, severity: SEVERITY.OK }
+    : {
+        name: 'node',
+        detail: `node ${process.versions.node} is too old, need >= ${MIN_NODE_MAJOR}`,
+        severity: SEVERITY.FAIL,
+      }
+}
+
+async function checkGit() {
+  try {
+    const { stdout } = await exec('git', ['--version'])
+    return { name: 'git', detail: stdout.trim(), severity: SEVERITY.OK }
+  } catch {
+    return { name: 'git', detail: 'git not found on PATH', severity: SEVERITY.FAIL }
+  }
+}
+
+async function checkGitRepo(dir) {
+  try {
+    return { name: 'git repo', detail: await gitx.root(dir), severity: SEVERITY.OK }
+  } catch {
+    return { name: 'git repo', detail: `${dir} is not inside a git repository`, severity: SEVERITY.FAIL }
+  }
+}
+
+function checkCpu() {
+  const n = cpus().length
+  return {
+    name: 'cpu',
+    detail: `${n} logical core(s) — ${cpus()[0]?.model ?? 'unknown model'}`,
+    severity: n >= 2 ? SEVERITY.OK : SEVERITY.WARN,
+  }
+}
+
+function checkLoad() {
+  const [one] = loadavg()
+  const perCore = one / Math.max(1, cpus().length)
+  return {
+    name: 'load',
+    detail:
+      `1-minute load ${one.toFixed(2)} (${perCore.toFixed(2)} per core)` +
+      (perCore > 0.5 ? ' — other work on this machine will show up as measurement noise' : ''),
+    severity: perCore > 0.5 ? SEVERITY.WARN : SEVERITY.OK,
+  }
+}
+
+/**
+ * Resolves vitest from the MEASURED repository, not this harness's own
+ * node_modules — a repo with no vitest installed must not look fine just
+ * because the harness happens to depend on it for its own tests.
+ */
+async function checkVitest(dir) {
+  // Check the exact file the adapter spawns, not a module resolution of it.
+  // require.resolve('vitest/vitest.mjs') honours the package `exports` map,
+  // and Vitest 2 exposes a `./*` wildcard there while Vitest 3 and 4 do not —
+  // so the resolution answered "not installed" for every Vitest 3/4 repository
+  // even though the file is present and benchmarks measure fine. doctor exists
+  // to predict whether measurement will work, so it must ask what measurement
+  // asks.
+  const bin = vitestBin(dir)
+  const present = await stat(bin).then(
+    (st) => st.isFile(),
+    () => false,
+  )
+  return present
+    ? { name: 'vitest', detail: 'vitest resolves in this repository', severity: SEVERITY.OK }
+    : {
+        name: 'vitest',
+        detail: `vitest is not installed here — benchmarks cannot be measured until it is (looked for ${bin})`,
+        severity: SEVERITY.WARN,
+      }
+}
+
+/**
+ * Windows is not a supported platform, and the reason is the stop path.
+ *
+ * 448 of 458 tests pass there, so most of the harness works — but Node cannot
+ * deliver SIGINT to a child process group on Windows the way it does on POSIX,
+ * so an interrupted `eval` does not reach the ABORTED path: it exits with a
+ * null code instead of 2, and the run may leave a claim behind. An unattended
+ * harness that cannot be reliably stopped is the wrong thing to be quiet about,
+ * so this says it up front rather than at 3am.
+ *
+ * WSL reports linux and is unaffected.
+ */
+function checkWindows() {
+  return {
+    name: 'platform',
+    detail:
+      'Windows is not a supported platform: an interrupted eval does not reach the ABORTED path, so a run ' +
+      'may not stop cleanly and can leave its claim behind. Measurement itself works. Use WSL for a supported setup.',
+    severity: SEVERITY.WARN,
+  }
+}
+
+function checkHeapHint() {
+  // Reported up front so a run does not discover mid-flight that the hint is
+  // unavailable, which would otherwise look like the benchmark allocating
+  // nothing. The driver always spawns its own child with --expose-gc, so
+  // this is OK regardless of whether gc() happens to be exposed in THIS
+  // process — but say so explicitly rather than implying it was probed here.
+  return {
+    name: 'heap hint',
+    detail: 'the driver spawns its own --expose-gc child, so the bytes/op hint should be available',
+    severity: SEVERITY.OK,
+  }
+}
+
+/**
+ * Runs a command and returns its trimmed stdout, or null if it failed —
+ * missing binary, non-zero exit, or unrecognized flag. Every macOS-specific
+ * probe below tolerates null so one absent tool degrades one note, not the
+ * whole finding.
+ */
+async function tryExec(cmd, args) {
+  try {
+    const { stdout } = await exec(cmd, args)
+    return stdout
+  } catch {
+    return null
+  }
+}
+
+async function checkDarwin() {
+  const notes = []
+  let severity = SEVERITY.OK
+
+  const batt = await tryExec('pmset', ['-g', 'batt'])
+  if (batt === null) {
+    notes.push('pmset unavailable — power source unknown')
+  } else if (/Battery Power/.test(batt)) {
+    notes.push('running on battery — macOS clocks down aggressively')
+    severity = SEVERITY.WARN
+  } else {
+    notes.push('on AC power')
+  }
+
+  const gen = await tryExec('pmset', ['-g'])
+  if (gen === null) {
+    notes.push('Low Power Mode unknown')
+  } else {
+    const m = gen.match(/lowpowermode\s+(\d+)/i)
+    if (m && m[1] !== '0') {
+      notes.push('Low Power Mode is on')
+      severity = SEVERITY.WARN
+    }
+  }
+
+  // `machdep.xcpm.cpu_thermal_level` is Intel-only (xcpm = Intel's power
+  // manager) and does not exist on Apple Silicon, where sysctl exits
+  // non-zero with "unknown oid". `pmset -g therm` works on both, but macOS
+  // only starts recording these fields once a thermal EVENT has actually
+  // occurred since boot — a fresh boot legitimately prints
+  // "No thermal warning level has been recorded", which means "nothing to
+  // report", not "unknown". Anything else unrecognized is reported as such
+  // rather than silently assumed fine.
+  const therm = await tryExec('pmset', ['-g', 'therm'])
+  if (therm === null) {
+    notes.push('thermal state unknown (pmset -g therm unavailable)')
+  } else {
+    const limit = therm.match(/CPU_Speed_Limit\s*=\s*(\d+)/)
+    if (limit) {
+      const pct = Number(limit[1])
+      if (pct < 100) {
+        notes.push(`CPU speed limited to ${pct}% by thermal pressure`)
+        severity = SEVERITY.WARN
+      } else {
+        notes.push('no thermal throttling')
+      }
+    } else if (/No thermal warning level has been recorded/i.test(therm)) {
+      notes.push('no thermal event recorded since boot')
+    } else {
+      notes.push('thermal state format not recognized on this machine')
+    }
+  }
+
+  notes.push('P/E core scheduling makes macOS numbers jump; a quiet Linux box gives cleaner results')
+  return { name: 'power', detail: notes.join('; '), severity }
+}
+
+async function checkLinux() {
+  const governor = await readFile('/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor', 'utf8').catch(() => null)
+  const noTurbo = await readFile('/sys/devices/system/cpu/intel_pstate/no_turbo', 'utf8').catch(() => null)
+  const notes = []
+  let severity = SEVERITY.OK
+  if (governor === null) {
+    notes.push('cpufreq governor not readable')
+  } else if (governor.trim() !== 'performance') {
+    notes.push(`cpufreq governor is "${governor.trim()}" — "performance" gives steadier numbers`)
+    severity = SEVERITY.WARN
+  } else {
+    notes.push('cpufreq governor is "performance"')
+  }
+  if (noTurbo !== null && noTurbo.trim() === '0') {
+    notes.push('turbo boost is enabled — clock varies with temperature')
+    severity = SEVERITY.WARN
+  }
+  return { name: 'cpu governor', detail: notes.join('; '), severity }
+}
+
+async function checkDisk(dir) {
+  const stats = await statfs(dir)
+  const freeGb = (stats.bavail * stats.bsize) / 1024 ** 3
+  return {
+    name: 'disk',
+    detail: `${freeGb.toFixed(1)} GB free`,
+    severity: freeGb < 2 ? SEVERITY.WARN : SEVERITY.OK,
+  }
+}

package/src/duration.js

@@ -0,0 +1,75 @@
+/**
+ * Duration parsing, used for `timeout` in the run configuration. Values are
+ * written as a number and a unit ("1s", "500ms", "15m"), optionally
+ * compounded ("1m30s") — the notation most performance tooling already uses,
+ * so there is nothing new to learn here.
+ */
+
+/** Milliseconds per unit. */
+const UNITS = {
+  ns: 1e-6,
+  us: 1e-3,
+  µs: 1e-3,
+  ms: 1,
+  s: 1000,
+  m: 60_000,
+  h: 3_600_000,
+}
+
+/** One `<number><unit>` term, e.g. "1h", "30.5s". */
+const TERM = /(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)/gy
+
+/**
+ * Parses a duration into milliseconds. Fractional values and compound terms
+ * ("1m30s") are both accepted; a negative or unitless value is not.
+ *
+ * @param {string} s
+ * @returns {number} milliseconds
+ * @throws {Error} when s is not a duration
+ */
+export function parseDuration(s) {
+  const bad = () => new Error(`${JSON.stringify(s)} is not a duration (want e.g. "500ms", "1s", "15m")`)
+  if (typeof s !== 'string' || s.length === 0) throw bad()
+
+  TERM.lastIndex = 0
+  let total = 0
+  let terms = 0
+  let lastIndex = 0
+  let match
+  while ((match = TERM.exec(s)) !== null) {
+    total += Number(match[1]) * UNITS[match[2]]
+    terms++
+    lastIndex = TERM.lastIndex
+  }
+  // The sticky flag anchors each term to the end of the previous one, so a
+  // full match means lastIndex reached the end of the string. Anything left
+  // over — a leading sign, a stray unit, trailing text — means this was not a
+  // duration, and must not be silently accepted as its parseable prefix.
+  if (terms === 0 || lastIndex !== s.length) throw bad()
+  return total
+}
+
+/**
+ * Renders milliseconds back into the same notation parseDuration accepts.
+ * Used only for human-facing output.
+ *
+ * @param {number} ms
+ * @returns {string}
+ */
+export function formatDuration(ms) {
+  if (ms < 1000) return `${round(ms)}ms`
+  const totalSeconds = ms / 1000
+  const hours = Math.floor(totalSeconds / 3600)
+  const minutes = Math.floor((totalSeconds % 3600) / 60)
+  const seconds = round(totalSeconds % 60)
+  let out = ''
+  if (hours) out += `${hours}h`
+  if (minutes) out += `${minutes}m`
+  if (seconds || out === '') out += `${seconds}s`
+  return out
+}
+
+/** Rounds to at most three decimal places, dropping a trailing ".0". */
+function round(n) {
+  return Number(n.toFixed(3))
+}

package/src/freeze.js

@@ -0,0 +1,234 @@
+/**
+ * Snapshots test and benchmark files at baseline time and restores them
+ * before every evaluation, so an agent cannot weaken its own success
+ * criteria — or rewrite the benchmark that defines the metric.
+ *
+ * Every path constant here is relative to the run's OUT-OF-TREE state
+ * directory, never to the repository root. The frozen store and its manifest
+ * are part of what the score depends on, so they must live where the agent
+ * being measured cannot reach them; a caller joining these onto the
+ * repository root would silently reintroduce that hole.
+ */
+import { createHash } from 'node:crypto'
+import { lstat, mkdir, readFile, writeFile } from 'node:fs/promises'
+import { dirname, join, posix, isAbsolute } from 'node:path'
+
+export const STORE_DIR = 'frozen'
+export const MANIFEST_PATH = 'frozen/manifest.json'
+
+/** `.code` on an error raised by a symlink anywhere along a frozen path. */
+export const ERR_SYMLINK = 'A3S_SYMLINK'
+
+/** `.code` on an error raised when a golden copy no longer matches its hash. */
+export const ERR_STORE_TAMPERED = 'A3S_STORE_TAMPERED'
+
+/**
+ * `.code` on an error raised by a HARD link at a frozen path.
+ *
+ * A hard link defeats the symlink defence entirely: `lstat().isSymbolicLink()`
+ * is false for one, so the path looks like an ordinary regular file. Writing
+ * to it writes to every other name for that inode — so an agent can point a
+ * frozen path at any file it may write and have `restore` clobber it with
+ * test source. That is an uncontrolled write outside the repository, which is
+ * exactly what this module exists to prevent.
+ */
+export const ERR_HARD_LINK = 'A3S_HARD_LINK'
+
+/**
+ * Copies each file into storeDir and records its hash.
+ *
+ * @param {string} repoRoot
+ * @param {string} storeDir
+ * @param {string[]} files repo-relative paths
+ * @returns {Promise<{files: Record<string, string>}>}
+ */
+export async function snapshot(repoRoot, storeDir, files) {
+  const manifest = { files: {} }
+  for (const rel of [...files].sort()) {
+    const src = safeJoin(repoRoot, rel, 'snapshot')
+    await refuseSymlink(repoRoot, rel, 'snapshot', 'in the repository')
+    // The store is harness-owned, but a directory left behind by an earlier
+    // attempt under the same tag is not necessarily pristine: refuse to write
+    // the golden copy through a link there either.
+    await refuseSymlink(storeDir, rel, 'snapshot', 'inside the frozen store')
+
+    const content = await readFile(src)
+    const dst = safeJoin(storeDir, rel, 'snapshot')
+    await mkdir(dirname(dst), { recursive: true })
+    await refuseHardLink(dst, rel, 'snapshot', 'inside the frozen store')
+    await writeFile(dst, content)
+    manifest.files[rel] = hash(content)
+  }
+  return manifest
+}
+
+/**
+ * Rewrites every frozen file in the working tree from the store, recreating
+ * files the agent deleted. Returns the paths it changed, sorted.
+ *
+ * Both sides are checked against the hash recorded at baseline, and the
+ * working tree is examined BEFORE the store is read. That ordering is what
+ * makes the common case — an eval where the agent touched no frozen file —
+ * cost one read per file instead of two: the destination already hashes to
+ * the manifest value, so the golden copy is never opened at all.
+ *
+ * @returns {Promise<string[]>}
+ */
+export async function restore(repoRoot, storeDir, manifest) {
+  const changed = []
+  for (const rel of sortedPaths(manifest)) {
+    // Before any read or write: readFile follows links exactly as writeFile
+    // does, so this has to come first, or the check would read THROUGH a link
+    // and conclude the file was fine.
+    await refuseSymlink(repoRoot, rel, 'restore', 'in the repository')
+    const dst = safeJoin(repoRoot, rel, 'restore')
+
+    const current = await readFile(dst).catch(() => null)
+    if (current && hash(current) === manifest.files[rel]) continue
+
+    await refuseSymlink(storeDir, rel, 'restore', 'inside the frozen store')
+    const src = safeJoin(storeDir, rel, 'restore')
+    const golden = await readFile(src)
+    const got = hash(golden)
+    if (got !== manifest.files[rel]) {
+      throw tagged(
+        `restore ${rel}: frozen store copy does not match the hash recorded at baseline ` +
+          `(store hashes to ${got}, manifest records ${manifest.files[rel]})`,
+        ERR_STORE_TAMPERED,
+      )
+    }
+
+    await mkdir(dirname(dst), { recursive: true })
+    await refuseHardLink(dst, rel, 'restore', 'in the repository')
+    await writeFile(dst, golden)
+    changed.push(rel)
+  }
+  return changed
+}
+
+/**
+ * Reports which frozen files currently differ from the baseline. A deleted
+ * file counts as changed, and so does one reached through a symlink — that is
+ * at least as suspicious as a deletion, and is reported rather than followed.
+ *
+ * @returns {Promise<string[]>}
+ */
+export async function verify(repoRoot, manifest) {
+  const changed = []
+  for (const rel of sortedPaths(manifest)) {
+    const linked = await symlinkComponent(repoRoot, rel)
+    if (linked) {
+      changed.push(rel)
+      continue
+    }
+    const content = await readFile(safeJoin(repoRoot, rel, 'verify')).catch(() => null)
+    if (content === null || hash(content) !== manifest.files[rel]) changed.push(rel)
+  }
+  return changed
+}
+
+/** Writes the manifest as indented JSON. */
+export async function saveManifest(path, manifest) {
+  await mkdir(dirname(path), { recursive: true })
+  await writeFile(path, `${JSON.stringify(manifest, null, 2)}\n`)
+}
+
+/** Reads a manifest written by saveManifest. */
+export async function loadManifest(path) {
+  let text
+  try {
+    text = await readFile(path, 'utf8')
+  } catch (err) {
+    throw new Error(`read manifest ${path}: ${err.message}`, { cause: err })
+  }
+  let parsed
+  try {
+    parsed = JSON.parse(text)
+  } catch (err) {
+    throw new Error(`parse manifest ${path}: ${err.message}`, { cause: err })
+  }
+  return { files: parsed.files ?? {} }
+}
+
+const sortedPaths = (manifest) => Object.keys(manifest.files).sort()
+
+const hash = (buffer) => createHash('sha256').update(buffer).digest('hex')
+
+/**
+ * Joins rel onto root, rejecting anything that would escape it. Manifest
+ * entries come from a JSON file on disk, so they are untrusted input:
+ * `restore` writes through them before every evaluation.
+ */
+function safeJoin(root, rel, op) {
+  if (isAbsolute(rel)) throw new Error(`${op} ${rel}: frozen path must be relative`)
+  const clean = posix.normalize(rel.split('\\').join('/'))
+  if (clean === '..' || clean.startsWith('../')) {
+    throw new Error(`${op} ${rel}: frozen path escapes the repository root`)
+  }
+  return join(root, clean)
+}
+
+/**
+ * Throws when any component of rel, beneath root, is a symlink.
+ *
+ * Checking only the FINAL component is not enough, and that is the hole this
+ * closes: readFile and writeFile resolve the whole path, so replacing a
+ * parent DIRECTORY with a link redirects the write just as effectively as
+ * replacing the file itself, landing the frozen content outside the
+ * repository. An lstat on the file then reports a perfectly ordinary regular
+ * file, because the link was already followed to get there.
+ *
+ * `root` itself is deliberately not examined: a repository legitimately
+ * reached through a symlinked ancestor — macOS's /tmp, a home directory on a
+ * linked volume — is not tampering, and refusing to work there would break
+ * ordinary setups.
+ */
+async function refuseSymlink(root, rel, op, where) {
+  const linked = await symlinkComponent(root, rel)
+  if (!linked) return
+  throw tagged(
+    `${op} ${rel}: ${linked} is a symlink ${where}; refusing to read or write through it, ` +
+      `which could reach a file outside the repository`,
+    ERR_SYMLINK,
+  )
+}
+
+/** The first component of rel, beneath root, that is a symlink — or null. */
+async function symlinkComponent(root, rel) {
+  const parts = posix.normalize(rel.split('\\').join('/')).split('/').filter((p) => p !== '' && p !== '.')
+  let path = root
+  for (const [i, part] of parts.entries()) {
+    path = join(path, part)
+    const stats = await lstat(path).catch(() => null)
+    if (stats?.isSymbolicLink()) return parts.slice(0, i + 1).join('/')
+  }
+  return null
+}
+
+/**
+ * Throws when the destination is a hard link — a file with more than one name.
+ *
+ * Checked immediately before every write, on both the working tree and the
+ * store. A regular file in a checkout has exactly one link; more than one
+ * means writing here also writes somewhere else, potentially outside the
+ * repository. Only an existing regular file is examined: a path that does not
+ * exist yet cannot be linked, and a directory's link count is unrelated.
+ */
+async function refuseHardLink(path, rel, op, where) {
+  const stats = await lstat(path).catch(() => null)
+  if (stats?.isFile() && stats.nlink > 1) {
+    throw tagged(
+      `${op} ${rel}: ${rel} has ${stats.nlink} names (a hard link) ${where}; refusing to write through ` +
+        `it, because that would also overwrite the other name(s) for this file — possibly outside the ` +
+        `repository. Replace it with a regular file and rerun.`,
+      ERR_HARD_LINK,
+    )
+  }
+}
+
+/** Builds an Error carrying a stable `.code` the pipeline branches on. */
+function tagged(message, code) {
+  const err = new Error(message)
+  err.code = code
+  return err
+}