mixdog 1.4.1 → 1.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mixdog",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "private": false,
   "type": "module",
   "description": "Open-source coding agent that matches Codex on Terminal-Bench 2.1 with 63% fewer tokens.",
@@ -53,6 +53,7 @@
     "src/",
     "!src/tui/dev",
     "!src/**/*.test.mjs",
+    "!src/**/*.test-hook.mjs",
     "!src/**/*.test.jsx"
   ],
   "scripts": {
@@ -182,7 +183,7 @@
     "wrap-ansi": "^10.0.0",
     "ws": "^8.21.0",
     "yaml": "^2.9.0",
-    "zod": "^3.25.76"
+    "zod": "^4.6.7"
   },
   "overrides": {
     "onnxruntime-node": {

package/scripts/build-shipped-history.mjs

@@ -13,7 +13,7 @@
  *       node scripts/build-shipped-history.mjs --check   (release workflow)
  * A shallow clone has no history: the committed file is left untouched.
  */
-import { execFileSync } from 'node:child_process';
+import { execFileSync, spawn } from 'node:child_process';
 import { readFileSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -39,12 +39,81 @@ if (git('rev-parse', '--is-shallow-repository').toString().trim() === 'true') {
   process.exit(0);
 }
 
-const blobs = new Map();
-const blob = (sha) => {
-  if (!blobs.has(sha)) blobs.set(sha, git('cat-file', 'blob', sha));
-  return blobs.get(sha);
+/**
+ * One long-lived `git cat-file --batch` answers every object read, so the whole
+ * history costs a single process. `read(spec)` resolves to {sha, type, body}
+ * or null when the object is missing.
+ */
+const catFile = spawn('git', ['cat-file', '--batch'], { cwd: ROOT, stdio: ['pipe', 'pipe', 'inherit'] });
+const waiting = [];
+let pending = Buffer.alloc(0);
+catFile.stdout.on('data', (chunk) => {
+  pending = Buffer.concat([pending, chunk]);
+  for (;;) {
+    const eol = pending.indexOf(10);
+    if (eol < 0) return;
+    const header = pending.toString('utf8', 0, eol);
+    if (header.endsWith(' missing')) {
+      pending = pending.subarray(eol + 1);
+      waiting.shift()(null);
+      continue;
+    }
+    const [sha, type, size] = header.split(' ');
+    const end = eol + 1 + Number(size);
+    if (pending.length < end + 1) return;
+    const body = pending.subarray(eol + 1, end);
+    pending = pending.subarray(end + 1);
+    waiting.shift()({ sha, type, body });
+  }
+});
+const read = (spec) =>
+  new Promise((resolve) => {
+    waiting.push(resolve);
+    catFile.stdin.write(`${spec}\n`);
+  });
+
+/** Entries of a tree object body: [{ name, tree, sha }] in git (ls-tree) order. */
+function parseTree(body) {
+  const entries = [];
+  for (let at = 0; at < body.length; ) {
+    const space = body.indexOf(32, at);
+    const nul = body.indexOf(0, space);
+    const mode = body.toString('latin1', at, space);
+    entries.push({
+      name: body.toString('utf8', space + 1, nul),
+      tree: mode === '40000',
+      sha: body.toString('hex', nul + 1, nul + 21),
+    });
+    at = nul + 21;
+  }
+  return entries;
+}
+
+const treeCache = new Map();
+const blobCache = new Map();
+const treeOf = async (sha) => {
+  if (!treeCache.has(sha)) treeCache.set(sha, read(sha).then(({ body }) => parseTree(body)));
+  return treeCache.get(sha);
+};
+const blobOf = async (sha) => {
+  if (!blobCache.has(sha)) blobCache.set(sha, read(sha).then(({ body }) => body));
+  return blobCache.get(sha);
 };
 
+/** Relative path -> blob sha for every file under a tree entry (a lone file keeps its own name). */
+async function filesOf({ name, tree, sha }) {
+  if (!tree) return new Map([[name, sha]]);
+  const files = new Map();
+  const walk = async (treeSha, base) => {
+    for (const entry of await treeOf(treeSha)) {
+      if (entry.tree) await walk(entry.sha, `${base}${entry.name}/`);
+      else files.set(`${base}${entry.name}`, entry.sha);
+    }
+  };
+  await walk(sha, '');
+  return files;
+}
+
 /** Names (path segment after `prefix`) that ever existed under prefix. */
 function everNames(prefix) {
   const names = new Set();
@@ -54,29 +123,39 @@ function everNames(prefix) {
   return [...names].sort();
 }
 
-/** Every version of `path`: Map of relative path -> blob sha, one per commit that touched it. */
-function versionsOf(path) {
-  const versions = [];
-  for (const commit of lines(git('log', '--format=%H', '--', path))) {
-    const files = new Map();
-    for (const row of lines(git('ls-tree', '-r', commit, '--', path))) {
-      const [meta, file] = row.split('\t');
-      files.set(file.slice(path.length + 1) || file.split('/').pop(), meta.split(' ')[2]);
+/**
+ * Every distinct state `prefix/<name>` had in the commits the path-limited
+ * `git log -- path` lists (its history simplification decides which commits
+ * count). Only the object reads are batched through `cat-file`.
+ */
+async function everVersions(prefix) {
+  const out = new Map();
+  for (const name of everNames(prefix)) {
+    const path = `${prefix}/${name}`;
+    const states = new Map();
+    for (const commit of lines(git('log', '--format=%H', '--', path))) {
+      const object = await read(`${commit}:${path}`);
+      if (object) states.set(object.sha, { name, tree: object.type === 'tree', sha: object.sha });
     }
-    if (files.size) versions.push(files);
+    const versions = [];
+    for (const entry of states.values()) {
+      const files = await filesOf(entry);
+      if (files.size) versions.push(files);
+    }
+    out.set(name, versions);
   }
-  return versions;
+  return out;
 }
-
 const unique = (hashes) => [...new Set(hashes)].sort();
 
-function packHistory(dir, entry) {
+async function packHistory(dir, entry) {
   const out = {};
-  for (const id of everNames(dir)) {
+  for (const [id, versions] of await everVersions(dir)) {
     const hashes = [];
-    for (const files of versionsOf(`${dir}/${id}`)) {
-      const read = (name) => (files.has(name) ? blob(files.get(name)).toString('utf8') : null);
-      const definition = packDefinitionFrom(read, entry, id);
+    for (const files of versions) {
+      const texts = new Map();
+      for (const [file, sha] of files) texts.set(file, (await blobOf(sha)).toString('utf8'));
+      const definition = packDefinitionFrom((name) => texts.get(name) ?? null, entry, id);
       if (definition) hashes.push(hashDefinition(definition));
     }
     if (hashes.length) out[id] = unique(hashes);
@@ -84,32 +163,35 @@ function packHistory(dir, entry) {
   return out;
 }
 
-function outputStyleHistory() {
+async function outputStyleHistory() {
   const out = {};
-  for (const name of everNames('src/output-styles')) {
-    const hashes = versionsOf(`src/output-styles/${name}`).map((files) => hashText(blob([...files.values()][0])));
+  for (const [name, versions] of await everVersions('src/output-styles')) {
+    const hashes = [];
+    for (const files of versions) hashes.push(hashText(await blobOf([...files.values()][0])));
     if (hashes.length) out[name] = unique(hashes);
   }
   return out;
 }
 
-function skillHistory() {
+async function skillHistory() {
   const out = {};
-  for (const name of everNames('src/defaults/skills')) {
-    const hashes = versionsOf(`src/defaults/skills/${name}`).map((files) =>
-      hashTree([...files].map(([rel, sha]) => [rel, blob(sha)]))
-    );
+  for (const [name, versions] of await everVersions('src/defaults/skills')) {
+    const hashes = [];
+    for (const files of versions) {
+      hashes.push(hashTree(await Promise.all([...files].map(async ([rel, sha]) => [rel, await blobOf(sha)]))));
+    }
     if (hashes.length) out[name] = unique(hashes);
   }
   return out;
 }
 
 const history = {
-  agents: packHistory('src/agents', 'AGENT.md'),
-  workflows: packHistory('src/workflows', 'WORKFLOW.md'),
-  outputStyles: outputStyleHistory(),
-  skills: skillHistory(),
+  agents: await packHistory('src/agents', 'AGENT.md'),
+  workflows: await packHistory('src/workflows', 'WORKFLOW.md'),
+  outputStyles: await outputStyleHistory(),
+  skills: await skillHistory(),
 };
+catFile.stdin.end();
 const text = `${JSON.stringify(history, null, 2)}\n`;
 if (CHECK) {
   let committed = '';

package/scripts/gate-local.mjs

@@ -1,12 +1,17 @@
 #!/usr/bin/env node
 // Local pre-push check. Mirrors the test legs of .github/workflows/release-gate.yml
-// (jobs: runtime, runtime-slow, desktop, desktop-tests fast/slow). It MUST be
-// updated together with that workflow whenever those commands change.
-// Up front, before any build or leg, it runs two fast source checks and stops
-// if either fails: the gate's shipped-history job (build-shipped-history.mjs
-// --check; needs a full clone, a shallow one keeps the committed file), and the
-// released-section changelog check the desktop fast lane asserts
-// (changelog-locales.test.mjs; Deploy's plan adds the Unreleased section).
+// (jobs: runtime, runtime-slow, desktop, desktop-tests fast/slow, shipped-history).
+// It MUST be updated together with that workflow whenever those commands change.
+// Up front it runs the released-section changelog check the desktop fast lane
+// asserts (changelog-locales.test.mjs; Deploy's plan adds the Unreleased
+// section) and stops if it fails. Everything else runs from one shared pool of
+// workers (scripts/lib/gate-scheduler.mjs): every test lane is cut into shards
+// of its own files (each run with the lane's own npm script command), and the
+// desktop build + daemon end-to-end test, the patch smoke and the gate's
+// shipped-history job (build-shipped-history.mjs --check; needs a full clone, a
+// shallow one keeps the committed file) are units of the same pool. A failed
+// shipped-history check stops new starts, so the gate still ends early on it.
+// The whole run is at below-normal priority so the desktop stays responsive.
 // Not covered locally: runtime-macos, computer-backend (cargo + Linux X11),
 // desktop-tests `computer` lane (windows-latest, src/main/computer/ only),
 // relay (apps/relay tests), graph (cargo test), patch (cargo test of the
@@ -15,23 +20,32 @@
 // unittest). The gate's path
 // filtering is not applied: every check and leg always runs. Run `npm ci`
 // (root and apps/desktop) first.
+// Usage: npm run gate:local [-- --test-concurrency=<workers>]
+// (MIXDOG_TEST_CONCURRENCY also sets the pool size; default: half the cores, at most 8).
 import { spawn, spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { gateTestBudget, runGateLegs } from './lib/gate-scheduler.mjs';
+import {
+  DEFAULT_ESTIMATE_MS,
+  gateTestBudget,
+  measuredFileTimes,
+  planShards,
+  readTimingHistory,
+  runPool,
+  saveTimingHistory,
+} from './lib/gate-scheduler.mjs';
 import { pruneStaleTempDirs, removeTempDirSync } from './lib/stale-temp.mjs';
 
 const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const started = Date.now();
+const budget = gateTestBudget(process.argv.slice(2));
+// Inherited by every process the gate starts (Windows and POSIX alike).
+os.setPriority(os.constants.priority.PRIORITY_BELOW_NORMAL);
 
-const preChecks = ['node scripts/build-shipped-history.mjs --check', 'node scripts/check-changelog-locales.mjs'];
-const failedPreChecks = preChecks.filter(
-  (cmd) => spawnSync(cmd, { cwd: root, stdio: 'inherit', shell: true }).status !== 0
-);
-if (failedPreChecks.length) {
-  console.error(`[gate:local] pre-check failed: ${failedPreChecks.join('; ')}`);
+if (spawnSync('node scripts/check-changelog-locales.mjs', { cwd: root, stdio: 'inherit', shell: true }).status !== 0) {
+  console.error('[gate:local] pre-check failed: node scripts/check-changelog-locales.mjs');
   process.exit(1);
 }
 
@@ -57,73 +71,170 @@ const RESOURCE_PREFIX = 'mixdog-gate-resources-';
 pruneStaleTempDirs([LOG_PREFIX, RESOURCE_PREFIX]);
 const logDir = fs.mkdtempSync(path.join(os.tmpdir(), LOG_PREFIX));
 const resourceDir = fs.mkdtempSync(path.join(os.tmpdir(), RESOURCE_PREFIX));
-const budget = gateTestBudget();
-// weight: relative worker share, roughly each leg's measured duration x its 3 workers
-// (runtime ~440s, runtime-slow ~545s, desktop fast ~390s, desktop slow ~280s).
-const legs = [
-  { name: 'runtime', weight: 3, cmds: ['npm test', 'npm run smoke:patch'] },
-  { name: 'desktop-tests-fast', weight: 3, cmds: ['npm test --prefix apps/desktop'] },
-  { name: 'runtime-slow', weight: 4, cmds: ['npm run test:slow'] },
-  { name: 'desktop-tests-slow', weight: 2, cmds: ['npm run test:slow --prefix apps/desktop'] },
-  { name: 'desktop', cmds: ['npm run build --prefix apps/desktop', 'npm run test:daemon:e2e --prefix apps/desktop'] },
-];
 
-function runCmd(cmd, logFd, concurrency) {
+// Measured times live in the repository's common git dir, so every worktree
+// and snapshot of the repo plans from the same history. Keys: a test file's
+// repo-relative path, or `cmd:` + a command unit's commands. Timings only
+// plan the run: without them (no git dir, unreadable or invalid history) the
+// gate plans from default estimates.
+const commonDir = spawnSync('git', ['rev-parse', '--path-format=absolute', '--git-common-dir'], {
+  cwd: root,
+  encoding: 'utf8',
+});
+const historyPath =
+  commonDir.status === 0 ? path.join(commonDir.stdout.trim(), 'mixdog-gate-timings.json') : null;
+if (!historyPath) console.warn(`[gate:local] warning: no git common dir, timings not kept: ${commonDir.stderr}`);
+const history = historyPath ? readTimingHistory(historyPath) : {};
+const measured = {};
+
+// Each test lane runs its package's own npm script command, given shard files
+// as filters (scripts/test.mjs), without npm or a shell between: a shell's
+// command line could not hold a shard's paths.
+const testLanes = [
+  { name: 'runtime', pkg: '.', script: 'test', speed: 'fast' },
+  { name: 'runtime-slow', pkg: '.', script: 'test:slow', speed: 'slow' },
+  { name: 'desktop-tests-fast', pkg: 'apps/desktop', script: 'test', speed: 'fast' },
+  { name: 'desktop-tests-slow', pkg: 'apps/desktop', script: 'test:slow', speed: 'slow' },
+].map((lane) => {
+  const dir = path.join(root, lane.pkg);
+  const command = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf8')).scripts[lane.script];
+  const [program, ...args] = command.split(/\s+/);
+  if (program !== 'node') throw new Error(`${lane.pkg}/package.json "${lane.script}" must be a node command: ${command}`);
+  const list = spawnSync(process.execPath, [...args, '--list'], { cwd: dir, encoding: 'utf8' });
+  if (list.status !== 0) throw new Error(`${command} --list failed: ${list.stderr}`);
+  const files = list.stdout
+    .split(/\r?\n/)
+    .filter(Boolean)
+    .map((line) => line.replace(/^\S+\s+/, ''));
+  const key = (file) => path.posix.join(lane.pkg, file);
+  return { ...lane, dir, args, files, key, estimateOf: (file) => history[key(file)] ?? DEFAULT_ESTIMATE_MS[lane.speed] };
+});
+const lanesByName = new Map(testLanes.map((lane) => [lane.name, lane]));
+const commandUnits = [
+  { lane: 'shipped-history', cmds: ['node scripts/build-shipped-history.mjs --check'], stopOnFail: true },
+  { lane: 'runtime', cmds: ['npm run smoke:patch'] },
+  // Only the daemon end-to-end test reads the build (apps/desktop/out); no
+  // test lane does, so the build shares the pool with them.
+  { lane: 'desktop', cmds: ['npm run build --prefix apps/desktop', 'npm run test:daemon:e2e --prefix apps/desktop'] },
+].map((unit) => {
+  const key = `cmd:${unit.cmds.join(' && ')}`;
+  return { ...unit, key, estimate: history[key] ?? DEFAULT_ESTIMATE_MS.command };
+});
+const units = [...planShards(testLanes, budget), ...commandUnits];
+const laneCounts = new Map();
+for (const unit of units) {
+  laneCounts.set(unit.lane, (laneCounts.get(unit.lane) ?? 0) + 1);
+  unit.id = `${unit.lane}.${laneCounts.get(unit.lane)}`;
+}
+const laneNames = [...laneCounts.keys()];
+const laneLog = (name) => path.join(logDir, `${name}.log`);
+
+// Every unit is one worker: a shard runs its files one at a time.
+const env = { ...process.env, MIXDOG_TEST_CONCURRENCY: '1', MIXDOG_TEST_OOXML_CACHE_DIR: resourceDir };
+function run(command, args, options, fd) {
   return new Promise((resolve) => {
-    fs.writeSync(logFd, `\n$ ${cmd}\n`);
-    const child = spawn(cmd, {
-      cwd: root,
-      shell: true,
-      windowsHide: true,
-      stdio: ['ignore', logFd, logFd],
-      env: {
-        ...process.env,
-        MIXDOG_TEST_CONCURRENCY: String(concurrency),
-        MIXDOG_TEST_OOXML_CACHE_DIR: resourceDir,
-      },
+    const child = spawn(command, args, { ...options, env, windowsHide: true, stdio: ['ignore', fd, fd] });
+    child.on('error', (error) => {
+      fs.writeSync(fd, `${error.stack}\n`);
+      resolve(1);
     });
-    child.on('error', () => resolve(1));
     child.on('close', (code) => resolve(code ?? 1));
   });
 }
 
-async function runLeg(leg, concurrency) {
-  const t0 = Date.now();
-  leg.log = path.join(logDir, `${leg.name}.log`);
-  const fd = fs.openSync(leg.log, 'w');
-  leg.ok = true;
-  for (const cmd of leg.cmds) {
-    if ((await runCmd(cmd, fd, concurrency)) !== 0) {
-      leg.ok = false;
+const TIMING_REPORTER = new URL('./lib/gate-timing-reporter.mjs', import.meta.url).href;
+async function runShard(unit, fd) {
+  const lane = lanesByName.get(unit.lane);
+  const timings = path.join(logDir, `${unit.id}.timings.jsonl`);
+  const args = [...lane.args, `--test-reporter=${TIMING_REPORTER}`, `--test-reporter-destination=${timings}`];
+  fs.writeSync(fd, `\n$ (${lane.pkg}) node ${[...args, ...unit.files].join(' ')}\n`);
+  const code = await run(process.execPath, [...args, ...unit.files], { cwd: lane.dir }, fd);
+  const reported = Object.fromEntries(
+    (fs.existsSync(timings) ? fs.readFileSync(timings, 'utf8') : '')
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => {
+        const { file, ms } = JSON.parse(line);
+        return [file, ms];
+      })
+  );
+  fs.rmSync(timings, { force: true });
+  for (const [file, ms] of Object.entries(measuredFileTimes(unit.files, reported, Date.now() - unit.start))) {
+    measured[lane.key(file)] = Math.round(ms);
+  }
+  return code === 0;
+}
+
+async function runCommands(unit, fd) {
+  let ok = true;
+  for (const cmd of unit.cmds) {
+    fs.writeSync(fd, `\n$ ${cmd}\n`);
+    if ((await run(cmd, [], { cwd: root, shell: true }, fd)) !== 0) {
+      ok = false;
       break;
     }
   }
-  fs.closeSync(fd);
-  leg.ms = Date.now() - t0;
+  measured[unit.key] = Date.now() - unit.start;
+  return ok;
 }
 
-console.log(`[gate:local] build separately, then tests sharing ${budget} workers; logs: ${logDir}`);
+let finished = 0;
+async function runUnit(unit) {
+  const part = path.join(logDir, `${unit.id}.part.log`);
+  const fd = fs.openSync(part, 'w');
+  unit.start = Date.now();
+  let ok = false;
+  try {
+    ok = await (unit.files ? runShard(unit, fd) : runCommands(unit, fd));
+  } finally {
+    fs.closeSync(fd);
+    unit.end = Date.now();
+    fs.appendFileSync(laneLog(unit.lane), fs.readFileSync(part));
+    fs.rmSync(part);
+  }
+  console.log(
+    `[gate:local] ${ok ? 'ok  ' : 'FAIL'} ${unit.id} ${((unit.end - unit.start) / 1000).toFixed(1)}s (${++finished}/${units.length})`
+  );
+  return ok;
+}
+
+console.log(
+  `[gate:local] ${units.length} units on ${budget} workers at below-normal priority; timings: ${historyPath}; logs: ${logDir}`
+);
+let notStarted = [];
 try {
-  // TypeScript/bundling must not compete with four independent test runners.
-  await runLeg(legs.at(-1), budget);
-  await runGateLegs(legs.slice(0, -1), budget, runLeg);
+  notStarted = await runPool(units, budget, runUnit);
 } finally {
   removeTempDirSync(resourceDir);
+  // The next run plans from these; work this run did not reach keeps its old time.
+  const keys = [...testLanes.flatMap((lane) => lane.files.map(lane.key)), ...commandUnits.map((unit) => unit.key)];
+  const next = Object.fromEntries(
+    keys.filter((key) => (measured[key] ?? history[key]) !== undefined).map((key) => [key, measured[key] ?? history[key]])
+  );
+  const saveError = historyPath && saveTimingHistory(historyPath, next);
+  if (saveError) console.warn(`[gate:local] warning: timings not saved (${saveError.message})`);
 }
 
 console.log('\n=== gate:local summary ===');
-for (const leg of legs) {
-  console.log(`${leg.ok ? 'PASS' : 'FAIL'}  ${leg.name}  ${(leg.ms / 1000).toFixed(1)}s  ${leg.log}`);
-  if (!leg.ok) {
-    const lines = fs
-      .readFileSync(leg.log, 'utf8')
-      .split(/\r?\n/)
-      .filter((l) => l.includes('[failure-summary]'));
-    for (const l of lines) console.log(`    ${l}`);
-  }
+for (const name of laneNames) {
+  const laneUnits = units.filter((unit) => unit.lane === name);
+  const ran = laneUnits.filter((unit) => unit.end);
+  const status = ran.some((unit) => !unit.ok) ? 'FAIL' : ran.length < laneUnits.length ? 'NOT RUN' : 'PASS';
+  const span = ran.length ? Math.max(...ran.map((u) => u.end)) - Math.min(...ran.map((u) => u.start)) : 0;
+  const work = ran.reduce((sum, unit) => sum + unit.end - unit.start, 0);
+  console.log(
+    `${status}  ${name}  ${(span / 1000).toFixed(1)}s (${(work / 1000).toFixed(1)}s worker time, ${ran.length}/${laneUnits.length} units)  ${laneLog(name)}`
+  );
+  if (status !== 'FAIL') continue;
+  const lines = fs
+    .readFileSync(laneLog(name), 'utf8')
+    .split(/\r?\n/)
+    .filter((l) => /\[failure-summary\] (?:RUN FAILED|HUNG|FAILED|FLAKY)/.test(l));
+  for (const l of lines) console.log(`    ${l}`);
 }
+if (notStarted.length) console.log(`Stopped after a failed ${units.find((u) => u.stopOnFail && u.ok === false).lane} check: ${notStarted.length} unit(s) not run`);
 console.log(
   'Not covered locally: runtime-macos, computer-backend, desktop-tests computer lane (Windows-only), relay, graph, patch, browser-import, mobile-scripts'
 );
 console.log(`Total: ${((Date.now() - started) / 1000).toFixed(1)}s`);
-process.exit(legs.every((l) => l.ok) ? 0 : 1);
+process.exit(units.every((unit) => unit.ok) ? 0 : 1);