@opengsd/gsd-core 1.7.0 → 1.8.0

package/scripts/gen-test-timings.cjs

@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+// Regenerate the per-file test timing table used to weight chunk packing in
+// scripts/run-tests.cjs (issue #2456).
+//
+// The chunk packer needs to know what each test file actually COSTS. Before
+// #2456 it guessed from the filename (`^(?:install|codex-)` scored 12x,
+// everything else 1) and was wrong in both directions — installer-migration-
+// authoring.test.cjs scored 12x but runs ~0.1s, while the two heaviest files in
+// the suite (run-tests-harness.test.cjs and release-tarball-smoke.install
+// .test.cjs) both scored 1. This script replaces the guess with measurement.
+//
+// Input is one or more node:test reporter event streams as emitted by
+// `gsd-test` (`~/.local/state/gsd-test/runs/<run-id>/test-events-<os>-node<v>
+// .jsonl`). Each stream carries one `test:summary` event per test FILE, whose
+// `data.duration_ms` is that file's total wall-clock and whose `data.file` is
+// its absolute in-container path.
+//
+// Usage:
+//   node scripts/gen-test-timings.cjs <events.jsonl> [<events.jsonl> ...]
+//   node scripts/gen-test-timings.cjs ~/.local/state/gsd-test/runs/*/test-events-*.jsonl
+//   node scripts/gen-test-timings.cjs events.jsonl --out tests/test-timings.json
+//
+// When several streams are supplied (multiple lanes, e.g. node22 + node24), a
+// file's recorded time is the MAX across them, not the mean: the packer exists
+// to keep the SLOWEST lane's slowest chunk away from the per-chunk timeout, so
+// the conservative bound is the right one to balance against.
+//
+// The table is ADVISORY and deliberately un-gated — there is no `--check` mode
+// and no CI lint that fails on staleness, because timing data legitimately
+// varies run to run. A file missing from the table falls back to the table's
+// median weight, so a stale table degrades chunk BALANCE gracefully instead of
+// failing the build. Regenerate it when the suite's cost profile has visibly
+// drifted, not on a schedule.
+'use strict';
+
+const fs = require('fs');
+const { basename, dirname, join } = require('path');
+const { ExitError, runMain } = require('./lib/cli-exit.cjs');
+
+const DEFAULT_OUT = join(__dirname, '..', 'tests', 'test-timings.json');
+const SCHEMA_VERSION = 1;
+
+function parseArgs(argv) {
+  const inputs = [];
+  let out = DEFAULT_OUT;
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === '--out') {
+      const value = argv[++i];
+      if (!value) return { error: '--out requires a path' };
+      out = value;
+    } else if (arg.startsWith('--out=')) {
+      const value = arg.slice('--out='.length);
+      if (!value) return { error: '--out requires a path' };
+      out = value;
+    } else if (arg.startsWith('-')) {
+      return { error: `unknown flag "${arg}"` };
+    } else {
+      inputs.push(arg);
+    }
+  }
+  if (inputs.length === 0) {
+    return { error: 'usage: gen-test-timings.cjs <reporter-events.jsonl> [...] [--out <path>]' };
+  }
+  return { inputs, out };
+}
+
+// Fold one reporter event stream into `acc`, keeping the MAX duration seen for
+// each test file. Returns per-stream counters plus any basename collisions
+// found WITHIN this stream.
+//
+// Keying is by BASENAME, matching how run-tests.cjs weights a selected file:
+// the reporter reports absolute in-container paths (/work/tests/foo.test.cjs)
+// while the harness carries paths relative to its test dir, so the basename is
+// the only stable join key between the two. A basename seen in two different
+// directories would make the table ambiguous, so it must fail loudly.
+//
+// Collision detection is scoped to a SINGLE stream deliberately. Every lane
+// writes its own stream under its own container root (`/work/tests` on Linux,
+// `C:/work/tests` on Windows), so comparing directories ACROSS streams reports
+// every shared basename as a collision — which is the script's own documented
+// usage (globbing `test-events-*.jsonl` across lanes). Within one stream the
+// root is constant, so a differing directory is a real collision.
+function foldStream(text, acc) {
+  const dirsByBase = new Map();
+  let files = 0;
+  let malformed = 0;
+  for (const line of text.split('\n')) {
+    if (line.trim() === '') continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      malformed++;
+      continue;
+    }
+    if (!event || event.type !== 'test:summary') continue;
+    const data = event.data;
+    if (!data || typeof data.file !== 'string') continue;
+    const ms = data.duration_ms;
+    if (typeof ms !== 'number' || !Number.isFinite(ms) || ms < 0) continue;
+    const path = data.file.replace(/\\/g, '/');
+    const base = basename(path);
+    if (!dirsByBase.has(base)) dirsByBase.set(base, new Set());
+    dirsByBase.get(base).add(dirname(path));
+    const prev = acc.get(base);
+    if (prev === undefined || ms > prev) acc.set(base, ms);
+    files++;
+  }
+  const collisions = [...dirsByBase.entries()]
+    .filter(([, dirs]) => dirs.size > 1)
+    .map(([base, dirs]) => `${base} (${[...dirs].sort().join(', ')})`);
+  return { files, malformed, collisions };
+}
+
+function main() {
+  const parsed = parseArgs(process.argv.slice(2));
+  if (parsed.error) throw new ExitError(2, `gen-test-timings: ${parsed.error}`);
+
+  const acc = new Map();
+  const allCollisions = new Set();
+  const sources = [];
+  for (const input of parsed.inputs) {
+    let text;
+    try {
+      text = fs.readFileSync(input, 'utf8');
+    } catch (err) {
+      throw new ExitError(2, `gen-test-timings: cannot read "${input}": ${err.message}`);
+    }
+    const { files, malformed, collisions } = foldStream(text, acc);
+    for (const c of collisions) allCollisions.add(c);
+    sources.push(basename(input));
+    console.error(
+      `gen-test-timings: ${basename(input)} — ${files} file summaries` +
+        (malformed > 0 ? `, ${malformed} unparseable lines skipped` : ''),
+    );
+  }
+
+  if (acc.size === 0) {
+    throw new ExitError(
+      2,
+      'gen-test-timings: no `test:summary` events with a file and duration_ms were found. ' +
+        'Check that the input is a node:test reporter event stream (test-events-<os>-node<v>.jsonl).',
+    );
+  }
+
+  // A basename that resolves to two different directories within one lane makes
+  // the table ambiguous: run-tests.cjs joins on basename alone, so one file's
+  // measured cost would silently be applied to the other. Fail rather than emit
+  // a table that lies.
+  if (allCollisions.size > 0) {
+    throw new ExitError(
+      2,
+      `gen-test-timings: basename collision — the table cannot key on basename alone:\n  ${[...allCollisions].sort().join('\n  ')}`,
+    );
+  }
+
+  // Every key must be a plain test-file basename. This is a data-integrity
+  // check on a stream we do not control (the reporter emits whatever path the
+  // runner saw), and it structurally excludes a computed key like `__proto__`
+  // or `constructor` from being written into the table object below — the
+  // `js/prototype-polluting-assignment` shape, even though the value here is
+  // always a number and could not actually pollute.
+  const SAFE_BASENAME_RE = /^[A-Za-z0-9._-]+\.test\.cjs$/;
+  const rejected = [...acc.keys()].filter((base) => !SAFE_BASENAME_RE.test(base));
+  if (rejected.length > 0) {
+    throw new ExitError(
+      2,
+      `gen-test-timings: refusing to emit non-test-file keys: ${rejected.sort().join(', ')}`,
+    );
+  }
+
+  // Sorted keys keep the checked-in diff reviewable: a regeneration shows only
+  // the files whose cost actually moved, not a reshuffled object.
+  const timings = Object.create(null);
+  for (const base of [...acc.keys()].sort()) {
+    timings[base] = Math.round(acc.get(base));
+  }
+
+  const payload = {
+    schema_version: SCHEMA_VERSION,
+    generated_by: 'scripts/gen-test-timings.cjs',
+    unit: 'ms',
+    sources: sources.sort(),
+    file_count: acc.size,
+    timings,
+  };
+
+  fs.writeFileSync(parsed.out, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+  const totalMs = [...acc.values()].reduce((sum, ms) => sum + ms, 0);
+  console.error(
+    `gen-test-timings: wrote ${parsed.out} — ${acc.size} files, ${(totalMs / 1000).toFixed(1)}s total`,
+  );
+  return 0;
+}
+
+if (require.main === module) {
+  runMain(main);
+}
+
+module.exports = { parseArgs, foldStream };

package/scripts/lint-portable-timeout.cjs

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * lint-portable-timeout.cjs — ban hardcoded GNU-`timeout` in gsd
+ * workflow / agent / reference / command markdown (#2351).
+ *
+ * ## Why
+ *
+ * `timeout` and `gtimeout` are GNU coreutils. Stock macOS ships NEITHER
+ * (`brew install coreutils` only provides `gtimeout`, and only if installed).
+ * A hardcoded `timeout <n> <cmd>` inside an executable workflow snippet exits
+ * 127 ("command not found") on such a host, and the gate that runs it — which
+ * only distinguishes 0 (pass) / 124 (timeout) / other (fail) — misreports a
+ * perfectly good build or test command as a FAILURE (#2351).
+ *
+ * The portable, coreutils-independent replacement is the
+ * `gsd_run run-with-timeout <secs> [--] <cmd…>` verb (gsd-core/bin/gsd-tools.cjs):
+ * a Node-based wall-clock cap that keeps GNU `timeout`'s exit-code contract
+ * (124 on timeout) on every platform. The resolution lives there ONCE and is
+ * reused by every call site instead of a per-file `command -v timeout` probe.
+ *
+ * This ratchet fails the build if a NEW bare `timeout`/`gtimeout` execution
+ * slips into any of these surfaces.
+ *
+ * ## What PASSES
+ *
+ * - `gsd_run run-with-timeout 300 -- bash -c "$CMD"` — the approved verb.
+ * - `command -v timeout` / `command -v gtimeout` / `which timeout` capability
+ *   PROBES — portable: they detect the binary, they do not unconditionally
+ *   execute it (see gsd-core/workflows/review.md's `_AGY_KILLER` fallback).
+ * - Prose ("timed out after 5 minutes"), config keys
+ *   (`workflow.test_gate_timeout`), CI `timeout-minutes:`, the agy
+ *   `--print-timeout` flag, `$TIMEOUT`-style variable names — none of which is
+ *   a bare timeout command invocation.
+ *
+ * ## What FAILS
+ *
+ * A bare `timeout <duration> …` / `gtimeout <duration> …` command invocation,
+ * where `<duration>` is a number, a `"$VAR"`, or a `${VAR}` (optionally with a
+ * leading `-k`/`-s` option).
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { ExitError, runMain } = require('./lib/cli-exit.cjs');
+
+const ROOT = path.join(__dirname, '..');
+
+// Surfaces whose markdown carries agent-executed bash. Kept broad so the guard
+// catches a regression anywhere a workflow snippet could bound a command.
+const DEFAULT_ROOTS = ['gsd-core/workflows', 'gsd-core/references', 'agents', 'commands'];
+
+// Capability probes to strip BEFORE testing for an invocation, so a portable
+// `command -v timeout` on the same line is never mistaken for a bare execution.
+const PROBE_RE = /command\s+-v\s+g?timeout|which\s+g?timeout/g;
+
+// A `timeout`/`gtimeout` token INVOKED AS A COMMAND with a duration argument.
+// Anchored to a command position — line start or right after `| & ; ( ` {` — so
+// prose ("increase the timeout 30 seconds") and the `--print-timeout` flag / the
+// approved `run-with-timeout` verb (no separator before "timeout") never match.
+// Between the token and the duration, allow any number of leading options in
+// short OR long form (`-k5`, `-k 5`, `--kill-after=5`, `--foreground`,
+// `--signal=KILL`). The duration is a digit, `$((arith))`, a `$VAR`, or `${VAR}`.
+const EXEC_RE = /(?:^|[|&;(`{])[ \t]*g?timeout[ \t]+(?:-{1,2}[\w-]+(?:=\S+)?[ \t]+)*["']?(?:\$\{?[A-Za-z_]|\$\(\(|\d)/;
+
+/**
+ * Locate bare `timeout`/`gtimeout` invocations in a block of text.
+ *
+ * Pure (no I/O): callers pass the file contents; the caller reads files. Returns
+ * structured findings so tests assert on typed values, never on grepped text.
+ *
+ * @param {string} text  file contents
+ * @returns {{ line: number, snippet: string }[]}  findings (empty array = clean)
+ */
+function findRawTimeoutInvocations(text) {
+  const findings = [];
+  const lines = String(text).split(/\r?\n/);
+  for (let i = 0; i < lines.length; i += 1) {
+    const stripped = lines[i].replace(PROBE_RE, '');
+    if (EXEC_RE.test(stripped)) findings.push({ line: i + 1, snippet: lines[i].trim() });
+  }
+  return findings;
+}
+
+function walkMarkdown(dir) {
+  const out = [];
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return out; // a missing root is not an error — some surfaces are optional
+  }
+  for (const entry of entries) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) out.push(...walkMarkdown(full));
+    else if (entry.isFile() && entry.name.endsWith('.md')) out.push(full);
+  }
+  return out;
+}
+
+/**
+ * Scan the given roots (repo-relative) for bare timeout invocations.
+ * @param {string[]} roots
+ * @returns {{ file: string, line: number, snippet: string }[]}
+ */
+function scan(roots = DEFAULT_ROOTS) {
+  const offenders = [];
+  for (const rel of roots) {
+    const abs = path.isAbsolute(rel) ? rel : path.join(ROOT, rel);
+    for (const file of walkMarkdown(abs)) {
+      const findings = findRawTimeoutInvocations(fs.readFileSync(file, 'utf8'));
+      for (const f of findings) {
+        offenders.push({ file: path.relative(ROOT, file), line: f.line, snippet: f.snippet });
+      }
+    }
+  }
+  return offenders;
+}
+
+function main() {
+  const rootsEnv = process.env.GSD_LINT_PORTABLE_TIMEOUT_ROOTS;
+  const roots = rootsEnv ? rootsEnv.split(path.delimiter).filter(Boolean) : DEFAULT_ROOTS;
+  const offenders = scan(roots);
+  if (offenders.length > 0) {
+    const detail = offenders.map((o) => `  ${o.file}:${o.line}  ${o.snippet}`).join('\n');
+    throw new ExitError(
+      1,
+      'lint-portable-timeout: hardcoded `timeout`/`gtimeout` is not portable — stock\n' +
+        'macOS ships no coreutils, so these exit 127 and misreport a passing command as a\n' +
+        'failure. Use `gsd_run run-with-timeout <secs> [--] <cmd…>` instead (#2351):\n' +
+        detail,
+    );
+  }
+  console.log(`ok lint-portable-timeout: no hardcoded timeout invocations in ${roots.length} root(s)`);
+}
+
+module.exports = { findRawTimeoutInvocations, scan, DEFAULT_ROOTS };
+
+if (require.main === module) runMain(main);

package/scripts/lint-test-file-count.allowlist.json

@@ -60,6 +60,7 @@
     "state": {
       "files": [
         "state-acquirestatelock-non-eexist.test.cjs",
+        "state-command-cutover.test.cjs",
         "state-prune.test.cjs",
         "state-rebuild-cli.test.cjs",
         "state-rebuild.test.cjs",

package/scripts/release-tarball-smoke.cjs

@@ -47,16 +47,23 @@ const os = require('os');
 const path = require('path');
 const { PACKAGE_NAME } = require('../gsd-core/bin/lib/package-identity.cjs');
 const { ExitError, runMain } = require('./lib/cli-exit.cjs');
-// 120 s proved too tight on Windows GitHub-hosted runners: cold-cache
-// `npm install -g` with a 1499-file tarball took ~120 s exactly, causing
-// spawnSync to fire SIGTERM and return { status: null, stdout: '', stderr: '' }
-// (Node docs: status is null when subprocess terminated due to a signal).
-// The INSTALL_FAILED branch checks `status !== 0`, which null satisfies, so the
-// test saw empty stdout/stderr and a spurious INSTALL_FAILED. Windows runners
-// are slower than Linux/macOS for filesystem-heavy operations (
-// https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
-// ). Raise to 600 s (the same ceiling the before() helper uses for pack+install).
-const CHILD_TIMEOUT_MS = process.platform === 'win32' ? 600_000 : 120_000;
+// 120 s proved too tight for cold-cache `npm install -g` of a 1499-file tarball:
+// spawnSync fires SIGTERM at the deadline and returns { status: null, stdout: '',
+// stderr: '' } (Node docs: status is null when a subprocess is terminated by a
+// signal). The INSTALL_FAILED branch checks `status !== 0`, which null satisfies,
+// so the test sees empty stdout/stderr and a spurious INSTALL_FAILED (surfaced as
+// `installError: spawnSync npm ETIMEDOUT`).
+//
+// First observed on Windows GitHub-hosted runners, which are slower for
+// filesystem-heavy work
+// (https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories),
+// then again on a slow Linux bench (cartographer: cold disk, constrained CPU) —
+// the same failure the before() helper's SLOW_HOST_TIMEOUT already guards for
+// pack+install. The slow-host reality is not platform-specific, so the ceiling is
+// now uniform 600 s across all platforms AND shared with before() via this
+// exported constant, so the two surfaces cannot diverge again. 600 s stays well
+// clear of a real cold install (3–6 min) without masking a genuine hang.
+const CHILD_TIMEOUT_MS = 600_000;
 const QUIET_NPM_ENV = Object.freeze({
   npm_config_loglevel: 'error',
   npm_config_update_notifier: 'false',
@@ -628,7 +635,7 @@ function cleanup(...dirs) {
 // Exports
 // ---------------------------------------------------------------------------
 
-module.exports = { SMOKE, runSmoke, binInvocation };
+module.exports = { SMOKE, runSmoke, binInvocation, CHILD_TIMEOUT_MS };
 
 if (require.main === module) {
   runMain(cliMain);