@astryxdesign/cli 0.7.2 → 0.7.3-canary.06872b4

package/foundation/runtime/package-set.test.mjs

@@ -0,0 +1,432 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Matching a project to a host's package sets (architecture:cli-surface
+ * INV30): one root per run, the first that qualifies in the host's order, the
+ * versions a project would install, nothing filled that ships an integration
+ * manifest, and the integrations a host approves for use outside a project.
+ */
+
+import {describe, it, expect, afterEach} from 'vitest';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {
+  cleanupFixtures,
+  project,
+  tempDir,
+  writeJson,
+} from '../../test-utils/release-fixtures.mjs';
+import {projectAstryxPins} from './locked-versions.mjs';
+import {qualifyingRoot, rootPackage, satisfiesInstall} from './package-set.mjs';
+import {scanComputerIntegrations} from '../integrations/computer-integrations.mjs';
+import {findCoreDir} from '../fs/paths.mjs';
+import {
+  _resetHostedRun,
+  activateHostedRun,
+  recordResolved,
+  recordResolvedIntegrations,
+  resolvedPackages,
+} from './package-root.mjs';
+
+afterEach(cleanupFixtures);
+
+/**
+ * Write a package into `dir`'s node_modules.
+ * @param {string} dir
+ * @param {string} name
+ * @param {Record<string, unknown>} pkg
+ * @param {{manifest?: boolean, bin?: boolean, hostEntry?: boolean}} [options]
+ * @returns {string} the package folder
+ */
+function install(dir, name, pkg, options = {}) {
+  const pkgDir = path.join(dir, 'node_modules', ...name.split('/'));
+  writeJson(path.join(pkgDir, 'package.json'), {
+    name,
+    ...pkg,
+    ...(options.bin ? {bin: {astryx: './bin.cjs'}} : {}),
+  });
+  if (options.bin) fs.writeFileSync(path.join(pkgDir, 'bin.cjs'), '');
+  if (options.hostEntry) {
+    fs.mkdirSync(path.join(pkgDir, 'host'), {recursive: true});
+    fs.writeFileSync(path.join(pkgDir, 'host', 'index.mjs'), '');
+  }
+  if (options.manifest) {
+    fs.writeFileSync(
+      path.join(pkgDir, 'astryx.integration.mjs'),
+      'export default {};\n',
+    );
+  }
+  return pkgDir;
+}
+
+/**
+ * A package set: a root holding a CLI and Core of `version`.
+ * @param {string} version
+ * @param {{hostEntry?: boolean, bin?: boolean}} [options]
+ * @returns {string}
+ */
+function packageSet(version, options = {}) {
+  const root = tempDir('set');
+  install(
+    root,
+    '@astryxdesign/cli',
+    {version},
+    {bin: options.bin ?? true, hostEntry: options.hostEntry},
+  );
+  install(root, '@astryxdesign/core', {version});
+  return root;
+}
+
+describe('the versions a project would install', () => {
+  it('reads each declared @astryxdesign/* package with its lockfile and installed versions', () => {
+    const root = project(tempDir(), {
+      '@astryxdesign/core': '^0.7.0',
+      '@astryxdesign/cli': '0.7.0',
+      react: '^19.0.0',
+    });
+    install(root, '@astryxdesign/cli', {version: '0.7.0'});
+    writeJson(path.join(root, 'package-lock.json'), {
+      lockfileVersion: 3,
+      packages: {
+        '': {},
+        'node_modules/@astryxdesign/core': {version: '0.7.1'},
+        'node_modules/@astryxdesign/cli': {version: '0.7.0'},
+      },
+    });
+    expect(projectAstryxPins(root)).toEqual([
+      {
+        name: '@astryxdesign/core',
+        declared: '^0.7.0',
+        locked: '0.7.1',
+        installed: null,
+      },
+      {
+        name: '@astryxdesign/cli',
+        declared: '0.7.0',
+        locked: '0.7.0',
+        installed: '0.7.0',
+      },
+    ]);
+  });
+});
+
+describe('a version the project would install for a range', () => {
+  it.each([
+    ['0.7.1', '^0.7.0', true],
+    ['0.8.0', '^0.7.0', false],
+    ['0.7.1-canary.1', '^0.7.0', false],
+    ['0.7.1-canary.1', '^0.7.1-canary.0', true],
+    ['0.7.1', 'file:../core', false],
+    ['0.7.1', 'workspace:*', false],
+    [null, '^0.7.0', false],
+  ])('%s for %s: %s', (version, range, expected) => {
+    expect(satisfiesInstall(version, range)).toBe(expected);
+  });
+});
+
+describe('the package set that serves a run another release answers', () => {
+  it('serves from the first set whose CLI and Core are that release', () => {
+    const older = packageSet('0.6.8');
+    const first = packageSet('0.7.0');
+    const second = packageSet('0.7.0');
+    const root = project(tempDir(), {'@astryxdesign/core': '0.7.0'});
+    const served = qualifyingRoot({
+      release: '0.7.0',
+      projectDir: root,
+      roots: [older, first, second],
+    });
+    expect(served).toMatchObject({root: first, hostEntry: null});
+    expect(/** @type {any} */ (served).bin).toBe(
+      path.join(first, 'node_modules', '@astryxdesign', 'cli', 'bin.cjs'),
+    );
+  });
+
+  it('skips a set whose Core is another release, or missing', () => {
+    const mixed = tempDir('set');
+    install(mixed, '@astryxdesign/cli', {version: '0.7.0'}, {bin: true});
+    install(mixed, '@astryxdesign/core', {version: '0.7.1'});
+    const bare = tempDir('set');
+    install(bare, '@astryxdesign/cli', {version: '0.7.0'}, {bin: true});
+    const root = project(tempDir(), {'@astryxdesign/core': '0.7.0'});
+    expect(
+      qualifyingRoot({
+        release: '0.7.0',
+        projectDir: root,
+        roots: [mixed, bare],
+      }),
+    ).toEqual({
+      reason:
+        "no package set the host passed holds CLI 0.7.0 with this project's packages",
+    });
+  });
+
+  it("holds the lockfile's version of each package it fills, else one in the declared range", () => {
+    const locked = project(tempDir(), {
+      '@astryxdesign/core': '^0.7.0',
+      '@astryxdesign/charts': '^0.7.0',
+    });
+    writeJson(path.join(locked, 'package-lock.json'), {
+      lockfileVersion: 3,
+      packages: {
+        '': {},
+        'node_modules/@astryxdesign/core': {version: '0.7.0'},
+        'node_modules/@astryxdesign/charts': {version: '0.7.0'},
+      },
+    });
+    const newer = packageSet('0.7.0');
+    install(newer, '@astryxdesign/charts', {version: '0.7.2'});
+    const exact = packageSet('0.7.0');
+    install(exact, '@astryxdesign/charts', {version: '0.7.0'});
+    expect(
+      qualifyingRoot({
+        release: '0.7.0',
+        projectDir: locked,
+        roots: [newer, exact],
+      }),
+    ).toMatchObject({root: exact});
+
+    const ranged = project(tempDir(), {
+      '@astryxdesign/core': '^0.7.0',
+      '@astryxdesign/charts': '^0.7.0',
+    });
+    expect(
+      qualifyingRoot({release: '0.7.0', projectDir: ranged, roots: [newer]}),
+    ).toMatchObject({root: newer});
+  });
+
+  it('keeps out a set whose copy the project declares as a path and has not locked', () => {
+    const root = project(tempDir(), {
+      '@astryxdesign/core': '0.7.0',
+      '@astryxdesign/charts': 'file:../charts',
+    });
+    const set = packageSet('0.7.0');
+    install(set, '@astryxdesign/charts', {version: '0.7.0'});
+    expect(
+      qualifyingRoot({release: '0.7.0', projectDir: root, roots: [set]}),
+    ).toHaveProperty('reason');
+  });
+
+  it('never fills an @astryxdesign/* integration, so its version never keeps a set out', () => {
+    const root = project(tempDir(), {
+      '@astryxdesign/core': '0.7.0',
+      '@astryxdesign/richtext': '^0.9.0',
+    });
+    const set = packageSet('0.7.0');
+    install(
+      set,
+      '@astryxdesign/richtext',
+      {version: '0.7.0'},
+      {manifest: true},
+    );
+    expect(
+      qualifyingRoot({release: '0.7.0', projectDir: root, roots: [set]}),
+    ).toMatchObject({root: set});
+  });
+
+  it('fills nothing the project installed', () => {
+    const root = project(tempDir(), {
+      '@astryxdesign/core': '0.7.0',
+      '@astryxdesign/charts': '^0.6.0',
+    });
+    install(root, '@astryxdesign/charts', {version: '0.6.3'});
+    const set = packageSet('0.7.0');
+    install(set, '@astryxdesign/charts', {version: '0.7.0'});
+    expect(
+      qualifyingRoot({release: '0.7.0', projectDir: root, roots: [set]}),
+    ).toMatchObject({root: set});
+  });
+
+  it('needs a host entry when asked to', () => {
+    const plain = packageSet('0.7.0');
+    const hosted = packageSet('0.7.0', {hostEntry: true});
+    const root = project(tempDir(), {'@astryxdesign/core': '0.7.0'});
+    expect(
+      qualifyingRoot({
+        release: '0.7.0',
+        projectDir: root,
+        roots: [plain, hosted],
+        needsHostEntry: true,
+      }),
+    ).toMatchObject({
+      root: hosted,
+      hostEntry: path.join(
+        hosted,
+        'node_modules',
+        '@astryxdesign',
+        'cli',
+        'host',
+        'index.mjs',
+      ),
+    });
+  });
+
+  it("reads a root's own node_modules only", () => {
+    const outer = tempDir('outer');
+    install(outer, '@astryxdesign/core', {version: '0.7.0'});
+    const set = path.join(outer, 'set');
+    install(set, '@astryxdesign/cli', {version: '0.7.0'}, {bin: true});
+    expect(rootPackage(set, '@astryxdesign/core')).toBeNull();
+    expect(
+      qualifyingRoot({release: '0.7.0', projectDir: null, roots: [set]}),
+    ).toHaveProperty('reason');
+  });
+});
+
+describe('the integrations a host approves', () => {
+  it('load by name from the package set that holds the CLI, with the same checks', () => {
+    const set = tempDir('set');
+    const cliDir = install(set, '@astryxdesign/cli', {version: '0.7.2'});
+    // The set declares only the CLI and Core, so FR12's scan finds nothing.
+    writeJson(path.join(set, 'package.json'), {
+      dependencies: {'@astryxdesign/cli': '0.7.2'},
+    });
+    install(set, 'acme-widgets', {version: '1.0.0'}, {manifest: true});
+    install(
+      set,
+      'acme-old',
+      {version: '1.0.0', peerDependencies: {'@astryxdesign/core': '^0.6.0'}},
+      {manifest: true},
+    );
+    install(set, 'acme-plain', {version: '1.0.0'});
+    expect(
+      scanComputerIntegrations({cliRoot: cliDir, release: '0.7.2'}),
+    ).toEqual([]);
+    const found = scanComputerIntegrations({
+      cliRoot: cliDir,
+      release: '0.7.2',
+      approved: ['acme-widgets', 'acme-old', 'acme-plain', 'acme-missing'],
+    });
+    expect(found.map(c => [c.name, c.incompatible])).toEqual([
+      ['acme-widgets', null],
+      [
+        'acme-old',
+        "it needs @astryxdesign/core ^0.6.0, and this computer's CLI is 0.7.2",
+      ],
+    ]);
+    expect(found[0].packageDir).toBe(
+      path.join(set, 'node_modules', 'acme-widgets'),
+    );
+  });
+
+  it('never takes an approved name from a folder above the package set', () => {
+    const outer = tempDir('outer');
+    install(outer, 'acme-outer', {version: '1.0.0'}, {manifest: true});
+    const set = path.join(outer, 'set');
+    const cliDir = install(set, '@astryxdesign/cli', {version: '0.7.2'});
+    writeJson(path.join(set, 'package.json'), {
+      dependencies: {'@astryxdesign/cli': '0.7.2'},
+    });
+    expect(
+      scanComputerIntegrations({
+        cliRoot: cliDir,
+        release: '0.7.2',
+        approved: ['acme-outer'],
+      }),
+    ).toEqual([]);
+  });
+});
+
+describe('where each package a hosted run resolved came from', () => {
+  afterEach(() => _resetHostedRun());
+
+  /**
+   * @param {string} root
+   * @param {string} name
+   */
+  const at = (root, name) =>
+    path.join(root, 'node_modules', ...name.split('/'));
+
+  it("names the project's packages as the project's when the host's CLI is installed in the project", () => {
+    const app = tempDir('app');
+    activateHostedRun({startingRoot: app, projectRoot: app});
+    recordResolved('@astryxdesign/cli', '0.7.2', at(app, '@astryxdesign/cli'));
+    recordResolved(
+      '@astryxdesign/core',
+      '0.7.2',
+      at(app, '@astryxdesign/core'),
+      'project',
+    );
+    recordResolvedIntegrations([
+      {
+        name: 'acme-widgets',
+        version: '1.0.0',
+        __packageDir: at(app, 'acme-widgets'),
+      },
+    ]);
+    expect(resolvedPackages()).toEqual([
+      {name: '@astryxdesign/cli', version: '0.7.2', source: 'project'},
+      {name: '@astryxdesign/core', version: '0.7.2', source: 'project'},
+      {name: 'acme-widgets', version: '1.0.0', source: 'project'},
+    ]);
+  });
+
+  it("names what a package set holds as the package set's, and the project's integrations as the project's", () => {
+    const set = tempDir('set');
+    const app = tempDir('app');
+    activateHostedRun({startingRoot: set, projectRoot: app});
+    recordResolved('@astryxdesign/cli', '0.7.2', at(set, '@astryxdesign/cli'));
+    recordResolved(
+      '@astryxdesign/core',
+      '0.7.2',
+      at(set, '@astryxdesign/core'),
+    );
+    recordResolvedIntegrations([
+      {
+        name: 'acme-tools',
+        version: '2.0.0',
+        __packageDir: at(set, 'acme-tools'),
+        __computer: true,
+      },
+      {
+        name: 'acme-widgets',
+        version: '1.0.0',
+        __packageDir: at(app, 'acme-widgets'),
+      },
+    ]);
+    expect(resolvedPackages()).toEqual([
+      {name: '@astryxdesign/cli', version: '0.7.2', source: 'package-set'},
+      {name: '@astryxdesign/core', version: '0.7.2', source: 'package-set'},
+      {name: 'acme-tools', version: '2.0.0', source: 'package-set'},
+      {name: 'acme-widgets', version: '1.0.0', source: 'project'},
+    ]);
+  });
+
+  it("names what the project loaded or found above itself, as a workspace hoists it, as the project's", () => {
+    // The host's CLI sits in the workspace root, which is its package set,
+    // and the project is a workspace package whose packages are hoisted there.
+    const mono = tempDir('mono');
+    writeJson(path.join(mono, 'package.json'), {
+      private: true,
+      workspaces: ['packages/*'],
+    });
+    const app = path.join(mono, 'packages', 'app');
+    writeJson(path.join(app, 'package.json'), {
+      name: 'app',
+      dependencies: {'@astryxdesign/core': '0.7.2', 'acme-widgets': '1.0.0'},
+    });
+    install(mono, '@astryxdesign/core', {version: '0.7.2'});
+    activateHostedRun({startingRoot: mono, projectRoot: app});
+    expect(findCoreDir(app)).toBe(at(mono, '@astryxdesign/core'));
+    recordResolvedIntegrations([
+      {
+        name: 'acme-widgets',
+        version: '1.0.0',
+        __packageDir: at(mono, 'acme-widgets'),
+      },
+    ]);
+    expect(resolvedPackages()).toEqual([
+      {name: '@astryxdesign/core', version: '0.7.2', source: 'project'},
+      {name: 'acme-widgets', version: '1.0.0', source: 'project'},
+    ]);
+  });
+
+  it('names a package set inside the project folder as a package set', () => {
+    const app = tempDir('app');
+    const set = path.join(app, '.tools', 'astryx');
+    activateHostedRun({startingRoot: set, projectRoot: app});
+    recordResolved('@astryxdesign/cli', '0.7.2', at(set, '@astryxdesign/cli'));
+    expect(resolvedPackages()).toEqual([
+      {name: '@astryxdesign/cli', version: '0.7.2', source: 'package-set'},
+    ]);
+  });
+});

package/host/door-conformance.test.mjs

@@ -0,0 +1,274 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Door conformance (spec AST-081 FR1): every command example, run
+ * through the bin and through the host entry, gives the same exit status,
+ * output, and file changes, in an empty folder, a project on the current
+ * minor, and a project on the previous minor.
+ *
+ * Each run is a child process with its own copy of the folder, its own home
+ * folder, and its own per-user cache. The package registry is unreachable and
+ * notices are off, so no run reaches the network. Output is compared once each
+ * run's own folders read as placeholders. Neither door is given a command
+ * name, so FR5's command names are the same through both, and no line FR5 or
+ * FR6 lets differ prints.
+ */
+
+import {spawn} from 'node:child_process';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import {fileURLToPath, pathToFileURL} from 'node:url';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const CLI_ROOT = path.resolve(HERE, '..');
+const CORE_ROOT = path.resolve(CLI_ROOT, '../core');
+const BIN = path.join(CLI_ROOT, 'clients', 'cli', 'bin', 'astryx.mjs');
+const HOST_URL = pathToFileURL(path.join(HERE, 'index.mjs')).href;
+const COMMANDS = path.join(CLI_ROOT, 'clients', 'cli', 'commands');
+const VERSION = JSON.parse(
+  fs.readFileSync(path.join(CLI_ROOT, 'package.json'), 'utf8'),
+).version;
+const [MAJOR, MINOR] = VERSION.split('.').map(Number);
+const PREVIOUS_MINOR = `${MAJOR}.${Math.max(MINOR - 1, 0)}.0`;
+/** How long one run may take before it counts as hung. */
+const RUN_TIMEOUT_MS = 90_000;
+
+/** Split an example's command line the way a shell would, quotes included. */
+function splitArgs(line) {
+  const out = [];
+  let current = '';
+  let quote = null;
+  let started = false;
+  for (const ch of line) {
+    if (quote) {
+      if (ch === quote) quote = null;
+      else current += ch;
+    } else if (ch === '"' || ch === "'") {
+      quote = ch;
+      started = true;
+    } else if (/\s/.test(ch)) {
+      if (current || started) out.push(current);
+      current = '';
+      started = false;
+    } else {
+      current += ch;
+    }
+  }
+  if (current || started) out.push(current);
+  return out;
+}
+
+/** @returns {Promise<Array<{cli: string, argv: string[]}>>} every distinct command example */
+async function commandExamples() {
+  /** @type {Map<string, string[]>} */
+  const byLine = new Map();
+  for (const name of fs.readdirSync(COMMANDS).sort()) {
+    if (!name.endsWith('.doc.mjs')) continue;
+    const {doc} = await import(pathToFileURL(path.join(COMMANDS, name)).href);
+    if (doc?.type !== 'command') continue;
+    for (const example of doc.examples ?? []) {
+      const words = splitArgs(example.cli);
+      byLine.set(example.cli, words[0] === 'astryx' ? words.slice(1) : words);
+    }
+  }
+  return [...byLine].map(([cli, argv]) => ({cli, argv}));
+}
+
+/**
+ * @param {string} file
+ * @param {unknown} value
+ */
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), {recursive: true});
+  fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n');
+}
+
+/** The folders every example runs in, each set up fresh for each run. */
+const FOLDERS = {
+  'an empty folder': () => {},
+  /** @param {string} dir */
+  'a project on the current minor': dir => {
+    writeJson(path.join(dir, 'package.json'), {
+      name: 'app',
+      private: true,
+      dependencies: {'@astryxdesign/core': VERSION},
+    });
+    const scope = path.join(dir, 'node_modules', '@astryxdesign');
+    fs.mkdirSync(scope, {recursive: true});
+    fs.symlinkSync(CORE_ROOT, path.join(scope, 'core'), 'dir');
+  },
+  /** @param {string} dir */
+  'a project on the previous minor': dir => {
+    writeJson(path.join(dir, 'package.json'), {
+      name: 'app',
+      private: true,
+      dependencies: {'@astryxdesign/core': PREVIOUS_MINOR},
+    });
+    writeJson(
+      path.join(dir, 'node_modules', '@astryxdesign', 'core', 'package.json'),
+      {name: '@astryxdesign/core', version: PREVIOUS_MINOR},
+    );
+  },
+};
+
+/** @type {string[]} */
+const bases = [];
+
+/**
+ * Every file under `root`, skipping node_modules, by path relative to `base`.
+ * @param {string} root
+ * @param {string} base
+ * @param {Map<string, string>} into
+ */
+function readTree(root, base, into = new Map()) {
+  if (!fs.existsSync(root)) return into;
+  for (const entry of fs.readdirSync(root, {withFileTypes: true})) {
+    const file = path.join(root, entry.name);
+    if (entry.isDirectory() && entry.name !== 'node_modules') {
+      readTree(file, base, into);
+    } else if (entry.isFile()) {
+      into.set(path.relative(base, file), fs.readFileSync(file, 'utf8'));
+    }
+  }
+  return into;
+}
+
+/**
+ * Run one example through one door in a fresh copy of a folder.
+ * @param {'bin' | 'host entry'} door
+ * @param {string[]} argv
+ * @param {(dir: string) => void} setUp
+ * @returns {Promise<{status: number | null, stdout: string, stderr: string, changes: Record<string, string | null>}>}
+ */
+async function runThrough(door, argv, setUp) {
+  const base = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-door-'));
+  bases.push(base);
+  const dir = path.join(base, 'app');
+  const home = path.join(base, 'home');
+  fs.mkdirSync(dir);
+  fs.mkdirSync(home);
+  setUp(dir);
+  const read = () => readTree(home, base, readTree(dir, base));
+  const before = read();
+  const args =
+    door === 'bin'
+      ? [BIN, ...argv]
+      : [
+          '--input-type=module',
+          '-e',
+          `import {runHostedCli} from ${JSON.stringify(HOST_URL)};\n` +
+            `const result = await runHostedCli({argv: ${JSON.stringify(argv)}, cwd: process.cwd()});\n` +
+            'process.exitCode = result.status;\n',
+        ];
+  const env = {...process.env};
+  for (const key of Object.keys(env)) {
+    if (key.startsWith('VITEST') || key === 'NODE_OPTIONS') delete env[key];
+  }
+  Object.assign(env, {
+    HOME: home,
+    USERPROFILE: home,
+    XDG_CACHE_HOME: path.join(base, 'cache'),
+    XDG_CONFIG_HOME: path.join(home, '.config'),
+    XDG_DATA_HOME: path.join(home, '.local', 'share'),
+    // Other tools' own state, such as the device id the gh tool writes when
+    // swizzle asks it whether it is signed in, and npm's cache and logs,
+    // stays outside the folders the runs compare.
+    XDG_STATE_HOME: path.join(base, 'state'),
+    npm_config_cache: path.join(base, 'npm-cache'),
+    CI: '1',
+    NO_UPDATE_NOTIFIER: '1',
+    NO_COLOR: '1',
+    npm_config_registry: 'http://127.0.0.1:9/',
+    npm_config_fetch_retries: '0',
+    npm_config_fetch_timeout: '2000',
+  });
+  const child = spawn(process.execPath, args, {
+    cwd: dir,
+    env,
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+  let stdout = '';
+  let stderr = '';
+  child.stdout.on('data', chunk => (stdout += chunk));
+  child.stderr.on('data', chunk => (stderr += chunk));
+  const timer = setTimeout(() => child.kill('SIGKILL'), RUN_TIMEOUT_MS);
+  const status = await new Promise(resolve => child.on('close', resolve));
+  clearTimeout(timer);
+  // Each run's own folders read as placeholders.
+  const real = fs.realpathSync(base);
+  /** @param {string} text */
+  const placeholders = text =>
+    text.split(real).join('<run>').split(base).join('<run>');
+  const after = read();
+  /** @type {Record<string, string | null>} */
+  const changes = {};
+  for (const [file, text] of after) {
+    if (before.get(file) !== text) changes[file] = placeholders(text);
+  }
+  for (const file of before.keys()) {
+    if (!after.has(file)) changes[file] = null;
+  }
+  return {
+    status,
+    stdout: placeholders(stdout),
+    stderr: placeholders(stderr),
+    changes,
+  };
+}
+
+/**
+ * Run `tasks` with at most `limit` at a time.
+ * @template T
+ * @param {Array<() => Promise<T>>} tasks
+ * @param {number} limit
+ * @returns {Promise<T[]>}
+ */
+async function inBatches(tasks, limit) {
+  const results = new Array(tasks.length);
+  let next = 0;
+  const worker = async () => {
+    while (next < tasks.length) {
+      const index = next++;
+      results[index] = await tasks[index]();
+    }
+  };
+  await Promise.all(Array.from({length: limit}, worker));
+  return results;
+}
+
+const examples = await commandExamples();
+
+afterAll(() => {
+  for (const base of bases) fs.rmSync(base, {recursive: true, force: true});
+});
+
+describe.each(Object.entries(FOLDERS))('in %s', (_folder, setUp) => {
+  /** @type {Map<string, {bin: Awaited<ReturnType<typeof runThrough>>, host: Awaited<ReturnType<typeof runThrough>>}>} */
+  const runs = new Map();
+
+  beforeAll(async () => {
+    const limit = Math.max(2, Math.min(16, os.availableParallelism()));
+    const results = await inBatches(
+      examples.flatMap(({argv}) => [
+        () => runThrough('bin', argv, setUp),
+        () => runThrough('host entry', argv, setUp),
+      ]),
+      limit,
+    );
+    examples.forEach(({cli}, index) => {
+      runs.set(cli, {bin: results[2 * index], host: results[2 * index + 1]});
+    });
+  }, 15 * 60_000);
+
+  it('runs every example', () => {
+    expect(examples.length).toBeGreaterThan(100);
+  });
+
+  it.each(examples.map(({cli}) => [cli]))('%s', cli => {
+    const {bin, host} = runs.get(cli) ?? {};
+    expect(bin?.status, `${cli} finished through the bin`).not.toBeNull();
+    expect(host).toEqual(bin);
+  });
+});