@git.zone/tsrust 1.9.1 → 1.10.0

package/ts/mod_matrix/classes.nativematrixbuilder.ts

@@ -0,0 +1,1472 @@
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {
+  ArtifactAssembler,
+  MAX_ARTIFACT_BYTES,
+  MAX_ASSEMBLY_INPUT_BYTES,
+  type IArtifactAssemblyResult,
+} from '../mod_artifact/index.js';
+import { CargoConfig } from '../mod_cargo/index.js';
+import {
+  assertGitSnapshotUnchanged,
+  captureGitSnapshot,
+  MAX_PROVENANCE_SIDECAR_BYTES,
+  PROVENANCE_SIDECAR_SUFFIX,
+  ProvenanceStore,
+} from '../mod_provenance/index.js';
+import { ToolchainManager } from '../mod_toolchain/index.js';
+import { friendlyTargetName, type ITsrustConfig } from '../mod_cli/helpers.targets.js';
+import { commitinfo } from '../00_commitinfo_data.js';
+import {
+  MatrixCommandRunner,
+  type IMatrixCommandOptions,
+  type IMatrixCommandRunner,
+} from './classes.matrixcommandrunner.js';
+import {
+  resolveMatrixPlan,
+  type IResolvedMatrixPlan,
+  type IResolvedMatrixWorker,
+  type TMatrixHost,
+} from './helpers.matrixconfig.js';
+
+const capabilityCommandTimeoutMs = 5 * 60 * 1000;
+const buildCommandTimeoutMs = 60 * 60 * 1000;
+const maximumRetainedDiagnosticRuns = 3;
+const sshConnectionArgs = [
+  '-o',
+  'ServerAliveInterval=15',
+  '-o',
+  'ServerAliveCountMax=2',
+];
+const matrixOwnerFileName = '.tsrust-matrix-owner';
+const forwardEnvironmentGrantName = 'TSRUST_MATRIX_ALLOWED_ENVIRONMENT';
+const defaultWorkerEnvironmentNames = [
+  'PATH',
+  'USER',
+  'LOGNAME',
+  'SHELL',
+  'TMPDIR',
+  'TMP',
+  'TEMP',
+  'LANG',
+  'LC_ALL',
+  'LC_CTYPE',
+  'TERM',
+  'COLORTERM',
+  'CI',
+  'RUSTUP_HOME',
+] as const;
+const transportEnvironmentNames = [
+  'PATH',
+  'HOME',
+  'USER',
+  'LOGNAME',
+  'SHELL',
+  'SSH_AUTH_SOCK',
+  'SSH_AGENT_PID',
+  'TMPDIR',
+  'TMP',
+  'TEMP',
+  'LANG',
+  'LC_ALL',
+  'LC_CTYPE',
+  'TERM',
+  'COLORTERM',
+  'CI',
+] as const;
+const bundledToolchainValidationSource = [
+  "const fs = require('node:fs');",
+  "const path = require('node:path');",
+  'const [root, ...executables] = process.argv.slice(1);',
+  'const rootEntry = fs.lstatSync(root);',
+  'if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) process.exit(1);',
+  'if (process.getuid && rootEntry.uid !== process.getuid()) process.exit(1);',
+  'if ((rootEntry.mode & 0o022) !== 0) process.exit(1);',
+  'const physicalRoot = fs.realpathSync(root);',
+  'for (const executable of executables) {',
+  '  const entry = fs.lstatSync(executable);',
+  '  if (!entry.isFile() && !entry.isSymbolicLink()) process.exit(1);',
+  '  const resolvedPath = fs.realpathSync(executable);',
+  '  const relativePath = path.relative(physicalRoot, resolvedPath);',
+  "  if (!relativePath || relativePath.startsWith('..') || path.isAbsolute(relativePath)) process.exit(1);",
+  '  const resolved = fs.statSync(resolvedPath);',
+  '  if (!resolved.isFile() || (resolved.mode & 0o111) === 0 || (resolved.mode & 0o022) !== 0) process.exit(1);',
+  '  if (process.getuid && resolved.uid !== process.getuid()) process.exit(1);',
+  '}',
+].join('\n');
+const remoteWatchdogSource = [
+  "const childProcess = require('node:child_process');",
+  'const shell = process.argv[1];',
+  'const script = process.argv[2];',
+  "const child = childProcess.spawn(shell, ['-lc', script], { detached: true, stdio: ['ignore', 'inherit', 'inherit'] });",
+  'let lastHeartbeat = Date.now();',
+  'let terminationDeadline;',
+  'let spawnError;',
+  'let finished = false;',
+  'const killGroup = (signal) => { if (!child.pid) return; try { process.kill(-child.pid, signal); } catch (error) { if (error.code !== "ESRCH") throw error; } };',
+  'const groupExists = () => { if (!child.pid) return false; try { process.kill(-child.pid, 0); return true; } catch (error) { if (error.code === "ESRCH") return false; throw error; } };',
+  'const terminate = (signal) => { if (terminationDeadline) return; killGroup(signal); terminationDeadline = Date.now() + 10000; };',
+  'const enforceEscalation = () => { if (terminationDeadline && Date.now() >= terminationDeadline) killGroup("SIGKILL"); };',
+  'const waitForGroupExit = async () => { const deadline = Math.max(Date.now(), terminationDeadline || 0) + 2000; while (groupExists() && Date.now() < deadline) { enforceEscalation(); await new Promise((resolve) => setTimeout(resolve, 100)); } enforceEscalation(); return !groupExists(); };',
+  'const signalHandlers = new Map();',
+  'for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { const handler = () => terminate(signal); signalHandlers.set(signal, handler); process.on(signal, handler); }',
+  'const heartbeatTimer = setInterval(() => { if (Date.now() - lastHeartbeat > 45000) terminate("SIGHUP"); enforceEscalation(); }, 1000);',
+  'process.stdin.on("data", () => { lastHeartbeat = Date.now(); });',
+  'process.stdin.on("end", () => terminate("SIGHUP"));',
+  'process.stdin.on("error", () => terminate("SIGHUP"));',
+  'process.stdin.resume();',
+  'child.once("error", (error) => { spawnError = error; });',
+  'child.once("close", (code, signal) => { void (async () => {',
+  '  if (finished) return;',
+  '  finished = true;',
+  '  const hadSurvivingGroup = groupExists();',
+  '  if (hadSurvivingGroup) terminate("SIGTERM");',
+  '  const stopped = await waitForGroupExit();',
+  '  clearInterval(heartbeatTimer);',
+  '  for (const [registeredSignal, handler] of signalHandlers) process.off(registeredSignal, handler);',
+  '  process.stdin.destroy();',
+  '  if (spawnError) console.error(spawnError);',
+  '  if (!stopped) console.error("Remote matrix command left a live process group");',
+  '  process.exitCode = spawnError || !stopped || hadSurvivingGroup ? 1 : (code ?? (signal ? 1 : 0));',
+  '})(); });',
+].join('\n');
+
+interface IResolvedSshWorker {
+  destination: string;
+  temporaryRoot: string;
+}
+
+interface IRemoteWorkspace {
+  worker: IResolvedMatrixWorker;
+  runtime: IResolvedSshWorker;
+  directory: string;
+  ownerToken: string;
+  gitCommit: string;
+}
+
+export interface INativeMatrixBuilderOptions {
+  workspace: string;
+  config: ITsrustConfig;
+  environment?: NodeJS.ProcessEnv;
+  commandRunner?: IMatrixCommandRunner;
+}
+
+export interface INativeMatrixCheckResult {
+  workers: TMatrixHost[];
+  targets: string[];
+}
+
+export interface INativeMatrixBuildResult extends INativeMatrixCheckResult {
+  gitCommit: string;
+  assembly: IArtifactAssemblyResult;
+}
+
+const shellQuote = (valueArg: string): string => `'${valueArg.replace(/'/g, `'"'"'`)}'`;
+
+const expectedHostDetails = (
+  hostArg: TMatrixHost,
+): { system: 'Linux' | 'Darwin'; machine: 'x86_64' | 'aarch64' | 'arm64' } => {
+  switch (hostArg) {
+    case 'linux_amd64':
+      return { system: 'Linux', machine: 'x86_64' };
+    case 'linux_arm64':
+      return { system: 'Linux', machine: 'aarch64' };
+    case 'macos_amd64':
+      return { system: 'Darwin', machine: 'x86_64' };
+    case 'macos_arm64':
+      return { system: 'Darwin', machine: 'arm64' };
+  }
+};
+
+const artifactNamesFor = (
+  workerArg: IResolvedMatrixWorker,
+  binariesArg: string[],
+): string[] =>
+  workerArg.targets
+    .flatMap((target) =>
+      binariesArg.flatMap((binary) => {
+        const binaryName = `${binary}_${target.friendly}`;
+        return [binaryName, `${binaryName}${PROVENANCE_SIDECAR_SUFFIX}`];
+      }),
+    )
+    .sort();
+
+const linuxProbeCompiler = (targetArg: string): string | undefined => {
+  switch (targetArg) {
+    case 'linux_amd64':
+      return 'x86_64-linux-gnu-gcc';
+    case 'linux_arm64':
+      return 'aarch64-linux-gnu-gcc';
+    case 'linux_amd64_musl':
+      return 'x86_64-linux-musl-gcc';
+    case 'linux_arm64_musl':
+      return 'aarch64-linux-musl-gcc';
+    default:
+      return undefined;
+  }
+};
+
+export class NativeMatrixBuilder {
+  private readonly workspace: string;
+  private readonly environment: NodeJS.ProcessEnv;
+  private readonly commandRunner: IMatrixCommandRunner;
+  private readonly plan: IResolvedMatrixPlan;
+
+  constructor(optionsArg: INativeMatrixBuilderOptions) {
+    this.workspace = path.resolve(optionsArg.workspace);
+    this.environment = { ...process.env, ...(optionsArg.environment || {}) };
+    this.commandRunner =
+      optionsArg.commandRunner || new MatrixCommandRunner(this.environment);
+    this.plan = resolveMatrixPlan(optionsArg.config);
+    this.assertForwardEnvironmentGrant();
+  }
+
+  public async check(): Promise<INativeMatrixCheckResult> {
+    await this.assertBundleCompatibleSource();
+    await this.commandRunner.execute('git', ['--version'], {
+      cwd: this.workspace,
+      capture: true,
+      timeoutMs: capabilityCommandTimeoutMs,
+    });
+    await this.commandRunner.execute('pnpm', ['--version'], {
+      cwd: this.workspace,
+      capture: true,
+      timeoutMs: capabilityCommandTimeoutMs,
+    });
+    await this.assertInstalledTsrustVersion(this.workspace);
+    for (const worker of this.plan.workers) {
+      if (worker.config.transport === 'local') {
+        await this.checkLocalWorker(worker);
+      } else {
+        await this.checkSshWorker(worker, this.resolveSshWorker(worker));
+      }
+    }
+    return this.checkResult();
+  }
+
+  public async build(): Promise<INativeMatrixBuildResult> {
+    const gitBefore = await captureGitSnapshot(this.workspace);
+    if (!gitBefore.available || gitBefore.commit === 'unknown') {
+      throw new Error('tsrust matrix build requires a Git checkout');
+    }
+    if (gitBefore.status.length > 0 || (gitBefore.unsafeIndexFlags?.length || 0) > 0) {
+      throw new Error('tsrust matrix build requires a clean Git worktree');
+    }
+    await this.assertPlanMatchesCommit(gitBefore.commit);
+    await this.check();
+    assertGitSnapshotUnchanged(gitBefore, await captureGitSnapshot(this.workspace));
+    await this.assertBundleCompatibleSource();
+    assertGitSnapshotUnchanged(gitBefore, await captureGitSnapshot(this.workspace));
+    const projectIdentity = this.readProjectIdentity();
+    const binaries = await this.readBinaryTargets();
+    const runParent = path.join(this.workspace, '.nogit', 'tsrust-matrix');
+    await this.ensureLocalRunParent(runParent);
+    const runDirectory = await fs.promises.mkdtemp(
+      path.join(runParent, `${gitBefore.commit.slice(0, 12)}-`),
+    );
+    const bundlePath = path.join(runDirectory, 'source.bundle');
+    const sourceDirectories: string[] = [];
+    const remoteWorkspaces: IRemoteWorkspace[] = [];
+    let completed = false;
+
+    try {
+      await this.commandRunner.execute('git', ['bundle', 'create', bundlePath, 'HEAD'], {
+        cwd: this.workspace,
+        timeoutMs: capabilityCommandTimeoutMs,
+      });
+      await this.commandRunner.execute('git', ['bundle', 'verify', bundlePath], {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+      });
+      assertGitSnapshotUnchanged(gitBefore, await captureGitSnapshot(this.workspace));
+
+      for (const worker of this.plan.workers) {
+        if (worker.config.transport === 'local') {
+          sourceDirectories.push(
+            await this.buildLocalWorker(
+              worker,
+              bundlePath,
+              runDirectory,
+              gitBefore.commit,
+              projectIdentity,
+              binaries,
+            ),
+          );
+        } else {
+          const remoteWorkspace = this.describeRemoteWorkspace(
+            worker,
+            this.resolveSshWorker(worker),
+            gitBefore.commit,
+          );
+          remoteWorkspaces.push(remoteWorkspace);
+          await this.createRemoteWorkspace(remoteWorkspace, gitBefore.commit);
+          sourceDirectories.push(
+            await this.buildSshWorker(
+              remoteWorkspace,
+              bundlePath,
+              runDirectory,
+              gitBefore.commit,
+              projectIdentity,
+              binaries,
+            ),
+          );
+        }
+      }
+
+      const gitAfterWorkers = await captureGitSnapshot(this.workspace);
+      assertGitSnapshotUnchanged(gitBefore, gitAfterWorkers);
+      this.assertProjectIdentityUnchanged(projectIdentity);
+      while (remoteWorkspaces.length > 0) {
+        const remoteWorkspace = remoteWorkspaces[0];
+        await this.cleanupRemoteWorkspace(remoteWorkspace, gitBefore.commit);
+        remoteWorkspaces.shift();
+      }
+      const gitAfterRemoteCleanup = await captureGitSnapshot(this.workspace);
+      assertGitSnapshotUnchanged(gitBefore, gitAfterRemoteCleanup);
+      this.assertProjectIdentityUnchanged(projectIdentity);
+      const assembly = await new ArtifactAssembler({
+        workspace: this.workspace,
+        sourceDirectories,
+        expectedTargets: this.plan.expectedTargets,
+        expectedBinaries: binaries,
+        expectedProjectName: projectIdentity.projectName,
+        expectedProjectVersion: projectIdentity.projectVersion,
+        expectedGitCommit: gitBefore.commit,
+        expectedTsrustVersion: commitinfo.version,
+      }).assemble();
+      if (assembly.cleanupPending) {
+        console.warn('Matrix assembly committed; stale transaction cleanup will retry next run.');
+      }
+      completed = true;
+      try {
+        await fs.promises.rm(runDirectory, { recursive: true });
+      } catch (error) {
+        console.warn(
+          `Matrix assembly succeeded but local staging cleanup failed at ${runDirectory}: ${
+            error instanceof Error ? error.message : String(error)
+          }`,
+        );
+      }
+      return {
+        ...this.checkResult(),
+        gitCommit: gitBefore.commit,
+        assembly,
+      };
+    } finally {
+      if (!completed) {
+        console.error(`Retained local tsrust matrix diagnostics at ${runDirectory}`);
+        for (const remoteWorkspace of remoteWorkspaces) {
+          console.error(
+            `${remoteWorkspace.worker.host} matrix diagnostics may remain at ${remoteWorkspace.directory}`,
+          );
+        }
+        console.error('Matrix diagnostics are not pruned automatically; inspect before removal.');
+      }
+    }
+  }
+
+  private checkResult(): INativeMatrixCheckResult {
+    return {
+      workers: this.plan.workers.map((worker) => worker.host),
+      targets: this.plan.expectedTargets.map((target) => target.friendly),
+    };
+  }
+
+  private assertForwardEnvironmentGrant(): void {
+    const granted = new Set(
+      (this.environment[forwardEnvironmentGrantName] || '')
+        .split(',')
+        .map((name) => name.trim())
+        .filter(Boolean),
+    );
+    const reserved = new Set<string>([
+      ...defaultWorkerEnvironmentNames,
+      ...transportEnvironmentNames,
+      'HOME',
+      'CARGO_HOME',
+      forwardEnvironmentGrantName,
+    ]);
+    const forbidden = this.plan.forwardEnvironment.filter(
+      (name) => reserved.has(name) || !granted.has(name),
+    );
+    if (forbidden.length > 0) {
+      throw new Error(
+        `matrix.forwardEnvironment is not granted by ${forwardEnvironmentGrantName}: ${forbidden.join(', ')}`,
+      );
+    }
+  }
+
+  private workerEnvironment(homeArg: string, cargoHomeArg: string): NodeJS.ProcessEnv {
+    const environment: NodeJS.ProcessEnv = {};
+    for (const name of [
+      ...defaultWorkerEnvironmentNames,
+      ...this.plan.forwardEnvironment,
+    ]) {
+      if (this.environment[name] !== undefined) environment[name] = this.environment[name];
+    }
+    environment.HOME = homeArg;
+    environment.CARGO_HOME = cargoHomeArg;
+    if (!environment.RUSTUP_HOME && this.environment.HOME) {
+      environment.RUSTUP_HOME = path.join(this.environment.HOME, '.rustup');
+    }
+    return environment;
+  }
+
+  private transportEnvironment(): NodeJS.ProcessEnv {
+    const environment: NodeJS.ProcessEnv = {};
+    for (const name of [...transportEnvironmentNames, ...this.plan.forwardEnvironment]) {
+      if (this.environment[name] !== undefined) environment[name] = this.environment[name];
+    }
+    return environment;
+  }
+
+  private remainingBuildTimeout(deadlineArg: number, workerArg: TMatrixHost): number {
+    const remaining = deadlineArg - Date.now();
+    if (remaining <= 0) {
+      throw new Error(`Matrix worker ${workerArg} exceeded its shared build timeout`);
+    }
+    return remaining;
+  }
+
+  private async assertInstalledTsrustVersion(workspaceArg: string): Promise<void> {
+    const packagePath = path.join(
+      workspaceArg,
+      'node_modules',
+      '@git.zone',
+      'tsrust',
+      'package.json',
+    );
+    const packageJson = JSON.parse(await fs.promises.readFile(packagePath, 'utf8')) as {
+      name?: unknown;
+      version?: unknown;
+    };
+    if (packageJson.name !== '@git.zone/tsrust' || packageJson.version !== commitinfo.version) {
+      throw new Error(
+        `Project-local tsrust ${String(packageJson.version || 'unknown')} does not match coordinator ${commitinfo.version}`,
+      );
+    }
+  }
+
+  private async assertPlanMatchesCommit(gitCommitArg: string): Promise<void> {
+    const committedFile = await this.commandRunner.execute(
+      'git',
+      ['show', `${gitCommitArg}:.smartconfig.json`],
+      {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+      },
+    );
+    const committedRoot = JSON.parse(committedFile.stdout) as Record<string, unknown>;
+    const committedConfig = committedRoot['@git.zone/tsrust'];
+    const committedPlan = resolveMatrixPlan(committedConfig as ITsrustConfig);
+    if (JSON.stringify(committedPlan) !== JSON.stringify(this.plan)) {
+      throw new Error('Matrix configuration does not match the exact Git commit');
+    }
+  }
+
+  private remoteWorkerEnvironmentFunction(homeArg: string, cargoHomeArg: string): string {
+    const names = [
+      ...new Set([...defaultWorkerEnvironmentNames, ...this.plan.forwardEnvironment]),
+    ];
+    const assignments = names.map((name) => `${name}="\${${name}-}"`).join(' ');
+    return `run_matrix_worker() { worker_rustup_home="\${RUSTUP_HOME:-\${HOME}/.rustup}"; env -i ${assignments} HOME=${shellQuote(homeArg)} CARGO_HOME=${shellQuote(cargoHomeArg)} RUSTUP_HOME="$worker_rustup_home" "$@"; }`;
+  }
+
+  private async assertBundleCompatibleSource(): Promise<void> {
+    const staged = await this.commandRunner.execute(
+      'git',
+      ['ls-files', '--stage', '-z'],
+      {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+      },
+    );
+    for (const entry of staged.stdout.split('\0').filter(Boolean)) {
+      const mode = entry.slice(0, entry.indexOf(' '));
+      if (mode === '160000') {
+        throw new Error('tsrust matrix builds do not support Git submodules');
+      }
+      if (mode === '120000') {
+        throw new Error('tsrust matrix builds do not support tracked symbolic links');
+      }
+    }
+
+    const tracked = await this.commandRunner.execute('git', ['ls-files', '-z'], {
+      cwd: this.workspace,
+      capture: true,
+      timeoutMs: capabilityCommandTimeoutMs,
+    });
+    if (tracked.stdout.length > 0) {
+      const attributes = await this.commandRunner.execute(
+        'git',
+        ['check-attr', '-z', '--stdin', 'filter'],
+        {
+          cwd: this.workspace,
+          capture: true,
+          input: tracked.stdout,
+          timeoutMs: capabilityCommandTimeoutMs,
+        },
+      );
+      const values = attributes.stdout.split('\0');
+      for (let index = 0; index + 2 < values.length; index += 3) {
+        if (!['unspecified', 'unset'].includes(values[index + 2])) {
+          throw new Error(
+            `tsrust matrix builds do not support checkout filters: ${values[index]}`,
+          );
+        }
+      }
+    }
+
+    const pointerCandidates = await this.commandRunner.execute(
+      'git',
+      [
+        'grep',
+        '-I',
+        '-l',
+        '-z',
+        '-e',
+        '^version https://',
+        'HEAD',
+        '--',
+      ],
+      {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+        allowedExitCodes: [0, 1],
+      },
+    );
+    for (const candidate of pointerCandidates.stdout.split('\0').filter(Boolean)) {
+      const filePath = candidate.startsWith('HEAD:') ? candidate.slice(5) : candidate;
+      const content = await this.commandRunner.execute('git', ['show', `HEAD:${filePath}`], {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+      });
+      if (
+        /^version https:\/\/[^\r\n]+\r?\n(?:ext-[^\r\n]+\r?\n)*oid sha256:[a-f0-9]{64}\r?\nsize [0-9]+(?:\r?\n|$)/.test(
+          content.stdout,
+        )
+      ) {
+        throw new Error(`tsrust matrix builds do not support Git LFS pointer ${filePath}`);
+      }
+    }
+  }
+
+  private async checkLocalWorker(workerArg: IResolvedMatrixWorker): Promise<void> {
+    const actualHost = friendlyTargetName(new ToolchainManager().getHostTriple());
+    if (actualHost !== workerArg.host) {
+      throw new Error(
+        `Local matrix worker expected ${workerArg.host} but coordinator is ${actualHost}`,
+      );
+    }
+    await this.checkLocalToolchain(workerArg.host);
+    await this.checkLocalExecution(workerArg);
+  }
+
+  private async checkLocalToolchain(hostArg: TMatrixHost): Promise<void> {
+    let hasSystemCargo = false;
+    try {
+      await this.commandRunner.execute('cargo', ['--version'], {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+      });
+      hasSystemCargo = true;
+    } catch {
+      hasSystemCargo = false;
+    }
+    if (hasSystemCargo) {
+      await this.commandRunner.execute('rustup', ['--version'], {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+      });
+    } else {
+      const toolchainManager = new ToolchainManager();
+      if (await toolchainManager.isInstalled()) {
+        await this.commandRunner.execute(
+          path.join(ToolchainManager.BIN_DIR, 'cargo'),
+          ['--version'],
+          {
+            cwd: this.workspace,
+            capture: true,
+            timeoutMs: capabilityCommandTimeoutMs,
+          },
+        );
+        await this.commandRunner.execute(
+          path.join(ToolchainManager.BIN_DIR, 'rustup'),
+          ['--version'],
+          {
+            cwd: this.workspace,
+            capture: true,
+            timeoutMs: capabilityCommandTimeoutMs,
+          },
+        );
+      } else {
+        await this.commandRunner.execute('curl', ['--version'], {
+          cwd: this.workspace,
+          capture: true,
+          timeoutMs: capabilityCommandTimeoutMs,
+        });
+      }
+    }
+    if (hostArg.startsWith('macos_')) {
+      await this.commandRunner.execute('xcrun', ['--find', 'clang'], {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+      });
+    } else {
+      await this.commandRunner.execute('cc', ['--version'], {
+        cwd: this.workspace,
+        capture: true,
+        timeoutMs: capabilityCommandTimeoutMs,
+      });
+    }
+  }
+
+  private async checkLocalExecution(workerArg: IResolvedMatrixWorker): Promise<void> {
+    for (const target of workerArg.targets) {
+      if (target.friendly === workerArg.host) continue;
+      if (workerArg.host.startsWith('macos_') && target.friendly.startsWith('macos_')) {
+        const architecture = target.friendly === 'macos_arm64' ? '-arm64' : '-x86_64';
+        await this.commandRunner.execute('/usr/bin/arch', [architecture, '/usr/bin/true'], {
+          cwd: this.workspace,
+          timeoutMs: capabilityCommandTimeoutMs,
+        });
+        continue;
+      }
+      if (workerArg.host.startsWith('linux_') && target.friendly.startsWith('linux_')) {
+        await this.checkLocalLinuxCrossExecution(target.friendly);
+        continue;
+      }
+      throw new Error(
+        `Matrix worker ${workerArg.host} cannot execute smoke tests for ${target.friendly}`,
+      );
+    }
+  }
+
+  private async checkLocalLinuxCrossExecution(targetArg: string): Promise<void> {
+    const compiler = linuxProbeCompiler(targetArg);
+    if (!compiler) {
+      throw new Error(`No matrix execution probe is available for ${targetArg}`);
+    }
+    const probeParent = path.join(this.workspace, '.nogit', 'tsrust-matrix-probes');
+    await fs.promises.mkdir(probeParent, { recursive: true, mode: 0o700 });
+    const probeDirectory = await fs.promises.mkdtemp(path.join(probeParent, `${targetArg}-`));
+    try {
+      const sourcePath = path.join(probeDirectory, 'probe.c');
+      const binaryPath = path.join(probeDirectory, 'probe');
+      await fs.promises.writeFile(sourcePath, 'int main(void) { return 0; }\n');
+      await this.commandRunner.execute(
+        compiler,
+        ['-static', '-O2', sourcePath, '-o', binaryPath],
+        {
+          cwd: this.workspace,
+          timeoutMs: capabilityCommandTimeoutMs,
+        },
+      );
+      await this.commandRunner.execute(binaryPath, [], {
+        cwd: this.workspace,
+        timeoutMs: capabilityCommandTimeoutMs,
+      });
+    } finally {
+      await fs.promises.rm(probeDirectory, { recursive: true, force: true });
+    }
+  }
+
+  private resolveSshWorker(workerArg: IResolvedMatrixWorker): IResolvedSshWorker {
+    if (workerArg.config.transport !== 'ssh') {
+      throw new Error(`Matrix worker ${workerArg.host} is not configured for SSH`);
+    }
+    const destination = this.environment[workerArg.config.destinationEnv]?.trim() || '';
+    const temporaryRoot = (
+      this.environment[workerArg.config.temporaryRootEnv]?.trim() || ''
+    ).replace(/\/+$/, '');
+    if (!/^[A-Za-z0-9._-]+(?:@[A-Za-z0-9._-]+)?$/.test(destination)) {
+      throw new Error(
+        `${workerArg.config.destinationEnv} must contain one SSH alias or user@host destination without options`,
+      );
+    }
+    if (
+      !/^\/[A-Za-z0-9._/-]+$/.test(temporaryRoot) ||
+      path.posix.normalize(temporaryRoot) !== temporaryRoot ||
+      temporaryRoot.split('/').filter(Boolean).length < 2
+    ) {
+      throw new Error(
+        `${workerArg.config.temporaryRootEnv} must contain a non-root absolute path without spaces`,
+      );
+    }
+    return { destination, temporaryRoot };
+  }
+
+  private async checkSshWorker(
+    workerArg: IResolvedMatrixWorker,
+    runtimeArg: IResolvedSshWorker,
+  ): Promise<void> {
+    const details = expectedHostDetails(workerArg.host);
+    const token = crypto.randomUUID();
+    const probeDirectory = `${runtimeArg.temporaryRoot}/tsrust-matrix.check.${token}`;
+    const commands = [
+      'set -eu',
+      'umask 077',
+      `test "$(uname -s)" = ${shellQuote(details.system)}`,
+      `test "$(uname -m)" = ${shellQuote(details.machine)}`,
+      'command -v git >/dev/null',
+      'command -v node >/dev/null',
+      'command -v pnpm >/dev/null',
+      this.remoteToolchainCapabilityCommand(),
+      ...(details.system === 'Darwin' ? ['command -v codesign >/dev/null'] : []),
+      ...(details.system === 'Darwin'
+        ? ['xcrun --find clang >/dev/null']
+        : ['command -v cc >/dev/null']),
+      ...this.remoteRootValidationCommands(runtimeArg.temporaryRoot),
+      `test "$(find "$root" -mindepth 1 -maxdepth 1 -type d -name 'tsrust-matrix.*' | wc -l | tr -d ' ')" -lt ${maximumRetainedDiagnosticRuns}`,
+      `probe=${shellQuote(probeDirectory)}`,
+      `test "$probe" = ${shellQuote(probeDirectory)}`,
+      'created=0',
+      'cleanup_probe() { if [ "$created" = 1 ]; then rm -rf "$probe"; fi; }',
+      'trap cleanup_probe EXIT HUP INT TERM',
+      'mkdir -m 700 "$probe"',
+      'created=1',
+      ...this.remoteExecutionProbeCommands(workerArg, probeDirectory),
+      'rm -rf "$probe"',
+      'created=0',
+      'trap - EXIT HUP INT TERM',
+    ];
+    try {
+      await this.runRemoteOwnedScript(
+        workerArg,
+        runtimeArg,
+        commands.join('\n'),
+        capabilityCommandTimeoutMs,
+      );
+    } catch (error) {
+      throw new Error(
+        `${workerArg.host} capability check failed; diagnostics may remain at ${probeDirectory}: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+        { cause: error },
+      );
+    }
+  }
+
+  private remoteToolchainCapabilityCommand(): string {
+    const root = ToolchainManager.TOOLCHAIN_DIR;
+    const cargo = path.join(ToolchainManager.BIN_DIR, 'cargo');
+    const rustup = path.join(ToolchainManager.BIN_DIR, 'rustup');
+    const installLock = `${root}.install.lock`;
+    return `if command -v cargo >/dev/null && cargo --version >/dev/null 2>&1; then command -v rustup >/dev/null; rustup --version >/dev/null; elif [ -e ${shellQuote(installLock)} ]; then exit 1; elif [ -x ${shellQuote(cargo)} ] && [ -x ${shellQuote(rustup)} ]; then node -e ${shellQuote(bundledToolchainValidationSource)} ${shellQuote(root)} ${shellQuote(cargo)} ${shellQuote(rustup)}; ${shellQuote(cargo)} --version >/dev/null; ${shellQuote(rustup)} --version >/dev/null; elif [ -e ${shellQuote(cargo)} ] || [ -e ${shellQuote(rustup)} ]; then exit 1; else command -v curl >/dev/null; fi`;
+  }
+
+  private remoteRootValidationCommands(rootArg: string): string[] {
+    return [
+      `root=${shellQuote(rootArg)}`,
+      'if [ -e "$root" ]; then test -d "$root"; test ! -L "$root"; else mkdir -p "$root"; fi',
+      'test "$(cd "$root" && pwd -P)" = "$root"',
+      'test -z "$(find "$root" -prune -perm -022 -print)"',
+      'if [ "$(uname -s)" = Darwin ]; then test "$(stat -f %u "$root")" = "$(id -u)"; else test "$(stat -c %u "$root")" = "$(id -u)"; fi',
+    ];
+  }
+
+  private remoteOwnershipValidationCommands(
+    remoteArg: IRemoteWorkspace,
+    gitCommitArg: string,
+  ): string[] {
+    const ownerPath = `${remoteArg.directory}/${matrixOwnerFileName}`;
+    const ownerContents = `${remoteArg.ownerToken}\n${gitCommitArg}\n${remoteArg.directory}`;
+    return [
+      `directory=${shellQuote(remoteArg.directory)}`,
+      `test "$directory" = ${shellQuote(remoteArg.directory)}`,
+      'test -d "$directory"',
+      'test ! -L "$directory"',
+      'test "$(cd "$directory/.." && pwd -P)" = "$root"',
+      'test -z "$(find "$directory" -prune -perm -077 -print)"',
+      'if [ "$(uname -s)" = Darwin ]; then test "$(stat -f %u "$directory")" = "$(id -u)"; else test "$(stat -c %u "$directory")" = "$(id -u)"; fi',
+      `test -f ${shellQuote(ownerPath)}`,
+      `test ! -L ${shellQuote(ownerPath)}`,
+      `test -z "$(find ${shellQuote(ownerPath)} -prune -perm -077 -print)"`,
+      `if [ "$(uname -s)" = Darwin ]; then test "$(stat -f %u ${shellQuote(ownerPath)})" = "$(id -u)"; else test "$(stat -c %u ${shellQuote(ownerPath)})" = "$(id -u)"; fi`,
+      `test "$(cat ${shellQuote(ownerPath)})" = ${shellQuote(ownerContents)}`,
+    ];
+  }
+
+  private remoteExecutionProbeCommands(
+    workerArg: IResolvedMatrixWorker,
+    probeDirectoryArg: string,
+  ): string[] {
+    const commands: string[] = [];
+    for (const target of workerArg.targets) {
+      if (target.friendly === workerArg.host) continue;
+      if (workerArg.host.startsWith('macos_') && target.friendly.startsWith('macos_')) {
+        const architecture = target.friendly === 'macos_arm64' ? '-arm64' : '-x86_64';
+        commands.push(`/usr/bin/arch ${architecture} /usr/bin/true`);
+        continue;
+      }
+      if (workerArg.host.startsWith('linux_') && target.friendly.startsWith('linux_')) {
+        const compiler = linuxProbeCompiler(target.friendly);
+        if (!compiler) {
+          throw new Error(`No matrix execution probe is available for ${target.friendly}`);
+        }
+        const sourcePath = `${probeDirectoryArg}/probe-${target.friendly}.c`;
+        const binaryPath = `${probeDirectoryArg}/probe-${target.friendly}`;
+        commands.push(
+          `printf '%s\n' 'int main(void) { return 0; }' > ${shellQuote(sourcePath)}`,
+          `${compiler} -static -O2 ${shellQuote(sourcePath)} -o ${shellQuote(binaryPath)}`,
+          shellQuote(binaryPath),
+        );
+        continue;
+      }
+      throw new Error(
+        `Matrix worker ${workerArg.host} cannot execute smoke tests for ${target.friendly}`,
+      );
+    }
+    return commands;
+  }
+
+  private async readBinaryTargets(): Promise<string[]> {
+    const rustDirectory = await this.detectRustDirectory();
+    const workspaceInfo = await new CargoConfig(rustDirectory).parse();
+    const binaries = [...new Set(workspaceInfo.binTargets)];
+    if (binaries.length === 0) {
+      throw new Error('tsrust matrix build requires at least one Cargo binary target');
+    }
+    for (const binary of binaries) {
+      if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(binary)) {
+        throw new Error(`Invalid Cargo binary name for matrix build: ${binary}`);
+      }
+    }
+    return binaries;
+  }
+
+  private async detectRustDirectory(): Promise<string> {
+    for (const directoryName of ['rust', 'ts_rust']) {
+      const candidate = path.join(this.workspace, directoryName);
+      try {
+        const stat = await fs.promises.stat(path.join(candidate, 'Cargo.toml'));
+        if (stat.isFile()) return candidate;
+      } catch {
+        // Try the next conventional Rust source directory.
+      }
+    }
+    throw new Error('No rust/ or ts_rust/ directory found with a Cargo.toml');
+  }
+
+  private readProjectIdentity(): { projectName: string; projectVersion: string } {
+    const packageJson = JSON.parse(
+      fs.readFileSync(path.join(this.workspace, 'package.json'), 'utf8'),
+    ) as { name?: unknown; version?: unknown };
+    if (typeof packageJson.name !== 'string' || typeof packageJson.version !== 'string') {
+      throw new Error('tsrust matrix build requires package.json name and version');
+    }
+    return { projectName: packageJson.name, projectVersion: packageJson.version };
+  }
+
+  private assertProjectIdentityUnchanged(identityArg: {
+    projectName: string;
+    projectVersion: string;
+  }): void {
+    const current = this.readProjectIdentity();
+    if (
+      current.projectName !== identityArg.projectName ||
+      current.projectVersion !== identityArg.projectVersion
+    ) {
+      throw new Error('Project package identity changed during the matrix build');
+    }
+  }
+
+  private async ensureLocalRunParent(parentArg: string): Promise<void> {
+    await fs.promises.mkdir(parentArg, { recursive: true, mode: 0o700 });
+    const stat = await fs.promises.lstat(parentArg);
+    if (!stat.isDirectory() || stat.isSymbolicLink()) {
+      throw new Error(`tsrust matrix state path is not a regular directory: ${parentArg}`);
+    }
+    const expectedPhysicalPath = path.join(
+      await fs.promises.realpath(this.workspace),
+      '.nogit',
+      'tsrust-matrix',
+    );
+    if ((await fs.promises.realpath(parentArg)) !== expectedPhysicalPath) {
+      throw new Error(`tsrust matrix state path traverses a symbolic link: ${parentArg}`);
+    }
+    if (
+      (typeof process.getuid === 'function' && stat.uid !== process.getuid()) ||
+      (stat.mode & 0o077) !== 0
+    ) {
+      throw new Error(`tsrust matrix state path has unsafe ownership or permissions: ${parentArg}`);
+    }
+    const retainedRuns = (await fs.promises.readdir(parentArg, { withFileTypes: true })).filter(
+      (entry) => entry.isDirectory(),
+    );
+    if (retainedRuns.length >= maximumRetainedDiagnosticRuns) {
+      throw new Error(
+        `tsrust matrix retains at most ${maximumRetainedDiagnosticRuns} failed local runs; inspect and remove the reported paths before retrying`,
+      );
+    }
+  }
+
+  private async buildLocalWorker(
+    workerArg: IResolvedMatrixWorker,
+    bundlePathArg: string,
+    runDirectoryArg: string,
+    gitCommitArg: string,
+    identityArg: { projectName: string; projectVersion: string },
+    binariesArg: string[],
+  ): Promise<string> {
+    const workerParent = path.join(runDirectoryArg, workerArg.host);
+    const workerWorkspace = path.join(workerParent, 'workspace');
+    const workerHome = path.join(workerParent, 'home');
+    const workerCargoHome = path.join(workerParent, 'cargo-home');
+    await fs.promises.mkdir(workerParent, { mode: 0o700 });
+    await fs.promises.mkdir(workerHome, { mode: 0o700 });
+    await fs.promises.mkdir(workerCargoHome, { mode: 0o700 });
+    const workerEnvironment = this.workerEnvironment(workerHome, workerCargoHome);
+    const buildDeadline = Date.now() + buildCommandTimeoutMs;
+    await this.commandRunner.execute(
+      'git',
+      ['-c', 'advice.detachedHead=false', 'clone', '--quiet', bundlePathArg, workerWorkspace],
+      {
+        cwd: this.workspace,
+        timeoutMs: Math.min(
+          capabilityCommandTimeoutMs,
+          this.remainingBuildTimeout(buildDeadline, workerArg.host),
+        ),
+      },
+    );
+    await this.assertWorkerSource(workerWorkspace, gitCommitArg, buildDeadline, workerArg.host);
+    await this.commandRunner.execute('pnpm', ['install', '--frozen-lockfile'], {
+      cwd: workerWorkspace,
+      timeoutMs: this.remainingBuildTimeout(buildDeadline, workerArg.host),
+      environment: workerEnvironment,
+    });
+    await this.assertInstalledTsrustVersion(workerWorkspace);
+    this.remainingBuildTimeout(buildDeadline, workerArg.host);
+    await this.commandRunner.execute('pnpm', this.workerTsrustArgs(workerArg), {
+      cwd: workerWorkspace,
+      timeoutMs: this.remainingBuildTimeout(buildDeadline, workerArg.host),
+      environment: workerEnvironment,
+    });
+    await this.assertWorkerSource(workerWorkspace, gitCommitArg, buildDeadline, workerArg.host);
+    const artifactDirectory = path.join(workerWorkspace, 'dist_rust');
+    await this.assertArtifactDirectory(
+      artifactDirectory,
+      workerArg,
+      binariesArg,
+      gitCommitArg,
+      identityArg,
+    );
+    this.remainingBuildTimeout(buildDeadline, workerArg.host);
+    await this.smokeLocalArtifacts(
+      artifactDirectory,
+      workerArg,
+      binariesArg,
+      workerEnvironment,
+      buildDeadline,
+    );
+    await this.assertWorkerSource(workerWorkspace, gitCommitArg, buildDeadline, workerArg.host);
+    return artifactDirectory;
+  }
+
+  private workerTsrustArgs(workerArg: IResolvedMatrixWorker): string[] {
+    return [
+      'exec',
+      'tsrust',
+      ...workerArg.targets.flatMap((target) => ['--target', target.friendly]),
+    ];
+  }
+
+  private async assertWorkerSource(
+    workspaceArg: string,
+    gitCommitArg: string,
+    buildDeadlineArg: number,
+    workerArg: TMatrixHost,
+  ): Promise<void> {
+    const commit = await this.commandRunner.execute('git', ['rev-parse', 'HEAD'], {
+      cwd: workspaceArg,
+      capture: true,
+      timeoutMs: Math.min(
+        capabilityCommandTimeoutMs,
+        this.remainingBuildTimeout(buildDeadlineArg, workerArg),
+      ),
+    });
+    if (commit.stdout.trim() !== gitCommitArg) {
+      throw new Error(`Matrix worker cloned the wrong Git commit: ${workspaceArg}`);
+    }
+    const status = await this.commandRunner.execute(
+      'git',
+      ['status', '--porcelain=v1', '--untracked-files=all'],
+      {
+        cwd: workspaceArg,
+        capture: true,
+        timeoutMs: Math.min(
+          capabilityCommandTimeoutMs,
+          this.remainingBuildTimeout(buildDeadlineArg, workerArg),
+        ),
+      },
+    );
+    if (status.stdout.length > 0) {
+      throw new Error(`Matrix worker source is dirty: ${workspaceArg}`);
+    }
+    const index = await this.commandRunner.execute('git', ['ls-files', '-v', '-z'], {
+      cwd: workspaceArg,
+      capture: true,
+      timeoutMs: Math.min(
+        capabilityCommandTimeoutMs,
+        this.remainingBuildTimeout(buildDeadlineArg, workerArg),
+      ),
+    });
+    if (
+      index.stdout
+        .split('\0')
+        .some((entry) => entry.length > 0 && !entry.startsWith('H '))
+    ) {
+      throw new Error(`Matrix worker source uses unsafe Git index flags: ${workspaceArg}`);
+    }
+  }
+
+  private async assertArtifactDirectory(
+    directoryArg: string,
+    workerArg: IResolvedMatrixWorker,
+    binariesArg: string[],
+    gitCommitArg: string,
+    identityArg: { projectName: string; projectVersion: string },
+  ): Promise<void> {
+    const expectedNames = artifactNamesFor(workerArg, binariesArg);
+    const actualNames = (await fs.promises.readdir(directoryArg)).sort();
+    if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) {
+      throw new Error(
+        `Unexpected ${workerArg.host} matrix artifacts: ${actualNames.join(', ')}`,
+      );
+    }
+    let aggregateBytes = 0;
+    for (const target of workerArg.targets) {
+      for (const binary of binariesArg) {
+        const binaryPath = path.join(directoryArg, `${binary}_${target.friendly}`);
+        const binaryStat = await fs.promises.lstat(binaryPath);
+        if (!binaryStat.isFile() || binaryStat.isSymbolicLink() || !(binaryStat.mode & 0o111)) {
+          throw new Error(`Matrix artifact is not an executable regular file: ${binaryPath}`);
+        }
+        if (binaryStat.size > MAX_ARTIFACT_BYTES) {
+          throw new Error(`Matrix artifact exceeds the size limit: ${binaryPath}`);
+        }
+        const sidecarStat = await fs.promises.lstat(ProvenanceStore.sidecarPath(binaryPath));
+        if (!sidecarStat.isFile() || sidecarStat.isSymbolicLink()) {
+          throw new Error(`Matrix provenance is not a regular file: ${binaryPath}`);
+        }
+        if (sidecarStat.size > MAX_PROVENANCE_SIDECAR_BYTES) {
+          throw new Error(`Matrix provenance exceeds the size limit: ${binaryPath}`);
+        }
+        aggregateBytes += binaryStat.size + sidecarStat.size;
+        if (aggregateBytes > MAX_ASSEMBLY_INPUT_BYTES) {
+          throw new Error('Matrix artifacts exceed the aggregate size limit');
+        }
+        const buildInfo = await ProvenanceStore.readSidecar(binaryPath);
+        if (
+          buildInfo.binary !== binary ||
+          buildInfo.target !== target.friendly ||
+          buildInfo.projectName !== identityArg.projectName ||
+          buildInfo.projectVersion !== identityArg.projectVersion ||
+          buildInfo.gitCommit !== gitCommitArg ||
+          buildInfo.gitDirty !== false ||
+          buildInfo.tsrustVersion !== commitinfo.version
+        ) {
+          throw new Error(`Matrix provenance does not match the build request: ${binaryPath}`);
+        }
+      }
+    }
+  }
+
+  private async smokeLocalArtifacts(
+    directoryArg: string,
+    workerArg: IResolvedMatrixWorker,
+    binariesArg: string[],
+    environmentArg: NodeJS.ProcessEnv,
+    buildDeadlineArg: number,
+  ): Promise<void> {
+    if (!this.plan.smokeTestArgs) return;
+    for (const target of workerArg.targets) {
+      for (const binary of binariesArg) {
+        const binaryPath = path.join(directoryArg, `${binary}_${target.friendly}`);
+        if (target.friendly.startsWith('macos_')) {
+          const architecture = target.friendly === 'macos_arm64' ? '-arm64' : '-x86_64';
+          await this.commandRunner.execute(
+            '/usr/bin/arch',
+            [architecture, binaryPath, ...this.plan.smokeTestArgs],
+            {
+              cwd: directoryArg,
+              timeoutMs: Math.min(
+                capabilityCommandTimeoutMs,
+                this.remainingBuildTimeout(buildDeadlineArg, workerArg.host),
+              ),
+              environment: environmentArg,
+            },
+          );
+        } else {
+          await this.commandRunner.execute(binaryPath, this.plan.smokeTestArgs, {
+            cwd: directoryArg,
+            timeoutMs: Math.min(
+              capabilityCommandTimeoutMs,
+              this.remainingBuildTimeout(buildDeadlineArg, workerArg.host),
+            ),
+            environment: environmentArg,
+          });
+        }
+      }
+    }
+  }
+
+  private describeRemoteWorkspace(
+    workerArg: IResolvedMatrixWorker,
+    runtimeArg: IResolvedSshWorker,
+    gitCommitArg: string,
+  ): IRemoteWorkspace {
+    const ownerToken = crypto.randomUUID();
+    const directory = `${runtimeArg.temporaryRoot}/tsrust-matrix.${gitCommitArg}.${workerArg.host}.${ownerToken}`;
+    return { worker: workerArg, runtime: runtimeArg, directory, ownerToken, gitCommit: gitCommitArg };
+  }
+
+  private async createRemoteWorkspace(
+    remoteArg: IRemoteWorkspace,
+    gitCommitArg: string,
+  ): Promise<void> {
+    const { ownerToken, directory } = remoteArg;
+    const ownerContents = `${ownerToken}\n${gitCommitArg}\n${directory}`;
+    const ownerPath = `${directory}/${matrixOwnerFileName}`;
+    const script = [
+      'set -eu',
+      'umask 077',
+      ...this.remoteRootValidationCommands(remoteArg.runtime.temporaryRoot),
+      `test "$(find "$root" -mindepth 1 -maxdepth 1 -type d -name 'tsrust-matrix.*' | wc -l | tr -d ' ')" -lt ${maximumRetainedDiagnosticRuns}`,
+      `directory=${shellQuote(directory)}`,
+      `test "$directory" = ${shellQuote(directory)}`,
+      'created=0',
+      `cleanup_unowned() { if [ "$created" = 1 ]; then rm -f ${shellQuote(ownerPath)}; rmdir "$directory" 2>/dev/null || true; fi; }`,
+      'trap cleanup_unowned EXIT HUP INT TERM',
+      'mkdir -m 700 "$directory"',
+      'created=1',
+      `printf '%s\n' ${shellQuote(ownerToken)} ${shellQuote(gitCommitArg)} ${shellQuote(directory)} > ${shellQuote(ownerPath)}`,
+      `chmod 600 ${shellQuote(ownerPath)}`,
+      `test ! -L ${shellQuote(ownerPath)}`,
+      `test "$(cat ${shellQuote(ownerPath)})" = ${shellQuote(ownerContents)}`,
+      'trap - EXIT HUP INT TERM',
+    ].join('\n');
+    await this.runRemoteLoginScript(
+      remoteArg.worker,
+      remoteArg.runtime,
+      script,
+      capabilityCommandTimeoutMs,
+    );
+  }
+
+  private async buildSshWorker(
+    remoteArg: IRemoteWorkspace,
+    bundlePathArg: string,
+    runDirectoryArg: string,
+    gitCommitArg: string,
+    identityArg: { projectName: string; projectVersion: string },
+    binariesArg: string[],
+  ): Promise<string> {
+    const buildDeadline = Date.now() + buildCommandTimeoutMs;
+    const remoteBundle = `${remoteArg.directory}/source.bundle`;
+    const remoteWorkspace = `${remoteArg.directory}/workspace`;
+    const remoteBundleTemporary = `${remoteBundle}.${crypto.randomUUID()}.tmp`;
+    await this.runRemoteTransferScript(
+      remoteArg,
+      [
+        `bundle=${shellQuote(remoteBundle)}`,
+        `bundle_temporary=${shellQuote(remoteBundleTemporary)}`,
+        'trap \'rm -f "$bundle_temporary"\' EXIT HUP INT TERM',
+        'cat > "$bundle_temporary"',
+        'chmod 600 "$bundle_temporary"',
+        'mv "$bundle_temporary" "$bundle"',
+        'trap - EXIT HUP INT TERM',
+      ].join('\n'),
+      {
+        inputFile: bundlePathArg,
+        timeoutMs: this.remainingBuildTimeout(buildDeadline, remoteArg.worker.host),
+      },
+    );
+    const expectedNames = artifactNamesFor(remoteArg.worker, binariesArg);
+    const remoteDist = `${remoteWorkspace}/dist_rust`;
+    const remoteHome = `${remoteArg.directory}/home`;
+    const remoteCargoHome = `${remoteArg.directory}/cargo-home`;
+    const buildCommands = [
+      'set -eu',
+      ...this.remoteRootValidationCommands(remoteArg.runtime.temporaryRoot),
+      ...this.remoteOwnershipValidationCommands(remoteArg, gitCommitArg),
+      `bundle=${shellQuote(remoteBundle)}`,
+      `workspace=${shellQuote(remoteWorkspace)}`,
+      `expected_commit=${shellQuote(gitCommitArg)}`,
+      `mkdir -m 700 ${shellQuote(remoteHome)} ${shellQuote(remoteCargoHome)}`,
+      'git -c advice.detachedHead=false clone --quiet "$bundle" "$workspace"',
+      'cd "$workspace"',
+      'test "$(git rev-parse HEAD)" = "$expected_commit"',
+      'test -z "$(git status --porcelain=v1 --untracked-files=all)"',
+      `test -z "$(git ls-files -v | grep -v '^H ' || true)"`,
+      this.remoteWorkerEnvironmentFunction(remoteHome, remoteCargoHome),
+      ['run_matrix_worker', 'pnpm', 'install', '--frozen-lockfile']
+        .map(shellQuote)
+        .join(' '),
+      [
+        'run_matrix_worker',
+        'node',
+        '-e',
+        `const value = require('./node_modules/@git.zone/tsrust/package.json'); if (value.name !== '@git.zone/tsrust' || value.version !== ${JSON.stringify(commitinfo.version)}) process.exit(1);`,
+      ]
+        .map(shellQuote)
+        .join(' '),
+      ['run_matrix_worker', 'pnpm', ...this.workerTsrustArgs(remoteArg.worker)]
+        .map(shellQuote)
+        .join(' '),
+      'test "$(git rev-parse HEAD)" = "$expected_commit"',
+      'test -z "$(git status --porcelain=v1 --untracked-files=all)"',
+      `test -z "$(git ls-files -v | grep -v '^H ' || true)"`,
+      `actual_names="$(find ${shellQuote(remoteDist)} -mindepth 1 -maxdepth 1 -exec basename {} \\; | LC_ALL=C sort)"`,
+      `test "$actual_names" = ${shellQuote(expectedNames.join('\n'))}`,
+      ...expectedNames.map(
+        (name) =>
+          `test -f ${shellQuote(`${remoteDist}/${name}`)}; test ! -L ${shellQuote(`${remoteDist}/${name}`)}`,
+      ),
+      'artifact_total=0',
+      ...expectedNames.map((name) => {
+        const artifactPath = `${remoteDist}/${name}`;
+        const maximumBytes = name.endsWith(PROVENANCE_SIDECAR_SUFFIX)
+          ? MAX_PROVENANCE_SIDECAR_BYTES
+          : MAX_ARTIFACT_BYTES;
+        return `if [ "$(uname -s)" = Darwin ]; then artifact_size="$(stat -f %z ${shellQuote(artifactPath)})"; else artifact_size="$(stat -c %s ${shellQuote(artifactPath)})"; fi; test "$artifact_size" -le ${maximumBytes}; artifact_total=$((artifact_total + artifact_size)); test "$artifact_total" -le ${MAX_ASSEMBLY_INPUT_BYTES}`;
+      }),
+      ...remoteArg.worker.targets.flatMap((target) =>
+        binariesArg.map(
+          (binary) => `test -x ${shellQuote(`${remoteDist}/${binary}_${target.friendly}`)}`,
+        ),
+      ),
+      ...this.remoteSmokeCommands(remoteArg.worker, remoteDist, binariesArg),
+      'test "$(git rev-parse HEAD)" = "$expected_commit"',
+      'test -z "$(git status --porcelain=v1 --untracked-files=all)"',
+      `test -z "$(git ls-files -v | grep -v '^H ' || true)"`,
+    ];
+    await this.runRemoteOwnedScript(
+      remoteArg.worker,
+      remoteArg.runtime,
+      buildCommands.join('\n'),
+      this.remainingBuildTimeout(buildDeadline, remoteArg.worker.host),
+    );
+
+    const localArtifactDirectory = path.join(runDirectoryArg, remoteArg.worker.host, 'artifacts');
+    await fs.promises.mkdir(localArtifactDirectory, { recursive: true, mode: 0o700 });
+    let downloadedBytes = 0;
+    for (const name of expectedNames) {
+      const remoteFile = `${remoteDist}/${name}`;
+      const localFile = path.join(localArtifactDirectory, name);
+      const maximumBytes = name.endsWith(PROVENANCE_SIDECAR_SUFFIX)
+        ? MAX_PROVENANCE_SIDECAR_BYTES
+        : MAX_ARTIFACT_BYTES;
+      const blockSize = 1024 * 1024;
+      const blockCount = Math.ceil(maximumBytes / blockSize) + 1;
+      try {
+        await this.runRemoteTransferScript(
+          remoteArg,
+          [
+            `file=${shellQuote(remoteFile)}`,
+            `test "$file" = ${shellQuote(remoteFile)}`,
+            'test -f "$file"',
+            'test ! -L "$file"',
+            'if [ "$(uname -s)" = Darwin ]; then file_size="$(stat -f %z "$file")"; else file_size="$(stat -c %s "$file")"; fi',
+            `test "$file_size" -le ${maximumBytes}`,
+            `dd if="$file" bs=${blockSize} count=${blockCount} 2>/dev/null`,
+          ].join('\n'),
+          {
+            outputFile: localFile,
+            maximumOutputBytes: maximumBytes,
+            timeoutMs: this.remainingBuildTimeout(buildDeadline, remoteArg.worker.host),
+          },
+        );
+        const downloaded = await fs.promises.lstat(localFile);
+        if (!downloaded.isFile() || downloaded.isSymbolicLink() || downloaded.size > maximumBytes) {
+          throw new Error(`Remote matrix artifact exceeds its size limit: ${name}`);
+        }
+        downloadedBytes += downloaded.size;
+        if (downloadedBytes > MAX_ASSEMBLY_INPUT_BYTES) {
+          throw new Error('Remote matrix artifacts exceed the aggregate size limit');
+        }
+        await fs.promises.chmod(
+          localFile,
+          name.endsWith(PROVENANCE_SIDECAR_SUFFIX) ? 0o644 : 0o755,
+        );
+      } catch (error) {
+        await fs.promises.rm(localFile, { force: true });
+        throw error;
+      }
+    }
+    await this.assertArtifactDirectory(
+      localArtifactDirectory,
+      remoteArg.worker,
+      binariesArg,
+      gitCommitArg,
+      identityArg,
+    );
+    this.remainingBuildTimeout(buildDeadline, remoteArg.worker.host);
+    return localArtifactDirectory;
+  }
+
+  private remoteSmokeCommands(
+    workerArg: IResolvedMatrixWorker,
+    directoryArg: string,
+    binariesArg: string[],
+  ): string[] {
+    if (!this.plan.smokeTestArgs) return [];
+    const commands: string[] = [];
+    for (const target of workerArg.targets) {
+      for (const binary of binariesArg) {
+        const binaryPath = `${directoryArg}/${binary}_${target.friendly}`;
+        if (target.friendly.startsWith('macos_')) {
+          const architecture = target.friendly === 'macos_arm64' ? '-arm64' : '-x86_64';
+          commands.push(
+            [
+              'run_matrix_worker',
+              '/usr/bin/arch',
+              architecture,
+              binaryPath,
+              ...this.plan.smokeTestArgs,
+            ]
+              .map(shellQuote)
+              .join(' '),
+          );
+        } else {
+          commands.push(
+            ['run_matrix_worker', binaryPath, ...this.plan.smokeTestArgs]
+              .map(shellQuote)
+              .join(' '),
+          );
+        }
+      }
+    }
+    return commands;
+  }
+
+  private async cleanupRemoteWorkspace(
+    remoteArg: IRemoteWorkspace,
+    gitCommitArg: string,
+  ): Promise<void> {
+    const script = [
+      'set -eu',
+      ...this.remoteRootValidationCommands(remoteArg.runtime.temporaryRoot),
+      ...this.remoteOwnershipValidationCommands(remoteArg, gitCommitArg),
+      'rm -rf "$directory"',
+    ].join('\n');
+    await this.runRemoteLoginScript(
+      remoteArg.worker,
+      remoteArg.runtime,
+      script,
+      capabilityCommandTimeoutMs,
+    );
+  }
+
+  private async runRemoteTransferScript(
+    remoteArg: IRemoteWorkspace,
+    scriptArg: string,
+    optionsArg: Pick<
+      IMatrixCommandOptions,
+      'inputFile' | 'outputFile' | 'maximumOutputBytes' | 'timeoutMs'
+    >,
+  ): Promise<void> {
+    const details = expectedHostDetails(remoteArg.worker.host);
+    const guardedScript = [
+      'set -eu',
+      `test "$(uname -s)" = ${shellQuote(details.system)}`,
+      `test "$(uname -m)" = ${shellQuote(details.machine)}`,
+      ...this.remoteRootValidationCommands(remoteArg.runtime.temporaryRoot),
+      ...this.remoteOwnershipValidationCommands(remoteArg, remoteArg.gitCommit),
+      scriptArg,
+    ].join('\n');
+    await this.commandRunner.execute(
+      'ssh',
+      [
+        '-T',
+        ...sshConnectionArgs,
+        '--',
+        remoteArg.runtime.destination,
+        `/bin/sh -c ${shellQuote(guardedScript)}`,
+      ],
+      {
+        cwd: this.workspace,
+        timeoutMs: optionsArg.timeoutMs,
+        inputFile: optionsArg.inputFile,
+        outputFile: optionsArg.outputFile,
+        maximumOutputBytes: optionsArg.maximumOutputBytes,
+        environment: this.transportEnvironment(),
+      },
+    );
+  }
+
+  private async runRemoteLoginScript(
+    workerArg: IResolvedMatrixWorker,
+    runtimeArg: IResolvedSshWorker,
+    scriptArg: string,
+    timeoutMsArg: number,
+  ): Promise<void> {
+    const shell = workerArg.host.startsWith('macos_') ? '/bin/zsh' : '/bin/bash';
+    const shellFlags = workerArg.host.startsWith('macos_') ? '-lic' : '-lc';
+    const details = expectedHostDetails(workerArg.host);
+    const guardedScript = [
+      'set -eu',
+      `test "$(uname -s)" = ${shellQuote(details.system)}`,
+      `test "$(uname -m)" = ${shellQuote(details.machine)}`,
+      scriptArg,
+    ].join('\n');
+    await this.commandRunner.execute(
+      'ssh',
+      [
+        ...sshConnectionArgs,
+        '--',
+        runtimeArg.destination,
+        `${shell} ${shellFlags} ${shellQuote(guardedScript)}`,
+      ],
+      {
+        cwd: this.workspace,
+        timeoutMs: timeoutMsArg,
+        environment: this.transportEnvironment(),
+      },
+    );
+  }
+
+  private async runRemoteOwnedScript(
+    workerArg: IResolvedMatrixWorker,
+    runtimeArg: IResolvedSshWorker,
+    scriptArg: string,
+    timeoutMsArg: number,
+  ): Promise<void> {
+    const shell = workerArg.host.startsWith('macos_') ? '/bin/zsh' : '/bin/bash';
+    const outerFlags = workerArg.host.startsWith('macos_') ? '-lic' : '-lc';
+    const details = expectedHostDetails(workerArg.host);
+    const guardedScript = [
+      'set -eu',
+      `test "$(uname -s)" = ${shellQuote(details.system)}`,
+      `test "$(uname -m)" = ${shellQuote(details.machine)}`,
+      scriptArg,
+    ].join('\n');
+    const watchdogCommand = ['node', '-e', remoteWatchdogSource, shell, guardedScript]
+      .map(shellQuote)
+      .join(' ');
+    await this.commandRunner.execute(
+      'ssh',
+      [
+        ...sshConnectionArgs,
+        '--',
+        runtimeArg.destination,
+        `${shell} ${outerFlags} ${shellQuote(watchdogCommand)}`,
+      ],
+      {
+        cwd: this.workspace,
+        timeoutMs: timeoutMsArg,
+        heartbeatStdin: true,
+        environment: this.transportEnvironment(),
+      },
+    );
+  }
+}