@zgeoff/atc 2.11.0 → 2.13.0

package/src/store/trail-entry.ts

@@ -18,6 +18,10 @@ interface MessageTrailEntry extends TrailEntryBase {
 interface ReportTrailEntry extends TrailEntryBase {
   readonly kind: 'report';
   readonly label: string;
+
+  // The id a remote session's reporter gave the report, so a resent
+  // report is stored once; absent for a report that carries none.
+  readonly reportID?: string;
 }
 
 /**

package/src/store/workspace-materialization.ts

@@ -0,0 +1,70 @@
+import type { SessionID } from '../shared/session-id';
+
+/**
+ * Where a spawn's workspace stands on its way to the execution target:
+ * resolving the source to a commit, cloning it on the daemon's host,
+ * transferring the archive to the target, and verifying the target's
+ * checkout, then `ready` or `failed`. A row a stopped daemon left in any
+ * other phase is reconciled to `failed` before the next daemon serves a
+ * request, so no workspace ever counts as ready without its verification.
+ */
+export type MaterializationPhase =
+  | 'resolving'
+  | 'cloning'
+  | 'transferring'
+  | 'verifying'
+  | 'ready'
+  | 'failed';
+
+/**
+ * One spawn's workspace materialization, keyed by the session it is for.
+ * The repository URL is the token-free form, and the row never holds a
+ * credential or the reference to one.
+ */
+export interface WorkspaceMaterialization {
+  readonly sessionID: SessionID;
+  readonly target: string;
+
+  // The directory on the target the workspace is materialized into.
+  readonly dir: string;
+  readonly sourceKind: 'path' | 'git';
+  readonly phase: MaterializationPhase;
+  readonly repoURL: string | null;
+  readonly sha: string | null;
+
+  // The branch or tag the commit was resolved from; null for a pinned
+  // commit or a detached checkout.
+  readonly ref: string | null;
+
+  // The refusal that failed the materialization; null unless failed.
+  readonly errorCode: string | null;
+  readonly startedAt: number;
+  readonly updatedAt: number;
+  readonly materializedAt: number | null;
+
+  // The environment variable names every harness the session starts goes
+  // without: the workspace credential's variable and the askpass context.
+  readonly withheldEnv: readonly string[];
+}
+
+// The fields a materialization rewrites as it moves through its phases.
+export interface MaterializationUpdate {
+  readonly phase: MaterializationPhase;
+  readonly repoURL?: string;
+  readonly sha?: string;
+  readonly ref?: string | null;
+  readonly errorCode?: string;
+  readonly materializedAt?: number;
+}
+
+/**
+ * What a session's ready workspace was built from: the token-free
+ * repository URL, the commit checked out, the branch or tag it came from,
+ * and when it became ready.
+ */
+export interface SessionWorkspace {
+  readonly repoURL: string;
+  readonly sha: string;
+  readonly ref?: string;
+  readonly materializedAt: number;
+}

package/src/tap.ts

@@ -1,6 +1,7 @@
 import { z } from 'zod';
 import { DaemonClient } from './client/daemon-client';
 import { DaemonError } from './protocol/daemon-error';
+import { runBridgeTap } from './run-bridge-tap';
 import { daemonSocketPath } from './shared/config';
 import { getBuild } from './shared/get-build';
 
@@ -28,6 +29,16 @@ interface InboxMessage {
  * another tap replaced it or the session was removed.
  */
 export async function runTap(session: string): Promise<void> {
+  const bridge = process.env['ATC_SOCKET'];
+
+  // Inside a remote host the daemon's own socket is out of reach, and the
+  // session bridge serves the tap instead.
+  if (process.env['ATC_BRIDGE'] === '1' && bridge !== undefined && bridge !== '') {
+    await runBridgeTap(bridge, process.env['ATC_OUTBOX'] ?? '');
+
+    return;
+  }
+
   let client: DaemonClient;
 
   try {

package/src/workspace/check-url-credentials.ts

@@ -0,0 +1,59 @@
+import { runGit } from './run-git';
+
+type URLCredentialFinding =
+  | { readonly ok: true }
+  | { readonly ok: false; readonly code: 'credential_in_url'; readonly message: string };
+
+/**
+ * Checks that a repository URL reaches git without a credential in it, so
+ * a token can never surface in git's own messages or in the clone. The URL
+ * itself must carry none, and no `url.<base>.insteadOf` rewrite in the git
+ * config read from `cwd` may expand it into one that does. git expands the
+ * URL without touching the network. An http(s) URL carries a credential in
+ * its userinfo, query, or fragment; any other scheme URL only in a
+ * password, since its user is a login name such as `git`.
+ */
+export async function checkURLCredentials(url: string, cwd: string): Promise<URLCredentialFinding> {
+  // git prints the URL as it would fetch it, with every rewrite applied.
+  const expanded = await runGit(['ls-remote', '--get-url', url], { cwd });
+
+  const fetched = expanded.exitCode === 0 ? expanded.stdout.trim() : url;
+
+  if (!hasURLCredential(fetched)) {
+    return { ok: true };
+  }
+
+  return {
+    ok: false,
+    code: 'credential_in_url',
+    message: hasURLCredential(url)
+      ? 'the repository URL carries a credential; pass it as a credentialRef instead'
+      : 'a url.<base>.insteadOf rewrite in the host git config puts a credential into the repository URL; remove the rewrite or pass the credential as a credentialRef',
+  };
+}
+
+const SCHEME_PATTERN = /^[a-z][a-z\d+.-]*:\/\//iu;
+
+const HTTP_PROTOCOLS: ReadonlySet<string> = new Set(['http:', 'https:']);
+
+function hasURLCredential(url: string): boolean {
+  if (!SCHEME_PATTERN.test(url)) {
+    return false;
+  }
+
+  let parsed: URL;
+
+  try {
+    parsed = new URL(url);
+  } catch {
+    return false;
+  }
+
+  if (HTTP_PROTOCOLS.has(parsed.protocol)) {
+    return (
+      parsed.username !== '' || parsed.password !== '' || parsed.search !== '' || parsed.hash !== ''
+    );
+  }
+
+  return parsed.password !== '';
+}

package/src/workspace/check-workspace-completeness.ts

@@ -0,0 +1,90 @@
+import { runGit } from './run-git';
+
+type CompletenessFinding =
+  | { readonly ok: true }
+  | {
+      readonly ok: false;
+      readonly code: 'has_submodules' | 'unreadable_tree';
+      readonly message: string;
+    }
+  | {
+      readonly ok: false;
+      readonly code: 'lfs_unsupported';
+      readonly message: string;
+      readonly count: number;
+      readonly paths: readonly string[];
+    };
+
+const LFS_PATHS_SHOWN = 5;
+
+/**
+ * Checks that a checked-out commit is the whole workspace, so a checkout is
+ * never handed on with holes in it. The commit's own tree must hold no
+ * gitlink and no `.gitmodules`, since submodules are never initialized, and
+ * no tracked path may carry the `lfs` filter, since its content would be a
+ * pointer file rather than the object it points at.
+ */
+export async function checkWorkspaceCompleteness(
+  dir: string,
+  sha: string,
+): Promise<CompletenessFinding> {
+  const tree = await runGit(['ls-tree', '-r', '--full-tree', sha], { cwd: dir, isolated: true });
+
+  if (tree.exitCode !== 0) {
+    return {
+      ok: false,
+      code: 'unreadable_tree',
+      message: `cannot list the tree of ${sha}: ${tree.stderr.trim()}`,
+    };
+  }
+
+  const entries = tree.stdout.split('\n');
+
+  if (entries.some((line) => line.startsWith('160000 ') || line.endsWith('\t.gitmodules'))) {
+    return { ok: false, code: 'has_submodules', message: `${sha} uses submodules` };
+  }
+
+  const tracked = await runGit(['ls-files', '-z'], { cwd: dir, isolated: true });
+
+  const attributes = await runGit(['check-attr', '--stdin', '-z', 'filter'], {
+    cwd: dir,
+    input: tracked.stdout,
+    isolated: true,
+  });
+
+  if (tracked.exitCode !== 0 || attributes.exitCode !== 0) {
+    return {
+      ok: false,
+      code: 'unreadable_tree',
+      message: `cannot read the attributes of ${sha}: ${tracked.stderr.trim()}${attributes.stderr.trim()}`,
+    };
+  }
+
+  const lfsPaths = collectLFSPaths(attributes.stdout);
+
+  if (lfsPaths.length > 0) {
+    return {
+      ok: false,
+      code: 'lfs_unsupported',
+      message: `${sha} tracks ${lfsPaths.length} path(s) through Git LFS`,
+      count: lfsPaths.length,
+      paths: lfsPaths.slice(0, LFS_PATHS_SHOWN),
+    };
+  }
+
+  return { ok: true };
+}
+
+// `check-attr -z` writes one path, attribute, value triple per path.
+function collectLFSPaths(output: string): string[] {
+  const fields = output.split('\0');
+  const paths: string[] = [];
+
+  for (let index = 0; index + 2 < fields.length; index += 3) {
+    if (fields[index + 2] === 'lfs') {
+      paths.push(fields[index] ?? '');
+    }
+  }
+
+  return paths;
+}

package/src/workspace/create-workspace-clone.ts

@@ -0,0 +1,251 @@
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { checkWorkspaceCompleteness } from './check-workspace-completeness';
+import { runGit } from './run-git';
+import type { WorkspaceSource } from './workspace-source';
+
+interface GitCredential {
+  readonly kind: 'env';
+  readonly name: string;
+}
+
+interface CloneRequest {
+  readonly source: Extract<WorkspaceSource, { readonly kind: 'git' }>;
+  readonly dir: string;
+  readonly credential?: GitCredential;
+}
+
+interface CreatedClone {
+  readonly ok: true;
+  readonly sha: string;
+  readonly branch: string | null;
+}
+
+interface CloneRefusal {
+  readonly ok: false;
+  readonly code: 'credential_missing' | 'ref_not_found' | 'clone_failed';
+  readonly message: string;
+}
+
+type IncompleteCheckout = Exclude<
+  Awaited<ReturnType<typeof checkWorkspaceCompleteness>>,
+  { readonly ok: true }
+>;
+
+/**
+ * Clones a repository into an empty directory and checks out the commit its
+ * ref points at that moment. A branch ref is checked out as that branch; a tag
+ * or a full commit id leaves HEAD detached at the commit. The clone copies
+ * objects rather than hard-linking them, takes no hook templates, and runs no
+ * hooks on checkout.
+ *
+ * A ref, or a full commit id, that the upstream does not have is refused as
+ * `ref_not_found`, and the directory is removed.
+ *
+ * The checked-out commit must be the whole workspace: one that uses
+ * submodules or tracks paths through Git LFS is refused and the directory is
+ * removed. The clone fetches without checking out, and the checkout then
+ * runs isolated from the host's system and global git config, so no filter
+ * the host configured, LFS or otherwise, runs or reaches the network.
+ *
+ * Without a credential, git authenticates through the host's own git
+ * config. An env credential is read from the named variable and handed to git
+ * only through a private askpass helper for the network commands, never
+ * through the command line or the URL, and the helper is deleted before this
+ * returns. The host's credential helpers are switched off for those commands
+ * so none of them stores the token.
+ */
+export async function createWorkspaceClone(
+  request: CloneRequest,
+): Promise<CloneRefusal | CreatedClone | IncompleteCheckout> {
+  const askpass = await createAskpass(request.credential);
+
+  if (!askpass.ok) {
+    return askpass;
+  }
+
+  try {
+    return await createCloneAtRef(request, askpass.env, askpass.args);
+  } finally {
+    await askpass[Symbol.asyncDispose]();
+  }
+}
+
+interface Askpass {
+  readonly ok: true;
+  readonly env: Readonly<Record<string, string>>;
+  readonly args: readonly string[];
+  [Symbol.asyncDispose]: () => Promise<void>;
+}
+
+const ASKPASS_SECRET_VAR = 'ATC_GIT_ASKPASS_SECRET';
+
+// Username prompts get a fixed name that GitHub and GitLab both accept
+// alongside a token; password prompts get the token.
+const ASKPASS_SCRIPT = `#!/bin/sh
+case "$1" in
+  Username*) printf '%s\\n' x-access-token ;;
+  *) printf '%s\\n' "$${ASKPASS_SECRET_VAR}" ;;
+esac
+`;
+
+async function createAskpass(
+  credential: GitCredential | undefined,
+): Promise<Askpass | CloneRefusal> {
+  if (credential === undefined) {
+    return { ok: true, env: {}, args: [], [Symbol.asyncDispose]: async () => {} };
+  }
+
+  const secret = process.env[credential.name];
+
+  if (secret === undefined || secret === '') {
+    return {
+      ok: false,
+      code: 'credential_missing',
+      message: 'the credential environment variable is unset or empty',
+    };
+  }
+
+  const dir = await mkdtemp(join(tmpdir(), 'atc-askpass-'));
+
+  const helper = join(dir, 'askpass');
+
+  await writeFile(helper, ASKPASS_SCRIPT, { mode: 0o700 });
+
+  return {
+    ok: true,
+    env: { GIT_ASKPASS: helper, [ASKPASS_SECRET_VAR]: secret },
+    args: ['-c', 'credential.helper='],
+    [Symbol.asyncDispose]: async () => {
+      await rm(dir, { recursive: true, force: true });
+    },
+  };
+}
+
+async function createCloneAtRef(
+  request: CloneRequest,
+  env: Readonly<Record<string, string>>,
+  args: readonly string[],
+): Promise<CloneRefusal | CreatedClone | IncompleteCheckout> {
+  const target = await resolveRef(request.source, env, args);
+
+  if (!target.ok) {
+    return target;
+  }
+
+  const clone = await runGit(
+    [
+      ...args,
+      'clone',
+      '--quiet',
+      '--no-local',
+      '--no-hardlinks',
+      '--no-checkout',
+      '--template=',
+      '--',
+      request.source.url,
+      request.dir,
+    ],
+    { env },
+  );
+
+  if (clone.exitCode !== 0) {
+    return { ok: false, code: 'clone_failed', message: clone.stderr.trim() };
+  }
+
+  const checkout = await runGit(
+    [
+      '-c',
+      'core.hooksPath=/dev/null',
+      'checkout',
+      '--quiet',
+      ...(target.branch === null ? ['--detach', target.sha] : ['-B', target.branch, target.sha]),
+    ],
+    { cwd: request.dir, isolated: true },
+  );
+
+  if (checkout.exitCode !== 0) {
+    // A pinned commit that no upstream ref reaches never arrives in the
+    // clone, so its checkout fails on a commit the upstream does not have.
+    const present = await runGit(['cat-file', '-e', `${target.sha}^{commit}`], {
+      cwd: request.dir,
+      isolated: true,
+    });
+
+    await rm(request.dir, { recursive: true, force: true });
+
+    return target.branch === null && present.exitCode !== 0
+      ? { ok: false, code: 'ref_not_found', message: `origin has no commit ${target.sha}` }
+      : { ok: false, code: 'clone_failed', message: checkout.stderr.trim() };
+  }
+
+  const complete = await checkWorkspaceCompleteness(request.dir, target.sha);
+
+  if (!complete.ok) {
+    await rm(request.dir, { recursive: true, force: true });
+
+    return complete;
+  }
+
+  return { ok: true, sha: target.sha, branch: target.branch };
+}
+
+interface ResolvedRef {
+  readonly ok: true;
+  readonly sha: string;
+  readonly branch: string | null;
+}
+
+const SHA_PATTERN = /^(?:[\da-f]{40}|[\da-f]{64})$/u;
+
+/**
+ * Pins a ref to a commit before cloning, so the checkout is exactly the
+ * commit the ref pointed at when asked, whatever lands on the branch meanwhile.
+ * A branch wins over a same-named tag, and an annotated tag resolves to the
+ * commit it points at.
+ */
+async function resolveRef(
+  source: Extract<WorkspaceSource, { readonly kind: 'git' }>,
+  env: Readonly<Record<string, string>>,
+  args: readonly string[],
+): Promise<CloneRefusal | ResolvedRef> {
+  if (SHA_PATTERN.test(source.ref)) {
+    return { ok: true, sha: source.ref, branch: null };
+  }
+
+  const listed = await runGit(
+    [...args, 'ls-remote', '--', source.url, source.ref, `${source.ref}^{}`],
+    { env },
+  );
+
+  if (listed.exitCode !== 0) {
+    return { ok: false, code: 'clone_failed', message: listed.stderr.trim() };
+  }
+
+  const refs = new Map(
+    listed.stdout
+      .split('\n')
+      .filter((line) => line !== '')
+      .map((line) => {
+        const [sha = '', name = ''] = line.split('\t');
+
+        return [name, sha] as const;
+      }),
+  );
+
+  const name = source.ref.replace(/^refs\/(?:heads|tags)\//u, '');
+  const branchSHA = refs.get(`refs/heads/${name}`);
+
+  if (branchSHA !== undefined && !source.ref.startsWith('refs/tags/')) {
+    return { ok: true, sha: branchSHA, branch: name };
+  }
+
+  const tagSHA = refs.get(`refs/tags/${name}^{}`) ?? refs.get(`refs/tags/${name}`);
+
+  if (tagSHA !== undefined && !source.ref.startsWith('refs/heads/')) {
+    return { ok: true, sha: tagSHA, branch: null };
+  }
+
+  return { ok: false, code: 'ref_not_found', message: `origin has no branch or tag '${name}'` };
+}

package/src/workspace/normalize-git-url.ts

@@ -0,0 +1,59 @@
+type NormalizedGitURL =
+  | { readonly ok: true; readonly url: string }
+  | { readonly ok: false; readonly code: 'invalid_git_url'; readonly message: string };
+
+const SCHEME_PATTERN = /^[a-z][a-z\d+.-]*:\/\//iu;
+const SCP_PATTERN = /^(?:[^@/:\s]+@)?[^@/:\s]+:[^@\s]+$/u;
+const SHORTHAND_PATTERN = /^[\w.-]+\/[\w.-]+$/u;
+
+/**
+ * Reduces a repository URL to a form that carries no credential, so it can be
+ * recorded, shown, and handed to another host. An http(s) URL loses its
+ * userinfo, query, and fragment, where tokens ride. Any other scheme URL
+ * loses its password but keeps its user, which is a login name such as
+ * `git`. An scp-style `git@host:owner/repo` and a local path pass through,
+ * and a bare `owner/repo` expands to its GitHub https URL.
+ */
+export function normalizeGitURL(raw: string): NormalizedGitURL {
+  const trimmed = raw.trim();
+
+  if (SCHEME_PATTERN.test(trimmed)) {
+    return normalizeSchemeURL(trimmed);
+  }
+
+  if (trimmed.startsWith('/')) {
+    return { ok: true, url: trimmed };
+  }
+
+  if (SHORTHAND_PATTERN.test(trimmed) && !trimmed.startsWith('.')) {
+    return { ok: true, url: `https://github.com/${trimmed.replace(/\.git$/u, '')}.git` };
+  }
+
+  if (SCP_PATTERN.test(trimmed)) {
+    return { ok: true, url: trimmed };
+  }
+
+  return { ok: false, code: 'invalid_git_url', message: 'not a git repository URL' };
+}
+
+const HTTP_PROTOCOLS: ReadonlySet<string> = new Set(['http:', 'https:']);
+
+function normalizeSchemeURL(raw: string): NormalizedGitURL {
+  let url: URL;
+
+  try {
+    url = new URL(raw);
+  } catch {
+    return { ok: false, code: 'invalid_git_url', message: 'not a parseable repository URL' };
+  }
+
+  url.password = '';
+
+  if (HTTP_PROTOCOLS.has(url.protocol)) {
+    url.username = '';
+    url.search = '';
+    url.hash = '';
+  }
+
+  return { ok: true, url: url.toString() };
+}

package/src/workspace/read-workspace-tar.ts

@@ -0,0 +1,38 @@
+type TarOutcome =
+  | { readonly ok: true }
+  | { readonly ok: false; readonly code: 'tar_failed'; readonly message: string };
+
+interface WorkspaceTar {
+  readonly stream: ReadableStream<Uint8Array>;
+  readonly done: Promise<TarOutcome>;
+}
+
+/**
+ * Streams an uncompressed tar of a directory, `.git` included, with entries
+ * relative to the directory. The archive is read as it is produced and never
+ * written to disk. A failure partway through still ends the stream, so a
+ * consumer drains it and then awaits the outcome before trusting what it
+ * received. Options the host passes to GNU tar through `TAR_OPTIONS` are
+ * ignored, since one could follow symlinks out of the directory or leave
+ * files out.
+ */
+export function readWorkspaceTar(dir: string): WorkspaceTar {
+  const { TAR_OPTIONS: _ignored, ...env } = process.env;
+
+  const proc = Bun.spawn(['tar', '-c', '-f', '-', '-C', dir, '.'], {
+    env,
+    stdin: 'ignore',
+    stdout: 'pipe',
+    stderr: 'pipe',
+  });
+
+  const done = (async (): Promise<TarOutcome> => {
+    const [stderr, exitCode] = await Promise.all([new Response(proc.stderr).text(), proc.exited]);
+
+    return exitCode === 0
+      ? { ok: true }
+      : { ok: false, code: 'tar_failed', message: stderr.trim() };
+  })();
+
+  return { stream: proc.stdout, done };
+}

package/src/workspace/repository-env-vars.ts

@@ -0,0 +1,22 @@
+/**
+ * The variables that pin git to a repository other than the one in its
+ * working directory, such as the `GIT_DIR` a git hook exports. A command
+ * that must act on its own directory runs without every one of them.
+ */
+export const REPOSITORY_ENV_VARS: ReadonlySet<string> = new Set([
+  'GIT_ALTERNATE_OBJECT_DIRECTORIES',
+  'GIT_COMMON_DIR',
+  'GIT_CONFIG',
+  'GIT_CONFIG_COUNT',
+  'GIT_CONFIG_PARAMETERS',
+  'GIT_DIR',
+  'GIT_GRAFT_FILE',
+  'GIT_IMPLICIT_WORK_TREE',
+  'GIT_INDEX_FILE',
+  'GIT_NO_REPLACE_OBJECTS',
+  'GIT_OBJECT_DIRECTORY',
+  'GIT_PREFIX',
+  'GIT_REPLACE_REF_BASE',
+  'GIT_SHALLOW_FILE',
+  'GIT_WORK_TREE',
+]);