@zgeoff/atc 2.12.0 → 2.14.0

package/src/store/state-store.ts

@@ -24,6 +24,11 @@ import type { MessageRecord } from './message-record';
 import { runMigrations } from './run-migrations';
 import type { StateStoreSchema } from './run-migrations';
 import type { TrailEntry } from './trail-entry';
+import type {
+  MaterializationUpdate,
+  SessionWorkspace,
+  WorkspaceMaterialization,
+} from './workspace-materialization';
 
 // Spelled as the partial index's predicate so SQLite can match them.
 const TRAIL_FILTER = sql<boolean>`kind IS NOT NULL AND kind != 'heartbeat'`;
@@ -58,6 +63,23 @@ export interface StoredEvent {
   readonly label?: string;
 }
 
+/**
+ * One report as the trail holds it. A report recorded before the trail kept
+ * whole texts has only its preview, so its text is that preview and
+ * `complete` is false.
+ */
+export interface StoredReport {
+  readonly id: number;
+
+  // Epoch ms the report arrived.
+  readonly at: number;
+  readonly atcID: SessionID;
+  readonly agentSessionID: AgentSessionID | null;
+  readonly label: string;
+  readonly text: string;
+  readonly complete: boolean;
+}
+
 /**
  * Daemon state in one SQLite store: the restorable fleet, the event trail
  * (hook events, message status changes, and reports) that events.read and
@@ -124,29 +146,43 @@ export class StateStore {
   async loadFleet(): Promise<FleetEntry[]> {
     const rows = await this.db
       .selectFrom('fleet')
-      .where('session_id', 'in', (eb) =>
+
+      // A session's ready workspace comes back with its row.
+      .leftJoin('workspace_materialization as workspace', (join) =>
+        join
+          .onRef('workspace.session_id', '=', 'fleet.session_id')
+          .on('workspace.phase', '=', 'ready'),
+      )
+      .where('fleet.session_id', 'in', (eb) =>
         eb.selectFrom('session_owner').select('session_id').where('daemon_id', '=', this.daemonID),
       )
 
       // Stored order, which a restore keeps for sessions with no recency.
-      .orderBy(sql`rowid`)
+      .orderBy(sql`fleet.rowid`)
       .select([
-        'session_id',
-        'agent_session_id',
-        'name',
-        'cwd',
-        'pinned',
-        'last_attached',
-        'agent',
-        'exited',
-        'parent_session_id',
-        'prompt',
-        'result',
-        'transcript_path',
-        'model',
-        'effort',
-        'target',
-        'target_identity',
+        'fleet.session_id',
+        'fleet.agent_session_id',
+        'fleet.name',
+        'fleet.cwd',
+        'fleet.pinned',
+        'fleet.last_attached',
+        'fleet.agent',
+        'fleet.exited',
+        'fleet.parent_session_id',
+        'fleet.prompt',
+        'fleet.result',
+        'fleet.transcript_path',
+        'fleet.model',
+        'fleet.effort',
+        'fleet.target',
+        'fleet.target_identity',
+        'workspace.repo_url',
+        'workspace.sha',
+        'workspace.ref',
+        'workspace.materialized_at',
+        'workspace.withheld_env',
+        'fleet.desired',
+        'fleet.host_key',
       ])
       .execute();
 
@@ -172,6 +208,10 @@ export class StateStore {
         ...(row.effort === null ? {} : { effort: row.effort }),
         ...(row.target === null ? {} : { target: row.target }),
         ...(row.target_identity === null ? {} : { targetIdentity: row.target_identity }),
+        ...buildWorkspaceField(row),
+        ...buildWithheldEnvField(row.withheld_env),
+        ...(row.desired === 'sleep' || row.desired === 'stop' ? { desired: row.desired } : {}),
+        ...(row.host_key === null ? {} : { hostKey: toSessionID(row.host_key) }),
       });
     }
 
@@ -249,6 +289,8 @@ export class StateStore {
             effort: entry.effort ?? null,
             target: entry.target ?? null,
             target_identity: entry.targetIdentity ?? null,
+            desired: entry.desired ?? null,
+            host_key: entry.hostKey ?? null,
           })
           .execute();
 
@@ -306,8 +348,10 @@ export class StateStore {
       .execute();
   }
 
-  async recordTrailEntry(entry: TrailEntry): Promise<void> {
-    await this.db
+  // Returns whether the entry was written: a report whose id the trail
+  // already holds is left out.
+  async recordTrailEntry(entry: TrailEntry): Promise<boolean> {
+    const result = await this.db
       .insertInto('events')
       .values({
         ts: new Date(entry.at).toISOString(),
@@ -319,8 +363,13 @@ export class StateStore {
         session_id: entry.agentSessionID,
         kind: entry.kind,
         detail: entry.detail,
+        report_id: entry.kind === 'report' ? (entry.reportID ?? null) : null,
+        report_text: entry.kind === 'report' ? entry.text : null,
       })
-      .execute();
+      .onConflict((oc) => oc.column('report_id').doNothing())
+      .executeTakeFirst();
+
+    return result.numInsertedOrUpdatedRows !== 0n;
   }
 
   async collectEventsAfter(
@@ -341,6 +390,32 @@ export class StateStore {
     return buildStoredEvents(rows);
   }
 
+  // A row that is not a report, or that lies outside the scope, misses as a
+  // row the trail never held does.
+  async findReport(id: number, scope: EventScope | null = null): Promise<StoredReport | null> {
+    const row = await this.db
+      .selectFrom('events')
+      .select(['id', 'ts', 'atc_id', 'session_id', 'message', 'detail', 'report_text'])
+      .where('id', '=', id)
+      .where('kind', '=', 'report')
+      .where((eb) => buildScopeMatch(eb, scope))
+      .executeTakeFirst();
+
+    if (row === undefined) {
+      return null;
+    }
+
+    return {
+      id: row.id,
+      at: Date.parse(row.ts),
+      atcID: toSessionID(row.atc_id),
+      agentSessionID: row.session_id === null ? null : toAgentSessionID(row.session_id),
+      label: row.message ?? '',
+      text: row.report_text ?? row.detail ?? '',
+      complete: row.report_text !== null,
+    };
+  }
+
   async collectLatestEvents(
     limit: number,
     scope: EventScope | null = null,
@@ -710,6 +785,76 @@ export class StateStore {
       .execute();
   }
 
+  // Records a materialization as it starts, in the resolving phase.
+  async createMaterialization(
+    row: Pick<
+      WorkspaceMaterialization,
+      'sessionID' | 'target' | 'dir' | 'sourceKind' | 'withheldEnv'
+    >,
+    at: number,
+  ): Promise<void> {
+    await this.db
+      .insertInto('workspace_materialization')
+      .values({
+        session_id: row.sessionID,
+        target: row.target,
+        dir: row.dir,
+        source_kind: row.sourceKind,
+        phase: 'resolving',
+        repo_url: null,
+        sha: null,
+        ref: null,
+        error_code: null,
+        started_at: at,
+        updated_at: at,
+        materialized_at: null,
+        withheld_env: JSON.stringify(row.withheldEnv),
+      })
+      .execute();
+  }
+
+  async updateMaterialization(
+    sessionID: SessionID,
+    fields: MaterializationUpdate,
+    at: number,
+  ): Promise<void> {
+    await this.db
+      .updateTable('workspace_materialization')
+      .set({
+        phase: fields.phase,
+        ...(fields.repoURL === undefined ? {} : { repo_url: fields.repoURL }),
+        ...(fields.sha === undefined ? {} : { sha: fields.sha }),
+        ...(fields.ref === undefined ? {} : { ref: fields.ref }),
+        ...(fields.errorCode === undefined ? {} : { error_code: fields.errorCode }),
+        ...(fields.materializedAt === undefined ? {} : { materialized_at: fields.materializedAt }),
+        updated_at: at,
+      })
+      .where('session_id', '=', sessionID)
+      .execute();
+  }
+
+  async findMaterialization(sessionID: SessionID): Promise<WorkspaceMaterialization | null> {
+    const row = await this.db
+      .selectFrom('workspace_materialization')
+      .selectAll()
+      .where('session_id', '=', sessionID)
+      .executeTakeFirst();
+
+    return row === undefined ? null : toWorkspaceMaterialization(row);
+  }
+
+  // Runs once as a daemon starts, before it serves a request: a
+  // materialization still short of ready or failed belonged to a daemon that
+  // stopped partway through it, and its workspace was never verified, so it
+  // fails as interrupted.
+  async reconcileMaterializations(at: number): Promise<void> {
+    await this.db
+      .updateTable('workspace_materialization')
+      .set({ phase: 'failed', error_code: 'workspace_interrupted', updated_at: at })
+      .where('phase', 'not in', ['ready', 'failed'])
+      .execute();
+  }
+
   async stop(): Promise<void> {
     await this.db.destroy();
 
@@ -748,6 +893,30 @@ export class StateStore {
   }
 }
 
+// A fleet row's ready workspace as its entry field, or nothing for a row
+// that ran without one.
+function buildWorkspaceField(
+  row: Readonly<{
+    repo_url: string | null;
+    sha: string | null;
+    ref: string | null;
+    materialized_at: number | null;
+  }>,
+): { readonly workspace?: SessionWorkspace } {
+  if (row.repo_url === null || row.sha === null || row.materialized_at === null) {
+    return {};
+  }
+
+  return {
+    workspace: {
+      repoURL: row.repo_url,
+      sha: row.sha,
+      ...(row.ref === null ? {} : { ref: row.ref }),
+      materializedAt: row.materialized_at,
+    },
+  };
+}
+
 // Two sessions share an agent session id when one resumes the other's agent
 // session. The fleet keeps one row per agent session id, the entry written
 // last, and relinks the survivors as a one-level hierarchy of rows it holds:
@@ -761,6 +930,22 @@ export class StateStore {
 //    row above it.
 //
 // Every walk takes at most one step per entry, so it ends on any input.
+// A ready workspace's withheld variable names as their entry field, or
+// nothing for a row that withholds none.
+function buildWithheldEnvField(stored: string | null): { readonly withheldEnv?: string[] } {
+  const names = parseWithheldEnv(stored);
+
+  return names.length === 0 ? {} : { withheldEnv: names };
+}
+
+function parseWithheldEnv(stored: string | null): string[] {
+  const parsed: unknown = stored === null ? [] : JSON.parse(stored);
+
+  return Array.isArray(parsed)
+    ? parsed.filter((name): name is string => typeof name === 'string')
+    : [];
+}
+
 function buildFleetWithoutReplacedRows(entries: readonly FleetEntry[]): FleetEntry[] {
   const survivors = new Map<AgentSessionID, SessionID>();
 
@@ -1033,3 +1218,23 @@ function toMessageRecord(row: Readonly<StateStoreSchema['messages']>): MessageRe
     ...(row.turn_id === null ? {} : { turn: row.turn_id }),
   };
 }
+
+function toWorkspaceMaterialization(
+  row: Readonly<StateStoreSchema['workspace_materialization']>,
+): WorkspaceMaterialization {
+  return {
+    sessionID: toSessionID(row.session_id),
+    target: row.target,
+    dir: row.dir,
+    sourceKind: row.source_kind,
+    phase: row.phase,
+    repoURL: row.repo_url,
+    sha: row.sha,
+    ref: row.ref,
+    errorCode: row.error_code,
+    startedAt: row.started_at,
+    updatedAt: row.updated_at,
+    materializedAt: row.materialized_at,
+    withheldEnv: parseWithheldEnv(row.withheld_env),
+  };
+}

package/src/store/trail-entry.ts

@@ -18,6 +18,13 @@ interface MessageTrailEntry extends TrailEntryBase {
 interface ReportTrailEntry extends TrailEntryBase {
   readonly kind: 'report';
   readonly label: string;
+
+  // The report's whole text, which the detail previews.
+  readonly text: string;
+
+  // The id a remote session's reporter gave the report, so a resent
+  // report is stored once; absent for a report that carries none.
+  readonly reportID?: string;
 }
 
 /**

package/src/store/workspace-materialization.ts

@@ -0,0 +1,70 @@
+import type { SessionID } from '../shared/session-id';
+
+/**
+ * Where a spawn's workspace stands on its way to the execution target:
+ * resolving the source to a commit, cloning it on the daemon's host,
+ * transferring the archive to the target, and verifying the target's
+ * checkout, then `ready` or `failed`. A row a stopped daemon left in any
+ * other phase is reconciled to `failed` before the next daemon serves a
+ * request, so no workspace ever counts as ready without its verification.
+ */
+export type MaterializationPhase =
+  | 'resolving'
+  | 'cloning'
+  | 'transferring'
+  | 'verifying'
+  | 'ready'
+  | 'failed';
+
+/**
+ * One spawn's workspace materialization, keyed by the session it is for.
+ * The repository URL is the token-free form, and the row never holds a
+ * credential or the reference to one.
+ */
+export interface WorkspaceMaterialization {
+  readonly sessionID: SessionID;
+  readonly target: string;
+
+  // The directory on the target the workspace is materialized into.
+  readonly dir: string;
+  readonly sourceKind: 'path' | 'git';
+  readonly phase: MaterializationPhase;
+  readonly repoURL: string | null;
+  readonly sha: string | null;
+
+  // The branch or tag the commit was resolved from; null for a pinned
+  // commit or a detached checkout.
+  readonly ref: string | null;
+
+  // The refusal that failed the materialization; null unless failed.
+  readonly errorCode: string | null;
+  readonly startedAt: number;
+  readonly updatedAt: number;
+  readonly materializedAt: number | null;
+
+  // The environment variable names every harness the session starts goes
+  // without: the workspace credential's variable and the askpass context.
+  readonly withheldEnv: readonly string[];
+}
+
+// The fields a materialization rewrites as it moves through its phases.
+export interface MaterializationUpdate {
+  readonly phase: MaterializationPhase;
+  readonly repoURL?: string;
+  readonly sha?: string;
+  readonly ref?: string | null;
+  readonly errorCode?: string;
+  readonly materializedAt?: number;
+}
+
+/**
+ * What a session's ready workspace was built from: the token-free
+ * repository URL, the commit checked out, the branch or tag it came from,
+ * and when it became ready.
+ */
+export interface SessionWorkspace {
+  readonly repoURL: string;
+  readonly sha: string;
+  readonly ref?: string;
+  readonly materializedAt: number;
+}

package/src/tap.ts

@@ -1,6 +1,7 @@
 import { z } from 'zod';
 import { DaemonClient } from './client/daemon-client';
 import { DaemonError } from './protocol/daemon-error';
+import { runBridgeTap } from './run-bridge-tap';
 import { daemonSocketPath } from './shared/config';
 import { getBuild } from './shared/get-build';
 
@@ -28,6 +29,16 @@ interface InboxMessage {
  * another tap replaced it or the session was removed.
  */
 export async function runTap(session: string): Promise<void> {
+  const bridge = process.env['ATC_SOCKET'];
+
+  // Inside a remote host the daemon's own socket is out of reach, and the
+  // session bridge serves the tap instead.
+  if (process.env['ATC_BRIDGE'] === '1' && bridge !== undefined && bridge !== '') {
+    await runBridgeTap(bridge, process.env['ATC_OUTBOX'] ?? '');
+
+    return;
+  }
+
   let client: DaemonClient;
 
   try {

package/src/workspace/check-url-credentials.ts

@@ -0,0 +1,59 @@
+import { runGit } from './run-git';
+
+type URLCredentialFinding =
+  | { readonly ok: true }
+  | { readonly ok: false; readonly code: 'credential_in_url'; readonly message: string };
+
+/**
+ * Checks that a repository URL reaches git without a credential in it, so
+ * a token can never surface in git's own messages or in the clone. The URL
+ * itself must carry none, and no `url.<base>.insteadOf` rewrite in the git
+ * config read from `cwd` may expand it into one that does. git expands the
+ * URL without touching the network. An http(s) URL carries a credential in
+ * its userinfo, query, or fragment; any other scheme URL only in a
+ * password, since its user is a login name such as `git`.
+ */
+export async function checkURLCredentials(url: string, cwd: string): Promise<URLCredentialFinding> {
+  // git prints the URL as it would fetch it, with every rewrite applied.
+  const expanded = await runGit(['ls-remote', '--get-url', url], { cwd });
+
+  const fetched = expanded.exitCode === 0 ? expanded.stdout.trim() : url;
+
+  if (!hasURLCredential(fetched)) {
+    return { ok: true };
+  }
+
+  return {
+    ok: false,
+    code: 'credential_in_url',
+    message: hasURLCredential(url)
+      ? 'the repository URL carries a credential; pass it as a credentialRef instead'
+      : 'a url.<base>.insteadOf rewrite in the host git config puts a credential into the repository URL; remove the rewrite or pass the credential as a credentialRef',
+  };
+}
+
+const SCHEME_PATTERN = /^[a-z][a-z\d+.-]*:\/\//iu;
+
+const HTTP_PROTOCOLS: ReadonlySet<string> = new Set(['http:', 'https:']);
+
+function hasURLCredential(url: string): boolean {
+  if (!SCHEME_PATTERN.test(url)) {
+    return false;
+  }
+
+  let parsed: URL;
+
+  try {
+    parsed = new URL(url);
+  } catch {
+    return false;
+  }
+
+  if (HTTP_PROTOCOLS.has(parsed.protocol)) {
+    return (
+      parsed.username !== '' || parsed.password !== '' || parsed.search !== '' || parsed.hash !== ''
+    );
+  }
+
+  return parsed.password !== '';
+}

package/src/workspace/create-workspace-clone.ts

@@ -40,6 +40,9 @@ type IncompleteCheckout = Exclude<
  * objects rather than hard-linking them, takes no hook templates, and runs no
  * hooks on checkout.
  *
+ * A ref, or a full commit id, that the upstream does not have is refused as
+ * `ref_not_found`, and the directory is removed.
+ *
  * The checked-out commit must be the whole workspace: one that uses
  * submodules or tracks paths through Git LFS is refused and the directory is
  * removed. The clone fetches without checking out, and the checkout then
@@ -163,9 +166,18 @@ async function createCloneAtRef(
   );
 
   if (checkout.exitCode !== 0) {
+    // A pinned commit that no upstream ref reaches never arrives in the
+    // clone, so its checkout fails on a commit the upstream does not have.
+    const present = await runGit(['cat-file', '-e', `${target.sha}^{commit}`], {
+      cwd: request.dir,
+      isolated: true,
+    });
+
     await rm(request.dir, { recursive: true, force: true });
 
-    return { ok: false, code: 'clone_failed', message: checkout.stderr.trim() };
+    return target.branch === null && present.exitCode !== 0
+      ? { ok: false, code: 'ref_not_found', message: `origin has no commit ${target.sha}` }
+      : { ok: false, code: 'clone_failed', message: checkout.stderr.trim() };
   }
 
   const complete = await checkWorkspaceCompleteness(request.dir, target.sha);

package/src/workspace/repository-env-vars.ts

@@ -0,0 +1,22 @@
+/**
+ * The variables that pin git to a repository other than the one in its
+ * working directory, such as the `GIT_DIR` a git hook exports. A command
+ * that must act on its own directory runs without every one of them.
+ */
+export const REPOSITORY_ENV_VARS: ReadonlySet<string> = new Set([
+  'GIT_ALTERNATE_OBJECT_DIRECTORIES',
+  'GIT_COMMON_DIR',
+  'GIT_CONFIG',
+  'GIT_CONFIG_COUNT',
+  'GIT_CONFIG_PARAMETERS',
+  'GIT_DIR',
+  'GIT_GRAFT_FILE',
+  'GIT_IMPLICIT_WORK_TREE',
+  'GIT_INDEX_FILE',
+  'GIT_NO_REPLACE_OBJECTS',
+  'GIT_OBJECT_DIRECTORY',
+  'GIT_PREFIX',
+  'GIT_REPLACE_REF_BASE',
+  'GIT_SHALLOW_FILE',
+  'GIT_WORK_TREE',
+]);

package/src/workspace/run-git.ts

@@ -1,3 +1,5 @@
+import { REPOSITORY_ENV_VARS } from './repository-env-vars';
+
 interface GitRunOptions {
   readonly cwd?: string;
   readonly env?: Readonly<Record<string, string>>;
@@ -73,24 +75,6 @@ export async function runGit(
   return { exitCode, stdout, stderr };
 }
 
-const REPOSITORY_ENV_VARS: ReadonlySet<string> = new Set([
-  'GIT_ALTERNATE_OBJECT_DIRECTORIES',
-  'GIT_COMMON_DIR',
-  'GIT_CONFIG',
-  'GIT_CONFIG_COUNT',
-  'GIT_CONFIG_PARAMETERS',
-  'GIT_DIR',
-  'GIT_GRAFT_FILE',
-  'GIT_IMPLICIT_WORK_TREE',
-  'GIT_INDEX_FILE',
-  'GIT_NO_REPLACE_OBJECTS',
-  'GIT_OBJECT_DIRECTORY',
-  'GIT_PREFIX',
-  'GIT_REPLACE_REF_BASE',
-  'GIT_SHALLOW_FILE',
-  'GIT_WORK_TREE',
-]);
-
 function collectHostEnv(): Record<string, string | undefined> {
   return Object.fromEntries(
     Object.entries(process.env).filter(