@zgeoff/atc 2.14.0 → 2.15.1

package/src/daemon/idempotency-ledger.ts

@@ -23,7 +23,9 @@ interface IdempotentCall<T> {
   // effect may still stand, so the claim is kept as outcome_unknown.
   readonly start: () => T | Promise<T>;
 
-  // Resolves once the effect is durable; the claim completes only after.
+  // Resolves once the effect is durable; the claim completes only after. A
+  // settle that throws leaves the effect standing, so the answer is
+  // outcome_unknown and the claim stays held.
   readonly settle: () => Promise<void>;
 
   // The answer to a retry of a completed key.
@@ -43,19 +45,27 @@ interface IdempotentCall<T> {
  * answer; one with a different payload is `idempotency_conflict`; one whose
  * effect a stopped daemon may or may not have run is `outcome_unknown` with
  * the effect id in `data.effectRef`, and never starts the effect again.
+ * An effect that may still stand once its request fails is answered with
+ * `outcome_unknown` and its effect id even when recording that outcome
+ * fails: the claim stays held either way, and the failed write goes to the
+ * log.
  */
 export class IdempotencyLedger {
   private readonly store: StateStore;
 
   private readonly principal: string;
 
+  // Where a failed write to the ledger is reported, one line at a time.
+  private readonly log: (line: string) => void;
+
   // One promise chain per key, so two requests under the same key never
   // interleave their claim and completion on one daemon.
   private readonly locks = new Map<string, Promise<void>>();
 
-  constructor(store: StateStore, principal: string) {
+  constructor(store: StateStore, principal: string, log: (line: string) => void) {
     this.store = store;
     this.principal = principal;
+    this.log = log;
   }
 
   async run<T extends Readonly<Record<string, unknown>>>(call: IdempotentCall<T>): Promise<T> {
@@ -114,13 +124,11 @@ export class IdempotencyLedger {
       result = await call.start();
     } catch (error) {
       if (error instanceof EffectRemainsError) {
-        await this.store.updateIdempotencyOutcomeUnknown(id, Date.now());
+        this.logFailure(call, 'start', error);
+
+        await this.tryRecordOutcomeUnknown(call, id);
 
-        throw new DaemonError(
-          'outcome_unknown',
-          `the ${call.operation} under idempotency key '${call.keyed.key}' failed and its effect may still stand; check ${call.effectRef} before retrying under a new key`,
-          { effectRef: call.effectRef },
-        );
+        throw buildOutcomeUnknownError(call, 'failed');
       }
 
       await this.store.removeIdempotencyKey(id);
@@ -128,20 +136,61 @@ export class IdempotencyLedger {
       throw error;
     }
 
-    // A settle that fails leaves the claim in progress: the effect started,
-    // so the next daemon start marks its outcome unknown rather than letting
-    // a retry run it again.
-    await call.settle();
+    // From here the effect stands. A settle or completion that fails keeps
+    // the claim in progress, so a retry answers outcome_unknown rather than
+    // running the effect again.
+    try {
+      await call.settle();
+    } catch (error) {
+      this.logFailure(call, 'settle', error);
+
+      await this.tryRecordOutcomeUnknown(call, id);
 
-    await this.store.updateIdempotencyCompleted(
-      id,
-      JSON.stringify(result),
-      Date.now(),
-      call.findEffectTarget?.(result) ?? null,
-    );
+      throw buildOutcomeUnknownError(call, 'could not be made durable');
+    }
+
+    try {
+      await this.store.updateIdempotencyCompleted(
+        id,
+        JSON.stringify(result),
+        Date.now(),
+        call.findEffectTarget?.(result) ?? null,
+      );
+    } catch (error) {
+      this.logFailure(call, 'completion write', error);
+      throw buildOutcomeUnknownError(call, 'could not be recorded as completed');
+    }
 
     return result;
   }
+
+  // A claim this write fails to update stays in progress, which a retry
+  // answers the same way, so the failure is logged and not thrown.
+  private async tryRecordOutcomeUnknown(
+    call: Pick<IdempotentCall<unknown>, 'operation' | 'keyed' | 'effectRef'>,
+    id: Pick<IdempotencyRecord, 'principal' | 'operation' | 'key'>,
+  ): Promise<void> {
+    try {
+      await this.store.updateIdempotencyOutcomeUnknown(id, Date.now());
+    } catch (error) {
+      this.logFailure(call, 'outcome_unknown write', error);
+    }
+  }
+
+  // The line holds the operation, key, effect id, the step that failed, and
+  // the error, never the request's payload or the effect's result.
+  private logFailure(
+    call: Pick<IdempotentCall<unknown>, 'operation' | 'keyed' | 'effectRef'>,
+    step: string,
+    error: unknown,
+  ): void {
+    const cause =
+      error instanceof Error && error.cause instanceof Error ? ` (${String(error.cause)})` : '';
+
+    this.log(
+      `atc ${call.operation} under idempotency key '${call.keyed.key}' (effect ${call.effectRef}): ${step} failed: ${String(error)}${cause}`,
+    );
+  }
 }
 
 function answerHeldKey<T>(call: IdempotentCall<T>, held: IdempotencyRecord): T | Promise<T> {
@@ -162,3 +211,14 @@ function answerHeldKey<T>(call: IdempotentCall<T>, held: IdempotencyRecord): T |
     { effectRef: held.effectRef },
   );
 }
+
+function buildOutcomeUnknownError(
+  call: Pick<IdempotentCall<unknown>, 'operation' | 'keyed' | 'effectRef'>,
+  what: string,
+): DaemonError {
+  return new DaemonError(
+    'outcome_unknown',
+    `the ${call.operation} under idempotency key '${call.keyed.key}' ${what} and its effect may still stand; check ${call.effectRef} before retrying under a new key`,
+    { effectRef: call.effectRef },
+  );
+}

package/src/daemon/imp-client-port.ts

@@ -18,12 +18,15 @@ import type {
 import { ImpPortError } from './imp-port-error';
 
 /**
- * Where impd listens and the token the daemon calls it with. The token
- * comes from the daemon's environment, never from a config file.
+ * Where impd listens, and how the daemon reads the token it calls impd
+ * with: null for no token. The port reads the token again before each
+ * call and connection, so a token that changes at its source takes effect
+ * on the next one. A read that throws fails that call without reaching
+ * impd.
  */
 export interface ImpClientPortOptions {
   readonly url: string;
-  readonly token: string | null;
+  readonly readToken: () => string | null;
 }
 
 /**
@@ -35,30 +38,26 @@ export interface ImpClientPortOptions {
 export class ImpClientPort implements ImpPort {
   private readonly url: string;
 
-  private readonly token: string | null;
+  private readonly readToken: () => string | null;
 
-  private readonly client: ImpClient;
+  // The client for the token read last, rebuilt when the token changes.
+  private client: Readonly<{ token: string | null; client: ImpClient }> | null = null;
 
   constructor(options: ImpClientPortOptions) {
     this.url = options.url;
-    this.token = options.token;
-
-    this.client = createImpClient({
-      url: options.url,
-      ...(options.token === null ? {} : { token: options.token }),
-    });
+    this.readToken = options.readToken;
   }
 
   // An impd from before the flags has neither.
   readonly readFeatures = async (): Promise<ImpFeatures> => {
-    const info = await this.tryCall(() => this.client.system.info());
+    const info = await this.tryCall((client) => client.system.info());
 
     return info.features ?? { sessionOffsets: false, leases: false };
   };
 
   readonly readImp = async (name: string): Promise<ImpView | null> => {
     try {
-      const imp = await this.tryCall(() => this.client.imps.get({ name }));
+      const imp = await this.tryCall((client) => client.imps.get({ name }));
 
       return {
         name: imp.name,
@@ -76,8 +75,8 @@ export class ImpClientPort implements ImpPort {
   };
 
   readonly createImp = async (spec: ImpCreateSpec): Promise<ImpView> => {
-    const imp = await this.tryCall(() =>
-      this.client.imps.create({
+    const imp = await this.tryCall((client) =>
+      client.imps.create({
         name: spec.name,
         ...(spec.image === undefined ? {} : { image: spec.image }),
         ...(spec.memoryMib === undefined ? {} : { memoryMib: spec.memoryMib }),
@@ -92,7 +91,9 @@ export class ImpClientPort implements ImpPort {
     label: string,
     ttlSeconds: number,
   ): Promise<ImpLease> => {
-    const lease = await this.tryCall(() => this.client.leases.acquire({ name, label, ttlSeconds }));
+    const lease = await this.tryCall((client) =>
+      client.leases.acquire({ name, label, ttlSeconds }),
+    );
 
     return toLease(lease);
   };
@@ -102,23 +103,23 @@ export class ImpClientPort implements ImpPort {
     label: string,
     ttlSeconds: number,
   ): Promise<ImpLease> => {
-    const lease = await this.tryCall(() => this.client.leases.renew({ name, label, ttlSeconds }));
+    const lease = await this.tryCall((client) => client.leases.renew({ name, label, ttlSeconds }));
 
     return toLease(lease);
   };
 
   readonly releaseLease = async (name: string, label: string): Promise<boolean> => {
-    const result = await this.tryCall(() => this.client.leases.release({ name, label }));
+    const result = await this.tryCall((client) => client.leases.release({ name, label }));
 
     return result.released;
   };
 
   readonly suspendImp = async (name: string): Promise<void> => {
-    await this.tryCall(() => this.client.imps.sleep({ name }));
+    await this.tryCall((client) => client.imps.sleep({ name }));
   };
 
   readonly destroyImp = async (name: string): Promise<void> => {
-    await this.tryCall(() => this.client.imps.destroy({ name }));
+    await this.tryCall((client) => client.imps.destroy({ name }));
   };
 
   // The exec WebSocket answers only after it opens, so nothing reaches the
@@ -127,9 +128,23 @@ export class ImpClientPort implements ImpPort {
     request: ImpSessionRequest,
     handlers: ImpSessionHandlers,
   ): ImpSessionConnection => {
+    const token = this.tryReadToken();
+
+    // A token that cannot be read ends the connection the way impd's
+    // refusal of a token does, without dialing impd.
+    if (token instanceof ImpPortError) {
+      return {
+        outcome: Promise.resolve({ kind: 'unauthorized' }),
+        write: () => {},
+        resize: () => {},
+        sendSignal: () => {},
+        close: () => {},
+      };
+    }
+
     const session = openExecSession({
       baseUrl: this.url,
-      token: this.token,
+      token,
       start:
         request.kind === 'start'
           ? {
@@ -179,8 +194,8 @@ export class ImpClientPort implements ImpPort {
 
   // oxlint-disable-next-line prefer-readonly-parameter-types -- the command's input bytes have no readonly form
   readonly runCommand = async (name: string, command: ImpCommand): Promise<ImpCommandResult> => {
-    const result = await this.tryCall(() =>
-      this.client.run(name, command.argv, {
+    const result = await this.tryCall((client) =>
+      client.run(name, command.argv, {
         ...(command.cwd === undefined ? {} : { cwd: command.cwd }),
         ...(command.stdin === undefined ? {} : { stdin: command.stdin }),
       }),
@@ -194,12 +209,30 @@ export class ImpClientPort implements ImpPort {
     guestPath: string,
     onConnection: (connection: ImpRelayConnection) => void,
   ): ImpReverseForward => {
+    const token = this.tryReadToken();
+
+    if (token instanceof ImpPortError) {
+      return { listening: Promise.reject(token), stop: () => {} };
+    }
+
     const forward = openReverseForward({
       baseUrl: this.url,
-      token: this.token,
+      token,
       name,
       guest: { network: 'unix', path: guestPath },
-      connect: (url, headers) => new WebSocket(url, { headers }),
+
+      // The client reuses the headers of the listening socket for every
+      // guest connection relay, so each socket reads the token afresh. A
+      // token that cannot be read dials without one, which impd refuses.
+      connect: (url, headers) => {
+        const { authorization: _, ...rest } = headers;
+        const current = this.tryReadToken();
+
+        return new WebSocket(url, {
+          headers:
+            typeof current === 'string' ? { ...rest, authorization: `Bearer ${current}` } : rest,
+        });
+      },
       onConnection: (accept) => {
         onConnection(openRelay(accept));
       },
@@ -215,13 +248,36 @@ export class ImpClientPort implements ImpPort {
 
   // Runs one client call, turning impd's refusals and transport failures
   // into port errors.
-  private async tryCall<T>(call: () => Promise<T>): Promise<T> {
+  private async tryCall<T>(call: (client: ImpClient) => Promise<T>): Promise<T> {
     try {
-      return await call();
+      return await call(this.loadClient());
     } catch (error) {
       throw toPortError(error);
     }
   }
+
+  // The token as it reads now, or the port error its read throws.
+  private tryReadToken(): string | null | ImpPortError {
+    try {
+      return this.readToken();
+    } catch (error) {
+      return toPortError(error);
+    }
+  }
+
+  // The client that calls impd with the token as it reads now.
+  private loadClient(): ImpClient {
+    const token = this.readToken();
+
+    if (this.client === null || this.client.token !== token) {
+      this.client = {
+        token,
+        client: createImpClient({ url: this.url, ...(token === null ? {} : { token }) }),
+      };
+    }
+
+    return this.client.client;
+  }
 }
 
 // A lease as the client returns it.

package/src/daemon/imp-harness.ts

@@ -99,6 +99,15 @@ export class ImpHarness implements HarnessHandle {
 
   private killPending = false;
 
+  // Whether the process exited, once the harness stops being followed, and
+  // the waits on that answer until then.
+  private exitConfirmed: boolean | null = null;
+
+  private readonly exitWaiters = new Set<{
+    readonly waited: PromiseWithResolvers<boolean>;
+    readonly timer: ReturnType<typeof setTimeout>;
+  }>();
+
   private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
 
   // The terminal size the open connection's request asked for.
@@ -175,6 +184,28 @@ export class ImpHarness implements HarnessHandle {
     }
   };
 
+  // Only an exit impd reports counts: a host that lost the process or went
+  // to sleep with it inside confirms nothing.
+  readonly waitForExit = (timeoutMs: number): Promise<boolean> => {
+    if (this.exitConfirmed !== null) {
+      return Promise.resolve(this.exitConfirmed);
+    }
+
+    const waited = Promise.withResolvers<boolean>();
+
+    const waiter = {
+      waited,
+      timer: setTimeout(() => {
+        this.exitWaiters.delete(waiter);
+        waited.resolve(false);
+      }, timeoutMs),
+    };
+
+    this.exitWaiters.add(waiter);
+
+    return waited.promise;
+  };
+
   readonly detach = (): void => {
     if (this.done) {
       return;
@@ -521,6 +552,8 @@ export class ImpHarness implements HarnessHandle {
   private emitExit(exit: HarnessExit): void {
     const listeners = [...this.exitListeners];
 
+    this.exitConfirmed = exit.reason === undefined || exit.reason === 'exited';
+
     this.stopFollowing();
 
     for (const listener of listeners) {
@@ -537,7 +570,15 @@ export class ImpHarness implements HarnessHandle {
 
     this.done = true;
     this.connection = null;
+    this.exitConfirmed ??= false;
+
+    for (const waiter of this.exitWaiters) {
+      clearTimeout(waiter.timer);
+
+      waiter.waited.resolve(this.exitConfirmed);
+    }
 
+    this.exitWaiters.clear();
     this.dataListeners.clear();
     this.exitListeners.clear();
     this.attachmentListeners.clear();

package/src/daemon/imp-provider.ts

@@ -1,4 +1,5 @@
 import { DaemonError } from '../protocol/daemon-error';
+import { LineDecoder } from '../protocol/line-decoder';
 import { isCompiledBinary } from '../shared/is-compiled-binary';
 import { buildTarArchive } from './build-tar-archive';
 import type {
@@ -523,22 +524,15 @@ function buildSocketName(sessionID: string): string {
 
 // Frames a guest connection's bytes into lines, both ways.
 function toHarnessRelay(connection: ImpRelayConnection): HarnessRelay {
-  const decoder = new TextDecoder();
+  const decoder = new LineDecoder();
   const encoder = new TextEncoder();
 
   const lineListeners: ((line: string) => void)[] = [];
-  let pending = '';
 
   connection.onData((data) => {
-    const lines = `${pending}${decoder.decode(data, { stream: true })}`.split('\n');
-
-    pending = lines.pop() ?? '';
-
-    for (const line of lines) {
-      if (line.trim() !== '') {
-        for (const listener of lineListeners) {
-          listener(line);
-        }
+    for (const line of decoder.splitChunk(data)) {
+      for (const listener of lineListeners) {
+        listener(line);
       }
     }
   });

package/src/daemon/local-pty-provider.ts

@@ -74,6 +74,35 @@ export class LocalPTYProvider implements ExecutionProvider {
       kill: () => {
         pty.kill();
       },
+
+      // A process already gone has nothing left to end.
+      killForced: () => {
+        try {
+          process.kill(pty.pid, 'SIGKILL');
+        } catch (error) {
+          if (!isMissingProcessError(error)) {
+            throw error;
+          }
+        }
+      },
+
+      // bun-pty's kill sends one SIGHUP and reports an exit at once, whether
+      // the process ended or not, so the exit is read from the process id
+      // instead: the library reaps its child, so the id stops answering a
+      // signal once the process is gone.
+      waitForExit: async (timeoutMs) => {
+        const deadline = Date.now() + timeoutMs;
+
+        while (isProcessRunning(pty.pid)) {
+          if (Date.now() >= deadline) {
+            return false;
+          }
+
+          await Bun.sleep(20);
+        }
+
+        return true;
+      },
       detach: () => {
         for (const subscription of subscriptions) {
           subscription.dispose();
@@ -140,3 +169,19 @@ export class LocalPTYProvider implements ExecutionProvider {
     return { exitCode, stdout, stderr };
   }
 }
+
+// A process another user owns still runs, so only a missing process counts
+// as gone.
+function isProcessRunning(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+
+    return true;
+  } catch (error) {
+    return !isMissingProcessError(error);
+  }
+}
+
+function isMissingProcessError(error: unknown): boolean {
+  return error instanceof Error && 'code' in error && error.code === 'ESRCH';
+}

package/src/daemon/parse-hook-line.ts

@@ -1,6 +1,6 @@
+import type { HookEvent } from '../protocol/hook-event';
 import { isRecord } from '../shared/report';
 import { toSessionID } from '../shared/to-session-id';
-import type { HookEvent } from './hooks';
 
 /**
  * One reporter line as a hook event, or null for a line that is not one.

package/src/daemon/read-imp-token.ts

@@ -0,0 +1,30 @@
+import { readFileSync } from 'node:fs';
+import { ImpPortError } from './imp-port-error';
+
+/**
+ * The impd token held in the file at `path`, without one trailing newline.
+ * An empty or unreadable file throws an `UNAUTHORIZED` port error whose
+ * message holds the path and the read's error code, never file content.
+ */
+export function readImpToken(path: string): string {
+  let text: string;
+
+  try {
+    text = readFileSync(path, 'utf8');
+  } catch (error) {
+    const code =
+      error instanceof Error && 'code' in error && typeof error.code === 'string'
+        ? error.code
+        : 'unknown error';
+
+    throw new ImpPortError('UNAUTHORIZED', `cannot read the impd token file ${path}: ${code}`);
+  }
+
+  const token = text.endsWith('\n') ? text.slice(0, -1) : text;
+
+  if (token === '') {
+    throw new ImpPortError('UNAUTHORIZED', `the impd token file ${path} is empty`);
+  }
+
+  return token;
+}