@zgeoff/atc 2.15.0 → 2.15.2

package/src/daemon/drain-inbox.ts

@@ -0,0 +1,85 @@
+import { PROTOCOL_V } from '../protocol/protocol';
+import type { SessionID } from '../shared/session-id';
+import type { MessageOwner } from '../store/message-owner';
+import type { MessageRecord } from '../store/message-record';
+import type { TapClient } from './daemon-context';
+import type { TapGeneration, TapRegistry } from './tap-registry';
+
+/**
+ * What a drain reads from: the taps, the owner a linked tap reads messages
+ * under (the session's atc id and agent session id, or null for a session
+ * the daemon does not hold), and the store's pending messages for an owner.
+ */
+export interface InboxSource {
+  readonly taps: TapRegistry<TapClient>;
+  readonly findLinkedOwner: (sessionID: SessionID) => MessageOwner | null;
+  readonly collectPendingMessages: (owner: MessageOwner) => Promise<MessageRecord[]>;
+}
+
+/**
+ * Hands the session's tap the oldest pending message it has not taken. The
+ * backlog is read before anything is sent, so a tap's ok response is always
+ * queued ahead of its first message, and one message goes per call: the
+ * tap's ack calls it again, so the backlog never outgrows the connection's
+ * outbound queue. The drain is bound to the tap it found before the read: a
+ * tap another attach made while the read waited gets nothing from it.
+ */
+// oxlint-disable-next-line prefer-readonly-parameter-types -- the drain claims deliveries in the tap registry it is given
+export async function drainInbox(sessionID: SessionID, source: InboxSource): Promise<void> {
+  const linkedOwner = source.findLinkedOwner(sessionID);
+  const tap = source.taps.findTap(sessionID);
+
+  if (linkedOwner === null || tap === null) {
+    return;
+  }
+
+  // A tap a principal holds takes only the messages sent to this atc id:
+  // another session that shares the agent session id may be out of its
+  // reach.
+  const owner = tap.linked ? linkedOwner : { atcID: linkedOwner.atcID };
+
+  try {
+    const pending = await source.collectPendingMessages(owner);
+
+    for (const record of pending) {
+      if (sendInboxMessage(sessionID, record, tap, source.taps)) {
+        return;
+      }
+    }
+  } catch {}
+}
+
+// Hands one pending message to the tap the drain was made for, once, and
+// reports whether it did. An unlinked tap takes only a message sent to the
+// session's own atc id. The event goes to the tap connection alone: it never
+// reaches other clients, the events socket, or hooks.
+function sendInboxMessage(
+  sessionID: SessionID,
+  record: MessageRecord,
+  tap: TapGeneration,
+
+  // oxlint-disable-next-line prefer-readonly-parameter-types -- a delivery claims the message in the registry
+  taps: TapRegistry<TapClient>,
+): boolean {
+  if (!tap.linked && record.atcID !== sessionID) {
+    return false;
+  }
+
+  const client = taps.claimDelivery(sessionID, record.id, tap.generation);
+
+  if (client === null) {
+    return false;
+  }
+
+  client.sendEvent({
+    v: PROTOCOL_V,
+    ev: 'InboxMessage',
+    s: sessionID,
+    message: record.id,
+    from: record.from,
+    text: record.text,
+    sentAt: record.sentAt,
+  });
+
+  return true;
+}

package/src/daemon/execution-provider.ts

@@ -167,8 +167,20 @@ export interface HarnessHandle {
   ) => HarnessSubscription;
   readonly write: (data: string) => void;
   readonly resize: (cols: number, rows: number) => void;
+
+  // Sends the harness's process the signal that ends it, and returns without
+  // waiting for it to exit.
   readonly kill: () => void;
 
+  // Ends the harness's process with a signal it cannot catch or ignore, on
+  // a provider that can send one; absent on a provider that cannot.
+  readonly killForced?: () => void;
+
+  // Resolves true once the harness's process has exited, and false when the
+  // wait runs out first or the harness stops being followed without an
+  // exit, such as a host that went to sleep with the process inside.
+  readonly waitForExit: (timeoutMs: number) => Promise<boolean>;
+
   // Stops following the harness and leaves its process running, for a host
   // that keeps it after the daemon lets go; on a host that cannot keep it,
   // the process ends as a kill ends it. No listener fires after a detach.

package/src/daemon/hooks.ts

@@ -1,15 +1,9 @@
 import { unlinkSync } from 'node:fs';
+import type { HookEvent } from '../protocol/hook-event';
 import { LineDecoder } from '../protocol/line-decoder';
 import { socketPath } from '../shared/config';
-import type { SessionID } from '../shared/session-id';
 import { parseHookLine } from './parse-hook-line';
 
-export interface HookEvent {
-  atcId: SessionID;
-  event: string;
-  payload: Record<string, unknown>;
-}
-
 // Per-connection read state: the connection's own line framing.
 interface HookConnection {
   readonly lines: LineDecoder;

package/src/daemon/idempotency-ledger.ts

@@ -23,7 +23,9 @@ interface IdempotentCall<T> {
   // effect may still stand, so the claim is kept as outcome_unknown.
   readonly start: () => T | Promise<T>;
 
-  // Resolves once the effect is durable; the claim completes only after.
+  // Resolves once the effect is durable; the claim completes only after. A
+  // settle that throws leaves the effect standing, so the answer is
+  // outcome_unknown and the claim stays held.
   readonly settle: () => Promise<void>;
 
   // The answer to a retry of a completed key.
@@ -43,19 +45,27 @@ interface IdempotentCall<T> {
  * answer; one with a different payload is `idempotency_conflict`; one whose
  * effect a stopped daemon may or may not have run is `outcome_unknown` with
  * the effect id in `data.effectRef`, and never starts the effect again.
+ * An effect that may still stand once its request fails is answered with
+ * `outcome_unknown` and its effect id even when recording that outcome
+ * fails: the claim stays held either way, and the failed write goes to the
+ * log.
  */
 export class IdempotencyLedger {
   private readonly store: StateStore;
 
   private readonly principal: string;
 
+  // Where a failed write to the ledger is reported, one line at a time.
+  private readonly log: (line: string) => void;
+
   // One promise chain per key, so two requests under the same key never
   // interleave their claim and completion on one daemon.
   private readonly locks = new Map<string, Promise<void>>();
 
-  constructor(store: StateStore, principal: string) {
+  constructor(store: StateStore, principal: string, log: (line: string) => void) {
     this.store = store;
     this.principal = principal;
+    this.log = log;
   }
 
   async run<T extends Readonly<Record<string, unknown>>>(call: IdempotentCall<T>): Promise<T> {
@@ -114,13 +124,11 @@ export class IdempotencyLedger {
       result = await call.start();
     } catch (error) {
       if (error instanceof EffectRemainsError) {
-        await this.store.updateIdempotencyOutcomeUnknown(id, Date.now());
+        this.logFailure(call, 'start', error);
+
+        await this.tryRecordOutcomeUnknown(call, id);
 
-        throw new DaemonError(
-          'outcome_unknown',
-          `the ${call.operation} under idempotency key '${call.keyed.key}' failed and its effect may still stand; check ${call.effectRef} before retrying under a new key`,
-          { effectRef: call.effectRef },
-        );
+        throw buildOutcomeUnknownError(call, 'failed');
       }
 
       await this.store.removeIdempotencyKey(id);
@@ -128,20 +136,61 @@ export class IdempotencyLedger {
       throw error;
     }
 
-    // A settle that fails leaves the claim in progress: the effect started,
-    // so the next daemon start marks its outcome unknown rather than letting
-    // a retry run it again.
-    await call.settle();
+    // From here the effect stands. A settle or completion that fails keeps
+    // the claim in progress, so a retry answers outcome_unknown rather than
+    // running the effect again.
+    try {
+      await call.settle();
+    } catch (error) {
+      this.logFailure(call, 'settle', error);
+
+      await this.tryRecordOutcomeUnknown(call, id);
 
-    await this.store.updateIdempotencyCompleted(
-      id,
-      JSON.stringify(result),
-      Date.now(),
-      call.findEffectTarget?.(result) ?? null,
-    );
+      throw buildOutcomeUnknownError(call, 'could not be made durable');
+    }
+
+    try {
+      await this.store.updateIdempotencyCompleted(
+        id,
+        JSON.stringify(result),
+        Date.now(),
+        call.findEffectTarget?.(result) ?? null,
+      );
+    } catch (error) {
+      this.logFailure(call, 'completion write', error);
+      throw buildOutcomeUnknownError(call, 'could not be recorded as completed');
+    }
 
     return result;
   }
+
+  // A claim this write fails to update stays in progress, which a retry
+  // answers the same way, so the failure is logged and not thrown.
+  private async tryRecordOutcomeUnknown(
+    call: Pick<IdempotentCall<unknown>, 'operation' | 'keyed' | 'effectRef'>,
+    id: Pick<IdempotencyRecord, 'principal' | 'operation' | 'key'>,
+  ): Promise<void> {
+    try {
+      await this.store.updateIdempotencyOutcomeUnknown(id, Date.now());
+    } catch (error) {
+      this.logFailure(call, 'outcome_unknown write', error);
+    }
+  }
+
+  // The line holds the operation, key, effect id, the step that failed, and
+  // the error, never the request's payload or the effect's result.
+  private logFailure(
+    call: Pick<IdempotentCall<unknown>, 'operation' | 'keyed' | 'effectRef'>,
+    step: string,
+    error: unknown,
+  ): void {
+    const cause =
+      error instanceof Error && error.cause instanceof Error ? ` (${String(error.cause)})` : '';
+
+    this.log(
+      `atc ${call.operation} under idempotency key '${call.keyed.key}' (effect ${call.effectRef}): ${step} failed: ${String(error)}${cause}`,
+    );
+  }
 }
 
 function answerHeldKey<T>(call: IdempotentCall<T>, held: IdempotencyRecord): T | Promise<T> {
@@ -162,3 +211,14 @@ function answerHeldKey<T>(call: IdempotentCall<T>, held: IdempotencyRecord): T |
     { effectRef: held.effectRef },
   );
 }
+
+function buildOutcomeUnknownError(
+  call: Pick<IdempotentCall<unknown>, 'operation' | 'keyed' | 'effectRef'>,
+  what: string,
+): DaemonError {
+  return new DaemonError(
+    'outcome_unknown',
+    `the ${call.operation} under idempotency key '${call.keyed.key}' ${what} and its effect may still stand; check ${call.effectRef} before retrying under a new key`,
+    { effectRef: call.effectRef },
+  );
+}

package/src/daemon/imp-harness.ts

@@ -99,6 +99,15 @@ export class ImpHarness implements HarnessHandle {
 
   private killPending = false;
 
+  // Whether the process exited, once the harness stops being followed, and
+  // the waits on that answer until then.
+  private exitConfirmed: boolean | null = null;
+
+  private readonly exitWaiters = new Set<{
+    readonly waited: PromiseWithResolvers<boolean>;
+    readonly timer: ReturnType<typeof setTimeout>;
+  }>();
+
   private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
 
   // The terminal size the open connection's request asked for.
@@ -175,6 +184,28 @@ export class ImpHarness implements HarnessHandle {
     }
   };
 
+  // Only an exit impd reports counts: a host that lost the process or went
+  // to sleep with it inside confirms nothing.
+  readonly waitForExit = (timeoutMs: number): Promise<boolean> => {
+    if (this.exitConfirmed !== null) {
+      return Promise.resolve(this.exitConfirmed);
+    }
+
+    const waited = Promise.withResolvers<boolean>();
+
+    const waiter = {
+      waited,
+      timer: setTimeout(() => {
+        this.exitWaiters.delete(waiter);
+        waited.resolve(false);
+      }, timeoutMs),
+    };
+
+    this.exitWaiters.add(waiter);
+
+    return waited.promise;
+  };
+
   readonly detach = (): void => {
     if (this.done) {
       return;
@@ -521,6 +552,8 @@ export class ImpHarness implements HarnessHandle {
   private emitExit(exit: HarnessExit): void {
     const listeners = [...this.exitListeners];
 
+    this.exitConfirmed = exit.reason === undefined || exit.reason === 'exited';
+
     this.stopFollowing();
 
     for (const listener of listeners) {
@@ -537,7 +570,15 @@ export class ImpHarness implements HarnessHandle {
 
     this.done = true;
     this.connection = null;
+    this.exitConfirmed ??= false;
+
+    for (const waiter of this.exitWaiters) {
+      clearTimeout(waiter.timer);
+
+      waiter.waited.resolve(this.exitConfirmed);
+    }
 
+    this.exitWaiters.clear();
     this.dataListeners.clear();
     this.exitListeners.clear();
     this.attachmentListeners.clear();

package/src/daemon/is-tree-in-reach.ts

@@ -0,0 +1,32 @@
+import type { SessionID } from '../shared/session-id';
+import type { TargetAccess, TargetGrant } from './target-access';
+
+// A session as the tree check reads it: its id, its parent if it has one,
+// and the target it is bound to.
+interface TreeMember extends TargetGrant {
+  readonly id: SessionID;
+  readonly parent: SessionID | null;
+}
+
+/**
+ * Whether the access reaches every session in the tree the given session
+ * belongs to: its top-level session and each sub-session of that one. A
+ * session the list does not hold, or one whose parent it does not hold,
+ * is out of reach.
+ */
+export function isTreeInReach(
+  sessions: readonly TreeMember[],
+  id: SessionID,
+  access: TargetAccess,
+): boolean {
+  const session = sessions.find((x) => x.id === id);
+
+  if (session === undefined) {
+    return false;
+  }
+
+  const root = session.parent ?? session.id;
+  const tree = sessions.filter((x) => x.id === root || x.parent === root);
+
+  return tree.some((x) => x.id === root) && tree.every((x) => access.canUse(x));
+}

package/src/daemon/local-pty-provider.ts

@@ -74,6 +74,35 @@ export class LocalPTYProvider implements ExecutionProvider {
       kill: () => {
         pty.kill();
       },
+
+      // A process already gone has nothing left to end.
+      killForced: () => {
+        try {
+          process.kill(pty.pid, 'SIGKILL');
+        } catch (error) {
+          if (!isMissingProcessError(error)) {
+            throw error;
+          }
+        }
+      },
+
+      // bun-pty's kill sends one SIGHUP and reports an exit at once, whether
+      // the process ended or not, so the exit is read from the process id
+      // instead: the library reaps its child, so the id stops answering a
+      // signal once the process is gone.
+      waitForExit: async (timeoutMs) => {
+        const deadline = Date.now() + timeoutMs;
+
+        while (isProcessRunning(pty.pid)) {
+          if (Date.now() >= deadline) {
+            return false;
+          }
+
+          await Bun.sleep(20);
+        }
+
+        return true;
+      },
       detach: () => {
         for (const subscription of subscriptions) {
           subscription.dispose();
@@ -140,3 +169,19 @@ export class LocalPTYProvider implements ExecutionProvider {
     return { exitCode, stdout, stderr };
   }
 }
+
+// A process another user owns still runs, so only a missing process counts
+// as gone.
+function isProcessRunning(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+
+    return true;
+  } catch (error) {
+    return !isMissingProcessError(error);
+  }
+}
+
+function isMissingProcessError(error: unknown): boolean {
+  return error instanceof Error && 'code' in error && error.code === 'ESRCH';
+}

package/src/daemon/parse-hook-line.ts

@@ -1,6 +1,6 @@
+import type { HookEvent } from '../protocol/hook-event';
 import { isRecord } from '../shared/report';
 import { toSessionID } from '../shared/to-session-id';
-import type { HookEvent } from './hooks';
 
 /**
  * One reporter line as a hook event, or null for a line that is not one.