@relayflows/sdk 2.0.37 → 2.0.39

package/src/worker-cli.ts

@@ -4,7 +4,7 @@ import { diffWorkspaceFiles, snapshotWorkspaceFiles } from './agent-artifacts.js
 import { claudeResultOutcome, decodeProviderResult, decodeWrapperResult, requirePricedUsage } from './worker-usage.js';
 import { openSidechannel, type SidechannelContext } from './pty-sidechannel.js';
 import { openTranscriptWriter, transcriptPath, type TranscriptDigest, type TranscriptFile, type TranscriptWriter } from './agent-transcript.js';
-import { spawn } from 'node:child_process';
+import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
 import { StringDecoder } from 'node:string_decoder';
 import { childStop } from './child-stop.js';
 import { reapOnExit } from './agent-reaper.js';
@@ -29,6 +29,7 @@ import {
   AgentRelayTransportError,
   type AgentTransport,
 } from './agent-relay-transport.js';
+import { pinCliAlias } from './cli/pinned-cli-alias.js';
 
 /** Present only when a dispatched agent step carries a journaled wake context. */
 export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT';
@@ -40,6 +41,11 @@ export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT';
  */
 export const MODEL_ENV = 'RELAYFLOW_MODEL';
 
+/** Preserve a multicall adapter's authored basename while executing pinned bytes. */
+export function cliInvocationArgv0(cli: string, cliIdentity?: string): string {
+  return basename(cliIdentity ?? cli);
+}
+
 export interface WorkerCliResult {
   relay_task?: import('./agent-relay-receipt.js').RelayTaskReceipt;
   tokens_input?: number;
@@ -92,13 +98,15 @@ export async function runAgentCli(
   transport: AgentTransport = 'direct',
   relayContext?: AgentRelayContext,
   processEnvironment: NodeJS.ProcessEnv = process.env,
+  cliIdentity?: string,
 ): Promise<WorkerCliResult> {
   signal?.throwIfAborted();
   if (signal !== undefined && process.platform === 'win32') {
     throw new Error('Lease-bound agent execution requires macOS or Linux process-group cancellation; Windows is unsupported.');
   }
-  const kind = cliAdapterKind(cli);
-  const effectiveModel = resolveCliModel(cli, model);
+  const kind = cliAdapterKind(cliIdentity ?? cli);
+  const effectiveModel = resolveCliModel(cliIdentity ?? cli, model);
+  const argv0 = cliInvocationArgv0(cli, cliIdentity);
 
   if (mode === 'agent' && transport === 'relay') {
     return runViaAgentRelay(kind, instruction, wakeContext, effectiveModel, relayContext, cwd, signal);
@@ -146,6 +154,7 @@ export async function runAgentCli(
       wrapperLimits,
       signal,
       cwd,
+      argv0,
     )), effectiveModel);
   }
 
@@ -176,7 +185,7 @@ export async function runAgentCli(
   const args = [...invocation.args];
   args.splice(args.length - 1, 0, ...(kind === 'claude' ? ['--output-format', 'stream-json', '--verbose'] : ['--json']));
   const completion = kind === 'claude' ? claudeResultOutcome : undefined;
-  return requirePricedUsage(decodeProviderResult(await spawnInvocation(cli, { ...invocation, args }, env, signal, sidechannel, cwd, completion), kind, env), effectiveModel);
+  return requirePricedUsage(decodeProviderResult(await spawnInvocation(cli, { ...invocation, args }, env, signal, sidechannel, cwd, completion, argv0), kind, env), effectiveModel);
   }
 }
 
@@ -315,40 +324,71 @@ async function spawnInvocation(
   sidechannel?: SidechannelContext,
   cwd?: string,
   completion?: (line: string) => { failed: boolean } | undefined,
+  argv0?: string,
 ): Promise<WorkerCliResult> {
   let writeInput: (bytes: Buffer) => Promise<boolean> = async () => false;
   let canDrive = () => false;
   let driven = false;
+  // Prove and pin the authored identity before opening any per-attempt
+  // resources. If pinning fails there is then nothing else to unwind.
+  const pinned = await pinCliAlias(cli, argv0 ?? basename(cli));
+  if (signal?.aborted) {
+    pinned.release();
+    signal.throwIfAborted();
+  }
   // The transcript file lives beside the PTY socket and is named by attempt.
   // Its absence (no data dir, no attempt, unwritable dir) costs the step
   // nothing: the digest is built from the buffered frames regardless. Opened
   // BEFORE the sidechannel: once `onReady` has fired, a drive peer's HELLO may
   // arrive at any time, and nothing may sit between that and the spawn that
   // arms `canDrive`.
-  const writer: TranscriptWriter | undefined = sidechannel?.attempt === undefined ? undefined
-    : await openTranscriptWriter(transcriptPath(sidechannel, sidechannel.attempt), env);
-  const channel = sidechannel === undefined ? undefined : await openSidechannel({
-    ...sidechannel,
-    onDrive() { driven = true; sidechannel.onDrive(); },
-  }, bytes => writeInput(bytes), () => canDrive());
-  // Tee the transcript into bounded tail files beside the socket. Evidence
-  // for `flows status --tail`, never the record; a failure here is a warning.
-  const tails = sidechannel === undefined ? undefined : openTails(sidechannel);
-  if (signal?.aborted) {
+  let writer: TranscriptWriter | undefined;
+  let channel: Awaited<ReturnType<typeof openSidechannel>> | undefined;
+  let tails: ReturnType<typeof openTails>;
+  const closeBeforeSpawn = async (): Promise<void> => {
     channel?.close();
-    void writer?.close();
-    void tails?.stdout.close(); void tails?.stderr.close();
+    const closing = Promise.allSettled([
+      ...(writer === undefined ? [] : [writer.close()]),
+      ...(tails === undefined ? [] : [tails.stdout.close(), tails.stderr.close()]),
+    ]);
+    pinned.release();
+    const deadline = new Promise<void>((done) => setTimeout(done, TAIL_CLOSE_TIMEOUT_MS).unref?.());
+    await Promise.race([closing, deadline]);
+  };
+  try {
+    writer = sidechannel?.attempt === undefined ? undefined
+      : await openTranscriptWriter(transcriptPath(sidechannel, sidechannel.attempt), env);
+    channel = sidechannel === undefined ? undefined : await openSidechannel({
+      ...sidechannel,
+      onDrive() { driven = true; sidechannel.onDrive(); },
+    }, bytes => writeInput(bytes), () => canDrive());
+    // Tee the transcript into bounded tail files beside the socket. Evidence
+    // for `flows status --tail`, never the record; a failure here is a warning.
+    tails = sidechannel === undefined ? undefined : openTails(sidechannel);
+  } catch (error) {
+    await closeBeforeSpawn();
+    throw error;
+  }
+  if (signal?.aborted) {
+    await closeBeforeSpawn();
     signal.throwIfAborted();
   }
-  return new Promise((resolve) => {
-    // Always a group of its own off Windows, so every stop — and
-    // `reapOnExit` — reaches the whole agent tree, lease-bound or not.
-    const ownsGroup = process.platform !== 'win32';
-    const child = spawn(cli, invocation.args, {
+  // Always a group of its own off Windows, so every stop — and
+  // `reapOnExit` — reaches the whole agent tree, lease-bound or not.
+  const ownsGroup = process.platform !== 'win32';
+  let child: ChildProcessWithoutNullStreams;
+  try {
+    child = spawn(pinned.executable, invocation.args, {
       stdio: ['pipe', 'pipe', 'pipe'], env,
       detached: ownsGroup,
+      ...(argv0 === undefined ? {} : { argv0 }),
       ...(cwd === undefined ? {} : { cwd }),
     });
+  } catch (error) {
+    await closeBeforeSpawn();
+    throw error;
+  }
+  return new Promise((resolve) => {
     child.stdin.on('error', () => {});
     if (channel === undefined) child.stdin.end();
     canDrive = () => !child.stdin.destroyed && !child.stdin.writableEnded;
@@ -363,8 +403,7 @@ async function spawnInvocation(
       // until they flush; the sidechannel pauses its reader in the meantime.
       child.stdin.write(bytes, error => resolve(!error));
     });
-    const stop = childStop(child, ownsGroup);
-    const release = ownsGroup ? reapOnExit(stop) : () => {};
+    let release = () => {};
     const stdout: Buffer[] = [];
     const stderr: Buffer[] = [];
     let settled = false;
@@ -413,17 +452,66 @@ async function spawnInvocation(
           ? result : { ...result, transcript: { file: transcriptFile } });
       }, () => resolve(result));
     };
+    let pendingStopResult: {
+      result: WorkerCliResult;
+      discardTranscript: boolean;
+      priority: 'normal' | 'abort';
+    } | undefined;
+    const stop = childStop(child, ownsGroup, undefined, (stopError) => {
+      if (stopError !== undefined) {
+        finish({ exit_code: null, stdout_tail: '', stderr_tail: stopError.message }, true);
+        return;
+      }
+      try {
+        pinned.release();
+      } catch (error) {
+        finish({
+          exit_code: null,
+          stdout_tail: '',
+          stderr_tail: `CLI invocation alias cleanup failed: ${String(error)}`,
+        }, true);
+        return;
+      }
+      if (pendingStopResult !== undefined) {
+        finish(pendingStopResult.result, pendingStopResult.discardTranscript);
+      }
+    });
+    release = ownsGroup ? reapOnExit(stop, () => {
+      try { pinned.release(); } catch { /* host exit cannot report another result */ }
+    }) : () => {};
+    const finishAfterStop = (
+      result: WorkerCliResult,
+      action: 'kill' | 'terminate',
+      discardTranscript = false,
+      priority: 'normal' | 'abort' = 'normal',
+    ): void => {
+      // Lease loss may replace an earlier result while its stop is pending;
+      // once it does, no later result-grace or execution timer may replace it.
+      if (pendingStopResult !== undefined) {
+        if (pendingStopResult.priority === 'abort' || priority === 'normal') return;
+      }
+      pendingStopResult = { result, discardTranscript, priority };
+      stop[action]();
+    };
     const onAbort = (): void => {
-      stop.kill();
-      finish({ exit_code: null, stdout_tail: '', stderr_tail: 'Agent execution aborted: lease ownership lost.' }, true);
+      if (timer !== undefined) clearTimeout(timer);
+      timer = undefined;
+      if (graceTimer !== undefined) clearTimeout(graceTimer);
+      graceTimer = undefined;
+      finishAfterStop(
+        { exit_code: null, stdout_tail: '', stderr_tail: 'Agent execution aborted: lease ownership lost.' },
+        'kill',
+        true,
+        'abort',
+      );
     };
     /**
      * Same invariant as `wrapper-session.ts`: `'close'` and `'error'` are
      * evidence about the DIRECT CHILD, so they may not settle over a pending
      * escalation, and only `maySettleOnChildExit` may drop one. This settle
-     * carries no deadline of its own because it needs none — the timeout below
-     * settles on the spot and lets its escalation outlive that, so refusing
-     * here can only defer to a `'close'` we are still going to get.
+     * carries no deadline of its own because it needs none — a stop-owned
+     * result now settles from the shared confirmation callback, while a normal
+     * child exit can only defer to a `'close'` we are still going to get.
      */
     const finishOnChildExit = (result: WorkerCliResult): void => {
       if (!stop.maySettleOnChildExit()) return;
@@ -441,12 +529,11 @@ async function spawnInvocation(
     const onResult = (outcome: { failed: boolean }): void => {
       if (graceTimer !== undefined || settled) return;
       graceTimer = setTimeout(() => {
-        stop.terminate();
-        finish({
+        finishAfterStop({
           exit_code: outcome.failed ? 1 : 0,
           stdout_tail: Buffer.concat(stdout).toString('utf8'),
           stderr_tail: `${Buffer.concat(stderr).toString('utf8')}\nCLI reported its final result but had not exited ${RESULT_EXIT_GRACE_MS}ms later; its process tree was stopped.`.trim(),
-        });
+        }, 'terminate');
       }, RESULT_EXIT_GRACE_MS);
     };
     child.stdout.on('data', (chunk: Buffer) => {
@@ -483,15 +570,14 @@ async function spawnInvocation(
     }));
     if (invocation.timeoutMs > 0) {
       timer = setTimeout(() => {
-        // The stop outlives this settle on purpose: `finish` resolves the step,
-        // but only the forced group kill releases the pipes a leaked descendant
-        // is holding, and until they are released `flows run` cannot exit.
-        stop.terminate();
-        finish({
+        // The stop owns this settle: a direct-child event is not proof that its
+        // group is gone, and an unprovable forced stop must fail the step rather
+        // than arriving after a successful result has already won the race.
+        finishAfterStop({
           exit_code: null,
           stdout_tail: Buffer.concat(stdout).toString('utf8'),
           stderr_tail: `CLI invocation timed out after ${invocation.timeoutMs}ms.`,
-        });
+        }, 'terminate');
       }, invocation.timeoutMs);
     }
   });

package/src/worker-lease.ts

@@ -2,8 +2,12 @@ import { JournalProtocolError, type JournalClient } from './journal-client.js';
 import type { StepDispatchEvent } from './protocol.js';
 
 export class WorkerLeaseLostError extends Error {
-  constructor(readonly reason: 'already_expired' | 'renewal_expired' | 'completion_expired', message: string) {
-    super(message);
+  constructor(
+    readonly reason: 'already_expired' | 'renewal_expired' | 'completion_expired',
+    message: string,
+    options?: { cause?: unknown },
+  ) {
+    super(message, options);
     this.name = 'WorkerLeaseLostError';
   }
 }
@@ -36,13 +40,19 @@ export async function withWorkerLease<T>(
 ): Promise<T> {
   const controller = new AbortController();
   let stopped = false;
-  let latestDeadline = dispatch.lease_deadline_ms;
+  // Every deadline below is on THIS process's monotonic clock. The daemon
+  // issues `lease_deadline_ms` on its own wall clock, and comparing it with
+  // ours made the lease exactly as good as the two clocks' agreement: a
+  // worker 45s ahead refused every dispatch as already expired, one 75s
+  // behind scheduled its first renewal after the daemon had swept the lease
+  // (customer rw_3a0fcb71; tests/worker-lease-drift-live.test.ts).
+  let latestDeadline = Number.NaN;
   let renewalTimer: NodeJS.Timeout | undefined;
   let expiryTimer: NodeJS.Timeout | undefined;
   let pending: Promise<void> = Promise.resolve();
   const fail = (error: unknown): void => { controller.abort(error); };
   const armExpiry = (deadline: number): number => {
-    const remaining = deadline - Date.now();
+    const remaining = deadline - performance.now();
     if (!Number.isFinite(remaining)) {
       throw new Error(`Agent lease is already expired for ${dispatch.run_id}/${dispatch.step_id}.`);
     }
@@ -57,16 +67,19 @@ export async function withWorkerLease<T>(
     return remaining;
   };
   const renew = async (): Promise<void> => {
+    // Anchored BEFORE the request: the daemon starts the renewed lease no
+    // earlier than it receives this, so the local deadline errs early.
+    const sentAt = performance.now();
     const result = await untilAborted(client.stepHeartbeat(
       dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.lease_id,
     ), controller.signal);
     controller.signal.throwIfAborted();
     // A response handled after local expiry cannot revive ownership, even
     // if its future deadline was issued before this event loop stalled.
-    if (Date.now() >= latestDeadline) {
+    if (performance.now() >= latestDeadline) {
       throw new WorkerLeaseLostError('renewal_expired', `Agent lease expired before renewal for ${dispatch.run_id}/${dispatch.step_id}.`);
     }
-    const remaining = armExpiry(result.lease_deadline_ms);
+    const remaining = armExpiry(localDeadline(result.lease_deadline_ms, result.lease_ttl_ms, sentAt));
     if (!stopped) {
       renewalTimer = setTimeout(() => {
         pending = renew().catch(fail);
@@ -74,10 +87,26 @@ export async function withWorkerLease<T>(
     }
   };
   try {
-    armExpiry(dispatch.lease_deadline_ms);
+    armExpiry(localDeadline(dispatch.lease_deadline_ms, dispatch.lease_ttl_ms, performance.now()));
     // Establish ownership before starting an effectful process.
     await renew();
-    const result = await execute(controller.signal);
+    let result: T;
+    try {
+      result = await execute(controller.signal);
+    } catch (error) {
+      // A body that rejected because the lease was lost under it reports
+      // whatever its wait happened to say ("worker wait canceled"). The lease
+      // is the cause, and callers branch on it (isLeaseLost): an attempt the
+      // kernel will retry must not be recorded as the body's own failure.
+      const lost = controller.signal.reason;
+      if (isLeaseLost(lost) && error !== lost) {
+        if (lost instanceof WorkerLeaseLostError) {
+          throw new WorkerLeaseLostError(lost.reason, lost.message, { cause: error });
+        }
+        throw lost;
+      }
+      throw error;
+    }
     stopped = true;
     if (renewalTimer !== undefined) clearTimeout(renewalTimer);
     // Drain any renewal before the caller sends step.complete. A renewal
@@ -86,7 +115,7 @@ export async function withWorkerLease<T>(
     controller.signal.throwIfAborted();
     // Timer callbacks can be delayed behind a resolved subprocess promise.
     // Check the clock itself before permitting step.complete.
-    if (Date.now() >= latestDeadline) {
+    if (performance.now() >= latestDeadline) {
       throw new WorkerLeaseLostError('completion_expired', `Agent lease expired before completion for ${dispatch.run_id}/${dispatch.step_id}.`);
     }
     return result;
@@ -99,6 +128,18 @@ export async function withWorkerLease<T>(
   }
 }
 
+/**
+ * A daemon lease deadline on this process's monotonic clock. With the
+ * daemon's `lease_ttl_ms` the wall clocks never meet: the duration is added to
+ * `anchor`, a moment no later than the daemon started it. An older daemon
+ * sends only the absolute deadline, which keeps the old wall-clock comparison
+ * -- read NOW, not at `anchor`, or the round trip would be subtracted twice.
+ */
+function localDeadline(daemonDeadlineMs: number, ttlMs: number | undefined, anchor: number): number {
+  if (typeof ttlMs === 'number' && Number.isFinite(ttlMs)) return anchor + ttlMs;
+  return performance.now() + (daemonDeadlineMs - Date.now());
+}
+
 function untilAborted<T>(request: Promise<T>, signal: AbortSignal): Promise<T> {
   return new Promise((resolve, reject) => {
     const abort = (): void => { reject(signal.reason); };

package/src/worker.ts

@@ -5,7 +5,7 @@ import type { WorkerCliResult } from './worker-cli.js';
 import { EventEmitter } from 'node:events';
 import type { JournalClient } from './journal-client.js';
 import type { Pins, StepDispatchEvent } from './protocol.js';
-import type { KernelAgentStep } from './spec.js';
+import type { ResolvedKernelAgentStep } from './resolved-cli-identity.js';
 import { runAgentCli } from './worker-cli.js';
 import { agentStepCwd } from './agent-cwd.js';
 import { resolveCliModel } from './cli-adapter.js';
@@ -114,7 +114,7 @@ export class AgentWorker extends EventEmitter {
   };
 
   private async execute(dispatch: StepDispatchEvent): Promise<void> {
-    const spec = dispatch.spec as Partial<KernelAgentStep>;
+    const spec = dispatch.spec as Partial<ResolvedKernelAgentStep>;
     const helper = helperCall(spec);
     if (helper !== undefined) {
       if (this.options.dataDir === undefined) throw new Error('Helper worker requires a data directory for durable receipts');
@@ -130,7 +130,9 @@ export class AgentWorker extends EventEmitter {
       return;
     }
     let humanIntervention = false;
-    const effectiveModel = typeof spec.cli === 'string' ? resolveCliModel(spec.cli, spec.model) : spec.model;
+    const effectiveModel = typeof spec.cli === 'string'
+      ? resolveCliModel(spec.cli_identity ?? spec.cli, spec.model)
+      : spec.model;
     // Resolved here, before the lease, because this is the process that shares
     // the agent's filesystem. A refusal completes the step the way a missing
     // CLI does — journaled as `worker_error` with the reason — rather than
@@ -147,7 +149,7 @@ export class AgentWorker extends EventEmitter {
             spec.transport === 'relay' ? 'relay' : 'direct',
             { runId: dispatch.run_id, stepId: dispatch.step_id, idempotencyKey: dispatch.idempotency_key,
               dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema },
-            this.options.environment)
+            this.options.environment, spec.cli_identity)
           : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' }));
     const { result, usage } = workerSpend(completed, effectiveModel);
     const cost = reportedCost(completed, effectiveModel);