@nifrajs/coding-agent 4.0.2 → 5.1.0

package/src/orchestration/dispatch.ts

@@ -15,7 +15,11 @@ export const DISPATCH_JOB_NAME = "nifra.agent.run-node"
 export const MAX_DISPATCH_STRING = 256
 export const MAX_DISPATCH_ATTEMPTS = 16
 
-export type DispatchBoundary = "before-effect" | "after-effect"
+/**
+ * `before-effect`: no side effect has happened, so a redelivery may run the effect. `effect-started`:
+ * the effect was invoked and may have acted, so a redelivery needs a proof. `after-effect`: it acted.
+ */
+export type DispatchBoundary = "before-effect" | "effect-started" | "after-effect"
 export type DispatchState =
   | "queued"
   | "leased"
@@ -300,7 +304,7 @@ export function parseRunCheckpoint(value: unknown): RunCheckpoint {
   if (input.version !== RUN_DISPATCH_VERSION)
     throw new TypeError("run checkpoint.version is unsupported")
   const boundary = input.boundary
-  if (boundary !== "before-effect" && boundary !== "after-effect")
+  if (boundary !== "before-effect" && boundary !== "effect-started" && boundary !== "after-effect")
     throw new TypeError("run checkpoint.boundary is invalid")
   return Object.freeze({
     version: RUN_DISPATCH_VERSION,

package/src/orchestration/evidence-store.ts

@@ -10,6 +10,7 @@
  * evidence only, and the file store persists exactly the content-free record it parsed.
  */
 
+import { createHash } from "node:crypto"
 import { appendFile } from "node:fs/promises"
 import type { ArtifactRef, RunEvidence } from "@nifrajs/agent-protocol"
 import { assertEvidenceSize, parseRunEvidence } from "@nifrajs/agent-protocol"
@@ -41,12 +42,10 @@ export interface EvidenceStore {
 }
 
 const DEFAULT_MAX_LIVE = 1024
-const encoder = new TextEncoder()
 
-async function sha256Bytes(text: string): Promise<Uint8Array> {
-  const digest = await crypto.subtle.digest("SHA-256", encoder.encode(text))
-  return new Uint8Array(digest)
-}
+// Synchronous: `crypto.subtle` hands every record to a worker thread and back, which on a loaded
+// machine made a 10k-record run take seconds.
+const sha256Bytes = (text: string): Uint8Array => createHash("sha256").update(text).digest()
 
 /**
  * The identity of one record for the terminal digest: node outcome only. `seq`, `durationMs`, and
@@ -97,7 +96,7 @@ class EvidenceAccumulator {
     } else this.failedCount++
     if (record.artifacts !== undefined)
       for (const ref of record.artifacts) this.artifactRefs.push(ref)
-    const hash = await sha256Bytes(digestKey(record))
+    const hash = sha256Bytes(digestKey(record))
     let carry = 0
     for (let i = 31; i >= 0; i--) {
       const sum = (this.acc[i] as number) + (hash[i] as number) + carry

package/src/orchestration/host.ts

@@ -72,6 +72,13 @@ export interface RunResult {
   readonly failureCode?: string
 }
 
+/** Why one node failed, handed to {@link OrchestrationHostOptions.onStepFailed}. */
+export interface StepFailure {
+  readonly code: "STEP_FAILED" | "GATE_REJECTED"
+  /** What the step threw (bounded), or a rejecting verify gate's string `reason`. Not masked. */
+  readonly message?: string
+}
+
 export interface SubmitOptions {
   /** Caller-supplied for reproducibility; defaults to a per-host monotonic id (no clock, no random). */
   readonly runId?: string
@@ -98,9 +105,17 @@ export interface OrchestrationHostOptions {
    * running. `limits.maxAbandonedSubagents` bounds how many may run at once across all runs.
    */
   onSubagentAbandoned?(runId: string, nodeId: string, abandonment: SubagentAbandonment): void
+  /**
+   * Called when a node fails, with what it threw or why a verify gate refused. Evidence and
+   * {@link RunResult} stay content-free, so this is the only place the text goes: mask anything
+   * sensitive before keeping it. A throw here is ignored.
+   */
+  onStepFailed?(runId: string, nodeId: string, failure: StepFailure): void
 }
 
 const DEFAULT_MAX_STEPS = 100_000
+// Matches the bound the workflow kernel puts on a thrown step's message.
+const MAX_FAILURE_MESSAGE = 4096
 const DEFAULT_MAX_DEPTH = 64
 
 interface RunRecord {
@@ -136,6 +151,7 @@ export class OrchestrationHost {
   private counter = 0
   private readonly subagentAbandonment: SubagentAbandonmentLedger = { abandoned: 0 }
   private readonly onSubagentAbandoned: OrchestrationHostOptions["onSubagentAbandoned"]
+  private readonly onStepFailed: OrchestrationHostOptions["onStepFailed"]
 
   constructor(options: OrchestrationHostOptions) {
     this.catalog = options.catalog
@@ -144,6 +160,7 @@ export class OrchestrationHost {
     this.maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH
     this.policy = options.policy
     this.onSubagentAbandoned = options.onSubagentAbandoned
+    this.onStepFailed = options.onStepFailed
   }
 
   /** Subagent executors abandoned by any run of this host that are still running. */
@@ -348,6 +365,9 @@ export class OrchestrationHost {
   }
 
   private async drive(run: RunRecord): Promise<void> {
+    // One map for every layer, so a node reads what an earlier layer set. It lives only as long as
+    // this drive: values are never written into the run's evidence.
+    const values = new Map<string, unknown>()
     try {
       for (; run.layerIndex < run.layers.length; run.layerIndex++) {
         while (run.state === "paused") await run.gate?.promise
@@ -355,6 +375,7 @@ export class OrchestrationHost {
         const layer = run.layers[run.layerIndex] as WorkflowStep
         const result = await new WorkflowRunner({
           signal: run.controller.signal,
+          values,
           maxSteps: this.maxSteps,
           maxDepth: this.maxDepth,
           onEvent: (event) => this.project(run, event),
@@ -397,6 +418,7 @@ export class OrchestrationHost {
       if (isGateRejection(run.kindById.get(event.id), event.output)) {
         run.failureCode = "E_GATE_REJECTED"
         await run.store.append({ ...base, status: "failed", errorCode: "GATE_REJECTED", ...effect })
+        this.reportFailure(run.runId, event.id, "GATE_REJECTED", gateReason(event.output))
         return
       }
       const refs = run.artifacts.get(event.id)
@@ -411,6 +433,21 @@ export class OrchestrationHost {
     if (event.type === "step.failed") {
       run.failureCode = "E_STEP_FAILED"
       await run.store.append({ ...base, status: "failed", errorCode: "STEP_FAILED", ...effect })
+      this.reportFailure(run.runId, event.id, "STEP_FAILED", event.error)
+    }
+  }
+
+  private reportFailure(
+    runId: string,
+    nodeId: string,
+    code: StepFailure["code"],
+    message: string | undefined,
+  ): void {
+    if (this.onStepFailed === undefined) return
+    try {
+      this.onStepFailed(runId, nodeId, message === undefined ? { code } : { code, message })
+    } catch {
+      // The caller's sink must not decide the run's outcome.
     }
   }
 
@@ -448,6 +485,13 @@ export class OrchestrationHost {
 }
 
 /** A verify/approve node whose output is not a pass. Other kinds never gate. */
+function gateReason(output: unknown): string | undefined {
+  const reason =
+    typeof output === "object" && output !== null && "reason" in output ? output.reason : undefined
+  if (typeof reason !== "string" || reason.length === 0) return undefined
+  return reason.length > MAX_FAILURE_MESSAGE ? `${reason.slice(0, MAX_FAILURE_MESSAGE)}…` : reason
+}
+
 function isGateRejection(kind: RunNodeKind | undefined, output: unknown): boolean {
   if (kind === "verify") {
     if (output === true) return false

package/src/orchestration/index.ts

@@ -104,6 +104,7 @@ export {
   type RunResult,
   type RunState,
   type RunStatus,
+  type StepFailure,
   type SubmitOptions,
 } from "./host.ts"
 export {

package/src/orchestration/recovery.ts

@@ -9,6 +9,7 @@
  */
 
 import {
+  type DispatchBoundary,
   type DispatchResultCode,
   type InjectedClock,
   type RunDispatchStore,
@@ -45,7 +46,11 @@ export interface CommittedEffect {
 export interface RunEffectContext {
   readonly lease: RunLease
   readonly signal: AbortSignal
-  /** Persist a content-free safe boundary; a failed write prevents the next unsafe step. */
+  /**
+   * Persist a content-free safe boundary; a failed write prevents the next unsafe step. Recovery
+   * marks the effect started before invoking it, so a worker lost mid-effect is dead-lettered
+   * unless proven; `before-effect` declares nothing has acted yet, so a redelivery may rerun it.
+   */
   readonly checkpoint: (boundary: "before-effect" | "after-effect") => Promise<boolean>
 }
 
@@ -168,7 +173,11 @@ export class RunRecoveryMachine {
     const inspection = await this.store.inspect(lease.dispatch.dispatchId)
     const checkpoint = inspection?.checkpoint
     const proof = await this.proofs.has(lease.idempotencyKey)
-    if (proof || checkpoint?.boundary === "after-effect") {
+    if (
+      proof ||
+      checkpoint?.boundary === "after-effect" ||
+      checkpoint?.boundary === "effect-started"
+    ) {
       if (!proof) {
         await this.store.deadLetter(lease, "idempotency_required")
         return "dead-lettered"
@@ -177,8 +186,8 @@ export class RunRecoveryMachine {
       return completed.ok ? "completed" : "waiting"
     }
 
-    const before = await this.storeCheckpoint(lease, "before-effect")
-    if (!before) return "waiting"
+    const started = await this.storeCheckpoint(lease, "effect-started")
+    if (!started) return "waiting"
     if (signal.aborted) {
       await this.store.cancel(lease.dispatch.runId, "cancelled")
       return "cancelled"
@@ -229,10 +238,7 @@ export class RunRecoveryMachine {
     }
   }
 
-  private async storeCheckpoint(
-    lease: RunLease,
-    boundary: "before-effect" | "after-effect",
-  ): Promise<boolean> {
+  private async storeCheckpoint(lease: RunLease, boundary: DispatchBoundary): Promise<boolean> {
     const outcome = await this.store.checkpoint(lease, {
       version: 1,
       dispatchId: lease.dispatch.dispatchId,

package/src/process.ts

@@ -3,9 +3,11 @@ export interface BoundedText {
   readonly truncated: boolean
 }
 
+/** Read at most `maxBytes` of `stream`; `signal` stops the read early with what arrived so far. */
 export async function readBoundedText(
   stream: ReadableStream<Uint8Array> | null | undefined | number,
   maxBytes: number,
+  signal?: AbortSignal,
 ): Promise<BoundedText> {
   if (stream === null || stream === undefined || typeof stream === "number")
     return { text: "", truncated: false }
@@ -16,6 +18,11 @@ export async function readBoundedText(
   const chunks: string[] = []
   let total = 0
   let truncated = false
+  const stop = (): void => {
+    reader.cancel().catch(() => {})
+  }
+  if (signal?.aborted) stop()
+  else signal?.addEventListener("abort", stop, { once: true })
   try {
     for (;;) {
       const { done, value } = await reader.read()
@@ -41,7 +48,89 @@ export async function readBoundedText(
       chunks.push(decoder.decode(value, { stream: true }))
     }
   } finally {
+    signal?.removeEventListener("abort", stop)
     reader.releaseLock()
   }
   return { text: chunks.join(""), truncated }
 }
+
+export interface BoundedProcessOptions {
+  readonly cwd: string
+  readonly env: Record<string, string>
+  readonly timeoutMs: number
+  readonly maxStdoutBytes: number
+  readonly maxStderrBytes: number
+}
+
+export interface BoundedProcessResult {
+  readonly stdout: BoundedText
+  readonly stderr: BoundedText
+  readonly status: number | null
+  readonly timedOut: boolean
+}
+
+const KILL_GRACE_MS = 2_000
+// How long the pipes are still read once the child has exited, for output it wrote just before.
+const DRAIN_GRACE_MS = 2_000
+const PROCESS_GROUPS = process.platform !== "win32"
+
+/**
+ * Run a trusted command to completion within `timeoutMs`. On POSIX the child leads its own process
+ * group, so a timeout reaches every process it started: SIGTERM, then SIGKILL after a grace period.
+ * The call settles when the child exits: a process it left running that still holds the pipes open
+ * is read from for a short grace and then no longer waited on, and `timedOut` stays `false`.
+ */
+export async function runBoundedProcess(
+  argv: readonly string[],
+  options: BoundedProcessOptions,
+): Promise<BoundedProcessResult> {
+  const proc = Bun.spawn([...argv], {
+    cwd: options.cwd,
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "pipe",
+    env: options.env,
+    detached: PROCESS_GROUPS,
+  })
+  const signalAll = (signal: "SIGTERM" | "SIGKILL"): void => {
+    try {
+      if (PROCESS_GROUPS) process.kill(-proc.pid, signal)
+      else proc.kill(signal)
+    } catch {
+      // The group is already gone.
+    }
+  }
+  const reading = new AbortController()
+  const graceMs = Math.min(KILL_GRACE_MS, options.timeoutMs)
+  let timedOut = false
+  let escalation: ReturnType<typeof setTimeout> | undefined
+  const timer = setTimeout(() => {
+    timedOut = true
+    signalAll("SIGTERM")
+    escalation = setTimeout(() => {
+      signalAll("SIGKILL")
+      void proc.exited.then(() => reading.abort())
+    }, graceMs)
+  }, options.timeoutMs)
+  let drain: ReturnType<typeof setTimeout> | undefined
+  // Awaited below, so the drain timer is armed before the `finally` that clears it can run.
+  const exited = proc.exited.then((status) => {
+    if (!timedOut) {
+      clearTimeout(timer)
+      drain = setTimeout(() => reading.abort(), DRAIN_GRACE_MS)
+    }
+    return status
+  })
+  try {
+    const [stdout, stderr, status] = await Promise.all([
+      readBoundedText(proc.stdout, options.maxStdoutBytes, reading.signal),
+      readBoundedText(proc.stderr, options.maxStderrBytes, reading.signal),
+      exited,
+    ])
+    return { stdout, stderr, status, timedOut }
+  } finally {
+    clearTimeout(timer)
+    clearTimeout(escalation)
+    clearTimeout(drain)
+  }
+}