@agent-compose/sdk 0.5.1 → 0.5.5

package/src/types/workflow.ts

@@ -210,7 +210,12 @@ function compileRunForm<TOutput, TInput extends Record<string, unknown>>(
         setMetadata:    stepCtx.setMetadata,
         step:           stepCtx.step,
         agentEvents:    stepCtx.agentEvents,
-        processors:     metadata.processors ?? [],
+        checkpoint:      stepCtx.checkpoint,
+        pause:           stepCtx.pause,
+        requestDecision: stepCtx.requestDecision,
+        sleep:           stepCtx.sleep,
+        waitForEvent:    stepCtx.waitForEvent,
+        processors:      metadata.processors ?? [],
       };
       const sandbox = stepCtx.sandbox;
       if (!sandbox) throw new Error("legacy run-form workflow requires a sandbox in StepContext");

package/src/utils/bundler.ts

@@ -111,8 +111,9 @@ async function bundle(path: string, label: string): Promise<string> {
 }
 
 /** Dynamically import a bundled source in-process and extract fields from its
- *  default export. Safe for CLI/test contexts. Returns `undefined` on
- *  resolver failure (missing deps) so callers fall through. */
+ *  default export. Safe for CLI/test contexts. Evaluation failures are
+ *  surfaced with their original cause so users fix the real import/runtime
+ *  problem instead of being told the default export shape is wrong. */
 async function extractFromBundle<T>(
   source: string,
   label: string,
@@ -123,7 +124,11 @@ async function extractFromBundle<T>(
     const loaded = await importSourceModule<Record<string, unknown>>(source, tmpPath);
     try { return pick(loaded.mod.default); }
     finally { await loaded.cleanup(); }
-  } catch { return undefined; }
+  } catch (err) {
+    throw new WorkflowSourceValidationError(
+      `Bundled ${label} could not be evaluated: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
 }
 
 /**

package/src/utils/schemas.ts

@@ -7,4 +7,6 @@ export const AgentStatusSchema = z.object({
   completed:    z.array(z.string()),
   blockers:     z.array(z.string()),
   exit_signal:  z.boolean(),
+  needs_input:  z.boolean().optional(),
+  question:     z.string().optional(),
 }) satisfies z.ZodType<AgentStatus>;

package/src/workflow-steps/index.ts

@@ -28,3 +28,6 @@ export { WORKFLOW_BRAND } from "./types.js";
 
 export { StepObservabilityCollector } from "./observability.js";
 export type { StepObservability, SubStepEvent } from "./observability.js";
+
+export { makeRunCallbackEmitterFromEnv } from "./run-callback.js";
+export type { LiveAgentEventEmitter } from "./run-callback.js";

package/src/workflow-steps/observability.ts

@@ -5,19 +5,34 @@
  * tokenised stdout sentinel; the activity persists the snapshot to the
  * run's metadata + lifecycle event tables.
  *
- * STREAMING — known gap. `agentEvents` are currently batched at step
- * end. For long-running steps that emit many agent events, this hides
- * progress from the dashboard until the step completes. The follow-up
- * is to bring back a minimal `/internal/runs/:runId/steps/:stepIndex/events`
- * route gated by a per-step JIT-signed token (mint in the activity,
- * stamp into the runner env, verify on receive) so `agentEvents.emit`
- * POSTs in real time. Until then `metadata` and `subSteps` are
- * effectively boundary events (no streaming need) and batching them
- * matches their natural granularity.
+ * Live streaming + batch backstop — the no-double-write design:
+ *
+ *   1. `agentEvents.emit` immediately fires the optional `liveEmitter`
+ *      (the runner's POST to `/internal/runs/.../events`).
+ *   2. The emitter returns `Promise<boolean>` — true means the server
+ *      accepted and persisted this event, false means it failed (POST
+ *      error, server 5xx, network blip).
+ *   3. The collector tracks which seqs were successfully ack'd.
+ *   4. At `snapshot()` time we await any in-flight emits (with a small
+ *      grace window so the agent loop's final-burst posts can finish),
+ *      then STRIP ack'd events from the returned `events` array.
+ *
+ *   The result: the snapshot's `events` array only contains events
+ *   that the live path didn't successfully deliver. The server's
+ *   batch-flush in `persistStepObservability` becomes a true backstop
+ *   for the FAILURE path — it never re-writes (and never re-notifies)
+ *   the events the live route already handled. No double pg_notify,
+ *   no dashboard duplicates.
+ *
+ *   `metadata` and `subSteps` remain batch-only because they're
+ *   naturally boundary events (no streaming benefit) and aren't
+ *   written by the live route at all.
  */
 
 import type { AgentLifecycleEvent } from "../agent/agent-loop.js";
 import type { AgentEventSink } from "../types/workflow.js";
+import type { LiveAgentEventEmitter } from "./run-callback.js";
+import { PauseSignal, isPauseSignal } from "../pause/pause-core.js";
 
 /** One named sub-step (from `ctx.step("name", async () => ...)`).
  *  Becomes a `workflow_substep_*` lifecycle event on the run timeline. */
@@ -39,6 +54,13 @@ export interface StepObservability {
   subSteps?: SubStepEvent[];
 }
 
+/** Maximum time we'll wait for in-flight live-emit POSTs to settle
+ *  before taking the snapshot. Tuned to be longer than a healthy POST
+ *  (~50ms) but short enough that a totally-broken live path doesn't
+ *  block step return — the batch backstop will handle whatever doesn't
+ *  resolve in time. */
+const LIVE_EMIT_DRAIN_TIMEOUT_MS = 1_500;
+
 /**
  * Append-only collector bound to a single step's `StepContext`. The
  * step's `setMetadata` / `step` / `agentEvents` properties all point at
@@ -53,6 +75,15 @@ export class StepObservabilityCollector {
   private metadata: Record<string, unknown> = {};
   private events:   AgentLifecycleEvent[]   = [];
   private subSteps: SubStepEvent[]          = [];
+  private readonly liveEmitter?: LiveAgentEventEmitter;
+  /** Seqs of events the server confirmed via the live route. */
+  private readonly ackedSeqs: Set<number> = new Set();
+  /** Promises for in-flight live emits — awaited at snapshot time. */
+  private readonly inFlight: Set<Promise<void>> = new Set();
+
+  constructor(opts: { liveEmitter?: LiveAgentEventEmitter } = {}) {
+    this.liveEmitter = opts.liveEmitter;
+  }
 
   readonly setMetadata = async (data: Record<string, unknown>): Promise<void> => {
     Object.assign(this.metadata, data);
@@ -70,6 +101,9 @@ export class StepObservabilityCollector {
       });
       return result;
     } catch (err) {
+      // A pause inside ctx.step is control flow, not a failed sub-step — let
+      // it propagate untouched so serveStep emits the pause sentinel.
+      if (isPauseSignal(err)) throw err;
       this.subSteps.push({
         name,
         startedAt,
@@ -83,20 +117,57 @@ export class StepObservabilityCollector {
 
   readonly agentEvents: AgentEventSink = {
     emit: (event: AgentLifecycleEvent) => {
+      // `seq` is the event's index in `events`, captured BEFORE push so
+      // it matches the index the server-side batch flush uses (its
+      // `.entries()` loop). Same index → same idempotency key on the
+      // server. The live route's `acceptedSeqs` response uses this seq;
+      // we strip those from the snapshot in `snapshot()`.
+      const seq = this.events.length;
       this.events.push(event);
+      if (this.liveEmitter) {
+        const tracked = this.liveEmitter(event, seq)
+          .then((ok) => { if (ok) this.ackedSeqs.add(seq); })
+          .catch(() => { /* leave unacked → batch backstop delivers */ });
+        this.inFlight.add(tracked);
+        // Self-clean so completed promises don't leak across long-
+        // running steps.
+        void tracked.finally(() => this.inFlight.delete(tracked));
+      }
     },
   };
 
+  /** Wait for in-flight live emits to settle (or timeout) so the
+   *  ackedSeqs set is maximally up-to-date before we filter. Used by
+   *  `snapshot()` — exposed separately for tests. */
+  private async drainInFlight(timeoutMs = LIVE_EMIT_DRAIN_TIMEOUT_MS): Promise<void> {
+    if (this.inFlight.size === 0) return;
+    await Promise.race([
+      Promise.allSettled([...this.inFlight]),
+      new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
+    ]);
+  }
+
   /** Snapshot the accumulated state. Returns `undefined` when nothing
-   *  was recorded so the wire payload can drop the field entirely. */
-  snapshot(): StepObservability | undefined {
+   *  was recorded so the wire payload can drop the field entirely.
+   *
+   *  Async because we drain in-flight live emits first. Any event the
+   *  server acknowledged is REMOVED from the returned `events` array
+   *  so the server-side batch flush doesn't re-write/re-notify it.
+   *  Events that failed live delivery (POST error, timeout) stay in
+   *  the array as the durable backstop. */
+  async snapshot(): Promise<StepObservability | undefined> {
+    await this.drainInFlight();
+
     const hasMetadata = Object.keys(this.metadata).length > 0;
-    const hasEvents   = this.events.length > 0;
     const hasSubSteps = this.subSteps.length > 0;
+    // Filter out ack'd events — the live route already wrote them.
+    const remainingEvents = this.events.filter((_, idx) => !this.ackedSeqs.has(idx));
+    const hasEvents = remainingEvents.length > 0;
+
     if (!hasMetadata && !hasEvents && !hasSubSteps) return undefined;
     const result: StepObservability = {};
     if (hasMetadata) result.metadata = { ...this.metadata };
-    if (hasEvents)   result.events   = [...this.events];
+    if (hasEvents)   result.events   = remainingEvents;
     if (hasSubSteps) result.subSteps = [...this.subSteps];
     return result;
   }

package/src/workflow-steps/run-callback.ts

@@ -0,0 +1,72 @@
+/**
+ * Runner → server live event emitter.
+ *
+ * Pairs with the server-side `/api/v1/internal/runs/:runId/steps/:stepIndex/events`
+ * route. The runner reads `AGENT_COMPOSE_URL`, `AGENT_COMPOSE_RUN_TOKEN`,
+ * and `RUN_ID` from env; if all three are present it builds a POST
+ * function that sends each agent lifecycle event to the server as
+ * soon as the agent loop emits it.
+ *
+ * Non-blocking: the agent loop doesn't await the emitter — the runtime
+ * keeps emitting events at full speed. But the emitter still returns a
+ * `Promise<boolean>` so the collector can later decide whether to
+ * include each event in the batch-end durable backstop. `true` = server
+ * accepted the event (don't re-deliver in batch); `false` = POST
+ * failed (DO re-deliver). The collector awaits these promises with a
+ * short grace window at snapshot time.
+ *
+ * This is the seam that lets us avoid the double-write problem: every
+ * successfully-ack'd live event is stripped from `observability.events`
+ * before the batch flush runs, so the durable backstop only carries
+ * events that genuinely failed live delivery. No more two paths writing
+ * the same row + double pg_notify.
+ */
+
+import type { AgentLifecycleEvent } from "../agent/agent-loop.js";
+
+export type LiveAgentEventEmitter =
+  (event: AgentLifecycleEvent, seq: number) => Promise<boolean>;
+
+/**
+ * Build a live emitter from process env + the caller-supplied stepIndex.
+ * Returns `undefined` when any required env var is missing (local tests,
+ * non-sandbox callers) so the caller can wire `undefined` straight
+ * through to the collector and get batch-only delivery without
+ * conditional plumbing.
+ *
+ * `stepIndex` is a parameter rather than an env read because
+ * `serveStep` scrubs `AC_STEP_INDEX` from `process.env` before invoking
+ * the workflow handler (to keep the transport envelope unreachable
+ * from user code); the handler still holds the parsed integer and
+ * passes it here.
+ */
+export function makeRunCallbackEmitterFromEnv(stepIndex: number): LiveAgentEventEmitter | undefined {
+  const baseUrl = process.env.AGENT_COMPOSE_URL;
+  const token   = process.env.AGENT_COMPOSE_RUN_TOKEN;
+  const runId   = process.env.RUN_ID;
+  if (!baseUrl || !token || !runId) return undefined;
+  if (!Number.isFinite(stepIndex) || stepIndex < 0) return undefined;
+  const url = `${baseUrl.replace(/\/+$/, "")}/api/v1/internal/runs/${runId}/steps/${stepIndex}/events`;
+  const headers = {
+    "content-type":  "application/json",
+    "authorization": `Bearer ${token}`,
+  } as const;
+  return async (event, seq) => {
+    try {
+      const res = await fetch(url, {
+        method:  "POST",
+        headers,
+        body:    JSON.stringify({ events: [{ ...event, seq }] }),
+      });
+      if (!res.ok) return false;
+      // Parse the route's response to confirm THIS seq was accepted.
+      // The route returns `{ acceptedSeqs: number[] }`; absence means
+      // the server logged but didn't persist (rare — DB hiccup), and
+      // we should leave the event in the batch backstop.
+      const body = await res.json().catch(() => null) as { acceptedSeqs?: number[] } | null;
+      return Array.isArray(body?.acceptedSeqs) && body.acceptedSeqs.includes(seq);
+    } catch {
+      return false;
+    }
+  };
+}

package/src/workflow-steps/runner.ts

@@ -28,6 +28,12 @@ import type { RequestContext } from "../request-context/request-context.js";
 import type { SandboxProvider } from "../types/sandbox.js";
 import type { WorkflowRun, WorkflowCtx } from "../types/workflow.js";
 import { StepObservabilityCollector, type StepObservability } from "./observability.js";
+import type { LiveAgentEventEmitter } from "./run-callback.js";
+import { scopedCheckpoint } from "../pause/checkpoint.js";
+import { corePause, PauseSignal, isPauseSignal, type PauseRequest } from "../pause/pause-core.js";
+import type { StepPauseRequest } from "../step-invocation/types.js";
+import { buildPauseWrappers, type PauseFn, type KindedPauseFn } from "../pause/wrappers.js";
+import { runWithActiveStep, setActiveStepBridge, restoreActiveStepBridge } from "../active-step.js";
 
 export class StepValidationError extends Error {
   readonly kind = "step-validation" as const;
@@ -84,6 +90,11 @@ export interface RunWorkflowStepsOpts<TInput, TOutput> {
   onStepStarted?(stepIndex: number, stepName: string): void | Promise<void>;
   /** Child workflow invocation implementation. Defaults to a clear unsupported error. */
   invokeChild?: WorkflowCtx["invokeChild"];
+  /** Optional live-stream emitter for agent lifecycle events. The runner
+   *  passes a fetch-based emitter wired to the per-run callback token so
+   *  the dashboard sees events as the agent loop produces them; tests
+   *  leave it undefined and get batch-only delivery. */
+  liveAgentEventEmitter?: LiveAgentEventEmitter;
 }
 
 export interface RunWorkflowStepsResult<TOutput> {
@@ -101,6 +112,8 @@ export interface RunWorkflowSingleStepOpts {
   sandbox?:       SandboxProvider;
   abortSignal?:   AbortSignal;
   invokeChild?:   WorkflowCtx["invokeChild"];
+  /** Optional live-stream emitter — see `RunWorkflowStepsOpts.liveAgentEventEmitter`. */
+  liveAgentEventEmitter?: LiveAgentEventEmitter;
 }
 
 /** Result of one step run — output plus whatever the step's observability
@@ -116,7 +129,14 @@ export async function runWorkflowSingleStep(opts: RunWorkflowSingleStepOpts): Pr
   if (!step) throw new Error(`Step index ${opts.stepIndex} not found in workflow "${opts.workflow.id}"`);
   const parsedInput = step.input.safeParse(opts.input);
   if (!parsedInput.success) throw new StepValidationError(step.name, "input", parsedInput.error);
-  const collector = new StepObservabilityCollector();
+  const collector = new StepObservabilityCollector({ liveEmitter: opts.liveAgentEventEmitter });
+  const coord = { runId: opts.run.id, stepIndex: opts.stepIndex };
+  const pauseWithKind: KindedPauseFn = function <T>(req: PauseRequest<T>, kind: StepPauseRequest["kind"]): Promise<T> {
+    return corePause(req, coord, kind);
+  };
+  const pause: PauseFn = function <T>(req: PauseRequest<T>): Promise<T> {
+    return corePause(req, coord);
+  };
   const stepCtx: StepContext<unknown> = {
     input: parsedInput.data,
     requestContext: opts.requestContext,
@@ -128,11 +148,35 @@ export async function runWorkflowSingleStep(opts: RunWorkflowSingleStepOpts): Pr
     setMetadata: collector.setMetadata,
     step:        collector.step,
     agentEvents: collector.agentEvents,
+    checkpoint:  scopedCheckpoint(`step${opts.stepIndex}`),
+    pause,
+    ...buildPauseWrappers(pauseWithKind),
   };
-  const output = await step.run(stepCtx);
+  let output: unknown;
+  try {
+    // Set the active step so `agent()` / `ctx.pause` derive deterministic
+    // ids from the un-scrubbed index (see active-step.ts). Async-local scope
+    // keeps concurrent in-process runs from clobbering each other. ALSO publish
+    // on the cross-instance bridge: a bundled workflow inlines its own SDK copy
+    // (a separate AsyncLocalStorage), so its `agent()` can't see the ALS we set
+    // here — the bridge carries the step across that boundary. One step per
+    // subprocess in the sandbox ⇒ no concurrency on the global slot.
+    const bridged = setActiveStepBridge({ stepIndex: opts.stepIndex });
+    try {
+      output = await runWithActiveStep({ stepIndex: opts.stepIndex }, () => Promise.resolve(step.run(stepCtx)));
+    } finally {
+      restoreActiveStepBridge(bridged);
+    }
+  } catch (err) {
+    if (isPauseSignal(err)) {
+      const observability = await collector.snapshot();
+      throw new PauseSignal(err.pauseId, err.pauseRequest, observability);
+    }
+    throw err;
+  }
   const parsedOutput = step.output.safeParse(output);
   if (!parsedOutput.success) throw new StepValidationError(step.name, "output", parsedOutput.error);
-  const observability = collector.snapshot();
+  const observability = await collector.snapshot();
   return observability === undefined
     ? { output: parsedOutput.data }
     : { output: parsedOutput.data, observability };
@@ -179,7 +223,14 @@ export async function runWorkflowSteps<TInput, TOutput>(
       throw err;
     }
 
-    const collector = new StepObservabilityCollector();
+    const collector = new StepObservabilityCollector({ liveEmitter: opts.liveAgentEventEmitter });
+    const coord = { runId: run.id, stepIndex: i };
+    const pauseWithKind: KindedPauseFn = function <T>(req: PauseRequest<T>, kind: StepPauseRequest["kind"]): Promise<T> {
+      return corePause(req, coord, kind);
+    };
+    const pause: PauseFn = function <T>(req: PauseRequest<T>): Promise<T> {
+      return corePause(req, coord);
+    };
     const stepCtx: StepContext<unknown> = {
       input:          parsedStepInput.data,
       requestContext,
@@ -191,16 +242,27 @@ export async function runWorkflowSteps<TInput, TOutput>(
       setMetadata:    collector.setMetadata,
       step:           collector.step,
       agentEvents:    collector.agentEvents,
+      checkpoint:     scopedCheckpoint(`step${i}`),
+      pause,
+      ...buildPauseWrappers(pauseWithKind),
     };
 
     const startedAt = Date.now();
     let output: unknown;
     try {
-      output = await step.run(stepCtx);
+      // Active step for deterministic agent/pause id derivation (active-step.ts).
+      // Async-local scope keeps concurrent in-process runs isolated.
+      output = await runWithActiveStep({ stepIndex: i }, () => Promise.resolve(step.run(stepCtx)));
     } catch (err) {
+      // A pause is control flow, not a failure — let it propagate so serveStep
+      // emits the pause sentinel instead of recording a failed step.
+      if (isPauseSignal(err)) {
+        const observability = await collector.snapshot();
+        throw new PauseSignal(err.pauseId, err.pauseRequest, observability);
+      }
       const wrapped = err instanceof Error ? err : new Error(String(err));
       const durationMs = Date.now() - startedAt;
-      const observability = collector.snapshot();
+      const observability = await collector.snapshot();
       stepResults.push({
         name: step.name, status: "failed", error: wrapped.message, durationMs,
         ...(observability ? { observability } : {}),
@@ -213,7 +275,7 @@ export async function runWorkflowSteps<TInput, TOutput>(
     if (!parsedOutput.success) {
       const err = new StepValidationError(step.name, "output", parsedOutput.error);
       const durationMs = Date.now() - startedAt;
-      const observability = collector.snapshot();
+      const observability = await collector.snapshot();
       stepResults.push({
         name: step.name, status: "failed", error: err.message, durationMs,
         ...(observability ? { observability } : {}),
@@ -224,7 +286,7 @@ export async function runWorkflowSteps<TInput, TOutput>(
 
     const durationMs = Date.now() - startedAt;
     current = parsedOutput.data;
-    const observability = collector.snapshot();
+    const observability = await collector.snapshot();
     stepResults.push({
       name: step.name, status: "completed", output: parsedOutput.data, durationMs,
       ...(observability ? { observability } : {}),

package/dist/utils/discovery.d.ts

@@ -1,2 +0,0 @@
-/** Find the runtime name referenced via runtime: "name" in workflow source. */
-export declare function discoverRuntimeName(source: string): string | null;

package/src/utils/discovery.ts

@@ -1,4 +0,0 @@
-/** Find the runtime name referenced via runtime: "name" in workflow source. */
-export function discoverRuntimeName(source: string): string | null {
-  return source.match(/runtime:\s*["']([^"']+)["']/)?.[1] ?? null;
-}