@stigmer/runner 3.15.1 → 3.15.2-dev.20260913223433

package/src/activities/execute-deep-agent/streaming-v3.ts

@@ -1,277 +0,0 @@
-/**
- * V3 streaming execution loop for ExecuteDeepAgent.
- *
- * Feature-flagged path activated by LANGGRAPH_STREAM_EVENTS_VERSION=v3.
- * Calls the v3 streamEvents API (two-arg, returns Promise<GraphRunStream>),
- * iterates raw ProtocolEvents, feeds them through the V3ProtocolNormalizer
- * and V3StatusBuilder, persists status on schedule, and handles terminal
- * conditions (STOP, pause, recursion limit) identically to v2.
- *
- * Phase 2: full orchestration parity with streamExecutionV2.
- */
-
-import type { StreamDependencies, StreamResult } from "./streaming.js";
-import { StallTimeoutError } from "./streaming.js";
-import { ExecutionControlSignal, ExecutionPhase } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-import { createV3EventRecorder, type V3ProtocolEvent } from "./v3-event-recorder.js";
-import { normalize } from "./v3-protocol-normalizer.js";
-import { V3StatusBuilder } from "./v3-status-builder.js";
-import { StreamingUpdateScheduler } from "../../shared/streaming-scheduler.js";
-import { persistStatus, slimStatus } from "../../shared/status.js";
-import { StreamingSideEffects } from "./streaming-side-effects.js";
-import {
-  handlePause,
-  handleStop,
-  handleRecursionLimit,
-  isGraphRecursionError,
-} from "./streaming-terminal.js";
-
-interface V3RunStream extends AsyncIterable<V3ProtocolEvent> {
-  readonly output: Promise<unknown>;
-  abort(reason?: unknown): void;
-  readonly signal: AbortSignal;
-}
-
-const DEFAULT_STALL_TIMEOUT_MS = 120_000;
-const HEARTBEAT_INTERVAL_MS = 2_000;
-const RUN_OUTPUT_TIMEOUT_MS = 30_000;
-
-export async function streamExecutionV3(
-  deps: StreamDependencies,
-): Promise<StreamResult> {
-  const {
-    agentGraph,
-    langgraphInput,
-    langgraphConfig,
-    executionId,
-    client,
-    initialStatus,
-    streamingConfig,
-    retryOptions,
-    offload,
-    stallTimeoutMs = DEFAULT_STALL_TIMEOUT_MS,
-    heartbeatFn,
-    isCancelledFn,
-    gracefulStop,
-    inlinePublisher,
-    writebackCoordinator,
-    approvalProvider,
-  } = deps;
-
-  const statusBuilder = new V3StatusBuilder(executionId, initialStatus);
-  if (approvalProvider) {
-    statusBuilder.setApprovalProvider(approvalProvider);
-  }
-
-  const scheduler = new StreamingUpdateScheduler(streamingConfig);
-  const recorder = createV3EventRecorder(executionId, process.env.V3_EVENT_RECORD_DIR);
-  const abortController = new AbortController();
-
-  const sideEffects = new StreamingSideEffects({
-    inlinePublisher,
-    writebackCoordinator,
-  });
-
-  // Pre-loop heartbeat to cover the initial await
-  sendHeartbeat(heartbeatFn, executionId, 0, statusBuilder);
-
-  const run = await (agentGraph as any).streamEvents(langgraphInput, {
-    ...langgraphConfig,
-    version: "v3",
-    signal: abortController.signal,
-  }) as V3RunStream;
-
-  let eventsProcessed = 0;
-  let lastActivityAt = performance.now();
-
-  const heartbeatTimer = setInterval(() => {
-    sendHeartbeat(heartbeatFn, executionId, eventsProcessed, statusBuilder);
-  }, HEARTBEAT_INTERVAL_MS);
-
-  try {
-    for await (const event of run as AsyncIterable<V3ProtocolEvent>) {
-      if (isCancelledFn?.()) {
-        abortController.abort("Cancelled by platform");
-        statusBuilder.cancelSubAgents();
-        return handlePause(
-          statusBuilder, eventsProcessed,
-          sideEffects.pendingPublishPromises,
-          sideEffects.pendingWritebackPromises,
-        );
-      }
-
-      lastActivityAt = performance.now();
-      recorder?.record(event, eventsProcessed);
-
-      for (const normalized of normalize(event)) {
-        statusBuilder.processEvent(normalized);
-      }
-      sideEffects.onProtocolEvent(event);
-      eventsProcessed++;
-
-      const shouldPersist = statusBuilder.forceNextUpdate ||
-        scheduler.shouldSendUpdate(eventsProcessed);
-
-      if (shouldPersist) {
-        if (statusBuilder.forceNextUpdate) {
-          statusBuilder.clearForceFlag();
-        }
-
-        statusBuilder.syncSubAgentExecutions();
-        // Mid-run live capture (DD-32): attach file_change_progress to the live
-        // status before it is persisted. Injected so this loop stays ignorant of
-        // file-review specifics; a no-op when the hook is absent or nothing changed.
-        const statusToPersist = statusBuilder.currentStatus;
-        await deps.beforePersist?.(statusToPersist);
-        const signal = await persistStatus(
-          client,
-          executionId,
-          statusToPersist,
-          { offload, retry: retryOptions },
-        );
-        scheduler.markUpdateSent(eventsProcessed);
-
-        if (signal === ExecutionControlSignal.STOP) {
-          console.warn(
-            `[streaming-v3] STOP signal received for execution ${executionId}`,
-          );
-          if (gracefulStop) {
-            gracefulStop.activate("Platform STOP signal");
-          } else {
-            return handleStop(
-              statusBuilder, eventsProcessed,
-              sideEffects.pendingPublishPromises,
-              sideEffects.pendingWritebackPromises,
-            );
-          }
-        }
-      }
-
-      checkStallTimeout(lastActivityAt, stallTimeoutMs, executionId);
-    }
-  } catch (err: unknown) {
-    if (isGraphRecursionError(err)) {
-      await recorder?.flush();
-      return handleRecursionLimit(
-        statusBuilder, eventsProcessed,
-        sideEffects.pendingPublishPromises,
-        sideEffects.pendingWritebackPromises,
-      );
-    }
-    throw err;
-  } finally {
-    clearInterval(heartbeatTimer);
-  }
-
-  await recorder?.flush();
-
-  if (eventsProcessed === 0) {
-    throw new Error(
-      "Stream completed without processing any events. " +
-      "This may indicate a configuration error or v3 API incompatibility.",
-    );
-  }
-
-  statusBuilder.syncSubAgentExecutions();
-
-  if (initialStatus.phase === ExecutionPhase.EXECUTION_WAITING_FOR_APPROVAL) {
-    console.log(
-      `[streaming-v3] execution=${executionId} stream ended with WAITING_FOR_APPROVAL. ` +
-      `Not setting COMPLETED. pending_approvals computed server-side.`,
-    );
-    return {
-      eventsProcessed,
-      terminalStatus: slimStatus(initialStatus),
-      pendingPublishPromises: sideEffects.pendingPublishPromises,
-      pendingWritebackPromises: sideEffects.pendingWritebackPromises,
-    };
-  }
-
-  console.log(
-    `[streaming-v3] execution=${executionId} stream finished — ` +
-    `processed ${eventsProcessed} events`,
-  );
-
-  const runOutput = await extractRunOutput(run, executionId);
-
-  return {
-    eventsProcessed,
-    runOutput,
-    pendingPublishPromises: sideEffects.pendingPublishPromises,
-    pendingWritebackPromises: sideEffects.pendingWritebackPromises,
-  };
-}
-
-// ── Run Output Extraction ────────────────────────────────────────────
-
-async function extractRunOutput(
-  run: { output: Promise<unknown> },
-  executionId: string,
-): Promise<Record<string, unknown> | undefined> {
-  try {
-    const finalState = await Promise.race([
-      run.output,
-      timeoutPromise(RUN_OUTPUT_TIMEOUT_MS),
-    ]);
-
-    if (finalState === TIMEOUT_SENTINEL) {
-      console.warn(
-        `[streaming-v3] execution=${executionId} — run.output did not resolve ` +
-        `within ${RUN_OUTPUT_TIMEOUT_MS}ms. Proceeding without final state.`,
-      );
-      return undefined;
-    }
-
-    const output = finalState as Record<string, unknown>;
-    console.log(
-      `[streaming-v3] execution=${executionId} — run.output resolved. ` +
-      `Keys: [${Object.keys(output ?? {}).join(", ")}]. ` +
-      `hasStructuredResponse=${output?.structuredResponse !== undefined}`,
-    );
-    return output;
-  } catch (err) {
-    console.warn(
-      `[streaming-v3] execution=${executionId} — run.output rejected: ${err}`,
-    );
-    return undefined;
-  }
-}
-
-const TIMEOUT_SENTINEL = Symbol("timeout");
-
-function timeoutPromise(ms: number): Promise<typeof TIMEOUT_SENTINEL> {
-  return new Promise((resolve) => setTimeout(() => resolve(TIMEOUT_SENTINEL), ms));
-}
-
-// ── Helpers ──────────────────────────────────────────────────────────
-
-function sendHeartbeat(
-  fn: ((details: Record<string, unknown>) => void) | undefined,
-  executionId: string,
-  eventsProcessed: number,
-  sb: V3StatusBuilder,
-): void {
-  if (!fn) return;
-  try {
-    fn({
-      executionId,
-      eventsProcessed,
-      messages: sb.currentStatus.messages.length,
-      phase: sb.currentStatus.phase,
-    });
-  } catch { /* swallow */ }
-}
-
-function checkStallTimeout(
-  lastActivityAt: number,
-  stallTimeoutMs: number,
-  executionId: string,
-): void {
-  const elapsed = performance.now() - lastActivityAt;
-  if (elapsed > stallTimeoutMs) {
-    throw new StallTimeoutError(
-      `Agent stream stalled: no events received for ${Math.round(elapsed / 1000)}s ` +
-      `for execution ${executionId}`,
-    );
-  }
-}

package/src/activities/execute-deep-agent/streaming.ts

@@ -1,313 +0,0 @@
-/**
- * Streaming execution loop for ExecuteDeepAgent.
- *
- * Replaces the Phase 3a invoke() approach with streamEvents() v2
- * consumption. Feeds events to the StatusBuilder, persists status
- * via the scheduler + retry executor, handles terminal conditions
- * (STOP signal, stall, recursion limit, cancellation).
- *
- * Phase 3b-iii: inline artifact publish and incremental git writeback
- * on file-modifying tool completions (write, edit, create).
- */
-
-import { create, toJson } from "@bufbuild/protobuf";
-import {
-  AgentExecutionStatusSchema,
-} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import { AgentMessageSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
-import {
-  ExecutionControlSignal,
-  ExecutionPhase,
-  MessageType,
-} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-
-import { StatusBuilder, type StreamEvent, type ApprovalPolicyProvider } from "./status-builder.js";
-import {
-  handlePause,
-  handleStop,
-  handleRecursionLimit,
-  isGraphRecursionError,
-} from "./streaming-terminal.js";
-import {
-  StreamingUpdateScheduler,
-  type StreamingConfig,
-} from "../../shared/streaming-scheduler.js";
-import { type RetryOptions } from "../../shared/grpc-retry.js";
-import { persistStatus, slimStatus, utcTimestamp } from "../../shared/status.js";
-import type { ToolOutputOffloadContext } from "../../shared/status-offload.js";
-import type { StigmerClient } from "../../client/stigmer-client.js";
-import type { GracefulStopMiddleware } from "../../middleware/index.js";
-import type { InlinePublisher } from "./inline-publisher.js";
-import type { WriteBackCoordinator } from "../../shared/workspace/writeback-coordinator.js";
-import { createV2EventRecorder } from "./event-recorder.js";
-import { streamExecutionV3 } from "./streaming-v3.js";
-import { extractFilePath, isFileModifyingTool } from "../../shared/file-tools.js";
-
-const DEFAULT_STALL_TIMEOUT_MS = 120_000;
-
-export interface StreamDependencies {
-  readonly agentGraph: {
-    streamEvents(
-      input: Record<string, unknown>,
-      config: Record<string, unknown>,
-      options: { version: string },
-    ): AsyncIterable<StreamEvent>;
-  };
-  readonly langgraphInput: Record<string, unknown>;
-  readonly langgraphConfig: Record<string, unknown>;
-  readonly executionId: string;
-  readonly client: StigmerClient;
-  readonly initialStatus: AgentExecutionStatus;
-  readonly streamingConfig?: StreamingConfig;
-  readonly retryOptions?: RetryOptions;
-  /**
-   * Offload context for the persist chokepoint. When set, oversized tool
-   * outputs (e.g. computer-use screenshots) are spilled to artifact storage so
-   * the UI can render them; when omitted, the aggregate size backstop still
-   * keeps the payload under the gRPC cap.
-   */
-  readonly offload?: ToolOutputOffloadContext;
-  readonly stallTimeoutMs?: number;
-  /** Temporal heartbeat. Injected so the loop is testable without Temporal. */
-  readonly heartbeatFn?: (details: Record<string, unknown>) => void;
-  /** Temporal cancellation check. */
-  readonly isCancelledFn?: () => boolean;
-  /** GracefulStopMiddleware instance for platform STOP signal handling. */
-  readonly gracefulStop?: GracefulStopMiddleware;
-  /** Inline artifact publisher for file-modifying tool completions. */
-  readonly inlinePublisher?: InlinePublisher;
-  /** Incremental git write-back coordinator. */
-  readonly writebackCoordinator?: WriteBackCoordinator;
-  /** Approval policy provider for StatusBuilder tool-call phase tracking. */
-  readonly approvalProvider?: ApprovalPolicyProvider;
-  /** Streaming protocol version. Defaults to "v2" if unset. */
-  readonly streamVersion?: "v2" | "v3";
-  /**
-   * Optional hook invoked with the live status just before each scheduled
-   * persist. Used by mid-run live capture (DD-32) to attach file_change_progress.
-   * Kept as an injected callback (closing over the turn's baseline tree / change
-   * set id in index.ts) so this loop stays ignorant of file-review specifics.
-   */
-  readonly beforePersist?: (status: AgentExecutionStatus) => Promise<void>;
-}
-
-export interface StreamResult {
-  readonly eventsProcessed: number;
-  readonly terminalStatus?: unknown;
-  readonly pendingPublishPromises: readonly Promise<void>[];
-  readonly pendingWritebackPromises: readonly Promise<void>[];
-  /** v3 only: the final state from run.output (includes structuredResponse if present). */
-  readonly runOutput?: Record<string, unknown>;
-}
-
-/**
- * Run the streaming loop: consume streamEvents, map to proto via
- * StatusBuilder, persist on schedule, and handle terminal conditions.
- *
- * Routes to v3 when deps.streamVersion === "v3".
- */
-export async function streamExecution(
-  deps: StreamDependencies,
-): Promise<StreamResult> {
-  if (deps.streamVersion === "v3") {
-    return streamExecutionV3(deps);
-  }
-  return streamExecutionV2(deps);
-}
-
-async function streamExecutionV2(
-  deps: StreamDependencies,
-): Promise<StreamResult> {
-  const {
-    agentGraph,
-    langgraphInput,
-    langgraphConfig,
-    executionId,
-    client,
-    initialStatus,
-    streamingConfig,
-    retryOptions,
-    offload,
-    stallTimeoutMs = DEFAULT_STALL_TIMEOUT_MS,
-    heartbeatFn,
-    isCancelledFn,
-    gracefulStop,
-    inlinePublisher,
-    writebackCoordinator,
-    approvalProvider,
-  } = deps;
-
-  const statusBuilder = new StatusBuilder(executionId, initialStatus);
-  if (approvalProvider) {
-    statusBuilder.setApprovalProvider(approvalProvider);
-  }
-  const scheduler = new StreamingUpdateScheduler(streamingConfig);
-  const recorder = createV2EventRecorder(executionId, process.env.V2_EVENT_RECORD_DIR);
-
-  let eventsProcessed = 0;
-  let lastEventTime = performance.now();
-  let lastHeartbeatTime = performance.now();
-  const heartbeatIntervalMs = 2000;
-
-  const pendingPublishPromises: Promise<void>[] = [];
-  const pendingWritebackPromises: Promise<void>[] = [];
-
-  try {
-    const stream = agentGraph.streamEvents(
-      langgraphInput,
-      langgraphConfig,
-      { version: "v2" },
-    );
-
-    for await (const event of stream) {
-      if (isCancelledFn?.()) {
-        return handlePause(statusBuilder, eventsProcessed, pendingPublishPromises, pendingWritebackPromises);
-      }
-
-      lastEventTime = performance.now();
-      recorder?.record(event, eventsProcessed);
-      statusBuilder.processEvent(event);
-      eventsProcessed++;
-
-      if (event.event === "on_tool_end" && (inlinePublisher || writebackCoordinator)) {
-        const filePath = extractFilePathFromToolEnd(event);
-        if (filePath) {
-          if (inlinePublisher) {
-            pendingPublishPromises.push(inlinePublisher.publish(filePath));
-          }
-          if (writebackCoordinator) {
-            pendingWritebackPromises.push(writebackCoordinator.onFileModified(filePath));
-          }
-        }
-      }
-
-      const now = performance.now();
-      if (heartbeatFn && (now - lastHeartbeatTime) >= heartbeatIntervalMs) {
-        sendHeartbeat(heartbeatFn, executionId, eventsProcessed, statusBuilder);
-        lastHeartbeatTime = now;
-      }
-
-      const shouldPersist = statusBuilder.forceNextUpdate ||
-        scheduler.shouldSendUpdate(eventsProcessed);
-
-      if (shouldPersist) {
-        if (statusBuilder.forceNextUpdate) {
-          statusBuilder.clearForceFlag();
-        }
-
-        // Mid-run live capture (DD-32): attach file_change_progress to the live
-        // status before persisting (see streaming-v3.ts for the rationale).
-        const statusToPersist = statusBuilder.currentStatus;
-        await deps.beforePersist?.(statusToPersist);
-        const signal = await persistStatus(
-          client,
-          executionId,
-          statusToPersist,
-          { offload, retry: retryOptions },
-        );
-        scheduler.markUpdateSent(eventsProcessed);
-
-        if (signal === ExecutionControlSignal.STOP) {
-          console.warn(
-            `[streaming] STOP signal received for execution ${executionId}`,
-          );
-          if (gracefulStop) {
-            gracefulStop.activate("Platform STOP signal");
-          } else {
-            return handleStop(statusBuilder, eventsProcessed, pendingPublishPromises, pendingWritebackPromises);
-          }
-        }
-      }
-
-      checkStallTimeout(lastEventTime, stallTimeoutMs, executionId);
-    }
-  } catch (err: unknown) {
-    if (isGraphRecursionError(err)) {
-      return handleRecursionLimit(statusBuilder, eventsProcessed, pendingPublishPromises, pendingWritebackPromises);
-    }
-    throw err;
-  }
-
-  await recorder?.flush();
-
-  if (eventsProcessed === 0) {
-    throw new Error(
-      "Stream completed without processing any events. " +
-      "This may indicate a configuration error.",
-    );
-  }
-
-  if (initialStatus.phase === ExecutionPhase.EXECUTION_WAITING_FOR_APPROVAL) {
-    console.log(
-      `[streaming] execution=${executionId} stream ended with WAITING_FOR_APPROVAL. ` +
-      `Not setting COMPLETED. pending_approvals computed server-side.`,
-    );
-    return {
-      eventsProcessed,
-      terminalStatus: slimStatus(initialStatus),
-      pendingPublishPromises,
-      pendingWritebackPromises,
-    };
-  }
-
-  console.log(
-    `[streaming] execution=${executionId} stream finished — ` +
-    `processed ${eventsProcessed} events`,
-  );
-
-  return { eventsProcessed, pendingPublishPromises, pendingWritebackPromises };
-}
-
-// Terminal state handlers imported from streaming-terminal.ts
-
-// ── Helpers ──────────────────────────────────────────────────────────
-
-function sendHeartbeat(
-  fn: (details: Record<string, unknown>) => void,
-  executionId: string,
-  eventsProcessed: number,
-  sb: StatusBuilder,
-): void {
-  try {
-    fn({
-      executionId,
-      eventsProcessed,
-      messages: sb.currentStatus.messages.length,
-      phase: sb.currentStatus.phase,
-    });
-  } catch (err) {
-    console.warn(`[streaming] Heartbeat failed for ${executionId}:`, err);
-  }
-}
-
-function checkStallTimeout(
-  lastEventTime: number,
-  stallTimeoutMs: number,
-  executionId: string,
-): void {
-  const elapsed = performance.now() - lastEventTime;
-  if (elapsed > stallTimeoutMs) {
-    throw new StallTimeoutError(
-      `Agent stream stalled: no events received for ${Math.round(elapsed / 1000)}s ` +
-      `for execution ${executionId}`,
-    );
-  }
-}
-
-function extractFilePathFromToolEnd(event: StreamEvent): string | null {
-  const toolName = event.name ?? "";
-  if (!isFileModifyingTool(toolName)) return null;
-
-  const input = event.data?.input as Record<string, unknown> | undefined;
-  if (!input) return null;
-
-  return extractFilePath(input);
-}
-
-export class StallTimeoutError extends Error {
-  constructor(message: string) {
-    super(message);
-    this.name = "StallTimeoutError";
-  }
-}