@studio-foundation/engine 0.3.0-beta.1 → 0.3.0-beta.6

package/src/engine.ts

@@ -1,491 +0,0 @@
-// Main pipeline engine — the orchestrator
-// Connects ralph (retry loop) + runner (agent execution) + contracts (validation)
-
-import { randomUUID } from 'node:crypto';
-import type {
-  PipelineEntry,
-  PipelineRun,
-  PipelineDefinition,
-  RunSpawner,
-  StageRun,
-  ToolCall,
-} from '@studio-foundation/contracts';
-import { isStageGroup } from '@studio-foundation/contracts';
-import {
-  type ToolRegistry,
-  type ProviderRegistry,
-  AnonymizationMiddleware,
-  createStudioRunTool,
-  STUDIO_RUN_PROMPT_SNIPPET,
-} from '@studio-foundation/runner';
-import { loadPipelineByName } from './pipeline/loader.js';
-import { executeStartupCommands } from './pipeline/startup-executor.js';
-import { loadInvariantsFile } from './pipeline/invariants-loader.js';
-import {
-  createInitialContext,
-  addStageOutput,
-  addStageToolResults,
-  clearGroupFeedback,
-} from './pipeline/context-propagation.js';
-import type { AnyRunStore } from './state/run-store.js';
-import type { EngineEvents } from './events.js';
-import { PipelineEventEmitter } from './events.js';
-import { resolveProjectPaths } from './pipeline/types.js';
-import { StageExecutor } from './pipeline/stage-executor.js';
-import { GroupOrchestrator } from './pipeline/group-orchestrator.js';
-
-export interface EngineConfig {
-  configsDir: string;
-  repoPath?: string;
-  providerRegistry: ProviderRegistry;
-  toolRegistry?: ToolRegistry;  // optional — not needed for script-only pipelines
-  db?: AnyRunStore;
-  providerOverride?: string;
-  /**
-   * Skills content from active plugins, keyed by plugin name.
-   * Each entry is an array of formatted markdown strings to inject
-   * into the system prompt of agents that declare the plugin.
-   */
-  pluginSkills?: Record<string, string[]>;
-  spawner?: RunSpawner;  // if set, studio-run tool is available to agents
-  maxDepth?: number;     // max nesting depth for spawned runs, default 3
-  defaultProvider?: string;  // fallback when agent YAML omits provider
-  defaultModel?: string;     // fallback when agent YAML omits model
-}
-
-export interface RunInput {
-  id?: string;          // ← pre-generated run ID (e.g. from the API)
-  pipeline?: string;    // pipeline name (loads from YAML file)
-  pipelineDef?: PipelineDefinition; // inline pipeline definition (skips YAML loading — for tests/programmatic use)
-  input?: string | Record<string, unknown>;
-  userInput?: string | Record<string, unknown>; // alias for input (used in tests and programmatic API)
-  meta?: Record<string, unknown>;
-  anonymize?: boolean;
-  signal?: AbortSignal;
-  depth?: number;        // nesting depth (0 = top-level)
-  parentRunId?: string;  // parent run ID if spawned by another pipeline
-  // Resume fields — used by `studio replay --restart`
-  resumeFromStage?: string;
-  priorStageOutputs?: Map<string, unknown>;
-  priorStageToolResults?: Map<string, ToolCall[]>;
-  originalRunId?: string;
-}
-
-/**
- * Collect all leaf stage names in pipeline order (group containers are transparent).
- */
-function collectLeafStageNames(entries: PipelineEntry[]): string[] {
-  const names: string[] = [];
-  for (const entry of entries) {
-    if (isStageGroup(entry)) {
-      for (const stage of entry.stages) names.push(stage.name);
-    } else {
-      names.push(entry.name);
-    }
-  }
-  return names;
-}
-
-/**
- * Returns the set of stage names that should be skipped (all leaf stages before resumeFromStage).
- * Throws if resumeFromStage is not found in the pipeline.
- */
-function buildSkipSet(entries: PipelineEntry[], resumeFromStage: string): Set<string> {
-  const leafNames = collectLeafStageNames(entries);
-  const targetIndex = leafNames.indexOf(resumeFromStage);
-  if (targetIndex < 0) {
-    throw new Error(`Stage "${resumeFromStage}" not found in pipeline`);
-  }
-  return new Set(leafNames.slice(0, targetIndex));
-}
-
-function countTotalStages(entries: PipelineEntry[]): number {
-  let count = 0;
-  for (const entry of entries) {
-    if (isStageGroup(entry)) {
-      count += entry.stages.length;
-    } else {
-      count++;
-    }
-  }
-  return count;
-}
-
-/**
- * Create a StageRun for a skipped stage (used during resume operations).
- * Uses randomUUID() to ensure unique IDs across multiple resume runs.
- */
-function makeSkippedStageRun(stageName: string, reason: string): StageRun {
-  const now = new Date().toISOString();
-  return {
-    id: randomUUID(),
-    stage_name: stageName,
-    status: 'skipped',
-    started_at: now,
-    completed_at: now,
-    tasks: [],
-    skipped_reason: reason,
-  };
-}
-
-export class PipelineEngine {
-  private emitter: PipelineEventEmitter;
-  private pipelineTotals = { tokens: 0, toolCalls: 0 };
-  private stageExecutor: StageExecutor;
-  private groupOrchestrator: GroupOrchestrator;
-
-  constructor(
-    private config: EngineConfig,
-    private events?: EngineEvents,
-  ) {
-    this.emitter = new PipelineEventEmitter();
-    this.stageExecutor = new StageExecutor({
-      events,
-      emitter: this.emitter,
-      providerRegistry: config.providerRegistry,
-      repoPath: config.repoPath,
-      configsDir: config.configsDir,
-      pluginSkills: config.pluginSkills,
-      providerOverride: config.providerOverride,
-      defaultProvider: config.defaultProvider,
-      defaultModel: config.defaultModel,
-    });
-    this.groupOrchestrator = new GroupOrchestrator({
-      events,
-      emitter: this.emitter,
-      stageExecutor: this.stageExecutor,
-    });
-  }
-
-  async run(input: RunInput): Promise<PipelineRun> {
-    const signal = input.signal;
-
-    // Resolve the effective user input (support both 'input' and 'userInput' aliases)
-    const userInputValue: string | Record<string, unknown> = input.userInput ?? input.input ?? '';
-
-    if (!input.pipeline && !input.pipelineDef) {
-      throw new Error('RunInput must provide either "pipeline" (pipeline name) or "pipelineDef" (inline definition)');
-    }
-
-    // 1. Resolve paths — configsDir is now the project root directly
-    const projectPaths = resolveProjectPaths(this.config.configsDir);
-
-    // 2. Load the pipeline — either from an inline definition or a YAML file
-    const pipeline: PipelineDefinition = input.pipelineDef
-      ? input.pipelineDef
-      : await loadPipelineByName(input.pipeline!, projectPaths.pipelinesDir);
-
-    // 2. Create the PipelineRun
-    const pipelineRun: PipelineRun = {
-      id: input.id ?? randomUUID(),
-      pipeline_name: pipeline.name,
-      status: 'running',
-      started_at: new Date().toISOString(),
-      stages: [],
-      ...(typeof userInputValue === 'object' && userInputValue !== null
-        ? { input: userInputValue as Record<string, unknown> }
-        : {}),
-      ...(input.parentRunId ? { parent_run_id: input.parentRunId } : {}),
-    };
-
-    // Reset totals for this run
-    this.pipelineTotals = { tokens: 0, toolCalls: 0 };
-    const pipelineStartTime = Date.now();
-
-    // Create anonymization middleware for this run if requested via RunInput flag
-    const runAnonymize = input.anonymize === true;
-    const runMiddleware = runAnonymize ? new AnonymizationMiddleware() : null;
-
-    // Persist the run immediately so log_path can be written before terminal states
-    await this.config.db?.savePipelineRun(pipelineRun);
-
-    // Build per-run tool registry: clone the shared registry and inject studio-run
-    // with run-specific context (run ID, depth) if a spawner is configured.
-    // For script-only pipelines, toolRegistry may be undefined.
-    const runToolRegistry = this.config.spawner && this.config.toolRegistry
-      ? (() => {
-          const registry = this.config.toolRegistry.clone();
-          registry.registerPlugin(
-            'studio_run',
-            createStudioRunTool({
-              spawner: this.config.spawner!,
-              currentRunId: pipelineRun.id,
-              currentDepth: input.depth ?? 0,
-              maxDepth: this.config.maxDepth ?? 3,
-            }),
-            STUDIO_RUN_PROMPT_SNIPPET
-          );
-          return registry;
-        })()
-      : this.config.toolRegistry;
-
-    this.events?.onPipelineStart?.({
-      pipeline_name: pipeline.name,
-      run_id: pipelineRun.id,
-    });
-    this.emitter.emit({ type: 'pipeline_start', pipelineId: pipelineRun.id });
-
-    // 3. Initialize context
-    const pipelineContext = createInitialContext(userInputValue, this.config.repoPath);
-
-    // Run on_pipeline_start commands to bootstrap dynamic context
-    if (pipeline.on_pipeline_start?.length) {
-      const cwd = this.config.repoPath ?? this.config.configsDir;
-      pipelineContext.startupContext = await executeStartupCommands(
-        pipeline.on_pipeline_start,
-        cwd
-      );
-    }
-
-    // Load .studio/invariants.md if present — injected into every agent's system_prompt
-    pipelineContext.invariantsContent = await loadInvariantsFile(projectPaths.projectDir);
-
-    // Pre-populate context from prior run if resuming
-    const skipSet: Set<string> =
-      input.resumeFromStage
-        ? buildSkipSet(pipeline.stages, input.resumeFromStage)
-        : new Set();
-
-    if (input.resumeFromStage) {
-      for (const [stageName, output] of input.priorStageOutputs ?? []) {
-        addStageOutput(pipelineContext, stageName, output);
-      }
-      for (const [stageName, toolCalls] of input.priorStageToolResults ?? []) {
-        addStageToolResults(pipelineContext, stageName, toolCalls);
-      }
-    }
-
-    // 4. Execute stages sequentially (handling groups)
-    const totalStages = countTotalStages(pipeline.stages);
-    let stageCounter = 0;
-    let previousStageName: string | undefined;
-
-    for (const entry of pipeline.stages) {
-      // Check for cancellation before each pipeline entry
-      if (signal?.aborted) {
-        pipelineRun.status = 'cancelled';
-        pipelineRun.completed_at = new Date().toISOString();
-        const lastStage = pipelineRun.stages[pipelineRun.stages.length - 1];
-        this.events?.onPipelineCancelled?.({
-          run_id: pipelineRun.id,
-          cancelled_at_stage: lastStage?.stage_name ?? 'before_first_stage',
-          duration_ms: Date.now() - pipelineStartTime,
-        });
-        await this.config.db?.savePipelineRun(pipelineRun);
-        this.events?.onPipelineComplete?.({
-          pipeline_name: pipeline.name,
-          run_id: pipelineRun.id,
-          status: 'cancelled',
-          duration_ms: Date.now() - pipelineStartTime,
-          total_tokens: this.pipelineTotals.tokens,
-          total_tool_calls: this.pipelineTotals.toolCalls,
-        });
-        this.emitter.emit({ type: 'pipeline_complete', pipelineId: pipelineRun.id });
-        return pipelineRun;
-      }
-
-      if (isStageGroup(entry)) {
-        // ========== GROUP ==========
-
-        // Check if all group stages should be skipped
-        const allGroupStagesSkipped = entry.stages.every(s => skipSet.has(s.name));
-
-        if (allGroupStagesSkipped) {
-          const groupSkipReason = input.originalRunId
-            ? `resumed from run ${input.originalRunId}`
-            : 'resumed from prior run';
-          for (const stage of entry.stages) {
-            stageCounter++;
-            const skippedRun = makeSkippedStageRun(stage.name, groupSkipReason);
-            this.events?.onStageComplete?.({
-              stage_name: stage.name,
-              stage_index: stageCounter - 1,
-              total_stages: totalStages,
-              status: 'skipped',
-              attempts: 0,
-              duration_ms: 0,
-              skipped_reason: groupSkipReason,
-            });
-            pipelineRun.stages.push(skippedRun);
-            previousStageName = stage.name;
-          }
-          clearGroupFeedback(pipelineContext);
-          continue;
-        }
-
-        const groupResult = await this.groupOrchestrator.run(
-          entry,
-          pipelineContext,
-          stageCounter,
-          totalStages,
-          userInputValue,
-          projectPaths,
-          runToolRegistry,
-          runMiddleware,
-          pipelineRun.id,
-          signal,
-          skipSet,
-          input.originalRunId,
-        );
-        this.pipelineTotals.tokens += groupResult.totalTokensDelta;
-        this.pipelineTotals.toolCalls += groupResult.totalToolCallsDelta;
-
-        pipelineRun.stages.push(...groupResult.stageRuns);
-        stageCounter += groupResult.stagesExecuted;
-        // Update previousStageName to the last stage in the group
-        if (entry.stages.length > 0) {
-          previousStageName = entry.stages[entry.stages.length - 1].name;
-        }
-
-        // Clear group feedback after group completes
-        clearGroupFeedback(pipelineContext);
-
-        if (groupResult.status === 'rejected' || groupResult.status === 'failed' || groupResult.status === 'cancelled') {
-          pipelineRun.status = groupResult.status;
-          pipelineRun.completed_at = new Date().toISOString();
-          if (groupResult.status === 'cancelled') {
-            const lastStage = pipelineRun.stages[pipelineRun.stages.length - 1];
-            this.events?.onPipelineCancelled?.({
-              run_id: pipelineRun.id,
-              cancelled_at_stage: lastStage?.stage_name ?? 'unknown',
-              duration_ms: Date.now() - pipelineStartTime,
-            });
-          }
-          await this.config.db?.savePipelineRun(pipelineRun);
-          if (runMiddleware) {
-            await this.persistKeymap(pipelineRun.id, runMiddleware.getKeymap());
-          }
-          this.events?.onPipelineComplete?.({
-            pipeline_name: pipeline.name,
-            run_id: pipelineRun.id,
-            status: pipelineRun.status,
-            duration_ms: Date.now() - pipelineStartTime,
-            total_tokens: this.pipelineTotals.tokens,
-            total_tool_calls: this.pipelineTotals.toolCalls,
-          });
-          this.emitter.emit({ type: 'pipeline_complete', pipelineId: pipelineRun.id });
-          return pipelineRun;
-        }
-      } else {
-        // ========== SIMPLE STAGE ==========
-        stageCounter++;
-
-        // Skip stage if resuming from a later stage
-        if (skipSet.has(entry.name)) {
-          const skippedReason = input.originalRunId
-            ? `resumed from run ${input.originalRunId}`
-            : 'resumed from prior run';
-          const skippedRun = makeSkippedStageRun(entry.name, skippedReason);
-          this.events?.onStageComplete?.({
-            stage_name: entry.name,
-            stage_index: stageCounter - 1,
-            total_stages: totalStages,
-            status: 'skipped',
-            attempts: 0,
-            duration_ms: 0,
-            skipped_reason: skippedReason,
-          });
-          pipelineRun.stages.push(skippedRun);
-          previousStageName = entry.name;
-          continue;
-        }
-
-        const result = await this.stageExecutor.execute(
-          entry,
-          pipelineContext,
-          previousStageName,
-          userInputValue,
-          stageCounter - 1,
-          totalStages,
-          projectPaths,
-          runToolRegistry,
-          runMiddleware,
-          pipelineRun.id,
-          signal,
-        );
-        this.pipelineTotals.tokens += result.tokensDelta ?? 0;
-        this.pipelineTotals.toolCalls += result.toolCallsDelta ?? 0;
-
-        pipelineRun.stages.push(result.stageRun);
-
-        if (result.status === 'failed' || result.status === 'rejected' || result.status === 'cancelled') {
-          pipelineRun.status = result.stageRun.status;
-          pipelineRun.completed_at = new Date().toISOString();
-          if (result.status === 'cancelled') {
-            this.events?.onPipelineCancelled?.({
-              run_id: pipelineRun.id,
-              cancelled_at_stage: result.stageRun.stage_name,
-              duration_ms: Date.now() - pipelineStartTime,
-            });
-          }
-          await this.config.db?.savePipelineRun(pipelineRun);
-          if (runMiddleware) {
-            await this.persistKeymap(pipelineRun.id, runMiddleware.getKeymap());
-          }
-          this.events?.onPipelineComplete?.({
-            pipeline_name: pipeline.name,
-            run_id: pipelineRun.id,
-            status: pipelineRun.status,
-            duration_ms: Date.now() - pipelineStartTime,
-            total_tokens: this.pipelineTotals.tokens,
-            total_tool_calls: this.pipelineTotals.toolCalls,
-          });
-          this.emitter.emit({ type: 'pipeline_complete', pipelineId: pipelineRun.id });
-          return pipelineRun;
-        }
-
-        if (result.lastAgentOutput !== undefined) {
-          addStageOutput(pipelineContext, entry.name, result.lastAgentOutput);
-        }
-        if (result.toolCalls && result.toolCalls.length > 0) {
-          addStageToolResults(pipelineContext, entry.name, result.toolCalls);
-        }
-
-        previousStageName = entry.name;
-      }
-    }
-
-    // 5. All stages succeeded
-    pipelineRun.status = 'success';
-    pipelineRun.completed_at = new Date().toISOString();
-
-    // 6. Persist then emit (DB must be updated before SSE fires, so spawnAndWait GET sees final status)
-    await this.config.db?.savePipelineRun(pipelineRun);
-    if (runMiddleware) {
-      await this.persistKeymap(pipelineRun.id, runMiddleware.getKeymap());
-    }
-    this.events?.onPipelineComplete?.({
-      pipeline_name: pipeline.name,
-      run_id: pipelineRun.id,
-      status: pipelineRun.status,
-      duration_ms: Date.now() - pipelineStartTime,
-      total_tokens: this.pipelineTotals.tokens,
-      total_tool_calls: this.pipelineTotals.toolCalls,
-    });
-    this.emitter.emit({ type: 'pipeline_complete', pipelineId: pipelineRun.id });
-
-    return pipelineRun;
-  }
-
-  /** For external listeners via the generic event bus */
-  onEvent(listener: (event: import('./events.js').PipelineEvent) => void): void {
-    this.emitter.on(listener);
-  }
-
-  // -- Private --
-
-  private async persistKeymap(runId: string, keymap: Record<string, string>): Promise<void> {
-    if (Object.keys(keymap).length === 0) return;
-    try {
-      const { mkdir, writeFile } = await import('node:fs/promises');
-      const { join } = await import('node:path');
-      // configsDir is .studio/ directly — keymap goes in .studio/runs/anonymization/
-      const anonDir = join(this.config.configsDir, 'runs', 'anonymization');
-      await mkdir(anonDir, { recursive: true });
-      const keymapPath = join(anonDir, `${runId}.keymap.json`);
-      await writeFile(keymapPath, JSON.stringify(keymap, null, 2), 'utf-8');
-    } catch {
-      // Non-fatal — keymap persistence is best-effort
-    }
-  }
-}

package/src/events.ts

@@ -1,167 +0,0 @@
-import type { ToolCall, ToolCallStartEvent, ToolCallCompleteEvent, AgentThinkingEvent, AgentProgressEvent, AgentTokenEvent } from '@studio-foundation/contracts';
-
-// Event types for pipeline observability
-// Dedicated event types — separate from contract types (PipelineRun, StageRun)
-
-export interface ToolCallSummary {
-  name: string;
-  arguments_summary: string;
-}
-
-export interface TokenUsage {
-  prompt_tokens: number;
-  completion_tokens: number;
-  total_tokens: number;
-}
-
-export interface PipelineStartEvent {
-  pipeline_name: string;
-  run_id: string;
-}
-
-export interface PipelineCompleteEvent {
-  pipeline_name: string;
-  run_id: string;
-  status: string;
-  duration_ms: number;
-  total_tokens: number;
-  total_tool_calls: number;
-}
-
-export interface PipelineCancelledEvent {
-  run_id: string;
-  cancelled_at_stage: string;
-  duration_ms: number;
-}
-
-export interface StageStartEvent {
-  stage_name: string;
-  stage_index: number;
-  total_stages: number;
-  max_attempts: number;
-}
-
-export interface StageCompleteEvent {
-  stage_name: string;
-  stage_index: number;
-  total_stages: number;
-  status: string;
-  attempts: number;
-  duration_ms: number;
-  output_summary?: string;
-  output?: unknown;
-  tool_calls?: ToolCall[];
-  token_usage?: TokenUsage;
-  rejection_reason?: string;
-  rejection_details?: string[];
-  skipped_reason?: string;
-}
-
-export interface StageRetryEvent {
-  stage: string;
-  attempt: number;
-  max_attempts: number;
-  failures: string[];
-  agent_output_raw?: string;
-  tool_calls_count?: number;
-}
-
-export interface GroupStartEvent {
-  group_name: string;
-  max_iterations: number;
-  parallel?: boolean;
-}
-
-export interface GroupIterationEvent {
-  group_name: string;
-  iteration: number;
-  max_iterations: number;
-}
-
-export interface GroupFeedbackEvent {
-  group_name: string;
-  iteration: number;
-  rejection_reason: string;
-  rejection_details: string[];
-}
-
-export interface GroupCompleteEvent {
-  group_name: string;
-  iterations: number;
-  status: string;
-}
-
-export interface StageContextEvent {
-  stage: string;
-  run_id: string;
-  context_keys: Record<string, number>;
-  context_content?: Record<string, unknown>;
-  system_prompt?: string;
-}
-
-export interface StagedToolCallStartEvent extends ToolCallStartEvent {
-  stage: string;
-}
-
-export interface StagedToolCallCompleteEvent extends ToolCallCompleteEvent {
-  stage: string;
-}
-
-export interface StagedAgentThinkingEvent extends AgentThinkingEvent {
-  stage: string;
-}
-
-export interface StagedAgentProgressEvent extends AgentProgressEvent {
-  stage: string;
-}
-
-export interface StagedAgentTokenEvent extends AgentTokenEvent {
-  stage: string;
-}
-
-export interface EngineEvents {
-  onPipelineStart?: (event: PipelineStartEvent) => void;
-  onPipelineComplete?: (event: PipelineCompleteEvent) => void;
-  onPipelineCancelled?: (event: PipelineCancelledEvent) => void;
-  onStageStart?: (event: StageStartEvent) => void;
-  onStageComplete?: (event: StageCompleteEvent) => void;
-  onTaskRetry?: (event: StageRetryEvent) => void;
-  onGroupStart?: (event: GroupStartEvent) => void;
-  onGroupIteration?: (event: GroupIterationEvent) => void;
-  onGroupFeedback?: (event: GroupFeedbackEvent) => void;
-  onGroupComplete?: (event: GroupCompleteEvent) => void;
-  onStageContext?: (event: StageContextEvent) => void;
-  // Real-time tool call streaming (used by --live mode)
-  onToolCallStart?: (event: StagedToolCallStartEvent) => void;
-  onToolCallComplete?: (event: StagedToolCallCompleteEvent) => void;
-  // Agent thinking/progress (text content emitted alongside tool calls)
-  onAgentThinking?: (event: StagedAgentThinkingEvent) => void;
-  onAgentProgress?: (event: StagedAgentProgressEvent) => void;
-  onAgentToken?: (event: StagedAgentTokenEvent) => void;
-}
-
-// Keep the generic event bus for other use cases
-export type PipelineEvent =
-  | { type: 'pipeline_start'; pipelineId: string }
-  | { type: 'pipeline_complete'; pipelineId: string }
-  | { type: 'stage_start'; stageId: string; stageName: string }
-  | { type: 'stage_complete'; stageId: string; stageName: string }
-  | { type: 'task_retry'; stageName: string; attempt: number; failures: string[]; rawOutput?: string }
-  | { type: 'group_start'; groupName: string; maxIterations: number }
-  | { type: 'group_iteration'; groupName: string; iteration: number; maxIterations: number }
-  | { type: 'group_feedback'; groupName: string; iteration: number; rejectionReason: string }
-  | { type: 'group_complete'; groupName: string; iterations: number; status: string };
-
-export class PipelineEventEmitter {
-  private listeners: Array<(event: PipelineEvent) => void> = [];
-
-  on(listener: (event: PipelineEvent) => void): void {
-    this.listeners.push(listener);
-  }
-
-  emit(event: PipelineEvent): void {
-    for (const listener of this.listeners) {
-      listener(event);
-    }
-  }
-}