orquesta-cli 0.1.12

package/src/eval/eval-runner.ts

@@ -0,0 +1,456 @@
+/**
+ * Eval Runner
+ *
+ * Core execution logic for --eval mode
+ * Output as NDJSON event stream
+ */
+
+import { Message, TodoItem } from '../types/index.js';
+import { LLMClient, createLLMClient } from '../core/llm/llm-client.js';
+import { configManager } from '../core/config/config-manager.js';
+import { PlanExecutor } from '../orchestration/plan-executor.js';
+import type { StateCallbacks } from '../orchestration/types.js';
+import type {
+  EvalInput,
+  EvalEvent,
+  EvalStartEvent,
+  EvalTodoEvent,
+  EvalToolCallEvent,
+  EvalToolResultEvent,
+  EvalResponseEvent,
+  EvalErrorEvent,
+  EvalEndEvent,
+} from './types.js';
+
+/**
+ * Output event to stdout as NDJSON
+ */
+function emitEvent(event: EvalEvent): void {
+  console.log(JSON.stringify(event));
+}
+
+/**
+ * Return current timestamp
+ */
+function now(): string {
+  return new Date().toISOString();
+}
+
+/**
+ * Eval Runner class
+ */
+export class EvalRunner {
+  private llmClient: LLMClient | null = null;
+  private planExecutor: PlanExecutor;
+  private startTime: number = 0;
+  private toolCallCount: number = 0;
+  private filesModified: Set<string> = new Set();
+  private lastResponse: string = '';
+  private todos: TodoItem[] = [];
+
+  constructor() {
+    this.planExecutor = new PlanExecutor();
+  }
+
+  /**
+   * Run evaluation
+   */
+  async run(input: EvalInput): Promise<void> {
+    this.startTime = Date.now();
+    this.toolCallCount = 0;
+    this.filesModified.clear();
+    this.lastResponse = '';
+    this.todos = [];
+
+    try {
+      // 1. Initialize
+      await configManager.initialize();
+
+      if (!configManager.hasEndpoints()) {
+        this.emitError('No LLM endpoint configured. Run: orquesta to setup.');
+        return;
+      }
+
+      this.llmClient = createLLMClient();
+      const modelInfo = this.llmClient.getModelInfo();
+
+      // Change working directory
+      if (input.working_dir) {
+        process.chdir(input.working_dir);
+      }
+
+      // 2. Start event
+      const startEvent: EvalStartEvent = {
+        event: 'start',
+        timestamp: now(),
+        data: {
+          prompt: input.prompt,
+          model: modelInfo.model,
+          endpoint: modelInfo.endpoint,
+          working_dir: process.cwd(),
+        },
+      };
+      emitEvent(startEvent);
+
+      // 3. Execute
+      await this.execute(input.prompt);
+
+      // 4. End event
+      this.emitEnd(true);
+
+    } catch (error) {
+      this.emitError(error instanceof Error ? error.message : String(error));
+      this.emitEnd(false);
+    }
+  }
+
+  /**
+   * Actual execution logic
+   */
+  private async execute(prompt: string): Promise<void> {
+    if (!this.llmClient) {
+      throw new Error('LLM Client not initialized');
+    }
+
+    const messages: Message[] = [];
+    const isInterruptedRef = { current: false };
+
+    // Connect StateCallbacks to events
+    const callbacks = this.createCallbacks();
+
+    // Execute in Auto mode (classify and select appropriate mode)
+    await this.planExecutor.executeAutoMode(
+      prompt,
+      this.llmClient,
+      messages,
+      this.todos,
+      isInterruptedRef,
+      callbacks
+    );
+  }
+
+  /**
+   * Create StateCallbacks for event output
+   */
+  private createCallbacks(): StateCallbacks {
+    let previousMessages: Message[] = [];
+
+    return {
+      setTodos: (todosOrFn) => {
+        const newTodos = typeof todosOrFn === 'function'
+          ? todosOrFn(this.todos)
+          : todosOrFn;
+
+        // Detect newly added TODOs
+        for (const todo of newTodos) {
+          const existing = this.todos.find(t => t.id === todo.id);
+          if (!existing) {
+            // New TODO
+            this.emitTodo('created', todo);
+          } else if (existing.status !== todo.status) {
+            // Status changed
+            if (todo.status === 'in_progress') {
+              this.emitTodo('started', todo);
+            } else if (todo.status === 'completed') {
+              this.emitTodo('completed', todo);
+            } else if (todo.status === 'failed') {
+              this.emitTodo('failed', todo);
+            }
+          }
+        }
+
+        this.todos = newTodos;
+      },
+
+      setCurrentTodoId: () => {
+        // Event not needed
+      },
+
+      setExecutionPhase: () => {
+        // Event not needed
+      },
+
+      setIsInterrupted: () => {
+        // Event not needed
+      },
+
+      setCurrentActivity: () => {
+        // Event not needed
+      },
+
+      setMessages: (messagesOrFn) => {
+        const newMessages = typeof messagesOrFn === 'function'
+          ? messagesOrFn(previousMessages)
+          : messagesOrFn;
+
+        // Detect tool calls and responses from newly added messages
+        const addedMessages = newMessages.slice(previousMessages.length);
+
+        for (const msg of addedMessages) {
+          // Detect tool call (tool_calls in assistant message)
+          if (msg.role === 'assistant' && msg.tool_calls) {
+            for (const toolCall of msg.tool_calls) {
+              this.emitToolCall(toolCall.function.name, toolCall.function.arguments);
+            }
+          }
+
+          // Detect tool result
+          if (msg.role === 'tool') {
+            const isSuccess = !msg.content?.startsWith('Error:');
+            this.emitToolResult(msg.tool_call_id || 'unknown', isSuccess, msg.content);
+
+            // Detect file modifications
+            if (isSuccess && msg.content) {
+              this.detectFileModification(msg.content);
+            }
+          }
+
+          // Assistant final response (without tool_calls)
+          // Skip intermediate "Created X tasks" messages
+          if (msg.role === 'assistant' && !msg.tool_calls && msg.content) {
+            const isIntermediateMessage = msg.content.includes('Created') && msg.content.includes('tasks') && msg.content.includes('Starting execution');
+            if (!isIntermediateMessage) {
+              this.lastResponse = msg.content;
+            }
+          }
+        }
+
+        previousMessages = newMessages;
+      },
+
+      setAskUserRequest: () => {
+        // User input not available in eval mode - auto skip
+      },
+    };
+  }
+
+  /**
+   * Output TODO event
+   */
+  private emitTodo(action: 'created' | 'started' | 'completed' | 'failed', todo: TodoItem): void {
+    const event: EvalTodoEvent = {
+      event: 'todo',
+      timestamp: now(),
+      data: {
+        action,
+        id: todo.id,
+        title: todo.title,
+        status: todo.status,
+      },
+    };
+    emitEvent(event);
+  }
+
+  /**
+   * Output tool call event
+   */
+  private emitToolCall(tool: string, argsJson: string): void {
+    this.toolCallCount++;
+
+    let args: Record<string, unknown> = {};
+    try {
+      args = JSON.parse(argsJson);
+    } catch {
+      args = { raw: argsJson };
+    }
+
+    const event: EvalToolCallEvent = {
+      event: 'tool_call',
+      timestamp: now(),
+      data: {
+        tool,
+        args,
+      },
+    };
+    emitEvent(event);
+  }
+
+  /**
+   * Output tool result event
+   */
+  private emitToolResult(tool: string, success: boolean, content?: string): void {
+    const event: EvalToolResultEvent = {
+      event: 'tool_result',
+      timestamp: now(),
+      data: {
+        tool,
+        success,
+        ...(success ? { result: content?.substring(0, 500) } : { error: content }),
+      },
+    };
+    emitEvent(event);
+  }
+
+  /**
+   * Detect file modifications
+   */
+  private detectFileModification(content: string): void {
+    // Extract file paths using patterns like "created", "written", "edited"
+    const patterns = [
+      /File created: (.+)/,
+      /File written: (.+)/,
+      /Successfully edited (.+)/,
+      /Created file: (.+)/,
+      /Updated file: (.+)/,
+    ];
+
+    for (const pattern of patterns) {
+      const match = content.match(pattern);
+      if (match && match[1]) {
+        this.filesModified.add(match[1].trim());
+      }
+    }
+  }
+
+  /**
+   * Output response event
+   */
+  private emitResponse(): void {
+    let responseContent = this.lastResponse;
+
+    // If no final response captured, generate a summary from completed tasks
+    if (!responseContent && this.todos.length > 0) {
+      const completedCount = this.todos.filter(t => t.status === 'completed').length;
+      const failedCount = this.todos.filter(t => t.status === 'failed').length;
+
+      responseContent = `✅ Task execution completed.\n\n`;
+      responseContent += `Completed: ${completedCount}/${this.todos.length} tasks\n`;
+      if (failedCount > 0) {
+        responseContent += `Failed: ${failedCount} tasks\n`;
+      }
+      responseContent += `\nTasks:\n`;
+      this.todos.forEach(todo => {
+        const icon = todo.status === 'completed' ? '✅' : todo.status === 'failed' ? '❌' : '⏸️';
+        responseContent += `${icon} ${todo.title}\n`;
+      });
+    }
+
+    if (responseContent) {
+      const event: EvalResponseEvent = {
+        event: 'response',
+        timestamp: now(),
+        data: {
+          content: responseContent,
+        },
+      };
+      emitEvent(event);
+    }
+  }
+
+  /**
+   * Output error event
+   */
+  private emitError(message: string): void {
+    const event: EvalErrorEvent = {
+      event: 'error',
+      timestamp: now(),
+      data: {
+        message,
+      },
+    };
+    emitEvent(event);
+  }
+
+  /**
+   * Output end event
+   */
+  private emitEnd(success: boolean): void {
+    // Output response event first
+    if (success) {
+      this.emitResponse();
+    }
+
+    const duration = Date.now() - this.startTime;
+
+    const todoStats = this.todos.length > 0 ? {
+      total: this.todos.length,
+      completed: this.todos.filter(t => t.status === 'completed').length,
+      failed: this.todos.filter(t => t.status === 'failed').length,
+    } : undefined;
+
+    const event: EvalEndEvent = {
+      event: 'end',
+      timestamp: now(),
+      data: {
+        success,
+        duration_ms: duration,
+        ...(todoStats && { todos: todoStats }),
+        ...(this.toolCallCount > 0 && { tool_calls: this.toolCallCount }),
+        ...(this.filesModified.size > 0 && { files_modified: Array.from(this.filesModified) }),
+      },
+    };
+    emitEvent(event);
+  }
+}
+
+/**
+ * Read JSON input from stdin
+ */
+export async function readStdinJson(): Promise<EvalInput> {
+  return new Promise((resolve, reject) => {
+    let data = '';
+
+    process.stdin.setEncoding('utf8');
+    process.stdin.on('data', (chunk) => {
+      data += chunk;
+    });
+
+    process.stdin.on('end', () => {
+      try {
+        const input = JSON.parse(data) as EvalInput;
+        if (!input.prompt) {
+          reject(new Error('Missing required field: prompt'));
+          return;
+        }
+        resolve(input);
+      } catch (error) {
+        reject(new Error(`Invalid JSON input: ${error instanceof Error ? error.message : String(error)}`));
+      }
+    });
+
+    process.stdin.on('error', (error) => {
+      reject(error);
+    });
+
+    // Timeout for stdin (30 seconds)
+    setTimeout(() => {
+      if (!data) {
+        reject(new Error('No input received (timeout after 30s)'));
+      }
+    }, 30000);
+  });
+}
+
+/**
+ * Run eval mode (called from CLI)
+ */
+export async function runEvalMode(): Promise<void> {
+  try {
+    const input = await readStdinJson();
+    const runner = new EvalRunner();
+    await runner.run(input);
+  } catch (error) {
+    // Output input errors as JSON events
+    const errorEvent: EvalErrorEvent = {
+      event: 'error',
+      timestamp: new Date().toISOString(),
+      data: {
+        message: error instanceof Error ? error.message : String(error),
+        code: 'INPUT_ERROR',
+      },
+    };
+    console.log(JSON.stringify(errorEvent));
+
+    const endEvent: EvalEndEvent = {
+      event: 'end',
+      timestamp: new Date().toISOString(),
+      data: {
+        success: false,
+        duration_ms: 0,
+      },
+    };
+    console.log(JSON.stringify(endEvent));
+
+    process.exit(1);
+  }
+}

package/src/eval/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Eval Module
+ *
+ * --eval mode exports
+ */
+
+export * from './types.js';
+export * from './eval-runner.js';

package/src/eval/types.ts

@@ -0,0 +1,139 @@
+/**
+ * Eval Mode Types
+ *
+ * --eval mode type definitions
+ */
+
+/**
+ * Eval input schema (stdin JSON)
+ */
+export interface EvalInput {
+  prompt: string;
+  working_dir?: string;
+}
+
+/**
+ * Event type
+ */
+export type EvalEventType =
+  | 'start'
+  | 'todo'
+  | 'tool_call'
+  | 'tool_result'
+  | 'response'
+  | 'error'
+  | 'end';
+
+/**
+ * Base event structure
+ */
+export interface EvalEventBase {
+  event: EvalEventType;
+  timestamp: string;
+}
+
+/**
+ * Start event
+ */
+export interface EvalStartEvent extends EvalEventBase {
+  event: 'start';
+  data: {
+    prompt: string;
+    model: string;
+    endpoint: string;
+    working_dir: string;
+  };
+}
+
+/**
+ * TODO event
+ */
+export interface EvalTodoEvent extends EvalEventBase {
+  event: 'todo';
+  data: {
+    action: 'created' | 'started' | 'completed' | 'failed';
+    id: string;
+    title: string;
+    status?: string;
+  };
+}
+
+/**
+ * Tool Call event
+ */
+export interface EvalToolCallEvent extends EvalEventBase {
+  event: 'tool_call';
+  data: {
+    tool: string;
+    args: Record<string, unknown>;
+  };
+}
+
+/**
+ * Tool Result event
+ */
+export interface EvalToolResultEvent extends EvalEventBase {
+  event: 'tool_result';
+  data: {
+    tool: string;
+    success: boolean;
+    result?: string;
+    error?: string;
+  };
+}
+
+/**
+ * Response event (final response)
+ */
+export interface EvalResponseEvent extends EvalEventBase {
+  event: 'response';
+  data: {
+    content: string;
+  };
+}
+
+/**
+ * Error event
+ */
+export interface EvalErrorEvent extends EvalEventBase {
+  event: 'error';
+  data: {
+    message: string;
+    code?: string;
+  };
+}
+
+/**
+ * End event
+ */
+export interface EvalEndEvent extends EvalEventBase {
+  event: 'end';
+  data: {
+    success: boolean;
+    duration_ms: number;
+    usage?: {
+      input_tokens: number;
+      output_tokens: number;
+      total_tokens: number;
+    };
+    todos?: {
+      total: number;
+      completed: number;
+      failed: number;
+    };
+    tool_calls?: number;
+    files_modified?: string[];
+  };
+}
+
+/**
+ * All event types
+ */
+export type EvalEvent =
+  | EvalStartEvent
+  | EvalTodoEvent
+  | EvalToolCallEvent
+  | EvalToolResultEvent
+  | EvalResponseEvent
+  | EvalErrorEvent
+  | EvalEndEvent;

package/src/index.ts

@@ -0,0 +1,22 @@
+/**
+ * LOCAL-CLI Main Export
+ *
+ * This file is the entry point when used as a library.
+ * CLI execution uses src/cli.ts.
+ */
+
+import { createRequire } from 'module';
+
+// Read version from package.json (single source of truth)
+const require = createRequire(import.meta.url);
+const packageJson = require('../package.json') as { version: string; name: string };
+
+export const version = packageJson.version;
+export const name = packageJson.name;
+
+/**
+ * Functions to be used as API will be exported here in the future.
+ */
+
+// Placeholder for future exports
+export {};

package/src/orchestration/index.ts

@@ -0,0 +1,30 @@
+/**
+ * Orchestration Module
+ *
+ * Plan & Execute workflow management
+ */
+
+// Types
+export type {
+  ExecutionPhase,
+  PlanExecutionState,
+  AskUserRequest,
+  AskUserResponse,
+  AskUserState,
+  StateCallbacks,
+  ExecutionContext,
+  ExecutionResult,
+  PlanExecutionActions,
+} from './types.js';
+
+// Utilities
+export {
+  formatErrorMessage,
+  buildTodoContext,
+  areAllTodosCompleted,
+  findActiveTodo,
+  getTodoStats,
+} from './utils.js';
+
+// Plan Executor
+export { PlanExecutor, planExecutor } from './plan-executor.js';