@metabob/minibob 0.1.2

package/src/search-first-executor.ts

@@ -0,0 +1,666 @@
+/**
+ * Search-First Goal Executor
+ *
+ * Implements dynamic step-by-step goal execution:
+ * 1. Decompose goal into steps using LLM
+ * 2. For each step:
+ *    - Search for existing activities that match
+ *    - If match: delegate to existing activity (minimal context)
+ *    - If no match: execute step directly
+ *    - Summarize result
+ *    - Pass summary to next step
+ *
+ * This prevents token accumulation by:
+ * - Using summaries instead of full traces between steps
+ * - Delegating to existing activities with minimal context
+ * - Breaking large goals into focused steps
+ */
+
+import type { LLMClient } from "./llm"
+import type { ActivityTemplate, ToolResult, ToolHandler } from "./types"
+import { createLLMClient } from "./llm"
+import { createToolHandlers, getAllToolDefinitions } from "./tools"
+import { getMCPClient, isMCPEnabled } from "./mcp"
+import { loadTemplateFromMCPOrLocal, ActivityExecutor } from "./activity"
+
+/**
+ * Safe wrapper for Bun.file() with better error handling
+ */
+function safeReadFile(path: string): ReturnType<typeof Bun.file> {
+  if (!path || typeof path !== 'string') {
+    throw new Error(`Invalid file path: expected string, got ${typeof path}`)
+  }
+  try {
+    return Bun.file(path)
+  } catch (error) {
+    throw new Error(`Failed to create file handle for '${path}': ${error instanceof Error ? error.message : String(error)}`)
+  }
+}
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+export interface StepValidation {
+  /** Commands to run for validation (exit code 0 = pass) */
+  commands?: string[]
+  /** Files that must exist after execution */
+  requiredFiles?: string[]
+  /** Patterns that must exist in specific files */
+  requiredPatterns?: Array<{ file: string; pattern: string }>
+  /** Description of what success looks like (for LLM self-validation) */
+  successCriteria?: string
+}
+
+export interface GoalStep {
+  id: string
+  description: string
+  intent: string
+  category: "feature" | "bugfix" | "refactor" | "tool" | "infrastructure" | "other"
+  dependencies: string[]
+  /** Validation criteria for this step */
+  validation?: StepValidation
+}
+
+export interface StepResult {
+  stepId: string
+  status: "completed" | "failed" | "skipped"
+  method: "existing_activity" | "direct_execution" | "skipped"
+  activityId?: string
+  summary: string
+  error?: string
+}
+
+export interface SearchFirstConfig {
+  provider: "anthropic" | "openai"
+  apiKey: string
+  model: string
+  workingDirectory: string
+  maxSteps?: number
+  maxTokensPerStep?: number
+}
+
+// =============================================================================
+// SEARCH-FIRST EXECUTOR
+// =============================================================================
+
+export class SearchFirstExecutor {
+  private config: SearchFirstConfig
+  private llm: LLMClient
+
+  constructor(config: SearchFirstConfig) {
+    this.config = {
+      ...config,
+      maxSteps: config.maxSteps ?? 5,
+      maxTokensPerStep: config.maxTokensPerStep ?? 4096,
+    }
+    this.llm = createLLMClient(config.provider, config.apiKey)
+  }
+
+  /**
+   * Decompose a goal into discrete steps using LLM
+   */
+  async decomposeGoal(goal: string, context?: Record<string, unknown>): Promise<GoalStep[]> {
+    const contextStr = context ? Object.entries(context)
+      .map(([k, v]) => `- ${k}: ${typeof v === 'string' ? v : JSON.stringify(v)}`)
+      .join('\n') : 'None provided'
+
+    const prompt = `Decompose this goal into 2-4 discrete, actionable steps WITH validation criteria.
+
+Goal: ${goal}
+
+Context:
+${contextStr}
+
+Rules:
+- Each step should be independently accomplishable
+- Steps should be sequential (later steps may depend on earlier ones)
+- Keep steps focused and specific
+- Use categories: feature, bugfix, refactor, tool, infrastructure
+- CRITICAL: Include validation criteria for each step - how do we VERIFY it succeeded?
+
+Respond with JSON array:
+[
+  {
+    "id": "step-1",
+    "description": "Brief description of step",
+    "intent": "What this step accomplishes",
+    "category": "refactor",
+    "dependencies": [],
+    "validation": {
+      "commands": ["bun run typecheck"],
+      "requiredFiles": ["src/new-file.ts"],
+      "successCriteria": "Code compiles without type errors"
+    }
+  }
+]
+
+Validation guidelines by category:
+- refactor: typecheck passes, tests pass, no regressions
+- bugfix: specific test passes, error no longer occurs
+- feature: new functionality works, tests pass
+- tool: tool executes successfully with expected output
+- infrastructure: service starts, health check passes
+
+Only output the JSON array, nothing else.`
+
+    const result = await this.llm.complete({
+      model: this.config.model,
+      messages: [
+        { role: "user", content: prompt }
+      ],
+      maxTokens: 1024,
+    })
+
+    try {
+      // Extract JSON from response
+      const jsonMatch = result.content.match(/\[[\s\S]*\]/)
+      if (!jsonMatch) {
+        throw new Error("No JSON array found in response")
+      }
+      const steps = JSON.parse(jsonMatch[0]) as GoalStep[]
+      return steps.slice(0, this.config.maxSteps)
+    } catch (e) {
+      console.error("[SearchFirst] Failed to parse steps:", e)
+      // Fallback: treat entire goal as single step
+      return [{
+        id: "step-1",
+        description: goal,
+        intent: goal,
+        category: "other",
+        dependencies: [],
+      }]
+    }
+  }
+
+  /**
+   * Search for existing activities that can accomplish a step
+   */
+  async searchForActivity(step: GoalStep): Promise<{ found: boolean; templateId?: string; score?: number }> {
+    if (!isMCPEnabled()) {
+      return { found: false }
+    }
+
+    const mcp = getMCPClient()
+    if (!mcp) {
+      return { found: false }
+    }
+
+    try {
+      // Get recommendations from backend
+      const recommendations = await mcp.recommendActivities(
+        step.intent,
+        step.category === "other" ? undefined : step.category,
+        [],
+        3
+      )
+
+      if (recommendations.length === 0) {
+        console.log(`[SearchFirst] No existing activities found for: ${step.description}`)
+        return { found: false }
+      }
+
+      // Check if top recommendation is a good match
+      const top = recommendations[0]
+      if (!top) {
+        return { found: false }
+      }
+
+      const score = top.selection_metadata?.score ?? 0
+
+      // Require minimum score for reuse (Thompson Sampling score > 0.3)
+      if (score < 0.3) {
+        console.log(`[SearchFirst] Top match score too low (${score}): ${top.template_id}`)
+        return { found: false }
+      }
+
+      console.log(`[SearchFirst] Found matching activity: ${top.template_id} (score: ${score})`)
+      return {
+        found: true,
+        templateId: top.template_id,
+        score,
+      }
+    } catch (e) {
+      console.error("[SearchFirst] Search failed:", e)
+      return { found: false }
+    }
+  }
+
+  /**
+   * Execute a step using an existing activity
+   */
+  async executeViaActivity(
+    step: GoalStep,
+    templateId: string,
+    previousResults: StepResult[]
+  ): Promise<StepResult> {
+    console.log(`[SearchFirst] Executing step "${step.id}" via activity: ${templateId}`)
+
+    try {
+      const template = await loadTemplateFromMCPOrLocal(templateId)
+      if (!template) {
+        return {
+          stepId: step.id,
+          status: "failed",
+          method: "existing_activity",
+          error: `Template not found: ${templateId}`,
+          summary: `Failed to load activity template ${templateId}`,
+        }
+      }
+
+      // Create minimal executor with isolated context
+      const executor = new ActivityExecutor({
+        provider: this.config.provider,
+        apiKey: this.config.apiKey,
+        model: this.config.model,
+        workingDirectory: this.config.workingDirectory,
+        maxNestingDepth: 1, // Prevent further nesting
+      })
+
+      // Build minimal context from previous step summaries
+      const contextSummary = previousResults
+        .filter(r => r.status === "completed")
+        .map(r => `- ${r.stepId}: ${r.summary}`)
+        .join('\n')
+
+      const result = await executor.execute({
+        template,
+        variables: {
+          goal: step.intent,
+          previousSteps: contextSummary || "This is the first step",
+        },
+        reason: step.description,
+      })
+
+      // Activity reported failure
+      if (result.status !== "completed") {
+        return {
+          stepId: step.id,
+          status: "failed",
+          method: "existing_activity",
+          activityId: templateId,
+          summary: `Failed: ${result.error?.substring(0, 100) ?? "unknown error"}`,
+          error: result.error,
+        }
+      }
+
+      // Activity completed - now run our own validation
+      const validationResult = await this.validateStep(step)
+
+      if (!validationResult.passed) {
+        console.log(`[SearchFirst] Activity completed but step validation FAILED`)
+
+        // Report validation failure to backend to update metrics
+        // This corrects the false "success" that the activity reported
+        await this.reportValidationFailure(templateId, step, validationResult.reason)
+
+        return {
+          stepId: step.id,
+          status: "failed",
+          method: "existing_activity",
+          activityId: templateId,
+          summary: `Activity completed but validation failed: ${validationResult.reason}`,
+          error: validationResult.reason,
+        }
+      }
+
+      return {
+        stepId: step.id,
+        status: "completed",
+        method: "existing_activity",
+        activityId: templateId,
+        summary: `Completed via ${templateId}`,
+      }
+    } catch (e) {
+      const error = e instanceof Error ? e.message : String(e)
+      return {
+        stepId: step.id,
+        status: "failed",
+        method: "existing_activity",
+        activityId: templateId,
+        summary: `Execution failed: ${error.substring(0, 100)}`,
+        error,
+      }
+    }
+  }
+
+  /**
+   * Execute a step directly (no existing activity found)
+   */
+  async executeDirect(
+    step: GoalStep,
+    previousResults: StepResult[]
+  ): Promise<StepResult> {
+    console.log(`[SearchFirst] Executing step "${step.id}" directly`)
+
+    try {
+      // Build context from previous step summaries
+      const contextSummary = previousResults
+        .filter(r => r.status === "completed")
+        .map(r => `- ${r.stepId}: ${r.summary}`)
+        .join('\n')
+
+      const prompt = `Complete this step:
+
+Step: ${step.description}
+Intent: ${step.intent}
+
+Previous steps completed:
+${contextSummary || "None - this is the first step"}
+
+Instructions:
+- Focus only on this specific step
+- Use available tools to accomplish the goal
+- Be concise and efficient`
+
+      // Create minimal tool handlers
+      const toolHandlers = createToolHandlers({
+        workingDirectory: this.config.workingDirectory,
+        // Don't allow nested activity execution from direct steps
+        onActivityExecute: undefined,
+        onSearchActivities: undefined,
+        onCreateActivity: undefined,
+      })
+
+      const result = await this.llm.completeWithTools(
+        {
+          model: this.config.model,
+          messages: [
+            {
+              role: "system",
+              content: "You are a focused task executor. Complete the given step using available tools. Be efficient and concise.",
+            },
+            {
+              role: "user",
+              content: prompt,
+            },
+          ],
+          tools: getAllToolDefinitions(),
+          maxTokens: this.config.maxTokensPerStep,
+        },
+        toolHandlers
+      )
+
+      // Summarize the result
+      const summary = await this.summarizeResult(step, result.content)
+
+      // Run validation if specified
+      const validationResult = await this.validateStep(step)
+
+      if (!validationResult.passed) {
+        console.log(`[SearchFirst] Step ${step.id} execution completed but validation FAILED`)
+        return {
+          stepId: step.id,
+          status: "failed",
+          method: "direct_execution",
+          summary: `Execution completed but validation failed: ${validationResult.reason}`,
+          error: validationResult.reason,
+        }
+      }
+
+      return {
+        stepId: step.id,
+        status: "completed",
+        method: "direct_execution",
+        summary,
+      }
+    } catch (e) {
+      const error = e instanceof Error ? e.message : String(e)
+      return {
+        stepId: step.id,
+        status: "failed",
+        method: "direct_execution",
+        summary: `Direct execution failed: ${error.substring(0, 100)}`,
+        error,
+      }
+    }
+  }
+
+  /**
+   * Summarize a step result to compress context for next step
+   */
+  async summarizeResult(step: GoalStep, fullResult: string): Promise<string> {
+    // If result is already short, use as-is
+    if (fullResult.length < 200) {
+      return fullResult
+    }
+
+    try {
+      const result = await this.llm.complete({
+        model: this.config.model,
+        messages: [
+          {
+            role: "user",
+            content: `Summarize this step result in 1-2 sentences (max 100 words):
+
+Step: ${step.description}
+
+Result:
+${fullResult.substring(0, 2000)}
+
+Summary:`,
+          },
+        ],
+        maxTokens: 150,
+      })
+
+      return result.content.trim()
+    } catch {
+      // Fallback: truncate
+      return fullResult.substring(0, 200) + "..."
+    }
+  }
+
+  /**
+   * Validate a step's execution result
+   *
+   * Returns { passed: true } if validation passes or no validation specified
+   * Returns { passed: false, reason: string } if validation fails
+   */
+  async validateStep(step: GoalStep): Promise<{ passed: boolean; reason?: string }> {
+    if (!step.validation) {
+      console.log(`[SearchFirst] No validation criteria for step ${step.id}`)
+      return { passed: true }
+    }
+
+    const validation = step.validation
+    console.log(`[SearchFirst] Validating step ${step.id}...`)
+
+    // 1. Check required files exist
+    if (validation.requiredFiles && validation.requiredFiles.length > 0) {
+      for (const filePath of validation.requiredFiles) {
+        const fullPath = filePath.startsWith("/")
+          ? filePath
+          : `${this.config.workingDirectory}/${filePath}`
+        const file = safeReadFile(fullPath)
+        if (!(await file.exists())) {
+          console.log(`[SearchFirst] Validation FAILED: Missing file ${filePath}`)
+          return { passed: false, reason: `Required file missing: ${filePath}` }
+        }
+      }
+      console.log(`[SearchFirst] ✓ Required files exist`)
+    }
+
+    // 2. Check required patterns in files
+    if (validation.requiredPatterns && validation.requiredPatterns.length > 0) {
+      for (const { file: filePath, pattern } of validation.requiredPatterns) {
+        const fullPath = filePath.startsWith("/")
+          ? filePath
+          : `${this.config.workingDirectory}/${filePath}`
+        const file = safeReadFile(fullPath)
+        if (!(await file.exists())) {
+          console.log(`[SearchFirst] Validation FAILED: File missing for pattern check ${filePath}`)
+          return { passed: false, reason: `File missing for pattern check: ${filePath}` }
+        }
+        const content = await file.text()
+        const regex = new RegExp(pattern)
+        if (!regex.test(content)) {
+          console.log(`[SearchFirst] Validation FAILED: Pattern not found in ${filePath}`)
+          return { passed: false, reason: `Pattern "${pattern}" not found in ${filePath}` }
+        }
+      }
+      console.log(`[SearchFirst] ✓ Required patterns found`)
+    }
+
+    // 3. Run validation commands
+    if (validation.commands && validation.commands.length > 0) {
+      for (const command of validation.commands) {
+        console.log(`[SearchFirst] Running validation command: ${command}`)
+        try {
+          const proc = Bun.spawn(["sh", "-c", command], {
+            cwd: this.config.workingDirectory,
+            stdout: "pipe",
+            stderr: "pipe",
+          })
+          const exitCode = await proc.exited
+
+          if (exitCode !== 0) {
+            const stderr = await new Response(proc.stderr).text()
+            console.log(`[SearchFirst] Validation FAILED: Command "${command}" exited with ${exitCode}`)
+            return {
+              passed: false,
+              reason: `Validation command failed: ${command}\nExit code: ${exitCode}\n${stderr.substring(0, 200)}`,
+            }
+          }
+          console.log(`[SearchFirst] ✓ Command passed: ${command}`)
+        } catch (e) {
+          const error = e instanceof Error ? e.message : String(e)
+          console.log(`[SearchFirst] Validation FAILED: Command error: ${error}`)
+          return { passed: false, reason: `Validation command error: ${error}` }
+        }
+      }
+    }
+
+    console.log(`[SearchFirst] ✓ All validations passed for step ${step.id}`)
+    return { passed: true }
+  }
+
+  /**
+   * Report validation failure to backend to correct metrics
+   *
+   * When an activity reports "completed" but our validation fails,
+   * we need to notify the backend so it can update Thompson Sampling
+   * parameters (increment beta to reduce the activity's score).
+   */
+  private async reportValidationFailure(
+    templateId: string,
+    step: GoalStep,
+    reason?: string
+  ): Promise<void> {
+    const mcp = getMCPClient()
+    if (!mcp) return
+
+    try {
+      console.log(`[SearchFirst] Reporting validation failure for ${templateId}`)
+
+      // Use the executions endpoint to record the failure
+      // This will increment beta in Thompson Sampling
+      const endpoint = (mcp as any).endpoint
+      const response = await fetch(`${endpoint}/v2/activities/executions`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          variant_id: templateId,
+          success: false,  // This increments beta in Thompson Sampling
+          duration_ms: 0,
+          cost: 0,
+          tokens: { input: 0, output: 0, cache: 0 },
+          error_message: `Validation failed: ${reason ?? "unknown"}`,
+          error_type: "validation_failure",
+          failed_task_id: step.id,
+        }),
+      })
+
+      if (response.ok) {
+        console.log(`[SearchFirst] ✓ Validation failure reported for ${templateId}`)
+      } else {
+        const text = await response.text()
+        console.warn(`[SearchFirst] Failed to report validation failure: ${response.status} - ${text}`)
+      }
+    } catch (e) {
+      console.error("[SearchFirst] Error reporting validation failure:", e)
+    }
+  }
+
+  /**
+   * Execute goal using search-first approach
+   */
+  async execute(
+    goal: string,
+    context?: Record<string, unknown>
+  ): Promise<{
+    goal: string
+    steps: GoalStep[]
+    results: StepResult[]
+    completed: boolean
+    summary: string
+  }> {
+    console.log(`[SearchFirst] Starting goal: ${goal}`)
+
+    // Step 1: Decompose goal into steps
+    const steps = await this.decomposeGoal(goal, context)
+    console.log(`[SearchFirst] Decomposed into ${steps.length} steps`)
+
+    const results: StepResult[] = []
+
+    // Step 2: Execute each step
+    for (const step of steps) {
+      console.log(`\n[SearchFirst] === Step ${step.id}: ${step.description} ===`)
+
+      // Check dependencies
+      const unmetDeps = step.dependencies.filter(
+        depId => !results.find(r => r.stepId === depId && r.status === "completed")
+      )
+      if (unmetDeps.length > 0) {
+        console.log(`[SearchFirst] Skipping - unmet dependencies: ${unmetDeps.join(", ")}`)
+        results.push({
+          stepId: step.id,
+          status: "skipped",
+          method: "skipped",
+          summary: `Skipped due to unmet dependencies: ${unmetDeps.join(", ")}`,
+        })
+        continue
+      }
+
+      // Search for existing activity
+      const searchResult = await this.searchForActivity(step)
+
+      let stepResult: StepResult
+
+      if (searchResult.found && searchResult.templateId) {
+        // Execute via existing activity
+        stepResult = await this.executeViaActivity(step, searchResult.templateId, results)
+      } else {
+        // Execute directly
+        stepResult = await this.executeDirect(step, results)
+      }
+
+      results.push(stepResult)
+      console.log(`[SearchFirst] Step ${step.id} ${stepResult.status}: ${stepResult.summary}`)
+
+      // Stop on failure (could be configurable)
+      if (stepResult.status === "failed") {
+        console.log(`[SearchFirst] Stopping due to step failure`)
+        break
+      }
+    }
+
+    // Generate final summary
+    const completedSteps = results.filter(r => r.status === "completed")
+    const completed = completedSteps.length === steps.length
+
+    const summary = completed
+      ? `Goal completed in ${steps.length} steps: ${completedSteps.map(r => r.summary).join("; ")}`
+      : `Goal partially completed (${completedSteps.length}/${steps.length} steps)`
+
+    console.log(`\n[SearchFirst] === Final: ${summary} ===`)
+
+    return {
+      goal,
+      steps,
+      results,
+      completed,
+      summary,
+    }
+  }
+}