@geekbeer/minion 2.32.0 → 2.42.3

package/core/lib/step-poller.js

@@ -0,0 +1,222 @@
+/**
+ * Step Poller
+ *
+ * Polling daemon that runs on every minion (including PM).
+ * Periodically checks HQ for pending workflow steps assigned to this
+ * minion's role, then fetches the skill and executes it.
+ *
+ * This enables the Pull model: minions autonomously pick up work
+ * when their turn comes, without needing a PM to push-dispatch.
+ * Handles minion absence gracefully — when a minion comes online,
+ * it simply picks up any pending steps waiting for its role.
+ *
+ * Flow per poll cycle:
+ * 1. GET /api/minion/pending-steps → list of actionable steps
+ * 2. For each step (one at a time):
+ *    a. POST /api/skills/fetch/:name → deploy skill locally
+ *    b. POST /api/skills/run → execute in tmux session
+ *    c. (step-complete is reported by the /api/skills/run post-execution hook)
+ */
+
+const { config, isHqConfigured } = require('../config')
+const api = require('../api')
+
+// Polling interval: 30 seconds (matches heartbeat frequency)
+const POLL_INTERVAL_MS = 30_000
+
+// Prevent concurrent poll cycles from overlapping
+let polling = false
+
+// Timer reference for cleanup
+let pollTimer = null
+
+// Track currently executing step to avoid double-dispatch
+let activeStepExecutionId = null
+
+/**
+ * Poll HQ for pending steps and execute them.
+ */
+async function pollOnce() {
+  if (!isHqConfigured()) return
+  if (polling) return
+
+  polling = true
+  try {
+    // 1. Fetch pending steps from HQ
+    const data = await api.request('/pending-steps')
+
+    if (!data.steps || data.steps.length === 0) {
+      return
+    }
+
+    console.log(`[StepPoller] Found ${data.steps.length} pending step(s)`)
+
+    // 2. Process steps one at a time (sequential execution)
+    for (const step of data.steps) {
+      // Skip if we're already executing this step
+      if (activeStepExecutionId === step.step_execution_id) {
+        console.log(`[StepPoller] Step ${step.step_execution_id} already in progress, skipping`)
+        continue
+      }
+
+      await executeStep(step)
+
+      // Only execute one step per poll cycle to avoid overloading
+      break
+    }
+  } catch (err) {
+    // Don't log network errors at error level — they're expected when HQ is temporarily unreachable
+    if (err.message?.includes('fetch failed') || err.message?.includes('ECONNREFUSED')) {
+      console.log(`[StepPoller] HQ unreachable, will retry next cycle`)
+    } else {
+      console.error(`[StepPoller] Poll error: ${err.message}`)
+    }
+  } finally {
+    polling = false
+  }
+}
+
+/**
+ * Execute a single pending step:
+ * 1. Claim the step by calling dispatch-self endpoint
+ * 2. Fetch the skill from HQ
+ * 3. Run the skill locally
+ *
+ * @param {object} step - Step info from pending-steps response
+ */
+async function executeStep(step) {
+  const {
+    step_execution_id,
+    execution_id,
+    workflow_name,
+    step_index,
+    skill_version_id,
+    assigned_role,
+    skill_name,
+    revision_feedback,
+  } = step
+
+  console.log(
+    `[StepPoller] Executing step ${step_index} of "${workflow_name}" ` +
+    `(skill: ${skill_name || skill_version_id}, role: ${assigned_role})`
+  )
+
+  activeStepExecutionId = step_execution_id
+
+  try {
+    // 1. Claim the step — tell HQ we're taking it
+    //    This sets status to 'running' and prevents other minions from picking it up
+    try {
+      await api.request('/claim-step', {
+        method: 'POST',
+        body: JSON.stringify({
+          execution_id,
+          step_index,
+        }),
+      })
+    } catch (claimErr) {
+      // 409 means step is no longer pending (already claimed or completed)
+      if (claimErr.statusCode === 409) {
+        console.log(`[StepPoller] Step ${step_index} already claimed, skipping`)
+        return
+      }
+      throw claimErr
+    }
+
+    // 2. Fetch the skill from HQ to ensure it's deployed locally
+    if (skill_name) {
+      try {
+        const fetchUrl = `http://localhost:${config.AGENT_PORT || 8080}/api/skills/fetch/${encodeURIComponent(skill_name)}`
+        const fetchResp = await fetch(fetchUrl, {
+          method: 'POST',
+          headers: { 'Authorization': `Bearer ${config.API_TOKEN}` },
+        })
+        if (!fetchResp.ok) {
+          console.error(`[StepPoller] Skill fetch failed: ${await fetchResp.text()}`)
+        }
+      } catch (fetchErr) {
+        console.error(`[StepPoller] Skill fetch error: ${fetchErr.message}`)
+        // Continue — skill may already be deployed locally
+      }
+    }
+
+    // 3. Run the skill via local API
+    const runPayload = {
+      skill_name,
+      execution_id,
+      step_index,
+      workflow_name,
+      role: assigned_role,
+    }
+    if (revision_feedback) {
+      runPayload.revision_feedback = revision_feedback
+    }
+
+    const runUrl = `http://localhost:${config.AGENT_PORT || 8080}/api/skills/run`
+    const runResp = await fetch(runUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${config.API_TOKEN}`,
+      },
+      body: JSON.stringify(runPayload),
+    })
+
+    if (!runResp.ok) {
+      const errData = await runResp.json().catch(() => ({}))
+      console.error(`[StepPoller] Skill run failed: ${errData.error || runResp.status}`)
+      // Report failure to HQ
+      try {
+        await api.reportStepComplete({
+          workflow_execution_id: execution_id,
+          step_index,
+          status: 'failed',
+          output_summary: `Step poller failed to start skill: ${errData.error || 'unknown error'}`,
+        })
+      } catch (reportErr) {
+        console.error(`[StepPoller] Failed to report step failure: ${reportErr.message}`)
+      }
+      return
+    }
+
+    const runData = await runResp.json()
+    console.log(
+      `[StepPoller] Skill "${skill_name}" started (session: ${runData.session_name}). ` +
+      `Step completion will be reported by the post-execution hook.`
+    )
+  } catch (err) {
+    console.error(`[StepPoller] Failed to execute step ${step_index}: ${err.message}`)
+  } finally {
+    activeStepExecutionId = null
+  }
+}
+
+/**
+ * Start the polling daemon.
+ */
+function start() {
+  if (!isHqConfigured()) {
+    console.log('[StepPoller] HQ not configured, step poller disabled')
+    return
+  }
+
+  // Initial poll after a short delay (let server fully start)
+  setTimeout(() => pollOnce(), 5000)
+
+  // Periodic polling
+  pollTimer = setInterval(() => pollOnce(), POLL_INTERVAL_MS)
+  console.log(`[StepPoller] Started (polling every ${POLL_INTERVAL_MS / 1000}s)`)
+}
+
+/**
+ * Stop the polling daemon.
+ */
+function stop() {
+  if (pollTimer) {
+    clearInterval(pollTimer)
+    pollTimer = null
+    console.log('[StepPoller] Stopped')
+  }
+}
+
+module.exports = { start, stop, pollOnce }

package/core/lib/strip-ansi.js

@@ -0,0 +1,18 @@
+/**
+ * Strip ANSI escape sequences from a string.
+ * Handles CSI sequences, OSC sequences, and other control codes
+ * that terminal emulators (node-pty) produce.
+ */
+
+// Match ANSI escape sequences:
+// - CSI: ESC[ ... (params including <>=:;?) ... (final byte)
+// - OSC: ESC] ... ST (string terminator: ESC\ or BEL, or next ESC)
+// - Other ESC sequences: ESC followed by a character
+// - Standalone control characters (except newline, carriage return, tab)
+const ANSI_REGEX = /(?:\x1B\[[0-9;?<>=:]*[A-Za-z~]|\x1B\][^\x07\x1B]*(?:\x07|\x1B\\|\x1B(?=\[|\]))|\x1B[^[\]()][A-Za-z]?|[\x00-\x08\x0B\x0C\x0E-\x1F\x7F])/g
+
+function stripAnsi(str) {
+  return str.replace(ANSI_REGEX, '')
+}
+
+module.exports = { stripAnsi }

package/core/lib/workflow-orchestrator.js

@@ -0,0 +1,382 @@
+/**
+ * Workflow Orchestrator
+ *
+ * Deterministic orchestration engine for multi-minion workflow execution.
+ * Replaces the previous LLM-based orchestration skill with code-driven
+ * step dispatch, status polling, and review gate handling.
+ *
+ * The only LLM call is for revision routing: when a reviewer requests
+ * changes, the orchestrator asks an LLM to decide which pipeline step
+ * to roll back to based on the review comment and pipeline structure.
+ */
+
+const { config } = require('../config')
+const api = require('../api')
+
+// Polling configuration
+const POLL_INTERVAL_MS = 30_000 // 30 seconds
+const STEP_TIMEOUT_MS = 30 * 60 * 1000 // 30 minutes per step
+const MAX_REVISIONS_PER_STEP = 3
+
+/**
+ * Sleep for specified milliseconds
+ * @param {number} ms
+ */
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/**
+ * Run the orchestration loop for a workflow execution.
+ * Dispatches steps sequentially, polls for completion, and handles review gates.
+ *
+ * @param {object} params
+ * @param {string} params.executionId - Workflow execution UUID
+ * @param {Array<{step_index: number, step_execution_id: string, skill_version_id: string, skill_name: string, assigned_role: string, requires_review: boolean}>} params.steps
+ * @param {string} params.hqUrl - HQ server URL
+ * @param {string} [params.revisionPolicy] - Optional PM revision policy text
+ * @returns {Promise<{success: boolean, error?: string}>}
+ */
+async function orchestrate({ executionId, steps, hqUrl, revisionPolicy }) {
+  // Always use the minion's own HQ_URL — it is guaranteed reachable from this
+  // network context.  The hqUrl passed from the trigger route may be a
+  // public URL that is not resolvable from inside a container.
+  const effectiveHqUrl = config.HQ_URL || hqUrl
+  if (!effectiveHqUrl) {
+    console.error('[Orchestrator] No HQ URL available (config.HQ_URL and hqUrl are both empty)')
+    return { success: false, error: 'No HQ URL configured' }
+  }
+
+  const apiToken = config.API_TOKEN
+  const headers = {
+    'Content-Type': 'application/json',
+    'Authorization': `Bearer ${apiToken}`,
+  }
+
+  console.log(`[Orchestrator] Starting execution ${executionId} with ${steps.length} steps`)
+
+  let currentStepIdx = 0
+
+  while (currentStepIdx < steps.length) {
+    const step = steps[currentStepIdx]
+    console.log(`[Orchestrator] Dispatching step ${step.step_index}: ${step.skill_name} (role: ${step.assigned_role})`)
+
+    // 1. Dispatch step
+    const dispatchResult = await dispatchStep(effectiveHqUrl, headers, executionId, step.step_index)
+    if (!dispatchResult.success) {
+      console.error(`[Orchestrator] Failed to dispatch step ${step.step_index}: ${dispatchResult.error}`)
+      return { success: false, error: `Dispatch failed at step ${step.step_index}: ${dispatchResult.error}` }
+    }
+
+    console.log(`[Orchestrator] Step ${step.step_index} dispatched to ${dispatchResult.minion_name || 'unknown'}`)
+
+    // 2. Poll until step completes
+    const pollResult = await pollStepCompletion(effectiveHqUrl, headers, executionId, step.step_index)
+    if (!pollResult.success) {
+      console.error(`[Orchestrator] Step ${step.step_index} failed: ${pollResult.error}`)
+      return { success: false, error: `Step ${step.step_index} failed: ${pollResult.error}` }
+    }
+
+    console.log(`[Orchestrator] Step ${step.step_index} completed (status: ${pollResult.status})`)
+
+    // 3. Handle review gate if required
+    if (step.requires_review) {
+      const reviewResult = await handleReviewGate(effectiveHqUrl, headers, executionId, step, steps, revisionPolicy)
+
+      if (reviewResult.action === 'continue') {
+        // Approved — proceed to next step
+        console.log(`[Orchestrator] Step ${step.step_index} approved, continuing`)
+      } else if (reviewResult.action === 'abort') {
+        console.log(`[Orchestrator] Step ${step.step_index} rejected, aborting workflow`)
+        return { success: false, error: `Workflow aborted: step ${step.step_index} rejected by reviewer` }
+      } else if (reviewResult.action === 'retry_from') {
+        // Revision requested — jump back to target step
+        console.log(`[Orchestrator] Revision requested, restarting from step ${reviewResult.targetStepIndex}`)
+        currentStepIdx = reviewResult.targetStepIndex
+        continue
+      } else if (reviewResult.action === 'error') {
+        return { success: false, error: reviewResult.error }
+      }
+    }
+
+    currentStepIdx++
+  }
+
+  console.log(`[Orchestrator] Execution ${executionId} completed successfully`)
+  return { success: true }
+}
+
+/**
+ * Dispatch a single step via HQ API.
+ */
+async function dispatchStep(hqUrl, headers, executionId, stepIndex) {
+  try {
+    const resp = await fetch(`${hqUrl}/api/minion/dispatch-step`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({ execution_id: executionId, step_index: stepIndex }),
+    })
+
+    const data = await resp.json()
+    if (!resp.ok) {
+      return { success: false, error: data.error || `HTTP ${resp.status}` }
+    }
+    return { success: true, minion_name: data.minion_name }
+  } catch (err) {
+    return { success: false, error: err.message }
+  }
+}
+
+/**
+ * Poll HQ for step completion. Returns when the step reaches a terminal state.
+ */
+async function pollStepCompletion(hqUrl, headers, executionId, stepIndex) {
+  const startTime = Date.now()
+
+  while (Date.now() - startTime < STEP_TIMEOUT_MS) {
+    try {
+      const resp = await fetch(`${hqUrl}/api/minion/execution/${executionId}/status`, {
+        headers,
+      })
+
+      if (!resp.ok) {
+        console.error(`[Orchestrator] Status poll failed: HTTP ${resp.status}`)
+        await sleep(POLL_INTERVAL_MS)
+        continue
+      }
+
+      const data = await resp.json()
+      const stepData = data.steps?.find((s) => s.step_index === stepIndex)
+
+      if (!stepData) {
+        console.error(`[Orchestrator] Step ${stepIndex} not found in status response`)
+        await sleep(POLL_INTERVAL_MS)
+        continue
+      }
+
+      if (stepData.status === 'completed') {
+        return { success: true, status: 'completed' }
+      }
+
+      if (stepData.status === 'failed') {
+        return { success: false, error: 'Step execution failed' }
+      }
+
+      // Still running or pending
+      await sleep(POLL_INTERVAL_MS)
+    } catch (err) {
+      console.error(`[Orchestrator] Poll error: ${err.message}`)
+      await sleep(POLL_INTERVAL_MS)
+    }
+  }
+
+  return { success: false, error: `Step ${stepIndex} timed out after ${STEP_TIMEOUT_MS / 60000} minutes` }
+}
+
+/**
+ * Handle review gate: poll for review decision and act accordingly.
+ * Reviewer can approve, reject, or request revisions.
+ *
+ * @returns {{ action: 'continue' | 'abort' | 'retry_from' | 'error', targetStepIndex?: number, error?: string }}
+ */
+async function handleReviewGate(hqUrl, headers, executionId, currentStep, allSteps, revisionPolicy) {
+  console.log(`[Orchestrator] Waiting for review on step ${currentStep.step_index}...`)
+
+  // Poll for review decision (no timeout — waits indefinitely for human input)
+  // In practice, the orchestrate caller or HQ can cancel the execution externally.
+  while (true) {
+    try {
+      const resp = await fetch(`${hqUrl}/api/minion/execution/${executionId}/status`, {
+        headers,
+      })
+
+      if (!resp.ok) {
+        await sleep(POLL_INTERVAL_MS)
+        continue
+      }
+
+      const data = await resp.json()
+      const stepData = data.steps?.find((s) => s.step_index === currentStep.step_index)
+
+      if (!stepData || !stepData.review_status) {
+        await sleep(POLL_INTERVAL_MS)
+        continue
+      }
+
+      if (stepData.review_status === 'approved') {
+        return { action: 'continue' }
+      }
+
+      if (stepData.review_status === 'rejected') {
+        return { action: 'abort' }
+      }
+
+      if (stepData.review_status === 'revision_requested') {
+        const reviewComment = stepData.review_comment || ''
+        console.log(`[Orchestrator] Revision requested: "${reviewComment}"`)
+
+        // Decide which step to roll back to
+        const targetStepIndex = await decideRevisionTarget(
+          allSteps,
+          reviewComment,
+          currentStep.step_index,
+          revisionPolicy
+        )
+
+        // Call revision-reset API
+        const resetResult = await resetForRevision(
+          hqUrl, headers, executionId,
+          targetStepIndex, currentStep.step_index, reviewComment
+        )
+
+        if (!resetResult.success) {
+          return { action: 'error', error: `Revision reset failed: ${resetResult.error}` }
+        }
+
+        return { action: 'retry_from', targetStepIndex }
+      }
+
+      await sleep(POLL_INTERVAL_MS)
+    } catch (err) {
+      console.error(`[Orchestrator] Review poll error: ${err.message}`)
+      await sleep(POLL_INTERVAL_MS)
+    }
+  }
+}
+
+/**
+ * Decide which step to roll back to when a reviewer requests revisions.
+ *
+ * Uses LLM to analyze the review comment and pipeline structure.
+ * Falls back to current step index if LLM call fails or returns invalid data.
+ *
+ * @param {Array} steps - All pipeline steps
+ * @param {string} reviewComment - Reviewer's feedback
+ * @param {number} currentStepIndex - The step that was reviewed
+ * @param {string} [revisionPolicy] - Optional PM-specific revision policy
+ * @returns {Promise<number>} Target step index to roll back to
+ */
+async function decideRevisionTarget(steps, reviewComment, currentStepIndex, revisionPolicy) {
+  // If only one step or current is first, no choice needed
+  if (currentStepIndex === 0 || steps.length <= 1) {
+    return currentStepIndex
+  }
+
+  // Build pipeline description for the LLM
+  const pipelineDesc = steps
+    .filter((s) => s.step_index <= currentStepIndex)
+    .map((s) => `Step ${s.step_index}: ${s.skill_name} (role: ${s.assigned_role})`)
+    .join('\n')
+
+  const systemPrompt = `You are analyzing a workflow pipeline to decide which step to roll back to after a reviewer requested changes.
+
+Given the pipeline steps and the reviewer's feedback, determine which step is the root cause that needs to be re-executed.
+- If the feedback targets the current step's output only, return the current step index.
+- If the feedback suggests an earlier step produced incorrect input, return that earlier step's index.
+- Always return the EARLIEST step that needs re-execution.
+
+Respond with ONLY a JSON object: {"target_step_index": <number>}
+Do not include any other text.`
+
+  const userPrompt = `## Pipeline (steps 0 through ${currentStepIndex})
+${pipelineDesc}
+
+## Reviewer Feedback
+${reviewComment}
+
+## Current Step (reviewed)
+Step ${currentStepIndex}
+${revisionPolicy ? `\n## PM Revision Policy\n${revisionPolicy}` : ''}`
+
+  try {
+    const result = await callLlmForJson(systemPrompt, userPrompt)
+
+    if (
+      result &&
+      typeof result.target_step_index === 'number' &&
+      Number.isInteger(result.target_step_index) &&
+      result.target_step_index >= 0 &&
+      result.target_step_index <= currentStepIndex
+    ) {
+      console.log(`[Orchestrator] LLM decided revision target: step ${result.target_step_index}`)
+      return result.target_step_index
+    }
+
+    console.warn(`[Orchestrator] LLM returned invalid target, falling back to current step ${currentStepIndex}`)
+    return currentStepIndex
+  } catch (err) {
+    console.error(`[Orchestrator] LLM call failed, falling back to current step: ${err.message}`)
+    return currentStepIndex
+  }
+}
+
+/**
+ * Call an LLM API to get a JSON response.
+ * Uses the Anthropic Messages API with ANTHROPIC_API_KEY from environment.
+ *
+ * @param {string} systemPrompt
+ * @param {string} userPrompt
+ * @returns {Promise<object>} Parsed JSON response
+ */
+async function callLlmForJson(systemPrompt, userPrompt) {
+  const apiKey = process.env.ANTHROPIC_API_KEY
+  if (!apiKey) {
+    throw new Error('ANTHROPIC_API_KEY not set — cannot make LLM call for revision routing')
+  }
+
+  const resp = await fetch('https://api.anthropic.com/v1/messages', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'x-api-key': apiKey,
+      'anthropic-version': '2023-06-01',
+    },
+    body: JSON.stringify({
+      model: 'claude-haiku-4-5-20251001',
+      max_tokens: 256,
+      system: systemPrompt,
+      messages: [{ role: 'user', content: userPrompt }],
+    }),
+  })
+
+  if (!resp.ok) {
+    const text = await resp.text()
+    throw new Error(`Anthropic API error: ${resp.status} ${text}`)
+  }
+
+  const data = await resp.json()
+  const content = data.content?.[0]?.text
+  if (!content) {
+    throw new Error('Empty response from Anthropic API')
+  }
+
+  return JSON.parse(content)
+}
+
+/**
+ * Call the revision-reset API on HQ to reset steps for re-execution.
+ */
+async function resetForRevision(hqUrl, headers, executionId, targetStepIndex, revisionStepIndex, revisionFeedback) {
+  try {
+    const resp = await fetch(`${hqUrl}/api/minion/revision-reset`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({
+        execution_id: executionId,
+        target_step_index: targetStepIndex,
+        revision_step_index: revisionStepIndex,
+        revision_feedback: revisionFeedback,
+      }),
+    })
+
+    const data = await resp.json()
+    if (!resp.ok) {
+      return { success: false, error: data.error || `HTTP ${resp.status}` }
+    }
+    return { success: true }
+  } catch (err) {
+    return { success: false, error: err.message }
+  }
+}
+
+module.exports = { orchestrate }