@geekbeer/minion 3.16.1 → 3.22.0

package/core/lib/dag-step-poller.js

@@ -7,8 +7,6 @@
  * - Supports concurrent node execution (up to MAX_CONCURRENT)
  * - Reports completion to /api/dag/minion/node-complete
  * - Injects input_data into skill context
- *
- * Feature flag: only starts when DAG_ENGINE_ENABLED=true
  */
 
 const { config, isHqConfigured } = require('../config')
@@ -98,13 +96,24 @@ async function pollOnce() {
 }
 
 /**
- * Execute a single pending DAG node:
+ * Execute a single pending DAG node.
+ * Routes to skill or transform execution based on node_type.
+ */
+async function executeNode(node) {
+  if (node.node_type === 'transform') {
+    return executeTransformNode(node)
+  }
+  return executeSkillNode(node)
+}
+
+/**
+ * Execute a skill node:
  * 1. Claim the node
  * 2. Fetch the skill from HQ
  * 3. Run the skill locally (with input_data injected)
  * 4. Report completion with output_data
  */
-async function executeNode(node) {
+async function executeSkillNode(node) {
   const {
     node_execution_id,
     execution_id,
@@ -119,7 +128,7 @@ async function executeNode(node) {
   } = node
 
   console.log(
-    `[DagPoller] Executing node "${node_id}" of DAG "${dag_workflow_name}" ` +
+    `[DagPoller] Executing skill node "${node_id}" of DAG "${dag_workflow_name}" ` +
     `(skill: ${resolvedSkillName || skill_version_id}, scope: "${scope_path || 'root'}", role: ${assigned_role})`
   )
 
@@ -212,7 +221,7 @@ async function executeNode(node) {
     // This will be handled by dag-node-executor.js which watches the session.
 
   } catch (err) {
-    console.error(`[DagPoller] Failed to execute node ${node_id}: ${err.message}`)
+    console.error(`[DagPoller] Failed to execute skill node ${node_id}: ${err.message}`)
     try {
       await reportNodeComplete(node_execution_id, 'failed', null, err.message)
     } catch {
@@ -221,6 +230,149 @@ async function executeNode(node) {
   }
 }
 
+/**
+ * Execute a transform node:
+ * 1. Claim the node
+ * 2. Create an ephemeral skill from the transform_instruction
+ * 3. Run the ephemeral skill via existing /api/skills/run
+ * 4. Clean up ephemeral skill directory
+ */
+async function executeTransformNode(node) {
+  const {
+    node_execution_id,
+    execution_id,
+    dag_workflow_name,
+    node_id,
+    scope_path,
+    assigned_role,
+    input_data,
+    transform_instruction,
+  } = node
+
+  console.log(
+    `[DagPoller] Executing transform node "${node_id}" of DAG "${dag_workflow_name}" ` +
+    `(scope: "${scope_path || 'root'}", role: ${assigned_role})`
+  )
+
+  const ephemeralName = `_dag_transform_${node_execution_id.substring(0, 8)}`
+  const fs = require('fs').promises
+  const path = require('path')
+  const { getActiveSkillDirs } = require('../llm-plugins/lib/skill-dirs')
+  const skillRoots = getActiveSkillDirs()
+  // Write the ephemeral skill to every active plugin's dir so any Primary
+  // can find it. Cleanup at the end sweeps the same set.
+  const ephemeralSkillDirs = skillRoots.map(root => path.join(root, ephemeralName))
+  const skillDir = ephemeralSkillDirs[0] // canonical for downstream paths
+
+  try {
+    // 1. Claim the node
+    try {
+      await dagRequest('/claim-node', {
+        method: 'POST',
+        body: JSON.stringify({ node_execution_id }),
+      })
+    } catch (claimErr) {
+      if (claimErr.statusCode === 409) {
+        console.log(`[DagPoller] Transform node ${node_id} already claimed, skipping`)
+        return
+      }
+      throw claimErr
+    }
+
+    // 2. Create ephemeral skill from transform_instruction.
+    // Write to every active plugin's skill dir so any Primary can find it.
+    const skillContent = buildTransformSkillContent(transform_instruction, input_data)
+    for (const dir of ephemeralSkillDirs) {
+      await fs.mkdir(dir, { recursive: true })
+      await fs.writeFile(path.join(dir, 'SKILL.md'), skillContent, 'utf-8')
+    }
+
+    // 3. Run the ephemeral skill
+    const runPayload = {
+      skill_name: ephemeralName,
+      execution_id,
+      workflow_name: dag_workflow_name,
+      role: assigned_role,
+      dag_node_id: node_id,
+      dag_input_data: input_data,
+      dag_node_execution_id: node_execution_id,
+    }
+
+    const runUrl = `http://localhost:${config.AGENT_PORT || 8080}/api/skills/run`
+    const runResp = await fetch(runUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${config.API_TOKEN}`,
+      },
+      body: JSON.stringify(runPayload),
+    })
+
+    if (!runResp.ok) {
+      const errData = await runResp.json().catch(() => ({}))
+      console.error(`[DagPoller] Transform run failed: ${errData.error || runResp.status}`)
+      await reportNodeComplete(
+        node_execution_id,
+        'failed',
+        null,
+        `Failed to start transform: ${errData.error || 'unknown error'}`
+      )
+      return
+    }
+
+    const runData = await runResp.json()
+    console.log(
+      `[DagPoller] Transform started for node "${node_id}" ` +
+      `(session: ${runData.session_name}). Completion reported by post-execution hook.`
+    )
+
+  } catch (err) {
+    console.error(`[DagPoller] Failed to execute transform node ${node_id}: ${err.message}`)
+    // Best-effort cleanup on synchronous failure (before skill hand-off).
+    // On the success path, cleanup is performed by the post-execution hook
+    // in skills.js once the session actually finishes.
+    for (const dir of ephemeralSkillDirs) {
+      try {
+        await fs.rm(dir, { recursive: true, force: true })
+      } catch {
+        // best-effort
+      }
+    }
+    try {
+      await reportNodeComplete(node_execution_id, 'failed', null, err.message)
+    } catch {
+      // best-effort
+    }
+  }
+}
+
+/**
+ * Build SKILL.md content for a transform node's ephemeral skill.
+ */
+function buildTransformSkillContent(instruction, inputData) {
+  return [
+    '---',
+    'name: dag-transform',
+    'description: DAG Transform Node',
+    '---',
+    '',
+    'You are a data transformation step in a DAG workflow.',
+    '',
+    '## Input Data',
+    '```json',
+    JSON.stringify(inputData, null, 2),
+    '```',
+    '',
+    '## Transform Instruction',
+    instruction,
+    '',
+    '## Task',
+    'Apply the transform instruction to the input data above.',
+    'Output the result as a JSON object in an "## Output Data" section with a json code block.',
+    'Do NOT output anything other than the Output Data section.',
+  ].join('\n')
+}
+
 /**
  * Resolve skill_version_id to skill name via HQ API.
  */

package/core/lib/llm-checker.js

@@ -69,8 +69,10 @@ function isClaudeAuthenticated() {
 function isGeminiAuthenticated() {
   if (process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY) return true
 
+  // Official @google/gemini-cli stores OAuth credentials at ~/.gemini/oauth_creds.json
+  // after `gemini` login. Fall back to ADC locations for gcloud-based setups.
   const possiblePaths = [
-    path.join(config.HOME_DIR, '.config', 'gemini'),
+    path.join(config.HOME_DIR, '.gemini', 'oauth_creds.json'),
     path.join(config.HOME_DIR, '.config', 'gcloud', 'application_default_credentials.json'),
     // Windows-specific locations
     ...(IS_WINDOWS ? [
@@ -80,12 +82,7 @@ function isGeminiAuthenticated() {
   for (const p of possiblePaths) {
     try {
       if (!fs.existsSync(p)) continue
-      const stat = fs.statSync(p)
-      if (stat.isDirectory()) {
-        if (fs.readdirSync(p).length > 0) return true
-      } else {
-        if (fs.readFileSync(p, 'utf-8').trim().length > 0) return true
-      }
+      if (fs.readFileSync(p, 'utf-8').trim().length > 0) return true
     } catch {
       // Ignore
     }

package/core/lib/template-expander.js

@@ -44,21 +44,26 @@ async function expandSkillTemplates(skillNames) {
   // If no variables defined, skip
   if (Object.keys(minionVars).length === 0) return new Map()
 
+  // originals key: `${root}::${name}` so we restore the right copy in each
+  // plugin's skill dir (multi-plugin distribution).
   const originals = new Map()
-  const skillsDir = path.join(config.HOME_DIR, '.claude', 'skills')
+  const { getActiveSkillDirs } = require('../llm-plugins/lib/skill-dirs')
+  const roots = getActiveSkillDirs()
 
-  for (const name of skillNames) {
-    const skillMdPath = path.join(skillsDir, name, 'SKILL.md')
-    try {
-      const original = await fs.readFile(skillMdPath, 'utf-8')
-      const expanded = expandTemplateVars(original, minionVars)
-      if (expanded !== original) {
-        originals.set(name, original)
-        await fs.writeFile(skillMdPath, expanded, 'utf-8')
-        console.log(`[TemplateExpander] Expanded {{VAR}} in skill: ${name}`)
+  for (const root of roots) {
+    for (const name of skillNames) {
+      const skillMdPath = path.join(root, name, 'SKILL.md')
+      try {
+        const original = await fs.readFile(skillMdPath, 'utf-8')
+        const expanded = expandTemplateVars(original, minionVars)
+        if (expanded !== original) {
+          originals.set(`${root}::${name}`, { skillMdPath, original })
+          await fs.writeFile(skillMdPath, expanded, 'utf-8')
+          console.log(`[TemplateExpander] Expanded {{VAR}} in skill: ${name} (${root})`)
+        }
+      } catch {
+        // Skill missing in this dir — skip
       }
-    } catch (err) {
-      console.error(`[TemplateExpander] Failed to expand ${name}: ${err.message}`)
     }
   }
 
@@ -68,16 +73,14 @@ async function expandSkillTemplates(skillNames) {
 /**
  * Restore original SKILL.md contents after execution.
  *
- * @param {Map<string, string>} originals - Map from expandSkillTemplates
+ * @param {Map<string, {skillMdPath: string, original: string}>} originals - Map from expandSkillTemplates
  */
 async function restoreSkillTemplates(originals) {
-  const skillsDir = path.join(config.HOME_DIR, '.claude', 'skills')
-  for (const [name, content] of originals) {
-    const skillMdPath = path.join(skillsDir, name, 'SKILL.md')
+  for (const { skillMdPath, original } of originals.values()) {
     try {
-      await fs.writeFile(skillMdPath, content, 'utf-8')
+      await fs.writeFile(skillMdPath, original, 'utf-8')
     } catch (err) {
-      console.error(`[TemplateExpander] Failed to restore ${name}: ${err.message}`)
+      console.error(`[TemplateExpander] Failed to restore ${skillMdPath}: ${err.message}`)
     }
   }
 }

package/core/llm-dispatch/mcp-server.js

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+/**
+ * Dispatch MCP server (stdio).
+ *
+ * Primary LLM clients (Claude Code, Codex, etc.) launch this as a child
+ * process via ~/.mcp.json. It exposes one MCP tool per enabled child plugin:
+ *
+ *   dispatch_to_<plugin>(prompt, opts)
+ *
+ * When invoked, the server acquires a slot from the child session pool,
+ * calls plugin.invoke() (or plugin.resume() when a prior slot is reused),
+ * and returns the standardized LlmOutput as an MCP tool result.
+ *
+ * Parallel tool calls from the Primary are supported: each concurrent call
+ * gets its own slot. The server's lifetime equals the Primary's lifetime
+ * (when the Primary exits, the MCP transport closes and this process dies,
+ * disposing the pool).
+ *
+ * A parent_session_id is generated on startup (random UUID). It represents
+ * "this Primary session" and scopes the pool.
+ *
+ * Protocol: JSON-RPC 2.0 over stdio, one message per line.
+ */
+
+const readline = require('readline')
+const crypto = require('crypto')
+const { loadAllPlugins, readConfig } = require('../llm-plugins/registry')
+const { SessionPool } = require('./session-pool')
+
+const PARENT_SESSION_ID = crypto.randomUUID()
+const pool = new SessionPool()
+
+const plugins = loadAllPlugins()
+const cfg = readConfig()
+
+/** Enabled child plugins (exclude the primary itself to avoid self-dispatch loops). */
+function enabledChildren() {
+  const enabled = new Set(cfg.enabled)
+  const out = []
+  for (const p of plugins.values()) {
+    if (!enabled.has(p.name)) continue
+    if (p.name === cfg.primary) continue
+    out.push(p)
+  }
+  return out
+}
+
+function toolNameFor(pluginName) {
+  return `dispatch_to_${pluginName}`
+}
+
+function toolsList() {
+  return enabledChildren().map(p => ({
+    name: toolNameFor(p.name),
+    description: `Dispatch a prompt to ${p.displayName}. Returns standardized LlmOutput { text, files, metadata }.`,
+    inputSchema: {
+      type: 'object',
+      properties: {
+        prompt: { type: 'string', description: 'Prompt text to send to the child LLM.' },
+        systemPrompt: { type: 'string', description: 'Optional system prompt.' },
+        model: { type: 'string', description: 'Optional model identifier.' },
+        continue: {
+          type: 'boolean',
+          description: 'Reuse the existing child session if one exists. Default true.',
+          default: true,
+        },
+      },
+      required: ['prompt'],
+    },
+  }))
+}
+
+async function handleToolCall(name, args) {
+  const children = enabledChildren()
+  const plugin = children.find(p => toolNameFor(p.name) === name)
+  if (!plugin) throw new Error(`Unknown tool: ${name}`)
+
+  const input = {
+    prompt: String(args.prompt || ''),
+    systemPrompt: args.systemPrompt,
+    model: args.model,
+  }
+  const continueSession = args.continue !== false
+
+  const forceFresh = !continueSession || !plugin.capabilities.sessionResume
+  const { slot } = pool.acquire(PARENT_SESSION_ID, plugin.name, { forceFresh })
+
+  let output
+  try {
+    if (slot.sessionId && continueSession && typeof plugin.resume === 'function') {
+      output = await plugin.resume(slot.sessionId, input)
+    } else {
+      output = await plugin.invoke(input)
+    }
+  } catch (err) {
+    pool.drop(PARENT_SESSION_ID, slot)
+    throw err
+  }
+
+  pool.release(slot, output?.metadata?.sessionId)
+  return output
+}
+
+// --- JSON-RPC transport ------------------------------------------------------
+
+function send(msg) {
+  process.stdout.write(JSON.stringify(msg) + '\n')
+}
+
+function sendResult(id, result) {
+  send({ jsonrpc: '2.0', id, result })
+}
+
+function sendError(id, code, message) {
+  send({ jsonrpc: '2.0', id, error: { code, message } })
+}
+
+async function dispatch(msg) {
+  const { id, method, params } = msg
+  try {
+    switch (method) {
+      case 'initialize':
+        sendResult(id, {
+          protocolVersion: '2024-11-05',
+          serverInfo: { name: 'minion-llm-dispatch', version: '1.0.0' },
+          capabilities: { tools: {} },
+        })
+        return
+
+      case 'notifications/initialized':
+        // Notification: no response
+        return
+
+      case 'tools/list':
+        sendResult(id, { tools: toolsList() })
+        return
+
+      case 'tools/call': {
+        const { name, arguments: args } = params || {}
+        const output = await handleToolCall(name, args || {})
+        sendResult(id, {
+          content: [{ type: 'text', text: JSON.stringify(output) }],
+          isError: !!output?.error,
+        })
+        return
+      }
+
+      case 'ping':
+        sendResult(id, {})
+        return
+
+      default:
+        if (id !== undefined) sendError(id, -32601, `Method not found: ${method}`)
+        return
+    }
+  } catch (err) {
+    if (id !== undefined) sendError(id, -32000, err.message || String(err))
+  }
+}
+
+function start() {
+  const rl = readline.createInterface({ input: process.stdin, terminal: false })
+  rl.on('line', line => {
+    const trimmed = line.trim()
+    if (!trimmed) return
+    let msg
+    try {
+      msg = JSON.parse(trimmed)
+    } catch {
+      // Ignore malformed lines
+      return
+    }
+    dispatch(msg)
+  })
+  rl.on('close', () => {
+    pool.disposeParent(PARENT_SESSION_ID)
+    process.exit(0)
+  })
+}
+
+if (require.main === module) {
+  start()
+}
+
+module.exports = { start, toolsList, handleToolCall, pool, PARENT_SESSION_ID }

package/core/llm-dispatch/session-pool.js

@@ -0,0 +1,97 @@
+/**
+ * Child session pool for dispatch.
+ *
+ * Keyed by (parentSessionId, pluginName). When a dispatch arrives while the
+ * existing pool entry is in-flight (parallel dispatch), a new slot is created
+ * so that the concurrent calls use independent child sessions.
+ *
+ * Parent termination disposes all slots at once.
+ */
+
+/**
+ * @typedef {Object} Slot
+ * @property {string} pluginName
+ * @property {string|null} sessionId     plugin-reported sessionId (for resume)
+ * @property {boolean} inFlight
+ */
+
+class SessionPool {
+  constructor() {
+    /** @type {Map<string, Slot[]>} parentSessionId -> slots */
+    this._slots = new Map()
+  }
+
+  _key(parentSessionId) {
+    return String(parentSessionId)
+  }
+
+  /**
+   * Acquire a slot for (parent, plugin). Reuses an idle slot if available,
+   * otherwise creates a new slot. Marks the slot in-flight.
+   * @param {string} parentSessionId
+   * @param {string} pluginName
+   * @param {{ forceFresh?: boolean }} [opts]
+   * @returns {{ slot: Slot, slotIndex: number, slots: Slot[] }}
+   */
+  acquire(parentSessionId, pluginName, opts = {}) {
+    const key = this._key(parentSessionId)
+    let slots = this._slots.get(key)
+    if (!slots) {
+      slots = []
+      this._slots.set(key, slots)
+    }
+
+    if (!opts.forceFresh) {
+      const idle = slots.find(s => s.pluginName === pluginName && !s.inFlight)
+      if (idle) {
+        idle.inFlight = true
+        return { slot: idle, slotIndex: slots.indexOf(idle), slots }
+      }
+    }
+
+    const slot = { pluginName, sessionId: null, inFlight: true }
+    slots.push(slot)
+    return { slot, slotIndex: slots.length - 1, slots }
+  }
+
+  /**
+   * Release a slot after a dispatch completes. Stores sessionId (if returned
+   * by the plugin) for subsequent reuse.
+   */
+  release(slot, sessionId) {
+    slot.inFlight = false
+    if (sessionId) slot.sessionId = sessionId
+  }
+
+  /**
+   * Drop a slot entirely (e.g. plugin reported hard failure or caller asked
+   * to reset the session).
+   */
+  drop(parentSessionId, slot) {
+    const key = this._key(parentSessionId)
+    const slots = this._slots.get(key)
+    if (!slots) return
+    const idx = slots.indexOf(slot)
+    if (idx >= 0) slots.splice(idx, 1)
+  }
+
+  /** Dispose all slots for a parent session (call on parent termination). */
+  disposeParent(parentSessionId) {
+    this._slots.delete(this._key(parentSessionId))
+  }
+
+  /** Inspection helper (for diagnostics). */
+  snapshot() {
+    const out = {}
+    for (const [parent, slots] of this._slots.entries()) {
+      out[parent] = slots.map(s => ({
+        pluginName: s.pluginName,
+        sessionId: s.sessionId,
+        inFlight: s.inFlight,
+      }))
+    }
+    return out
+  }
+}
+
+module.exports = { SessionPool }