agent-state-machine 2.2.2 → 2.4.0

package/bin/cli.js

@@ -99,6 +99,7 @@ Options:
   --new, -n       Generate a new remote follow path
   --full-auto, -a Auto-select first option for choice interactions (no blocking)
   --delay, -d     Seconds to wait before auto-select in full-auto mode (default: 20)
+  --non-verbose, -q  Suppress per-agent token usage display (show only final summary)
   -reset          Reset workflow state before running
   -reset-hard     Hard reset workflow before running
   --help, -h      Show help
@@ -108,7 +109,7 @@ Environment Variables:
   STATE_MACHINE_REMOTE_URL    Override the default remote server URL (for local dev testing)
 
 Workflow Structure:
-  workflows/<name>/
+  .workflows/<name>/
   ├── workflow.js        # Native JS workflow (async/await)
   ├── config.js          # Model/API key configuration
   ├── package.json       # Sets "type": "module" for this workflow folder
@@ -142,7 +143,7 @@ async function confirmHardReset(workflowName) {
 }
 
 function workflowsRoot() {
-  return path.join(process.cwd(), 'workflows');
+  return path.join(process.cwd(), '.workflows');
 }
 
 function resolveWorkflowDir(workflowName) {
@@ -190,11 +191,55 @@ function summarizeStatus(state) {
   return state.status ? ` [${state.status}]` : '';
 }
 
+/**
+ * Display usage summary after workflow completion
+ */
+function displayUsageSummary(runtime) {
+  const u = runtime._usageTotals;
+  if (!u || (!u.totalInputTokens && !u.totalOutputTokens)) return;
+
+  const C = {
+    bold: '\x1b[1m',
+    dim: '\x1b[2m',
+    cyan: '\x1b[36m',
+    reset: '\x1b[0m'
+  };
+
+  const formatTokens = (count) => {
+    if (count >= 1000000) return `${(count / 1000000).toFixed(1)}M`;
+    if (count >= 10000) return `${Math.round(count / 1000)}k`;
+    if (count >= 1000) return `${(count / 1000).toFixed(1)}k`;
+    return count.toString();
+  };
+
+  console.log(`\n${C.bold}Token Usage Summary${C.reset}`);
+  console.log(`${C.dim}${'─'.repeat(40)}${C.reset}`);
+  console.log(`  Input:  ${formatTokens(u.totalInputTokens)}`);
+  console.log(`  Output: ${formatTokens(u.totalOutputTokens)}`);
+  if (u.totalCachedTokens > 0) {
+    console.log(`  Cached: ${formatTokens(u.totalCachedTokens)}`);
+  }
+  console.log(`  ${C.bold}Total:  ${formatTokens(u.totalInputTokens + u.totalOutputTokens)}${C.reset}`);
+  if (u.totalCost > 0) {
+    console.log(`  ${C.cyan}Cost:   $${u.totalCost.toFixed(4)}${C.reset}`);
+  }
+
+  // Show per-model breakdown if multiple models used
+  const models = Object.keys(u.modelUsage || {});
+  if (models.length > 1) {
+    console.log(`\n${C.dim}By Model:${C.reset}`);
+    for (const model of models) {
+      const m = u.modelUsage[model];
+      console.log(`  ${model}: ${formatTokens(m.inputTokens)} in / ${formatTokens(m.outputTokens)} out`);
+    }
+  }
+}
+
 function listWorkflows() {
   const root = workflowsRoot();
 
   if (!fs.existsSync(root)) {
-    console.log('No workflows directory found.');
+    console.log('No .workflows directory found.');
     console.log('Run `state-machine --setup <name>` to create your first workflow.');
     return;
   }
@@ -242,7 +287,8 @@ async function runOrResume(
     preReset = false,
     preResetHard = false,
     fullAuto = false,
-    autoSelectDelay = null
+    autoSelectDelay = null,
+    nonVerbose = false
   } = {}
 ) {
   const workflowDir = resolveWorkflowDir(workflowName);
@@ -308,6 +354,11 @@ async function runOrResume(
     console.log(`\n\x1b[36m\x1b[1m⚡ Full-auto mode enabled\x1b[0m - Agent will auto-select recommended options after ${delay}s countdown`);
   }
 
+  // Set non-verbose mode from CLI flag
+  if (nonVerbose) {
+    runtime.workflowConfig.nonVerbose = true;
+  }
+
   // Prevent system sleep while workflow runs (macOS only)
   // Display can still sleep, but system stays awake for remote follow
   const stopCaffeinate = preventSleep();
@@ -317,6 +368,9 @@ async function runOrResume(
 
   try {
     await runtime.runWorkflow(workflowUrl);
+
+    // Display usage summary after workflow completion
+    displayUsageSummary(runtime);
   } finally {
     // Allow sleep again
     if (stopCaffeinate) {
@@ -385,6 +439,7 @@ async function main() {
         const preReset = args.includes('-reset');
         const preResetHard = args.includes('-reset-hard');
         const fullAuto = args.includes('--full-auto') || args.includes('-a');
+        const nonVerbose = args.includes('--non-verbose') || args.includes('-q') || args.includes('--quiet');
         const remoteEnabled = !useLocalServer; // Use Vercel if not local
 
         // Parse --delay or -d flag
@@ -405,7 +460,8 @@ async function main() {
             preReset,
             preResetHard,
             fullAuto,
-            autoSelectDelay
+            autoSelectDelay,
+            nonVerbose
           });
         } catch (err) {
           console.error('Error:', err.message || String(err));

package/lib/file-tree.js

@@ -21,7 +21,7 @@ export const DEFAULT_IGNORE = [
   'coverage/**',
   '.next/**',
   '.cache/**',
-  'workflows/**',
+  '.workflows/**',
   '*.log',
   '.DS_Store'
 ];

package/lib/llm.js

@@ -12,6 +12,111 @@ import { resolveUnknownModel } from './runtime/model-resolution.js';
 
 const require = createRequire(import.meta.url);
 
+/**
+ * Parse Claude CLI JSON output
+ * @param {string} output - Raw JSON output from claude --output-format json
+ * @returns {{ text: string, model: string|null, usage: object|null }}
+ */
+function parseClaudeOutput(output) {
+  try {
+    const json = JSON.parse(output);
+    const modelUsage = json.modelUsage || {};
+    const modelName = Object.keys(modelUsage)[0] || null;
+
+    const usage = json.usage ? {
+      inputTokens: json.usage.input_tokens || 0,
+      outputTokens: json.usage.output_tokens || 0,
+      cacheReadInputTokens: json.usage.cache_read_input_tokens || 0,
+      cacheCreationInputTokens: json.usage.cache_creation_input_tokens || 0,
+      cost: json.total_cost_usd || null
+    } : null;
+
+    return {
+      text: json.result || output,
+      model: modelName,
+      usage
+    };
+  } catch {
+    return { text: output, model: null, usage: null };
+  }
+}
+
+/**
+ * Parse Gemini CLI JSON output
+ * @param {string} output - Raw JSON output from gemini --output-format json
+ * @returns {{ text: string, model: string|null, usage: object|null }}
+ */
+function parseGeminiOutput(output) {
+  try {
+    const json = JSON.parse(output);
+    const stats = json.stats?.models || {};
+    const modelName = Object.keys(stats)[0] || null;
+    const tokens = modelName ? stats[modelName]?.tokens || {} : {};
+
+    const usage = {
+      inputTokens: tokens.input || tokens.prompt || 0,
+      outputTokens: tokens.candidates || 0,
+      cachedTokens: tokens.cached || 0,
+      thoughtTokens: tokens.thoughts || 0
+    };
+
+    return {
+      text: json.response || output,
+      model: modelName,
+      usage
+    };
+  } catch {
+    return { text: output, model: null, usage: null };
+  }
+}
+
+/**
+ * Parse Codex CLI JSON output (NDJSON format)
+ * @param {string} output - Raw NDJSON output from codex --json
+ * @returns {{ text: string, model: string|null, usage: object|null }}
+ */
+function parseCodexOutput(output) {
+  const lines = output.trim().split('\n');
+  let text = '';
+  let usage = null;
+
+  for (const line of lines) {
+    try {
+      const json = JSON.parse(line);
+      // Extract text from agent_message items
+      if (json.type === 'item.completed' && json.item?.type === 'agent_message') {
+        text = json.item.text || text;
+      }
+      // Extract usage from turn.completed event
+      if (json.type === 'turn.completed' && json.usage) {
+        usage = {
+          inputTokens: json.usage.input_tokens || 0,
+          outputTokens: json.usage.output_tokens || 0,
+          cachedInputTokens: json.usage.cached_input_tokens || 0
+        };
+      }
+    } catch {
+      // Non-JSON line - might be the actual response text
+      if (!text && line.trim()) text = line;
+    }
+  }
+
+  return { text, model: null, usage };
+}
+
+/**
+ * Parse CLI output based on tool type
+ * @param {string} output - Raw CLI output
+ * @param {string} baseCmd - Base command (claude, gemini, codex)
+ * @returns {{ text: string, model: string|null, usage: object|null }}
+ */
+function parseCLIOutput(output, baseCmd) {
+  if (baseCmd === 'claude') return parseClaudeOutput(output);
+  if (baseCmd === 'gemini') return parseGeminiOutput(output);
+  if (baseCmd === 'codex') return parseCodexOutput(output);
+  return { text: output.trim(), model: null, usage: null };
+}
+
 /**
  * LLM Helper Module
  *
@@ -250,17 +355,15 @@ async function executeCLI(command, promptText, options = {}, apiKeys = {}) {
     if (baseCmd === 'claude') {
       args.push('--print');
       args.push('--permission-mode', 'acceptEdits');
+      args.push('--output-format', 'json');
       // Input via stdin
     } else if (baseCmd === 'gemini') {
       args.push('--approval-mode', 'auto_edit');
+      args.push('--output-format', 'json');
       // Input via stdin
     } else if (baseCmd === 'codex') {
       ensureCodexExec();
-      const lastMessageFile = path.join(
-        os.tmpdir(),
-        `codex-last-message-${process.pid}-${Date.now()}.txt`
-      );
-      args.push('--output-last-message', lastMessageFile);
+      args.push('--json');
       args.push('-'); // Explicitly read from stdin
     } else {
       // Generic CLI: Fallback to temp file if not a known stdin consumer
@@ -310,24 +413,23 @@ async function executeCLI(command, promptText, options = {}, apiKeys = {}) {
       }
 
       if (code === 0) {
-        if (baseCmd === 'codex') {
-          const outputFlagIndex = args.findIndex(a => a === '--output-last-message' || a === '-o');
-          const outputFile = outputFlagIndex >= 0 ? args[outputFlagIndex + 1] : null;
-          if (outputFile && fs.existsSync(outputFile)) {
-            try {
-              stdout = fs.readFileSync(outputFile, 'utf-8');
-            } finally {
-              try { fs.unlinkSync(outputFile); } catch {}
-            }
-          }
+        // Parse JSON output for standard CLI tools
+        if (isStandardCLI) {
+          const parsed = parseCLIOutput(stdout, baseCmd);
+          resolve({
+            text: parsed.text,
+            model: parsed.model || command,
+            provider: 'cli',
+            usage: parsed.usage
+          });
+        } else {
+          resolve({
+            text: stdout.trim(),
+            model: command,
+            provider: 'cli',
+            usage: null
+          });
         }
-
-        resolve({
-          text: stdout.trim(),
-          model: command,
-          provider: 'cli',
-          usage: null
-        });
       } else {
         reject(new Error(`CLI command failed (exit ${code}): ${stderr || stdout}`));
       }
@@ -482,6 +584,16 @@ export async function llm(context, options) {
     result = await executeCLI(modelConfig, fullPrompt, options, apiKeys);
   }
 
+  // Record usage in agent tracker (if active)
+  if (result.usage) {
+    try {
+      const { recordLLMUsage } = await import('./runtime/agent.js');
+      recordLLMUsage(result.usage, result.model, result.provider);
+    } catch {
+      // Agent tracking not available (outside agent context)
+    }
+  }
+
   return { ...result, fullPrompt };
 }
 

package/lib/runtime/agent.js

@@ -16,6 +16,74 @@ import { withChangeTracking } from './track-changes.js';
 
 const require = createRequire(import.meta.url);
 
+/**
+ * Token Usage Tracking
+ *
+ * Tracks LLM token usage across all calls within a single agent execution.
+ * The tracker is cleared before each agent runs and aggregated after completion.
+ */
+const AGENT_USAGE_KEY = Symbol.for('agent-state-machine.agent-usage');
+
+function getAgentUsageTracker() {
+  return globalThis[AGENT_USAGE_KEY] || (globalThis[AGENT_USAGE_KEY] = []);
+}
+
+export function clearAgentUsageTracker() {
+  globalThis[AGENT_USAGE_KEY] = [];
+}
+
+/**
+ * Record usage from an LLM call (called from llm.js)
+ */
+export function recordLLMUsage(usage, model, provider) {
+  if (!usage) return;
+  const tracker = getAgentUsageTracker();
+  tracker.push({ usage, model, provider, timestamp: new Date().toISOString() });
+}
+
+/**
+ * Aggregate all recorded usage into a summary
+ */
+export function aggregateUsage() {
+  const tracker = getAgentUsageTracker();
+  if (tracker.length === 0) return null;
+
+  const agg = {
+    inputTokens: 0,
+    outputTokens: 0,
+    cachedTokens: 0,
+    cost: 0,
+    calls: tracker.length,
+    models: {}
+  };
+
+  for (const { usage, model } of tracker) {
+    agg.inputTokens += usage.inputTokens || 0;
+    agg.outputTokens += usage.outputTokens || 0;
+    agg.cachedTokens += usage.cachedTokens || usage.cacheReadInputTokens || usage.cachedInputTokens || 0;
+    if (usage.cost) agg.cost += usage.cost;
+
+    const m = model || 'unknown';
+    if (!agg.models[m]) {
+      agg.models[m] = { inputTokens: 0, outputTokens: 0 };
+    }
+    agg.models[m].inputTokens += usage.inputTokens || 0;
+    agg.models[m].outputTokens += usage.outputTokens || 0;
+  }
+
+  return agg;
+}
+
+/**
+ * Format token count for display
+ */
+function formatTokens(count) {
+  if (count >= 1000000) return `${(count / 1000000).toFixed(1)}M`;
+  if (count >= 10000) return `${Math.round(count / 1000)}k`;
+  if (count >= 1000) return `${(count / 1000).toFixed(1)}k`;
+  return count.toString();
+}
+
 /**
  * Run an agent with context
  * @param {string} name - Agent name (file basename)
@@ -43,20 +111,49 @@ export async function agent(name, params = {}, options = {}) {
         console.log(`  [Agent: ${name}] Starting...`);
       }
 
+      // Clear usage tracker before each attempt
+      clearAgentUsageTracker();
+
       const result = await executeAgent(runtime, name, params, options);
 
       if (result && typeof result === 'object' && result._debug_prompt) {
         delete result._debug_prompt;
       }
 
+      // Aggregate token usage from all LLM calls in this agent
+      const usage = aggregateUsage();
+
       console.log(`  [Agent: ${name}] Completed`);
+
+      // Display token usage (unless non-verbose mode)
+      if (usage && !runtime.workflowConfig?.nonVerbose) {
+        let usageLine = `    Tokens: ${formatTokens(usage.inputTokens)} in / ${formatTokens(usage.outputTokens)} out`;
+        if (usage.cachedTokens > 0) {
+          usageLine += ` (${formatTokens(usage.cachedTokens)} cached)`;
+        }
+        if (usage.cost) {
+          usageLine += ` $${usage.cost.toFixed(4)}`;
+        }
+        console.log(usageLine);
+      }
+
+      // Get primary model from usage
+      const primaryModel = usage?.models ? Object.keys(usage.models)[0] : null;
+
       await runtime.prependHistory({
         event: 'AGENT_COMPLETED',
         agent: name,
         output: result,
-        attempts: attempt + 1
+        attempts: attempt + 1,
+        usage: usage,
+        model: primaryModel
       });
 
+      // Update running totals
+      if (usage && runtime.updateUsageTotals) {
+        runtime.updateUsageTotals(name, usage);
+      }
+
       return result;
     } catch (error) {
       lastError = error;

package/lib/runtime/runtime.js

@@ -68,6 +68,9 @@ export class WorkflowRuntime {
     this.status = savedState.status || 'IDLE';
     this.startedAt = savedState.startedAt || null;
 
+    // Load usage totals
+    this._usageTotals = savedState.usage || null;
+
     // Create memory proxy for auto-persistence
     this.memory = createMemoryProxy(this._rawMemory, () => this.persist());
 
@@ -197,12 +200,67 @@ export class WorkflowRuntime {
       memory: this._rawMemory,
       _error: this._error,
       startedAt: this.startedAt,
-      lastUpdatedAt: new Date().toISOString()
+      lastUpdatedAt: new Date().toISOString(),
+      usage: this._usageTotals
     };
 
     fs.writeFileSync(this.stateFile, JSON.stringify(state, null, 2));
   }
 
+  /**
+   * Update running token usage totals
+   * @param {string} agentName - Name of the agent that generated the usage
+   * @param {object} usage - Usage object with inputTokens, outputTokens, etc.
+   */
+  updateUsageTotals(agentName, usage) {
+    if (!usage) return;
+
+    if (!this._usageTotals) {
+      this._usageTotals = {
+        totalInputTokens: 0,
+        totalOutputTokens: 0,
+        totalCachedTokens: 0,
+        totalCost: 0,
+        agentUsage: {},
+        modelUsage: {}
+      };
+    }
+
+    // Update totals
+    this._usageTotals.totalInputTokens += usage.inputTokens || 0;
+    this._usageTotals.totalOutputTokens += usage.outputTokens || 0;
+    this._usageTotals.totalCachedTokens += usage.cachedTokens || 0;
+    this._usageTotals.totalCost += usage.cost || 0;
+
+    // Per-agent tracking
+    if (!this._usageTotals.agentUsage[agentName]) {
+      this._usageTotals.agentUsage[agentName] = { inputTokens: 0, outputTokens: 0, calls: 0 };
+    }
+    this._usageTotals.agentUsage[agentName].inputTokens += usage.inputTokens || 0;
+    this._usageTotals.agentUsage[agentName].outputTokens += usage.outputTokens || 0;
+    this._usageTotals.agentUsage[agentName].calls += usage.calls || 1;
+
+    // Per-model tracking
+    if (usage.models) {
+      for (const [model, modelUsage] of Object.entries(usage.models)) {
+        if (!this._usageTotals.modelUsage[model]) {
+          this._usageTotals.modelUsage[model] = { inputTokens: 0, outputTokens: 0 };
+        }
+        this._usageTotals.modelUsage[model].inputTokens += modelUsage.inputTokens || 0;
+        this._usageTotals.modelUsage[model].outputTokens += modelUsage.outputTokens || 0;
+      }
+    }
+
+    this.persist();
+  }
+
+  /**
+   * Reset usage totals
+   */
+  resetUsageTotals() {
+    this._usageTotals = null;
+  }
+
   /**
    * Prepend an event to history.jsonl (newest first)
    */
@@ -562,14 +620,14 @@ export class WorkflowRuntime {
   showStatus() {
     console.log(`\n${C.bold}Workflow: ${C.cyan}${this.workflowName}${C.reset}`);
     console.log(`${C.dim}${'─'.repeat(40)}${C.reset}`);
-    
+
     let statusColor = C.reset;
     if (this.status === 'COMPLETED') statusColor = C.green;
     if (this.status === 'FAILED') statusColor = C.red;
     if (this.status === 'STOPPED') statusColor = C.yellow;
     if (this.status === 'RUNNING') statusColor = C.blue;
     if (this.status === 'IDLE') statusColor = C.gray;
-    
+
     console.log(`Status: ${statusColor}${this.status}${C.reset}`);
 
     if (this.startedAt) {
@@ -580,6 +638,36 @@ export class WorkflowRuntime {
       console.log(`Error: ${C.red}${this._error}${C.reset}`);
     }
 
+    // Display token usage if available
+    if (this._usageTotals && (this._usageTotals.totalInputTokens > 0 || this._usageTotals.totalOutputTokens > 0)) {
+      console.log(`\n${C.bold}Token Usage:${C.reset}`);
+      const formatTokens = (count) => {
+        if (count >= 1000000) return `${(count / 1000000).toFixed(1)}M`;
+        if (count >= 10000) return `${Math.round(count / 1000)}k`;
+        if (count >= 1000) return `${(count / 1000).toFixed(1)}k`;
+        return count.toString();
+      };
+      console.log(`  Input:  ${formatTokens(this._usageTotals.totalInputTokens)}`);
+      console.log(`  Output: ${formatTokens(this._usageTotals.totalOutputTokens)}`);
+      if (this._usageTotals.totalCachedTokens > 0) {
+        console.log(`  Cached: ${formatTokens(this._usageTotals.totalCachedTokens)}`);
+      }
+      console.log(`  ${C.bold}Total:  ${formatTokens(this._usageTotals.totalInputTokens + this._usageTotals.totalOutputTokens)}${C.reset}`);
+      if (this._usageTotals.totalCost > 0) {
+        console.log(`  ${C.cyan}Cost:   $${this._usageTotals.totalCost.toFixed(4)}${C.reset}`);
+      }
+
+      // Show per-model breakdown if multiple models used
+      const models = Object.keys(this._usageTotals.modelUsage || {});
+      if (models.length > 1) {
+        console.log(`\n${C.dim}By Model:${C.reset}`);
+        for (const model of models) {
+          const m = this._usageTotals.modelUsage[model];
+          console.log(`  ${model}: ${formatTokens(m.inputTokens)} in / ${formatTokens(m.outputTokens)} out`);
+        }
+      }
+    }
+
     const memoryKeys = Object.keys(this._rawMemory).filter((k) => !k.startsWith('_'));
     console.log(`\nMemory Keys: ${C.yellow}${memoryKeys.length}${C.reset}`);
     if (memoryKeys.length > 0) {
@@ -633,6 +721,7 @@ export class WorkflowRuntime {
     this._error = null;
     this.status = 'IDLE';
     this.startedAt = null;
+    this.resetUsageTotals();
 
     // Recreate memory proxy
     this.memory = createMemoryProxy(this._rawMemory, () => this.persist());
@@ -663,6 +752,7 @@ export class WorkflowRuntime {
     this._error = null;
     this.status = 'IDLE';
     this.startedAt = null;
+    this.resetUsageTotals();
 
     // Recreate memory proxy
     this.memory = createMemoryProxy(this._rawMemory, () => this.persist());

package/lib/setup.js

@@ -70,7 +70,7 @@ function copyTemplateDir(srcDir, destDir, replacements, createdPaths) {
  * Setup a new workflow with directory structure
  */
 async function setup(workflowName, options = {}) {
-  const workflowsDir = path.join(process.cwd(), 'workflows');
+  const workflowsDir = path.join(process.cwd(), '.workflows');
   const workflowDir = path.join(workflowsDir, workflowName);
   const templateName = options.template || DEFAULT_TEMPLATE;
 
@@ -111,9 +111,9 @@ async function setup(workflowName, options = {}) {
   console.log('─'.repeat(40));
   console.log(`\n✓ Workflow '${workflowName}' created successfully!\n`);
   console.log('Next steps:');
-  console.log(`  1. Edit workflows/${workflowName}/workflow.js to implement your flow`);
-  console.log(`  2. Edit workflows/${workflowName}/config.js to set models/API keys`);
-  console.log(`  3. Add custom agents in workflows/${workflowName}/agents/`);
+  console.log(`  1. Edit .workflows/${workflowName}/workflow.js to implement your flow`);
+  console.log(`  2. Edit .workflows/${workflowName}/config.js to set models/API keys`);
+  console.log(`  3. Add custom agents in .workflows/${workflowName}/agents/`);
   console.log(`  4. Run: state-machine run ${workflowName}\n`);
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-state-machine",
-  "version": "2.2.2",
+  "version": "2.4.0",
   "type": "module",
   "description": "A workflow orchestrator for running agents and scripts in sequence with state management",
   "main": "lib/index.js",

package/templates/project-builder/config.js

@@ -1,9 +1,9 @@
 export const config = {
   models: {
-    fast: "gemini -m gemini-2.5-flash-lite",
-    low: "gemini -m gemini-2.5-flash-lite",
-    med: "gemini -m gemini-2.5-flash-lite",
-    high: "gemini -m gemini-2.5-flash-lite",
+    fast: "gemini -m gemini-2.5-pro",
+    low: "gemini -m gemini-2.5-pro",
+    med: "gemini -m gemini-2.5-pro",
+    high: "gemini -m gemini-2.5-pro",
   },
   apiKeys: {
     gemini: process.env.GEMINI_API_KEY,