@litmers/cursorflow-orchestrator 0.1.13 → 0.1.15

package/scripts/test-streaming-multi-task.sh

@@ -0,0 +1,247 @@
+#!/bin/bash
+#
+# Comprehensive Streaming Output Test with Multi-Task Dependencies
+# 
+# This test verifies:
+# 1. Streaming output from cursor-agent (not just final JSON)
+# 2. Multiple tasks execution in sequence
+# 3. Dependency handling between tasks
+# 4. Raw log capture and parsed log quality
+#
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+MAGENTA='\033[0;35m'
+NC='\033[0m'
+
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo -e "${CYAN}  🧪 Streaming Multi-Task Test${NC}"
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo ""
+
+# Check cursor-agent
+if ! command -v cursor-agent &> /dev/null; then
+    echo -e "${RED}❌ cursor-agent not found${NC}"
+    exit 1
+fi
+
+# Check auth
+echo -e "${YELLOW}Checking cursor-agent authentication...${NC}"
+if ! cursor-agent create-chat &> /dev/null; then
+    echo -e "${RED}❌ cursor-agent authentication failed${NC}"
+    exit 1
+fi
+echo -e "${GREEN}✓ cursor-agent authenticated${NC}"
+
+cd "$PROJECT_ROOT"
+
+# Build
+echo -e "${YELLOW}Building project...${NC}"
+npm run build > /dev/null 2>&1
+echo -e "${GREEN}✓ Build complete${NC}"
+
+# Setup test directory
+TEST_DIR="$PROJECT_ROOT/_test-streaming"
+rm -rf "$TEST_DIR"
+mkdir -p "$TEST_DIR/lane"
+
+# Create multi-task configuration with dependencies
+cat > "$TEST_DIR/multi-task.json" << 'EOF'
+{
+  "baseBranch": "main",
+  "branchPrefix": "test/stream-",
+  "timeout": 120000,
+  "dependencyPolicy": {
+    "allowDependencyChange": false,
+    "lockfileReadOnly": true
+  },
+  "tasks": [
+    {
+      "name": "task-1-analyze",
+      "prompt": "Analyze this codebase. List the top 3 most important files and briefly explain what each does. Keep your response under 200 words. Do not make any file changes.",
+      "model": "sonnet-4.5"
+    },
+    {
+      "name": "task-2-summary",
+      "prompt": "Based on your previous analysis, write a 2-sentence summary of what this project does. Do not make any file changes.",
+      "model": "sonnet-4.5"
+    }
+  ]
+}
+EOF
+
+echo -e "${GREEN}✓ Test configuration created (2 tasks)${NC}"
+echo ""
+
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo -e "${CYAN}  Running orchestration with STREAMING enabled...${NC}"
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo ""
+
+LANE_DIR="$TEST_DIR/lane"
+
+# Run spawnLane directly to test streaming
+node -e "
+const { spawnLane, waitChild } = require('./dist/core/orchestrator');
+const fs = require('fs');
+const path = require('path');
+
+const tasksFile = path.join('$TEST_DIR', 'multi-task.json');
+const laneRunDir = '$LANE_DIR';
+
+console.log('Spawning lane with streaming output enabled...');
+console.log('');
+
+const result = spawnLane({
+  laneName: 'stream-test',
+  tasksFile,
+  laneRunDir,
+  executor: 'cursor-agent',
+  startIndex: 0,
+  enhancedLogConfig: {
+    enabled: true,
+    stripAnsi: true,
+    streamOutput: true,  // Enable streaming!
+    addTimestamps: true,
+    keepRawLogs: true,
+    writeJsonLog: true,
+    timestampFormat: 'iso',
+  },
+});
+
+console.log('Lane spawned, PID:', result.child.pid);
+console.log('Waiting for completion (may take 2-3 minutes)...');
+console.log('');
+
+const timeout = setTimeout(() => {
+  console.log('\\nTimeout reached (3 min)');
+  result.child.kill('SIGTERM');
+}, 180000);
+
+waitChild(result.child).then((code) => {
+  clearTimeout(timeout);
+  console.log('');
+  console.log('Lane completed with exit code:', code);
+  result.logManager?.close();
+  process.exit(code === 0 ? 0 : 1);
+}).catch(e => {
+  clearTimeout(timeout);
+  console.error('Error:', e);
+  process.exit(1);
+});
+" 2>&1
+
+EXIT_CODE=$?
+
+echo ""
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo -e "${CYAN}  📋 Log File Analysis${NC}"
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo ""
+
+# Check files
+echo -e "${YELLOW}Log files created:${NC}"
+ls -la "$LANE_DIR"/*.log "$LANE_DIR"/*.jsonl 2>/dev/null || echo "No log files found"
+echo ""
+
+# Count streaming entries in JSON log
+if [ -f "$LANE_DIR/terminal.jsonl" ]; then
+    TOTAL_ENTRIES=$(wc -l < "$LANE_DIR/terminal.jsonl")
+    STDOUT_ENTRIES=$(grep -c '"level":"stdout"' "$LANE_DIR/terminal.jsonl" || echo "0")
+    STREAMING_ENTRIES=$(grep -c '"type":' "$LANE_DIR/terminal.jsonl" || echo "0")
+    
+    echo -e "${YELLOW}JSON Log Statistics:${NC}"
+    echo "  Total entries: $TOTAL_ENTRIES"
+    echo "  stdout entries: $STDOUT_ENTRIES"
+    echo "  Entries with streaming 'type': $STREAMING_ENTRIES"
+    echo ""
+fi
+
+# Check for streaming content markers in raw log
+if [ -f "$LANE_DIR/terminal-raw.log" ]; then
+    RAW_SIZE=$(wc -c < "$LANE_DIR/terminal-raw.log")
+    RAW_LINES=$(wc -l < "$LANE_DIR/terminal-raw.log")
+    
+    # Look for streaming indicators
+    HAS_TYPE_TEXT=$(grep -c '"type":"text"' "$LANE_DIR/terminal-raw.log" 2>/dev/null || echo "0")
+    HAS_TYPE_RESULT=$(grep -c '"type":"result"' "$LANE_DIR/terminal-raw.log" 2>/dev/null || echo "0")
+    HAS_CONTENT=$(grep -c '"content":' "$LANE_DIR/terminal-raw.log" 2>/dev/null || echo "0")
+    
+    echo -e "${YELLOW}Raw Log Statistics:${NC}"
+    echo "  Size: $RAW_SIZE bytes"
+    echo "  Lines: $RAW_LINES"
+    echo ""
+    echo -e "${YELLOW}Streaming Markers Found:${NC}"
+    echo "  type:'text' entries: $HAS_TYPE_TEXT"
+    echo "  type:'result' entries: $HAS_TYPE_RESULT"
+    echo "  content entries: $HAS_CONTENT"
+    echo ""
+fi
+
+# Show sample of streaming output
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo -e "${CYAN}  📝 Sample Streaming Output (first 30 lines of stdout)${NC}"
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo ""
+
+if [ -f "$LANE_DIR/terminal.log" ]; then
+    # Show lines containing cursor-agent output (streaming JSON)
+    echo -e "${MAGENTA}Looking for streaming JSON output...${NC}"
+    grep -E '^\[.*\] \{"type":' "$LANE_DIR/terminal.log" | head -20 || echo "No streaming JSON found"
+    echo ""
+fi
+
+# Test the logs CLI command
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo -e "${CYAN}  🔍 Testing cursorflow logs command${NC}"
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo ""
+
+# Create a fake runs directory structure for the CLI
+FAKE_RUN_DIR="$TEST_DIR/runs/run-test"
+mkdir -p "$FAKE_RUN_DIR/lanes"
+ln -sf "$LANE_DIR" "$FAKE_RUN_DIR/lanes/stream-test"
+
+echo -e "${YELLOW}Testing: cursorflow logs --tail 15${NC}"
+node dist/cli/index.js logs "$FAKE_RUN_DIR" --tail 15 2>&1 || true
+
+echo ""
+echo -e "${YELLOW}Testing: cursorflow logs --filter 'type.*text'${NC}"
+node dist/cli/index.js logs "$FAKE_RUN_DIR" --filter '"type":"text"' --tail 10 2>&1 || true
+
+echo ""
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo -e "${CYAN}  📊 Test Summary${NC}"
+echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+echo ""
+
+# Determine if streaming worked
+if [ "$HAS_TYPE_TEXT" -gt "0" ] || [ "$HAS_CONTENT" -gt "10" ]; then
+    echo -e "${GREEN}✅ Streaming output captured successfully!${NC}"
+    echo "   Found $HAS_TYPE_TEXT text stream entries"
+    echo "   Found $HAS_CONTENT content entries"
+else
+    echo -e "${YELLOW}⚠️  Limited streaming content detected${NC}"
+    echo "   This may indicate:"
+    echo "   - stream-json format not outputting expected data"
+    echo "   - Or cursor-agent only provides final results"
+fi
+
+echo ""
+echo "Log files saved at: $LANE_DIR"
+echo ""
+
+# Cleanup git
+git worktree remove _cursorflow/worktrees/test/stream-* --force 2>/dev/null || true
+git branch -D $(git branch | grep "test/stream-") 2>/dev/null || true
+
+exit $EXIT_CODE
+

package/src/cli/clean.ts

@@ -8,7 +8,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as logger from '../utils/logger';
 import * as git from '../utils/git';
-import { loadConfig, getLogsDir } from '../utils/config';
+import { loadConfig, getLogsDir, getTasksDir } from '../utils/config';
 
 interface CleanOptions {
   type?: string;
@@ -17,6 +17,8 @@ interface CleanOptions {
   force: boolean;
   all: boolean;
   help: boolean;
+  keepLatest: boolean;
+  includeLatest: boolean;
 }
 
 function printHelp(): void {
@@ -29,26 +31,43 @@ Types:
   branches               Remove local feature branches
   worktrees              Remove temporary Git worktrees
   logs                   Clear log directories
+  tasks                  Clear task directories
   all                    Remove all of the above (default)
 
 Options:
   --dry-run              Show what would be removed without deleting
   --force                Force removal (ignore uncommitted changes)
+  --include-latest       Also remove the most recent item (by default, latest is kept)
   --help, -h             Show help
   `);
 }
 
 function parseArgs(args: string[]): CleanOptions {
+  const includeLatest = args.includes('--include-latest');
   return {
-    type: args.find(a => ['branches', 'worktrees', 'logs', 'all'].includes(a)),
+    type: args.find(a => ['branches', 'worktrees', 'logs', 'tasks', 'all'].includes(a)),
     pattern: null,
     dryRun: args.includes('--dry-run'),
     force: args.includes('--force'),
     all: args.includes('--all'),
     help: args.includes('--help') || args.includes('-h'),
+    keepLatest: !includeLatest, // Default: keep latest, unless --include-latest is specified
+    includeLatest,
   };
 }
 
+/**
+ * Get the modification time of a path (directory or file)
+ */
+function getModTime(targetPath: string): number {
+  try {
+    const stat = fs.statSync(targetPath);
+    return stat.mtimeMs;
+  } catch {
+    return 0;
+  }
+}
+
 async function clean(args: string[]): Promise<void> {
   const options = parseArgs(args);
   
@@ -68,12 +87,15 @@ async function clean(args: string[]): Promise<void> {
     await cleanWorktrees(config, repoRoot, options);
     await cleanBranches(config, repoRoot, options);
     await cleanLogs(config, options);
+    await cleanTasks(config, options);
   } else if (type === 'worktrees') {
     await cleanWorktrees(config, repoRoot, options);
   } else if (type === 'branches') {
     await cleanBranches(config, repoRoot, options);
   } else if (type === 'logs') {
     await cleanLogs(config, options);
+  } else if (type === 'tasks') {
+    await cleanTasks(config, options);
   }
 
   logger.success('\n✨ Cleaning complete!');
@@ -84,7 +106,7 @@ async function cleanWorktrees(config: any, repoRoot: string, options: CleanOptio
   const worktrees = git.listWorktrees(repoRoot);
   
   const worktreeRoot = path.join(repoRoot, config.worktreeRoot || '_cursorflow/worktrees');
-  const toRemove = worktrees.filter(wt => {
+  let toRemove = worktrees.filter(wt => {
     // Skip main worktree
     if (wt.path === repoRoot) return false;
     
@@ -99,6 +121,15 @@ async function cleanWorktrees(config: any, repoRoot: string, options: CleanOptio
     return;
   }
 
+  // If keepLatest is set, keep the most recent worktree
+  if (options.keepLatest && toRemove.length > 1) {
+    // Sort by modification time (newest first)
+    toRemove.sort((a, b) => getModTime(b.path) - getModTime(a.path));
+    const kept = toRemove[0];
+    toRemove = toRemove.slice(1);
+    logger.info(`  Keeping latest worktree: ${kept.path} (${kept.branch || 'no branch'})`);
+  }
+
   for (const wt of toRemove) {
     if (options.dryRun) {
       logger.info(`  [DRY RUN] Would remove worktree: ${wt.path} (${wt.branch || 'no branch'})`);
@@ -123,6 +154,21 @@ async function cleanWorktrees(config: any, repoRoot: string, options: CleanOptio
   }
 }
 
+/**
+ * Get the commit timestamp of a branch
+ */
+function getBranchCommitTime(branch: string, repoRoot: string): number {
+  try {
+    const result = git.runGitResult(['log', '-1', '--format=%ct', branch], { cwd: repoRoot });
+    if (result.success && result.stdout.trim()) {
+      return parseInt(result.stdout.trim(), 10) * 1000; // Convert to milliseconds
+    }
+  } catch {
+    // Ignore errors
+  }
+  return 0;
+}
+
 async function cleanBranches(config: any, repoRoot: string, options: CleanOptions) {
   logger.info('\nChecking branches...');
   
@@ -136,13 +182,22 @@ async function cleanBranches(config: any, repoRoot: string, options: CleanOption
     .filter(b => b && b !== 'main' && b !== 'master');
 
   const prefix = config.branchPrefix || 'feature/';
-  const toDelete = branches.filter(b => b.startsWith(prefix));
+  let toDelete = branches.filter(b => b.startsWith(prefix));
 
   if (toDelete.length === 0) {
     logger.info('  No branches found to clean.');
     return;
   }
 
+  // If keepLatest is set, keep the most recent branch
+  if (options.keepLatest && toDelete.length > 1) {
+    // Sort by commit time (newest first)
+    toDelete.sort((a, b) => getBranchCommitTime(b, repoRoot) - getBranchCommitTime(a, repoRoot));
+    const kept = toDelete[0];
+    toDelete = toDelete.slice(1);
+    logger.info(`  Keeping latest branch: ${kept}`);
+  }
+
   for (const branch of toDelete) {
     if (options.dryRun) {
       logger.info(`  [DRY RUN] Would delete branch: ${branch}`);
@@ -166,16 +221,118 @@ async function cleanLogs(config: any, options: CleanOptions) {
     return;
   }
 
-  if (options.dryRun) {
-    logger.info(`  [DRY RUN] Would remove logs directory: ${logsDir}`);
+  // If keepLatest is set, keep the most recent log directory/file
+  if (options.keepLatest) {
+    const entries = fs.readdirSync(logsDir, { withFileTypes: true });
+    let items = entries.map(entry => ({
+      name: entry.name,
+      path: path.join(logsDir, entry.name),
+      isDir: entry.isDirectory(),
+      mtime: getModTime(path.join(logsDir, entry.name))
+    }));
+
+    if (items.length <= 1) {
+      logger.info('  Only one or no log entries found, nothing to clean.');
+      return;
+    }
+
+    // Sort by modification time (newest first)
+    items.sort((a, b) => b.mtime - a.mtime);
+    const kept = items[0];
+    const toRemove = items.slice(1);
+
+    logger.info(`  Keeping latest log: ${kept.name}`);
+
+    for (const item of toRemove) {
+      if (options.dryRun) {
+        logger.info(`  [DRY RUN] Would remove log: ${item.name}`);
+      } else {
+        try {
+          logger.info(`  Removing log: ${item.name}...`);
+          fs.rmSync(item.path, { recursive: true, force: true });
+        } catch (e: any) {
+          logger.error(`  Failed to remove log ${item.name}: ${e.message}`);
+        }
+      }
+    }
   } else {
-    try {
-      logger.info(`  Removing logs...`);
-      fs.rmSync(logsDir, { recursive: true, force: true });
-      fs.mkdirSync(logsDir, { recursive: true });
-      logger.info(`  Logs cleared.`);
-    } catch (e: any) {
-      logger.error(`  Failed to clean logs: ${e.message}`);
+    if (options.dryRun) {
+      logger.info(`  [DRY RUN] Would remove logs directory: ${logsDir}`);
+    } else {
+      try {
+        logger.info(`  Removing logs...`);
+        fs.rmSync(logsDir, { recursive: true, force: true });
+        fs.mkdirSync(logsDir, { recursive: true });
+        logger.info(`  Logs cleared.`);
+      } catch (e: any) {
+        logger.error(`  Failed to clean logs: ${e.message}`);
+      }
+    }
+  }
+}
+
+async function cleanTasks(config: any, options: CleanOptions) {
+  const tasksDir = getTasksDir(config);
+  logger.info(`\nChecking tasks in ${tasksDir}...`);
+
+  if (!fs.existsSync(tasksDir)) {
+    logger.info('  Tasks directory does not exist.');
+    return;
+  }
+
+  // If keepLatest is set, keep the most recent task directory/file
+  if (options.keepLatest) {
+    const entries = fs.readdirSync(tasksDir, { withFileTypes: true });
+    // Skip example task if it exists and there are other tasks
+    let items = entries
+      .filter(entry => entry.name !== 'example')
+      .map(entry => ({
+        name: entry.name,
+        path: path.join(tasksDir, entry.name),
+        isDir: entry.isDirectory(),
+        mtime: getModTime(path.join(tasksDir, entry.name))
+      }));
+
+    if (items.length <= 1) {
+      logger.info('  Only one or no user task entries found, nothing to clean.');
+      return;
+    }
+
+    // Sort by modification time (newest first)
+    items.sort((a, b) => b.mtime - a.mtime);
+    const kept = items[0];
+    const toRemove = items.slice(1);
+
+    logger.info(`  Keeping latest task: ${kept.name}`);
+
+    for (const item of toRemove) {
+      if (options.dryRun) {
+        logger.info(`  [DRY RUN] Would remove task: ${item.name}`);
+      } else {
+        try {
+          logger.info(`  Removing task: ${item.name}...`);
+          fs.rmSync(item.path, { recursive: true, force: true });
+        } catch (e: any) {
+          logger.error(`  Failed to remove task ${item.name}: ${e.message}`);
+        }
+      }
+    }
+  } else {
+    if (options.dryRun) {
+      logger.info(`  [DRY RUN] Would remove tasks in directory: ${tasksDir} (except example)`);
+    } else {
+      try {
+        const entries = fs.readdirSync(tasksDir, { withFileTypes: true });
+        for (const entry of entries) {
+          if (entry.name === 'example') continue;
+          const itemPath = path.join(tasksDir, entry.name);
+          logger.info(`  Removing task: ${entry.name}...`);
+          fs.rmSync(itemPath, { recursive: true, force: true });
+        }
+        logger.info(`  Tasks cleared.`);
+      } catch (e: any) {
+        logger.error(`  Failed to clean tasks: ${e.message}`);
+      }
     }
   }
 }

package/src/cli/index.ts

@@ -20,6 +20,7 @@ const COMMANDS: Record<string, CommandFn> = {
   doctor: require('./doctor'),
   signal: require('./signal'),
   models: require('./models'),
+  logs: require('./logs'),
   setup: require('./setup-commands').main,
   'setup-commands': require('./setup-commands').main,
 };
@@ -37,10 +38,11 @@ function printHelp(): void {
   \x1b[33mrun\x1b[0m <tasks-dir> [options]   Run orchestration (DAG-based)
   \x1b[33mmonitor\x1b[0m [run-dir] [options] \x1b[36mInteractive\x1b[0m lane dashboard
   \x1b[33mclean\x1b[0m <type> [options]      Clean branches/worktrees/logs
-  \x1b[33mresume\x1b[0m <lane> [options]     Resume interrupted lane
+  \x1b[33mresume\x1b[0m [lane] [options]     Resume lane(s) - use --all for batch resume
   \x1b[33mdoctor\x1b[0m [options]            Check environment and preflight
   \x1b[33msignal\x1b[0m <lane> <msg>         Directly intervene in a running lane
   \x1b[33mmodels\x1b[0m [options]            List available AI models
+  \x1b[33mlogs\x1b[0m [run-dir] [options]    View, export, and follow logs
 
 \x1b[1mGLOBAL OPTIONS\x1b[0m
   --config <path>             Config file path
@@ -114,3 +116,4 @@ if (require.main === module) {
 
 export default main;
 export { main };
+export { events } from '../utils/events';