maestro-flow 0.4.8 → 0.4.10

package/.agy/skills/skill-iter-tune/phases/01-setup.md

@@ -0,0 +1,144 @@
+# Phase 1: Setup
+
+Initialize workspace, backup skills, parse inputs.
+
+## Objective
+
+- Parse skill path(s) and test scenario from user input
+- Validate all skill paths exist and contain SKILL.md
+- Create isolated workspace directory structure
+- Backup original skill files
+- Initialize iteration-state.json
+
+## Execution
+
+### Step 1.1: Parse Input
+
+Parse `$ARGUMENTS` to extract skill paths and test scenario.
+
+```javascript
+// Parse skill paths (first argument or comma-separated)
+const args = $ARGUMENTS.trim();
+const pathMatch = args.match(/^([^\s]+)/);
+const rawPaths = pathMatch ? pathMatch[1].split(',') : [];
+
+// Parse test scenario
+const scenarioMatch = args.match(/(?:--scenario|--test)\s+"([^"]+)"/);
+const scenarioText = scenarioMatch ? scenarioMatch[1] : args.replace(rawPaths.join(','), '').trim();
+
+// Record chain order (preserves input order for chain mode)
+const chainOrder = rawPaths.map(p => p.startsWith('.claude/') ? p.split('/').pop() : p);
+
+// If no scenario, ask user
+if (!scenarioText) {
+  const response = ask_question({
+    questions: [{
+      question: "Please describe the test scenario for evaluating this skill:",
+      header: "Test Scenario",
+      multiSelect: false,
+      options: [
+        { label: "General quality test", description: "Evaluate overall skill quality with a generic task" },
+        { label: "Specific scenario", description: "I'll describe a specific test case" }
+      ]
+    }]
+  });
+  // Use response to construct testScenario
+}
+```
+
+### Step 1.2: Validate Skill Paths
+
+```javascript
+const targetSkills = [];
+for (const rawPath of rawPaths) {
+  const skillPath = rawPath.startsWith('.claude/') ? rawPath : `.claude/skills/${rawPath}`;
+
+  // Validate SKILL.md exists
+  const skillFiles = grep_search(`${skillPath}/SKILL.md`);
+  if (skillFiles.length === 0) {
+    throw new Error(`Skill not found at: ${skillPath} -- SKILL.md missing`);
+  }
+
+  // Collect all skill files
+  const allFiles = grep_search(`${skillPath}/**/*.md`);
+  targetSkills.push({
+    name: skillPath.split('/').pop(),
+    path: skillPath,
+    files: allFiles.map(f => f.replace(skillPath + '/', '')),
+    primary_file: 'SKILL.md'
+  });
+}
+```
+
+### Step 1.3: Create Workspace
+
+```javascript
+const ts = Date.now();
+const workDir = `.workflow/.scratchpad/skill-iter-tune-${ts}`;
+
+run_command(`mkdir -p "${workDir}/backups" "${workDir}/iterations"`);
+```
+
+### Step 1.4: Backup Original Skills
+
+```javascript
+for (const skill of targetSkills) {
+  run_command(`cp -r "${skill.path}" "${workDir}/backups/${skill.name}"`);
+}
+```
+
+### Step 1.5: Initialize State
+
+Write `iteration-state.json` with initial state:
+
+```javascript
+const initialState = {
+  status: 'running',
+  started_at: new Date().toISOString(),
+  updated_at: new Date().toISOString(),
+  target_skills: targetSkills,
+  test_scenario: {
+    description: scenarioText,
+    // Parse --requirements and --input-args from $ARGUMENTS if provided
+    // e.g., --requirements "clear output,no errors" --input-args "my-skill --scenario test"
+    requirements: parseListArg(args, '--requirements') || [],
+    input_args: parseStringArg(args, '--input-args') || '',
+    success_criteria: parseStringArg(args, '--success-criteria') || 'Produces correct, high-quality output'
+  },
+  execution_mode: workflowPreferences.executionMode || 'single',
+  chain_order: workflowPreferences.executionMode === 'chain'
+    ? targetSkills.map(s => s.name)
+    : [],
+  current_iteration: 0,
+  max_iterations: workflowPreferences.maxIterations,
+  quality_threshold: workflowPreferences.qualityThreshold,
+  latest_score: 0,
+  score_trend: [],
+  converged: false,
+  iterations: [],
+  errors: [],
+  error_count: 0,
+  max_errors: 3,
+  work_dir: workDir,
+  backup_dir: `${workDir}/backups`
+};
+
+write_to_file(`${workDir}/iteration-state.json`, JSON.stringify(initialState, null, 2));
+
+// Chain mode: create per-skill tracking tasks
+if (initialState.execution_mode === 'chain') {
+  for (const skill of targetSkills) {
+    TaskCreate({
+      subject: `Chain: ${skill.name}`,
+      activeForm: `Tracking ${skill.name}`,
+      description: `Skill chain member: ${skill.path} | Position: ${targetSkills.indexOf(skill) + 1}/${targetSkills.length}`
+    });
+  }
+}
+```
+
+## Output
+
+- **Variables**: `workDir`, `targetSkills[]`, `testScenario`, `chainOrder` (chain mode)
+- **Files**: `iteration-state.json`, `backups/` directory with skill copies
+- **TodoWrite**: Mark Phase 1 completed, start Iteration Loop. Chain mode: per-skill tracking tasks created

package/.agy/skills/skill-iter-tune/phases/02-execute.md

@@ -0,0 +1,292 @@
+# Phase 2: Execute Skill
+
+> **COMPACT SENTINEL [Phase 2: Execute]**
+> This phase contains 4 execution steps (Step 2.1 -- 2.4).
+> If you can read this sentinel but cannot find the full Step protocol below, context has been compressed.
+> Recovery: `view_file("phases/02-execute.md")`
+
+Execute the target skill against the test scenario using `ccw cli --tool claude --mode write`. Claude receives the full skill definition and simulates producing its expected output artifacts.
+
+## Objective
+
+- Snapshot current skill version before execution
+- Construct execution prompt with full skill content + test scenario
+- Execute via ccw cli Claude
+- Collect output artifacts
+
+## Execution
+
+### Step 2.1: Snapshot Current Skill
+
+```javascript
+const N = state.current_iteration;
+const iterDir = `${state.work_dir}/iterations/iteration-${N}`;
+run_command(`mkdir -p "${iterDir}/skill-snapshot" "${iterDir}/artifacts"`);
+
+// Chain mode: create per-skill artifact directories
+if (state.execution_mode === 'chain') {
+  for (const skillName of state.chain_order) {
+    run_command(`mkdir -p "${iterDir}/artifacts/${skillName}"`);
+  }
+}
+
+// Snapshot current skill state (so we can compare/rollback)
+for (const skill of state.target_skills) {
+  run_command(`cp -r "${skill.path}" "${iterDir}/skill-snapshot/${skill.name}"`);
+}
+```
+
+### Step 2.2: Construct Execution Prompt (Single Mode)
+
+Read the execute-prompt template and substitute variables.
+
+> Skip to Step 2.2b if `state.execution_mode === 'chain'`.
+
+```javascript
+// Ref: templates/execute-prompt.md
+
+// Build skillContent by reading only executable skill files (SKILL.md, phases/, specs/)
+// Exclude README.md, docs/, and other non-executable files to save tokens
+const skillContent = state.target_skills.map(skill => {
+  const skillMd = view_file(`${skill.path}/SKILL.md`);
+  const phaseFiles = grep_search(`${skill.path}/phases/*.md`).sort().map(f => ({
+    relativePath: f.replace(skill.path + '/', ''),
+    content: view_file(f)
+  }));
+  const specFiles = grep_search(`${skill.path}/specs/*.md`).map(f => ({
+    relativePath: f.replace(skill.path + '/', ''),
+    content: view_file(f)
+  }));
+
+  return `### File: SKILL.md\n${skillMd}\n\n` +
+    phaseFiles.map(f => `### File: ${f.relativePath}\n${f.content}`).join('\n\n') +
+    (specFiles.length > 0 ? '\n\n' + specFiles.map(f => `### File: ${f.relativePath}\n${f.content}`).join('\n\n') : '');
+}).join('\n\n---\n\n');
+
+// Construct full prompt using template
+const executePrompt = `PURPOSE: Simulate executing the following workflow skill against a test scenario. Produce all expected output artifacts as if the skill were invoked with the given input.
+
+SKILL CONTENT:
+${skillContent}
+
+TEST SCENARIO:
+Description: ${state.test_scenario.description}
+Input Arguments: ${state.test_scenario.input_args}
+Requirements: ${state.test_scenario.requirements.join('; ')}
+Success Criteria: ${state.test_scenario.success_criteria}
+
+TASK:
+1. Study the complete skill structure (SKILL.md + all phase files)
+2. Follow the skill execution flow sequentially
+3. For each phase, produce the artifacts that phase would generate
+4. Write all output artifacts to the current working directory
+5. Create a manifest.json listing all produced artifacts
+
+MODE: write
+CONTEXT: @**/*
+EXPECTED: All artifacts written to disk + manifest.json
+CONSTRAINTS: Follow skill flow exactly, produce realistic output, not placeholders`;
+```
+
+### Step 2.3: Execute via ccw cli
+
+> **CHECKPOINT**: Before executing CLI, verify:
+> 1. This phase is TodoWrite `in_progress`
+> 2. `iterDir/artifacts/` directory exists
+> 3. Prompt is properly escaped
+
+```javascript
+function escapeForShell(str) {
+  return str.replace(/"/g, '\\"').replace(/\$/g, '\\$').replace(/`/g, '\\`');
+}
+
+const cliCommand = `ccw cli -p "${escapeForShell(executePrompt)}" --tool claude --mode write --cd "${iterDir}/artifacts"`;
+
+// Execute in background, wait for hook callback
+run_command({
+  command: cliCommand,
+  run_in_background: true,
+  timeout: 600000  // 10 minutes max
+});
+
+// STOP HERE -- wait for hook callback to resume
+// After callback, verify artifacts were produced
+```
+
+### Step 2.2b: Chain Execution Path
+
+> Skip this step if `state.execution_mode === 'single'`.
+
+In chain mode, execute each skill sequentially. Each skill receives the previous skill's artifacts as input context.
+
+```javascript
+// Chain execution: iterate through chain_order
+let previousArtifacts = '';  // Accumulates upstream output
+
+for (let i = 0; i < state.chain_order.length; i++) {
+  const skillName = state.chain_order[i];
+  const skill = state.target_skills.find(s => s.name === skillName);
+  const skillArtifactDir = `${iterDir}/artifacts/${skillName}`;
+
+  // Build this skill's content
+  const skillMd = view_file(`${skill.path}/SKILL.md`);
+  const phaseFiles = grep_search(`${skill.path}/phases/*.md`).sort().map(f => ({
+    relativePath: f.replace(skill.path + '/', ''),
+    content: view_file(f)
+  }));
+  const specFiles = grep_search(`${skill.path}/specs/*.md`).map(f => ({
+    relativePath: f.replace(skill.path + '/', ''),
+    content: view_file(f)
+  }));
+
+  const singleSkillContent = `### File: SKILL.md\n${skillMd}\n\n` +
+    phaseFiles.map(f => `### File: ${f.relativePath}\n${f.content}`).join('\n\n') +
+    (specFiles.length > 0 ? '\n\n' + specFiles.map(f => `### File: ${f.relativePath}\n${f.content}`).join('\n\n') : '');
+
+  // Build chain context from previous skill's artifacts
+  const chainInputContext = previousArtifacts
+    ? `\nPREVIOUS CHAIN OUTPUT (from upstream skill "${state.chain_order[i - 1]}"):\n${previousArtifacts}\n\nIMPORTANT: Use the above output as input context for this skill's execution.\n`
+    : '';
+
+  // Construct per-skill execution prompt
+  // Ref: templates/execute-prompt.md
+  const chainPrompt = `PURPOSE: Simulate executing the following workflow skill against a test scenario. Produce all expected output artifacts.
+
+SKILL CONTENT (${skillName} — chain position ${i + 1}/${state.chain_order.length}):
+${singleSkillContent}
+${chainInputContext}
+TEST SCENARIO:
+Description: ${state.test_scenario.description}
+Input Arguments: ${state.test_scenario.input_args}
+Requirements: ${state.test_scenario.requirements.join('; ')}
+Success Criteria: ${state.test_scenario.success_criteria}
+
+TASK:
+1. Study the complete skill structure
+2. Follow the skill execution flow sequentially
+3. Produce all expected artifacts
+4. Write output to the current working directory
+5. Create manifest.json listing all produced artifacts
+
+MODE: write
+CONTEXT: @**/*
+CONSTRAINTS: Follow skill flow exactly, produce realistic output`;
+
+  function escapeForShell(str) {
+    return str.replace(/"/g, '\\"').replace(/\$/g, '\\$').replace(/`/g, '\\`');
+  }
+
+  const cliCommand = `ccw cli -p "${escapeForShell(chainPrompt)}" --tool claude --mode write --cd "${skillArtifactDir}"`;
+
+  // Execute in background
+  run_command({
+    command: cliCommand,
+    run_in_background: true,
+    timeout: 600000
+  });
+
+  // STOP -- wait for hook callback
+
+  // After callback: collect artifacts for next skill in chain
+  const artifacts = grep_search(`${skillArtifactDir}/**/*`);
+  const skillSuccess = artifacts.length > 0;
+
+  if (skillSuccess) {
+    previousArtifacts = artifacts.slice(0, 10).map(f => {
+      const relPath = f.replace(skillArtifactDir + '/', '');
+      const content = view_file(f, { limit: 100 });
+      return `--- ${relPath} ---\n${content}`;
+    }).join('\n\n');
+  } else {
+    // Mid-chain failure: keep previous artifacts for downstream skills
+    // Log warning but continue chain — downstream skills receive last successful output
+    state.errors.push({
+      phase: 'execute',
+      message: `Chain skill "${skillName}" (position ${i + 1}) produced no artifacts. Downstream skills will receive upstream output from "${state.chain_order[i - 1] || 'none'}" instead.`,
+      timestamp: new Date().toISOString()
+    });
+    state.error_count++;
+    // previousArtifacts remains from last successful skill (or empty if first)
+  }
+
+  // Update per-skill TodoWrite
+  // TaskUpdate chain skill task with execution status
+
+  // Record per-skill execution
+  if (!state.iterations[N - 1].execution.chain_executions) {
+    state.iterations[N - 1].execution.chain_executions = [];
+  }
+  state.iterations[N - 1].execution.chain_executions.push({
+    skill_name: skillName,
+    cli_command: cliCommand,
+    artifacts_dir: skillArtifactDir,
+    success: skillSuccess
+  });
+
+  // Check error budget: abort chain if too many consecutive failures
+  if (state.error_count >= 3) {
+    state.errors.push({
+      phase: 'execute',
+      message: `Chain execution aborted at skill "${skillName}" — error limit reached (${state.error_count} errors).`,
+      timestamp: new Date().toISOString()
+    });
+    break;
+  }
+}
+```
+
+### Step 2.4: Collect Artifacts
+
+After CLI completes (hook callback received):
+
+```javascript
+// List produced artifacts
+const artifactFiles = grep_search(`${iterDir}/artifacts/**/*`);
+
+// Chain mode: check per-skill artifacts
+if (state.execution_mode === 'chain') {
+  const chainSuccess = state.iterations[N - 1].execution.chain_executions?.every(e => e.success) ?? false;
+  state.iterations[N - 1].execution.success = chainSuccess;
+  state.iterations[N - 1].execution.artifacts_dir = `${iterDir}/artifacts`;
+} else {
+
+if (artifactFiles.length === 0) {
+  // Execution produced nothing -- record error
+  state.iterations[N - 1].execution = {
+    cli_command: cliCommand,
+    started_at: new Date().toISOString(),
+    completed_at: new Date().toISOString(),
+    artifacts_dir: `${iterDir}/artifacts`,
+    success: false
+  };
+  state.error_count++;
+  // Continue to Phase 3 anyway -- Gemini can evaluate the skill even without artifacts
+} else {
+  state.iterations[N - 1].execution = {
+    cli_command: cliCommand,
+    started_at: new Date().toISOString(),
+    completed_at: new Date().toISOString(),
+    artifacts_dir: `${iterDir}/artifacts`,
+    success: true
+  };
+}
+
+} // end single mode branch
+
+// Update state
+write_to_file(`${state.work_dir}/iteration-state.json`, JSON.stringify(state, null, 2));
+```
+
+## Error Handling
+
+| Error | Recovery |
+|-------|----------|
+| CLI timeout (10min) | Record failure, continue to Phase 3 without artifacts |
+| CLI crash | Retry once with simplified prompt (SKILL.md only, no phase files) |
+| No artifacts produced | Continue to Phase 3, evaluation focuses on skill definition quality |
+
+## Output
+
+- **Files**: `iteration-{N}/skill-snapshot/`, `iteration-{N}/artifacts/`
+- **State**: `iterations[N-1].execution` updated
+- **Next**: Phase 3 (Evaluate)