@pennyfarthing/core 7.7.0 → 7.8.0

package/pennyfarthing-dist/scripts/misc/migrate_bmad_workflow.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+"""
+migrate_bmad_workflow.py - Migrate BMAD workflows to Pennyfarthing format
+
+Converts BMAD workflow format to Pennyfarthing stepped workflow format:
+- Parses BMAD workflow.md with YAML frontmatter
+- Extracts step files and their frontmatter
+- Converts variable syntax ({var-name} -> {var_name})
+- Generates workflow.yaml with Pennyfarthing schema
+- Preserves tri-modal structure (steps-c, steps-v, steps-e)
+
+Usage: python migrate_bmad_workflow.py [--dry-run] <source-dir> [target-dir]
+"""
+
+import argparse
+import re
+import shutil
+import sys
+from pathlib import Path
+
+
+class Colors:
+    RED = '\033[31m'
+    GREEN = '\033[32m'
+    YELLOW = '\033[33m'
+    BLUE = '\033[34m'
+    NC = '\033[0m'
+
+
+def log_info(msg: str) -> None:
+    print(f"{Colors.BLUE}INFO{Colors.NC}: {msg}")
+
+
+def log_success(msg: str) -> None:
+    print(f"{Colors.GREEN}SUCCESS{Colors.NC}: {msg}")
+
+
+def log_warn(msg: str) -> None:
+    print(f"{Colors.YELLOW}WARN{Colors.NC}: {msg}")
+
+
+def log_error(msg: str) -> None:
+    print(f"{Colors.RED}ERROR{Colors.NC}: {msg}", file=sys.stderr)
+
+
+def extract_frontmatter(content: str) -> tuple[dict, str]:
+    """Extract YAML frontmatter from markdown content."""
+    lines = content.split('\n')
+
+    if not lines or lines[0].strip() != '---':
+        return {}, content
+
+    end_index = -1
+    for i, line in enumerate(lines[1:], 1):
+        if line.strip() == '---':
+            end_index = i
+            break
+
+    if end_index == -1:
+        return {}, content
+
+    frontmatter = {}
+    for line in lines[1:end_index]:
+        match = re.match(r'^([a-zA-Z_][a-zA-Z0-9_]*):\s*(.*)$', line)
+        if match:
+            key, value = match.groups()
+            value = value.strip().strip('"\'')
+            frontmatter[key] = value
+
+    rest_content = '\n'.join(lines[end_index + 1:])
+    return frontmatter, rest_content
+
+
+def convert_variables(text: str) -> str:
+    """Convert BMAD-style dashed variables to Pennyfarthing underscore style."""
+    result = text
+    prev = None
+
+    while result != prev:
+        prev = result
+        result = re.sub(
+            r'\{([a-zA-Z0-9_]+)-([a-zA-Z0-9_-]+)\}',
+            lambda m: '{' + m.group(1) + '_' + m.group(2).replace('-', '_') + '}',
+            result
+        )
+
+    return result
+
+
+def is_trimodal(source_dir: Path) -> bool:
+    """Check if workflow is tri-modal (has steps-c, steps-v, or steps-e)."""
+    return any(
+        (source_dir / d).exists()
+        for d in ['steps-c', 'steps-v', 'steps-e']
+    )
+
+
+def get_step_directories(source_dir: Path) -> list[str]:
+    """Get step directories to process."""
+    if is_trimodal(source_dir):
+        return [d for d in ['steps-c', 'steps-v', 'steps-e']
+                if (source_dir / d).exists()]
+    elif (source_dir / 'steps').exists():
+        return ['steps']
+    return []
+
+
+def get_supporting_directories(source_dir: Path) -> list[str]:
+    """Get supporting directories (templates, data, etc.)."""
+    return [d for d in ['templates', 'data', 'assets']
+            if (source_dir / d).exists()]
+
+
+def get_root_template_files(source_dir: Path) -> list[str]:
+    """Get root-level template files."""
+    return [f.name for f in source_dir.iterdir()
+            if f.is_file() and '-template' in f.name and f.suffix == '.md']
+
+
+def copy_dir_with_transform(src_dir: Path, dest_dir: Path,
+                            transform: callable, dry_run: bool) -> None:
+    """Recursively copy directory with file transformation."""
+    if not src_dir.exists():
+        return
+
+    if not dry_run:
+        dest_dir.mkdir(parents=True, exist_ok=True)
+
+    for entry in src_dir.iterdir():
+        dest_path = dest_dir / entry.name
+
+        if entry.is_dir():
+            copy_dir_with_transform(entry, dest_path, transform, dry_run)
+        elif entry.is_file():
+            content = entry.read_text()
+            transformed = transform(content, entry)
+
+            if dry_run:
+                if content != transformed:
+                    log_info(f"[DRY-RUN] Would convert variables in: {entry}")
+            else:
+                dest_path.write_text(transformed)
+                log_success(f"Copied: {dest_path.name}")
+
+
+def generate_workflow_yaml(name: str, description: str, trimodal: bool) -> str:
+    """Generate workflow.yaml content."""
+    converted_desc = convert_variables(description)
+
+    if trimodal:
+        step_config = """    path: ./steps-c/
+    pattern: step-*.md"""
+        modes_config = """
+  # Tri-modal workflow paths
+  modes:
+    create: ./steps-c/
+    validate: ./steps-v/
+    edit: ./steps-e/"""
+    else:
+        step_config = """    path: ./steps/
+    pattern: step-*.md"""
+        modes_config = ""
+
+    return f"""# {name} Workflow - Migrated from BMAD format
+# Generated by migrate_bmad_workflow.py
+
+workflow:
+  name: {name}
+  description: {converted_desc}
+  version: "1.0.0"
+  type: stepped
+
+  # Step configuration
+  steps:
+{step_config}
+{modes_config}
+  # Variables available in step files
+  variables:
+    project_root: .
+    planning_artifacts: ./artifacts
+    output_file: artifacts/{name}.md
+
+  # Agent assignment (customize as needed)
+  agent: architect
+
+  # Triggers - when to suggest this workflow
+  triggers:
+    types: [{name}]
+    tags: [{name}, stepped]
+"""
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Migrate BMAD workflows to Pennyfarthing stepped workflow format"
+    )
+    parser.add_argument('--dry-run', action='store_true',
+                        help='Show what would be done without making changes')
+    parser.add_argument('source_dir', type=Path,
+                        help='Path to BMAD workflow directory (must contain workflow.md)')
+    parser.add_argument('target_dir', type=Path, nargs='?', default=Path('.'),
+                        help='Path to output directory (default: current directory)')
+    args = parser.parse_args()
+
+    source_dir = args.source_dir.resolve()
+    target_dir = args.target_dir.resolve()
+    dry_run = args.dry_run
+
+    if not source_dir.exists():
+        log_error(f"Source directory not found: {source_dir}")
+        return 1
+
+    workflow_md = source_dir / 'workflow.md'
+    if not workflow_md.exists():
+        log_error(f"workflow.md not found in {source_dir}")
+        return 1
+
+    log_info('BMAD Workflow Migration')
+    log_info('=======================')
+    log_info(f'Source: {source_dir}')
+    log_info(f'Target: {target_dir}')
+
+    if dry_run:
+        log_warn('DRY-RUN MODE - No changes will be made')
+
+    print('')
+
+    # Parse workflow.md
+    content = workflow_md.read_text()
+    frontmatter, _ = extract_frontmatter(content)
+
+    name = frontmatter.get('name')
+    description = frontmatter.get('description', '')
+
+    if not name:
+        log_error('Could not extract workflow name from workflow.md')
+        return 1
+
+    log_info(f'Parsed workflow: {name}')
+    log_info(f'Description: {description}')
+    print('')
+
+    # Create target directory
+    if not dry_run:
+        target_dir.mkdir(parents=True, exist_ok=True)
+
+    # Generate workflow.yaml
+    log_info('Generating workflow.yaml')
+    trimodal = is_trimodal(source_dir)
+    workflow_yaml = generate_workflow_yaml(name, description, trimodal)
+
+    if dry_run:
+        log_info('[DRY-RUN] Would create: workflow.yaml')
+    else:
+        (target_dir / 'workflow.yaml').write_text(workflow_yaml)
+        log_success('Created workflow.yaml')
+
+    print('')
+
+    # Copy step directories
+    def md_transform(content: str, filepath: Path) -> str:
+        if filepath.suffix == '.md':
+            return convert_variables(content)
+        return content
+
+    for dir_name in get_step_directories(source_dir):
+        log_info(f'Processing step directory: {dir_name}')
+        if dry_run:
+            log_info(f'[DRY-RUN] Would copy and transform: {dir_name}/')
+        copy_dir_with_transform(
+            source_dir / dir_name,
+            target_dir / dir_name,
+            md_transform,
+            dry_run
+        )
+
+    print('')
+
+    # Copy supporting directories
+    for dir_name in get_supporting_directories(source_dir):
+        log_info(f'Processing supporting directory: {dir_name}')
+        if dry_run:
+            log_info(f'[DRY-RUN] Would copy: {dir_name}/')
+        copy_dir_with_transform(
+            source_dir / dir_name,
+            target_dir / dir_name,
+            md_transform,
+            dry_run
+        )
+
+    print('')
+
+    # Copy root-level template files
+    for filename in get_root_template_files(source_dir):
+        log_info(f'Processing root template file: {filename}')
+        src_path = source_dir / filename
+        dest_path = target_dir / filename
+
+        if dry_run:
+            log_info(f'[DRY-RUN] Would copy: {filename}')
+        else:
+            content = src_path.read_text()
+            transformed = convert_variables(content)
+            dest_path.write_text(transformed)
+            log_success(f'Copied: {filename}')
+
+    print('')
+
+    if dry_run:
+        log_info('DRY-RUN complete. No files were modified.')
+    else:
+        log_success('Migration complete!')
+        log_info(f'Output directory: {target_dir}')
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/pennyfarthing-dist/scripts/sprint/import-epic-to-future.sh

@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
-# import-epic-to-future.sh - Thin wrapper for import-epic-to-future.mjs
+# import-epic-to-future.sh - Import epics-and-stories workflow output to future.yaml
 #
 # Usage: ./scripts/sprint/import-epic-to-future.sh [--dry-run] <epics-md-file> [initiative-name]
-#
-# Delegates to Node script for cleaner markdown parsing and YAML generation.
 
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-exec node "${SCRIPT_DIR}/import-epic-to-future.mjs" "$@"
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+exec python3 "$SCRIPT_DIR/import_epic_to_future.py" "$@"

package/pennyfarthing-dist/scripts/sprint/import_epic_to_future.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""
+import_epic_to_future.py - Import epics-and-stories workflow output to future.yaml
+
+Transforms the markdown output from the epics-and-stories workflow into
+the YAML format used by sprint/future.yaml.
+
+Usage: python import_epic_to_future.py [--dry-run] <epics-md-file> [initiative-name]
+
+Example:
+  python import_epic_to_future.py docs/planning/reflector-epics-and-stories.md "Reflector Consolidation"
+"""
+
+import argparse
+import re
+import sys
+from datetime import date
+from pathlib import Path
+
+
+class Colors:
+    RED = '\033[31m'
+    GREEN = '\033[32m'
+    YELLOW = '\033[33m'
+    BLUE = '\033[34m'
+    NC = '\033[0m'
+
+
+def log_info(msg: str) -> None:
+    print(f"{Colors.BLUE}INFO{Colors.NC}: {msg}")
+
+
+def log_success(msg: str) -> None:
+    print(f"{Colors.GREEN}SUCCESS{Colors.NC}: {msg}")
+
+
+def log_error(msg: str) -> None:
+    print(f"{Colors.RED}ERROR{Colors.NC}: {msg}", file=sys.stderr)
+
+
+def find_project_root() -> Path:
+    """Find project root by looking for .claude directory."""
+    current = Path.cwd()
+    while current != current.parent:
+        if (current / ".claude").is_dir():
+            return current
+        current = current.parent
+    return Path.cwd()
+
+
+def parse_epics_markdown(content: str) -> dict:
+    """Parse epics markdown file."""
+    result = {
+        'title': '',
+        'description': '',
+        'total_points': 0,
+        'epic_title': '',
+        'epic_description': '',
+        'stories': [],
+    }
+
+    # Extract title from first # heading
+    title_match = re.search(r'^# (.+?)( - Epic Breakdown)?$', content, re.MULTILINE)
+    if title_match:
+        result['title'] = title_match.group(1).replace(' - Epics and Stories', '')
+
+    # Extract description from Overview section
+    overview_match = re.search(r'## Overview\s*\n\s*\n(.+?)(?=\n\n##|\n##)', content, re.DOTALL)
+    if overview_match:
+        result['description'] = overview_match.group(1).strip()
+
+    # Extract total points
+    points_match = re.search(r'\*\*Points:\*\*\s*(\d+)', content)
+    if points_match:
+        result['total_points'] = int(points_match.group(1))
+    else:
+        effort_match = re.search(r'Total Effort.*?(\d+)\s*story points', content, re.IGNORECASE)
+        if effort_match:
+            result['total_points'] = int(effort_match.group(1))
+
+    # Extract epic title
+    epic_title_match = re.search(r'^## Epic \d+:\s*(.+)$', content, re.MULTILINE)
+    if epic_title_match:
+        result['epic_title'] = epic_title_match.group(1)
+
+    # Extract epic description (User Outcome)
+    user_outcome_match = re.search(r'\*\*User Outcome:\*\*\s*(.+)', content)
+    if user_outcome_match:
+        result['epic_description'] = user_outcome_match.group(1)
+
+    # Parse stories
+    story_pattern = re.compile(r'^### Story \d+\.(\d+):\s*(.+)$', re.MULTILINE)
+    i_want_pattern = re.compile(r'^I want \*\*(.+?)\*\*,', re.MULTILINE)
+
+    story_positions = []
+    for match in story_pattern.finditer(content):
+        story_positions.append({
+            'num': int(match.group(1)),
+            'title': match.group(2),
+            'index': match.start(),
+        })
+
+    for i, story in enumerate(story_positions):
+        start_index = story['index']
+        end_index = story_positions[i + 1]['index'] if i < len(story_positions) - 1 else len(content)
+        story_content = content[start_index:end_index]
+
+        i_want_match = i_want_pattern.search(story_content)
+        description = i_want_match.group(1) if i_want_match else ''
+
+        result['stories'].append({
+            'num': story['num'],
+            'title': story['title'],
+            'description': description,
+        })
+
+    return result
+
+
+def get_next_epic_number(future_yaml_path: Path) -> int:
+    """Get next epic number from future.yaml."""
+    if not future_yaml_path.exists():
+        return 60
+
+    content = future_yaml_path.read_text()
+
+    # Look for "Next Available Epic Number: XX"
+    next_match = re.search(r'Next Available Epic Number:\s*(\d+)', content, re.IGNORECASE)
+    if next_match:
+        return int(next_match.group(1))
+
+    # Fallback: find highest epic-XX number
+    max_num = 59
+    for match in re.finditer(r'epic-(\d+)', content):
+        num = int(match.group(1))
+        if num > max_num:
+            max_num = num
+
+    return max_num + 1
+
+
+def generate_yaml(parsed: dict, epic_num: int, initiative_name: str, epics_file: str) -> str:
+    """Generate YAML for initiative."""
+    today = date.today().isoformat()
+
+    yaml = f"""
+    # ==========================================================================
+    # {initiative_name}
+    # Imported from: {epics_file}
+    # Date: {today}
+    # ==========================================================================
+    - name: "{initiative_name}"
+      description: |
+        {parsed['description']}
+      status: ready
+      blocked_by: null
+      total_points: {parsed['total_points']}
+      prd: docs/planning/reflector-prd.md
+      epics_doc: {epics_file}
+      epics:
+        - id: epic-{epic_num}
+          title: "{parsed['epic_title'] or initiative_name}"
+          description: |
+            {parsed['epic_description'] or parsed['description']}
+          points: {parsed['total_points']}
+          priority: P1
+          marker: "reflector"
+          repos: pennyfarthing
+          status: planning
+          stories:
+"""
+
+    for story in parsed['stories']:
+        yaml += f"""            - id: "{epic_num}-{story['num']}"
+              title: "{story['title']}"
+              description: |
+                {story['description'] or story['title']}
+              points: 1
+              priority: P0
+              status: planning
+              repos: pennyfarthing
+"""
+
+    return yaml
+
+
+def update_future_yaml(future_yaml_path: Path, new_content: str, next_epic_num: int) -> str:
+    """Update future.yaml."""
+    content = future_yaml_path.read_text()
+
+    # Update next epic number comment
+    content = re.sub(
+        r'Next Available Epic Number:\s*\d+',
+        f'Next Available Epic Number: {next_epic_num + 1}',
+        content,
+        flags=re.IGNORECASE
+    )
+
+    # Find insertion point (before SUMMARY section)
+    summary_marker = '# =============================================================================\n# SUMMARY'
+    summary_index = content.find(summary_marker)
+
+    if summary_index == -1:
+        content += new_content
+    else:
+        content = content[:summary_index] + new_content + '\n' + content[summary_index:]
+
+    return content
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Import epics-and-stories workflow output to future.yaml"
+    )
+    parser.add_argument('--dry-run', action='store_true',
+                        help='Print YAML to stdout instead of updating future.yaml')
+    parser.add_argument('epics_file', type=str,
+                        help='Path to the markdown file from epics-and-stories workflow')
+    parser.add_argument('initiative_name', type=str, nargs='?', default='',
+                        help='Name for the initiative (optional, extracted from file if not provided)')
+    args = parser.parse_args()
+
+    project_root = find_project_root()
+
+    epics_path = Path(args.epics_file)
+    if not epics_path.is_absolute():
+        epics_path = project_root / epics_path
+
+    if not epics_path.exists():
+        log_error(f"File not found: {epics_path}")
+        return 1
+
+    future_yaml_path = project_root / 'sprint' / 'future.yaml'
+
+    # Read and parse epics file
+    content = epics_path.read_text()
+    parsed = parse_epics_markdown(content)
+
+    # Use provided name or extract from file
+    initiative_name = args.initiative_name or parsed['title'] or 'Imported Initiative'
+
+    # Get next epic number
+    next_epic_num = get_next_epic_number(future_yaml_path)
+
+    log_info(f"Next epic number: {next_epic_num}")
+    log_info(f"Initiative name: {initiative_name}")
+    log_info(f"Total points: {parsed['total_points']}")
+    log_info(f"Epic title: {parsed['epic_title']}")
+    log_info(f"Stories found: {len(parsed['stories'])}")
+
+    # Generate YAML
+    new_yaml = generate_yaml(parsed, next_epic_num, initiative_name, args.epics_file)
+
+    if args.dry_run:
+        print(f"\n{Colors.YELLOW}=== DRY RUN: Would append to future.yaml ==={Colors.NC}")
+        print(new_yaml)
+        print(f"{Colors.YELLOW}=== End of YAML ==={Colors.NC}\n")
+        print(f"{Colors.GREEN}To apply, run without --dry-run{Colors.NC}")
+    else:
+        updated_content = update_future_yaml(future_yaml_path, new_yaml, next_epic_num)
+        future_yaml_path.write_text(updated_content)
+
+        log_success(f"Added epic-{next_epic_num} to {future_yaml_path}")
+        log_success(f"Next available epic number is now: {next_epic_num + 1}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/pennyfarthing-dist/scripts/test/ensure-swebench-data.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# ensure-swebench-data.sh - Downloads SWE-bench data if not present
+#
+# Usage: ensure-swebench-data.sh [--force]
+#
+# Downloads SWE-bench Verified dataset from HuggingFace to /tmp/swebench_all.json
+# This is a dependency for:
+#   - swebench-judge.py
+#   - ground-truth-judge.py
+#
+# Options:
+#   --force    Re-download even if file exists
+
+set -euo pipefail
+
+CACHE_PATH="/tmp/swebench_all.json"
+DATASET_URL="https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified/resolve/main/data/test.jsonl"
+
+force=false
+if [[ "${1:-}" == "--force" ]]; then
+    force=true
+fi
+
+# Check if already present
+if [[ -f "$CACHE_PATH" ]] && [[ "$force" == "false" ]]; then
+    echo "SWE-bench data already cached at $CACHE_PATH"
+    exit 0
+fi
+
+echo "Downloading SWE-bench Verified dataset..."
+
+# Download JSONL and convert to JSON array
+if command -v curl &>/dev/null; then
+    curl -sL "$DATASET_URL" | python3 -c "
+import json
+import sys
+lines = [json.loads(line) for line in sys.stdin if line.strip()]
+print(json.dumps(lines, indent=2))
+" > "$CACHE_PATH"
+elif command -v wget &>/dev/null; then
+    wget -qO- "$DATASET_URL" | python3 -c "
+import json
+import sys
+lines = [json.loads(line) for line in sys.stdin if line.strip()]
+print(json.dumps(lines, indent=2))
+" > "$CACHE_PATH"
+else
+    echo "Error: curl or wget required to download SWE-bench data"
+    exit 1
+fi
+
+# Verify download
+if [[ -f "$CACHE_PATH" ]]; then
+    count=$(python3 -c "import json; print(len(json.load(open('$CACHE_PATH'))))")
+    echo "Downloaded $count SWE-bench scenarios to $CACHE_PATH"
+else
+    echo "Error: Failed to download SWE-bench data"
+    exit 1
+fi

package/pennyfarthing-dist/scripts/theme/compute-theme-tiers.sh

@@ -1,11 +1,13 @@
-#!/bin/bash
-# compute-theme-tiers.sh - Shell wrapper for compute-theme-tiers.js
-# Computes tier rankings from job-fair results and updates theme files
+#!/usr/bin/env bash
+# compute-theme-tiers.sh - Compute tier rankings from job-fair results
 #
 # Uses the MOST COMPLETE run for each theme (most matrix entries),
 # not the most recent. This prevents incomplete runs from overriding good data.
 #
-# All logic is implemented in compute-theme-tiers.js (Node.js)
+# Usage: compute-theme-tiers.sh [--dry-run] [--verbose] [--min-entries N]
 
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-exec node "$SCRIPT_DIR/compute-theme-tiers.js" "$@"
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+exec python3 "$SCRIPT_DIR/compute_theme_tiers.py" "$@"