@pennyfarthing/core 7.4.0 → 7.4.1

package/pennyfarthing-dist/scripts/story/size-story.sh

@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+# size-story.sh - Display story sizing guidelines
+# Usage: size-story.sh [points]
+#   No args: Show all sizing guidelines
+#   points:  Show specific guidance for that point value
+
+set -euo pipefail
+
+POINTS="${1:-}"
+
+print_all() {
+    cat << 'EOF'
+# Story Sizing Guidelines
+
+| Points | Scale | Complexity | Testing |
+|--------|-------|------------|---------|
+| 1-2 | Trivial | Single file, config | Minimal/none |
+| 3 | Small | Few files, single component | Some tests |
+| 5 | Medium | Multiple files, cross-layer | Comprehensive |
+| 8 | Large | Significant scope, cross-repo | Extensive |
+| 13+ | Epic | **SPLIT THIS** | N/A |
+
+## 1-2 Points (Trivial)
+- Single file change
+- Config update
+- Simple bug fix
+- Minimal or no new tests needed
+
+## 3 Points (Small)
+- Few files changed
+- Single component/handler
+- Straightforward implementation
+- Some tests needed
+
+## 5 Points (Medium)
+- Multiple files across layers
+- New feature component
+- API + UI changes
+- Comprehensive tests
+
+## 8 Points (Large)
+- Significant feature
+- Multiple components
+- Database changes possible
+- Cross-repo coordination
+
+## 13+ Points (Epic-level)
+- **SPLIT THIS STORY**
+- Too complex for single story
+- Multiple unknowns
+- Break into 3-5 smaller stories
+EOF
+}
+
+print_specific() {
+    local pts="$1"
+    case "$pts" in
+        1|2)
+            cat << 'EOF'
+## 1-2 Points (Trivial)
+
+**Characteristics:**
+- Single file change
+- Config update
+- Simple bug fix
+- Minimal or no new tests needed
+
+**Examples:**
+- Fix typo in documentation
+- Update environment variable
+- Add missing import
+- Simple CSS adjustment
+
+**YAML:**
+```yaml
+- id: PROJ-XXXXX
+  title: "Fix typo in README"
+  points: 1
+  workflow: trivial
+```
+EOF
+            ;;
+        3)
+            cat << 'EOF'
+## 3 Points (Small)
+
+**Characteristics:**
+- Few files changed
+- Single component/handler
+- Straightforward implementation
+- Some tests needed
+
+**Examples:**
+- Add validation to existing endpoint
+- Fix bug with tests
+- Add new utility function
+
+**YAML:**
+```yaml
+- id: PROJ-XXXXX
+  title: "Add input validation"
+  points: 3
+  workflow: tdd
+```
+EOF
+            ;;
+        5)
+            cat << 'EOF'
+## 5 Points (Medium)
+
+**Characteristics:**
+- Multiple files across layers
+- New feature component
+- API + UI changes
+- Comprehensive tests
+
+**Examples:**
+- New admin page
+- New API endpoint with tests
+- Refactor with behavioral tests
+
+**YAML:**
+```yaml
+- id: PROJ-XXXXX
+  title: "Add settings page"
+  points: 5
+  workflow: tdd
+```
+EOF
+            ;;
+        8)
+            cat << 'EOF'
+## 8 Points (Large)
+
+**Characteristics:**
+- Significant feature
+- Multiple components
+- Database changes possible
+- Cross-repo coordination
+
+**Examples:**
+- New integration
+- Major refactor
+- New subsystem
+
+**YAML:**
+```yaml
+- id: PROJ-XXXXX
+  title: "Add notification system"
+  points: 8
+  workflow: tdd
+```
+
+**Consider:** Can this be split into 2-3 smaller stories?
+EOF
+            ;;
+        13|21|*)
+            if [[ "$pts" -ge 13 ]]; then
+                cat << 'EOF'
+## 13+ Points - TOO LARGE
+
+**This story should be SPLIT.**
+
+Stories over 8 points have:
+- Multiple unknowns
+- High risk of scope creep
+- Difficult to estimate accurately
+- Hard to review in one PR
+
+**Action:** Break into 3-5 smaller stories:
+1. Identify distinct deliverables
+2. Create a story per deliverable
+3. Size each story 3-5 points
+4. Link stories to parent epic
+
+**Example split:**
+```
+Epic: User Dashboard (13 pts total)
+├── Story 1: Dashboard layout (3 pts)
+├── Story 2: User stats API (5 pts)
+├── Story 3: Activity feed (3 pts)
+└── Story 4: Export feature (2 pts)
+```
+EOF
+            else
+                echo "Unknown point value: $pts"
+                echo "Valid values: 1, 2, 3, 5, 8, 13+"
+                exit 1
+            fi
+            ;;
+    esac
+}
+
+if [[ -z "$POINTS" ]]; then
+    print_all
+else
+    print_specific "$POINTS"
+fi

package/pennyfarthing-dist/scripts/story/story-template.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# story-template.sh - Show story templates by type
+# Usage: story-template.sh [bug|feature|refactor|chore]
+#   No args: Show all templates
+#   type:    Show specific template
+
+set -euo pipefail
+
+TYPE="${1:-}"
+
+print_bug() {
+    cat << 'EOF'
+## Bug Fix Story Template
+
+```yaml
+- id: PROJ-XXXXX
+  title: "[Component] Bug description"
+  description: |
+    **Symptom:** What user sees
+    **Error:** Error message or behavior
+    **Affected:** Pages/endpoints impacted
+  points: 2-3
+  priority: P1
+  status: backlog
+  workflow: tdd
+  acceptance_criteria:
+    - Bug no longer occurs under [conditions]
+    - Regression test added covering the bug
+    - Related code audited for same issue
+```
+
+**Sizing hints:**
+- Simple fix with clear cause: 2 pts
+- Investigation needed: 3 pts
+- Multiple components affected: 5 pts
+EOF
+}
+
+print_feature() {
+    cat << 'EOF'
+## Feature Story Template
+
+```yaml
+- id: PROJ-XXXXX
+  title: "[Action verb] [what] for [who/where]"
+  description: |
+    User can [action] to [benefit].
+    [Additional context if needed]
+  points: 3-8
+  priority: P2
+  status: backlog
+  workflow: tdd
+  acceptance_criteria:
+    - Feature works as specified
+    - UI matches design (if applicable)
+    - API documented (if applicable)
+    - Tests cover happy path and edge cases
+    - Error handling implemented
+```
+
+**Sizing hints:**
+- Single component: 3 pts
+- API + UI: 5 pts
+- Cross-repo/complex: 8 pts
+EOF
+}
+
+print_refactor() {
+    cat << 'EOF'
+## Refactor Story Template
+
+```yaml
+- id: PROJ-XXXXX
+  title: "Refactor [component] to [improvement]"
+  description: |
+    Current: [what exists now]
+    Target: [what it should become]
+    Why: [reason for change]
+  points: 3-5
+  priority: P2
+  status: backlog
+  workflow: tdd
+  acceptance_criteria:
+    - All existing tests pass
+    - No behavior changes (unless explicitly scoped)
+    - Code follows new pattern
+    - Documentation updated
+```
+
+**Sizing hints:**
+- Extract/rename: 2-3 pts
+- Restructure module: 5 pts
+- Architecture change: 8 pts (consider splitting)
+EOF
+}
+
+print_chore() {
+    cat << 'EOF'
+## Chore Story Template
+
+```yaml
+- id: PROJ-XXXXX
+  title: "[Action] [what]"
+  description: |
+    [What needs to be done and why]
+  points: 1-2
+  priority: P2
+  status: backlog
+  workflow: trivial
+  acceptance_criteria:
+    - Task completed
+    - No regressions introduced
+```
+
+**Examples:**
+- Update dependencies
+- Fix typo in docs
+- Remove dead code
+- Update config values
+
+**Note:** Chores typically use a lightweight workflow.
+EOF
+}
+
+print_all() {
+    cat << 'EOF'
+# Story Templates
+
+Use these templates when adding stories to sprint YAML.
+
+EOF
+    print_bug
+    echo ""
+    print_feature
+    echo ""
+    print_refactor
+    echo ""
+    print_chore
+}
+
+case "$TYPE" in
+    bug)
+        print_bug
+        ;;
+    feature)
+        print_feature
+        ;;
+    refactor)
+        print_refactor
+        ;;
+    chore)
+        print_chore
+        ;;
+    "")
+        print_all
+        ;;
+    *)
+        echo "Unknown type: $TYPE"
+        echo "Valid types: bug, feature, refactor, chore"
+        exit 1
+        ;;
+esac

package/pennyfarthing-dist/scripts/test/README.md

@@ -0,0 +1,23 @@
+# Test Scripts
+
+Scripts for test infrastructure and benchmarking.
+
+## Scripts
+
+| Script | Purpose |
+|--------|---------|
+| `test-setup.sh` | Set up test environment |
+| `test-cache.sh` | Manage test result caching |
+| `ground-truth-judge.py` | Python judge for ground truth evaluation |
+| `swebench-judge.py` | SWE-bench style evaluation |
+
+## Usage
+
+```bash
+.pennyfarthing/scripts/core/run.sh test/test-setup.sh
+```
+
+## Ownership
+
+- **Primary users:** TEA agent, testing-runner subagent
+- **Maintained by:** Core Pennyfarthing team

package/pennyfarthing-dist/scripts/test/ground-truth-judge.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+"""
+Ground-truth judge for SWE-bench scenarios.
+
+Compares Claude's proposed fix against the actual SWE-bench patch.
+Scores based on:
+- File identification (20%)
+- Function/location identification (20%)
+- Fix logic match (40%)
+- Completeness (20%)
+"""
+
+import json
+import re
+import sys
+from pathlib import Path
+from difflib import SequenceMatcher
+
+def load_swebench_data(cache_path="/tmp/swebench_all.json"):
+    """Load SWE-bench data from cache."""
+    with open(cache_path, 'r') as f:
+        return json.load(f)
+
+def find_scenario(data, scenario_name):
+    """Find scenario in SWE-bench data by name."""
+    # Normalize name (flask-5014 -> pallets__flask-5014)
+    for item in data:
+        instance_id = item.get('instance_id', '')
+        # Try various matching strategies
+        if scenario_name in instance_id.replace('__', '-'):
+            return item
+        if scenario_name.replace('-', '__') in instance_id:
+            return item
+    return None
+
+def extract_patch_elements(patch_text):
+    """Extract key elements from a patch."""
+    elements = {
+        'files': [],
+        'functions': [],
+        'additions': [],
+        'deletions': [],
+        'key_patterns': []
+    }
+
+    current_file = None
+    for line in patch_text.split('\n'):
+        # File changes
+        if line.startswith('diff --git'):
+            match = re.search(r'b/(.+)$', line)
+            if match:
+                current_file = match.group(1)
+                elements['files'].append(current_file)
+
+        # Function/class context
+        if line.startswith('@@'):
+            match = re.search(r'@@.*@@\s*(.+)$', line)
+            if match:
+                elements['functions'].append(match.group(1).strip())
+
+        # Additions
+        if line.startswith('+') and not line.startswith('+++'):
+            clean_line = line[1:].strip()
+            if clean_line and not clean_line.startswith('#'):
+                elements['additions'].append(clean_line)
+                # Extract key patterns (function calls, variable names, etc.)
+                patterns = re.findall(r'\b\w+\b', clean_line)
+                elements['key_patterns'].extend(patterns)
+
+        # Deletions
+        if line.startswith('-') and not line.startswith('---'):
+            clean_line = line[1:].strip()
+            if clean_line and not clean_line.startswith('#'):
+                elements['deletions'].append(clean_line)
+
+    # Deduplicate
+    elements['key_patterns'] = list(set(elements['key_patterns']))
+
+    return elements
+
+def score_response(response_text, ground_truth):
+    """Score a response against ground truth patch."""
+    gt_elements = extract_patch_elements(ground_truth['patch'])
+
+    scores = {
+        'file_identification': 0,
+        'location_identification': 0,
+        'fix_logic_match': 0,
+        'completeness': 0,
+        'details': {}
+    }
+
+    response_lower = response_text.lower()
+
+    # 1. FILE IDENTIFICATION (20 points)
+    files_found = 0
+    for f in gt_elements['files']:
+        # Check various forms of the filename
+        filename = Path(f).name
+        if filename.lower() in response_lower or f.lower() in response_lower:
+            files_found += 1
+
+    if gt_elements['files']:
+        file_score = (files_found / len(gt_elements['files'])) * 20
+        scores['file_identification'] = min(20, file_score)
+        scores['details']['files_expected'] = gt_elements['files']
+        scores['details']['files_found'] = files_found
+    else:
+        scores['file_identification'] = 20  # No specific file in patch
+
+    # 2. LOCATION IDENTIFICATION (20 points)
+    # Look for function/class names mentioned in the patch
+    locations_found = 0
+    for func in gt_elements['functions']:
+        # Extract the function/class name
+        func_match = re.search(r'(def|class)\s+(\w+)', func)
+        if func_match:
+            func_name = func_match.group(2)
+            if func_name.lower() in response_lower:
+                locations_found += 1
+        elif func.strip() and func.strip().split()[0] in response_lower:
+            locations_found += 1
+
+    if gt_elements['functions']:
+        loc_score = (locations_found / len(gt_elements['functions'])) * 20
+        scores['location_identification'] = min(20, loc_score)
+        scores['details']['locations_expected'] = gt_elements['functions'][:3]
+        scores['details']['locations_found'] = locations_found
+    else:
+        scores['location_identification'] = 10  # Partial credit
+
+    # 3. FIX LOGIC MATCH (40 points)
+    # Check if key code patterns from the fix appear in the response
+    key_patterns = gt_elements['key_patterns']
+    # Filter to meaningful patterns (not common words)
+    common_words = {'if', 'else', 'return', 'self', 'def', 'class', 'for', 'in', 'not', 'and', 'or', 'is', 'none', 'true', 'false'}
+    meaningful_patterns = [p for p in key_patterns if p.lower() not in common_words and len(p) > 2]
+
+    patterns_found = 0
+    for pattern in meaningful_patterns:
+        if pattern.lower() in response_lower:
+            patterns_found += 1
+
+    if meaningful_patterns:
+        pattern_score = (patterns_found / len(meaningful_patterns)) * 20
+        scores['details']['patterns_expected'] = meaningful_patterns[:10]
+        scores['details']['patterns_found'] = patterns_found
+    else:
+        pattern_score = 10
+
+    # Check for actual code additions
+    additions_matched = 0
+    for addition in gt_elements['additions'][:5]:  # Check first 5 additions
+        # Normalize and check
+        addition_normalized = re.sub(r'\s+', ' ', addition.lower())
+        response_normalized = re.sub(r'\s+', ' ', response_lower)
+
+        # Use fuzzy matching
+        similarity = SequenceMatcher(None, addition_normalized, response_normalized).ratio()
+        if similarity > 0.6 or addition_normalized in response_normalized:
+            additions_matched += 1
+
+    if gt_elements['additions']:
+        addition_score = (additions_matched / min(5, len(gt_elements['additions']))) * 20
+        scores['details']['additions_matched'] = additions_matched
+    else:
+        addition_score = 10
+
+    scores['fix_logic_match'] = min(40, pattern_score + addition_score)
+
+    # 4. COMPLETENESS (20 points)
+    # Does the response have all the elements of a good fix?
+    completeness_score = 0
+
+    # Has code block?
+    if '```' in response_text:
+        completeness_score += 5
+
+    # Has test considerations?
+    if 'test' in response_lower:
+        completeness_score += 5
+
+    # Mentions the specific error/issue?
+    problem_keywords = extract_problem_keywords(ground_truth.get('problem_statement', ''))
+    keywords_found = sum(1 for kw in problem_keywords if kw.lower() in response_lower)
+    if problem_keywords:
+        completeness_score += min(5, (keywords_found / len(problem_keywords)) * 5)
+    else:
+        completeness_score += 2.5
+
+    # Has explanation of why fix works?
+    explanation_words = ['because', 'this fixes', 'this resolves', 'the issue', 'the problem', 'solution']
+    if any(word in response_lower for word in explanation_words):
+        completeness_score += 5
+
+    scores['completeness'] = min(20, completeness_score)
+
+    # Total
+    scores['total'] = round(
+        scores['file_identification'] +
+        scores['location_identification'] +
+        scores['fix_logic_match'] +
+        scores['completeness']
+    , 1)
+
+    return scores
+
+def extract_problem_keywords(problem_statement):
+    """Extract key technical terms from problem statement."""
+    if not problem_statement:
+        return []
+
+    # Find quoted strings, function names, error messages
+    keywords = []
+
+    # Find quoted terms
+    quoted = re.findall(r'[`\'"]([^`\'"]+)[`\'"]', problem_statement)
+    keywords.extend(quoted)
+
+    # Find CamelCase or snake_case identifiers
+    identifiers = re.findall(r'\b[A-Z][a-z]+[A-Z]\w*\b|\b\w+_\w+\b', problem_statement)
+    keywords.extend(identifiers)
+
+    return list(set(keywords))[:10]
+
+def main():
+    if len(sys.argv) < 3:
+        print("Usage: ground-truth-judge.py <scenario_name> <response_file>")
+        print("Example: ground-truth-judge.py flask-5014 run_20260102T134237Z.json")
+        sys.exit(1)
+
+    scenario_name = sys.argv[1]
+    response_file = sys.argv[2]
+
+    # Load SWE-bench data
+    swebench_data = load_swebench_data()
+
+    # Find scenario
+    scenario = find_scenario(swebench_data, scenario_name)
+    if not scenario:
+        print(f"Error: Scenario '{scenario_name}' not found in SWE-bench data")
+        sys.exit(1)
+
+    # Load response
+    with open(response_file, 'r') as f:
+        response_data = json.load(f)
+
+    response_text = response_data.get('result', '')
+    if not response_text:
+        print("Error: No 'result' field in response file")
+        sys.exit(1)
+
+    # Score
+    scores = score_response(response_text, scenario)
+
+    # Output
+    print(f"\n{'='*60}")
+    print(f"GROUND TRUTH EVALUATION: {scenario_name}")
+    print(f"{'='*60}")
+    print(f"\nScores:")
+    print(f"  File Identification:     {scores['file_identification']:5.1f}/20")
+    print(f"  Location Identification: {scores['location_identification']:5.1f}/20")
+    print(f"  Fix Logic Match:         {scores['fix_logic_match']:5.1f}/40")
+    print(f"  Completeness:            {scores['completeness']:5.1f}/20")
+    print(f"  {'─'*40}")
+    print(f"  TOTAL:                   {scores['total']:5.1f}/100")
+
+    print(f"\nDetails:")
+    for key, value in scores['details'].items():
+        print(f"  {key}: {value}")
+
+    # Output JSON for programmatic use
+    output = {
+        'scenario': scenario_name,
+        'instance_id': scenario.get('instance_id'),
+        'scores': scores,
+        'ground_truth_patch_preview': scenario.get('patch', '')[:300]
+    }
+
+    # Save judge output
+    output_path = response_file.replace('run_', 'gt_judge_')
+    with open(output_path, 'w') as f:
+        json.dump(output, f, indent=2)
+    print(f"\nSaved to: {output_path}")
+
+    return scores
+
+if __name__ == '__main__':
+    main()