jfl 0.4.4 → 0.6.0

package/scripts/train/v2/transform_buffer.py

@@ -0,0 +1,227 @@
+"""
+Transform v1 training buffer (state, action, reward) to v2 format (current_state, goal, correct_tool).
+
+Reads .jfl/training-buffer.jsonl and .jfl/agents/*.toml to produce v2 training data
+with 70/15/15 train/val/test splits.
+"""
+
+import json
+import os
+import sys
+import random
+import argparse
+from pathlib import Path
+
+ACTION_TYPE_MAP = {
+    "fix": "fix_bug",
+    "refactor": "refactor_code",
+    "feature": "add_feature",
+    "test": "add_tests",
+    "config": "update_config",
+    "experiment": "run_experiment",
+}
+
+def load_domain(domain_path: str) -> dict:
+    with open(domain_path) as f:
+        return json.load(f)
+
+def load_agent_goals(agents_dir: str) -> dict[str, str]:
+    goals = {}
+    agents_path = Path(agents_dir)
+    if not agents_path.exists():
+        return goals
+
+    for toml_file in agents_path.glob("*.toml"):
+        name = None
+        metric = None
+        direction = None
+        scope = None
+
+        for line in toml_file.read_text().splitlines():
+            line = line.strip()
+            if line.startswith("name"):
+                name = line.split("=", 1)[1].strip().strip('"')
+            elif line.startswith("metric"):
+                metric = line.split("=", 1)[1].strip().strip('"')
+            elif line.startswith("direction"):
+                direction = line.split("=", 1)[1].strip().strip('"')
+            elif line.startswith("scope") and "=" in line and not line.startswith("scope_files"):
+                scope = line.split("=", 1)[1].strip().strip('"')
+
+        if name and metric and direction:
+            verb = "Improve" if direction == "maximize" else "Reduce"
+            scope_str = f" for {scope}" if scope else ""
+            goals[name] = f"{verb} {metric.replace('_', ' ')}{scope_str}"
+
+    return goals
+
+def format_state_text(state: dict) -> str:
+    dims = state.get("dimension_scores", {})
+    dims_str = ", ".join(f"{k}={v:.4f}" for k, v in dims.items()) if dims else "none"
+    deltas = state.get("recent_deltas", [])
+    deltas_str = ", ".join(f"{'+' if d >= 0 else ''}{d:.4f}" for d in deltas) if deltas else "none"
+    return "\n".join([
+        f"Agent: {state.get('agent', 'unknown')}",
+        f"Composite: {state.get('composite_score', 0):.4f}",
+        f"Tests: {state.get('tests_passing', 0)}/{state.get('tests_total', 0)}",
+        f"Trajectory: {state.get('trajectory_length', 0)}",
+        f"Dimensions: {dims_str}",
+        f"Recent deltas: {deltas_str}",
+    ])
+
+def map_action_type(action_type: str) -> str | None:
+    return ACTION_TYPE_MAP.get(action_type)
+
+def transform_entry(entry: dict, agent_goals: dict, valid_tools: set) -> dict | None:
+    state = entry.get("state", {})
+    action = entry.get("action", {})
+    reward = entry.get("reward", {})
+
+    action_type = action.get("type", "")
+    tool_name = map_action_type(action_type)
+    if not tool_name or tool_name not in valid_tools:
+        return None
+
+    agent_name = entry.get("agent", state.get("agent", ""))
+    goal = agent_goals.get(agent_name)
+    if not goal:
+        metric_hint = action.get("description", "")[:80]
+        goal = f"Improve codebase quality: {metric_hint}" if metric_hint else "Improve overall codebase quality"
+
+    current_state = format_state_text(state)
+
+    return {
+        "current_state": current_state,
+        "goal": goal,
+        "correct_tool": tool_name,
+        "source": "buffer",
+        "agent": agent_name,
+        "improved": reward.get("improved", False),
+        "reward_delta": reward.get("composite_delta", 0),
+    }
+
+def transform_buffer(
+    buffer_path: str,
+    agents_dir: str,
+    domain_path: str,
+    output_dir: str,
+    improved_only: bool = True,
+    seed: int = 42,
+) -> dict:
+    domain = load_domain(domain_path)
+    valid_tools = {t["name"] for t in domain["tools"]}
+    agent_goals = load_agent_goals(agents_dir)
+
+    entries = []
+    with open(buffer_path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entries.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+
+    print(f"Loaded {len(entries)} raw training buffer entries")
+    if agent_goals:
+        print(f"Agent goals: {json.dumps(agent_goals, indent=2)}")
+
+    examples = []
+    skipped_type = 0
+    skipped_improved = 0
+
+    for entry in entries:
+        transformed = transform_entry(entry, agent_goals, valid_tools)
+        if transformed is None:
+            skipped_type += 1
+            continue
+
+        if improved_only and not transformed["improved"]:
+            skipped_improved += 1
+            continue
+
+        examples.append({
+            "current_state": transformed["current_state"],
+            "goal": transformed["goal"],
+            "correct_tool": transformed["correct_tool"],
+            "source": transformed["source"],
+        })
+
+    print(f"Transformed: {len(examples)} examples")
+    print(f"Skipped: {skipped_type} (unmapped type), {skipped_improved} (not improved)")
+
+    if not examples:
+        print("No examples to write. Check action types and improved filter.")
+        return {"train": 0, "val": 0, "test": 0}
+
+    random.seed(seed)
+    random.shuffle(examples)
+
+    n = len(examples)
+    train_end = int(n * 0.7)
+    val_end = int(n * 0.85)
+
+    splits = {
+        "train": examples[:train_end],
+        "val": examples[train_end:val_end],
+        "test": examples[val_end:],
+    }
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    counts = {}
+    for split_name, split_data in splits.items():
+        path = os.path.join(output_dir, f"{split_name}.jsonl")
+        with open(path, "w") as f:
+            for ex in split_data:
+                f.write(json.dumps(ex) + "\n")
+        counts[split_name] = len(split_data)
+        print(f"  {split_name}: {len(split_data)} examples -> {path}")
+
+    # Class balance report
+    from collections import Counter
+    tool_counts = Counter(ex["correct_tool"] for ex in examples)
+    print(f"\nClass distribution:")
+    for tool, count in tool_counts.most_common():
+        pct = count / len(examples) * 100
+        print(f"  {tool:25s} {count:4d} ({pct:5.1f}%)")
+
+    return counts
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Transform v1 training buffer to v2 format")
+    parser.add_argument("--buffer", default=".jfl/training-buffer.jsonl", help="Path to training buffer JSONL")
+    parser.add_argument("--agents", default=".jfl/agents", help="Path to agents directory")
+    parser.add_argument("--domain", default=None, help="Path to domain.json")
+    parser.add_argument("--output", default=".jfl/v2-data", help="Output directory for split data")
+    parser.add_argument("--all", action="store_true", help="Include entries where improved=false")
+    parser.add_argument("--seed", type=int, default=42, help="Random seed for split")
+    args = parser.parse_args()
+
+    # Default domain path: relative to this script
+    domain_path = args.domain
+    if domain_path is None:
+        domain_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "domain.json")
+
+    if not os.path.exists(args.buffer):
+        print(f"Training buffer not found: {args.buffer}")
+        sys.exit(1)
+
+    if not os.path.exists(domain_path):
+        print(f"Domain file not found: {domain_path}")
+        sys.exit(1)
+
+    transform_buffer(
+        buffer_path=args.buffer,
+        agents_dir=args.agents,
+        domain_path=domain_path,
+        output_dir=args.output,
+        improved_only=not args.all,
+        seed=args.seed,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/train/v2/validate_data.py

@@ -0,0 +1,115 @@
+"""
+Validate v2 training data quality before training.
+Checks class balance, duplicates, empty fields, and text lengths.
+"""
+
+import json
+import sys
+import os
+import argparse
+from collections import Counter
+
+def validate_dataset(path: str) -> dict:
+    examples = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            examples.append(json.loads(line))
+
+    results = {"total": len(examples), "issues": []}
+
+    print(f"Total examples: {len(examples)}")
+
+    tool_counts = Counter(ex["correct_tool"] for ex in examples)
+    print("\nClass distribution:")
+    for tool, count in tool_counts.most_common():
+        pct = count / len(examples) * 100
+        bar = "█" * int(pct)
+        print(f"  {tool:30s} {count:4d} ({pct:5.1f}%) {bar}")
+
+    max_count = max(tool_counts.values()) if tool_counts else 0
+    min_count = min(tool_counts.values()) if tool_counts else 0
+    ratio = max_count / min_count if min_count > 0 else float("inf")
+    if ratio > 3.0:
+        msg = f"Class imbalance: {ratio:.1f}x ratio (max/min)"
+        print(f"\n  ⚠️  {msg}")
+        results["issues"].append(msg)
+    else:
+        print(f"\n  ✅ Class balance OK: {ratio:.1f}x ratio")
+
+    results["class_ratio"] = ratio
+    results["tool_counts"] = dict(tool_counts)
+
+    goal_set = set()
+    dupes = 0
+    for ex in examples:
+        key = (ex["goal"], ex["correct_tool"])
+        if key in goal_set:
+            dupes += 1
+        goal_set.add(key)
+    if dupes > 0:
+        msg = f"{dupes} duplicate (goal, tool) pairs"
+        print(f"\n  ⚠️  {msg}")
+        results["issues"].append(msg)
+    else:
+        print("\n  ✅ No duplicate (goal, tool) pairs")
+    results["duplicates"] = dupes
+
+    empty_states = sum(1 for ex in examples if not ex.get("current_state", "").strip())
+    empty_goals = sum(1 for ex in examples if not ex.get("goal", "").strip())
+    if empty_states or empty_goals:
+        msg = f"Empty fields: {empty_states} states, {empty_goals} goals"
+        print(f"\n  ⚠️  {msg}")
+        results["issues"].append(msg)
+    else:
+        print("  ✅ No empty fields")
+
+    if examples:
+        avg_state_len = sum(len(ex.get("current_state", "")) for ex in examples) / len(examples)
+        avg_goal_len = sum(len(ex.get("goal", "")) for ex in examples) / len(examples)
+        print(f"\n  Avg state length: {avg_state_len:.0f} chars")
+        print(f"  Avg goal length:  {avg_goal_len:.0f} chars")
+        results["avg_state_len"] = avg_state_len
+        results["avg_goal_len"] = avg_goal_len
+
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Validate v2 training data quality")
+    parser.add_argument("--data-dir", default=".jfl/v2-data", help="Directory with train/val/test JSONL files")
+    parser.add_argument("--json", action="store_true", help="Output results as JSON")
+    args = parser.parse_args()
+
+    all_results = {}
+    has_issues = False
+
+    for split in ["train", "val", "test"]:
+        path = os.path.join(args.data_dir, f"{split}.jsonl")
+        if not os.path.exists(path):
+            print(f"\nSkipping {split} (not found)")
+            continue
+
+        print(f"\n{'=' * 60}")
+        print(f"  {split.upper()} SPLIT")
+        print(f"{'=' * 60}")
+
+        results = validate_dataset(path)
+        all_results[split] = results
+        if results["issues"]:
+            has_issues = True
+
+    if args.json:
+        print(json.dumps(all_results, indent=2))
+
+    if has_issues:
+        print(f"\n⚠️  Issues found. Review above warnings.")
+        sys.exit(1)
+    else:
+        print(f"\n✅ All validation checks passed.")
+
+
+if __name__ == "__main__":
+    main()