@jenga-ai/agent 1.0.1 → 1.1.1

package/skills/train/assets/results-parsers/classifiers.py

@@ -1,84 +0,0 @@
-"""
-classifiers.py — Results parser for sklearn-based classifier jobs.
-
-Extracts: accuracy, precision, recall, f1_score, model_type.
-Reads from results.json or scans training-results/ for JSON files.
-"""
-import json
-from pathlib import Path
-
-
-def parse(job_dir: Path) -> dict:
-    """
-    Parse training results for a classifiers job.
-
-    Returns a dict with keys:
-        accuracy, precision, recall, f1_score, model_type
-    Returns empty dict if no results can be found; never raises.
-    """
-    job_dir = Path(job_dir)
-    metrics = {}
-
-    # 1. Try results.json at root
-    results_json = job_dir / "results.json"
-    if results_json.exists():
-        try:
-            data = json.loads(results_json.read_text())
-            metrics = _extract_from_dict(data)
-            if metrics:
-                return metrics
-        except Exception:
-            pass
-
-    # 2. Scan training-results/ for any JSON files
-    training_results_dir = job_dir / "training-results"
-    if training_results_dir.exists():
-        for json_file in sorted(training_results_dir.rglob("*.json")):
-            try:
-                data = json.loads(json_file.read_text())
-                metrics = _extract_from_dict(data)
-                if metrics:
-                    return metrics
-            except Exception:
-                continue
-
-    return metrics
-
-
-def _extract_from_dict(data: dict) -> dict:
-    """Extract classifier-relevant keys from a parsed dict."""
-    metrics = {}
-    if not isinstance(data, dict):
-        return metrics
-
-    # Common field names produced by sklearn classification_report / custom scripts
-    field_map = {
-        "accuracy": ["accuracy", "test_accuracy", "val_accuracy", "acc"],
-        "precision": ["precision", "weighted avg.precision", "macro avg.precision"],
-        "recall": ["recall", "weighted avg.recall", "macro avg.recall"],
-        "f1_score": ["f1_score", "f1", "weighted avg.f1-score", "macro avg.f1-score"],
-        "model_type": ["model_type", "model", "algorithm", "classifier"],
-    }
-
-    for target_key, candidates in field_map.items():
-        for candidate in candidates:
-            # Support dot-path lookup (e.g. "weighted avg.precision")
-            value = _deep_get(data, candidate)
-            if value is not None:
-                metrics[target_key] = value
-                break
-
-    return metrics
-
-
-def _deep_get(data: dict, dotted_key: str):
-    """Retrieve a value from a nested dict using a dot-separated key path."""
-    keys = dotted_key.split(".")
-    current = data
-    for k in keys:
-        if not isinstance(current, dict):
-            return None
-        current = current.get(k)
-        if current is None:
-            return None
-    return current

package/skills/train/assets/results-parsers/nlp.py

@@ -1,88 +0,0 @@
-"""
-nlp.py — Results parser for spaCy / NLP pipeline training jobs.
-
-Extracts: f1, precision, recall, model_name, task, iterations.
-Reads from results.json or scans training-results/ for JSON files.
-"""
-import json
-from pathlib import Path
-
-
-def parse(job_dir: Path) -> dict:
-    """
-    Parse training results for an nlp job.
-
-    Returns a dict with keys:
-        f1, precision, recall, model_name, task, iterations
-    Returns empty dict if no results can be found; never raises.
-    """
-    job_dir = Path(job_dir)
-    metrics = {}
-
-    # 1. Try results.json at root
-    results_json = job_dir / "results.json"
-    if results_json.exists():
-        try:
-            data = json.loads(results_json.read_text())
-            metrics = _extract_from_dict(data)
-            if metrics:
-                return metrics
-        except Exception:
-            pass
-
-    # 2. Scan training-results/ for JSON files
-    training_results_dir = job_dir / "training-results"
-    if training_results_dir.exists():
-        for json_file in sorted(training_results_dir.rglob("*.json")):
-            try:
-                data = json.loads(json_file.read_text())
-                metrics = _extract_from_dict(data)
-                if metrics:
-                    return metrics
-            except Exception:
-                continue
-
-    # 3. Try spaCy training output (scores.json or metrics.json)
-    for scores_file in sorted(job_dir.rglob("scores.json")) + sorted(job_dir.rglob("metrics.json")):
-        try:
-            data = json.loads(scores_file.read_text())
-            metrics = _extract_from_dict(data)
-            if metrics:
-                return metrics
-        except Exception:
-            continue
-
-    return metrics
-
-
-def _extract_from_dict(data: dict) -> dict:
-    """Extract NLP-relevant keys from a parsed dict."""
-    metrics = {}
-    if not isinstance(data, dict):
-        return metrics
-
-    field_map = {
-        "f1": ["f1", "f1_score", "ents_f", "token_f", "tag_f", "sents_f", "score"],
-        "precision": ["precision", "ents_p", "token_p", "tag_p"],
-        "recall": ["recall", "ents_r", "token_r", "tag_r"],
-        "model_name": ["model_name", "model", "base_model"],
-        "task": ["task", "pipeline_component", "component"],
-        "iterations": ["iterations", "n_iter", "steps", "batches_trained"],
-    }
-
-    for target_key, candidates in field_map.items():
-        for candidate in candidates:
-            value = data.get(candidate)
-            if value is None:
-                # Try nested under "scores" or "results" key
-                for wrapper in ("scores", "results", "metrics"):
-                    nested = data.get(wrapper, {})
-                    if isinstance(nested, dict):
-                        value = nested.get(candidate)
-                    if value is not None:
-                        break
-            if value is not None:
-                metrics[target_key] = value
-                break
-
-    return metrics

package/skills/train/assets/results-parsers/reporter.py

@@ -1,154 +0,0 @@
-"""
-reporter.py — Surface training results to terminal, summary.md, and results.json.
-
-Usage:
-    from skills.train.assets.results_parsers.reporter import surface_results
-    surface_results(job_dir, job_type)
-"""
-import importlib
-import json
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-
-_PARSER_PKG = Path(__file__).resolve().parent
-sys.path.insert(0, str(_PARSER_PKG.parent.parent.parent))  # ensure skills/ is on path
-
-
-def _load_parser(job_type: str):
-    """Dynamically import the type-specific parser module."""
-    try:
-        spec_path = _PARSER_PKG / f"{job_type}.py"
-        import importlib.util
-        spec = importlib.util.spec_from_file_location(job_type, spec_path)
-        mod = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(mod)
-        return mod
-    except Exception as e:
-        return None
-
-
-def _format_value(v) -> str:
-    if isinstance(v, float):
-        return f"{v:.4f}"
-    return str(v)
-
-
-def _print_terminal_summary(job_dir: Path, job_type: str, metrics: dict):
-    """Print a box-formatted summary to stdout."""
-    job_name = job_dir.name
-    width = 54
-    border = "─" * width
-    print(f"\n┌{border}┐")
-    print(f"│  📊 Training Summary — {job_name:<{width - 25}}│")
-    print(f"│  Type: {job_type:<{width - 9}}│")
-    print(f"├{border}┤")
-    if metrics:
-        for k, v in metrics.items():
-            label = k.replace("_", " ").title()
-            value = _format_value(v)
-            line = f"  {label}: {value}"
-            print(f"│{line:<{width + 1}}│")
-    else:
-        print(f"│  (no metrics found — check training-results/ or results.json)  │")
-    print(f"└{border}┘\n")
-
-
-def _write_summary_md(job_dir: Path, job_type: str, metrics: dict):
-    """Write a Markdown summary file to the job directory."""
-    job_name = job_dir.name
-    timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
-    lines = [
-        f"# Training Summary — {job_name}",
-        f"",
-        f"**Type:** {job_type}  ",
-        f"**Generated:** {timestamp}  ",
-        f"",
-        f"## Metrics",
-        f"",
-        f"| Metric | Value |",
-        f"|--------|-------|",
-    ]
-    if metrics:
-        for k, v in metrics.items():
-            label = k.replace("_", " ").title()
-            lines.append(f"| {label} | {_format_value(v)} |")
-    else:
-        lines.append("| — | No metrics found |")
-
-    summary_path = job_dir / "summary.md"
-    summary_path.write_text("\n".join(lines) + "\n")
-    return summary_path
-
-
-def _write_results_json(job_dir: Path, job_type: str, metrics: dict):
-    """Write or update results.json in the job directory."""
-    results_path = job_dir / "results.json"
-
-    # Merge with existing results.json if present
-    existing = {}
-    if results_path.exists():
-        try:
-            existing = json.loads(results_path.read_text())
-        except Exception:
-            pass
-
-    existing.update({
-        "job_name": job_dir.name,
-        "job_type": job_type,
-        "generated_at": datetime.now(timezone.utc).isoformat(),
-        "metrics": metrics,
-    })
-
-    results_path.write_text(json.dumps(existing, indent=2))
-    return results_path
-
-
-def surface_results(job_dir, job_type: str):
-    """
-    Parse training results for the given job and surface them:
-    1. Print terminal summary
-    2. Write summary.md to job directory
-    3. Write/update results.json in job directory
-
-    Args:
-        job_dir: Path (or str) to the job directory
-        job_type: One of 'classifiers', 'transformers', 'nlp'
-    """
-    job_dir = Path(job_dir)
-
-    parser = _load_parser(job_type)
-    if parser is None:
-        print(f"⚠️  No parser found for type '{job_type}' — skipping result surfacing.")
-        return {}
-
-    try:
-        metrics = parser.parse(job_dir) or {}
-    except Exception as e:
-        print(f"⚠️  Parser error for '{job_type}': {e}")
-        metrics = {}
-
-    _print_terminal_summary(job_dir, job_type, metrics)
-
-    try:
-        summary_path = _write_summary_md(job_dir, job_type, metrics)
-        print(f"📄 summary.md written to {summary_path}")
-    except Exception as e:
-        print(f"⚠️  Could not write summary.md: {e}")
-
-    try:
-        results_path = _write_results_json(job_dir, job_type, metrics)
-        print(f"📄 results.json written to {results_path}")
-    except Exception as e:
-        print(f"⚠️  Could not write results.json: {e}")
-
-    return metrics
-
-
-if __name__ == "__main__":
-    import argparse
-    p = argparse.ArgumentParser(description="Surface training results for a job directory.")
-    p.add_argument("job_dir", help="Path to the job directory")
-    p.add_argument("job_type", choices=["classifiers", "transformers", "nlp"])
-    ns = p.parse_args()
-    surface_results(ns.job_dir, ns.job_type)

package/skills/train/assets/results-parsers/transformers.py

@@ -1,120 +0,0 @@
-"""
-transformers.py — Results parser for HuggingFace Transformers fine-tuning jobs.
-
-Extracts: perplexity, eval_loss, train_loss, epochs, model_name.
-Reads from results.json or scans training-results/ and trainer_state.json.
-"""
-import json
-import math
-from pathlib import Path
-
-
-def parse(job_dir: Path) -> dict:
-    """
-    Parse training results for a transformers job.
-
-    Returns a dict with keys:
-        perplexity, eval_loss, train_loss, epochs, model_name
-    Returns empty dict if no results can be found; never raises.
-    """
-    job_dir = Path(job_dir)
-    metrics = {}
-
-    # 1. Try results.json at root
-    results_json = job_dir / "results.json"
-    if results_json.exists():
-        try:
-            data = json.loads(results_json.read_text())
-            metrics = _extract_from_dict(data)
-            if metrics:
-                return metrics
-        except Exception:
-            pass
-
-    # 2. Try trainer_state.json (HuggingFace Trainer output)
-    for trainer_state in sorted(job_dir.rglob("trainer_state.json")):
-        try:
-            data = json.loads(trainer_state.read_text())
-            metrics = _extract_from_trainer_state(data)
-            if metrics:
-                return metrics
-        except Exception:
-            continue
-
-    # 3. Scan training-results/ for JSON files
-    training_results_dir = job_dir / "training-results"
-    if training_results_dir.exists():
-        for json_file in sorted(training_results_dir.rglob("*.json")):
-            try:
-                data = json.loads(json_file.read_text())
-                metrics = _extract_from_dict(data)
-                if metrics:
-                    return metrics
-            except Exception:
-                continue
-
-    return metrics
-
-
-def _extract_from_dict(data: dict) -> dict:
-    """Extract transformer-relevant keys from a parsed dict."""
-    metrics = {}
-    if not isinstance(data, dict):
-        return metrics
-
-    field_map = {
-        "eval_loss": ["eval_loss", "validation_loss", "val_loss"],
-        "train_loss": ["train_loss", "training_loss", "loss"],
-        "epochs": ["epochs", "num_train_epochs", "epoch"],
-        "model_name": ["model_name", "model", "base_model", "pretrained_model_name_or_path"],
-        "perplexity": ["perplexity", "eval_perplexity"],
-    }
-
-    for target_key, candidates in field_map.items():
-        for candidate in candidates:
-            value = data.get(candidate)
-            if value is not None:
-                metrics[target_key] = value
-                break
-
-    # Derive perplexity from eval_loss if not already present
-    if "perplexity" not in metrics and "eval_loss" in metrics:
-        try:
-            metrics["perplexity"] = round(math.exp(float(metrics["eval_loss"])), 4)
-        except (ValueError, OverflowError):
-            pass
-
-    return metrics
-
-
-def _extract_from_trainer_state(data: dict) -> dict:
-    """Extract metrics from HuggingFace Trainer's trainer_state.json."""
-    metrics = {}
-    if not isinstance(data, dict):
-        return metrics
-
-    # Best metrics
-    best_metric = data.get("best_metric")
-    if best_metric is not None:
-        metrics["eval_loss"] = best_metric
-
-    # Epoch count
-    epoch = data.get("epoch")
-    if epoch is not None:
-        metrics["epochs"] = epoch
-
-    # Last eval from log history
-    log_history = data.get("log_history", [])
-    for entry in reversed(log_history):
-        if "eval_loss" in entry:
-            metrics["eval_loss"] = entry["eval_loss"]
-            break
-
-    # Derive perplexity
-    if "perplexity" not in metrics and "eval_loss" in metrics:
-        try:
-            metrics["perplexity"] = round(math.exp(float(metrics["eval_loss"])), 4)
-        except (ValueError, OverflowError):
-            pass
-
-    return metrics