@jaguilar87/gaia 5.0.2 → 5.0.5

package/tools/agentic-loop/decide-status.py

@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-"""
-decide-status.py
-
-Mechanically decide what to do based on numbers alone.  No LLM judgment.
-
-Usage:
-    python3 decide-status.py \
-        --current 94.5 \
-        --best 92.0 \
-        --threshold 98 \
-        --direction higher \
-        --consecutive-discards 2 \
-        --pivot-count 1
-
-Output JSON:
-    {
-      "decision": "keep",
-      "reason": "Metric improved from 92.0 to 94.5",
-      "improved": true,
-      "gap_remaining": 3.5
-    }
-
-Decision precedence (evaluated top-to-bottom, first match wins):
-  1. pivot_count >= 3                          → stop
-  2. consecutive_discards >= 5                 → pivot   (also a discard)
-  3. consecutive_discards >= 3                 → refine  (also a discard)
-  4. current meets or passes threshold         → threshold_reached
-  5. current improved vs best (per direction)  → keep
-  6. current same or worse                     → discard
-
-Exit codes:
-  0  success (decision emitted as JSON)
-  1  invalid input
-"""
-
-import argparse
-import json
-import sys
-
-
-Decision = str  # type alias for readability
-
-
-def _is_improved(current: float, best: float, direction: str) -> bool:
-    """Return True if *current* is strictly better than *best* per direction."""
-    if direction == "higher":
-        return current > best
-    return current < best  # lower is better
-
-
-def _threshold_reached(current: float, threshold: float, direction: str) -> bool:
-    """Return True if *current* has met or surpassed *threshold*."""
-    if direction == "higher":
-        return current >= threshold
-    return current <= threshold
-
-
-def _gap_remaining(current: float, threshold: float, direction: str) -> float:
-    """Absolute gap between current value and threshold."""
-    if direction == "higher":
-        return max(0.0, threshold - current)
-    return max(0.0, current - threshold)
-
-
-def decide(
-    current: float,
-    best: float,
-    threshold: float,
-    direction: str,
-    consecutive_discards: int,
-    pivot_count: int,
-) -> dict:
-    """Pure function: return decision dict from numeric inputs."""
-
-    gap = _gap_remaining(current, threshold, direction)
-    improved = _is_improved(current, best, direction)
-
-    # --- Precedence 1: hard stop on too many pivots ---
-    if pivot_count >= 3:
-        return {
-            "decision": "stop",
-            "reason": f"pivot_count={pivot_count} has reached the maximum of 3; halting loop",
-            "improved": improved,
-            "gap_remaining": gap,
-        }
-
-    # --- Precedence 2 & 3: discard streak escalations ---
-    # Evaluated before threshold/keep so an ongoing failing streak is flagged
-    # even if the current run happens to reach the threshold.
-    if consecutive_discards >= 5:
-        return {
-            "decision": "pivot",
-            "reason": (
-                f"consecutive_discards={consecutive_discards} >= 5; "
-                "strategy is not working, force a pivot"
-            ),
-            "improved": improved,
-            "gap_remaining": gap,
-        }
-
-    if consecutive_discards >= 3:
-        return {
-            "decision": "refine",
-            "reason": (
-                f"consecutive_discards={consecutive_discards} >= 3; "
-                "current approach needs refinement before continuing"
-            ),
-            "improved": improved,
-            "gap_remaining": gap,
-        }
-
-    # --- Precedence 4: threshold reached ---
-    if _threshold_reached(current, threshold, direction):
-        return {
-            "decision": "threshold_reached",
-            "reason": (
-                f"current={current} {'≥' if direction == 'higher' else '≤'} "
-                f"threshold={threshold}; goal achieved"
-            ),
-            "improved": improved,
-            "gap_remaining": 0.0,
-        }
-
-    # --- Precedence 5 & 6: standard keep/discard ---
-    if improved:
-        return {
-            "decision": "keep",
-            "reason": f"Metric improved from {best} to {current}",
-            "improved": True,
-            "gap_remaining": gap,
-        }
-
-    return {
-        "decision": "discard",
-        "reason": f"Metric did not improve (current={current}, best={best})",
-        "improved": False,
-        "gap_remaining": gap,
-    }
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Compute the next agentic-loop decision from metric numbers only.",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Decisions:
-  keep               current improved vs best
-  discard            current same or worse
-  refine             3+ consecutive discards (improvement needed in approach)
-  pivot              5+ consecutive discards (strategy change required)
-  stop               3+ pivots already attempted
-  threshold_reached  current meets or surpasses the goal threshold
-
-Direction values:
-  higher   larger numbers are better  (e.g. accuracy, passing tests)
-  lower    smaller numbers are better (e.g. error rate, latency ms)
-        """,
-    )
-    parser.add_argument("--current", required=True, type=float, help="Metric value for the current run")
-    parser.add_argument("--best", required=True, type=float, help="Best metric seen so far (from state.json)")
-    parser.add_argument("--threshold", required=True, type=float, help="Target threshold to reach")
-    parser.add_argument(
-        "--direction",
-        required=True,
-        choices=["higher", "lower"],
-        help="Whether higher or lower values are better",
-    )
-    parser.add_argument(
-        "--consecutive-discards",
-        required=True,
-        type=int,
-        metavar="N",
-        help="Number of consecutive discard outcomes so far (from state.json)",
-    )
-    parser.add_argument(
-        "--pivot-count",
-        required=True,
-        type=int,
-        metavar="N",
-        help="Number of pivots executed so far (from state.json)",
-    )
-    args = parser.parse_args()
-
-    # --- Input validation ---
-    errors = []
-    if args.consecutive_discards < 0:
-        errors.append("--consecutive-discards must be >= 0")
-    if args.pivot_count < 0:
-        errors.append("--pivot-count must be >= 0")
-
-    if errors:
-        for err in errors:
-            print(f"error: {err}", file=sys.stderr)
-        sys.exit(1)
-
-    result = decide(
-        current=args.current,
-        best=args.best,
-        threshold=args.threshold,
-        direction=args.direction,
-        consecutive_discards=args.consecutive_discards,
-        pivot_count=args.pivot_count,
-    )
-
-    print(json.dumps(result, indent=2))
-
-
-if __name__ == "__main__":
-    main()

package/tools/agentic-loop/parse-metric.py

@@ -1,106 +0,0 @@
-#!/usr/bin/env python3
-"""
-parse-metric.py
-
-Read stdout from eval_command and extract METRIC lines.
-
-Usage:
-    echo "output" | python3 parse-metric.py --metric accuracy
-    python3 parse-metric.py --metric accuracy --file /tmp/eval-output.txt
-
-Input lines must match: METRIC {name}={number}
-Output: JSON to stdout with metric name, numeric value, and raw line.
-"""
-
-import argparse
-import json
-import re
-import sys
-from typing import Optional
-
-
-METRIC_PATTERN = re.compile(r"^METRIC\s+(\w+)=([\d.]+)\s*$")
-
-
-def parse_lines(lines: list[str]) -> list[dict]:
-    """Extract all METRIC entries from a sequence of lines."""
-    results = []
-    for line in lines:
-        stripped = line.rstrip("\n")
-        match = METRIC_PATTERN.match(stripped)
-        if match:
-            name = match.group(1)
-            raw_value = match.group(2)
-            # Preserve int vs float from the source text.
-            value: int | float
-            if "." in raw_value:
-                value = float(raw_value)
-            else:
-                value = int(raw_value)
-            results.append(
-                {
-                    "metric": name,
-                    "value": value,
-                    "raw_line": stripped,
-                }
-            )
-    return results
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Extract METRIC lines from eval_command output.",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  echo "METRIC accuracy=94.5" | python3 parse-metric.py --metric accuracy
-  python3 parse-metric.py --metric passing_tests --file /tmp/out.txt
-  python3 parse-metric.py --file /tmp/out.txt          # returns all metrics
-        """,
-    )
-    parser.add_argument(
-        "--metric",
-        metavar="NAME",
-        help="Return only this named metric (case-sensitive). Exits 1 if not found.",
-    )
-    parser.add_argument(
-        "--file",
-        metavar="PATH",
-        help="Read from file instead of stdin.",
-    )
-    args = parser.parse_args()
-
-    # --- Read input ---
-    try:
-        if args.file:
-            with open(args.file, "r") as fh:
-                lines = fh.readlines()
-        else:
-            lines = sys.stdin.readlines()
-    except OSError as exc:
-        print(f"error: cannot read input: {exc}", file=sys.stderr)
-        sys.exit(1)
-
-    # --- Parse ---
-    all_metrics = parse_lines(lines)
-
-    if args.metric:
-        # Filter to the requested metric name.
-        matches = [m for m in all_metrics if m["metric"] == args.metric]
-        if not matches:
-            print(
-                f"error: metric '{args.metric}' not found in input",
-                file=sys.stderr,
-            )
-            sys.exit(1)
-        # Return the last occurrence if there are duplicates.
-        result = matches[-1]
-    else:
-        # Return all metrics as a list when no --metric filter is given.
-        result = all_metrics  # type: ignore[assignment]
-
-    print(json.dumps(result, indent=2))
-
-
-if __name__ == "__main__":
-    main()

package/tools/agentic-loop/record-iteration.py

@@ -1,223 +0,0 @@
-#!/usr/bin/env python3
-"""
-record-iteration.py
-
-Atomically update state.json and append to worklog.md after each iteration.
-The LLM never writes state.json directly — this script is the only writer.
-
-Usage:
-    python3 record-iteration.py \
-        --state-file state.json \
-        --worklog worklog.md \
-        --iteration 5 \
-        --metric-value 94.5 \
-        --status keep \
-        --description "Handle hyphenated verbs" \
-        --insight "delete-objects splits correctly" \
-        --next "Check camelCase+hyphen combined"
-
-    Optional flags:
-        --changed TEXT      What was modified (default: same as description)
-        --metric-name TEXT  Name of the metric recorded (default: "metric")
-
-Atomic write guarantee: state.json is written to a .tmp sibling, fsynced,
-then renamed over the original.  Either the full write lands or the original
-is untouched.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sys
-import tempfile
-from datetime import datetime, timezone
-
-
-def load_state(path: str) -> dict:
-    """Load existing state.json or return an empty skeleton."""
-    if not os.path.exists(path):
-        return {
-            "iteration": 0,
-            "current_metric": None,
-            "best_metric": None,
-            "consecutive_discards": 0,
-            "pivot_count": 0,
-            "timestamp": None,
-            "status": None,
-        }
-    try:
-        with open(path, "r") as fh:
-            data = json.load(fh)
-        return data
-    except (OSError, json.JSONDecodeError) as exc:
-        print(f"error: cannot read state file '{path}': {exc}", file=sys.stderr)
-        sys.exit(1)
-
-
-def atomic_write_json(path: str, data: dict) -> None:
-    """Write *data* to *path* atomically using write-fsync-rename."""
-    dir_name = os.path.dirname(os.path.abspath(path))
-    # Use a temp file in the same directory so rename is on the same filesystem.
-    try:
-        fd, tmp_path = tempfile.mkstemp(dir=dir_name, suffix=".tmp")
-        try:
-            with os.fdopen(fd, "w") as fh:
-                json.dump(data, fh, indent=2)
-                fh.write("\n")
-                fh.flush()
-                os.fsync(fh.fileno())
-            os.replace(tmp_path, path)
-        except Exception:
-            # Clean up orphaned temp file on failure.
-            try:
-                os.unlink(tmp_path)
-            except OSError:
-                pass
-            raise
-    except OSError as exc:
-        print(f"error: atomic write to '{path}' failed: {exc}", file=sys.stderr)
-        sys.exit(1)
-
-
-def append_worklog(
-    path: str,
-    iteration: int,
-    description: str,
-    metric_name: str,
-    metric_value: float,
-    status: str,
-    changed: str,
-    insight: str,
-    next_step: str,
-    best_metric: float | None,
-) -> None:
-    """Append a structured run entry to worklog.md."""
-    status_upper = status.upper()
-
-    # Build result sentence
-    if best_metric is None:
-        result_text = f"{metric_name}={metric_value} (first run, no prior best)"
-    else:
-        comparison = (
-            f"improved from {best_metric}"
-            if metric_value > best_metric
-            else (
-                f"unchanged from {best_metric}"
-                if metric_value == best_metric
-                else f"regressed from {best_metric}"
-            )
-        )
-        result_text = f"{metric_name}={metric_value} ({comparison})"
-
-    entry = (
-        f"\n### Run {iteration}: {description} — {metric_name}={metric_value} ({status_upper})\n"
-        f"- **Changed:** {changed}\n"
-        f"- **Result:** {result_text}\n"
-        f"- **Insight:** {insight}\n"
-        f"- **Next:** {next_step}\n"
-    )
-
-    try:
-        with open(path, "a") as fh:
-            fh.write(entry)
-    except OSError as exc:
-        print(f"error: cannot append to worklog '{path}': {exc}", file=sys.stderr)
-        sys.exit(1)
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Atomically record an agentic-loop iteration into state.json and worklog.md.",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Status values:
-  keep     — metric improved; best is updated, consecutive_discards reset to 0
-  discard  — metric did not improve; consecutive_discards incremented
-  pivot    — forced strategy change (also increments pivot_count)
-  stop     — terminal state; loop should halt
-
-Exit codes:
-  0  success
-  1  error (message on stderr)
-        """,
-    )
-    parser.add_argument("--state-file", required=True, metavar="PATH", help="Path to state.json")
-    parser.add_argument("--worklog", required=True, metavar="PATH", help="Path to worklog.md (append-only)")
-    parser.add_argument("--iteration", required=True, type=int, help="Current iteration number (1-based)")
-    parser.add_argument("--metric-value", required=True, type=float, metavar="NUM", help="Numeric metric value this run")
-    parser.add_argument(
-        "--status",
-        required=True,
-        choices=["keep", "discard", "pivot", "stop"],
-        help="Outcome classification for this iteration",
-    )
-    parser.add_argument("--description", required=True, help="Short description of what changed this run")
-    parser.add_argument("--insight", required=True, help="What was learned from this run")
-    parser.add_argument("--next", required=True, dest="next_step", help="What to try in the next iteration")
-    parser.add_argument(
-        "--changed",
-        default=None,
-        metavar="TEXT",
-        help="What was specifically modified (defaults to --description)",
-    )
-    parser.add_argument(
-        "--metric-name",
-        default="metric",
-        metavar="NAME",
-        help="Name label for the metric (default: metric)",
-    )
-    args = parser.parse_args()
-
-    changed = args.changed if args.changed is not None else args.description
-
-    # --- Load current state ---
-    state = load_state(args.state_file)
-
-    prev_best: float | None = state.get("best_metric")
-
-    # --- Compute new state values ---
-    state["iteration"] = args.iteration
-    state["current_metric"] = args.metric_value
-    state["status"] = args.status
-    state["timestamp"] = datetime.now(tz=timezone.utc).isoformat()
-
-    if args.status == "keep":
-        # Keep: this run is better; promote to best.
-        state["best_metric"] = args.metric_value
-        state["consecutive_discards"] = 0
-    elif args.status == "discard":
-        # Do not update best; increment discard counter.
-        state["consecutive_discards"] = int(state.get("consecutive_discards") or 0) + 1
-    elif args.status == "pivot":
-        # Pivot: counts as a discard for streak purposes, but also advances pivot_count.
-        state["consecutive_discards"] = int(state.get("consecutive_discards") or 0) + 1
-        state["pivot_count"] = int(state.get("pivot_count") or 0) + 1
-    elif args.status == "stop":
-        # Terminal — no counter changes needed beyond recording.
-        pass
-
-    # --- Atomic write ---
-    atomic_write_json(args.state_file, state)
-
-    # --- Append worklog ---
-    append_worklog(
-        path=args.worklog,
-        iteration=args.iteration,
-        description=args.description,
-        metric_name=args.metric_name,
-        metric_value=args.metric_value,
-        status=args.status,
-        changed=changed,
-        insight=args.insight,
-        next_step=args.next_step,
-        best_metric=prev_best,
-    )
-
-    # Emit updated state summary for easy inspection.
-    print(json.dumps(state, indent=2))
-
-
-if __name__ == "__main__":
-    main()