@giamme/forge 0.0.0-stage → 0.9.0

package/scripts/forge_jev/review.py

@@ -0,0 +1,73 @@
+"""Post-review annotation of a FAIL.
+
+Reads a reviewer's FAIL and says whether it looks sound. It cannot change the verdict,
+and no caller is given a way to. Forge's second invariant is reviewer independence: a
+model that can talk another model out of a FAIL is worth less than no reviewer at all,
+because what it produces is a confident PASS on code nobody checked.
+
+So the output is an annotation for a human, and the status file is never touched. The
+asymmetry is deliberate in the other direction too -- nothing here ever annotates a PASS
+into looking suspect, because a PASS that gets merged is the outcome that needs the
+reviewer's judgement left alone.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from . import threshold
+from .client import ask, noul
+from .questions import failure_is_correctness, findings_cite_the_diff
+
+# A diff large enough to exceed this is one no reviewer read carefully either. Truncating
+# is honest: the annotation is advisory, and a partial read is flagged in the output.
+DIFF_CHARS = 24000
+REVIEW_CHARS = 12000
+
+
+def _read(path, limit: int) -> tuple[str, bool]:
+    try:
+        text = Path(path).read_text(errors='replace')
+    except OSError:
+        return '', False
+    return text[:limit], len(text) > limit
+
+
+def annotate_failure(*, review_path, diff_path, run_dir=None, config: dict | None = None):
+    """Judge a FAIL. Returns None when no judgment could be made.
+
+    Both questions travel in one request. The result carries `suspect` when either
+    probability falls below `gate_warn`, which is the only thing a caller should branch
+    on -- and the only branch available is what to print.
+    """
+    review, review_truncated = _read(review_path, REVIEW_CHARS)
+    diff, diff_truncated = _read(diff_path, DIFF_CHARS)
+    if not review.strip() or not diff.strip():
+        return None
+
+    questions = dict(correctness=failure_is_correctness(), citation=findings_cite_the_diff())
+    state = dict(review=review, diff=diff,
+                 truncated=dict(review=review_truncated, diff=diff_truncated))
+    result = ask(state, questions, site='jev-review-triage', run_dir=run_dir, config=config)
+    if result is None:
+        return None
+
+    correctness = noul(result, 'correctness')
+    citation = noul(result, 'citation')
+    if correctness is None and citation is None:
+        return None
+
+    warn_at = threshold('gate_warn', config=config)
+    reasons = []
+    citation_bad = citation is not None and citation < warn_at
+    if citation_bad:
+        reasons.append('the review cites code that may not be in this diff')
+    elif correctness is not None and correctness < warn_at:
+        # Only when the citations hold up. A review that points at a file the diff does
+        # not contain scores low on correctness too, but reporting that as "these are
+        # style preferences" describes the wrong problem: nothing can be said about the
+        # correctness of code that is not there. Measured on a hallucinated review --
+        # citation 0.02, correctness 0.45 -- where both reasons printed and one was wrong.
+        reasons.append('the findings look like style preferences rather than correctness bugs')
+    return dict(correctness=correctness, citation=citation, suspect=bool(reasons),
+                reasons=reasons, truncated=review_truncated or diff_truncated)
+

package/scripts/forge_jev/routing.py

@@ -0,0 +1,271 @@
+"""Plan-time difficulty scoring.
+
+Jev rates five properties of a task; **this file**, not the model, turns them into a
+tier. That split is the whole design. Weights below can be re-tuned against accrued
+outcomes without spending a single request, because re-weighting replays recorded
+scores rather than re-asking.
+
+Nothing here writes to tasks.tsv. Scoring is advisory at best and silent in shadow
+mode; see references/jev.md for how the result reaches (or does not reach) a decision.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from . import enabled, threshold
+from .client import ask, noul, score
+from .questions import (blast_radius, design_judgment, independently_verifiable,
+                        prompt_adequacy, spec_clarity, state_subtlety, test_coverage)
+
+# Weights sum to 1.0. design_judgment carries the most because decompose.md defines
+# difficulty by it first ("rated ... by what the task demands of the model"), and
+# state_subtlety comes second because that same table names concurrency explicitly as a
+# thing that makes a ten-line diff `high`.
+#
+# test_coverage is inverted -- good coverage LOWERS the tier -- and weighted lightly on
+# purpose. Coverage does not reduce the judgment a task demands; it only reduces what a
+# mistake costs. Letting it dominate would route a genuinely hard change to a cheap
+# model because the area happens to be well tested.
+WEIGHTS = dict(design_judgment=0.35, state_subtlety=0.25, blast_radius=0.20,
+               spec_clarity=0.15, test_coverage=-0.05)
+
+QUESTIONS = dict(design_judgment=design_judgment, state_subtlety=state_subtlety,
+                 blast_radius=blast_radius, spec_clarity=spec_clarity,
+                 test_coverage=test_coverage)
+
+# Composite cut points. A task exactly on a boundary is the case most likely to be wrong
+# in either direction, so BOUNDARY_BAND either side of a cut escalates a tier: paying
+# for a stronger model on an ambiguous task is the cheap mistake, and under-routing a
+# hard task is the expensive one.
+LOW_MAX = 0.33
+MEDIUM_MAX = 0.63
+BOUNDARY_BAND = 0.05
+
+# Below this, the judgment is too uncertain to act on in either direction, so it
+# escalates for the same reason a boundary does.
+CONFIDENCE_FLOOR = 0.55
+
+FILE_EXCERPT_CHARS = 600
+MAX_EXCERPT_FILES = 6
+MEMORY_CHARS = 4000
+
+
+def _normalized(result, qid: str, levels: int):
+    """A Score's position mapped onto 0.0-1.0, with its confidence.
+
+    Score returns a probability-weighted position across ordered levels, so a task
+    sitting between 'some' and 'substantial' lands between them rather than being forced
+    onto one. Dividing by (levels - 1) makes rubrics with different level counts
+    comparable before they are weighted against each other.
+    """
+    position, confidence = score(result, qid)
+    if position is None or levels < 2:
+        return None, None
+    return max(0.0, min(1.0, float(position) / (levels - 1))), confidence
+
+
+def tier_for(composite: float) -> str:
+    if composite < LOW_MAX:
+        return 'low'
+    if composite < MEDIUM_MAX:
+        return 'medium'
+    return 'high'
+
+
+def _escalate(tier: str) -> str:
+    return dict(low='medium', medium='high', high='high')[tier]
+
+
+def _near_boundary(composite: float) -> bool:
+    """Is this close enough to a cut, from BELOW, that it should round up to it?
+
+    Only from below. A composite just above a cut is already on the higher side of that
+    boundary, and escalating it again promotes it a whole tier past the line it was
+    merely near -- measured on a real plan, a task at 0.373 sat 0.043 above LOW_MAX,
+    scored `medium`, and was escalated to `high`. Rounding up to a boundary is a
+    tie-break; jumping the next one is not a tie-break at all.
+    """
+    # Strictly below: at exactly the cut, tier_for already returns the higher tier.
+    return any(0.0 < cut - composite <= BOUNDARY_BAND for cut in (LOW_MAX, MEDIUM_MAX))
+
+
+def combine(dimensions: dict) -> dict | None:
+    """Weighted composite -> tier, plus why it may have been escalated.
+
+    `dimensions` maps a rubric name to (normalized_position, confidence). A missing
+    dimension is not fatal: the remaining weights are renormalized, because a partial
+    answer that is honest about being partial beats refusing to say anything.
+    """
+    usable = {name: value for name, value in dimensions.items()
+              if value and value[0] is not None}
+    if not usable:
+        return None
+    total_weight = sum(abs(WEIGHTS[name]) for name in usable)
+    if not total_weight:
+        return None
+
+    # test_coverage's weight is negative, so its contribution is inverted here: a
+    # well-covered surface pulls the composite down.
+    raw = 0.0
+    for name, (position, _confidence) in usable.items():
+        weight = WEIGHTS[name]
+        raw += (1.0 - position) * abs(weight) if weight < 0 else position * weight
+    composite = max(0.0, min(1.0, raw / total_weight))
+
+    # Confidence is weighted by the same weights as the composite, because that is what
+    # it is the confidence OF. Measured: taking the minimum instead let one low-weight
+    # dimension veto a well-determined answer -- on a real task, blast_radius came back
+    # at 0.0 while design_judgment, which carries 0.35, was 0.94, and the minimum
+    # reported 0.0 for a composite that was mostly settled. The weakest dimension is
+    # still reported separately as `weakest`, since it is the one worth looking at.
+    scored = [(name, value) for name, value in usable.items() if value[1] is not None]
+    confidence = weakest = None
+    if scored:
+        weight_sum = sum(abs(WEIGHTS[name]) for name, _ in scored)
+        confidence = round(sum(abs(WEIGHTS[name]) * value[1]
+                               for name, value in scored) / weight_sum, 3)
+        weakest = min(scored, key=lambda item: item[1][1])[0]
+
+    tier = tier_for(composite)
+    escalated = None
+    if _near_boundary(composite):
+        escalated = 'boundary'
+    elif confidence is not None and confidence < CONFIDENCE_FLOOR:
+        escalated = 'low-confidence'
+    if escalated:
+        tier = _escalate(tier)
+    return dict(tier=tier, composite=round(composite, 3), confidence=confidence,
+                weakest=weakest, escalated=escalated, base_tier=tier_for(composite),
+                dimensions={name: round(value[0], 3) for name, value in usable.items()},
+                missing=sorted(set(WEIGHTS) - set(usable)))
+
+
+def _excerpt(repo: Path, files: str) -> dict:
+    out = {}
+    for rel in (files or '').replace(',', ' ').split():
+        if len(out) >= MAX_EXCERPT_FILES:
+            break
+        path = repo / rel
+        if not path.is_file():
+            # A file the task will CREATE is normal and informative in itself.
+            out[rel] = '(does not exist yet)'
+            continue
+        try:
+            out[rel] = path.read_text(errors='replace')[:FILE_EXCERPT_CHARS]
+        except OSError:
+            continue
+    return out
+
+
+def _memory(repo: Path) -> str:
+    path = repo / '.forge' / 'memory.md'
+    try:
+        return path.read_text(errors='replace')[:MEMORY_CHARS] if path.is_file() else ''
+    except OSError:
+        return ''
+
+
+# The other tasks in the plan, as sent to the model. Capped because a large plan would
+# otherwise dominate the payload, and truncated per field for the same reason: the
+# question these answer is "does this task need one of those to land first", which needs
+# their identity and file scope, not their full text.
+MAX_SIBLINGS = 40
+SIBLING_TITLE_CHARS = 120
+
+
+def _siblings(siblings, task_id: str) -> list[dict]:
+    """Compact view of the rest of the plan, excluding the task being scored."""
+    out = []
+    for sibling in siblings or []:
+        if sibling.get('id') == task_id:
+            continue
+        out.append(dict(id=sibling.get('id', ''),
+                        title=str(sibling.get('title', ''))[:SIBLING_TITLE_CHARS],
+                        files=sibling.get('files', ''),
+                        deps=sibling.get('deps', '')))
+        if len(out) >= MAX_SIBLINGS:
+            break
+    return out
+
+
+def score_task(repo, *, goal: str, task_id: str, title: str, files: str,
+               approach: str = '', prompt: str = '', declared_difficulty: str = '',
+               deps: str = '', siblings: list[dict] | None = None,
+               run_dir=None, config: dict | None = None) -> dict | None:
+    """One request per task. All five rubrics evaluate in parallel server-side.
+
+    Returns None whenever anything at all goes wrong, because a plan that cannot be
+    scored must still be a plan.
+    """
+    repo_path = Path(repo)
+    questions = {name: builder() for name, builder in QUESTIONS.items()}
+    # `prompt` is the task's requirements -- tasks/<id>/prompt.md, the text the dwarf is
+    # dispatched with and the reviewer reads. It was missing from this state until a real
+    # plan was scored, and its absence was not visible from any unit test: every rubric
+    # still answered, plausibly, about the one-line title.
+    #
+    # It mattered most to the gate that exists to catch under-specified tasks. Asked
+    # whether "the task states requirements specific enough that a reviewer could judge
+    # correctness", with only a title in state, the honest answer is no -- so a 460-word
+    # spec with a declared output shape and named edge cases scored 0.23 and tripped its
+    # own warning. The gate could not have done anything else; it was never shown the
+    # requirements it was asked about.
+    # The rest of the plan. independently_verifiable asks whether confirming this task
+    # needs "work declared in a different task" -- and until now no different task was
+    # ever in state, so the model could only guess whether such a task existed. That is
+    # why its two classes overlapped (0.34-0.56 unverifiable against 0.43-0.90
+    # verifiable) and no threshold divided them: same failure as prompt.md, a rubric
+    # scored against evidence it was never shown.
+    plan_tasks = _siblings(siblings, task_id)
+    state = dict(goal=goal, task=dict(id=task_id, title=title, files=files, deps=deps),
+                 prompt=prompt, approach=approach, plan_tasks=plan_tasks,
+                 file_excerpts=_excerpt(repo_path, files), memory=_memory(repo_path))
+    # The planner's own rating is deliberately NOT sent. Jev is here to be a second,
+    # independent opinion on the same evidence; showing it the answer first would buy an
+    # agreement rate instead of a judgment.
+    # Phase 4's plan-time gates need exactly this state, and questions in one request
+    # evaluate in parallel, so folding them in here costs no extra round trip -- the
+    # alternative is a second request per task carrying an identical payload.
+    # (An effort question rode along here until 2026-10-17: it carried no signal beyond the
+    # tier -- within-tier AUC 0.50-0.54, 81% of answers xhigh or higher -- and never acted.
+    # Effort is now measured the way models are, by exploring pools that name efforts.)
+
+    gate_ids = {}
+    if enabled('gates', config=config):
+        gate_ids = dict(prompt_adequacy=prompt_adequacy,
+                        independently_verifiable=independently_verifiable)
+        for name, builder in gate_ids.items():
+            questions[name] = builder()
+
+    result = ask(state, questions, site='jev-score-plan', run_dir=run_dir, config=config)
+    if result is None:
+        return None
+
+    dimensions = {}
+    for name in QUESTIONS:
+        dimensions[name] = _normalized(result, name, len(questions[name]['criteria']))
+    combined = combine(dimensions)
+    if combined is None:
+        return None
+    combined.update(id=task_id, declared=declared_difficulty)
+
+    # Warnings, not gates: recorded below the threshold and silent above it. A LOW
+    # probability is what is worth saying out loud here.
+    #
+    # Each rubric gets its own threshold because they are not on one scale, which is the
+    # same trap `tests_act` set earlier. Measured over 15 hand-labelled task prompts:
+    # genuinely vague prompts ("add tests", "make it faster") all landed at 0.04 while
+    # specific ones ran 0.55-0.87, and task fragments that need a sibling to land first
+    # scored 0.11-0.24 against 0.51-0.72 for self-contained work. The shared gate_warn
+    # of 0.60 -- which IS right for drift, where it was measured -- would have warned on
+    # 5 of 9 and 5 of 11 perfectly good tasks.
+    warn_levels = dict(prompt_adequacy=threshold('prompt_warn', config=config),
+                       independently_verifiable=threshold('verifiable_warn', config=config))
+    warnings = {}
+    for name in gate_ids:
+        probability = noul(result, name)
+        if probability is not None and probability < warn_levels[name]:
+            warnings[name] = round(probability, 3)
+    if warnings:
+        combined['warnings'] = warnings
+    return combined

package/scripts/forge_jev/subset.py

@@ -0,0 +1,123 @@
+"""Run the tests a change most likely broke, early, in a throwaway copy.
+
+Read the deferral note in references/jev.md first: there is no safe window to run tests
+inside a task worktree. An artifact created before the diff is captured reaches QA as if
+the dwarf wrote it; one created after trips the fingerprint re-checks in `do_task` and
+`merge_task`, which write INVALIDATED and block the merge. So this never touches the
+worktree. It exports the reviewed commit to a disposable directory and runs there.
+
+**This is early feedback, not verification, and not a speed-up.** The full suite still
+runs at integration exactly as before, so this ADDS a run rather than replacing one. What
+it buys is that a reviewer learns "your diff breaks test_x" while it can still act on
+that, instead of the failure surfacing after the wave. Anyone reading this expecting the
+wall-clock lever the plan described should know the lever was not available: it required
+replacing the full verification, which the fingerprint rules forbid.
+"""
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+from pathlib import Path
+
+from . import threshold
+from .client import ask, noul
+from .questions import test_relevance
+
+CANDIDATE_CAP = 100
+HEADER_LINES = 12
+
+# Two conditions, not one. Matching "anything under tests/" pulled in README.md, JSON
+# fixtures and benchmark output -- files no runner will execute, each costing a question
+# and a slot under the cap.
+_TEST_PATH_RE = re.compile(r'(^|/)(tests?|spec|specs|__tests__)/'
+                           r'|(^|/)test_[^/]*$|_test\.[^/]+$|\.(test|spec)\.[^/]+$')
+_CODE_SUFFIXES = ('.py', '.go', '.js', '.jsx', '.ts', '.tsx', '.rb', '.rs', '.java',
+                  '.kt', '.php', '.sh', '.mjs', '.cjs')
+
+# Copied rather than rebuilt when present. Rebuilding a dependency tree per task would
+# cost far more than the feedback is worth; a symlink keeps it near-free.
+_DEP_DIRS = ('node_modules', '.venv', 'venv', 'vendor')
+
+
+def candidate_tests(repo) -> list[str]:
+    """Tracked test files, sorted and capped, from the repo's own index."""
+    try:
+        out = subprocess.run(['git', '-C', str(repo), 'ls-files', '-z'],
+                             capture_output=True, text=True, timeout=20)
+    except (OSError, subprocess.SubprocessError):
+        return []
+    if out.returncode != 0:
+        return []
+    found = sorted(name for name in out.stdout.split('\0')
+                   if name and name.endswith(_CODE_SUFFIXES) and _TEST_PATH_RE.search(name))
+    return found[:CANDIDATE_CAP]
+
+
+def _header(repo: Path, rel: str) -> str:
+    try:
+        with (repo / rel).open(errors='replace') as handle:
+            return ''.join(next(handle, '') for _ in range(HEADER_LINES))
+    except OSError:
+        return ''
+
+
+def select(repo, *, task: str, changed: list, candidates: list, run_dir=None,
+           config: dict | None = None):
+    """The tests worth running for this change, most likely first.
+
+    Returns None when no judgment could be made. The threshold is `tests_act`'s sibling
+    `gate_warn` deliberately NOT reused here -- see the caller, which uses the measured
+    per-repo value when one exists and otherwise runs nothing rather than guess.
+    """
+    candidates = [c for c in (candidates or []) if c]
+    if not candidates or not (task or '').strip():
+        return None
+    repo_path = Path(repo)
+    questions = {f't{i}': test_relevance(name, _header(repo_path, name))
+                 for i, name in enumerate(candidates)}
+    state = dict(task=task, changed_files=list(changed or []))
+    result = ask(state, questions, site='jev-test-subset', run_dir=run_dir, config=config)
+    if result is None:
+        return None
+    scored = []
+    for i, name in enumerate(candidates):
+        probability = noul(result, f't{i}')
+        if probability is not None:
+            scored.append((probability, name))
+    scored.sort(reverse=True)
+    return scored
+
+
+def export(repo, commit: str, destination) -> bool:
+    """Copy one commit's tree into a throwaway directory. Never touches the worktree."""
+    destination = Path(destination)
+    try:
+        destination.mkdir(parents=True, exist_ok=True)
+        archive = subprocess.run(['git', '-C', str(repo), 'archive', commit],
+                                 capture_output=True, timeout=120)
+        if archive.returncode != 0:
+            return False
+        extract = subprocess.run(['tar', '-x', '-C', str(destination)],
+                                 input=archive.stdout, capture_output=True, timeout=120)
+        if extract.returncode != 0:
+            return False
+    except (OSError, subprocess.SubprocessError):
+        return False
+
+    # Dependencies are not in the tree. Link them so the export can actually run; a
+    # missing link simply means the command fails and the caller reports nothing.
+    for name in _DEP_DIRS:
+        source = Path(repo) / name
+        target = destination / name
+        if source.is_dir() and not target.exists():
+            try:
+                os.symlink(source, target)
+            except OSError:
+                pass
+    return True
+
+
+def cleanup(destination) -> None:
+    shutil.rmtree(str(destination), ignore_errors=True)