bmad-module-skill-forge 2.0.0 → 2.0.1

package/src/skf-test-skill/scripts/compute-score.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # /// script
 # requires-python = ">=3.10"
 # dependencies = []
@@ -25,9 +26,19 @@ Input schema (one object):
       "externalValidation": <0-100>
     },
     "threshold": <0-100, optional, default 80>,
-    "evidenceCount": <int, optional, used for INCONCLUSIVE floor>
+    "evidenceCount": <int, optional, used for INCONCLUSIVE floor>,
+    "analysisConfidence": "<optional string; 'degraded' fires the tooling cap>",
+    "toolingStatus": "<optional string; a '*-missing' marker fires the tooling cap>"
   }
 
+Verdict override (post-score caps + threshold fallback — see compute_score):
+  When a cap or the threshold fallback engages, the output additionally carries
+  `effectiveResult` (the final verdict after caps/fallback), `capReason`
+  (string or null), `thresholdFallback` (bool), and `originalThreshold` (the
+  pre-fallback threshold, or null). When neither engages, those keys are omitted
+  and `result` is the final verdict. `result` itself is never mutated — it is
+  always the pre-cap/pre-fallback score-vs-threshold (or evidence-floor) verdict.
+
 Exit codes:
   0  — input was parsed; either a score was computed (verdict may be
        PASS/FAIL/INCONCLUSIVE) or a schema/validation error object
@@ -71,6 +82,59 @@ CATEGORIES = [
 DEFAULT_THRESHOLD = 80
 
 
+# --- Redistribution equivalence classes (moved here from scoring-rules.md) ---
+#
+# Determinism note for maintainers — NOT reader-model context. The redistributed
+# `weights` / `activeCategories` / `skippedCategories` output depends ONLY on
+# three things: the base table (contextual vs naive), whether Signature Accuracy
+# + Type Coverage are skipped (Quick tier OR docsOnly OR state2 OR stackSkill OR
+# referenceApp), and whether External Validation is skipped (externalValidation
+# is null). Every (mode × tier × docsOnly × state2 × stackSkill × referenceApp)
+# cell with an identical skip set + base table therefore reduces to the same
+# equivalence class below and emits identical final weights for identical input
+# scores — only the `skipReasons` string differs. The prompt (score.md) never
+# names a class or fixture; it sets the flags and reads the script's output back.
+# The rightmost column pins each class to a representative fixture in
+# test/fixtures/compute-score-contract.json (see test-compute-score-contract.py);
+# "(equiv.)" cells reduce algebraically to a listed representative.
+#
+# |  # | mode       | tier   | docsOnly | state2 | base       | sig/type skip          | ext skip | class | representative fixture           |
+# |----|------------|--------|----------|--------|------------|------------------------|----------|-------|----------------------------------|
+# |  1 | contextual | Deep   | F        | F      | contextual | —                      | if null  | A     | suite_a_all_active               |
+# |  2 | contextual | Forge+ | F        | F      | contextual | —                      | if null  | A     | suite_k_forge_plus               |
+# |  3 | contextual | Forge  | F        | F      | contextual | —                      | if null  | A     | suite_p_contextual_forge         |
+# |  4 | contextual | Quick  | F        | F      | contextual | Quick tier             | if null  | B     | suite_c_quick_tier               |
+# |  5 | contextual | Deep   | T        | F      | contextual | docs-only              | if null  | B     | suite_r_contextual_deep_docsonly |
+# |  6 | contextual | Forge+ | T        | F      | contextual | docs-only              | if null  | B     | (equiv.)                         |
+# |  7 | contextual | Forge  | T        | F      | contextual | docs-only              | if null  | B     | (equiv.)                         |
+# |  8 | contextual | Quick  | T        | F      | contextual | Quick tier + docs-only | if null  | B     | suite_f_docs_only                |
+# |  9 | contextual | Deep   | F        | T      | contextual | State 2                | if null  | B     | suite_g_state2                   |
+# | 10 | contextual | Forge+ | F        | T      | contextual | State 2                | if null  | B     | (equiv.)                         |
+# | 11 | contextual | Forge  | F        | T      | contextual | State 2                | if null  | B     | (equiv.)                         |
+# | 12 | contextual | Quick  | F        | T      | contextual | Quick + State 2        | if null  | B     | (equiv.)                         |
+# | 13 | contextual | *      | T        | T      | contextual | docs-only + State 2    | if null  | B     | (equiv.)                         |
+# | 14 | naive      | Deep   | F        | F      | naive      | —                      | if null  | C     | suite_q_naive_deep               |
+# | 15 | naive      | Forge+ | F        | F      | naive      | —                      | if null  | C     | (equiv.)                         |
+# | 16 | naive      | Forge  | F        | F      | naive      | —                      | if null  | C     | suite_b_naive                    |
+# | 17 | naive      | Quick  | F        | F      | naive      | Quick tier             | if null  | D     | suite_e_triple_skip              |
+# | 18 | naive      | Deep   | T        | F      | naive      | docs-only              | if null  | D     | suite_s_naive_deep_docsonly      |
+# | 19 | naive      | Forge+ | T        | F      | naive      | docs-only              | if null  | D     | (equiv.)                         |
+# | 20 | naive      | Forge  | T        | F      | naive      | docs-only              | if null  | D     | (equiv.)                         |
+# | 21 | naive      | Quick  | T        | F      | naive      | Quick + docs-only      | if null  | D     | suite_o_input_echo               |
+# | 22 | naive      | Deep   | F        | T      | naive      | State 2                | if null  | D     | suite_t_naive_state2             |
+# | 23 | naive      | Forge+ | F        | T      | naive      | State 2                | if null  | D     | (equiv.)                         |
+# | 24 | naive      | Forge  | F        | T      | naive      | State 2                | if null  | D     | (equiv.)                         |
+# | 25 | naive      | Quick  | F        | T      | naive      | Quick + State 2        | if null  | D     | (equiv.)                         |
+# | 26 | naive      | *      | T        | T      | naive      | docs-only + State 2    | if null  | D     | (equiv.)                         |
+#
+# Stack skills (stackSkill) and reference-app skills (referenceApp) share the
+# skip set of the docsOnly/state2 rows: contextual → class B, naive → class D;
+# only skipReasons differs (fixtures suite_u_stack_skill_deep,
+# suite_v_reference_app_deep). Quick-tier rows (4, 8, 12, 17, 21, 25) then feed
+# the minimum-evidence floor in the result block below (§7 of compute_score),
+# which can force INCONCLUSIVE after this pre-floor redistribution.
+
+
 # --- Helpers ---
 
 
@@ -130,6 +194,11 @@ def validate_input(inp):
         if not isinstance(threshold, (int, float)) or threshold < 0 or threshold > 100:
             return "threshold must be a number between 0 and 100"
 
+    for str_field in ("analysisConfidence", "toolingStatus"):
+        val = inp.get(str_field)
+        if val is not None and not isinstance(val, str):
+            return f"{str_field} must be a string or null, got: {type(val).__name__}"
+
     for cat in CATEGORIES:
         score = inp["scores"].get(cat)
         if score is not None:
@@ -277,6 +346,58 @@ def compute_score(inp):
     else:
         result = "PASS" if total_score >= threshold else "FAIL"
 
+    # 7b. Post-score caps + threshold fallback — deterministic verdict override.
+    # Lifted from score.md §3d/§4b so the cap<->fallback interaction is centralized
+    # and unit-tested here rather than re-derived in the prompt. The minimum-
+    # evidence floor (INCONCLUSIVE) is a gate that NO cap or fallback may override,
+    # so all of this is skipped when result == "INCONCLUSIVE".
+    analysis_confidence = inp.get("analysisConfidence")
+    tooling_status = inp.get("toolingStatus")
+
+    effective_result = result
+    cap_reason = None
+    threshold_fallback = False
+    original_threshold = None
+    cap_fired = False
+
+    if result != "INCONCLUSIVE":
+        cap_reasons = []
+        # Cap 1 — tooling degraded: analysis confidence is "degraded", or a
+        # missing-helper marker (e.g. python3-missing, frontmatter-validator-missing).
+        tooling_degraded = analysis_confidence == "degraded" or (
+            isinstance(tooling_status, str) and "missing" in tooling_status
+        )
+        if tooling_degraded:
+            cap_fired = True
+            cap_reasons.append(
+                "tooling degraded — capped below threshold until helper restored"
+            )
+        # Cap 2 — docs-only mode with no external validators available.
+        if docs_only and scores.get("externalValidation") is None:
+            cap_fired = True
+            cap_reasons.append(
+                "docs-only without external validators — capped below threshold"
+            )
+        if cap_fired:
+            cap_reason = "; ".join(cap_reasons)
+            # A cap only flips a PASS into FAIL; a pre-existing FAIL stays FAIL.
+            if result == "PASS":
+                effective_result = "FAIL"
+
+        # Threshold fallback — convert a FAIL into PASS at the 80 floor when the
+        # RAW totalScore clears 80 and the effective threshold was above 80. The
+        # fallback reads the raw totalScore (matching score.md §4b's documented
+        # `totalScore >= 80`), not the capped value; because it also requires
+        # threshold > 80 (so threshold - 1 >= 80), min(totalScore, threshold - 1)
+        # is itself >= 80 whenever totalScore >= 80, so raw-vs-capped is the same
+        # decision — raw is used for spec fidelity and testability.
+        if effective_result == "FAIL" and total_score >= 80 and threshold > 80:
+            threshold_fallback = True
+            original_threshold = threshold
+            effective_result = "PASS"
+
+    overrides_engaged = cap_fired or threshold_fallback
+
     # Build scores echo with null preservation
     scores_echo = {}
     for cat in CATEGORIES:
@@ -303,6 +424,15 @@ def compute_score(inp):
         "weightSum": weight_sum,
     }
 
+    # Verdict-override fields — emitted as an atomic group only when a post-score
+    # cap or the threshold fallback engaged (mirrors the conditional `warnings` /
+    # `inconclusiveReasons` fields). When absent, `result` is the final verdict.
+    if overrides_engaged:
+        output["effectiveResult"] = effective_result
+        output["capReason"] = cap_reason
+        output["thresholdFallback"] = threshold_fallback
+        output["originalThreshold"] = original_threshold
+
     if warnings:
         output["warnings"] = warnings
 

package/src/skf-test-skill/scripts/reconcile-coverage.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# ///
+"""Deterministic Documented-vs-Source coverage reconciliation.
+
+Set-reconciliation helper for the SKF test-skill workflow (coverage-check.md
+§2c). Distinct from compute-score.py (which owns weight tables, redistribution,
+and the INCONCLUSIVE floor): this script performs the coverage *numerator*
+arithmetic that §2c calls the "Deterministic Intersection" — turning two
+already-extracted name lists (the §1 documented inventory and the §2 source
+barrel) into a reproducible {documented, missing, stale, exportCoverage}
+result, so the numerator no longer swings between runs on split-body skills.
+
+It covers the three branches §2c enumerates:
+
+  * "barrel"  (enumerated path) — intersect documented_set against the source
+    barrel_set: Documented = |documented_set ∩ barrel_set|,
+    Missing = barrel_set − documented_set, Stale = documented_set − barrel_set,
+    Export Coverage = |Documented| / |barrel_set| * 100.
+
+  * "scalar"  (§4 priority-1 effective_denominator, no enumerated name set) —
+    grep each documented name across SKILL.md ∪ references/*.md;
+    Documented = count present, denominator = effective_denominator,
+    Missing = denominator − Documented, Stale not enumerable (empty),
+    Export Coverage = Documented / denominator * 100.
+
+  * "stack"   (skill_type == "stack", empty source barrel) — grep each
+    composition-surface name (provenance-map cited contracts, ::-excluded, or
+    libraries + integration_pairs — resolved upstream) across
+    SKILL.md ∪ references/*.md; denominator = stack_denominator,
+    Missing = denominator − Documented, Stale not enumerable,
+    Export Coverage = Documented / stack_denominator * 100.
+
+CLI usage:
+  uv run reconcile-coverage.py '<JSON>'                  # JSON literal positional
+  uv run reconcile-coverage.py --json-input '<JSON>'     # explicit flag form
+  cat input.json | uv run reconcile-coverage.py --stdin  # piped input
+
+Input schema (one object):
+  {
+    "denominatorSource": "barrel" | "scalar" | "stack",   # required
+    "exports": [ {"name": "...", "kind": "..."}, ... ],    # §1 inventory
+    "barrelSet": ["a", "b", ...],                          # barrel: resolved name set
+    "perFileResults": [ {"exports_found": ["a", ...]} ],   # barrel: union'd if no barrelSet
+    "denominatorValue": <int>,                             # scalar/stack: resolved denominator
+    "compositionNames": ["lib::x", ...],                   # stack: names to grep
+    "skillPackagePath": "/path/to/skill"                   # scalar/stack: SKILL.md ∪ references
+  }
+
+Output (stdout, one object):
+  {
+    "branch": "enumerated" | "scalar" | "stack",
+    "denominatorSource": "barrel" | "scalar" | "stack",
+    "denominator": <int>,
+    "documented": <int>,
+    "missing": [names],          # enumerated: source names not documented; scalar/stack: []
+    "missingCount": <int>,       # always present
+    "stale": [names],            # enumerated: documented names not in source; scalar/stack: []
+    "staleCount": <int>,         # always present
+    "staleApplicable": <bool>,   # false for scalar/stack (no barrel to enumerate)
+    "exportCoverage": <float>
+  }
+
+Exit codes:
+  0  — reconciliation emitted successfully
+  1  — no input / input could not be parsed as JSON
+  2  — input parsed but schema/semantics invalid (error object emitted as JSON)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+VALID_SOURCES = ("barrel", "scalar", "stack")
+
+
+def round2(value):
+    """Round to 2 decimals with JS-compatible half-up rounding (matches compute-score.py)."""
+    return math.floor(value * 100 + 0.5) / 100
+
+
+def make_error(message):
+    return {"error": message, "code": "INVALID_INPUT"}
+
+
+# --- Documented-set derivation ---------------------------------------------
+
+
+def build_documented_set(exports):
+    """De-duplicated set of `name` from the §1 inventory, excluding kind:"method".
+
+    Methods are members of an already-counted class/type, not top-level barrel
+    exports (coverage-check.md §2c step 1).
+    """
+    names = set()
+    for entry in exports or []:
+        if not isinstance(entry, dict):
+            continue
+        if entry.get("kind") == "method":
+            continue
+        name = entry.get("name")
+        if isinstance(name, str) and name:
+            names.add(name)
+    return names
+
+
+def build_barrel_set(inp):
+    """barrel_set = explicit `barrelSet` if given, else union of exports_found[]."""
+    if inp.get("barrelSet") is not None:
+        return {n for n in inp["barrelSet"] if isinstance(n, str) and n}
+    barrel = set()
+    for result in inp.get("perFileResults") or []:
+        if not isinstance(result, dict):
+            continue
+        for name in result.get("exports_found") or []:
+            if isinstance(name, str) and name:
+                barrel.add(name)
+    return barrel
+
+
+# --- Grep (documented-name presence) ---------------------------------------
+
+
+def load_doc_text(skill_package_path):
+    """Concatenate SKILL.md ∪ references/*.md text for name-presence grepping.
+
+    Deterministic: references are read in sorted order. Reads as UTF-8 so
+    non-ASCII exports do not mojibake on Windows (cp1252 default).
+    """
+    root = Path(skill_package_path)
+    parts = []
+    skill_md = root / "SKILL.md"
+    if skill_md.is_file():
+        parts.append(skill_md.read_text(encoding="utf-8"))
+    refs_dir = root / "references"
+    if refs_dir.is_dir():
+        for ref in sorted(refs_dir.glob("*.md")):
+            parts.append(ref.read_text(encoding="utf-8"))
+    return "\n".join(parts)
+
+
+def names_present(names, doc_text):
+    """Return the sorted, de-duplicated set of `names` that appear in doc_text.
+
+    Substring containment, case-sensitive — mirrors `grep "{name}"`.
+    """
+    present = {n for n in names if isinstance(n, str) and n and n in doc_text}
+    return sorted(present)
+
+
+# --- Core reconciliation ----------------------------------------------------
+
+
+def _validate(inp):
+    if inp is None or not isinstance(inp, dict):
+        return "Input must be a JSON object"
+    source = inp.get("denominatorSource")
+    if source not in VALID_SOURCES:
+        return (
+            "Missing or invalid required field: denominatorSource "
+            f"(must be one of: {', '.join(VALID_SOURCES)})"
+        )
+    if source == "barrel":
+        if inp.get("barrelSet") is None and inp.get("perFileResults") is None:
+            return "barrel branch requires either `barrelSet` or `perFileResults`"
+    else:  # scalar / stack
+        dv = inp.get("denominatorValue")
+        if not isinstance(dv, int) or isinstance(dv, bool) or dv < 0:
+            return f"{source} branch requires integer `denominatorValue` >= 0"
+        if not inp.get("skillPackagePath"):
+            return f"{source} branch requires `skillPackagePath` to grep SKILL.md ∪ references"
+        if source == "stack" and inp.get("compositionNames") is None:
+            return "stack branch requires `compositionNames` (composition-surface names to grep)"
+    return None
+
+
+def reconcile(inp, doc_text=None):
+    """Pure reconciliation. `doc_text` may be injected (tests); else loaded from disk."""
+    err = _validate(inp)
+    if err:
+        return make_error(err)
+
+    source = inp["denominatorSource"]
+
+    if source == "barrel":
+        documented_set = build_documented_set(inp.get("exports"))
+        barrel_set = build_barrel_set(inp)
+        if not barrel_set:
+            return make_error(
+                "barrel branch: barrel_set is empty (denominator 0) — "
+                "Export Coverage is undefined; upstream §2b zero-exports guard should HALT"
+            )
+        documented_names = documented_set & barrel_set
+        missing = sorted(barrel_set - documented_set)
+        stale = sorted(documented_set - barrel_set)
+        denominator = len(barrel_set)
+        documented = len(documented_names)
+        return {
+            "branch": "enumerated",
+            "denominatorSource": source,
+            "denominator": denominator,
+            "documented": documented,
+            "missing": missing,
+            "missingCount": len(missing),
+            "stale": stale,
+            "staleCount": len(stale),
+            "staleApplicable": True,
+            "exportCoverage": round2(documented / denominator * 100),
+        }
+
+    # scalar / stack — grep-based numerator against SKILL.md ∪ references
+    denominator = inp["denominatorValue"]
+    if doc_text is None:
+        doc_text = load_doc_text(inp["skillPackagePath"])
+
+    if source == "scalar":
+        candidates = sorted(build_documented_set(inp.get("exports")))
+    else:  # stack
+        candidates = [n for n in inp["compositionNames"] if isinstance(n, str) and n]
+
+    present = names_present(candidates, doc_text)
+    documented = len(present)
+
+    if denominator == 0:
+        return make_error(
+            f"{source} branch: denominatorValue is 0 — Export Coverage is undefined; "
+            "upstream §2b guard should HALT before reconciliation"
+        )
+
+    missing_count = denominator - documented
+    return {
+        "branch": source,
+        "denominatorSource": source,
+        "denominator": denominator,
+        "documented": documented,
+        "missing": [],
+        "missingCount": missing_count,
+        "stale": [],
+        "staleCount": 0,
+        "staleApplicable": False,
+        "exportCoverage": round2(documented / denominator * 100),
+    }
+
+
+# --- CLI --------------------------------------------------------------------
+
+
+def _build_parser():
+    parser = argparse.ArgumentParser(
+        prog="reconcile-coverage",
+        description=(
+            "Deterministic Documented-vs-Source coverage reconciliation "
+            "(coverage-check.md §2c). Consumes the §1 documented inventory and "
+            "the §2 source barrel (or a resolved scalar/stack denominator + the "
+            "skill package path) and emits {documented, missing, stale, "
+            "exportCoverage} so the coverage numerator is reproducible."
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=(
+            "Example (barrel branch):\n"
+            "  uv run reconcile-coverage.py "
+            "'{\"denominatorSource\":\"barrel\","
+            "\"exports\":[{\"name\":\"a\",\"kind\":\"function\"}],"
+            "\"barrelSet\":[\"a\",\"b\"]}'"
+        ),
+    )
+    src = parser.add_mutually_exclusive_group()
+    src.add_argument(
+        "json_input",
+        nargs="?",
+        help="JSON object as a positional argument (single-quote it on the shell).",
+    )
+    src.add_argument(
+        "--json-input",
+        dest="json_input_flag",
+        help="JSON object passed via flag (overrides positional).",
+    )
+    src.add_argument(
+        "--stdin",
+        action="store_true",
+        help="Read the JSON object from stdin.",
+    )
+    return parser
+
+
+def _resolve_input(args):
+    if args.stdin:
+        return sys.stdin.read()
+    if args.json_input_flag is not None:
+        return args.json_input_flag
+    if args.json_input is not None:
+        return args.json_input
+    return ""
+
+
+def main(argv=None):
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    raw = _resolve_input(args)
+    if not raw.strip():
+        parser.print_usage(file=sys.stderr)
+        print(
+            "error: no input provided (positional arg, --json-input, or --stdin)",
+            file=sys.stderr,
+        )
+        return 1
+
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        print(json.dumps(make_error(f"Invalid JSON: {exc.msg}"), indent=2))
+        return 1
+
+    result = reconcile(data)
+    print(json.dumps(result, indent=2))
+    if isinstance(result, dict) and result.get("code") == "INVALID_INPUT":
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())