create-pmos 0.4.0 → 0.6.0

package/payload/scripts/metrics.py

@@ -5,19 +5,61 @@ Reads pmos/state/{runs,evals,acceptances}.jsonl and prints:
   B1  north star  — human corrections per ACCEPTED run (lower is better; D10, counted over
                     accepted runs only so leniency can't deflate it)
   B2  throughput  — accepted runs (guards against winning B1 by doing less)
-  --  eval pass rate — share of evals with verdict PASS / PASS-WITH-FINDINGS
+  --  eval pass rate — share of evals passing the Review-Gate rule (see review_gate_pass)
   B4  calibration — evaluator verdict vs PM label, joined by run_id: raw agreement, TPR, TNR,
-                    Cohen's kappa (the D40 classifier method, computed locally)
+                    Cohen's kappa (the D40 classifier method, computed locally). κ is broken out
+                    PER evaluator_model, with a mixed-model warning — a κ is a property of the
+                    evaluator model and is not inherited across a model swap (harness-model-agnosticism)
   --  Goodhart tripwire — fires when eval_pass_rate >= 0.8 while avg corrections >= 1.5 (the
                     pass-happy-judge signal; anti-optimism: read the red first)
 
 No network, no secrets. Exit 0 always (a reporting tool, not a gate). Anti-optimism: reds print first.
+
+Import-safe (D64 Track D1): every helper is module-level and the report runs only under __main__, so
+kit/fixtures/test_parsers.py can import review_gate_pass + leading_frontmatter and prove the Python
+parsers agree with the JavaScript ones in workspace/lib/.
 """
 import datetime, glob, json, os, re, sys
 
 KIT = os.environ.get("KIT_ROOT") or os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 STATE = os.path.join(KIT, "state")
 
+# ── CANONICAL Review-Gate pass rule — the MIRROR of workspace/lib/review-gate.js ──────────────
+# Rule: verdict ILIKE 'pass%' OR score >= 0.85. This is the single source of truth's Python half;
+# the JS half is workspace/lib/review-gate.js. kit/fixtures/test_parsers.py runs BOTH against
+# kit/fixtures/review-gate-pass-rule.json and fails if they ever disagree — the mirror can't drift.
+PASS_VERDICT_PREFIX = "pass"
+SCORE_THRESHOLD = 0.85
+
+
+def review_gate_pass(verdict, score):
+    """A recorded verdict is AUTHORITATIVE (pass iff it starts with "pass"); an explicit FAIL is never
+    overridden by a high score. Only when NO verdict was recorded does the score decide.
+    AMENDED 2026-07-26: was `verdict ILIKE 'pass%' OR score >= 0.85`, which let a FAIL at score 0.93
+    through the gate (blind-plant round 2, specimen be58). Mirror of workspace/lib/review-gate.js
+    isReviewPass; agreement enforced by kit/fixtures/test_parsers.py."""
+    v = str("" if verdict is None else verdict).strip().lower()
+    if v:
+        return v.startswith(PASS_VERDICT_PREFIX)
+    try:
+        return float(score) >= SCORE_THRESHOLD
+    except (TypeError, ValueError):
+        return False
+
+
+def leading_frontmatter(path):
+    """The LEADING frontmatter block only — body text cannot fake a declaration. Mirror of
+    workspace/lib/file-mode-parse.js parseLeadingFrontmatter (agreement enforced by the fixture test)."""
+    fm = {}
+    with open(path, encoding="utf-8") as fh:
+        for line in fh:
+            m = re.match(r"^([a-z_]+):\s*(.+?)\s*$", line.rstrip("\n"))
+            if m:
+                fm.setdefault(m.group(1), m.group(2))
+            elif line.strip() and not line.startswith("---"):
+                break  # frontmatter is the LEADING block only
+    return fm
+
 
 def load(name):
     path = os.path.join(STATE, name)
@@ -34,38 +76,27 @@ def load(name):
     return rows
 
 
-runs = load("runs.jsonl")
-evals = load("evals.jsonl")
-acc = load("acceptances.jsonl")
-
-accepted = [a for a in acc if a.get("accepted") is True]
-corrections_by_run = {r.get("run_id"): int(r.get("human_corrections") or 0) for r in runs}
-acc_corr = [corrections_by_run.get(a.get("run_id"), 0) for a in accepted]
-
-passing = ("PASS", "PASS-WITH-FINDINGS")
-n_pass = sum(1 for e in evals if str(e.get("verdict", "")).upper() in passing)
-pass_rate = (n_pass / len(evals)) if evals else None
-
-# --- B4: verdict-vs-PM-label confusion matrix, joined by run_id (D40) ---
-label_by_run = {a.get("run_id"): str(a.get("pm_label", "")).lower() for a in acc if a.get("pm_label")}
-tp = tn = fp = fn = 0
-for e in evals:
-    lab = label_by_run.get(e.get("run_id"))
-    if lab not in ("pass", "fail"):
-        continue
-    pred_pass = str(e.get("verdict", "")).upper() in passing
-    if pred_pass and lab == "pass":
-        tp += 1
-    elif pred_pass and lab == "fail":
-        fp += 1
-    elif not pred_pass and lab == "fail":
-        tn += 1
-    else:
-        fn += 1
-n = tp + tn + fp + fn
-
-
-def kappa():
+def confusion(rows, label_by_run):
+    """(tp, tn, fp, fn) over evals joined to a pass/fail PM label by run_id, using the canonical rule."""
+    tp = tn = fp = fn = 0
+    for e in rows:
+        lab = label_by_run.get(e.get("run_id"))
+        if lab not in ("pass", "fail"):
+            continue
+        pred_pass = review_gate_pass(e.get("verdict"), e.get("score"))
+        if pred_pass and lab == "pass":
+            tp += 1
+        elif pred_pass and lab == "fail":
+            fp += 1
+        elif not pred_pass and lab == "fail":
+            tn += 1
+        else:
+            fn += 1
+    return tp, tn, fp, fn
+
+
+def kappa_of(tp, tn, fp, fn):
+    n = tp + tn + fp + fn
     if n == 0:
         return None
     po = (tp + tn) / n
@@ -73,72 +104,102 @@ def kappa():
     return None if pe == 1 else (po - pe) / (1 - pe)
 
 
-avg_corr = (sum(corrections_by_run.values()) / len(corrections_by_run)) if corrections_by_run else None
-tripwire = pass_rate is not None and avg_corr is not None and pass_rate >= 0.8 and avg_corr >= 1.5
-
-print(f"PMOS kit metrics — {len(runs)} run(s), {len(evals)} eval(s), {len(acc)} acceptance record(s)\n")
-if tripwire:
-    print("🔴 GOODHART TRIPWIRE FIRED — evaluator passes >=80% while humans average >=1.5 corrections/run.")
-    print("   The north star is being gamed; treat the Review Gate as NOT calibrated (D40).\n")
-if not accepted:
-    print("🔴 B1 north star: UNMEASURABLE — no accepted runs yet (record acceptances to pmos/state/acceptances.jsonl).")
-else:
-    ns = sum(acc_corr) / len(accepted)
-    print(f"B1 north star: {ns:.2f} human corrections per accepted run (over {len(accepted)} accepted run(s); lower is better)")
-print(f"B2 throughput: {len(accepted)} accepted run(s)")
-print(f"   eval pass rate: {'n/a — no evals yet' if pass_rate is None else f'{pass_rate:.0%} ({n_pass}/{len(evals)})'}")
-k = kappa()
-if n == 0:
-    print("🔴 B4 calibration: UNMEASURABLE — no eval joined to a PM-labelled acceptance yet; the Review Gate is uncalibrated.")
-else:
-    tpr = tp / (tp + fn) if (tp + fn) else None
-    tnr = tn / (tn + fp) if (tn + fp) else None
-    print(f"B4 calibration (n={n}): raw agreement {(tp + tn) / n:.0%}, "
-          f"TPR {'n/a' if tpr is None else f'{tpr:.0%}'}, "
-          f"TNR {'n/a — no PM fails seen (leniency blind spot)' if tnr is None else f'{tnr:.0%}'}, "
-          f"kappa {'undefined' if k is None else f'{k:.2f}'} (target >= 0.60 — verdict ADVISORY below it, D40)")
-
-# --- D57 outcome checks (accepted-but-unvalidated; due prints red) -----------
-# Declarations live in initiative frontmatter (outcome_metric / outcome_horizon); dispositions are
-# appended to state/outcome_checks.jsonl ({initiative_id, disposition, evidence, date}). The latest
-# line per initiative wins; history is never rewritten. Auto-validation is prohibited (D57) — this
-# section only REPORTS; recording a disposition is always the PM's word.
-checks = load("outcome_checks.jsonl")
-latest_disp = {}
-for c in checks:
-    if c.get("initiative_id"):
-        latest_disp[c["initiative_id"]] = str(c.get("disposition", "")).lower()
-
-declared = []
-for path in sorted(glob.glob(os.path.join(STATE, "initiatives", "*.md"))):
-    fm = {}
-    for line in open(path, encoding="utf-8"):
-        m = re.match(r"^([a-z_]+):\s*(.+?)\s*$", line)
-        if m:
-            fm.setdefault(m.group(1), m.group(2))
-        elif line.strip() and not line.startswith("---"):
-            break  # frontmatter is the LEADING block only — body text cannot fake a declaration
-    if not fm.get("id"):
+def main():
+    runs = load("runs.jsonl")
+    evals = load("evals.jsonl")
+    acc = load("acceptances.jsonl")
+
+    accepted = [a for a in acc if a.get("accepted") is True]
+    corrections_by_run = {r.get("run_id"): int(r.get("human_corrections") or 0) for r in runs}
+    acc_corr = [corrections_by_run.get(a.get("run_id"), 0) for a in accepted]
+
+    n_pass = sum(1 for e in evals if review_gate_pass(e.get("verdict"), e.get("score")))
+    pass_rate = (n_pass / len(evals)) if evals else None
+
+    # --- B4: verdict-vs-PM-label confusion matrix, joined by run_id (D40) ---
+    label_by_run = {a.get("run_id"): str(a.get("pm_label", "")).lower() for a in acc if a.get("pm_label")}
+    tp, tn, fp, fn = confusion(evals, label_by_run)
+    n = tp + tn + fp + fn
+    k = kappa_of(tp, tn, fp, fn)
+
+    avg_corr = (sum(corrections_by_run.values()) / len(corrections_by_run)) if corrections_by_run else None
+    tripwire = pass_rate is not None and avg_corr is not None and pass_rate >= 0.8 and avg_corr >= 1.5
+
+    print(f"PMOS kit metrics — {len(runs)} run(s), {len(evals)} eval(s), {len(acc)} acceptance record(s)\n")
+    if tripwire:
+        print("🔴 GOODHART TRIPWIRE FIRED — evaluator passes >=80% while humans average >=1.5 corrections/run.")
+        print("   The north star is being gamed; treat the Review Gate as NOT calibrated (D40).\n")
+    if not accepted:
+        print("🔴 B1 north star: UNMEASURABLE — no accepted runs yet (record acceptances to pmos/state/acceptances.jsonl).")
+    else:
+        ns = sum(acc_corr) / len(accepted)
+        print(f"B1 north star: {ns:.2f} human corrections per accepted run (over {len(accepted)} accepted run(s); lower is better)")
+    print(f"B2 throughput: {len(accepted)} accepted run(s)")
+    print(f"   eval pass rate: {'n/a — no evals yet' if pass_rate is None else f'{pass_rate:.0%} ({n_pass}/{len(evals)})'}")
+    if n == 0:
+        print("🔴 B4 calibration: UNMEASURABLE — no eval joined to a PM-labelled acceptance yet; the Review Gate is uncalibrated.")
+    else:
+        tpr = tp / (tp + fn) if (tp + fn) else None
+        tnr = tn / (tn + fp) if (tn + fp) else None
+        print(f"B4 calibration (n={n}): raw agreement {(tp + tn) / n:.0%}, "
+              f"TPR {'n/a' if tpr is None else f'{tpr:.0%}'}, "
+              f"TNR {'n/a — no PM fails seen (leniency blind spot)' if tnr is None else f'{tnr:.0%}'}, "
+              f"kappa {'undefined' if k is None else f'{k:.2f}'} (target >= 0.60 — verdict ADVISORY below it, D40)")
+
+    # --- per-evaluator-model calibration (harness-model-agnosticism) -------------
+    labeled_evals = [e for e in evals if label_by_run.get(e.get("run_id")) in ("pass", "fail")]
+    by_model = {}
+    for e in labeled_evals:
+        by_model.setdefault(e.get("evaluator_model") or "unknown", []).append(e)
+    if len(by_model) > 1:
+        print(f"\n🔴 MIXED-MODEL CALIBRATION — labeled evals span {len(by_model)} evaluator models; the "
+              "aggregate κ above POOLS them and is not any single evaluator's calibration (κ is per "
+              "evaluator model, D40). Per model:")
+        for m in sorted(by_model):
+            c = confusion(by_model[m], label_by_run)
+            km = kappa_of(*c)
+            print(f"   - {m}: n={sum(c)}, kappa {'undefined' if km is None else f'{km:.2f}'} "
+                  f"(tp{c[0]} tn{c[1]} fp{c[2]} fn{c[3]})")
+    elif by_model and set(by_model) == {"unknown"}:
+        print("\n⚠ evaluator_model unrecorded on every labeled eval — κ cannot be attributed to a model "
+              "and silently pools across any model swaps. Pass the evaluator model as the last arg to "
+              "`pmos-log.sh eval` (harness-model-agnosticism).")
+
+    # --- D57 outcome checks (accepted-but-unvalidated; due prints red) -----------
+    checks = load("outcome_checks.jsonl")
+    latest_disp = {}
+    for c in checks:
+        if c.get("initiative_id"):
+            latest_disp[c["initiative_id"]] = str(c.get("disposition", "")).lower()
+
+    declared = []
+    for path in sorted(glob.glob(os.path.join(STATE, "initiatives", "*.md"))):
+        fm = leading_frontmatter(path)
+        if not fm.get("id"):
+            if fm.get("outcome_metric"):
+                print(f"⚠ {os.path.basename(path)} declares an outcome but has no id: — it can never be disposed; fix the frontmatter")
+            continue
         if fm.get("outcome_metric"):
-            print(f"⚠ {os.path.basename(path)} declares an outcome but has no id: — it can never be disposed; fix the frontmatter")
-        continue
-    if fm.get("outcome_metric"):
-        declared.append(fm)
-
-if declared:
-    today = datetime.date.today().isoformat()
-    undisposed = [f for f in declared if f.get("id") not in latest_disp]
-    # a missing horizon errs RED (counts as due): a bet with no judgment date can never come due
-    # by itself — horizon-gaming-by-omission (mirrors the hosted CHECK requiring a horizon).
-    due = [f for f in undisposed if str(f.get("outcome_horizon", "0000-00-00")) < today]
-    pending = [f for f in undisposed if f not in due]
-    print()
-    if due:
-        print(f"🔴 D57 outcome checks DUE ({len(due)}) — accepted bets past horizon, never validated:")
-        for f in due:
-            print(f"   - {f.get('id')}: \"{f.get('outcome_metric')}\" (horizon {f.get('outcome_horizon', '?')})"
-                  " -> dispose validated|refuted|inconclusive to state/outcome_checks.jsonl")
-    refuted = sum(1 for v in latest_disp.values() if v == "refuted")
-    print(f"D57 outcomes: {len(declared)} declared — {len(pending)} pending, {len(due)} due, "
-          f"{len(latest_disp)} disposed"
-          + (f" ({refuted} refuted — recorded, not hidden)" if refuted else ""))
+            declared.append(fm)
+
+    if declared:
+        today = datetime.date.today().isoformat()
+        undisposed = [f for f in declared if f.get("id") not in latest_disp]
+        # a missing horizon errs RED (counts as due): a bet with no judgment date can never come due
+        # by itself — horizon-gaming-by-omission (mirrors the hosted CHECK requiring a horizon).
+        due = [f for f in undisposed if str(f.get("outcome_horizon", "0000-00-00")) < today]
+        pending = [f for f in undisposed if f not in due]
+        print()
+        if due:
+            print(f"🔴 D57 outcome checks DUE ({len(due)}) — accepted bets past horizon, never validated:")
+            for f in due:
+                print(f"   - {f.get('id')}: \"{f.get('outcome_metric')}\" (horizon {f.get('outcome_horizon', '?')})"
+                      " -> dispose validated|refuted|inconclusive to state/outcome_checks.jsonl")
+        refuted = sum(1 for v in latest_disp.values() if v == "refuted")
+        print(f"D57 outcomes: {len(declared)} declared — {len(pending)} pending, {len(due)} due, "
+              f"{len(latest_disp)} disposed"
+              + (f" ({refuted} refuted — recorded, not hidden)" if refuted else ""))
+
+
+if __name__ == "__main__":
+    main()

package/payload/scripts/pmos-log.sh

@@ -6,8 +6,8 @@
 # script only ever `>>`s; a correction is a NEW record. No dependencies beyond POSIX + python3
 # (used solely for safe JSON string escaping).
 #
-#   pmos-log.sh run      <run_id> <initiative_id> [human_corrections]
-#   pmos-log.sh eval     <run_id> <initiative_id> <score 0..1> <PASS|PASS-WITH-FINDINGS|FAIL> <rubric_path>
+#   pmos-log.sh run      <run_id> <initiative_id> [human_corrections] [model]
+#   pmos-log.sh eval     <run_id> <initiative_id> <score 0..1> <PASS|PASS-WITH-FINDINGS|FAIL> <rubric_path> [evaluator_model]
 #   pmos-log.sh accept   <run_id> <initiative_id> <pass|fail> [note]
 #   pmos-log.sh outcome  <initiative_id> <validated|refuted|inconclusive> [evidence]
 #   pmos-log.sh friction <note...>
@@ -33,20 +33,30 @@ append() { # append <file> <json-line>
 cmd="${1:-}"; shift || true
 case "$cmd" in
   run)
-    [ $# -ge 2 ] || fail "usage: run <run_id> <initiative_id> [human_corrections]"
+    [ $# -ge 2 ] || fail "usage: run <run_id> <initiative_id> [human_corrections] [model]"
     nonempty run_id "$1"; nonempty initiative_id "$2"
     corr="${3:-0}"
     printf '%s' "$corr" | grep -qE '^[0-9]+$' || fail "human_corrections must be a non-negative integer, got '$corr'."
-    append runs.jsonl "{\"run_id\":$(js "$1"),\"initiative_id\":$(js "$2"),\"human_corrections\":$corr,\"date\":\"$today\"}"
+    # optional builder-model provenance (which model produced the run). Omitted when absent — the
+    # metric layer treats a missing model as 'unknown'. if-form, not `&&`: an empty $4 under set -e
+    # would abort. Keeps calibration honest across model switches (harness-model-agnosticism).
+    model_field=""
+    if [ -n "${4:-}" ]; then model_field=",\"model\":$(js "$4")"; fi
+    append runs.jsonl "{\"run_id\":$(js "$1"),\"initiative_id\":$(js "$2"),\"human_corrections\":$corr,\"date\":\"$today\"$model_field}"
     ;;
   eval)
-    [ $# -ge 5 ] || fail "usage: eval <run_id> <initiative_id> <score 0..1> <PASS|PASS-WITH-FINDINGS|FAIL> <rubric_path>"
+    [ $# -ge 5 ] || fail "usage: eval <run_id> <initiative_id> <score 0..1> <PASS|PASS-WITH-FINDINGS|FAIL> <rubric_path> [evaluator_model]"
     nonempty run_id "$1"; nonempty initiative_id "$2"
     printf '%s' "$3" | grep -qE '^(0?\.[0-9]+|0|1(\.0+)?)$' || fail "score must be 0..1 written as 0, 1, N.N or .N — got '$3'."
     case "$4" in PASS|PASS-WITH-FINDINGS|FAIL) ;; *) fail "verdict must be PASS | PASS-WITH-FINDINGS | FAIL, got '$4'." ;; esac
     score="$3"; case "$score" in .*) score="0$score" ;; esac  # ".5" -> "0.5": bare .5 is not valid JSON
     [ -f "$KIT/$5" ] || [ -f "$5" ] || fail "rubric_path '$5' not found (relative to the kit or the repo)."
-    append evals.jsonl "{\"run_id\":$(js "$1"),\"initiative_id\":$(js "$2"),\"score\":$score,\"verdict\":$(js "$4"),\"rubric_path\":$(js "$5"),\"date\":\"$today\"}"
+    # optional evaluator-model provenance (which model rendered the verdict). Calibration (κ) is a
+    # property of the evaluator MODEL, so metrics.py segments κ by it; a missing value counts as
+    # 'unknown'. if-form, not `&&` (empty $6 under set -e aborts).
+    emodel_field=""
+    if [ -n "${6:-}" ]; then emodel_field=",\"evaluator_model\":$(js "$6")"; fi
+    append evals.jsonl "{\"run_id\":$(js "$1"),\"initiative_id\":$(js "$2"),\"score\":$score,\"verdict\":$(js "$4"),\"rubric_path\":$(js "$5"),\"date\":\"$today\"$emodel_field}"
     ;;
   accept)
     [ $# -ge 3 ] || fail "usage: accept <run_id> <initiative_id> <pass|fail> [note]"

package/payload/skills/SKILL-FORMAT.md

@@ -1,3 +1,9 @@
+---
+type: Reference
+title: "SKILL-FORMAT.md — Required Format for PMOS Skills"
+timestamp: 2026-07-23
+---
+
 # SKILL-FORMAT.md — Required Format for PMOS Skills
 
 This document defines the **required** structure for every `.skill` file in PMOS. It is the
@@ -36,13 +42,36 @@ Rules:
   (specified, non-executable) → graduates to active (key removed) by a PM decision in `DECISIONS.md`.
 - **Optional Phase-B registry metadata (D49)** — a skill may also carry `version`, `owner`, `risk`,
   `category`, and `scope`. These are **optional**, producer-defined registry fields that
-  [`gen_skills_seed.sh`](/supabase/seed/gen_skills_seed.sh) sources into the `skills` table so the skill
+  the resync generator [`gen_okf_mirror_seed.py`](/supabase/gen_okf_mirror_seed.py) sources into the `skills` table so the skill
   library can group/version/scope skills. A skill with only `name` + `description` is still fully
   conformant (e.g. `build-skills/evaluator.skill`); carry the registry keys on durable `skills/core/`
   entries for consistency with their siblings.
+- `lifecycle` — **optional**, in the same D49 registry slot. Values: `active` (the default when the
+  key is absent) | `deprecated`. A deprecated skill **stays on disk and stays in the registry** —
+  `get_skill` returns it in full with a `deprecated_notice` so a retrieving agent is *told* not to
+  start new work on it. Retirement is a state you can read; deletion is a separate act (see the
+  invariant below). Distinct from `maturity_mode`, which is about a skill not yet executable —
+  `lifecycle` is about one on its way out.
 - **No other L1 keys.** Beyond the required `name` + `description`, the optional `maturity_mode`, and the
   optional Phase-B registry keys above, procedural detail does not belong in L1 — it lives in the body.
 
+### The disk ⇄ registry invariant (enforced in CI)
+
+**Every `.skill` on disk is seeded, and every seeded skill exists on disk — exactly.** The `skills`
+table is a *mirror*, and `get_skill` serves it to agents in other people's repos, so a mirror that
+has quietly drifted is a wrong answer delivered confidently.
+
+The `REGISTRY` check in [`check_okf_drift.py`](/supabase/check_okf_drift.py) fails the build on:
+adding or renaming a `.skill` without regenerating the seed; a seeded skill whose file is gone; and
+any registry-metadata value that disagrees with the file's own frontmatter. That last one is not
+theoretical — a generator whose INSERT omitted the Phase-B columns silently nulled
+`version/owner/risk/category/scope` for **all 21 skills for about three weeks** in 2026-07, and was
+caught only by an incidental version pin in an RLS test. So:
+
+> **When you add, rename, delete, or edit the frontmatter of a `.skill`, regenerate the seed in the
+> same PR** ([okf-sync playbook](/okf/core/playbooks/okf-sync.md)) — and sweep `supabase/tests/` for
+> pins on that skill's registry metadata before you do (the prd-grilling F1 rule).
+
 ### Level 2 — SKILL.md body (loaded when relevant)
 
 The body below the frontmatter holds the **full procedural detail**: steps, heuristics, required
@@ -143,15 +172,49 @@ cheap:
 - Level-3 sub-files **≅** the standard's bundled files.
 
 (One nuance for the follow-on below: PMOS's *optional* D49 registry keys —
-`version`/`owner`/`risk`/`category`/`scope` — are top-level custom keys; the standard would expect
+`version`/`owner`/`risk`/`category`/`scope`/`lifecycle` — are top-level custom keys; the standard would expect
 producer-defined extensions under a `metadata:` map. The **core** shape is lossless; relocating the
 registry keys belongs to the native-format packaging follow-on, not this flat form.)
 
-**Named follow-on (not built here):** shipping the **kit** in native Agent-Skills directory format so
-a stranger's Claude Code / Codex harness auto-discovers PMOS skills. It touches `build-kit.sh`,
-`get_skill`, `gen_skills_seed.sh`, and every skill path, so it is its own initiative — revisit when
-native-discovery demand in kit repos is proven. (This mirrors the mattpocock/skills plugin decision:
-keep the editable flat-file form, add a native-package path only when the demand is real.)
+### Kit native-format packaging (E6 — shipped)
+
+The **kit** now ships every skill in native Agent-Skills directory form *as well as* the flat file —
+a lossless portability layer, generated **from** the flat `.skill` at build time (never
+hand-authored, so it cannot drift from its source). For each shipped skill, `kit/build-kit.sh` emits
+`skills-native/<name>/SKILL.md` where:
+
+- the directory name **equals** the frontmatter `name` (the standard's hard rule), and required
+  `name` + `description` are present;
+- PMOS's D49 registry keys (`version`/`owner`/`risk`/`category`/`scope`/`lifecycle`) are relocated
+  **under a `metadata:` map with a `pmos-` prefix** — the standard's producer-extension slot — never
+  as top-level custom keys, so a conformant runtime ignores them cleanly;
+- a **fail-closed conformance self-check** in `build-kit.sh` aborts the build on any violation
+  (name≠dirname, missing required field, or a top-level custom key), the same posture as the
+  containment check.
+
+The flat `.skill` files remain the file-mode source of truth (the AGENTS.md state adapter references
+`pmos/skills/<name>.skill`); the native tree is the ecosystem-portable mirror. **Still a named
+follow-on (not built here):** wiring `create-pmos` to install the native tree into a runtime's
+auto-discovery locations (`.claude/skills/`, `.agents/skills/`). E6 delivers the conformant
+*packaging*; the *install location* is a separate change, revisited when native-discovery demand in
+kit repos is proven.
+
+### Per-skill eval harness (E8 — convention shipped, benchmark deferred)
+
+A skill may carry an **opt-in** eval file at `skills/evals/<name>.evals.json` (mapping to the
+standard's `<name>/evals/evals.json` in native packaging), adopting the skill-creator harness shape:
+
+- **`description_triggers`** — `should_fire` / `should_not_fire` prompt lists that test whether the
+  L1 description fires on the right requests and stays silent on the wrong ones (trigger-accuracy).
+- **`cases`** — each a `{ name, prompt, expect, with_without }`, where `expect` is an **outcome
+  property** a grader checks (never a script) and `with_without` toggles whether the case is meant
+  for with-vs-without-the-skill benchmarking.
+
+`scripts/check_skill_evals.py` (run in the `okf-drift` job) validates the **shape** of any present
+eval file and that it names a real skill. It **never runs a benchmark** — actually scoring
+with-vs-without needs labeled outcomes + compute and is **deferred** (there is no negative class
+yet); running models in CI would make the gate nondeterministic. The convention and a worked example
+(`skills/evals/prd.evals.json`) ship now; the verdict "this skill measurably helps" does not.
 
 ## File naming
 
@@ -170,8 +233,8 @@ keep the editable flat-file form, add a native-package path only when the demand
 A skill is well-formed when:
 
 1. Level 1 frontmatter has `name` + `description` (both required), plus optionally `maturity_mode` (on
-   non-active skills) and the Phase-B registry keys `version`/`owner`/`risk`/`category`/`scope` (D49);
-   no other L1 keys.
+   non-active skills) and the Phase-B registry keys `version`/`owner`/`risk`/`category`/`scope`/
+   `lifecycle` (D49); no other L1 keys.
 2. `name` matches the filename stem.
 3. `description` states both what the skill does and when to use it.
 4. The body (Level 2) carries the full procedure and is self-sufficient for the common case.

package/payload/skills/discovery.skill

@@ -35,6 +35,10 @@ fake rubric, or escaped the record entirely.
 1. **Anchor and create the initiative** with `type: discovery`
    (`create_initiative({id, title, parent_kr, type: 'discovery'})`; file mode: `type: discovery`
    in the initiative frontmatter).
+1b. **Existing surface? Understand it first.** If answering the question honestly requires reading
+   code that already exists, run an [explain-surface](/build-skills/explain-surface.skill) pass on
+   that region — in a fresh context, **before** writing the brief. A spike aimed at a system the PM
+   does not understand produces a learning they cannot act on. Greenfield → skip.
 2. **Write the spike brief** from
    [spike-brief-template](/okf/core/templates/spike-brief-template.md) into
    `/planning/prd/<id>-spike.md`; set it as the initiative's `prd_path`. The Initiative Gate

package/payload/skills/eval-rubric.skill

@@ -43,7 +43,7 @@ Acceptance Gate is the PM's and is not graded by the rubric.
 
 Every rubric document carries, at minimum:
 
-- `type: Rubric` (or as the bundle/planning convention requires)
+- **OKF frontmatter** — the file opens with `type: Rubric`, `title`, `timestamp` (repo-wide convention, D63; [frontmatter-format](/okf/core/concepts/frontmatter-format.md)).
 - **`eval_type: capability | regression`** —
   - `capability` — tests whether the agent can do something new / harder. Used to push the frontier.
   - `regression` — guards an already-passing capability against backsliding. Saturated capability

package/payload/skills/evaluator.skill

@@ -8,7 +8,9 @@ description: Adversarially evaluate an agent run's output against its rubric and
 This skill is PMOS's **Review Gate** made a first-class, repeatable capability (P4). It formalizes the
 adversarial independent evaluator run by hand through P1–P3: judge what an agent run produced against a
 rubric, write a structured verdict to the `evals` table, and surface it on the radar/board. It runs in
-**Claude Code** (heavy judgment), not an Edge Function ([D34](/DECISIONS.md)).
+the **interactive agent runtime** — any capable coding agent; heavy judgment belongs in the agent, not
+an Edge Function ([D09](/DECISIONS.md)/[D34](/DECISIONS.md), amended 2026-07-23). PMOS-self runs it in
+Claude Code, but the runtime is not Claude-specific.
 
 > **Format:** three-level progressive disclosure ([SKILL-FORMAT](/skills/core/SKILL-FORMAT.md)). Level 1
 > frontmatter above is the trigger. This Level 2 body is the full procedure.
@@ -23,12 +25,21 @@ rubric, write a structured verdict to the `evals` table, and surface it on the r
 - **Outcome-only ([D14](/DECISIONS.md)).** Grade what the run *produced* — the artifact, the diff, the
   live behavior — **not** the tool-call sequence or the path taken. A right answer reached "the wrong
   way" still passes; a wrong answer reached tidily still fails.
-- **Self-preference bias — distrust your own praise.** You are a **Claude evaluator judging
-  Claude-built work**, so you carry a built-in bias to over-reward outputs in your own family's style
-  ([D40](/DECISIONS.md)). Counter it: **default to skeptical**, and when a pass feels easy or you are
-  uncertain, **say so explicitly** and flag the run for the cross-family spot-check / minority-veto jury
-  ([calibration-method](/planning/evals/calibration-method.md)). Do not let stylistic familiarity read as
-  quality.
+- **Self-preference bias — distrust your own praise.** An LLM judge over-rewards outputs from **its own
+  model family's style** ([D40](/DECISIONS.md)). Read the run's recorded `builder_model` and your own
+  `evaluator_model`: when they **share a family** you carry a built-in bias to over-reward (the common
+  PMOS-self case — a Claude evaluator on Claude-built work); when they **differ**, the bias flips and you
+  may under-reward an unfamiliar style. Either way: **default to skeptical**, and when a pass feels easy
+  or you are uncertain, **say so explicitly** and flag the run for the cross-family spot-check /
+  minority-veto jury ([calibration-method](/planning/evals/calibration-method.md)). Do not let stylistic
+  (un)familiarity read as quality. **Record your `evaluator_model` on the `save_eval` verdict** — κ is
+  calibrated per evaluator model, and an unrecorded model silently pools calibration across model swaps.
+- **Capability floor — this role assumes a frontier-tier reasoner.** The adversarial passes below
+  (default-to-refute, both-ways pairwise, semantic-pass-not-grep, injection resistance) degrade
+  **silently** to leniency and grep-clean on a weaker model — the verdict still looks well-formed while
+  the safety property is already gone. If you cannot confidently perform them, **say so in the verdict**
+  and score the affected dimensions `unverified` rather than passing on trust; the per-model κ
+  ([v_calibration_by_model], file-mode `metrics.py`) is what surfaces a too-weak evaluator over time.
 - **Position, order, and verbosity bias — judge the artifact, not its packaging.** When comparing
   options or scoring against exemplars, substance decides, not presentation order or length.
   **Order must not sway a comparison** — never let "first-seen" or "last-seen" win; for any **pairwise**
@@ -56,6 +67,30 @@ rubric, write a structured verdict to the `evals` table, and surface it on the r
   PM labels** (TPR/TNR, not a streak) per [D40](/DECISIONS.md), which **supersedes** D20's "3 consecutive
   agreements" standard. Say in the verdict that the gate is advisory until that kappa threshold is met.
 
+## Two-axis verdict (Spec × Standards)
+
+Score on **two independent axes**, because "did it do what was asked?" and "is it well-built?" fail
+separately and a single blended number lets one hide the other:
+
+- **Spec axis** — does the output satisfy the `pre_run_contract` done-criteria? Outcome-only (D14):
+  the right thing was produced, judged against each criterion's `test_method`. This axis is the
+  per-criterion binary backbone of step 5.
+- **Standards axis** — is it well-built? Craft, security, tests, maintainability, and the rubric's
+  quality dimensions (step 6). A run can meet every done-criterion and still be shoddy here; a
+  beautifully-built run can miss the spec.
+
+Each axis gets its **own pass/fail and its own one-line rationale**, and the overall verdict is a
+**stated function of the two: a FAIL on *either* axis cannot become an overall PASS.** (Spec-fail →
+FAIL; Standards-fail → at best PASS-WITH-FINDINGS only when every defect is non-blocking, otherwise
+FAIL — never a clean PASS.) The split is **persisted**, not just narrated (step 9).
+
+> **Calibration of the axis split is DEFERRED (stated, not done).** Whether separating the axes
+> actually *moves* Cohen's κ against PM labels is the E4 research question — and it is unanswerable
+> until the Review Gate has faced a **negative class** (all labeled pairs are currently `pass`; TNR
+> is undefined). Emit the two axes now for legibility; do **not** claim the split is calibrated or
+> validated. It becomes measurable once `pmos-seeded-known-bad-runs` (or real fails) gives κ a
+> denominator. ([D40](/DECISIONS.md), [calibration-method](/planning/evals/calibration-method.md).)
+
 ## Procedure
 
 1. **Read [AGENTS.md](/AGENTS.md)** and the run's `pre_run_contract`. *Done when* you can state the
@@ -111,17 +146,25 @@ rubric, write a structured verdict to the `evals` table, and surface it on the r
 7. **Enumerate defects** — each with `severity` (critical / must-fix / should-fix / low / nit), the
    specific file+line or behavior, why it's a problem, and a concrete fix. Lead with the single most
    important finding.
-8. **Decide a verdict**: `PASS` / `PASS-WITH-FINDINGS` / `FAIL`. PASS-WITH-FINDINGS is correct when the
-   work is acceptable but carries non-blocking defects. Include a one-line **pass-fraction
-   disclosure**: the share of pass-side verdicts you (this gate) have recently issued — read it
-   from `v_calibration` / `v_goodhart_tripwire` where a backend is available, or state
-   `pass-fraction: unknown (no backend)` in kit mode. A high pass-fraction is a leniency-drift
-   warning to the PM, not a compliment to the builders
+8. **Decide the two axis verdicts, then the overall.** State a **Spec** pass/fail (from the step-5
+   per-criterion binaries — any FAILed done-criterion fails the axis) and a **Standards** pass/fail
+   (from the step-6 dimensions + defect severities — a critical/must-fix defect fails the axis),
+   each with a one-line rationale. Then combine into `PASS` / `PASS-WITH-FINDINGS` / `FAIL` under the
+   **stated rule: a FAIL on either axis cannot be an overall PASS** — Spec-fail ⇒ FAIL;
+   Standards-fail ⇒ FAIL unless every defect is non-blocking, in which case PASS-WITH-FINDINGS; only
+   both axes passing yields PASS (clean PASS only with no should-fix-or-worse defects). Include a
+   one-line **pass-fraction disclosure**: the share of pass-side verdicts you (this gate) have
+   recently issued — read it from `v_calibration` / `v_goodhart_tripwire` where a backend is
+   available, or state `pass-fraction: unknown (no backend)` in kit mode. A high pass-fraction is a
+   leniency-drift warning to the PM, not a compliment to the builders
    ([llm-judge-calibration](/planning/research/llm-judge-calibration.md)).
 9. **Persist it** — `save_eval({ initiative_id, run_id, rubric_path, score, verdict, dimensions })`,
-   where `dimensions` is `{ <name>: { score, weight, note, evidence } }` — `evidence` carries the
-   quote/pointer from step 6 (or the string `unverified`). This is what makes the gate self-hosting
-   — the verdict lands in `evals` and surfaces on `v_eval_gate_status` (radar/board).
+   where `dimensions` is `{ <name>: { score, weight, note, evidence, axis } }` — `evidence` carries
+   the quote/pointer from step 6 (or the string `unverified`), and **`axis` is `"spec"` or
+   `"standards"`** so the two-axis split is persisted, not just narrated. Put the two axis pass/fail
+   summaries in the top-level `note`/report and tag every dimension with its axis. This is what makes
+   the gate self-hosting — the verdict lands in `evals` and surfaces on `v_eval_gate_status`
+   (radar/board).
 10. **Hand to the PM.** The Acceptance Gate is theirs, and the PM records the ground-truth `pm_label` to
    `acceptances` — paired against your verdict by `v_calibration` to compute the kappa that calibrates this
    gate ([D40](/DECISIONS.md), [calibration-method](/planning/evals/calibration-method.md)). If the PM
@@ -130,7 +173,9 @@ rubric, write a structured verdict to the `evals` table, and surface it on the r
 
 ## Output shape
 
-A short report: **most important finding first**, then VERDICT (score vs threshold) with the
-pass-fraction disclosure, the calibration exemplars used (step 3), the numbered defects, and the
-per-dimension scores **with their evidence quotes/pointers** — followed by the `save_eval` call. Be
-concrete; cite specifics; if it is genuinely solid, say so — but only after a real refutation attempt.
+A short report: **most important finding first**, then the **two axis verdicts (Spec: pass/fail —
+one line; Standards: pass/fail — one line)** and the **overall VERDICT** (score vs threshold, under
+the combine rule) with the pass-fraction disclosure, the calibration exemplars used (step 3), the
+numbered defects, and the per-dimension scores **with their evidence quotes/pointers and `axis`
+tag** — followed by the `save_eval` call. Be concrete; cite specifics; if it is genuinely solid, say
+so — but only after a real refutation attempt.