@techierathore/techieflow 1.0.2 → 1.0.4

package/.tfcore/utils/tf-emit.sh

@@ -19,6 +19,12 @@
 #   tf-emit.sh --amend MISS-App-20260828-01 why_missed missing-checklist-item
 #                                               # fills a field that is still null on a miss already on
 #                                               # the stream (SCHEMA.md §5.5.7). Never overwrites.
+#   tf-emit.sh --void-run build-phase 2026-09-09T11:40:00Z "the start time was guessed, not measured"
+#                                               # marks a WRONG run record as not to be counted: both
+#                                               # records stay, every figure skips it and says how many
+#                                               # and why (SCHEMA.md §2.7). Refuses if no such run.
+#   tf-emit.sh --origin-of REQ-UI-014           # prints "<started> <cmd> <agent>" of the last build or fix
+#                                               # run that touched the row, nothing when there is none
 #   tf-emit.sh --where                          # prints the resolved docs/metrics dir
 #
 # STREAMS: runs | gates | sessions | commits | misses   (anything else is dropped)
@@ -131,6 +137,39 @@ PY
   exit 0
 fi
 
+# --- read helper: which run caused a row's defect ------------------------
+# Prints "<started> <cmd> <agent>" for the most recent non-backfilled build-phase
+# or fix-issues run whose reqs_touched holds the REQ, nothing when there is none
+# (the caller then leaves origin_run_id out and the record is marked inferred).
+# Used by tf-log-miss.sh and tf-triage.sh (Sitting 4c, 2026-09-06).
+if [[ "$1" == "--origin-of" ]]; then
+  REQ="$2"
+  [[ -n "$REQ" ]] || exit 0
+  python3 - "$MET_DIR/runs.jsonl" "$REQ" 2>"$TF_ERR" <<'PY2' || true
+import json, sys
+path, req = sys.argv[1], sys.argv[2].upper()
+best = None
+try:
+    with open(path, "r", encoding="utf-8") as fh:
+        for line in fh:
+            try:
+                r = json.loads(line)
+            except Exception:
+                continue
+            if r.get("kind") != "run" or r.get("backfilled") or r.get("cmd") not in ("build-phase", "fix-issues"):
+                continue
+            if req in [str(x).upper() for x in (r.get("reqs_touched") or [])]:
+                if best is None or r.get("started", "") >= best.get("started", ""):
+                    best = r
+except FileNotFoundError:
+    pass
+if best:
+    subs = best.get("subagents") or []
+    print(best.get("started", ""), best.get("cmd"), subs[0] if subs else "flow-master")
+PY2
+  exit 0
+fi
+
 # --- read helper: next RUN attempt number for a cmd + REQ set ------------
 # run attempt = 1 + count of prior NON-BACKFILLED `run` records with the same
 # cmd whose reqs_touched intersects the given REQ IDs. Same counting rule that
@@ -170,9 +209,10 @@ fi
 # `miss-fix`, and a collision would silently merge two defects into one lifecycle.
 if [[ "$1" == "--next-miss-id" ]]; then
   python3 - "$MET_DIR/misses.jsonl" "$ROOT" 2>"$TF_ERR" <<'PY' || true
-import datetime, glob, json, os, sys
+import datetime, glob, json, os, re, sys
 path, root = sys.argv[1], sys.argv[2]
-hits = glob.glob(os.path.join(root, "docs", "*-Checklist.md"))
+hits = [h for h in glob.glob(os.path.join(root, "docs", "*-Checklist.md"))
+        if not re.search(r"-(Deployment|P\d+)-Checklist\.md$", h)]  # only the phase-1 work list names the app
 app = (os.path.basename(hits[0])[: -len("-Checklist.md")] if len(hits) == 1
        else os.path.basename(os.path.abspath(root)))
 day = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d")
@@ -234,6 +274,61 @@ fi
 # patience rather than quality. Open = no miss-fix, or the latest miss-fix
 # closed with a verdict_after other than "Verified". Prints "<miss_id>
 # <miss_class>" for the most recent open miss, or nothing at all.
+# --- list the open misses whose deficient artifact is a document ----------
+# `*amend-docs` is the command that edits a BRD, an Architecture or a mockup, so it is
+# the only command that can close a miss whose artifact is one of those -- and it had no
+# way to find them. Every other close path works off a checklist row the verifier touched,
+# which a BRD edit never is, so a document miss stayed open forever (TF-016: twelve
+# finished TfLens items showed as outstanding for up to two weeks).
+#
+#   tf-emit.sh --open-misses <App> [--artifact-class doc]
+#
+# Prints one line per open miss: "<miss_id> <artifact> <what>". Open means no miss-fix
+# carries verdict_after Verified, the same test --open-miss applies.
+if [[ "$1" == "--open-misses" ]]; then
+  OMAPP="${2:-}"; OMCLASS=""
+  [[ "${3:-}" == "--artifact-class" ]] && OMCLASS="${4:-}"
+  TF_OMAPP="$OMAPP" TF_OMCLASS="$OMCLASS" python3 - "$MET_DIR/misses.jsonl" 2>"$TF_ERR" <<'PY2' || true
+import json, os, sys
+DOC = {"brd", "architecture", "uidesign", "checklist", "devguide"}
+app, klass = os.environ.get("TF_OMAPP") or "", os.environ.get("TF_OMCLASS") or ""
+opened, fixes = [], {}
+try:
+    with open(sys.argv[1], encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                r = json.loads(line)
+            except Exception:
+                continue
+            if r.get("kind") == "miss":
+                opened.append(r)
+            elif r.get("kind") == "miss-fix":
+                mid = r.get("miss_id")
+                if mid:
+                    prev = fixes.get(mid)
+                    if prev is None or (r.get("ts") or "") >= (prev.get("ts") or ""):
+                        fixes[mid] = r
+except FileNotFoundError:
+    pass
+for r in opened:
+    if app and r.get("app") and r.get("app") != app:
+        continue
+    art = r.get("artifact") or ""
+    if klass == "doc" and art not in DOC:
+        continue
+    if klass and klass != "doc" and art != klass:
+        continue
+    fix = fixes.get(r.get("miss_id"))
+    if fix is not None and fix.get("verdict_after") == "Verified":
+        continue
+    print("%s %s %s" % (r.get("miss_id"), art or "-", (r.get("what") or "")[:110]))
+PY2
+  exit 0
+fi
+
 if [[ "$1" == "--open-miss" ]]; then
   REQ="$2"
   [[ -n "$REQ" ]] || exit 0
@@ -304,10 +399,23 @@ AMENDABLE = {
     "why_missed": ("missing-checklist-item", "insufficient-verify-method",
                    "code-audit-limitation", "ambiguous-acceptance",
                    "dependency-not-declared", "instruction-ignored", "other"),
+    # whose gap it was, the four-question sort (FR-32, Session 5): a judgement a reader can
+    # still make later, so a miss logged before the field existed can be sorted now.
+    "sort": ("spec", "unsaid", "weak-check", "ignored"),
 }
-if fld not in AMENDABLE:
+    # `what` is the miss's own sentence, and it is FREE TEXT -- the one field here that
+    # is not a closed vocabulary. It is allowed because the protection was never the
+    # vocabulary: an amend may only fill a field that is still null and can never
+    # overwrite one that is set (below), so free text cannot rewrite a fact, only
+    # supply one that was missing. Validated as non-empty text instead.
+    # MISS-TechieFlow-20260909-02; tests/regression/run.sh tf_018.
+AMENDABLE_TEXT = ("what",)
+if fld not in AMENDABLE and fld not in AMENDABLE_TEXT:
     print("REFUSED %s is not an amendable field (SCHEMA.md §5.5.7)" % fld); raise SystemExit(0)
-if val not in AMENDABLE[fld]:
+if fld in AMENDABLE_TEXT:
+    if not isinstance(val, str) or not val.strip():
+        print("REFUSED %s must be a non-empty sentence" % fld); raise SystemExit(0)
+elif val not in AMENDABLE[fld]:
     print("REFUSED %r is not in the closed vocabulary for %s" % (val, fld)); raise SystemExit(0)
 
 parent, current = None, None
@@ -350,8 +458,77 @@ PY
   exit 0
 fi
 
+# --- write helper: mark a run record as not to be counted -----------------
+# SCHEMA.md §2.7. The same doctrine as --amend, one stream over: a run record that
+# is WRONG cannot be edited or deleted, because the stream is append-only and the
+# evidence that it happened is worth keeping. So the correction is another record.
+#
+#   tf-emit.sh --void-run <cmd> <started> "<why it is wrong>"
+#
+# It names the run by its own `cmd` and `started` — the pair every run record
+# carries and the key the readers already use — and every figure then skips it,
+# with the count and the reasons printed. A void that names no run on this stream
+# is refused: it would be a claim about nothing. It is deliberately NOT a delete:
+# both records stay, and a reader can see what was corrected and why.
+if [[ "$1" == "--void-run" ]]; then
+  VCMD="$2"; VSTART="$3"; VWHY="$4"
+  if [[ -z "$VCMD" || -z "$VSTART" || -z "$VWHY" ]]; then
+    echo "tf-emit: --void-run needs <cmd> <started> \"<reason>\""
+    exit 0
+  fi
+  VREC="$(python3 - "$MET_DIR/runs.jsonl" "$VCMD" "$VSTART" "$VWHY" 2>"$TF_ERR" <<'PY' || true
+import json, sys
+path, cmd, started, why = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
+target, voided, app = None, False, None
+try:
+    with open(path, encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                r = json.loads(line)
+            except Exception:
+                continue
+            if r.get("cmd") == cmd and r.get("started") == started:
+                if r.get("kind") == "run-void":
+                    voided = True
+                elif r.get("kind", "run") == "run":
+                    target = r
+                    app = r.get("app")
+except FileNotFoundError:
+    pass
+
+if target is None:
+    print("REFUSED no run record %s started %s on this stream" % (cmd, started)); raise SystemExit(0)
+if voided:
+    print("REFUSED %s started %s is already voided — one void is enough" % (cmd, started)); raise SystemExit(0)
+if not why.strip():
+    print("REFUSED a void needs a reason a reader can check"); raise SystemExit(0)
+
+print("OK " + json.dumps({"kind": "run-void", "app": app, "cmd": cmd,
+                          "started": started, "reason": why.strip()},
+                         separators=(",", ":"), ensure_ascii=False))
+PY
+)"
+  case "$VREC" in
+    "OK "*) printf '%s' "${VREC#OK }" | bash "${BASH_SOURCE[0]}" runs
+            echo "tf-emit: voided $VCMD started $VSTART — $VWHY"
+            echo "         both records stay on the stream; every figure now skips the run and says so" ;;
+    "REFUSED "*) echo "tf-emit: void refused — ${VREC#REFUSED }" ;;
+    *) echo "tf-emit: void could not be evaluated — nothing written" ;;
+  esac
+  exit 0
+fi
+
 # --- append path ----------------------------------------------------------
 STREAM="$1"
+# --allow-overlap: this run really did happen while another was still open (a second machine
+# appending to the same merge=union stream is the only case that has ever come up). Without it a
+# run whose `started` precedes the previous run's `ended` is refused — see _overlap_ok below.
+TF_ALLOW_OVERLAP=0
+for _a in "$@"; do [[ "$_a" == "--allow-overlap" ]] && TF_ALLOW_OVERLAP=1; done
+export TF_ALLOW_OVERLAP
 case "$STREAM" in
   runs|gates|sessions|commits|misses) ;;
   *) _warn "unknown stream '${STREAM:-<none>}' — event dropped"; exit 0 ;;
@@ -360,7 +537,7 @@ esac
 # The program is captured into a variable rather than fed on stdin — a heredoc
 # on `python3 -` would REPLACE the caller's piped JSON and silently drop every event.
 TF_PROG="$(cat <<'PY'
-import json, os, sys, datetime, glob, re
+import json, os, sys, datetime, glob, re, subprocess, time
 
 met_dir, stream, root = sys.argv[1], sys.argv[2], sys.argv[3]
 dbg = os.environ.get("TF_METRICS_DEBUG")
@@ -456,11 +633,8 @@ def _detect_harness():
     # the OTHER harness inherits that harness's markers (e.g. OpenCode started
     # from a Claude Code bash still carries CLAUDE_PROJECT_DIR).
     tf = os.environ.get("TF_HARNESS")
-    if tf in ("claude-code", "opencode", "codex"):
+    if tf in ("claude-code", "opencode"):
         return tf
-    for k in ("CODEX_THREAD_ID", "CODEX_SESSION_ID"):
-        if os.environ.get(k):
-            return "codex"
     for k in ("CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_SESSION_ID",
               "CLAUDE_PROJECT_DIR"):
         if os.environ.get(k):
@@ -488,8 +662,6 @@ def _detect_harness():
                     name, ppid = os.path.basename(parts[0]), int(parts[1])
             if name and "opencode" in name.lower():
                 return "opencode"
-            if name and "codex" in name.lower():
-                return "codex"
             if name and "claude" in name.lower():
                 return "claude-code"
             pid = ppid
@@ -505,7 +677,11 @@ except Exception:
 # app name — inferred from the one checklist, else the repo directory name
 def _app_name():
     try:
-        hits = glob.glob(os.path.join(root, "docs", "*-Checklist.md"))
+        # The Deployment Checklist and a Large project's phase-n checklists share the suffix;
+        # only <App>-Checklist.md names the app, else a copy such as TfLens-oc was recorded
+        # under its folder name (MISS-TechieFlow-20260906-04).
+        hits = [h for h in glob.glob(os.path.join(root, "docs", "*-Checklist.md"))
+                if not re.search(r"-(Deployment|P\d+)-Checklist\.md$", h)]
         if len(hits) == 1:
             return os.path.basename(hits[0])[: -len("-Checklist.md")]
     except Exception:
@@ -524,6 +700,80 @@ def enrich(rec):
     DECISIONS.md 2026-08-20)."""
     rec.setdefault("v", 1)
     rec.setdefault("ts", NOW)
+    # A run record says whether YOLO was on (D-12, FR-35; Sitting 4c, 2026-09-06): the task
+    # never has to remember it. TF_YOLO=1 (the supervisor) or a live .tfcore/.session/yolo.json.
+    if rec.get("kind") == "run" and "yolo" not in rec:
+        on = os.environ.get("TF_YOLO") == "1"
+        if not on:
+            try:
+                fl = os.path.join(root, ".tfcore", ".session", "yolo.json")
+                on = os.path.isfile(fl) and (time.time() - os.path.getmtime(fl)) < 24 * 3600
+            except Exception:
+                on = False
+        rec["yolo"] = bool(on)
+    # A run record without `started` takes it from the command marker written by
+    # tf-phase.sh start (step 0 of every task), when the marker names the same command.
+    if rec.get("kind") == "run" and not rec.get("started"):
+        try:
+            with open(os.path.join(root, ".tfcore", ".session", "phase.json"), encoding="utf-8") as fh:
+                mk = json.load(fh) or {}
+            if mk.get("started") and (not rec.get("cmd") or mk.get("cmd") == rec.get("cmd")):
+                rec["started"] = mk["started"]
+                rec.setdefault("ended", NOW)
+                sys.stdout.write("tf-emit: started taken from the command marker (%s)\n" % mk["started"])
+        except Exception:
+            pass
+    # A run's `ended` is when the record is written, never a guess: an ended in the
+    # future (more than a minute past now) or before started is replaced with now and
+    # duration_s recomputed (MISS-TechieFlow-20260905-11: a day-1 run wrote 17:05 at 16:40).
+    # The same rule the other way round: a run record with NO `ended` was silently
+    # accepted and could never be costed — the effort-per-phase figure just lost it
+    # (MISS-TechieFlow-20260907-09, the session-6 record itself). `ended` is when the
+    # record is written, so an absent one is filled in, never left empty.
+    if rec.get("kind") == "run" and not isinstance(rec.get("ended"), str):
+        rec["ended"] = NOW
+    if rec.get("kind") == "run" and isinstance(rec.get("ended"), str):
+        t_end, t_now = _iso_ms(rec["ended"]), _iso_ms(NOW)
+        t_start = _iso_ms(rec["started"]) if isinstance(rec.get("started"), str) else None
+        # Two different faults, and they must not share a remedy.
+        #
+        # (a) `ended` a little in the future: `started` is still trustworthy, so clamp to
+        #     now and re-derive (MISS-TechieFlow-20260905-11, a day-1 run wrote 17:05 at 16:40).
+        # (b) `ended` BEFORE `started`: the record contradicts itself and there is no way to
+        #     tell which of the two is wrong, so it carries NO duration at all. Deriving one
+        #     from the surviving timestamp invents a figure -- a TechieBlog record started
+        #     2026-08-22 would have been "measured" at 1,515,038 seconds on the day it was
+        #     re-emitted. An honest gap beats a plausible number: SCHEMA §2.5.
+        if t_end is not None and t_now is not None and t_start is not None and t_end < t_start:
+            sys.stdout.write("tf-emit: ended %s precedes started %s; the record contradicts "
+                             "itself, so no duration is stored\n" % (rec["ended"], rec["started"]))
+            rec["ended"] = NOW
+            rec.pop("duration_s", None)
+            rec["duration_unmeasured"] = "ended-before-started"
+        elif t_end is not None and t_now is not None and t_end > t_now + 60_000:
+            sys.stdout.write("tf-emit: ended %s is not a measurement (it lies in the future); "
+                             "set to now, %s\n" % (rec["ended"], NOW))
+            rec["ended"] = NOW
+            if t_start is not None and "duration_s" in rec:
+                rec["duration_s"] = max(0, (t_now - t_start) // 1000)
+        elif t_end is not None and t_start is not None and isinstance(rec.get("duration_s"), (int, float)):
+            # A supplied duration that bears no relation to its own timestamps. Thirteen of
+            # TechieBlog's fourteen impossible records hid here, storing a plausible round
+            # number (3600, 2700, 1800, 900, 600) that nothing downstream could question --
+            # only the ONE record that stored the negative arithmetic was ever detectable.
+            span = (t_end - t_start) // 1000
+            if abs(span - rec["duration_s"]) > 1:
+                sys.stdout.write("tf-emit: duration_s %s disagrees with started/ended (%ss); "
+                                 "using the timestamps\n" % (rec["duration_s"], span))
+                rec["duration_s"] = span
+    # A run record that omits duration_s gets it from started and ended (the devguide and
+    # refresh-status records of 2026-09-06 had none; MISS-TechieFlow-20260906-10).
+    if rec.get("kind") == "run" and "duration_s" not in rec and rec.get("started") and rec.get("ended") \
+            and not rec.get("duration_unmeasured"):
+        try:
+            rec["duration_s"] = max(0, (_iso_ms(rec["ended"]) - _iso_ms(rec["started"])) // 1000)
+        except Exception:
+            pass
     if "project_type" not in rec:
         if PTYPE:
             rec["project_type"] = PTYPE
@@ -531,7 +781,10 @@ def enrich(rec):
             # Default, but NEVER silently — the report labels these unclassified.
             rec["project_type"] = "app"
             rec["project_type_inferred"] = True
-    rec.setdefault("harness", HARNESS)
+    if HARNESS:
+        rec["harness"] = HARNESS          # detected, never declared (SCHEMA.md §1)
+    else:
+        rec.setdefault("harness", None)
     rec.setdefault("app", APP)
     return rec
 
@@ -574,6 +827,52 @@ def _routing():
 
 ROUTING = _routing()
 
+# --- how a model is paid for (SCHEMA.md §2.5b, added 2026-09-10) -----------
+# One cost column cannot mean three things at once. A run on a flat-fee subscription records
+# zero marginal dollars, a run on a metered API key records real money, and a run on a local
+# model records zero because there is no bill — and until this field existed a reader could
+# not tell those three zeroes apart, so every "what did it cost" figure quietly averaged real
+# spending with free runs. The answer is resolved by tf-model-pick.sh, which reads OpenCode's
+# own auth.json and model catalog, so it is looked up and never declared by an agent — the
+# same rule as `harness` (§1) and `origin_model` (§5.5.1).
+_BILLING_CACHE = {}
+
+
+def _billing(model, harness):
+    key = (model, harness)
+    if key not in _BILLING_CACHE:
+        mode, source = "unknown", "none"
+        pick = os.path.join(root, ".tfcore", "utils", "tf-model-pick.sh")
+        if model and os.path.isfile(pick):
+            try:
+                out = subprocess.run(["bash", pick, "billing", model, harness or "claude"],
+                                     capture_output=True, text=True, timeout=20,
+                                     env=dict(os.environ, TF_PROJECT_DIR=root))
+                parts = (out.stdout or "").strip().split("\t")
+                if parts and parts[0] in ("subscription", "plan", "metered", "local", "unknown"):
+                    mode = parts[0]
+                    source = parts[1] if len(parts) > 1 else "tf-model-pick"
+            except Exception:
+                pass
+        _BILLING_CACHE[key] = (mode, source)
+    return _BILLING_CACHE[key]
+
+
+def _was_limited(model, harness, at_epoch):
+    """Did this run start while the tier's own model was parked? Then it is a deliberate
+    fallback, not routing drift, and `routed:false` should not be read as one."""
+    pick = os.path.join(root, ".tfcore", "utils", "tf-model-pick.sh")
+    if not model or not os.path.isfile(pick) or at_epoch is None:
+        return False
+    try:
+        out = subprocess.run(["bash", pick, "was-limited", harness or "claude", model, str(int(at_epoch))],
+                             capture_output=True, text=True, timeout=20,
+                             env=dict(os.environ, TF_PROJECT_DIR=root))
+        return out.returncode == 0
+    except Exception:
+        return False
+
+
 def _iso_ms(s):
     try:
         return int(datetime.datetime.strptime(
@@ -747,6 +1046,97 @@ def _prior_runs():
             pass
     return _PRIOR_RUNS
 
+# --- a run may not start before the last one finished (FR-72, 2026-09-10) --
+# `--void-run` (§2.7) gave a way to CORRECT a run record written with a start time nobody
+# measured. Nothing refused one, so the same mistake happened twice more the day the correction
+# mechanism shipped — two records whose `started` preceded the previous record's `ended`, one by
+# 42 minutes and one by nearly three hours, each silently adding that overlap to every total
+# (MISS-TechieFlow-20260910-04, sorted `ignored`: it was written down and not followed).
+#
+# A stream is append-only, so the only place to stop it is before the write. The rule: for one
+# app, a live `run` record may not begin before the last live `run` record ended. Voided and
+# backfilled records are ignored, which is what makes the void-then-re-emit flow work. Records
+# with no `started` are untouched.
+#
+# The one legitimate overlap is two machines appending to the same merge=union stream, so
+# `--allow-overlap` exists and the refusal message names it. It is deliberately a flag and not a
+# fallback: the mistake this refuses is easy to make and invisible afterwards.
+_LAST_END = None
+
+
+def _last_live_end(app):
+    global _LAST_END
+    if _LAST_END is None:
+        _LAST_END = {}
+        voided = set()
+        rows = []
+        try:
+            with open(os.path.join(met_dir, "runs.jsonl"), encoding="utf-8") as fh:
+                for line in fh:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        r = json.loads(line)
+                    except Exception:
+                        continue
+                    if r.get("kind") == "run-void":
+                        voided.add((r.get("app"), r.get("cmd"), r.get("started")))
+                    elif r.get("kind", "run") == "run" and not r.get("backfilled"):
+                        rows.append(r)
+        except Exception:
+            pass
+        for r in rows:
+            if (r.get("app"), r.get("cmd"), r.get("started")) in voided:
+                continue
+            # `ended` ONLY. `ts` is when the record was WRITTEN, not when the run finished, so
+            # falling back to it would make every historical record emitted today look like an
+            # overlap with the record written a second ago. A run with no `ended` has no window
+            # and constrains nothing.
+            #
+            # And only a record whose window is COHERENT: a self-contradicting one (ended before
+            # started) has its `ended` replaced with the write time by the check above, and
+            # marked by a null `duration_s`. That placeholder is not a measurement, so it must
+            # not become a boundary — it would block every later record for the same app.
+            if not isinstance(r.get("duration_s"), (int, float)):
+                continue
+            end = r.get("ended")
+            if not end:
+                continue
+            a = r.get("app")
+            if a not in _LAST_END or end > _LAST_END[a][0]:
+                _LAST_END[a] = (end, r.get("cmd"), r.get("started"))
+    return _LAST_END.get(app)
+
+
+def _overlap_ok(rec):
+    """False (and the reason printed) when this run would start before the last one ended."""
+    if stream != "runs" or rec.get("kind", "run") != "run" or rec.get("backfilled"):
+        return True
+    if os.environ.get("TF_ALLOW_OVERLAP") == "1":
+        return True
+    started = rec.get("started")
+    if not started:
+        return True
+    prev = _last_live_end(rec.get("app"))
+    if not prev:
+        return True
+    a, b = _iso_ms(started), _iso_ms(prev[0])
+    if a is not None and b is not None:
+        if a >= b:
+            return True
+    elif started >= prev[0]:        # unparseable timestamp: fall back to the string order
+        return True
+    sys.stdout.write(
+        "tf-emit: REFUSED — this run starts at %s, before the previous run for %s ended at %s "
+        "(%s started %s). The two would overlap and every total over them would be that much too "
+        "big. Use the previous run's `ended` as this one's `started`, or pass --allow-overlap if "
+        "they really did run at the same time (a second machine on the same stream). "
+        "Nothing was appended.\n"
+        % (started, rec.get("app"), prev[0], prev[1], prev[2]))
+    return False
+
+
 def stamp_attempt(rec):
     if stream != "runs" or "attempt" in rec or rec.get("backfilled"):
         return rec
@@ -771,7 +1161,7 @@ def enrich_run(rec):
             tier = ROUTING["phases"].get(cmd)
             if tier and tier != "inherit":
                 rec["tier"] = tier
-                key = {"claude-code": "claude", "opencode": "opencode", "codex": "codex"}.get(HARNESS)
+                key = {"claude-code": "claude", "opencode": "opencode"}.get(HARNESS)
                 tm = ROUTING["tiers"].get(tier, {}).get(key) if key else None
                 if tm:
                     rec["tier_model"] = tm
@@ -812,8 +1202,21 @@ def enrich_run(rec):
                 rec["tokens_out_subagents"] = win.get("sub_out", 0)
                 if rec.get("tier_model") and rec.get("model"):
                     rec["routed"] = rec["model"] == rec["tier_model"]
+                # What a cost figure from this window MEANS. Several models in one window may
+                # be paid for differently — a fallback from a subscription to a metered key is
+                # exactly that case — so a mixed window says so rather than picking a winner.
+                modes = {_billing(m, HARNESS)[0] for m in models} or {_billing(rec.get("model"), HARNESS)[0]}
+                rec["billing_mode"] = modes.pop() if len(modes) == 1 else "mixed"
+                rec["cost_source"] = "opencode-db" if HARNESS == "opencode" else "none"
+                # `routed:false` because the tier model was out of quota is a different fact from
+                # `routed:false` because something drifted. Only the first names a model here.
+                tm = rec.get("tier_model")
+                if tm and rec.get("model") and rec["model"] != tm and _was_limited(tm, HARNESS, t0 / 1000.0):
+                    rec["fallback_from"] = tm
             else:
                 rec["tokens_scope"] = "none"
+                rec["billing_mode"] = _billing(rec.get("tier_model"), HARNESS)[0]
+                rec["cost_source"] = "none"
     except Exception as e:
         warn("run enrichment failed (%s) — record kept unenriched" % e)
     return rec
@@ -881,7 +1284,7 @@ def _runs_by_start():
     return _RUNS_BY_START
 
 _COST_FIELDS = ("tokens_in", "tokens_out", "tokens_cache_read", "tokens_cache_write",
-                "cost_usd", "tokens_scope", "model")
+                "cost_usd", "tokens_scope", "model", "billing_mode", "cost_source")
 
 # Kept identical to the --amend branch above. Two doors, ONE enforcement: an
 # agent that hand-writes a miss-amend record onto the stream faces the same
@@ -891,14 +1294,23 @@ _AMENDABLE = {
     "why_missed": ("missing-checklist-item", "insufficient-verify-method",
                    "code-audit-limitation", "ambiguous-acceptance",
                    "dependency-not-declared", "instruction-ignored", "other"),
+    "sort": ("spec", "unsaid", "weak-check", "ignored"),
 }
+_AMENDABLE_TEXT = ("what",)   # free text, guarded by the never-overwrite rule (see --amend)
 
 def _amend_ok(rec):
     """True when this miss-amend may be appended. Returning False DROPS it —
     an invalid amend writes nothing rather than landing as a record the reader
     then has to defend itself against."""
     mid, fld, val = rec.get("miss_id"), rec.get("field"), rec.get("value")
-    if fld not in _AMENDABLE or val not in _AMENDABLE[fld] or not mid:
+    if not mid:
+        warn("amend refused — no miss_id")
+        return False
+    if fld in _AMENDABLE_TEXT:
+        if not isinstance(val, str) or not val.strip():
+            warn("amend refused — %s must be a non-empty sentence" % fld)
+            return False
+    elif fld not in _AMENDABLE or val not in _AMENDABLE[fld]:
         warn("amend refused — %r/%r outside the §5.5.7 allowlist" % (fld, val))
         return False
     parent, current = None, None
@@ -952,6 +1364,14 @@ _MISS_ENUMS = {
     "why_missed": ("missing-checklist-item", "insufficient-verify-method",
                    "code-audit-limitation", "ambiguous-acceptance",
                    "dependency-not-declared", "instruction-ignored", "other"),
+    # whose gap (FR-32, Session 5 2026-09-07): spec = the app's spec did not say it (fix the
+    # checklist line) · unsaid = the framework never said it (a requirement line plus a check) ·
+    # weak-check = a check existed and did not catch it (fix the check) · ignored = it was written
+    # and not followed (a hook, or delete the rule)
+    "sort": ("spec", "unsaid", "weak-check", "ignored"),
+}
+_REVIEW_ENUMS = {
+    "phase": ("day1-review", "build-review", "verify-review", "handoff-review"),
 }
 _FIX_ENUMS = {
     "verdict_after": ("Verified", "Needs re-verify", "FAIL", "deferred", "wont-fix"),
@@ -975,6 +1395,81 @@ def _enums_ok(rec, table):
     return True
 
 
+# --- shape check (added 2026-09-05, reset Sitting 4a) ------------------------
+# A record may carry only the fields SCHEMA.md defines for its kind (plus the
+# ones this script injects). A `miss` must carry a miss_id from --next-miss-id.
+# Anything else is REFUSED on stdout and never appended, the same choice as
+# _enums_ok(): a field nobody defined is a field no report can read.
+_COMMON = {"v", "ts", "kind", "app", "project_type", "project_type_inferred", "harness", "backfilled"}
+_ALLOWED = {
+    ("runs", "run"): {"cmd", "mode", "yolo", "started", "ended", "duration_s", "reqs_touched", "reqs_count",
+                      "subagents", "files_written", "build_result", "tier", "tier_model", "model", "models",
+                      "routed", "tokens_in", "tokens_out", "tokens_cache_read", "tokens_cache_write",
+                      "cost_usd", "tokens_scope", "attempt", "subagent_runs", "tokens_out_subagents",
+                      "model_tokens_out", "session_id", "usage_start_event_ts", "usage_end_event_ts",
+                      "billing_mode", "cost_source", "fallback_from"},
+    # A correction, not a deletion: it names a run record that is wrong by the pair every
+    # run carries (`cmd` + `started`) and says why, so every figure can skip it while the
+    # record itself stays where it is. Written by --void-run, which refuses one that names
+    # no run. SCHEMA.md §2.7; tests/regression/run.sh tf_void.
+    ("runs", "run-void"): {"cmd", "started", "reason"},
+    ("gates", "gate"): {"run_id", "req_id", "req_class", "attempt", "verdict", "gate", "gates_run",
+                        "failure_class", "prior_verdict", "proof", "escaped", "phase", "result", "detail", "at",
+                        "tier", "tier_model", "model", "models", "tokens_in", "tokens_out", "tokens_cache_read",
+                        "tokens_cache_write", "cost_usd", "tokens_scope", "model_tokens_out", "routed",
+                        "subagent_runs", "tokens_out_subagents", "session_id",
+                        "billing_mode", "cost_source", "fallback_from"},
+    ("sessions", "session"): {"session_id", "model", "duration_s", "input_tokens", "output_tokens",
+                              "cache_read_tokens", "cache_creation_tokens", "cost_usd", "children_sessions"},
+    ("commits", "commit"): {"sha", "files", "insertions", "deletions", "subject_prefix", "branch"},
+    ("misses", "miss"): {"miss_id", "req_id", "req_class", "miss_class", "artifact", "severity", "origin_phase",
+                         "origin_agent", "origin_run_id", "origin_model", "origin_harness", "origin_confidence",
+                         "why_missed", "found_by", "found_phase", "found_gate", "found_run_id", "failure_class",
+                         "what", "sort"},
+    ("misses", "miss-fix"): {"miss_id", "req_id", "fix_run_id", "fix_cmd", "fix_attempt", "verdict_after",
+                             "reopened", "cost_attribution", "tokens_in", "tokens_out", "tokens_cache_read",
+                             "tokens_cache_write", "cost_usd", "tokens_scope", "model",
+                             "billing_mode", "cost_source"},
+    ("misses", "miss-amend"): {"miss_id", "field", "value"},
+    # an owner review of a phase's output (FR-36, built 2026-09-06): how many corrections were
+    # given, what producing the reviewed output cost, what applying the corrections cost. The two
+    # costs are copied from the runs named by reviewed_run_id and correction_run_id (their `started`).
+    ("misses", "review"): {"phase", "reviewed_run_id", "correction_run_id", "corrections", "what",
+                           "tokens_produce", "tokens_correct", "cost_produce_usd", "cost_correct_usd",
+                           "model_produce", "model_correct"},
+}
+# Fields the emitter derives for runs/gates: a caller's value is discarded.
+_DERIVED = ("harness", "tier", "tier_model", "model", "models", "model_tokens_out", "routed",
+            "tokens_in", "tokens_out", "tokens_cache_read", "tokens_cache_write", "cost_usd",
+            "tokens_scope", "subagent_runs", "tokens_out_subagents",
+            "billing_mode", "cost_source", "fallback_from")
+
+
+def _shape_ok(rec):
+    kind = rec.get("kind")
+    if not isinstance(rec, dict) or not kind:
+        sys.stdout.write("tf-emit: REFUSED — record has no \"kind\" (SCHEMA.md §1). Nothing was appended.\n")
+        return False
+    allowed = _ALLOWED.get((stream, kind))
+    if allowed is None:
+        sys.stdout.write("tf-emit: REFUSED — kind %r is not a record kind of the %s stream. Nothing was appended.\n" % (kind, stream))
+        return False
+    unknown = sorted(set(rec) - allowed - _COMMON)
+    if unknown:
+        sys.stdout.write("tf-emit: REFUSED — field(s) not in SCHEMA.md for a %s record: %s. Add the field to the schema first, or drop it. Nothing was appended.\n" % (kind, ", ".join(unknown)))
+        return False
+    if kind == "run" and rec.get("build_result") not in (None, "pass", "fail", "not-run"):
+        sys.stdout.write("tf-emit: REFUSED — build_result must be pass, fail or not-run (SCHEMA.md §2); got %r. Put the detail in the checklist Remarks or Known blockers, not here. Nothing was appended.\n" % (rec.get("build_result"),))
+        return False
+    if kind == "miss" and not rec.get("miss_id"):
+        sys.stdout.write("tf-emit: REFUSED — a miss record needs a miss_id; get one with: bash .tfcore/utils/tf-emit.sh --next-miss-id. Nothing was appended.\n")
+        return False
+    if stream in ("runs", "gates") and not rec.get("backfilled"):
+        for f in _DERIVED:
+            rec.pop(f, None)
+    return True
+
+
 def enrich_miss(rec):
     if stream != "misses":
         return rec
@@ -984,6 +1479,21 @@ def enrich_miss(rec):
         return None
     if rec.get("kind") == "miss-fix" and not _enums_ok(rec, _FIX_ENUMS):
         return None
+    if rec.get("kind") == "review":
+        if not _enums_ok(rec, _REVIEW_ENUMS):
+            return None
+        if not isinstance(rec.get("corrections"), int) or rec["corrections"] < 0:
+            sys.stdout.write("tf-emit: REFUSED — a review record needs \"corrections\", a whole number of corrections the owner gave. Nothing was appended.\n")
+            return None
+        for f in ("tokens_produce", "tokens_correct", "cost_produce_usd", "cost_correct_usd", "model_produce", "model_correct"):
+            rec.pop(f, None)            # derived, never a caller's figure
+        runs = _runs_by_start()
+        for side, key in (("produce", "reviewed_run_id"), ("correct", "correction_run_id")):
+            src = runs.get(rec.get(key)) if rec.get(key) else None
+            rec["tokens_" + side] = src.get("tokens_out") if src else None
+            rec["cost_" + side + "_usd"] = src.get("cost_usd") if src else None
+            rec["model_" + side] = src.get("model") if src else None
+        return rec
     try:
         kind = rec.get("kind")
         runs = _runs_by_start()
@@ -1003,9 +1513,11 @@ def enrich_miss(rec):
                 rec["origin_harness"] = None
         elif kind == "miss-fix":
             src = runs.get(rec.get("fix_run_id")) if rec.get("fix_run_id") else None
+            for f in _COST_FIELDS:
+                rec.pop(f, None)            # a caller's cost figure is never kept
             if src:
                 for f in _COST_FIELDS:
-                    if f in src and f not in rec:
+                    if f in src:
                         rec[f] = src[f]
                 if "cost_attribution" not in rec:
                     touched = src.get("reqs_touched") or []
@@ -1048,7 +1560,14 @@ def enrich_miss(rec):
 # .gitattributes block pins to LF. These files are LF on every platform.
 lines = []
 for rec in records:
-    out = enrich_miss(enrich_run(stamp_attempt(enrich(rec))))
+    if not _shape_ok(rec):
+        continue
+    # enrich() first, because it is what fills `app` when the caller left it out — the overlap
+    # rule is per app and would otherwise compare a named run against an unnamed one.
+    base = enrich(rec)
+    if not _overlap_ok(base):
+        continue
+    out = enrich_miss(enrich_run(stamp_attempt(base)))
     if out is None:                 # a refused miss-amend (§5.5.7) — dropped, never appended
         continue
     line = json.dumps(out, separators=(",", ":"), ensure_ascii=False)
@@ -1072,4 +1591,12 @@ PY
 )"
 python3 -c "$TF_PROG" "$MET_DIR" "$STREAM" "$ROOT" 2>"$TF_ERR"
 
+# The readable miss list beside the record (FR-31, Session 5 2026-09-07): after any write to the
+# misses stream, docs/<App>-Misses.md is rebuilt from the stream, so the file is never older than
+# the record. Markdown only — the miss log is an agent document and is never rendered to HTML
+# (owner, 2026-09-08). Best effort, like everything else here.
+if [[ "$STREAM" == "misses" ]]; then
+  python3 "$(dirname "${BASH_SOURCE[0]}")/tf-misses-md.py" --root "$ROOT" --quiet 2>"$TF_ERR" || true
+fi
+
 exit 0