@techierathore/techieflow 1.0.3 → 1.1.0

package/.tfcore/utils/tf-status-facts.py

@@ -27,6 +27,7 @@ import re
 import sys
 
 TERMINAL = {"verified", "done (pre-existing)", "n/a"}
+NA = {"n/a"}                         # terminal, but never counted as verified (TF-018)
 PASS_THROUGH = {"blocked"}           # a library gap; counts as closed for the ladder
 BUILT = {"implemented", "needs re-verify"}
 OWNER = {"owner-uat"}                # only the owner can close it, from the UsageGuide test plan
@@ -61,6 +62,9 @@ def read(path):
 
 
 def checklist_rows(text):
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    import tf_roadmap
+    marked = tf_roadmap.ids(text)
     rows = []
     for line in text.splitlines():
         if not re.match(r"^\s*\|\s*`?REQ-", line):
@@ -69,22 +73,76 @@ def checklist_rows(text):
         if len(cells) < 3:
             continue
         rid = cells[0].strip("`* ")
-        rows.append({"id": rid, "name": cells[1].strip("`* "), "status": norm_status(cells[2])})
+        status = norm_status(cells[2])
+        rows.append({"id": rid, "name": cells[1].strip("`* "), "status": status,
+                     "remarks": cells[4] if len(cells) > 4 else "",
+                     "roadmap": rid.upper() in marked and status.lower() not in TERMINAL})
     return rows
 
 
+def figures(rows):
+    """(verified, counted, not applicable). An N/A row is terminal but was never verified, so it is
+    left out of both numbers and named: AppManager read "89 of 89 verified" with REQ-NFR-008 at N/A
+    (TF-018). tests/regression/run.sh am_018."""
+    na = sum(1 for r in rows if r["status"].lower() in NA)
+    return sum(1 for r in rows if r["status"].lower() in TERMINAL - NA), len(rows) - na, na
+
+
+# The defect marks live in tf_defect.py. A bare ⚠ is not enough: sending "not verifiable here" to a
+# build is the TfLens loop again.
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from tf_defect import DEFECT  # noqa: E402
+
+
+def has_defect(r):
+    """A Needs re-verify row whose Remarks carry a framework defect mark holds a defect no fix has
+    cleared, and a verify of the acceptance line cannot see it. So it is fix work, as build-phase step 7
+    and tf-build-list say, never a verify (AppManager TF-019). tests/regression/run.sh am_019."""
+    return r["status"].lower() == "needs re-verify" and bool(DEFECT.search(r.get("remarks", "")))
+
+
 def ladder_rank(status):
     return LADDER.index(status) if status in LADDER else 0
 
 
+def ledger(root):
+    """docs/.last-verify.json as a dict, or {} when there is none.
+
+    The verifier writes it with `json.dump(..., indent=1)`, so it is ONE pretty-printed
+    object spanning many lines. This reader used to take `splitlines()[-1]` — written for
+    an append-per-line ledger that no version of the verifier has ever produced — so it
+    parsed the closing brace, threw, and every project reported `last_verified_build:
+    not-run` however many verify runs it had behind it (TfLens: 69 of 73 rows Verified and
+    a full ledger, reported as never verified). Whole file first, last line second for a
+    legacy JSONL ledger. tests/regression/run.sh tf_ledger."""
+    path = os.path.join(root, "docs", ".last-verify.json")
+    if not os.path.isfile(path):
+        return {}
+    text = read(path).strip()
+    for candidate in (text, text.splitlines()[-1] if text else ""):
+        try:
+            data = json.loads(candidate)
+            if isinstance(data, dict):
+                return data
+        except Exception:
+            continue
+    return {}
+
+
+def ledger_verdicts(root):
+    """{REQ id: verdict} from the last verify run, or {} when none has run.
+
+    The verifier already writes every row's verdict into the ledger. Reading it here is
+    what lets the status line tell "built, waiting for a verify" apart from "verified as
+    far as it can be, and the rest needs a decision only the owner can make"."""
+    rows = ledger(root).get("rows")
+    return rows if isinstance(rows, dict) else {}
+
+
 def last_verify(root, app):
     """(date, build_result) from docs/.last-verify.json and gates.jsonl; ('', 'not-run') if none."""
-    ledger = os.path.join(root, "docs", ".last-verify.json")
-    if not os.path.isfile(ledger):
-        return "", "not-run"
-    try:
-        data = json.loads(read(ledger).strip().splitlines()[-1])
-    except Exception:
+    data = ledger(root)
+    if not data:
         return "", "not-run"
     date = str(data.get("date", ""))[:10]
     result = "PASS"
@@ -117,19 +175,13 @@ def handoff_ran(log_rows):
 
 
 def feedback_lines(root, app):
-    out = []
-    for path in sorted(glob.glob(os.path.join(root, "docs", f"{app}-*-Feedback.md"))):
-        lib = re.sub(rf"^{re.escape(app)}-(.+)-Feedback\.md$", r"\1", os.path.basename(path))
-        text = read(path)
-        entries = re.split(r"(?m)^###\s+", text)[1:]
-        open_n = 0
-        for e in entries:
-            closed = re.search(r"(?im)^\s*[-*]\s*\*\*(status|resolved|fixed)[^:]*:\*\*\s*(fixed|resolved|closed|done)", e)
-            if not closed:
-                open_n += 1
-        rel = os.path.relpath(path, root).replace(os.sep, "/")
-        out.append(f"- {lib}: {open_n} open of {len(entries)} — {rel}")
-    return out or ["- None"]
+    # One reader for every script (tf_feedback.py). This function used to count every `###`
+    # heading as an entry and accept only a closing line no file used, so TfLens's status read
+    # "TechieFlow: 62 open of 62" for 27 entries, 19 of them fixed or closed, and the agent told
+    # the owner two fixed problems were open (MISS-TechieFlow-20260911-04).
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    import tf_feedback
+    return [tf_feedback.summary_line(p, app) for p in tf_feedback.files(root, app)] or ["- None"]
 
 
 def config_phase(root):
@@ -161,8 +213,19 @@ def phase_row(root, app, phase):
     return name, total
 
 
-def next_command(app, rows, log_rows, phase=1):
+def usage_guide(app):
+    """UsageGuide or Usage-Guide, whichever exists: tf-verify-list.py's reader (AppManager TF-022)."""
+    import importlib.util
+    spec = importlib.util.spec_from_file_location("tf_verify_list", os.path.join(os.path.dirname(os.path.abspath(__file__)), "tf-verify-list.py"))
+    vl = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(vl)
+    return vl.usage_guide(app)
+
+
+def next_command(app, rows, log_rows, phase=1, verdicts=None):
     """Returns (phase, qualifier, cc_line, oc_line, reason)."""
+    verdicts = verdicts or {}
+
     def ids(lst):
         return ", ".join(r["id"] for r in lst[:6]) + (" …" if len(lst) > 6 else "")
 
@@ -175,26 +238,49 @@ def next_command(app, rows, log_rows, phase=1):
                 f"{CC}analyst *day1-greenfield {app}", f"{OC}flow-analyst *day1-greenfield {app}",
                 "no BRD and no checklist exist; day-1 stage 1 has not run")
 
-    unbuilt = [r for r in rows if r["status"].lower() not in TERMINAL | PASS_THROUGH | BUILT | OWNER]
-    built = [r for r in rows if r["status"].lower() in BUILT]
-    owner = [r for r in rows if r["status"].lower() in OWNER]
-    total = len(rows)
-    closed = sum(1 for r in rows if r["status"].lower() in TERMINAL)
-    q = f"{closed} of {total} verified"
-
-    if unbuilt:
-        return ("Build", f"{len(unbuilt)} not built, {q}",
-                f"{CC}flow-master *build-phase {app}", f"{OC}flow-master *build-phase {app}",
-                f"{len(unbuilt)} rows are not built yet: {ids(unbuilt)}")
+    # A roadmap row is not this phase's work, so it never decides the next command (TF-017).
+    now = [r for r in rows if not r.get("roadmap")]
+    unbuilt = [r for r in now if r["status"].lower() not in TERMINAL | PASS_THROUGH | BUILT | OWNER]
+    defects = [r for r in now if has_defect(r)]
+    built = [r for r in now if r["status"].lower() in BUILT and r not in defects]
+    owner = [r for r in now if r["status"].lower() in OWNER]
+    verified, counted, na = figures(rows)
+    q = f"{verified} of {counted} verified" + (f", {na} not applicable" if na else "")
+
+    if unbuilt or defects:
+        qual = ", ".join(x for x in (f"{len(unbuilt)} not built" if unbuilt else "",
+                                     f"{len(defects)} to fix" if defects else "") if x)
+        why = "; ".join(x for x in (f"{len(unbuilt)} rows are not built yet: {ids(unbuilt)}" if unbuilt else "",
+                                    f"{len(defects)} rows carry a defect (⚠ in Remarks) that a fix must clear before "
+                                    f"a verify: {ids(defects)}" if defects else "") if x)
+        return ("Build", f"{qual}, {q}",
+                f"{CC}flow-master *build-phase {app}", f"{OC}flow-master *build-phase {app}", why)
     if built:
-        ui = [r for r in built if r["id"].startswith("REQ-UI")]
-        scope = "ui" if len(ui) == len(built) else ("functional" if not ui else "all")
-        return ("Verify", f"{len(built)} to verify, {q}",
+        # A row the last verify could not MEASURE is not a row waiting for another verify.
+        # Its tests declined — the state they need does not exist in this environment — so
+        # running the verifier again produces the same NOT-TESTED and the project sits in a
+        # loop that looks like progress. Say what it actually needs: data, a changed
+        # acceptance line, or the row marked N/A. Only the owner can choose. FR-69's sibling
+        # case; the verdicts come from the ledger the verifier already writes.
+        unmeasured = [r for r in built if verdicts.get(r["id"]) == "NOT-TESTED"]
+        waiting = [r for r in built if r not in unmeasured]
+        if unmeasured and not waiting:
+            line = (f"(owner) {len(unmeasured)} row(s) cannot be measured here — create the data, "
+                    f"change the acceptance line, or mark the row N/A: {ids(unmeasured)}")
+            return ("Verify", f"{len(unmeasured)} not measurable, {q}", line, line,
+                    f"every test carrying these ids was skipped, so another verify run changes nothing: {ids(unmeasured)}")
+        ui = [r for r in waiting if r["id"].startswith("REQ-UI")]
+        scope = "ui" if len(ui) == len(waiting) else ("functional" if not ui else "all")
+        reason = f"{len(waiting)} rows are built and not verified: {ids(waiting)}"
+        if unmeasured:
+            reason += (f"; {len(unmeasured)} more could not be measured (every test skipped) "
+                       f"and need the owner: {ids(unmeasured)}")
+        return ("Verify", f"{len(waiting)} to verify, {q}",
                 f"{CC}verifier *verify {scope} {app}", f"{OC}flow-verifier *verify {scope} {app}",
-                f"{len(built)} rows are built and not verified: {ids(built)}")
+                reason)
     if owner:
         if handoff_ran(log_rows):
-            line = f"(owner-run) docs/{app}-UsageGuide.md — {ids(owner)}"
+            line = f"(owner-run) {usage_guide(app)} — {ids(owner)}"
             return ("UAT", f"{len(owner)} owner-run, {q}", line, line,
                     f"{len(owner)} rows are Owner-UAT and handoff has run; the owner closes them")
         return ("Handoff", f"{len(owner)} owner-run, {q}",
@@ -203,10 +289,10 @@ def next_command(app, rows, log_rows, phase=1):
     if handoff_ran(log_rows):
         line = "(owner) set current_phase to Released after UAT — no agent command"
         return ("UAT", f"handoff done, {q}", line, line,
-                "every row is terminal and handoff has run; waiting on the owner")
+                "every row in this phase's scope is terminal and handoff has run; waiting on the owner")
     return ("Handoff", q,
             f"{CC}flow-master *handoff-phase {app}", f"{OC}flow-master *handoff-phase {app}",
-            "every row is terminal and handoff has not run yet")
+            "every row in this phase's scope is terminal and handoff has not run yet")
 
 
 def main(argv):
@@ -234,15 +320,18 @@ def main(argv):
     st_text = read(st_path) if os.path.isfile(st_path) else ""
     log_rows = existing_log_rows(st_text)
     vdate, vresult = last_verify(root, app)
-    phase, qual, cc, oc, reason = next_command(app, rows, log_rows, phase_n)
+    phase, qual, cc, oc, reason = next_command(app, rows, log_rows, phase_n, ledger_verdicts(root))
+    roadmap = [r["id"] for r in rows if r["roadmap"]]
+    if roadmap:
+        reason += (f"; {len(roadmap)} roadmap row(s) are not in this phase's scope and are left for the owner "
+                   f"(*amend-docs): {', '.join(roadmap)}")
     pname, ptotal = phase_row(root, app, phase_n)
     if ptotal or phase_n > 1:
         tag = f"Phase {phase_n} of {ptotal or '?'}" + (f" ({pname})" if pname else "")
         phase = f"{tag} · {phase}"
         reason = f"{reason}; working {cl_rel}"
 
-    total = len(rows)
-    closed = sum(1 for r in rows if r["status"].lower() in TERMINAL)
+    verified, counted, na = figures(rows)
     counts = {}
     for r in rows:
         counts[r["status"]] = counts.get(r["status"], 0) + 1
@@ -273,7 +362,7 @@ def main(argv):
     print()
     if open_rows:
         for r in open_rows[:10]:
-            print(f"- [ ] {r['id']} — {r['name']} ({r['status']})")
+            print(f"- [ ] {r['id']} — {r['name']} ({r['status']}{', roadmap — not in this phase' if r['roadmap'] else ''})")
         if len(open_rows) > 10:
             print(f"- ({len(open_rows) - 10} more open rows in {cl_rel})")
     else:
@@ -283,7 +372,7 @@ def main(argv):
     print("Last five passes; older passes live in `docs/metrics/gates.jsonl`.")
     print()
     print("| Date | Phase | Result | Status table |\n|---|---|---|---|")
-    new_row = f"| {today} | {label} | {closed}/{total} Verified | {cl_rel}#requirements-status |"
+    new_row = f"| {today} | {label} | {verified}/{counted} Verified{f', {na} N/A' if na else ''} | {cl_rel}#requirements-status |"
     for r in (log_rows[-4:] if len(log_rows) > 4 else log_rows):
         print(r)
     print(new_row)

package/.tfcore/utils/tf-triage.py

@@ -8,6 +8,9 @@
     bash .tfcore/utils/tf-triage.sh <App> note <REQ> "<could not reproduce: what was tried>"
     bash .tfcore/utils/tf-triage.sh <App> close [--started <ISO>] [--verify-ran] [--cmd triage-issues|fix-issues]
 
+demote, new and note take --phase N for another phase's checklist; without it demote and note find the
+row in whichever docs/<App>-Checklist.md or docs/<App>-P<N>-Checklist.md holds it.
+
 demote  the row goes to Needs re-verify (% capped at 75) with a dated Remark holding the symptom in the
         reporter's words, the evidence path and the kind.
 new     a defect no row covers: the next free id in the prefix, a Not Started row, a detail entry with the
@@ -20,7 +23,8 @@ close   what triage owes the telemetry, from the actions above (tests/.artifacts
         spec for a new row, which the spec never had) (duplicates collapsed through
         tf-emit.sh --open-miss; the origin run looked up with --origin-of), then the run record. It also
         lists every file under src/, source/ or tests/ changed since the start: a triage that edited code
-        is reported and logged as an instruction-ignored miss (FR-28).
+        is reported and logged as an instruction-ignored miss (FR-28). Only actions logged since the start
+        count, and a close empties the list (TrBlazeUI TF-001).
 Exit 0 · 2 could not run.
 """
 import datetime
@@ -58,12 +62,36 @@ def cfg_phase():
     return 1
 
 
-def checklist(app):
-    ph = cfg_phase()
-    p = os.path.join("docs", f"{app}-Checklist.md" if ph <= 1 else f"{app}-P{ph}-Checklist.md")
-    if not os.path.isfile(p):
-        die(f"{p} does not exist")
-    return p
+def phase_checklist(app, ph):
+    return os.path.join("docs", f"{app}-Checklist.md" if ph <= 1 else f"{app}-P{ph}-Checklist.md")
+
+
+def checklist(app, argv, ce, rid=None):
+    """--phase N names the checklist, as tf-verify-list.py takes it. Without it: the appPhase
+    checklist, or, for a row that is not there, the one earlier-phase checklist that holds it —
+    a Phase 3 verify finds Phase 1 rows failing, and demote could not reach them (TF-051)."""
+    if "--phase" in argv:
+        v = opt(argv, "--phase", "")
+        if not v.isdigit():
+            die("--phase needs a number")
+        p = phase_checklist(app, int(v))
+        if not os.path.isfile(p):
+            die(f"{p} does not exist")
+        return p
+    p = phase_checklist(app, cfg_phase())
+    if rid is None or (os.path.isfile(p) and rid in ce.rows(ce.read(p))):
+        if not os.path.isfile(p):
+            die(f"{p} does not exist")
+        return p
+    import glob
+    every = sorted(set(glob.glob(os.path.join("docs", f"{app}-Checklist.md")) + glob.glob(os.path.join("docs", f"{app}-P*-Checklist.md"))))
+    hits = [c for c in every if rid in ce.rows(ce.read(c))]
+    if len(hits) == 1:
+        print(f"note: {rid} is a row of {hits[0]}, not of {p}; written there")
+        return hits[0]
+    if len(hits) > 1:
+        die(f"{rid} is a row of {', '.join(hits)}; name one with --phase N")
+    die(f"{rid} is not a row of {', '.join(every) or p}")
 
 
 def log_load():
@@ -112,7 +140,6 @@ def main(argv):
         return 0 if len(argv) > 1 else 2
     app, verb = argv[1], argv[2]
     ce = mod()
-    cl = checklist(app)
     log = log_load()
     log.setdefault("app", app)
     now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -121,6 +148,7 @@ def main(argv):
         if len(argv) < 5:
             die("demote needs <REQ> \"<symptom>\"")
         rid, symptom = argv[3].upper(), argv[4]
+        cl = checklist(app, argv, ce, rid)
         kind = opt(argv, "--kind"); ev = opt(argv, "--evidence"); src = opt(argv, "--source", "owner")
         prior = ce.demote(cl, rid, symptom, kind=kind, evidence=ev, source=src, prefix="⚠ prod bug" if src == "production" else "⚠ UAT bug")
         if prior is None:
@@ -136,6 +164,7 @@ def main(argv):
         if len(argv) < 5:
             die("new needs \"<title>\" \"<acceptance>\"")
         title, acc = argv[3], argv[4]
+        cl = checklist(app, argv, ce)
         prefix = opt(argv, "--prefix", "FN").upper(); section = opt(argv, "--section"); ev = opt(argv, "--evidence")
         src = opt(argv, "--source", "owner"); mockup = opt(argv, "--mockup")
         if prefix == "UI" and not mockup:
@@ -152,6 +181,7 @@ def main(argv):
         if len(argv) < 5:
             die("note needs <REQ> \"<text>\"")
         rid = argv[3].upper()
+        cl = checklist(app, argv, ce, rid)
         if ce.note(cl, rid, argv[4]) is None:
             die(f"{rid} is not a row of {cl}")
         log["actions"].append({"verb": "note", "req_id": rid, "symptom": argv[4], "ts": now})
@@ -162,9 +192,11 @@ def main(argv):
     if verb == "close":
         started = opt(argv, "--started") or subprocess.run(["bash", os.path.join(HERE, "tf-phase.sh"), "show"], capture_output=True, text=True).stdout
         if started.startswith("{"):
-            import re
-            m = re.search(r'"started":"([^"]*)"', started)
-            started = m.group(1) if m else ""
+            # the marker's own started, never its "outer" block's (TF-052 follow-up)
+            try:
+                started = (json.loads(started) or {}).get("started") or ""
+            except ValueError:
+                started = ""
         started = started.strip() if started and not started.startswith("none") else ""
         cmd = opt(argv, "--cmd", "triage-issues")
         # code untouched?
@@ -173,16 +205,25 @@ def main(argv):
             t0 = datetime.datetime.strptime(started, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=datetime.timezone.utc).timestamp()
             for top in ("src", "source", "tests"):
                 for d, dirs, files in os.walk(top):
-                    dirs[:] = [x for x in dirs if x not in ("bin", "obj", "node_modules", ".artifacts", "TestResults")]
+                    # a running app's own log files are not code (AppManager TF-026)
+                    dirs[:] = [x for x in dirs if x not in ("bin", "obj", "node_modules", ".artifacts", "TestResults")
+                               and x.lower() not in ("logs", "log")]
                     for f in files:
+                        if f.lower().endswith(".log"):
+                            continue
                         p = os.path.join(d, f)
                         try:
                             if os.path.getmtime(p) > t0:
                                 changed.append(p.replace("\\", "/"))
                         except OSError:
                             pass
-        acts = [a for a in log.get("actions", []) if a["verb"] in ("demote", "new")]
+        # only this triage's actions: an earlier run's leftovers were logged again under today's run (TrBlazeUI TF-001)
+        stale = [a for a in log.get("actions", []) if started and (a.get("ts") or "") < started]
+        if stale:
+            print(f"  skipped {len(stale)} action(s) from before {started} — an earlier triage's, already closed or abandoned")
+        acts = [a for a in log.get("actions", []) if a["verb"] in ("demote", "new") and a not in stale]
         gates = misses = 0
+        failed = []
         for a in acts:
             rid = a["req_id"]; cls_req = rid.split("-")[1]
             att = q("--next-attempt", rid) or "1"
@@ -192,7 +233,9 @@ def main(argv):
                    "gate": "escaped", "gates_run": [], "failure_class": fc, "prior_verdict": a.get("prior")}
             if emit("gates", rec):
                 gates += 1
-            mc = ("unspecified-gap" if a["verb"] == "new" else "regression" if (a.get("prior") or "").lower().startswith("verified") else "wrong-behaviour")
+            else:
+                failed.append(a)
+            mc =("unspecified-gap" if a["verb"] == "new" else "regression" if (a.get("prior") or "").lower().startswith("verified") else "wrong-behaviour")
             opened = q("--open-miss", rid)
             if opened and opened.split()[-1] == mc:
                 print(f"  {rid}: already open as {opened.split()[0]} ({mc}); no second miss")
@@ -210,9 +253,11 @@ def main(argv):
             if emit("misses", m):
                 misses += 1
                 a["miss_id"] = mid
-        if changed and cmd == "triage-issues" and not log.get("code_miss_id"):
+            elif a not in failed:
+                failed.append(a)
+        if changed and cmd == "triage-issues" and log.get("code_miss_run") != started:
             mid = q("--next-miss-id")
-            log["code_miss_id"] = mid
+            log["code_miss_id"] = mid; log["code_miss_run"] = started
             emit("misses", {"kind": "miss", "miss_id": mid, "req_id": None, "req_class": None, "miss_class": "other", "artifact": "src",
                             "severity": "major", "why_missed": "instruction-ignored", "origin_phase": "triage-issues", "origin_agent": "flow-master",
                             "found_by": "gate", "found_phase": "triage-issues", "found_gate": None, "found_run_id": started, "failure_class": "other",
@@ -232,6 +277,7 @@ def main(argv):
         else:
             ok = None
         log["closed"] = now
+        log["actions"] = failed          # a closed action is never closed again; one whose record was refused waits for the next close
         log_save(log)
         print(f"triage: {len(acts)} row(s) logged — {gates} gate record(s) (escaped), {misses} miss(es)"
               + (f", run record {'written' if ok else 'not written'}" if ok is not None else "") + f"; code untouched: {'yes' if not changed else 'NO'}")

package/.tfcore/utils/tf-triage.sh

@@ -6,6 +6,8 @@
 #   bash .tfcore/utils/tf-triage.sh <App> note <REQ> "<could not reproduce: what was tried>"
 #   bash .tfcore/utils/tf-triage.sh <App> close [--started <ISO>] [--cmd triage-issues|fix-issues]
 #
+# demote, new and note take --phase N; without it a row of an earlier phase's checklist is found (TF-051).
+#
 # demote: Needs re-verify with a dated Remark in the reporter's words. new: a Not Started row with its
 # acceptance line and BRD-pending. note: a remark, no status change. close: the escaped gate records,
 # one miss per row (found by the owner, the symptom as `what`, duplicates collapsed), the run record,