@r1z4x/tezgah 1.6.0 → 1.7.0

package/hooks/tezgah_rank.py

@@ -13,12 +13,15 @@ import math
 import re
 from collections import Counter
 
+from tezgah_paths import packs
+
 # Lowercased words, Turkish letters kept; one- and two-letter words carry no topic.
 # Dotted and dotless i fold together (İ, I, ı -> i) so "İSTEK" meets "istek" and
 # "KIRMIZI" meets "kırmızı" while English capitals stay "index"; plain lower()
-# would turn İ into i + U+0307 and split the word. Circumflexes fold too.
+# would turn İ into i + U+0307 and split the word. A pack's `fold` pairs (tr:
+# dotless i and the circumflexes) fold too.
 WORD = re.compile(r"[a-zçğöşü0-9_]+")
-FOLD = str.maketrans("ıâîû", "iaiu")
+FOLD = str.maketrans({pair[0]: pair[1] for p in packs().values() for pair in p["fold"]})
 MIN_WORD = 3
 K1 = 1.2
 B = 0.75
@@ -33,32 +36,22 @@ def words(text):
 # The lessons path's own filter, passed as `max_df` (`tezgah_context.LESSON_MAX_DF`):
 # a word in most of the ledger ("the", "and" in 41 of 47 lines) ranked lines 9,
 # 7 and 34 for "update the changelog", a prompt no line is about. Function words
-# go first, in both languages the ledger is written in (folded the way `words`
-# folds them); then any term in more than `max_df` of the texts - once there are
+# go first, every pack's `rank_stopwords` (folded the way `words` folds them);
+# then any term in more than `max_df` of the texts - once there are
 # MIN_CAP_TEXTS of them, because on one or two texts a 50% cap drops every term.
 # The docs fallback passes no cap.
-STOPWORDS = frozenset((
-    "the", "and", "that", "this", "with", "for", "from", "are", "was", "were",
-    "not", "but", "its", "into", "than", "then", "they", "them", "their",
-    "have", "has", "had", "will", "would", "can", "could", "should", "you",
-    "your", "what", "when", "which", "who", "how", "all", "any", "out",
-    "bir", "bunu", "buna", "bunlar", "şunu", "için", "ile", "ama", "gibi",
-    "daha", "çok", "kadar", "veya", "değil", "her", "hem", "olan", "olarak",
-    "sonra", "önce", "ise", "yani", "nasil", "neden"))
+STOPWORDS = frozenset(w for p in packs().values() for w in p["rank_stopwords"])
 MIN_CAP_TEXTS = 10
 
 
-def rank(query, texts, k, max_df=None):
-    """Indices of the `k` texts that score highest for `query`, best first.
-
-    Only a text sharing at least one term with the query is returned (score > 0),
-    so an unrelated query gets [] rather than an arbitrary order. `max_df` (a
-    fraction) drops STOPWORDS and, on MIN_CAP_TEXTS or more texts, every query
-    term found in more than that share of them."""
+def scores(query, texts, max_df=None):
+    """Every text's BM25 score for `query`, in input order; 0.0 for a text that
+    shares no term with it. `rank` orders these; the research dedup reads them as
+    numbers (`tezgah_research.similarity_matrix`)."""
     docs = [Counter(words(t)) for t in texts]
     terms = set(words(query))
     if not docs or not terms:
-        return []
+        return [0.0] * len(docs)
     avg = sum(sum(d.values()) for d in docs) / len(docs) or 1.0
     df = {t: sum(1 for d in docs if t in d) for t in terms}
     if max_df is not None:
@@ -66,12 +59,21 @@ def rank(query, texts, k, max_df=None):
                  if len(docs) < MIN_CAP_TEXTS or df[t] <= max_df * len(docs)}
     idf = {t: math.log(1 + (len(docs) - df[t] + 0.5) / (df[t] + 0.5))
            for t in terms}
-    scored = []
-    for i, d in enumerate(docs):
+    out = []
+    for d in docs:
         size = sum(d.values())
-        score = sum(idf[t] * d[t] * (K1 + 1)
-                    / (d[t] + K1 * (1 - B + B * size / avg))
-                    for t in terms if t in d)
-        if score > 0:
-            scored.append((-score, i))
+        out.append(sum(idf[t] * d[t] * (K1 + 1)
+                       / (d[t] + K1 * (1 - B + B * size / avg))
+                       for t in terms if t in d))
+    return out
+
+
+def rank(query, texts, k, max_df=None):
+    """Indices of the `k` texts that score highest for `query`, best first.
+
+    Only a text sharing at least one term with the query is returned (score > 0),
+    so an unrelated query gets [] rather than an arbitrary order. `max_df` (a
+    fraction) drops STOPWORDS and, on MIN_CAP_TEXTS or more texts, every query
+    term found in more than that share of them."""
+    scored = [(-s, i) for i, s in enumerate(scores(query, texts, max_df)) if s > 0]
     return [i for _s, i in sorted(scored)[:k]]

package/hooks/tezgah_replay.py

@@ -3,8 +3,9 @@
 code, a blind label sheet drawn from them, and the report that reads the labels
 (plan 055, roadmap R12; corpus spec REPORT.md §4.4.3).
 
-  tezgah-gate replay [--cutoff WHEN] [--since WHEN] [--json]
-  tezgah-gate replay --report [--run DIR] [--labels FILE ...] [--json]
+  tezgah-gate replay [--cutoff WHEN] [--since WHEN] [--split search|holdout] [--json]
+  tezgah-gate replay --report [--run DIR] [--labels FILE ...]
+                     [--split search|holdout] [--json]
   tezgah-gate replay --sheet [--run DIR] [--seed N] [--rules FILE]
   tezgah-gate replay --label-model --prompt FILE [--provider jev|deepseek]
                      [--model ID] [--run DIR]
@@ -44,9 +45,20 @@ inputs), `results.jsonl` and the sheet files. The run's summary is its row in
 the cache database's replay index (`tezgah_store.add_replay`), whose newest
 row is the latest run. The sandbox HOME and the row stream are deleted when
 the child returns.
+
+**Split.** `--split search|holdout` reads one side of the run (idea 113): a
+session is on `search` when the first byte of the sha256 of its session id is
+even, else on `holdout` (`split_of`), so a session's side is the same in every
+process. The replay still runs the whole corpus and the indexed summary stays
+the whole corpus; the split recomputes the item counts, join, fidelity and race
+family over that side's items, and the report's label rates over that side's
+sheet rows. The ledger, exclusion and denial-budget counts stay whole-corpus.
+The report and a split run name the side they read (`split`, `all` with no
+`--split`) and its `sessions`.
 """
 import argparse
 import datetime
+import hashlib
 import importlib.machinery
 import importlib.util
 import json
@@ -90,7 +102,7 @@ RACE_BAR_UNMEASURABLE = ("not measurable on this corpus: no race item is on the
 # disk-state when the command reads a message file.
 STRATA = {"ledger": ("loop", "retry", "order", "drift", "race", "piped",
                      "attribution", "secret", "shortcut"),
-          "disk": ("task", "plan", "workspace", "explorer"),
+          "disk": ("task", "plan", "workspace", "explorer", "lease"),
           "disk_on_write": ("shortcut",),
           "disk_on_file": ("lang",)}
 MESSAGE_FILE = re.compile(r"(?:^|\s)(?:-F|--file)(?:\s|=|$)")
@@ -102,6 +114,12 @@ SEED = 55
 CUT_PROMPT, CUT_INPUT, CUT_REPLY = 1500, 2000, 3000
 STOP_CONTEXT = ("run", "edit", "verify", "verify_ok", "verify_fail",
                 "interrupted", "external")
+# A shell call that steps outward on the user's remote (`outward_without_grant`).
+OUTWARD_COMMAND = re.compile(r"\bgit\s+push\b|\b(?:npm|pnpm|yarn)\s+publish\b|\bgh\s+"
+                             r"(?:release\s+create|pr\s+(?:create|merge))\b|\btwine\s+upload\b")
+# How Claude writes a Stop hook's refusal back into the transcript: a user row,
+# but no user prompt, so no new turn.
+STOP_FEEDBACK = "Stop hook feedback:"
 
 
 def replay_root():
@@ -256,6 +274,96 @@ def _transcripts(wanted):
     return found, parent_of
 
 
+def residual_turns(cutoff, since=None):
+    """Residual sections per user turn over the top-level omp and Claude
+    transcripts (a subagent's reply is not user-facing). A turn starts at a user
+    prompt that is not a harness prompt nor a Stop hook's feedback (Claude
+    writes a refusal as a user row); its final reply is the last reply before
+    the next one, and it counts when that reply's ts is in `(since, cutoff]`.
+    A session in a fixture workspace (`ti.FIXTURE_WORKSPACE`, the cache) is
+    left out, as the corpus leaves its ledger out: a probe or a consult run is
+    no user turn. A turn did work when one of its calls `ti.classify`es.
+    `unblocked_work_turns` reads each reply with no ledger rows, so an item
+    whose evidence is a failed command counts as unblocked: an upper bound.
+    `question_endings` watches doable work rephrased as a question.
+    The ask policy's four counters (plan 086, S3): `process_questions`, the
+    final replies the Stop rule's `needless ask` reads as a process question
+    (no grant row exists here, so a re-ask after a grant is not counted);
+    `cheap_answers`, the prompts that only approve ("devam", "evet", "go on")
+    right after a reply that ended on a question; `asks_without_default`, the
+    ask-tool questions (omp `ask`, Claude AskUserQuestion) with no recommended
+    option (`ti.undefaulted`); `outward_without_grant`, the turns that ran an
+    outward shell call (OUTWARD_COMMAND) when no prompt of the session so far
+    had granted or named an outward step, nor approved a question that named
+    one - the flip condition of the S3 V3 gate reads this one.
+    Whole-corpus even under `--split`."""
+    import tezgah_context as tc  # the grant reader (`user_grants`)
+    taste = _taste()
+    out = Counter()
+
+    def counted(ts):
+        return ts is not None and ts <= cutoff and (since is None or ts > since)
+
+    def close(reply, work, outward):
+        """(ended on a question, the question named an outward step) for a
+        counted turn, else None."""
+        if reply is None or not counted(reply[1]):
+            return None
+        text = reply[0]
+        found = ti.residual_sections(text)
+        lines = [ln.strip().rstrip("*_` ") for ln in text.splitlines() if ln.strip()]
+        asked = bool(lines and ti.QUESTION_END.search(lines[-1]))
+        out["turns"] += 1
+        out["work_turns"] += work
+        out["residual_turns"] += bool(found)
+        out["residual_work_turns"] += bool(found) and work
+        out["sections"] += len(found)
+        out["unblocked_work_turns"] += work and bool(ti.unblocked_items(text, []))
+        out["question_endings"] += asked
+        out["process_questions"] += bool(ti.needless_ask(text))
+        out["outward_without_grant"] += outward
+        return asked, asked and bool(ti.outward_classes(lines[-1]))
+
+    for host in ("omp", "claude"):
+        walk = taste.HOSTS[host][2]
+        for path, cwd in taste.sessions(host, "", nested=False):
+            if ti.FIXTURE_WORKSPACE.search(cwd or "") or (cwd or "").startswith(tp.CACHE):
+                continue
+            reply, work, started, outward, granted = None, False, False, False, False
+            for event in walk(path, calls=True):
+                if event[0] == "user" and not taste.harness_prompt(event[1]) \
+                        and not event[1].lstrip().startswith(STOP_FEEDBACK):
+                    prev = close(reply, work, outward) if started else None
+                    prompt = event[1]
+                    if prev and prev[0] and ti.CHEAP_ANSWER.fullmatch(prompt.strip()):
+                        out["cheap_answers"] += 1
+                        granted = granted or prev[1]
+                    grants = tc.user_grants(prompt)
+                    if grants:
+                        granted = not grants[-1][1].startswith("lift:")
+                    elif ti.outward_classes(prompt):
+                        granted = True
+                    reply, work, started, outward = None, False, True, False
+                elif event[0] == "reply":
+                    reply = (event[1], event[2])
+                elif event[0] == "call":
+                    name, inp = str(event[1] or "").lower(), event[2] or {}
+                    work = work or ti.classify(event[1], inp) is not None
+                    outward = outward or (not granted and name in ti.BASH_TOOLS and bool(
+                        OUTWARD_COMMAND.search(str(inp.get("command") or ""))))
+                    if name in ti.ASK_TOOLS and counted(event[-1]):
+                        asked, defaulted = ti.undefaulted(inp)
+                        out["asks_without_default"] += asked - defaulted
+            if started:
+                close(reply, work, outward)
+    keys = ("turns", "work_turns", "residual_turns", "residual_work_turns", "sections",
+            "unblocked_work_turns", "question_endings", "process_questions",
+            "cheap_answers", "asks_without_default", "outward_without_grant")
+    res = {k: int(out[k]) for k in keys}
+    res["per_turn"] = round(res["sections"] / res["turns"], 3) if res["turns"] else None
+    return res
+
+
 def _nearest(candidates, ts):
     best = None
     for cand in candidates:
@@ -572,15 +680,8 @@ def run_replay(cutoff=None, since=None):
     if proc.returncode:
         raise RuntimeError("replay child failed: %s" % proc.stderr[-2000:])
     results = _read_jsonl(os.path.join(run, "results.jsonl"))
-    join = Counter()
-    for it in items:
-        key = it["kind"] if it["kind"] == "claim" else "%s:%s" % (
-            it["kind"], str(it["tool"] or "unknown").lower())
-        join[key + ":n"] += 1
-        join[key + ":joined"] += bool(it["join"])
     by = {r["i"]: r for r in results}
-    text_ok = sum(1 for it in items if by.get(it["i"], {}).get("text_ok"))
-    family = race_family(items, results, parent_of)
+    race_family(items, results, parent_of)
     for it in items:
         writers = by.get(it["i"], {}).get("writers")
         if writers:
@@ -589,16 +690,51 @@ def run_replay(cutoff=None, since=None):
     summary = {"run": run, "cutoff": cutoff, "cutoff_local": _stamp(cutoff),
                "since": since, "since_release": SINCE_RELEASE,
                "ledgers": dict(counts["ledgers"]), "candidates": counts["candidates"],
-               "excluded": dict(counts["excluded"]), "items": len(items),
-               "joined": sum(1 for it in items if it["join"]), "join": dict(join),
-               "stop_text_fidelity": text_ok,
-               "fidelity": fidelity(items, results),
-               "race_family": family,
-               "deny_runs": folds}
+               "excluded": dict(counts["excluded"])}
+    summary.update(_item_stats(items, results))
+    summary["deny_runs"] = folds
+    summary["residual"] = residual_turns(cutoff, since)
     tezgah_store.add_replay(_run_key(run), summary, tp.CACHE)
     return summary
 
 
+def _item_stats(items, results):
+    """The summary fields computed from the items and their replay results."""
+    by = {r["i"]: r for r in results}
+    join = Counter()
+    for it in items:
+        key = it["kind"] if it["kind"] == "claim" else "%s:%s" % (
+            it["kind"], str(it["tool"] or "unknown").lower())
+        join[key + ":n"] += 1
+        join[key + ":joined"] += bool(it["join"])
+    return {"items": len(items), "joined": sum(1 for it in items if it["join"]),
+            "join": dict(join),
+            "stop_text_fidelity": sum(1 for it in items
+                                      if by.get(it["i"], {}).get("text_ok")),
+            "fidelity": fidelity(items, results),
+            "race_family": _family_counts(items, by)}
+
+
+def split_of(sid):
+    """`search` or `holdout`: the first byte of sha256(session id), even or odd.
+    Not `hash()`, which is salted per process."""
+    return "holdout" if hashlib.sha256(str(sid).encode("utf-8")).digest()[0] % 2 else "search"
+
+
+def _split_view(summary, run, split):
+    """`summary` named by the side it reads; with a split, its item fields
+    recomputed over that side's items (module docstring, Split)."""
+    items = _read_jsonl(os.path.join(run, "corpus.jsonl"))
+    out = dict(summary, split=split or "all")
+    if split:
+        items = [it for it in items if split_of(it["sid"]) == split]
+        keep = {it["i"] for it in items}
+        out.update(_item_stats(items, [r for r in _read_jsonl(
+            os.path.join(run, "results.jsonl")) if r["i"] in keep]))
+    out["sessions"] = len({it["sid"] for it in items})
+    return out
+
+
 def relation(a, b, parent_of):
     """parent/child, sibling, unrelated, or undecided for two ledger stems."""
     pa, pb = parent_of.get(a, "unknown"), parent_of.get(b, "unknown")
@@ -616,6 +752,18 @@ def race_family(items, results, parent_of):
     the replay found (plan 055 part 6, D32a): intra-family when every writer is
     family, cross-family when any is unrelated."""
     by = {r["i"]: r for r in results}
+    for it in items:
+        if it["live"] != "deny" or it["live_rule"] != RACE_RULE:
+            continue
+        writers = by.get(it["i"], {}).get("writers") or it.get("foreign") or []
+        rels = {relation(it["stem"], w, parent_of) for w in writers}
+        it["family"] = ("no-writer" if not rels else "cross" if "unrelated" in rels
+                        else "undecided" if "undecided" in rels else "intra")
+    return _family_counts(items, by)
+
+
+def _family_counts(items, by):
+    """The race family counts over the items `race_family` labelled."""
     out = Counter()
     for it in items:
         if it["live"] != "deny" or it["live_rule"] != RACE_RULE:
@@ -624,10 +772,7 @@ def race_family(items, results, parent_of):
         writers = res.get("writers") or it.get("foreign") or []
         out["from replay" if res.get("writers") else "from live detail"
             if writers else "no writer named"] += 1
-        rels = {relation(it["stem"], w, parent_of) for w in writers}
         out["n"] += 1
-        it["family"] = ("no-writer" if not rels else "cross" if "unrelated" in rels
-                        else "undecided" if "undecided" in rels else "intra")
         out[it["family"]] += 1
     return dict(out)
 
@@ -786,12 +931,13 @@ def _rate(k, n):
             "ci95": [round(ci[0], 4), round(ci[1], 4)] if ci else None}
 
 
-def report(run, label_files=()):
-    summary = _summary(run)
-    out = {k: summary[k] for k in ("run", "cutoff", "cutoff_local", "ledgers", "candidates",
-                                   "excluded", "items", "joined", "join",
-                                   "stop_text_fidelity", "fidelity", "race_family",
+def report(run, label_files=(), split=None):
+    summary = _split_view(_summary(run), run, split)
+    out = {k: summary[k] for k in ("run", "split", "sessions", "cutoff", "cutoff_local",
+                                   "ledgers", "candidates", "excluded", "items", "joined",
+                                   "join", "stop_text_fidelity", "fidelity", "race_family",
                                    "deny_runs")}
+    out["residual"] = summary.get("residual")
     keyfile = os.path.join(run, "sheet-key.jsonl")
     labels = _labels(label_files)
     raters = sorted(labels)
@@ -803,7 +949,9 @@ def report(run, label_files=()):
         out["labels"] = "unverifiable: %s" % ("no sheet drawn" if not os.path.exists(keyfile)
                                               else "no labels file")
         return out
-    key = {(k["set"], k["n"]): k for k in _read_jsonl(keyfile)}
+    items = {it["i"]: it for it in _read_jsonl(os.path.join(run, "corpus.jsonl"))}
+    key = {(k["set"], k["n"]): k for k in _read_jsonl(keyfile)
+           if not split or split_of(items.get(k["item"], {}).get("sid")) == split}
     usable = {}
     if len(raters) >= 2:
         a, b = labels[raters[0]], labels[raters[1]]
@@ -826,7 +974,6 @@ def report(run, label_files=()):
         return out
     rules = defaultdict(Counter)
     stop = Counter()
-    items = {it["i"]: it for it in _read_jsonl(os.path.join(run, "corpus.jsonl"))}
     race_intra = Counter()
     for x, label in usable.items():
         k = key[x]
@@ -844,8 +991,10 @@ def report(run, label_files=()):
             stop[side + ":hit"] += label == ("false" if side == "allow" else "honest")
     # a rule whose joined denies fell short of its quota is pooled with the
     # other rules (H2 protocol, stated before the draw); a rule with no item on
-    # the sheet is neither pooled nor rated (H2 amendment 2026-10-06)
-    drawn = Counter(k["bucket"] for k in key.values())
+    # the sheet is neither pooled nor rated (H2 amendment 2026-10-06). The draw
+    # is a property of the whole sheet, so a --split side counts it unsplit: one
+    # side holds about half of a rule's quota and would read as short.
+    drawn = Counter(k["bucket"] for k in _read_jsonl(keyfile))
     short = [(b.split(":", 1)[1], drawn[b]) for b, q in QUOTA.items()
              if b.startswith("deny:") and b != "deny:other" and drawn[b] < q]
     out["pooled"] = sorted(rule for rule, n in short if n)
@@ -979,6 +1128,8 @@ def label_model(run, prompt_file, provider="jev", model=None):
 
 def _print_summary(s):
     print("replay run: %s" % s["run"])
+    if "split" in s:
+        print("split: %s (%d sessions)" % (s["split"], s["sessions"]))
     print("cutoff: %s (epoch %d)" % (s["cutoff_local"], s["cutoff"]))
     led = s["ledgers"]
     print("ledgers: read %d, fixture %d, test-polluted %d, kept %d" % (
@@ -1008,6 +1159,22 @@ def _print_summary(s):
           "sessions with %d+ denies: %d" % (
               runs["run_min"], runs["runs"] or "none", runs["longest"] or "none",
               runs["session_min"], runs["sessions_over"]))
+    res = s.get("residual")
+    if res:
+        print("residual sections%s: %d turns (%d with work); %d with a residual section "
+              "(%d in work turns), %d sections (%s per turn); %d work turns with an "
+              "unblocked item; %d question endings" % (
+                  " (whole corpus)" if s.get("split") not in (None, "all") else "",
+                  res["turns"], res["work_turns"], res["residual_turns"],
+                  res["residual_work_turns"], res["sections"],
+                  "n/a" if res["per_turn"] is None else "%.3f" % res["per_turn"],
+                  res["unblocked_work_turns"], res["question_endings"]))
+        # a summary written before plan 086 S3 carries no ask counters
+        print("ask policy: %d process questions, %d cheap answers after a question, "
+              "%d ask-tool questions without a default, %d outward turns without a "
+              "grant" % tuple(res.get(k, 0) for k in (
+                  "process_questions", "cheap_answers", "asks_without_default",
+                  "outward_without_grant")))
 
 
 def _pct(k, n):
@@ -1052,6 +1219,7 @@ def main(argv):
     p.add_argument("--labels", action="append", default=[])
     p.add_argument("--seed", type=int, default=SEED)
     p.add_argument("--rules")
+    p.add_argument("--split", choices=("search", "holdout"))
     p.add_argument("--json", action="store_true")
     args = p.parse_args(argv)
     if args.label_model:
@@ -1077,7 +1245,7 @@ def main(argv):
         print("race denies left off (written before the ledger stored `target`): %d" % left)
         return 0
     if args.report:
-        out = report(_run_dir(args.run), args.labels)
+        out = report(_run_dir(args.run), args.labels, args.split)
         if args.json:
             print(json.dumps(out, indent=2))
             return 0
@@ -1085,6 +1253,8 @@ def main(argv):
         _print_labels(out)
         return 0
     summary = run_replay(_when(args.cutoff), _when(args.since))
+    if args.split:
+        summary = _split_view(summary, summary["run"], args.split)
     print(json.dumps(summary, indent=2) if args.json else "", end="")
     if not args.json:
         _print_summary(summary)