@r1z4x/tezgah 1.0.0

package/hooks/tezgah_shapes.py

@@ -0,0 +1,305 @@
+"""The recurring refusal shapes the local ledgers hold, folded.
+
+`failure_shapes()` reads every ledger on this machine - the same corpus walk
+`tezgah_integrity.counters_all()` makes, through the same two readers
+(`ledgers()` and `events_path()`) - and groups the `deny` rows by the rule label
+its writer put before the first colon, which is the label `_counts` already
+folds by. A shape is reported only when it recurs in at least
+`RECURRING_SESSIONS` distinct sessions: one session's repeated refusal is an
+anecdote about that session, and a report that could not tell the two apart
+would recommend a change to a shape that met its rule once.
+
+**It is a report, not a gate.** Nothing here refuses a call, no caller reads it
+on the hot path, and the CLI that prints it always exits 0. The corpus size it
+read is printed with the answer, because an empty report must not read as
+"nothing recurs" when the fold read nothing at all.
+
+**The precision proxy is not optional, and raw frequency must never rank this
+list.** Every shape carries `fires_per_session` = its fires over the sessions it
+fired in, so a count inflated by one long session repeating itself (a value well
+above 1.0) is visible beside the count that inflated it. The trap the field
+exists for is the highest-firing rule on this machine, `drift`: it fires on long
+careful turns whether or not anything is drifting
+(an internal hardening research line, C04/C11 - 181 fires in 143
+sessions, and it fired three times while that experiment was written), so a
+report ranked by fires would hand its reader the one rule whose count carries
+the least signal about a real failure. The ledger stores no ground truth for
+whether a refusal was a failure worth fixing - the detail is the reason text and
+the row says nothing about what the turn was doing - so `fires_per_session` is a
+proxy for how much of a shape's count is repetition, and never a precision
+measurement.
+
+`rule_yield()` is the other half of the same reading: the live gate rules that
+stopped firing. A rule missing from `failure_shapes` may be dead or may be a day
+old, and only its exposure - the real ledgers written since it shipped - tells
+the two apart.
+"""
+from collections import Counter, defaultdict
+import ast
+import os
+import re
+import subprocess
+
+import tezgah_integrity as ti
+
+# The floor a shape must clear to be reported at all: distinct sessions, not
+# rows. Below it, a shape is one session's habit.
+RECURRING_SESSIONS = 5
+
+# How many of a shape's most common command shapes come with it.
+MAX_COMMANDS = 3
+
+# The one normalization a shape text gets: a digit run is a variant of one
+# shape, not a shape of its own (`drift`'s `Long turn (25 work rows...)`,
+# `(26 work rows...)` and `(27 ...)` are one shape, three fires).
+DIGITS = re.compile(r"\d+")
+
+
+def _label(detail):
+    """The rule label of a deny row's detail: the text before the first colon,
+    or `other` when there is none - the derivation `_counts` folds by, so a
+    shape's name and a counter's key are the same string."""
+    return str(detail or "").split(":", 1)[0].strip() or "other"
+
+
+def _command(detail):
+    """One deny row's command shape: what its writer recorded after the label -
+    the refused call's own reason text, already credential-redacted by the one
+    append every row goes through (`tezgah_integrity.note_path`), which is why
+    no second redactor lives here - with digit runs collapsed and whitespace
+    folded, so the variants of one refusal read as one shape."""
+    text = str(detail or "")
+    tail = text.split(":", 1)[1] if ":" in text else ""
+    return DIGITS.sub("N", " ".join(ti.redact(tail).split()))
+
+
+def _contract(value):
+    """A row's contract version as the report names it: `v1`, or `unversioned`
+    for a row written before the writer stamped one."""
+    return "unversioned" if value is None else "v%s" % value
+
+
+def failure_shapes():
+    """Every shape recurring in at least `RECURRING_SESSIONS` distinct sessions.
+
+    Returns `ledgers` (how many files the fold read), `recurring_sessions` (the
+    floor it applied) and `shapes`: one object per shape with `rule`,
+    `sessions`, `fires`, `fires_per_session` and up to `MAX_COMMANDS` of its
+    most common command shapes, most common first, ties by text so two runs of
+    the report agree. Ranked by distinct sessions descending - the
+    cross-instance recurrence this exists to find - never by fires, and the
+    module docstring says why.
+
+    `row_versions` counts the rows the fold read by the contract that wrote them
+    (`unversioned` for a row predating the field): a shape can move because the
+    rule changed or because a row's meaning did, and a report that cannot tell
+    the two apart would attribute one to the other.
+
+    Bound: none, like `counters_all`, and for the same reason - a window would
+    make the report contradict the corpus it claims to be about."""
+    files = ti.ledgers()
+    sessions = defaultdict(set)
+    fires = Counter()
+    commands = defaultdict(Counter)
+    versions = Counter()
+    for path in files:
+        # every session's ledger, so one damaged file costs its own rows and
+        # not the report (audit GAP-02 / M-7: the same reader shape turned the
+        # cross-session write gate off machine-wide)
+        for row in ti._foreign_rows(path):
+            # A string label, not the raw value: a None key beside int keys makes
+            # `json.dumps(..., sort_keys=True)` raise, and the report is printed
+            # as JSON by the CLI.
+            versions[_contract(row.get("v"))] += 1
+            if row.get("kind") != "deny":
+                continue
+            label = _label(row.get("detail"))
+            fires[label] += 1
+            sessions[label].add(path)
+            commands[label][_command(row.get("detail"))] += 1
+    shapes = [{"rule": label, "sessions": len(paths), "fires": fires[label],
+               "fires_per_session": round(fires[label] / len(paths), 2),
+               "commands": [text for text, _ in sorted(
+                   commands[label].items(),
+                   key=lambda kv: (-kv[1], kv[0]))[:MAX_COMMANDS]]}
+              for label, paths in sessions.items()
+              if len(paths) >= RECURRING_SESSIONS]
+    shapes.sort(key=lambda s: (-s["sessions"], s["rule"]))
+    return {"ledgers": len(files), "recurring_sessions": RECURRING_SESSIONS,
+            "shapes": shapes,
+            # Which row contracts the fold actually read, for the same reason the
+            # ledger count is here: a shape can move because the rule changed or
+            # because the row's meaning did, and a reader that cannot tell the two
+            # apart would attribute one to the other. None is the pre-version rows.
+            "row_versions": dict(versions)}
+
+
+# A live rule is marked `retire-candidate` only at or below this many fires over
+# its exposure window. Zero, not "low": a rule whose design is to be rare (a
+# credential on its way into a file) is working when it fires twice in a
+# thousand sessions, and the ledger holds no ground truth that would tell a rare
+# true refusal from a rule nobody needs.
+RETIRE_FIRES = 0
+
+# The exposure floor, in real ledgers written since the rule shipped. A rule
+# firing in 1% of sessions leaves no fire in 300 of them with probability
+# 0.99 ** 300 ~= 0.05, so below the floor zero fires is not evidence of a dead
+# rule - it is a young one, and the report says `low-exposure` instead.
+MIN_EXPOSURE = 300
+
+GATE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tezgah_gate.py")
+
+
+def gate_rules(source):
+    """The rule labels one revision of the gate's source refuses under: the
+    string literal every `_deny(session_id, "<rule>", ...)` call passes. Read
+    from the code, so the list cannot drift from the rules that write rows."""
+    return {node.args[1].value for node in ast.walk(ast.parse(source))
+            if isinstance(node, ast.Call)
+            and getattr(node.func, "id", None) == "_deny"
+            and len(node.args) > 1 and isinstance(node.args[1], ast.Constant)
+            and isinstance(node.args[1].value, str)}
+
+
+def continuous_since(revisions):
+    """{rule: the commit time it has been in the gate without a gap since}, over
+    `revisions` as (commit time, source) oldest first. A rule removed and brought
+    back dates from its return: it could not fire while it was gone."""
+    since = {}
+    for when, source in revisions:
+        try:
+            rules = gate_rules(source)
+        except SyntaxError:
+            continue
+        since = {rule: since.get(rule, when) for rule in rules}
+    return since
+
+
+def rule_ship_times():
+    """{live rule: epoch it shipped, or None when that cannot be read}.
+
+    Git is the only honest source here, and that is why this shells out: no
+    ledger row carries the harness revision that wrote it, and the earliest row
+    carrying a rule does not exist for the rule this report is about - one that
+    never fired. A hand-kept table of ship dates would be the second list the
+    code already is. The history this checkout holds starts at its first commit,
+    so a rule older than that dates from it, which only shrinks its exposure -
+    the direction that keeps a rule, never the one that retires it. Without git
+    (a packaged install) every date is None and no rule is read as dead."""
+    here = os.path.dirname(GATE)
+    with open(GATE, encoding="utf-8") as fh:
+        live = gate_rules(fh.read())
+
+    def git(*args):
+        return subprocess.run(["git", "-C", here] + list(args),
+                              capture_output=True, text=True, timeout=30,
+                              check=True).stdout
+    try:
+        log = git("log", "--reverse", "--format=%H %ct", "--", "tezgah_gate.py")
+        revisions = [(int(ct), git("show", "%s:./tezgah_gate.py" % sha))
+                     for sha, ct in (line.split() for line in log.splitlines())]
+    except (OSError, ValueError, subprocess.SubprocessError):
+        revisions = []
+    since = continuous_since(revisions)
+    # a rule only in the working tree has not shipped: None, never "old"
+    return {rule: since.get(rule) for rule in live}
+
+
+def rule_yield(files=None, shipped=None):
+    """Every live gate rule's fires over its exposure window.
+
+    `files` defaults to every ledger (`ti.ledgers()`), `shipped` to
+    `rule_ship_times()`. Fixture ledgers are left out by `ti.fixture_ledger`, the
+    exclusion `counters_all` applies, because a probe run measures the harness
+    and not use. A rule's exposure is the real ledgers last written at or after
+    it shipped (the file's mtime), and its fires are the deny rows under its
+    label in those ledgers. `mark` is `retire-candidate` at or below
+    `RETIRE_FIRES` fires over at least `MIN_EXPOSURE` ledgers, `low-exposure` at
+    or below `RETIRE_FIRES` over fewer (or an unknown ship date), else None.
+
+    A report, not a gate, like `failure_shapes`: `ledgers`, `fixtures` and
+    `real` are returned with the answer so an empty window cannot read as a
+    quiet rule."""
+    files = ti.ledgers() if files is None else files
+    shipped = rule_ship_times() if shipped is None else shipped
+    real = []
+    for path in files:
+        rows = ti._foreign_rows(path)
+        if ti.fixture_ledger(rows):
+            continue
+        try:
+            when = os.path.getmtime(path)
+        except OSError:
+            continue
+        real.append((when, Counter(_label(row.get("detail")) for row in rows
+                                   if row.get("kind") == "deny")))
+    rules = []
+    for rule, since in shipped.items():
+        window = [] if since is None else [c for when, c in real if when >= since]
+        fires = sum(c[rule] for c in window)
+        exposure = None if since is None else len(window)
+        mark = None
+        if fires <= RETIRE_FIRES:
+            mark = ("retire-candidate" if (exposure or 0) >= MIN_EXPOSURE
+                    else "low-exposure")
+        rules.append({"rule": rule, "shipped": since, "exposure": exposure,
+                      "fires": fires, "mark": mark})
+    rules.sort(key=lambda r: (r["fires"], -(r["exposure"] or 0), r["rule"]))
+    return {"ledgers": len(files), "fixtures": len(files) - len(real),
+            "real": len(real), "retire_fires": RETIRE_FIRES,
+            "min_exposure": MIN_EXPOSURE, "rules": rules}
+
+
+# Claude Code auto mode's escalation thresholds: a person is asked after 3
+# consecutive or 20 total denials. They are example values for the denial-budget
+# fold (plan 055 part 8a), not a tezgah measurement.
+RUN_MIN = 3
+SESSION_DENIES = 20
+
+# The rows the gate writes beside a refusal (`drift_reason`'s mark, the nudge
+# mark, and the call's `began`, written before any rule since plan 051): a
+# refused call leaves them in front of its deny row, so they continue a run
+# instead of ending it. Every other row - a call that ran, a turn marker, a
+# reply - ends the run.
+GATE_SIDE_KINDS = frozenset(("deny", "drift", "nudge", "began"))
+
+
+def deny_runs(ledgers, run_min=RUN_MIN, session_min=SESSION_DENIES):
+    """The denial-budget fold over `ledgers`, an iterable of row lists, one per
+    session: per rule, the runs of at least `run_min` consecutive denies of that
+    rule inside one turn, the longest run, and how many sessions hold at least
+    `session_min` denies. A report only: nothing reads it on the hot path and no
+    gate rule changes on it (ADR 010).
+
+    A run is consecutive deny rows under one `_deny_rule` label; a deny of
+    another rule starts a new run, and any row outside `GATE_SIDE_KINDS` (a
+    step, a `turn` marker) ends it."""
+    runs, longest, sessions, read = Counter(), Counter(), 0, 0
+    for rows in ledgers:
+        read += 1
+        rule, length, denies = None, 0, 0
+
+        def close():
+            if rule is not None and length >= run_min:
+                runs[rule] += 1
+            if rule is not None:
+                longest[rule] = max(longest[rule], length)
+
+        for row in rows:
+            kind = row.get("kind")
+            if kind == "deny":
+                denies += 1
+                label = ti._deny_rule(str(row.get("detail") or ""))
+                if label == rule:
+                    length += 1
+                else:
+                    close()
+                    rule, length = label, 1
+            elif kind not in GATE_SIDE_KINDS:
+                close()
+                rule, length = None, 0
+        close()
+        sessions += denies >= session_min
+    return {"ledgers": read, "run_min": run_min, "session_min": session_min,
+            "runs": dict(runs), "longest": dict(longest),
+            "sessions_over": sessions}

package/hooks/tezgah_skill_pick.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""One cheap judgement in front of the skill choice: at most one skill, one line.
+
+A session meets the roster as an index of one-line descriptions, truncated by the
+host, so the skill that authors a file reads like the one that edits it, and a
+list of names invites a guess even on a turn where nothing in it fits. TypeSafe's
+own skill-suggestion cookbook puts a ranking in front of that choice and reports
+the effect over 488 requests: wrong skill loads 16.8% -> 7.3%, needless loads
+9.8% -> 4.0%.
+
+This is the smallest version of that recipe that fits a hook: ONE batched request
+carrying a Choice over the installed skill names (with `none` as an option) and a
+Noul asking whether the turn needs a skill at all. The winner becomes one
+`<skill_relevance>` line appended to the per-turn text, naming what the skill is
+for and saying the line is a hint to look at first. The roster itself is left
+untouched - the cookbook keeps it so the host's prefix cache over it still holds,
+and on five of the six hosts tezgah does not write that text anyway.
+
+The roster is the plugin's own `skills/` - the 13 this checkout ships and every
+host links - because a name no host can open would be a hint to nothing.
+
+OFF UNLESS ARMED: `skill-suggest-on` in `~/.config/tezgah` turns it on. The
+roster here is 13 skills, and an independent chooser was measured at 0 of 20
+wrong on a labelled set WITHOUT any hint, so the line's benefit on this roster is
+unproven while its cost is not (about 78 tokens and 0.8 s per fresh prompt), and a
+capability that costs a certain amount for an unmeasured gain is opt-in. Total by
+design: no credential, a timeout, a refused request, a reply that is not the
+documented shape, a cache dir that cannot be written - every one of them returns
+"" instead of raising, since this runs on the prompt path of every turn and a
+missing hint must cost the turn nothing.
+"""
+import hashlib
+import json
+import os
+import re
+
+import tezgah_integrity as ti
+import tezgah_judge
+import tezgah_paths as tp
+
+ARM = "skill-suggest-on"
+# Below this, the turn's own gate question says no skill is wanted and nothing is
+# suggested - the cookbook's threshold, which is also where the triage selection
+# sits: with `none` in the Choice, a forced nearest-neighbour pick is what the
+# gate exists to prevent.
+GATE = 0.30
+# One attempt and a 4 s wall clock (`ask`'s `deadline`), about 4x the measured
+# worst case (0.3-1.1 s live): omp's bridge kills a hook at 10 s, a killed hook
+# loses the whole per-turn injection, and the rest of the prompt hook runs after
+# this call - so a slow reply costs the hint and never the turn.
+ASK_DEADLINE = 4.0
+# The prompt goes out redacted (`ti.redact`) and cut to taste's own cap: the
+# choice needs the request's gist, not a pasted log.
+PROMPT_MAX = 2000
+# The criteria the judge reads per skill, and what the appended line says the
+# skill is for. One line: it must not cost more than the index entry it points at
+# (the host's own entry is 60 characters), and 120 keeps the whole appended block
+# under 320 characters (measured 305-317). The router line in bin/tezgah-setup
+# caps its own copy of this sentence at 140.
+CLAUSE_CAP = 120
+NONE = "none"
+CHOICE_INSTRUCTIONS = ("Which of these skills, if any, is the right one to load "
+                       "to help with the user's latest request?")
+# What "needs a skill" means for THIS roster: tezgah's 13 skills are working
+# rules (scope, output shape, workflow, evidence), not the external procedures
+# the cookbook's roster holds, so its "documented procedure" question is the wrong
+# one here - measured on 8 prompts it put `ponytail` at 0.05-0.24 and lost two
+# correct picks. This wording keeps them at 0.47-0.67 and separates the five that
+# want a skill (0.80-0.98) from the three that do not (0.01-0.15).
+GATE_INSTRUCTIONS = ("Does this request ask for work on the user's repository, "
+                     "code, plans, analysis or writing, rather than a question "
+                     "answered from general knowledge?")
+# One session's answered prompts. Bounded because a session can be long, and keyed
+# on the prompt so an identical submission is never paid for twice.
+KEPT = 200
+_FRONTMATTER = re.compile(r"^---\s*\n(.*?)\n---", re.S)
+_FOLDED = re.compile(r"description:\s*[>|]-?\s*\n((?:\s+.*\n?)+)")
+_INLINE = re.compile(r"description:\s*(.+)")
+
+
+def slug(text):
+    """A filename-safe form of a host's session id."""
+    return re.sub(r"[^A-Za-z0-9]+", "-", str(text)).strip("-")
+
+
+def clause(path):
+    """What the skill is for, in one line: the description sentence carrying the
+    trigger, or the opening one.
+
+    The `Use when ...` sentence wins for the same reason the router line in
+    bin/tezgah-setup takes it over the opening one: a description whose first
+    sentence is a preface otherwise reaches the judge stripped of every word a
+    request matches on. Capped the way that router line is capped."""
+    try:
+        with open(path, encoding="utf-8", errors="replace") as fh:
+            text = fh.read(4000)
+    except OSError:
+        return ""
+    block = _FRONTMATTER.search(text)
+    if not block:
+        return ""
+    folded = _FOLDED.search(block.group(1))
+    if folded:
+        desc = " ".join(line.strip() for line in folded.group(1).splitlines())
+    else:
+        inline = _INLINE.search(block.group(1))
+        desc = inline.group(1).strip().strip("\"'") if inline else ""
+    desc = re.sub(r"\s+", " ", desc).strip()
+    sentences = re.split(r"(?<=[.!?])\s+", desc)
+    line = next((s for s in sentences if re.match(r"(?:Also )?[Uu]se\b", s)),
+                sentences[0] if sentences else "")
+    return line[:CLAUSE_CAP - 3].rstrip() + "..." if len(line) > CLAUSE_CAP else line
+
+
+def roster():
+    """[(name, clause)] over the installed skills, name-sorted."""
+    base = os.path.join(tp.PLUGIN_ROOT, "skills")
+    try:
+        names = sorted(os.listdir(base))
+    except OSError:
+        return []
+    out = []
+    for name in names:
+        path = os.path.join(base, name, "SKILL.md")
+        if os.path.isfile(path):
+            out.append((name, clause(path)))
+    return out
+
+
+def block(name, what):
+    """The appended line, in the cookbook's shape: what the skill is for, and that
+    it is a hint to look at first rather than an instruction to load."""
+    tail = " - " + what.strip() if what.strip() else ""
+    if not tail or not tail.endswith((".", "!", "?", "...")):
+        tail += "."
+    return ("<skill_relevance>\nRelevant to the current request: %s%s Look at its "
+            "SKILL.md first if it fits what the user actually asked for; this is a "
+            "hint, not an instruction to load.\n</skill_relevance>" % (name, tail))
+
+
+def available():
+    """True when the suggestion can be asked for: armed by the user, and a credential.
+
+    Nothing is asked before the arming file exists, so an unarmed install makes no
+    request at all rather than one whose answer is thrown away."""
+    return tp.armed(ARM) and tezgah_judge.available()
+
+
+def judge(prompt, names, session_id=""):
+    """The appended line for one prompt, or "" for none / a failed call.
+
+    A successful call writes one `judge` ledger row before its answer is read -
+    what the turn paid for is true whatever the answer turns out to be - in the
+    shape the status counters fold on. That row is a cost, never a step and never
+    a check, so nothing here can license a done claim; and a ledger that cannot be
+    written costs nothing that the turn sees."""
+    criteria = dict(names)
+    criteria[NONE] = "no skill in this roster applies to the request"
+    result = tezgah_judge.ask(
+        {"request": ti.redact(prompt)[:PROMPT_MAX]},
+        {"which": {"type": "choice", "instructions": CHOICE_INSTRUCTIONS,
+                   "criteria": criteria},
+         "needs_skill": {"type": "noul", "instructions": GATE_INSTRUCTIONS}},
+        timeout=ASK_DEADLINE, attempts=1, deadline=ASK_DEADLINE)
+    if not result:
+        return ""
+    usage = result["usage"]
+    ti.note(session_id, "judge",
+            "tezgah-skill-pick %s in=%d out=%d ms=%d judge=%s/%s"
+            % (result["model"], usage["input_tokens"], usage["output_tokens"],
+               result["latency_ms"], result["provider"], result["model"]))
+    chosen = tezgah_judge.choice(result, "which")
+    gate = tezgah_judge.noul(result, "needs_skill")
+    if chosen is None or chosen == NONE:
+        return ""
+    if gate is None or gate < GATE:
+        return ""
+    options = dict(names)
+    return block(chosen, options[chosen]) if chosen in options else ""
+
+
+def _path(session_id):
+    return os.path.join(tp.cache_dir(), "skill-pick",
+                        slug(session_id) + ".json")
+
+
+def _remembered(session_id):
+    try:
+        with open(_path(session_id), encoding="utf-8") as fh:
+            data = json.load(fh)
+    except (OSError, ValueError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def _remember(session_id, digest, line):
+    """Best effort: a host may sandbox hook writes, and a cache that cannot be
+    written costs one more call, never the line."""
+    data = _remembered(session_id)
+    data[digest] = line
+    for old in list(data)[:max(0, len(data) - KEPT)]:
+        del data[old]
+    path = _path(session_id)
+    try:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(data, fh)
+    except OSError:
+        pass
+
+
+def suggest(prompt, session_id=""):
+    """The `<skill_relevance>` line this prompt earns, or "".
+
+    Nothing is asked for a slash command (the user named the work, not the task
+    class), and nothing is asked twice for one prompt in one session."""
+    prompt = (prompt or "").strip()
+    if not prompt or prompt.startswith("/") or not available():
+        return ""
+    digest = hashlib.sha1(prompt.encode()).hexdigest()
+    known = _remembered(session_id)
+    if digest in known:
+        return known[digest] if isinstance(known[digest], str) else ""
+    names = roster()
+    line = judge(prompt, names, session_id) if names else ""
+    _remember(session_id, digest, line)
+    return line