@r1z4x/tezgah 1.0.0

package/hooks/tezgah_clarity.py

@@ -0,0 +1,171 @@
+"""The clarity ratchet: countable prose rules over the English docs, held to a baseline.
+
+Four rules are counted per file over docs/*.md and CHANGELOG.md: sentences over
+25 words, semicolons, a short list of words with a plainer replacement, and
+passive voice. Only English prose written for people is read: fenced code,
+code spans (including one broken across lines), link targets, HTML comments,
+tables, headings and any line holding a Turkish letter are blanked first, with
+their newlines kept so a hit still names its line.
+
+The counts are a ratchet, not a style gate: `docs/clarity-baseline.json` holds
+each file's count per rule as measured, a file may not exceed it, a file the
+baseline does not list is held to 0, and a count that drops is written back
+(lowered) on the next run. Raising a count needs `tezgah-docs --clarity --update`.
+The rules and the word list are the frozen measurer of the research line that
+chose this check (E1, `measure.py` k1), with the multi-line span fix it required.
+"""
+import glob
+import json
+import os
+import re
+
+RULES = ("long_sentences", "semicolons", "replace_words", "passive")
+LONG = 25
+REPLACE = ("commence", "initiate", "utilize", "prior to", "in order to", "ensure",
+           "replenish", "indicate", "verify")
+REPLACE_RE = re.compile(r"\b(?:%s)\b" % "|".join(re.escape(w) for w in REPLACE), re.I)
+PASSIVE = re.compile(r"\b(?:is|are|was|were|be|been|being)\s+(?:\w+ly\s+)?\w+(?:ed|en)\b", re.I)
+SENT = re.compile(r"(?<=[.!?])\s+(?=[A-Z(\"'`*])")
+ABBREV = re.compile(r"\b(e\.g|i\.e|etc|vs|cf)\.")
+WORD = re.compile(r"[A-Za-z0-9][\w'./-]*")
+FENCE = re.compile(r"^\s*(```|~~~)")
+# a span closes on a backtick run of the same length, across lines inside its paragraph
+SPAN = re.compile(r"(?<!`)(`+)(?!`)(.+?)(?<!`)\1(?!`)", re.S)
+COMMENT = re.compile(r"<!--.*?-->", re.S)
+LINK_TARGET = re.compile(r"\]\([^)]*\)|\]\[[^\]]*\]|<https?://[^>]*>|https?://\S+")
+TURKISH = re.compile(r"[ğĞışŞçÇöÖüÜİ]")
+SKIP_LINE = re.compile(r"^\s*(?:#|\||\[[^\]]+\]:\s)")
+ITEM = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s+")
+BASELINE = os.path.join("docs", "clarity-baseline.json")
+
+
+def _blank(match, keep=""):
+    return keep + "\n" * match.group(0).count("\n")
+
+
+def prose(text):
+    """The text with everything but English prose blanked, line count unchanged."""
+    lines, fenced = text.split("\n"), None
+    for i, line in enumerate(lines):
+        m = FENCE.match(line)
+        if fenced or m:
+            if m and (fenced is None or m.group(1) == fenced):
+                fenced = None if fenced else m.group(1)
+            lines[i] = ""
+    out = []
+    # a code span cannot cross a blank line, so an unmatched backtick stays in its paragraph
+    for para in re.split(r"(\n[ \t]*\n)", "\n".join(lines)):
+        out.append(SPAN.sub(lambda m: _blank(m, " CODE "), para))
+    text = LINK_TARGET.sub(lambda m: "]" if m.group(0)[0] == "]" else "",
+                           COMMENT.sub(_blank, "".join(out)))
+    return "\n".join("" if SKIP_LINE.match(line) or TURKISH.search(line) else line
+                     for line in text.split("\n"))
+
+
+def units(text):
+    """(first line number, text) per paragraph or list item of the prose."""
+    found, cur, start = [], [], 0
+    for n, line in enumerate(prose(text).split("\n") + [""], 1):
+        body = ITEM.sub("", line.strip().lstrip(">").lstrip()).strip()
+        if not body or ITEM.match(line.strip().lstrip(">").lstrip()):
+            if cur:
+                found.append((start, " ".join(cur)))
+            cur, start = ([body], n) if body else ([], 0)
+            continue
+        if not cur:
+            start = n
+        cur.append(body)
+    return found
+
+
+def sentences(unit):
+    flat = ABBREV.sub(r"\1", " ".join(unit.split()))
+    return [s for s in SENT.split(flat) if WORD.search(s)]
+
+
+def measure(text):
+    """(counts per rule, [(line, rule, snippet)]) for one file's text."""
+    counts, hits = dict.fromkeys(RULES, 0), []
+    for line, unit in units(text):
+        plain = re.sub(r"\bCODE\b", " ", unit)
+        found = [("semicolons", m) for m in re.findall(";", plain)]
+        found += [("replace_words", m) for m in REPLACE_RE.findall(plain)]
+        found += [("passive", m) for m in PASSIVE.findall(plain)]
+        found += [("long_sentences", s[:60] + "...") for s in sentences(unit)
+                  if len(WORD.findall(s)) > LONG]
+        for rule, snippet in found:
+            counts[rule] += 1
+            hits.append((line, rule, snippet))
+    return counts, hits
+
+
+def files(root):
+    return sorted(os.path.relpath(p, root).replace(os.sep, "/")
+                  for p in glob.glob(os.path.join(root, "docs", "*.md"))
+                  + glob.glob(os.path.join(root, "CHANGELOG.md")))
+
+
+def tree(root):
+    """{relpath: (counts, hits)} for every file the ratchet reads."""
+    out = {}
+    for rel in files(root):
+        with open(os.path.join(root, rel), encoding="utf-8") as fh:
+            out[rel] = measure(fh.read())
+    return out
+
+
+def load(path):
+    try:
+        with open(path, encoding="utf-8") as fh:
+            return json.load(fh)
+    except FileNotFoundError:
+        return {}
+
+
+def save(path, baseline):
+    with open(path, "w", encoding="utf-8") as fh:
+        json.dump(baseline, fh, indent=2, sort_keys=True)
+        fh.write("\n")
+
+
+def over(measured, baseline, rules=RULES):
+    """[(file, rule, count, allowed)] for each count above its baseline (absent: 0).
+    `rules` lets the citation ratchet (`bin/tezgah-docs`) reuse it for its one count."""
+    return [(rel, rule, counts[rule], baseline.get(rel, {}).get(rule, 0))
+            for rel, (counts, _) in sorted(measured.items()) for rule in rules
+            if counts[rule] > baseline.get(rel, {}).get(rule, 0)]
+
+
+def lowered(measured, baseline):
+    """The baseline with every count that dropped written down, nothing raised."""
+    new = {}
+    for rel, allowed in baseline.items():
+        if rel in measured:
+            counts = measured[rel][0]
+            new[rel] = {r: min(allowed.get(r, 0), counts[r]) for r in RULES}
+    return new
+
+
+def report(root, update=False, out=print):
+    """Print the per-file counts, the hits of each rule over baseline and one
+    summary line; lower (or with `update`, rewrite) the baseline. 0 or 1."""
+    path = os.path.join(root, BASELINE)
+    measured, baseline = tree(root), load(path)
+    for rel, (counts, _) in sorted(measured.items()):
+        out("%s  %s" % (rel, " ".join("%s=%d" % (r, counts[r]) for r in RULES)))
+    bad = [] if update else over(measured, baseline)
+    for rel, rule, count, allowed in bad:
+        for line, hit_rule, snippet in measured[rel][1]:
+            if hit_rule == rule:
+                out("%s:%d  %s  %s" % (rel, line, rule, snippet))
+        out("%s  %s: %d, baseline %d" % (rel, rule, count, allowed))
+    new = ({rel: counts for rel, (counts, _) in measured.items()} if update
+           else lowered(measured, baseline))
+    if new != baseline:
+        save(path, new)
+    totals = {r: sum(c[r] for c, _ in measured.values()) for r in RULES}
+    out("clarity: %d files, %s; %d rule(s) over baseline%s"
+        % (len(measured), ", ".join("%s %d" % (r, totals[r]) for r in RULES), len(bad),
+           " (baseline rewritten)" if update and new != baseline
+           else " (baseline lowered)" if new != baseline else ""))
+    return 1 if bad else 0

package/hooks/tezgah_components.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""The layer's editable components: one manifest, and AHE's seven beside it.
+
+AHE's first pillar gives every editable harness component a file-level,
+revertible representation, so the action space of a refinement loop is explicit
+rather than implied (arXiv 2604.25850). This module is that representation for
+this layer: `COMPONENTS`, one entry per editable component, and `keys()`, the
+list a prediction row may name.
+
+One entry, six fields, each one a fact a reader can check against the file it
+names (a test pins the fields against their own other definitions, so the
+manifest cannot drift into being a second, stale copy of the rules):
+
+- `key` - the name a prediction row uses. Unique.
+- `label` - the `CORE_RULES` rule this component owns, or the surface name when
+  it owns no rule. Every rule key appears exactly once.
+- `files` - repository-relative paths the component lives in: a file, a
+  directory, or a glob.
+- `switch` - the kill switch (`~/.config/tezgah/<name>`) or per-repo mark (`.<name>`
+  in the repo tree) whose presence drops this component's rule from the injected
+  text, or `none` when it has neither. `none` is honest and not a claim of
+  immutability: a broader switch such as `pretooluse-off` still turns off the
+  enforcement of a rule that no switch removes.
+- `editable` - whether the refinement loop may edit the component without a human
+  grant: true exactly when every file it declares is outside `FROZEN_PATHS`
+  (`hooks/tezgah_research.py`). A component that reaches a frozen file is false,
+  because the loop may not change a rule's text while the checker that reads it
+  moves underneath.
+- `revert` - how a change is undone. `git` when git tracks the declared files;
+  `snapshot` when the edit target is state git does not carry (the lessons
+  ledger), where the only revert is the pre-write copy the gate's `capture`
+  took, restored by `bin/tezgah-rollback`.
+
+AHE's seven components, each row read from the paper's own list and pointed at
+this repository. `not-owned` means the row exists here only as a mount or a
+subscription, so no edit in this checkout changes it:
+
+| AHE component | This repository |
+|---|---|
+| system prompt | owned - `hooks/tezgah_policy.py::CORE`, injected by `hooks/tezgah_context.py::core_for` |
+| tool descriptions | not-owned - each MCP server authors its own schema and this layer mounts it (`bin/tezgah-setup`); no text here reaches a model as a tool description |
+| tool implementations | not-owned - the tool body runs inside the host CLI and this layer only refuses a call before it (`hooks/tezgah_gate.py::decision`) |
+| middleware | not-owned - the host's own agent loop; this layer subscribes to the host's hook events and never sits in the loop |
+| skills | owned - `skills/<name>/SKILL.md`, symlinked into each host's skill directory by `bin/tezgah-setup` |
+| sub-agents | owned - `hooks/tezgah_agents.py::sync_root` renders the per-host briefs |
+| long-term memory | owned - `.tezgah/lessons.md`, read by `hooks/tezgah_context.py::lessons` at session start and `hooks/tezgah_context.py::relevant_lessons` per turn, plus `.tezgah/plans/` in the repository |
+
+The claim this table makes checkable is the one that was an argument before it:
+AHE's ablation localizes the gain to tools, middleware and long-term memory, not
+to the system prompt, and the reading offered was that those components sit
+inside the host rather than in this layer. Two of the three are indeed the
+host's - tools (descriptions and implementations) and middleware are
+`not-owned` - and one is not: long-term memory is this layer's own, and its
+ledger is a component here (`memory`, the one entry whose revert is a snapshot
+rather than git). So the boundary is a row a reader can point at, not a
+sentence to be re-argued.
+
+Import nothing heavy: this module is standard library only and imports no other
+tezgah module, because `hooks/tezgah_research.py` imports it lazily from a hot
+path and the rule text's builder has no business being pulled in by a manifest.
+"""
+
+COMPONENTS = [
+    {"key": "exec", "label": "exec",
+     "files": ["hooks/tezgah_policy.py"],
+     "switch": "exec-mode.off", "editable": True, "revert": "git"},
+    {"key": "ponytail", "label": "ponytail",
+     "files": ["hooks/tezgah_policy.py", "skills/ponytail/SKILL.md"],
+     "switch": "ponytail-auto.off", "editable": True, "revert": "git"},
+    {"key": "adhd", "label": "adhd",
+     "files": ["hooks/tezgah_policy.py", "skills/i-have-adhd/SKILL.md"],
+     "switch": "adhd-off", "editable": True, "revert": "git"},
+    {"key": "fidelity", "label": "fidelity",
+     "files": ["hooks/tezgah_policy.py"],
+     "switch": "none", "editable": True, "revert": "git"},
+    {"key": "integrity", "label": "integrity",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_integrity.py"],
+     "switch": "verify-off", "editable": False, "revert": "git"},
+    {"key": "loop", "label": "loop",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_gate.py"],
+     "switch": "none", "editable": False, "revert": "git"},
+    {"key": "safety", "label": "safety",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_gate.py"],
+     "switch": "none", "editable": False, "revert": "git"},
+    {"key": "scope", "label": "scope",
+     "files": ["hooks/tezgah_policy.py"],
+     "switch": "none", "editable": True, "revert": "git"},
+    {"key": "spec", "label": "spec",
+     "files": ["hooks/tezgah_policy.py"],
+     "switch": "spec-off", "editable": True, "revert": "git"},
+    {"key": "lessons", "label": "lessons",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_context.py"],
+     "switch": ".no-lessons", "editable": True, "revert": "git"},
+    {"key": "graph", "label": "graph",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_gate.py"],
+     "switch": ".no-graph", "editable": False, "revert": "git"},
+    {"key": "consult", "label": "consult",
+     "files": ["hooks/tezgah_policy.py", "bin/consult"],
+     "switch": "consult-off", "editable": True, "revert": "git"},
+    {"key": "research", "label": "research",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_research.py",
+               "bin/tezgah-research", ".tezgah/research/"],
+     "switch": "research-off", "editable": False, "revert": "git"},
+    {"key": "product", "label": "product",
+     "files": ["hooks/tezgah_policy.py", "skills/product-analysis/SKILL.md",
+               "skills/feature-audit/SKILL.md"],
+     "switch": "research-off", "editable": True, "revert": "git"},
+    {"key": "attribution", "label": "attribution",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_gate.py"],
+     "switch": "none", "editable": False, "revert": "git"},
+    {"key": "lang", "label": "lang",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_gate.py"],
+     "switch": "lang-off", "editable": False, "revert": "git"},
+    {"key": "workspace", "label": "workspace",
+     "files": ["hooks/tezgah_policy.py", "hooks/tezgah_gate.py"],
+     "switch": "workspace-off", "editable": False, "revert": "git"},
+    {"key": "skills", "label": "skills",
+     "files": ["skills/"],
+     "switch": "none", "editable": True, "revert": "git"},
+    {"key": "subagents", "label": "subagents",
+     "files": ["hooks/tezgah_agents.py"],
+     "switch": "none", "editable": True, "revert": "git"},
+    {"key": "memory", "label": "memory",
+     "files": [".tezgah/lessons.md"],
+     "switch": ".no-lessons", "editable": True, "revert": "snapshot"},
+    # The gate's own advisory surface: the long-turn notice (`drift`) is not a
+    # CORE paragraph, so it owns no rule label. It is declared with no switch of
+    # its own: `reminder-off` does drop it, but declaring that would say the
+    # manifest owns the per-turn reminder, and the switch test's own model reads
+    # every declared switch as one that leaves the rest of the turn intact.
+    # Added when the drift-notice experiment needed a component to name.
+    {"key": "drift-notice", "label": "drift-notice",
+     "files": ["hooks/tezgah_gate.py"],
+     "switch": "none", "editable": False, "revert": "git"},
+]
+
+
+def keys():
+    """The component keys, in manifest order: what a prediction row may name."""
+    return [c["key"] for c in COMPONENTS]