@r1z4x/tezgah 1.0.0

package/hooks/tezgah_lang.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""A non-English identifier tezgah is about to write: a heuristic, not a
+language detector.
+
+What this answers: the contract has always said code, commits, branches and docs
+are English, and nothing checked it - a plan branch was created as
+`plan/004-admin-durum-onarimi`, and the repository carries that folded slug in
+its history for the rest of its life. `hooks/tezgah_gate.py` refuses the command
+that would create such an identifier (the language rule in `decision`); this
+module is the detector behind it, so the predicate and the word list live in one
+place and a test can read them without a session.
+
+Two halves, and they answer different failures:
+
+- **Any letter outside ASCII.** The contract is "identifiers and messages stay
+  English", not "no Turkish", so a Russian, Greek, Arabic, Chinese or
+  accented-Latin identifier is refused for the same reason a Turkish one is -
+  `non_english()` is a predicate rather than a list of one language's letters.
+- **A curated Turkish word list.** Turkish is the language this repository has
+  actually seen folded into ASCII, where the alphabet half cannot see it
+  (`durum`, `onarim`, `hazirlik`). The list carries the English each stem should
+  be written as, so a refusal can name the replacement instead of leaving the
+  session to guess.
+
+What this is NOT: it does not read meaning and it is not a language detector. A
+word outside the two halves passes - an ASCII-folded word from any other
+language (`naprawa-stanu`, `reparatur`) is invisible here, and that is the
+ceiling this module states rather than hides: catching it would take a language
+package or a model, which is a dependency and a nondeterministic answer on a
+denial path. A proper noun, a product name and the user's own term are the cases
+no list can decide, which is why the refusal says it is a heuristic and hands the
+call back to the user instead of asserting a language.
+"""
+import re
+
+# The characters an English identifier never contains: a letter or digit outside
+# ASCII. A predicate rather than a list, because the contract is "identifiers and
+# messages stay English" and not "no Turkish": a Russian, Greek, Arabic, Chinese
+# or accented-Latin identifier is refused for the same reason `durum-onarimi` is,
+# and Turkish is one case of it. Measured before this change: every one of
+# `статус`, `إصلاح`, `状态修复`, `κατάσταση`, `kế-hoạch` and `état` passed, while
+# ASCII `I` - the letter `İ` folds to - was refused, which refused `CI`, `ID` and
+# `I1` as well. The cost is stated in the refusal: a proper noun or a product name
+# in another language is refused too, and the user's own term is theirs to settle.
+def non_english(token):
+    """True when the token holds a character English identifiers do not: any
+    alphanumeric outside ASCII. Digits and `-`/`_`/`/` are boundaries, so a slug
+    keeps its words whole and a version like `4.2.1` is untouched."""
+    return any(ord(ch) > 127 and ch.isalnum() for ch in token)
+
+# The ASCII a Turkish letter folds to when a slug or a filename strips it. The
+# reverse direction is what makes the word list work on `onarım` as it works on
+# `onarim`: the token is folded before it is looked up, so the English the list
+# knows is offered for either spelling.
+_FOLD = str.maketrans({"ı": "i", "İ": "I", "ş": "s", "Ş": "S", "ğ": "g",
+                       "Ğ": "G", "ü": "u", "Ü": "U", "ö": "o", "Ö": "O",
+                       "ç": "c", "Ç": "C"})
+
+# A token is a run of letters and digits; everything else - `-`, `_`, `/`,
+# space, a dot, a colon - is a boundary. That is the set the identifier
+# vocabulary is built from, so `plan/004-admin-durum-onarimi` yields `durum` and
+# `onarimi` and not one long string.
+# A token is a run of ASCII or non-ASCII alphanumerics; `-`, `_`, `/`, space
+# and punctuation are boundaries. Splitting on ASCII-alnum alone is what keeps
+# a non-Latin word whole, so the refusal names it instead of a fragment.
+_TOKENS = re.compile(r"[^0-9A-Za-z\u00c0-\U0010ffff]+")
+
+# The Turkish stems the list knows, and the English each one is asking for. The
+# replacements are the point of the list: the refusal can name what to write
+# instead, so a refused session is not left guessing at a translation.
+#
+# Every key is a word English does not have, which is what keeps the list from
+# refusing English text. `data` and `test` are the shape to avoid: they are
+# English words AND Turkish ones, so a list that held them would refuse an
+# ordinary commit subject.
+ENGLISH = {
+    "durum": "status",
+    "kural": "rule",
+    "dosya": "file",
+    "sayfa": "page",
+    "hata": "error",
+    "rapor": "report",
+    "ozet": "summary",
+    "sonuc": "result",
+    "kullanim": "usage",
+    "calisma": "work",
+    "onarim": "repair",
+    "deneme": "experiment",
+    "incele": "review",
+    "kontrol": "check",
+    "listele": "list",
+    "goster": "show",
+    "tasarla": "design",
+    "duzelt": "fix",
+    "ekle": "add",
+    "kaldir": "remove",
+    "guncelle": "update",
+    "hazirla": "prepare",
+    "olustur": "create",
+    "degistir": "change",
+    "temizle": "clean",
+    "iyilestir": "improve",
+    "karsilastir": "compare",
+    "tasi": "move",
+    "sil": "delete",
+    # `sil` on its own is exact-matched (see SHORT below), so the suffixed form
+    # Turkish actually uses needs its own entry: `silme` is the word a plan
+    # says, and `sil*` would also swallow `silent` and `silk`.
+    "silme": "deletion",
+    # The second group: the words a 57-word probe of ordinary Turkish found
+    # missing. Each one is here because the English side of that probe stayed
+    # clean - no English word grows out of the stem - and the three that DO have
+    # an English word growing out of them are in WHOLE below instead.
+    "hazirlik": "preparation",
+    "silin": "deleted",
+    "degis": "change",
+    "temizlik": "cleanup",
+    "tasarim": "design",
+    "baglanti": "link",
+    "ayar": "settings",
+    "kullanici": "user",
+    "yetki": "permission",
+    "guvenlik": "security",
+    "testler": "tests",
+    "senaryo": "scenario",
+    "aciklama": "description",
+    "ornek": "example",
+    "cikti": "output",
+    # the three WHOLE stems (below): listed here so the word itself is still
+    # named and translated, and read whole-token so the English they grow is not
+    "arama": "search",
+    "veri": "data",
+    "girdi": "input",
+    "donusum": "conversion",
+    "yapilandirma": "configuration",
+    "surum": "version",
+    "yayin": "release",
+    "baslat": "start",
+    "durdur": "stop",
+    "bekle": "wait",
+    "hiz": "speed",
+    "hizmet": "service",
+    "hizli": "fast",
+    "boyut": "size",
+}
+
+# A stem this short is matched whole-token rather than as a prefix. English
+# words are shorter than Turkish suffixes suggest: `sil` prefixes `silent` and
+# `silk`, while every longer stem in the list above has no English word growing
+# out of it, so the prefix match - which is what catches `onarimi`, `duzeltmesi`
+# - costs nothing there. Raising this number would start missing Turkish
+# suffixes; lowering it would start refusing English words.
+SHORT = 4
+
+# The stems long enough for the prefix match that must not use it, because the
+# prefix IS an English word: `veri` grows `verify`, `verification` and
+# `verifiable`, `arama` grows `Aramaic`, `girdi` grows `girding`. They are read
+# whole-token, which still catches the word on its own and in every slug or
+# subject that separates it (`veri-tabani`, `arama-sonuclari`, `girdi-dosyasi`)
+# and misses only the inflections Turkish glues to it (`veriler`, `aramasi`) -
+# the price of not refusing English. The English side of the probe is the guard
+# on this whole list, pinned in tests/test_lang.py.
+WHOLE = ("arama", "veri", "girdi")
+
+DENY = (
+    "Non-English identifier: the %s this command would create is not "
+    "English (`%s`). Code, commits, branches, PR and issue text stay English, "
+    "because an identifier outlives the session that typed it: a branch name "
+    "and a commit subject are in the repository, and often in a public "
+    "history, from the moment they exist. Say the same thing in English and "
+    "re-issue the command with it.%s This is a heuristic - a letter set plus a "
+    "curated word list, not a language detector - so it cannot tell a proper "
+    "noun or a product name in another language from a word that ought to be "
+    "English, and a word it lists may be the user's own term. Where the wording "
+    "is the user's own, say so plainly and let them settle it: their explicit "
+    "instruction overrides this rule, and re-issuing the same command is not "
+    "that.")
+
+
+def _folded(token):
+    return token.translate(_FOLD).lower()
+
+
+def replacement(token):
+    """The English the list holds for this token, or None when it does not know
+    it - a token the letter set caught, a word the user chose, a proper noun."""
+    folded = _folded(token)
+    for stem, english in ENGLISH.items():
+        whole = len(stem) < SHORT or stem in WHOLE
+        if folded == stem or (not whole and folded.startswith(stem)):
+            return english
+    return None
+
+
+def offending(text):
+    """The tokens of `text` that are not English, each once, in order.
+
+    Empty for English text and for an empty string, so a caller can treat a
+    non-empty answer as the whole condition."""
+    out = []
+    for token in _TOKENS.split(str(text or "")):
+        if not token or token in out:
+            continue
+        if non_english(token) or replacement(token):
+            out.append(token)
+    return out
+
+
+def _hint(tokens):
+    """The English to write instead, for the tokens the list knows. Empty when
+    it knows none of them - then the non-ASCII letter caught them and the
+    refusal says only which token it read."""
+    known = ["`%s` -> `%s`" % (token, replacement(token))
+             for token in tokens if replacement(token)]
+    if not known:
+        return ""
+    return " The English this list knows: %s." % ", ".join(known)
+
+
+def refusal(what, text):
+    """The refusal for a `text` that would become `what`, or None when the text
+    reads as English.
+
+    `what` is the identifier the command is about to create, in the words the
+    session wrote it with ("branch name", "commit subject", "PR title"), so the
+    refusal names the thing that will exist and not the regex that caught it."""
+    tokens = offending(text)
+    if not tokens:
+        return None
+    return DENY % (what, "`, `".join(tokens), _hint(tokens))

package/hooks/tezgah_lessons.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""The lesson ledger's one reader: `.tezgah/lessons.md` as entries, and the key
+each entry is remembered by.
+
+The session block, the per-turn block and the per-turn digest
+(`tezgah_context`), the gate and the tidy CLI (`bin/tezgah-lessons`) all read the
+ledger here, so they agree on what a lesson is and on its key. A module of its
+own because the gate must not import `tezgah_context` (its import cost, and the
+rule `hooks/tezgah_task.py::repo_root` states): this one imports only
+`tezgah_paths` at load, and `tezgah_gate` lazily, only for a line that names a
+gate rule as its enforcer.
+
+Stdlib only."""
+import hashlib
+import json
+import os
+import re
+
+from tezgah_paths import cache_dir, off
+
+# A line carries its metadata after its text, never in it: from the first
+# `|| check:` or `|| enforced_by:` clause outside a code span to the end of the
+# line (in any order), and an `@<sha> <path>` stamp just before those clauses.
+# `parse` cuts both off before anything else reads the line, so neither enters
+# the 200-character cut, `lesson_key`, the per-turn digest or the ranking. Prose
+# keeps the rest: a bare `||` (`pytest || true`), a clause inside backticks
+# (`a||check:b`), and an `@<sha>` followed by a word that is not a path (`git
+# show @cafebabe1 HEAD` - a path holds a `/` or a `.`). `check` is plan 061's
+# compiled check, which nothing reads yet.
+SUFFIX = re.compile(r"\s*\|\|\s*(?:check|enforced_by):")
+STAMP = re.compile(r"\s+@[0-9a-f]{7,40}\s+\S*[/.]\S*$")
+# A line a gate rule or a test already enforces carries `|| enforced_by:
+# <slug|test>` and leaves the injected pool while that enforcer is armed: a gate
+# rule slug from `tezgah_gate.DENY_RULES`, or a test named from the repository
+# root (`tests.test_research.Unfinished`) that this repository still defines. An
+# unknown name keeps the line. Read off the tail `parse` cut.
+ENFORCED = re.compile(r"\|\|\s*enforced_by:\s*(\S+)\s*(?=\|\||$)")
+# A ledger written with markdown bullets must not render as "- - ...".
+BULLET = re.compile(r"^[-*+]\s+|^\d+[.)]\s+")
+
+
+def ledger(root):
+    """The ledger's path in the repository at `root`."""
+    return os.path.join(root, ".tezgah", "lessons.md")
+
+
+def lesson_key(line):
+    """The short id a lesson is remembered by: the session's turn stamp, the
+    `lesson` row and the `lesson_tainted` row."""
+    return hashlib.sha1(line.encode("utf-8", "replace")).hexdigest()[:8]
+
+
+# The top-level class and function names of a test module, per (path, mtime,
+# size): a ledger's retired lines name one module many times, and parsing a
+# large test file once per line would cost every session start.
+_TEST_NAMES = {}
+
+
+def _test_names(path):
+    """The names `unittest` can load from `path` as `module.NAME`: its top-level
+    classes and functions, by `ast`, so a nested def or a `class X:` inside a
+    string is not one. Empty when the file is missing or does not parse."""
+    try:
+        st = os.stat(path)
+        key = (path, st.st_mtime_ns, st.st_size)
+        if key not in _TEST_NAMES:
+            import ast  # lazy: only a line retired by a test pays for it
+            with open(path, encoding="utf-8", errors="replace") as fh:
+                tree = ast.parse(fh.read(), filename=path)
+            _TEST_NAMES[key] = frozenset(
+                n.name for n in tree.body
+                if isinstance(n, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)))
+        return _TEST_NAMES[key]
+    except (OSError, SyntaxError, ValueError):
+        return frozenset()
+
+
+def _enforced(value, root):
+    """Whether the enforcer a retired lesson names is armed now: a test only
+    while `<root>/tests/test_x.py` still defines it at top level, a gate rule
+    only while neither its own switch nor `pretooluse-off` is set."""
+    test = re.match(r"tests\.(test_\w+)\.(\w+)$", value)
+    if test:
+        return test.group(2) in _test_names(
+            os.path.join(root, "tests", test.group(1) + ".py"))
+    from tezgah_gate import DENY_RULES  # lazy: only a retired line pays for it
+    if value not in DENY_RULES or off("pretooluse-off"):
+        return False
+    return not (DENY_RULES[value] and off(DENY_RULES[value]))
+
+
+def parse(line):
+    """(text, tail) for one ledger line - the lesson as every reader sees it, and
+    the clauses cut off its end - or None for a blank line or a `#` heading."""
+    s = line.strip()
+    if not s or s.startswith("#"):
+        return None
+    s = BULLET.sub("", s)
+    m = next((m for m in SUFFIX.finditer(s)
+              if s.count("`", 0, m.start()) % 2 == 0), None)
+    text, tail = (s[:m.start()], s[m.start():]) if m else (s, "")
+    return STAMP.sub("", text), tail
+
+
+def entries(text):
+    """The lessons a text holds, each as `lines` returns it: what a write adds
+    to the ledger, read the way the ledger will be read."""
+    return [got[0] for got in map(parse, text.splitlines()) if got and got[0]]
+
+
+def is_ledger(path):
+    """True when the absolute, unresolved `path` names a repository's ledger."""
+    return (os.path.basename(path) == "lessons.md"
+            and os.path.basename(os.path.dirname(path)) == ".tezgah")
+
+
+def taint_path(root):
+    """The `lesson_tainted` index of the repository at `root`: the gate keeps
+    each keyed row here as well as in the session's ledger, so the context of a
+    later session reads one small file for the per-line label instead of every
+    ledger on the machine (3173 files, 30 MB on the measuring machine,
+    2026-10-07)."""
+    name = hashlib.sha1(os.path.realpath(root).encode("utf-8", "replace"))
+    return os.path.join(cache_dir(), "lessons", name.hexdigest()[:16] + ".jsonl")
+
+
+def _index_rows(path):
+    """The parseable keyed rows of an index file, oldest first; [] without one.
+    A damaged line costs only itself."""
+    rows = []
+    try:
+        with open(path, encoding="utf-8", errors="replace") as fh:
+            for ln in fh:
+                try:
+                    row = json.loads(ln)
+                except ValueError:
+                    continue
+                if isinstance(row, dict) and row.get("key"):
+                    rows.append(row)
+    except OSError:
+        pass
+    return rows
+
+
+def write_taint(root, keep, rows):
+    """Rewrite the index of the repository at `root`: the rows it holds whose key
+    is in `keep` (the ledger's lines plus the write's new ones), then `rows`. So
+    the file never outgrows the ledger it labels. Best effort, like a ledger row.
+    ponytail: two sessions rewriting one index in the same instant can lose one
+    row (one label); the session ledger keeps its own copy of every row."""
+    path = taint_path(root)
+    kept = [r for r in _index_rows(path) if r["key"] in keep] + rows
+    tmp = "%s.%d.tmp" % (path, os.getpid())
+    try:
+        os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
+        fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            fh.write("".join(json.dumps(r) + "\n" for r in kept))
+        os.replace(tmp, path)
+    except OSError:
+        pass
+
+
+# The index as last read, per (path, mtime, size): the session block, the
+# per-turn block and the digest of one prompt each ask for it, and a hook
+# process serves one prompt.
+_TAINTED = {}
+
+
+def tainted(root):
+    """{key: source} for the lessons of the repository at `root` that a gate
+    recorded as written in a turn that had read untrusted text (`taint_path`);
+    {} when there is none. Read once while the file is unchanged."""
+    path = taint_path(root)
+    try:
+        st = os.stat(path)
+    except OSError:
+        return {}
+    key = (path, st.st_mtime_ns, st.st_size)
+    if key not in _TAINTED:
+        _TAINTED[key] = {str(r["key"]): str(r.get("source") or "")
+                         for r in _index_rows(path)}
+    return _TAINTED[key]
+
+
+def lines(root, retired=None):
+    """The lesson ledger as entries: one per line, markdown bullets, the clause
+    tail and the stamp stripped (`parse`).
+
+    The one reader, so every surface agrees on what is a lesson: a line whose
+    enforcer is armed (ENFORCED) is left out and appended to `retired` when
+    given; one whose enforcer is off comes back without its tail."""
+    try:
+        with open(ledger(root), encoding="utf-8", errors="replace") as fh:
+            raw = fh.read().splitlines()
+    except OSError:
+        return []
+    out = []
+    for ln in raw:
+        got = parse(ln)
+        if not got or not got[0]:
+            continue
+        text, tail = got
+        m = ENFORCED.search(tail)
+        if m and _enforced(m.group(1), root):
+            if retired is not None:
+                retired.append(text)
+            continue
+        out.append(text)
+    return out