@r1z4x/tezgah 1.5.1 → 1.7.0

package/hooks/tezgah_models.py

@@ -37,6 +37,7 @@ tier from the static table, and with no phase the middle tier is used.
 
 Stdlib only; every network or subprocess failure is a fallback, never a raise.
 """
+import contextlib
 import datetime
 import json
 import os
@@ -232,18 +233,331 @@ def _omp_stored_providers():
     return {row[0] for row in rows}
 
 
-def funded_families():
+def funded_families(live=True):
     """The `OMP_FAMILIES` with a credential this machine shows: an omp auth-store
-    entry, an env key omp reads, or (for `any`) `openrouter_ready`.
+    entry, an env key omp reads, or (for `any`) `openrouter_ready` - minus, with
+    `live`, the families the health record holds as down (`down_families`).
 
     A credential is not a balance - a funded-looking key can still answer 402 -
-    which is why each agent gets a chain rather than one more pin. A family whose
-    credential cannot be detected is left out, never guessed in."""
+    which is why each agent gets a chain rather than one more pin, and why a
+    family whose provider answered a limit leaves the chain until its reset. A
+    family whose credential cannot be detected is left out, never guessed in."""
     stored = _omp_stored_providers()
+    down = down_families() if live else set()
     return [family for family in OMP_FAMILIES
-            if OMP_AUTH[family][0] in stored
-            or any(os.environ.get(env, "").strip() for env in OMP_AUTH[family][1])
-            or (family == "any" and openrouter_ready())]
+            if family not in down
+            and (OMP_AUTH[family][0] in stored
+                 or any(os.environ.get(env, "").strip() for env in OMP_AUTH[family][1])
+                 or (family == "any" and openrouter_ready()))]
+
+
+# ------------------------------------------------------------ provider health
+#
+# A family with a credential can still be spent: on 2026-10-11 every subagent
+# chain read `anthropic, openrouter`, one Anthropic 429 moved the subagents onto
+# an OpenRouter key past its limit (403), and they died; zai, funded in omp's
+# store, answered `429 Weekly/Monthly Limit Exhausted. Your limit will reset at
+# 2026-10-12 02:02:35`. The record below holds which family is down and until
+# when; `funded_families` leaves those out, and `tezgah-route --health` - started
+# in the background at session start when due - probes the non-mode families
+# and re-applies omp's chain, so a family rejoins it once its reset has passed.
+HEALTH_FILE = "model-health.json"
+HEALTH_EVERY = 6 * 3600       # re-probe at most this often, unless a reset passed
+HEALTH_DOWN = 6 * 3600        # how long a limit with no reset time keeps a family out
+HEALTH_RATE = 600             # the same for a rate limit (429): it lifts in minutes
+HEALTH_DOWN_MAX = 7 * 86400   # the longest a named reset is believed
+OPENROUTER_KEY = "https://openrouter.ai/api/v1/key"
+_RESET_AT = re.compile(r"reset[s]? (?:at|on) (\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})")
+_RETRY_MS = re.compile(r"retry-after-ms=(\d+)")
+_LIMITED = re.compile(r"\b(?:429|402|403)\b|rate[ _-]?limit|quota|limit exhausted|"
+                      r"key limit exceeded|usage limit|insufficient (?:credit|balance)",
+                      re.I)
+
+
+def _health_path():
+    return os.path.join(tp.cache_dir(), HEALTH_FILE)
+
+
+def read_health():
+    """{"checked": epoch or None, "down": {family: {"until": epoch, "reason": str}}};
+    an unreadable record reads as empty - nothing known down."""
+    try:
+        with open(_health_path(), encoding="utf-8") as fh:
+            data = json.load(fh)
+    except (OSError, ValueError):
+        data = {}
+    if not isinstance(data, dict):
+        data = {}
+    down = data.get("down") if isinstance(data.get("down"), dict) else {}
+    return {"checked": data.get("checked"), "down": down}
+
+
+def write_health(data):
+    """Write the record atomically; False when the cache dir cannot be written."""
+    path = _health_path()
+    try:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        tmp = "%s.%d.tmp" % (path, os.getpid())
+        with open(tmp, "w", encoding="utf-8") as fh:
+            json.dump(data, fh, indent=1, sort_keys=True)
+        os.replace(tmp, path)
+        return True
+    except OSError:
+        return False
+
+
+def down_families(now=None, data=None):
+    """The families the record holds as down whose reset has not passed yet."""
+    import time
+    now = time.time() if now is None else now
+    data = read_health() if data is None else data
+    return {family for family, row in data["down"].items()
+            if isinstance(row, dict) and isinstance(row.get("until"), (int, float))
+            and row["until"] > now}
+
+
+def limit_until(text, now=None, default=HEALTH_DOWN):
+    """When a provider's limit answer says it lifts, as an epoch, or None when the
+    text is no limit answer at all. A named reset (`reset at 2026-10-12 02:02:35`,
+    read as local time) or a `retry-after-ms` wins; else `default` from now. A
+    reset further than HEALTH_DOWN_MAX away is capped: a wrong year must not lock a
+    family out for good."""
+    import time
+    now = time.time() if now is None else now
+    text = str(text or "")
+    if not _LIMITED.search(text):
+        return None
+    until = None
+    hit = _RESET_AT.search(text)
+    if hit:
+        try:
+            until = time.mktime(time.strptime("%s %s" % hit.groups(), "%Y-%m-%d %H:%M:%S"))
+        except (ValueError, OverflowError):
+            until = None
+        if until is not None and until <= now:
+            # a named reset already past is a clock we misread (the provider's
+            # zone is not stated): a minute's hold would re-probe every prompt
+            until = now + default
+    if until is None:
+        ms = _RETRY_MS.search(text)
+        until = now + int(ms.group(1)) / 1000.0 if ms else now + default
+    return min(max(until, now + 60), now + HEALTH_DOWN_MAX)
+
+
+@contextlib.contextmanager
+def _health_locked():
+    """An exclusive flock on the record's lock file around one re-read, change
+    and write, so two writers (a slice's stop, the probe) do not drop each
+    other's hold. Unlocked where fcntl is missing or the file cannot be opened."""
+    fh = None
+    try:
+        import fcntl
+        os.makedirs(os.path.dirname(_health_path()), exist_ok=True)
+        fh = open(_health_path() + ".lock", "a", encoding="utf-8")
+        fcntl.flock(fh, fcntl.LOCK_EX)
+    except (ImportError, OSError):
+        pass
+    try:
+        yield
+    finally:
+        if fh is not None:
+            fh.close()
+
+
+def _family(provider="", model="", default=""):
+    """The OMP_FAMILIES entry a provider id (omp's auth-store id), else a model id
+    (`_vendor`), else `default` runs under; None when none is one."""
+    by_provider = {auth[0]: family for family, auth in OMP_AUTH.items()}
+    for family in (by_provider.get(str(provider or "").lower()),
+                   _vendor(str(model)) if model else None, default):
+        if family in OMP_FAMILIES:
+            return family
+    return None
+
+
+def note_limit(text, provider="", model="", transient=False, default="", now=None):
+    """Hold down the family a slice stopped on with a limit answer (`text`), and
+    re-apply omp's chain, so the next spawn of this session leaves that family
+    instead of the next session start's probe moving it (research
+    orchestrator-flow-audit S2, F3a). A rate limit (`transient`) with no reset
+    named holds HEALTH_RATE, a quota HEALTH_DOWN. Returns the family newly held,
+    else None: no family resolves (`_family`), `text` is no limit answer
+    (`limit_until`), the family is already held, or TEZGAH_HEALTH_CHECK=0 (the
+    suite's switch)."""
+    import time
+    family = _family(provider, model, default)
+    if not family or os.environ.get("TEZGAH_HEALTH_CHECK") == "0":
+        return None
+    now = time.time() if now is None else now
+    until = limit_until(text, now, HEALTH_RATE if transient else HEALTH_DOWN)
+    if until is None:
+        return None
+    with _health_locked():
+        data = read_health()
+        if family in down_families(now, data):
+            return None
+        data["down"][family] = {"until": until,
+                                "reason": " ".join(str(text).split())[:200]}
+        if not write_health(data):
+            return None
+    apply_omp()
+    return family
+
+
+def other_family_up(text):
+    """Where work a limit stopped on can move: the first family the routing lets
+    a chain fall back to (`_chain` under `tp.fallback_policy()`, every slot) that
+    is funded and not down (`funded_families`) and that `text` does not name
+    (`named_families`), else None. None with routing off or under
+    TEZGAH_HEALTH_CHECK=0, so the suite never reads this machine's credentials."""
+    if os.environ.get("TEZGAH_HEALTH_CHECK") == "0":
+        return None
+    mode = omp_mode()
+    if mode not in OMP_FAMILIES:
+        return None
+    funded = funded_families(live=True)
+    chained = {sel for slot in SLOTS for sel in _chain(mode, slot, funded)}
+    named = named_families(text)
+    return next((family for family in funded if family not in named and any(
+        sel.startswith(OMP_AUTH[family][0] + "/") for sel in chained)), None)
+
+
+def named_families(text):
+    """The OMP_FAMILIES `text` names by an auth-store provider id (a whole word)
+    or a model stem (`VENDOR_STEMS`, a word start): `openrouter 403` names `any`,
+    `glm-5.3` names `zai`."""
+    low = str(text or "").lower()
+    return {family for family in OMP_FAMILIES
+            if re.search(r"\b%s\b" % re.escape(OMP_AUTH[family][0]), low)
+            or any(re.search(r"\b%s" % re.escape(s), low)
+                   for s, v in VENDOR_STEMS if v == family)}
+
+
+def _openrouter_key():
+    env, home = tp.PROVIDER_KEYS["openrouter"]
+    key = os.environ.get(env, "").strip()
+    if key:
+        return key
+    try:
+        with open(os.path.join(tp.HOME, home), encoding="utf-8") as fh:
+            return fh.read().strip()
+    except OSError:
+        return ""
+
+
+def probe_family(family, timeout=60):
+    """(up, reason) for one family: None for up when the probe could not tell.
+
+    `any` asks OpenRouter's key endpoint, which costs no tokens: a key whose
+    `limit_remaining` is 0 or less is down. A family omp runs is asked one short
+    prompt on that family's cheap row through `omp -p --no-session`; exit 0 is
+    up, a limit answer is down (its reset in `reason`), anything else is unknown."""
+    if family == "any":
+        key = _openrouter_key()
+        if not key:
+            return None, "no OpenRouter key"
+        req = urllib.request.Request(OPENROUTER_KEY,
+                                     headers={"Authorization": "Bearer " + key})
+        try:
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                info = (json.loads(resp.read().decode("utf-8")) or {}).get("data") or {}
+        except Exception as exc:  # noqa: BLE001 - any network failure is "cannot tell"
+            text = str(exc)
+            return (False, text) if _LIMITED.search(text) else (None, text)
+        left = info.get("limit_remaining")
+        if isinstance(left, (int, float)) and left <= 0:
+            return False, "OpenRouter key limit exhausted (limit %s, usage %s)" % (
+                info.get("limit"), info.get("usage"))
+        return True, "OpenRouter key has room"
+    exe = tp.omp_bin()
+    if not exe:
+        return None, "omp not found"
+    # TEZGAH_NESTED, as consult and the judge set it: a hook that does load in
+    # the child records nothing and refuses nothing
+    env = dict(os.environ, PI_CODING_AGENT_DIR=OMP_AGENT, TEZGAH_NESTED="1")
+    try:
+        # the consult argv: no tools, rules, skills, extensions or title call, so
+        # the probe is one short request and fires no nested tezgah hooks
+        argv = [exe] + tp.CONSULT_CLIS["omp"][0][1:]
+        proc = subprocess.run(argv + ["--model", _selector(family, "cheap"),
+                                      "--max-time=%d" % timeout, "Reply with exactly: OK"],
+                              capture_output=True, text=True, timeout=timeout + 15,
+                              env=env, cwd=tp.HOME)
+    except (OSError, subprocess.SubprocessError) as exc:
+        return None, str(exc)
+    text = (proc.stdout or "") + (proc.stderr or "")
+    if proc.returncode == 0:
+        return True, "answered"
+    tail = " ".join(text.split())[-200:]
+    return (False, tail) if _LIMITED.search(text) else (None, tail)
+
+
+def refresh_health(mode=None, probe=probe_family, now=None):
+    """Probe every family with a credential except the mode's own (it leads the
+    chain whatever its state, and the session is running on it), record the
+    downs with their resets, and return the record. A family the probe cannot
+    judge keeps the state it had. The probes run unlocked; their results land on
+    the record re-read under the lock (`_health_locked`), so a hold a slice's
+    stop wrote meanwhile stays."""
+    import time
+    now = time.time() if now is None else now
+    mode = mode or omp_mode()
+    seen = {family: probe(family) for family in funded_families(live=False)
+            if family != mode}
+    with _health_locked():
+        down = dict(read_health()["down"])
+        for family, (up, reason) in seen.items():
+            if up is True:
+                down.pop(family, None)
+            elif up is False:
+                down[family] = {"until": limit_until(reason, now) or now + HEALTH_DOWN,
+                                "reason": reason[:200]}
+        data = {"checked": now, "down": {f: r for f, r in down.items()
+                                         if isinstance(r, dict) and r.get("until", 0) > now}}
+        write_health(data)
+    return data
+
+
+def health_due(data=None, now=None):
+    """True when the record was never checked, is older than HEALTH_EVERY, or
+    holds a down family whose reset has passed since it was checked. Never with
+    TEZGAH_HEALTH_CHECK=0 (the test suite's switch, like TEZGAH_UPDATE_CHECK)."""
+    import time
+    if os.environ.get("TEZGAH_HEALTH_CHECK") == "0":
+        return False
+    now = time.time() if now is None else now
+    data = read_health() if data is None else data
+    checked = data.get("checked")
+    if not isinstance(checked, (int, float)) or now - checked >= HEALTH_EVERY:
+        return True
+    return any(isinstance(row, dict) and checked < row.get("until", 0) <= now
+               for row in data["down"].values())
+
+
+def start_health():
+    """Stamp the record, then start one detached `tezgah-route --health`, so the
+    session starts that follow read a fresh stamp and start nothing. False when
+    nothing started."""
+    import sys
+    import time
+    with _health_locked():
+        data = read_health()
+        data["checked"] = time.time()
+        if not write_health(data):
+            return False
+    route = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
+                         "bin", "tezgah-route")
+    kwargs = {"stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL,
+              "stderr": subprocess.DEVNULL, "close_fds": True}
+    if os.name == "posix":
+        kwargs["start_new_session"] = True
+    else:  # pragma: no cover - Windows: no console, outlives its parent
+        kwargs["creationflags"] = 0x00000008 | 0x00000200
+    try:
+        subprocess.Popen([sys.executable, route, "--health"], **kwargs)
+        return True
+    except OSError:
+        return False
 
 
 def omp_mode(default_selector=None):
@@ -308,11 +622,14 @@ def _chain(mode, slot, funded, agent=""):
     because a session on Opus that ran its subagents on GPT and GLM after one
     Anthropic 429 is the failure the owner reported (2026-10-07). `none`: the
     mode's selector alone. `any`: every other funded family, each on that
-    slot's own row, rotated by the agent's own name (`agent`): identical chains
-    for every agent meant one family's 429 moved all of them onto the same next
-    family in the same moment, which is the cascade the chain exists to break
-    (independent review, 2026-10-01). The rotation is deterministic - the same
-    agent resolves the same order on every machine - so nothing here is random."""
+    slot's own row. The families omp runs on an account (zai) come before
+    OpenRouter, which bills per token - the owner's ask (2026-10-11) was to use
+    zai when it is up - and among those the order is rotated by the agent's own
+    name (`agent`): identical chains for every agent meant one family's 429
+    moved all of them onto the same next family in the same moment, which is the
+    cascade the chain exists to break (independent review, 2026-10-01). The
+    rotation is deterministic - the same agent resolves the same order on every
+    machine - so nothing here is random."""
     policy = tp.fallback_policy()
     families = [mode] + [f for f in OMP_FAMILIES if f != mode and f in funded]
     if policy == "none":
@@ -327,10 +644,11 @@ def _chain(mode, slot, funded, agent=""):
                 out.append("openrouter/" + OPENROUTER_ID[model]
                            + (":" + effort if effort else ""))
         return list(dict.fromkeys(out))
-    tail = families[1:]
+    tail = [f for f in families[1:] if f != "any"]
     if agent and len(tail) > 1:
         shift = sum(ord(c) for c in agent) % len(tail)
-        families = families[:1] + tail[shift:] + tail[:shift]
+        tail = tail[shift:] + tail[:shift]
+    families = families[:1] + tail + [f for f in families[1:] if f == "any"]
     return [_selector(family, slot) for family in families]
 
 
@@ -552,7 +870,11 @@ def apply_omp(remove=False):
     owns.update(owns_chains)
     merged_chains = dict(chains)
     for key in flat_chains:
-        role = key.split(".", 1)[1]
+        role = key.rsplit(".", 1)[1]
+        # an earlier release wrote the chain under `fallbackChains.<role>`, a key
+        # omp never reads (the `split(".", 1)` of the flat key); it is tezgah's
+        # own and goes
+        merged_chains.pop("fallbackChains." + role, None)
         value = new_chains.get(key)
         if value is None:
             merged_chains.pop(role, None)

package/hooks/tezgah_paths.py

@@ -70,7 +70,8 @@ OFF_DIRS = (CONFIG_DIR, os.path.join(HOME, ".claude"))
 # the opencode plugin's own `off()` and the gate's control rule all answer from
 # it, and tests/test_paths.py fails when any of them names a switch it lacks.
 # CORE_SWITCHES are the sixteen the always-on contract names. ADR 007 leaves
-# UNCLASSIFIED_SWITCHES open - two `off()` names CORE does not list and two
+# UNCLASSIFIED_SWITCHES open - three `off()` names CORE does not list (the
+# third, `slices-off`, stops the slice journal: hooks/tezgah_task.py) and two
 # opt-in markers read with `armed()` - and until the owner classifies them the
 # control rule protects them like the rest (bin/tezgah-docs carries the same
 # set for its inventory check; the test pins the two equal).
@@ -78,7 +79,7 @@ CORE_SWITCHES = ("adhd-off", "consult-off", "docs-judge-off", "exec-mode.off",
                  "judge-off", "lang-off", "orchestrate-off", "ponytail-auto.off",
                  "pretooluse-off", "reminder-off", "research-off", "spec-off",
                  "task-off", "triage-off", "verify-off", "workspace-off")
-UNCLASSIFIED_SWITCHES = ("agents-off", "skill-suggest-on", "taste-on",
+UNCLASSIFIED_SWITCHES = ("agents-off", "skill-suggest-on", "slices-off", "taste-on",
                          "update-check-off")
 SWITCHES = CORE_SWITCHES + UNCLASSIFIED_SWITCHES
 REPO_MARKS = (".no-adhd", ".no-graph", ".no-lessons", ".no-plan-gate",
@@ -737,17 +738,99 @@ def codex_bin():
     return which_user(os.environ.get("TEZGAH_CODEX_BIN") or "codex")
 
 
-# The values `reply_lang` (config.json) takes. The value itself is the switch:
-# `tr` (the default, and what an install without the key reads) asks for Turkish
-# replies, `en` for English ones, and `any` drops the Stop language check.
-REPLY_LANGS = ("tr", "en", "any")
+# The reply language (config.json `language`; an older config's `reply_lang`
+# still reads and is never rewritten). The value is the switch: a code with a
+# pack under hooks/lang/ asks for replies in that language, `any` for the
+# user's own, and only a pack that ships reply `markers` (tr) is judged by the
+# Stop language check. A config naming neither key reads as `tr`, the default
+# every install made before the packs keeps; a first install writes the
+# environment's language instead (`env_language`, bin/tezgah-setup).
+LANG_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "lang")
+_PACKS = []
 
 
-def reply_lang():
-    """The configured reply language, `tr` when unset or not one of REPLY_LANGS."""
+def languages():
+    """The values `language` takes: every shipped pack's code, then `any`."""
+    return tuple(packs()) + ("any",)
+
+
+def language():
+    """The configured reply language, `tr` when unset or not one of `languages()`."""
     cfg = config()
-    value = cfg.get("reply_lang") if isinstance(cfg, dict) else None
-    return value if value in REPLY_LANGS else "tr"
+    for key in ("language", "reply_lang"):
+        value = cfg.get(key) if isinstance(cfg, dict) else None
+        if value in languages():
+            return value
+    return "tr"
+
+
+def spend_cap():
+    """The standing spend cap in USD, `spend_cap` in config.json: a paid run at
+    or under it is a process parameter the agent picks, one over it is asked
+    and waits (the ask policy's never-default class). 0 when unset, negative
+    or not a number, so every spend is asked."""
+    cfg = config()
+    value = cfg.get("spend_cap") if isinstance(cfg, dict) else None
+    ok = isinstance(value, (int, float)) and not isinstance(value, bool)
+    return float(value) if ok and value > 0 else 0.0
+
+
+def env_language(environ=None):
+    """The language a first install asks for: the first of LANGUAGE (its own
+    first `:` entry), LC_ALL, LC_MESSAGES and LANG that is set and not empty -
+    GNU gettext's order - cut to its language code (`tr_TR.UTF-8` -> `tr`), or
+    `en` when that code has no pack (`C`, `POSIX`, a language not shipped)."""
+    environ = os.environ if environ is None else environ
+    for key in ("LANGUAGE", "LC_ALL", "LC_MESSAGES", "LANG"):
+        value = (environ.get(key) or "").split(":")[0]
+        if value:
+            code = value.split(".")[0].split("@")[0].split("_")[0].lower()
+            return code if code in packs() else "en"
+    return "en"
+
+
+def packs():
+    """{code: pack} for every hooks/lang/<code>.json, read once per process.
+
+    A pack is data: one language's half of every detection class (S5's
+    inventory) and the contract's own words (`say`), keyed alike in every pack
+    (tests/test_lang_packs.py). The detectors compile the union of all packs,
+    so a reply or prompt in any shipped language arms the same classes whatever
+    `language` says; a language is added by one file, never a regex edit."""
+    if not _PACKS:
+        found = {}
+        for name in sorted(os.listdir(LANG_DIR)):
+            if name.endswith(".json"):
+                with open(os.path.join(LANG_DIR, name), encoding="utf-8") as fh:
+                    found[name[:-5]] = json.load(fh)
+        _PACKS.append(found)
+    return _PACKS[0]
+
+
+def pack_values(*path):
+    """Each pack's value at `path` (nested keys), in pack order, skipping a
+    pack that lacks it."""
+    for pack in packs().values():
+        node = pack
+        for key in path:
+            node = node.get(key) if isinstance(node, dict) else None
+        if node is not None:
+            yield node
+
+
+def alts(*path):
+    """The fragments every pack lists at `path`, joined into one alternation;
+    `(?!)`, which never matches, when no pack lists one."""
+    return "|".join(f for frags in pack_values(*path) for f in frags) or "(?!)"
+
+
+def say(code=None):
+    """The contract's words for `code` (the configured language by default):
+    the English pack's `say` map with the code's own pack laid over it, so a
+    language with no pack, or `any`, speaks the canonical English tokens."""
+    words = dict(packs().get("en", {}).get("say", {}))
+    words.update(packs().get(code or language(), {}).get("say", {}))
+    return words
 
 
 # Switch latching (plan 051, ADR 018). A hook process that knows its session
@@ -861,6 +944,16 @@ def pony_level():
     return value if value in PONY_LEVELS else "full"
 
 
+# The router's input-side tokens (input plus cache reads and writes) on its
+# newest step, above which the per-turn reminder asks it to push heavy reads
+# through a slice (tezgah_context.context_nudge). Measured on 12,576 omp
+# main-thread steps (orchestrator-flow-audit S2 F1): median 396 k, p90 724 k. The
+# median is the normal working state and would fire on every other turn, so the
+# line sits just under p90: it marks the heaviest tenth of steps and stays rare
+# enough to be read.
+CONTEXT_NUDGE_TOKENS = 700000
+
+
 def tool(name):
     """Stable installed path for a tezgah CLI, else the plugin's own copy, so
     the text injected into a session names something that actually exists."""