@r1z4x/tezgah 1.0.0 → 1.2.0

package/hooks/tezgah_judge.py

@@ -29,15 +29,19 @@ non-interactive shell where `.zshenv` never runs - the export is absent there,
 and the key file is the channel that survives. That is why the file is read
 rather than the environment trusted.
 
-Behind the credential there are two providers. TypeSafe answers in its own
-schema; when no TypeSafe key resolves, the same questions go to an
-OpenAI-compatible chat endpoint - OpenRouter, whose key rides the same two
-channels (`OPENROUTER_API_KEY`, then `~/.config/openrouter/key`) - and its JSON
-reply is mapped back into the shapes the callers already read. That path is a
-fallback, never a peer: it is chosen only when `key()` is empty, it sends the
-same state to a different host, and a reply that does not carry a question's
-answer leaves that question absent, which every caller already reads as no
-judgement.
+Behind the credential there are three providers, asked in this order. First the
+session's own CLI - `omp` or `claude`, the host this process runs under
+(`tp.session_cli()`) - run headless with no tools, so the judgement is paid
+with the session's own credential, OAuth subscription included, and answered by
+the session's own vendor. TypeSafe and OpenRouter are third parties: they are
+asked only as the `fallback` setting allows (`tp.fallback_policy()`): by
+default (`vendor`) only when there is no session CLI at all, with `any` also
+after the session CLI failed, with `none` never. A third party that answers,
+and a fallback that was refused, is never silent: one stderr line, the
+`fallback` field on the result, and the last-use record `tezgah-status
+--judge` prints (`_record`, `last_use`). A reply that does not carry a
+question's answer leaves that question absent, which every caller already reads
+as no judgement.
 
 Kill switch: `judge-off`, honoured by `available()`, so a caller can fall back to
 its own deterministic path without knowing why the judgement is gone. It is the
@@ -64,9 +68,10 @@ FALLBACK_MODEL = "deepseek/deepseek-v4-flash"
 # The contract the fallback asks a chat model to answer in, one shape per
 # question type this repository's callers send (`bin/tezgah-triage` and
 # `hooks/tezgah_skill_pick.py` ask `noul`, `bin/tezgah-docs` and the triage's
-# pair rows ask `choice`). It is prose because a chat endpoint has no schema
-# field to carry it; every field it names is validated on the way back, and a
-# question of any other type reads as unanswered rather than as a guess.
+# pair rows ask `choice`, `bin/tezgah-taste` asks `text` for a learning's
+# readable line). It is prose because a chat endpoint has no schema field to
+# carry it; every field it names is validated on the way back, and a question
+# of any other type reads as unanswered rather than as a guess.
 CHAT_SYSTEM = (
     "You answer questions about the state that follows. Reply with one JSON "
     "object and nothing else: {\"answers\": {\"<id>\": {...}}}, one entry per "
@@ -74,6 +79,7 @@ CHAT_SYSTEM = (
     "noul   -> {\"noul\": <probability 0..1 that the statement is true>}\n"
     "choice -> {\"choice\": \"<the chosen label>\", \"probabilities\": "
     "{\"<label>\": <probability>, ...}, \"confidence\": <0..1>}\n"
+    "text   -> {\"text\": \"<the answer as plain prose>\"}\n"
     "Every probability is a number in 0..1 and the ones you list for one "
     "question sum to 1. Answer every id you were given.")
 
@@ -148,58 +154,139 @@ def fallback_model():
     return os.environ.get("TEZGAH_JUDGE_MODEL", "").strip() or cheap_default()
 
 
-def credential():
-    """`(provider, secret)` for the first channel that resolves, else `(None, None)`.
+# The session CLIs the seam can ask, as the argv between the binary and the
+# prompt: no tools, no session file, no project rules, skills, extensions, MCP
+# servers or settings, and the chat fallback's own contract as the system
+# prompt. Measured 2026-10-07 on a one-word answer: claude went from 37,038
+# prompt tokens ($0.30) to 725 ($0.007); omp from 43,735 written to cache
+# ($0.35) to about 37,500, 36,700 of them read from cache ($0.015) - omp has
+# no flag here that drops the rest.
+SESSION_ARGV = {
+    "omp": ["-p", "--no-tools", "--no-lsp", "--no-session", "--no-rules",
+            "--no-skills", "--no-extensions", "--no-title", "--thinking", "off",
+            "--mode", "json", "--system-prompt", CHAT_SYSTEM],
+    "claude": ["-p", "--tools", "", "--no-session-persistence",
+               "--strict-mcp-config", "--disable-slash-commands",
+               "--setting-sources", "", "--output-format", "json",
+               "--system-prompt", CHAT_SYSTEM],
+}
+NO_SESSION = "no session CLI to ask (OMPCODE/CLAUDECODE unset or the CLI missing)"
+
+
+def providers():
+    """[(provider, secret), ...] in the order `ask()` tries them, or [].
+
+    The session's own CLI first (its `secret` is the binary's path); then the
+    third parties - TypeSafe, then OpenRouter - only as `tp.fallback_policy()`
+    allows: `vendor` (the default) asks them only when no session CLI exists,
+    `any` after the session CLI too, `none` never. TypeSafe still wins over
+    OpenRouter whenever both resolve."""
+    session = tp.session_cli()
+    policy = tp.fallback_policy()
+    out = [(session, getattr(tp, session + "_bin")())] if session else []
+    if policy == "none" or (session and policy == "vendor"):
+        return out
+    return out + [(name, secret) for name, secret in
+                  (("typesafe", key()), ("openrouter", openrouter_key())) if secret]
+
+
+def named(only):
+    """`providers()` for a caller that requires particular ones: every provider
+    in `only` that resolves, in `only`'s order. The `vendor` policy's
+    session-first order does not apply - the caller named the provider, so
+    nothing is a silent fallback - but `none` still keeps every third party
+    out."""
+    session = tp.session_cli()
+    third = tp.fallback_policy() != "none"
+    found = {session: getattr(tp, session + "_bin")()} if session else {}
+    if third:
+        found.update(typesafe=key(), openrouter=openrouter_key())
+    return [(name, found[name]) for name in only if found.get(name)]
+
 
-    TypeSafe wins whenever it resolves, so adding a fallback cannot move a
-    machine that already judges with Jev. The pair is returned rather than the
-    secret alone so that `available()` and `ask()` cannot disagree about which
-    provider a call will use."""
-    secret = key()
-    if secret:
-        return "typesafe", secret
-    secret = openrouter_key()
-    return ("openrouter", secret) if secret else (None, None)
+def credential():
+    """`(provider, secret)` for the first provider `ask()` will try, else
+    `(None, None)` - so `available()` and `ask()` cannot disagree."""
+    found = providers()
+    return found[0] if found else (None, None)
 
 
 def available():
-    """True when a judgement can be asked for: a key resolves and no kill switch."""
+    """True when a judgement can be asked for: a provider resolves and no kill switch."""
     return bool(credential()[1]) and not tp.off("judge-off")
 
 
-def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None):
+def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
+        only=None):
     """One batched call over `questions`; `{"answers", "usage", "latency_ms",
-    "model", "provider"}`.
+    "model", "provider", "fallback"}`.
 
     `questions` is the API's own map - id -> `{type, instructions, criteria}` -
     so a caller that needs a per-line or per-state pass sends every question in
     one request and pays for the state once. The reply's usage is carried back
     because a caller quotes what the judgement cost, and the model and provider
-    beside it because the fallback answers with a different one than the caller
-    named. The model is the one the reply names, else the one asked for: an alias
-    such as `jev-latest` hides a silent upgrade the reply's own field shows.
-
-    One request per call in the normal case, to whichever provider the credential
-    resolves - TypeSafe first, the chat fallback only when `key()` is empty. A
-    transient failure - a timeout, a connection error, a 5xx - gets one more
-    attempt, because a live measurement saw 2 of 50 calls lost that way while the
-    same cells answered on retry; a 4xx (a refused credential, a rejected body)
-    and a reply that parsed malformed are never retried, since the second request
-    would fail identically and only a call that already worked must not be billed
-    twice. The ceiling is one, so the worst case is one duplicated request on a
-    call that answered nothing anyway. A caller on a hook's budget passes
-    `attempts=1` and a `deadline` in seconds: urllib's `timeout` bounds one
-    socket operation, not the call, so only the deadline bounds its wall time.
-
-    A call that ends on a 401, 402 or 5xx marks the provider down for
-    `DOWN_FOR` seconds, and until then every call returns None without a
-    request (`_down`), so a dead credential is not re-paid on every prompt.
-
-    Total by design: no key, an unreadable state, a refused request, a timeout,
-    a reply that is not the documented shape - all `None`, never an exception."""
-    provider, secret = credential()
-    if not secret:
+    beside it because the provider that answers is not the caller's choice. The
+    model is the one the reply names, else the one asked for: an alias such as
+    `jev-latest` hides a silent upgrade the reply's own field shows. `model` is
+    TypeSafe's alone; the session CLI and the chat fallback ignore it.
+
+    The providers are tried in `providers()` order - the session's own CLI
+    first - and the next one is asked only when the previous failed. `fallback`
+    is None when the session answered, else why someone else did; that answer,
+    and a refusal, is recorded (`_record`) and said on stderr. A transient
+    failure - a timeout, a connection error, a 5xx - gets one more attempt on
+    the same provider, because a live measurement saw 2 of 50 calls lost that
+    way while the same cells answered on retry; a 4xx, a CLI that exited
+    non-zero and a reply that parsed malformed are never retried, since the
+    second request would fail identically and only a call that already worked
+    must not be billed twice. A caller on a hook's budget passes `attempts=1`
+    and a `deadline` in seconds: urllib's `timeout` bounds one socket operation,
+    not the call, so only the deadline bounds its wall time.
+
+    A call that ends on a 401, 402, 5xx or a session CLI's non-zero exit marks
+    that provider down for `DOWN_FOR` seconds, and until then it is skipped
+    without a request (`_down`), so a dead credential is not re-paid on every
+    prompt.
+
+    Total by design: no provider, an unreadable state, a refused request, a
+    timeout, a reply that is not the documented shape - all `None`, never an
+    exception.
+
+    `only` names the providers the caller requires (`named()`); a caller whose
+    decision must come from a typed model passes `only=("typesafe",)` and gets
+    None rather than another provider's answer."""
+    tried = providers() if only is None else named(only)
+    if any(isinstance(q, dict) and q.get("type") == "text" for q in questions.values()):
+        # TypeSafe answers no prose: left out up front, not logged as a failure
+        tried = [(p, s) for p, s in tried if p != "typesafe"]
+    if not tried or not tried[0][1]:
         return None
+    stop = None if deadline is None else time.monotonic() + deadline
+    failed = []
+    for provider, secret in tried:
+        result, why = _ask_one(provider, secret, state, questions, model,
+                               timeout, attempts, stop)
+        if result is None:
+            failed.append("%s: %s" % (provider, why))
+            continue
+        # a provider the caller named is not a stand-in: no note, unless an
+        # earlier one it named failed
+        note = "; ".join(failed) or (None if only is not None or provider in SESSION_ARGV
+                                     else NO_SESSION)
+        result["fallback"] = note
+        _record(result["provider"], result["model"], fallback=note)
+        return result
+    refused = None
+    session = tp.session_cli()
+    if session and (key() or openrouter_key()):
+        refused = ("fallback=%s refuses a third-party judge after %s failed"
+                   % (tp.fallback_policy(), session))
+    _record(None, None, failed="; ".join(failed), refused=refused)
+    return None
+
+
+def _ask_one(provider, secret, state, questions, model, timeout, attempts, stop):
+    """One provider's attempts: `(result, None)`, or `(None, why)`."""
     try:
         if provider == "typesafe":
             used, url = model, endpoint()
@@ -208,34 +295,43 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None)
 
             def send():
                 return _request(secret, body, timeout)
-        else:
+        elif provider == "openrouter":
             used, url = fallback_model(), openrouter_url()
             body = json.dumps(_chat_body(state, questions, used)).encode()
 
             def send():
                 return _chat_request(secret, body, timeout, questions)
+        else:
+            used, url = None, secret
+            prompt = json.dumps({"state": state, "questions": questions})
+
+            def send():
+                left = timeout if stop is None else max(0.1, min(timeout, stop - time.monotonic()))
+                return _session_request(provider, secret, prompt, left, questions)
         marker = _down_marker(provider, url, secret)
         if _down(marker):
-            return None
-    except Exception:
-        return None
-    stop = None if deadline is None else time.monotonic() + deadline
+            return None, "marked down for %ds after a refusal" % DOWN_FOR
+    except Exception as exc:
+        return None, _why(exc)
     for attempt in range(attempts):
+        if stop is not None and time.monotonic() >= stop:
+            return None, "deadline passed"
         try:
             result = _bounded(send, stop)
         except Exception as exc:
             late = stop is not None and time.monotonic() >= stop
             if attempt + 1 >= attempts or late or not _transient(exc):
                 code = getattr(exc, "code", None)
-                if code in (401, 402) or (isinstance(code, int) and code >= 500):
+                if code in (401, 402) or (isinstance(code, int) and code >= 500) \
+                        or isinstance(exc, SessionFailed):
                     _mark_down(marker)
-                return None
+                return None, _why(exc)
             continue
         named = result.get("model")
-        result["model"] = named if isinstance(named, str) and named else used
+        result["model"] = named if isinstance(named, str) and named else used or "-"
         result["provider"] = provider
-        return result
-    return None
+        return result, None
+    return None, "no attempt"
 
 
 def _answer(result, id):
@@ -260,6 +356,14 @@ def choice(result, id):
     return value if isinstance(value, str) else None
 
 
+def text(result, id):
+    """The prose a `text` question came back with, stripped, or None. Only a
+    generative provider answers one; TypeSafe is never sent one."""
+    answer = _answer(result, id)
+    value = answer.get("text") if answer else None
+    return (value.strip() or None) if isinstance(value, str) else None
+
+
 def noul(result, id, option=None):
     """The probability a Noul question came back with, or None.
 
@@ -380,17 +484,20 @@ def _chat_answers(parsed, questions):
 def _clean_answer(raw, kind):
     """One answer in the shape its question type names, or None.
 
-    The two shapes this repository's callers send - `noul`, the probability a
-    statement holds, and `choice`, the label taken - map onto exactly what
-    `choice()`/`noul()` already read, so a fallback answer and a TypeSafe answer
-    are indistinguishable downstream. Any other type reads as no answer rather
-    than as a guess. Probabilities are kept only when they are numbers, because
-    `noul()` does the arithmetic on them."""
+    The shapes this repository's callers send - `noul`, the probability a
+    statement holds, `choice`, the label taken, and `text`, prose - map onto
+    exactly what `choice()`/`noul()`/`text()` already read, so a fallback answer
+    and a TypeSafe answer are indistinguishable downstream. Any other type reads
+    as no answer rather than as a guess. Probabilities are kept only when they
+    are numbers, because `noul()` does the arithmetic on them."""
     if not isinstance(raw, dict):
         return None
     if kind == "noul":
         value = raw.get("noul")
         return {"noul": float(value)} if isinstance(value, (int, float)) else None
+    if kind == "text":
+        value = raw.get("text")
+        return {"text": value} if isinstance(value, str) and value.strip() else None
     if kind != "choice":
         return None
     if not isinstance(raw.get("choice"), str):
@@ -485,3 +592,118 @@ def _bounded(send, stop):
     if "err" in box:
         raise box["err"]
     return box["ok"]
+
+
+class SessionFailed(RuntimeError):
+    """The session CLI exited non-zero or printed no answer: a logged-out CLI or
+    an exhausted plan fails the same way on the next call, so it is never
+    retried and it marks the provider down like a 401."""
+
+
+def _session_request(name, exe, prompt, timeout, questions):
+    """One headless run of the session's own CLI, mapped into the documented
+    return. It starts in an empty temp dir with stdin closed - the answer rests
+    on the prompt alone, and a CLI that wants a login exits instead of waiting -
+    with TEZGAH_NESTED set so tezgah's own hooks stand down in the child. The
+    answer is read from the CLI's JSON output: the model and the usage it names
+    are what the call really used."""
+    import shutil
+    import subprocess
+    import tempfile
+    cwd = tempfile.mkdtemp(prefix="tezgah-judge-")
+    started = time.monotonic()
+    try:
+        proc = subprocess.run([exe] + SESSION_ARGV[name] + [prompt], cwd=cwd,
+                              stdin=subprocess.DEVNULL, capture_output=True,
+                              text=True, timeout=timeout,
+                              env=dict(os.environ, TEZGAH_NESTED="1"))
+    finally:
+        shutil.rmtree(cwd, ignore_errors=True)
+    if proc.returncode:
+        raise SessionFailed("%s exit %d: %s" % (name, proc.returncode,
+                                                 (proc.stderr or proc.stdout).strip()[-200:]))
+    if name == "claude":
+        data = json.loads(proc.stdout)
+        if data.get("is_error"):
+            raise SessionFailed("claude: %s" % str(data.get("result"))[:200])
+        text = data.get("result") or ""
+        model = next(iter(data.get("modelUsage") or {}), None)
+        usage = data.get("usage") or {}
+        tokens_in = sum(int(usage.get(k) or 0) for k in (
+            "input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"))
+        tokens_out = int(usage.get("output_tokens") or 0)
+    else:
+        message = None
+        for line in proc.stdout.splitlines():
+            try:
+                event = json.loads(line)
+            except ValueError:
+                continue
+            if isinstance(event, dict) and event.get("type") == "message_end" \
+                    and (event.get("message") or {}).get("role") == "assistant":
+                message = event["message"]
+        if message is None:
+            raise SessionFailed("omp printed no answer")
+        text = "".join(part.get("text") or "" for part in message.get("content") or []
+                       if isinstance(part, dict) and part.get("type") == "text")
+        model = "/".join(str(message[k]) for k in ("provider", "model") if message.get(k))
+        usage = message.get("usage") or {}
+        tokens_in = sum(int(usage.get(k) or 0) for k in ("input", "cacheRead", "cacheWrite"))
+        tokens_out = int(usage.get("output") or 0)
+    # A chat model may fence its object; the outermost braces are the object.
+    answers = _chat_answers(json.loads(text[text.find("{"):text.rfind("}") + 1]), questions)
+    return {"answers": answers,
+            "usage": {"input_tokens": tokens_in, "output_tokens": tokens_out},
+            "latency_ms": int((time.monotonic() - started) * 1000),
+            "model": model}
+
+
+def _why(exc):
+    """A failure as one short reason: the HTTP code, else the exception's text."""
+    code = getattr(exc, "code", None)
+    if isinstance(code, int):
+        return "http %d" % code
+    return ("%s: %s" % (type(exc).__name__, exc))[:200]
+
+
+# The last judgement's record, one file: what answered, and why it was not the
+# session's own CLI when it was not. `tezgah-status --judge` prints it.
+LAST = "judge-last.json"
+
+
+def _say(line):
+    try:
+        import sys
+        sys.stderr.write("tezgah-judge: %s\n" % line)
+    except Exception:
+        pass
+
+
+def _record(provider, model, fallback=None, failed=None, refused=None):
+    """Write the last-use record and say any fallback or refusal on stderr.
+    Best effort like every hook write: a cache that cannot be written loses
+    the record, never the answer."""
+    if fallback and provider:
+        _say("answered by %s/%s - %s" % (provider, model, fallback))
+    elif refused:
+        _say("no judgement: %s (%s)" % (refused, failed))
+    row = {"at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "provider": provider,
+           "model": model, "fallback": fallback, "failed": failed or None,
+           "refused": refused, "policy": tp.fallback_policy()}
+    try:
+        path = os.path.join(tp.cache_dir(), LAST)
+        with open(path + ".tmp", "w", encoding="utf-8") as fh:
+            json.dump(row, fh)
+        os.replace(path + ".tmp", path)
+    except (OSError, TypeError, ValueError):
+        pass
+
+
+def last_use():
+    """The last judgement's record (see `_record`), or None when none was kept."""
+    try:
+        with open(os.path.join(tp.cache_dir(), LAST), encoding="utf-8") as fh:
+            row = json.load(fh)
+    except (OSError, ValueError):
+        return None
+    return row if isinstance(row, dict) else None

package/hooks/tezgah_models.py

@@ -105,7 +105,10 @@ SLOTS = {
                  "any": ("anthropic/claude-opus-5.5", "high")},
 }
 AGENT_SLOT = {"tezgah-cheap": "cheap", "tezgah-standard": "standard",
-              "tezgah-frontier": "frontier", "tezgah-reviewer": "frontier"}
+              "tezgah-frontier": "frontier", "tezgah-reviewer": "frontier",
+              "tezgah-tester": "cheap", "tezgah-docs": "standard",
+              "tezgah-security": "frontier", "tezgah-ui": "standard",
+              "tezgah-researcher": "frontier"}
 # The OpenRouter id of every model the table names, with its list price per 1M
 # tokens (input, output) on READ_ON, so `refresh` can say what moved.
 SNAPSHOT = {"anthropic/claude-opus-5.5": (4.0, 20.0),
@@ -280,17 +283,50 @@ def _selector(family, slot):
     return FAMILY_PREFIX[family] + model + (":" + effort if effort else "")
 
 
+# The OpenRouter id of a host-column model: the route the default `vendor`
+# fallback keeps - the same model through another provider and account - when
+# the `any` column names another vendor's model on that slot.
+OPENROUTER_ID = {"claude-opus-5-5": "anthropic/claude-opus-5.5",
+                 "zai/glm-5.3-flash": "z-ai/glm-5.3-flash"}
+VENDOR_STEMS = (("claude", "anthropic"), ("glm", "zai"), ("gpt", "openai"),
+                ("deepseek", "deepseek"))
+
+
+def _vendor(model):
+    """The vendor a table id's model comes from, or the id itself."""
+    name = model.split("/")[-1].lower()
+    return next((v for stem, v in VENDOR_STEMS if name.startswith(stem)), name)
+
+
 def _chain(mode, slot, funded, agent=""):
     """[selector, ...] for one slot: the mode's family first - the user picked it,
-    so it leads even when its credential is not visible from here - then every
-    other funded family, each on that slot's own row.
-
-    The fallbacks are rotated by the agent's own name (`agent`): identical chains
+    so it leads even when its credential is not visible from here - then the
+    fallbacks the `fallback` setting allows (`tp.fallback_policy()`).
+
+    `vendor`, the default: only the mode's own model vendor - the same model
+    through OpenRouter, or another funded column whose model is that vendor's -
+    because a session on Opus that ran its subagents on GPT and GLM after one
+    Anthropic 429 is the failure the owner reported (2026-10-07). `none`: the
+    mode's selector alone. `any`: every other funded family, each on that
+    slot's own row, rotated by the agent's own name (`agent`): identical chains
     for every agent meant one family's 429 moved all of them onto the same next
     family in the same moment, which is the cascade the chain exists to break
     (independent review, 2026-10-01). The rotation is deterministic - the same
     agent resolves the same order on every machine - so nothing here is random."""
+    policy = tp.fallback_policy()
     families = [mode] + [f for f in OMP_FAMILIES if f != mode and f in funded]
+    if policy == "none":
+        return [_selector(mode, slot)]
+    if policy == "vendor":
+        model, effort = SLOTS[slot][mode]
+        out = [_selector(mode, slot)]
+        for family in families[1:]:
+            if _vendor(SLOTS[slot][family][0]) == _vendor(model):
+                out.append(_selector(family, slot))
+            elif family == "any" and model in OPENROUTER_ID:
+                out.append("openrouter/" + OPENROUTER_ID[model]
+                           + (":" + effort if effort else ""))
+        return list(dict.fromkeys(out))
     tail = families[1:]
     if agent and len(tail) > 1:
         shift = sum(ord(c) for c in agent) % len(tail)