@r1z4x/tezgah 1.4.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +120 -1
- package/MANIFEST +0 -1
- package/README.md +6 -4
- package/bin/tezgah-docs +3 -3
- package/bin/tezgah-route +12 -6
- package/bin/tezgah-setup +67 -24
- package/bin/tezgah-status +3 -1
- package/bin/tezgah-task +132 -39
- package/bin/tezgah-taste +49 -31
- package/bin/tezgah-triage +12 -7
- package/docs/architecture.md +1 -1
- package/docs/clarity-baseline.json +7 -7
- package/docs/contract.md +21 -11
- package/docs/environment.md +3 -1
- package/docs/evidence.md +4 -3
- package/docs/glossary.md +1 -1
- package/docs/hosts.md +9 -0
- package/docs/judge.md +95 -27
- package/docs/layers.md +1 -1
- package/docs/models.md +17 -7
- package/docs/operations.md +23 -13
- package/docs/skills.md +1 -1
- package/docs/testing.md +1 -1
- package/docs/vision.md +10 -5
- package/hooks/tezgah_agents.py +149 -105
- package/hooks/tezgah_context.py +42 -7
- package/hooks/tezgah_judge.py +252 -79
- package/hooks/tezgah_models.py +50 -3
- package/hooks/tezgah_paths.py +180 -27
- package/hooks/tezgah_policy.py +41 -10
- package/hooks/tezgah_replay.py +26 -15
- package/hooks/tezgah_task.py +160 -46
- package/hooks/tezgah_taste.py +2 -2
- package/hooks/tezgah_taste_ledger.py +13 -4
- package/hosts/codex/hook.py +5 -0
- package/package.json +1 -2
- package/skills/harness/SKILL.md +4 -0
- package/skills/plan-add/SKILL.md +23 -7
- package/skills/plan-add/render_table.py +3 -2
- package/skills/plan-status/SKILL.md +12 -4
- package/skills/plan-sync/SKILL.md +19 -9
- package/skills/tezgah-contract/SKILL.md +29 -14
- package/agents/tezgah-reviewer.md +0 -70
package/hooks/tezgah_judge.py
CHANGED
|
@@ -10,8 +10,10 @@ attempt and no more; a 4xx and a malformed reply never do (`ask()` below). A hoo
|
|
|
10
10
|
may import this module, and a hook that raises takes a session down, so every
|
|
11
11
|
failure is a `None` instead.
|
|
12
12
|
|
|
13
|
-
Privacy, plainly: the state and the questions leave this machine for
|
|
14
|
-
|
|
13
|
+
Privacy, plainly: the state and the questions leave this machine for the host
|
|
14
|
+
of whichever provider answers - the session's vendor, or the chosen Jev
|
|
15
|
+
carrier's (`api.typesafe.ai`, `openrouter.ai`, `api.cloudflare.com`, or the
|
|
16
|
+
`JEV_API_BASE_URL` a user set) - for the triage the state IS the screen's own text (an admin
|
|
15
17
|
table or a mobile view as captured, plus the `--select` task sentence), so a
|
|
16
18
|
screen carrying personal data is read by a third party. Nothing else leaves -
|
|
17
19
|
no session id, no workspace path, no environment - and the seam does not
|
|
@@ -23,17 +25,20 @@ prompt-path skill hint sends the user's prompt redacted and cut at 2,000
|
|
|
23
25
|
characters, because a brief or a prompt can quote a token and neither is a
|
|
24
26
|
state a person chose to send.
|
|
25
27
|
|
|
26
|
-
|
|
27
|
-
an interactive shell, but a hook or a bin tool started
|
|
28
|
-
non-interactive shell where `.zshenv` never runs - the
|
|
29
|
-
and the key file is the channel that survives
|
|
30
|
-
|
|
28
|
+
Each credential has a file channel beside its env var on purpose. `~/.zshenv`
|
|
29
|
+
exports the env var for an interactive shell, but a hook or a bin tool started
|
|
30
|
+
by a host runs in a non-interactive shell where `.zshenv` never runs - the
|
|
31
|
+
export is absent there, and the key file is the channel that survives (TypeSafe
|
|
32
|
+
also reads the active omp login). The resolvers live in `tezgah_paths`
|
|
33
|
+
(`typesafe_key`, `jev_carriers`), so `have_judge_key()` asks the same ones.
|
|
31
34
|
|
|
32
|
-
Behind the credential there are three
|
|
33
|
-
session's own CLI - `omp` or `
|
|
35
|
+
Behind the credential there are three kinds of provider, asked in this order. First the
|
|
36
|
+
session's own CLI - `omp`, `claude`, `opencode`, `cursor` or `codex`, the host
|
|
37
|
+
this process runs under or the one `TEZGAH_JUDGE_CLI`/`judge_cli` picks
|
|
34
38
|
(`tp.session_cli()`) - run headless with no tools, so the judgement is paid
|
|
35
39
|
with the session's own credential, OAuth subscription included, and answered by
|
|
36
|
-
the session's own vendor.
|
|
40
|
+
the session's own vendor. The Jev carriers (`JEV_CARRIERS`, as chosen by
|
|
41
|
+
`tp.jev_choice()`) and the OpenRouter chat fallback are third parties: they are
|
|
37
42
|
asked only as the `fallback` setting allows (`tp.fallback_policy()`): by
|
|
38
43
|
default (`vendor`) only when there is no session CLI at all, with `any` also
|
|
39
44
|
after the session CLI failed, with `none` never. A third party that answers,
|
|
@@ -58,13 +63,20 @@ import tezgah_paths as tp
|
|
|
58
63
|
import tezgah_store
|
|
59
64
|
|
|
60
65
|
URL = "https://api.typesafe.ai/v1/systemone"
|
|
61
|
-
KEY_FILE = "~/.config/typesafe/key"
|
|
62
66
|
MODEL = "jev-latest"
|
|
67
|
+
# The other Jev carriers (`JEV_CARRIERS`): OpenRouter's System One endpoint,
|
|
68
|
+
# which takes TypeSafe's body and maps a bare `jev-*` id onto `typesafe/` itself
|
|
69
|
+
# (`typesafe/jev-latest` is no model there: a live 400 on 2026-10-09); and
|
|
70
|
+
# Cloudflare Workers AI, whose run route wraps the state and questions in
|
|
71
|
+
# `input` and serves the one alias `typesafe/jev`. A System One-compatible URL
|
|
72
|
+
# is the user's own (`JEV_API_BASE_URL`), so it has no constant here.
|
|
73
|
+
JEV_OPENROUTER_URL = "https://openrouter.ai/api/v1/systemone"
|
|
74
|
+
CLOUDFLARE_URL = "https://api.cloudflare.com/client/v4"
|
|
75
|
+
CLOUDFLARE_MODEL = "typesafe/jev"
|
|
63
76
|
# The fallback provider: an OpenAI-compatible chat endpoint, its own key
|
|
64
77
|
# channels, and the cheap judge model this repository already quotes in its
|
|
65
78
|
# measured arm rows (`openrouter/deepseek/deepseek-v4-flash`).
|
|
66
79
|
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
|
67
|
-
OPENROUTER_KEY_FILE = "~/.config/openrouter/key"
|
|
68
80
|
FALLBACK_MODEL = "deepseek/deepseek-v4-flash"
|
|
69
81
|
# The contract the fallback asks a chat model to answer in, one shape per
|
|
70
82
|
# question type this repository's callers send (`bin/tezgah-triage` and
|
|
@@ -110,19 +122,11 @@ def endpoint():
|
|
|
110
122
|
|
|
111
123
|
|
|
112
124
|
def key():
|
|
113
|
-
"""
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
value = os.environ.get("TYPESAFE_API_KEY", "").strip()
|
|
119
|
-
if value:
|
|
120
|
-
return value
|
|
121
|
-
try:
|
|
122
|
-
with open(os.path.expanduser(KEY_FILE), encoding="utf-8") as fh:
|
|
123
|
-
return fh.read().strip() or None
|
|
124
|
-
except OSError:
|
|
125
|
-
return None
|
|
125
|
+
"""TypeSafe's credential: TYPESAFE_API_KEY, else `~/.config/typesafe/key`,
|
|
126
|
+
else the active omp login, else None (`tp.typesafe_key`). Stripped: a
|
|
127
|
+
newline inside an Authorization header is a header-injection attempt to
|
|
128
|
+
the HTTP layer, not a typo it will forgive."""
|
|
129
|
+
return tp.typesafe_key()
|
|
126
130
|
|
|
127
131
|
|
|
128
132
|
def openrouter_url():
|
|
@@ -131,19 +135,37 @@ def openrouter_url():
|
|
|
131
135
|
|
|
132
136
|
|
|
133
137
|
def openrouter_key():
|
|
134
|
-
"""
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
138
|
+
"""OpenRouter's credential, for its System One carrier and the chat
|
|
139
|
+
fallback alike: OPENROUTER_API_KEY, else `~/.config/openrouter/key`, else
|
|
140
|
+
None (`tp.openrouter_key`), stripped for the same reason as `key()`."""
|
|
141
|
+
return tp.openrouter_key()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _jev_call(provider, model, state, questions):
|
|
145
|
+
"""`(model used, url, body)` for one Jev carrier: TypeSafe's own body
|
|
146
|
+
(`state`, `model`, `questions`) everywhere but Cloudflare, whose run route
|
|
147
|
+
takes its one alias and the pair under `input`. Clef and Clef-flash, its own
|
|
148
|
+
System One models, have a route each and take the bare body with `model`
|
|
149
|
+
as the selector. Every URL routes through `override()`, so a plain-http one
|
|
150
|
+
off this machine raises here."""
|
|
151
|
+
if provider in ("jev-cloudflare", "clef", "clef-flash"):
|
|
152
|
+
account = urllib.parse.quote(tp.cloudflare_account() or "", safe="")
|
|
153
|
+
url = override("TEZGAH_CLOUDFLARE_URL", CLOUDFLARE_URL) + \
|
|
154
|
+
"/accounts/%s/ai/run" % account
|
|
155
|
+
if provider != "jev-cloudflare":
|
|
156
|
+
return provider, url + "/@cf/cloudflare/" + provider, {
|
|
157
|
+
"model": provider, "state": state, "questions": questions}
|
|
158
|
+
return CLOUDFLARE_MODEL, url, {"model": CLOUDFLARE_MODEL, "input": {
|
|
159
|
+
"state": state, "questions": questions}}
|
|
160
|
+
if provider == "jev-openrouter":
|
|
161
|
+
used = model
|
|
162
|
+
url = override("TEZGAH_JEV_OPENROUTER_URL", JEV_OPENROUTER_URL)
|
|
163
|
+
elif provider == "jev-compatible":
|
|
164
|
+
used = os.environ.get("TEZGAH_JEV_MODEL", "").strip() or model
|
|
165
|
+
url = override("JEV_API_BASE_URL", "")
|
|
166
|
+
else:
|
|
167
|
+
used, url = model, endpoint()
|
|
168
|
+
return used, url, {"state": state, "model": used, "questions": questions}
|
|
147
169
|
|
|
148
170
|
|
|
149
171
|
def fallback_model():
|
|
@@ -170,25 +192,56 @@ SESSION_ARGV = {
|
|
|
170
192
|
"--strict-mcp-config", "--disable-slash-commands",
|
|
171
193
|
"--setting-sources", "", "--output-format", "json",
|
|
172
194
|
"--system-prompt", CHAT_SYSTEM],
|
|
195
|
+
# None of the next three takes a system prompt or drops its tools, so
|
|
196
|
+
# `_session_request` puts the system prompt in front of the prompt text and
|
|
197
|
+
# the empty temp cwd is what keeps tools from reaching a repository. Each
|
|
198
|
+
# set is from the CLI's own `--help` (checked 2026-10-09). opencode 1.18.31
|
|
199
|
+
# `run`: `--pure` "run without external plugins", `--format json` "raw JSON
|
|
200
|
+
# events"; it has no flag that skips the session record.
|
|
201
|
+
"opencode": ["run", "--pure", "--format", "json"],
|
|
202
|
+
# cursor-agent 2026.10.01: `-p` print, `--output-format json` (only with
|
|
203
|
+
# --print), `--mode ask` "Q&A style ... (read-only)", `--trust` "Trust the
|
|
204
|
+
# current workspace without prompting" - the temp cwd is never trusted.
|
|
205
|
+
"cursor": ["-p", "--output-format", "json", "--mode", "ask", "--trust"],
|
|
206
|
+
# codex-cli 0.153.4 `exec`: `--json` "Print events to stdout as JSONL",
|
|
207
|
+
# `--ephemeral` "without persisting session files", `--skip-git-repo-check`
|
|
208
|
+
# (the temp cwd is no repository), `--ignore-rules` (no execpolicy rules),
|
|
209
|
+
# `-s read-only`, `--color never`: consult's argv (`tp.CONSULT_CLIS`) plus
|
|
210
|
+
# `--json` and `--ignore-rules`.
|
|
211
|
+
"codex": ["exec", "--json", "--ephemeral", "--skip-git-repo-check",
|
|
212
|
+
"--ignore-rules", "-s", "read-only", "--color", "never"],
|
|
213
|
+
}
|
|
214
|
+
# What the CLI's environment adds to the call. `opencode run` has no flag that
|
|
215
|
+
# drops its tools, and a probe on 2026-10-09 saw it run a bash command the
|
|
216
|
+
# prompt asked for; the judge's state can be a screen's untrusted text, so the
|
|
217
|
+
# run gets an inline config (`OPENCODE_CONFIG_CONTENT`) that denies every tool -
|
|
218
|
+
# the same probe with it wrote nothing. cursor-agent's `--mode ask` refused the
|
|
219
|
+
# same shell call, and codex's `-s read-only` sandbox has no network. Inline
|
|
220
|
+
# config overrides the user's files only key by key, and an agent's own
|
|
221
|
+
# `permission` wins over the top-level one, so the deny is repeated for the two
|
|
222
|
+
# built-in agents `run` uses.
|
|
223
|
+
_DENY_ALL = {"*": "deny", "bash": "deny", "edit": "deny", "webfetch": "deny",
|
|
224
|
+
"external_directory": "deny"}
|
|
225
|
+
SESSION_ENV = {
|
|
226
|
+
"opencode": {"OPENCODE_CONFIG_CONTENT": json.dumps({
|
|
227
|
+
"permission": _DENY_ALL, "tools": {"*": False},
|
|
228
|
+
"agent": {name: {"permission": _DENY_ALL, "tools": {"*": False}}
|
|
229
|
+
for name in ("build", "plan")}})},
|
|
173
230
|
}
|
|
174
|
-
NO_SESSION = "no session CLI to ask (
|
|
231
|
+
NO_SESSION = ("no session CLI to ask (no omp, Claude Code, opencode, Cursor or Codex "
|
|
232
|
+
"session marker or TEZGAH_JUDGE_CLI/judge_cli pick, or that CLI missing)")
|
|
175
233
|
|
|
176
234
|
|
|
177
235
|
def providers():
|
|
178
236
|
"""[(provider, secret), ...] in the order `ask()` tries them, or [].
|
|
179
237
|
|
|
180
238
|
The session's own CLI first (its `secret` is the binary's path); then the
|
|
181
|
-
third parties -
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
out = [(session, getattr(tp, session + "_bin")())] if session else []
|
|
188
|
-
if policy == "none" or (session and policy == "vendor"):
|
|
189
|
-
return out
|
|
190
|
-
return out + [(name, secret) for name, secret in
|
|
191
|
-
(("typesafe", key()), ("openrouter", openrouter_key())) if secret]
|
|
239
|
+
third parties - the chosen Jev carriers (`tp.jev_carriers()`), then the
|
|
240
|
+
OpenRouter chat fallback - only as `tp.fallback_policy()` allows: `vendor`
|
|
241
|
+
(the default) asks them only when no session CLI exists, `any` after the
|
|
242
|
+
session CLI too, `none` never. A Jev carrier still wins over the chat
|
|
243
|
+
model whenever both resolve."""
|
|
244
|
+
return [(p, s) for p, s in _plan() if s]
|
|
192
245
|
|
|
193
246
|
|
|
194
247
|
def named(only):
|
|
@@ -196,13 +249,67 @@ def named(only):
|
|
|
196
249
|
in `only` that resolves, in `only`'s order. The `vendor` policy's
|
|
197
250
|
session-first order does not apply - the caller named the provider, so
|
|
198
251
|
nothing is a silent fallback - but `none` still keeps every third party
|
|
199
|
-
out.
|
|
252
|
+
out. The name `jev` stands for every chosen Jev carrier that resolves
|
|
253
|
+
(`tp.jev_carriers()`, in the chosen order), so a caller that needs a typed
|
|
254
|
+
answer names the model family, not one carrier's address."""
|
|
255
|
+
return [(p, s) for p, s in _plan(only) if s]
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _plan(only=None):
|
|
259
|
+
"""`providers()` (`only` None) or `named(only)`, with one difference: a
|
|
260
|
+
forced Jev carrier whose credential is missing stays in as `(carrier,
|
|
261
|
+
None)`, so `ask()` records why no Jev carrier was asked instead of
|
|
262
|
+
silently asking another one."""
|
|
200
263
|
session = tp.session_cli()
|
|
201
|
-
|
|
264
|
+
policy = tp.fallback_policy()
|
|
265
|
+
jev = tp.jev_carriers() if policy != "none" else []
|
|
266
|
+
if only is None:
|
|
267
|
+
out = [(session, getattr(tp, session + "_bin")())] if session else []
|
|
268
|
+
if policy == "none" or (session and policy == "vendor"):
|
|
269
|
+
return out
|
|
270
|
+
return out + jev + [("openrouter", s) for s in (openrouter_key(),) if s]
|
|
202
271
|
found = {session: getattr(tp, session + "_bin")()} if session else {}
|
|
203
|
-
if
|
|
204
|
-
found.update(
|
|
205
|
-
|
|
272
|
+
if policy != "none":
|
|
273
|
+
found.update(jev, openrouter=openrouter_key())
|
|
274
|
+
out = []
|
|
275
|
+
for name in only:
|
|
276
|
+
for one, secret in (jev if name == "jev" else [(name, found.get(name))]):
|
|
277
|
+
if (secret or one in dict(jev)) and (one, secret) not in out:
|
|
278
|
+
out.append((one, secret))
|
|
279
|
+
return out
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
# Every carrier that answers with a typed System One model - Jev itself, or
|
|
283
|
+
# Cloudflare's Clef family, which takes the same request and answers in the
|
|
284
|
+
# same shape - as opposed to a chat model asked to imitate it: TypeSafe direct,
|
|
285
|
+
# OpenRouter's System One endpoint, Cloudflare Workers AI, any System
|
|
286
|
+
# One-compatible URL, Clef and Clef-flash. A decision a caller requires to be
|
|
287
|
+
# typed checks `is_jev(result["provider"])`.
|
|
288
|
+
JEV_CARRIERS = ("typesafe", "jev-openrouter", "jev-cloudflare", "jev-compatible",
|
|
289
|
+
"clef", "clef-flash")
|
|
290
|
+
# Dollars per 1M input tokens, output free on every one: Jev's listed price on
|
|
291
|
+
# each of its carriers, Clef's and Clef-flash's from their Workers AI pages
|
|
292
|
+
# (2026-10-09).
|
|
293
|
+
JEV_INPUT_PRICE = 0.042
|
|
294
|
+
CLEF_INPUT_PRICE = {"clef": 0.24, "clef-flash": 0.09}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def is_jev(provider):
|
|
298
|
+
"""True when `provider` is a typed System One carrier (`JEV_CARRIERS`)."""
|
|
299
|
+
return provider in JEV_CARRIERS
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def input_price(provider):
|
|
303
|
+
"""The carrier's input price per 1M tokens, None for anything not typed."""
|
|
304
|
+
return CLEF_INPUT_PRICE.get(provider, JEV_INPUT_PRICE) if is_jev(provider) else None
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def jev_summary():
|
|
308
|
+
"""One line naming the Jev choice and the carriers it resolves to now, for
|
|
309
|
+
`tezgah-status --judge` and `tezgah-setup --jev`: a forced carrier without
|
|
310
|
+
its credential is named as such, never hidden."""
|
|
311
|
+
names = ", ".join(p if s else p + " (no credential)" for p, s in tp.jev_carriers())
|
|
312
|
+
return "jev=%s: %s" % (tp.jev_choice(), names or "no carrier resolves")
|
|
206
313
|
|
|
207
314
|
|
|
208
315
|
def credential():
|
|
@@ -229,7 +336,8 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
229
336
|
beside it because the provider that answers is not the caller's choice. The
|
|
230
337
|
model is the one the reply names, else the one asked for: an alias such as
|
|
231
338
|
`jev-latest` hides a silent upgrade the reply's own field shows. `model` is
|
|
232
|
-
|
|
339
|
+
the Jev carriers' alone (Cloudflare serves its one alias whatever is asked);
|
|
340
|
+
the session CLI and the chat fallback ignore it.
|
|
233
341
|
|
|
234
342
|
The providers are tried in `providers()` order - the session's own CLI
|
|
235
343
|
first - and the next one is asked only when the previous failed. `fallback`
|
|
@@ -244,7 +352,7 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
244
352
|
and a `deadline` in seconds: urllib's `timeout` bounds one socket operation,
|
|
245
353
|
not the call, so only the deadline bounds its wall time.
|
|
246
354
|
|
|
247
|
-
A call that ends on a 401, 402, 5xx or a session CLI's non-zero exit marks
|
|
355
|
+
A call that ends on a 401, 402, 403, 5xx or a session CLI's non-zero exit marks
|
|
248
356
|
that provider down for `DOWN_FOR` seconds, and until then it is skipped
|
|
249
357
|
without a request (`_down`), so a dead credential is not re-paid on every
|
|
250
358
|
prompt.
|
|
@@ -254,17 +362,22 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
254
362
|
exception.
|
|
255
363
|
|
|
256
364
|
`only` names the providers the caller requires (`named()`); a caller whose
|
|
257
|
-
decision must come from a typed model passes `only=("
|
|
258
|
-
None rather than another provider's answer.
|
|
259
|
-
|
|
365
|
+
decision must come from a typed model passes `only=("jev",)` and gets
|
|
366
|
+
None rather than another provider's answer. A forced Jev carrier
|
|
367
|
+
(`tp.jev_choice()`) whose credential is missing is asked nothing and
|
|
368
|
+
recorded as a failure, never replaced by another carrier."""
|
|
369
|
+
tried = _plan(only)
|
|
260
370
|
if any(isinstance(q, dict) and q.get("type") == "text" for q in questions.values()):
|
|
261
|
-
#
|
|
262
|
-
tried = [(p, s) for p, s in tried if p
|
|
263
|
-
if not tried
|
|
371
|
+
# no Jev carrier answers prose: left out up front, not logged as a failure
|
|
372
|
+
tried = [(p, s) for p, s in tried if not is_jev(p)]
|
|
373
|
+
if not tried:
|
|
264
374
|
return None
|
|
265
375
|
stop = None if deadline is None else time.monotonic() + deadline
|
|
266
376
|
failed = []
|
|
267
377
|
for provider, secret in tried:
|
|
378
|
+
if not secret:
|
|
379
|
+
failed.append("%s: no credential (jev=%s)" % (provider, tp.jev_choice()))
|
|
380
|
+
continue
|
|
268
381
|
result, why = _ask_one(provider, secret, state, questions, model,
|
|
269
382
|
timeout, attempts, stop)
|
|
270
383
|
if result is None:
|
|
@@ -279,7 +392,9 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
279
392
|
return result
|
|
280
393
|
refused = None
|
|
281
394
|
session = tp.session_cli()
|
|
282
|
-
|
|
395
|
+
# only a session CLI that was asked can have failed before a refusal
|
|
396
|
+
if session in (p for p, _ in tried) and (openrouter_key()
|
|
397
|
+
or any(s for _, s in tp.jev_carriers())):
|
|
283
398
|
refused = ("fallback=%s refuses a third-party judge after %s failed"
|
|
284
399
|
% (tp.fallback_policy(), session))
|
|
285
400
|
_record(None, None, failed="; ".join(failed), refused=refused)
|
|
@@ -289,13 +404,12 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
289
404
|
def _ask_one(provider, secret, state, questions, model, timeout, attempts, stop):
|
|
290
405
|
"""One provider's attempts: `(result, None)`, or `(None, why)`."""
|
|
291
406
|
try:
|
|
292
|
-
if provider
|
|
293
|
-
used, url = model,
|
|
294
|
-
body = json.dumps(
|
|
295
|
-
"questions": questions}).encode()
|
|
407
|
+
if is_jev(provider):
|
|
408
|
+
used, url, payload = _jev_call(provider, model, state, questions)
|
|
409
|
+
body = json.dumps(payload).encode()
|
|
296
410
|
|
|
297
411
|
def send():
|
|
298
|
-
return _request(secret, body, timeout)
|
|
412
|
+
return _request(secret, body, timeout, url)
|
|
299
413
|
elif provider == "openrouter":
|
|
300
414
|
used, url = fallback_model(), openrouter_url()
|
|
301
415
|
body = json.dumps(_chat_body(state, questions, used)).encode()
|
|
@@ -323,7 +437,7 @@ def _ask_one(provider, secret, state, questions, model, timeout, attempts, stop)
|
|
|
323
437
|
late = stop is not None and time.monotonic() >= stop
|
|
324
438
|
if attempt + 1 >= attempts or late or not _transient(exc):
|
|
325
439
|
code = getattr(exc, "code", None)
|
|
326
|
-
if code in (401, 402) or (isinstance(code, int) and code >= 500) \
|
|
440
|
+
if code in (401, 402, 403) or (isinstance(code, int) and code >= 500) \
|
|
327
441
|
or isinstance(exc, SessionFailed):
|
|
328
442
|
_mark_down(down)
|
|
329
443
|
return None, _why(exc)
|
|
@@ -397,19 +511,24 @@ def _transient(exc):
|
|
|
397
511
|
return isinstance(exc, OSError)
|
|
398
512
|
|
|
399
513
|
|
|
400
|
-
def _request(secret, body, timeout):
|
|
401
|
-
"""One POST
|
|
514
|
+
def _request(secret, body, timeout, url=None):
|
|
515
|
+
"""One POST to a Jev carrier - TypeSafe's unless `url` names another: the
|
|
516
|
+
documented return, or the exception `ask()` decides on. Cloudflare's REST
|
|
517
|
+
API may wrap the documented body as `{"result": {...}, "success": true}`;
|
|
518
|
+
both shapes are read.
|
|
402
519
|
|
|
403
520
|
Split from `ask()` so the retry re-sends the body already serialized rather
|
|
404
521
|
than rebuilding it, and so the URL parse stays inside `ask()`'s try: a
|
|
405
522
|
scheme-less override raises ValueError there and becomes a None like any
|
|
406
523
|
other failure rather than escaping."""
|
|
407
|
-
request = urllib.request.Request(endpoint(), data=body, headers={
|
|
524
|
+
request = urllib.request.Request(url or endpoint(), data=body, headers={
|
|
408
525
|
"Authorization": "Bearer " + secret,
|
|
409
526
|
"Content-Type": "application/json"})
|
|
410
527
|
started = time.monotonic()
|
|
411
528
|
with OPENER.open(request, timeout=timeout) as response:
|
|
412
529
|
data = json.load(response)
|
|
530
|
+
if "answers" not in data and isinstance(data.get("result"), dict):
|
|
531
|
+
data = data["result"]
|
|
413
532
|
usage = data["usage"]
|
|
414
533
|
return {"answers": data["answers"],
|
|
415
534
|
"usage": {"input_tokens": int(usage["input_tokens"]),
|
|
@@ -533,7 +652,7 @@ def cheap_default():
|
|
|
533
652
|
return row[0] if row else FALLBACK_MODEL
|
|
534
653
|
|
|
535
654
|
|
|
536
|
-
# How long a provider stays marked down after a 401, 402 or 5xx: five minutes,
|
|
655
|
+
# How long a provider stays marked down after a 401, 402, 403 or 5xx: five minutes,
|
|
537
656
|
# so a dead key or an empty account costs one refusal per five minutes rather
|
|
538
657
|
# than one per prompt, and a provider that comes back is asked again within the
|
|
539
658
|
# same working session. The mark is keyed on the provider, the endpoint and a
|
|
@@ -601,22 +720,28 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
601
720
|
on the prompt alone, and a CLI that wants a login exits instead of waiting -
|
|
602
721
|
with TEZGAH_NESTED set so tezgah's own hooks stand down in the child. The
|
|
603
722
|
answer is read from the CLI's JSON output: the model and the usage it names
|
|
604
|
-
are what the call really used.
|
|
723
|
+
are what the call really used. A CLI with no system-prompt flag gets the
|
|
724
|
+
system prompt in front of the prompt text instead."""
|
|
605
725
|
import shutil
|
|
606
726
|
import subprocess
|
|
607
727
|
import tempfile
|
|
728
|
+
if CHAT_SYSTEM not in SESSION_ARGV[name]:
|
|
729
|
+
prompt = CHAT_SYSTEM + "\n\n" + prompt
|
|
608
730
|
cwd = tempfile.mkdtemp(prefix="tezgah-judge-")
|
|
609
731
|
started = time.monotonic()
|
|
610
732
|
try:
|
|
611
733
|
proc = subprocess.run([exe] + SESSION_ARGV[name] + [prompt], cwd=cwd,
|
|
612
734
|
stdin=subprocess.DEVNULL, capture_output=True,
|
|
613
735
|
text=True, timeout=timeout,
|
|
614
|
-
env=dict(os.environ, TEZGAH_NESTED="1"
|
|
736
|
+
env=dict(os.environ, TEZGAH_NESTED="1",
|
|
737
|
+
**SESSION_ENV.get(name, {})))
|
|
615
738
|
finally:
|
|
616
739
|
shutil.rmtree(cwd, ignore_errors=True)
|
|
617
740
|
if proc.returncode:
|
|
618
|
-
|
|
619
|
-
|
|
741
|
+
# stdout last: codex prints a stdin banner on stderr and its reason as
|
|
742
|
+
# the last JSON event on stdout
|
|
743
|
+
raise SessionFailed("%s exit %d: %s" % (name, proc.returncode, (
|
|
744
|
+
(proc.stderr or "") + (proc.stdout or "")).strip()[-200:]))
|
|
620
745
|
if name == "claude":
|
|
621
746
|
data = json.loads(proc.stdout)
|
|
622
747
|
if data.get("is_error"):
|
|
@@ -627,7 +752,7 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
627
752
|
tokens_in = sum(int(usage.get(k) or 0) for k in (
|
|
628
753
|
"input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"))
|
|
629
754
|
tokens_out = int(usage.get("output_tokens") or 0)
|
|
630
|
-
|
|
755
|
+
elif name == "omp":
|
|
631
756
|
message = None
|
|
632
757
|
for line in proc.stdout.splitlines():
|
|
633
758
|
try:
|
|
@@ -645,6 +770,11 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
645
770
|
usage = message.get("usage") or {}
|
|
646
771
|
tokens_in = sum(int(usage.get(k) or 0) for k in ("input", "cacheRead", "cacheWrite"))
|
|
647
772
|
tokens_out = int(usage.get("output") or 0)
|
|
773
|
+
else:
|
|
774
|
+
model = None # none of the three names the model it ran
|
|
775
|
+
text, tokens_in, tokens_out = _cli_answer(name, proc.stdout)
|
|
776
|
+
if not text.strip():
|
|
777
|
+
raise SessionFailed("%s printed no answer" % name)
|
|
648
778
|
# A chat model may fence its object; the outermost braces are the object.
|
|
649
779
|
answers = _chat_answers(json.loads(text[text.find("{"):text.rfind("}") + 1]), questions)
|
|
650
780
|
return {"answers": answers,
|
|
@@ -653,6 +783,49 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
653
783
|
"model": model}
|
|
654
784
|
|
|
655
785
|
|
|
786
|
+
def _cli_answer(name, out):
|
|
787
|
+
"""(text, input tokens, output tokens) from opencode's, cursor-agent's
|
|
788
|
+
or codex's JSON output, the shapes each printed on 2026-10-09; text is ""
|
|
789
|
+
when no answer came. None of the three names the model it ran. Input counts
|
|
790
|
+
the cache reads and writes, as the claude and omp branches do: opencode and
|
|
791
|
+
cursor-agent report them beside the input (opencode's `total` is the sum of
|
|
792
|
+
all its fields), codex inside `input_tokens` (`cached_input_tokens` is a
|
|
793
|
+
share of it, as `reasoning_output_tokens` is of `output_tokens`)."""
|
|
794
|
+
text, tokens_in, tokens_out = "", 0, 0
|
|
795
|
+
for line in out.splitlines():
|
|
796
|
+
try:
|
|
797
|
+
event = json.loads(line)
|
|
798
|
+
except ValueError:
|
|
799
|
+
continue
|
|
800
|
+
if not isinstance(event, dict):
|
|
801
|
+
continue
|
|
802
|
+
kind = event.get("type")
|
|
803
|
+
part, item = event.get("part") or {}, event.get("item") or {}
|
|
804
|
+
usage = event.get("usage") or {}
|
|
805
|
+
if name == "cursor" and kind == "result":
|
|
806
|
+
if event.get("is_error"):
|
|
807
|
+
raise SessionFailed("cursor: %s" % str(event.get("result"))[:200])
|
|
808
|
+
text = event.get("result") or ""
|
|
809
|
+
tokens_in = sum(int(usage.get(k) or 0) for k in (
|
|
810
|
+
"inputTokens", "cacheReadTokens", "cacheWriteTokens"))
|
|
811
|
+
tokens_out = int(usage.get("outputTokens") or 0)
|
|
812
|
+
elif name == "opencode" and kind == "text":
|
|
813
|
+
text = part.get("text") or text
|
|
814
|
+
elif name == "opencode" and kind == "step_finish":
|
|
815
|
+
tokens = part.get("tokens") or {}
|
|
816
|
+
cache = tokens.get("cache") or {}
|
|
817
|
+
tokens_in += sum(int(v or 0) for v in (tokens.get("input"), cache.get("read"),
|
|
818
|
+
cache.get("write")))
|
|
819
|
+
tokens_out += int(tokens.get("output") or 0) + int(tokens.get("reasoning") or 0)
|
|
820
|
+
elif name == "codex" and kind == "item.completed" \
|
|
821
|
+
and item.get("type") == "agent_message":
|
|
822
|
+
text = item.get("text") or text
|
|
823
|
+
elif name == "codex" and kind == "turn.completed":
|
|
824
|
+
tokens_in += int(usage.get("input_tokens") or 0)
|
|
825
|
+
tokens_out += int(usage.get("output_tokens") or 0)
|
|
826
|
+
return text, tokens_in, tokens_out
|
|
827
|
+
|
|
828
|
+
|
|
656
829
|
def _why(exc):
|
|
657
830
|
"""A failure as one short reason: the HTTP code, else the exception's text."""
|
|
658
831
|
code = getattr(exc, "code", None)
|
package/hooks/tezgah_models.py
CHANGED
|
@@ -709,6 +709,39 @@ TIER_QUESTION = {
|
|
|
709
709
|
"a hard-to-reverse decision",
|
|
710
710
|
},
|
|
711
711
|
}
|
|
712
|
+
# The second question in the same batched call: whether the brief is one of the
|
|
713
|
+
# specialists' jobs. Before it existed route() could only name a tier, and from
|
|
714
|
+
# 2026-10-07 to 10-09 the five specialists were spawned 0 times in 121
|
|
715
|
+
# delegations while the generic `task` agent took 86. Every specialist but the
|
|
716
|
+
# docs writer is read-only or check-only, so a brief that changes code is `none`.
|
|
717
|
+
ROLE_QUESTION = {
|
|
718
|
+
"type": "choice",
|
|
719
|
+
"instructions": (
|
|
720
|
+
"Is `task` the whole job of one of these specialist agents? Pick `none` "
|
|
721
|
+
"when the task writes or changes source code, or mixes a specialist job "
|
|
722
|
+
"with code changes."),
|
|
723
|
+
"criteria": {
|
|
724
|
+
"none": "writing or changing code, or anything not listed below",
|
|
725
|
+
"tester": "only running the checks or tests a change reaches and reporting "
|
|
726
|
+
"their output; no edits",
|
|
727
|
+
"docs": "writing or repairing documentation pages or their path:line "
|
|
728
|
+
"citations",
|
|
729
|
+
"security": "a read-only security review of a change or surface",
|
|
730
|
+
"ui": "inspecting a running web or mobile app's screens against a design "
|
|
731
|
+
"floor or accessibility",
|
|
732
|
+
"researcher": "a research line: hypotheses, literature, experiments, a report",
|
|
733
|
+
"reviewer": "an adversarial read-only review of a diff, branch or pull request",
|
|
734
|
+
},
|
|
735
|
+
}
|
|
736
|
+
ROLE_AGENT = {role: "tezgah-" + role for role in ROLE_QUESTION["criteria"] if role != "none"}
|
|
737
|
+
# Without a judgement, the phase names the specialist where it is one.
|
|
738
|
+
PHASE_AGENT = {"review": "tezgah-reviewer", "research": "tezgah-researcher"}
|
|
739
|
+
# An override brief never leaves the machine, so the rule decides the one
|
|
740
|
+
# specialist it can name: a security review that writes nothing.
|
|
741
|
+
REVIEW_ONLY = re.compile(r"\b(?:review|audit|assess)\w*\b", re.I)
|
|
742
|
+
# `change` and `update` are left out: in "review the auth change" they are nouns.
|
|
743
|
+
WRITES = re.compile(r"\b(?:fix|implement|edit|write|add|refactor|patch|migrate|"
|
|
744
|
+
r"rename|delete|remove|rotate)\w*\b", re.I)
|
|
712
745
|
|
|
713
746
|
|
|
714
747
|
def route(brief, phase=None, ask=None):
|
|
@@ -724,23 +757,37 @@ def route(brief, phase=None, ask=None):
|
|
|
724
757
|
reply is the same as no reply and this never raises."""
|
|
725
758
|
hit = OVERRIDE.search(brief or "")
|
|
726
759
|
if hit:
|
|
760
|
+
if REVIEW_ONLY.search(brief) and not WRITES.search(brief):
|
|
761
|
+
return _agent("tezgah-security", "override %r, review only" % hit.group(0),
|
|
762
|
+
"rule")
|
|
727
763
|
return _pick("frontier", "override: %r" % hit.group(0), "rule")
|
|
728
|
-
result = ask({"task": ti.redact(brief or "")},
|
|
764
|
+
result = ask({"task": ti.redact(brief or "")},
|
|
765
|
+
{"tier": TIER_QUESTION, "role": ROLE_QUESTION}) if ask else None
|
|
729
766
|
choice = tj.choice(result, "tier")
|
|
730
767
|
if choice in JEV_TIER:
|
|
731
768
|
chance = tj.noul(result, "tier", choice)
|
|
732
|
-
|
|
733
|
-
|
|
769
|
+
why = "jev %s%s" % (choice, "" if chance is None else " %.2f" % chance)
|
|
770
|
+
role = tj.choice(result, "role")
|
|
771
|
+
out = (_agent(ROLE_AGENT[role], "%s, role %s" % (why, role), "jev")
|
|
772
|
+
if role in ROLE_AGENT else _pick(JEV_TIER[choice], why, "jev"))
|
|
734
773
|
out["judge"] = "%s/%s" % (result.get("provider") or "-",
|
|
735
774
|
result.get("model") or "-")
|
|
736
775
|
out["judged"] = result
|
|
737
776
|
return out
|
|
777
|
+
if phase in PHASE_AGENT:
|
|
778
|
+
return _agent(PHASE_AGENT[phase], "no judgement; phase %s names it" % phase,
|
|
779
|
+
"phase")
|
|
738
780
|
if phase in PHASE_TIER:
|
|
739
781
|
return _pick(PHASE_TIER[phase], "no judgement; static table for phase %s" % phase,
|
|
740
782
|
"phase")
|
|
741
783
|
return _pick("standard", "no judgement and no phase; the middle tier", "default")
|
|
742
784
|
|
|
743
785
|
|
|
786
|
+
def _agent(agent, why, via):
|
|
787
|
+
"""A specialist, on the tier its model slot names (`AGENT_SLOT`)."""
|
|
788
|
+
return dict(_pick(AGENT_SLOT[agent], why, via), agent=agent)
|
|
789
|
+
|
|
790
|
+
|
|
744
791
|
def _pick(tier, why, via):
|
|
745
792
|
return {"agent": "tezgah-" + tier, "tier": tier, "why": why, "via": via,
|
|
746
793
|
"judge": None, "judged": None}
|