@r1z4x/tezgah 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +103 -1
- package/MANIFEST +3 -0
- package/agents/tezgah-reviewer.md +19 -10
- package/bin/tezgah-docs +3 -3
- package/bin/tezgah-doctor +75 -18
- package/bin/tezgah-route +5 -2
- package/bin/tezgah-setup +30 -18
- package/bin/tezgah-status +3 -1
- package/bin/tezgah-task +130 -37
- package/bin/tezgah-taste +74 -55
- package/bin/tezgah-triage +12 -7
- package/docs/architecture.md +4 -4
- package/docs/clarity-baseline.json +9 -9
- package/docs/contract.md +22 -12
- package/docs/environment.md +4 -1
- package/docs/evidence.md +71 -34
- package/docs/gate.md +21 -21
- package/docs/glossary.md +5 -5
- package/docs/hosts.md +8 -8
- package/docs/judge.md +93 -27
- package/docs/operations.md +69 -30
- package/docs/skills.md +2 -2
- package/docs/status-line.md +7 -7
- package/docs/vision.md +10 -5
- package/hooks/tezgah_agents.py +22 -11
- package/hooks/tezgah_attest.py +15 -22
- package/hooks/tezgah_context.py +84 -124
- package/hooks/tezgah_gate.py +26 -23
- package/hooks/tezgah_guard.py +9 -11
- package/hooks/tezgah_integrity.py +77 -178
- package/hooks/tezgah_judge.py +270 -122
- package/hooks/tezgah_lessons.py +18 -63
- package/hooks/tezgah_paths.py +183 -55
- package/hooks/tezgah_policy.py +35 -8
- package/hooks/tezgah_replay.py +57 -33
- package/hooks/tezgah_shapes.py +4 -4
- package/hooks/tezgah_skill_pick.py +6 -35
- package/hooks/tezgah_snapshot.py +25 -46
- package/hooks/tezgah_store.py +1268 -0
- package/hooks/tezgah_task.py +146 -40
- package/hooks/tezgah_taste.py +21 -18
- package/hooks/tezgah_taste_ledger.py +63 -88
- package/hooks/tezgah_update.py +6 -26
- package/hosts/codex/hook.py +5 -0
- package/hosts/cursor/hook.py +19 -31
- package/hosts/opencode/plugins/tezgah.js +309 -108
- package/package.json +1 -1
- package/skills/harness/SKILL.md +4 -0
- package/skills/plan-add/SKILL.md +23 -7
- package/skills/plan-add/render_table.py +3 -2
- package/skills/plan-status/SKILL.md +12 -4
- package/skills/plan-sync/SKILL.md +19 -9
- package/skills/tezgah-contract/SKILL.md +25 -12
package/hooks/tezgah_judge.py
CHANGED
|
@@ -10,8 +10,10 @@ attempt and no more; a 4xx and a malformed reply never do (`ask()` below). A hoo
|
|
|
10
10
|
may import this module, and a hook that raises takes a session down, so every
|
|
11
11
|
failure is a `None` instead.
|
|
12
12
|
|
|
13
|
-
Privacy, plainly: the state and the questions leave this machine for
|
|
14
|
-
|
|
13
|
+
Privacy, plainly: the state and the questions leave this machine for the host
|
|
14
|
+
of whichever provider answers - the session's vendor, or the chosen Jev
|
|
15
|
+
carrier's (`api.typesafe.ai`, `openrouter.ai`, `api.cloudflare.com`, or the
|
|
16
|
+
`JEV_API_BASE_URL` a user set) - for the triage the state IS the screen's own text (an admin
|
|
15
17
|
table or a mobile view as captured, plus the `--select` task sentence), so a
|
|
16
18
|
screen carrying personal data is read by a third party. Nothing else leaves -
|
|
17
19
|
no session id, no workspace path, no environment - and the seam does not
|
|
@@ -23,17 +25,20 @@ prompt-path skill hint sends the user's prompt redacted and cut at 2,000
|
|
|
23
25
|
characters, because a brief or a prompt can quote a token and neither is a
|
|
24
26
|
state a person chose to send.
|
|
25
27
|
|
|
26
|
-
|
|
27
|
-
an interactive shell, but a hook or a bin tool started
|
|
28
|
-
non-interactive shell where `.zshenv` never runs - the
|
|
29
|
-
and the key file is the channel that survives
|
|
30
|
-
|
|
28
|
+
Each credential has a file channel beside its env var on purpose. `~/.zshenv`
|
|
29
|
+
exports the env var for an interactive shell, but a hook or a bin tool started
|
|
30
|
+
by a host runs in a non-interactive shell where `.zshenv` never runs - the
|
|
31
|
+
export is absent there, and the key file is the channel that survives (TypeSafe
|
|
32
|
+
also reads the active omp login). The resolvers live in `tezgah_paths`
|
|
33
|
+
(`typesafe_key`, `jev_carriers`), so `have_judge_key()` asks the same ones.
|
|
31
34
|
|
|
32
|
-
Behind the credential there are three
|
|
33
|
-
session's own CLI - `omp` or `
|
|
35
|
+
Behind the credential there are three kinds of provider, asked in this order. First the
|
|
36
|
+
session's own CLI - `omp`, `claude`, `opencode`, `cursor` or `codex`, the host
|
|
37
|
+
this process runs under or the one `TEZGAH_JUDGE_CLI`/`judge_cli` picks
|
|
34
38
|
(`tp.session_cli()`) - run headless with no tools, so the judgement is paid
|
|
35
39
|
with the session's own credential, OAuth subscription included, and answered by
|
|
36
|
-
the session's own vendor.
|
|
40
|
+
the session's own vendor. The Jev carriers (`JEV_CARRIERS`, as chosen by
|
|
41
|
+
`tp.jev_choice()`) and the OpenRouter chat fallback are third parties: they are
|
|
37
42
|
asked only as the `fallback` setting allows (`tp.fallback_policy()`): by
|
|
38
43
|
default (`vendor`) only when there is no session CLI at all, with `any` also
|
|
39
44
|
after the session CLI failed, with `none` never. A third party that answers,
|
|
@@ -55,15 +60,22 @@ import urllib.parse
|
|
|
55
60
|
import urllib.request
|
|
56
61
|
|
|
57
62
|
import tezgah_paths as tp
|
|
63
|
+
import tezgah_store
|
|
58
64
|
|
|
59
65
|
URL = "https://api.typesafe.ai/v1/systemone"
|
|
60
|
-
KEY_FILE = "~/.config/typesafe/key"
|
|
61
66
|
MODEL = "jev-latest"
|
|
67
|
+
# The other Jev carriers (`JEV_CARRIERS`): OpenRouter's System One endpoint,
|
|
68
|
+
# which takes TypeSafe's body and namespaces the model `typesafe/<id>`; and
|
|
69
|
+
# Cloudflare Workers AI, whose run route wraps the state and questions in
|
|
70
|
+
# `input` and serves the one alias `typesafe/jev`. A System One-compatible URL
|
|
71
|
+
# is the user's own (`JEV_API_BASE_URL`), so it has no constant here.
|
|
72
|
+
JEV_OPENROUTER_URL = "https://openrouter.ai/api/v1/systemone"
|
|
73
|
+
CLOUDFLARE_URL = "https://api.cloudflare.com/client/v4"
|
|
74
|
+
CLOUDFLARE_MODEL = "typesafe/jev"
|
|
62
75
|
# The fallback provider: an OpenAI-compatible chat endpoint, its own key
|
|
63
76
|
# channels, and the cheap judge model this repository already quotes in its
|
|
64
77
|
# measured arm rows (`openrouter/deepseek/deepseek-v4-flash`).
|
|
65
78
|
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
|
66
|
-
OPENROUTER_KEY_FILE = "~/.config/openrouter/key"
|
|
67
79
|
FALLBACK_MODEL = "deepseek/deepseek-v4-flash"
|
|
68
80
|
# The contract the fallback asks a chat model to answer in, one shape per
|
|
69
81
|
# question type this repository's callers send (`bin/tezgah-triage` and
|
|
@@ -109,19 +121,11 @@ def endpoint():
|
|
|
109
121
|
|
|
110
122
|
|
|
111
123
|
def key():
|
|
112
|
-
"""
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
value = os.environ.get("TYPESAFE_API_KEY", "").strip()
|
|
118
|
-
if value:
|
|
119
|
-
return value
|
|
120
|
-
try:
|
|
121
|
-
with open(os.path.expanduser(KEY_FILE), encoding="utf-8") as fh:
|
|
122
|
-
return fh.read().strip() or None
|
|
123
|
-
except OSError:
|
|
124
|
-
return None
|
|
124
|
+
"""TypeSafe's credential: TYPESAFE_API_KEY, else `~/.config/typesafe/key`,
|
|
125
|
+
else the active omp login, else None (`tp.typesafe_key`). Stripped: a
|
|
126
|
+
newline inside an Authorization header is a header-injection attempt to
|
|
127
|
+
the HTTP layer, not a typo it will forgive."""
|
|
128
|
+
return tp.typesafe_key()
|
|
125
129
|
|
|
126
130
|
|
|
127
131
|
def openrouter_url():
|
|
@@ -130,19 +134,37 @@ def openrouter_url():
|
|
|
130
134
|
|
|
131
135
|
|
|
132
136
|
def openrouter_key():
|
|
133
|
-
"""
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
137
|
+
"""OpenRouter's credential, for its System One carrier and the chat
|
|
138
|
+
fallback alike: OPENROUTER_API_KEY, else `~/.config/openrouter/key`, else
|
|
139
|
+
None (`tp.openrouter_key`), stripped for the same reason as `key()`."""
|
|
140
|
+
return tp.openrouter_key()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _jev_call(provider, model, state, questions):
|
|
144
|
+
"""`(model used, url, body)` for one Jev carrier: TypeSafe's own body
|
|
145
|
+
(`state`, `model`, `questions`) everywhere but Cloudflare, whose run route
|
|
146
|
+
takes its one alias and the pair under `input`. Clef and Clef-flash, its own
|
|
147
|
+
System One models, have a route each and take the bare body with `model`
|
|
148
|
+
as the selector. Every URL routes through `override()`, so a plain-http one
|
|
149
|
+
off this machine raises here."""
|
|
150
|
+
if provider in ("jev-cloudflare", "clef", "clef-flash"):
|
|
151
|
+
account = urllib.parse.quote(tp.cloudflare_account() or "", safe="")
|
|
152
|
+
url = override("TEZGAH_CLOUDFLARE_URL", CLOUDFLARE_URL) + \
|
|
153
|
+
"/accounts/%s/ai/run" % account
|
|
154
|
+
if provider != "jev-cloudflare":
|
|
155
|
+
return provider, url + "/@cf/cloudflare/" + provider, {
|
|
156
|
+
"model": provider, "state": state, "questions": questions}
|
|
157
|
+
return CLOUDFLARE_MODEL, url, {"model": CLOUDFLARE_MODEL, "input": {
|
|
158
|
+
"state": state, "questions": questions}}
|
|
159
|
+
if provider == "jev-openrouter":
|
|
160
|
+
used = model if "/" in model else "typesafe/" + model
|
|
161
|
+
url = override("TEZGAH_JEV_OPENROUTER_URL", JEV_OPENROUTER_URL)
|
|
162
|
+
elif provider == "jev-compatible":
|
|
163
|
+
used = os.environ.get("TEZGAH_JEV_MODEL", "").strip() or model
|
|
164
|
+
url = override("JEV_API_BASE_URL", "")
|
|
165
|
+
else:
|
|
166
|
+
used, url = model, endpoint()
|
|
167
|
+
return used, url, {"state": state, "model": used, "questions": questions}
|
|
146
168
|
|
|
147
169
|
|
|
148
170
|
def fallback_model():
|
|
@@ -169,25 +191,51 @@ SESSION_ARGV = {
|
|
|
169
191
|
"--strict-mcp-config", "--disable-slash-commands",
|
|
170
192
|
"--setting-sources", "", "--output-format", "json",
|
|
171
193
|
"--system-prompt", CHAT_SYSTEM],
|
|
194
|
+
# None of the next three takes a system prompt or drops its tools, so
|
|
195
|
+
# `_session_request` puts the system prompt in front of the prompt text and
|
|
196
|
+
# the empty temp cwd is what keeps tools from reaching a repository. Each
|
|
197
|
+
# set is from the CLI's own `--help` (checked 2026-10-09). opencode 1.18.31
|
|
198
|
+
# `run`: `--pure` "run without external plugins", `--format json` "raw JSON
|
|
199
|
+
# events"; it has no flag that skips the session record.
|
|
200
|
+
"opencode": ["run", "--pure", "--format", "json"],
|
|
201
|
+
# cursor-agent 2026.10.01: `-p` print, `--output-format json` (only with
|
|
202
|
+
# --print), `--mode ask` "Q&A style ... (read-only)", `--trust` "Trust the
|
|
203
|
+
# current workspace without prompting" - the temp cwd is never trusted.
|
|
204
|
+
"cursor": ["-p", "--output-format", "json", "--mode", "ask", "--trust"],
|
|
205
|
+
# codex-cli 0.153.4 `exec`: `--json` "Print events to stdout as JSONL",
|
|
206
|
+
# `--ephemeral` "without persisting session files", `--skip-git-repo-check`
|
|
207
|
+
# (the temp cwd is no repository), `--ignore-rules` (no execpolicy rules),
|
|
208
|
+
# `-s read-only`, `--color never`: consult's argv (`tp.CONSULT_CLIS`) plus
|
|
209
|
+
# `--json` and `--ignore-rules`.
|
|
210
|
+
"codex": ["exec", "--json", "--ephemeral", "--skip-git-repo-check",
|
|
211
|
+
"--ignore-rules", "-s", "read-only", "--color", "never"],
|
|
172
212
|
}
|
|
173
|
-
|
|
213
|
+
# What the CLI's environment adds to the call. `opencode run` has no flag that
|
|
214
|
+
# drops its tools, and a probe on 2026-10-09 saw it run a bash command the
|
|
215
|
+
# prompt asked for; the judge's state can be a screen's untrusted text, so the
|
|
216
|
+
# run gets an inline config (`OPENCODE_CONFIG_CONTENT`) that denies every tool -
|
|
217
|
+
# the same probe with it wrote nothing. cursor-agent's `--mode ask` refused the
|
|
218
|
+
# same shell call, and codex's `-s read-only` sandbox has no network.
|
|
219
|
+
SESSION_ENV = {
|
|
220
|
+
"opencode": {"OPENCODE_CONFIG_CONTENT": json.dumps({
|
|
221
|
+
"permission": {"*": "deny", "bash": "deny", "edit": "deny", "webfetch": "deny",
|
|
222
|
+
"external_directory": "deny"},
|
|
223
|
+
"tools": {"*": False}})},
|
|
224
|
+
}
|
|
225
|
+
NO_SESSION = ("no session CLI to ask (no omp, Claude Code, opencode, Cursor or Codex "
|
|
226
|
+
"session marker or TEZGAH_JUDGE_CLI/judge_cli pick, or that CLI missing)")
|
|
174
227
|
|
|
175
228
|
|
|
176
229
|
def providers():
|
|
177
230
|
"""[(provider, secret), ...] in the order `ask()` tries them, or [].
|
|
178
231
|
|
|
179
232
|
The session's own CLI first (its `secret` is the binary's path); then the
|
|
180
|
-
third parties -
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
out = [(session, getattr(tp, session + "_bin")())] if session else []
|
|
187
|
-
if policy == "none" or (session and policy == "vendor"):
|
|
188
|
-
return out
|
|
189
|
-
return out + [(name, secret) for name, secret in
|
|
190
|
-
(("typesafe", key()), ("openrouter", openrouter_key())) if secret]
|
|
233
|
+
third parties - the chosen Jev carriers (`tp.jev_carriers()`), then the
|
|
234
|
+
OpenRouter chat fallback - only as `tp.fallback_policy()` allows: `vendor`
|
|
235
|
+
(the default) asks them only when no session CLI exists, `any` after the
|
|
236
|
+
session CLI too, `none` never. A Jev carrier still wins over the chat
|
|
237
|
+
model whenever both resolve."""
|
|
238
|
+
return [(p, s) for p, s in _plan() if s]
|
|
191
239
|
|
|
192
240
|
|
|
193
241
|
def named(only):
|
|
@@ -195,13 +243,67 @@ def named(only):
|
|
|
195
243
|
in `only` that resolves, in `only`'s order. The `vendor` policy's
|
|
196
244
|
session-first order does not apply - the caller named the provider, so
|
|
197
245
|
nothing is a silent fallback - but `none` still keeps every third party
|
|
198
|
-
out.
|
|
246
|
+
out. The name `jev` stands for every chosen Jev carrier that resolves
|
|
247
|
+
(`tp.jev_carriers()`, in the chosen order), so a caller that needs a typed
|
|
248
|
+
answer names the model family, not one carrier's address."""
|
|
249
|
+
return [(p, s) for p, s in _plan(only) if s]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _plan(only=None):
|
|
253
|
+
"""`providers()` (`only` None) or `named(only)`, with one difference: a
|
|
254
|
+
forced Jev carrier whose credential is missing stays in as `(carrier,
|
|
255
|
+
None)`, so `ask()` records why no Jev carrier was asked instead of
|
|
256
|
+
silently asking another one."""
|
|
199
257
|
session = tp.session_cli()
|
|
200
|
-
|
|
258
|
+
policy = tp.fallback_policy()
|
|
259
|
+
jev = tp.jev_carriers() if policy != "none" else []
|
|
260
|
+
if only is None:
|
|
261
|
+
out = [(session, getattr(tp, session + "_bin")())] if session else []
|
|
262
|
+
if policy == "none" or (session and policy == "vendor"):
|
|
263
|
+
return out
|
|
264
|
+
return out + jev + [("openrouter", s) for s in (openrouter_key(),) if s]
|
|
201
265
|
found = {session: getattr(tp, session + "_bin")()} if session else {}
|
|
202
|
-
if
|
|
203
|
-
found.update(
|
|
204
|
-
|
|
266
|
+
if policy != "none":
|
|
267
|
+
found.update(jev, openrouter=openrouter_key())
|
|
268
|
+
out = []
|
|
269
|
+
for name in only:
|
|
270
|
+
for one, secret in (jev if name == "jev" else [(name, found.get(name))]):
|
|
271
|
+
if (secret or one in dict(jev)) and (one, secret) not in out:
|
|
272
|
+
out.append((one, secret))
|
|
273
|
+
return out
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
# Every carrier that answers with a typed System One model - Jev itself, or
|
|
277
|
+
# Cloudflare's Clef family, which takes the same request and answers in the
|
|
278
|
+
# same shape - as opposed to a chat model asked to imitate it: TypeSafe direct,
|
|
279
|
+
# OpenRouter's System One endpoint, Cloudflare Workers AI, any System
|
|
280
|
+
# One-compatible URL, Clef and Clef-flash. A decision a caller requires to be
|
|
281
|
+
# typed checks `is_jev(result["provider"])`.
|
|
282
|
+
JEV_CARRIERS = ("typesafe", "jev-openrouter", "jev-cloudflare", "jev-compatible",
|
|
283
|
+
"clef", "clef-flash")
|
|
284
|
+
# Dollars per 1M input tokens, output free on every one: Jev's listed price on
|
|
285
|
+
# each of its carriers, Clef's and Clef-flash's from their Workers AI pages
|
|
286
|
+
# (2026-10-09).
|
|
287
|
+
JEV_INPUT_PRICE = 0.042
|
|
288
|
+
CLEF_INPUT_PRICE = {"clef": 0.24, "clef-flash": 0.09}
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def is_jev(provider):
|
|
292
|
+
"""True when `provider` is a typed System One carrier (`JEV_CARRIERS`)."""
|
|
293
|
+
return provider in JEV_CARRIERS
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def input_price(provider):
|
|
297
|
+
"""The carrier's input price per 1M tokens, None for anything not typed."""
|
|
298
|
+
return CLEF_INPUT_PRICE.get(provider, JEV_INPUT_PRICE) if is_jev(provider) else None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def jev_summary():
|
|
302
|
+
"""One line naming the Jev choice and the carriers it resolves to now, for
|
|
303
|
+
`tezgah-status --judge` and `tezgah-setup --jev`: a forced carrier without
|
|
304
|
+
its credential is named as such, never hidden."""
|
|
305
|
+
names = ", ".join(p if s else p + " (no credential)" for p, s in tp.jev_carriers())
|
|
306
|
+
return "jev=%s: %s" % (tp.jev_choice(), names or "no carrier resolves")
|
|
205
307
|
|
|
206
308
|
|
|
207
309
|
def credential():
|
|
@@ -228,7 +330,8 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
228
330
|
beside it because the provider that answers is not the caller's choice. The
|
|
229
331
|
model is the one the reply names, else the one asked for: an alias such as
|
|
230
332
|
`jev-latest` hides a silent upgrade the reply's own field shows. `model` is
|
|
231
|
-
|
|
333
|
+
the Jev carriers' alone (Cloudflare serves its one alias whatever is asked);
|
|
334
|
+
the session CLI and the chat fallback ignore it.
|
|
232
335
|
|
|
233
336
|
The providers are tried in `providers()` order - the session's own CLI
|
|
234
337
|
first - and the next one is asked only when the previous failed. `fallback`
|
|
@@ -243,7 +346,7 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
243
346
|
and a `deadline` in seconds: urllib's `timeout` bounds one socket operation,
|
|
244
347
|
not the call, so only the deadline bounds its wall time.
|
|
245
348
|
|
|
246
|
-
A call that ends on a 401, 402, 5xx or a session CLI's non-zero exit marks
|
|
349
|
+
A call that ends on a 401, 402, 403, 5xx or a session CLI's non-zero exit marks
|
|
247
350
|
that provider down for `DOWN_FOR` seconds, and until then it is skipped
|
|
248
351
|
without a request (`_down`), so a dead credential is not re-paid on every
|
|
249
352
|
prompt.
|
|
@@ -253,17 +356,22 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
253
356
|
exception.
|
|
254
357
|
|
|
255
358
|
`only` names the providers the caller requires (`named()`); a caller whose
|
|
256
|
-
decision must come from a typed model passes `only=("
|
|
257
|
-
None rather than another provider's answer.
|
|
258
|
-
|
|
359
|
+
decision must come from a typed model passes `only=("jev",)` and gets
|
|
360
|
+
None rather than another provider's answer. A forced Jev carrier
|
|
361
|
+
(`tp.jev_choice()`) whose credential is missing is asked nothing and
|
|
362
|
+
recorded as a failure, never replaced by another carrier."""
|
|
363
|
+
tried = _plan(only)
|
|
259
364
|
if any(isinstance(q, dict) and q.get("type") == "text" for q in questions.values()):
|
|
260
|
-
#
|
|
261
|
-
tried = [(p, s) for p, s in tried if p
|
|
262
|
-
if not tried
|
|
365
|
+
# no Jev carrier answers prose: left out up front, not logged as a failure
|
|
366
|
+
tried = [(p, s) for p, s in tried if not is_jev(p)]
|
|
367
|
+
if not tried:
|
|
263
368
|
return None
|
|
264
369
|
stop = None if deadline is None else time.monotonic() + deadline
|
|
265
370
|
failed = []
|
|
266
371
|
for provider, secret in tried:
|
|
372
|
+
if not secret:
|
|
373
|
+
failed.append("%s: no credential (jev=%s)" % (provider, tp.jev_choice()))
|
|
374
|
+
continue
|
|
267
375
|
result, why = _ask_one(provider, secret, state, questions, model,
|
|
268
376
|
timeout, attempts, stop)
|
|
269
377
|
if result is None:
|
|
@@ -278,7 +386,8 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
278
386
|
return result
|
|
279
387
|
refused = None
|
|
280
388
|
session = tp.session_cli()
|
|
281
|
-
if session and (
|
|
389
|
+
if only is None and session and (openrouter_key()
|
|
390
|
+
or any(s for _, s in tp.jev_carriers())):
|
|
282
391
|
refused = ("fallback=%s refuses a third-party judge after %s failed"
|
|
283
392
|
% (tp.fallback_policy(), session))
|
|
284
393
|
_record(None, None, failed="; ".join(failed), refused=refused)
|
|
@@ -288,13 +397,12 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
288
397
|
def _ask_one(provider, secret, state, questions, model, timeout, attempts, stop):
|
|
289
398
|
"""One provider's attempts: `(result, None)`, or `(None, why)`."""
|
|
290
399
|
try:
|
|
291
|
-
if provider
|
|
292
|
-
used, url = model,
|
|
293
|
-
body = json.dumps(
|
|
294
|
-
"questions": questions}).encode()
|
|
400
|
+
if is_jev(provider):
|
|
401
|
+
used, url, payload = _jev_call(provider, model, state, questions)
|
|
402
|
+
body = json.dumps(payload).encode()
|
|
295
403
|
|
|
296
404
|
def send():
|
|
297
|
-
return _request(secret, body, timeout)
|
|
405
|
+
return _request(secret, body, timeout, url)
|
|
298
406
|
elif provider == "openrouter":
|
|
299
407
|
used, url = fallback_model(), openrouter_url()
|
|
300
408
|
body = json.dumps(_chat_body(state, questions, used)).encode()
|
|
@@ -308,8 +416,8 @@ def _ask_one(provider, secret, state, questions, model, timeout, attempts, stop)
|
|
|
308
416
|
def send():
|
|
309
417
|
left = timeout if stop is None else max(0.1, min(timeout, stop - time.monotonic()))
|
|
310
418
|
return _session_request(provider, secret, prompt, left, questions)
|
|
311
|
-
|
|
312
|
-
if _down(
|
|
419
|
+
down = _down_key(provider, url, secret)
|
|
420
|
+
if _down(down):
|
|
313
421
|
return None, "marked down for %ds after a refusal" % DOWN_FOR
|
|
314
422
|
except Exception as exc:
|
|
315
423
|
return None, _why(exc)
|
|
@@ -322,9 +430,9 @@ def _ask_one(provider, secret, state, questions, model, timeout, attempts, stop)
|
|
|
322
430
|
late = stop is not None and time.monotonic() >= stop
|
|
323
431
|
if attempt + 1 >= attempts or late or not _transient(exc):
|
|
324
432
|
code = getattr(exc, "code", None)
|
|
325
|
-
if code in (401, 402) or (isinstance(code, int) and code >= 500) \
|
|
433
|
+
if code in (401, 402, 403) or (isinstance(code, int) and code >= 500) \
|
|
326
434
|
or isinstance(exc, SessionFailed):
|
|
327
|
-
_mark_down(
|
|
435
|
+
_mark_down(down)
|
|
328
436
|
return None, _why(exc)
|
|
329
437
|
continue
|
|
330
438
|
named = result.get("model")
|
|
@@ -396,19 +504,24 @@ def _transient(exc):
|
|
|
396
504
|
return isinstance(exc, OSError)
|
|
397
505
|
|
|
398
506
|
|
|
399
|
-
def _request(secret, body, timeout):
|
|
400
|
-
"""One POST
|
|
507
|
+
def _request(secret, body, timeout, url=None):
|
|
508
|
+
"""One POST to a Jev carrier - TypeSafe's unless `url` names another: the
|
|
509
|
+
documented return, or the exception `ask()` decides on. Cloudflare's REST
|
|
510
|
+
API may wrap the documented body as `{"result": {...}, "success": true}`;
|
|
511
|
+
both shapes are read.
|
|
401
512
|
|
|
402
513
|
Split from `ask()` so the retry re-sends the body already serialized rather
|
|
403
514
|
than rebuilding it, and so the URL parse stays inside `ask()`'s try: a
|
|
404
515
|
scheme-less override raises ValueError there and becomes a None like any
|
|
405
516
|
other failure rather than escaping."""
|
|
406
|
-
request = urllib.request.Request(endpoint(), data=body, headers={
|
|
517
|
+
request = urllib.request.Request(url or endpoint(), data=body, headers={
|
|
407
518
|
"Authorization": "Bearer " + secret,
|
|
408
519
|
"Content-Type": "application/json"})
|
|
409
520
|
started = time.monotonic()
|
|
410
521
|
with OPENER.open(request, timeout=timeout) as response:
|
|
411
522
|
data = json.load(response)
|
|
523
|
+
if "answers" not in data and isinstance(data.get("result"), dict):
|
|
524
|
+
data = data["result"]
|
|
412
525
|
usage = data["usage"]
|
|
413
526
|
return {"answers": data["answers"],
|
|
414
527
|
"usage": {"input_tokens": int(usage["input_tokens"]),
|
|
@@ -532,38 +645,32 @@ def cheap_default():
|
|
|
532
645
|
return row[0] if row else FALLBACK_MODEL
|
|
533
646
|
|
|
534
647
|
|
|
535
|
-
# How long a provider stays marked down after a 401, 402 or 5xx: five minutes,
|
|
648
|
+
# How long a provider stays marked down after a 401, 402, 403 or 5xx: five minutes,
|
|
536
649
|
# so a dead key or an empty account costs one refusal per five minutes rather
|
|
537
650
|
# than one per prompt, and a provider that comes back is asked again within the
|
|
538
|
-
# same working session. The
|
|
539
|
-
# digest of the credential, so a rotated key is asked at once
|
|
651
|
+
# same working session. The mark is keyed on the provider, the endpoint and a
|
|
652
|
+
# digest of the credential, so a rotated key is asked at once; the store keeps
|
|
653
|
+
# the time it expires.
|
|
540
654
|
DOWN_FOR = 300
|
|
541
655
|
|
|
542
656
|
|
|
543
|
-
def
|
|
657
|
+
def _down_key(provider, url, secret):
|
|
544
658
|
import hashlib
|
|
545
|
-
|
|
546
|
-
return os.path.join(tp.cache_dir(), "judge-down", digest[:16])
|
|
659
|
+
return hashlib.sha256("\0".join((provider, url, secret)).encode()).hexdigest()[:16]
|
|
547
660
|
|
|
548
661
|
|
|
549
|
-
def _down(
|
|
550
|
-
"""True while the
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
return False
|
|
662
|
+
def _down(key):
|
|
663
|
+
"""True while the mark of `key` has not expired; one expiring more than
|
|
664
|
+
`DOWN_FOR` from now (a clock step back, a copied cache) and any read error
|
|
665
|
+
are up."""
|
|
666
|
+
until = tezgah_store.down_until(key)
|
|
667
|
+
return until is not None and 0 < until - time.time() <= DOWN_FOR
|
|
556
668
|
|
|
557
669
|
|
|
558
|
-
def _mark_down(
|
|
559
|
-
"""Best effort, like every other write a hook makes: a
|
|
670
|
+
def _mark_down(key):
|
|
671
|
+
"""Best effort, like every other write a hook makes: a store that cannot be
|
|
560
672
|
written costs the next call a request, never an exception."""
|
|
561
|
-
|
|
562
|
-
os.makedirs(os.path.dirname(marker), exist_ok=True)
|
|
563
|
-
with open(marker, "w", encoding="utf-8"):
|
|
564
|
-
pass
|
|
565
|
-
except OSError:
|
|
566
|
-
pass
|
|
673
|
+
tezgah_store.set_down(key, time.time() + DOWN_FOR)
|
|
567
674
|
|
|
568
675
|
|
|
569
676
|
def _bounded(send, stop):
|
|
@@ -606,22 +713,28 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
606
713
|
on the prompt alone, and a CLI that wants a login exits instead of waiting -
|
|
607
714
|
with TEZGAH_NESTED set so tezgah's own hooks stand down in the child. The
|
|
608
715
|
answer is read from the CLI's JSON output: the model and the usage it names
|
|
609
|
-
are what the call really used.
|
|
716
|
+
are what the call really used. A CLI with no system-prompt flag gets the
|
|
717
|
+
system prompt in front of the prompt text instead."""
|
|
610
718
|
import shutil
|
|
611
719
|
import subprocess
|
|
612
720
|
import tempfile
|
|
721
|
+
if CHAT_SYSTEM not in SESSION_ARGV[name]:
|
|
722
|
+
prompt = CHAT_SYSTEM + "\n\n" + prompt
|
|
613
723
|
cwd = tempfile.mkdtemp(prefix="tezgah-judge-")
|
|
614
724
|
started = time.monotonic()
|
|
615
725
|
try:
|
|
616
726
|
proc = subprocess.run([exe] + SESSION_ARGV[name] + [prompt], cwd=cwd,
|
|
617
727
|
stdin=subprocess.DEVNULL, capture_output=True,
|
|
618
728
|
text=True, timeout=timeout,
|
|
619
|
-
env=dict(os.environ, TEZGAH_NESTED="1"
|
|
729
|
+
env=dict(os.environ, TEZGAH_NESTED="1",
|
|
730
|
+
**SESSION_ENV.get(name, {})))
|
|
620
731
|
finally:
|
|
621
732
|
shutil.rmtree(cwd, ignore_errors=True)
|
|
622
733
|
if proc.returncode:
|
|
623
|
-
|
|
624
|
-
|
|
734
|
+
# stdout last: codex prints a stdin banner on stderr and its reason as
|
|
735
|
+
# the last JSON event on stdout
|
|
736
|
+
raise SessionFailed("%s exit %d: %s" % (name, proc.returncode, (
|
|
737
|
+
(proc.stderr or "") + (proc.stdout or "")).strip()[-200:]))
|
|
625
738
|
if name == "claude":
|
|
626
739
|
data = json.loads(proc.stdout)
|
|
627
740
|
if data.get("is_error"):
|
|
@@ -632,7 +745,7 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
632
745
|
tokens_in = sum(int(usage.get(k) or 0) for k in (
|
|
633
746
|
"input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"))
|
|
634
747
|
tokens_out = int(usage.get("output_tokens") or 0)
|
|
635
|
-
|
|
748
|
+
elif name == "omp":
|
|
636
749
|
message = None
|
|
637
750
|
for line in proc.stdout.splitlines():
|
|
638
751
|
try:
|
|
@@ -650,6 +763,11 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
650
763
|
usage = message.get("usage") or {}
|
|
651
764
|
tokens_in = sum(int(usage.get(k) or 0) for k in ("input", "cacheRead", "cacheWrite"))
|
|
652
765
|
tokens_out = int(usage.get("output") or 0)
|
|
766
|
+
else:
|
|
767
|
+
model = None # none of the three names the model it ran
|
|
768
|
+
text, tokens_in, tokens_out = _cli_answer(name, proc.stdout)
|
|
769
|
+
if not text.strip():
|
|
770
|
+
raise SessionFailed("%s printed no answer" % name)
|
|
653
771
|
# A chat model may fence its object; the outermost braces are the object.
|
|
654
772
|
answers = _chat_answers(json.loads(text[text.find("{"):text.rfind("}") + 1]), questions)
|
|
655
773
|
return {"answers": answers,
|
|
@@ -658,6 +776,49 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
658
776
|
"model": model}
|
|
659
777
|
|
|
660
778
|
|
|
779
|
+
def _cli_answer(name, out):
|
|
780
|
+
"""(text, input tokens, output tokens) from opencode's, cursor-agent's
|
|
781
|
+
or codex's JSON output, the shapes each printed on 2026-10-09; text is ""
|
|
782
|
+
when no answer came. None of the three names the model it ran. Input counts
|
|
783
|
+
the cache reads and writes, as the claude and omp branches do: opencode and
|
|
784
|
+
cursor-agent report them beside the input (opencode's `total` is the sum of
|
|
785
|
+
all its fields), codex inside `input_tokens` (`cached_input_tokens` is a
|
|
786
|
+
share of it, as `reasoning_output_tokens` is of `output_tokens`)."""
|
|
787
|
+
text, tokens_in, tokens_out = "", 0, 0
|
|
788
|
+
for line in out.splitlines():
|
|
789
|
+
try:
|
|
790
|
+
event = json.loads(line)
|
|
791
|
+
except ValueError:
|
|
792
|
+
continue
|
|
793
|
+
if not isinstance(event, dict):
|
|
794
|
+
continue
|
|
795
|
+
kind = event.get("type")
|
|
796
|
+
part, item = event.get("part") or {}, event.get("item") or {}
|
|
797
|
+
usage = event.get("usage") or {}
|
|
798
|
+
if name == "cursor" and kind == "result":
|
|
799
|
+
if event.get("is_error"):
|
|
800
|
+
raise SessionFailed("cursor: %s" % str(event.get("result"))[:200])
|
|
801
|
+
text = event.get("result") or ""
|
|
802
|
+
tokens_in = sum(int(usage.get(k) or 0) for k in (
|
|
803
|
+
"inputTokens", "cacheReadTokens", "cacheWriteTokens"))
|
|
804
|
+
tokens_out = int(usage.get("outputTokens") or 0)
|
|
805
|
+
elif name == "opencode" and kind == "text":
|
|
806
|
+
text = part.get("text") or text
|
|
807
|
+
elif name == "opencode" and kind == "step_finish":
|
|
808
|
+
tokens = part.get("tokens") or {}
|
|
809
|
+
cache = tokens.get("cache") or {}
|
|
810
|
+
tokens_in += sum(int(v or 0) for v in (tokens.get("input"), cache.get("read"),
|
|
811
|
+
cache.get("write")))
|
|
812
|
+
tokens_out += int(tokens.get("output") or 0) + int(tokens.get("reasoning") or 0)
|
|
813
|
+
elif name == "codex" and kind == "item.completed" \
|
|
814
|
+
and item.get("type") == "agent_message":
|
|
815
|
+
text = item.get("text") or text
|
|
816
|
+
elif name == "codex" and kind == "turn.completed":
|
|
817
|
+
tokens_in += int(usage.get("input_tokens") or 0)
|
|
818
|
+
tokens_out += int(usage.get("output_tokens") or 0)
|
|
819
|
+
return text, tokens_in, tokens_out
|
|
820
|
+
|
|
821
|
+
|
|
661
822
|
def _why(exc):
|
|
662
823
|
"""A failure as one short reason: the HTTP code, else the exception's text."""
|
|
663
824
|
code = getattr(exc, "code", None)
|
|
@@ -666,11 +827,6 @@ def _why(exc):
|
|
|
666
827
|
return ("%s: %s" % (type(exc).__name__, exc))[:200]
|
|
667
828
|
|
|
668
829
|
|
|
669
|
-
# The last judgement's record, one file: what answered, and why it was not the
|
|
670
|
-
# session's own CLI when it was not. `tezgah-status --judge` prints it.
|
|
671
|
-
LAST = "judge-last.json"
|
|
672
|
-
|
|
673
|
-
|
|
674
830
|
def _say(line):
|
|
675
831
|
try:
|
|
676
832
|
import sys
|
|
@@ -680,9 +836,11 @@ def _say(line):
|
|
|
680
836
|
|
|
681
837
|
|
|
682
838
|
def _record(provider, model, fallback=None, failed=None, refused=None):
|
|
683
|
-
"""Write the last-use record
|
|
684
|
-
|
|
685
|
-
|
|
839
|
+
"""Write the last-use record (the `judge_last` store's one document: what
|
|
840
|
+
answered, and why it was not the session's own CLI when it was not;
|
|
841
|
+
`tezgah-status --judge` prints it) and say any fallback or refusal on
|
|
842
|
+
stderr. Best effort like every hook write: a store that cannot be written
|
|
843
|
+
loses the record, never the answer."""
|
|
686
844
|
if fallback and provider:
|
|
687
845
|
_say("answered by %s/%s - %s" % (provider, model, fallback))
|
|
688
846
|
elif refused:
|
|
@@ -690,20 +848,10 @@ def _record(provider, model, fallback=None, failed=None, refused=None):
|
|
|
690
848
|
row = {"at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "provider": provider,
|
|
691
849
|
"model": model, "fallback": fallback, "failed": failed or None,
|
|
692
850
|
"refused": refused, "policy": tp.fallback_policy()}
|
|
693
|
-
|
|
694
|
-
path = os.path.join(tp.cache_dir(), LAST)
|
|
695
|
-
with open(path + ".tmp", "w", encoding="utf-8") as fh:
|
|
696
|
-
json.dump(row, fh)
|
|
697
|
-
os.replace(path + ".tmp", path)
|
|
698
|
-
except (OSError, TypeError, ValueError):
|
|
699
|
-
pass
|
|
851
|
+
tezgah_store.put_doc("judge_last", "", row)
|
|
700
852
|
|
|
701
853
|
|
|
702
854
|
def last_use():
|
|
703
855
|
"""The last judgement's record (see `_record`), or None when none was kept."""
|
|
704
|
-
|
|
705
|
-
with open(os.path.join(tp.cache_dir(), LAST), encoding="utf-8") as fh:
|
|
706
|
-
row = json.load(fh)
|
|
707
|
-
except (OSError, ValueError):
|
|
708
|
-
return None
|
|
856
|
+
row = tezgah_store.doc("judge_last", "")
|
|
709
857
|
return row if isinstance(row, dict) else None
|