@r1z4x/tezgah 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -1
- package/agents/tezgah-reviewer.md +19 -10
- package/bin/tezgah-docs +3 -3
- package/bin/tezgah-route +5 -2
- package/bin/tezgah-setup +24 -1
- package/bin/tezgah-status +3 -1
- package/bin/tezgah-task +130 -37
- package/bin/tezgah-taste +49 -31
- package/bin/tezgah-triage +12 -7
- package/docs/clarity-baseline.json +6 -6
- package/docs/contract.md +21 -11
- package/docs/environment.md +3 -1
- package/docs/evidence.md +4 -3
- package/docs/glossary.md +1 -1
- package/docs/judge.md +93 -27
- package/docs/operations.md +23 -12
- package/docs/skills.md +1 -1
- package/docs/vision.md +10 -5
- package/hooks/tezgah_agents.py +22 -11
- package/hooks/tezgah_context.py +40 -6
- package/hooks/tezgah_judge.py +245 -79
- package/hooks/tezgah_paths.py +165 -27
- package/hooks/tezgah_policy.py +35 -8
- package/hooks/tezgah_replay.py +26 -15
- package/hooks/tezgah_task.py +146 -40
- package/hooks/tezgah_taste.py +2 -2
- package/hooks/tezgah_taste_ledger.py +13 -4
- package/hosts/codex/hook.py +5 -0
- package/package.json +1 -1
- package/skills/harness/SKILL.md +4 -0
- package/skills/plan-add/SKILL.md +23 -7
- package/skills/plan-add/render_table.py +3 -2
- package/skills/plan-status/SKILL.md +12 -4
- package/skills/plan-sync/SKILL.md +19 -9
- package/skills/tezgah-contract/SKILL.md +25 -12
package/hooks/tezgah_judge.py
CHANGED
|
@@ -10,8 +10,10 @@ attempt and no more; a 4xx and a malformed reply never do (`ask()` below). A hoo
|
|
|
10
10
|
may import this module, and a hook that raises takes a session down, so every
|
|
11
11
|
failure is a `None` instead.
|
|
12
12
|
|
|
13
|
-
Privacy, plainly: the state and the questions leave this machine for
|
|
14
|
-
|
|
13
|
+
Privacy, plainly: the state and the questions leave this machine for the host
|
|
14
|
+
of whichever provider answers - the session's vendor, or the chosen Jev
|
|
15
|
+
carrier's (`api.typesafe.ai`, `openrouter.ai`, `api.cloudflare.com`, or the
|
|
16
|
+
`JEV_API_BASE_URL` a user set) - for the triage the state IS the screen's own text (an admin
|
|
15
17
|
table or a mobile view as captured, plus the `--select` task sentence), so a
|
|
16
18
|
screen carrying personal data is read by a third party. Nothing else leaves -
|
|
17
19
|
no session id, no workspace path, no environment - and the seam does not
|
|
@@ -23,17 +25,20 @@ prompt-path skill hint sends the user's prompt redacted and cut at 2,000
|
|
|
23
25
|
characters, because a brief or a prompt can quote a token and neither is a
|
|
24
26
|
state a person chose to send.
|
|
25
27
|
|
|
26
|
-
|
|
27
|
-
an interactive shell, but a hook or a bin tool started
|
|
28
|
-
non-interactive shell where `.zshenv` never runs - the
|
|
29
|
-
and the key file is the channel that survives
|
|
30
|
-
|
|
28
|
+
Each credential has a file channel beside its env var on purpose. `~/.zshenv`
|
|
29
|
+
exports the env var for an interactive shell, but a hook or a bin tool started
|
|
30
|
+
by a host runs in a non-interactive shell where `.zshenv` never runs - the
|
|
31
|
+
export is absent there, and the key file is the channel that survives (TypeSafe
|
|
32
|
+
also reads the active omp login). The resolvers live in `tezgah_paths`
|
|
33
|
+
(`typesafe_key`, `jev_carriers`), so `have_judge_key()` asks the same ones.
|
|
31
34
|
|
|
32
|
-
Behind the credential there are three
|
|
33
|
-
session's own CLI - `omp` or `
|
|
35
|
+
Behind the credential there are three kinds of provider, asked in this order. First the
|
|
36
|
+
session's own CLI - `omp`, `claude`, `opencode`, `cursor` or `codex`, the host
|
|
37
|
+
this process runs under or the one `TEZGAH_JUDGE_CLI`/`judge_cli` picks
|
|
34
38
|
(`tp.session_cli()`) - run headless with no tools, so the judgement is paid
|
|
35
39
|
with the session's own credential, OAuth subscription included, and answered by
|
|
36
|
-
the session's own vendor.
|
|
40
|
+
the session's own vendor. The Jev carriers (`JEV_CARRIERS`, as chosen by
|
|
41
|
+
`tp.jev_choice()`) and the OpenRouter chat fallback are third parties: they are
|
|
37
42
|
asked only as the `fallback` setting allows (`tp.fallback_policy()`): by
|
|
38
43
|
default (`vendor`) only when there is no session CLI at all, with `any` also
|
|
39
44
|
after the session CLI failed, with `none` never. A third party that answers,
|
|
@@ -58,13 +63,19 @@ import tezgah_paths as tp
|
|
|
58
63
|
import tezgah_store
|
|
59
64
|
|
|
60
65
|
URL = "https://api.typesafe.ai/v1/systemone"
|
|
61
|
-
KEY_FILE = "~/.config/typesafe/key"
|
|
62
66
|
MODEL = "jev-latest"
|
|
67
|
+
# The other Jev carriers (`JEV_CARRIERS`): OpenRouter's System One endpoint,
|
|
68
|
+
# which takes TypeSafe's body and namespaces the model `typesafe/<id>`; and
|
|
69
|
+
# Cloudflare Workers AI, whose run route wraps the state and questions in
|
|
70
|
+
# `input` and serves the one alias `typesafe/jev`. A System One-compatible URL
|
|
71
|
+
# is the user's own (`JEV_API_BASE_URL`), so it has no constant here.
|
|
72
|
+
JEV_OPENROUTER_URL = "https://openrouter.ai/api/v1/systemone"
|
|
73
|
+
CLOUDFLARE_URL = "https://api.cloudflare.com/client/v4"
|
|
74
|
+
CLOUDFLARE_MODEL = "typesafe/jev"
|
|
63
75
|
# The fallback provider: an OpenAI-compatible chat endpoint, its own key
|
|
64
76
|
# channels, and the cheap judge model this repository already quotes in its
|
|
65
77
|
# measured arm rows (`openrouter/deepseek/deepseek-v4-flash`).
|
|
66
78
|
OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
|
67
|
-
OPENROUTER_KEY_FILE = "~/.config/openrouter/key"
|
|
68
79
|
FALLBACK_MODEL = "deepseek/deepseek-v4-flash"
|
|
69
80
|
# The contract the fallback asks a chat model to answer in, one shape per
|
|
70
81
|
# question type this repository's callers send (`bin/tezgah-triage` and
|
|
@@ -110,19 +121,11 @@ def endpoint():
|
|
|
110
121
|
|
|
111
122
|
|
|
112
123
|
def key():
|
|
113
|
-
"""
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
value = os.environ.get("TYPESAFE_API_KEY", "").strip()
|
|
119
|
-
if value:
|
|
120
|
-
return value
|
|
121
|
-
try:
|
|
122
|
-
with open(os.path.expanduser(KEY_FILE), encoding="utf-8") as fh:
|
|
123
|
-
return fh.read().strip() or None
|
|
124
|
-
except OSError:
|
|
125
|
-
return None
|
|
124
|
+
"""TypeSafe's credential: TYPESAFE_API_KEY, else `~/.config/typesafe/key`,
|
|
125
|
+
else the active omp login, else None (`tp.typesafe_key`). Stripped: a
|
|
126
|
+
newline inside an Authorization header is a header-injection attempt to
|
|
127
|
+
the HTTP layer, not a typo it will forgive."""
|
|
128
|
+
return tp.typesafe_key()
|
|
126
129
|
|
|
127
130
|
|
|
128
131
|
def openrouter_url():
|
|
@@ -131,19 +134,37 @@ def openrouter_url():
|
|
|
131
134
|
|
|
132
135
|
|
|
133
136
|
def openrouter_key():
|
|
134
|
-
"""
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
137
|
+
"""OpenRouter's credential, for its System One carrier and the chat
|
|
138
|
+
fallback alike: OPENROUTER_API_KEY, else `~/.config/openrouter/key`, else
|
|
139
|
+
None (`tp.openrouter_key`), stripped for the same reason as `key()`."""
|
|
140
|
+
return tp.openrouter_key()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _jev_call(provider, model, state, questions):
|
|
144
|
+
"""`(model used, url, body)` for one Jev carrier: TypeSafe's own body
|
|
145
|
+
(`state`, `model`, `questions`) everywhere but Cloudflare, whose run route
|
|
146
|
+
takes its one alias and the pair under `input`. Clef and Clef-flash, its own
|
|
147
|
+
System One models, have a route each and take the bare body with `model`
|
|
148
|
+
as the selector. Every URL routes through `override()`, so a plain-http one
|
|
149
|
+
off this machine raises here."""
|
|
150
|
+
if provider in ("jev-cloudflare", "clef", "clef-flash"):
|
|
151
|
+
account = urllib.parse.quote(tp.cloudflare_account() or "", safe="")
|
|
152
|
+
url = override("TEZGAH_CLOUDFLARE_URL", CLOUDFLARE_URL) + \
|
|
153
|
+
"/accounts/%s/ai/run" % account
|
|
154
|
+
if provider != "jev-cloudflare":
|
|
155
|
+
return provider, url + "/@cf/cloudflare/" + provider, {
|
|
156
|
+
"model": provider, "state": state, "questions": questions}
|
|
157
|
+
return CLOUDFLARE_MODEL, url, {"model": CLOUDFLARE_MODEL, "input": {
|
|
158
|
+
"state": state, "questions": questions}}
|
|
159
|
+
if provider == "jev-openrouter":
|
|
160
|
+
used = model if "/" in model else "typesafe/" + model
|
|
161
|
+
url = override("TEZGAH_JEV_OPENROUTER_URL", JEV_OPENROUTER_URL)
|
|
162
|
+
elif provider == "jev-compatible":
|
|
163
|
+
used = os.environ.get("TEZGAH_JEV_MODEL", "").strip() or model
|
|
164
|
+
url = override("JEV_API_BASE_URL", "")
|
|
165
|
+
else:
|
|
166
|
+
used, url = model, endpoint()
|
|
167
|
+
return used, url, {"state": state, "model": used, "questions": questions}
|
|
147
168
|
|
|
148
169
|
|
|
149
170
|
def fallback_model():
|
|
@@ -170,25 +191,51 @@ SESSION_ARGV = {
|
|
|
170
191
|
"--strict-mcp-config", "--disable-slash-commands",
|
|
171
192
|
"--setting-sources", "", "--output-format", "json",
|
|
172
193
|
"--system-prompt", CHAT_SYSTEM],
|
|
194
|
+
# None of the next three takes a system prompt or drops its tools, so
|
|
195
|
+
# `_session_request` puts the system prompt in front of the prompt text and
|
|
196
|
+
# the empty temp cwd is what keeps tools from reaching a repository. Each
|
|
197
|
+
# set is from the CLI's own `--help` (checked 2026-10-09). opencode 1.18.31
|
|
198
|
+
# `run`: `--pure` "run without external plugins", `--format json` "raw JSON
|
|
199
|
+
# events"; it has no flag that skips the session record.
|
|
200
|
+
"opencode": ["run", "--pure", "--format", "json"],
|
|
201
|
+
# cursor-agent 2026.10.01: `-p` print, `--output-format json` (only with
|
|
202
|
+
# --print), `--mode ask` "Q&A style ... (read-only)", `--trust` "Trust the
|
|
203
|
+
# current workspace without prompting" - the temp cwd is never trusted.
|
|
204
|
+
"cursor": ["-p", "--output-format", "json", "--mode", "ask", "--trust"],
|
|
205
|
+
# codex-cli 0.153.4 `exec`: `--json` "Print events to stdout as JSONL",
|
|
206
|
+
# `--ephemeral` "without persisting session files", `--skip-git-repo-check`
|
|
207
|
+
# (the temp cwd is no repository), `--ignore-rules` (no execpolicy rules),
|
|
208
|
+
# `-s read-only`, `--color never`: consult's argv (`tp.CONSULT_CLIS`) plus
|
|
209
|
+
# `--json` and `--ignore-rules`.
|
|
210
|
+
"codex": ["exec", "--json", "--ephemeral", "--skip-git-repo-check",
|
|
211
|
+
"--ignore-rules", "-s", "read-only", "--color", "never"],
|
|
212
|
+
}
|
|
213
|
+
# What the CLI's environment adds to the call. `opencode run` has no flag that
|
|
214
|
+
# drops its tools, and a probe on 2026-10-09 saw it run a bash command the
|
|
215
|
+
# prompt asked for; the judge's state can be a screen's untrusted text, so the
|
|
216
|
+
# run gets an inline config (`OPENCODE_CONFIG_CONTENT`) that denies every tool -
|
|
217
|
+
# the same probe with it wrote nothing. cursor-agent's `--mode ask` refused the
|
|
218
|
+
# same shell call, and codex's `-s read-only` sandbox has no network.
|
|
219
|
+
SESSION_ENV = {
|
|
220
|
+
"opencode": {"OPENCODE_CONFIG_CONTENT": json.dumps({
|
|
221
|
+
"permission": {"*": "deny", "bash": "deny", "edit": "deny", "webfetch": "deny",
|
|
222
|
+
"external_directory": "deny"},
|
|
223
|
+
"tools": {"*": False}})},
|
|
173
224
|
}
|
|
174
|
-
NO_SESSION = "no session CLI to ask (
|
|
225
|
+
NO_SESSION = ("no session CLI to ask (no omp, Claude Code, opencode, Cursor or Codex "
|
|
226
|
+
"session marker or TEZGAH_JUDGE_CLI/judge_cli pick, or that CLI missing)")
|
|
175
227
|
|
|
176
228
|
|
|
177
229
|
def providers():
|
|
178
230
|
"""[(provider, secret), ...] in the order `ask()` tries them, or [].
|
|
179
231
|
|
|
180
232
|
The session's own CLI first (its `secret` is the binary's path); then the
|
|
181
|
-
third parties -
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
out = [(session, getattr(tp, session + "_bin")())] if session else []
|
|
188
|
-
if policy == "none" or (session and policy == "vendor"):
|
|
189
|
-
return out
|
|
190
|
-
return out + [(name, secret) for name, secret in
|
|
191
|
-
(("typesafe", key()), ("openrouter", openrouter_key())) if secret]
|
|
233
|
+
third parties - the chosen Jev carriers (`tp.jev_carriers()`), then the
|
|
234
|
+
OpenRouter chat fallback - only as `tp.fallback_policy()` allows: `vendor`
|
|
235
|
+
(the default) asks them only when no session CLI exists, `any` after the
|
|
236
|
+
session CLI too, `none` never. A Jev carrier still wins over the chat
|
|
237
|
+
model whenever both resolve."""
|
|
238
|
+
return [(p, s) for p, s in _plan() if s]
|
|
192
239
|
|
|
193
240
|
|
|
194
241
|
def named(only):
|
|
@@ -196,13 +243,67 @@ def named(only):
|
|
|
196
243
|
in `only` that resolves, in `only`'s order. The `vendor` policy's
|
|
197
244
|
session-first order does not apply - the caller named the provider, so
|
|
198
245
|
nothing is a silent fallback - but `none` still keeps every third party
|
|
199
|
-
out.
|
|
246
|
+
out. The name `jev` stands for every chosen Jev carrier that resolves
|
|
247
|
+
(`tp.jev_carriers()`, in the chosen order), so a caller that needs a typed
|
|
248
|
+
answer names the model family, not one carrier's address."""
|
|
249
|
+
return [(p, s) for p, s in _plan(only) if s]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _plan(only=None):
|
|
253
|
+
"""`providers()` (`only` None) or `named(only)`, with one difference: a
|
|
254
|
+
forced Jev carrier whose credential is missing stays in as `(carrier,
|
|
255
|
+
None)`, so `ask()` records why no Jev carrier was asked instead of
|
|
256
|
+
silently asking another one."""
|
|
200
257
|
session = tp.session_cli()
|
|
201
|
-
|
|
258
|
+
policy = tp.fallback_policy()
|
|
259
|
+
jev = tp.jev_carriers() if policy != "none" else []
|
|
260
|
+
if only is None:
|
|
261
|
+
out = [(session, getattr(tp, session + "_bin")())] if session else []
|
|
262
|
+
if policy == "none" or (session and policy == "vendor"):
|
|
263
|
+
return out
|
|
264
|
+
return out + jev + [("openrouter", s) for s in (openrouter_key(),) if s]
|
|
202
265
|
found = {session: getattr(tp, session + "_bin")()} if session else {}
|
|
203
|
-
if
|
|
204
|
-
found.update(
|
|
205
|
-
|
|
266
|
+
if policy != "none":
|
|
267
|
+
found.update(jev, openrouter=openrouter_key())
|
|
268
|
+
out = []
|
|
269
|
+
for name in only:
|
|
270
|
+
for one, secret in (jev if name == "jev" else [(name, found.get(name))]):
|
|
271
|
+
if (secret or one in dict(jev)) and (one, secret) not in out:
|
|
272
|
+
out.append((one, secret))
|
|
273
|
+
return out
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
# Every carrier that answers with a typed System One model - Jev itself, or
|
|
277
|
+
# Cloudflare's Clef family, which takes the same request and answers in the
|
|
278
|
+
# same shape - as opposed to a chat model asked to imitate it: TypeSafe direct,
|
|
279
|
+
# OpenRouter's System One endpoint, Cloudflare Workers AI, any System
|
|
280
|
+
# One-compatible URL, Clef and Clef-flash. A decision a caller requires to be
|
|
281
|
+
# typed checks `is_jev(result["provider"])`.
|
|
282
|
+
JEV_CARRIERS = ("typesafe", "jev-openrouter", "jev-cloudflare", "jev-compatible",
|
|
283
|
+
"clef", "clef-flash")
|
|
284
|
+
# Dollars per 1M input tokens, output free on every one: Jev's listed price on
|
|
285
|
+
# each of its carriers, Clef's and Clef-flash's from their Workers AI pages
|
|
286
|
+
# (2026-10-09).
|
|
287
|
+
JEV_INPUT_PRICE = 0.042
|
|
288
|
+
CLEF_INPUT_PRICE = {"clef": 0.24, "clef-flash": 0.09}
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def is_jev(provider):
|
|
292
|
+
"""True when `provider` is a typed System One carrier (`JEV_CARRIERS`)."""
|
|
293
|
+
return provider in JEV_CARRIERS
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def input_price(provider):
|
|
297
|
+
"""The carrier's input price per 1M tokens, None for anything not typed."""
|
|
298
|
+
return CLEF_INPUT_PRICE.get(provider, JEV_INPUT_PRICE) if is_jev(provider) else None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def jev_summary():
|
|
302
|
+
"""One line naming the Jev choice and the carriers it resolves to now, for
|
|
303
|
+
`tezgah-status --judge` and `tezgah-setup --jev`: a forced carrier without
|
|
304
|
+
its credential is named as such, never hidden."""
|
|
305
|
+
names = ", ".join(p if s else p + " (no credential)" for p, s in tp.jev_carriers())
|
|
306
|
+
return "jev=%s: %s" % (tp.jev_choice(), names or "no carrier resolves")
|
|
206
307
|
|
|
207
308
|
|
|
208
309
|
def credential():
|
|
@@ -229,7 +330,8 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
229
330
|
beside it because the provider that answers is not the caller's choice. The
|
|
230
331
|
model is the one the reply names, else the one asked for: an alias such as
|
|
231
332
|
`jev-latest` hides a silent upgrade the reply's own field shows. `model` is
|
|
232
|
-
|
|
333
|
+
the Jev carriers' alone (Cloudflare serves its one alias whatever is asked);
|
|
334
|
+
the session CLI and the chat fallback ignore it.
|
|
233
335
|
|
|
234
336
|
The providers are tried in `providers()` order - the session's own CLI
|
|
235
337
|
first - and the next one is asked only when the previous failed. `fallback`
|
|
@@ -244,7 +346,7 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
244
346
|
and a `deadline` in seconds: urllib's `timeout` bounds one socket operation,
|
|
245
347
|
not the call, so only the deadline bounds its wall time.
|
|
246
348
|
|
|
247
|
-
A call that ends on a 401, 402, 5xx or a session CLI's non-zero exit marks
|
|
349
|
+
A call that ends on a 401, 402, 403, 5xx or a session CLI's non-zero exit marks
|
|
248
350
|
that provider down for `DOWN_FOR` seconds, and until then it is skipped
|
|
249
351
|
without a request (`_down`), so a dead credential is not re-paid on every
|
|
250
352
|
prompt.
|
|
@@ -254,17 +356,22 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
254
356
|
exception.
|
|
255
357
|
|
|
256
358
|
`only` names the providers the caller requires (`named()`); a caller whose
|
|
257
|
-
decision must come from a typed model passes `only=("
|
|
258
|
-
None rather than another provider's answer.
|
|
259
|
-
|
|
359
|
+
decision must come from a typed model passes `only=("jev",)` and gets
|
|
360
|
+
None rather than another provider's answer. A forced Jev carrier
|
|
361
|
+
(`tp.jev_choice()`) whose credential is missing is asked nothing and
|
|
362
|
+
recorded as a failure, never replaced by another carrier."""
|
|
363
|
+
tried = _plan(only)
|
|
260
364
|
if any(isinstance(q, dict) and q.get("type") == "text" for q in questions.values()):
|
|
261
|
-
#
|
|
262
|
-
tried = [(p, s) for p, s in tried if p
|
|
263
|
-
if not tried
|
|
365
|
+
# no Jev carrier answers prose: left out up front, not logged as a failure
|
|
366
|
+
tried = [(p, s) for p, s in tried if not is_jev(p)]
|
|
367
|
+
if not tried:
|
|
264
368
|
return None
|
|
265
369
|
stop = None if deadline is None else time.monotonic() + deadline
|
|
266
370
|
failed = []
|
|
267
371
|
for provider, secret in tried:
|
|
372
|
+
if not secret:
|
|
373
|
+
failed.append("%s: no credential (jev=%s)" % (provider, tp.jev_choice()))
|
|
374
|
+
continue
|
|
268
375
|
result, why = _ask_one(provider, secret, state, questions, model,
|
|
269
376
|
timeout, attempts, stop)
|
|
270
377
|
if result is None:
|
|
@@ -279,7 +386,8 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
279
386
|
return result
|
|
280
387
|
refused = None
|
|
281
388
|
session = tp.session_cli()
|
|
282
|
-
if session and (
|
|
389
|
+
if only is None and session and (openrouter_key()
|
|
390
|
+
or any(s for _, s in tp.jev_carriers())):
|
|
283
391
|
refused = ("fallback=%s refuses a third-party judge after %s failed"
|
|
284
392
|
% (tp.fallback_policy(), session))
|
|
285
393
|
_record(None, None, failed="; ".join(failed), refused=refused)
|
|
@@ -289,13 +397,12 @@ def ask(state, questions, *, model=MODEL, timeout=30, attempts=2, deadline=None,
|
|
|
289
397
|
def _ask_one(provider, secret, state, questions, model, timeout, attempts, stop):
|
|
290
398
|
"""One provider's attempts: `(result, None)`, or `(None, why)`."""
|
|
291
399
|
try:
|
|
292
|
-
if provider
|
|
293
|
-
used, url = model,
|
|
294
|
-
body = json.dumps(
|
|
295
|
-
"questions": questions}).encode()
|
|
400
|
+
if is_jev(provider):
|
|
401
|
+
used, url, payload = _jev_call(provider, model, state, questions)
|
|
402
|
+
body = json.dumps(payload).encode()
|
|
296
403
|
|
|
297
404
|
def send():
|
|
298
|
-
return _request(secret, body, timeout)
|
|
405
|
+
return _request(secret, body, timeout, url)
|
|
299
406
|
elif provider == "openrouter":
|
|
300
407
|
used, url = fallback_model(), openrouter_url()
|
|
301
408
|
body = json.dumps(_chat_body(state, questions, used)).encode()
|
|
@@ -323,7 +430,7 @@ def _ask_one(provider, secret, state, questions, model, timeout, attempts, stop)
|
|
|
323
430
|
late = stop is not None and time.monotonic() >= stop
|
|
324
431
|
if attempt + 1 >= attempts or late or not _transient(exc):
|
|
325
432
|
code = getattr(exc, "code", None)
|
|
326
|
-
if code in (401, 402) or (isinstance(code, int) and code >= 500) \
|
|
433
|
+
if code in (401, 402, 403) or (isinstance(code, int) and code >= 500) \
|
|
327
434
|
or isinstance(exc, SessionFailed):
|
|
328
435
|
_mark_down(down)
|
|
329
436
|
return None, _why(exc)
|
|
@@ -397,19 +504,24 @@ def _transient(exc):
|
|
|
397
504
|
return isinstance(exc, OSError)
|
|
398
505
|
|
|
399
506
|
|
|
400
|
-
def _request(secret, body, timeout):
|
|
401
|
-
"""One POST
|
|
507
|
+
def _request(secret, body, timeout, url=None):
|
|
508
|
+
"""One POST to a Jev carrier - TypeSafe's unless `url` names another: the
|
|
509
|
+
documented return, or the exception `ask()` decides on. Cloudflare's REST
|
|
510
|
+
API may wrap the documented body as `{"result": {...}, "success": true}`;
|
|
511
|
+
both shapes are read.
|
|
402
512
|
|
|
403
513
|
Split from `ask()` so the retry re-sends the body already serialized rather
|
|
404
514
|
than rebuilding it, and so the URL parse stays inside `ask()`'s try: a
|
|
405
515
|
scheme-less override raises ValueError there and becomes a None like any
|
|
406
516
|
other failure rather than escaping."""
|
|
407
|
-
request = urllib.request.Request(endpoint(), data=body, headers={
|
|
517
|
+
request = urllib.request.Request(url or endpoint(), data=body, headers={
|
|
408
518
|
"Authorization": "Bearer " + secret,
|
|
409
519
|
"Content-Type": "application/json"})
|
|
410
520
|
started = time.monotonic()
|
|
411
521
|
with OPENER.open(request, timeout=timeout) as response:
|
|
412
522
|
data = json.load(response)
|
|
523
|
+
if "answers" not in data and isinstance(data.get("result"), dict):
|
|
524
|
+
data = data["result"]
|
|
413
525
|
usage = data["usage"]
|
|
414
526
|
return {"answers": data["answers"],
|
|
415
527
|
"usage": {"input_tokens": int(usage["input_tokens"]),
|
|
@@ -533,7 +645,7 @@ def cheap_default():
|
|
|
533
645
|
return row[0] if row else FALLBACK_MODEL
|
|
534
646
|
|
|
535
647
|
|
|
536
|
-
# How long a provider stays marked down after a 401, 402 or 5xx: five minutes,
|
|
648
|
+
# How long a provider stays marked down after a 401, 402, 403 or 5xx: five minutes,
|
|
537
649
|
# so a dead key or an empty account costs one refusal per five minutes rather
|
|
538
650
|
# than one per prompt, and a provider that comes back is asked again within the
|
|
539
651
|
# same working session. The mark is keyed on the provider, the endpoint and a
|
|
@@ -601,22 +713,28 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
601
713
|
on the prompt alone, and a CLI that wants a login exits instead of waiting -
|
|
602
714
|
with TEZGAH_NESTED set so tezgah's own hooks stand down in the child. The
|
|
603
715
|
answer is read from the CLI's JSON output: the model and the usage it names
|
|
604
|
-
are what the call really used.
|
|
716
|
+
are what the call really used. A CLI with no system-prompt flag gets the
|
|
717
|
+
system prompt in front of the prompt text instead."""
|
|
605
718
|
import shutil
|
|
606
719
|
import subprocess
|
|
607
720
|
import tempfile
|
|
721
|
+
if CHAT_SYSTEM not in SESSION_ARGV[name]:
|
|
722
|
+
prompt = CHAT_SYSTEM + "\n\n" + prompt
|
|
608
723
|
cwd = tempfile.mkdtemp(prefix="tezgah-judge-")
|
|
609
724
|
started = time.monotonic()
|
|
610
725
|
try:
|
|
611
726
|
proc = subprocess.run([exe] + SESSION_ARGV[name] + [prompt], cwd=cwd,
|
|
612
727
|
stdin=subprocess.DEVNULL, capture_output=True,
|
|
613
728
|
text=True, timeout=timeout,
|
|
614
|
-
env=dict(os.environ, TEZGAH_NESTED="1"
|
|
729
|
+
env=dict(os.environ, TEZGAH_NESTED="1",
|
|
730
|
+
**SESSION_ENV.get(name, {})))
|
|
615
731
|
finally:
|
|
616
732
|
shutil.rmtree(cwd, ignore_errors=True)
|
|
617
733
|
if proc.returncode:
|
|
618
|
-
|
|
619
|
-
|
|
734
|
+
# stdout last: codex prints a stdin banner on stderr and its reason as
|
|
735
|
+
# the last JSON event on stdout
|
|
736
|
+
raise SessionFailed("%s exit %d: %s" % (name, proc.returncode, (
|
|
737
|
+
(proc.stderr or "") + (proc.stdout or "")).strip()[-200:]))
|
|
620
738
|
if name == "claude":
|
|
621
739
|
data = json.loads(proc.stdout)
|
|
622
740
|
if data.get("is_error"):
|
|
@@ -627,7 +745,7 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
627
745
|
tokens_in = sum(int(usage.get(k) or 0) for k in (
|
|
628
746
|
"input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"))
|
|
629
747
|
tokens_out = int(usage.get("output_tokens") or 0)
|
|
630
|
-
|
|
748
|
+
elif name == "omp":
|
|
631
749
|
message = None
|
|
632
750
|
for line in proc.stdout.splitlines():
|
|
633
751
|
try:
|
|
@@ -645,6 +763,11 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
645
763
|
usage = message.get("usage") or {}
|
|
646
764
|
tokens_in = sum(int(usage.get(k) or 0) for k in ("input", "cacheRead", "cacheWrite"))
|
|
647
765
|
tokens_out = int(usage.get("output") or 0)
|
|
766
|
+
else:
|
|
767
|
+
model = None # none of the three names the model it ran
|
|
768
|
+
text, tokens_in, tokens_out = _cli_answer(name, proc.stdout)
|
|
769
|
+
if not text.strip():
|
|
770
|
+
raise SessionFailed("%s printed no answer" % name)
|
|
648
771
|
# A chat model may fence its object; the outermost braces are the object.
|
|
649
772
|
answers = _chat_answers(json.loads(text[text.find("{"):text.rfind("}") + 1]), questions)
|
|
650
773
|
return {"answers": answers,
|
|
@@ -653,6 +776,49 @@ def _session_request(name, exe, prompt, timeout, questions):
|
|
|
653
776
|
"model": model}
|
|
654
777
|
|
|
655
778
|
|
|
779
|
+
def _cli_answer(name, out):
|
|
780
|
+
"""(text, input tokens, output tokens) from opencode's, cursor-agent's
|
|
781
|
+
or codex's JSON output, the shapes each printed on 2026-10-09; text is ""
|
|
782
|
+
when no answer came. None of the three names the model it ran. Input counts
|
|
783
|
+
the cache reads and writes, as the claude and omp branches do: opencode and
|
|
784
|
+
cursor-agent report them beside the input (opencode's `total` is the sum of
|
|
785
|
+
all its fields), codex inside `input_tokens` (`cached_input_tokens` is a
|
|
786
|
+
share of it, as `reasoning_output_tokens` is of `output_tokens`)."""
|
|
787
|
+
text, tokens_in, tokens_out = "", 0, 0
|
|
788
|
+
for line in out.splitlines():
|
|
789
|
+
try:
|
|
790
|
+
event = json.loads(line)
|
|
791
|
+
except ValueError:
|
|
792
|
+
continue
|
|
793
|
+
if not isinstance(event, dict):
|
|
794
|
+
continue
|
|
795
|
+
kind = event.get("type")
|
|
796
|
+
part, item = event.get("part") or {}, event.get("item") or {}
|
|
797
|
+
usage = event.get("usage") or {}
|
|
798
|
+
if name == "cursor" and kind == "result":
|
|
799
|
+
if event.get("is_error"):
|
|
800
|
+
raise SessionFailed("cursor: %s" % str(event.get("result"))[:200])
|
|
801
|
+
text = event.get("result") or ""
|
|
802
|
+
tokens_in = sum(int(usage.get(k) or 0) for k in (
|
|
803
|
+
"inputTokens", "cacheReadTokens", "cacheWriteTokens"))
|
|
804
|
+
tokens_out = int(usage.get("outputTokens") or 0)
|
|
805
|
+
elif name == "opencode" and kind == "text":
|
|
806
|
+
text = part.get("text") or text
|
|
807
|
+
elif name == "opencode" and kind == "step_finish":
|
|
808
|
+
tokens = part.get("tokens") or {}
|
|
809
|
+
cache = tokens.get("cache") or {}
|
|
810
|
+
tokens_in += sum(int(v or 0) for v in (tokens.get("input"), cache.get("read"),
|
|
811
|
+
cache.get("write")))
|
|
812
|
+
tokens_out += int(tokens.get("output") or 0) + int(tokens.get("reasoning") or 0)
|
|
813
|
+
elif name == "codex" and kind == "item.completed" \
|
|
814
|
+
and item.get("type") == "agent_message":
|
|
815
|
+
text = item.get("text") or text
|
|
816
|
+
elif name == "codex" and kind == "turn.completed":
|
|
817
|
+
tokens_in += int(usage.get("input_tokens") or 0)
|
|
818
|
+
tokens_out += int(usage.get("output_tokens") or 0)
|
|
819
|
+
return text, tokens_in, tokens_out
|
|
820
|
+
|
|
821
|
+
|
|
656
822
|
def _why(exc):
|
|
657
823
|
"""A failure as one short reason: the HTTP code, else the exception's text."""
|
|
658
824
|
code = getattr(exc, "code", None)
|