@r1z4x/tezgah 1.4.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +120 -1
- package/MANIFEST +0 -1
- package/README.md +6 -4
- package/bin/tezgah-docs +3 -3
- package/bin/tezgah-route +12 -6
- package/bin/tezgah-setup +67 -24
- package/bin/tezgah-status +3 -1
- package/bin/tezgah-task +132 -39
- package/bin/tezgah-taste +49 -31
- package/bin/tezgah-triage +12 -7
- package/docs/architecture.md +1 -1
- package/docs/clarity-baseline.json +7 -7
- package/docs/contract.md +21 -11
- package/docs/environment.md +3 -1
- package/docs/evidence.md +4 -3
- package/docs/glossary.md +1 -1
- package/docs/hosts.md +9 -0
- package/docs/judge.md +95 -27
- package/docs/layers.md +1 -1
- package/docs/models.md +17 -7
- package/docs/operations.md +23 -13
- package/docs/skills.md +1 -1
- package/docs/testing.md +1 -1
- package/docs/vision.md +10 -5
- package/hooks/tezgah_agents.py +149 -105
- package/hooks/tezgah_context.py +42 -7
- package/hooks/tezgah_judge.py +252 -79
- package/hooks/tezgah_models.py +50 -3
- package/hooks/tezgah_paths.py +180 -27
- package/hooks/tezgah_policy.py +41 -10
- package/hooks/tezgah_replay.py +26 -15
- package/hooks/tezgah_task.py +160 -46
- package/hooks/tezgah_taste.py +2 -2
- package/hooks/tezgah_taste_ledger.py +13 -4
- package/hosts/codex/hook.py +5 -0
- package/package.json +1 -2
- package/skills/harness/SKILL.md +4 -0
- package/skills/plan-add/SKILL.md +23 -7
- package/skills/plan-add/render_table.py +3 -2
- package/skills/plan-status/SKILL.md +12 -4
- package/skills/plan-sync/SKILL.md +19 -9
- package/skills/tezgah-contract/SKILL.md +29 -14
- package/agents/tezgah-reviewer.md +0 -70
package/hooks/tezgah_paths.py
CHANGED
|
@@ -499,32 +499,144 @@ def have_typesafe_key():
|
|
|
499
499
|
def have_judge_key():
|
|
500
500
|
"""True when tezgah's own judgement seam can ask anyone: the session's own
|
|
501
501
|
CLI (`session_cli()`), else - when the `fallback` setting lets a third party
|
|
502
|
-
answer -
|
|
503
|
-
two channels (`OPENROUTER_API_KEY`,
|
|
504
|
-
the channels `hooks/tezgah_judge.py`'s
|
|
505
|
-
what `bin/tezgah-triage`,
|
|
506
|
-
|
|
502
|
+
answer - a chosen Jev carrier's credential (`jev_carriers()`) or the
|
|
503
|
+
OpenRouter chat fallback's own two channels (`OPENROUTER_API_KEY`,
|
|
504
|
+
`~/.config/openrouter/key`), which are the channels `hooks/tezgah_judge.py`'s
|
|
505
|
+
`providers()` picks between. This is what `bin/tezgah-triage`,
|
|
506
|
+
`bin/tezgah-docs` and the skill picker spend.
|
|
507
507
|
|
|
508
508
|
Kept next to `have_typesafe_key()` because the two are asked together and a
|
|
509
509
|
report that showed one answer under both questions was wrong in both
|
|
510
|
-
directions - a key file alone
|
|
511
|
-
|
|
512
|
-
|
|
510
|
+
directions - a key file alone serves the seam and not omp. A blank file is
|
|
511
|
+
not a credential here either, so this stays in step with the seam's own
|
|
512
|
+
`strip() or None`."""
|
|
513
513
|
if session_cli():
|
|
514
514
|
return True
|
|
515
515
|
if fallback_policy() == "none":
|
|
516
516
|
return False
|
|
517
|
-
for
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
517
|
+
return bool(openrouter_key() or any(s for _, s in jev_carriers()))
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def key_from(envs, path):
|
|
521
|
+
"""The first non-blank env var of `envs`, else the file at `path` (`~`
|
|
522
|
+
expanded), else None - stripped either way, because a newline inside an
|
|
523
|
+
Authorization header is an injection, not a typo. The file is the channel a
|
|
524
|
+
hook needs: it runs where `~/.zshenv` never exported the variable."""
|
|
525
|
+
for env in envs:
|
|
526
|
+
value = os.environ.get(env, "").strip()
|
|
527
|
+
if value:
|
|
528
|
+
return value
|
|
529
|
+
try:
|
|
530
|
+
with open(os.path.expanduser(path), encoding="utf-8") as fh:
|
|
531
|
+
return fh.read().strip() or None
|
|
532
|
+
except OSError:
|
|
533
|
+
return None
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def omp_login_key(provider):
|
|
537
|
+
"""The API key omp's login store holds for `provider` (`omp auth login`),
|
|
538
|
+
stripped, else None: the newest enabled `api_key` row's `data.key`, read
|
|
539
|
+
read-only. Any error - no store, another schema, a locked file - is None."""
|
|
540
|
+
import sqlite3 # deferred: see have_typesafe_key()
|
|
541
|
+
store = os.path.join(os.path.expanduser("~"), ".omp", "agent", "agent.db")
|
|
542
|
+
try:
|
|
543
|
+
conn = sqlite3.connect("file:%s?mode=ro" % store, uri=True)
|
|
521
544
|
try:
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
545
|
+
rows = conn.execute(
|
|
546
|
+
"select data from auth_credentials where provider = ? and "
|
|
547
|
+
"disabled_cause is null order by id desc", (provider,)).fetchall()
|
|
548
|
+
finally:
|
|
549
|
+
conn.close()
|
|
550
|
+
except (sqlite3.Error, OSError):
|
|
551
|
+
return None
|
|
552
|
+
for (data,) in rows:
|
|
553
|
+
try:
|
|
554
|
+
value = json.loads(data).get("key")
|
|
555
|
+
except (TypeError, ValueError, AttributeError):
|
|
556
|
+
continue
|
|
557
|
+
if isinstance(value, str) and value.strip():
|
|
558
|
+
return value.strip()
|
|
559
|
+
return None
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def typesafe_key():
|
|
563
|
+
"""TypeSafe's credential: TYPESAFE_API_KEY, else `~/.config/typesafe/key`,
|
|
564
|
+
else the active omp login (`omp_login_key("typesafe")`), else None."""
|
|
565
|
+
return (key_from(("TYPESAFE_API_KEY",), "~/.config/typesafe/key")
|
|
566
|
+
or omp_login_key("typesafe"))
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def openrouter_key():
|
|
570
|
+
"""OpenRouter's credential: OPENROUTER_API_KEY, else its key file."""
|
|
571
|
+
return key_from(("OPENROUTER_API_KEY",), "~/.config/openrouter/key")
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def cloudflare_account():
|
|
575
|
+
"""The Workers AI account id: CLOUDFLARE_ACCOUNT_ID, else its file."""
|
|
576
|
+
return key_from(("CLOUDFLARE_ACCOUNT_ID",), "~/.config/cloudflare/account_id")
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def cloudflare_token():
|
|
580
|
+
"""The Workers AI token: JEV_CLOUDFLARE_API_TOKEN, else CLOUDFLARE_API_TOKEN,
|
|
581
|
+
else `~/.config/cloudflare/token`."""
|
|
582
|
+
return key_from(("JEV_CLOUDFLARE_API_TOKEN", "CLOUDFLARE_API_TOKEN"),
|
|
583
|
+
"~/.config/cloudflare/token")
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
# The Jev carrier a user may choose (`jev_choice()`), and the order `auto`
|
|
587
|
+
# tries them in: TypeSafe direct, OpenRouter's System One endpoint, Cloudflare
|
|
588
|
+
# Workers AI, any System One-compatible URL (JEV_API_BASE_URL + JEV_API_KEY,
|
|
589
|
+
# the names the public jev-mcp server reads), then Cloudflare's own System One
|
|
590
|
+
# models Clef and Clef-flash on Workers AI (the same Cloudflare credential).
|
|
591
|
+
JEV_CHOICES = ("auto", "typesafe", "openrouter", "cloudflare", "compatible",
|
|
592
|
+
"clef", "clef-flash")
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def jev_name(value):
|
|
596
|
+
"""A carrier choice as written by a user, lowercased and with the Workers AI
|
|
597
|
+
catalog prefix dropped: `@cf/cloudflare/clef-flash` is what Cloudflare's
|
|
598
|
+
dashboard and docs show, and it names the `clef-flash` carrier."""
|
|
599
|
+
value = value.strip().lower()
|
|
600
|
+
return value[len("@cf/cloudflare/"):] if value.startswith("@cf/cloudflare/") else value
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def jev_choice():
|
|
604
|
+
"""TEZGAH_JEV_PROVIDER, else JEV_PROVIDER (jev-mcp's name), else config.json
|
|
605
|
+
`jev`, else `auto` - through `jev_name`. An unknown value is kept, so it
|
|
606
|
+
resolves to no carrier instead of silently to another one."""
|
|
607
|
+
for env in ("TEZGAH_JEV_PROVIDER", "JEV_PROVIDER"):
|
|
608
|
+
value = jev_name(os.environ.get(env, ""))
|
|
609
|
+
if value:
|
|
610
|
+
return value
|
|
611
|
+
cfg = config()
|
|
612
|
+
value = cfg.get("jev") if isinstance(cfg, dict) else None
|
|
613
|
+
value = jev_name(value) if isinstance(value, str) else ""
|
|
614
|
+
return value or "auto"
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def jev_carriers():
|
|
618
|
+
"""[(carrier, secret), ...] the chosen Jev carriers in the order the seam
|
|
619
|
+
asks them. `auto`: every carrier whose credential resolves. A named one: that
|
|
620
|
+
carrier alone, its secret None when it does not resolve - the seam records
|
|
621
|
+
why rather than substituting another carrier. Only the chosen carrier's
|
|
622
|
+
credential is read: the status line asks this on every redraw, and the
|
|
623
|
+
TypeSafe channel ends in omp's login database."""
|
|
624
|
+
def compatible():
|
|
625
|
+
return (os.environ.get("JEV_API_KEY", "").strip() or None
|
|
626
|
+
if os.environ.get("JEV_API_BASE_URL", "").strip() else None)
|
|
627
|
+
|
|
628
|
+
def workers_ai():
|
|
629
|
+
return cloudflare_token() if cloudflare_account() else None
|
|
630
|
+
found = [("typesafe", typesafe_key), ("jev-openrouter", openrouter_key),
|
|
631
|
+
("jev-cloudflare", workers_ai), ("jev-compatible", compatible),
|
|
632
|
+
("clef", workers_ai), ("clef-flash", workers_ai)]
|
|
633
|
+
choice = jev_choice()
|
|
634
|
+
if choice == "auto":
|
|
635
|
+
return [(name, secret) for name, secret in ((n, read()) for n, read in found)
|
|
636
|
+
if secret]
|
|
637
|
+
name = choice if choice in ("typesafe", "clef", "clef-flash") else "jev-" + choice
|
|
638
|
+
read = dict(found).get(name)
|
|
639
|
+
return [(name, read() if read else None)]
|
|
528
640
|
|
|
529
641
|
|
|
530
642
|
# How far a model call may leave the session's own vendor (config.json
|
|
@@ -544,16 +656,39 @@ def fallback_policy():
|
|
|
544
656
|
return value if value in FALLBACKS else "vendor"
|
|
545
657
|
|
|
546
658
|
|
|
659
|
+
# The session CLIs the judgement seam can ask, in detection order, each with
|
|
660
|
+
# the env vars its host sets in the processes it spawns. omp sets CLAUDECODE
|
|
661
|
+
# too, so it comes first. opencode sets OPENCODE=1 in its own process at start
|
|
662
|
+
# (`process.env.OPENCODE="1"` in the 1.18.31 binary), so its plugin's children
|
|
663
|
+
# and its shell tool inherit it. cursor-agent gives its shell tool CURSOR_AGENT=1
|
|
664
|
+
# and its hooks CURSOR_VERSION (`buildHookEnvironment` in the 2026.10.01
|
|
665
|
+
# bundle). codex gives its shell tool CODEX_THREAD_ID (codex-rs
|
|
666
|
+
# protocol/src/shell_environment.rs `populate_env`); its hooks get only codex's
|
|
667
|
+
# own process env (codex-rs hooks/src/registry.rs), so a codex hook names no
|
|
668
|
+
# session - `judge_cli` picks it there.
|
|
669
|
+
SESSION_CLIS = (("omp", ("OMPCODE",)), ("claude", ("CLAUDECODE",)),
|
|
670
|
+
("opencode", ("OPENCODE",)), ("cursor", ("CURSOR_AGENT", "CURSOR_VERSION")),
|
|
671
|
+
("codex", ("CODEX_THREAD_ID",)))
|
|
672
|
+
|
|
673
|
+
|
|
547
674
|
def session_cli():
|
|
548
|
-
"""
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
675
|
+
"""The session CLI the judgement seam asks - `omp`, `claude`, `opencode`,
|
|
676
|
+
`cursor` or `codex` - else None.
|
|
677
|
+
|
|
678
|
+
TEZGAH_JUDGE_CLI, else config.json `judge_cli`, picks one by name; `auto`
|
|
679
|
+
(the default, and any other value) reads the host this process runs under:
|
|
680
|
+
the first CLI in SESSION_CLIS whose marker is set. Either way the answer is
|
|
681
|
+
None when that CLI's binary is missing - an omp session is not handed to
|
|
682
|
+
whatever `claude` is logged in as, and a picked CLI has no stand-in."""
|
|
683
|
+
bins = {"omp": omp_bin, "claude": claude_bin, "opencode": opencode_bin,
|
|
684
|
+
"cursor": cursor_bin, "codex": codex_bin}
|
|
685
|
+
cfg = config()
|
|
686
|
+
pick = (os.environ.get("TEZGAH_JUDGE_CLI", "").strip()
|
|
687
|
+
or (cfg.get("judge_cli") if isinstance(cfg, dict) else None))
|
|
688
|
+
if not isinstance(pick, str) or pick not in bins:
|
|
689
|
+
pick = next((name for name, markers in SESSION_CLIS
|
|
690
|
+
if any(os.environ.get(m) for m in markers)), None)
|
|
691
|
+
return pick if pick and bins[pick]() else None
|
|
557
692
|
|
|
558
693
|
|
|
559
694
|
def orx_bin():
|
|
@@ -584,6 +719,24 @@ def claude_bin():
|
|
|
584
719
|
return which_user(os.environ.get("TEZGAH_CLAUDE_BIN") or "claude")
|
|
585
720
|
|
|
586
721
|
|
|
722
|
+
def opencode_bin():
|
|
723
|
+
"""The opencode CLI (TEZGAH_OPENCODE_BIN, else `opencode`), or None - the
|
|
724
|
+
lookup `omp_bin()` makes."""
|
|
725
|
+
return which_user(os.environ.get("TEZGAH_OPENCODE_BIN") or "opencode")
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def cursor_bin():
|
|
729
|
+
"""Cursor's agent CLI (TEZGAH_CURSOR_BIN, else `cursor-agent`), or None -
|
|
730
|
+
the lookup `omp_bin()` makes."""
|
|
731
|
+
return which_user(os.environ.get("TEZGAH_CURSOR_BIN") or "cursor-agent")
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def codex_bin():
|
|
735
|
+
"""The Codex CLI (TEZGAH_CODEX_BIN, else `codex`), or None - the lookup
|
|
736
|
+
`omp_bin()` makes."""
|
|
737
|
+
return which_user(os.environ.get("TEZGAH_CODEX_BIN") or "codex")
|
|
738
|
+
|
|
739
|
+
|
|
587
740
|
# The values `reply_lang` (config.json) takes. The value itself is the switch:
|
|
588
741
|
# `tr` (the default, and what an install without the key reads) asks for Turkish
|
|
589
742
|
# replies, `en` for English ones, and `any` drops the Stop language check.
|
package/hooks/tezgah_policy.py
CHANGED
|
@@ -237,7 +237,15 @@ rewrite it plainly.
|
|
|
237
237
|
on a merge. Review first with a FRESH reviewer (never my own read of my own
|
|
238
238
|
code), run the full suite, and if the review is clean and every test passes,
|
|
239
239
|
merge, move the plan to `.tezgah/plans/done/`, and report the outcome. Bring it back
|
|
240
|
-
to the user only on a critical or
|
|
240
|
+
to the user only on a critical or major finding left after round two, or a
|
|
241
|
+
failing test. One review per plan, over its whole diff in the verification
|
|
242
|
+
phase; at most two rounds - only a confirmed critical or major finding opens
|
|
243
|
+
round two, a minor or suggestion one is recorded fix-later, and after round two
|
|
244
|
+
what is left goes to the user or is recorded fix-later, never a third round.
|
|
245
|
+
A round spawns its reviewers in ONE message, one per dimension (correctness
|
|
246
|
+
and contract, security, tests and performance) or file group; a diff under 300
|
|
247
|
+
changed lines gets one reviewer. Per-task work gets tests, not a reviewer, and
|
|
248
|
+
`tezgah-task review` refuses a third round.
|
|
241
249
|
|
|
242
250
|
**Attribution: none, anywhere, ever.** Nothing you persist or publish may name
|
|
243
251
|
the assistant, model, vendor or "AI" as author, co-author, generator or helper -
|
|
@@ -283,9 +291,11 @@ independent work remains. Fan out only when
|
|
|
283
291
|
the subtasks share no mutable file and no interface: if two of them would edit
|
|
284
292
|
the same file, or one's answer decides the other's, keep them in one context or
|
|
285
293
|
sequence them. The shared artifact is the coordination channel, not chatter -
|
|
286
|
-
naming a lead coordinates nothing by itself.
|
|
287
|
-
|
|
288
|
-
|
|
294
|
+
naming a lead coordinates nothing by itself. Only a single item - one file, one
|
|
295
|
+
bounded change, a strictly serial chain - is done directly; a list of
|
|
296
|
+
independent items is never worked one by one on the main thread, however small
|
|
297
|
+
each item is, and that holds at every phase: a plan's tasks, its acceptance
|
|
298
|
+
commands and the sources of a survey fan out the same way.
|
|
289
299
|
A slice that needs its own checkout: inside Orca (the session-start `Orca:`
|
|
290
300
|
line) create it with `orca worktree create --name <slug> --parent-worktree
|
|
291
301
|
active --json` and track it with `orca worktree ps --json` or `tezgah-status
|
|
@@ -293,8 +303,10 @@ active --json` and track it with `orca worktree ps --json` or `tezgah-status
|
|
|
293
303
|
status in Orca; outside Orca, `git worktree add`. Either way the brief names
|
|
294
304
|
absolute paths under that checkout.
|
|
295
305
|
Routing: `{ROUTE_BIN} "<brief>"` names the worker - tezgah-cheap, -standard or
|
|
296
|
-
-frontier
|
|
297
|
-
|
|
306
|
+
-frontier, or the specialist whose whole job the brief is (tezgah-tester,
|
|
307
|
+
-docs, -security, -ui, -researcher, -reviewer) - and the spawn uses that name,
|
|
308
|
+
never the host's generic task agent (overrides for stored data, credentials,
|
|
309
|
+
security first; then a Jev judgement; the static phase table with no key). A worker that answers
|
|
298
310
|
`ESCALATE:` is restarted on tezgah-frontier with the original brief, never
|
|
299
311
|
continued; the main thread never switches model (the cache is per model). A
|
|
300
312
|
code-discovery brief names codegraph's tools (`codegraph callers`, `callees`,
|
|
@@ -330,7 +342,8 @@ guards against.
|
|
|
330
342
|
|
|
331
343
|
### The gate, for anything either tier produces
|
|
332
344
|
Read the diff yourself. Then tests, lint and type check must pass, and anything
|
|
333
|
-
non-trivial gets an independent review before it lands
|
|
345
|
+
non-trivial gets an independent review before it lands - once per plan, not
|
|
346
|
+
per task or commit. A delegate's output is
|
|
334
347
|
a draft until the router has evidence; "the agent said it works" is not
|
|
335
348
|
evidence.
|
|
336
349
|
|
|
@@ -897,8 +910,14 @@ project root and never `git add -f` anything under `.tezgah/`. Off:
|
|
|
897
910
|
**Parallel by default: fan out independent items.** A request with two or
|
|
898
911
|
more independent items - a numbered or bulleted list, several files or areas, a
|
|
899
912
|
todo list whose entries share no file and no interface - is delegated without
|
|
900
|
-
being asked: one subagent per item
|
|
901
|
-
|
|
913
|
+
being asked: count the independent items, spawn one subagent per item in ONE
|
|
914
|
+
message, and keep the main thread for decomposing, integrating, verifying and
|
|
915
|
+
reporting. It holds at every phase, not only the first message: a plan's tasks,
|
|
916
|
+
its acceptance commands and the sources of a survey are items too, and checks
|
|
917
|
+
that share no state are tool calls in one message, never a queue (one suite run
|
|
918
|
+
at a time: two runs of one suite storm the CPU and race on its temp state).
|
|
919
|
+
No item is too small to delegate while another runs beside it; only a single
|
|
920
|
+
item is done inline. Every writing slice gets
|
|
902
921
|
its own branch and checkout - inside Orca `orca worktree create --name <slug>
|
|
903
922
|
--parent-worktree active --json`, else `git worktree add -b <branch> <absolute
|
|
904
923
|
path>`; its brief names every read, edit and shell `cwd` as an absolute path
|
|
@@ -907,7 +926,8 @@ not grow. Read-only slices need no worktree. Pick each agent by task class from
|
|
|
907
926
|
the specialists line; `{ROUTE_BIN}` names the tier for code work. Serialize
|
|
908
927
|
only a real dependency - one slice's output is another's input, or two slices
|
|
909
928
|
edit one file - and say which. Never wait idle on a spawned slice while
|
|
910
|
-
independent work remains.
|
|
929
|
+
independent work remains. A subagent does its own slice and never fans out
|
|
930
|
+
again. Off: `orchestrate-off`.
|
|
911
931
|
|
|
912
932
|
**Kill switches:** each one removes its own rule from this text, not just the
|
|
913
933
|
status mark. `~/.config/tezgah/`: `exec-mode.off`, `orchestrate-off`,
|
|
@@ -928,6 +948,17 @@ The ponytail intensity level is not a switch: `tezgah-pony lite|full|ultra`.
|
|
|
928
948
|
# CORE_RULES labels in hooks/tezgah_context.py.
|
|
929
949
|
CONDITIONAL_KEYS = ("spec", "consult", "research", "product", "graph", "fanout")
|
|
930
950
|
|
|
951
|
+
# The fan-out rule's action, which the one-line repeat of an armed paragraph
|
|
952
|
+
# (hooks/tezgah_context.armed_again) carries: a bare "Armed again" named a rule
|
|
953
|
+
# the session had to recall, and the second list of a session was worked one
|
|
954
|
+
# item at a time. It is the paragraph's own clause, cut to one sentence.
|
|
955
|
+
FANOUT_ACTION = ("count the independent items, spawn one subagent per item in "
|
|
956
|
+
"ONE message, each the agent `tezgah-route` names - a specialist "
|
|
957
|
+
"by name, never the generic task agent - (each writing slice in "
|
|
958
|
+
"its own worktree, absolute paths), and keep the main thread for "
|
|
959
|
+
"integrating and verifying; a subagent does its own slice and "
|
|
960
|
+
"never fans out again.")
|
|
961
|
+
|
|
931
962
|
# The always-on replacement for the conditional paragraphs: one line each so a
|
|
932
963
|
# host without a per-turn hook still knows the rule exists and where the full
|
|
933
964
|
# text lives.
|
package/hooks/tezgah_replay.py
CHANGED
|
@@ -6,7 +6,7 @@ code, a blind label sheet drawn from them, and the report that reads the labels
|
|
|
6
6
|
tezgah-gate replay [--cutoff WHEN] [--since WHEN] [--json]
|
|
7
7
|
tezgah-gate replay --report [--run DIR] [--labels FILE ...] [--json]
|
|
8
8
|
tezgah-gate replay --sheet [--run DIR] [--seed N] [--rules FILE]
|
|
9
|
-
tezgah-gate replay --label-model --prompt FILE [--provider
|
|
9
|
+
tezgah-gate replay --label-model --prompt FILE [--provider jev|deepseek]
|
|
10
10
|
[--model ID] [--run DIR]
|
|
11
11
|
|
|
12
12
|
**Corpus.** Every ledger under the cache's `evidence/` is read read-only; a
|
|
@@ -864,9 +864,9 @@ def report(run, label_files=()):
|
|
|
864
864
|
|
|
865
865
|
# ----------------------------------------------------------------- raters --
|
|
866
866
|
|
|
867
|
-
# Rater 2 (H2 amendment 2026-10-07):
|
|
868
|
-
#
|
|
869
|
-
# `bin/tezgah-triage::PRICE_PER_MILLION` quotes.
|
|
867
|
+
# Rater 2 (H2 amendment 2026-10-07): the Jev model through the judge seam, any
|
|
868
|
+
# Jev carrier (`tezgah_judge.JEV_CARRIERS`), billed per million input tokens and
|
|
869
|
+
# nothing for output, at the price `bin/tezgah-triage::PRICE_PER_MILLION` quotes.
|
|
870
870
|
RATER_MODEL = "jev-latest"
|
|
871
871
|
JEV_PRICE_PER_MILLION = 0.042
|
|
872
872
|
LABEL_WORKERS = 4
|
|
@@ -903,9 +903,10 @@ def _deepseek(secret, model):
|
|
|
903
903
|
return ask
|
|
904
904
|
|
|
905
905
|
|
|
906
|
-
def label_model(run, prompt_file, provider="
|
|
906
|
+
def label_model(run, prompt_file, provider="jev", model=None):
|
|
907
907
|
"""One `choice` question per blind sheet row, the row's text redacted
|
|
908
|
-
(`ti.redact`). `typesafe` asks
|
|
908
|
+
(`ti.redact`). `jev` (old name `typesafe`) asks the Jev model through the
|
|
909
|
+
judge seam, whichever carrier resolves, and writes
|
|
909
910
|
`labels-model.jsonl` (rater 2); `deepseek` asks DeepSeek's chat endpoint and
|
|
910
911
|
writes `labels-deepseek.jsonl` (rater 1). Rows already labelled there are not
|
|
911
912
|
asked again; a reply without a label among the prompt's criteria stays
|
|
@@ -920,13 +921,15 @@ def label_model(run, prompt_file, provider="typesafe", model=None):
|
|
|
920
921
|
ask, rater = _deepseek(secret, model or "deepseek-v4-pro"), "deepseek"
|
|
921
922
|
price = DEEPSEEK_PRICE_PER_MILLION
|
|
922
923
|
else:
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
"
|
|
924
|
+
if tj.tp.off("judge-off") or not tj.named(("jev",)):
|
|
925
|
+
raise SystemExit("label-model: the judge seam must answer through a Jev "
|
|
926
|
+
"carrier (TypeSafe, OpenRouter System One, Cloudflare "
|
|
927
|
+
"Workers AI or JEV_API_BASE_URL; TEZGAH_JEV_PROVIDER picks "
|
|
928
|
+
"one), and none resolves")
|
|
927
929
|
|
|
928
930
|
def ask(state, questions):
|
|
929
|
-
return tj.ask(state, questions, model=model or RATER_MODEL, timeout=60
|
|
931
|
+
return tj.ask(state, questions, model=model or RATER_MODEL, timeout=60,
|
|
932
|
+
only=("jev",))
|
|
930
933
|
rater, price = "model", (JEV_PRICE_PER_MILLION, 0.0)
|
|
931
934
|
with open(prompt_file, encoding="utf-8") as fh:
|
|
932
935
|
prompt = json.load(fh)
|
|
@@ -942,7 +945,7 @@ def label_model(run, prompt_file, provider="typesafe", model=None):
|
|
|
942
945
|
label = tj.choice(result, "label")
|
|
943
946
|
return row, result, label if label in question["criteria"] else None
|
|
944
947
|
|
|
945
|
-
tokens, out_tokens, models, new = 0, 0, Counter(), 0
|
|
948
|
+
tokens, out_tokens, models, new, spent, prices = 0, 0, Counter(), 0, 0.0, set()
|
|
946
949
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
|
|
947
950
|
os.fchmod(fd, 0o600)
|
|
948
951
|
# Each answered row is appended as it arrives, so a run that dies mid-way
|
|
@@ -953,6 +956,12 @@ def label_model(run, prompt_file, provider="typesafe", model=None):
|
|
|
953
956
|
tokens += result["usage"]["input_tokens"]
|
|
954
957
|
out_tokens += result["usage"]["output_tokens"]
|
|
955
958
|
models["%s/%s" % (result["provider"], result["model"])] += 1
|
|
959
|
+
# a typed carrier is billed at its own price (Clef's is not Jev's)
|
|
960
|
+
typed = tj.input_price(result["provider"])
|
|
961
|
+
row_price = (typed, 0.0) if typed is not None else price
|
|
962
|
+
prices.add(row_price)
|
|
963
|
+
spent += (result["usage"]["input_tokens"] * row_price[0]
|
|
964
|
+
+ result["usage"]["output_tokens"] * row_price[1]) / 1e6
|
|
956
965
|
if label:
|
|
957
966
|
fh.write(json.dumps({"set": row["set"], "n": row["n"], "label": label,
|
|
958
967
|
"rater": rater, "model": result["model"],
|
|
@@ -962,8 +971,9 @@ def label_model(run, prompt_file, provider="typesafe", model=None):
|
|
|
962
971
|
new += 1
|
|
963
972
|
return {"path": path, "asked": len(todo), "labelled": new,
|
|
964
973
|
"total": len(done) + new, "input_tokens": tokens, "output_tokens": out_tokens,
|
|
965
|
-
|
|
966
|
-
"
|
|
974
|
+
# the printed formula names the rate the rows were billed at, when one
|
|
975
|
+
"price": prices.pop() if len(prices) == 1 else price,
|
|
976
|
+
"cost_usd": spent,
|
|
967
977
|
"models": dict(models)}
|
|
968
978
|
|
|
969
979
|
|
|
@@ -1034,7 +1044,8 @@ def main(argv):
|
|
|
1034
1044
|
p.add_argument("--report", action="store_true")
|
|
1035
1045
|
p.add_argument("--label-model", action="store_true")
|
|
1036
1046
|
p.add_argument("--prompt")
|
|
1037
|
-
|
|
1047
|
+
# `typesafe` is the old name of `jev`, kept for the commands already written down
|
|
1048
|
+
p.add_argument("--provider", choices=("jev", "typesafe", "deepseek"), default="jev")
|
|
1038
1049
|
p.add_argument("--model")
|
|
1039
1050
|
p.add_argument("--sheet", action="store_true")
|
|
1040
1051
|
p.add_argument("--run")
|