@r1z4x/tezgah 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -1
- package/MANIFEST +2 -0
- package/bin/tezgah-taste +509 -5
- package/docs/clarity-baseline.json +1 -1
- package/docs/index.json +5 -2
- package/docs/judge.md +23 -9
- package/docs/operations.md +79 -21
- package/hooks/projects-posttooluse.py +7 -0
- package/hooks/tezgah_context.py +28 -1
- package/hooks/tezgah_judge.py +53 -14
- package/hooks/tezgah_taste.py +86 -10
- package/hooks/tezgah_taste_ledger.py +628 -0
- package/hosts/codex/hook.py +6 -1
- package/hosts/cursor/hook.py +6 -1
- package/hosts/omp/hook.py +6 -0
- package/hosts/opencode/plugins/tezgah.js +10 -6
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,36 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [1.3.0] - 2026-10-07
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Armed taste learns by itself.** Session start runs `tezgah-taste learn` in
|
|
14
|
+
the background when the repository's signals grew, at most once an hour. It
|
|
15
|
+
runs only when a TypeSafe key resolves, with the new `--no-fallback`, so it
|
|
16
|
+
asks no other provider. It does not run on Windows yet. Its log goes under
|
|
17
|
+
the cache dir, in `taste-learn/`.
|
|
18
|
+
|
|
19
|
+
## [1.2.0] - 2026-10-07
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- **Taste learns and applies, not only records.** Arm it with `taste-on`.
|
|
24
|
+
`tezgah-taste learn` then turns each correction after a writing turn into one
|
|
25
|
+
typed decision, and only TypeSafe may give it. The decision names a
|
|
26
|
+
preference, a defect or neither. It picks one of eleven fixed categories and
|
|
27
|
+
a scope, and relates the correction to each learning already held. A ledger
|
|
28
|
+
computes each learning's confidence from its evidence and forgets it over
|
|
29
|
+
time. A contradiction flags a learning and never overwrites it. A learning
|
|
30
|
+
goes active in its second session. Session start injects active learnings as
|
|
31
|
+
rules or hints. Every host notes the in-scope ones on a write, each once per
|
|
32
|
+
session. A rule needs a measured calibration bound. A defect never enters the
|
|
33
|
+
ledger. A before/after gate stops injection when corrections do not fall.
|
|
34
|
+
`list`, `accept`, `reject`, `edit` and `export` give the user control, and
|
|
35
|
+
`learn --from-transcripts` bootstraps from history. The judge seam gains
|
|
36
|
+
`only=` and a `text` question. Two commands never rewrite the ledger at once:
|
|
37
|
+
a second `learn`, `accept`, `reject` or `edit` exits 2 while one runs.
|
|
38
|
+
|
|
9
39
|
## [1.1.0] - 2026-10-07
|
|
10
40
|
|
|
11
41
|
### Added
|
|
@@ -803,6 +833,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
803
833
|
the copy could not find `hooks/` and died on import. It now links into the
|
|
804
834
|
installed tree, and also installs the command as `tezgah`.
|
|
805
835
|
|
|
806
|
-
[Unreleased]: https://github.com/r1z4x/tezgah/compare/v1.
|
|
836
|
+
[Unreleased]: https://github.com/r1z4x/tezgah/compare/v1.3.0...HEAD
|
|
837
|
+
[1.3.0]: https://github.com/r1z4x/tezgah/releases/tag/v1.3.0
|
|
838
|
+
[1.2.0]: https://github.com/r1z4x/tezgah/releases/tag/v1.2.0
|
|
807
839
|
[1.1.0]: https://github.com/r1z4x/tezgah/releases/tag/v1.1.0
|
|
808
840
|
[1.0.0]: https://github.com/r1z4x/tezgah/releases/tag/v1.0.0
|
package/MANIFEST
CHANGED
|
@@ -144,6 +144,7 @@ hooks/tezgah_stack.py
|
|
|
144
144
|
hooks/tezgah_stopspec.py
|
|
145
145
|
hooks/tezgah_task.py
|
|
146
146
|
hooks/tezgah_taste.py
|
|
147
|
+
hooks/tezgah_taste_ledger.py
|
|
147
148
|
hooks/tezgah_untrusted.py
|
|
148
149
|
hooks/tezgah_update.py
|
|
149
150
|
hosts/codex/hook.py
|
|
@@ -752,6 +753,7 @@ tests/test_switch_latching.py
|
|
|
752
753
|
tests/test_task.py
|
|
753
754
|
tests/test_taste.py
|
|
754
755
|
tests/test_taste_cli.py
|
|
756
|
+
tests/test_taste_ledger.py
|
|
755
757
|
tests/test_tezgah_mcp_wiring.py
|
|
756
758
|
tests/test_triage.py
|
|
757
759
|
tests/test_untrusted.py
|
package/bin/tezgah-taste
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""tezgah-taste - mine and
|
|
2
|
+
"""tezgah-taste - mine, measure, learn and apply the coding-taste signals.
|
|
3
3
|
|
|
4
4
|
tezgah-taste mine [--host omp|claude|all] [--root DIR] [--stats] [--out FILE]
|
|
5
5
|
tezgah-taste measure --samples FILE --labels FILE [--model M] [--json]
|
|
6
6
|
tezgah-taste rate --in MINED [--limit N] [--json]
|
|
7
|
+
tezgah-taste learn [--repo DIR] [--from-transcripts] [--host omp|claude|all]
|
|
8
|
+
[--limit N] [--no-fallback] [--json]
|
|
9
|
+
tezgah-taste list [--repo DIR] [--all] [--json]
|
|
10
|
+
tezgah-taste show|accept|reject ID [--repo DIR]
|
|
11
|
+
tezgah-taste edit ID --text TEXT [--repo DIR]
|
|
12
|
+
tezgah-taste export [--repo DIR]
|
|
13
|
+
tezgah-taste label [ID LABEL] [--n N] [--repo DIR]
|
|
14
|
+
tezgah-taste calibrate|gate [--repo DIR] [--json]
|
|
7
15
|
|
|
8
16
|
`mine` reads the session transcripts on this machine - omp's top-level session
|
|
9
17
|
files under `~/.omp/agent/sessions/*/` (a nested file is a subagent session) and
|
|
@@ -25,13 +33,36 @@ writing-turn prompt, per repo and overall. Both send the sample text to the
|
|
|
25
33
|
judge's provider (`hooks/tezgah_judge.py` names it); with no credential or
|
|
26
34
|
`judge-off` set they send nothing and exit 2.
|
|
27
35
|
|
|
36
|
+
`learn` is taste-1's pipeline on tezgah (research line taste-architecture,
|
|
37
|
+
design T1P). Each prompt after a writing turn - from the capture rows in
|
|
38
|
+
`<repo>/.tezgah/taste/signals.jsonl`, or with `--from-transcripts` from this
|
|
39
|
+
repository's omp and Claude transcripts - becomes one structured signal built by
|
|
40
|
+
code: the prompt, the files written, the agent's edits and the user's own edit
|
|
41
|
+
between two agent writes. One typed decision per signal, required from TypeSafe
|
|
42
|
+
(`only=("typesafe",)`): preference, defect or none; the category; the scope; and
|
|
43
|
+
how it relates to each learning held. A decision any other provider gave is
|
|
44
|
+
recorded `unverified` and never applied. A defect goes to `defects.jsonl`, never
|
|
45
|
+
into the ledger. The ledger itself is `hooks/tezgah_taste_ledger.py`. A
|
|
46
|
+
learning that becomes active gets one readable line from a generative provider
|
|
47
|
+
(a `text` question; the user's own words when none answers).
|
|
48
|
+
|
|
49
|
+
`list`, `show`, `accept`, `reject` and `edit` are the user's control; `export`
|
|
50
|
+
writes the accepted learnings into AGENTS.md. `label` draws an independent
|
|
51
|
+
random sample of decided signals for the user to label, `calibrate` prints the
|
|
52
|
+
Clopper-Pearson bound a rule needs, and `gate` runs the pooled before/after test
|
|
53
|
+
that stops injection when the corrections do not fall.
|
|
54
|
+
|
|
28
55
|
Exit codes: 0 done, 2 misuse or no judge.
|
|
29
56
|
"""
|
|
30
57
|
import argparse
|
|
31
58
|
import datetime
|
|
59
|
+
import difflib
|
|
32
60
|
import glob
|
|
61
|
+
import hashlib
|
|
33
62
|
import json
|
|
34
63
|
import os
|
|
64
|
+
import random
|
|
65
|
+
import subprocess
|
|
35
66
|
import sys
|
|
36
67
|
|
|
37
68
|
HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
|
@@ -39,6 +70,7 @@ sys.path.insert(0, os.path.join(HERE, "hooks"))
|
|
|
39
70
|
import tezgah_integrity as ti # noqa: E402
|
|
40
71
|
import tezgah_judge # noqa: E402
|
|
41
72
|
import tezgah_models # noqa: E402
|
|
73
|
+
import tezgah_taste_ledger as tl # noqa: E402
|
|
42
74
|
|
|
43
75
|
WRITE_TOOLS = {"omp": {"edit", "write"},
|
|
44
76
|
"claude": {"Edit", "Write", "MultiEdit", "NotebookEdit"}}
|
|
@@ -282,7 +314,7 @@ def read_jsonl(path):
|
|
|
282
314
|
raise SystemExit(misuse("cannot read %s: %s" % (path, exc)))
|
|
283
315
|
|
|
284
316
|
|
|
285
|
-
def classify(texts, model):
|
|
317
|
+
def classify(texts, model, only=None):
|
|
286
318
|
"""One label per text (None when the judge gave none) and the usage.
|
|
287
319
|
|
|
288
320
|
The rule rides the state, so each request pays for it once and the
|
|
@@ -294,7 +326,7 @@ def classify(texts, model):
|
|
|
294
326
|
"instructions": "Label this prompt under the rule "
|
|
295
327
|
"in the state:\n" + ti.redact(text)}
|
|
296
328
|
for i, text in enumerate(chunk)}
|
|
297
|
-
result = tezgah_judge.ask(RULE, questions, model=model, timeout=120)
|
|
329
|
+
result = tezgah_judge.ask(RULE, questions, model=model, timeout=120, only=only)
|
|
298
330
|
if result is not None:
|
|
299
331
|
tokens_in += result["usage"]["input_tokens"]
|
|
300
332
|
tokens_out += result["usage"]["output_tokens"]
|
|
@@ -336,7 +368,8 @@ def cmd_measure(args):
|
|
|
336
368
|
return misuse("no sample joins a label on (set, n)")
|
|
337
369
|
if not tezgah_judge.available():
|
|
338
370
|
return misuse("no judge: no credential resolves or judge-off is set; nothing sent")
|
|
339
|
-
got, usage = classify([text for text, _gold in pairs], args.model
|
|
371
|
+
got, usage = classify([text for text, _gold in pairs], args.model,
|
|
372
|
+
only=("typesafe",) if args.typed else None)
|
|
340
373
|
confusion = {g: {p: 0 for p in CLASSES} for g in CLASSES}
|
|
341
374
|
unjudged = 0
|
|
342
375
|
for (_text, gold), pred in zip(pairs, got):
|
|
@@ -396,6 +429,433 @@ def cmd_rate(args):
|
|
|
396
429
|
print(usage_line(usage))
|
|
397
430
|
return 0
|
|
398
431
|
|
|
432
|
+
# ------------------------------------------------------------ learn, apply ----
|
|
433
|
+
|
|
434
|
+
SCOPE_CRITERIA = {
|
|
435
|
+
"path": "only files under the directory the agent wrote",
|
|
436
|
+
"language": "files of the language or extension the agent wrote",
|
|
437
|
+
"repository": "everything in this repository",
|
|
438
|
+
"user": "every repository this user works in"}
|
|
439
|
+
RELATION_CRITERIA = {
|
|
440
|
+
"supports": "the prompt asks for the same preference as this learning",
|
|
441
|
+
"contradicts": "the prompt asks for the opposite of this learning",
|
|
442
|
+
"narrows": "the prompt keeps this learning but makes an exception for a narrower place",
|
|
443
|
+
"unrelated": "the prompt is about something else"}
|
|
444
|
+
CATEGORY_CRITERIA = {c: c.replace("-", " ") for c in tl.CATEGORIES}
|
|
445
|
+
HELD_MAX = 30
|
|
446
|
+
DIFF_MAX = 1500
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def repo_root(path):
|
|
450
|
+
out = subprocess.run(["git", "-C", path, "rev-parse", "--show-toplevel"],
|
|
451
|
+
capture_output=True, text=True)
|
|
452
|
+
return out.stdout.strip() if out.returncode == 0 and out.stdout.strip() else None
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def sig_id(*parts):
|
|
456
|
+
return hashlib.sha1("\0".join(str(p) for p in parts).encode()).hexdigest()[:12]
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def iso_day(ts):
|
|
460
|
+
"""A capture row's ISO timestamp as days since the epoch, or None."""
|
|
461
|
+
try:
|
|
462
|
+
return datetime.datetime.fromisoformat(str(ts).replace("Z", "+00:00")) \
|
|
463
|
+
.timestamp() / 86400.0
|
|
464
|
+
except ValueError:
|
|
465
|
+
return None
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _snapshots():
|
|
469
|
+
"""{absolute path: [(ts, id, after)]} over the snapshot store, oldest first."""
|
|
470
|
+
import tezgah_snapshot as ts
|
|
471
|
+
out = {}
|
|
472
|
+
for d in ts._dirs():
|
|
473
|
+
try:
|
|
474
|
+
with open(os.path.join(d, "meta.json"), encoding="utf-8") as fh:
|
|
475
|
+
meta = json.load(fh)
|
|
476
|
+
except (OSError, ValueError):
|
|
477
|
+
continue
|
|
478
|
+
if isinstance(meta, dict) and meta.get("path"):
|
|
479
|
+
out.setdefault(meta["path"], []).append(
|
|
480
|
+
(int(meta.get("ts") or 0), os.path.basename(d), bool(meta.get("after"))))
|
|
481
|
+
for rows in out.values():
|
|
482
|
+
rows.sort()
|
|
483
|
+
return out
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _read_blob(sid):
|
|
487
|
+
import tezgah_snapshot as ts
|
|
488
|
+
try:
|
|
489
|
+
with open(ts._blob(sid), "rb") as fh:
|
|
490
|
+
return fh.read().decode("utf-8", "replace")
|
|
491
|
+
except OSError:
|
|
492
|
+
return None
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def user_diff(repo, rel, after_sid, snaps):
|
|
496
|
+
"""The user's own change to `rel` after the agent's write: the after-blob
|
|
497
|
+
against the pre-write bytes of the next agent write to it, or against the
|
|
498
|
+
file on disk when none came. "" when nothing changed in between."""
|
|
499
|
+
after = _read_blob(after_sid)
|
|
500
|
+
if after is None:
|
|
501
|
+
return ""
|
|
502
|
+
apath = os.path.realpath(os.path.join(repo, rel))
|
|
503
|
+
rows = snaps.get(apath) or []
|
|
504
|
+
mine = next((t for t, sid, _a in rows if sid == after_sid), None)
|
|
505
|
+
later = next((sid for t, sid, a in rows if mine is not None and t >= mine
|
|
506
|
+
and not a and sid != after_sid), None)
|
|
507
|
+
if later:
|
|
508
|
+
now = _read_blob(later)
|
|
509
|
+
else:
|
|
510
|
+
try:
|
|
511
|
+
with open(apath, encoding="utf-8", errors="replace") as fh:
|
|
512
|
+
now = fh.read()
|
|
513
|
+
except OSError:
|
|
514
|
+
return ""
|
|
515
|
+
if now is None or now == after:
|
|
516
|
+
return ""
|
|
517
|
+
diff = "".join(difflib.unified_diff(after.splitlines(True), now.splitlines(True),
|
|
518
|
+
"agent/" + rel, "user/" + rel, n=1))
|
|
519
|
+
return ti.redact(diff)[:DIFF_MAX]
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def capture_signals(repo):
|
|
523
|
+
"""One signal per prompt that followed a writing turn, from the capture rows."""
|
|
524
|
+
snaps, turns, out = None, {}, []
|
|
525
|
+
for row in tl.rows(os.path.join(tl.store_dir(repo), "signals.jsonl")):
|
|
526
|
+
session = row.get("session") or ""
|
|
527
|
+
turn = turns.setdefault(session, {"edits": [], "after": []})
|
|
528
|
+
if row.get("kind") == "edit":
|
|
529
|
+
turn["edits"].append(row)
|
|
530
|
+
elif row.get("kind") == "after":
|
|
531
|
+
turn["after"].append(row)
|
|
532
|
+
elif row.get("kind") == "prompt":
|
|
533
|
+
if turn["edits"]:
|
|
534
|
+
if snaps is None:
|
|
535
|
+
snaps = _snapshots()
|
|
536
|
+
diffs = [d for d in (user_diff(repo, a.get("path"), a.get("snapshot"), snaps)
|
|
537
|
+
for a in turn["after"] if a.get("path")) if d]
|
|
538
|
+
out.append({
|
|
539
|
+
"id": sig_id(session, row.get("ts"), row.get("text")),
|
|
540
|
+
"at": iso_day(row.get("ts")),
|
|
541
|
+
"session": session, "text": row.get("text") or "",
|
|
542
|
+
"paths": sorted({e.get("path") for e in turn["edits"] if e.get("path")}),
|
|
543
|
+
"edits": [(e.get("path"), (e.get("old") or "")[:300],
|
|
544
|
+
(e.get("new") or "")[:300]) for e in turn["edits"][-4:]],
|
|
545
|
+
"diffs": diffs})
|
|
546
|
+
turns[session] = {"edits": [], "after": []}
|
|
547
|
+
return out
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _call_path(args):
|
|
551
|
+
for key in ("path", "file_path", "notebook_path"):
|
|
552
|
+
if isinstance(args.get(key), str):
|
|
553
|
+
return args[key]
|
|
554
|
+
return None
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def transcript_signals(repo, host):
|
|
558
|
+
"""The same signals from this repository's top-level host transcripts: the
|
|
559
|
+
prompt and the files written; no edit text or user diff (the history
|
|
560
|
+
bootstrap, Command Code's `/learn-taste`)."""
|
|
561
|
+
out = []
|
|
562
|
+
for name in (("omp", "claude") if host == "all" else (host,)):
|
|
563
|
+
for path, cwd in sessions(name, os.path.dirname(repo)):
|
|
564
|
+
if cwd != repo and not cwd.startswith(repo + os.sep):
|
|
565
|
+
continue
|
|
566
|
+
session = session_id(name, path)[0] or os.path.basename(path)
|
|
567
|
+
written, n, k, at = [], 0, 0, None
|
|
568
|
+
for event in HOSTS[name][2](path, calls=True):
|
|
569
|
+
if event[0] in ("call", "reply") and isinstance(event[-1], (int, float)):
|
|
570
|
+
at = event[-1] / 86400.0
|
|
571
|
+
if event[0] == "call" and event[1] in WRITE_TOOLS[name]:
|
|
572
|
+
target = _call_path(event[2])
|
|
573
|
+
if target:
|
|
574
|
+
rel = os.path.relpath(os.path.join(cwd, target), repo)
|
|
575
|
+
if not rel.startswith(".."):
|
|
576
|
+
written.append(rel)
|
|
577
|
+
elif event[0] == "write":
|
|
578
|
+
n += 1
|
|
579
|
+
elif event[0] == "user":
|
|
580
|
+
if n and not harness_prompt(event[1]):
|
|
581
|
+
text = ti.redact(event[1])[:CUT]
|
|
582
|
+
# keyed inside its own session, so another session's
|
|
583
|
+
# new turns never shift this id and re-decide it
|
|
584
|
+
k += 1
|
|
585
|
+
out.append({"id": sig_id(name, session, k, text), "at": at,
|
|
586
|
+
"session": session, "text": text,
|
|
587
|
+
"paths": sorted(set(written)), "edits": [], "diffs": []})
|
|
588
|
+
written, n = [], 0
|
|
589
|
+
return out
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def held(ledgers):
|
|
593
|
+
"""The learnings a new signal is compared against: every one not retired,
|
|
594
|
+
the most supported first, capped so one call stays one call."""
|
|
595
|
+
alive = [v for v in tl.every(ledgers).values() if v["state"] != "retired"]
|
|
596
|
+
alive.sort(key=lambda v: -v.get("r", 0.0))
|
|
597
|
+
return alive[:HELD_MAX]
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def decision_request(sig, learnings):
|
|
601
|
+
state = ["A user prompt that followed a turn in which a coding agent wrote files.",
|
|
602
|
+
"Prompt: " + ti.redact(sig["text"])[:CUT]]
|
|
603
|
+
if sig["paths"]:
|
|
604
|
+
state.append("Files the agent wrote: " + ", ".join(sig["paths"][:12]))
|
|
605
|
+
for path, old, new in sig.get("edits") or ():
|
|
606
|
+
state.append("Agent edit to %s:\n- %s\n+ %s" % (path, ti.redact(old), ti.redact(new)))
|
|
607
|
+
for diff in sig.get("diffs") or ():
|
|
608
|
+
state.append("The user's own edit after the agent's write:\n" + diff)
|
|
609
|
+
questions = {
|
|
610
|
+
"kind": {"type": "choice", "instructions": RULE, "criteria": CRITERIA},
|
|
611
|
+
"category": {"type": "choice", "criteria": CATEGORY_CRITERIA,
|
|
612
|
+
"instructions": "If the prompt states a coding preference, which "
|
|
613
|
+
"category does it belong to?"},
|
|
614
|
+
"scope": {"type": "choice", "criteria": SCOPE_CRITERIA,
|
|
615
|
+
"instructions": "Where should this preference apply, judging only from "
|
|
616
|
+
"what the prompt says?"}}
|
|
617
|
+
for learning in learnings:
|
|
618
|
+
questions["rel_" + learning["id"]] = {
|
|
619
|
+
"type": "choice", "criteria": RELATION_CRITERIA,
|
|
620
|
+
"instructions": "How does the prompt relate to this learning (%s, %s): %s"
|
|
621
|
+
% (learning["category"], learning["scope"],
|
|
622
|
+
" ".join(learning["text"].split())[:300])}
|
|
623
|
+
return "\n\n".join(state), questions
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def picked(result, qid, default):
|
|
627
|
+
choice = tezgah_judge.choice(result, qid)
|
|
628
|
+
if choice is None:
|
|
629
|
+
return default, 0.0
|
|
630
|
+
p = tezgah_judge.noul(result, qid, choice)
|
|
631
|
+
return choice, (p if p is not None else 0.5)
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def decide(sig, learnings, fallback=True):
|
|
635
|
+
"""(decision, result): the typed decision, TypeSafe required; when it cannot
|
|
636
|
+
answer, the seam's own providers, recorded unverified - only on a signal's
|
|
637
|
+
first try, since an unverified answer is never applied and a rerun would pay
|
|
638
|
+
for it again."""
|
|
639
|
+
state, questions = decision_request(sig, learnings)
|
|
640
|
+
result = tezgah_judge.ask(state, questions, only=("typesafe",), timeout=60)
|
|
641
|
+
if result is None and fallback:
|
|
642
|
+
result = tezgah_judge.ask(state, questions, timeout=120)
|
|
643
|
+
if result is None:
|
|
644
|
+
return None, None
|
|
645
|
+
dec = {"kind": picked(result, "kind", "none"),
|
|
646
|
+
"category": picked(result, "category", ""),
|
|
647
|
+
"scope": picked(result, "scope", "repository"),
|
|
648
|
+
"relations": {lr["id"]: picked(result, "rel_" + lr["id"], "unrelated")
|
|
649
|
+
for lr in learnings},
|
|
650
|
+
"verified": result.get("provider") == "typesafe"}
|
|
651
|
+
return dec, result
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def write_line(learning):
|
|
655
|
+
"""One readable line for a learning that became active, from a generative
|
|
656
|
+
provider; None when none answers (the user's own words stay)."""
|
|
657
|
+
evidence = " ".join(learning["text"].split())[:600]
|
|
658
|
+
result = tezgah_judge.ask(
|
|
659
|
+
"A user corrected a coding agent with: " + evidence,
|
|
660
|
+
{"line": {"type": "text", "instructions":
|
|
661
|
+
"Write the coding preference this correction states as one imperative "
|
|
662
|
+
"line for a coding agent, under 20 words. Use only what the correction "
|
|
663
|
+
"says; add no preference it does not state."}},
|
|
664
|
+
timeout=120)
|
|
665
|
+
return tezgah_judge.text(result, "line")
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def store_or_exit(args):
|
|
669
|
+
repo = repo_root(os.path.abspath(args.repo or os.getcwd()))
|
|
670
|
+
if not repo:
|
|
671
|
+
raise SystemExit(misuse("not inside a git repository"))
|
|
672
|
+
return repo
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def learn_refusal(repo):
|
|
676
|
+
"""Why `learn` sends and writes nothing here, or None - read before the
|
|
677
|
+
store lock, so a refused run leaves no lock file behind."""
|
|
678
|
+
if tezgah_judge.tp.off("judge-off") or not (
|
|
679
|
+
tezgah_judge.available() or tezgah_judge.named(("typesafe",))):
|
|
680
|
+
return "no judge: no credential resolves or judge-off is set; nothing sent"
|
|
681
|
+
import tezgah_context
|
|
682
|
+
if ".no-taste" in tezgah_context.repo_marks(repo)[1]:
|
|
683
|
+
return ".no-taste marks this repository; nothing learned"
|
|
684
|
+
return None
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def cmd_learn(args):
|
|
688
|
+
repo = store_or_exit(args)
|
|
689
|
+
signals = (transcript_signals(repo, args.host) if args.from_transcripts
|
|
690
|
+
else capture_signals(repo))
|
|
691
|
+
decided = {}
|
|
692
|
+
for r in tl.rows(os.path.join(tl.store_dir(repo), "decisions.jsonl")):
|
|
693
|
+
decided[r.get("id")] = decided.get(r.get("id")) or r.get("provider") == "typesafe"
|
|
694
|
+
# a typed decision is final; an unverified one is retried, TypeSafe only
|
|
695
|
+
todo = [s for s in signals if not decided.get(s["id"])]
|
|
696
|
+
if args.limit is not None:
|
|
697
|
+
todo = todo[:args.limit]
|
|
698
|
+
ledgers = tl.load(repo)
|
|
699
|
+
before = {k for k, v in tl.every(ledgers).items() if v["state"] == "active"}
|
|
700
|
+
counts = {"signals": len(todo), "typesafe": 0, "unverified": 0, "unanswered": 0,
|
|
701
|
+
"preference": 0, "defect": 0, "none": 0}
|
|
702
|
+
tokens = [0, 0]
|
|
703
|
+
for sig in todo:
|
|
704
|
+
dec, result = decide(sig, held(ledgers),
|
|
705
|
+
fallback=not args.no_fallback and sig["id"] not in decided)
|
|
706
|
+
if dec is None:
|
|
707
|
+
counts["unanswered"] += 1
|
|
708
|
+
continue
|
|
709
|
+
tokens[0] += result["usage"]["input_tokens"]
|
|
710
|
+
tokens[1] += result["usage"]["output_tokens"]
|
|
711
|
+
day = tl.now_day()
|
|
712
|
+
tl.append(os.path.join(tl.store_dir(repo), "decisions.jsonl"), {
|
|
713
|
+
"id": sig["id"], "day": day, "at": sig.get("at"), "session": sig["session"],
|
|
714
|
+
"provider": result.get("provider"), "model": result.get("model"),
|
|
715
|
+
"verified": dec["verified"], "kind": dec["kind"][0], "p": dec["kind"][1],
|
|
716
|
+
"category": dec["category"][0], "scope": dec["scope"][0],
|
|
717
|
+
"relations": {k: v[0] for k, v in dec["relations"].items() if v[0] != "unrelated"},
|
|
718
|
+
"text": ti.redact(sig["text"])[:300], "paths": sig["paths"][:8]}, strict=True)
|
|
719
|
+
if not dec["verified"]:
|
|
720
|
+
counts["unverified"] += 1
|
|
721
|
+
continue
|
|
722
|
+
counts["typesafe"] += 1
|
|
723
|
+
counts[dec["kind"][0] if dec["kind"][0] in CLASSES else "none"] += 1
|
|
724
|
+
if dec["kind"][0] == "defect":
|
|
725
|
+
tl.append(os.path.join(tl.store_dir(repo), "defects.jsonl"),
|
|
726
|
+
{"id": sig["id"], "day": day, "session": sig["session"],
|
|
727
|
+
"text": ti.redact(sig["text"])[:600], "paths": sig["paths"][:8]},
|
|
728
|
+
strict=True)
|
|
729
|
+
tl.apply(ledgers, sig, dec, day, applied=tl.injected(repo, sig["session"]))
|
|
730
|
+
for lid, learning in tl.every(ledgers).items():
|
|
731
|
+
if learning["state"] == "active" and lid not in before \
|
|
732
|
+
and not learning.get("written") and not learning.get("edited"):
|
|
733
|
+
# --no-fallback asks nothing of a generative provider: the user's
|
|
734
|
+
# own words stay the learning's line
|
|
735
|
+
line = None if args.no_fallback else write_line(learning)
|
|
736
|
+
if line:
|
|
737
|
+
learning.update(text=" ".join(line.split())[:400], written=True)
|
|
738
|
+
tl.save(repo, ledgers)
|
|
739
|
+
tl.markdown(repo)
|
|
740
|
+
states = {}
|
|
741
|
+
for learning in tl.every(ledgers).values():
|
|
742
|
+
states[learning["state"]] = states.get(learning["state"], 0) + 1
|
|
743
|
+
report = dict(counts, states=states, input_tokens=tokens[0], output_tokens=tokens[1],
|
|
744
|
+
cost_usd=cost(tezgah_judge.MODEL, tokens[0], tokens[1]))
|
|
745
|
+
if args.json:
|
|
746
|
+
print(json.dumps(report, indent=2))
|
|
747
|
+
return 0
|
|
748
|
+
print("signals %d: typesafe %d, unverified %d, unanswered %d" % (
|
|
749
|
+
counts["signals"], counts["typesafe"], counts["unverified"], counts["unanswered"]))
|
|
750
|
+
print("decided: preference %d, defect %d, none %d" % (
|
|
751
|
+
counts["preference"], counts["defect"], counts["none"]))
|
|
752
|
+
print("ledger: " + (", ".join("%s %d" % kv for kv in sorted(states.items())) or "empty"))
|
|
753
|
+
print("typed decisions: %d input tokens, $%s" % (tokens[0], report["cost_usd"]))
|
|
754
|
+
return 0
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
def cmd_list(args):
|
|
758
|
+
repo = store_or_exit(args)
|
|
759
|
+
ledgers = tl.load(repo)
|
|
760
|
+
modes = {m[1]["id"]: m[0] for m in tl.usable(repo)}
|
|
761
|
+
rows = [v for v in tl.every(ledgers).values() if args.all or v["state"] != "retired"]
|
|
762
|
+
rows.sort(key=lambda v: (v["state"] != "active", -tl.confidence(v)))
|
|
763
|
+
if args.json:
|
|
764
|
+
print(json.dumps([dict(v, confidence=round(tl.confidence(v), 4),
|
|
765
|
+
mode=modes.get(v["id"])) for v in rows], indent=2))
|
|
766
|
+
return 0
|
|
767
|
+
if not rows:
|
|
768
|
+
print("no learnings")
|
|
769
|
+
return 0
|
|
770
|
+
for v in rows:
|
|
771
|
+
print("%-6s %-11s %-4s %.2f %-10s %-17s %s" % (
|
|
772
|
+
v["id"], v["state"], modes.get(v["id"], "-"), tl.confidence(v),
|
|
773
|
+
v["scope"] + ("" if v["scope"] in ("repository", "user") else ":" + v["where"]),
|
|
774
|
+
v["category"], " ".join(v["text"].split())[:80]))
|
|
775
|
+
return 0
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def cmd_one(args):
|
|
779
|
+
repo = store_or_exit(args)
|
|
780
|
+
ledgers = tl.load(repo)
|
|
781
|
+
if args.cmd == "show":
|
|
782
|
+
learning = tl.every(ledgers).get(args.id)
|
|
783
|
+
if learning is None:
|
|
784
|
+
return misuse("no learning %s" % args.id)
|
|
785
|
+
print(json.dumps(dict(learning, confidence=round(tl.confidence(learning), 4)),
|
|
786
|
+
indent=2, ensure_ascii=False))
|
|
787
|
+
return 0
|
|
788
|
+
if not tl.decide_user(ledgers, args.id, args.cmd, tl.now_day(),
|
|
789
|
+
text=getattr(args, "text", None)):
|
|
790
|
+
return misuse("no learning %s" % args.id)
|
|
791
|
+
tl.save(repo, ledgers)
|
|
792
|
+
tl.markdown(repo)
|
|
793
|
+
print("%s %s" % (args.id, {"accept": "accepted", "reject": "rejected",
|
|
794
|
+
"edit": "edited"}[args.cmd]))
|
|
795
|
+
return 0
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def cmd_export(args):
|
|
799
|
+
repo = store_or_exit(args)
|
|
800
|
+
print("AGENTS.md: %d accepted learning(s)" % tl.export_agents(repo))
|
|
801
|
+
return 0
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
def cmd_label(args):
|
|
805
|
+
"""With ID and LABEL, record the user's label; without, print a random
|
|
806
|
+
sample of unlabelled typed decisions - drawn without regard to what was
|
|
807
|
+
injected, so the calibration bound rests on an unselected sample."""
|
|
808
|
+
repo = store_or_exit(args)
|
|
809
|
+
store = tl.store_dir(repo)
|
|
810
|
+
decided = {r["id"]: r for r in tl.rows(os.path.join(store, "decisions.jsonl"))
|
|
811
|
+
if r.get("provider") == "typesafe" and r.get("id")}
|
|
812
|
+
labelled = {r.get("id") for r in tl.rows(os.path.join(store, "labels.jsonl"))}
|
|
813
|
+
if args.id:
|
|
814
|
+
if args.label not in CLASSES:
|
|
815
|
+
return misuse("label is one of %s" % ", ".join(CLASSES))
|
|
816
|
+
row = decided.get(args.id)
|
|
817
|
+
if row is None:
|
|
818
|
+
return misuse("no typed decision %s" % args.id)
|
|
819
|
+
tl.append(os.path.join(store, "labels.jsonl"),
|
|
820
|
+
{"id": args.id, "label": args.label, "decided": row.get("kind"),
|
|
821
|
+
"provider": row.get("provider"), "day": tl.now_day()}, strict=True)
|
|
822
|
+
print("%s labelled %s (decided %s)" % (args.id, args.label, row.get("kind")))
|
|
823
|
+
return 0
|
|
824
|
+
pool = sorted(i for i in decided if i not in labelled)
|
|
825
|
+
for i in random.Random().sample(pool, min(args.n, len(pool))):
|
|
826
|
+
print("%s %s" % (i, " ".join(decided[i].get("text", "").split())[:200]))
|
|
827
|
+
print("label each: tezgah-taste label <id> preference|defect|none")
|
|
828
|
+
return 0
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def cmd_calibrate(args):
|
|
832
|
+
repo = store_or_exit(args)
|
|
833
|
+
x, n, bound = tl.calibration(repo)
|
|
834
|
+
report = {"agree": x, "labelled_preference_decisions": n, "bound": round(bound, 4),
|
|
835
|
+
"rules_allowed": bound >= tl.BOUND_AT}
|
|
836
|
+
if args.json:
|
|
837
|
+
print(json.dumps(report, indent=2))
|
|
838
|
+
else:
|
|
839
|
+
print("preference precision %d/%d, Clopper-Pearson 95%% lower bound %.4f: %s" % (
|
|
840
|
+
x, n, bound, "rules allowed" if report["rules_allowed"]
|
|
841
|
+
else "hints only (a rule needs >= %.1f)" % tl.BOUND_AT))
|
|
842
|
+
return 0
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def cmd_gate(args):
|
|
846
|
+
repo = store_or_exit(args)
|
|
847
|
+
report = tl.gate(repo)
|
|
848
|
+
if args.json:
|
|
849
|
+
print(json.dumps(report, indent=2))
|
|
850
|
+
return 0
|
|
851
|
+
b, a = report["before"], report["after"]
|
|
852
|
+
print("before: %d/%d preference per writing turn; after: %d/%d; one-sided p %s; "
|
|
853
|
+
"need %d per arm; injection %s" % (
|
|
854
|
+
b["preference"], b["turns"], a["preference"], a["turns"], report["p"],
|
|
855
|
+
report["need"], "STOPPED" if report["stopped"] else "on"))
|
|
856
|
+
return 0
|
|
857
|
+
|
|
858
|
+
|
|
399
859
|
|
|
400
860
|
def main(argv):
|
|
401
861
|
parser = argparse.ArgumentParser(prog="tezgah-taste", description=__doc__,
|
|
@@ -411,12 +871,56 @@ def main(argv):
|
|
|
411
871
|
measure.add_argument("--labels", required=True)
|
|
412
872
|
measure.add_argument("--model", default=tezgah_judge.MODEL)
|
|
413
873
|
measure.add_argument("--json", action="store_true")
|
|
874
|
+
measure.add_argument("--typed", action="store_true",
|
|
875
|
+
help="require TypeSafe (the decision T1P's ledger trusts)")
|
|
414
876
|
rate = sub.add_parser("rate")
|
|
415
877
|
rate.add_argument("--in", dest="inp", required=True)
|
|
416
878
|
rate.add_argument("--limit", type=int)
|
|
417
879
|
rate.add_argument("--json", action="store_true")
|
|
880
|
+
learn = sub.add_parser("learn")
|
|
881
|
+
learn.add_argument("--from-transcripts", action="store_true")
|
|
882
|
+
learn.add_argument("--host", choices=("omp", "claude", "all"), default="all")
|
|
883
|
+
learn.add_argument("--limit", type=int)
|
|
884
|
+
learn.add_argument("--no-fallback", action="store_true",
|
|
885
|
+
help="TypeSafe only: never pay another provider for a decision")
|
|
886
|
+
listing = sub.add_parser("list")
|
|
887
|
+
listing.add_argument("--all", action="store_true")
|
|
888
|
+
for name in ("show", "accept", "reject", "edit"):
|
|
889
|
+
one = sub.add_parser(name)
|
|
890
|
+
one.add_argument("id")
|
|
891
|
+
if name == "edit":
|
|
892
|
+
one.add_argument("--text", required=True)
|
|
893
|
+
sub.add_parser("export")
|
|
894
|
+
label = sub.add_parser("label")
|
|
895
|
+
label.add_argument("id", nargs="?")
|
|
896
|
+
label.add_argument("label", nargs="?")
|
|
897
|
+
label.add_argument("--n", type=int, default=10)
|
|
898
|
+
sub.add_parser("calibrate")
|
|
899
|
+
sub.add_parser("gate")
|
|
900
|
+
for name, p in sub.choices.items():
|
|
901
|
+
if name not in ("mine", "measure", "rate"):
|
|
902
|
+
p.add_argument("--repo")
|
|
903
|
+
if name in ("learn", "list", "calibrate", "gate"):
|
|
904
|
+
p.add_argument("--json", action="store_true")
|
|
418
905
|
args = parser.parse_args(argv[1:])
|
|
419
|
-
|
|
906
|
+
handler = {"mine": cmd_mine, "measure": cmd_measure, "rate": cmd_rate,
|
|
907
|
+
"learn": cmd_learn, "list": cmd_list, "show": cmd_one, "accept": cmd_one,
|
|
908
|
+
"reject": cmd_one, "edit": cmd_one, "export": cmd_export,
|
|
909
|
+
"label": cmd_label, "calibrate": cmd_calibrate,
|
|
910
|
+
"gate": cmd_gate}[args.cmd]
|
|
911
|
+
if args.cmd not in ("learn", "accept", "reject", "edit", "label"):
|
|
912
|
+
return handler(args)
|
|
913
|
+
# the commands that rewrite ledger.json: one at a time, or the last one
|
|
914
|
+
# to finish would silently drop what the other wrote
|
|
915
|
+
try:
|
|
916
|
+
repo = store_or_exit(args)
|
|
917
|
+
refused = learn_refusal(repo) if args.cmd == "learn" else None
|
|
918
|
+
if refused:
|
|
919
|
+
return misuse(refused)
|
|
920
|
+
with tl.locked():
|
|
921
|
+
return handler(args)
|
|
922
|
+
except tl.Busy as busy:
|
|
923
|
+
return misuse("the taste ledger is in use (%s); run this again when it ends" % busy)
|
|
420
924
|
|
|
421
925
|
|
|
422
926
|
if __name__ == "__main__":
|