@r1z4x/tezgah 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -6,6 +6,36 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [1.3.0] - 2026-10-07
10
+
11
+ ### Added
12
+
13
+ - **Armed taste learns by itself.** Session start runs `tezgah-taste learn` in
14
+ the background when the repository's signals grew, at most once an hour. It
15
+ runs only when a TypeSafe key resolves, with the new `--no-fallback`, so it
16
+ asks no other provider. It does not run on Windows yet. Its log goes under
17
+ the cache dir, in `taste-learn/`.
18
+
19
+ ## [1.2.0] - 2026-10-07
20
+
21
+ ### Added
22
+
23
+ - **Taste learns and applies, not only records.** Arm it with `taste-on`.
24
+ `tezgah-taste learn` then turns each correction after a writing turn into one
25
+ typed decision, and only TypeSafe may give it. The decision names a
26
+ preference, a defect or neither. It picks one of eleven fixed categories and
27
+ a scope, and relates the correction to each learning already held. A ledger
28
+ computes each learning's confidence from its evidence and forgets it over
29
+ time. A contradiction flags a learning and never overwrites it. A learning
30
+ goes active in its second session. Session start injects active learnings as
31
+ rules or hints. Every host notes the in-scope ones on a write, each once per
32
+ session. A rule needs a measured calibration bound. A defect never enters the
33
+ ledger. A before/after gate stops injection when corrections do not fall.
34
+ `list`, `accept`, `reject`, `edit` and `export` give the user control, and
35
+ `learn --from-transcripts` bootstraps from history. The judge seam gains
36
+ `only=` and a `text` question. Two commands never rewrite the ledger at once:
37
+ a second `learn`, `accept`, `reject` or `edit` exits 2 while one runs.
38
+
9
39
  ## [1.1.0] - 2026-10-07
10
40
 
11
41
  ### Added
@@ -803,6 +833,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
803
833
  the copy could not find `hooks/` and died on import. It now links into the
804
834
  installed tree, and also installs the command as `tezgah`.
805
835
 
806
- [Unreleased]: https://github.com/r1z4x/tezgah/compare/v1.1.0...HEAD
836
+ [Unreleased]: https://github.com/r1z4x/tezgah/compare/v1.3.0...HEAD
837
+ [1.3.0]: https://github.com/r1z4x/tezgah/releases/tag/v1.3.0
838
+ [1.2.0]: https://github.com/r1z4x/tezgah/releases/tag/v1.2.0
807
839
  [1.1.0]: https://github.com/r1z4x/tezgah/releases/tag/v1.1.0
808
840
  [1.0.0]: https://github.com/r1z4x/tezgah/releases/tag/v1.0.0
package/MANIFEST CHANGED
@@ -144,6 +144,7 @@ hooks/tezgah_stack.py
144
144
  hooks/tezgah_stopspec.py
145
145
  hooks/tezgah_task.py
146
146
  hooks/tezgah_taste.py
147
+ hooks/tezgah_taste_ledger.py
147
148
  hooks/tezgah_untrusted.py
148
149
  hooks/tezgah_update.py
149
150
  hosts/codex/hook.py
@@ -752,6 +753,7 @@ tests/test_switch_latching.py
752
753
  tests/test_task.py
753
754
  tests/test_taste.py
754
755
  tests/test_taste_cli.py
756
+ tests/test_taste_ledger.py
755
757
  tests/test_tezgah_mcp_wiring.py
756
758
  tests/test_triage.py
757
759
  tests/test_untrusted.py
package/bin/tezgah-taste CHANGED
@@ -1,9 +1,17 @@
1
1
  #!/usr/bin/env python3
2
- """tezgah-taste - mine and measure the coding-taste signals in local sessions.
2
+ """tezgah-taste - mine, measure, learn and apply the coding-taste signals.
3
3
 
4
4
  tezgah-taste mine [--host omp|claude|all] [--root DIR] [--stats] [--out FILE]
5
5
  tezgah-taste measure --samples FILE --labels FILE [--model M] [--json]
6
6
  tezgah-taste rate --in MINED [--limit N] [--json]
7
+ tezgah-taste learn [--repo DIR] [--from-transcripts] [--host omp|claude|all]
8
+ [--limit N] [--no-fallback] [--json]
9
+ tezgah-taste list [--repo DIR] [--all] [--json]
10
+ tezgah-taste show|accept|reject ID [--repo DIR]
11
+ tezgah-taste edit ID --text TEXT [--repo DIR]
12
+ tezgah-taste export [--repo DIR]
13
+ tezgah-taste label [ID LABEL] [--n N] [--repo DIR]
14
+ tezgah-taste calibrate|gate [--repo DIR] [--json]
7
15
 
8
16
  `mine` reads the session transcripts on this machine - omp's top-level session
9
17
  files under `~/.omp/agent/sessions/*/` (a nested file is a subagent session) and
@@ -25,13 +33,36 @@ writing-turn prompt, per repo and overall. Both send the sample text to the
25
33
  judge's provider (`hooks/tezgah_judge.py` names it); with no credential or
26
34
  `judge-off` set they send nothing and exit 2.
27
35
 
36
+ `learn` is taste-1's pipeline on tezgah (research line taste-architecture,
37
+ design T1P). Each prompt after a writing turn - from the capture rows in
38
+ `<repo>/.tezgah/taste/signals.jsonl`, or with `--from-transcripts` from this
39
+ repository's omp and Claude transcripts - becomes one structured signal built by
40
+ code: the prompt, the files written, the agent's edits and the user's own edit
41
+ between two agent writes. One typed decision per signal, required from TypeSafe
42
+ (`only=("typesafe",)`): preference, defect or none; the category; the scope; and
43
+ how it relates to each learning held. A decision any other provider gave is
44
+ recorded `unverified` and never applied. A defect goes to `defects.jsonl`, never
45
+ into the ledger. The ledger itself is `hooks/tezgah_taste_ledger.py`. A
46
+ learning that becomes active gets one readable line from a generative provider
47
+ (a `text` question; the user's own words when none answers).
48
+
49
+ `list`, `show`, `accept`, `reject` and `edit` are the user's control; `export`
50
+ writes the accepted learnings into AGENTS.md. `label` draws an independent
51
+ random sample of decided signals for the user to label, `calibrate` prints the
52
+ Clopper-Pearson bound a rule needs, and `gate` runs the pooled before/after test
53
+ that stops injection when the corrections do not fall.
54
+
28
55
  Exit codes: 0 done, 2 misuse or no judge.
29
56
  """
30
57
  import argparse
31
58
  import datetime
59
+ import difflib
32
60
  import glob
61
+ import hashlib
33
62
  import json
34
63
  import os
64
+ import random
65
+ import subprocess
35
66
  import sys
36
67
 
37
68
  HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@@ -39,6 +70,7 @@ sys.path.insert(0, os.path.join(HERE, "hooks"))
39
70
  import tezgah_integrity as ti # noqa: E402
40
71
  import tezgah_judge # noqa: E402
41
72
  import tezgah_models # noqa: E402
73
+ import tezgah_taste_ledger as tl # noqa: E402
42
74
 
43
75
  WRITE_TOOLS = {"omp": {"edit", "write"},
44
76
  "claude": {"Edit", "Write", "MultiEdit", "NotebookEdit"}}
@@ -282,7 +314,7 @@ def read_jsonl(path):
282
314
  raise SystemExit(misuse("cannot read %s: %s" % (path, exc)))
283
315
 
284
316
 
285
- def classify(texts, model):
317
+ def classify(texts, model, only=None):
286
318
  """One label per text (None when the judge gave none) and the usage.
287
319
 
288
320
  The rule rides the state, so each request pays for it once and the
@@ -294,7 +326,7 @@ def classify(texts, model):
294
326
  "instructions": "Label this prompt under the rule "
295
327
  "in the state:\n" + ti.redact(text)}
296
328
  for i, text in enumerate(chunk)}
297
- result = tezgah_judge.ask(RULE, questions, model=model, timeout=120)
329
+ result = tezgah_judge.ask(RULE, questions, model=model, timeout=120, only=only)
298
330
  if result is not None:
299
331
  tokens_in += result["usage"]["input_tokens"]
300
332
  tokens_out += result["usage"]["output_tokens"]
@@ -336,7 +368,8 @@ def cmd_measure(args):
336
368
  return misuse("no sample joins a label on (set, n)")
337
369
  if not tezgah_judge.available():
338
370
  return misuse("no judge: no credential resolves or judge-off is set; nothing sent")
339
- got, usage = classify([text for text, _gold in pairs], args.model)
371
+ got, usage = classify([text for text, _gold in pairs], args.model,
372
+ only=("typesafe",) if args.typed else None)
340
373
  confusion = {g: {p: 0 for p in CLASSES} for g in CLASSES}
341
374
  unjudged = 0
342
375
  for (_text, gold), pred in zip(pairs, got):
@@ -396,6 +429,433 @@ def cmd_rate(args):
396
429
  print(usage_line(usage))
397
430
  return 0
398
431
 
432
+ # ------------------------------------------------------------ learn, apply ----
433
+
434
+ SCOPE_CRITERIA = {
435
+ "path": "only files under the directory the agent wrote",
436
+ "language": "files of the language or extension the agent wrote",
437
+ "repository": "everything in this repository",
438
+ "user": "every repository this user works in"}
439
+ RELATION_CRITERIA = {
440
+ "supports": "the prompt asks for the same preference as this learning",
441
+ "contradicts": "the prompt asks for the opposite of this learning",
442
+ "narrows": "the prompt keeps this learning but makes an exception for a narrower place",
443
+ "unrelated": "the prompt is about something else"}
444
+ CATEGORY_CRITERIA = {c: c.replace("-", " ") for c in tl.CATEGORIES}
445
+ HELD_MAX = 30
446
+ DIFF_MAX = 1500
447
+
448
+
449
+ def repo_root(path):
450
+ out = subprocess.run(["git", "-C", path, "rev-parse", "--show-toplevel"],
451
+ capture_output=True, text=True)
452
+ return out.stdout.strip() if out.returncode == 0 and out.stdout.strip() else None
453
+
454
+
455
+ def sig_id(*parts):
456
+ return hashlib.sha1("\0".join(str(p) for p in parts).encode()).hexdigest()[:12]
457
+
458
+
459
+ def iso_day(ts):
460
+ """A capture row's ISO timestamp as days since the epoch, or None."""
461
+ try:
462
+ return datetime.datetime.fromisoformat(str(ts).replace("Z", "+00:00")) \
463
+ .timestamp() / 86400.0
464
+ except ValueError:
465
+ return None
466
+
467
+
468
+ def _snapshots():
469
+ """{absolute path: [(ts, id, after)]} over the snapshot store, oldest first."""
470
+ import tezgah_snapshot as ts
471
+ out = {}
472
+ for d in ts._dirs():
473
+ try:
474
+ with open(os.path.join(d, "meta.json"), encoding="utf-8") as fh:
475
+ meta = json.load(fh)
476
+ except (OSError, ValueError):
477
+ continue
478
+ if isinstance(meta, dict) and meta.get("path"):
479
+ out.setdefault(meta["path"], []).append(
480
+ (int(meta.get("ts") or 0), os.path.basename(d), bool(meta.get("after"))))
481
+ for rows in out.values():
482
+ rows.sort()
483
+ return out
484
+
485
+
486
+ def _read_blob(sid):
487
+ import tezgah_snapshot as ts
488
+ try:
489
+ with open(ts._blob(sid), "rb") as fh:
490
+ return fh.read().decode("utf-8", "replace")
491
+ except OSError:
492
+ return None
493
+
494
+
495
+ def user_diff(repo, rel, after_sid, snaps):
496
+ """The user's own change to `rel` after the agent's write: the after-blob
497
+ against the pre-write bytes of the next agent write to it, or against the
498
+ file on disk when none came. "" when nothing changed in between."""
499
+ after = _read_blob(after_sid)
500
+ if after is None:
501
+ return ""
502
+ apath = os.path.realpath(os.path.join(repo, rel))
503
+ rows = snaps.get(apath) or []
504
+ mine = next((t for t, sid, _a in rows if sid == after_sid), None)
505
+ later = next((sid for t, sid, a in rows if mine is not None and t >= mine
506
+ and not a and sid != after_sid), None)
507
+ if later:
508
+ now = _read_blob(later)
509
+ else:
510
+ try:
511
+ with open(apath, encoding="utf-8", errors="replace") as fh:
512
+ now = fh.read()
513
+ except OSError:
514
+ return ""
515
+ if now is None or now == after:
516
+ return ""
517
+ diff = "".join(difflib.unified_diff(after.splitlines(True), now.splitlines(True),
518
+ "agent/" + rel, "user/" + rel, n=1))
519
+ return ti.redact(diff)[:DIFF_MAX]
520
+
521
+
522
+ def capture_signals(repo):
523
+ """One signal per prompt that followed a writing turn, from the capture rows."""
524
+ snaps, turns, out = None, {}, []
525
+ for row in tl.rows(os.path.join(tl.store_dir(repo), "signals.jsonl")):
526
+ session = row.get("session") or ""
527
+ turn = turns.setdefault(session, {"edits": [], "after": []})
528
+ if row.get("kind") == "edit":
529
+ turn["edits"].append(row)
530
+ elif row.get("kind") == "after":
531
+ turn["after"].append(row)
532
+ elif row.get("kind") == "prompt":
533
+ if turn["edits"]:
534
+ if snaps is None:
535
+ snaps = _snapshots()
536
+ diffs = [d for d in (user_diff(repo, a.get("path"), a.get("snapshot"), snaps)
537
+ for a in turn["after"] if a.get("path")) if d]
538
+ out.append({
539
+ "id": sig_id(session, row.get("ts"), row.get("text")),
540
+ "at": iso_day(row.get("ts")),
541
+ "session": session, "text": row.get("text") or "",
542
+ "paths": sorted({e.get("path") for e in turn["edits"] if e.get("path")}),
543
+ "edits": [(e.get("path"), (e.get("old") or "")[:300],
544
+ (e.get("new") or "")[:300]) for e in turn["edits"][-4:]],
545
+ "diffs": diffs})
546
+ turns[session] = {"edits": [], "after": []}
547
+ return out
548
+
549
+
550
+ def _call_path(args):
551
+ for key in ("path", "file_path", "notebook_path"):
552
+ if isinstance(args.get(key), str):
553
+ return args[key]
554
+ return None
555
+
556
+
557
+ def transcript_signals(repo, host):
558
+ """The same signals from this repository's top-level host transcripts: the
559
+ prompt and the files written; no edit text or user diff (the history
560
+ bootstrap, Command Code's `/learn-taste`)."""
561
+ out = []
562
+ for name in (("omp", "claude") if host == "all" else (host,)):
563
+ for path, cwd in sessions(name, os.path.dirname(repo)):
564
+ if cwd != repo and not cwd.startswith(repo + os.sep):
565
+ continue
566
+ session = session_id(name, path)[0] or os.path.basename(path)
567
+ written, n, k, at = [], 0, 0, None
568
+ for event in HOSTS[name][2](path, calls=True):
569
+ if event[0] in ("call", "reply") and isinstance(event[-1], (int, float)):
570
+ at = event[-1] / 86400.0
571
+ if event[0] == "call" and event[1] in WRITE_TOOLS[name]:
572
+ target = _call_path(event[2])
573
+ if target:
574
+ rel = os.path.relpath(os.path.join(cwd, target), repo)
575
+ if not rel.startswith(".."):
576
+ written.append(rel)
577
+ elif event[0] == "write":
578
+ n += 1
579
+ elif event[0] == "user":
580
+ if n and not harness_prompt(event[1]):
581
+ text = ti.redact(event[1])[:CUT]
582
+ # keyed inside its own session, so another session's
583
+ # new turns never shift this id and re-decide it
584
+ k += 1
585
+ out.append({"id": sig_id(name, session, k, text), "at": at,
586
+ "session": session, "text": text,
587
+ "paths": sorted(set(written)), "edits": [], "diffs": []})
588
+ written, n = [], 0
589
+ return out
590
+
591
+
592
+ def held(ledgers):
593
+ """The learnings a new signal is compared against: every one not retired,
594
+ the most supported first, capped so one call stays one call."""
595
+ alive = [v for v in tl.every(ledgers).values() if v["state"] != "retired"]
596
+ alive.sort(key=lambda v: -v.get("r", 0.0))
597
+ return alive[:HELD_MAX]
598
+
599
+
600
+ def decision_request(sig, learnings):
601
+ state = ["A user prompt that followed a turn in which a coding agent wrote files.",
602
+ "Prompt: " + ti.redact(sig["text"])[:CUT]]
603
+ if sig["paths"]:
604
+ state.append("Files the agent wrote: " + ", ".join(sig["paths"][:12]))
605
+ for path, old, new in sig.get("edits") or ():
606
+ state.append("Agent edit to %s:\n- %s\n+ %s" % (path, ti.redact(old), ti.redact(new)))
607
+ for diff in sig.get("diffs") or ():
608
+ state.append("The user's own edit after the agent's write:\n" + diff)
609
+ questions = {
610
+ "kind": {"type": "choice", "instructions": RULE, "criteria": CRITERIA},
611
+ "category": {"type": "choice", "criteria": CATEGORY_CRITERIA,
612
+ "instructions": "If the prompt states a coding preference, which "
613
+ "category does it belong to?"},
614
+ "scope": {"type": "choice", "criteria": SCOPE_CRITERIA,
615
+ "instructions": "Where should this preference apply, judging only from "
616
+ "what the prompt says?"}}
617
+ for learning in learnings:
618
+ questions["rel_" + learning["id"]] = {
619
+ "type": "choice", "criteria": RELATION_CRITERIA,
620
+ "instructions": "How does the prompt relate to this learning (%s, %s): %s"
621
+ % (learning["category"], learning["scope"],
622
+ " ".join(learning["text"].split())[:300])}
623
+ return "\n\n".join(state), questions
624
+
625
+
626
+ def picked(result, qid, default):
627
+ choice = tezgah_judge.choice(result, qid)
628
+ if choice is None:
629
+ return default, 0.0
630
+ p = tezgah_judge.noul(result, qid, choice)
631
+ return choice, (p if p is not None else 0.5)
632
+
633
+
634
+ def decide(sig, learnings, fallback=True):
635
+ """(decision, result): the typed decision, TypeSafe required; when it cannot
636
+ answer, the seam's own providers, recorded unverified - only on a signal's
637
+ first try, since an unverified answer is never applied and a rerun would pay
638
+ for it again."""
639
+ state, questions = decision_request(sig, learnings)
640
+ result = tezgah_judge.ask(state, questions, only=("typesafe",), timeout=60)
641
+ if result is None and fallback:
642
+ result = tezgah_judge.ask(state, questions, timeout=120)
643
+ if result is None:
644
+ return None, None
645
+ dec = {"kind": picked(result, "kind", "none"),
646
+ "category": picked(result, "category", ""),
647
+ "scope": picked(result, "scope", "repository"),
648
+ "relations": {lr["id"]: picked(result, "rel_" + lr["id"], "unrelated")
649
+ for lr in learnings},
650
+ "verified": result.get("provider") == "typesafe"}
651
+ return dec, result
652
+
653
+
654
+ def write_line(learning):
655
+ """One readable line for a learning that became active, from a generative
656
+ provider; None when none answers (the user's own words stay)."""
657
+ evidence = " ".join(learning["text"].split())[:600]
658
+ result = tezgah_judge.ask(
659
+ "A user corrected a coding agent with: " + evidence,
660
+ {"line": {"type": "text", "instructions":
661
+ "Write the coding preference this correction states as one imperative "
662
+ "line for a coding agent, under 20 words. Use only what the correction "
663
+ "says; add no preference it does not state."}},
664
+ timeout=120)
665
+ return tezgah_judge.text(result, "line")
666
+
667
+
668
+ def store_or_exit(args):
669
+ repo = repo_root(os.path.abspath(args.repo or os.getcwd()))
670
+ if not repo:
671
+ raise SystemExit(misuse("not inside a git repository"))
672
+ return repo
673
+
674
+
675
+ def learn_refusal(repo):
676
+ """Why `learn` sends and writes nothing here, or None - read before the
677
+ store lock, so a refused run leaves no lock file behind."""
678
+ if tezgah_judge.tp.off("judge-off") or not (
679
+ tezgah_judge.available() or tezgah_judge.named(("typesafe",))):
680
+ return "no judge: no credential resolves or judge-off is set; nothing sent"
681
+ import tezgah_context
682
+ if ".no-taste" in tezgah_context.repo_marks(repo)[1]:
683
+ return ".no-taste marks this repository; nothing learned"
684
+ return None
685
+
686
+
687
+ def cmd_learn(args):
688
+ repo = store_or_exit(args)
689
+ signals = (transcript_signals(repo, args.host) if args.from_transcripts
690
+ else capture_signals(repo))
691
+ decided = {}
692
+ for r in tl.rows(os.path.join(tl.store_dir(repo), "decisions.jsonl")):
693
+ decided[r.get("id")] = decided.get(r.get("id")) or r.get("provider") == "typesafe"
694
+ # a typed decision is final; an unverified one is retried, TypeSafe only
695
+ todo = [s for s in signals if not decided.get(s["id"])]
696
+ if args.limit is not None:
697
+ todo = todo[:args.limit]
698
+ ledgers = tl.load(repo)
699
+ before = {k for k, v in tl.every(ledgers).items() if v["state"] == "active"}
700
+ counts = {"signals": len(todo), "typesafe": 0, "unverified": 0, "unanswered": 0,
701
+ "preference": 0, "defect": 0, "none": 0}
702
+ tokens = [0, 0]
703
+ for sig in todo:
704
+ dec, result = decide(sig, held(ledgers),
705
+ fallback=not args.no_fallback and sig["id"] not in decided)
706
+ if dec is None:
707
+ counts["unanswered"] += 1
708
+ continue
709
+ tokens[0] += result["usage"]["input_tokens"]
710
+ tokens[1] += result["usage"]["output_tokens"]
711
+ day = tl.now_day()
712
+ tl.append(os.path.join(tl.store_dir(repo), "decisions.jsonl"), {
713
+ "id": sig["id"], "day": day, "at": sig.get("at"), "session": sig["session"],
714
+ "provider": result.get("provider"), "model": result.get("model"),
715
+ "verified": dec["verified"], "kind": dec["kind"][0], "p": dec["kind"][1],
716
+ "category": dec["category"][0], "scope": dec["scope"][0],
717
+ "relations": {k: v[0] for k, v in dec["relations"].items() if v[0] != "unrelated"},
718
+ "text": ti.redact(sig["text"])[:300], "paths": sig["paths"][:8]}, strict=True)
719
+ if not dec["verified"]:
720
+ counts["unverified"] += 1
721
+ continue
722
+ counts["typesafe"] += 1
723
+ counts[dec["kind"][0] if dec["kind"][0] in CLASSES else "none"] += 1
724
+ if dec["kind"][0] == "defect":
725
+ tl.append(os.path.join(tl.store_dir(repo), "defects.jsonl"),
726
+ {"id": sig["id"], "day": day, "session": sig["session"],
727
+ "text": ti.redact(sig["text"])[:600], "paths": sig["paths"][:8]},
728
+ strict=True)
729
+ tl.apply(ledgers, sig, dec, day, applied=tl.injected(repo, sig["session"]))
730
+ for lid, learning in tl.every(ledgers).items():
731
+ if learning["state"] == "active" and lid not in before \
732
+ and not learning.get("written") and not learning.get("edited"):
733
+ # --no-fallback asks nothing of a generative provider: the user's
734
+ # own words stay the learning's line
735
+ line = None if args.no_fallback else write_line(learning)
736
+ if line:
737
+ learning.update(text=" ".join(line.split())[:400], written=True)
738
+ tl.save(repo, ledgers)
739
+ tl.markdown(repo)
740
+ states = {}
741
+ for learning in tl.every(ledgers).values():
742
+ states[learning["state"]] = states.get(learning["state"], 0) + 1
743
+ report = dict(counts, states=states, input_tokens=tokens[0], output_tokens=tokens[1],
744
+ cost_usd=cost(tezgah_judge.MODEL, tokens[0], tokens[1]))
745
+ if args.json:
746
+ print(json.dumps(report, indent=2))
747
+ return 0
748
+ print("signals %d: typesafe %d, unverified %d, unanswered %d" % (
749
+ counts["signals"], counts["typesafe"], counts["unverified"], counts["unanswered"]))
750
+ print("decided: preference %d, defect %d, none %d" % (
751
+ counts["preference"], counts["defect"], counts["none"]))
752
+ print("ledger: " + (", ".join("%s %d" % kv for kv in sorted(states.items())) or "empty"))
753
+ print("typed decisions: %d input tokens, $%s" % (tokens[0], report["cost_usd"]))
754
+ return 0
755
+
756
+
757
+ def cmd_list(args):
758
+ repo = store_or_exit(args)
759
+ ledgers = tl.load(repo)
760
+ modes = {m[1]["id"]: m[0] for m in tl.usable(repo)}
761
+ rows = [v for v in tl.every(ledgers).values() if args.all or v["state"] != "retired"]
762
+ rows.sort(key=lambda v: (v["state"] != "active", -tl.confidence(v)))
763
+ if args.json:
764
+ print(json.dumps([dict(v, confidence=round(tl.confidence(v), 4),
765
+ mode=modes.get(v["id"])) for v in rows], indent=2))
766
+ return 0
767
+ if not rows:
768
+ print("no learnings")
769
+ return 0
770
+ for v in rows:
771
+ print("%-6s %-11s %-4s %.2f %-10s %-17s %s" % (
772
+ v["id"], v["state"], modes.get(v["id"], "-"), tl.confidence(v),
773
+ v["scope"] + ("" if v["scope"] in ("repository", "user") else ":" + v["where"]),
774
+ v["category"], " ".join(v["text"].split())[:80]))
775
+ return 0
776
+
777
+
778
+ def cmd_one(args):
779
+ repo = store_or_exit(args)
780
+ ledgers = tl.load(repo)
781
+ if args.cmd == "show":
782
+ learning = tl.every(ledgers).get(args.id)
783
+ if learning is None:
784
+ return misuse("no learning %s" % args.id)
785
+ print(json.dumps(dict(learning, confidence=round(tl.confidence(learning), 4)),
786
+ indent=2, ensure_ascii=False))
787
+ return 0
788
+ if not tl.decide_user(ledgers, args.id, args.cmd, tl.now_day(),
789
+ text=getattr(args, "text", None)):
790
+ return misuse("no learning %s" % args.id)
791
+ tl.save(repo, ledgers)
792
+ tl.markdown(repo)
793
+ print("%s %s" % (args.id, {"accept": "accepted", "reject": "rejected",
794
+ "edit": "edited"}[args.cmd]))
795
+ return 0
796
+
797
+
798
+ def cmd_export(args):
799
+ repo = store_or_exit(args)
800
+ print("AGENTS.md: %d accepted learning(s)" % tl.export_agents(repo))
801
+ return 0
802
+
803
+
804
+ def cmd_label(args):
805
+ """With ID and LABEL, record the user's label; without, print a random
806
+ sample of unlabelled typed decisions - drawn without regard to what was
807
+ injected, so the calibration bound rests on an unselected sample."""
808
+ repo = store_or_exit(args)
809
+ store = tl.store_dir(repo)
810
+ decided = {r["id"]: r for r in tl.rows(os.path.join(store, "decisions.jsonl"))
811
+ if r.get("provider") == "typesafe" and r.get("id")}
812
+ labelled = {r.get("id") for r in tl.rows(os.path.join(store, "labels.jsonl"))}
813
+ if args.id:
814
+ if args.label not in CLASSES:
815
+ return misuse("label is one of %s" % ", ".join(CLASSES))
816
+ row = decided.get(args.id)
817
+ if row is None:
818
+ return misuse("no typed decision %s" % args.id)
819
+ tl.append(os.path.join(store, "labels.jsonl"),
820
+ {"id": args.id, "label": args.label, "decided": row.get("kind"),
821
+ "provider": row.get("provider"), "day": tl.now_day()}, strict=True)
822
+ print("%s labelled %s (decided %s)" % (args.id, args.label, row.get("kind")))
823
+ return 0
824
+ pool = sorted(i for i in decided if i not in labelled)
825
+ for i in random.Random().sample(pool, min(args.n, len(pool))):
826
+ print("%s %s" % (i, " ".join(decided[i].get("text", "").split())[:200]))
827
+ print("label each: tezgah-taste label <id> preference|defect|none")
828
+ return 0
829
+
830
+
831
+ def cmd_calibrate(args):
832
+ repo = store_or_exit(args)
833
+ x, n, bound = tl.calibration(repo)
834
+ report = {"agree": x, "labelled_preference_decisions": n, "bound": round(bound, 4),
835
+ "rules_allowed": bound >= tl.BOUND_AT}
836
+ if args.json:
837
+ print(json.dumps(report, indent=2))
838
+ else:
839
+ print("preference precision %d/%d, Clopper-Pearson 95%% lower bound %.4f: %s" % (
840
+ x, n, bound, "rules allowed" if report["rules_allowed"]
841
+ else "hints only (a rule needs >= %.1f)" % tl.BOUND_AT))
842
+ return 0
843
+
844
+
845
+ def cmd_gate(args):
846
+ repo = store_or_exit(args)
847
+ report = tl.gate(repo)
848
+ if args.json:
849
+ print(json.dumps(report, indent=2))
850
+ return 0
851
+ b, a = report["before"], report["after"]
852
+ print("before: %d/%d preference per writing turn; after: %d/%d; one-sided p %s; "
853
+ "need %d per arm; injection %s" % (
854
+ b["preference"], b["turns"], a["preference"], a["turns"], report["p"],
855
+ report["need"], "STOPPED" if report["stopped"] else "on"))
856
+ return 0
857
+
858
+
399
859
 
400
860
  def main(argv):
401
861
  parser = argparse.ArgumentParser(prog="tezgah-taste", description=__doc__,
@@ -411,12 +871,56 @@ def main(argv):
411
871
  measure.add_argument("--labels", required=True)
412
872
  measure.add_argument("--model", default=tezgah_judge.MODEL)
413
873
  measure.add_argument("--json", action="store_true")
874
+ measure.add_argument("--typed", action="store_true",
875
+ help="require TypeSafe (the decision T1P's ledger trusts)")
414
876
  rate = sub.add_parser("rate")
415
877
  rate.add_argument("--in", dest="inp", required=True)
416
878
  rate.add_argument("--limit", type=int)
417
879
  rate.add_argument("--json", action="store_true")
880
+ learn = sub.add_parser("learn")
881
+ learn.add_argument("--from-transcripts", action="store_true")
882
+ learn.add_argument("--host", choices=("omp", "claude", "all"), default="all")
883
+ learn.add_argument("--limit", type=int)
884
+ learn.add_argument("--no-fallback", action="store_true",
885
+ help="TypeSafe only: never pay another provider for a decision")
886
+ listing = sub.add_parser("list")
887
+ listing.add_argument("--all", action="store_true")
888
+ for name in ("show", "accept", "reject", "edit"):
889
+ one = sub.add_parser(name)
890
+ one.add_argument("id")
891
+ if name == "edit":
892
+ one.add_argument("--text", required=True)
893
+ sub.add_parser("export")
894
+ label = sub.add_parser("label")
895
+ label.add_argument("id", nargs="?")
896
+ label.add_argument("label", nargs="?")
897
+ label.add_argument("--n", type=int, default=10)
898
+ sub.add_parser("calibrate")
899
+ sub.add_parser("gate")
900
+ for name, p in sub.choices.items():
901
+ if name not in ("mine", "measure", "rate"):
902
+ p.add_argument("--repo")
903
+ if name in ("learn", "list", "calibrate", "gate"):
904
+ p.add_argument("--json", action="store_true")
418
905
  args = parser.parse_args(argv[1:])
419
- return {"mine": cmd_mine, "measure": cmd_measure, "rate": cmd_rate}[args.cmd](args)
906
+ handler = {"mine": cmd_mine, "measure": cmd_measure, "rate": cmd_rate,
907
+ "learn": cmd_learn, "list": cmd_list, "show": cmd_one, "accept": cmd_one,
908
+ "reject": cmd_one, "edit": cmd_one, "export": cmd_export,
909
+ "label": cmd_label, "calibrate": cmd_calibrate,
910
+ "gate": cmd_gate}[args.cmd]
911
+ if args.cmd not in ("learn", "accept", "reject", "edit", "label"):
912
+ return handler(args)
913
+ # the commands that rewrite ledger.json: one at a time, or the last one
914
+ # to finish would silently drop what the other wrote
915
+ try:
916
+ repo = store_or_exit(args)
917
+ refused = learn_refusal(repo) if args.cmd == "learn" else None
918
+ if refused:
919
+ return misuse(refused)
920
+ with tl.locked():
921
+ return handler(args)
922
+ except tl.Busy as busy:
923
+ return misuse("the taste ledger is in use (%s); run this again when it ends" % busy)
420
924
 
421
925
 
422
926
  if __name__ == "__main__":
@@ -60,7 +60,7 @@
60
60
  "semicolons": 18
61
61
  },
62
62
  "docs/judge.md": {
63
- "long_sentences": 23,
63
+ "long_sentences": 22,
64
64
  "passive": 15,
65
65
  "replace_words": 0,
66
66
  "semicolons": 14