@kaydash9999/banker-plugins 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/CHANGELOG.md +66 -0
  4. package/README.md +23 -11
  5. package/codex/manifest.json +5 -5
  6. package/codex/transform-matrix.md +5 -5
  7. package/commands/front-qa.md +2 -2
  8. package/commands/setup.md +13 -4
  9. package/hooks/graceful-pause.mjs +222 -0
  10. package/hooks/hooks.json +2 -1
  11. package/hooks/register.mjs +12 -0
  12. package/package.json +4 -1
  13. package/skills/3d-intro-build/SKILL.md +1 -1
  14. package/skills/3d-intro-setup/SKILL.md +1 -1
  15. package/skills/all-in-one/SKILL.md +1 -1
  16. package/skills/append_wiki/SKILL.md +1 -1
  17. package/skills/arch-diagram/SKILL.md +1 -1
  18. package/skills/audit-mock/SKILL.md +1 -1
  19. package/skills/audit-security/SKILL.md +1 -1
  20. package/skills/audit-web-page/SKILL.md +1 -1
  21. package/skills/cleansing-memory/SKILL.md +1 -1
  22. package/skills/compact-copy/SKILL.md +1 -1
  23. package/skills/compact-wiki/SKILL.md +1 -1
  24. package/skills/curation/SKILL.md +1 -1
  25. package/skills/deep-init/SKILL.md +1 -1
  26. package/skills/deep-research/SKILL.md +1 -1
  27. package/skills/docs-setup/SKILL.md +1 -1
  28. package/skills/humanizer/SKILL.md +1 -1
  29. package/skills/interval-report/SKILL.md +1 -1
  30. package/skills/lineage/SKILL.md +152 -16
  31. package/skills/lineage/lineage.py +1210 -108
  32. package/skills/make-notion-guide/SKILL.md +1 -1
  33. package/skills/motion-graphic-make/SKILL.md +1 -1
  34. package/skills/motion-graphic-setup/SKILL.md +1 -1
  35. package/skills/nothing-design/SKILL.md +1 -1
  36. package/skills/obsidizer/SKILL.md +1 -1
  37. package/skills/omc-reference/SKILL.md +1 -1
  38. package/skills/payload-mon/SKILL.md +1 -1
  39. package/skills/pdf-vision-extract/SKILL.md +1 -1
  40. package/skills/play-qa/SKILL.md +1 -1
  41. package/skills/ralph-qa/SKILL.md +304 -112
  42. package/skills/ralph-qa/references/gemini-read-only.toml +9 -0
  43. package/skills/ralph-qa/references/gemini-seat.mjs +207 -0
  44. package/skills/ralph-qa/references/payload-scan.mjs +154 -0
  45. package/skills/ralph-qa/references/verifier-probe.mjs +881 -780
  46. package/skills/ready-compact/SKILL.md +1 -1
  47. package/skills/refresh-git-ignore/SKILL.md +1 -1
  48. package/skills/refresh-readme/SKILL.md +1 -1
  49. package/skills/rfp-author/SKILL.md +1 -1
  50. package/skills/setup-bypass-permissions/SKILL.md +123 -0
  51. package/skills/setup-bypass-permissions/agents/openai.yaml +4 -0
  52. package/skills/setup-bypass-permissions/scripts/bypass-permissions.mjs +381 -0
  53. package/skills/setup-bypass-permissions/scripts/fallback/bypass-permissions.ps1 +247 -0
  54. package/skills/setup-bypass-permissions/scripts/fallback/bypass-permissions.py +241 -0
  55. package/skills/{harness-factory → setup-harness-factory}/SKILL.md +3 -3
  56. package/skills/setup-insane-search/SKILL.md +1 -1
  57. package/skills/setup-java/SKILL.md +1 -1
  58. package/skills/setup-lsp/SKILL.md +1 -1
  59. package/skills/setup-mcp/SKILL.md +1 -1
  60. package/skills/setup-node/SKILL.md +1 -1
  61. package/skills/setup-omc/SKILL.md +1 -1
  62. package/skills/setup-omc-hud/SKILL.md +15 -1
  63. package/skills/setup-omc-hud/scripts/claude-update-last.mjs +247 -0
  64. package/skills/setup-playwright/SKILL.md +1 -1
  65. package/skills/setup-pwsh/SKILL.md +1 -1
  66. package/skills/setup-python/SKILL.md +1 -1
  67. package/skills/setup-sandbox/SKILL.md +1 -1
  68. package/skills/setup-stitch/SKILL.md +1 -1
  69. package/skills/setup-tmux/SKILL.md +1 -1
  70. package/skills/smart-compact/SKILL.md +1 -1
  71. package/skills/summary-wiki/SKILL.md +1 -1
  72. package/skills/tone-compact/SKILL.md +1 -1
  73. package/skills/ultra-init/SKILL.md +1 -1
  74. package/skills/ultra-interview/SKILL.md +1 -1
  75. package/skills/ultra-ui-qa/SKILL.md +1 -1
  76. package/skills/update-banker/SKILL.md +1 -1
  77. package/skills/vertical-pptx/SKILL.md +1 -1
  78. package/skills/vertical-pptx-setup/SKILL.md +1 -1
  79. package/skills/visual-ralph/SKILL.md +1 -1
  80. package/skills/graceful_pause/SKILL.md +0 -95
@@ -18,6 +18,7 @@ import os
18
18
  import pathlib
19
19
  import re
20
20
  import sys
21
+ import tempfile
21
22
  from html.parser import HTMLParser
22
23
 
23
24
  if sys.version_info < (3, 7):
@@ -32,6 +33,16 @@ ECHO_ASK = 40 # echo-exchange: user question length ceiling (A-4)
32
33
  ECHO_REPLY = 5 # echo-exchange: assistant reply length ceiling (A-4)
33
34
  _BLOCKQUOTE_MAX_DEPTH = 32 # blockquote recursion cap (C-2)
34
35
  CACHE_BASE = pathlib.Path.home() / ".cache" / "lineage"
36
+ REVIEW_SCHEMA = "lineage-review/1" # --emit-review pack, read back by --apply-review
37
+ REVIEW_PART_SCHEMA = "lineage-review-part/1"
38
+ REVIEW_PART = 40 # turns per part file: one reviewer's share
39
+ PREVIEW_HEAD = 1500 # a long turn's preview: head + tail, where conclusions sit
40
+ PREVIEW_TAIL = 700
41
+ SUMMARY_MAX = 120 # a reviewer's summary is cut to this, after redaction
42
+ LLM_CACHE_VERSION = 1 # bump to drop every cached reviewer decision
43
+ REVIEWER_TIMEOUT = 60 # --reviewer-timeout default, seconds
44
+ DECISIONS_MAX = 1_000_000 # bytes in one decisions file (a 40-turn part answers in some 20 KB)
45
+ DECISIONS_TRIES = 200 # places a wrapped answer's list may start, tried at most
35
46
 
36
47
  # ---------------------------------------------------------------- Noise / classify
37
48
  # NOTE: JSONL records carry RAW `<...>` tags. Every regex below uses raw `<`/`>`
@@ -155,7 +166,7 @@ def _mask(s: str) -> str:
155
166
  return (s[:4] + "****" + s[-4:]) if len(s) > 8 else "[REDACTED]"
156
167
 
157
168
 
158
- def redact(text: str, extra: str = None, mode: str = "full"):
169
+ def redact(text: str, extra: "str | None" = None, mode: str = "full"):
159
170
  """Multi-layer redaction. Returns (redacted_text, count_by_kind)."""
160
171
  counts = {}
161
172
  out = text
@@ -218,6 +229,72 @@ def redact(text: str, extra: str = None, mode: str = "full"):
218
229
  return out, counts
219
230
 
220
231
 
232
+ # A character of curl's user before the colon: no space, : or =, and a quote, backtick or ( only
233
+ # where no flag follows (-u, -Xu or --user, as either branch below starts). A match starts only
234
+ # there, so no user runs past the place where the next match may start: a long run stays linear,
235
+ # and a user joined to a variable by a - ("$USER"-bot) is read whole.
236
+ _CU_USER = r"(?:[^\s:=\"'`(]|[\"'`(](?!-(?:u|[A-Za-z0-9]{1,6}u[\s\"']|-user[\s=\"'])))"
237
+ # The quotes that may open the value, bare or escaped (\" and \\\" in a quoted command, '\'' and
238
+ # '"'"' from bash and shlex.quote, ^" in cmd, `" in PowerShell, $' in bash): up to six in a row.
239
+ _CU_LEAD = r"(?:[\\^`$]{0,7}[\"']){0,6}"
240
+ # Values read as a whole token, past the quotes that open them, that hold no user and password: a
241
+ # uid:gid (docker), a date format (date -u +%H:%M), two references ($UID:$GID, ${UID}:${GID},
242
+ # %USER%:%PASS%, $(id -u):$(id -g), whose inner `-u)` is a match of its own) and a path default
243
+ # (mktemp -u "${TMPDIR:-/tmp}/x"). A literal password, or a default that is no path or holds a
244
+ # colon, is hidden.
245
+ _CU_REF = r"(?:\$\{?\w+\}?|%\w+%|\$\([^\s()]*(?:\s+[^\s()]+)*\))"
246
+ _CU_SKIP = (r"(?!" + _CU_LEAD + r"(?:\+[\"']?%[-_0^#]?[A-Za-z%]|(?:\d+:\d+|(?:" + _CU_REF
247
+ + r"|[^\s:=\"'`(]*\)):" + _CU_REF + r")(?![^\s\"'`;|&)\\^])"
248
+ r"|\$\{\w+:[-=?+]?[/~.][^\s}\"'`(:]*\}[^\s:]*(?!\S)))")
249
+
250
+ # Extra patterns for text a reviewer model reads (part files) and nothing else: the page
251
+ # and --rulebase keep the patterns above, so their output stays as it was.
252
+ REVIEW_SECRET_PATTERNS = [
253
+ ("AnthropicKey", re.compile(r"(?<![A-Za-z0-9])sk-ant-[A-Za-z0-9_-]{20,}")),
254
+ ("OpenAIKey", re.compile(r"(?<![A-Za-z0-9-])sk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}")),
255
+ ("GoogleKey", re.compile(r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}")),
256
+ # bounded repeats: a long run of `key` or `a.` would otherwise take quadratic time
257
+ ("HexKey", re.compile(r"(?i)(?:key|token|secret)[\w-]{0,100}\s*[:=]\s*['\"]?[0-9a-f]{32,}")),
258
+ ("PasswordBare", re.compile(
259
+ r"(?i)(?:password|passwd|passcode|pwd|암호|비번|비밀\s?번호|패스워드)\s*[:=]\s*[^\s'\"]{6,}")),
260
+ # the password runs to the last @ before the host: it may hold an @ of its own
261
+ ("UrlCredential", re.compile(r"(?i)[a-z][a-z0-9+.-]{0,30}://[^\s:@/]*:[^\s/]{3,}@")),
262
+ ("Bearer", re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{16,}")),
263
+ # Authorization then `:`, `=`, `=>` or headers["Authorization"] =, a call's ("Authorization",
264
+ # "...") and a HAR name/value pair, or a space before a quote (nginx, Apache)
265
+ ("BasicAuth", re.compile(
266
+ r"(?i)\bauthorization"
267
+ r"(?:[\"']?\]?\s*(?:=>|[:=])"
268
+ r"|[\"']\s*,\s*(?:[\"']value[\"']\s*:\s*)?(?=[\"'])"
269
+ r"|\s+(?=[\"']))"
270
+ r"\s*[\"']?basic\s+[A-Za-z0-9+/]{8,}={0,2}")),
271
+ # curl's -u and --user flags, alone or last in a bundle (-su, -4u), with a user and password
272
+ # after a space, an = or nothing, quoted together or apart ("user":"pw"), with quotes bare,
273
+ # escaped or nested (_CU_LEAD) or none, in inline code or parentheses, or as two items of an
274
+ # argument list. An empty user with a token, and a
275
+ # long key as the user with no password, count too. The separators share no character with
276
+ # the user name, so a long run of them stays linear. The values _CU_SKIP names and a
277
+ # host:/path (rsync) are left.
278
+ ("CurlUser", re.compile(
279
+ r"(?<![^\s\"'`(])(?:-u\s*|-[A-Za-z0-9]{1,6}u\s+|--user(?:\s+|=))" + _CU_SKIP + _CU_LEAD
280
+ + r"(?:" + _CU_USER + r"*:(?!/)\S{3,}|" + _CU_USER + r"{15,}:(?=[\s\"'`).,;\\]|$))"
281
+ r"|([\"'])(?:-[A-Za-z0-9]{0,6}u|--user)\1\s*,\s*([\"'])" + _CU_SKIP
282
+ + r"(?:" + _CU_USER + r"*:[^\s\"']{3,}|" + _CU_USER + r"{15,}:)\2")),
283
+ ]
284
+
285
+
286
+ def review_redact(text, extra=None):
287
+ """Text as a reviewer model reads it: the reviewer-only patterns first, while the keys
288
+ are whole (the entropy rule would otherwise cut a key and leave the rest), then fully
289
+ redacted (never masked, whatever --redact-mode says). Returns (text, count)."""
290
+ n = 0
291
+ for name, pat in REVIEW_SECRET_PATTERNS:
292
+ text, k = pat.subn(f"[REDACTED:{name}]", text)
293
+ n += k
294
+ red, found = redact(text, extra=extra, mode="full")
295
+ return red, n + sum(found.values())
296
+
297
+
221
298
  # ---------------------------------------------------------------- Discovery
222
299
  _TITLE_RE = re.compile(r'"customTitle"\s*:\s*"([^"]+)"')
223
300
 
@@ -372,7 +449,8 @@ def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
372
449
  yield {"role": role, "text": text, "ts": ts, "uuid": uuid_,
373
450
  "tools": tools, "line_no": line_no,
374
451
  "parts": [text] if text else [], # R1: always seed parts
375
- "session": session_id, "session_name": session_name}
452
+ "session": session_id, "session_name": session_name,
453
+ "meta": bool(obj.get("isMeta"))} # a harness-injected body
376
454
 
377
455
 
378
456
  # ---------------------------------------------------------------- Classify (A-1..A-4)
@@ -443,10 +521,10 @@ def merge_assistant_runs(turns):
443
521
  return merged
444
522
 
445
523
 
446
- def drop_echo_exchanges(turns):
447
- """Drop no-op exchanges: a short user question + a tool-less token reply.
448
- Structural, not keyword-based (A-4). Same-session only. Dropping a reply
449
- also drops its question (else the question is orphaned).
524
+ def echo_indices(turns):
525
+ """Indices of no-op exchanges: a short user question + a tool-less token reply.
526
+ Structural, not keyword-based (A-4). Same-session only. A reply's question is
527
+ included with it (else the question is orphaned).
450
528
  """
451
529
  drop = set()
452
530
  for i, t in enumerate(turns):
@@ -457,6 +535,12 @@ def drop_echo_exchanges(turns):
457
535
  and len(n["text"].strip()) <= ECHO_REPLY
458
536
  and n.get("session") == t.get("session")):
459
537
  drop |= {i, i + 1}
538
+ return drop
539
+
540
+
541
+ def drop_echo_exchanges(turns):
542
+ """Drop the no-op exchanges echo_indices finds."""
543
+ drop = echo_indices(turns)
460
544
  return [t for i, t in enumerate(turns) if i not in drop]
461
545
 
462
546
 
@@ -567,23 +651,35 @@ def read_or_summarize(turn: dict, session_id: str, rebuild: bool = False,
567
651
  digest = hashlib.sha256(
568
652
  (turn["text"] + "\x00s" + str(SUMMARIZER_VERSION)).encode()
569
653
  ).hexdigest()[:8]
570
- p = cache_dir(turn.get("session") or session_id) / f"{turn['uuid']}-{digest}.txt"
571
- if p.exists() and not rebuild:
654
+ try: # ids come from the transcript: keep them in the cache
655
+ p = (cache_dir(_safe_name(turn.get("session") or session_id))
656
+ / f"{_safe_name(turn['uuid'])}-{digest}.txt")
657
+ except OSError as e: # no writable cache: summarize anyway
658
+ _warn_once("_no_cache", f"cache unavailable ({e}); summaries are not cached")
659
+ p = None
660
+ if p is not None and p.exists() and not rebuild:
572
661
  try:
573
662
  return p.read_text(encoding="utf-8").strip(), True
574
663
  except (OSError, UnicodeDecodeError):
575
664
  pass
576
665
  raw_summary = summarize_turn(turn)
577
666
  redacted, _ = redact(raw_summary, extra=redact_extra, mode=redact_mode)
578
- try:
579
- fd = os.open(str(p), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
580
- with os.fdopen(fd, "w", encoding="utf-8") as f:
581
- f.write(redacted)
582
- except OSError:
583
- pass
667
+ if p is not None:
668
+ try:
669
+ fd = os.open(str(p), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
670
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
671
+ f.write(redacted)
672
+ except OSError:
673
+ pass
584
674
  return redacted, False
585
675
 
586
676
 
677
+ def _warn_once(key, message):
678
+ if not getattr(_warn_once, key, False):
679
+ print(f"[lineage] WARN: {message}", file=sys.stderr)
680
+ setattr(_warn_once, key, True)
681
+
682
+
587
683
  # ---------------------------------------------------------------- Markdown (C-1..C-3)
588
684
  # Carve code spans and https-only links to opaque sentinels BEFORE emphasis, so
589
685
  # no emphasis rule ever runs over emitted markup (C-2). Sentinel chars exclude
@@ -920,12 +1016,32 @@ HTML_TEMPLATE = r"""<!DOCTYPE html>
920
1016
 
921
1017
 
922
1018
  # ---------------------------------------------------------------- Render rows
1019
+ def user_fold_summary(redacted_text):
1020
+ """The summary line of a user turn long enough to fold (USER_FOLD)."""
1021
+ return _cut(redacted_text.replace("\n", " "), 90)
1022
+
1023
+
1024
+ def _page_summary(t, summaries, session_id, rebuild, redact_extra, redact_mode):
1025
+ """(summary, cache_hit) of a bot turn as the page shows it: the reviewer's when
1026
+ `summaries` has one (cache_hit None: no rule summary was looked up), else the
1027
+ rules' (cached)."""
1028
+ if summaries is not None and t.get("uuid") in summaries:
1029
+ return summaries[t["uuid"]], None
1030
+ return read_or_summarize(t, session_id, rebuild=rebuild,
1031
+ redact_extra=redact_extra, redact_mode=redact_mode)
1032
+
1033
+
1034
+ def _reviewed_fold(t, summaries):
1035
+ """A reviewer's summary for a folded user turn, or None."""
1036
+ return summaries.get(t.get("uuid")) if summaries is not None else None
1037
+
1038
+
923
1039
  def _fmt_time(ts):
924
1040
  return ts[11:16] if ts and len(ts) >= 16 else ""
925
1041
 
926
1042
 
927
1043
  def render_rows(turns, session_id, redact_extra, redact_mode, rebuild,
928
- open_details=False, markdown=True, all_sessions=False):
1044
+ open_details=False, markdown=True, all_sessions=False, summaries=None):
929
1045
  rows = []
930
1046
  last_date = None
931
1047
  last_session = None
@@ -968,7 +1084,8 @@ def render_rows(turns, session_id, redact_extra, redact_mode, rebuild,
968
1084
  if role == "user":
969
1085
  red = _red(t["text"])
970
1086
  if len(red) > USER_FOLD:
971
- summary = html.escape(_cut(red.replace("\n", " "), 90))
1087
+ reviewed = _reviewed_fold(t, summaries)
1088
+ summary = html.escape(_red(reviewed) if reviewed else user_fold_summary(red))
972
1089
  body = render_body(red, markdown=markdown)
973
1090
  rows.append(
974
1091
  '<div class="row me"><span class="time">%s</span>'
@@ -984,12 +1101,11 @@ def render_rows(turns, session_id, redact_extra, redact_mode, rebuild,
984
1101
  continue
985
1102
 
986
1103
  # assistant or agent bubble
987
- cache_total += 1
988
- summary, hit = read_or_summarize(
989
- t, session_id, rebuild=rebuild,
990
- redact_extra=redact_extra, redact_mode=redact_mode)
991
- if hit:
992
- cache_hits += 1
1104
+ summary, hit = _page_summary(t, summaries, session_id, rebuild,
1105
+ redact_extra, redact_mode)
1106
+ if hit is not None:
1107
+ cache_total += 1
1108
+ cache_hits += bool(hit)
993
1109
  esc_sum = html.escape(_red(summary))
994
1110
  detail_body = render_body(_red(t["text"]), markdown=markdown)
995
1111
 
@@ -1105,6 +1221,693 @@ def self_verify(html_text: str):
1105
1221
 
1106
1222
 
1107
1223
  # ---------------------------------------------------------------- CLI
1224
+ # ---------------------------------------------------------------- Review pack
1225
+ # The default /lineage flow: --emit-review packs the turns for the session's model,
1226
+ # its reviewers write keep/summary decisions, --apply-review renders from them. The
1227
+ # pack holds redacted text only. Noise the rules are certain of (wrapper blocks, hook
1228
+ # feedback, injected bodies) is gone before the pack; the rules' judgement calls (echo
1229
+ # exchanges, tool-only turns) stay in it, marked, for the reviewers to confirm or undo.
1230
+ def _is_keep(v):
1231
+ return v is None or isinstance(v, bool)
1232
+
1233
+
1234
+ def _is_summary(v):
1235
+ return v is None or isinstance(v, str)
1236
+
1237
+
1238
+ def _first(*values):
1239
+ for v in values:
1240
+ if v is not None:
1241
+ return v
1242
+ return None
1243
+
1244
+
1245
+ def _bad(message):
1246
+ print(f"[lineage] ERROR: {message}", file=sys.stderr)
1247
+ return None
1248
+
1249
+
1250
+ def _write_private(path, text):
1251
+ """Write `text` to `path` as a fresh 0600 file, replaced whole. The temporary file
1252
+ comes from mkstemp (created exclusively, 0600), so a link planted at a guessable name
1253
+ is never followed, and it is removed when the write fails."""
1254
+ path = pathlib.Path(path)
1255
+ path.parent.mkdir(parents=True, exist_ok=True)
1256
+ fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")
1257
+ try:
1258
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
1259
+ f.write(text)
1260
+ os.replace(tmp, str(path))
1261
+ except BaseException:
1262
+ try:
1263
+ os.unlink(tmp)
1264
+ except OSError:
1265
+ pass
1266
+ raise
1267
+
1268
+
1269
+ def _safe_name(s):
1270
+ """A file-name-safe form of an id read from a transcript or a pack."""
1271
+ s = re.sub(r"[^A-Za-z0-9_.-]", "_", str(s or "default"))[:80]
1272
+ return s if s.strip(".") else "_"
1273
+
1274
+
1275
+ def _decision_context(why, keeps):
1276
+ """What a cached decision rests on beside the text: the rules' call on the turn (a
1277
+ `keep: null` defers to it) and the keep flags the reviewer was told to honour."""
1278
+ return "|".join([why or "", str(keeps.get("keep_trivia") is True),
1279
+ str(keeps.get("keep_tool_only") is True)])
1280
+
1281
+
1282
+ def _llm_cache_path(turn_id, session, redacted_text, context):
1283
+ digest = hashlib.sha256((redacted_text + "\x00llm" + str(LLM_CACHE_VERSION)
1284
+ + "\x00" + context).encode()).hexdigest()[:12]
1285
+ return cache_dir(_safe_name(session)) / f"{_safe_name(turn_id)}-{digest}-llm.json"
1286
+
1287
+
1288
+ def read_llm_cache(turn_id, session, redacted_text, context):
1289
+ """A reviewer's earlier {keep, summary} for this exact (redacted) text under the same
1290
+ rule call and keep flags (`context`), or None when no reviewer saw it so. Both values
1291
+ null: the reviewer left the turn to the rules."""
1292
+ try:
1293
+ path = _llm_cache_path(turn_id, session, redacted_text, context)
1294
+ except OSError as e: # no cache folder (a read-only HOME, say)
1295
+ _warn_once("_no_llm_cache", f"reviewer decision cache unavailable ({e}); "
1296
+ "every turn is reviewed")
1297
+ return None
1298
+ try:
1299
+ got = json.loads(path.read_text(encoding="utf-8"))
1300
+ except (OSError, ValueError, UnicodeDecodeError):
1301
+ return None
1302
+ if not isinstance(got, dict):
1303
+ return None
1304
+ keep, summary = got.get("keep"), got.get("summary")
1305
+ if not (_is_keep(keep) and _is_summary(summary)):
1306
+ return None
1307
+ return {"keep": keep, "summary": summary}
1308
+
1309
+
1310
+ def write_llm_cache(turn_id, session, redacted_text, context, keep, summary):
1311
+ try:
1312
+ _write_private(_llm_cache_path(turn_id, session, redacted_text, context),
1313
+ json.dumps({"keep": keep, "summary": summary}, ensure_ascii=False))
1314
+ except OSError as e:
1315
+ _warn_once("_no_llm_cache_write", f"reviewer decisions not cached ({e}); the next "
1316
+ "run reviews this run's turns again (decisions cached before still apply)")
1317
+
1318
+
1319
+ def heuristic_drops(turns, args):
1320
+ """{index: why} for the rules' judgement calls: echo exchanges, tool-only turns."""
1321
+ drops = {}
1322
+ if args.drop_trivia:
1323
+ drops.update((i, "echo") for i in echo_indices(turns))
1324
+ if args.hide_tool_only:
1325
+ for i, t in enumerate(turns):
1326
+ if i not in drops and t["role"] == "assistant" and not t["text"].strip():
1327
+ drops[i] = "tool-only"
1328
+ return drops
1329
+
1330
+
1331
+ def review_window(turns, drops, args):
1332
+ """Indices for the pack: the turns the range flags pick among those the rules keep
1333
+ (what a --rulebase run shows), the rule-dropped turns between them, and those past
1334
+ either end the pick reaches. None on an invalid --turns."""
1335
+ cand = [i for i, t in enumerate(turns) if _in_time(t, args)]
1336
+ kept = [i for i in cand if i not in drops]
1337
+ pool = kept or cand
1338
+ sl = _count_slice(len(pool), args)
1339
+ if sl is None:
1340
+ return None
1341
+ picked = pool[sl]
1342
+ if not kept or not picked:
1343
+ return picked
1344
+ lo = cand[0] if picked[0] == kept[0] else picked[0]
1345
+ hi = cand[-1] if picked[-1] == kept[-1] else picked[-1]
1346
+ chosen = set(picked)
1347
+ return [i for i in cand if lo <= i <= hi and (i in chosen or i in drops)]
1348
+
1349
+
1350
+ def _preview(text):
1351
+ """The reviewer's view of a turn: whole when short, else head + tail."""
1352
+ if len(text) <= PREVIEW_HEAD + PREVIEW_TAIL + 200:
1353
+ return text, False
1354
+ cut = len(text) - PREVIEW_HEAD - PREVIEW_TAIL
1355
+ return (text[:PREVIEW_HEAD].rstrip() + f"\n\n[중간 {cut}자 생략]\n\n"
1356
+ + text[-PREVIEW_TAIL:].lstrip()), True
1357
+
1358
+
1359
+ def _public_counts(counts):
1360
+ """Redaction counts by kind with the --redact-extra keywords folded into `custom`:
1361
+ the keywords themselves stay out of the files lineage writes."""
1362
+ out = {}
1363
+ for k, v in counts.items():
1364
+ key = "custom" if k.startswith("custom:") else k
1365
+ out[key] = out.get(key, 0) + v
1366
+ return out
1367
+
1368
+
1369
+ def _rule_summary(t, red, args):
1370
+ """The rules' summary for the page, made afresh and redacted with THIS run's settings.
1371
+ The summary cache is keyed on the text alone, so a cached one may predate a
1372
+ --redact-extra keyword or carry another --redact-mode."""
1373
+ if t["role"] in ("assistant", "agent"):
1374
+ return redact(summarize_turn(t), extra=args.redact_extra, mode=args.redact_mode)[0]
1375
+ if t["role"] == "user" and len(red) > USER_FOLD:
1376
+ return user_fold_summary(red)
1377
+ return None
1378
+
1379
+
1380
+ def _redacted_turn(t, extra):
1381
+ """`t` with its text and parts fully redacted, so a summary cut from it cannot cut a
1382
+ secret in two and keep the half no pattern recognises."""
1383
+ def clean(s):
1384
+ return review_redact(s, extra)[0]
1385
+ return dict(t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []])
1386
+
1387
+
1388
+ def _review_view(t, red, args):
1389
+ """What a reviewer reads beside the preview: the rules' summary and the agent's
1390
+ name, fully redacted (the page's copies may be masked)."""
1391
+ if t["role"] in ("assistant", "agent"):
1392
+ summary = review_redact(summarize_turn(_redacted_turn(t, args.redact_extra)),
1393
+ args.redact_extra)[0]
1394
+ elif t["role"] == "user" and len(red) > USER_FOLD:
1395
+ summary = user_fold_summary(review_redact(t["text"], args.redact_extra)[0])
1396
+ else:
1397
+ summary = None
1398
+ who = review_redact(t["agent_from"], args.redact_extra)[0] if t.get("agent_from") else None
1399
+ return {"summary": summary, "agent_from": who}
1400
+
1401
+
1402
+ def _pack_turn(n, t, tid, why, session_id, args, counts):
1403
+ """One turn of the pack under its final id `tid` (redacted text, the rules' decision,
1404
+ the reviewer's slot), and the reviewer's view of it."""
1405
+ red, found = redact(t["text"], extra=args.redact_extra, mode=args.redact_mode)
1406
+ for k, v in found.items():
1407
+ counts[k] = counts.get(k, 0) + v
1408
+ preview, clipped = _preview(review_redact(t["text"], args.redact_extra)[0])
1409
+ session = t.get("session") or session_id
1410
+ context = _decision_context(why, _keeps(args))
1411
+ cached = None if args.rebuild_summaries else read_llm_cache(tid, session, red, context)
1412
+ view = _review_view(t, red, args)
1413
+ turn = {"id": tid, "n": n, "role": t["role"], "ts": t.get("ts") or "",
1414
+ "tools": dict(t.get("tools") or {}), "text": red,
1415
+ "redactions": _public_counts(found), "preview": preview, "clipped": clipped,
1416
+ "rule": {"keep": why is None, "why": why,
1417
+ "summary": _rule_summary(t, red, args),
1418
+ "review_summary": view["summary"]},
1419
+ "llm": dict(cached, cached=True) if cached else {"keep": None, "summary": None}}
1420
+ if t.get("meta"):
1421
+ turn["meta"] = True
1422
+ for k in ("session", "session_name", "agent_from"):
1423
+ if t.get(k):
1424
+ turn[k] = t[k]
1425
+ return turn, view
1426
+
1427
+
1428
+ def _final_ids(turns, window):
1429
+ """Pack ids, fixed before anything reads the decision cache: the turn uuid, with #N
1430
+ on a repeat (stdin uuids are content hashes, so equal texts share one). Counted over
1431
+ every turn, not the window, so a turn keeps its id whichever range is picked."""
1432
+ seen, ids = {}, {}
1433
+ for i, t in enumerate(turns):
1434
+ u = t["uuid"]
1435
+ seen[u] = seen.get(u, 0) + 1
1436
+ ids[i] = u if seen[u] == 1 else f"{u}#{seen[u]}"
1437
+ return [ids[i] for i in window]
1438
+
1439
+
1440
+ def _split_even(items, size):
1441
+ """Parts of at most `size` items, as even as possible (41: 21 + 20, not 40 + 1)."""
1442
+ if not items:
1443
+ return []
1444
+ step = -(-len(items) // -(-len(items) // size))
1445
+ return [items[k:k + step] for k in range(0, len(items), step)]
1446
+
1447
+
1448
+ def _decisions_name(part_path):
1449
+ return part_path.with_name(part_path.stem + ".decisions.json").name
1450
+
1451
+
1452
+ def _clear_parts(pack_path):
1453
+ """Remove the part and decisions files an earlier --emit-review left beside the
1454
+ pack, and an earlier gate's samples: that run's decisions must not be applied to this
1455
+ one, and its samples would stop this pack's rerun. Returns the files it could not
1456
+ remove."""
1457
+ prefix = pack_path.stem + ".part-"
1458
+ samples = f"{pack_path.stem}.reviewer-input.json"
1459
+ try:
1460
+ olds = [p for p in pack_path.parent.iterdir()
1461
+ if (p.name.startswith(prefix) and p.name.endswith(".json")) or p.name == samples]
1462
+ except OSError:
1463
+ return []
1464
+ failed = []
1465
+ for p in olds:
1466
+ try:
1467
+ p.unlink()
1468
+ except OSError:
1469
+ failed.append(p)
1470
+ return failed
1471
+
1472
+
1473
+ def _hide_keywords(s, extra):
1474
+ """`s` with each --redact-extra keyword replaced (for short names such as tool names)."""
1475
+ for kw in [k.strip() for k in (extra or "").split(",") if k.strip()]:
1476
+ s = re.sub(re.escape(kw), "[REDACTED]", s, flags=re.IGNORECASE)
1477
+ return s
1478
+
1479
+
1480
+ def _keeps(args):
1481
+ """The keep flags the reviewers must honour: with them, the rules keep those turns."""
1482
+ return {"keep_trivia": not args.drop_trivia, "keep_tool_only": not args.hide_tool_only}
1483
+
1484
+
1485
+ def _sheet(k, of, part_path, group, args):
1486
+ """Part file `k`: the turns one reviewer reads, without the page-only fields."""
1487
+ turns = []
1488
+ for x, view in group:
1489
+ y = {key: v for key, v in x.items()
1490
+ if key not in ("text", "session", "session_name", "agent_from", "redactions")}
1491
+ y["rule"] = {k: v for k, v in x["rule"].items() if k != "review_summary"}
1492
+ y["rule"]["summary"] = view["summary"]
1493
+ y["tools"] = {_hide_keywords(name, args.redact_extra): n for name, n in x["tools"].items()}
1494
+ if x["llm"].get("summary"): # cached: redacted for the page, maybe masked
1495
+ s = _MASKED.sub("[REDACTED]", x["llm"]["summary"])
1496
+ y["llm"] = dict(x["llm"], summary=review_redact(s, args.redact_extra)[0])
1497
+ if view["agent_from"]:
1498
+ y["agent_from"] = view["agent_from"]
1499
+ turns.append(y)
1500
+ return {"schema": REVIEW_PART_SCHEMA, "part": k, "of": of, **_keeps(args),
1501
+ "decisions": _decisions_name(part_path), "turns": turns}
1502
+
1503
+
1504
+ def _pack_header(session_id, output, args, counts):
1505
+ """The pack's settings: what --apply-review renders with and the quality gate it runs."""
1506
+ return {"schema": REVIEW_SCHEMA, "source": session_id, "output": str(output),
1507
+ "title": args.title, "markdown": args.markdown, "open": args.open_details,
1508
+ "redact_mode": args.redact_mode, "redact_extra": bool(args.redact_extra),
1509
+ "all_sessions": bool(args.all_sessions), "redactions": _public_counts(counts),
1510
+ **_keeps(args),
1511
+ "gate": {"skip_reviewer": bool(args.skip_reviewer),
1512
+ "reviewer_output": args.reviewer_output,
1513
+ "reviewer_timeout": args.reviewer_timeout}}
1514
+
1515
+
1516
+ def _report_emit(pack_path, items, paths, groups):
1517
+ dropped = sum(1 for x in items if not x["rule"]["keep"])
1518
+ cached = sum(1 for x in items if x["llm"].get("cached"))
1519
+ print(f"[lineage] review pack: {pack_path} (turns={len(items)} "
1520
+ f"rule-dropped={dropped} cached={cached})", file=sys.stderr)
1521
+ for k, (p, g) in enumerate(zip(paths, groups), 1):
1522
+ todo = sum(1 for x, _ in g if not x["llm"].get("cached"))
1523
+ print(f"[lineage] part {k}/{len(groups)}: {p} (to review: {todo}) "
1524
+ f"-> {p.with_name(_decisions_name(p))}", file=sys.stderr)
1525
+ print(f"[lineage] next: write each part's decisions, then "
1526
+ f"--apply-review {pack_path}", file=sys.stderr)
1527
+
1528
+
1529
+ def emit_review(turns, drops, window, session_id, output, args):
1530
+ """Write the pack and one part file per reviewer; print them and the next step. A
1531
+ --reviewer-output that holds something other than a verdict list stops it here, before
1532
+ any reviewer runs: the gate would refuse it only after them."""
1533
+ rop = pathlib.Path(args.reviewer_output) if args.reviewer_output and not args.skip_reviewer else None
1534
+ if rop is not None and rop.exists() and not _is_verdict_file(rop):
1535
+ _bad(f"{rop} is there and is not a verdict list; give a --reviewer-output that does not exist yet")
1536
+ return 2
1537
+ pack_path = pathlib.Path(args.emit_review)
1538
+ counts, pairs = {}, []
1539
+ for n, (i, tid) in enumerate(zip(window, _final_ids(turns, window)), 1):
1540
+ pairs.append(_pack_turn(n, turns[i], tid, drops.get(i), session_id, args, counts))
1541
+ groups = _split_even(pairs, REVIEW_PART)
1542
+ paths = [pack_path.with_name(f"{pack_path.stem}.part-{k}.json")
1543
+ for k in range(1, len(groups) + 1)]
1544
+ pack = _pack_header(session_id, _with_timestamp_suffix(output), args, counts)
1545
+ pack["parts"] = [{"part": k, "file": p.name, "decisions": _decisions_name(p),
1546
+ "ids": [x["id"] for x, _ in g]}
1547
+ for k, (p, g) in enumerate(zip(paths, groups), 1)]
1548
+ pack["turns"] = [x for x, _ in pairs]
1549
+ stuck = _clear_parts(pack_path)
1550
+ if stuck:
1551
+ _bad(f"cannot remove an earlier run's files: {', '.join(map(str, stuck))}")
1552
+ return 2
1553
+ try:
1554
+ _write_private(pack_path, json.dumps(pack, ensure_ascii=False, indent=1))
1555
+ for k, (p, g) in enumerate(zip(paths, groups), 1):
1556
+ _write_private(p, json.dumps(_sheet(k, len(groups), p, g, args),
1557
+ ensure_ascii=False, indent=1))
1558
+ except OSError as e:
1559
+ _bad(f"cannot write the review pack: {e}")
1560
+ return 2
1561
+ _report_emit(pack_path, pack["turns"], paths, groups)
1562
+ return 0
1563
+
1564
+
1565
+ def _is_pack_turn(x):
1566
+ return (isinstance(x, dict) and isinstance(x.get("id"), str)
1567
+ and x.get("role") in ("user", "assistant", "agent", "mark")
1568
+ and isinstance(x.get("text"), str)
1569
+ and isinstance(x.get("rule"), dict) and isinstance(x["rule"].get("keep"), bool)
1570
+ and _is_summary(x["rule"].get("summary"))
1571
+ and isinstance(x.get("llm"), dict) and _is_keep(x["llm"].get("keep"))
1572
+ and _is_summary(x["llm"].get("summary")))
1573
+
1574
+
1575
+ def _load_pack(path):
1576
+ try:
1577
+ pack = json.loads(path.read_text(encoding="utf-8"))
1578
+ except (OSError, ValueError, UnicodeDecodeError) as e:
1579
+ return _bad(f"cannot read review pack {path}: {e}")
1580
+ if not isinstance(pack, dict) or pack.get("schema") != REVIEW_SCHEMA:
1581
+ return _bad(f"{path} is not a {REVIEW_SCHEMA} pack")
1582
+ if not isinstance(pack.get("turns"), list) or not all(map(_is_pack_turn, pack["turns"])):
1583
+ return _bad(f"{path}: malformed turns")
1584
+ return pack
1585
+
1586
+
1587
+ def _decision_files(pack_path, pack, explicit):
1588
+ """The parts' decisions files that exist beside the pack, then each --decisions.
1589
+ Those beside the pack are made 0600: the session wrote them with its own umask."""
1590
+ beside = [pack_path.with_name(p["decisions"]) for p in pack.get("parts") or []
1591
+ if isinstance(p, dict) and isinstance(p.get("decisions"), str)]
1592
+ files, seen = [], set()
1593
+ for f in [f for f in beside if f.exists()] + [pathlib.Path(f) for f in explicit]:
1594
+ if os.path.realpath(str(f)) not in seen:
1595
+ seen.add(os.path.realpath(str(f)))
1596
+ files.append(f)
1597
+ for f in beside:
1598
+ try:
1599
+ os.chmod(str(f), 0o600)
1600
+ except OSError:
1601
+ pass
1602
+ return files
1603
+
1604
+
1605
+ # Keys of a part file's turns: a list of objects carrying one is the reviewer quoting its
1606
+ # input, not its answer.
1607
+ _PART_TURN_KEYS = frozenset({"n", "role", "ts", "tools", "text", "preview", "clipped",
1608
+ "rule", "llm", "meta", "agent_from"})
1609
+
1610
+
1611
+ def _is_answer(d):
1612
+ """A decision, extra keys and all; a part turn quoted back (its keep and summary sit
1613
+ under rule and llm, not on top) is not one."""
1614
+ return (isinstance(d, dict) and "id" in d
1615
+ and ("keep" in d or "summary" in d or not _PART_TURN_KEYS & d.keys()))
1616
+
1617
+
1618
+ def _decisions_text(text):
1619
+ """The JSON list in a reviewer's answer: the whole text, or, when the answer came
1620
+ wrapped (a code fence, a sentence with brackets of its own, a list of files or a quote
1621
+ of the part's turns around it), the longest list of {id, ...} objects in it that is
1622
+ not such a quote (the last of equal ones: an example quoted before the answer loses),
1623
+ else an empty list it holds."""
1624
+ try:
1625
+ return json.loads(text)
1626
+ except (ValueError, RecursionError):
1627
+ pass
1628
+ lists, empty = [], False
1629
+ decoder = json.JSONDecoder()
1630
+ # only where a list of objects or an empty list can start: a run of `[` is tried once,
1631
+ # and only so many starts (each failed try parses up to the nesting limit)
1632
+ for k, m in enumerate(re.finditer(r"\[(?=\s*[{\]])", text)):
1633
+ if k == DECISIONS_TRIES:
1634
+ raise ValueError(f"more than {DECISIONS_TRIES} places a list may start; "
1635
+ "write the bare JSON array")
1636
+ try:
1637
+ got, _ = decoder.raw_decode(text, m.start())
1638
+ except (ValueError, RecursionError):
1639
+ continue
1640
+ if got == []:
1641
+ empty = True
1642
+ elif isinstance(got, list) and all(_is_answer(d) for d in got):
1643
+ lists.append(got)
1644
+ if lists:
1645
+ return max(reversed(lists), key=len)
1646
+ if empty:
1647
+ return []
1648
+ raise ValueError("no JSON list of {id, keep, summary} objects in it")
1649
+
1650
+
1651
+ def _decision_problem(d):
1652
+ """Why item `d` of a decisions file is not a decision, or ""."""
1653
+ if not (isinstance(d, dict) and isinstance(d.get("id"), str)
1654
+ and _is_keep(d.get("keep")) and _is_summary(d.get("summary"))):
1655
+ return 'want {"id": str, "keep": true|false|null, "summary": str|null}'
1656
+ if not _is_answer(d):
1657
+ return "a turn of the part file, not a decision"
1658
+ return ""
1659
+
1660
+
1661
+ def _load_decisions(files):
1662
+ """{id: {keep, summary}} from the files in order, a later value winning; None when a
1663
+ file is not a JSON list of {id, keep, summary}. A file whose every decision is null
1664
+ is read, with a WARN: the rules then decide all its turns."""
1665
+ merged = {}
1666
+ for f in files:
1667
+ try:
1668
+ if f.stat().st_size > DECISIONS_MAX:
1669
+ raise ValueError(f"larger than {DECISIONS_MAX} bytes")
1670
+ items = _decisions_text(f.read_text(encoding="utf-8"))
1671
+ except (OSError, ValueError, UnicodeDecodeError, RecursionError) as e:
1672
+ return _bad(f"cannot read decisions {f}: {e}")
1673
+ if not isinstance(items, list):
1674
+ return _bad(f"{f}: decisions must be a JSON list of {{id, keep, summary}}")
1675
+ for k, d in enumerate(items):
1676
+ why = _decision_problem(d)
1677
+ if why:
1678
+ return _bad(f"{f}[{k}]: {why}")
1679
+ cur = merged.setdefault(d["id"], {})
1680
+ for key in ("keep", "summary"):
1681
+ if d.get(key) is not None:
1682
+ cur[key] = d[key]
1683
+ if items and all(d.get("keep") is None and d.get("summary") is None for d in items):
1684
+ print(f"[lineage] WARN: {f}: all {len(items)} decisions are null; the rules "
1685
+ "decide those turns", file=sys.stderr)
1686
+ return merged
1687
+
1688
+
1689
+ def _clean_summary(s, extra, mode):
1690
+ """A reviewer's summary as the page shows it (one line, redacted, then cut), and how
1691
+ many secret-like strings that redaction hid. The reviewer patterns go first and hide
1692
+ a key whole, whatever --redact-mode says: the summary is cached and goes back to the
1693
+ next reviewer and the gate's critic, so no part of a key may stay in it."""
1694
+ pre, n = review_redact(" ".join(s.split()), extra)
1695
+ red, found = redact(pre, extra=extra, mode=mode)
1696
+ return (_cut(red, SUMMARY_MAX) or None), n + sum(found.values())
1697
+
1698
+
1699
+ def _warn_coverage(pack, decided):
1700
+ """Name the parts that left turns undecided: the rules decide those, and since only
1701
+ decided turns are cached, they are reviewed again next time."""
1702
+ for p in pack.get("parts") or []:
1703
+ ids = p.get("ids") if isinstance(p, dict) else None
1704
+ if not isinstance(ids, list):
1705
+ continue
1706
+ todo = [i for i in ids if i not in decided]
1707
+ if todo:
1708
+ more = ", ..." if len(todo) > 3 else ""
1709
+ print(f"[lineage] WARN: part {p.get('part')}: {len(todo)}/{len(ids)} turns "
1710
+ f"undecided ({', '.join(map(str, todo[:3]))}{more}); the rules decide them",
1711
+ file=sys.stderr)
1712
+
1713
+
1714
+ def _reviewer_summary(x, d, args, stats):
1715
+ """The reviewer's summary for pack turn `x` (this run's decision `d`, else the cached
1716
+ one), cleaned; secret-like strings in it are counted and the turn is named."""
1717
+ raw = _first(d.get("summary"), x["llm"].get("summary"))
1718
+ if not raw:
1719
+ return None
1720
+ s, n = _clean_summary(raw, args.redact_extra, args.redact_mode)
1721
+ if n:
1722
+ stats["hidden"] += n
1723
+ print(f"[lineage] WARN: the reviewer summary for {x['id']} held {n} "
1724
+ "secret-like string(s); redacted", file=sys.stderr)
1725
+ return s
1726
+
1727
+
1728
+ def _show(x, llm_summary, turns, summaries, stats, prior):
1729
+ """Put kept pack turn `x` on the page with the summary it shows; `prior` adds up what
1730
+ --emit-review redacted."""
1731
+ shows = x["role"] in ("assistant", "agent") or (
1732
+ x["role"] == "user" and len(x["text"]) > USER_FOLD)
1733
+ if llm_summary and not shows:
1734
+ stats["ignored"] += 1
1735
+ summary = (llm_summary if shows else None) or x["rule"].get("summary")
1736
+ if summary:
1737
+ summaries[x["id"]] = summary
1738
+ stats["reviewer" if llm_summary and shows else "rule"] += bool(summary)
1739
+ for k, v in (x.get("redactions") or {}).items():
1740
+ if isinstance(v, int):
1741
+ prior[k] = prior.get(k, 0) + v
1742
+ # what the gate's critic reads: the reviewer's summary, else the rules' cut from the
1743
+ # fully redacted text (the page's is cut first and may keep half a secret)
1744
+ sample = (llm_summary if shows else None) or x["rule"].get("review_summary")
1745
+ turns.append({"uuid": x["id"], "role": x["role"], "text": x["text"],
1746
+ "ts": x.get("ts") or None, "tools": dict(x.get("tools") or {}),
1747
+ "parts": [], "session": x.get("session"),
1748
+ "session_name": x.get("session_name"),
1749
+ "agent_from": x.get("agent_from"), "preview": x.get("preview"),
1750
+ "sample_summary": sample})
1751
+
1752
+
1753
+ def _kept_by_flag(pack, x):
1754
+ """True when a keep flag given at --emit-review keeps pack turn `x` whatever the
1755
+ reviewer says: --keep-trivia keeps every turn the rules keep, --keep-tool-only the
1756
+ tool-only ones (the rules keep those only with that flag)."""
1757
+ if not x["rule"]["keep"]:
1758
+ return False
1759
+ if pack.get("keep_trivia") is True:
1760
+ return True
1761
+ tool_only = x["role"] == "assistant" and not x["text"].strip()
1762
+ return pack.get("keep_tool_only") is True and tool_only
1763
+
1764
+
1765
+ def _review_result(pack, merged, args):
1766
+ """(turns to render, their summaries, cache writes, counts, prior redactions, the ids of
1767
+ turns the user typed that the reviewers dropped) from the pack and the decisions: a
1768
+ reviewer's value first, then the pack's cached one, then the rules'. A keep flag outranks
1769
+ a reviewer's `false`; the cache still gets what was said."""
1770
+ turns, summaries, writes, prior, typed = [], {}, [], {}, []
1771
+ stats = {"restored": 0, "dropped": 0, "reviewer": 0, "rule": 0, "ignored": 0,
1772
+ "hidden": 0, "flag-kept": 0}
1773
+ for x in pack["turns"]:
1774
+ d = merged.get(x["id"], {})
1775
+ llm_keep = _first(d.get("keep"), x["llm"].get("keep"))
1776
+ held = llm_keep is False and _kept_by_flag(pack, x)
1777
+ stats["flag-kept"] += held
1778
+ keep = bool(_first(None if held else llm_keep, x["rule"]["keep"]))
1779
+ if not keep and x["role"] == "user" and x["rule"]["keep"] and not x.get("meta"):
1780
+ typed.append(x["id"]) # typed by the user: the rules keep these
1781
+ llm_summary = _reviewer_summary(x, d, args, stats)
1782
+ if x["id"] in merged: # reviewed, even if left to the rules
1783
+ writes.append((x["id"], x.get("session") or pack.get("source"), x["text"],
1784
+ _decision_context(x["rule"].get("why"), pack), llm_keep, llm_summary))
1785
+ stats["restored"] += keep and not x["rule"]["keep"]
1786
+ stats["dropped"] += x["rule"]["keep"] and not keep
1787
+ if keep:
1788
+ _show(x, llm_summary, turns, summaries, stats, prior)
1789
+ return turns, summaries, writes, stats, prior, typed
1790
+
1791
+
1792
+ def _gate_from_pack(args, pack):
1793
+ """The quality-gate flags given at --emit-review, unless this run gives its own."""
1794
+ gate = pack.get("gate") if isinstance(pack.get("gate"), dict) else {}
1795
+ args.skip_reviewer = bool(args.skip_reviewer or gate.get("skip_reviewer") is True)
1796
+ if args.reviewer_output is None and isinstance(gate.get("reviewer_output"), str):
1797
+ args.reviewer_output = gate["reviewer_output"]
1798
+ if args.reviewer_timeout is None and isinstance(gate.get("reviewer_timeout"), int):
1799
+ args.reviewer_timeout = gate["reviewer_timeout"]
1800
+
1801
+
1802
+ def _check_pack(pack, merged, args):
1803
+ """Warnings before the render, and the settings the pack carries from --emit-review."""
1804
+ ids = {x["id"] for x in pack["turns"]}
1805
+ unknown = [i for i in merged if i not in ids]
1806
+ if unknown:
1807
+ print(f"[lineage] WARN: decisions name {len(unknown)} turn(s) not in the pack, "
1808
+ f"ignored: {', '.join(unknown[:5])}", file=sys.stderr)
1809
+ _warn_coverage(pack, set(merged) | {x["id"] for x in pack["turns"]
1810
+ if x["llm"].get("cached")})
1811
+ if pack.get("redact_mode") in ("full", "mask"):
1812
+ args.redact_mode = pack["redact_mode"]
1813
+ if pack.get("redact_extra") is True and not args.redact_extra:
1814
+ print("[lineage] WARN: --emit-review had --redact-extra; give the same keywords "
1815
+ "here too, or reviewer summaries miss that redaction", file=sys.stderr)
1816
+ _gate_from_pack(args, pack)
1817
+
1818
+
1819
+ def _remove_review_files(pack_path, pack):
1820
+ """After a good render the pack, its parts and the decisions beside it go: they hold
1821
+ the redacted session."""
1822
+ samples = f"{pack_path.stem}.reviewer-input.json" # a failed gate's, beside the pack
1823
+ names = [pack_path.name, samples]
1824
+ for p in pack.get("parts") or []:
1825
+ if isinstance(p, dict):
1826
+ names += [p.get(k) for k in ("file", "decisions") if isinstance(p.get(k), str)]
1827
+ prefix = pack_path.stem + ".part-"
1828
+ left = []
1829
+ for name in names:
1830
+ if name not in (pack_path.name, samples) and not name.startswith(prefix):
1831
+ continue # only what --emit-review named
1832
+ try:
1833
+ pack_path.with_name(name).unlink()
1834
+ except FileNotFoundError:
1835
+ pass
1836
+ except (OSError, ValueError):
1837
+ left.append(name)
1838
+ if left:
1839
+ print(f"[lineage] WARN: could not remove review files: {', '.join(left)}",
1840
+ file=sys.stderr)
1841
+
1842
+
1843
+ def _samples_left_unread(pack_path, args):
1844
+ """True, with the reason, when a gated run left its samples beside the pack and this run
1845
+ gives neither a verdict path nor --skip-reviewer: it would pass over that run's verdict."""
1846
+ left = pack_path.with_name(f"{pack_path.stem}.reviewer-input.json")
1847
+ if left.exists() and not (args.reviewer_output or args.skip_reviewer):
1848
+ _bad(f"{left} was left by a gated run; give its --reviewer-output again, or --skip-reviewer")
1849
+ return True
1850
+ return False
1851
+
1852
+
1853
+ def _review_output(args, pack):
1854
+ """The page path: the pack's when this run gives none, else the given one with the pack's
1855
+ stamp, so the gate's runs write one page."""
1856
+ if args.output is None:
1857
+ return pack.get("output") or "work/lineage-review.html"
1858
+ return _with_pack_stamp(args.output, pack.get("output"))
1859
+
1860
+
1861
+ def _report_review(stats, turns, pack, typed):
1862
+ """What the render took from the reviewers, and the turns typed by the user they dropped."""
1863
+ print(f"[lineage] review applied: shown={len(turns)}/{len(pack['turns'])} "
1864
+ f"restored={stats['restored']} dropped={stats['dropped']} "
1865
+ f"summaries reviewer={stats['reviewer']} rule={stats['rule']}", file=sys.stderr)
1866
+ if stats["ignored"]:
1867
+ print(f"[lineage] note: {stats['ignored']} summaries for turns the page shows "
1868
+ "in full were ignored", file=sys.stderr)
1869
+ if stats["flag-kept"]:
1870
+ print(f"[lineage] note: {stats['flag-kept']} turn(s) a reviewer marked keep: false "
1871
+ "stay, as --keep-trivia or --keep-tool-only asked", file=sys.stderr)
1872
+ if typed:
1873
+ print(f"[lineage] WARN: reviewers dropped {len(typed)} typed user turn(s): {', '.join(typed[:10])}"
1874
+ f"{' ...' if len(typed) > 10 else ''}; tell the user", file=sys.stderr)
1875
+
1876
+
1877
+ def apply_review(args):
1878
+ """Render from a review pack and its decisions; cache the reviewers' values and, after
1879
+ a good render, remove the review files."""
1880
+ pack_path = pathlib.Path(args.apply_review)
1881
+ pack = _load_pack(pack_path)
1882
+ if pack is None:
1883
+ return 2
1884
+ merged = _load_decisions(_decision_files(pack_path, pack, args.decisions))
1885
+ if merged is None:
1886
+ return 2
1887
+ _check_pack(pack, merged, args)
1888
+ if _samples_left_unread(pack_path, args):
1889
+ return 2
1890
+ turns, summaries, writes, stats, prior, typed = _review_result(pack, merged, args)
1891
+ args.output = _review_output(args, pack)
1892
+ if stats["hidden"]:
1893
+ prior["reviewer-summary"] = stats["hidden"]
1894
+ try:
1895
+ rc = render_and_write(turns, pack.get("source") or "session", args,
1896
+ all_sessions=bool(pack.get("all_sessions")),
1897
+ markdown=pack.get("markdown") is not False,
1898
+ open_details=bool(pack.get("open")),
1899
+ title=str(pack.get("title") or "Session Lineage"),
1900
+ summaries=summaries, prior_redactions=prior)
1901
+ except SystemExit as e: # the quality gate failed
1902
+ rc = int(e.code) if isinstance(e.code, int) else 2
1903
+ if rc == 0:
1904
+ for w in writes:
1905
+ write_llm_cache(*w)
1906
+ _remove_review_files(pack_path, pack)
1907
+ _report_review(stats, turns, pack, typed)
1908
+ return rc
1909
+
1910
+
1108
1911
  def build_arg_parser() -> argparse.ArgumentParser:
1109
1912
  ap = argparse.ArgumentParser(
1110
1913
  prog="lineage",
@@ -1133,9 +1936,11 @@ def build_arg_parser() -> argparse.ArgumentParser:
1133
1936
  help="skip reviewer quality gate (warn-only)")
1134
1937
  ap.add_argument("--reviewer-output",
1135
1938
  help="path to Critic response JSON; enforces the quality gate")
1136
- ap.add_argument("--reviewer-timeout", type=int, default=60,
1137
- help="seconds to wait for reviewer-output (default 60)")
1939
+ ap.add_argument("--reviewer-timeout", type=int, default=None,
1940
+ help="seconds to wait for reviewer-output (default 60 with --rulebase; "
1941
+ "the reviewed flow waits only when this is given)")
1138
1942
  ap.add_argument("--title", default="Session Lineage", help="HTML title")
1943
+ _add_review_args(ap)
1139
1944
  # Readability defaults are ON. Opt-out flags restore raw/older behavior.
1140
1945
  ap.add_argument("--hide-tool-only", dest="hide_tool_only", action="store_true",
1141
1946
  help="(DEFAULT) drop assistant turns that are only tool calls")
@@ -1155,6 +1960,19 @@ def build_arg_parser() -> argparse.ArgumentParser:
1155
1960
  return ap
1156
1961
 
1157
1962
 
1963
+ def _add_review_args(ap):
1964
+ """The flags of the reviewed flow (3.0.0): --emit-review, the session's reviewers,
1965
+ then --apply-review."""
1966
+ ap.add_argument("--rulebase", action="store_true",
1967
+ help="one pass, rules only (the default when no review flag is given)")
1968
+ ap.add_argument("--emit-review", metavar="PACK",
1969
+ help="write the turns for the session model to review (JSON) and stop")
1970
+ ap.add_argument("--apply-review", metavar="PACK",
1971
+ help="render from a review pack and the reviewer's decisions")
1972
+ ap.add_argument("--decisions", action="append", default=[], metavar="FILE",
1973
+ help="reviewer decisions [{id, keep, summary}] for --apply-review (repeatable)")
1974
+
1975
+
1158
1976
  def _open_stream(path):
1159
1977
  return open(path, encoding="utf-8", errors="strict")
1160
1978
 
@@ -1172,6 +1990,21 @@ def _load_turns_from(path, unsafe_schema, session_id, session_name):
1172
1990
  return []
1173
1991
 
1174
1992
 
1993
+ def _flag_clash(args):
1994
+ """The review flags that cannot go together, as one message; None when they can."""
1995
+ if args.emit_review == "" or args.apply_review == "":
1996
+ return "--emit-review and --apply-review need a file path"
1997
+ if args.purge_cache and (args.emit_review or args.apply_review):
1998
+ return "--purge-cache is its own run: it takes no review pack"
1999
+ if args.emit_review and args.apply_review:
2000
+ return "--emit-review and --apply-review are two separate runs"
2001
+ if args.rulebase and (args.emit_review or args.apply_review):
2002
+ return "--rulebase is the one-pass run: it takes no review pack"
2003
+ if args.decisions and not args.apply_review:
2004
+ return "--decisions goes with --apply-review"
2005
+ return None
2006
+
2007
+
1175
2008
  def main(argv=None) -> int:
1176
2009
  args = build_arg_parser().parse_args(argv)
1177
2010
 
@@ -1180,6 +2013,11 @@ def main(argv=None) -> int:
1180
2013
  args.redact_extra = ",".join(
1181
2014
  p for p in (args.redact_extra, env_extra) if p)
1182
2015
 
2016
+ clash = _flag_clash(args)
2017
+ if clash:
2018
+ print(f"[lineage] ERROR: {clash}", file=sys.stderr)
2019
+ return 2
2020
+
1183
2021
  if args.purge_cache:
1184
2022
  import shutil
1185
2023
  if CACHE_BASE.exists():
@@ -1189,6 +2027,13 @@ def main(argv=None) -> int:
1189
2027
  print("[lineage] no cache to purge", file=sys.stderr)
1190
2028
  return 0
1191
2029
 
2030
+ if args.apply_review:
2031
+ if any((args.last, args.from_, args.to, args.turns, args.session,
2032
+ args.all_sessions, args.from_transcript)):
2033
+ print("[lineage] note: --apply-review renders the pack; source and range "
2034
+ "flags were applied at --emit-review", file=sys.stderr)
2035
+ return apply_review(args)
2036
+
1192
2037
  # ---- Resolve input → raw turns (parse + parts + session) ----
1193
2038
  turns = []
1194
2039
  session_id = "session"
@@ -1249,51 +2094,95 @@ def main(argv=None) -> int:
1249
2094
  # stdin → derive per-turn uuid from content hash so cache works
1250
2095
  if args.from_transcript == "-":
1251
2096
  for t in turns:
1252
- t["uuid"] = hashlib.sha256(t["text"].encode()).hexdigest()[:16]
2097
+ t["uuid"] = hashlib.sha256(str(t["text"]).encode()).hexdigest()[:16]
1253
2098
 
1254
2099
  # ---- Pipeline order (R6): classify → merge → echo → hide-tool-only → range ----
1255
2100
  turns = classify_turns(turns, drop_trivia=args.drop_trivia)
1256
2101
  turns = merge_assistant_runs(turns)
2102
+ if args.emit_review:
2103
+ drops = heuristic_drops(turns, args)
2104
+ window = review_window(turns, drops, args)
2105
+ if window is None:
2106
+ return 2
2107
+ return emit_review(turns, drops, window, session_id,
2108
+ default_output(args, jsonl_path, session_id), args)
1257
2109
  if args.drop_trivia:
1258
2110
  turns = drop_echo_exchanges(turns)
1259
2111
  if args.hide_tool_only:
1260
2112
  turns = [t for t in turns
1261
2113
  if not (t["role"] == "assistant" and not t["text"].strip())]
1262
- if args.from_:
1263
- turns = [t for t in turns if (t["ts"] or "") >= args.from_]
1264
- if args.to:
1265
- # A bare date bound is inclusive of the WHOLE end day: a full ISO
1266
- # timestamp sorts AFTER the date string ("...T10:..." <= "2026-08-09"
1267
- # is False), so append a high sentinel for date-only input.
1268
- to_bound = args.to if "T" in args.to else args.to + "T99"
1269
- turns = [t for t in turns if (t["ts"] or "") <= to_bound]
2114
+ turns = select_range(turns, args)
2115
+ if turns is None:
2116
+ return 2
2117
+ if args.output is None:
2118
+ args.output = default_output(args, jsonl_path, session_id)
2119
+ return render_and_write(turns, session_id, args, all_sessions=args.all_sessions,
2120
+ markdown=args.markdown, open_details=args.open_details,
2121
+ title=args.title)
2122
+
2123
+
2124
+ def _in_time(t, args):
2125
+ """--from / --to on one turn. A bare date bound is inclusive of the WHOLE end
2126
+ day: a full ISO timestamp sorts AFTER the date string ("...T10:..." <=
2127
+ "2026-08-09" is False), so a high sentinel is appended for date-only input."""
2128
+ ts = t["ts"] or ""
2129
+ if args.from_ and ts < args.from_:
2130
+ return False
2131
+ if args.to and ts > (args.to if "T" in args.to else args.to + "T99"):
2132
+ return False
2133
+ return True
2134
+
2135
+
2136
+ def _count_slice(n, args):
2137
+ """--turns then --last over n items, as a slice; None on an invalid --turns."""
2138
+ lo, hi = 0, n
1270
2139
  if args.turns:
1271
2140
  m = re.match(r"^(\d+)-(\d+)$", args.turns)
1272
2141
  if not m:
1273
2142
  print(f"[lineage] invalid --turns '{args.turns}'", file=sys.stderr)
1274
- return 2
1275
- lo, hi = max(1, int(m.group(1))), int(m.group(2))
1276
- turns = turns[lo - 1:hi]
2143
+ return None
2144
+ lo, hi = max(1, int(m.group(1))) - 1, min(int(m.group(2)), n)
1277
2145
  if args.last and args.last > 0:
1278
- turns = turns[-args.last:]
2146
+ lo = max(lo, hi - args.last)
2147
+ return slice(lo, hi)
2148
+
1279
2149
 
2150
+ def select_range(turns, args):
2151
+ """--from / --to / --turns / --last, in that order; None on an invalid --turns."""
2152
+ turns = [t for t in turns if _in_time(t, args)]
2153
+ sl = _count_slice(len(turns), args)
2154
+ if sl is None:
2155
+ return None
2156
+ turns = turns[sl]
1280
2157
  if len(turns) > 100:
1281
2158
  print(f"[lineage] WARN: {len(turns)} turns is large — render may be slow",
1282
2159
  file=sys.stderr)
2160
+ return turns
1283
2161
 
1284
- # ---- Resolve default output path ----
1285
- if args.output is None:
1286
- session_name = discover_session_name(jsonl_path) if jsonl_path else None
1287
- slug = ("all-sessions" if args.all_sessions
1288
- else (session_name or session_id[:8]))
1289
- args.output = f"work/lineage-{slug}.html"
1290
- if session_name:
1291
- print(f"[lineage] session name: {session_name}", file=sys.stderr)
1292
2162
 
2163
+ def default_output(args, jsonl_path, session_id):
2164
+ """work/lineage-<session name or id>.html, or args.output when given."""
2165
+ if args.output is not None:
2166
+ return args.output
2167
+ session_name = discover_session_name(jsonl_path) if jsonl_path else None
2168
+ slug = ("all-sessions" if args.all_sessions
2169
+ else (session_name or session_id[:8]))
2170
+ if session_name:
2171
+ print(f"[lineage] session name: {session_name}", file=sys.stderr)
2172
+ return f"work/lineage-{slug}.html"
2173
+
2174
+
2175
+ def render_and_write(turns, session_id, args, all_sessions=False, markdown=True,
2176
+ open_details=False, title="Session Lineage", summaries=None,
2177
+ prior_redactions=None):
2178
+ """Render the rows, fill the template, self-verify, write, run the reviewer gate.
2179
+ `prior_redactions`: what --emit-review redacted before the text reached the pack."""
1293
2180
  rows, redact_counts, hits, total = render_rows(
1294
2181
  turns, session_id, args.redact_extra, args.redact_mode,
1295
- args.rebuild_summaries, open_details=args.open_details,
1296
- markdown=args.markdown, all_sessions=args.all_sessions)
2182
+ args.rebuild_summaries, open_details=open_details,
2183
+ markdown=markdown, all_sessions=all_sessions, summaries=summaries)
2184
+ for k, v in (prior_redactions or {}).items():
2185
+ redact_counts[k] = redact_counts.get(k, 0) + v
1297
2186
 
1298
2187
  date_range = ""
1299
2188
  dated = [t for t in turns if t.get("ts")]
@@ -1303,8 +2192,8 @@ def main(argv=None) -> int:
1303
2192
  date_range = first if first == last else f"{first} ~ {last}"
1304
2193
 
1305
2194
  html_doc = HTML_TEMPLATE
1306
- html_doc = html_doc.replace("{{TITLE}}", html.escape(args.title), 1)
1307
- html_doc = html_doc.replace("{{HEADER_TITLE}}", html.escape(args.title), 1)
2195
+ html_doc = html_doc.replace("{{TITLE}}", html.escape(title), 1)
2196
+ html_doc = html_doc.replace("{{HEADER_TITLE}}", html.escape(title), 1)
1308
2197
  html_doc = html_doc.replace("{{DATE_RANGE}}", html.escape(date_range), 1)
1309
2198
  html_doc = html_doc.replace("{{TURNS}}", "\n".join(rows), 1)
1310
2199
 
@@ -1316,7 +2205,7 @@ def main(argv=None) -> int:
1316
2205
  out.parent.mkdir(parents=True, exist_ok=True)
1317
2206
  out.write_text(html_doc, encoding="utf-8")
1318
2207
 
1319
- _run_reviewer_gate(turns, out, session_id, args)
2208
+ _run_reviewer_gate(turns, out, session_id, args, summaries=summaries)
1320
2209
 
1321
2210
  tool_total = sum(sum(t.get("tools", {}).values()) for t in turns)
1322
2211
  print(f"[lineage] turns={len(turns)} redacted={sum(redact_counts.values())} "
@@ -1328,6 +2217,16 @@ def main(argv=None) -> int:
1328
2217
  return 0
1329
2218
 
1330
2219
 
2220
+ def _with_pack_stamp(output, pack_output):
2221
+ """`output` with the `_YYMMDD+HHMM` stamp of the pack's own output, so the gate's runs
2222
+ minutes apart write one page; as given when it has a stamp or the pack's output has none."""
2223
+ stamp = re.search(r"_\d{6}\+\d{4}$", pathlib.Path(str(pack_output or "")).stem)
2224
+ out = pathlib.Path(output)
2225
+ if stamp and not re.search(r"_\d{6}\+\d{4}$", out.stem):
2226
+ return str(out.with_name(f"{out.stem}{stamp.group(0)}{out.suffix}"))
2227
+ return output
2228
+
2229
+
1331
2230
  def _with_timestamp_suffix(path_str):
1332
2231
  """Append `_YYMMDD+HHMM` before the extension (idempotent)."""
1333
2232
  import datetime as _dt
@@ -1338,70 +2237,273 @@ def _with_timestamp_suffix(path_str):
1338
2237
  return p.with_name(f"{p.stem}{stamp}{p.suffix}")
1339
2238
 
1340
2239
 
1341
- def _run_reviewer_gate(turns, out, session_id, args):
1342
- """Write reviewer samples and (if --reviewer-output) enforce the gate.
1343
- Uses the new summarizer; the input/output contract is unchanged (N3)."""
1344
- bots = [t for t in turns if t["role"] in ("assistant", "agent")]
1345
- if not args.skip_reviewer and bots:
1346
- import random
1347
- sample = random.sample(bots, min(5, len(bots)))
1348
- review_samples = []
1349
- for i, t in enumerate(sample):
1350
- s, _ = read_or_summarize(t, session_id, rebuild=False,
1351
- redact_extra=args.redact_extra,
1352
- redact_mode=args.redact_mode)
2240
+ def _gate_samples(bots, summaries, session_id, args):
2241
+ """Up to 5 bot turns as the gate's critic reads them. --rulebase keeps the 2.x shape
2242
+ (random turns, page redaction). With reviewer summaries (the default flow) the pick is
2243
+ seeded by the turns, so a rerun on the same turns samples the same ones, and a sample
2244
+ names its turn and shows the head and tail the reviewer saw, fully redacted."""
2245
+ import random
2246
+ if summaries is None:
2247
+ out = []
2248
+ for i, t in enumerate(random.sample(bots, min(5, len(bots)))):
2249
+ s, _ = _page_summary(t, summaries, session_id, False,
2250
+ args.redact_extra, args.redact_mode)
1353
2251
  red_s, _ = redact(s, extra=args.redact_extra, mode=args.redact_mode)
1354
- red_d, _ = redact(t["text"][:500], extra=args.redact_extra,
1355
- mode=args.redact_mode)
1356
- review_samples.append({"idx": i, "original_detail": red_d,
1357
- "generated_summary": red_s})
1358
- review_path = out.parent / f".{out.stem}.reviewer-input.json"
2252
+ red_d, _ = redact(t["text"][:500], extra=args.redact_extra, mode=args.redact_mode)
2253
+ out.append({"idx": i, "original_detail": red_d, "generated_summary": red_s})
2254
+ return out
2255
+ seed = hashlib.sha256("\x00".join(t["uuid"] for t in bots).encode()).hexdigest()
2256
+ out = []
2257
+ for i, t in enumerate(random.Random(seed).sample(bots, min(5, len(bots)))):
2258
+ s = t.get("sample_summary") or _page_summary(t, summaries, session_id, False,
2259
+ args.redact_extra, args.redact_mode)[0]
2260
+ detail = t.get("preview") or _preview(review_redact(t["text"], args.redact_extra)[0])[0]
2261
+ gen = review_redact(_MASKED.sub("[REDACTED]", s), args.redact_extra)[0]
2262
+ key = hashlib.sha256("\x00".join([t["uuid"], detail, gen]).encode()).hexdigest()[:12]
2263
+ out.append({"idx": i, "id": t["uuid"], "key": key, "original_detail": detail,
2264
+ "generated_summary": gen})
2265
+ return out
2266
+
2267
+
2268
+ # What --redact-mode mask leaves of a secret (4 characters, ****, 4 characters): the page
2269
+ # keeps it, a model reading the gate samples gets none of it.
2270
+ _MASKED = re.compile(r"[^\s*]{4}\*{4}[^\s*]{4}")
2271
+
2272
+
2273
+ def _renew_samples(path, text, rop):
2274
+ """Write the reviewed flow's samples unless they are there as they are. New samples set
2275
+ aside a verdict list already at `rop` (_refuse_non_verdict has stopped on anything
2276
+ else): no critic judged them in it (a --rulebase gate or an earlier sample set left
2277
+ it). The same samples keep a verdict: a critic answered an earlier run, and this run
2278
+ reads that answer."""
2279
+ try:
2280
+ if path.read_text(encoding="utf-8") == text:
2281
+ return
2282
+ except (OSError, UnicodeDecodeError):
2283
+ pass
2284
+ _write_private(path, text)
2285
+ old = pathlib.Path(rop) if rop else None
2286
+ if old is not None and _is_verdict_file(old):
2287
+ print(f"[lineage] WARN: {rop} predates these samples; it is not read",
2288
+ file=sys.stderr)
2289
+ _set_aside(old)
2290
+
2291
+
2292
+ def _refuse_non_verdict(rop):
2293
+ """Stop (exit 2) before any sample or instruction when the reviewed flow's verdict path
2294
+ holds something other than a verdict list: whoever put it there, nothing printed here
2295
+ may lead the session to write over it."""
2296
+ p = pathlib.Path(rop) if rop else None
2297
+ if p is None or not p.exists() or _is_verdict_file(p):
2298
+ return
2299
+ if _json_list(p): # step 8: written again, the same answer fails the same way
2300
+ why = ("it is a JSON array where no entry names idx; if the session wrote the critic's "
2301
+ "answer, have the critic judge the samples again (a failed gate run)")
2302
+ else:
2303
+ why = ("if the session wrote it, write it again as a bare JSON array of "
2304
+ "{idx, id, key, recoverable, reason}")
2305
+ print(f"[lineage] ERROR: {rop} is not a verdict list; {why}, else give a --reviewer-output "
2306
+ "that does not exist yet", file=sys.stderr)
2307
+ raise SystemExit(2)
2308
+
2309
+
2310
+ def _json_list(p):
2311
+ """True when `p` is a file holding a JSON array, whatever its entries."""
2312
+ try:
2313
+ return p.is_file() and isinstance(json.loads(p.read_text(encoding="utf-8")), list)
2314
+ except (OSError, ValueError, UnicodeDecodeError, RecursionError):
2315
+ return False
2316
+
2317
+
2318
+ def _is_verdict_file(p, every=False):
2319
+ """True when `p` is a file holding a critic's verdict: a JSON list with at least one
2320
+ object that names an `idx`. A list of anything else (a user's data file) is not one; a
2321
+ verdict with an entry short of its idx is one, and the coverage check says what it
2322
+ misses. With `every`, each entry must be such an object: only that file may be written
2323
+ over, so a user's list holding one such object among other data stays."""
2324
+ if not p.is_file():
2325
+ return False
2326
+ try:
2327
+ got = json.loads(p.read_text(encoding="utf-8"))
2328
+ except (OSError, ValueError, UnicodeDecodeError, RecursionError):
2329
+ return False
2330
+ if not isinstance(got, list):
2331
+ return False
2332
+ named = [isinstance(v, dict) and "idx" in v for v in got]
2333
+ return bool(named) and all(named) if every else any(named)
2334
+
2335
+
2336
+ def _write_gate_samples(bots, out, summaries, session_id, args):
2337
+ """Write the samples for the critic and say what to do with them; their path, or None.
2338
+ The reviewed flow writes them beside the pack, under one name across its reruns, and
2339
+ stops (exit 2) when it cannot: no critic could judge them."""
2340
+ if summaries is not None:
2341
+ _refuse_non_verdict(args.reviewer_output)
2342
+ path = out.parent / f".{out.stem}.reviewer-input.json"
2343
+ if summaries is not None and getattr(args, "apply_review", None):
2344
+ pack = pathlib.Path(args.apply_review)
2345
+ path = pack.with_name(f"{pack.stem}.reviewer-input.json")
2346
+ text = json.dumps(_gate_samples(bots, summaries, session_id, args),
2347
+ ensure_ascii=False, indent=2)
2348
+ try:
2349
+ if summaries is None:
2350
+ _write_private(path, text)
2351
+ else:
2352
+ _renew_samples(path, text, args.reviewer_output)
2353
+ except OSError as e:
2354
+ if summaries is not None:
2355
+ print(f"[lineage] ERROR: cannot write the gate samples {path}: {e}", file=sys.stderr)
2356
+ raise SystemExit(2)
2357
+ return None
2358
+ print(f"[lineage] reviewer samples: {path}", file=sys.stderr, flush=True)
2359
+ if summaries is None:
2360
+ print("[lineage] next: invoke Skill('oh-my-claudecode:critic') with "
2361
+ "the JSON above; expected [{idx, recoverable, reason}, ...] "
2362
+ "(PASS = 5/5 recoverable)", file=sys.stderr, flush=True)
2363
+ elif pathlib.Path(args.reviewer_output).exists():
2364
+ print(f"[lineage] reading the verdict at {args.reviewer_output}", file=sys.stderr, flush=True)
2365
+ else:
2366
+ print(f"[lineage] next: have a critic agent judge the JSON above and write its "
2367
+ f"[{{idx, id, key, recoverable, reason}}, ...] to {args.reviewer_output} "
2368
+ "(one per sample; PASS = all recoverable)", file=sys.stderr, flush=True)
2369
+ return path
2370
+
2371
+
2372
+ def _read_verdict(rop, timeout, set_aside):
2373
+ """The critic's verdict list from `rop`, waited for up to `timeout` seconds (None: not
2374
+ waited for, the next run reads it). With `set_aside` a verdict list is renamed once read,
2375
+ so a rerun waits for a fresh one instead of reusing it; anything else stays where it is.
2376
+ Exits 2 when it is missing, unreadable or not a non-empty list."""
2377
+ import time as _time
2378
+ deadline = _time.time() + (0 if timeout is None else max(1, timeout))
2379
+ while not rop.exists() and _time.time() < deadline:
2380
+ _time.sleep(1)
2381
+ if not rop.exists():
2382
+ if timeout is None:
2383
+ print(f"[lineage] ERROR: no verdict at {rop} yet; have a critic judge the samples, "
2384
+ "write its answer there and run this again", file=sys.stderr)
2385
+ else:
2386
+ print(f"[lineage] ERROR: reviewer-output not found within "
2387
+ f"{timeout}s: {rop}", file=sys.stderr)
2388
+ raise SystemExit(2)
2389
+ try:
2390
+ verdict = json.loads(rop.read_text(encoding="utf-8"))
2391
+ except (json.JSONDecodeError, OSError, UnicodeDecodeError, RecursionError) as e:
2392
+ print(f"[lineage] ERROR: reviewer-output parse failed: {e}",
2393
+ file=sys.stderr)
2394
+ raise SystemExit(2)
2395
+ if not isinstance(verdict, list) or not verdict:
2396
+ print("[lineage] ERROR: reviewer-output is not a non-empty array",
2397
+ file=sys.stderr)
2398
+ raise SystemExit(2)
2399
+ if set_aside:
2400
+ _set_aside(rop)
2401
+ return verdict
2402
+
2403
+
2404
+ def _set_aside(rop):
2405
+ """Rename the verdict file to `<name>.used` and say so; a folder or other non-file at
2406
+ that path is left alone. Something other than an earlier verdict at `<name>.used` is
2407
+ kept too: the verdict then goes to the first `<name>.used.N` that is free or holds an
2408
+ earlier verdict."""
2409
+ if not rop.is_file():
2410
+ print(f"[lineage] WARN: {rop} is not a file; left in place", file=sys.stderr)
2411
+ return
2412
+ dest = rop.with_name(rop.name + ".used")
2413
+ k = 0
2414
+ while dest.exists() and not _is_verdict_file(dest, every=True):
2415
+ k += 1
2416
+ dest = rop.with_name(f"{rop.name}.used.{k}")
2417
+ try:
2418
+ os.replace(str(rop), str(dest))
2419
+ except OSError as e:
2420
+ print(f"[lineage] WARN: could not set the verdict aside ({e}); remove {rop} "
2421
+ "before the next run", file=sys.stderr)
2422
+ return
2423
+ print(f"[lineage] note: verdict moved to {dest}", file=sys.stderr)
2424
+
2425
+
2426
+ def _verdict_gaps(verdict, samples):
2427
+ """Why a verdict does not answer these samples, or "": it needs one entry per sample
2428
+ idx and no other, each carrying the `id` and `key` of that sample (its turn and its
2429
+ content: an answer to the samples from before a fix carries an old key)."""
2430
+ want = {s["idx"]: s for s in samples if isinstance(s, dict) and type(s.get("idx")) is int}
2431
+ seen, stray = set(), []
2432
+ for v in verdict:
2433
+ idx = v.get("idx") if isinstance(v, dict) else None
2434
+ if (type(idx) is not int or idx not in want or idx in seen
2435
+ or any(v.get(k) != want[idx][k] for k in ("id", "key") if k in want[idx])):
2436
+ stray.append(idx)
2437
+ else:
2438
+ seen.add(idx)
2439
+ missing = sorted(set(want) - seen)
2440
+ if not (missing or stray):
2441
+ return ""
2442
+ return f"missing idx {missing}, entries that match no sample {stray}"
2443
+
2444
+
2445
+ def _check_coverage(verdict, samples):
2446
+ """Exit 2 unless the verdict answers each of the samples in file `samples`."""
2447
+ try:
2448
+ gaps = _verdict_gaps(verdict, json.loads(samples.read_text(encoding="utf-8")))
2449
+ except (OSError, ValueError, UnicodeDecodeError, AttributeError) as e:
2450
+ gaps = f"cannot read the samples: {e}"
2451
+ if gaps:
2452
+ print(f"[lineage] ERROR: the verdict does not answer {samples} ({gaps}); have the "
2453
+ "critic judge every sample again", file=sys.stderr)
2454
+ for v in verdict:
2455
+ if isinstance(v, dict) and v.get("recoverable") is not True:
2456
+ print(f" - not recoverable in it: idx={v.get('idx', '?')}: "
2457
+ f"{v.get('reason', '(no reason)')}", file=sys.stderr)
2458
+ raise SystemExit(2)
2459
+
2460
+
2461
+ def _enforce_gate(rop, timeout, samples=None, reviewed=False):
2462
+ """PASS when every verdict entry is recoverable, else exit 2 with the reasons. In the
2463
+ reviewed flow the verdict is set aside once read, it must answer each sample, and the
2464
+ samples go after a PASS."""
2465
+ verdict = _read_verdict(rop, timeout, set_aside=reviewed)
2466
+ if reviewed and samples is not None:
2467
+ _check_coverage(verdict, samples)
2468
+ fails = [v for v in verdict
2469
+ if not (isinstance(v, dict) and v.get("recoverable") is True)]
2470
+ if fails:
2471
+ print(f"[lineage] FAIL: quality gate {len(verdict) - len(fails)}/"
2472
+ f"{len(verdict)} recoverable. Reasons:", file=sys.stderr)
2473
+ for f in fails:
2474
+ if isinstance(f, dict):
2475
+ print(f" - idx={f.get('idx', '?')}: "
2476
+ f"{f.get('reason', '(no reason)')}", file=sys.stderr)
2477
+ raise SystemExit(2)
2478
+ print(f"[lineage] PASS: quality gate {len(verdict)}/{len(verdict)} "
2479
+ "recoverable", file=sys.stderr)
2480
+ if reviewed and samples is not None:
1359
2481
  try:
1360
- review_path.write_text(
1361
- json.dumps(review_samples, ensure_ascii=False, indent=2),
1362
- encoding="utf-8")
1363
- print(f"[lineage] reviewer samples: {review_path}", file=sys.stderr)
1364
- print("[lineage] next: invoke Skill('oh-my-claudecode:critic') with "
1365
- "the JSON above; expected [{idx, recoverable, reason}, ...] "
1366
- "(PASS = 5/5 recoverable)", file=sys.stderr)
2482
+ samples.unlink()
1367
2483
  except OSError:
1368
2484
  pass
1369
- elif args.skip_reviewer:
2485
+
2486
+
2487
+ def _run_reviewer_gate(turns, out, session_id, args, summaries=None):
2488
+ """Write reviewer samples and (if --reviewer-output) enforce the gate. Samples the
2489
+ summaries the page shows (`summaries` overrides, as in render_rows); the input/output
2490
+ contract is unchanged (N3). The reviewed flow (`summaries` given) already had a model
2491
+ review every turn, so it samples only for a gate it enforces."""
2492
+ bots = [t for t in turns if t["role"] in ("assistant", "agent")]
2493
+ reviewed = summaries is not None
2494
+ if args.skip_reviewer:
1370
2495
  print("[lineage] WARN: --skip-reviewer — quality gate not enforced",
1371
2496
  file=sys.stderr)
1372
-
1373
- if (not args.skip_reviewer) and bots and args.reviewer_output:
1374
- import time as _time
1375
- rop = pathlib.Path(args.reviewer_output)
1376
- deadline = _time.time() + max(1, args.reviewer_timeout)
1377
- while not rop.exists() and _time.time() < deadline:
1378
- _time.sleep(1)
1379
- if not rop.exists():
1380
- print(f"[lineage] ERROR: reviewer-output not found within "
1381
- f"{args.reviewer_timeout}s: {rop}", file=sys.stderr)
1382
- raise SystemExit(2)
1383
- try:
1384
- verdict = json.loads(rop.read_text(encoding="utf-8"))
1385
- except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e:
1386
- print(f"[lineage] ERROR: reviewer-output parse failed: {e}",
1387
- file=sys.stderr)
1388
- raise SystemExit(2)
1389
- if not isinstance(verdict, list) or not verdict:
1390
- print("[lineage] ERROR: reviewer-output is not a non-empty array",
1391
- file=sys.stderr)
1392
- raise SystemExit(2)
1393
- fails = [v for v in verdict
1394
- if not (isinstance(v, dict) and v.get("recoverable") is True)]
1395
- if fails:
1396
- print(f"[lineage] FAIL: quality gate {len(verdict) - len(fails)}/"
1397
- f"{len(verdict)} recoverable. Reasons:", file=sys.stderr)
1398
- for f in fails:
1399
- if isinstance(f, dict):
1400
- print(f" - idx={f.get('idx', '?')}: "
1401
- f"{f.get('reason', '(no reason)')}", file=sys.stderr)
1402
- raise SystemExit(2)
1403
- print(f"[lineage] PASS: quality gate {len(verdict)}/{len(verdict)} "
1404
- "recoverable", file=sys.stderr)
2497
+ return
2498
+ if not bots or (reviewed and not args.reviewer_output):
2499
+ return
2500
+ samples = _write_gate_samples(bots, out, summaries, session_id, args)
2501
+ if args.reviewer_output:
2502
+ if reviewed and args.reviewer_timeout is None:
2503
+ timeout = None # the run after the critic reads the verdict; this one does not wait
2504
+ else:
2505
+ timeout = REVIEWER_TIMEOUT if args.reviewer_timeout is None else args.reviewer_timeout
2506
+ _enforce_gate(pathlib.Path(args.reviewer_output), timeout, samples, reviewed)
1405
2507
 
1406
2508
 
1407
2509
  if __name__ == "__main__":