@kaydash9999/banker-plugins 0.20.5 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/CHANGELOG.md +52 -0
  4. package/PRIVACY.md +25 -1
  5. package/README.md +63 -18
  6. package/bin/banker.js +173 -14
  7. package/bin/lib/codex-hooks.mjs +254 -0
  8. package/bin/lib/codex-routes.mjs +112 -0
  9. package/bin/lib/command-skill.cjs +19 -0
  10. package/bin/lib/telemetry-config.mjs +4 -1
  11. package/codex/command-skills/front-qa/SKILL.md +56 -0
  12. package/codex/command-skills/setup/SKILL.md +87 -0
  13. package/codex/hooks.json +29 -0
  14. package/codex/manifest.json +13 -10
  15. package/codex/skills/graceful-pause/SKILL.md +30 -0
  16. package/codex/skills/progress/SKILL.md +42 -0
  17. package/codex/skills/progress/scripts/progress-plan.mjs +722 -0
  18. package/codex/skills/progress/scripts/progress.mjs +295 -0
  19. package/codex/transform-matrix.md +18 -13
  20. package/commands/setup.md +2 -2
  21. package/hooks/obsidize-hook.mjs +116 -4
  22. package/hooks/progress-plan.mjs +1 -1
  23. package/hooks/telemetry-count-codex.mjs +309 -0
  24. package/hooks/update-notify.mjs +105 -14
  25. package/package.json +1 -1
  26. package/skills/3d-intro-build/references/verify-intro.mjs +15 -1
  27. package/skills/anti-drm/SKILL.md +1 -1
  28. package/skills/anti-drm/scripts/instruction-toggle.mjs +57 -4
  29. package/skills/append_wiki/SKILL.md +34 -8
  30. package/skills/arch-diagram/build_pptx_template.py +47 -28
  31. package/skills/audit-web-page/SKILL.md +1 -1
  32. package/skills/audit-web-page/harness.py +0 -1
  33. package/skills/compact-wiki/SKILL.md +37 -6
  34. package/skills/deep-research/SKILL.md +6 -0
  35. package/skills/lfm/SKILL.md +148 -0
  36. package/skills/lineage/SKILL.md +14 -4
  37. package/skills/lineage/lineage.py +1582 -516
  38. package/skills/motion-graphic-make/SKILL.md +1 -1
  39. package/skills/obsidizer/SKILL.md +44 -10
  40. package/skills/obsidizer/obsidize.mjs +42 -11
  41. package/skills/omc-patch/SKILL.md +6 -0
  42. package/skills/payload-mon/SKILL.md +3 -1
  43. package/skills/play-qa/SKILL.md +1 -1
  44. package/skills/ralph-qa/SKILL.md +1 -1
  45. package/skills/remains/SKILL.md +1 -1
  46. package/skills/rfp-author/SKILL.md +5 -3
  47. package/skills/rfp-author/references/vendor-rfp-structure.md +1 -1
  48. package/skills/rfp-author/scripts/md2docx.py +63 -34
  49. package/skills/setup-bypass-permissions/SKILL.md +28 -10
  50. package/skills/setup-bypass-permissions/scripts/codex-bypass.mjs +135 -0
  51. package/skills/setup-bypass-permissions/scripts/codex-config.mjs +210 -0
  52. package/skills/setup-bypass-permissions/scripts/fallback/bypass-permissions.py +66 -15
  53. package/skills/setup-omc-hud/SKILL.md +33 -3
  54. package/skills/setup-omc-hud/assets/codex-config.mjs +210 -0
  55. package/skills/setup-omc-hud/assets/codex-status-line.mjs +90 -0
  56. package/skills/smart-compact/SKILL.md +59 -13
  57. package/skills/smart-compact/scripts/codex-gate.mjs +385 -0
  58. package/skills/theme-selection/SKILL.md +1 -1
  59. package/skills/tone-compact/SKILL.md +1 -1
  60. package/skills/tone-compact/scripts/instruction-toggle.mjs +57 -4
  61. package/skills/ultra-chain/SKILL.md +10 -6
  62. package/skills/ultra-interview/SKILL.md +13 -2
  63. package/skills/ultra-ui-qa/SKILL.md +1 -1
  64. package/skills/update-banker/SKILL.md +31 -6
  65. package/skills/vertical-pptx/SKILL.md +1 -1
  66. package/skills/vertical-pptx/build_a4p.js +20 -2
  67. package/skills/vertical-pptx/build_a4p.py +560 -359
@@ -9,6 +9,7 @@ self-verifies the output markup. See SKILL.md for the full design and the
9
9
  Runtime-agnostic: works from any transcript via --session / --from-transcript.
10
10
  Auto-discovery assumes Claude Code's ~/.claude/projects/<encoded-cwd>/ layout.
11
11
  """
12
+
12
13
  import argparse
13
14
  import bisect
14
15
  import hashlib
@@ -29,43 +30,59 @@ from html.parser import HTMLParser
29
30
  from typing import Any
30
31
 
31
32
  if sys.version_info < (3, 7):
32
- sys.stderr.write("lineage requires Python 3.7+ (found %s)\n"
33
- % sys.version.split()[0])
33
+ sys.stderr.write(
34
+ "lineage requires Python 3.7+ (found %s)\n" % sys.version.split()[0]
35
+ )
34
36
  sys.exit(2)
35
37
 
36
- SCHEMA_VERSION = 2 # cache dir schema (v2: redacted summaries only)
37
- SUMMARIZER_VERSION = 4 # bump when summarize_turn or the page redaction changes (B-3 cache key)
38
- # 3: cut from redacted text (3.0.2); 4: split secrets joined first (3.0.8)
39
- USER_FOLD = 400 # collapse user messages longer than this (E-1)
40
- ECHO_ASK = 40 # echo-exchange: user question length ceiling (A-4)
41
- ECHO_REPLY = 5 # echo-exchange: assistant reply length ceiling (A-4)
38
+ SCHEMA_VERSION = 2 # cache dir schema (v2: redacted summaries only)
39
+ SUMMARIZER_VERSION = (
40
+ 4 # bump when summarize_turn or the page redaction changes (B-3 cache key)
41
+ )
42
+ # 3: cut from redacted text (3.0.2); 4: split secrets joined first (3.0.8)
43
+ USER_FOLD = 400 # collapse user messages longer than this (E-1)
44
+ ECHO_ASK = 40 # echo-exchange: user question length ceiling (A-4)
45
+ ECHO_REPLY = 5 # echo-exchange: assistant reply length ceiling (A-4)
42
46
  _BLOCKQUOTE_MAX_DEPTH = 32 # blockquote recursion cap (C-2)
43
47
  CACHE_BASE = pathlib.Path.home() / ".cache" / "lineage"
44
- REVIEW_SCHEMA = "lineage-review/2" # --emit-review pack, read back by --apply-review; 2: llm_key, pages redacted (3.0.2)
48
+ REVIEW_SCHEMA = "lineage-review/2" # --emit-review pack, read back by --apply-review; 2: llm_key, pages redacted (3.0.2)
45
49
  REVIEW_PART_SCHEMA = "lineage-review-part/1"
46
- REVIEW_PART = 40 # turns per part file: one reviewer's share
47
- PREVIEW_HEAD = 1500 # a long turn's preview: head + tail, where conclusions sit
50
+ REVIEW_PART = 40 # turns per part file: one reviewer's share
51
+ PREVIEW_HEAD = 1500 # a long turn's preview: head + tail, where conclusions sit
48
52
  PREVIEW_TAIL = 700
49
- SUMMARY_MAX = 120 # a reviewer's summary is cut to this, after redaction
50
- LLM_CACHE_VERSION = 1 # bump to drop every cached reviewer decision
51
- REVIEWER_TIMEOUT = 60 # --reviewer-timeout default, seconds
52
- DECISIONS_MAX = 1_000_000 # bytes in one decisions file (a 40-turn part answers in some 20 KB)
53
- DECISIONS_TRIES = 200 # places a wrapped answer's list may start, tried at most
53
+ SUMMARY_MAX = 120 # a reviewer's summary is cut to this, after redaction
54
+ LLM_CACHE_VERSION = 1 # bump to drop every cached reviewer decision
55
+ REVIEWER_TIMEOUT = 60 # --reviewer-timeout default, seconds
56
+ DECISIONS_MAX = (
57
+ 1_000_000 # bytes in one decisions file (a 40-turn part answers in some 20 KB)
58
+ )
59
+ DECISIONS_TRIES = 200 # places a wrapped answer's list may start, tried at most
54
60
 
55
61
  # ---------------------------------------------------------------- Noise / classify
56
62
  # NOTE: JSONL records carry RAW `<...>` tags. Every regex below uses raw `<`/`>`
57
63
  # (NOT the HTML-escaped `&lt;` seen in the display-only change-request document).
58
- _NOISE_TAGS = ("system-reminder", "local-command-caveat", "local-command-stdout",
59
- "local-command-stderr", "teammate-message", "command-message",
60
- "command-args", "command-name", "task-notification")
64
+ _NOISE_TAGS = (
65
+ "system-reminder",
66
+ "local-command-caveat",
67
+ "local-command-stdout",
68
+ "local-command-stderr",
69
+ "teammate-message",
70
+ "command-message",
71
+ "command-args",
72
+ "command-name",
73
+ "task-notification",
74
+ )
61
75
  _NOISE_BLOCK_RE = re.compile(r"<(%s)\b[^>]*>.*?</\1>" % "|".join(_NOISE_TAGS), re.S)
62
76
  _NOISE_CLOSE_RE = re.compile(r"</(%s)>" % "|".join(_NOISE_TAGS))
63
- _NOISE_OPEN_RE = re.compile(r"<(?:%s)\b" % "|".join(_NOISE_TAGS)) # leading orphan open (R7)
77
+ _NOISE_OPEN_RE = re.compile(
78
+ r"<(?:%s)\b" % "|".join(_NOISE_TAGS)
79
+ ) # leading orphan open (R7)
64
80
 
65
81
  _SKILL_BODY_RE = re.compile(r"^\s*Base directory for this skill:")
66
82
  _WORKFLOW_BODY_RE = re.compile(r'^\s*Run the "[^"\n]+" workflow\.')
67
83
  _COMPACTION_RE = re.compile(
68
- r"^\s*This session is being continued from a previous conversation")
84
+ r"^\s*This session is being continued from a previous conversation"
85
+ )
69
86
  _HOOK_FEEDBACK_RE = re.compile(r"^\s*Stop hook feedback:")
70
87
  _INTERRUPT_RE = re.compile(r"^\s*\[Request interrupted")
71
88
  _IMAGE_NOTE_RE = re.compile(r"^\s*\[Image:\s*original\s+\d+x\d+")
@@ -74,18 +91,48 @@ _SYS_ERROR_RE = re.compile(r"^\s*(?:Login expired|Please run /login|API Error)")
74
91
  # Manipulation commands: judged by NAME, not by "has no args" (A-4). Plugin
75
92
  # commands (name contains ':') are never manipulation — always preserved.
76
93
  _HARNESS_CMDS = {
77
- "agents", "bug", "clear", "compact", "config", "context", "copy", "cost",
78
- "doctor", "effort", "exit", "export", "help", "hooks", "ide",
79
- "install-github-app", "login", "logout", "mcp", "memory",
80
- "migrate-installer", "model", "output-style", "permissions",
81
- "privacy-settings", "quit", "release-notes", "resume", "status",
82
- "statusline", "terminal-setup", "todos", "upgrade", "usage", "vim",
94
+ "agents",
95
+ "bug",
96
+ "clear",
97
+ "compact",
98
+ "config",
99
+ "context",
100
+ "copy",
101
+ "cost",
102
+ "doctor",
103
+ "effort",
104
+ "exit",
105
+ "export",
106
+ "help",
107
+ "hooks",
108
+ "ide",
109
+ "install-github-app",
110
+ "login",
111
+ "logout",
112
+ "mcp",
113
+ "memory",
114
+ "migrate-installer",
115
+ "model",
116
+ "output-style",
117
+ "permissions",
118
+ "privacy-settings",
119
+ "quit",
120
+ "release-notes",
121
+ "resume",
122
+ "status",
123
+ "statusline",
124
+ "terminal-setup",
125
+ "todos",
126
+ "upgrade",
127
+ "usage",
128
+ "vim",
83
129
  }
84
130
  # Deliberately NOT in the set: /add-dir (changes readable dirs), /init (writes
85
131
  # CLAUDE.md) — these do real work and are kept.
86
132
 
87
133
  _AGENT_WRAP_RE = re.compile(
88
- r"<(agent-message|teammate-message)\b([^>]*)>(.*?)</\1>", re.S)
134
+ r"<(agent-message|teammate-message)\b([^>]*)>(.*?)</\1>", re.S
135
+ )
89
136
  _AGENT_FROM_RE = re.compile(r'(?:from|teammate_id)="([^"]+)"')
90
137
  _PEER_LEAD_RE = re.compile(r"^\s*Another Claude session sent a message:\s*")
91
138
  _BARE_CMD_RE = re.compile(r"^/([\w-]+)\s*$")
@@ -105,7 +152,7 @@ def clean_user_text(text, drop_harness=True):
105
152
  """
106
153
  out = _NOISE_CLOSE_RE.sub("", _NOISE_BLOCK_RE.sub("", text)).strip()
107
154
 
108
- mb = _BARE_CMD_RE.match(out) # bare `/name` with no wrapper
155
+ mb = _BARE_CMD_RE.match(out) # bare `/name` with no wrapper
109
156
  if mb and drop_harness and mb.group(1) in _HARNESS_CMDS:
110
157
  return ""
111
158
 
@@ -140,7 +187,7 @@ def split_agent_message(text):
140
187
  return None, None
141
188
  who = _AGENT_FROM_RE.search(m.group(2) or "")
142
189
  body = m.group(3).strip()
143
- if body.startswith('{"type"'): # idle/status notification
190
+ if body.startswith('{"type"'): # idle/status notification
144
191
  return (who.group(1) if who else "agent"), ""
145
192
  return (who.group(1) if who else "agent"), body
146
193
 
@@ -152,10 +199,18 @@ SECRET_PATTERNS = [
152
199
  ("GitHubPAT", re.compile(r"gh[posru]_[A-Za-z0-9]{36}")),
153
200
  ("Slack", re.compile(r"xox[bpars]-[A-Za-z0-9-]{10,}")),
154
201
  ("JWT", re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+")),
155
- ("PrivateKey", re.compile(
156
- r"-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]+?-----END [A-Z ]+PRIVATE KEY-----")),
157
- ("Password", re.compile(
158
- r"(?i)(?:password|암호|비번|패스워드)\s*[:=]\s*['\"]([^'\"\n]{4,})['\"]")),
202
+ (
203
+ "PrivateKey",
204
+ re.compile(
205
+ r"-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]+?-----END [A-Z ]+PRIVATE KEY-----"
206
+ ),
207
+ ),
208
+ (
209
+ "Password",
210
+ re.compile(
211
+ r"(?i)(?:password|암호|비번|패스워드)\s*[:=]\s*['\"]([^'\"\n]{4,})['\"]"
212
+ ),
213
+ ),
159
214
  ]
160
215
  ENTROPY_LONG = re.compile(r"[A-Za-z0-9+/=]{32,}")
161
216
 
@@ -194,7 +249,7 @@ class _Trail:
194
249
  if r.startswith("[REDACTED"):
195
250
  out += new * len(r)
196
251
  elif r:
197
- out += pos[s:s + 4] + new * (len(r) - 8) + pos[e - 4:e]
252
+ out += pos[s : s + 4] + new * (len(r) - 8) + pos[e - 4 : e]
198
253
  at = e
199
254
  self.pos = out + pos[at:]
200
255
 
@@ -207,6 +262,7 @@ class _Trail:
207
262
  if r != m.group(0):
208
263
  edits.append((m.start(), m.end(), r))
209
264
  return r
265
+
210
266
  out = pat.sub(noted, text)
211
267
  self.carry(edits)
212
268
  return out
@@ -234,7 +290,9 @@ def _sub_for(trail):
234
290
  def _shows_more(trail, shown):
235
291
  """Whether the text `trail` follows shows a character of the text it started as that `shown` (a _Trail.shown of
236
292
  another text from the same start) has hidden."""
237
- return (int.from_bytes(trail.shown(), "little") & ~int.from_bytes(shown, "little")) != 0
293
+ return (
294
+ int.from_bytes(trail.shown(), "little") & ~int.from_bytes(shown, "little")
295
+ ) != 0
238
296
 
239
297
 
240
298
  # Characters that cut a value for the patterns while no person reads them as a separator: Unicode's default-ignorable
@@ -244,14 +302,46 @@ def _shows_more(trail, shown):
244
302
  # (a lone surrogate became U+FFFD). A value cut by them is put back together where a pattern finds it once they
245
303
  # are gone (3.0.8).
246
304
  _SPLITTER_RANGES = (
247
- (0x0000, 0x0008), (0x000E, 0x001B), (0x007F, 0x0084), (0x0086, 0x009F), (0x00A0, 0x00A0), (0x00AD, 0x00AD),
248
- (0x034F, 0x034F), (0x0600, 0x0605), (0x061C, 0x061C), (0x06DD, 0x06DD), (0x070F, 0x070F), (0x0890, 0x0891),
249
- (0x08E2, 0x08E2), (0x115F, 0x1160), (0x1680, 0x1680), (0x17B4, 0x17B5), (0x180B, 0x180F), (0x2000, 0x200F),
250
- (0x202A, 0x202F), (0x205F, 0x206F), (0x2800, 0x2800), (0x3000, 0x3000), (0x3164, 0x3164), (0xE000, 0xF8FF),
251
- (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), (0xFFA0, 0xFFA0), (0xFFF0, 0xFFFD), (0x110BD, 0x110BD), (0x110CD, 0x110CD),
252
- (0x13430, 0x1343F), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), (0xF0000, 0xFFFFD),
253
- (0x100000, 0x10FFFD))
254
- _SPLITTERS = "".join(chr(a) if a == b else chr(a) + "-" + chr(b) for a, b in _SPLITTER_RANGES)
305
+ (0x0000, 0x0008),
306
+ (0x000E, 0x001B),
307
+ (0x007F, 0x0084),
308
+ (0x0086, 0x009F),
309
+ (0x00A0, 0x00A0),
310
+ (0x00AD, 0x00AD),
311
+ (0x034F, 0x034F),
312
+ (0x0600, 0x0605),
313
+ (0x061C, 0x061C),
314
+ (0x06DD, 0x06DD),
315
+ (0x070F, 0x070F),
316
+ (0x0890, 0x0891),
317
+ (0x08E2, 0x08E2),
318
+ (0x115F, 0x1160),
319
+ (0x1680, 0x1680),
320
+ (0x17B4, 0x17B5),
321
+ (0x180B, 0x180F),
322
+ (0x2000, 0x200F),
323
+ (0x202A, 0x202F),
324
+ (0x205F, 0x206F),
325
+ (0x2800, 0x2800),
326
+ (0x3000, 0x3000),
327
+ (0x3164, 0x3164),
328
+ (0xE000, 0xF8FF),
329
+ (0xFE00, 0xFE0F),
330
+ (0xFEFF, 0xFEFF),
331
+ (0xFFA0, 0xFFA0),
332
+ (0xFFF0, 0xFFFD),
333
+ (0x110BD, 0x110BD),
334
+ (0x110CD, 0x110CD),
335
+ (0x13430, 0x1343F),
336
+ (0x1BCA0, 0x1BCA3),
337
+ (0x1D173, 0x1D17A),
338
+ (0xE0000, 0xE0FFF),
339
+ (0xF0000, 0xFFFFD),
340
+ (0x100000, 0x10FFFD),
341
+ )
342
+ _SPLITTERS = "".join(
343
+ chr(a) if a == b else chr(a) + "-" + chr(b) for a, b in _SPLITTER_RANGES
344
+ )
255
345
  _SPLITTER_RE = re.compile("[" + _SPLITTERS + "]")
256
346
  _UNSPLIT_RE = re.compile("[^" + _SPLITTERS + "]+")
257
347
  # A character a value can hold: past a splitter next to a found value, one of these means another value begins.
@@ -273,7 +363,9 @@ def _matches(text, patterns, entropy):
273
363
  """The matches of `patterns` in `text`, and with `entropy` the runs the entropy rule hides."""
274
364
  found = [m for pat in patterns for m in pat.finditer(text)]
275
365
  if entropy:
276
- found += [m for m in ENTROPY_LONG.finditer(text) if shannon_entropy(m.group(0)) >= 4.5]
366
+ found += [
367
+ m for m in ENTROPY_LONG.finditer(text) if shannon_entropy(m.group(0)) >= 4.5
368
+ ]
277
369
  return found
278
370
 
279
371
 
@@ -281,9 +373,11 @@ def _cut_spans(text, joined, firsts, origins, matches):
281
373
  """Where in `text` the values `matches` found in `joined` lie, for those splitters cut or touch. A stretch takes
282
374
  in the splitters next to its value when the character past them is no value character (so the value's
283
375
  neighbours are those it had in `joined`); a splitter between the value and another value stays."""
376
+
284
377
  def at(j): # where joined[j] is in text
285
378
  k = bisect.bisect_right(firsts, j) - 1
286
379
  return origins[k] + j - firsts[k]
380
+
287
381
  spans = []
288
382
  for m in matches:
289
383
  s, e = at(m.start()), at(m.end() - 1) + 1
@@ -310,7 +404,9 @@ def _kept(spans, plain):
310
404
  at += 1
311
405
  while heap and heap[0][0] <= s:
312
406
  heapq.heappop(heap)
313
- if not any((o_s <= s and e <= o_e) or not (s <= o_s and o_e <= e) for o_e, o_s in heap):
407
+ if not any(
408
+ (o_s <= s and e <= o_e) or not (s <= o_s and o_e <= e) for o_e, o_s in heap
409
+ ):
314
410
  out.append((s, e))
315
411
  return out
316
412
 
@@ -333,10 +429,18 @@ def _rejoin_split(text, patterns, entropy=False, trail=None):
333
429
  if not _SPLITTER_RE.search(text):
334
430
  return text
335
431
  joined, firsts, origins = _unsplit_runs(text)
336
- spans = _cut_spans(text, joined, firsts, origins, _matches(joined, patterns, entropy))
432
+ spans = _cut_spans(
433
+ text, joined, firsts, origins, _matches(joined, patterns, entropy)
434
+ )
337
435
  if spans:
338
- spans = _kept(spans, [(m.start(), m.end()) for m in _matches(text, patterns, entropy)])
339
- drops = [(m.start(), m.end(), "") for s, e in _merged(sorted(spans)) for m in _SPLITTER_RE.finditer(text, s, e)]
436
+ spans = _kept(
437
+ spans, [(m.start(), m.end()) for m in _matches(text, patterns, entropy)]
438
+ )
439
+ drops = [
440
+ (m.start(), m.end(), "")
441
+ for s, e in _merged(sorted(spans))
442
+ for m in _SPLITTER_RE.finditer(text, s, e)
443
+ ]
340
444
  if trail is not None:
341
445
  trail.carry(drops)
342
446
  out, at = [], 0
@@ -350,9 +454,11 @@ def _hide_patterns(out, mode, sub, counts):
350
454
  """`out` with each value the page patterns and the entropy rule find replaced by its marker, or by a mask with
351
455
  mode mask; `counts` adds them up by name."""
352
456
  for name, pat in SECRET_PATTERNS:
457
+
353
458
  def repl(m, n=name):
354
459
  counts[n] = counts.get(n, 0) + 1
355
460
  return _mask(m.group(0)) if mode == "mask" else f"[REDACTED:{n}]"
461
+
356
462
  out = sub(pat, repl, out)
357
463
 
358
464
  def ent_repl(m):
@@ -361,6 +467,7 @@ def _hide_patterns(out, mode, sub, counts):
361
467
  counts["entropy"] = counts.get("entropy", 0) + 1
362
468
  return _mask(s) if mode == "mask" else "[REDACTED:entropy]"
363
469
  return s
470
+
364
471
  return sub(ENTROPY_LONG, ent_repl, out)
365
472
 
366
473
 
@@ -371,6 +478,7 @@ def _scan_detected(held, sub, counts):
371
478
  detect-secrets hid anything."""
372
479
  from detect_secrets import SecretsCollection
373
480
  from detect_secrets.settings import default_settings
481
+
374
482
  tf = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
375
483
  tmp = tf.name
376
484
  try:
@@ -383,9 +491,10 @@ def _scan_detected(held, sub, counts):
383
491
  for s in col[fname]:
384
492
  sv = s.secret_value
385
493
  if sv and sv in held[0]:
386
- held[0] = sub(re.compile(re.escape(sv)), lambda m: "[REDACTED:ds]", held[0])
387
- counts["ds:detect-secrets"] = counts.get(
388
- "ds:detect-secrets", 0) + 1
494
+ held[0] = sub(
495
+ re.compile(re.escape(sv)), lambda m: "[REDACTED:ds]", held[0]
496
+ )
497
+ counts["ds:detect-secrets"] = counts.get("ds:detect-secrets", 0) + 1
389
498
  finally:
390
499
  try:
391
500
  os.unlink(tmp)
@@ -400,13 +509,18 @@ def _hide_detected(out, sub, counts):
400
509
  _scan_detected(held, sub, counts)
401
510
  except ImportError:
402
511
  if not getattr(redact, "_ds_warned", False):
403
- print("[lineage] WARN: detect-secrets unavailable, using fallback "
404
- "(pip install 'detect-secrets>=1.5')", file=sys.stderr)
512
+ print(
513
+ "[lineage] WARN: detect-secrets unavailable, using fallback "
514
+ "(pip install 'detect-secrets>=1.5')",
515
+ file=sys.stderr,
516
+ )
405
517
  redact._ds_warned = True
406
518
  except Exception as e:
407
519
  if not getattr(redact, "_ds_err_warned", False):
408
- print(f"[lineage] WARN: detect-secrets error ({e}), continuing "
409
- "with fallback", file=sys.stderr)
520
+ print(
521
+ f"[lineage] WARN: detect-secrets error ({e}), continuing with fallback",
522
+ file=sys.stderr,
523
+ )
410
524
  redact._ds_err_warned = True
411
525
  return held[0]
412
526
 
@@ -417,8 +531,15 @@ def _hide_extra(out, extra, sub, counts):
417
531
  for kw in [k.strip() for k in extra.split(",") if k.strip()]:
418
532
  pat = re.compile(r"\[REDACTED(?::[^\]\s]*)?\]|" + re.escape(kw), re.IGNORECASE)
419
533
  hits = []
420
- out = sub(pat, lambda m: m.group(0) if m.group(0).upper().startswith("[REDACTED")
421
- else hits.append(1) or "[REDACTED]", out)
534
+ out = sub(
535
+ pat,
536
+ lambda m: (
537
+ m.group(0)
538
+ if m.group(0).upper().startswith("[REDACTED")
539
+ else hits.append(1) or "[REDACTED]"
540
+ ),
541
+ out,
542
+ )
422
543
  if hits:
423
544
  counts[f"custom:{kw}"] = len(hits)
424
545
  return out
@@ -440,8 +561,10 @@ def redact(text: str, extra: "str | None" = None, mode: str = "full", trail=None
440
561
  # follow (the end of an item in an argument list written without spaces). A match starts only
441
562
  # there, so no user runs past the place where the next match may start: a long run stays linear,
442
563
  # and a user joined to a variable by a - ("$USER"-bot) is read whole.
443
- _CU_USER = (r"(?:[^\s:=\"'`(]|[\"'`(](?!-(?:u|[A-Za-z0-9]{1,6}u[\s\"']|-user[\s=\"'])"
444
- r"|,\\{0,7}[\"']))")
564
+ _CU_USER = (
565
+ r"(?:[^\s:=\"'`(]|[\"'`(](?!-(?:u|[A-Za-z0-9]{1,6}u[\s\"']|-user[\s=\"'])"
566
+ r"|,\\{0,7}[\"']))"
567
+ )
445
568
  # The quotes that may open the value, bare or escaped (\" and \\\" in a quoted command, '\'' and
446
569
  # '"'"' from bash and shlex.quote, ^" in cmd, `" in PowerShell, $' in bash): up to six in a row,
447
570
  # none that closes an item of an argument list.
@@ -452,9 +575,16 @@ _CU_LEAD = r"(?:[\\^`$]{0,7}[\"'](?!,\\{0,7}[\"'])){0,6}"
452
575
  # (mktemp -u "${TMPDIR:-/tmp}/x"). A literal password, or a default that is no path or holds a
453
576
  # colon, is hidden.
454
577
  _CU_REF = r"(?:\$\{?\w+\}?|%\w+%|\$\([^\s()]*(?:\s+[^\s()]+)*\))"
455
- _CU_SKIP = (r"(?!" + _CU_LEAD + r"(?:\+[\"']?%[-_0^#]?[A-Za-z%]|(?:\d+:\d+|(?:" + _CU_REF
456
- + r"|[^\s:=\"'`(]*\)):" + _CU_REF + r")(?![^\s\"'`;|&)\\^])"
457
- r"|\$\{\w+:[-=?+]?[/~.][^\s}\"'`(:]*\}[^\s:]*(?!\S)))")
578
+ _CU_SKIP = (
579
+ r"(?!"
580
+ + _CU_LEAD
581
+ + r"(?:\+[\"']?%[-_0^#]?[A-Za-z%]|(?:\d+:\d+|(?:"
582
+ + _CU_REF
583
+ + r"|[^\s:=\"'`(]*\)):"
584
+ + _CU_REF
585
+ + r")(?![^\s\"'`;|&)\\^])"
586
+ r"|\$\{\w+:[-=?+]?[/~.][^\s}\"'`(:]*\}[^\s:]*(?!\S)))"
587
+ )
458
588
 
459
589
  # Patterns every page (the reviewed flow and --rulebase) and every part file hides with before the
460
590
  # page patterns above, while keys are whole (3.0.2).
@@ -473,28 +603,52 @@ _MARKER_UNIT = r"\[REDACTED:\w+\]"
473
603
 
474
604
  REVIEW_SECRET_PATTERNS = [
475
605
  ("AnthropicKey", re.compile(r"(?<![A-Za-z0-9])sk-ant-[A-Za-z0-9_-]{20,}")),
476
- ("OpenAIKey", re.compile(r"(?<![A-Za-z0-9-])sk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}")),
606
+ (
607
+ "OpenAIKey",
608
+ re.compile(r"(?<![A-Za-z0-9-])sk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}"),
609
+ ),
477
610
  ("GoogleKey", re.compile(r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}")),
478
611
  # bounded repeats: a long run of `key` or `a.` would otherwise take quadratic time
479
- ("HexKey", re.compile(r"(?i)(?:key|token|secret)[\w-]{0,100}\s*[:=]\s*['\"]?[0-9a-f]{32,}")),
612
+ (
613
+ "HexKey",
614
+ re.compile(
615
+ r"(?i)(?:key|token|secret)[\w-]{0,100}\s*[:=]\s*['\"]?[0-9a-f]{32,}"
616
+ ),
617
+ ),
480
618
  # a value, not a path, a variable or a redaction ($PWD is a variable, PGPASSWORD a name);
481
619
  # markdown, a table cell or a bracket that closes after the value stays outside the match
482
- ("PasswordBare", re.compile(
483
- r"(?i)(?<!\$)(?:password|passwd|passcode|pwd|암호|비번|비밀\s?번호|패스워드)\s*[:=]\s*"
484
- r"(?![/~$])" + _NOT_REDACTED + r"(?=[^\s'\"]{6})[^\s'\"]*[^\s'\"`|*)]")),
620
+ (
621
+ "PasswordBare",
622
+ re.compile(
623
+ r"(?i)(?<!\$)(?:password|passwd|passcode|pwd|암호|비번|비밀\s?번호|패스워드)\s*[:=]\s*"
624
+ r"(?![/~$])" + _NOT_REDACTED + r"(?=[^\s'\"]{6})[^\s'\"]*[^\s'\"`|*)]"
625
+ ),
626
+ ),
485
627
  # the password runs to the last @ before the host: it may hold an @ of its own. A marker is
486
628
  # no user or password, so a second pass leaves a token-only URL (https://[REDACTED:...]@host).
487
- ("UrlCredential", re.compile(r"(?i)[a-z][a-z0-9+.-]{0,30}://(?:" + _MARKER_UNIT + r"|[^\s:@/])*:" + _NOT_MARKED
488
- + r"(?!\[REDACTED)[^\s/]{3,}@")),
629
+ (
630
+ "UrlCredential",
631
+ re.compile(
632
+ r"(?i)[a-z][a-z0-9+.-]{0,30}://(?:"
633
+ + _MARKER_UNIT
634
+ + r"|[^\s:@/])*:"
635
+ + _NOT_MARKED
636
+ + r"(?!\[REDACTED)[^\s/]{3,}@"
637
+ ),
638
+ ),
489
639
  ("Bearer", re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{16,}")),
490
640
  # Authorization then `:`, `=`, `=>` or headers["Authorization"] =, a call's ("Authorization",
491
641
  # "...") and a HAR name/value pair, or a space before a quote (nginx, Apache)
492
- ("BasicAuth", re.compile(
493
- r"(?i)\bauthorization"
494
- r"(?:[\"']?\]?\s*(?:=>|[:=])"
495
- r"|[\"']\s*,\s*(?:[\"']value[\"']\s*:\s*)?(?=[\"'])"
496
- r"|\s+(?=[\"']))"
497
- r"\s*[\"']?basic\s+[A-Za-z0-9+/]{8,}={0,2}")),
642
+ (
643
+ "BasicAuth",
644
+ re.compile(
645
+ r"(?i)\bauthorization"
646
+ r"(?:[\"']?\]?\s*(?:=>|[:=])"
647
+ r"|[\"']\s*,\s*(?:[\"']value[\"']\s*:\s*)?(?=[\"'])"
648
+ r"|\s+(?=[\"']))"
649
+ r"\s*[\"']?basic\s+[A-Za-z0-9+/]{8,}={0,2}"
650
+ ),
651
+ ),
498
652
  # curl's -u and --user flags, alone or last in a bundle (-su, -4u), with a user and password
499
653
  # after a space, an = or nothing, quoted together or apart ("user":"pw"), with quotes bare,
500
654
  # escaped or nested (_CU_LEAD) or none, in inline code or parentheses, or as two items of an
@@ -503,15 +657,38 @@ REVIEW_SECRET_PATTERNS = [
503
657
  # long key as the user with no password, count too. The separators share no character with
504
658
  # the user name, so a long run of them stays linear. The values _CU_SKIP names and a
505
659
  # host:/path (rsync) are left, and so is a marker an earlier pass left (-u [REDACTED:JWT]).
506
- ("CurlUser", re.compile(
507
- r"(?<![^\s\"'`(])(?:-u\s*|-[A-Za-z0-9]{1,6}u\s+|--user(?:\s+|=))" + _CU_SKIP + _CU_LEAD
508
- + r"(?:(?:" + _MARKER_UNIT + r"|" + _CU_USER + r")*:" + _NOT_MARKED + r"(?!/|\[REDACTED)\S{3,}|" + _CU_USER
509
- + r"{15,}:" + _NOT_MARKED
510
- + r"(?=[\s\"'`).,;\\]|$))"
511
- r"|(\\{0,7}[\"'])(?:-[A-Za-z0-9]{0,6}u|--user)\1\s*,\s*(\\{0,7}[\"'])" + _CU_SKIP
512
- + r"(?:(?:" + _MARKER_UNIT + r"|" + _CU_USER + r")*:" + _NOT_MARKED + r"(?!/|\[REDACTED)[^\s\"']{3,}|"
513
- + _CU_USER + r"{15,}:"
514
- + _NOT_MARKED + r")\2")),
660
+ (
661
+ "CurlUser",
662
+ re.compile(
663
+ r"(?<![^\s\"'`(])(?:-u\s*|-[A-Za-z0-9]{1,6}u\s+|--user(?:\s+|=))"
664
+ + _CU_SKIP
665
+ + _CU_LEAD
666
+ + r"(?:(?:"
667
+ + _MARKER_UNIT
668
+ + r"|"
669
+ + _CU_USER
670
+ + r")*:"
671
+ + _NOT_MARKED
672
+ + r"(?!/|\[REDACTED)\S{3,}|"
673
+ + _CU_USER
674
+ + r"{15,}:"
675
+ + _NOT_MARKED
676
+ + r"(?=[\s\"'`).,;\\]|$))"
677
+ r"|(\\{0,7}[\"'])(?:-[A-Za-z0-9]{0,6}u|--user)\1\s*,\s*(\\{0,7}[\"'])"
678
+ + _CU_SKIP
679
+ + r"(?:(?:"
680
+ + _MARKER_UNIT
681
+ + r"|"
682
+ + _CU_USER
683
+ + r")*:"
684
+ + _NOT_MARKED
685
+ + r"(?!/|\[REDACTED)[^\s\"']{3,}|"
686
+ + _CU_USER
687
+ + r"{15,}:"
688
+ + _NOT_MARKED
689
+ + r")\2"
690
+ ),
691
+ ),
515
692
  ]
516
693
 
517
694
 
@@ -521,7 +698,9 @@ def _reviewer_patterns(text, trail=None):
521
698
  counts, sub = {}, _sub_for(trail)
522
699
  for name, pat in REVIEW_SECRET_PATTERNS:
523
700
  hits = []
524
- text = sub(pat, lambda m, marker=f"[REDACTED:{name}]": hits.append(1) or marker, text)
701
+ text = sub(
702
+ pat, lambda m, marker=f"[REDACTED:{name}]": hits.append(1) or marker, text
703
+ )
525
704
  if hits:
526
705
  counts[name] = len(hits)
527
706
  return text, counts
@@ -531,8 +710,11 @@ def _rejoin_secrets(text, extra=None, trail=None):
531
710
  """`text` with each secret a splitter cuts put back together, for every pattern a page or a reviewer's text is
532
711
  hidden with: the reviewer's, the page's, the entropy rule and the --redact-extra keywords (3.0.8)."""
533
712
  keywords = [k.strip() for k in (extra or "").split(",") if k.strip()]
534
- patterns = ([p for _, p in REVIEW_SECRET_PATTERNS] + [p for _, p in SECRET_PATTERNS]
535
- + [re.compile(re.escape(k), re.IGNORECASE) for k in keywords])
713
+ patterns = (
714
+ [p for _, p in REVIEW_SECRET_PATTERNS]
715
+ + [p for _, p in SECRET_PATTERNS]
716
+ + [re.compile(re.escape(k), re.IGNORECASE) for k in keywords]
717
+ )
536
718
  return _rejoin_split(text, patterns, entropy=True, trail=trail)
537
719
 
538
720
 
@@ -601,7 +783,9 @@ def _hidden_by(floor, base):
601
783
  j = i
602
784
  while j < n and pos[j] < 0:
603
785
  j += 1
604
- out.append((pos[i - 1] + 1 if i else 0, pos[j] if j < n else base.n, floor[i:j]))
786
+ out.append(
787
+ (pos[i - 1] + 1 if i else 0, pos[j] if j < n else base.n, floor[i:j])
788
+ )
605
789
  i = j
606
790
  return out
607
791
 
@@ -634,7 +818,10 @@ def _with_rejoins(text, extra, one_pass):
634
818
  [found])."""
635
819
  red, founds, trail = _join_first(text, extra, one_pass)
636
820
  if trail is None:
637
- return red, founds # nothing joined: that was the one pass over the text as it stands
821
+ return (
822
+ red,
823
+ founds,
824
+ ) # nothing joined: that was the one pass over the text as it stands
638
825
  base = _Trail(len(text))
639
826
  floor, found = one_pass(text, base)
640
827
  joined = _kinds(founds)
@@ -662,7 +849,9 @@ def review_redact(text, extra=None):
662
849
  are whole (the entropy rule would otherwise cut a key and leave the rest), then fully
663
850
  redacted (never masked, whatever --redact-mode says). A secret a splitter cuts is put back
664
851
  together (3.0.8). Returns (text, count)."""
665
- red, founds = _with_rejoins(text, extra, lambda t, trail=None: _review_pass(t, extra, trail))
852
+ red, founds = _with_rejoins(
853
+ text, extra, lambda t, trail=None: _review_pass(t, extra, trail)
854
+ )
666
855
  return red, sum(sum(f.values()) for f in founds)
667
856
 
668
857
 
@@ -681,7 +870,9 @@ def page_redact(text, extra=None, mode="full"):
681
870
  splitter cuts is put back together (3.0.8). Returns (text, count_by_kind). A second pass
682
871
  over its output finds nothing, but for two values glued with no separator at all (a key
683
872
  run into `Bearer`), where the second shows only once the first is a marker, as in 3.0.7."""
684
- red, founds = _with_rejoins(text, extra, lambda t, trail=None: _page_pass(t, extra, mode, trail))
873
+ red, founds = _with_rejoins(
874
+ text, extra, lambda t, trail=None: _page_pass(t, extra, mode, trail)
875
+ )
685
876
  counts = {}
686
877
  for found in founds:
687
878
  for k, v in found.items():
@@ -723,23 +914,111 @@ def encode_cwd(cwd) -> str:
723
914
  return re.sub(r"[^A-Za-z0-9]", "-", str(cwd))
724
915
 
725
916
 
917
+ def codex_sessions_dir():
918
+ """Where Codex keeps its session records (rollouts): <CODEX_HOME>/sessions/YYYY/MM/DD/rollout-*.jsonl."""
919
+ home = os.environ.get("CODEX_HOME") or str(pathlib.Path.home() / ".codex")
920
+ return pathlib.Path(home) / "sessions"
921
+
922
+
923
+ def _codex_thread():
924
+ """This Codex session's id when lineage runs inside Codex (Codex gives its shell CODEX_THREAD_ID, the id at the
925
+ end of the rollout file name), else None."""
926
+ tid = os.environ.get("CODEX_THREAD_ID", "")
927
+ return tid if re.fullmatch(r"[0-9A-Za-z-]{8,64}", tid) else None
928
+
929
+
930
+ def _mtime(path):
931
+ """A file's change time, or 0 for one that went away between the listing and the look."""
932
+ try:
933
+ return path.stat().st_mtime
934
+ except OSError:
935
+ return 0
936
+
937
+
938
+ def _codex_rollouts(pattern):
939
+ try:
940
+ found = list(codex_sessions_dir().glob(f"*/*/*/{pattern}"))
941
+ except OSError:
942
+ return []
943
+ return sorted(found, key=_mtime)
944
+
945
+
946
+ def _codex_meta(path):
947
+ """The session_meta payload a rollout opens with, else None (not a Codex rollout, or unreadable). Only a regular
948
+ file is opened, so a named pipe is not waited on."""
949
+ try:
950
+ if not stat.S_ISREG(os.stat(str(path)).st_mode):
951
+ return None
952
+ with open(path, encoding="utf-8") as f:
953
+ obj = json.loads(f.readline())
954
+ return (
955
+ obj["payload"]
956
+ if obj.get("type") == "session_meta"
957
+ and isinstance(obj.get("payload"), dict)
958
+ else None
959
+ )
960
+ except (
961
+ OSError,
962
+ UnicodeDecodeError,
963
+ ValueError,
964
+ RecursionError,
965
+ KeyError,
966
+ TypeError,
967
+ AttributeError,
968
+ ):
969
+ return None
970
+
971
+
972
+ def _codex_cwd(path):
973
+ """The cwd a rollout's first record (session_meta) names, else None. A subagent's rollout (its source names
974
+ the parent thread) gives None: what the parent handed it reads as a user turn, so it is no session of the user."""
975
+ try:
976
+ with open(path, encoding="utf-8") as f:
977
+ obj = json.loads(f.readline())
978
+ meta = obj["payload"] if obj.get("type") == "session_meta" else {}
979
+ source = meta.get("source")
980
+ if isinstance(source, dict) and "subagent" in source:
981
+ return None
982
+ return meta.get("cwd")
983
+ except (
984
+ OSError,
985
+ UnicodeDecodeError,
986
+ ValueError,
987
+ RecursionError,
988
+ KeyError,
989
+ TypeError,
990
+ AttributeError,
991
+ ):
992
+ return None
993
+
994
+
726
995
  def auto_discover_jsonl():
727
- projdir = pathlib.Path.home() / ".claude" / "projects" / encode_cwd(
728
- pathlib.Path.cwd())
996
+ tid = _codex_thread()
997
+ if tid: # inside Codex: this session or nothing, never a Claude Code one
998
+ found = _codex_rollouts(f"rollout-*-{tid}.jsonl")
999
+ return found[-1] if found else None
1000
+ projdir = (
1001
+ pathlib.Path.home() / ".claude" / "projects" / encode_cwd(pathlib.Path.cwd())
1002
+ )
729
1003
  if not projdir.exists():
730
1004
  return None
731
1005
  try:
732
- files = sorted(projdir.glob("*.jsonl"),
733
- key=lambda p: p.stat().st_mtime, reverse=True)
1006
+ files = sorted(
1007
+ projdir.glob("*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True
1008
+ )
734
1009
  except OSError:
735
1010
  return None
736
1011
  return files[0] if files else None
737
1012
 
738
1013
 
739
1014
  def project_jsonl_files():
740
- """All *.jsonl in the encoded project dir, for --all-sessions."""
741
- projdir = pathlib.Path.home() / ".claude" / "projects" / encode_cwd(
742
- pathlib.Path.cwd())
1015
+ """All *.jsonl in the encoded project dir, for --all-sessions. Inside Codex, the Codex sessions of this folder."""
1016
+ if _codex_thread():
1017
+ here = str(pathlib.Path.cwd())
1018
+ return [p for p in _codex_rollouts("rollout-*.jsonl") if _codex_cwd(p) == here]
1019
+ projdir = (
1020
+ pathlib.Path.home() / ".claude" / "projects" / encode_cwd(pathlib.Path.cwd())
1021
+ )
743
1022
  if not projdir.exists():
744
1023
  return []
745
1024
  try:
@@ -766,7 +1045,9 @@ def session_start(path):
766
1045
  # ---------------------------------------------------------------- JSONL parse
767
1046
  KNOWN_TYPES = {"user", "assistant", "agent-setting", "permission-mode", "summary"}
768
1047
  KNOWN_CONTENT_ITEMS = {"text", "tool_use", "tool_result", "thinking"}
769
- _CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]") # strip control chars (keeps \t\n)
1048
+ _CTRL_RE = re.compile(
1049
+ r"[\x00-\x08\x0b\x0c\x0e-\x1f]"
1050
+ ) # strip control chars (keeps \t\n)
770
1051
  # What a JSON escape for a lone surrogate (U+D800 to U+DFFF with no pair) leaves in a str. UTF-8 cannot hold one,
771
1052
  # so a page, a review pack or a cache key that has it stops with UnicodeEncodeError.
772
1053
  _LONE_SURROGATE_RE = re.compile("[" + chr(0xD800) + "-" + chr(0xDFFF) + "]")
@@ -780,13 +1061,18 @@ def _without_lone_surrogates(value: Any) -> Any:
780
1061
  if isinstance(value, list):
781
1062
  return [_without_lone_surrogates(v) for v in value]
782
1063
  if isinstance(value, dict):
783
- return {_without_lone_surrogates(k): _without_lone_surrogates(v) for k, v in value.items()}
1064
+ return {
1065
+ _without_lone_surrogates(k): _without_lone_surrogates(v)
1066
+ for k, v in value.items()
1067
+ }
784
1068
  return value
785
1069
 
786
1070
 
787
1071
  def _lone_to_fffd(value):
788
1072
  """A str with each lone surrogate as U+FFFD; anything else as it is (no walk into it)."""
789
- return _LONE_SURROGATE_RE.sub(chr(0xFFFD), value) if isinstance(value, str) else value
1073
+ return (
1074
+ _LONE_SURROGATE_RE.sub(chr(0xFFFD), value) if isinstance(value, str) else value
1075
+ )
790
1076
 
791
1077
 
792
1078
  def _shown_id(value):
@@ -798,9 +1084,14 @@ def _shown_id(value):
798
1084
  def _clean_strings(line_no, role, text, tools, ts, uuid_):
799
1085
  """A record's role, text, tool counts, timestamp and uuid with each lone surrogate as U+FFFD, after one WARN for
800
1086
  the line. Only text is touched: a uuid of another shape is replaced later, whole."""
801
- if not any(isinstance(s, str) and _LONE_SURROGATE_RE.search(s) for s in (role, text, ts, uuid_, *tools)):
1087
+ if not any(
1088
+ isinstance(s, str) and _LONE_SURROGATE_RE.search(s)
1089
+ for s in (role, text, ts, uuid_, *tools)
1090
+ ):
802
1091
  return role, text, tools, ts, uuid_
803
- print(f"[WARN] line {line_no}: lone surrogate replaced with U+FFFD", file=sys.stderr)
1092
+ print(
1093
+ f"[WARN] line {line_no}: lone surrogate replaced with U+FFFD", file=sys.stderr
1094
+ )
804
1095
  counts = {}
805
1096
  for name, n in tools.items():
806
1097
  name = _lone_to_fffd(name)
@@ -815,7 +1106,10 @@ def _turn_id(uuid_, line_no, ts):
815
1106
  if isinstance(uuid_, str) and uuid_:
816
1107
  return uuid_
817
1108
  if uuid_ is not None and uuid_ != "":
818
- print(f"[WARN] line {line_no}: uuid that is not text replaced by an id from the line", file=sys.stderr)
1109
+ print(
1110
+ f"[WARN] line {line_no}: uuid that is not text replaced by an id from the line",
1111
+ file=sys.stderr,
1112
+ )
819
1113
  return hashlib.sha256(f"{line_no}:{ts}".encode()).hexdigest()[:16]
820
1114
 
821
1115
 
@@ -825,7 +1119,7 @@ def _json_object(raw, line_no):
825
1119
  something other than an object is skipped, not fatal."""
826
1120
  try:
827
1121
  obj = json.loads(raw)
828
- except (ValueError, RecursionError): # JSONDecodeError is a ValueError
1122
+ except (ValueError, RecursionError): # JSONDecodeError is a ValueError
829
1123
  print(f"[WARN] line {line_no}: invalid JSON skipped", file=sys.stderr)
830
1124
  return None
831
1125
  if not isinstance(obj, dict):
@@ -841,18 +1135,27 @@ def _is_turn_record(obj, seen_unknown_type, unsafe_schema, line_no):
841
1135
  if obj.get("isSidechain") or obj.get("teamName") or obj.get("agentName"):
842
1136
  return False
843
1137
  t = obj.get("type")
844
- if (t and not isinstance(t, str)) or (t in ("user", "assistant") and _odd_message(obj.get("message"))):
845
- print(f"[WARN] line {line_no}: record of an unknown shape skipped", file=sys.stderr)
1138
+ if (t and not isinstance(t, str)) or (
1139
+ t in ("user", "assistant") and _odd_message(obj.get("message"))
1140
+ ):
1141
+ print(
1142
+ f"[WARN] line {line_no}: record of an unknown shape skipped",
1143
+ file=sys.stderr,
1144
+ )
846
1145
  return False
847
1146
  if t in ("user", "assistant"):
848
1147
  return True
849
1148
  if t and t not in KNOWN_TYPES and t not in seen_unknown_type:
850
1149
  seen_unknown_type.add(t)
851
- print(f"[WARN] unknown record type {t!r} skipped", file=sys.stderr) # repr: no raw control character
1150
+ print(
1151
+ f"[WARN] unknown record type {t!r} skipped", file=sys.stderr
1152
+ ) # repr: no raw control character
852
1153
  if not unsafe_schema and t.startswith("v"):
853
- print(f"[ERROR] schema marker {t!r} unrecognized — "
854
- "rerun with --unsafe-schema to proceed",
855
- file=sys.stderr)
1154
+ print(
1155
+ f"[ERROR] schema marker {t!r} unrecognized — "
1156
+ "rerun with --unsafe-schema to proceed",
1157
+ file=sys.stderr,
1158
+ )
856
1159
  raise SystemExit(2)
857
1160
  return False
858
1161
 
@@ -870,8 +1173,11 @@ def _odd_message(msg):
870
1173
  def _odd_item(it, text, name):
871
1174
  """Whether a content item's type, text or tool name has a shape a transcript does not give. A missing or empty
872
1175
  text reads as empty and a missing or null tool name as '?'."""
873
- return ((it is not None and not isinstance(it, str)) or (it == "text" and not isinstance(text, str))
874
- or (it == "tool_use" and not isinstance(name, str)))
1176
+ return (
1177
+ (it is not None and not isinstance(it, str))
1178
+ or (it == "text" and not isinstance(text, str))
1179
+ or (it == "tool_use" and not isinstance(name, str))
1180
+ )
875
1181
 
876
1182
 
877
1183
  def _take_item(item, line_no, text_parts, tools):
@@ -882,14 +1188,19 @@ def _take_item(item, line_no, text_parts, tools):
882
1188
  it, text, name = item.get("type"), item.get("text") or "", item.get("name")
883
1189
  name = "?" if name is None else name
884
1190
  if _odd_item(it, text, name):
885
- print(f"[WARN] line {line_no}: content item of an unknown shape skipped", file=sys.stderr)
1191
+ print(
1192
+ f"[WARN] line {line_no}: content item of an unknown shape skipped",
1193
+ file=sys.stderr,
1194
+ )
886
1195
  elif it == "text":
887
1196
  text_parts.append(text)
888
1197
  elif it == "tool_use":
889
1198
  tools[name] = tools.get(name, 0) + 1
890
1199
  elif it not in KNOWN_CONTENT_ITEMS:
891
- print(f"[WARN] line {line_no}: unknown content type "
892
- f"{it!r} skipped", file=sys.stderr)
1200
+ print(
1201
+ f"[WARN] line {line_no}: unknown content type {it!r} skipped",
1202
+ file=sys.stderr,
1203
+ )
893
1204
 
894
1205
 
895
1206
  def _content_parts(content, line_no):
@@ -903,14 +1214,115 @@ def _content_parts(content, line_no):
903
1214
  _take_item(item, line_no, text_parts, tools)
904
1215
  else:
905
1216
  if content is not None:
906
- print(f"[WARN] line {line_no}: unknown content shape "
907
- f"({type(content).__name__})", file=sys.stderr)
1217
+ print(
1218
+ f"[WARN] line {line_no}: unknown content shape "
1219
+ f"({type(content).__name__})",
1220
+ file=sys.stderr,
1221
+ )
908
1222
  return None
909
1223
  return text_parts, tools
910
1224
 
911
1225
 
912
- def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
913
- session_name=None):
1226
+ def _is_codex_meta(obj):
1227
+ """Whether a record opens a Codex session record (rollout): a session_meta with a payload."""
1228
+ return obj.get("type") == "session_meta" and isinstance(obj.get("payload"), dict)
1229
+
1230
+
1231
+ _CODEX_ITEM_ROLE = {"UserMessage": "user", "AgentMessage": "assistant"}
1232
+ _CODEX_INNER_TOOL_RE = re.compile(r"\btools\.([A-Za-z_][A-Za-z0-9_]{0,63})\(")
1233
+
1234
+
1235
+ def _codex_item_text(item):
1236
+ """The text parts of a completed item (Codex 0.160: content [{type: text|Text, text}])."""
1237
+ content = item.get("content")
1238
+ if not isinstance(content, list):
1239
+ return ""
1240
+ return "\n".join(
1241
+ c["text"]
1242
+ for c in content
1243
+ if isinstance(c, dict) and isinstance(c.get("text"), str)
1244
+ )
1245
+
1246
+
1247
+ def _codex_tools(p, pt):
1248
+ """Tool counts of one call record. Codex 0.160 runs tools from code (`exec` with `tools.exec_command(...)` in its
1249
+ input), so the inner tool names count, not the wrapper."""
1250
+ name = p.get("name")
1251
+ name = name if isinstance(name, str) and name else pt[:-5]
1252
+ inner = (
1253
+ _CODEX_INNER_TOOL_RE.findall(p.get("input"))
1254
+ if isinstance(p.get("input"), str)
1255
+ else []
1256
+ )
1257
+ counts = {}
1258
+ for n in inner or [name]:
1259
+ counts[n] = counts.get(n, 0) + 1
1260
+ return counts
1261
+
1262
+
1263
+ def _codex_record(obj):
1264
+ """(role, text, tools) a Codex rollout record shows as a turn, else None. What the user typed and what the
1265
+ model said come from the event records (0.144: user_message, agent_message; 0.160: item_completed with a
1266
+ UserMessage or AgentMessage item), so what Codex puts into the model input (environment, instructions, an
1267
+ injected skill body) is not a turn; a tool call is an assistant record with no text, merged into the turn around
1268
+ it like a Claude Code tool_use; a compaction is a divider."""
1269
+ p = obj.get("payload")
1270
+ kind = obj.get("type")
1271
+ if kind == "compacted":
1272
+ return "mark", "compaction", {}
1273
+ if not isinstance(p, dict):
1274
+ return None
1275
+ pt = p.get("type")
1276
+ if kind == "event_msg" and pt in ("user_message", "agent_message"):
1277
+ msg = p.get("message")
1278
+ return (
1279
+ ("user" if pt == "user_message" else "assistant"),
1280
+ (msg if isinstance(msg, str) else ""),
1281
+ {},
1282
+ )
1283
+ item = p.get("item")
1284
+ if (
1285
+ kind == "event_msg"
1286
+ and pt == "item_completed"
1287
+ and isinstance(item, dict)
1288
+ and item.get("type") in _CODEX_ITEM_ROLE
1289
+ ):
1290
+ return _CODEX_ITEM_ROLE[item["type"]], _codex_item_text(item), {}
1291
+ if kind == "response_item" and isinstance(pt, str) and pt.endswith("_call"):
1292
+ return "assistant", "", _codex_tools(p, pt)
1293
+ return None
1294
+
1295
+
1296
+ def _codex_turn(obj, line_no, session_id, session_name):
1297
+ """The turn dict of one Codex rollout record, else None."""
1298
+ got = _codex_record(obj)
1299
+ if got is None:
1300
+ return None
1301
+ role, text, tools = got
1302
+ text = _CTRL_RE.sub("", text.strip())
1303
+ if not text and not tools:
1304
+ return None
1305
+ role, text, tools, ts, uuid_ = _clean_strings(
1306
+ line_no, role, text, tools, _stamp(obj.get("timestamp")), None
1307
+ )
1308
+ uuid_ = _turn_id(uuid_, line_no, ts)
1309
+ return {
1310
+ "role": role,
1311
+ "text": text,
1312
+ "ts": ts,
1313
+ "uuid": uuid_,
1314
+ "tools": tools,
1315
+ "line_no": line_no,
1316
+ "parts": [text] if text else [],
1317
+ "session": session_id,
1318
+ "session_name": session_name,
1319
+ "meta": False,
1320
+ }
1321
+
1322
+
1323
+ def parse_turns(
1324
+ stream, unsafe_schema: bool = False, session_id=None, session_name=None
1325
+ ):
914
1326
  """Yield dicts: {role, text, ts, uuid, tools, line_no, parts, session,
915
1327
  session_name}.
916
1328
 
@@ -919,12 +1331,28 @@ def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
919
1331
  needs it) and `session` (session guards for merge/echo).
920
1332
  """
921
1333
  seen_unknown_type = set()
1334
+ codex = None # set by the first record: a Codex rollout or a Claude transcript
1335
+ last_said = (
1336
+ None # Codex: (role, text) of the last message, so one said twice counts once
1337
+ )
922
1338
  for line_no, raw in enumerate(stream, 1):
923
1339
  raw = raw.strip()
924
1340
  if not raw:
925
1341
  continue
926
1342
  obj = _json_object(raw, line_no)
927
- if obj is None or not _is_turn_record(obj, seen_unknown_type, unsafe_schema, line_no):
1343
+ if obj is not None and codex is None:
1344
+ codex = _is_codex_meta(obj)
1345
+ if obj is not None and codex:
1346
+ turn = _codex_turn(obj, line_no, session_id, session_name)
1347
+ said = (turn["role"], turn["text"]) if turn and turn["text"] else None
1348
+ if turn and not (said and said == last_said):
1349
+ yield turn
1350
+ if said:
1351
+ last_said = said
1352
+ continue
1353
+ if obj is None or not _is_turn_record(
1354
+ obj, seen_unknown_type, unsafe_schema, line_no
1355
+ ):
928
1356
  continue
929
1357
  msg = obj.get("message") or {}
930
1358
  got = _content_parts(msg.get("content"), line_no)
@@ -935,16 +1363,26 @@ def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
935
1363
  if not text and not tools:
936
1364
  continue
937
1365
  role = msg.get("role")
938
- role = role if role in ("user", "assistant") else obj.get("type") # the type _is_turn_record let through
1366
+ role = (
1367
+ role if role in ("user", "assistant") else obj.get("type")
1368
+ ) # the type _is_turn_record let through
939
1369
  role, text, tools, ts, uuid_ = _clean_strings(
940
- line_no, role, text, tools, _stamp(obj.get("timestamp")), obj.get("uuid"))
1370
+ line_no, role, text, tools, _stamp(obj.get("timestamp")), obj.get("uuid")
1371
+ )
941
1372
  uuid_ = _turn_id(uuid_, line_no, ts)
942
1373
 
943
- yield {"role": role, "text": text, "ts": ts, "uuid": uuid_,
944
- "tools": tools, "line_no": line_no,
945
- "parts": [text] if text else [], # R1: always seed parts
946
- "session": session_id, "session_name": session_name,
947
- "meta": bool(obj.get("isMeta"))} # a harness-injected body
1374
+ yield {
1375
+ "role": role,
1376
+ "text": text,
1377
+ "ts": ts,
1378
+ "uuid": uuid_,
1379
+ "tools": tools,
1380
+ "line_no": line_no,
1381
+ "parts": [text] if text else [], # R1: always seed parts
1382
+ "session": session_id,
1383
+ "session_name": session_name,
1384
+ "meta": bool(obj.get("isMeta")),
1385
+ } # a harness-injected body
948
1386
 
949
1387
 
950
1388
  # ---------------------------------------------------------------- Classify (A-1..A-4)
@@ -966,9 +1404,11 @@ def classify_turns(turns, drop_trivia=True):
966
1404
  if _HOOK_FEEDBACK_RE.match(text) or _INTERRUPT_RE.match(text):
967
1405
  continue
968
1406
  # 2. injected skill/workflow body & harness errors → drop if trivia
969
- if drop_trivia and (_SKILL_BODY_RE.match(text)
970
- or _WORKFLOW_BODY_RE.match(text)
971
- or _SYS_ERROR_RE.match(text)):
1407
+ if drop_trivia and (
1408
+ _SKILL_BODY_RE.match(text)
1409
+ or _WORKFLOW_BODY_RE.match(text)
1410
+ or _SYS_ERROR_RE.match(text)
1411
+ ):
972
1412
  continue
973
1413
  # 3. image note → replace with attachment marker
974
1414
  if _IMAGE_NOTE_RE.match(text):
@@ -981,7 +1421,7 @@ def classify_turns(turns, drop_trivia=True):
981
1421
  # 5. agent / teammate wrapper → agent bubble
982
1422
  who, body = split_agent_message(text)
983
1423
  if who is not None:
984
- if not body: # idle notification
1424
+ if not body: # idle notification
985
1425
  continue
986
1426
  out.append(dict(t, role="agent", text=body, agent_from=who))
987
1427
  continue
@@ -1003,8 +1443,12 @@ def merge_assistant_runs(turns):
1003
1443
  merged = []
1004
1444
  for t in turns:
1005
1445
  p = merged[-1] if merged else None
1006
- if (p and t["role"] == "assistant" and p["role"] == "assistant"
1007
- and t.get("session") == p.get("session")):
1446
+ if (
1447
+ p
1448
+ and t["role"] == "assistant"
1449
+ and p["role"] == "assistant"
1450
+ and t.get("session") == p.get("session")
1451
+ ):
1008
1452
  if t["text"]:
1009
1453
  p["text"] = (p["text"] + "\n\n" + t["text"]).strip()
1010
1454
  p["parts"].append(t["text"])
@@ -1025,9 +1469,13 @@ def echo_indices(turns):
1025
1469
  if t["role"] != "user" or len(t["text"]) > ECHO_ASK:
1026
1470
  continue
1027
1471
  n = turns[i + 1] if i + 1 < len(turns) else None
1028
- if (n and n["role"] == "assistant" and not n["tools"]
1029
- and len(n["text"].strip()) <= ECHO_REPLY
1030
- and n.get("session") == t.get("session")):
1472
+ if (
1473
+ n
1474
+ and n["role"] == "assistant"
1475
+ and not n["tools"]
1476
+ and len(n["text"].strip()) <= ECHO_REPLY
1477
+ and n.get("session") == t.get("session")
1478
+ ):
1031
1479
  drop |= {i, i + 1}
1032
1480
  return drop
1033
1481
 
@@ -1057,7 +1505,8 @@ def fill_missing_ts(turns):
1057
1505
  _SENT_SPLIT = re.compile(r"(?<=[.!?…])(?<![0-9][.!?…])(?=\s|$)\s*|\n+")
1058
1506
  _INTENT_ONLY_RE = re.compile(
1059
1507
  r"^.{0,30}(?:하겠습니다|합니다|해보겠습니다|확인합니다|봅니다|"
1060
- r"보겠습니다|시작합니다|진행합니다)\.?$")
1508
+ r"보겠습니다|시작합니다|진행합니다)\.?$"
1509
+ )
1061
1510
  SHORT_DETAIL = 160
1062
1511
  SINGLE_SPLIT_MIN = SHORT_DETAIL
1063
1512
 
@@ -1069,7 +1518,7 @@ def _cut(s, n):
1069
1518
  s = s.strip()
1070
1519
  if len(s) <= n:
1071
1520
  return s
1072
- head = _OPEN_MARKER_RE.sub("", s[:max(1, n - 1)]) # a marker cut in two goes whole
1521
+ head = _OPEN_MARKER_RE.sub("", s[: max(1, n - 1)]) # a marker cut in two goes whole
1073
1522
  return head.rstrip() + "…"
1074
1523
 
1075
1524
 
@@ -1146,10 +1595,14 @@ def _stdin_id_key():
1146
1595
  CACHE_BASE.mkdir(parents=True, exist_ok=True)
1147
1596
  os.chmod(str(CACHE_BASE), 0o700)
1148
1597
  # O_EXCL: a name that is already there (a link planted at this guessable name among them) is refused
1149
- fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0), 0o600)
1598
+ fd = os.open(
1599
+ str(tmp),
1600
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0),
1601
+ 0o600,
1602
+ )
1150
1603
  with os.fdopen(fd, "wb") as f:
1151
1604
  f.write(key)
1152
- os.replace(str(tmp), str(path)) # whole or not at all
1605
+ os.replace(str(tmp), str(path)) # whole or not at all
1153
1606
  except OSError:
1154
1607
  try:
1155
1608
  os.unlink(str(tmp))
@@ -1172,17 +1625,38 @@ def cache_dir(session_id: str):
1172
1625
  def _page_clean_turn(t, extra, mode):
1173
1626
  """`t` with its text and parts as a page shows them, so a summary cut from it cannot cut a
1174
1627
  secret in two and keep the half no pattern recognises."""
1628
+
1175
1629
  def clean(s):
1176
1630
  return page_redact(s, extra, mode)[0]
1177
- return dict(t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []])
1631
+
1632
+ return dict(
1633
+ t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []]
1634
+ )
1178
1635
 
1179
1636
 
1180
1637
  # Letters that print as nothing: the Hangul fillers, the braille blank, and the combining marks Unicode ignores by
1181
1638
  # default (the grapheme joiner, the Khmer inherent vowels, the Mongolian free variation selectors). With the control,
1182
1639
  # format, private-use, unassigned and separator categories, a cache made only of them holds no summary a person can
1183
1640
  # read (3.0.5).
1184
- _BLANK_LETTERS = frozenset(map(chr, (0x115F, 0x1160, 0x3164, 0xFFA0, 0x2800,
1185
- 0x034F, 0x17B4, 0x17B5, 0x180B, 0x180C, 0x180D, 0x180F)))
1641
+ _BLANK_LETTERS = frozenset(
1642
+ map(
1643
+ chr,
1644
+ (
1645
+ 0x115F,
1646
+ 0x1160,
1647
+ 0x3164,
1648
+ 0xFFA0,
1649
+ 0x2800,
1650
+ 0x034F,
1651
+ 0x17B4,
1652
+ 0x17B5,
1653
+ 0x180B,
1654
+ 0x180C,
1655
+ 0x180D,
1656
+ 0x180F,
1657
+ ),
1658
+ )
1659
+ )
1186
1660
  # Cn follows the Unicode table of the Python that runs: a character newer than that table reads as unassigned, so
1187
1661
  # a cache made only of such characters is summarized again every run (the page is the same).
1188
1662
  _BLANK_CATEGORIES = frozenset(("Cc", "Cf", "Co", "Cn", "Zs", "Zl", "Zp"))
@@ -1191,7 +1665,11 @@ _BLANK_CATEGORIES = frozenset(("Cc", "Cf", "Co", "Cn", "Zs", "Zl", "Zp"))
1191
1665
  def _blank(ch):
1192
1666
  """Whether ch prints as nothing: whitespace, a blank-looking letter, or a variation selector.
1193
1667
  White space is in the blank categories too (Zs, Zl, Zp, Cc); str.isspace() only answers it sooner."""
1194
- if ch.isspace() or ch in _BLANK_LETTERS or unicodedata.category(ch) in _BLANK_CATEGORIES:
1668
+ if (
1669
+ ch.isspace()
1670
+ or ch in _BLANK_LETTERS
1671
+ or unicodedata.category(ch) in _BLANK_CATEGORIES
1672
+ ):
1195
1673
  return True
1196
1674
  o = ord(ch)
1197
1675
  return 0xFE00 <= o <= 0xFE0F or 0xE0100 <= o <= 0xE01EF
@@ -1202,7 +1680,9 @@ def _has_visible(text):
1202
1680
  return not all(_blank(ch) for ch in text)
1203
1681
 
1204
1682
 
1205
- CACHE_READ_MAX = 64 * 1024 # bytes: a cached summary or decision is some hundred (120 characters), so this is generous
1683
+ CACHE_READ_MAX = (
1684
+ 64 * 1024
1685
+ ) # bytes: a cached summary or decision is some hundred (120 characters), so this is generous
1206
1686
 
1207
1687
 
1208
1688
  def _read_small(path, limit=CACHE_READ_MAX):
@@ -1210,10 +1690,16 @@ def _read_small(path, limit=CACHE_READ_MAX):
1210
1690
  folder, a larger file). Opened without blocking and without following a link, so a pipe or a link left at a
1211
1691
  cache place neither stops a run nor sends it to another file. Windows has neither flag: there the lstat
1212
1692
  turns a link away, but a link swapped in between the lstat and the open is followed."""
1213
- flags = (os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0)
1214
- | getattr(os, "O_BINARY", 0))
1693
+ flags = (
1694
+ os.O_RDONLY
1695
+ | getattr(os, "O_NONBLOCK", 0)
1696
+ | getattr(os, "O_NOFOLLOW", 0)
1697
+ | getattr(os, "O_BINARY", 0)
1698
+ )
1215
1699
  try:
1216
- if not stat.S_ISREG(os.lstat(str(path)).st_mode): # a link (Windows has no O_NOFOLLOW), pipe, folder
1700
+ if not stat.S_ISREG(
1701
+ os.lstat(str(path)).st_mode
1702
+ ): # a link (Windows has no O_NOFOLLOW), pipe, folder
1217
1703
  return None
1218
1704
  fd = os.open(str(path), flags)
1219
1705
  with os.fdopen(fd, "rb") as f:
@@ -1247,8 +1733,13 @@ def _read_cached(p):
1247
1733
  return None
1248
1734
 
1249
1735
 
1250
- def read_or_summarize(turn: dict, session_id: str, rebuild: bool = False,
1251
- redact_extra=None, redact_mode: str = "full"):
1736
+ def read_or_summarize(
1737
+ turn: dict,
1738
+ session_id: str,
1739
+ rebuild: bool = False,
1740
+ redact_extra=None,
1741
+ redact_mode: str = "full",
1742
+ ):
1252
1743
  """Return (summary, cache_hit). Cache key includes the content hash AND the
1253
1744
  summarizer version (B-3), so changing the summarizer invalidates old entries.
1254
1745
  Cached text is always redacted (secret hygiene). It is cut from the redacted text (3.0.2).
@@ -1256,10 +1747,12 @@ def read_or_summarize(turn: dict, session_id: str, rebuild: bool = False,
1256
1747
  digest = hashlib.sha256(
1257
1748
  (turn["text"] + "\x00s" + str(SUMMARIZER_VERSION)).encode()
1258
1749
  ).hexdigest()[:8]
1259
- try: # ids come from the transcript: keep them in the cache
1260
- p = (cache_dir(_safe_name(turn.get("session") or session_id))
1261
- / f"{_safe_name(turn['uuid'])}-{digest}.md")
1262
- except OSError as e: # no writable cache: summarize anyway
1750
+ try: # ids come from the transcript: keep them in the cache
1751
+ p = (
1752
+ cache_dir(_safe_name(turn.get("session") or session_id))
1753
+ / f"{_safe_name(turn['uuid'])}-{digest}.md"
1754
+ )
1755
+ except OSError as e: # no writable cache: summarize anyway
1263
1756
  _warn_once("_no_cache", f"cache unavailable ({e}); summaries are not cached")
1264
1757
  p = None
1265
1758
  cached = None if p is None or rebuild else _read_cached(p)
@@ -1288,7 +1781,8 @@ _SENTINEL = "\x00%d\x00"
1288
1781
  _SENTINEL_RE = re.compile(r"\x00(\d+)\x00")
1289
1782
  _MD_CARVE = re.compile(
1290
1783
  r"`(?P<code>[^`\n]+)`"
1291
- r"|\[(?P<label>[^\]\n]+)\]\((?P<url>https?://[^\s)]+)\)")
1784
+ r"|\[(?P<label>[^\]\n]+)\]\((?P<url>https?://[^\s)]+)\)"
1785
+ )
1292
1786
  _MD_BOLD = re.compile(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", re.S)
1293
1787
  _MD_DEL = re.compile(r"~~(?=\S)([^<]+?)(?<=\S)~~")
1294
1788
  _MD_ITAL = re.compile(r"(?<![*\w])\*(?=\S)([^*<\n]+?)(?<=\S)\*(?![*\w])")
@@ -1311,7 +1805,8 @@ def _md_inline(escaped):
1311
1805
  else:
1312
1806
  stash.append(
1313
1807
  '<a href="%s" target="_blank" rel="noopener noreferrer">%s</a>'
1314
- % (m.group("url"), m.group("label")))
1808
+ % (m.group("url"), m.group("label"))
1809
+ )
1315
1810
  return _SENTINEL % (len(stash) - 1)
1316
1811
 
1317
1812
  s = _MD_CARVE.sub(carve, escaped)
@@ -1323,16 +1818,25 @@ def _md_inline(escaped):
1323
1818
 
1324
1819
  def _is_block_start(lines, i):
1325
1820
  line = lines[i]
1326
- if (_FENCE_RE.match(line) or _HEAD_RE.match(line) or _HR_RE.match(line)
1327
- or _QUOTE_RE.match(line) or _LIST_RE.match(line)):
1821
+ if (
1822
+ _FENCE_RE.match(line)
1823
+ or _HEAD_RE.match(line)
1824
+ or _HR_RE.match(line)
1825
+ or _QUOTE_RE.match(line)
1826
+ or _LIST_RE.match(line)
1827
+ ):
1328
1828
  return True
1329
1829
  return _is_table_start(lines, i)
1330
1830
 
1331
1831
 
1332
1832
  def _is_table_start(lines, i):
1333
1833
  """Whether lines[i] starts a table: it has a pipe AND the next line is a delimiter row."""
1334
- return bool("|" in lines[i] and i + 1 < len(lines)
1335
- and "-" in lines[i + 1] and _TABLE_DELIM_RE.match(lines[i + 1]))
1834
+ return bool(
1835
+ "|" in lines[i]
1836
+ and i + 1 < len(lines)
1837
+ and "-" in lines[i + 1]
1838
+ and _TABLE_DELIM_RE.match(lines[i + 1])
1839
+ )
1336
1840
 
1337
1841
 
1338
1842
  def _render_table(lines, i):
@@ -1344,11 +1848,12 @@ def _render_table(lines, i):
1344
1848
  i += 1
1345
1849
  thead = "".join("<th>%s</th>" % _md_inline(html.escape(c)) for c in header)
1346
1850
  body = "".join(
1347
- "<tr>%s</tr>" % "".join("<td>%s</td>" % _md_inline(html.escape(c))
1348
- for c in r)
1349
- for r in rows)
1350
- return ("<table><thead><tr>%s</tr></thead><tbody>%s</tbody></table>"
1351
- % (thead, body)), i
1851
+ "<tr>%s</tr>" % "".join("<td>%s</td>" % _md_inline(html.escape(c)) for c in r)
1852
+ for r in rows
1853
+ )
1854
+ return (
1855
+ "<table><thead><tr>%s</tr></thead><tbody>%s</tbody></table>" % (thead, body)
1856
+ ), i
1352
1857
 
1353
1858
 
1354
1859
  def _md_fence(lines, i, fence):
@@ -1408,8 +1913,12 @@ def _md_block(lines, i, depth):
1408
1913
  return _md_fence(lines, i, mf.group(1))
1409
1914
  mh = _HEAD_RE.match(line)
1410
1915
  if mh:
1411
- hl = max(3, min(6, len(mh.group(1)))) # h3..h6 (avoid page-header clash)
1412
- return "<h%d>%s</h%d>" % (hl, _md_inline(html.escape(mh.group(2).strip())), hl), i + 1
1916
+ hl = max(3, min(6, len(mh.group(1)))) # h3..h6 (avoid page-header clash)
1917
+ return "<h%d>%s</h%d>" % (
1918
+ hl,
1919
+ _md_inline(html.escape(mh.group(2).strip())),
1920
+ hl,
1921
+ ), i + 1
1413
1922
  if _HR_RE.match(line):
1414
1923
  return "<hr>", i + 1
1415
1924
  if _QUOTE_RE.match(line):
@@ -1634,8 +2143,13 @@ def _page_summary(t, summaries, session_id, rebuild, redact_extra, redact_mode):
1634
2143
  rules' (cached)."""
1635
2144
  if summaries is not None and t.get("uuid") in summaries:
1636
2145
  return summaries[t["uuid"]], None
1637
- return read_or_summarize(t, session_id, rebuild=rebuild,
1638
- redact_extra=redact_extra, redact_mode=redact_mode)
2146
+ return read_or_summarize(
2147
+ t,
2148
+ session_id,
2149
+ rebuild=rebuild,
2150
+ redact_extra=redact_extra,
2151
+ redact_mode=redact_mode,
2152
+ )
1639
2153
 
1640
2154
 
1641
2155
  def _reviewed_fold(t, summaries):
@@ -1670,18 +2184,23 @@ def _pills(rs, t, all_sessions):
1670
2184
  turn `t`."""
1671
2185
  if all_sessions and t.get("session") != rs.last_session:
1672
2186
  rs.last_session = t.get("session")
1673
- name = html.escape(rs.red(t.get("session_name") or rs.last_session or "session"))
1674
- rs.rows.append('<div class="day" data-mark="%s">'
1675
- '<span class="pill pill-session">%s</span></div>'
1676
- % (name, name))
2187
+ name = html.escape(
2188
+ rs.red(t.get("session_name") or rs.last_session or "session")
2189
+ )
2190
+ rs.rows.append(
2191
+ '<div class="day" data-mark="%s">'
2192
+ '<span class="pill pill-session">%s</span></div>' % (name, name)
2193
+ )
1677
2194
  rs.last_date = None
1678
2195
  ts = t.get("ts") or ""
1679
2196
  if ts:
1680
2197
  d = ts[:10]
1681
2198
  if d != rs.last_date:
1682
- rs.rows.append('<div class="day" data-mark="%s">'
1683
- '<span class="pill pill-date">%s</span></div>'
1684
- % (html.escape(d), html.escape(d)))
2199
+ rs.rows.append(
2200
+ '<div class="day" data-mark="%s">'
2201
+ '<span class="pill pill-date">%s</span></div>'
2202
+ % (html.escape(d), html.escape(d))
2203
+ )
1685
2204
  rs.last_date = d
1686
2205
 
1687
2206
 
@@ -1692,24 +2211,31 @@ def _user_row(rs, t, summaries, time_hm):
1692
2211
  reviewed = _reviewed_fold(t, summaries)
1693
2212
  summary = html.escape(rs.red(reviewed) if reviewed else user_fold_summary(red))
1694
2213
  body = render_body(red, markdown=rs.markdown)
1695
- return ('<div class="row me"><span class="time">%s</span>'
1696
- '<div class="bubble"><details%s>'
1697
- '<summary><span class="sum">%s</span></summary>'
1698
- '<div class="detail">%s</div></details></div></div>'
1699
- % (time_hm, rs.open_attr, summary, body))
1700
- return ('<div class="row me"><span class="time">%s</span>'
1701
- '<div class="bubble"><div class="plain">%s</div></div></div>'
1702
- % (time_hm, render_body(red, markdown=rs.markdown)))
2214
+ return (
2215
+ '<div class="row me"><span class="time">%s</span>'
2216
+ '<div class="bubble"><details%s>'
2217
+ '<summary><span class="sum">%s</span></summary>'
2218
+ '<div class="detail">%s</div></details></div></div>'
2219
+ % (time_hm, rs.open_attr, summary, body)
2220
+ )
2221
+ return (
2222
+ '<div class="row me"><span class="time">%s</span>'
2223
+ '<div class="bubble"><div class="plain">%s</div></div></div>'
2224
+ % (time_hm, render_body(red, markdown=rs.markdown))
2225
+ )
1703
2226
 
1704
2227
 
1705
2228
  def _tools_line(tools, redact_extra):
1706
2229
  """The `🔧 도구 N건` line of a bot bubble ("" without tools), the --redact-extra keywords hidden in the names."""
1707
2230
  if not tools:
1708
2231
  return ""
1709
- ts_list = ", ".join(f"{_hide_keywords(k, redact_extra)}×{v}"
1710
- for k, v in sorted(tools.items()))
1711
- return ('<div class="tools">🔧 도구 %d건: %s</div>'
1712
- % (sum(tools.values()), html.escape(ts_list)))
2232
+ ts_list = ", ".join(
2233
+ f"{_hide_keywords(k, redact_extra)}×{v}" for k, v in sorted(tools.items())
2234
+ )
2235
+ return '<div class="tools">🔧 도구 %d건: %s</div>' % (
2236
+ sum(tools.values()),
2237
+ html.escape(ts_list),
2238
+ )
1713
2239
 
1714
2240
 
1715
2241
  def _bot_row(rs, t, summary, time_hm):
@@ -1720,32 +2246,59 @@ def _bot_row(rs, t, summary, time_hm):
1720
2246
  role = t["role"]
1721
2247
  if role == "agent":
1722
2248
  who = html.escape(rs.red(t.get("agent_from") or "agent"))
1723
- avatar, from_line, row_cls = "🤝", '<div class="from">%s</div>' % who, "row bot agent"
2249
+ avatar, from_line, row_cls = (
2250
+ "🤝",
2251
+ '<div class="from">%s</div>' % who,
2252
+ "row bot agent",
2253
+ )
1724
2254
  else:
1725
2255
  avatar, from_line, row_cls = "C", "", "row bot"
1726
- return ('<div class="%s"><div class="avatar%s">%s</div>'
1727
- '<div class="bubble"><details%s>'
1728
- '<summary><span class="sum">%s</span></summary>'
1729
- '<div class="detail">%s%s%s</div></details></div>'
1730
- '<span class="time">%s</span></div>'
1731
- % (row_cls, " agent" if role == "agent" else "", avatar,
1732
- rs.open_attr, esc_sum, from_line, detail_body, tools_html, time_hm))
1733
-
1734
-
1735
- def render_rows(turns, session_id, redact_extra, redact_mode, rebuild,
1736
- open_details=False, markdown=True, all_sessions=False, summaries=None):
2256
+ return (
2257
+ '<div class="%s"><div class="avatar%s">%s</div>'
2258
+ '<div class="bubble"><details%s>'
2259
+ '<summary><span class="sum">%s</span></summary>'
2260
+ '<div class="detail">%s%s%s</div></details></div>'
2261
+ '<span class="time">%s</span></div>'
2262
+ % (
2263
+ row_cls,
2264
+ " agent" if role == "agent" else "",
2265
+ avatar,
2266
+ rs.open_attr,
2267
+ esc_sum,
2268
+ from_line,
2269
+ detail_body,
2270
+ tools_html,
2271
+ time_hm,
2272
+ )
2273
+ )
2274
+
2275
+
2276
+ def render_rows(
2277
+ turns,
2278
+ session_id,
2279
+ redact_extra,
2280
+ redact_mode,
2281
+ rebuild,
2282
+ open_details=False,
2283
+ markdown=True,
2284
+ all_sessions=False,
2285
+ summaries=None,
2286
+ ):
1737
2287
  rs = _Rows(redact_extra, redact_mode, open_details, markdown)
1738
2288
  for t in turns:
1739
2289
  _pills(rs, t, all_sessions)
1740
2290
  time_hm = html.escape(_fmt_time(t.get("ts") or ""))
1741
2291
  if t["role"] == "mark":
1742
- rs.rows.append('<div class="day"><span class="pill pill-compact">'
1743
- '⋯ 이전 대화 요약(compaction) ⋯</span></div>')
2292
+ rs.rows.append(
2293
+ '<div class="day"><span class="pill pill-compact">'
2294
+ "⋯ 이전 대화 요약(compaction) ⋯</span></div>"
2295
+ )
1744
2296
  elif t["role"] == "user":
1745
2297
  rs.rows.append(_user_row(rs, t, summaries, time_hm))
1746
2298
  else:
1747
- summary, hit = _page_summary(t, summaries, session_id, rebuild,
1748
- redact_extra, redact_mode)
2299
+ summary, hit = _page_summary(
2300
+ t, summaries, session_id, rebuild, redact_extra, redact_mode
2301
+ )
1749
2302
  if hit is not None:
1750
2303
  rs.total += 1
1751
2304
  rs.hits += bool(hit)
@@ -1770,10 +2323,34 @@ def _residual_secrets(rows_html):
1770
2323
 
1771
2324
  # ---------------------------------------------------------------- Self-verify (F-1)
1772
2325
  _VOID = frozenset(
1773
- "area base br col embed hr img input link meta param source track wbr".split())
1774
- _VERIFY_TAGS = ("details", "div", "p", "ul", "ol", "li", "table", "thead",
1775
- "tbody", "tr", "th", "td", "blockquote", "pre", "code",
1776
- "strong", "em", "del", "a", "summary", "h3", "h4", "h5", "h6")
2326
+ "area base br col embed hr img input link meta param source track wbr".split()
2327
+ )
2328
+ _VERIFY_TAGS = (
2329
+ "details",
2330
+ "div",
2331
+ "p",
2332
+ "ul",
2333
+ "ol",
2334
+ "li",
2335
+ "table",
2336
+ "thead",
2337
+ "tbody",
2338
+ "tr",
2339
+ "th",
2340
+ "td",
2341
+ "blockquote",
2342
+ "pre",
2343
+ "code",
2344
+ "strong",
2345
+ "em",
2346
+ "del",
2347
+ "a",
2348
+ "summary",
2349
+ "h3",
2350
+ "h4",
2351
+ "h5",
2352
+ "h6",
2353
+ )
1777
2354
 
1778
2355
 
1779
2356
  class _BalanceParser(HTMLParser):
@@ -1818,8 +2395,12 @@ def self_verify(html_text: str):
1818
2395
  errs = []
1819
2396
  # strip script/style so their bodies aren't parsed as markup (avoids
1820
2397
  # false <a> counts from inline JS like `top<a-8`)
1821
- markup = re.sub(r"<script\b[\s\S]*?</script>|<style\b[\s\S]*?</style>",
1822
- "", html_text, flags=re.I)
2398
+ markup = re.sub(
2399
+ r"<script\b[\s\S]*?</script>|<style\b[\s\S]*?</style>",
2400
+ "",
2401
+ html_text,
2402
+ flags=re.I,
2403
+ )
1823
2404
  try:
1824
2405
  HTMLParser().feed(html_text)
1825
2406
  except Exception as e:
@@ -1843,9 +2424,11 @@ def self_verify(html_text: str):
1843
2424
  errs.append("external script src found")
1844
2425
  # Event-handler attributes are detected via the parser (real tag attrs),
1845
2426
  # NOT a text regex — escaped content like `&lt;img onerror=` is inert text.
1846
- for marker, ph in (("<title>{{TITLE}}</title>", "{{TITLE}}"),
1847
- ("<h1>{{HEADER_TITLE}}</h1>", "{{HEADER_TITLE}}"),
1848
- ('id="now">{{DATE_RANGE}}</div>', "{{DATE_RANGE}}")):
2427
+ for marker, ph in (
2428
+ ("<title>{{TITLE}}</title>", "{{TITLE}}"),
2429
+ ("<h1>{{HEADER_TITLE}}</h1>", "{{HEADER_TITLE}}"),
2430
+ ('id="now">{{DATE_RANGE}}</div>', "{{DATE_RANGE}}"),
2431
+ ):
1849
2432
  if marker in html_text:
1850
2433
  errs.append(f"unsubstituted template placeholder: {ph}")
1851
2434
  return errs
@@ -1884,10 +2467,14 @@ def _write_private(path, text):
1884
2467
  is never followed, and it is removed when the write fails."""
1885
2468
  path = pathlib.Path(path)
1886
2469
  path.parent.mkdir(parents=True, exist_ok=True)
1887
- fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")
2470
+ fd, tmp = tempfile.mkstemp(
2471
+ dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp"
2472
+ )
1888
2473
  try:
1889
2474
  with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f:
1890
- f.write(_lone_to_fffd(text)) # a title or a file name may still bring one: UTF-8 cannot hold it
2475
+ f.write(
2476
+ _lone_to_fffd(text)
2477
+ ) # a title or a file name may still bring one: UTF-8 cannot hold it
1891
2478
  os.replace(tmp, str(path))
1892
2479
  except BaseException:
1893
2480
  try:
@@ -1901,7 +2488,9 @@ def _write_text_lf(path, text):
1901
2488
  """Write `text` to `path` as UTF-8 with LF line ends on every OS. Text mode on Windows
1902
2489
  would write CRLF, so the same transcript would give a different page there."""
1903
2490
  with open(str(path), "w", encoding="utf-8", newline="\n") as f:
1904
- f.write(_lone_to_fffd(text)) # a title or a file name may still bring one: UTF-8 cannot hold it
2491
+ f.write(
2492
+ _lone_to_fffd(text)
2493
+ ) # a title or a file name may still bring one: UTF-8 cannot hold it
1905
2494
 
1906
2495
 
1907
2496
  def _safe_name(s):
@@ -1913,19 +2502,28 @@ def _safe_name(s):
1913
2502
  def _decision_context(why, keeps):
1914
2503
  """What a cached decision rests on beside the text: the rules' call on the turn (a
1915
2504
  `keep: null` defers to it) and the keep flags the reviewer was told to honour."""
1916
- return "|".join([why or "", str(keeps.get("keep_trivia") is True),
1917
- str(keeps.get("keep_tool_only") is True)])
2505
+ return "|".join(
2506
+ [
2507
+ why or "",
2508
+ str(keeps.get("keep_trivia") is True),
2509
+ str(keeps.get("keep_tool_only") is True),
2510
+ ]
2511
+ )
1918
2512
 
1919
2513
 
1920
2514
  def _llm_digest(redacted_text, context):
1921
2515
  """The decision cache key: the text with the page's patterns only (as 3.0.1 keyed it, so
1922
2516
  no turn is reviewed again for 3.0.2), the cache version and what the decision rests on."""
1923
- return hashlib.sha256((redacted_text + "\x00llm" + str(LLM_CACHE_VERSION)
1924
- + "\x00" + context).encode()).hexdigest()[:12]
2517
+ return hashlib.sha256(
2518
+ (redacted_text + "\x00llm" + str(LLM_CACHE_VERSION) + "\x00" + context).encode()
2519
+ ).hexdigest()[:12]
1925
2520
 
1926
2521
 
1927
2522
  def _llm_cache_path(turn_id, session, digest):
1928
- return cache_dir(_safe_name(session)) / f"{_safe_name(turn_id)}-{_safe_name(digest)}-llm.json"
2523
+ return (
2524
+ cache_dir(_safe_name(session))
2525
+ / f"{_safe_name(turn_id)}-{_safe_name(digest)}-llm.json"
2526
+ )
1929
2527
 
1930
2528
 
1931
2529
  def read_llm_cache(turn_id, session, digest):
@@ -1934,9 +2532,11 @@ def read_llm_cache(turn_id, session, digest):
1934
2532
  null: the reviewer left the turn to the rules."""
1935
2533
  try:
1936
2534
  path = _llm_cache_path(turn_id, session, digest)
1937
- except OSError as e: # no cache folder (a read-only HOME, say)
1938
- _warn_once("_no_llm_cache", f"reviewer decision cache unavailable ({e}); "
1939
- "every turn is reviewed")
2535
+ except OSError as e: # no cache folder (a read-only HOME, say)
2536
+ _warn_once(
2537
+ "_no_llm_cache",
2538
+ f"reviewer decision cache unavailable ({e}); every turn is reviewed",
2539
+ )
1940
2540
  return None
1941
2541
  try:
1942
2542
  got = json.loads(_read_text(path) or "")
@@ -1952,11 +2552,16 @@ def read_llm_cache(turn_id, session, digest):
1952
2552
 
1953
2553
  def write_llm_cache(turn_id, session, digest, keep, summary):
1954
2554
  try:
1955
- _write_private(_llm_cache_path(turn_id, session, digest),
1956
- json.dumps({"keep": keep, "summary": summary}, ensure_ascii=False))
2555
+ _write_private(
2556
+ _llm_cache_path(turn_id, session, digest),
2557
+ json.dumps({"keep": keep, "summary": summary}, ensure_ascii=False),
2558
+ )
1957
2559
  except OSError as e:
1958
- _warn_once("_no_llm_cache_write", f"reviewer decisions not cached ({e}); the next "
1959
- "run reviews this run's turns again (decisions cached before still apply)")
2560
+ _warn_once(
2561
+ "_no_llm_cache_write",
2562
+ f"reviewer decisions not cached ({e}); the next "
2563
+ "run reviews this run's turns again (decisions cached before still apply)",
2564
+ )
1960
2565
 
1961
2566
 
1962
2567
  def heuristic_drops(turns, args):
@@ -1995,8 +2600,11 @@ def _preview(text):
1995
2600
  if len(text) <= PREVIEW_HEAD + PREVIEW_TAIL + 200:
1996
2601
  return text, False
1997
2602
  cut = len(text) - PREVIEW_HEAD - PREVIEW_TAIL
1998
- return (text[:PREVIEW_HEAD].rstrip() + f"\n\n[중간 {cut}자 생략]\n\n"
1999
- + text[-PREVIEW_TAIL:].lstrip()), True
2603
+ return (
2604
+ text[:PREVIEW_HEAD].rstrip()
2605
+ + f"\n\n[중간 {cut}자 생략]\n\n"
2606
+ + text[-PREVIEW_TAIL:].lstrip()
2607
+ ), True
2000
2608
 
2001
2609
 
2002
2610
  def _public_counts(counts):
@@ -2023,22 +2631,31 @@ def _rule_summary(t, red, args):
2023
2631
  def _redacted_turn(t, extra):
2024
2632
  """`t` with its text and parts fully redacted, so a summary cut from it cannot cut a
2025
2633
  secret in two and keep the half no pattern recognises."""
2634
+
2026
2635
  def clean(s):
2027
2636
  return review_redact(s, extra)[0]
2028
- return dict(t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []])
2637
+
2638
+ return dict(
2639
+ t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []]
2640
+ )
2029
2641
 
2030
2642
 
2031
2643
  def _review_view(t, red, args):
2032
2644
  """What a reviewer reads beside the preview: the rules' summary and the agent's
2033
2645
  name, fully redacted (the page's copies may be masked)."""
2034
2646
  if t["role"] in ("assistant", "agent"):
2035
- summary = review_redact(summarize_turn(_redacted_turn(t, args.redact_extra)),
2036
- args.redact_extra)[0]
2647
+ summary = review_redact(
2648
+ summarize_turn(_redacted_turn(t, args.redact_extra)), args.redact_extra
2649
+ )[0]
2037
2650
  elif t["role"] == "user" and len(red) > USER_FOLD:
2038
2651
  summary = user_fold_summary(review_redact(t["text"], args.redact_extra)[0])
2039
2652
  else:
2040
2653
  summary = None
2041
- who = review_redact(t["agent_from"], args.redact_extra)[0] if t.get("agent_from") else None
2654
+ who = (
2655
+ review_redact(t["agent_from"], args.redact_extra)[0]
2656
+ if t.get("agent_from")
2657
+ else None
2658
+ )
2042
2659
  return {"summary": summary, "agent_from": who}
2043
2660
 
2044
2661
 
@@ -2052,28 +2669,53 @@ def _pack_turn(n, t, tid, why, session_id, args, counts):
2052
2669
  for k, v in found.items():
2053
2670
  counts[k] = counts.get(k, 0) + v
2054
2671
  hit = any(name in found for name, _ in REVIEW_SECRET_PATTERNS)
2055
- key_text = (redact(_rejoin_secrets(t["text"], args.redact_extra), extra=args.redact_extra,
2056
- mode=args.redact_mode)[0] if hit else red)
2672
+ key_text = (
2673
+ redact(
2674
+ _rejoin_secrets(t["text"], args.redact_extra),
2675
+ extra=args.redact_extra,
2676
+ mode=args.redact_mode,
2677
+ )[0]
2678
+ if hit
2679
+ else red
2680
+ )
2057
2681
  preview, clipped = _preview(review_redact(t["text"], args.redact_extra)[0])
2058
2682
  session = t.get("session") or session_id
2059
2683
  context = _decision_context(why, _keeps(args))
2060
2684
  llm_key = _llm_digest(key_text, context)
2061
2685
  cached = None if args.rebuild_summaries else read_llm_cache(tid, session, llm_key)
2062
2686
  view = _review_view(t, red, args)
2063
- turn = {"id": tid, "n": n, "role": t["role"], "ts": t.get("ts") or "",
2064
- "tools": dict(t.get("tools") or {}), "text": red, "llm_key": llm_key,
2065
- "redactions": _public_counts(found), "preview": preview, "clipped": clipped,
2066
- "rule": {"keep": why is None, "why": why,
2067
- "summary": _rule_summary(t, red, args),
2068
- "review_summary": view["summary"]},
2069
- "llm": dict(cached, cached=True) if cached else {"keep": None, "summary": None}}
2687
+ turn = {
2688
+ "id": tid,
2689
+ "n": n,
2690
+ "role": t["role"],
2691
+ "ts": t.get("ts") or "",
2692
+ "tools": dict(t.get("tools") or {}),
2693
+ "text": red,
2694
+ "llm_key": llm_key,
2695
+ "redactions": _public_counts(found),
2696
+ "preview": preview,
2697
+ "clipped": clipped,
2698
+ "rule": {
2699
+ "keep": why is None,
2700
+ "why": why,
2701
+ "summary": _rule_summary(t, red, args),
2702
+ "review_summary": view["summary"],
2703
+ },
2704
+ "llm": dict(cached, cached=True) if cached else {"keep": None, "summary": None},
2705
+ }
2070
2706
  if t.get("meta"):
2071
2707
  turn["meta"] = True
2072
2708
  if t["role"] == "user" and _bang_output(t["text"]):
2073
- turn["bang_output"] = True # for the dropped-turn WARN only: no part file carries it
2709
+ turn["bang_output"] = (
2710
+ True # for the dropped-turn WARN only: no part file carries it
2711
+ )
2074
2712
  for k in ("session", "session_name", "agent_from"):
2075
2713
  if t.get(k):
2076
- turn[k] = t[k] if k == "session" else page_redact(t[k], args.redact_extra, args.redact_mode)[0]
2714
+ turn[k] = (
2715
+ t[k]
2716
+ if k == "session"
2717
+ else page_redact(t[k], args.redact_extra, args.redact_mode)[0]
2718
+ )
2077
2719
  return turn, view
2078
2720
 
2079
2721
 
@@ -2094,7 +2736,7 @@ def _split_even(items, size):
2094
2736
  if not items:
2095
2737
  return []
2096
2738
  step = -(-len(items) // -(-len(items) // size))
2097
- return [items[k:k + step] for k in range(0, len(items), step)]
2739
+ return [items[k : k + step] for k in range(0, len(items), step)]
2098
2740
 
2099
2741
 
2100
2742
  def _decisions_name(part_path):
@@ -2109,8 +2751,12 @@ def _clear_parts(pack_path):
2109
2751
  prefix = pack_path.stem + ".part-"
2110
2752
  samples = f"{pack_path.stem}.reviewer-input.json"
2111
2753
  try:
2112
- olds = [p for p in pack_path.parent.iterdir()
2113
- if (p.name.startswith(prefix) and p.name.endswith(".json")) or p.name == samples]
2754
+ olds = [
2755
+ p
2756
+ for p in pack_path.parent.iterdir()
2757
+ if (p.name.startswith(prefix) and p.name.endswith(".json"))
2758
+ or p.name == samples
2759
+ ]
2114
2760
  except OSError:
2115
2761
  return []
2116
2762
  failed = []
@@ -2131,73 +2777,129 @@ def _hide_keywords(s, extra):
2131
2777
 
2132
2778
  def _keeps(args):
2133
2779
  """The keep flags the reviewers must honour: with them, the rules keep those turns."""
2134
- return {"keep_trivia": not args.drop_trivia, "keep_tool_only": not args.hide_tool_only}
2780
+ return {
2781
+ "keep_trivia": not args.drop_trivia,
2782
+ "keep_tool_only": not args.hide_tool_only,
2783
+ }
2135
2784
 
2136
2785
 
2137
2786
  def _sheet(k, of, part_path, group, args):
2138
2787
  """Part file `k`: the turns one reviewer reads, without the page-only fields."""
2139
2788
  turns = []
2140
2789
  for x, view in group:
2141
- y = {key: v for key, v in x.items()
2142
- if key not in ("text", "session", "session_name", "agent_from", "redactions", "llm_key",
2143
- "bang_output")}
2790
+ y = {
2791
+ key: v
2792
+ for key, v in x.items()
2793
+ if key
2794
+ not in (
2795
+ "text",
2796
+ "session",
2797
+ "session_name",
2798
+ "agent_from",
2799
+ "redactions",
2800
+ "llm_key",
2801
+ "bang_output",
2802
+ )
2803
+ }
2144
2804
  y["rule"] = {k: v for k, v in x["rule"].items() if k != "review_summary"}
2145
2805
  y["rule"]["summary"] = view["summary"]
2146
- y["tools"] = {_hide_keywords(name, args.redact_extra): n for name, n in x["tools"].items()}
2147
- if x["llm"].get("summary"): # cached: redacted for the page, maybe masked
2806
+ y["tools"] = {
2807
+ _hide_keywords(name, args.redact_extra): n for name, n in x["tools"].items()
2808
+ }
2809
+ if x["llm"].get("summary"): # cached: redacted for the page, maybe masked
2148
2810
  s = _MASKED.sub("[REDACTED]", x["llm"]["summary"])
2149
2811
  y["llm"] = dict(x["llm"], summary=review_redact(s, args.redact_extra)[0])
2150
2812
  if view["agent_from"]:
2151
2813
  y["agent_from"] = view["agent_from"]
2152
2814
  turns.append(y)
2153
- return {"schema": REVIEW_PART_SCHEMA, "part": k, "of": of, **_keeps(args),
2154
- "decisions": _decisions_name(part_path), "turns": turns}
2815
+ return {
2816
+ "schema": REVIEW_PART_SCHEMA,
2817
+ "part": k,
2818
+ "of": of,
2819
+ **_keeps(args),
2820
+ "decisions": _decisions_name(part_path),
2821
+ "turns": turns,
2822
+ }
2155
2823
 
2156
2824
 
2157
2825
  def _pack_header(session_id, output, args, counts):
2158
2826
  """The pack's settings: what --apply-review renders with and the quality gate it runs."""
2159
- return {"schema": REVIEW_SCHEMA, "source": session_id, "output": str(output),
2160
- "title": args.title, "markdown": args.markdown, "open": args.open_details,
2161
- "redact_mode": args.redact_mode, "redact_extra": bool(args.redact_extra),
2162
- "all_sessions": bool(args.all_sessions), "redactions": _public_counts(counts),
2163
- **_keeps(args),
2164
- "gate": {"skip_reviewer": bool(args.skip_reviewer),
2165
- "reviewer_output": args.reviewer_output,
2166
- "reviewer_timeout": args.reviewer_timeout}}
2827
+ return {
2828
+ "schema": REVIEW_SCHEMA,
2829
+ "source": session_id,
2830
+ "output": str(output),
2831
+ "title": args.title,
2832
+ "markdown": args.markdown,
2833
+ "open": args.open_details,
2834
+ "redact_mode": args.redact_mode,
2835
+ "redact_extra": bool(args.redact_extra),
2836
+ "all_sessions": bool(args.all_sessions),
2837
+ "redactions": _public_counts(counts),
2838
+ **_keeps(args),
2839
+ "gate": {
2840
+ "skip_reviewer": bool(args.skip_reviewer),
2841
+ "reviewer_output": args.reviewer_output,
2842
+ "reviewer_timeout": args.reviewer_timeout,
2843
+ },
2844
+ }
2167
2845
 
2168
2846
 
2169
2847
  def _report_emit(pack_path, items, paths, groups):
2170
2848
  dropped = sum(1 for x in items if not x["rule"]["keep"])
2171
2849
  cached = sum(1 for x in items if x["llm"].get("cached"))
2172
- print(f"[lineage] review pack: {pack_path} (turns={len(items)} "
2173
- f"rule-dropped={dropped} cached={cached})", file=sys.stderr)
2850
+ print(
2851
+ f"[lineage] review pack: {pack_path} (turns={len(items)} "
2852
+ f"rule-dropped={dropped} cached={cached})",
2853
+ file=sys.stderr,
2854
+ )
2174
2855
  for k, (p, g) in enumerate(zip(paths, groups), 1):
2175
2856
  todo = sum(1 for x, _ in g if not x["llm"].get("cached"))
2176
- print(f"[lineage] part {k}/{len(groups)}: {p} (to review: {todo}) "
2177
- f"-> {p.with_name(_decisions_name(p))}", file=sys.stderr)
2178
- print(f"[lineage] next: write each part's decisions, then "
2179
- f"--apply-review {pack_path}", file=sys.stderr)
2857
+ print(
2858
+ f"[lineage] part {k}/{len(groups)}: {p} (to review: {todo}) "
2859
+ f"-> {p.with_name(_decisions_name(p))}",
2860
+ file=sys.stderr,
2861
+ )
2862
+ print(
2863
+ f"[lineage] next: write each part's decisions, then --apply-review {pack_path}",
2864
+ file=sys.stderr,
2865
+ )
2180
2866
 
2181
2867
 
2182
2868
  def emit_review(turns, drops, window, session_id, output, args):
2183
2869
  """Write the pack and one part file per reviewer; print them and the next step. A
2184
2870
  --reviewer-output that holds something other than a verdict list stops it here, before
2185
2871
  any reviewer runs: the gate would refuse it only after them."""
2186
- rop = pathlib.Path(args.reviewer_output) if args.reviewer_output and not args.skip_reviewer else None
2872
+ rop = (
2873
+ pathlib.Path(args.reviewer_output)
2874
+ if args.reviewer_output and not args.skip_reviewer
2875
+ else None
2876
+ )
2187
2877
  if rop is not None and rop.exists() and not _is_verdict_file(rop):
2188
- _bad(f"{rop} is there and is not a verdict list; give a --reviewer-output that does not exist yet")
2878
+ _bad(
2879
+ f"{rop} is there and is not a verdict list; give a --reviewer-output that does not exist yet"
2880
+ )
2189
2881
  return 2
2190
2882
  pack_path = pathlib.Path(args.emit_review)
2191
2883
  counts, pairs = {}, []
2192
2884
  for n, (i, tid) in enumerate(zip(window, _final_ids(turns, window)), 1):
2193
- pairs.append(_pack_turn(n, turns[i], tid, drops.get(i), session_id, args, counts))
2885
+ pairs.append(
2886
+ _pack_turn(n, turns[i], tid, drops.get(i), session_id, args, counts)
2887
+ )
2194
2888
  groups = _split_even(pairs, REVIEW_PART)
2195
- paths = [pack_path.with_name(f"{pack_path.stem}.part-{k}.json")
2196
- for k in range(1, len(groups) + 1)]
2889
+ paths = [
2890
+ pack_path.with_name(f"{pack_path.stem}.part-{k}.json")
2891
+ for k in range(1, len(groups) + 1)
2892
+ ]
2197
2893
  pack = _pack_header(session_id, _with_timestamp_suffix(output), args, counts)
2198
- pack["parts"] = [{"part": k, "file": p.name, "decisions": _decisions_name(p),
2199
- "ids": [x["id"] for x, _ in g]}
2200
- for k, (p, g) in enumerate(zip(paths, groups), 1)]
2894
+ pack["parts"] = [
2895
+ {
2896
+ "part": k,
2897
+ "file": p.name,
2898
+ "decisions": _decisions_name(p),
2899
+ "ids": [x["id"] for x, _ in g],
2900
+ }
2901
+ for k, (p, g) in enumerate(zip(paths, groups), 1)
2902
+ ]
2201
2903
  pack["turns"] = [x for x, _ in pairs]
2202
2904
  stuck = _clear_parts(pack_path)
2203
2905
  if stuck:
@@ -2206,8 +2908,12 @@ def emit_review(turns, drops, window, session_id, output, args):
2206
2908
  try:
2207
2909
  _write_private(pack_path, json.dumps(pack, ensure_ascii=False, indent=1))
2208
2910
  for k, (p, g) in enumerate(zip(paths, groups), 1):
2209
- _write_private(p, json.dumps(_sheet(k, len(groups), p, g, args),
2210
- ensure_ascii=False, indent=1))
2911
+ _write_private(
2912
+ p,
2913
+ json.dumps(
2914
+ _sheet(k, len(groups), p, g, args), ensure_ascii=False, indent=1
2915
+ ),
2916
+ )
2211
2917
  except OSError as e:
2212
2918
  _bad(f"cannot write the review pack: {e}")
2213
2919
  return 2
@@ -2216,13 +2922,19 @@ def emit_review(turns, drops, window, session_id, output, args):
2216
2922
 
2217
2923
 
2218
2924
  def _is_pack_turn(x):
2219
- return (isinstance(x, dict) and isinstance(x.get("id"), str)
2220
- and x.get("role") in ("user", "assistant", "agent", "mark")
2221
- and isinstance(x.get("text"), str) and isinstance(x.get("llm_key"), str)
2222
- and isinstance(x.get("rule"), dict) and isinstance(x["rule"].get("keep"), bool)
2223
- and _is_summary(x["rule"].get("summary"))
2224
- and isinstance(x.get("llm"), dict) and _is_keep(x["llm"].get("keep"))
2225
- and _is_summary(x["llm"].get("summary")))
2925
+ return (
2926
+ isinstance(x, dict)
2927
+ and isinstance(x.get("id"), str)
2928
+ and x.get("role") in ("user", "assistant", "agent", "mark")
2929
+ and isinstance(x.get("text"), str)
2930
+ and isinstance(x.get("llm_key"), str)
2931
+ and isinstance(x.get("rule"), dict)
2932
+ and isinstance(x["rule"].get("keep"), bool)
2933
+ and _is_summary(x["rule"].get("summary"))
2934
+ and isinstance(x.get("llm"), dict)
2935
+ and _is_keep(x["llm"].get("keep"))
2936
+ and _is_summary(x["llm"].get("summary"))
2937
+ )
2226
2938
 
2227
2939
 
2228
2940
  def _load_pack(path):
@@ -2231,9 +2943,13 @@ def _load_pack(path):
2231
2943
  except (OSError, ValueError, UnicodeDecodeError, RecursionError) as e:
2232
2944
  return _bad(f"cannot read review pack {path}: {e}")
2233
2945
  if not isinstance(pack, dict) or pack.get("schema") != REVIEW_SCHEMA:
2234
- return _bad(f"{path} is not a {REVIEW_SCHEMA} pack (made by another lineage version? "
2235
- "run --emit-review again)")
2236
- if not isinstance(pack.get("turns"), list) or not all(map(_is_pack_turn, pack["turns"])):
2946
+ return _bad(
2947
+ f"{path} is not a {REVIEW_SCHEMA} pack (made by another lineage version? "
2948
+ "run --emit-review again)"
2949
+ )
2950
+ if not isinstance(pack.get("turns"), list) or not all(
2951
+ map(_is_pack_turn, pack["turns"])
2952
+ ):
2237
2953
  return _bad(f"{path}: malformed turns")
2238
2954
  return pack
2239
2955
 
@@ -2241,8 +2957,11 @@ def _load_pack(path):
2241
2957
  def _decision_files(pack_path, pack, explicit):
2242
2958
  """The parts' decisions files that exist beside the pack, then each --decisions.
2243
2959
  Those beside the pack are made 0600: the session wrote them with its own umask."""
2244
- beside = [pack_path.with_name(p["decisions"]) for p in pack.get("parts") or []
2245
- if isinstance(p, dict) and isinstance(p.get("decisions"), str)]
2960
+ beside = [
2961
+ pack_path.with_name(p["decisions"])
2962
+ for p in pack.get("parts") or []
2963
+ if isinstance(p, dict) and isinstance(p.get("decisions"), str)
2964
+ ]
2246
2965
  files, seen = [], set()
2247
2966
  for f in [f for f in beside if f.exists()] + [pathlib.Path(f) for f in explicit]:
2248
2967
  if os.path.realpath(str(f)) not in seen:
@@ -2258,15 +2977,31 @@ def _decision_files(pack_path, pack, explicit):
2258
2977
 
2259
2978
  # Keys of a part file's turns: a list of objects carrying one is the reviewer quoting its
2260
2979
  # input, not its answer.
2261
- _PART_TURN_KEYS = frozenset({"n", "role", "ts", "tools", "text", "preview", "clipped",
2262
- "rule", "llm", "meta", "agent_from"})
2980
+ _PART_TURN_KEYS = frozenset(
2981
+ {
2982
+ "n",
2983
+ "role",
2984
+ "ts",
2985
+ "tools",
2986
+ "text",
2987
+ "preview",
2988
+ "clipped",
2989
+ "rule",
2990
+ "llm",
2991
+ "meta",
2992
+ "agent_from",
2993
+ }
2994
+ )
2263
2995
 
2264
2996
 
2265
2997
  def _is_answer(d):
2266
2998
  """A decision, extra keys and all; a part turn quoted back (its keep and summary sit
2267
2999
  under rule and llm, not on top) is not one."""
2268
- return (isinstance(d, dict) and "id" in d
2269
- and ("keep" in d or "summary" in d or not _PART_TURN_KEYS & d.keys()))
3000
+ return (
3001
+ isinstance(d, dict)
3002
+ and "id" in d
3003
+ and ("keep" in d or "summary" in d or not _PART_TURN_KEYS & d.keys())
3004
+ )
2270
3005
 
2271
3006
 
2272
3007
  def _decisions_text(text):
@@ -2285,8 +3020,10 @@ def _decisions_text(text):
2285
3020
  # and only so many starts (each failed try parses up to the nesting limit)
2286
3021
  for k, m in enumerate(re.finditer(r"\[(?=\s*[{\]])", text)):
2287
3022
  if k == DECISIONS_TRIES:
2288
- raise ValueError(f"more than {DECISIONS_TRIES} places a list may start; "
2289
- "write the bare JSON array")
3023
+ raise ValueError(
3024
+ f"more than {DECISIONS_TRIES} places a list may start; "
3025
+ "write the bare JSON array"
3026
+ )
2290
3027
  try:
2291
3028
  got, _ = decoder.raw_decode(text, m.start())
2292
3029
  except (ValueError, RecursionError):
@@ -2304,8 +3041,12 @@ def _decisions_text(text):
2304
3041
 
2305
3042
  def _decision_problem(d):
2306
3043
  """Why item `d` of a decisions file is not a decision, or ""."""
2307
- if not (isinstance(d, dict) and isinstance(d.get("id"), str)
2308
- and _is_keep(d.get("keep")) and _is_summary(d.get("summary"))):
3044
+ if not (
3045
+ isinstance(d, dict)
3046
+ and isinstance(d.get("id"), str)
3047
+ and _is_keep(d.get("keep"))
3048
+ and _is_summary(d.get("summary"))
3049
+ ):
2309
3050
  return 'want {"id": str, "keep": true|false|null, "summary": str|null}'
2310
3051
  if not _is_answer(d):
2311
3052
  return "a turn of the part file, not a decision"
@@ -2330,13 +3071,20 @@ def _load_decisions(files):
2330
3071
  why = _decision_problem(d)
2331
3072
  if why:
2332
3073
  return _bad(f"{f}[{k}]: {why}")
2333
- cur = merged.setdefault(_lone_to_fffd(d["id"]), {}) # a model's JSON may hold a lone surrogate
3074
+ cur = merged.setdefault(
3075
+ _lone_to_fffd(d["id"]), {}
3076
+ ) # a model's JSON may hold a lone surrogate
2334
3077
  for key in ("keep", "summary"):
2335
3078
  if d.get(key) is not None:
2336
3079
  cur[key] = _lone_to_fffd(d[key])
2337
- if items and all(d.get("keep") is None and d.get("summary") is None for d in items):
2338
- print(f"[lineage] WARN: {f}: all {len(items)} decisions are null; the rules "
2339
- "decide those turns", file=sys.stderr)
3080
+ if items and all(
3081
+ d.get("keep") is None and d.get("summary") is None for d in items
3082
+ ):
3083
+ print(
3084
+ f"[lineage] WARN: {f}: all {len(items)} decisions are null; the rules "
3085
+ "decide those turns",
3086
+ file=sys.stderr,
3087
+ )
2340
3088
  return merged
2341
3089
 
2342
3090
 
@@ -2361,9 +3109,11 @@ def _warn_coverage(pack, decided):
2361
3109
  todo = [i for i in ids if i not in decided]
2362
3110
  if todo:
2363
3111
  more = ", ..." if len(todo) > 3 else ""
2364
- print(f"[lineage] WARN: part {p.get('part')}: {len(todo)}/{len(ids)} turns "
2365
- f"undecided ({', '.join(map(str, todo[:3]))}{more}); the rules decide them",
2366
- file=sys.stderr)
3112
+ print(
3113
+ f"[lineage] WARN: part {p.get('part')}: {len(todo)}/{len(ids)} turns "
3114
+ f"undecided ({', '.join(map(str, todo[:3]))}{more}); the rules decide them",
3115
+ file=sys.stderr,
3116
+ )
2367
3117
 
2368
3118
 
2369
3119
  def _reviewer_summary(x, d, args, stats):
@@ -2375,8 +3125,11 @@ def _reviewer_summary(x, d, args, stats):
2375
3125
  s, n = _clean_summary(raw, args.redact_extra, args.redact_mode)
2376
3126
  if n:
2377
3127
  stats["hidden"] += n
2378
- print(f"[lineage] WARN: the reviewer summary for {x['id']} held {n} "
2379
- "secret-like string(s); redacted", file=sys.stderr)
3128
+ print(
3129
+ f"[lineage] WARN: the reviewer summary for {x['id']} held {n} "
3130
+ "secret-like string(s); redacted",
3131
+ file=sys.stderr,
3132
+ )
2380
3133
  return s
2381
3134
 
2382
3135
 
@@ -2384,7 +3137,8 @@ def _show(x, llm_summary, turns, summaries, stats, prior):
2384
3137
  """Put kept pack turn `x` on the page with the summary it shows; `prior` adds up what
2385
3138
  --emit-review redacted."""
2386
3139
  shows = x["role"] in ("assistant", "agent") or (
2387
- x["role"] == "user" and len(x["text"]) > USER_FOLD)
3140
+ x["role"] == "user" and len(x["text"]) > USER_FOLD
3141
+ )
2388
3142
  if llm_summary and not shows:
2389
3143
  stats["ignored"] += 1
2390
3144
  summary = (llm_summary if shows else None) or x["rule"].get("summary")
@@ -2397,12 +3151,21 @@ def _show(x, llm_summary, turns, summaries, stats, prior):
2397
3151
  # what the gate's critic reads: the reviewer's summary, else the rules' cut from the
2398
3152
  # fully redacted text (the page's may hold a --redact-mode mask: 4 + 4 characters of a value)
2399
3153
  sample = (llm_summary if shows else None) or x["rule"].get("review_summary")
2400
- turns.append({"uuid": x["id"], "role": x["role"], "text": x["text"],
2401
- "ts": x.get("ts") or None, "tools": dict(x.get("tools") or {}),
2402
- "parts": [], "session": x.get("session"),
2403
- "session_name": x.get("session_name"),
2404
- "agent_from": x.get("agent_from"), "preview": x.get("preview"),
2405
- "sample_summary": sample})
3154
+ turns.append(
3155
+ {
3156
+ "uuid": x["id"],
3157
+ "role": x["role"],
3158
+ "text": x["text"],
3159
+ "ts": x.get("ts") or None,
3160
+ "tools": dict(x.get("tools") or {}),
3161
+ "parts": [],
3162
+ "session": x.get("session"),
3163
+ "session_name": x.get("session_name"),
3164
+ "agent_from": x.get("agent_from"),
3165
+ "preview": x.get("preview"),
3166
+ "sample_summary": sample,
3167
+ }
3168
+ )
2406
3169
 
2407
3170
 
2408
3171
  def _kept_by_flag(pack, x):
@@ -2430,8 +3193,12 @@ def _typed_by_user(x):
2430
3193
  """Whether pack turn `x` holds words the user typed: a user turn the rules keep that is neither a body the
2431
3194
  harness injected (meta) nor what Claude Code writes after a `!` command (bang_output, judged on the text before
2432
3195
  redaction: a hidden value at the end of the output takes the closing tags with it)."""
2433
- return (x["role"] == "user" and x["rule"]["keep"] and not x.get("meta")
2434
- and x.get("bang_output") is not True)
3196
+ return (
3197
+ x["role"] == "user"
3198
+ and x["rule"]["keep"]
3199
+ and not x.get("meta")
3200
+ and x.get("bang_output") is not True
3201
+ )
2435
3202
 
2436
3203
 
2437
3204
  def _review_result(pack, merged, args):
@@ -2440,8 +3207,15 @@ def _review_result(pack, merged, args):
2440
3207
  reviewer's value first, then the pack's cached one, then the rules'. A keep flag outranks
2441
3208
  a reviewer's `false`; the cache still gets what was said."""
2442
3209
  turns, summaries, writes, prior, typed = [], {}, [], {}, []
2443
- stats = {"restored": 0, "dropped": 0, "reviewer": 0, "rule": 0, "ignored": 0,
2444
- "hidden": 0, "flag-kept": 0}
3210
+ stats = {
3211
+ "restored": 0,
3212
+ "dropped": 0,
3213
+ "reviewer": 0,
3214
+ "rule": 0,
3215
+ "ignored": 0,
3216
+ "hidden": 0,
3217
+ "flag-kept": 0,
3218
+ }
2445
3219
  for x in pack["turns"]:
2446
3220
  d = merged.get(x["id"], {})
2447
3221
  llm_keep = _first(d.get("keep"), x["llm"].get("keep"))
@@ -2449,11 +3223,18 @@ def _review_result(pack, merged, args):
2449
3223
  stats["flag-kept"] += held
2450
3224
  keep = bool(_first(None if held else llm_keep, x["rule"]["keep"]))
2451
3225
  if not keep and _typed_by_user(x):
2452
- typed.append(x["id"]) # typed by the user: the rules keep these
3226
+ typed.append(x["id"]) # typed by the user: the rules keep these
2453
3227
  llm_summary = _reviewer_summary(x, d, args, stats)
2454
- if x["id"] in merged: # reviewed, even if left to the rules
2455
- writes.append((x["id"], x.get("session") or pack.get("source"), x["llm_key"],
2456
- llm_keep, llm_summary))
3228
+ if x["id"] in merged: # reviewed, even if left to the rules
3229
+ writes.append(
3230
+ (
3231
+ x["id"],
3232
+ x.get("session") or pack.get("source"),
3233
+ x["llm_key"],
3234
+ llm_keep,
3235
+ llm_summary,
3236
+ )
3237
+ )
2457
3238
  stats["restored"] += keep and not x["rule"]["keep"]
2458
3239
  stats["dropped"] += x["rule"]["keep"] and not keep
2459
3240
  if keep:
@@ -2476,31 +3257,42 @@ def _check_pack(pack, merged, args):
2476
3257
  ids = {x["id"] for x in pack["turns"]}
2477
3258
  unknown = [i for i in merged if i not in ids]
2478
3259
  if unknown:
2479
- print(f"[lineage] WARN: decisions name {len(unknown)} turn(s) not in the pack, "
2480
- f"ignored: {', '.join(_shown_id(i) for i in unknown[:5])}", file=sys.stderr)
2481
- _warn_coverage(pack, set(merged) | {x["id"] for x in pack["turns"]
2482
- if x["llm"].get("cached")})
3260
+ print(
3261
+ f"[lineage] WARN: decisions name {len(unknown)} turn(s) not in the pack, "
3262
+ f"ignored: {', '.join(_shown_id(i) for i in unknown[:5])}",
3263
+ file=sys.stderr,
3264
+ )
3265
+ _warn_coverage(
3266
+ pack, set(merged) | {x["id"] for x in pack["turns"] if x["llm"].get("cached")}
3267
+ )
2483
3268
  if pack.get("redact_mode") in ("full", "mask"):
2484
3269
  args.redact_mode = pack["redact_mode"]
2485
3270
  if pack.get("redact_extra") is True and not args.redact_extra:
2486
- print("[lineage] WARN: --emit-review had --redact-extra; give the same keywords "
2487
- "here too, or reviewer summaries miss that redaction", file=sys.stderr)
3271
+ print(
3272
+ "[lineage] WARN: --emit-review had --redact-extra; give the same keywords "
3273
+ "here too, or reviewer summaries miss that redaction",
3274
+ file=sys.stderr,
3275
+ )
2488
3276
  _gate_from_pack(args, pack)
2489
3277
 
2490
3278
 
2491
3279
  def _remove_review_files(pack_path, pack):
2492
3280
  """After a good render the pack, its parts and the decisions beside it go: they hold
2493
3281
  the redacted session."""
2494
- samples = f"{pack_path.stem}.reviewer-input.json" # a failed gate's, beside the pack
3282
+ samples = (
3283
+ f"{pack_path.stem}.reviewer-input.json" # a failed gate's, beside the pack
3284
+ )
2495
3285
  names = [pack_path.name, samples]
2496
3286
  for p in pack.get("parts") or []:
2497
3287
  if isinstance(p, dict):
2498
- names += [p.get(k) for k in ("file", "decisions") if isinstance(p.get(k), str)]
3288
+ names += [
3289
+ p.get(k) for k in ("file", "decisions") if isinstance(p.get(k), str)
3290
+ ]
2499
3291
  prefix = pack_path.stem + ".part-"
2500
3292
  left = []
2501
3293
  for name in names:
2502
3294
  if name not in (pack_path.name, samples) and not name.startswith(prefix):
2503
- continue # only what --emit-review named
3295
+ continue # only what --emit-review named
2504
3296
  try:
2505
3297
  pack_path.with_name(name).unlink()
2506
3298
  except FileNotFoundError:
@@ -2508,8 +3300,10 @@ def _remove_review_files(pack_path, pack):
2508
3300
  except (OSError, ValueError):
2509
3301
  left.append(name)
2510
3302
  if left:
2511
- print(f"[lineage] WARN: could not remove review files: {', '.join(left)}",
2512
- file=sys.stderr)
3303
+ print(
3304
+ f"[lineage] WARN: could not remove review files: {', '.join(left)}",
3305
+ file=sys.stderr,
3306
+ )
2513
3307
 
2514
3308
 
2515
3309
  def _samples_left_unread(pack_path, args):
@@ -2517,7 +3311,9 @@ def _samples_left_unread(pack_path, args):
2517
3311
  gives neither a verdict path nor --skip-reviewer: it would pass over that run's verdict."""
2518
3312
  left = pack_path.with_name(f"{pack_path.stem}.reviewer-input.json")
2519
3313
  if left.exists() and not (args.reviewer_output or args.skip_reviewer):
2520
- _bad(f"{left} was left by a gated run; give its --reviewer-output again, or --skip-reviewer")
3314
+ _bad(
3315
+ f"{left} was left by a gated run; give its --reviewer-output again, or --skip-reviewer"
3316
+ )
2521
3317
  return True
2522
3318
  return False
2523
3319
 
@@ -2532,18 +3328,30 @@ def _review_output(args, pack):
2532
3328
 
2533
3329
  def _report_review(stats, turns, pack, typed):
2534
3330
  """What the render took from the reviewers, and the turns typed by the user they dropped."""
2535
- print(f"[lineage] review applied: shown={len(turns)}/{len(pack['turns'])} "
2536
- f"restored={stats['restored']} dropped={stats['dropped']} "
2537
- f"summaries reviewer={stats['reviewer']} rule={stats['rule']}", file=sys.stderr)
3331
+ print(
3332
+ f"[lineage] review applied: shown={len(turns)}/{len(pack['turns'])} "
3333
+ f"restored={stats['restored']} dropped={stats['dropped']} "
3334
+ f"summaries reviewer={stats['reviewer']} rule={stats['rule']}",
3335
+ file=sys.stderr,
3336
+ )
2538
3337
  if stats["ignored"]:
2539
- print(f"[lineage] note: {stats['ignored']} summaries for turns the page shows "
2540
- "in full were ignored", file=sys.stderr)
3338
+ print(
3339
+ f"[lineage] note: {stats['ignored']} summaries for turns the page shows "
3340
+ "in full were ignored",
3341
+ file=sys.stderr,
3342
+ )
2541
3343
  if stats["flag-kept"]:
2542
- print(f"[lineage] note: {stats['flag-kept']} turn(s) a reviewer marked keep: false "
2543
- "stay, as --keep-trivia or --keep-tool-only asked", file=sys.stderr)
3344
+ print(
3345
+ f"[lineage] note: {stats['flag-kept']} turn(s) a reviewer marked keep: false "
3346
+ "stay, as --keep-trivia or --keep-tool-only asked",
3347
+ file=sys.stderr,
3348
+ )
2544
3349
  if typed:
2545
- print(f"[lineage] WARN: reviewers dropped {len(typed)} typed user turn(s): {', '.join(_shown_id(i) for i in typed[:10])}"
2546
- f"{' ...' if len(typed) > 10 else ''}; tell the user", file=sys.stderr)
3350
+ print(
3351
+ f"[lineage] WARN: reviewers dropped {len(typed)} typed user turn(s): {', '.join(_shown_id(i) for i in typed[:10])}"
3352
+ f"{' ...' if len(typed) > 10 else ''}; tell the user",
3353
+ file=sys.stderr,
3354
+ )
2547
3355
 
2548
3356
 
2549
3357
  def apply_review(args):
@@ -2564,13 +3372,18 @@ def apply_review(args):
2564
3372
  if stats["hidden"]:
2565
3373
  prior["reviewer-summary"] = stats["hidden"]
2566
3374
  try:
2567
- rc = render_and_write(turns, pack.get("source") or "session", args,
2568
- all_sessions=bool(pack.get("all_sessions")),
2569
- markdown=pack.get("markdown") is not False,
2570
- open_details=bool(pack.get("open")),
2571
- title=str(pack.get("title") or "Session Lineage"),
2572
- summaries=summaries, prior_redactions=prior)
2573
- except SystemExit as e: # the quality gate failed
3375
+ rc = render_and_write(
3376
+ turns,
3377
+ pack.get("source") or "session",
3378
+ args,
3379
+ all_sessions=bool(pack.get("all_sessions")),
3380
+ markdown=pack.get("markdown") is not False,
3381
+ open_details=bool(pack.get("open")),
3382
+ title=str(pack.get("title") or "Session Lineage"),
3383
+ summaries=summaries,
3384
+ prior_redactions=prior,
3385
+ )
3386
+ except SystemExit as e: # the quality gate failed
2574
3387
  rc = int(e.code) if isinstance(e.code, int) else 2
2575
3388
  if rc == 0:
2576
3389
  for w in writes:
@@ -2583,66 +3396,133 @@ def apply_review(args):
2583
3396
  def build_arg_parser() -> argparse.ArgumentParser:
2584
3397
  ap = argparse.ArgumentParser(
2585
3398
  prog="lineage",
2586
- description="Export Claude Code session(s) as a KakaoTalk-style HTML file.")
3399
+ description="Export Claude Code session(s) as a KakaoTalk-style HTML file.",
3400
+ )
2587
3401
  ap.add_argument("--last", type=int, help="keep last N turns")
2588
3402
  ap.add_argument("--from", dest="from_", help="ISO timestamp from (inclusive)")
2589
3403
  ap.add_argument("--to", help="ISO timestamp to (inclusive)")
2590
3404
  ap.add_argument("--turns", help='range like "10-50" (1-indexed inclusive)')
2591
- ap.add_argument("--output", default=None,
2592
- help="output HTML path (default: work/lineage-{name}_YYMMDD+HHMM.html)")
3405
+ ap.add_argument(
3406
+ "--output",
3407
+ default=None,
3408
+ help="output HTML path (default: work/lineage-{name}_YYMMDD+HHMM.html)",
3409
+ )
2593
3410
  ap.add_argument("--session", help="jsonl path (overrides auto-discover)")
2594
- ap.add_argument("--all-sessions", action="store_true",
2595
- help="merge ALL sessions in the project dir, chronologically")
2596
- ap.add_argument("--from-transcript",
2597
- help="read transcript from PATH or '-' for stdin")
3411
+ ap.add_argument(
3412
+ "--all-sessions",
3413
+ action="store_true",
3414
+ help="merge ALL sessions in the project dir, chronologically",
3415
+ )
3416
+ ap.add_argument(
3417
+ "--from-transcript", help="read transcript from PATH or '-' for stdin"
3418
+ )
2598
3419
  ap.add_argument("--redact-extra", help="comma-separated keywords to also redact")
2599
- ap.add_argument("--redact-mode", choices=["full", "mask"], default="full",
2600
- help="redaction mode (full=[REDACTED], mask=abcd****wxyz)")
2601
- ap.add_argument("--unsafe-schema", action="store_true",
2602
- help="proceed even on unknown schema markers")
2603
- ap.add_argument("--rebuild-summaries", action="store_true",
2604
- help="ignore summary cache and re-summarize")
2605
- ap.add_argument("--purge-cache", action="store_true",
2606
- help="purge ~/.cache/lineage then exit")
2607
- ap.add_argument("--skip-reviewer", action="store_true",
2608
- help="skip reviewer quality gate (warn-only)")
2609
- ap.add_argument("--reviewer-output",
2610
- help="path to Critic response JSON; enforces the quality gate")
2611
- ap.add_argument("--reviewer-timeout", type=int, default=None,
2612
- help="seconds to wait for reviewer-output (default 60 with --rulebase; "
2613
- "the reviewed flow waits only when this is given)")
3420
+ ap.add_argument(
3421
+ "--redact-mode",
3422
+ choices=["full", "mask"],
3423
+ default="full",
3424
+ help="redaction mode (full=[REDACTED], mask=abcd****wxyz)",
3425
+ )
3426
+ ap.add_argument(
3427
+ "--unsafe-schema",
3428
+ action="store_true",
3429
+ help="proceed even on unknown schema markers",
3430
+ )
3431
+ ap.add_argument(
3432
+ "--rebuild-summaries",
3433
+ action="store_true",
3434
+ help="ignore summary cache and re-summarize",
3435
+ )
3436
+ ap.add_argument(
3437
+ "--purge-cache", action="store_true", help="purge ~/.cache/lineage then exit"
3438
+ )
3439
+ ap.add_argument(
3440
+ "--skip-reviewer",
3441
+ action="store_true",
3442
+ help="skip reviewer quality gate (warn-only)",
3443
+ )
3444
+ ap.add_argument(
3445
+ "--reviewer-output",
3446
+ help="path to Critic response JSON; enforces the quality gate",
3447
+ )
3448
+ ap.add_argument(
3449
+ "--reviewer-timeout",
3450
+ type=int,
3451
+ default=None,
3452
+ help="seconds to wait for reviewer-output (default 60 with --rulebase; "
3453
+ "the reviewed flow waits only when this is given)",
3454
+ )
2614
3455
  ap.add_argument("--title", default="Session Lineage", help="HTML title")
2615
3456
  _add_review_args(ap)
2616
3457
  # Readability defaults are ON. Opt-out flags restore raw/older behavior.
2617
- ap.add_argument("--hide-tool-only", dest="hide_tool_only", action="store_true",
2618
- help="(DEFAULT) drop assistant turns that are only tool calls")
2619
- ap.add_argument("--keep-tool-only", dest="hide_tool_only", action="store_false",
2620
- help="keep tool-only assistant turns (override default)")
2621
- ap.add_argument("--no-markdown", dest="markdown", action="store_false",
2622
- help="disable markdown rendering (default: render ON)")
2623
- ap.add_argument("--keep-trivia", dest="drop_trivia", action="store_false",
2624
- help="keep manipulation commands / injected bodies / echo / "
2625
- "harness errors (default: filter ON)")
2626
- ap.add_argument("--open", dest="open_details", action="store_true",
2627
- help="render bubbles expanded (default: collapsed)")
2628
- ap.add_argument("--collapse", dest="open_details", action="store_false",
2629
- help="(DEFAULT) render bubbles collapsed")
2630
- ap.set_defaults(hide_tool_only=True, open_details=False,
2631
- markdown=True, drop_trivia=True)
3458
+ ap.add_argument(
3459
+ "--hide-tool-only",
3460
+ dest="hide_tool_only",
3461
+ action="store_true",
3462
+ help="(DEFAULT) drop assistant turns that are only tool calls",
3463
+ )
3464
+ ap.add_argument(
3465
+ "--keep-tool-only",
3466
+ dest="hide_tool_only",
3467
+ action="store_false",
3468
+ help="keep tool-only assistant turns (override default)",
3469
+ )
3470
+ ap.add_argument(
3471
+ "--no-markdown",
3472
+ dest="markdown",
3473
+ action="store_false",
3474
+ help="disable markdown rendering (default: render ON)",
3475
+ )
3476
+ ap.add_argument(
3477
+ "--keep-trivia",
3478
+ dest="drop_trivia",
3479
+ action="store_false",
3480
+ help="keep manipulation commands / injected bodies / echo / "
3481
+ "harness errors (default: filter ON)",
3482
+ )
3483
+ ap.add_argument(
3484
+ "--open",
3485
+ dest="open_details",
3486
+ action="store_true",
3487
+ help="render bubbles expanded (default: collapsed)",
3488
+ )
3489
+ ap.add_argument(
3490
+ "--collapse",
3491
+ dest="open_details",
3492
+ action="store_false",
3493
+ help="(DEFAULT) render bubbles collapsed",
3494
+ )
3495
+ ap.set_defaults(
3496
+ hide_tool_only=True, open_details=False, markdown=True, drop_trivia=True
3497
+ )
2632
3498
  return ap
2633
3499
 
2634
3500
 
2635
3501
  def _add_review_args(ap):
2636
3502
  """The flags of the reviewed flow (3.0.0): --emit-review, the session's reviewers,
2637
3503
  then --apply-review."""
2638
- ap.add_argument("--rulebase", action="store_true",
2639
- help="one pass, rules only (the default when no review flag is given)")
2640
- ap.add_argument("--emit-review", metavar="PACK",
2641
- help="write the turns for the session model to review (JSON) and stop")
2642
- ap.add_argument("--apply-review", metavar="PACK",
2643
- help="render from a review pack and the reviewer's decisions")
2644
- ap.add_argument("--decisions", action="append", default=[], metavar="FILE",
2645
- help="reviewer decisions [{id, keep, summary}] for --apply-review (repeatable)")
3504
+ ap.add_argument(
3505
+ "--rulebase",
3506
+ action="store_true",
3507
+ help="one pass, rules only (the default when no review flag is given)",
3508
+ )
3509
+ ap.add_argument(
3510
+ "--emit-review",
3511
+ metavar="PACK",
3512
+ help="write the turns for the session model to review (JSON) and stop",
3513
+ )
3514
+ ap.add_argument(
3515
+ "--apply-review",
3516
+ metavar="PACK",
3517
+ help="render from a review pack and the reviewer's decisions",
3518
+ )
3519
+ ap.add_argument(
3520
+ "--decisions",
3521
+ action="append",
3522
+ default=[],
3523
+ metavar="FILE",
3524
+ help="reviewer decisions [{id, keep, summary}] for --apply-review (repeatable)",
3525
+ )
2646
3526
 
2647
3527
 
2648
3528
  def _open_stream(path):
@@ -2653,12 +3533,18 @@ def _load_turns_from(path, unsafe_schema, session_id, session_name):
2653
3533
  """Parse one jsonl file, isolating per-file decode errors (D-3)."""
2654
3534
  try:
2655
3535
  with _open_stream(path) as stream:
2656
- return list(parse_turns(stream, unsafe_schema=unsafe_schema,
2657
- session_id=session_id,
2658
- session_name=session_name))
3536
+ return list(
3537
+ parse_turns(
3538
+ stream,
3539
+ unsafe_schema=unsafe_schema,
3540
+ session_id=session_id,
3541
+ session_name=session_name,
3542
+ )
3543
+ )
2659
3544
  except (OSError, UnicodeDecodeError) as e:
2660
- print(f"[lineage] WARN: skipping unreadable session {path}: {e}",
2661
- file=sys.stderr)
3545
+ print(
3546
+ f"[lineage] WARN: skipping unreadable session {path}: {e}", file=sys.stderr
3547
+ )
2662
3548
  return []
2663
3549
 
2664
3550
 
@@ -2681,12 +3567,12 @@ def _merge_env_extra(args):
2681
3567
  """LINEAGE_REDACT_EXTRA's keywords after --redact-extra's."""
2682
3568
  env_extra = os.environ.get("LINEAGE_REDACT_EXTRA", "").strip()
2683
3569
  if env_extra:
2684
- args.redact_extra = ",".join(
2685
- p for p in (args.redact_extra, env_extra) if p)
3570
+ args.redact_extra = ",".join(p for p in (args.redact_extra, env_extra) if p)
2686
3571
 
2687
3572
 
2688
3573
  def _purge_cache():
2689
3574
  import shutil
3575
+
2690
3576
  if CACHE_BASE.exists():
2691
3577
  shutil.rmtree(CACHE_BASE)
2692
3578
  print(f"[lineage] cache purged: {CACHE_BASE}", file=sys.stderr)
@@ -2696,10 +3582,22 @@ def _purge_cache():
2696
3582
 
2697
3583
 
2698
3584
  def _run_apply(args):
2699
- if any((args.last, args.from_, args.to, args.turns, args.session,
2700
- args.all_sessions, args.from_transcript)):
2701
- print("[lineage] note: --apply-review renders the pack; source and range "
2702
- "flags were applied at --emit-review", file=sys.stderr)
3585
+ if any(
3586
+ (
3587
+ args.last,
3588
+ args.from_,
3589
+ args.to,
3590
+ args.turns,
3591
+ args.session,
3592
+ args.all_sessions,
3593
+ args.from_transcript,
3594
+ )
3595
+ ):
3596
+ print(
3597
+ "[lineage] note: --apply-review renders the pack; source and range "
3598
+ "flags were applied at --emit-review",
3599
+ file=sys.stderr,
3600
+ )
2703
3601
  return apply_review(args)
2704
3602
 
2705
3603
 
@@ -2707,30 +3605,42 @@ def _read_all_sessions(args):
2707
3605
  """Every session of the project folder, merged record by record in time order."""
2708
3606
  files = project_jsonl_files()
2709
3607
  if not files:
2710
- print("[lineage] --all-sessions: no jsonl found in project dir",
2711
- file=sys.stderr)
3608
+ print(
3609
+ "[lineage] --all-sessions: no jsonl found in project dir", file=sys.stderr
3610
+ )
2712
3611
  return 2, [], "session", None
2713
3612
  starts = {p: session_start(p) for p in files}
2714
3613
  turns = []
2715
3614
  for p in files:
2716
- name = discover_session_name(p) or p.stem[:8]
3615
+ name = (
3616
+ discover_session_name(p)
3617
+ or (p.stem[-36:] if p.stem.startswith("rollout-") else p.stem)[:8]
3618
+ )
2717
3619
  part = _load_turns_from(p, args.unsafe_schema, p.stem, name)
3620
+ meta = _codex_meta(p)
3621
+ if meta is not None and not any(t["role"] == "user" for t in part):
3622
+ print(
3623
+ f"[lineage] WARN: {p.name}: no user turn read from this Codex {meta.get('cli_version', '?')} "
3624
+ "record (a new record shape?); left out",
3625
+ file=sys.stderr,
3626
+ )
3627
+ continue
2718
3628
  fill_missing_ts(part)
2719
3629
  for t in part:
2720
3630
  t["session_ts"] = starts[p] or "9999"
2721
3631
  turns.extend(part)
2722
3632
  # record-level chronological sort (stable → same-ts keeps file order)
2723
3633
  turns.sort(key=lambda t: t.get("ts") or t.get("session_ts") or "9999")
2724
- print(f"[lineage] --all-sessions: merged {len(files)} sessions",
2725
- file=sys.stderr)
3634
+ print(f"[lineage] --all-sessions: merged {len(files)} sessions", file=sys.stderr)
2726
3635
  return None, turns, "all-sessions", None
2727
3636
 
2728
3637
 
2729
3638
  def _read_transcript(args):
2730
3639
  """A transcript pasted on stdin ('-') or read from a file."""
2731
3640
  if args.from_transcript == "-":
2732
- turns = list(parse_turns(sys.stdin, unsafe_schema=args.unsafe_schema,
2733
- session_id="stdin"))
3641
+ turns = list(
3642
+ parse_turns(sys.stdin, unsafe_schema=args.unsafe_schema, session_id="stdin")
3643
+ )
2734
3644
  return None, turns, "stdin", None
2735
3645
  jsonl_path = pathlib.Path(args.from_transcript)
2736
3646
  session_id = "file-" + jsonl_path.stem
@@ -2742,13 +3652,15 @@ def _read_discovered(args):
2742
3652
  """The newest session of the project folder."""
2743
3653
  f = auto_discover_jsonl()
2744
3654
  if not f:
2745
- print("[lineage] No jsonl found. Use one of:",
2746
- "\n --session FILE explicit jsonl path",
2747
- "\n --all-sessions merge all project sessions",
2748
- "\n --from-transcript - paste transcript via stdin",
2749
- "\n --from-transcript FILE read transcript from file",
2750
- "\n(non-ASCII cwd may not auto-encode — use --session)",
2751
- file=sys.stderr)
3655
+ print(
3656
+ "[lineage] No jsonl found. Use one of:",
3657
+ "\n --session FILE explicit jsonl path",
3658
+ "\n --all-sessions merge all project sessions",
3659
+ "\n --from-transcript - paste transcript via stdin",
3660
+ "\n --from-transcript FILE read transcript from file",
3661
+ "\n(non-ASCII cwd may not auto-encode — use --session)",
3662
+ file=sys.stderr,
3663
+ )
2752
3664
  return 2, [], "session", None
2753
3665
  turns = _load_turns_from(f, args.unsafe_schema, f.stem, None)
2754
3666
  print(f"[lineage] auto-discovered session: {f.name}", file=sys.stderr)
@@ -2756,8 +3668,26 @@ def _read_discovered(args):
2756
3668
 
2757
3669
 
2758
3670
  def _read_input(args):
2759
- """(exit code or None, raw turns, session id, jsonl path) from the source the flags name. The code is 2 when
2760
- there is no source, or the code a schema marker stopped the read with."""
3671
+ """(exit code or None, raw turns, session id, jsonl path) from the source the flags name: a Claude Code
3672
+ transcript or a Codex session record (rollout), told apart by its first record. The code is 2 when there is no
3673
+ source, or the code a schema marker stopped the read with. A Codex rollout that gives no user turn stops with 2
3674
+ too: every Codex session starts with what the user typed, so none means this Codex version writes its record in a
3675
+ shape lineage does not read, and an empty page would hide that."""
3676
+ rc, turns, session_id, jsonl_path = _read_source(args)
3677
+ meta = _codex_meta(jsonl_path) if rc is None and jsonl_path is not None else None
3678
+ if meta is not None and not any(t["role"] == "user" for t in turns):
3679
+ print(
3680
+ f"[lineage] ERROR: {jsonl_path.name} is a Codex {meta.get('cli_version', '?')} session record, but no "
3681
+ "user turn was read from it; this Codex version may write its record in a new shape "
3682
+ "(nothing written; update banker or report it)",
3683
+ file=sys.stderr,
3684
+ )
3685
+ return 2, [], session_id, None
3686
+ return rc, turns, session_id, jsonl_path
3687
+
3688
+
3689
+ def _read_source(args):
3690
+ """(exit code or None, raw turns, session id, jsonl path) as read, before the Codex shape check."""
2761
3691
  try:
2762
3692
  if args.all_sessions:
2763
3693
  return _read_all_sessions(args)
@@ -2765,7 +3695,9 @@ def _read_input(args):
2765
3695
  return _read_transcript(args)
2766
3696
  if args.session:
2767
3697
  jsonl_path = pathlib.Path(args.session)
2768
- turns = _load_turns_from(jsonl_path, args.unsafe_schema, jsonl_path.stem, None)
3698
+ turns = _load_turns_from(
3699
+ jsonl_path, args.unsafe_schema, jsonl_path.stem, None
3700
+ )
2769
3701
  return None, turns, jsonl_path.stem, jsonl_path
2770
3702
  return _read_discovered(args)
2771
3703
  except SystemExit as e:
@@ -2776,7 +3708,9 @@ def _stdin_ids(turns):
2776
3708
  """stdin → derive per-turn uuid from the content so the caches work (keyed: see _stdin_id_key)."""
2777
3709
  key = _stdin_id_key()
2778
3710
  for t in turns:
2779
- t["uuid"] = hmac.new(key, str(t["text"]).encode(), hashlib.sha256).hexdigest()[:16]
3711
+ t["uuid"] = hmac.new(key, str(t["text"]).encode(), hashlib.sha256).hexdigest()[
3712
+ :16
3713
+ ]
2780
3714
 
2781
3715
 
2782
3716
  def _run_emit(turns, session_id, jsonl_path, args):
@@ -2784,8 +3718,14 @@ def _run_emit(turns, session_id, jsonl_path, args):
2784
3718
  window = review_window(turns, drops, args)
2785
3719
  if window is None:
2786
3720
  return 2
2787
- return emit_review(turns, drops, window, session_id,
2788
- default_output(args, jsonl_path, session_id), args)
3721
+ return emit_review(
3722
+ turns,
3723
+ drops,
3724
+ window,
3725
+ session_id,
3726
+ default_output(args, jsonl_path, session_id),
3727
+ args,
3728
+ )
2789
3729
 
2790
3730
 
2791
3731
  def _rulebase_turns(turns, args):
@@ -2794,8 +3734,9 @@ def _rulebase_turns(turns, args):
2794
3734
  if args.drop_trivia:
2795
3735
  turns = drop_echo_exchanges(turns)
2796
3736
  if args.hide_tool_only:
2797
- turns = [t for t in turns
2798
- if not (t["role"] == "assistant" and not t["text"].strip())]
3737
+ turns = [
3738
+ t for t in turns if not (t["role"] == "assistant" and not t["text"].strip())
3739
+ ]
2799
3740
  return select_range(turns, args)
2800
3741
 
2801
3742
 
@@ -2824,9 +3765,15 @@ def main(argv=None) -> int:
2824
3765
  return 2
2825
3766
  if args.output is None:
2826
3767
  args.output = default_output(args, jsonl_path, session_id)
2827
- return render_and_write(turns, session_id, args, all_sessions=args.all_sessions,
2828
- markdown=args.markdown, open_details=args.open_details,
2829
- title=args.title)
3768
+ return render_and_write(
3769
+ turns,
3770
+ session_id,
3771
+ args,
3772
+ all_sessions=args.all_sessions,
3773
+ markdown=args.markdown,
3774
+ open_details=args.open_details,
3775
+ title=args.title,
3776
+ )
2830
3777
 
2831
3778
 
2832
3779
  def _in_time(t, args):
@@ -2863,8 +3810,10 @@ def select_range(turns, args):
2863
3810
  return None
2864
3811
  turns = turns[sl]
2865
3812
  if len(turns) > 100:
2866
- print(f"[lineage] WARN: {len(turns)} turns is large — render may be slow",
2867
- file=sys.stderr)
3813
+ print(
3814
+ f"[lineage] WARN: {len(turns)} turns is large — render may be slow",
3815
+ file=sys.stderr,
3816
+ )
2868
3817
  return turns
2869
3818
 
2870
3819
 
@@ -2875,11 +3824,13 @@ def default_output(args, jsonl_path, session_id):
2875
3824
  return args.output
2876
3825
  session_name = discover_session_name(jsonl_path) if jsonl_path else None
2877
3826
  if session_name and page_redact(session_name)[0] != session_name:
2878
- print(f"[lineage] session name: {page_redact(session_name)[0]} "
2879
- "(holds a secret-like string; the file is named by the session id)", file=sys.stderr)
3827
+ print(
3828
+ f"[lineage] session name: {page_redact(session_name)[0]} "
3829
+ "(holds a secret-like string; the file is named by the session id)",
3830
+ file=sys.stderr,
3831
+ )
2880
3832
  session_name = None
2881
- slug = ("all-sessions" if args.all_sessions
2882
- else (session_name or session_id[:8]))
3833
+ slug = "all-sessions" if args.all_sessions else (session_name or session_id[:8])
2883
3834
  if session_name:
2884
3835
  print(f"[lineage] session name: {session_name}", file=sys.stderr)
2885
3836
  return f"work/lineage-{slug}.html"
@@ -2888,31 +3839,50 @@ def default_output(args, jsonl_path, session_id):
2888
3839
  # The controls that embed, override or isolate text direction (U+202A..U+202E, U+2066..U+2069). In a page they
2889
3840
  # reorder what a reader sees and can reach past the element that holds them, so the page shows each as a visible
2890
3841
  # [U+XXXX] instead. Plain letters of right-to-left scripts, and the direction marks, stay.
2891
- _BIDI_CONTROLS = re.compile("[" + "".join(map(chr, (*range(0x202A, 0x202F), *range(0x2066, 0x206A)))) + "]")
3842
+ _BIDI_CONTROLS = re.compile(
3843
+ "[" + "".join(map(chr, (*range(0x202A, 0x202F), *range(0x2066, 0x206A)))) + "]"
3844
+ )
2892
3845
 
2893
3846
 
2894
3847
  def _show_bidi_controls(text):
2895
3848
  return _BIDI_CONTROLS.sub(lambda m: "[U+%04X]" % ord(m.group()), text)
2896
3849
 
2897
3850
 
2898
- def render_and_write(turns, session_id, args, all_sessions=False, markdown=True,
2899
- open_details=False, title="Session Lineage", summaries=None,
2900
- prior_redactions=None):
3851
+ def render_and_write(
3852
+ turns,
3853
+ session_id,
3854
+ args,
3855
+ all_sessions=False,
3856
+ markdown=True,
3857
+ open_details=False,
3858
+ title="Session Lineage",
3859
+ summaries=None,
3860
+ prior_redactions=None,
3861
+ ):
2901
3862
  """Render the rows, fill the template, self-verify, write, run the reviewer gate.
2902
3863
  `prior_redactions`: what --emit-review redacted before the text reached the pack."""
2903
3864
  rows, redact_counts, hits, total = render_rows(
2904
- turns, session_id, args.redact_extra, args.redact_mode,
2905
- args.rebuild_summaries, open_details=open_details,
2906
- markdown=markdown, all_sessions=all_sessions, summaries=summaries)
3865
+ turns,
3866
+ session_id,
3867
+ args.redact_extra,
3868
+ args.redact_mode,
3869
+ args.rebuild_summaries,
3870
+ open_details=open_details,
3871
+ markdown=markdown,
3872
+ all_sessions=all_sessions,
3873
+ summaries=summaries,
3874
+ )
2907
3875
  for k, v in (prior_redactions or {}).items():
2908
3876
  redact_counts[k] = redact_counts.get(k, 0) + v
2909
3877
  left = _residual_secrets("\n".join(rows))
2910
3878
  if left:
2911
- print("[lineage] WARN: the page still holds secret-like text ("
2912
- + ", ".join(f"{k}={v}" for k, v in sorted(left.items()))
2913
- + "); check the page before sharing it, or run again with the value in LINEAGE_REDACT_EXTRA"
2914
- " (with --rulebase, add --rebuild-summaries)",
2915
- file=sys.stderr)
3879
+ print(
3880
+ "[lineage] WARN: the page still holds secret-like text ("
3881
+ + ", ".join(f"{k}={v}" for k, v in sorted(left.items()))
3882
+ + "); check the page before sharing it, or run again with the value in LINEAGE_REDACT_EXTRA"
3883
+ " (with --rulebase, add --rebuild-summaries)",
3884
+ file=sys.stderr,
3885
+ )
2916
3886
 
2917
3887
  date_range = ""
2918
3888
  dated = [t for t in turns if t.get("ts")]
@@ -2923,8 +3893,12 @@ def render_and_write(turns, session_id, args, all_sessions=False, markdown=True,
2923
3893
 
2924
3894
  html_doc = HTML_TEMPLATE
2925
3895
  html_doc = html_doc.replace("{{TITLE}}", html.escape(_show_bidi_controls(title)), 1)
2926
- html_doc = html_doc.replace("{{HEADER_TITLE}}", html.escape(_show_bidi_controls(title)), 1)
2927
- html_doc = html_doc.replace("{{DATE_RANGE}}", html.escape(_show_bidi_controls(date_range)), 1)
3896
+ html_doc = html_doc.replace(
3897
+ "{{HEADER_TITLE}}", html.escape(_show_bidi_controls(title)), 1
3898
+ )
3899
+ html_doc = html_doc.replace(
3900
+ "{{DATE_RANGE}}", html.escape(_show_bidi_controls(date_range)), 1
3901
+ )
2928
3902
  html_doc = html_doc.replace("{{TURNS}}", _show_bidi_controls("\n".join(rows)), 1)
2929
3903
 
2930
3904
  errs = self_verify(html_doc)
@@ -2938,9 +3912,12 @@ def render_and_write(turns, session_id, args, all_sessions=False, markdown=True,
2938
3912
  _run_reviewer_gate(turns, out, session_id, args, summaries=summaries)
2939
3913
 
2940
3914
  tool_total = sum(sum(t.get("tools", {}).values()) for t in turns)
2941
- print(f"[lineage] turns={len(turns)} redacted={sum(redact_counts.values())} "
2942
- f"tool_calls={tool_total} cache_hits={hits}/{total} "
2943
- f"size={out.stat().st_size}B output={out}", file=sys.stderr)
3915
+ print(
3916
+ f"[lineage] turns={len(turns)} redacted={sum(redact_counts.values())} "
3917
+ f"tool_calls={tool_total} cache_hits={hits}/{total} "
3918
+ f"size={out.stat().st_size}B output={out}",
3919
+ file=sys.stderr,
3920
+ )
2944
3921
  if redact_counts:
2945
3922
  details = ", ".join(f"{k}={v}" for k, v in sorted(redact_counts.items()))
2946
3923
  print(f"[lineage] redaction breakdown: {details}", file=sys.stderr)
@@ -2960,6 +3937,7 @@ def _with_pack_stamp(output, pack_output):
2960
3937
  def _with_timestamp_suffix(path_str):
2961
3938
  """Append `_YYMMDD+HHMM` before the extension (idempotent)."""
2962
3939
  import datetime as _dt
3940
+
2963
3941
  p = pathlib.Path(path_str)
2964
3942
  if re.search(r"_\d{6}\+\d{4}$", p.stem):
2965
3943
  return p
@@ -2974,11 +3952,13 @@ def _gate_samples(bots, summaries, session_id, args):
2974
3952
  seeded by the turns, so a rerun on the same turns samples the same ones, and a sample
2975
3953
  names its turn and shows the head and tail the reviewer saw, fully redacted."""
2976
3954
  import random
3955
+
2977
3956
  if summaries is None:
2978
3957
  out = []
2979
3958
  for i, t in enumerate(random.sample(bots, min(5, len(bots)))):
2980
- s, _ = _page_summary(t, summaries, session_id, False,
2981
- args.redact_extra, args.redact_mode)
3959
+ s, _ = _page_summary(
3960
+ t, summaries, session_id, False, args.redact_extra, args.redact_mode
3961
+ )
2982
3962
  red_s = review_redact(_MASKED.sub("[REDACTED]", s), args.redact_extra)[0]
2983
3963
  red_d = review_redact(t["text"], args.redact_extra)[0][:500]
2984
3964
  out.append({"idx": i, "original_detail": red_d, "generated_summary": red_s})
@@ -2986,13 +3966,29 @@ def _gate_samples(bots, summaries, session_id, args):
2986
3966
  seed = hashlib.sha256("\x00".join(t["uuid"] for t in bots).encode()).hexdigest()
2987
3967
  out = []
2988
3968
  for i, t in enumerate(random.Random(seed).sample(bots, min(5, len(bots)))):
2989
- s = t.get("sample_summary") or _page_summary(t, summaries, session_id, False,
2990
- args.redact_extra, args.redact_mode)[0]
2991
- detail = t.get("preview") or _preview(review_redact(t["text"], args.redact_extra)[0])[0]
3969
+ s = (
3970
+ t.get("sample_summary")
3971
+ or _page_summary(
3972
+ t, summaries, session_id, False, args.redact_extra, args.redact_mode
3973
+ )[0]
3974
+ )
3975
+ detail = (
3976
+ t.get("preview")
3977
+ or _preview(review_redact(t["text"], args.redact_extra)[0])[0]
3978
+ )
2992
3979
  gen = review_redact(_MASKED.sub("[REDACTED]", s), args.redact_extra)[0]
2993
- key = hashlib.sha256("\x00".join([t["uuid"], detail, gen]).encode()).hexdigest()[:12]
2994
- out.append({"idx": i, "id": t["uuid"], "key": key, "original_detail": detail,
2995
- "generated_summary": gen})
3980
+ key = hashlib.sha256(
3981
+ "\x00".join([t["uuid"], detail, gen]).encode()
3982
+ ).hexdigest()[:12]
3983
+ out.append(
3984
+ {
3985
+ "idx": i,
3986
+ "id": t["uuid"],
3987
+ "key": key,
3988
+ "original_detail": detail,
3989
+ "generated_summary": gen,
3990
+ }
3991
+ )
2996
3992
  return out
2997
3993
 
2998
3994
 
@@ -3015,8 +4011,10 @@ def _renew_samples(path, text, rop):
3015
4011
  _write_private(path, text)
3016
4012
  old = pathlib.Path(rop) if rop else None
3017
4013
  if old is not None and _is_verdict_file(old):
3018
- print(f"[lineage] WARN: {rop} predates these samples; it is not read",
3019
- file=sys.stderr)
4014
+ print(
4015
+ f"[lineage] WARN: {rop} predates these samples; it is not read",
4016
+ file=sys.stderr,
4017
+ )
3020
4018
  _set_aside(old)
3021
4019
 
3022
4020
 
@@ -3027,21 +4025,30 @@ def _refuse_non_verdict(rop):
3027
4025
  p = pathlib.Path(rop) if rop else None
3028
4026
  if p is None or not p.exists() or _is_verdict_file(p):
3029
4027
  return
3030
- if _json_list(p): # step 8: written again, the same answer fails the same way
3031
- why = ("it is a JSON array where no entry names idx; if the session wrote the critic's "
3032
- "answer, have the critic judge the samples again (a failed gate run)")
4028
+ if _json_list(p): # step 8: written again, the same answer fails the same way
4029
+ why = (
4030
+ "it is a JSON array where no entry names idx; if the session wrote the critic's "
4031
+ "answer, have the critic judge the samples again (a failed gate run)"
4032
+ )
3033
4033
  else:
3034
- why = ("if the session wrote it, write it again as a bare JSON array of "
3035
- "{idx, id, key, recoverable, reason}")
3036
- print(f"[lineage] ERROR: {rop} is not a verdict list; {why}, else give a --reviewer-output "
3037
- "that does not exist yet", file=sys.stderr)
4034
+ why = (
4035
+ "if the session wrote it, write it again as a bare JSON array of "
4036
+ "{idx, id, key, recoverable, reason}"
4037
+ )
4038
+ print(
4039
+ f"[lineage] ERROR: {rop} is not a verdict list; {why}, else give a --reviewer-output "
4040
+ "that does not exist yet",
4041
+ file=sys.stderr,
4042
+ )
3038
4043
  raise SystemExit(2)
3039
4044
 
3040
4045
 
3041
4046
  def _json_list(p):
3042
4047
  """True when `p` is a file holding a JSON array, whatever its entries."""
3043
4048
  try:
3044
- return p.is_file() and isinstance(json.loads(p.read_text(encoding="utf-8")), list)
4049
+ return p.is_file() and isinstance(
4050
+ json.loads(p.read_text(encoding="utf-8")), list
4051
+ )
3045
4052
  except (OSError, ValueError, UnicodeDecodeError, RecursionError):
3046
4053
  return False
3047
4054
 
@@ -3074,8 +4081,9 @@ def _write_gate_samples(bots, out, summaries, session_id, args):
3074
4081
  if summaries is not None and getattr(args, "apply_review", None):
3075
4082
  pack = pathlib.Path(args.apply_review)
3076
4083
  path = pack.with_name(f"{pack.stem}.reviewer-input.json")
3077
- text = json.dumps(_gate_samples(bots, summaries, session_id, args),
3078
- ensure_ascii=False, indent=2)
4084
+ text = json.dumps(
4085
+ _gate_samples(bots, summaries, session_id, args), ensure_ascii=False, indent=2
4086
+ )
3079
4087
  try:
3080
4088
  if summaries is None:
3081
4089
  _write_private(path, text)
@@ -3083,20 +4091,35 @@ def _write_gate_samples(bots, out, summaries, session_id, args):
3083
4091
  _renew_samples(path, text, args.reviewer_output)
3084
4092
  except OSError as e:
3085
4093
  if summaries is not None:
3086
- print(f"[lineage] ERROR: cannot write the gate samples {path}: {e}", file=sys.stderr)
4094
+ print(
4095
+ f"[lineage] ERROR: cannot write the gate samples {path}: {e}",
4096
+ file=sys.stderr,
4097
+ )
3087
4098
  raise SystemExit(2)
3088
4099
  return None
3089
4100
  print(f"[lineage] reviewer samples: {path}", file=sys.stderr, flush=True)
3090
4101
  if summaries is None:
3091
- print("[lineage] next: invoke Skill('oh-my-claudecode:critic') with "
3092
- "the JSON above; expected [{idx, recoverable, reason}, ...] "
3093
- "(PASS = 5/5 recoverable)", file=sys.stderr, flush=True)
4102
+ print(
4103
+ "[lineage] next: invoke Skill('oh-my-claudecode:critic') with "
4104
+ "the JSON above; expected [{idx, recoverable, reason}, ...] "
4105
+ "(PASS = 5/5 recoverable)",
4106
+ file=sys.stderr,
4107
+ flush=True,
4108
+ )
3094
4109
  elif pathlib.Path(args.reviewer_output).exists():
3095
- print(f"[lineage] reading the verdict at {args.reviewer_output}", file=sys.stderr, flush=True)
4110
+ print(
4111
+ f"[lineage] reading the verdict at {args.reviewer_output}",
4112
+ file=sys.stderr,
4113
+ flush=True,
4114
+ )
3096
4115
  else:
3097
- print(f"[lineage] next: have a critic agent judge the JSON above and write its "
3098
- f"[{{idx, id, key, recoverable, reason}}, ...] to {args.reviewer_output} "
3099
- "(one per sample; PASS = all recoverable)", file=sys.stderr, flush=True)
4116
+ print(
4117
+ f"[lineage] next: have a critic agent judge the JSON above and write its "
4118
+ f"[{{idx, id, key, recoverable, reason}}, ...] to {args.reviewer_output} "
4119
+ "(one per sample; PASS = all recoverable)",
4120
+ file=sys.stderr,
4121
+ flush=True,
4122
+ )
3100
4123
  return path
3101
4124
 
3102
4125
 
@@ -3106,26 +4129,32 @@ def _read_verdict(rop, timeout, set_aside):
3106
4129
  so a rerun waits for a fresh one instead of reusing it; anything else stays where it is.
3107
4130
  Exits 2 when it is missing, unreadable or not a non-empty list."""
3108
4131
  import time as _time
4132
+
3109
4133
  deadline = _time.time() + (0 if timeout is None else max(1, timeout))
3110
4134
  while not rop.exists() and _time.time() < deadline:
3111
4135
  _time.sleep(1)
3112
4136
  if not rop.exists():
3113
4137
  if timeout is None:
3114
- print(f"[lineage] ERROR: no verdict at {rop} yet; have a critic judge the samples, "
3115
- "write its answer there and run this again", file=sys.stderr)
4138
+ print(
4139
+ f"[lineage] ERROR: no verdict at {rop} yet; have a critic judge the samples, "
4140
+ "write its answer there and run this again",
4141
+ file=sys.stderr,
4142
+ )
3116
4143
  else:
3117
- print(f"[lineage] ERROR: reviewer-output not found within "
3118
- f"{timeout}s: {rop}", file=sys.stderr)
4144
+ print(
4145
+ f"[lineage] ERROR: reviewer-output not found within {timeout}s: {rop}",
4146
+ file=sys.stderr,
4147
+ )
3119
4148
  raise SystemExit(2)
3120
4149
  try:
3121
4150
  verdict = json.loads(rop.read_text(encoding="utf-8"))
3122
4151
  except (json.JSONDecodeError, OSError, UnicodeDecodeError, RecursionError) as e:
3123
- print(f"[lineage] ERROR: reviewer-output parse failed: {e}",
3124
- file=sys.stderr)
4152
+ print(f"[lineage] ERROR: reviewer-output parse failed: {e}", file=sys.stderr)
3125
4153
  raise SystemExit(2)
3126
4154
  if not isinstance(verdict, list) or not verdict:
3127
- print("[lineage] ERROR: reviewer-output is not a non-empty array",
3128
- file=sys.stderr)
4155
+ print(
4156
+ "[lineage] ERROR: reviewer-output is not a non-empty array", file=sys.stderr
4157
+ )
3129
4158
  raise SystemExit(2)
3130
4159
  if set_aside:
3131
4160
  _set_aside(rop)
@@ -3148,8 +4177,11 @@ def _set_aside(rop):
3148
4177
  try:
3149
4178
  os.replace(str(rop), str(dest))
3150
4179
  except OSError as e:
3151
- print(f"[lineage] WARN: could not set the verdict aside ({e}); remove {rop} "
3152
- "before the next run", file=sys.stderr)
4180
+ print(
4181
+ f"[lineage] WARN: could not set the verdict aside ({e}); remove {rop} "
4182
+ "before the next run",
4183
+ file=sys.stderr,
4184
+ )
3153
4185
  return
3154
4186
  print(f"[lineage] note: verdict moved to {dest}", file=sys.stderr)
3155
4187
 
@@ -3158,12 +4190,20 @@ def _verdict_gaps(verdict, samples):
3158
4190
  """Why a verdict does not answer these samples, or "": it needs one entry per sample
3159
4191
  idx and no other, each carrying the `id` and `key` of that sample (its turn and its
3160
4192
  content: an answer to the samples from before a fix carries an old key)."""
3161
- want = {s["idx"]: s for s in samples if isinstance(s, dict) and type(s.get("idx")) is int}
4193
+ want = {
4194
+ s["idx"]: s
4195
+ for s in samples
4196
+ if isinstance(s, dict) and type(s.get("idx")) is int
4197
+ }
3162
4198
  seen, stray = set(), []
3163
4199
  for v in verdict:
3164
4200
  idx = v.get("idx") if isinstance(v, dict) else None
3165
- if (type(idx) is not int or idx not in want or idx in seen
3166
- or any(v.get(k) != want[idx][k] for k in ("id", "key") if k in want[idx])):
4201
+ if (
4202
+ type(idx) is not int
4203
+ or idx not in want
4204
+ or idx in seen
4205
+ or any(v.get(k) != want[idx][k] for k in ("id", "key") if k in want[idx])
4206
+ ):
3167
4207
  stray.append(idx)
3168
4208
  else:
3169
4209
  seen.add(idx)
@@ -3177,15 +4217,27 @@ def _check_coverage(verdict, samples):
3177
4217
  """Exit 2 unless the verdict answers each of the samples in file `samples`."""
3178
4218
  try:
3179
4219
  gaps = _verdict_gaps(verdict, json.loads(samples.read_text(encoding="utf-8")))
3180
- except (OSError, ValueError, UnicodeDecodeError, AttributeError, RecursionError) as e:
4220
+ except (
4221
+ OSError,
4222
+ ValueError,
4223
+ UnicodeDecodeError,
4224
+ AttributeError,
4225
+ RecursionError,
4226
+ ) as e:
3181
4227
  gaps = f"cannot read the samples: {e}"
3182
4228
  if gaps:
3183
- print(f"[lineage] ERROR: the verdict does not answer {samples} ({gaps}); have the "
3184
- "critic judge every sample again", file=sys.stderr)
4229
+ print(
4230
+ f"[lineage] ERROR: the verdict does not answer {samples} ({gaps}); have the "
4231
+ "critic judge every sample again",
4232
+ file=sys.stderr,
4233
+ )
3185
4234
  for v in verdict:
3186
4235
  if isinstance(v, dict) and v.get("recoverable") is not True:
3187
- print(f" - not recoverable in it: idx={v.get('idx', '?')}: "
3188
- f"{v.get('reason', '(no reason)')}", file=sys.stderr)
4236
+ print(
4237
+ f" - not recoverable in it: idx={v.get('idx', '?')}: "
4238
+ f"{v.get('reason', '(no reason)')}",
4239
+ file=sys.stderr,
4240
+ )
3189
4241
  raise SystemExit(2)
3190
4242
 
3191
4243
 
@@ -3196,18 +4248,26 @@ def _enforce_gate(rop, timeout, samples=None, reviewed=False):
3196
4248
  verdict = _read_verdict(rop, timeout, set_aside=reviewed)
3197
4249
  if reviewed and samples is not None:
3198
4250
  _check_coverage(verdict, samples)
3199
- fails = [v for v in verdict
3200
- if not (isinstance(v, dict) and v.get("recoverable") is True)]
4251
+ fails = [
4252
+ v for v in verdict if not (isinstance(v, dict) and v.get("recoverable") is True)
4253
+ ]
3201
4254
  if fails:
3202
- print(f"[lineage] FAIL: quality gate {len(verdict) - len(fails)}/"
3203
- f"{len(verdict)} recoverable. Reasons:", file=sys.stderr)
4255
+ print(
4256
+ f"[lineage] FAIL: quality gate {len(verdict) - len(fails)}/"
4257
+ f"{len(verdict)} recoverable. Reasons:",
4258
+ file=sys.stderr,
4259
+ )
3204
4260
  for f in fails:
3205
4261
  if isinstance(f, dict):
3206
- print(f" - idx={f.get('idx', '?')}: "
3207
- f"{f.get('reason', '(no reason)')}", file=sys.stderr)
4262
+ print(
4263
+ f" - idx={f.get('idx', '?')}: {f.get('reason', '(no reason)')}",
4264
+ file=sys.stderr,
4265
+ )
3208
4266
  raise SystemExit(2)
3209
- print(f"[lineage] PASS: quality gate {len(verdict)}/{len(verdict)} "
3210
- "recoverable", file=sys.stderr)
4267
+ print(
4268
+ f"[lineage] PASS: quality gate {len(verdict)}/{len(verdict)} recoverable",
4269
+ file=sys.stderr,
4270
+ )
3211
4271
  if reviewed and samples is not None:
3212
4272
  try:
3213
4273
  samples.unlink()
@@ -3223,17 +4283,23 @@ def _run_reviewer_gate(turns, out, session_id, args, summaries=None):
3223
4283
  bots = [t for t in turns if t["role"] in ("assistant", "agent")]
3224
4284
  reviewed = summaries is not None
3225
4285
  if args.skip_reviewer:
3226
- print("[lineage] WARN: --skip-reviewer — quality gate not enforced",
3227
- file=sys.stderr)
4286
+ print(
4287
+ "[lineage] WARN: --skip-reviewer — quality gate not enforced",
4288
+ file=sys.stderr,
4289
+ )
3228
4290
  return
3229
4291
  if not bots or (reviewed and not args.reviewer_output):
3230
4292
  return
3231
4293
  samples = _write_gate_samples(bots, out, summaries, session_id, args)
3232
4294
  if args.reviewer_output:
3233
4295
  if reviewed and args.reviewer_timeout is None:
3234
- timeout = None # the run after the critic reads the verdict; this one does not wait
4296
+ timeout = None # the run after the critic reads the verdict; this one does not wait
3235
4297
  else:
3236
- timeout = REVIEWER_TIMEOUT if args.reviewer_timeout is None else args.reviewer_timeout
4298
+ timeout = (
4299
+ REVIEWER_TIMEOUT
4300
+ if args.reviewer_timeout is None
4301
+ else args.reviewer_timeout
4302
+ )
3237
4303
  _enforce_gate(pathlib.Path(args.reviewer_output), timeout, samples, reviewed)
3238
4304
 
3239
4305