@kaydash9999/banker-plugins 0.20.5 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +52 -0
- package/PRIVACY.md +25 -1
- package/README.md +63 -18
- package/bin/banker.js +173 -14
- package/bin/lib/codex-hooks.mjs +254 -0
- package/bin/lib/codex-routes.mjs +112 -0
- package/bin/lib/command-skill.cjs +19 -0
- package/bin/lib/telemetry-config.mjs +4 -1
- package/codex/command-skills/front-qa/SKILL.md +56 -0
- package/codex/command-skills/setup/SKILL.md +87 -0
- package/codex/hooks.json +29 -0
- package/codex/manifest.json +13 -10
- package/codex/skills/graceful-pause/SKILL.md +30 -0
- package/codex/skills/progress/SKILL.md +42 -0
- package/codex/skills/progress/scripts/progress-plan.mjs +722 -0
- package/codex/skills/progress/scripts/progress.mjs +295 -0
- package/codex/transform-matrix.md +18 -13
- package/commands/setup.md +2 -2
- package/hooks/obsidize-hook.mjs +116 -4
- package/hooks/progress-plan.mjs +1 -1
- package/hooks/telemetry-count-codex.mjs +309 -0
- package/hooks/update-notify.mjs +105 -14
- package/package.json +1 -1
- package/skills/3d-intro-build/references/verify-intro.mjs +15 -1
- package/skills/anti-drm/SKILL.md +1 -1
- package/skills/anti-drm/scripts/instruction-toggle.mjs +57 -4
- package/skills/append_wiki/SKILL.md +34 -8
- package/skills/arch-diagram/build_pptx_template.py +47 -28
- package/skills/audit-web-page/SKILL.md +1 -1
- package/skills/audit-web-page/harness.py +0 -1
- package/skills/compact-wiki/SKILL.md +37 -6
- package/skills/deep-research/SKILL.md +6 -0
- package/skills/lfm/SKILL.md +148 -0
- package/skills/lineage/SKILL.md +14 -4
- package/skills/lineage/lineage.py +1582 -516
- package/skills/motion-graphic-make/SKILL.md +1 -1
- package/skills/obsidizer/SKILL.md +44 -10
- package/skills/obsidizer/obsidize.mjs +42 -11
- package/skills/omc-patch/SKILL.md +6 -0
- package/skills/payload-mon/SKILL.md +3 -1
- package/skills/play-qa/SKILL.md +1 -1
- package/skills/ralph-qa/SKILL.md +1 -1
- package/skills/remains/SKILL.md +1 -1
- package/skills/rfp-author/SKILL.md +5 -3
- package/skills/rfp-author/references/vendor-rfp-structure.md +1 -1
- package/skills/rfp-author/scripts/md2docx.py +63 -34
- package/skills/setup-bypass-permissions/SKILL.md +28 -10
- package/skills/setup-bypass-permissions/scripts/codex-bypass.mjs +135 -0
- package/skills/setup-bypass-permissions/scripts/codex-config.mjs +210 -0
- package/skills/setup-bypass-permissions/scripts/fallback/bypass-permissions.py +66 -15
- package/skills/setup-omc-hud/SKILL.md +33 -3
- package/skills/setup-omc-hud/assets/codex-config.mjs +210 -0
- package/skills/setup-omc-hud/assets/codex-status-line.mjs +90 -0
- package/skills/smart-compact/SKILL.md +59 -13
- package/skills/smart-compact/scripts/codex-gate.mjs +385 -0
- package/skills/theme-selection/SKILL.md +1 -1
- package/skills/tone-compact/SKILL.md +1 -1
- package/skills/tone-compact/scripts/instruction-toggle.mjs +57 -4
- package/skills/ultra-chain/SKILL.md +10 -6
- package/skills/ultra-interview/SKILL.md +13 -2
- package/skills/ultra-ui-qa/SKILL.md +1 -1
- package/skills/update-banker/SKILL.md +31 -6
- package/skills/vertical-pptx/SKILL.md +1 -1
- package/skills/vertical-pptx/build_a4p.js +20 -2
- package/skills/vertical-pptx/build_a4p.py +560 -359
|
@@ -9,6 +9,7 @@ self-verifies the output markup. See SKILL.md for the full design and the
|
|
|
9
9
|
Runtime-agnostic: works from any transcript via --session / --from-transcript.
|
|
10
10
|
Auto-discovery assumes Claude Code's ~/.claude/projects/<encoded-cwd>/ layout.
|
|
11
11
|
"""
|
|
12
|
+
|
|
12
13
|
import argparse
|
|
13
14
|
import bisect
|
|
14
15
|
import hashlib
|
|
@@ -29,43 +30,59 @@ from html.parser import HTMLParser
|
|
|
29
30
|
from typing import Any
|
|
30
31
|
|
|
31
32
|
if sys.version_info < (3, 7):
|
|
32
|
-
sys.stderr.write(
|
|
33
|
-
|
|
33
|
+
sys.stderr.write(
|
|
34
|
+
"lineage requires Python 3.7+ (found %s)\n" % sys.version.split()[0]
|
|
35
|
+
)
|
|
34
36
|
sys.exit(2)
|
|
35
37
|
|
|
36
|
-
SCHEMA_VERSION = 2
|
|
37
|
-
SUMMARIZER_VERSION =
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
SCHEMA_VERSION = 2 # cache dir schema (v2: redacted summaries only)
|
|
39
|
+
SUMMARIZER_VERSION = (
|
|
40
|
+
4 # bump when summarize_turn or the page redaction changes (B-3 cache key)
|
|
41
|
+
)
|
|
42
|
+
# 3: cut from redacted text (3.0.2); 4: split secrets joined first (3.0.8)
|
|
43
|
+
USER_FOLD = 400 # collapse user messages longer than this (E-1)
|
|
44
|
+
ECHO_ASK = 40 # echo-exchange: user question length ceiling (A-4)
|
|
45
|
+
ECHO_REPLY = 5 # echo-exchange: assistant reply length ceiling (A-4)
|
|
42
46
|
_BLOCKQUOTE_MAX_DEPTH = 32 # blockquote recursion cap (C-2)
|
|
43
47
|
CACHE_BASE = pathlib.Path.home() / ".cache" / "lineage"
|
|
44
|
-
REVIEW_SCHEMA = "lineage-review/2"
|
|
48
|
+
REVIEW_SCHEMA = "lineage-review/2" # --emit-review pack, read back by --apply-review; 2: llm_key, pages redacted (3.0.2)
|
|
45
49
|
REVIEW_PART_SCHEMA = "lineage-review-part/1"
|
|
46
|
-
REVIEW_PART = 40
|
|
47
|
-
PREVIEW_HEAD = 1500
|
|
50
|
+
REVIEW_PART = 40 # turns per part file: one reviewer's share
|
|
51
|
+
PREVIEW_HEAD = 1500 # a long turn's preview: head + tail, where conclusions sit
|
|
48
52
|
PREVIEW_TAIL = 700
|
|
49
|
-
SUMMARY_MAX = 120
|
|
50
|
-
LLM_CACHE_VERSION = 1
|
|
51
|
-
REVIEWER_TIMEOUT = 60
|
|
52
|
-
DECISIONS_MAX =
|
|
53
|
-
|
|
53
|
+
SUMMARY_MAX = 120 # a reviewer's summary is cut to this, after redaction
|
|
54
|
+
LLM_CACHE_VERSION = 1 # bump to drop every cached reviewer decision
|
|
55
|
+
REVIEWER_TIMEOUT = 60 # --reviewer-timeout default, seconds
|
|
56
|
+
DECISIONS_MAX = (
|
|
57
|
+
1_000_000 # bytes in one decisions file (a 40-turn part answers in some 20 KB)
|
|
58
|
+
)
|
|
59
|
+
DECISIONS_TRIES = 200 # places a wrapped answer's list may start, tried at most
|
|
54
60
|
|
|
55
61
|
# ---------------------------------------------------------------- Noise / classify
|
|
56
62
|
# NOTE: JSONL records carry RAW `<...>` tags. Every regex below uses raw `<`/`>`
|
|
57
63
|
# (NOT the HTML-escaped `<` seen in the display-only change-request document).
|
|
58
|
-
_NOISE_TAGS = (
|
|
59
|
-
|
|
60
|
-
|
|
64
|
+
_NOISE_TAGS = (
|
|
65
|
+
"system-reminder",
|
|
66
|
+
"local-command-caveat",
|
|
67
|
+
"local-command-stdout",
|
|
68
|
+
"local-command-stderr",
|
|
69
|
+
"teammate-message",
|
|
70
|
+
"command-message",
|
|
71
|
+
"command-args",
|
|
72
|
+
"command-name",
|
|
73
|
+
"task-notification",
|
|
74
|
+
)
|
|
61
75
|
_NOISE_BLOCK_RE = re.compile(r"<(%s)\b[^>]*>.*?</\1>" % "|".join(_NOISE_TAGS), re.S)
|
|
62
76
|
_NOISE_CLOSE_RE = re.compile(r"</(%s)>" % "|".join(_NOISE_TAGS))
|
|
63
|
-
_NOISE_OPEN_RE = re.compile(
|
|
77
|
+
_NOISE_OPEN_RE = re.compile(
|
|
78
|
+
r"<(?:%s)\b" % "|".join(_NOISE_TAGS)
|
|
79
|
+
) # leading orphan open (R7)
|
|
64
80
|
|
|
65
81
|
_SKILL_BODY_RE = re.compile(r"^\s*Base directory for this skill:")
|
|
66
82
|
_WORKFLOW_BODY_RE = re.compile(r'^\s*Run the "[^"\n]+" workflow\.')
|
|
67
83
|
_COMPACTION_RE = re.compile(
|
|
68
|
-
r"^\s*This session is being continued from a previous conversation"
|
|
84
|
+
r"^\s*This session is being continued from a previous conversation"
|
|
85
|
+
)
|
|
69
86
|
_HOOK_FEEDBACK_RE = re.compile(r"^\s*Stop hook feedback:")
|
|
70
87
|
_INTERRUPT_RE = re.compile(r"^\s*\[Request interrupted")
|
|
71
88
|
_IMAGE_NOTE_RE = re.compile(r"^\s*\[Image:\s*original\s+\d+x\d+")
|
|
@@ -74,18 +91,48 @@ _SYS_ERROR_RE = re.compile(r"^\s*(?:Login expired|Please run /login|API Error)")
|
|
|
74
91
|
# Manipulation commands: judged by NAME, not by "has no args" (A-4). Plugin
|
|
75
92
|
# commands (name contains ':') are never manipulation — always preserved.
|
|
76
93
|
_HARNESS_CMDS = {
|
|
77
|
-
"agents",
|
|
78
|
-
"
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
94
|
+
"agents",
|
|
95
|
+
"bug",
|
|
96
|
+
"clear",
|
|
97
|
+
"compact",
|
|
98
|
+
"config",
|
|
99
|
+
"context",
|
|
100
|
+
"copy",
|
|
101
|
+
"cost",
|
|
102
|
+
"doctor",
|
|
103
|
+
"effort",
|
|
104
|
+
"exit",
|
|
105
|
+
"export",
|
|
106
|
+
"help",
|
|
107
|
+
"hooks",
|
|
108
|
+
"ide",
|
|
109
|
+
"install-github-app",
|
|
110
|
+
"login",
|
|
111
|
+
"logout",
|
|
112
|
+
"mcp",
|
|
113
|
+
"memory",
|
|
114
|
+
"migrate-installer",
|
|
115
|
+
"model",
|
|
116
|
+
"output-style",
|
|
117
|
+
"permissions",
|
|
118
|
+
"privacy-settings",
|
|
119
|
+
"quit",
|
|
120
|
+
"release-notes",
|
|
121
|
+
"resume",
|
|
122
|
+
"status",
|
|
123
|
+
"statusline",
|
|
124
|
+
"terminal-setup",
|
|
125
|
+
"todos",
|
|
126
|
+
"upgrade",
|
|
127
|
+
"usage",
|
|
128
|
+
"vim",
|
|
83
129
|
}
|
|
84
130
|
# Deliberately NOT in the set: /add-dir (changes readable dirs), /init (writes
|
|
85
131
|
# CLAUDE.md) — these do real work and are kept.
|
|
86
132
|
|
|
87
133
|
_AGENT_WRAP_RE = re.compile(
|
|
88
|
-
r"<(agent-message|teammate-message)\b([^>]*)>(.*?)</\1>", re.S
|
|
134
|
+
r"<(agent-message|teammate-message)\b([^>]*)>(.*?)</\1>", re.S
|
|
135
|
+
)
|
|
89
136
|
_AGENT_FROM_RE = re.compile(r'(?:from|teammate_id)="([^"]+)"')
|
|
90
137
|
_PEER_LEAD_RE = re.compile(r"^\s*Another Claude session sent a message:\s*")
|
|
91
138
|
_BARE_CMD_RE = re.compile(r"^/([\w-]+)\s*$")
|
|
@@ -105,7 +152,7 @@ def clean_user_text(text, drop_harness=True):
|
|
|
105
152
|
"""
|
|
106
153
|
out = _NOISE_CLOSE_RE.sub("", _NOISE_BLOCK_RE.sub("", text)).strip()
|
|
107
154
|
|
|
108
|
-
mb = _BARE_CMD_RE.match(out)
|
|
155
|
+
mb = _BARE_CMD_RE.match(out) # bare `/name` with no wrapper
|
|
109
156
|
if mb and drop_harness and mb.group(1) in _HARNESS_CMDS:
|
|
110
157
|
return ""
|
|
111
158
|
|
|
@@ -140,7 +187,7 @@ def split_agent_message(text):
|
|
|
140
187
|
return None, None
|
|
141
188
|
who = _AGENT_FROM_RE.search(m.group(2) or "")
|
|
142
189
|
body = m.group(3).strip()
|
|
143
|
-
if body.startswith('{"type"'):
|
|
190
|
+
if body.startswith('{"type"'): # idle/status notification
|
|
144
191
|
return (who.group(1) if who else "agent"), ""
|
|
145
192
|
return (who.group(1) if who else "agent"), body
|
|
146
193
|
|
|
@@ -152,10 +199,18 @@ SECRET_PATTERNS = [
|
|
|
152
199
|
("GitHubPAT", re.compile(r"gh[posru]_[A-Za-z0-9]{36}")),
|
|
153
200
|
("Slack", re.compile(r"xox[bpars]-[A-Za-z0-9-]{10,}")),
|
|
154
201
|
("JWT", re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+")),
|
|
155
|
-
(
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
202
|
+
(
|
|
203
|
+
"PrivateKey",
|
|
204
|
+
re.compile(
|
|
205
|
+
r"-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]+?-----END [A-Z ]+PRIVATE KEY-----"
|
|
206
|
+
),
|
|
207
|
+
),
|
|
208
|
+
(
|
|
209
|
+
"Password",
|
|
210
|
+
re.compile(
|
|
211
|
+
r"(?i)(?:password|암호|비번|패스워드)\s*[:=]\s*['\"]([^'\"\n]{4,})['\"]"
|
|
212
|
+
),
|
|
213
|
+
),
|
|
159
214
|
]
|
|
160
215
|
ENTROPY_LONG = re.compile(r"[A-Za-z0-9+/=]{32,}")
|
|
161
216
|
|
|
@@ -194,7 +249,7 @@ class _Trail:
|
|
|
194
249
|
if r.startswith("[REDACTED"):
|
|
195
250
|
out += new * len(r)
|
|
196
251
|
elif r:
|
|
197
|
-
out += pos[s:s + 4] + new * (len(r) - 8) + pos[e - 4:e]
|
|
252
|
+
out += pos[s : s + 4] + new * (len(r) - 8) + pos[e - 4 : e]
|
|
198
253
|
at = e
|
|
199
254
|
self.pos = out + pos[at:]
|
|
200
255
|
|
|
@@ -207,6 +262,7 @@ class _Trail:
|
|
|
207
262
|
if r != m.group(0):
|
|
208
263
|
edits.append((m.start(), m.end(), r))
|
|
209
264
|
return r
|
|
265
|
+
|
|
210
266
|
out = pat.sub(noted, text)
|
|
211
267
|
self.carry(edits)
|
|
212
268
|
return out
|
|
@@ -234,7 +290,9 @@ def _sub_for(trail):
|
|
|
234
290
|
def _shows_more(trail, shown):
|
|
235
291
|
"""Whether the text `trail` follows shows a character of the text it started as that `shown` (a _Trail.shown of
|
|
236
292
|
another text from the same start) has hidden."""
|
|
237
|
-
return (
|
|
293
|
+
return (
|
|
294
|
+
int.from_bytes(trail.shown(), "little") & ~int.from_bytes(shown, "little")
|
|
295
|
+
) != 0
|
|
238
296
|
|
|
239
297
|
|
|
240
298
|
# Characters that cut a value for the patterns while no person reads them as a separator: Unicode's default-ignorable
|
|
@@ -244,14 +302,46 @@ def _shows_more(trail, shown):
|
|
|
244
302
|
# (a lone surrogate became U+FFFD). A value cut by them is put back together where a pattern finds it once they
|
|
245
303
|
# are gone (3.0.8).
|
|
246
304
|
_SPLITTER_RANGES = (
|
|
247
|
-
(0x0000, 0x0008),
|
|
248
|
-
(
|
|
249
|
-
(
|
|
250
|
-
(
|
|
251
|
-
(
|
|
252
|
-
(
|
|
253
|
-
(
|
|
254
|
-
|
|
305
|
+
(0x0000, 0x0008),
|
|
306
|
+
(0x000E, 0x001B),
|
|
307
|
+
(0x007F, 0x0084),
|
|
308
|
+
(0x0086, 0x009F),
|
|
309
|
+
(0x00A0, 0x00A0),
|
|
310
|
+
(0x00AD, 0x00AD),
|
|
311
|
+
(0x034F, 0x034F),
|
|
312
|
+
(0x0600, 0x0605),
|
|
313
|
+
(0x061C, 0x061C),
|
|
314
|
+
(0x06DD, 0x06DD),
|
|
315
|
+
(0x070F, 0x070F),
|
|
316
|
+
(0x0890, 0x0891),
|
|
317
|
+
(0x08E2, 0x08E2),
|
|
318
|
+
(0x115F, 0x1160),
|
|
319
|
+
(0x1680, 0x1680),
|
|
320
|
+
(0x17B4, 0x17B5),
|
|
321
|
+
(0x180B, 0x180F),
|
|
322
|
+
(0x2000, 0x200F),
|
|
323
|
+
(0x202A, 0x202F),
|
|
324
|
+
(0x205F, 0x206F),
|
|
325
|
+
(0x2800, 0x2800),
|
|
326
|
+
(0x3000, 0x3000),
|
|
327
|
+
(0x3164, 0x3164),
|
|
328
|
+
(0xE000, 0xF8FF),
|
|
329
|
+
(0xFE00, 0xFE0F),
|
|
330
|
+
(0xFEFF, 0xFEFF),
|
|
331
|
+
(0xFFA0, 0xFFA0),
|
|
332
|
+
(0xFFF0, 0xFFFD),
|
|
333
|
+
(0x110BD, 0x110BD),
|
|
334
|
+
(0x110CD, 0x110CD),
|
|
335
|
+
(0x13430, 0x1343F),
|
|
336
|
+
(0x1BCA0, 0x1BCA3),
|
|
337
|
+
(0x1D173, 0x1D17A),
|
|
338
|
+
(0xE0000, 0xE0FFF),
|
|
339
|
+
(0xF0000, 0xFFFFD),
|
|
340
|
+
(0x100000, 0x10FFFD),
|
|
341
|
+
)
|
|
342
|
+
_SPLITTERS = "".join(
|
|
343
|
+
chr(a) if a == b else chr(a) + "-" + chr(b) for a, b in _SPLITTER_RANGES
|
|
344
|
+
)
|
|
255
345
|
_SPLITTER_RE = re.compile("[" + _SPLITTERS + "]")
|
|
256
346
|
_UNSPLIT_RE = re.compile("[^" + _SPLITTERS + "]+")
|
|
257
347
|
# A character a value can hold: past a splitter next to a found value, one of these means another value begins.
|
|
@@ -273,7 +363,9 @@ def _matches(text, patterns, entropy):
|
|
|
273
363
|
"""The matches of `patterns` in `text`, and with `entropy` the runs the entropy rule hides."""
|
|
274
364
|
found = [m for pat in patterns for m in pat.finditer(text)]
|
|
275
365
|
if entropy:
|
|
276
|
-
found += [
|
|
366
|
+
found += [
|
|
367
|
+
m for m in ENTROPY_LONG.finditer(text) if shannon_entropy(m.group(0)) >= 4.5
|
|
368
|
+
]
|
|
277
369
|
return found
|
|
278
370
|
|
|
279
371
|
|
|
@@ -281,9 +373,11 @@ def _cut_spans(text, joined, firsts, origins, matches):
|
|
|
281
373
|
"""Where in `text` the values `matches` found in `joined` lie, for those splitters cut or touch. A stretch takes
|
|
282
374
|
in the splitters next to its value when the character past them is no value character (so the value's
|
|
283
375
|
neighbours are those it had in `joined`); a splitter between the value and another value stays."""
|
|
376
|
+
|
|
284
377
|
def at(j): # where joined[j] is in text
|
|
285
378
|
k = bisect.bisect_right(firsts, j) - 1
|
|
286
379
|
return origins[k] + j - firsts[k]
|
|
380
|
+
|
|
287
381
|
spans = []
|
|
288
382
|
for m in matches:
|
|
289
383
|
s, e = at(m.start()), at(m.end() - 1) + 1
|
|
@@ -310,7 +404,9 @@ def _kept(spans, plain):
|
|
|
310
404
|
at += 1
|
|
311
405
|
while heap and heap[0][0] <= s:
|
|
312
406
|
heapq.heappop(heap)
|
|
313
|
-
if not any(
|
|
407
|
+
if not any(
|
|
408
|
+
(o_s <= s and e <= o_e) or not (s <= o_s and o_e <= e) for o_e, o_s in heap
|
|
409
|
+
):
|
|
314
410
|
out.append((s, e))
|
|
315
411
|
return out
|
|
316
412
|
|
|
@@ -333,10 +429,18 @@ def _rejoin_split(text, patterns, entropy=False, trail=None):
|
|
|
333
429
|
if not _SPLITTER_RE.search(text):
|
|
334
430
|
return text
|
|
335
431
|
joined, firsts, origins = _unsplit_runs(text)
|
|
336
|
-
spans = _cut_spans(
|
|
432
|
+
spans = _cut_spans(
|
|
433
|
+
text, joined, firsts, origins, _matches(joined, patterns, entropy)
|
|
434
|
+
)
|
|
337
435
|
if spans:
|
|
338
|
-
spans = _kept(
|
|
339
|
-
|
|
436
|
+
spans = _kept(
|
|
437
|
+
spans, [(m.start(), m.end()) for m in _matches(text, patterns, entropy)]
|
|
438
|
+
)
|
|
439
|
+
drops = [
|
|
440
|
+
(m.start(), m.end(), "")
|
|
441
|
+
for s, e in _merged(sorted(spans))
|
|
442
|
+
for m in _SPLITTER_RE.finditer(text, s, e)
|
|
443
|
+
]
|
|
340
444
|
if trail is not None:
|
|
341
445
|
trail.carry(drops)
|
|
342
446
|
out, at = [], 0
|
|
@@ -350,9 +454,11 @@ def _hide_patterns(out, mode, sub, counts):
|
|
|
350
454
|
"""`out` with each value the page patterns and the entropy rule find replaced by its marker, or by a mask with
|
|
351
455
|
mode mask; `counts` adds them up by name."""
|
|
352
456
|
for name, pat in SECRET_PATTERNS:
|
|
457
|
+
|
|
353
458
|
def repl(m, n=name):
|
|
354
459
|
counts[n] = counts.get(n, 0) + 1
|
|
355
460
|
return _mask(m.group(0)) if mode == "mask" else f"[REDACTED:{n}]"
|
|
461
|
+
|
|
356
462
|
out = sub(pat, repl, out)
|
|
357
463
|
|
|
358
464
|
def ent_repl(m):
|
|
@@ -361,6 +467,7 @@ def _hide_patterns(out, mode, sub, counts):
|
|
|
361
467
|
counts["entropy"] = counts.get("entropy", 0) + 1
|
|
362
468
|
return _mask(s) if mode == "mask" else "[REDACTED:entropy]"
|
|
363
469
|
return s
|
|
470
|
+
|
|
364
471
|
return sub(ENTROPY_LONG, ent_repl, out)
|
|
365
472
|
|
|
366
473
|
|
|
@@ -371,6 +478,7 @@ def _scan_detected(held, sub, counts):
|
|
|
371
478
|
detect-secrets hid anything."""
|
|
372
479
|
from detect_secrets import SecretsCollection
|
|
373
480
|
from detect_secrets.settings import default_settings
|
|
481
|
+
|
|
374
482
|
tf = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
|
|
375
483
|
tmp = tf.name
|
|
376
484
|
try:
|
|
@@ -383,9 +491,10 @@ def _scan_detected(held, sub, counts):
|
|
|
383
491
|
for s in col[fname]:
|
|
384
492
|
sv = s.secret_value
|
|
385
493
|
if sv and sv in held[0]:
|
|
386
|
-
held[0] = sub(
|
|
387
|
-
|
|
388
|
-
|
|
494
|
+
held[0] = sub(
|
|
495
|
+
re.compile(re.escape(sv)), lambda m: "[REDACTED:ds]", held[0]
|
|
496
|
+
)
|
|
497
|
+
counts["ds:detect-secrets"] = counts.get("ds:detect-secrets", 0) + 1
|
|
389
498
|
finally:
|
|
390
499
|
try:
|
|
391
500
|
os.unlink(tmp)
|
|
@@ -400,13 +509,18 @@ def _hide_detected(out, sub, counts):
|
|
|
400
509
|
_scan_detected(held, sub, counts)
|
|
401
510
|
except ImportError:
|
|
402
511
|
if not getattr(redact, "_ds_warned", False):
|
|
403
|
-
print(
|
|
404
|
-
|
|
512
|
+
print(
|
|
513
|
+
"[lineage] WARN: detect-secrets unavailable, using fallback "
|
|
514
|
+
"(pip install 'detect-secrets>=1.5')",
|
|
515
|
+
file=sys.stderr,
|
|
516
|
+
)
|
|
405
517
|
redact._ds_warned = True
|
|
406
518
|
except Exception as e:
|
|
407
519
|
if not getattr(redact, "_ds_err_warned", False):
|
|
408
|
-
print(
|
|
409
|
-
|
|
520
|
+
print(
|
|
521
|
+
f"[lineage] WARN: detect-secrets error ({e}), continuing with fallback",
|
|
522
|
+
file=sys.stderr,
|
|
523
|
+
)
|
|
410
524
|
redact._ds_err_warned = True
|
|
411
525
|
return held[0]
|
|
412
526
|
|
|
@@ -417,8 +531,15 @@ def _hide_extra(out, extra, sub, counts):
|
|
|
417
531
|
for kw in [k.strip() for k in extra.split(",") if k.strip()]:
|
|
418
532
|
pat = re.compile(r"\[REDACTED(?::[^\]\s]*)?\]|" + re.escape(kw), re.IGNORECASE)
|
|
419
533
|
hits = []
|
|
420
|
-
out = sub(
|
|
421
|
-
|
|
534
|
+
out = sub(
|
|
535
|
+
pat,
|
|
536
|
+
lambda m: (
|
|
537
|
+
m.group(0)
|
|
538
|
+
if m.group(0).upper().startswith("[REDACTED")
|
|
539
|
+
else hits.append(1) or "[REDACTED]"
|
|
540
|
+
),
|
|
541
|
+
out,
|
|
542
|
+
)
|
|
422
543
|
if hits:
|
|
423
544
|
counts[f"custom:{kw}"] = len(hits)
|
|
424
545
|
return out
|
|
@@ -440,8 +561,10 @@ def redact(text: str, extra: "str | None" = None, mode: str = "full", trail=None
|
|
|
440
561
|
# follow (the end of an item in an argument list written without spaces). A match starts only
|
|
441
562
|
# there, so no user runs past the place where the next match may start: a long run stays linear,
|
|
442
563
|
# and a user joined to a variable by a - ("$USER"-bot) is read whole.
|
|
443
|
-
_CU_USER = (
|
|
444
|
-
|
|
564
|
+
_CU_USER = (
|
|
565
|
+
r"(?:[^\s:=\"'`(]|[\"'`(](?!-(?:u|[A-Za-z0-9]{1,6}u[\s\"']|-user[\s=\"'])"
|
|
566
|
+
r"|,\\{0,7}[\"']))"
|
|
567
|
+
)
|
|
445
568
|
# The quotes that may open the value, bare or escaped (\" and \\\" in a quoted command, '\'' and
|
|
446
569
|
# '"'"' from bash and shlex.quote, ^" in cmd, `" in PowerShell, $' in bash): up to six in a row,
|
|
447
570
|
# none that closes an item of an argument list.
|
|
@@ -452,9 +575,16 @@ _CU_LEAD = r"(?:[\\^`$]{0,7}[\"'](?!,\\{0,7}[\"'])){0,6}"
|
|
|
452
575
|
# (mktemp -u "${TMPDIR:-/tmp}/x"). A literal password, or a default that is no path or holds a
|
|
453
576
|
# colon, is hidden.
|
|
454
577
|
_CU_REF = r"(?:\$\{?\w+\}?|%\w+%|\$\([^\s()]*(?:\s+[^\s()]+)*\))"
|
|
455
|
-
_CU_SKIP = (
|
|
456
|
-
|
|
457
|
-
|
|
578
|
+
_CU_SKIP = (
|
|
579
|
+
r"(?!"
|
|
580
|
+
+ _CU_LEAD
|
|
581
|
+
+ r"(?:\+[\"']?%[-_0^#]?[A-Za-z%]|(?:\d+:\d+|(?:"
|
|
582
|
+
+ _CU_REF
|
|
583
|
+
+ r"|[^\s:=\"'`(]*\)):"
|
|
584
|
+
+ _CU_REF
|
|
585
|
+
+ r")(?![^\s\"'`;|&)\\^])"
|
|
586
|
+
r"|\$\{\w+:[-=?+]?[/~.][^\s}\"'`(:]*\}[^\s:]*(?!\S)))"
|
|
587
|
+
)
|
|
458
588
|
|
|
459
589
|
# Patterns every page (the reviewed flow and --rulebase) and every part file hides with before the
|
|
460
590
|
# page patterns above, while keys are whole (3.0.2).
|
|
@@ -473,28 +603,52 @@ _MARKER_UNIT = r"\[REDACTED:\w+\]"
|
|
|
473
603
|
|
|
474
604
|
REVIEW_SECRET_PATTERNS = [
|
|
475
605
|
("AnthropicKey", re.compile(r"(?<![A-Za-z0-9])sk-ant-[A-Za-z0-9_-]{20,}")),
|
|
476
|
-
(
|
|
606
|
+
(
|
|
607
|
+
"OpenAIKey",
|
|
608
|
+
re.compile(r"(?<![A-Za-z0-9-])sk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}"),
|
|
609
|
+
),
|
|
477
610
|
("GoogleKey", re.compile(r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}")),
|
|
478
611
|
# bounded repeats: a long run of `key` or `a.` would otherwise take quadratic time
|
|
479
|
-
(
|
|
612
|
+
(
|
|
613
|
+
"HexKey",
|
|
614
|
+
re.compile(
|
|
615
|
+
r"(?i)(?:key|token|secret)[\w-]{0,100}\s*[:=]\s*['\"]?[0-9a-f]{32,}"
|
|
616
|
+
),
|
|
617
|
+
),
|
|
480
618
|
# a value, not a path, a variable or a redaction ($PWD is a variable, PGPASSWORD a name);
|
|
481
619
|
# markdown, a table cell or a bracket that closes after the value stays outside the match
|
|
482
|
-
(
|
|
483
|
-
|
|
484
|
-
|
|
620
|
+
(
|
|
621
|
+
"PasswordBare",
|
|
622
|
+
re.compile(
|
|
623
|
+
r"(?i)(?<!\$)(?:password|passwd|passcode|pwd|암호|비번|비밀\s?번호|패스워드)\s*[:=]\s*"
|
|
624
|
+
r"(?![/~$])" + _NOT_REDACTED + r"(?=[^\s'\"]{6})[^\s'\"]*[^\s'\"`|*)]"
|
|
625
|
+
),
|
|
626
|
+
),
|
|
485
627
|
# the password runs to the last @ before the host: it may hold an @ of its own. A marker is
|
|
486
628
|
# no user or password, so a second pass leaves a token-only URL (https://[REDACTED:...]@host).
|
|
487
|
-
(
|
|
488
|
-
|
|
629
|
+
(
|
|
630
|
+
"UrlCredential",
|
|
631
|
+
re.compile(
|
|
632
|
+
r"(?i)[a-z][a-z0-9+.-]{0,30}://(?:"
|
|
633
|
+
+ _MARKER_UNIT
|
|
634
|
+
+ r"|[^\s:@/])*:"
|
|
635
|
+
+ _NOT_MARKED
|
|
636
|
+
+ r"(?!\[REDACTED)[^\s/]{3,}@"
|
|
637
|
+
),
|
|
638
|
+
),
|
|
489
639
|
("Bearer", re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{16,}")),
|
|
490
640
|
# Authorization then `:`, `=`, `=>` or headers["Authorization"] =, a call's ("Authorization",
|
|
491
641
|
# "...") and a HAR name/value pair, or a space before a quote (nginx, Apache)
|
|
492
|
-
(
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
642
|
+
(
|
|
643
|
+
"BasicAuth",
|
|
644
|
+
re.compile(
|
|
645
|
+
r"(?i)\bauthorization"
|
|
646
|
+
r"(?:[\"']?\]?\s*(?:=>|[:=])"
|
|
647
|
+
r"|[\"']\s*,\s*(?:[\"']value[\"']\s*:\s*)?(?=[\"'])"
|
|
648
|
+
r"|\s+(?=[\"']))"
|
|
649
|
+
r"\s*[\"']?basic\s+[A-Za-z0-9+/]{8,}={0,2}"
|
|
650
|
+
),
|
|
651
|
+
),
|
|
498
652
|
# curl's -u and --user flags, alone or last in a bundle (-su, -4u), with a user and password
|
|
499
653
|
# after a space, an = or nothing, quoted together or apart ("user":"pw"), with quotes bare,
|
|
500
654
|
# escaped or nested (_CU_LEAD) or none, in inline code or parentheses, or as two items of an
|
|
@@ -503,15 +657,38 @@ REVIEW_SECRET_PATTERNS = [
|
|
|
503
657
|
# long key as the user with no password, count too. The separators share no character with
|
|
504
658
|
# the user name, so a long run of them stays linear. The values _CU_SKIP names and a
|
|
505
659
|
# host:/path (rsync) are left, and so is a marker an earlier pass left (-u [REDACTED:JWT]).
|
|
506
|
-
(
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
660
|
+
(
|
|
661
|
+
"CurlUser",
|
|
662
|
+
re.compile(
|
|
663
|
+
r"(?<![^\s\"'`(])(?:-u\s*|-[A-Za-z0-9]{1,6}u\s+|--user(?:\s+|=))"
|
|
664
|
+
+ _CU_SKIP
|
|
665
|
+
+ _CU_LEAD
|
|
666
|
+
+ r"(?:(?:"
|
|
667
|
+
+ _MARKER_UNIT
|
|
668
|
+
+ r"|"
|
|
669
|
+
+ _CU_USER
|
|
670
|
+
+ r")*:"
|
|
671
|
+
+ _NOT_MARKED
|
|
672
|
+
+ r"(?!/|\[REDACTED)\S{3,}|"
|
|
673
|
+
+ _CU_USER
|
|
674
|
+
+ r"{15,}:"
|
|
675
|
+
+ _NOT_MARKED
|
|
676
|
+
+ r"(?=[\s\"'`).,;\\]|$))"
|
|
677
|
+
r"|(\\{0,7}[\"'])(?:-[A-Za-z0-9]{0,6}u|--user)\1\s*,\s*(\\{0,7}[\"'])"
|
|
678
|
+
+ _CU_SKIP
|
|
679
|
+
+ r"(?:(?:"
|
|
680
|
+
+ _MARKER_UNIT
|
|
681
|
+
+ r"|"
|
|
682
|
+
+ _CU_USER
|
|
683
|
+
+ r")*:"
|
|
684
|
+
+ _NOT_MARKED
|
|
685
|
+
+ r"(?!/|\[REDACTED)[^\s\"']{3,}|"
|
|
686
|
+
+ _CU_USER
|
|
687
|
+
+ r"{15,}:"
|
|
688
|
+
+ _NOT_MARKED
|
|
689
|
+
+ r")\2"
|
|
690
|
+
),
|
|
691
|
+
),
|
|
515
692
|
]
|
|
516
693
|
|
|
517
694
|
|
|
@@ -521,7 +698,9 @@ def _reviewer_patterns(text, trail=None):
|
|
|
521
698
|
counts, sub = {}, _sub_for(trail)
|
|
522
699
|
for name, pat in REVIEW_SECRET_PATTERNS:
|
|
523
700
|
hits = []
|
|
524
|
-
text = sub(
|
|
701
|
+
text = sub(
|
|
702
|
+
pat, lambda m, marker=f"[REDACTED:{name}]": hits.append(1) or marker, text
|
|
703
|
+
)
|
|
525
704
|
if hits:
|
|
526
705
|
counts[name] = len(hits)
|
|
527
706
|
return text, counts
|
|
@@ -531,8 +710,11 @@ def _rejoin_secrets(text, extra=None, trail=None):
|
|
|
531
710
|
"""`text` with each secret a splitter cuts put back together, for every pattern a page or a reviewer's text is
|
|
532
711
|
hidden with: the reviewer's, the page's, the entropy rule and the --redact-extra keywords (3.0.8)."""
|
|
533
712
|
keywords = [k.strip() for k in (extra or "").split(",") if k.strip()]
|
|
534
|
-
patterns = (
|
|
535
|
-
|
|
713
|
+
patterns = (
|
|
714
|
+
[p for _, p in REVIEW_SECRET_PATTERNS]
|
|
715
|
+
+ [p for _, p in SECRET_PATTERNS]
|
|
716
|
+
+ [re.compile(re.escape(k), re.IGNORECASE) for k in keywords]
|
|
717
|
+
)
|
|
536
718
|
return _rejoin_split(text, patterns, entropy=True, trail=trail)
|
|
537
719
|
|
|
538
720
|
|
|
@@ -601,7 +783,9 @@ def _hidden_by(floor, base):
|
|
|
601
783
|
j = i
|
|
602
784
|
while j < n and pos[j] < 0:
|
|
603
785
|
j += 1
|
|
604
|
-
out.append(
|
|
786
|
+
out.append(
|
|
787
|
+
(pos[i - 1] + 1 if i else 0, pos[j] if j < n else base.n, floor[i:j])
|
|
788
|
+
)
|
|
605
789
|
i = j
|
|
606
790
|
return out
|
|
607
791
|
|
|
@@ -634,7 +818,10 @@ def _with_rejoins(text, extra, one_pass):
|
|
|
634
818
|
[found])."""
|
|
635
819
|
red, founds, trail = _join_first(text, extra, one_pass)
|
|
636
820
|
if trail is None:
|
|
637
|
-
return
|
|
821
|
+
return (
|
|
822
|
+
red,
|
|
823
|
+
founds,
|
|
824
|
+
) # nothing joined: that was the one pass over the text as it stands
|
|
638
825
|
base = _Trail(len(text))
|
|
639
826
|
floor, found = one_pass(text, base)
|
|
640
827
|
joined = _kinds(founds)
|
|
@@ -662,7 +849,9 @@ def review_redact(text, extra=None):
|
|
|
662
849
|
are whole (the entropy rule would otherwise cut a key and leave the rest), then fully
|
|
663
850
|
redacted (never masked, whatever --redact-mode says). A secret a splitter cuts is put back
|
|
664
851
|
together (3.0.8). Returns (text, count)."""
|
|
665
|
-
red, founds = _with_rejoins(
|
|
852
|
+
red, founds = _with_rejoins(
|
|
853
|
+
text, extra, lambda t, trail=None: _review_pass(t, extra, trail)
|
|
854
|
+
)
|
|
666
855
|
return red, sum(sum(f.values()) for f in founds)
|
|
667
856
|
|
|
668
857
|
|
|
@@ -681,7 +870,9 @@ def page_redact(text, extra=None, mode="full"):
|
|
|
681
870
|
splitter cuts is put back together (3.0.8). Returns (text, count_by_kind). A second pass
|
|
682
871
|
over its output finds nothing, but for two values glued with no separator at all (a key
|
|
683
872
|
run into `Bearer`), where the second shows only once the first is a marker, as in 3.0.7."""
|
|
684
|
-
red, founds = _with_rejoins(
|
|
873
|
+
red, founds = _with_rejoins(
|
|
874
|
+
text, extra, lambda t, trail=None: _page_pass(t, extra, mode, trail)
|
|
875
|
+
)
|
|
685
876
|
counts = {}
|
|
686
877
|
for found in founds:
|
|
687
878
|
for k, v in found.items():
|
|
@@ -723,23 +914,111 @@ def encode_cwd(cwd) -> str:
|
|
|
723
914
|
return re.sub(r"[^A-Za-z0-9]", "-", str(cwd))
|
|
724
915
|
|
|
725
916
|
|
|
917
|
+
def codex_sessions_dir():
|
|
918
|
+
"""Where Codex keeps its session records (rollouts): <CODEX_HOME>/sessions/YYYY/MM/DD/rollout-*.jsonl."""
|
|
919
|
+
home = os.environ.get("CODEX_HOME") or str(pathlib.Path.home() / ".codex")
|
|
920
|
+
return pathlib.Path(home) / "sessions"
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def _codex_thread():
|
|
924
|
+
"""This Codex session's id when lineage runs inside Codex (Codex gives its shell CODEX_THREAD_ID, the id at the
|
|
925
|
+
end of the rollout file name), else None."""
|
|
926
|
+
tid = os.environ.get("CODEX_THREAD_ID", "")
|
|
927
|
+
return tid if re.fullmatch(r"[0-9A-Za-z-]{8,64}", tid) else None
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def _mtime(path):
|
|
931
|
+
"""A file's change time, or 0 for one that went away between the listing and the look."""
|
|
932
|
+
try:
|
|
933
|
+
return path.stat().st_mtime
|
|
934
|
+
except OSError:
|
|
935
|
+
return 0
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
def _codex_rollouts(pattern):
|
|
939
|
+
try:
|
|
940
|
+
found = list(codex_sessions_dir().glob(f"*/*/*/{pattern}"))
|
|
941
|
+
except OSError:
|
|
942
|
+
return []
|
|
943
|
+
return sorted(found, key=_mtime)
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def _codex_meta(path):
|
|
947
|
+
"""The session_meta payload a rollout opens with, else None (not a Codex rollout, or unreadable). Only a regular
|
|
948
|
+
file is opened, so a named pipe is not waited on."""
|
|
949
|
+
try:
|
|
950
|
+
if not stat.S_ISREG(os.stat(str(path)).st_mode):
|
|
951
|
+
return None
|
|
952
|
+
with open(path, encoding="utf-8") as f:
|
|
953
|
+
obj = json.loads(f.readline())
|
|
954
|
+
return (
|
|
955
|
+
obj["payload"]
|
|
956
|
+
if obj.get("type") == "session_meta"
|
|
957
|
+
and isinstance(obj.get("payload"), dict)
|
|
958
|
+
else None
|
|
959
|
+
)
|
|
960
|
+
except (
|
|
961
|
+
OSError,
|
|
962
|
+
UnicodeDecodeError,
|
|
963
|
+
ValueError,
|
|
964
|
+
RecursionError,
|
|
965
|
+
KeyError,
|
|
966
|
+
TypeError,
|
|
967
|
+
AttributeError,
|
|
968
|
+
):
|
|
969
|
+
return None
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def _codex_cwd(path):
|
|
973
|
+
"""The cwd a rollout's first record (session_meta) names, else None. A subagent's rollout (its source names
|
|
974
|
+
the parent thread) gives None: what the parent handed it reads as a user turn, so it is no session of the user."""
|
|
975
|
+
try:
|
|
976
|
+
with open(path, encoding="utf-8") as f:
|
|
977
|
+
obj = json.loads(f.readline())
|
|
978
|
+
meta = obj["payload"] if obj.get("type") == "session_meta" else {}
|
|
979
|
+
source = meta.get("source")
|
|
980
|
+
if isinstance(source, dict) and "subagent" in source:
|
|
981
|
+
return None
|
|
982
|
+
return meta.get("cwd")
|
|
983
|
+
except (
|
|
984
|
+
OSError,
|
|
985
|
+
UnicodeDecodeError,
|
|
986
|
+
ValueError,
|
|
987
|
+
RecursionError,
|
|
988
|
+
KeyError,
|
|
989
|
+
TypeError,
|
|
990
|
+
AttributeError,
|
|
991
|
+
):
|
|
992
|
+
return None
|
|
993
|
+
|
|
994
|
+
|
|
726
995
|
def auto_discover_jsonl():
|
|
727
|
-
|
|
728
|
-
|
|
996
|
+
tid = _codex_thread()
|
|
997
|
+
if tid: # inside Codex: this session or nothing, never a Claude Code one
|
|
998
|
+
found = _codex_rollouts(f"rollout-*-{tid}.jsonl")
|
|
999
|
+
return found[-1] if found else None
|
|
1000
|
+
projdir = (
|
|
1001
|
+
pathlib.Path.home() / ".claude" / "projects" / encode_cwd(pathlib.Path.cwd())
|
|
1002
|
+
)
|
|
729
1003
|
if not projdir.exists():
|
|
730
1004
|
return None
|
|
731
1005
|
try:
|
|
732
|
-
files = sorted(
|
|
733
|
-
|
|
1006
|
+
files = sorted(
|
|
1007
|
+
projdir.glob("*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True
|
|
1008
|
+
)
|
|
734
1009
|
except OSError:
|
|
735
1010
|
return None
|
|
736
1011
|
return files[0] if files else None
|
|
737
1012
|
|
|
738
1013
|
|
|
739
1014
|
def project_jsonl_files():
|
|
740
|
-
"""All *.jsonl in the encoded project dir, for --all-sessions."""
|
|
741
|
-
|
|
742
|
-
pathlib.Path.cwd())
|
|
1015
|
+
"""All *.jsonl in the encoded project dir, for --all-sessions. Inside Codex, the Codex sessions of this folder."""
|
|
1016
|
+
if _codex_thread():
|
|
1017
|
+
here = str(pathlib.Path.cwd())
|
|
1018
|
+
return [p for p in _codex_rollouts("rollout-*.jsonl") if _codex_cwd(p) == here]
|
|
1019
|
+
projdir = (
|
|
1020
|
+
pathlib.Path.home() / ".claude" / "projects" / encode_cwd(pathlib.Path.cwd())
|
|
1021
|
+
)
|
|
743
1022
|
if not projdir.exists():
|
|
744
1023
|
return []
|
|
745
1024
|
try:
|
|
@@ -766,7 +1045,9 @@ def session_start(path):
|
|
|
766
1045
|
# ---------------------------------------------------------------- JSONL parse
|
|
767
1046
|
KNOWN_TYPES = {"user", "assistant", "agent-setting", "permission-mode", "summary"}
|
|
768
1047
|
KNOWN_CONTENT_ITEMS = {"text", "tool_use", "tool_result", "thinking"}
|
|
769
|
-
_CTRL_RE = re.compile(
|
|
1048
|
+
_CTRL_RE = re.compile(
|
|
1049
|
+
r"[\x00-\x08\x0b\x0c\x0e-\x1f]"
|
|
1050
|
+
) # strip control chars (keeps \t\n)
|
|
770
1051
|
# What a JSON escape for a lone surrogate (U+D800 to U+DFFF with no pair) leaves in a str. UTF-8 cannot hold one,
|
|
771
1052
|
# so a page, a review pack or a cache key that has it stops with UnicodeEncodeError.
|
|
772
1053
|
_LONE_SURROGATE_RE = re.compile("[" + chr(0xD800) + "-" + chr(0xDFFF) + "]")
|
|
@@ -780,13 +1061,18 @@ def _without_lone_surrogates(value: Any) -> Any:
|
|
|
780
1061
|
if isinstance(value, list):
|
|
781
1062
|
return [_without_lone_surrogates(v) for v in value]
|
|
782
1063
|
if isinstance(value, dict):
|
|
783
|
-
return {
|
|
1064
|
+
return {
|
|
1065
|
+
_without_lone_surrogates(k): _without_lone_surrogates(v)
|
|
1066
|
+
for k, v in value.items()
|
|
1067
|
+
}
|
|
784
1068
|
return value
|
|
785
1069
|
|
|
786
1070
|
|
|
787
1071
|
def _lone_to_fffd(value):
|
|
788
1072
|
"""A str with each lone surrogate as U+FFFD; anything else as it is (no walk into it)."""
|
|
789
|
-
return
|
|
1073
|
+
return (
|
|
1074
|
+
_LONE_SURROGATE_RE.sub(chr(0xFFFD), value) if isinstance(value, str) else value
|
|
1075
|
+
)
|
|
790
1076
|
|
|
791
1077
|
|
|
792
1078
|
def _shown_id(value):
|
|
@@ -798,9 +1084,14 @@ def _shown_id(value):
|
|
|
798
1084
|
def _clean_strings(line_no, role, text, tools, ts, uuid_):
|
|
799
1085
|
"""A record's role, text, tool counts, timestamp and uuid with each lone surrogate as U+FFFD, after one WARN for
|
|
800
1086
|
the line. Only text is touched: a uuid of another shape is replaced later, whole."""
|
|
801
|
-
if not any(
|
|
1087
|
+
if not any(
|
|
1088
|
+
isinstance(s, str) and _LONE_SURROGATE_RE.search(s)
|
|
1089
|
+
for s in (role, text, ts, uuid_, *tools)
|
|
1090
|
+
):
|
|
802
1091
|
return role, text, tools, ts, uuid_
|
|
803
|
-
print(
|
|
1092
|
+
print(
|
|
1093
|
+
f"[WARN] line {line_no}: lone surrogate replaced with U+FFFD", file=sys.stderr
|
|
1094
|
+
)
|
|
804
1095
|
counts = {}
|
|
805
1096
|
for name, n in tools.items():
|
|
806
1097
|
name = _lone_to_fffd(name)
|
|
@@ -815,7 +1106,10 @@ def _turn_id(uuid_, line_no, ts):
|
|
|
815
1106
|
if isinstance(uuid_, str) and uuid_:
|
|
816
1107
|
return uuid_
|
|
817
1108
|
if uuid_ is not None and uuid_ != "":
|
|
818
|
-
print(
|
|
1109
|
+
print(
|
|
1110
|
+
f"[WARN] line {line_no}: uuid that is not text replaced by an id from the line",
|
|
1111
|
+
file=sys.stderr,
|
|
1112
|
+
)
|
|
819
1113
|
return hashlib.sha256(f"{line_no}:{ts}".encode()).hexdigest()[:16]
|
|
820
1114
|
|
|
821
1115
|
|
|
@@ -825,7 +1119,7 @@ def _json_object(raw, line_no):
|
|
|
825
1119
|
something other than an object is skipped, not fatal."""
|
|
826
1120
|
try:
|
|
827
1121
|
obj = json.loads(raw)
|
|
828
|
-
except (ValueError, RecursionError):
|
|
1122
|
+
except (ValueError, RecursionError): # JSONDecodeError is a ValueError
|
|
829
1123
|
print(f"[WARN] line {line_no}: invalid JSON skipped", file=sys.stderr)
|
|
830
1124
|
return None
|
|
831
1125
|
if not isinstance(obj, dict):
|
|
@@ -841,18 +1135,27 @@ def _is_turn_record(obj, seen_unknown_type, unsafe_schema, line_no):
|
|
|
841
1135
|
if obj.get("isSidechain") or obj.get("teamName") or obj.get("agentName"):
|
|
842
1136
|
return False
|
|
843
1137
|
t = obj.get("type")
|
|
844
|
-
if (t and not isinstance(t, str)) or (
|
|
845
|
-
|
|
1138
|
+
if (t and not isinstance(t, str)) or (
|
|
1139
|
+
t in ("user", "assistant") and _odd_message(obj.get("message"))
|
|
1140
|
+
):
|
|
1141
|
+
print(
|
|
1142
|
+
f"[WARN] line {line_no}: record of an unknown shape skipped",
|
|
1143
|
+
file=sys.stderr,
|
|
1144
|
+
)
|
|
846
1145
|
return False
|
|
847
1146
|
if t in ("user", "assistant"):
|
|
848
1147
|
return True
|
|
849
1148
|
if t and t not in KNOWN_TYPES and t not in seen_unknown_type:
|
|
850
1149
|
seen_unknown_type.add(t)
|
|
851
|
-
print(
|
|
1150
|
+
print(
|
|
1151
|
+
f"[WARN] unknown record type {t!r} skipped", file=sys.stderr
|
|
1152
|
+
) # repr: no raw control character
|
|
852
1153
|
if not unsafe_schema and t.startswith("v"):
|
|
853
|
-
print(
|
|
854
|
-
|
|
855
|
-
|
|
1154
|
+
print(
|
|
1155
|
+
f"[ERROR] schema marker {t!r} unrecognized — "
|
|
1156
|
+
"rerun with --unsafe-schema to proceed",
|
|
1157
|
+
file=sys.stderr,
|
|
1158
|
+
)
|
|
856
1159
|
raise SystemExit(2)
|
|
857
1160
|
return False
|
|
858
1161
|
|
|
@@ -870,8 +1173,11 @@ def _odd_message(msg):
|
|
|
870
1173
|
def _odd_item(it, text, name):
|
|
871
1174
|
"""Whether a content item's type, text or tool name has a shape a transcript does not give. A missing or empty
|
|
872
1175
|
text reads as empty and a missing or null tool name as '?'."""
|
|
873
|
-
return (
|
|
874
|
-
|
|
1176
|
+
return (
|
|
1177
|
+
(it is not None and not isinstance(it, str))
|
|
1178
|
+
or (it == "text" and not isinstance(text, str))
|
|
1179
|
+
or (it == "tool_use" and not isinstance(name, str))
|
|
1180
|
+
)
|
|
875
1181
|
|
|
876
1182
|
|
|
877
1183
|
def _take_item(item, line_no, text_parts, tools):
|
|
@@ -882,14 +1188,19 @@ def _take_item(item, line_no, text_parts, tools):
|
|
|
882
1188
|
it, text, name = item.get("type"), item.get("text") or "", item.get("name")
|
|
883
1189
|
name = "?" if name is None else name
|
|
884
1190
|
if _odd_item(it, text, name):
|
|
885
|
-
print(
|
|
1191
|
+
print(
|
|
1192
|
+
f"[WARN] line {line_no}: content item of an unknown shape skipped",
|
|
1193
|
+
file=sys.stderr,
|
|
1194
|
+
)
|
|
886
1195
|
elif it == "text":
|
|
887
1196
|
text_parts.append(text)
|
|
888
1197
|
elif it == "tool_use":
|
|
889
1198
|
tools[name] = tools.get(name, 0) + 1
|
|
890
1199
|
elif it not in KNOWN_CONTENT_ITEMS:
|
|
891
|
-
print(
|
|
892
|
-
|
|
1200
|
+
print(
|
|
1201
|
+
f"[WARN] line {line_no}: unknown content type {it!r} skipped",
|
|
1202
|
+
file=sys.stderr,
|
|
1203
|
+
)
|
|
893
1204
|
|
|
894
1205
|
|
|
895
1206
|
def _content_parts(content, line_no):
|
|
@@ -903,14 +1214,115 @@ def _content_parts(content, line_no):
|
|
|
903
1214
|
_take_item(item, line_no, text_parts, tools)
|
|
904
1215
|
else:
|
|
905
1216
|
if content is not None:
|
|
906
|
-
print(
|
|
907
|
-
|
|
1217
|
+
print(
|
|
1218
|
+
f"[WARN] line {line_no}: unknown content shape "
|
|
1219
|
+
f"({type(content).__name__})",
|
|
1220
|
+
file=sys.stderr,
|
|
1221
|
+
)
|
|
908
1222
|
return None
|
|
909
1223
|
return text_parts, tools
|
|
910
1224
|
|
|
911
1225
|
|
|
912
|
-
def
|
|
913
|
-
|
|
1226
|
+
def _is_codex_meta(obj):
|
|
1227
|
+
"""Whether a record opens a Codex session record (rollout): a session_meta with a payload."""
|
|
1228
|
+
return obj.get("type") == "session_meta" and isinstance(obj.get("payload"), dict)
|
|
1229
|
+
|
|
1230
|
+
|
|
1231
|
+
_CODEX_ITEM_ROLE = {"UserMessage": "user", "AgentMessage": "assistant"}
|
|
1232
|
+
_CODEX_INNER_TOOL_RE = re.compile(r"\btools\.([A-Za-z_][A-Za-z0-9_]{0,63})\(")
|
|
1233
|
+
|
|
1234
|
+
|
|
1235
|
+
def _codex_item_text(item):
|
|
1236
|
+
"""The text parts of a completed item (Codex 0.160: content [{type: text|Text, text}])."""
|
|
1237
|
+
content = item.get("content")
|
|
1238
|
+
if not isinstance(content, list):
|
|
1239
|
+
return ""
|
|
1240
|
+
return "\n".join(
|
|
1241
|
+
c["text"]
|
|
1242
|
+
for c in content
|
|
1243
|
+
if isinstance(c, dict) and isinstance(c.get("text"), str)
|
|
1244
|
+
)
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
def _codex_tools(p, pt):
|
|
1248
|
+
"""Tool counts of one call record. Codex 0.160 runs tools from code (`exec` with `tools.exec_command(...)` in its
|
|
1249
|
+
input), so the inner tool names count, not the wrapper."""
|
|
1250
|
+
name = p.get("name")
|
|
1251
|
+
name = name if isinstance(name, str) and name else pt[:-5]
|
|
1252
|
+
inner = (
|
|
1253
|
+
_CODEX_INNER_TOOL_RE.findall(p.get("input"))
|
|
1254
|
+
if isinstance(p.get("input"), str)
|
|
1255
|
+
else []
|
|
1256
|
+
)
|
|
1257
|
+
counts = {}
|
|
1258
|
+
for n in inner or [name]:
|
|
1259
|
+
counts[n] = counts.get(n, 0) + 1
|
|
1260
|
+
return counts
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
def _codex_record(obj):
|
|
1264
|
+
"""(role, text, tools) a Codex rollout record shows as a turn, else None. What the user typed and what the
|
|
1265
|
+
model said come from the event records (0.144: user_message, agent_message; 0.160: item_completed with a
|
|
1266
|
+
UserMessage or AgentMessage item), so what Codex puts into the model input (environment, instructions, an
|
|
1267
|
+
injected skill body) is not a turn; a tool call is an assistant record with no text, merged into the turn around
|
|
1268
|
+
it like a Claude Code tool_use; a compaction is a divider."""
|
|
1269
|
+
p = obj.get("payload")
|
|
1270
|
+
kind = obj.get("type")
|
|
1271
|
+
if kind == "compacted":
|
|
1272
|
+
return "mark", "compaction", {}
|
|
1273
|
+
if not isinstance(p, dict):
|
|
1274
|
+
return None
|
|
1275
|
+
pt = p.get("type")
|
|
1276
|
+
if kind == "event_msg" and pt in ("user_message", "agent_message"):
|
|
1277
|
+
msg = p.get("message")
|
|
1278
|
+
return (
|
|
1279
|
+
("user" if pt == "user_message" else "assistant"),
|
|
1280
|
+
(msg if isinstance(msg, str) else ""),
|
|
1281
|
+
{},
|
|
1282
|
+
)
|
|
1283
|
+
item = p.get("item")
|
|
1284
|
+
if (
|
|
1285
|
+
kind == "event_msg"
|
|
1286
|
+
and pt == "item_completed"
|
|
1287
|
+
and isinstance(item, dict)
|
|
1288
|
+
and item.get("type") in _CODEX_ITEM_ROLE
|
|
1289
|
+
):
|
|
1290
|
+
return _CODEX_ITEM_ROLE[item["type"]], _codex_item_text(item), {}
|
|
1291
|
+
if kind == "response_item" and isinstance(pt, str) and pt.endswith("_call"):
|
|
1292
|
+
return "assistant", "", _codex_tools(p, pt)
|
|
1293
|
+
return None
|
|
1294
|
+
|
|
1295
|
+
|
|
1296
|
+
def _codex_turn(obj, line_no, session_id, session_name):
|
|
1297
|
+
"""The turn dict of one Codex rollout record, else None."""
|
|
1298
|
+
got = _codex_record(obj)
|
|
1299
|
+
if got is None:
|
|
1300
|
+
return None
|
|
1301
|
+
role, text, tools = got
|
|
1302
|
+
text = _CTRL_RE.sub("", text.strip())
|
|
1303
|
+
if not text and not tools:
|
|
1304
|
+
return None
|
|
1305
|
+
role, text, tools, ts, uuid_ = _clean_strings(
|
|
1306
|
+
line_no, role, text, tools, _stamp(obj.get("timestamp")), None
|
|
1307
|
+
)
|
|
1308
|
+
uuid_ = _turn_id(uuid_, line_no, ts)
|
|
1309
|
+
return {
|
|
1310
|
+
"role": role,
|
|
1311
|
+
"text": text,
|
|
1312
|
+
"ts": ts,
|
|
1313
|
+
"uuid": uuid_,
|
|
1314
|
+
"tools": tools,
|
|
1315
|
+
"line_no": line_no,
|
|
1316
|
+
"parts": [text] if text else [],
|
|
1317
|
+
"session": session_id,
|
|
1318
|
+
"session_name": session_name,
|
|
1319
|
+
"meta": False,
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
|
|
1323
|
+
def parse_turns(
|
|
1324
|
+
stream, unsafe_schema: bool = False, session_id=None, session_name=None
|
|
1325
|
+
):
|
|
914
1326
|
"""Yield dicts: {role, text, ts, uuid, tools, line_no, parts, session,
|
|
915
1327
|
session_name}.
|
|
916
1328
|
|
|
@@ -919,12 +1331,28 @@ def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
|
|
|
919
1331
|
needs it) and `session` (session guards for merge/echo).
|
|
920
1332
|
"""
|
|
921
1333
|
seen_unknown_type = set()
|
|
1334
|
+
codex = None # set by the first record: a Codex rollout or a Claude transcript
|
|
1335
|
+
last_said = (
|
|
1336
|
+
None # Codex: (role, text) of the last message, so one said twice counts once
|
|
1337
|
+
)
|
|
922
1338
|
for line_no, raw in enumerate(stream, 1):
|
|
923
1339
|
raw = raw.strip()
|
|
924
1340
|
if not raw:
|
|
925
1341
|
continue
|
|
926
1342
|
obj = _json_object(raw, line_no)
|
|
927
|
-
if obj is None
|
|
1343
|
+
if obj is not None and codex is None:
|
|
1344
|
+
codex = _is_codex_meta(obj)
|
|
1345
|
+
if obj is not None and codex:
|
|
1346
|
+
turn = _codex_turn(obj, line_no, session_id, session_name)
|
|
1347
|
+
said = (turn["role"], turn["text"]) if turn and turn["text"] else None
|
|
1348
|
+
if turn and not (said and said == last_said):
|
|
1349
|
+
yield turn
|
|
1350
|
+
if said:
|
|
1351
|
+
last_said = said
|
|
1352
|
+
continue
|
|
1353
|
+
if obj is None or not _is_turn_record(
|
|
1354
|
+
obj, seen_unknown_type, unsafe_schema, line_no
|
|
1355
|
+
):
|
|
928
1356
|
continue
|
|
929
1357
|
msg = obj.get("message") or {}
|
|
930
1358
|
got = _content_parts(msg.get("content"), line_no)
|
|
@@ -935,16 +1363,26 @@ def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
|
|
|
935
1363
|
if not text and not tools:
|
|
936
1364
|
continue
|
|
937
1365
|
role = msg.get("role")
|
|
938
|
-
role =
|
|
1366
|
+
role = (
|
|
1367
|
+
role if role in ("user", "assistant") else obj.get("type")
|
|
1368
|
+
) # the type _is_turn_record let through
|
|
939
1369
|
role, text, tools, ts, uuid_ = _clean_strings(
|
|
940
|
-
line_no, role, text, tools, _stamp(obj.get("timestamp")), obj.get("uuid")
|
|
1370
|
+
line_no, role, text, tools, _stamp(obj.get("timestamp")), obj.get("uuid")
|
|
1371
|
+
)
|
|
941
1372
|
uuid_ = _turn_id(uuid_, line_no, ts)
|
|
942
1373
|
|
|
943
|
-
yield {
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
1374
|
+
yield {
|
|
1375
|
+
"role": role,
|
|
1376
|
+
"text": text,
|
|
1377
|
+
"ts": ts,
|
|
1378
|
+
"uuid": uuid_,
|
|
1379
|
+
"tools": tools,
|
|
1380
|
+
"line_no": line_no,
|
|
1381
|
+
"parts": [text] if text else [], # R1: always seed parts
|
|
1382
|
+
"session": session_id,
|
|
1383
|
+
"session_name": session_name,
|
|
1384
|
+
"meta": bool(obj.get("isMeta")),
|
|
1385
|
+
} # a harness-injected body
|
|
948
1386
|
|
|
949
1387
|
|
|
950
1388
|
# ---------------------------------------------------------------- Classify (A-1..A-4)
|
|
@@ -966,9 +1404,11 @@ def classify_turns(turns, drop_trivia=True):
|
|
|
966
1404
|
if _HOOK_FEEDBACK_RE.match(text) or _INTERRUPT_RE.match(text):
|
|
967
1405
|
continue
|
|
968
1406
|
# 2. injected skill/workflow body & harness errors → drop if trivia
|
|
969
|
-
if drop_trivia and (
|
|
970
|
-
|
|
971
|
-
|
|
1407
|
+
if drop_trivia and (
|
|
1408
|
+
_SKILL_BODY_RE.match(text)
|
|
1409
|
+
or _WORKFLOW_BODY_RE.match(text)
|
|
1410
|
+
or _SYS_ERROR_RE.match(text)
|
|
1411
|
+
):
|
|
972
1412
|
continue
|
|
973
1413
|
# 3. image note → replace with attachment marker
|
|
974
1414
|
if _IMAGE_NOTE_RE.match(text):
|
|
@@ -981,7 +1421,7 @@ def classify_turns(turns, drop_trivia=True):
|
|
|
981
1421
|
# 5. agent / teammate wrapper → agent bubble
|
|
982
1422
|
who, body = split_agent_message(text)
|
|
983
1423
|
if who is not None:
|
|
984
|
-
if not body:
|
|
1424
|
+
if not body: # idle notification
|
|
985
1425
|
continue
|
|
986
1426
|
out.append(dict(t, role="agent", text=body, agent_from=who))
|
|
987
1427
|
continue
|
|
@@ -1003,8 +1443,12 @@ def merge_assistant_runs(turns):
|
|
|
1003
1443
|
merged = []
|
|
1004
1444
|
for t in turns:
|
|
1005
1445
|
p = merged[-1] if merged else None
|
|
1006
|
-
if (
|
|
1007
|
-
|
|
1446
|
+
if (
|
|
1447
|
+
p
|
|
1448
|
+
and t["role"] == "assistant"
|
|
1449
|
+
and p["role"] == "assistant"
|
|
1450
|
+
and t.get("session") == p.get("session")
|
|
1451
|
+
):
|
|
1008
1452
|
if t["text"]:
|
|
1009
1453
|
p["text"] = (p["text"] + "\n\n" + t["text"]).strip()
|
|
1010
1454
|
p["parts"].append(t["text"])
|
|
@@ -1025,9 +1469,13 @@ def echo_indices(turns):
|
|
|
1025
1469
|
if t["role"] != "user" or len(t["text"]) > ECHO_ASK:
|
|
1026
1470
|
continue
|
|
1027
1471
|
n = turns[i + 1] if i + 1 < len(turns) else None
|
|
1028
|
-
if (
|
|
1029
|
-
|
|
1030
|
-
|
|
1472
|
+
if (
|
|
1473
|
+
n
|
|
1474
|
+
and n["role"] == "assistant"
|
|
1475
|
+
and not n["tools"]
|
|
1476
|
+
and len(n["text"].strip()) <= ECHO_REPLY
|
|
1477
|
+
and n.get("session") == t.get("session")
|
|
1478
|
+
):
|
|
1031
1479
|
drop |= {i, i + 1}
|
|
1032
1480
|
return drop
|
|
1033
1481
|
|
|
@@ -1057,7 +1505,8 @@ def fill_missing_ts(turns):
|
|
|
1057
1505
|
_SENT_SPLIT = re.compile(r"(?<=[.!?…])(?<![0-9][.!?…])(?=\s|$)\s*|\n+")
|
|
1058
1506
|
_INTENT_ONLY_RE = re.compile(
|
|
1059
1507
|
r"^.{0,30}(?:하겠습니다|합니다|해보겠습니다|확인합니다|봅니다|"
|
|
1060
|
-
r"보겠습니다|시작합니다|진행합니다)\.?$"
|
|
1508
|
+
r"보겠습니다|시작합니다|진행합니다)\.?$"
|
|
1509
|
+
)
|
|
1061
1510
|
SHORT_DETAIL = 160
|
|
1062
1511
|
SINGLE_SPLIT_MIN = SHORT_DETAIL
|
|
1063
1512
|
|
|
@@ -1069,7 +1518,7 @@ def _cut(s, n):
|
|
|
1069
1518
|
s = s.strip()
|
|
1070
1519
|
if len(s) <= n:
|
|
1071
1520
|
return s
|
|
1072
|
-
head = _OPEN_MARKER_RE.sub("", s[:max(1, n - 1)])
|
|
1521
|
+
head = _OPEN_MARKER_RE.sub("", s[: max(1, n - 1)]) # a marker cut in two goes whole
|
|
1073
1522
|
return head.rstrip() + "…"
|
|
1074
1523
|
|
|
1075
1524
|
|
|
@@ -1146,10 +1595,14 @@ def _stdin_id_key():
|
|
|
1146
1595
|
CACHE_BASE.mkdir(parents=True, exist_ok=True)
|
|
1147
1596
|
os.chmod(str(CACHE_BASE), 0o700)
|
|
1148
1597
|
# O_EXCL: a name that is already there (a link planted at this guessable name among them) is refused
|
|
1149
|
-
fd = os.open(
|
|
1598
|
+
fd = os.open(
|
|
1599
|
+
str(tmp),
|
|
1600
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0),
|
|
1601
|
+
0o600,
|
|
1602
|
+
)
|
|
1150
1603
|
with os.fdopen(fd, "wb") as f:
|
|
1151
1604
|
f.write(key)
|
|
1152
|
-
os.replace(str(tmp), str(path))
|
|
1605
|
+
os.replace(str(tmp), str(path)) # whole or not at all
|
|
1153
1606
|
except OSError:
|
|
1154
1607
|
try:
|
|
1155
1608
|
os.unlink(str(tmp))
|
|
@@ -1172,17 +1625,38 @@ def cache_dir(session_id: str):
|
|
|
1172
1625
|
def _page_clean_turn(t, extra, mode):
|
|
1173
1626
|
"""`t` with its text and parts as a page shows them, so a summary cut from it cannot cut a
|
|
1174
1627
|
secret in two and keep the half no pattern recognises."""
|
|
1628
|
+
|
|
1175
1629
|
def clean(s):
|
|
1176
1630
|
return page_redact(s, extra, mode)[0]
|
|
1177
|
-
|
|
1631
|
+
|
|
1632
|
+
return dict(
|
|
1633
|
+
t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []]
|
|
1634
|
+
)
|
|
1178
1635
|
|
|
1179
1636
|
|
|
1180
1637
|
# Letters that print as nothing: the Hangul fillers, the braille blank, and the combining marks Unicode ignores by
|
|
1181
1638
|
# default (the grapheme joiner, the Khmer inherent vowels, the Mongolian free variation selectors). With the control,
|
|
1182
1639
|
# format, private-use, unassigned and separator categories, a cache made only of them holds no summary a person can
|
|
1183
1640
|
# read (3.0.5).
|
|
1184
|
-
_BLANK_LETTERS = frozenset(
|
|
1185
|
-
|
|
1641
|
+
_BLANK_LETTERS = frozenset(
|
|
1642
|
+
map(
|
|
1643
|
+
chr,
|
|
1644
|
+
(
|
|
1645
|
+
0x115F,
|
|
1646
|
+
0x1160,
|
|
1647
|
+
0x3164,
|
|
1648
|
+
0xFFA0,
|
|
1649
|
+
0x2800,
|
|
1650
|
+
0x034F,
|
|
1651
|
+
0x17B4,
|
|
1652
|
+
0x17B5,
|
|
1653
|
+
0x180B,
|
|
1654
|
+
0x180C,
|
|
1655
|
+
0x180D,
|
|
1656
|
+
0x180F,
|
|
1657
|
+
),
|
|
1658
|
+
)
|
|
1659
|
+
)
|
|
1186
1660
|
# Cn follows the Unicode table of the Python that runs: a character newer than that table reads as unassigned, so
|
|
1187
1661
|
# a cache made only of such characters is summarized again every run (the page is the same).
|
|
1188
1662
|
_BLANK_CATEGORIES = frozenset(("Cc", "Cf", "Co", "Cn", "Zs", "Zl", "Zp"))
|
|
@@ -1191,7 +1665,11 @@ _BLANK_CATEGORIES = frozenset(("Cc", "Cf", "Co", "Cn", "Zs", "Zl", "Zp"))
|
|
|
1191
1665
|
def _blank(ch):
|
|
1192
1666
|
"""Whether ch prints as nothing: whitespace, a blank-looking letter, or a variation selector.
|
|
1193
1667
|
White space is in the blank categories too (Zs, Zl, Zp, Cc); str.isspace() only answers it sooner."""
|
|
1194
|
-
if
|
|
1668
|
+
if (
|
|
1669
|
+
ch.isspace()
|
|
1670
|
+
or ch in _BLANK_LETTERS
|
|
1671
|
+
or unicodedata.category(ch) in _BLANK_CATEGORIES
|
|
1672
|
+
):
|
|
1195
1673
|
return True
|
|
1196
1674
|
o = ord(ch)
|
|
1197
1675
|
return 0xFE00 <= o <= 0xFE0F or 0xE0100 <= o <= 0xE01EF
|
|
@@ -1202,7 +1680,9 @@ def _has_visible(text):
|
|
|
1202
1680
|
return not all(_blank(ch) for ch in text)
|
|
1203
1681
|
|
|
1204
1682
|
|
|
1205
|
-
CACHE_READ_MAX =
|
|
1683
|
+
CACHE_READ_MAX = (
|
|
1684
|
+
64 * 1024
|
|
1685
|
+
) # bytes: a cached summary or decision is some hundred (120 characters), so this is generous
|
|
1206
1686
|
|
|
1207
1687
|
|
|
1208
1688
|
def _read_small(path, limit=CACHE_READ_MAX):
|
|
@@ -1210,10 +1690,16 @@ def _read_small(path, limit=CACHE_READ_MAX):
|
|
|
1210
1690
|
folder, a larger file). Opened without blocking and without following a link, so a pipe or a link left at a
|
|
1211
1691
|
cache place neither stops a run nor sends it to another file. Windows has neither flag: there the lstat
|
|
1212
1692
|
turns a link away, but a link swapped in between the lstat and the open is followed."""
|
|
1213
|
-
flags = (
|
|
1214
|
-
|
|
1693
|
+
flags = (
|
|
1694
|
+
os.O_RDONLY
|
|
1695
|
+
| getattr(os, "O_NONBLOCK", 0)
|
|
1696
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
1697
|
+
| getattr(os, "O_BINARY", 0)
|
|
1698
|
+
)
|
|
1215
1699
|
try:
|
|
1216
|
-
if not stat.S_ISREG(
|
|
1700
|
+
if not stat.S_ISREG(
|
|
1701
|
+
os.lstat(str(path)).st_mode
|
|
1702
|
+
): # a link (Windows has no O_NOFOLLOW), pipe, folder
|
|
1217
1703
|
return None
|
|
1218
1704
|
fd = os.open(str(path), flags)
|
|
1219
1705
|
with os.fdopen(fd, "rb") as f:
|
|
@@ -1247,8 +1733,13 @@ def _read_cached(p):
|
|
|
1247
1733
|
return None
|
|
1248
1734
|
|
|
1249
1735
|
|
|
1250
|
-
def read_or_summarize(
|
|
1251
|
-
|
|
1736
|
+
def read_or_summarize(
|
|
1737
|
+
turn: dict,
|
|
1738
|
+
session_id: str,
|
|
1739
|
+
rebuild: bool = False,
|
|
1740
|
+
redact_extra=None,
|
|
1741
|
+
redact_mode: str = "full",
|
|
1742
|
+
):
|
|
1252
1743
|
"""Return (summary, cache_hit). Cache key includes the content hash AND the
|
|
1253
1744
|
summarizer version (B-3), so changing the summarizer invalidates old entries.
|
|
1254
1745
|
Cached text is always redacted (secret hygiene). It is cut from the redacted text (3.0.2).
|
|
@@ -1256,10 +1747,12 @@ def read_or_summarize(turn: dict, session_id: str, rebuild: bool = False,
|
|
|
1256
1747
|
digest = hashlib.sha256(
|
|
1257
1748
|
(turn["text"] + "\x00s" + str(SUMMARIZER_VERSION)).encode()
|
|
1258
1749
|
).hexdigest()[:8]
|
|
1259
|
-
try:
|
|
1260
|
-
p = (
|
|
1261
|
-
|
|
1262
|
-
|
|
1750
|
+
try: # ids come from the transcript: keep them in the cache
|
|
1751
|
+
p = (
|
|
1752
|
+
cache_dir(_safe_name(turn.get("session") or session_id))
|
|
1753
|
+
/ f"{_safe_name(turn['uuid'])}-{digest}.md"
|
|
1754
|
+
)
|
|
1755
|
+
except OSError as e: # no writable cache: summarize anyway
|
|
1263
1756
|
_warn_once("_no_cache", f"cache unavailable ({e}); summaries are not cached")
|
|
1264
1757
|
p = None
|
|
1265
1758
|
cached = None if p is None or rebuild else _read_cached(p)
|
|
@@ -1288,7 +1781,8 @@ _SENTINEL = "\x00%d\x00"
|
|
|
1288
1781
|
_SENTINEL_RE = re.compile(r"\x00(\d+)\x00")
|
|
1289
1782
|
_MD_CARVE = re.compile(
|
|
1290
1783
|
r"`(?P<code>[^`\n]+)`"
|
|
1291
|
-
r"|\[(?P<label>[^\]\n]+)\]\((?P<url>https?://[^\s)]+)\)"
|
|
1784
|
+
r"|\[(?P<label>[^\]\n]+)\]\((?P<url>https?://[^\s)]+)\)"
|
|
1785
|
+
)
|
|
1292
1786
|
_MD_BOLD = re.compile(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", re.S)
|
|
1293
1787
|
_MD_DEL = re.compile(r"~~(?=\S)([^<]+?)(?<=\S)~~")
|
|
1294
1788
|
_MD_ITAL = re.compile(r"(?<![*\w])\*(?=\S)([^*<\n]+?)(?<=\S)\*(?![*\w])")
|
|
@@ -1311,7 +1805,8 @@ def _md_inline(escaped):
|
|
|
1311
1805
|
else:
|
|
1312
1806
|
stash.append(
|
|
1313
1807
|
'<a href="%s" target="_blank" rel="noopener noreferrer">%s</a>'
|
|
1314
|
-
% (m.group("url"), m.group("label"))
|
|
1808
|
+
% (m.group("url"), m.group("label"))
|
|
1809
|
+
)
|
|
1315
1810
|
return _SENTINEL % (len(stash) - 1)
|
|
1316
1811
|
|
|
1317
1812
|
s = _MD_CARVE.sub(carve, escaped)
|
|
@@ -1323,16 +1818,25 @@ def _md_inline(escaped):
|
|
|
1323
1818
|
|
|
1324
1819
|
def _is_block_start(lines, i):
|
|
1325
1820
|
line = lines[i]
|
|
1326
|
-
if (
|
|
1327
|
-
|
|
1821
|
+
if (
|
|
1822
|
+
_FENCE_RE.match(line)
|
|
1823
|
+
or _HEAD_RE.match(line)
|
|
1824
|
+
or _HR_RE.match(line)
|
|
1825
|
+
or _QUOTE_RE.match(line)
|
|
1826
|
+
or _LIST_RE.match(line)
|
|
1827
|
+
):
|
|
1328
1828
|
return True
|
|
1329
1829
|
return _is_table_start(lines, i)
|
|
1330
1830
|
|
|
1331
1831
|
|
|
1332
1832
|
def _is_table_start(lines, i):
|
|
1333
1833
|
"""Whether lines[i] starts a table: it has a pipe AND the next line is a delimiter row."""
|
|
1334
|
-
return bool(
|
|
1335
|
-
|
|
1834
|
+
return bool(
|
|
1835
|
+
"|" in lines[i]
|
|
1836
|
+
and i + 1 < len(lines)
|
|
1837
|
+
and "-" in lines[i + 1]
|
|
1838
|
+
and _TABLE_DELIM_RE.match(lines[i + 1])
|
|
1839
|
+
)
|
|
1336
1840
|
|
|
1337
1841
|
|
|
1338
1842
|
def _render_table(lines, i):
|
|
@@ -1344,11 +1848,12 @@ def _render_table(lines, i):
|
|
|
1344
1848
|
i += 1
|
|
1345
1849
|
thead = "".join("<th>%s</th>" % _md_inline(html.escape(c)) for c in header)
|
|
1346
1850
|
body = "".join(
|
|
1347
|
-
"<tr>%s</tr>" % "".join("<td>%s</td>" % _md_inline(html.escape(c))
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
return (
|
|
1351
|
-
|
|
1851
|
+
"<tr>%s</tr>" % "".join("<td>%s</td>" % _md_inline(html.escape(c)) for c in r)
|
|
1852
|
+
for r in rows
|
|
1853
|
+
)
|
|
1854
|
+
return (
|
|
1855
|
+
"<table><thead><tr>%s</tr></thead><tbody>%s</tbody></table>" % (thead, body)
|
|
1856
|
+
), i
|
|
1352
1857
|
|
|
1353
1858
|
|
|
1354
1859
|
def _md_fence(lines, i, fence):
|
|
@@ -1408,8 +1913,12 @@ def _md_block(lines, i, depth):
|
|
|
1408
1913
|
return _md_fence(lines, i, mf.group(1))
|
|
1409
1914
|
mh = _HEAD_RE.match(line)
|
|
1410
1915
|
if mh:
|
|
1411
|
-
hl = max(3, min(6, len(mh.group(1))))
|
|
1412
|
-
return "<h%d>%s</h%d>" % (
|
|
1916
|
+
hl = max(3, min(6, len(mh.group(1)))) # h3..h6 (avoid page-header clash)
|
|
1917
|
+
return "<h%d>%s</h%d>" % (
|
|
1918
|
+
hl,
|
|
1919
|
+
_md_inline(html.escape(mh.group(2).strip())),
|
|
1920
|
+
hl,
|
|
1921
|
+
), i + 1
|
|
1413
1922
|
if _HR_RE.match(line):
|
|
1414
1923
|
return "<hr>", i + 1
|
|
1415
1924
|
if _QUOTE_RE.match(line):
|
|
@@ -1634,8 +2143,13 @@ def _page_summary(t, summaries, session_id, rebuild, redact_extra, redact_mode):
|
|
|
1634
2143
|
rules' (cached)."""
|
|
1635
2144
|
if summaries is not None and t.get("uuid") in summaries:
|
|
1636
2145
|
return summaries[t["uuid"]], None
|
|
1637
|
-
return read_or_summarize(
|
|
1638
|
-
|
|
2146
|
+
return read_or_summarize(
|
|
2147
|
+
t,
|
|
2148
|
+
session_id,
|
|
2149
|
+
rebuild=rebuild,
|
|
2150
|
+
redact_extra=redact_extra,
|
|
2151
|
+
redact_mode=redact_mode,
|
|
2152
|
+
)
|
|
1639
2153
|
|
|
1640
2154
|
|
|
1641
2155
|
def _reviewed_fold(t, summaries):
|
|
@@ -1670,18 +2184,23 @@ def _pills(rs, t, all_sessions):
|
|
|
1670
2184
|
turn `t`."""
|
|
1671
2185
|
if all_sessions and t.get("session") != rs.last_session:
|
|
1672
2186
|
rs.last_session = t.get("session")
|
|
1673
|
-
name = html.escape(
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
2187
|
+
name = html.escape(
|
|
2188
|
+
rs.red(t.get("session_name") or rs.last_session or "session")
|
|
2189
|
+
)
|
|
2190
|
+
rs.rows.append(
|
|
2191
|
+
'<div class="day" data-mark="%s">'
|
|
2192
|
+
'<span class="pill pill-session">%s</span></div>' % (name, name)
|
|
2193
|
+
)
|
|
1677
2194
|
rs.last_date = None
|
|
1678
2195
|
ts = t.get("ts") or ""
|
|
1679
2196
|
if ts:
|
|
1680
2197
|
d = ts[:10]
|
|
1681
2198
|
if d != rs.last_date:
|
|
1682
|
-
rs.rows.append(
|
|
1683
|
-
|
|
1684
|
-
|
|
2199
|
+
rs.rows.append(
|
|
2200
|
+
'<div class="day" data-mark="%s">'
|
|
2201
|
+
'<span class="pill pill-date">%s</span></div>'
|
|
2202
|
+
% (html.escape(d), html.escape(d))
|
|
2203
|
+
)
|
|
1685
2204
|
rs.last_date = d
|
|
1686
2205
|
|
|
1687
2206
|
|
|
@@ -1692,24 +2211,31 @@ def _user_row(rs, t, summaries, time_hm):
|
|
|
1692
2211
|
reviewed = _reviewed_fold(t, summaries)
|
|
1693
2212
|
summary = html.escape(rs.red(reviewed) if reviewed else user_fold_summary(red))
|
|
1694
2213
|
body = render_body(red, markdown=rs.markdown)
|
|
1695
|
-
return (
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
2214
|
+
return (
|
|
2215
|
+
'<div class="row me"><span class="time">%s</span>'
|
|
2216
|
+
'<div class="bubble"><details%s>'
|
|
2217
|
+
'<summary><span class="sum">%s</span></summary>'
|
|
2218
|
+
'<div class="detail">%s</div></details></div></div>'
|
|
2219
|
+
% (time_hm, rs.open_attr, summary, body)
|
|
2220
|
+
)
|
|
2221
|
+
return (
|
|
2222
|
+
'<div class="row me"><span class="time">%s</span>'
|
|
2223
|
+
'<div class="bubble"><div class="plain">%s</div></div></div>'
|
|
2224
|
+
% (time_hm, render_body(red, markdown=rs.markdown))
|
|
2225
|
+
)
|
|
1703
2226
|
|
|
1704
2227
|
|
|
1705
2228
|
def _tools_line(tools, redact_extra):
|
|
1706
2229
|
"""The `🔧 도구 N건` line of a bot bubble ("" without tools), the --redact-extra keywords hidden in the names."""
|
|
1707
2230
|
if not tools:
|
|
1708
2231
|
return ""
|
|
1709
|
-
ts_list = ", ".join(
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
2232
|
+
ts_list = ", ".join(
|
|
2233
|
+
f"{_hide_keywords(k, redact_extra)}×{v}" for k, v in sorted(tools.items())
|
|
2234
|
+
)
|
|
2235
|
+
return '<div class="tools">🔧 도구 %d건: %s</div>' % (
|
|
2236
|
+
sum(tools.values()),
|
|
2237
|
+
html.escape(ts_list),
|
|
2238
|
+
)
|
|
1713
2239
|
|
|
1714
2240
|
|
|
1715
2241
|
def _bot_row(rs, t, summary, time_hm):
|
|
@@ -1720,32 +2246,59 @@ def _bot_row(rs, t, summary, time_hm):
|
|
|
1720
2246
|
role = t["role"]
|
|
1721
2247
|
if role == "agent":
|
|
1722
2248
|
who = html.escape(rs.red(t.get("agent_from") or "agent"))
|
|
1723
|
-
avatar, from_line, row_cls =
|
|
2249
|
+
avatar, from_line, row_cls = (
|
|
2250
|
+
"🤝",
|
|
2251
|
+
'<div class="from">%s</div>' % who,
|
|
2252
|
+
"row bot agent",
|
|
2253
|
+
)
|
|
1724
2254
|
else:
|
|
1725
2255
|
avatar, from_line, row_cls = "C", "", "row bot"
|
|
1726
|
-
return (
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
2256
|
+
return (
|
|
2257
|
+
'<div class="%s"><div class="avatar%s">%s</div>'
|
|
2258
|
+
'<div class="bubble"><details%s>'
|
|
2259
|
+
'<summary><span class="sum">%s</span></summary>'
|
|
2260
|
+
'<div class="detail">%s%s%s</div></details></div>'
|
|
2261
|
+
'<span class="time">%s</span></div>'
|
|
2262
|
+
% (
|
|
2263
|
+
row_cls,
|
|
2264
|
+
" agent" if role == "agent" else "",
|
|
2265
|
+
avatar,
|
|
2266
|
+
rs.open_attr,
|
|
2267
|
+
esc_sum,
|
|
2268
|
+
from_line,
|
|
2269
|
+
detail_body,
|
|
2270
|
+
tools_html,
|
|
2271
|
+
time_hm,
|
|
2272
|
+
)
|
|
2273
|
+
)
|
|
2274
|
+
|
|
2275
|
+
|
|
2276
|
+
def render_rows(
|
|
2277
|
+
turns,
|
|
2278
|
+
session_id,
|
|
2279
|
+
redact_extra,
|
|
2280
|
+
redact_mode,
|
|
2281
|
+
rebuild,
|
|
2282
|
+
open_details=False,
|
|
2283
|
+
markdown=True,
|
|
2284
|
+
all_sessions=False,
|
|
2285
|
+
summaries=None,
|
|
2286
|
+
):
|
|
1737
2287
|
rs = _Rows(redact_extra, redact_mode, open_details, markdown)
|
|
1738
2288
|
for t in turns:
|
|
1739
2289
|
_pills(rs, t, all_sessions)
|
|
1740
2290
|
time_hm = html.escape(_fmt_time(t.get("ts") or ""))
|
|
1741
2291
|
if t["role"] == "mark":
|
|
1742
|
-
rs.rows.append(
|
|
1743
|
-
|
|
2292
|
+
rs.rows.append(
|
|
2293
|
+
'<div class="day"><span class="pill pill-compact">'
|
|
2294
|
+
"⋯ 이전 대화 요약(compaction) ⋯</span></div>"
|
|
2295
|
+
)
|
|
1744
2296
|
elif t["role"] == "user":
|
|
1745
2297
|
rs.rows.append(_user_row(rs, t, summaries, time_hm))
|
|
1746
2298
|
else:
|
|
1747
|
-
summary, hit = _page_summary(
|
|
1748
|
-
|
|
2299
|
+
summary, hit = _page_summary(
|
|
2300
|
+
t, summaries, session_id, rebuild, redact_extra, redact_mode
|
|
2301
|
+
)
|
|
1749
2302
|
if hit is not None:
|
|
1750
2303
|
rs.total += 1
|
|
1751
2304
|
rs.hits += bool(hit)
|
|
@@ -1770,10 +2323,34 @@ def _residual_secrets(rows_html):
|
|
|
1770
2323
|
|
|
1771
2324
|
# ---------------------------------------------------------------- Self-verify (F-1)
|
|
1772
2325
|
_VOID = frozenset(
|
|
1773
|
-
"area base br col embed hr img input link meta param source track wbr".split()
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
2326
|
+
"area base br col embed hr img input link meta param source track wbr".split()
|
|
2327
|
+
)
|
|
2328
|
+
_VERIFY_TAGS = (
|
|
2329
|
+
"details",
|
|
2330
|
+
"div",
|
|
2331
|
+
"p",
|
|
2332
|
+
"ul",
|
|
2333
|
+
"ol",
|
|
2334
|
+
"li",
|
|
2335
|
+
"table",
|
|
2336
|
+
"thead",
|
|
2337
|
+
"tbody",
|
|
2338
|
+
"tr",
|
|
2339
|
+
"th",
|
|
2340
|
+
"td",
|
|
2341
|
+
"blockquote",
|
|
2342
|
+
"pre",
|
|
2343
|
+
"code",
|
|
2344
|
+
"strong",
|
|
2345
|
+
"em",
|
|
2346
|
+
"del",
|
|
2347
|
+
"a",
|
|
2348
|
+
"summary",
|
|
2349
|
+
"h3",
|
|
2350
|
+
"h4",
|
|
2351
|
+
"h5",
|
|
2352
|
+
"h6",
|
|
2353
|
+
)
|
|
1777
2354
|
|
|
1778
2355
|
|
|
1779
2356
|
class _BalanceParser(HTMLParser):
|
|
@@ -1818,8 +2395,12 @@ def self_verify(html_text: str):
|
|
|
1818
2395
|
errs = []
|
|
1819
2396
|
# strip script/style so their bodies aren't parsed as markup (avoids
|
|
1820
2397
|
# false <a> counts from inline JS like `top<a-8`)
|
|
1821
|
-
markup = re.sub(
|
|
1822
|
-
|
|
2398
|
+
markup = re.sub(
|
|
2399
|
+
r"<script\b[\s\S]*?</script>|<style\b[\s\S]*?</style>",
|
|
2400
|
+
"",
|
|
2401
|
+
html_text,
|
|
2402
|
+
flags=re.I,
|
|
2403
|
+
)
|
|
1823
2404
|
try:
|
|
1824
2405
|
HTMLParser().feed(html_text)
|
|
1825
2406
|
except Exception as e:
|
|
@@ -1843,9 +2424,11 @@ def self_verify(html_text: str):
|
|
|
1843
2424
|
errs.append("external script src found")
|
|
1844
2425
|
# Event-handler attributes are detected via the parser (real tag attrs),
|
|
1845
2426
|
# NOT a text regex — escaped content like `<img onerror=` is inert text.
|
|
1846
|
-
for marker, ph in (
|
|
1847
|
-
|
|
1848
|
-
|
|
2427
|
+
for marker, ph in (
|
|
2428
|
+
("<title>{{TITLE}}</title>", "{{TITLE}}"),
|
|
2429
|
+
("<h1>{{HEADER_TITLE}}</h1>", "{{HEADER_TITLE}}"),
|
|
2430
|
+
('id="now">{{DATE_RANGE}}</div>', "{{DATE_RANGE}}"),
|
|
2431
|
+
):
|
|
1849
2432
|
if marker in html_text:
|
|
1850
2433
|
errs.append(f"unsubstituted template placeholder: {ph}")
|
|
1851
2434
|
return errs
|
|
@@ -1884,10 +2467,14 @@ def _write_private(path, text):
|
|
|
1884
2467
|
is never followed, and it is removed when the write fails."""
|
|
1885
2468
|
path = pathlib.Path(path)
|
|
1886
2469
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1887
|
-
fd, tmp = tempfile.mkstemp(
|
|
2470
|
+
fd, tmp = tempfile.mkstemp(
|
|
2471
|
+
dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp"
|
|
2472
|
+
)
|
|
1888
2473
|
try:
|
|
1889
2474
|
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f:
|
|
1890
|
-
f.write(
|
|
2475
|
+
f.write(
|
|
2476
|
+
_lone_to_fffd(text)
|
|
2477
|
+
) # a title or a file name may still bring one: UTF-8 cannot hold it
|
|
1891
2478
|
os.replace(tmp, str(path))
|
|
1892
2479
|
except BaseException:
|
|
1893
2480
|
try:
|
|
@@ -1901,7 +2488,9 @@ def _write_text_lf(path, text):
|
|
|
1901
2488
|
"""Write `text` to `path` as UTF-8 with LF line ends on every OS. Text mode on Windows
|
|
1902
2489
|
would write CRLF, so the same transcript would give a different page there."""
|
|
1903
2490
|
with open(str(path), "w", encoding="utf-8", newline="\n") as f:
|
|
1904
|
-
f.write(
|
|
2491
|
+
f.write(
|
|
2492
|
+
_lone_to_fffd(text)
|
|
2493
|
+
) # a title or a file name may still bring one: UTF-8 cannot hold it
|
|
1905
2494
|
|
|
1906
2495
|
|
|
1907
2496
|
def _safe_name(s):
|
|
@@ -1913,19 +2502,28 @@ def _safe_name(s):
|
|
|
1913
2502
|
def _decision_context(why, keeps):
|
|
1914
2503
|
"""What a cached decision rests on beside the text: the rules' call on the turn (a
|
|
1915
2504
|
`keep: null` defers to it) and the keep flags the reviewer was told to honour."""
|
|
1916
|
-
return "|".join(
|
|
1917
|
-
|
|
2505
|
+
return "|".join(
|
|
2506
|
+
[
|
|
2507
|
+
why or "",
|
|
2508
|
+
str(keeps.get("keep_trivia") is True),
|
|
2509
|
+
str(keeps.get("keep_tool_only") is True),
|
|
2510
|
+
]
|
|
2511
|
+
)
|
|
1918
2512
|
|
|
1919
2513
|
|
|
1920
2514
|
def _llm_digest(redacted_text, context):
|
|
1921
2515
|
"""The decision cache key: the text with the page's patterns only (as 3.0.1 keyed it, so
|
|
1922
2516
|
no turn is reviewed again for 3.0.2), the cache version and what the decision rests on."""
|
|
1923
|
-
return hashlib.sha256(
|
|
1924
|
-
|
|
2517
|
+
return hashlib.sha256(
|
|
2518
|
+
(redacted_text + "\x00llm" + str(LLM_CACHE_VERSION) + "\x00" + context).encode()
|
|
2519
|
+
).hexdigest()[:12]
|
|
1925
2520
|
|
|
1926
2521
|
|
|
1927
2522
|
def _llm_cache_path(turn_id, session, digest):
|
|
1928
|
-
return
|
|
2523
|
+
return (
|
|
2524
|
+
cache_dir(_safe_name(session))
|
|
2525
|
+
/ f"{_safe_name(turn_id)}-{_safe_name(digest)}-llm.json"
|
|
2526
|
+
)
|
|
1929
2527
|
|
|
1930
2528
|
|
|
1931
2529
|
def read_llm_cache(turn_id, session, digest):
|
|
@@ -1934,9 +2532,11 @@ def read_llm_cache(turn_id, session, digest):
|
|
|
1934
2532
|
null: the reviewer left the turn to the rules."""
|
|
1935
2533
|
try:
|
|
1936
2534
|
path = _llm_cache_path(turn_id, session, digest)
|
|
1937
|
-
except OSError as e:
|
|
1938
|
-
_warn_once(
|
|
1939
|
-
|
|
2535
|
+
except OSError as e: # no cache folder (a read-only HOME, say)
|
|
2536
|
+
_warn_once(
|
|
2537
|
+
"_no_llm_cache",
|
|
2538
|
+
f"reviewer decision cache unavailable ({e}); every turn is reviewed",
|
|
2539
|
+
)
|
|
1940
2540
|
return None
|
|
1941
2541
|
try:
|
|
1942
2542
|
got = json.loads(_read_text(path) or "")
|
|
@@ -1952,11 +2552,16 @@ def read_llm_cache(turn_id, session, digest):
|
|
|
1952
2552
|
|
|
1953
2553
|
def write_llm_cache(turn_id, session, digest, keep, summary):
|
|
1954
2554
|
try:
|
|
1955
|
-
_write_private(
|
|
1956
|
-
|
|
2555
|
+
_write_private(
|
|
2556
|
+
_llm_cache_path(turn_id, session, digest),
|
|
2557
|
+
json.dumps({"keep": keep, "summary": summary}, ensure_ascii=False),
|
|
2558
|
+
)
|
|
1957
2559
|
except OSError as e:
|
|
1958
|
-
_warn_once(
|
|
1959
|
-
|
|
2560
|
+
_warn_once(
|
|
2561
|
+
"_no_llm_cache_write",
|
|
2562
|
+
f"reviewer decisions not cached ({e}); the next "
|
|
2563
|
+
"run reviews this run's turns again (decisions cached before still apply)",
|
|
2564
|
+
)
|
|
1960
2565
|
|
|
1961
2566
|
|
|
1962
2567
|
def heuristic_drops(turns, args):
|
|
@@ -1995,8 +2600,11 @@ def _preview(text):
|
|
|
1995
2600
|
if len(text) <= PREVIEW_HEAD + PREVIEW_TAIL + 200:
|
|
1996
2601
|
return text, False
|
|
1997
2602
|
cut = len(text) - PREVIEW_HEAD - PREVIEW_TAIL
|
|
1998
|
-
return (
|
|
1999
|
-
|
|
2603
|
+
return (
|
|
2604
|
+
text[:PREVIEW_HEAD].rstrip()
|
|
2605
|
+
+ f"\n\n[중간 {cut}자 생략]\n\n"
|
|
2606
|
+
+ text[-PREVIEW_TAIL:].lstrip()
|
|
2607
|
+
), True
|
|
2000
2608
|
|
|
2001
2609
|
|
|
2002
2610
|
def _public_counts(counts):
|
|
@@ -2023,22 +2631,31 @@ def _rule_summary(t, red, args):
|
|
|
2023
2631
|
def _redacted_turn(t, extra):
|
|
2024
2632
|
"""`t` with its text and parts fully redacted, so a summary cut from it cannot cut a
|
|
2025
2633
|
secret in two and keep the half no pattern recognises."""
|
|
2634
|
+
|
|
2026
2635
|
def clean(s):
|
|
2027
2636
|
return review_redact(s, extra)[0]
|
|
2028
|
-
|
|
2637
|
+
|
|
2638
|
+
return dict(
|
|
2639
|
+
t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []]
|
|
2640
|
+
)
|
|
2029
2641
|
|
|
2030
2642
|
|
|
2031
2643
|
def _review_view(t, red, args):
|
|
2032
2644
|
"""What a reviewer reads beside the preview: the rules' summary and the agent's
|
|
2033
2645
|
name, fully redacted (the page's copies may be masked)."""
|
|
2034
2646
|
if t["role"] in ("assistant", "agent"):
|
|
2035
|
-
summary = review_redact(
|
|
2036
|
-
|
|
2647
|
+
summary = review_redact(
|
|
2648
|
+
summarize_turn(_redacted_turn(t, args.redact_extra)), args.redact_extra
|
|
2649
|
+
)[0]
|
|
2037
2650
|
elif t["role"] == "user" and len(red) > USER_FOLD:
|
|
2038
2651
|
summary = user_fold_summary(review_redact(t["text"], args.redact_extra)[0])
|
|
2039
2652
|
else:
|
|
2040
2653
|
summary = None
|
|
2041
|
-
who =
|
|
2654
|
+
who = (
|
|
2655
|
+
review_redact(t["agent_from"], args.redact_extra)[0]
|
|
2656
|
+
if t.get("agent_from")
|
|
2657
|
+
else None
|
|
2658
|
+
)
|
|
2042
2659
|
return {"summary": summary, "agent_from": who}
|
|
2043
2660
|
|
|
2044
2661
|
|
|
@@ -2052,28 +2669,53 @@ def _pack_turn(n, t, tid, why, session_id, args, counts):
|
|
|
2052
2669
|
for k, v in found.items():
|
|
2053
2670
|
counts[k] = counts.get(k, 0) + v
|
|
2054
2671
|
hit = any(name in found for name, _ in REVIEW_SECRET_PATTERNS)
|
|
2055
|
-
key_text = (
|
|
2056
|
-
|
|
2672
|
+
key_text = (
|
|
2673
|
+
redact(
|
|
2674
|
+
_rejoin_secrets(t["text"], args.redact_extra),
|
|
2675
|
+
extra=args.redact_extra,
|
|
2676
|
+
mode=args.redact_mode,
|
|
2677
|
+
)[0]
|
|
2678
|
+
if hit
|
|
2679
|
+
else red
|
|
2680
|
+
)
|
|
2057
2681
|
preview, clipped = _preview(review_redact(t["text"], args.redact_extra)[0])
|
|
2058
2682
|
session = t.get("session") or session_id
|
|
2059
2683
|
context = _decision_context(why, _keeps(args))
|
|
2060
2684
|
llm_key = _llm_digest(key_text, context)
|
|
2061
2685
|
cached = None if args.rebuild_summaries else read_llm_cache(tid, session, llm_key)
|
|
2062
2686
|
view = _review_view(t, red, args)
|
|
2063
|
-
turn = {
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2687
|
+
turn = {
|
|
2688
|
+
"id": tid,
|
|
2689
|
+
"n": n,
|
|
2690
|
+
"role": t["role"],
|
|
2691
|
+
"ts": t.get("ts") or "",
|
|
2692
|
+
"tools": dict(t.get("tools") or {}),
|
|
2693
|
+
"text": red,
|
|
2694
|
+
"llm_key": llm_key,
|
|
2695
|
+
"redactions": _public_counts(found),
|
|
2696
|
+
"preview": preview,
|
|
2697
|
+
"clipped": clipped,
|
|
2698
|
+
"rule": {
|
|
2699
|
+
"keep": why is None,
|
|
2700
|
+
"why": why,
|
|
2701
|
+
"summary": _rule_summary(t, red, args),
|
|
2702
|
+
"review_summary": view["summary"],
|
|
2703
|
+
},
|
|
2704
|
+
"llm": dict(cached, cached=True) if cached else {"keep": None, "summary": None},
|
|
2705
|
+
}
|
|
2070
2706
|
if t.get("meta"):
|
|
2071
2707
|
turn["meta"] = True
|
|
2072
2708
|
if t["role"] == "user" and _bang_output(t["text"]):
|
|
2073
|
-
turn["bang_output"] =
|
|
2709
|
+
turn["bang_output"] = (
|
|
2710
|
+
True # for the dropped-turn WARN only: no part file carries it
|
|
2711
|
+
)
|
|
2074
2712
|
for k in ("session", "session_name", "agent_from"):
|
|
2075
2713
|
if t.get(k):
|
|
2076
|
-
turn[k] =
|
|
2714
|
+
turn[k] = (
|
|
2715
|
+
t[k]
|
|
2716
|
+
if k == "session"
|
|
2717
|
+
else page_redact(t[k], args.redact_extra, args.redact_mode)[0]
|
|
2718
|
+
)
|
|
2077
2719
|
return turn, view
|
|
2078
2720
|
|
|
2079
2721
|
|
|
@@ -2094,7 +2736,7 @@ def _split_even(items, size):
|
|
|
2094
2736
|
if not items:
|
|
2095
2737
|
return []
|
|
2096
2738
|
step = -(-len(items) // -(-len(items) // size))
|
|
2097
|
-
return [items[k:k + step] for k in range(0, len(items), step)]
|
|
2739
|
+
return [items[k : k + step] for k in range(0, len(items), step)]
|
|
2098
2740
|
|
|
2099
2741
|
|
|
2100
2742
|
def _decisions_name(part_path):
|
|
@@ -2109,8 +2751,12 @@ def _clear_parts(pack_path):
|
|
|
2109
2751
|
prefix = pack_path.stem + ".part-"
|
|
2110
2752
|
samples = f"{pack_path.stem}.reviewer-input.json"
|
|
2111
2753
|
try:
|
|
2112
|
-
olds = [
|
|
2113
|
-
|
|
2754
|
+
olds = [
|
|
2755
|
+
p
|
|
2756
|
+
for p in pack_path.parent.iterdir()
|
|
2757
|
+
if (p.name.startswith(prefix) and p.name.endswith(".json"))
|
|
2758
|
+
or p.name == samples
|
|
2759
|
+
]
|
|
2114
2760
|
except OSError:
|
|
2115
2761
|
return []
|
|
2116
2762
|
failed = []
|
|
@@ -2131,73 +2777,129 @@ def _hide_keywords(s, extra):
|
|
|
2131
2777
|
|
|
2132
2778
|
def _keeps(args):
|
|
2133
2779
|
"""The keep flags the reviewers must honour: with them, the rules keep those turns."""
|
|
2134
|
-
return {
|
|
2780
|
+
return {
|
|
2781
|
+
"keep_trivia": not args.drop_trivia,
|
|
2782
|
+
"keep_tool_only": not args.hide_tool_only,
|
|
2783
|
+
}
|
|
2135
2784
|
|
|
2136
2785
|
|
|
2137
2786
|
def _sheet(k, of, part_path, group, args):
|
|
2138
2787
|
"""Part file `k`: the turns one reviewer reads, without the page-only fields."""
|
|
2139
2788
|
turns = []
|
|
2140
2789
|
for x, view in group:
|
|
2141
|
-
y = {
|
|
2142
|
-
|
|
2143
|
-
|
|
2790
|
+
y = {
|
|
2791
|
+
key: v
|
|
2792
|
+
for key, v in x.items()
|
|
2793
|
+
if key
|
|
2794
|
+
not in (
|
|
2795
|
+
"text",
|
|
2796
|
+
"session",
|
|
2797
|
+
"session_name",
|
|
2798
|
+
"agent_from",
|
|
2799
|
+
"redactions",
|
|
2800
|
+
"llm_key",
|
|
2801
|
+
"bang_output",
|
|
2802
|
+
)
|
|
2803
|
+
}
|
|
2144
2804
|
y["rule"] = {k: v for k, v in x["rule"].items() if k != "review_summary"}
|
|
2145
2805
|
y["rule"]["summary"] = view["summary"]
|
|
2146
|
-
y["tools"] = {
|
|
2147
|
-
|
|
2806
|
+
y["tools"] = {
|
|
2807
|
+
_hide_keywords(name, args.redact_extra): n for name, n in x["tools"].items()
|
|
2808
|
+
}
|
|
2809
|
+
if x["llm"].get("summary"): # cached: redacted for the page, maybe masked
|
|
2148
2810
|
s = _MASKED.sub("[REDACTED]", x["llm"]["summary"])
|
|
2149
2811
|
y["llm"] = dict(x["llm"], summary=review_redact(s, args.redact_extra)[0])
|
|
2150
2812
|
if view["agent_from"]:
|
|
2151
2813
|
y["agent_from"] = view["agent_from"]
|
|
2152
2814
|
turns.append(y)
|
|
2153
|
-
return {
|
|
2154
|
-
|
|
2815
|
+
return {
|
|
2816
|
+
"schema": REVIEW_PART_SCHEMA,
|
|
2817
|
+
"part": k,
|
|
2818
|
+
"of": of,
|
|
2819
|
+
**_keeps(args),
|
|
2820
|
+
"decisions": _decisions_name(part_path),
|
|
2821
|
+
"turns": turns,
|
|
2822
|
+
}
|
|
2155
2823
|
|
|
2156
2824
|
|
|
2157
2825
|
def _pack_header(session_id, output, args, counts):
|
|
2158
2826
|
"""The pack's settings: what --apply-review renders with and the quality gate it runs."""
|
|
2159
|
-
return {
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2827
|
+
return {
|
|
2828
|
+
"schema": REVIEW_SCHEMA,
|
|
2829
|
+
"source": session_id,
|
|
2830
|
+
"output": str(output),
|
|
2831
|
+
"title": args.title,
|
|
2832
|
+
"markdown": args.markdown,
|
|
2833
|
+
"open": args.open_details,
|
|
2834
|
+
"redact_mode": args.redact_mode,
|
|
2835
|
+
"redact_extra": bool(args.redact_extra),
|
|
2836
|
+
"all_sessions": bool(args.all_sessions),
|
|
2837
|
+
"redactions": _public_counts(counts),
|
|
2838
|
+
**_keeps(args),
|
|
2839
|
+
"gate": {
|
|
2840
|
+
"skip_reviewer": bool(args.skip_reviewer),
|
|
2841
|
+
"reviewer_output": args.reviewer_output,
|
|
2842
|
+
"reviewer_timeout": args.reviewer_timeout,
|
|
2843
|
+
},
|
|
2844
|
+
}
|
|
2167
2845
|
|
|
2168
2846
|
|
|
2169
2847
|
def _report_emit(pack_path, items, paths, groups):
|
|
2170
2848
|
dropped = sum(1 for x in items if not x["rule"]["keep"])
|
|
2171
2849
|
cached = sum(1 for x in items if x["llm"].get("cached"))
|
|
2172
|
-
print(
|
|
2173
|
-
|
|
2850
|
+
print(
|
|
2851
|
+
f"[lineage] review pack: {pack_path} (turns={len(items)} "
|
|
2852
|
+
f"rule-dropped={dropped} cached={cached})",
|
|
2853
|
+
file=sys.stderr,
|
|
2854
|
+
)
|
|
2174
2855
|
for k, (p, g) in enumerate(zip(paths, groups), 1):
|
|
2175
2856
|
todo = sum(1 for x, _ in g if not x["llm"].get("cached"))
|
|
2176
|
-
print(
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2857
|
+
print(
|
|
2858
|
+
f"[lineage] part {k}/{len(groups)}: {p} (to review: {todo}) "
|
|
2859
|
+
f"-> {p.with_name(_decisions_name(p))}",
|
|
2860
|
+
file=sys.stderr,
|
|
2861
|
+
)
|
|
2862
|
+
print(
|
|
2863
|
+
f"[lineage] next: write each part's decisions, then --apply-review {pack_path}",
|
|
2864
|
+
file=sys.stderr,
|
|
2865
|
+
)
|
|
2180
2866
|
|
|
2181
2867
|
|
|
2182
2868
|
def emit_review(turns, drops, window, session_id, output, args):
|
|
2183
2869
|
"""Write the pack and one part file per reviewer; print them and the next step. A
|
|
2184
2870
|
--reviewer-output that holds something other than a verdict list stops it here, before
|
|
2185
2871
|
any reviewer runs: the gate would refuse it only after them."""
|
|
2186
|
-
rop =
|
|
2872
|
+
rop = (
|
|
2873
|
+
pathlib.Path(args.reviewer_output)
|
|
2874
|
+
if args.reviewer_output and not args.skip_reviewer
|
|
2875
|
+
else None
|
|
2876
|
+
)
|
|
2187
2877
|
if rop is not None and rop.exists() and not _is_verdict_file(rop):
|
|
2188
|
-
_bad(
|
|
2878
|
+
_bad(
|
|
2879
|
+
f"{rop} is there and is not a verdict list; give a --reviewer-output that does not exist yet"
|
|
2880
|
+
)
|
|
2189
2881
|
return 2
|
|
2190
2882
|
pack_path = pathlib.Path(args.emit_review)
|
|
2191
2883
|
counts, pairs = {}, []
|
|
2192
2884
|
for n, (i, tid) in enumerate(zip(window, _final_ids(turns, window)), 1):
|
|
2193
|
-
pairs.append(
|
|
2885
|
+
pairs.append(
|
|
2886
|
+
_pack_turn(n, turns[i], tid, drops.get(i), session_id, args, counts)
|
|
2887
|
+
)
|
|
2194
2888
|
groups = _split_even(pairs, REVIEW_PART)
|
|
2195
|
-
paths = [
|
|
2196
|
-
|
|
2889
|
+
paths = [
|
|
2890
|
+
pack_path.with_name(f"{pack_path.stem}.part-{k}.json")
|
|
2891
|
+
for k in range(1, len(groups) + 1)
|
|
2892
|
+
]
|
|
2197
2893
|
pack = _pack_header(session_id, _with_timestamp_suffix(output), args, counts)
|
|
2198
|
-
pack["parts"] = [
|
|
2199
|
-
|
|
2200
|
-
|
|
2894
|
+
pack["parts"] = [
|
|
2895
|
+
{
|
|
2896
|
+
"part": k,
|
|
2897
|
+
"file": p.name,
|
|
2898
|
+
"decisions": _decisions_name(p),
|
|
2899
|
+
"ids": [x["id"] for x, _ in g],
|
|
2900
|
+
}
|
|
2901
|
+
for k, (p, g) in enumerate(zip(paths, groups), 1)
|
|
2902
|
+
]
|
|
2201
2903
|
pack["turns"] = [x for x, _ in pairs]
|
|
2202
2904
|
stuck = _clear_parts(pack_path)
|
|
2203
2905
|
if stuck:
|
|
@@ -2206,8 +2908,12 @@ def emit_review(turns, drops, window, session_id, output, args):
|
|
|
2206
2908
|
try:
|
|
2207
2909
|
_write_private(pack_path, json.dumps(pack, ensure_ascii=False, indent=1))
|
|
2208
2910
|
for k, (p, g) in enumerate(zip(paths, groups), 1):
|
|
2209
|
-
_write_private(
|
|
2210
|
-
|
|
2911
|
+
_write_private(
|
|
2912
|
+
p,
|
|
2913
|
+
json.dumps(
|
|
2914
|
+
_sheet(k, len(groups), p, g, args), ensure_ascii=False, indent=1
|
|
2915
|
+
),
|
|
2916
|
+
)
|
|
2211
2917
|
except OSError as e:
|
|
2212
2918
|
_bad(f"cannot write the review pack: {e}")
|
|
2213
2919
|
return 2
|
|
@@ -2216,13 +2922,19 @@ def emit_review(turns, drops, window, session_id, output, args):
|
|
|
2216
2922
|
|
|
2217
2923
|
|
|
2218
2924
|
def _is_pack_turn(x):
|
|
2219
|
-
return (
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2925
|
+
return (
|
|
2926
|
+
isinstance(x, dict)
|
|
2927
|
+
and isinstance(x.get("id"), str)
|
|
2928
|
+
and x.get("role") in ("user", "assistant", "agent", "mark")
|
|
2929
|
+
and isinstance(x.get("text"), str)
|
|
2930
|
+
and isinstance(x.get("llm_key"), str)
|
|
2931
|
+
and isinstance(x.get("rule"), dict)
|
|
2932
|
+
and isinstance(x["rule"].get("keep"), bool)
|
|
2933
|
+
and _is_summary(x["rule"].get("summary"))
|
|
2934
|
+
and isinstance(x.get("llm"), dict)
|
|
2935
|
+
and _is_keep(x["llm"].get("keep"))
|
|
2936
|
+
and _is_summary(x["llm"].get("summary"))
|
|
2937
|
+
)
|
|
2226
2938
|
|
|
2227
2939
|
|
|
2228
2940
|
def _load_pack(path):
|
|
@@ -2231,9 +2943,13 @@ def _load_pack(path):
|
|
|
2231
2943
|
except (OSError, ValueError, UnicodeDecodeError, RecursionError) as e:
|
|
2232
2944
|
return _bad(f"cannot read review pack {path}: {e}")
|
|
2233
2945
|
if not isinstance(pack, dict) or pack.get("schema") != REVIEW_SCHEMA:
|
|
2234
|
-
return _bad(
|
|
2235
|
-
|
|
2236
|
-
|
|
2946
|
+
return _bad(
|
|
2947
|
+
f"{path} is not a {REVIEW_SCHEMA} pack (made by another lineage version? "
|
|
2948
|
+
"run --emit-review again)"
|
|
2949
|
+
)
|
|
2950
|
+
if not isinstance(pack.get("turns"), list) or not all(
|
|
2951
|
+
map(_is_pack_turn, pack["turns"])
|
|
2952
|
+
):
|
|
2237
2953
|
return _bad(f"{path}: malformed turns")
|
|
2238
2954
|
return pack
|
|
2239
2955
|
|
|
@@ -2241,8 +2957,11 @@ def _load_pack(path):
|
|
|
2241
2957
|
def _decision_files(pack_path, pack, explicit):
|
|
2242
2958
|
"""The parts' decisions files that exist beside the pack, then each --decisions.
|
|
2243
2959
|
Those beside the pack are made 0600: the session wrote them with its own umask."""
|
|
2244
|
-
beside = [
|
|
2245
|
-
|
|
2960
|
+
beside = [
|
|
2961
|
+
pack_path.with_name(p["decisions"])
|
|
2962
|
+
for p in pack.get("parts") or []
|
|
2963
|
+
if isinstance(p, dict) and isinstance(p.get("decisions"), str)
|
|
2964
|
+
]
|
|
2246
2965
|
files, seen = [], set()
|
|
2247
2966
|
for f in [f for f in beside if f.exists()] + [pathlib.Path(f) for f in explicit]:
|
|
2248
2967
|
if os.path.realpath(str(f)) not in seen:
|
|
@@ -2258,15 +2977,31 @@ def _decision_files(pack_path, pack, explicit):
|
|
|
2258
2977
|
|
|
2259
2978
|
# Keys of a part file's turns: a list of objects carrying one is the reviewer quoting its
|
|
2260
2979
|
# input, not its answer.
|
|
2261
|
-
_PART_TURN_KEYS = frozenset(
|
|
2262
|
-
|
|
2980
|
+
_PART_TURN_KEYS = frozenset(
|
|
2981
|
+
{
|
|
2982
|
+
"n",
|
|
2983
|
+
"role",
|
|
2984
|
+
"ts",
|
|
2985
|
+
"tools",
|
|
2986
|
+
"text",
|
|
2987
|
+
"preview",
|
|
2988
|
+
"clipped",
|
|
2989
|
+
"rule",
|
|
2990
|
+
"llm",
|
|
2991
|
+
"meta",
|
|
2992
|
+
"agent_from",
|
|
2993
|
+
}
|
|
2994
|
+
)
|
|
2263
2995
|
|
|
2264
2996
|
|
|
2265
2997
|
def _is_answer(d):
|
|
2266
2998
|
"""A decision, extra keys and all; a part turn quoted back (its keep and summary sit
|
|
2267
2999
|
under rule and llm, not on top) is not one."""
|
|
2268
|
-
return (
|
|
2269
|
-
|
|
3000
|
+
return (
|
|
3001
|
+
isinstance(d, dict)
|
|
3002
|
+
and "id" in d
|
|
3003
|
+
and ("keep" in d or "summary" in d or not _PART_TURN_KEYS & d.keys())
|
|
3004
|
+
)
|
|
2270
3005
|
|
|
2271
3006
|
|
|
2272
3007
|
def _decisions_text(text):
|
|
@@ -2285,8 +3020,10 @@ def _decisions_text(text):
|
|
|
2285
3020
|
# and only so many starts (each failed try parses up to the nesting limit)
|
|
2286
3021
|
for k, m in enumerate(re.finditer(r"\[(?=\s*[{\]])", text)):
|
|
2287
3022
|
if k == DECISIONS_TRIES:
|
|
2288
|
-
raise ValueError(
|
|
2289
|
-
|
|
3023
|
+
raise ValueError(
|
|
3024
|
+
f"more than {DECISIONS_TRIES} places a list may start; "
|
|
3025
|
+
"write the bare JSON array"
|
|
3026
|
+
)
|
|
2290
3027
|
try:
|
|
2291
3028
|
got, _ = decoder.raw_decode(text, m.start())
|
|
2292
3029
|
except (ValueError, RecursionError):
|
|
@@ -2304,8 +3041,12 @@ def _decisions_text(text):
|
|
|
2304
3041
|
|
|
2305
3042
|
def _decision_problem(d):
|
|
2306
3043
|
"""Why item `d` of a decisions file is not a decision, or ""."""
|
|
2307
|
-
if not (
|
|
2308
|
-
|
|
3044
|
+
if not (
|
|
3045
|
+
isinstance(d, dict)
|
|
3046
|
+
and isinstance(d.get("id"), str)
|
|
3047
|
+
and _is_keep(d.get("keep"))
|
|
3048
|
+
and _is_summary(d.get("summary"))
|
|
3049
|
+
):
|
|
2309
3050
|
return 'want {"id": str, "keep": true|false|null, "summary": str|null}'
|
|
2310
3051
|
if not _is_answer(d):
|
|
2311
3052
|
return "a turn of the part file, not a decision"
|
|
@@ -2330,13 +3071,20 @@ def _load_decisions(files):
|
|
|
2330
3071
|
why = _decision_problem(d)
|
|
2331
3072
|
if why:
|
|
2332
3073
|
return _bad(f"{f}[{k}]: {why}")
|
|
2333
|
-
cur = merged.setdefault(
|
|
3074
|
+
cur = merged.setdefault(
|
|
3075
|
+
_lone_to_fffd(d["id"]), {}
|
|
3076
|
+
) # a model's JSON may hold a lone surrogate
|
|
2334
3077
|
for key in ("keep", "summary"):
|
|
2335
3078
|
if d.get(key) is not None:
|
|
2336
3079
|
cur[key] = _lone_to_fffd(d[key])
|
|
2337
|
-
if items and all(
|
|
2338
|
-
|
|
2339
|
-
|
|
3080
|
+
if items and all(
|
|
3081
|
+
d.get("keep") is None and d.get("summary") is None for d in items
|
|
3082
|
+
):
|
|
3083
|
+
print(
|
|
3084
|
+
f"[lineage] WARN: {f}: all {len(items)} decisions are null; the rules "
|
|
3085
|
+
"decide those turns",
|
|
3086
|
+
file=sys.stderr,
|
|
3087
|
+
)
|
|
2340
3088
|
return merged
|
|
2341
3089
|
|
|
2342
3090
|
|
|
@@ -2361,9 +3109,11 @@ def _warn_coverage(pack, decided):
|
|
|
2361
3109
|
todo = [i for i in ids if i not in decided]
|
|
2362
3110
|
if todo:
|
|
2363
3111
|
more = ", ..." if len(todo) > 3 else ""
|
|
2364
|
-
print(
|
|
2365
|
-
|
|
2366
|
-
|
|
3112
|
+
print(
|
|
3113
|
+
f"[lineage] WARN: part {p.get('part')}: {len(todo)}/{len(ids)} turns "
|
|
3114
|
+
f"undecided ({', '.join(map(str, todo[:3]))}{more}); the rules decide them",
|
|
3115
|
+
file=sys.stderr,
|
|
3116
|
+
)
|
|
2367
3117
|
|
|
2368
3118
|
|
|
2369
3119
|
def _reviewer_summary(x, d, args, stats):
|
|
@@ -2375,8 +3125,11 @@ def _reviewer_summary(x, d, args, stats):
|
|
|
2375
3125
|
s, n = _clean_summary(raw, args.redact_extra, args.redact_mode)
|
|
2376
3126
|
if n:
|
|
2377
3127
|
stats["hidden"] += n
|
|
2378
|
-
print(
|
|
2379
|
-
|
|
3128
|
+
print(
|
|
3129
|
+
f"[lineage] WARN: the reviewer summary for {x['id']} held {n} "
|
|
3130
|
+
"secret-like string(s); redacted",
|
|
3131
|
+
file=sys.stderr,
|
|
3132
|
+
)
|
|
2380
3133
|
return s
|
|
2381
3134
|
|
|
2382
3135
|
|
|
@@ -2384,7 +3137,8 @@ def _show(x, llm_summary, turns, summaries, stats, prior):
|
|
|
2384
3137
|
"""Put kept pack turn `x` on the page with the summary it shows; `prior` adds up what
|
|
2385
3138
|
--emit-review redacted."""
|
|
2386
3139
|
shows = x["role"] in ("assistant", "agent") or (
|
|
2387
|
-
x["role"] == "user" and len(x["text"]) > USER_FOLD
|
|
3140
|
+
x["role"] == "user" and len(x["text"]) > USER_FOLD
|
|
3141
|
+
)
|
|
2388
3142
|
if llm_summary and not shows:
|
|
2389
3143
|
stats["ignored"] += 1
|
|
2390
3144
|
summary = (llm_summary if shows else None) or x["rule"].get("summary")
|
|
@@ -2397,12 +3151,21 @@ def _show(x, llm_summary, turns, summaries, stats, prior):
|
|
|
2397
3151
|
# what the gate's critic reads: the reviewer's summary, else the rules' cut from the
|
|
2398
3152
|
# fully redacted text (the page's may hold a --redact-mode mask: 4 + 4 characters of a value)
|
|
2399
3153
|
sample = (llm_summary if shows else None) or x["rule"].get("review_summary")
|
|
2400
|
-
turns.append(
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
3154
|
+
turns.append(
|
|
3155
|
+
{
|
|
3156
|
+
"uuid": x["id"],
|
|
3157
|
+
"role": x["role"],
|
|
3158
|
+
"text": x["text"],
|
|
3159
|
+
"ts": x.get("ts") or None,
|
|
3160
|
+
"tools": dict(x.get("tools") or {}),
|
|
3161
|
+
"parts": [],
|
|
3162
|
+
"session": x.get("session"),
|
|
3163
|
+
"session_name": x.get("session_name"),
|
|
3164
|
+
"agent_from": x.get("agent_from"),
|
|
3165
|
+
"preview": x.get("preview"),
|
|
3166
|
+
"sample_summary": sample,
|
|
3167
|
+
}
|
|
3168
|
+
)
|
|
2406
3169
|
|
|
2407
3170
|
|
|
2408
3171
|
def _kept_by_flag(pack, x):
|
|
@@ -2430,8 +3193,12 @@ def _typed_by_user(x):
|
|
|
2430
3193
|
"""Whether pack turn `x` holds words the user typed: a user turn the rules keep that is neither a body the
|
|
2431
3194
|
harness injected (meta) nor what Claude Code writes after a `!` command (bang_output, judged on the text before
|
|
2432
3195
|
redaction: a hidden value at the end of the output takes the closing tags with it)."""
|
|
2433
|
-
return (
|
|
2434
|
-
|
|
3196
|
+
return (
|
|
3197
|
+
x["role"] == "user"
|
|
3198
|
+
and x["rule"]["keep"]
|
|
3199
|
+
and not x.get("meta")
|
|
3200
|
+
and x.get("bang_output") is not True
|
|
3201
|
+
)
|
|
2435
3202
|
|
|
2436
3203
|
|
|
2437
3204
|
def _review_result(pack, merged, args):
|
|
@@ -2440,8 +3207,15 @@ def _review_result(pack, merged, args):
|
|
|
2440
3207
|
reviewer's value first, then the pack's cached one, then the rules'. A keep flag outranks
|
|
2441
3208
|
a reviewer's `false`; the cache still gets what was said."""
|
|
2442
3209
|
turns, summaries, writes, prior, typed = [], {}, [], {}, []
|
|
2443
|
-
stats = {
|
|
2444
|
-
|
|
3210
|
+
stats = {
|
|
3211
|
+
"restored": 0,
|
|
3212
|
+
"dropped": 0,
|
|
3213
|
+
"reviewer": 0,
|
|
3214
|
+
"rule": 0,
|
|
3215
|
+
"ignored": 0,
|
|
3216
|
+
"hidden": 0,
|
|
3217
|
+
"flag-kept": 0,
|
|
3218
|
+
}
|
|
2445
3219
|
for x in pack["turns"]:
|
|
2446
3220
|
d = merged.get(x["id"], {})
|
|
2447
3221
|
llm_keep = _first(d.get("keep"), x["llm"].get("keep"))
|
|
@@ -2449,11 +3223,18 @@ def _review_result(pack, merged, args):
|
|
|
2449
3223
|
stats["flag-kept"] += held
|
|
2450
3224
|
keep = bool(_first(None if held else llm_keep, x["rule"]["keep"]))
|
|
2451
3225
|
if not keep and _typed_by_user(x):
|
|
2452
|
-
typed.append(x["id"])
|
|
3226
|
+
typed.append(x["id"]) # typed by the user: the rules keep these
|
|
2453
3227
|
llm_summary = _reviewer_summary(x, d, args, stats)
|
|
2454
|
-
if x["id"] in merged:
|
|
2455
|
-
writes.append(
|
|
2456
|
-
|
|
3228
|
+
if x["id"] in merged: # reviewed, even if left to the rules
|
|
3229
|
+
writes.append(
|
|
3230
|
+
(
|
|
3231
|
+
x["id"],
|
|
3232
|
+
x.get("session") or pack.get("source"),
|
|
3233
|
+
x["llm_key"],
|
|
3234
|
+
llm_keep,
|
|
3235
|
+
llm_summary,
|
|
3236
|
+
)
|
|
3237
|
+
)
|
|
2457
3238
|
stats["restored"] += keep and not x["rule"]["keep"]
|
|
2458
3239
|
stats["dropped"] += x["rule"]["keep"] and not keep
|
|
2459
3240
|
if keep:
|
|
@@ -2476,31 +3257,42 @@ def _check_pack(pack, merged, args):
|
|
|
2476
3257
|
ids = {x["id"] for x in pack["turns"]}
|
|
2477
3258
|
unknown = [i for i in merged if i not in ids]
|
|
2478
3259
|
if unknown:
|
|
2479
|
-
print(
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
3260
|
+
print(
|
|
3261
|
+
f"[lineage] WARN: decisions name {len(unknown)} turn(s) not in the pack, "
|
|
3262
|
+
f"ignored: {', '.join(_shown_id(i) for i in unknown[:5])}",
|
|
3263
|
+
file=sys.stderr,
|
|
3264
|
+
)
|
|
3265
|
+
_warn_coverage(
|
|
3266
|
+
pack, set(merged) | {x["id"] for x in pack["turns"] if x["llm"].get("cached")}
|
|
3267
|
+
)
|
|
2483
3268
|
if pack.get("redact_mode") in ("full", "mask"):
|
|
2484
3269
|
args.redact_mode = pack["redact_mode"]
|
|
2485
3270
|
if pack.get("redact_extra") is True and not args.redact_extra:
|
|
2486
|
-
print(
|
|
2487
|
-
|
|
3271
|
+
print(
|
|
3272
|
+
"[lineage] WARN: --emit-review had --redact-extra; give the same keywords "
|
|
3273
|
+
"here too, or reviewer summaries miss that redaction",
|
|
3274
|
+
file=sys.stderr,
|
|
3275
|
+
)
|
|
2488
3276
|
_gate_from_pack(args, pack)
|
|
2489
3277
|
|
|
2490
3278
|
|
|
2491
3279
|
def _remove_review_files(pack_path, pack):
|
|
2492
3280
|
"""After a good render the pack, its parts and the decisions beside it go: they hold
|
|
2493
3281
|
the redacted session."""
|
|
2494
|
-
samples =
|
|
3282
|
+
samples = (
|
|
3283
|
+
f"{pack_path.stem}.reviewer-input.json" # a failed gate's, beside the pack
|
|
3284
|
+
)
|
|
2495
3285
|
names = [pack_path.name, samples]
|
|
2496
3286
|
for p in pack.get("parts") or []:
|
|
2497
3287
|
if isinstance(p, dict):
|
|
2498
|
-
names += [
|
|
3288
|
+
names += [
|
|
3289
|
+
p.get(k) for k in ("file", "decisions") if isinstance(p.get(k), str)
|
|
3290
|
+
]
|
|
2499
3291
|
prefix = pack_path.stem + ".part-"
|
|
2500
3292
|
left = []
|
|
2501
3293
|
for name in names:
|
|
2502
3294
|
if name not in (pack_path.name, samples) and not name.startswith(prefix):
|
|
2503
|
-
continue
|
|
3295
|
+
continue # only what --emit-review named
|
|
2504
3296
|
try:
|
|
2505
3297
|
pack_path.with_name(name).unlink()
|
|
2506
3298
|
except FileNotFoundError:
|
|
@@ -2508,8 +3300,10 @@ def _remove_review_files(pack_path, pack):
|
|
|
2508
3300
|
except (OSError, ValueError):
|
|
2509
3301
|
left.append(name)
|
|
2510
3302
|
if left:
|
|
2511
|
-
print(
|
|
2512
|
-
|
|
3303
|
+
print(
|
|
3304
|
+
f"[lineage] WARN: could not remove review files: {', '.join(left)}",
|
|
3305
|
+
file=sys.stderr,
|
|
3306
|
+
)
|
|
2513
3307
|
|
|
2514
3308
|
|
|
2515
3309
|
def _samples_left_unread(pack_path, args):
|
|
@@ -2517,7 +3311,9 @@ def _samples_left_unread(pack_path, args):
|
|
|
2517
3311
|
gives neither a verdict path nor --skip-reviewer: it would pass over that run's verdict."""
|
|
2518
3312
|
left = pack_path.with_name(f"{pack_path.stem}.reviewer-input.json")
|
|
2519
3313
|
if left.exists() and not (args.reviewer_output or args.skip_reviewer):
|
|
2520
|
-
_bad(
|
|
3314
|
+
_bad(
|
|
3315
|
+
f"{left} was left by a gated run; give its --reviewer-output again, or --skip-reviewer"
|
|
3316
|
+
)
|
|
2521
3317
|
return True
|
|
2522
3318
|
return False
|
|
2523
3319
|
|
|
@@ -2532,18 +3328,30 @@ def _review_output(args, pack):
|
|
|
2532
3328
|
|
|
2533
3329
|
def _report_review(stats, turns, pack, typed):
|
|
2534
3330
|
"""What the render took from the reviewers, and the turns typed by the user they dropped."""
|
|
2535
|
-
print(
|
|
2536
|
-
|
|
2537
|
-
|
|
3331
|
+
print(
|
|
3332
|
+
f"[lineage] review applied: shown={len(turns)}/{len(pack['turns'])} "
|
|
3333
|
+
f"restored={stats['restored']} dropped={stats['dropped']} "
|
|
3334
|
+
f"summaries reviewer={stats['reviewer']} rule={stats['rule']}",
|
|
3335
|
+
file=sys.stderr,
|
|
3336
|
+
)
|
|
2538
3337
|
if stats["ignored"]:
|
|
2539
|
-
print(
|
|
2540
|
-
|
|
3338
|
+
print(
|
|
3339
|
+
f"[lineage] note: {stats['ignored']} summaries for turns the page shows "
|
|
3340
|
+
"in full were ignored",
|
|
3341
|
+
file=sys.stderr,
|
|
3342
|
+
)
|
|
2541
3343
|
if stats["flag-kept"]:
|
|
2542
|
-
print(
|
|
2543
|
-
|
|
3344
|
+
print(
|
|
3345
|
+
f"[lineage] note: {stats['flag-kept']} turn(s) a reviewer marked keep: false "
|
|
3346
|
+
"stay, as --keep-trivia or --keep-tool-only asked",
|
|
3347
|
+
file=sys.stderr,
|
|
3348
|
+
)
|
|
2544
3349
|
if typed:
|
|
2545
|
-
print(
|
|
2546
|
-
|
|
3350
|
+
print(
|
|
3351
|
+
f"[lineage] WARN: reviewers dropped {len(typed)} typed user turn(s): {', '.join(_shown_id(i) for i in typed[:10])}"
|
|
3352
|
+
f"{' ...' if len(typed) > 10 else ''}; tell the user",
|
|
3353
|
+
file=sys.stderr,
|
|
3354
|
+
)
|
|
2547
3355
|
|
|
2548
3356
|
|
|
2549
3357
|
def apply_review(args):
|
|
@@ -2564,13 +3372,18 @@ def apply_review(args):
|
|
|
2564
3372
|
if stats["hidden"]:
|
|
2565
3373
|
prior["reviewer-summary"] = stats["hidden"]
|
|
2566
3374
|
try:
|
|
2567
|
-
rc = render_and_write(
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
3375
|
+
rc = render_and_write(
|
|
3376
|
+
turns,
|
|
3377
|
+
pack.get("source") or "session",
|
|
3378
|
+
args,
|
|
3379
|
+
all_sessions=bool(pack.get("all_sessions")),
|
|
3380
|
+
markdown=pack.get("markdown") is not False,
|
|
3381
|
+
open_details=bool(pack.get("open")),
|
|
3382
|
+
title=str(pack.get("title") or "Session Lineage"),
|
|
3383
|
+
summaries=summaries,
|
|
3384
|
+
prior_redactions=prior,
|
|
3385
|
+
)
|
|
3386
|
+
except SystemExit as e: # the quality gate failed
|
|
2574
3387
|
rc = int(e.code) if isinstance(e.code, int) else 2
|
|
2575
3388
|
if rc == 0:
|
|
2576
3389
|
for w in writes:
|
|
@@ -2583,66 +3396,133 @@ def apply_review(args):
|
|
|
2583
3396
|
def build_arg_parser() -> argparse.ArgumentParser:
|
|
2584
3397
|
ap = argparse.ArgumentParser(
|
|
2585
3398
|
prog="lineage",
|
|
2586
|
-
description="Export Claude Code session(s) as a KakaoTalk-style HTML file."
|
|
3399
|
+
description="Export Claude Code session(s) as a KakaoTalk-style HTML file.",
|
|
3400
|
+
)
|
|
2587
3401
|
ap.add_argument("--last", type=int, help="keep last N turns")
|
|
2588
3402
|
ap.add_argument("--from", dest="from_", help="ISO timestamp from (inclusive)")
|
|
2589
3403
|
ap.add_argument("--to", help="ISO timestamp to (inclusive)")
|
|
2590
3404
|
ap.add_argument("--turns", help='range like "10-50" (1-indexed inclusive)')
|
|
2591
|
-
ap.add_argument(
|
|
2592
|
-
|
|
3405
|
+
ap.add_argument(
|
|
3406
|
+
"--output",
|
|
3407
|
+
default=None,
|
|
3408
|
+
help="output HTML path (default: work/lineage-{name}_YYMMDD+HHMM.html)",
|
|
3409
|
+
)
|
|
2593
3410
|
ap.add_argument("--session", help="jsonl path (overrides auto-discover)")
|
|
2594
|
-
ap.add_argument(
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
3411
|
+
ap.add_argument(
|
|
3412
|
+
"--all-sessions",
|
|
3413
|
+
action="store_true",
|
|
3414
|
+
help="merge ALL sessions in the project dir, chronologically",
|
|
3415
|
+
)
|
|
3416
|
+
ap.add_argument(
|
|
3417
|
+
"--from-transcript", help="read transcript from PATH or '-' for stdin"
|
|
3418
|
+
)
|
|
2598
3419
|
ap.add_argument("--redact-extra", help="comma-separated keywords to also redact")
|
|
2599
|
-
ap.add_argument(
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
ap.add_argument(
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
3420
|
+
ap.add_argument(
|
|
3421
|
+
"--redact-mode",
|
|
3422
|
+
choices=["full", "mask"],
|
|
3423
|
+
default="full",
|
|
3424
|
+
help="redaction mode (full=[REDACTED], mask=abcd****wxyz)",
|
|
3425
|
+
)
|
|
3426
|
+
ap.add_argument(
|
|
3427
|
+
"--unsafe-schema",
|
|
3428
|
+
action="store_true",
|
|
3429
|
+
help="proceed even on unknown schema markers",
|
|
3430
|
+
)
|
|
3431
|
+
ap.add_argument(
|
|
3432
|
+
"--rebuild-summaries",
|
|
3433
|
+
action="store_true",
|
|
3434
|
+
help="ignore summary cache and re-summarize",
|
|
3435
|
+
)
|
|
3436
|
+
ap.add_argument(
|
|
3437
|
+
"--purge-cache", action="store_true", help="purge ~/.cache/lineage then exit"
|
|
3438
|
+
)
|
|
3439
|
+
ap.add_argument(
|
|
3440
|
+
"--skip-reviewer",
|
|
3441
|
+
action="store_true",
|
|
3442
|
+
help="skip reviewer quality gate (warn-only)",
|
|
3443
|
+
)
|
|
3444
|
+
ap.add_argument(
|
|
3445
|
+
"--reviewer-output",
|
|
3446
|
+
help="path to Critic response JSON; enforces the quality gate",
|
|
3447
|
+
)
|
|
3448
|
+
ap.add_argument(
|
|
3449
|
+
"--reviewer-timeout",
|
|
3450
|
+
type=int,
|
|
3451
|
+
default=None,
|
|
3452
|
+
help="seconds to wait for reviewer-output (default 60 with --rulebase; "
|
|
3453
|
+
"the reviewed flow waits only when this is given)",
|
|
3454
|
+
)
|
|
2614
3455
|
ap.add_argument("--title", default="Session Lineage", help="HTML title")
|
|
2615
3456
|
_add_review_args(ap)
|
|
2616
3457
|
# Readability defaults are ON. Opt-out flags restore raw/older behavior.
|
|
2617
|
-
ap.add_argument(
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
ap.add_argument(
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
3458
|
+
ap.add_argument(
|
|
3459
|
+
"--hide-tool-only",
|
|
3460
|
+
dest="hide_tool_only",
|
|
3461
|
+
action="store_true",
|
|
3462
|
+
help="(DEFAULT) drop assistant turns that are only tool calls",
|
|
3463
|
+
)
|
|
3464
|
+
ap.add_argument(
|
|
3465
|
+
"--keep-tool-only",
|
|
3466
|
+
dest="hide_tool_only",
|
|
3467
|
+
action="store_false",
|
|
3468
|
+
help="keep tool-only assistant turns (override default)",
|
|
3469
|
+
)
|
|
3470
|
+
ap.add_argument(
|
|
3471
|
+
"--no-markdown",
|
|
3472
|
+
dest="markdown",
|
|
3473
|
+
action="store_false",
|
|
3474
|
+
help="disable markdown rendering (default: render ON)",
|
|
3475
|
+
)
|
|
3476
|
+
ap.add_argument(
|
|
3477
|
+
"--keep-trivia",
|
|
3478
|
+
dest="drop_trivia",
|
|
3479
|
+
action="store_false",
|
|
3480
|
+
help="keep manipulation commands / injected bodies / echo / "
|
|
3481
|
+
"harness errors (default: filter ON)",
|
|
3482
|
+
)
|
|
3483
|
+
ap.add_argument(
|
|
3484
|
+
"--open",
|
|
3485
|
+
dest="open_details",
|
|
3486
|
+
action="store_true",
|
|
3487
|
+
help="render bubbles expanded (default: collapsed)",
|
|
3488
|
+
)
|
|
3489
|
+
ap.add_argument(
|
|
3490
|
+
"--collapse",
|
|
3491
|
+
dest="open_details",
|
|
3492
|
+
action="store_false",
|
|
3493
|
+
help="(DEFAULT) render bubbles collapsed",
|
|
3494
|
+
)
|
|
3495
|
+
ap.set_defaults(
|
|
3496
|
+
hide_tool_only=True, open_details=False, markdown=True, drop_trivia=True
|
|
3497
|
+
)
|
|
2632
3498
|
return ap
|
|
2633
3499
|
|
|
2634
3500
|
|
|
2635
3501
|
def _add_review_args(ap):
|
|
2636
3502
|
"""The flags of the reviewed flow (3.0.0): --emit-review, the session's reviewers,
|
|
2637
3503
|
then --apply-review."""
|
|
2638
|
-
ap.add_argument(
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
3504
|
+
ap.add_argument(
|
|
3505
|
+
"--rulebase",
|
|
3506
|
+
action="store_true",
|
|
3507
|
+
help="one pass, rules only (the default when no review flag is given)",
|
|
3508
|
+
)
|
|
3509
|
+
ap.add_argument(
|
|
3510
|
+
"--emit-review",
|
|
3511
|
+
metavar="PACK",
|
|
3512
|
+
help="write the turns for the session model to review (JSON) and stop",
|
|
3513
|
+
)
|
|
3514
|
+
ap.add_argument(
|
|
3515
|
+
"--apply-review",
|
|
3516
|
+
metavar="PACK",
|
|
3517
|
+
help="render from a review pack and the reviewer's decisions",
|
|
3518
|
+
)
|
|
3519
|
+
ap.add_argument(
|
|
3520
|
+
"--decisions",
|
|
3521
|
+
action="append",
|
|
3522
|
+
default=[],
|
|
3523
|
+
metavar="FILE",
|
|
3524
|
+
help="reviewer decisions [{id, keep, summary}] for --apply-review (repeatable)",
|
|
3525
|
+
)
|
|
2646
3526
|
|
|
2647
3527
|
|
|
2648
3528
|
def _open_stream(path):
|
|
@@ -2653,12 +3533,18 @@ def _load_turns_from(path, unsafe_schema, session_id, session_name):
|
|
|
2653
3533
|
"""Parse one jsonl file, isolating per-file decode errors (D-3)."""
|
|
2654
3534
|
try:
|
|
2655
3535
|
with _open_stream(path) as stream:
|
|
2656
|
-
return list(
|
|
2657
|
-
|
|
2658
|
-
|
|
3536
|
+
return list(
|
|
3537
|
+
parse_turns(
|
|
3538
|
+
stream,
|
|
3539
|
+
unsafe_schema=unsafe_schema,
|
|
3540
|
+
session_id=session_id,
|
|
3541
|
+
session_name=session_name,
|
|
3542
|
+
)
|
|
3543
|
+
)
|
|
2659
3544
|
except (OSError, UnicodeDecodeError) as e:
|
|
2660
|
-
print(
|
|
2661
|
-
|
|
3545
|
+
print(
|
|
3546
|
+
f"[lineage] WARN: skipping unreadable session {path}: {e}", file=sys.stderr
|
|
3547
|
+
)
|
|
2662
3548
|
return []
|
|
2663
3549
|
|
|
2664
3550
|
|
|
@@ -2681,12 +3567,12 @@ def _merge_env_extra(args):
|
|
|
2681
3567
|
"""LINEAGE_REDACT_EXTRA's keywords after --redact-extra's."""
|
|
2682
3568
|
env_extra = os.environ.get("LINEAGE_REDACT_EXTRA", "").strip()
|
|
2683
3569
|
if env_extra:
|
|
2684
|
-
args.redact_extra = ",".join(
|
|
2685
|
-
p for p in (args.redact_extra, env_extra) if p)
|
|
3570
|
+
args.redact_extra = ",".join(p for p in (args.redact_extra, env_extra) if p)
|
|
2686
3571
|
|
|
2687
3572
|
|
|
2688
3573
|
def _purge_cache():
|
|
2689
3574
|
import shutil
|
|
3575
|
+
|
|
2690
3576
|
if CACHE_BASE.exists():
|
|
2691
3577
|
shutil.rmtree(CACHE_BASE)
|
|
2692
3578
|
print(f"[lineage] cache purged: {CACHE_BASE}", file=sys.stderr)
|
|
@@ -2696,10 +3582,22 @@ def _purge_cache():
|
|
|
2696
3582
|
|
|
2697
3583
|
|
|
2698
3584
|
def _run_apply(args):
|
|
2699
|
-
if any(
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
3585
|
+
if any(
|
|
3586
|
+
(
|
|
3587
|
+
args.last,
|
|
3588
|
+
args.from_,
|
|
3589
|
+
args.to,
|
|
3590
|
+
args.turns,
|
|
3591
|
+
args.session,
|
|
3592
|
+
args.all_sessions,
|
|
3593
|
+
args.from_transcript,
|
|
3594
|
+
)
|
|
3595
|
+
):
|
|
3596
|
+
print(
|
|
3597
|
+
"[lineage] note: --apply-review renders the pack; source and range "
|
|
3598
|
+
"flags were applied at --emit-review",
|
|
3599
|
+
file=sys.stderr,
|
|
3600
|
+
)
|
|
2703
3601
|
return apply_review(args)
|
|
2704
3602
|
|
|
2705
3603
|
|
|
@@ -2707,30 +3605,42 @@ def _read_all_sessions(args):
|
|
|
2707
3605
|
"""Every session of the project folder, merged record by record in time order."""
|
|
2708
3606
|
files = project_jsonl_files()
|
|
2709
3607
|
if not files:
|
|
2710
|
-
print(
|
|
2711
|
-
|
|
3608
|
+
print(
|
|
3609
|
+
"[lineage] --all-sessions: no jsonl found in project dir", file=sys.stderr
|
|
3610
|
+
)
|
|
2712
3611
|
return 2, [], "session", None
|
|
2713
3612
|
starts = {p: session_start(p) for p in files}
|
|
2714
3613
|
turns = []
|
|
2715
3614
|
for p in files:
|
|
2716
|
-
name =
|
|
3615
|
+
name = (
|
|
3616
|
+
discover_session_name(p)
|
|
3617
|
+
or (p.stem[-36:] if p.stem.startswith("rollout-") else p.stem)[:8]
|
|
3618
|
+
)
|
|
2717
3619
|
part = _load_turns_from(p, args.unsafe_schema, p.stem, name)
|
|
3620
|
+
meta = _codex_meta(p)
|
|
3621
|
+
if meta is not None and not any(t["role"] == "user" for t in part):
|
|
3622
|
+
print(
|
|
3623
|
+
f"[lineage] WARN: {p.name}: no user turn read from this Codex {meta.get('cli_version', '?')} "
|
|
3624
|
+
"record (a new record shape?); left out",
|
|
3625
|
+
file=sys.stderr,
|
|
3626
|
+
)
|
|
3627
|
+
continue
|
|
2718
3628
|
fill_missing_ts(part)
|
|
2719
3629
|
for t in part:
|
|
2720
3630
|
t["session_ts"] = starts[p] or "9999"
|
|
2721
3631
|
turns.extend(part)
|
|
2722
3632
|
# record-level chronological sort (stable → same-ts keeps file order)
|
|
2723
3633
|
turns.sort(key=lambda t: t.get("ts") or t.get("session_ts") or "9999")
|
|
2724
|
-
print(f"[lineage] --all-sessions: merged {len(files)} sessions",
|
|
2725
|
-
file=sys.stderr)
|
|
3634
|
+
print(f"[lineage] --all-sessions: merged {len(files)} sessions", file=sys.stderr)
|
|
2726
3635
|
return None, turns, "all-sessions", None
|
|
2727
3636
|
|
|
2728
3637
|
|
|
2729
3638
|
def _read_transcript(args):
|
|
2730
3639
|
"""A transcript pasted on stdin ('-') or read from a file."""
|
|
2731
3640
|
if args.from_transcript == "-":
|
|
2732
|
-
turns = list(
|
|
2733
|
-
|
|
3641
|
+
turns = list(
|
|
3642
|
+
parse_turns(sys.stdin, unsafe_schema=args.unsafe_schema, session_id="stdin")
|
|
3643
|
+
)
|
|
2734
3644
|
return None, turns, "stdin", None
|
|
2735
3645
|
jsonl_path = pathlib.Path(args.from_transcript)
|
|
2736
3646
|
session_id = "file-" + jsonl_path.stem
|
|
@@ -2742,13 +3652,15 @@ def _read_discovered(args):
|
|
|
2742
3652
|
"""The newest session of the project folder."""
|
|
2743
3653
|
f = auto_discover_jsonl()
|
|
2744
3654
|
if not f:
|
|
2745
|
-
print(
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
3655
|
+
print(
|
|
3656
|
+
"[lineage] No jsonl found. Use one of:",
|
|
3657
|
+
"\n --session FILE explicit jsonl path",
|
|
3658
|
+
"\n --all-sessions merge all project sessions",
|
|
3659
|
+
"\n --from-transcript - paste transcript via stdin",
|
|
3660
|
+
"\n --from-transcript FILE read transcript from file",
|
|
3661
|
+
"\n(non-ASCII cwd may not auto-encode — use --session)",
|
|
3662
|
+
file=sys.stderr,
|
|
3663
|
+
)
|
|
2752
3664
|
return 2, [], "session", None
|
|
2753
3665
|
turns = _load_turns_from(f, args.unsafe_schema, f.stem, None)
|
|
2754
3666
|
print(f"[lineage] auto-discovered session: {f.name}", file=sys.stderr)
|
|
@@ -2756,8 +3668,26 @@ def _read_discovered(args):
|
|
|
2756
3668
|
|
|
2757
3669
|
|
|
2758
3670
|
def _read_input(args):
|
|
2759
|
-
"""(exit code or None, raw turns, session id, jsonl path) from the source the flags name
|
|
2760
|
-
|
|
3671
|
+
"""(exit code or None, raw turns, session id, jsonl path) from the source the flags name: a Claude Code
|
|
3672
|
+
transcript or a Codex session record (rollout), told apart by its first record. The code is 2 when there is no
|
|
3673
|
+
source, or the code a schema marker stopped the read with. A Codex rollout that gives no user turn stops with 2
|
|
3674
|
+
too: every Codex session starts with what the user typed, so none means this Codex version writes its record in a
|
|
3675
|
+
shape lineage does not read, and an empty page would hide that."""
|
|
3676
|
+
rc, turns, session_id, jsonl_path = _read_source(args)
|
|
3677
|
+
meta = _codex_meta(jsonl_path) if rc is None and jsonl_path is not None else None
|
|
3678
|
+
if meta is not None and not any(t["role"] == "user" for t in turns):
|
|
3679
|
+
print(
|
|
3680
|
+
f"[lineage] ERROR: {jsonl_path.name} is a Codex {meta.get('cli_version', '?')} session record, but no "
|
|
3681
|
+
"user turn was read from it; this Codex version may write its record in a new shape "
|
|
3682
|
+
"(nothing written; update banker or report it)",
|
|
3683
|
+
file=sys.stderr,
|
|
3684
|
+
)
|
|
3685
|
+
return 2, [], session_id, None
|
|
3686
|
+
return rc, turns, session_id, jsonl_path
|
|
3687
|
+
|
|
3688
|
+
|
|
3689
|
+
def _read_source(args):
|
|
3690
|
+
"""(exit code or None, raw turns, session id, jsonl path) as read, before the Codex shape check."""
|
|
2761
3691
|
try:
|
|
2762
3692
|
if args.all_sessions:
|
|
2763
3693
|
return _read_all_sessions(args)
|
|
@@ -2765,7 +3695,9 @@ def _read_input(args):
|
|
|
2765
3695
|
return _read_transcript(args)
|
|
2766
3696
|
if args.session:
|
|
2767
3697
|
jsonl_path = pathlib.Path(args.session)
|
|
2768
|
-
turns = _load_turns_from(
|
|
3698
|
+
turns = _load_turns_from(
|
|
3699
|
+
jsonl_path, args.unsafe_schema, jsonl_path.stem, None
|
|
3700
|
+
)
|
|
2769
3701
|
return None, turns, jsonl_path.stem, jsonl_path
|
|
2770
3702
|
return _read_discovered(args)
|
|
2771
3703
|
except SystemExit as e:
|
|
@@ -2776,7 +3708,9 @@ def _stdin_ids(turns):
|
|
|
2776
3708
|
"""stdin → derive per-turn uuid from the content so the caches work (keyed: see _stdin_id_key)."""
|
|
2777
3709
|
key = _stdin_id_key()
|
|
2778
3710
|
for t in turns:
|
|
2779
|
-
t["uuid"] = hmac.new(key, str(t["text"]).encode(), hashlib.sha256).hexdigest()[
|
|
3711
|
+
t["uuid"] = hmac.new(key, str(t["text"]).encode(), hashlib.sha256).hexdigest()[
|
|
3712
|
+
:16
|
|
3713
|
+
]
|
|
2780
3714
|
|
|
2781
3715
|
|
|
2782
3716
|
def _run_emit(turns, session_id, jsonl_path, args):
|
|
@@ -2784,8 +3718,14 @@ def _run_emit(turns, session_id, jsonl_path, args):
|
|
|
2784
3718
|
window = review_window(turns, drops, args)
|
|
2785
3719
|
if window is None:
|
|
2786
3720
|
return 2
|
|
2787
|
-
return emit_review(
|
|
2788
|
-
|
|
3721
|
+
return emit_review(
|
|
3722
|
+
turns,
|
|
3723
|
+
drops,
|
|
3724
|
+
window,
|
|
3725
|
+
session_id,
|
|
3726
|
+
default_output(args, jsonl_path, session_id),
|
|
3727
|
+
args,
|
|
3728
|
+
)
|
|
2789
3729
|
|
|
2790
3730
|
|
|
2791
3731
|
def _rulebase_turns(turns, args):
|
|
@@ -2794,8 +3734,9 @@ def _rulebase_turns(turns, args):
|
|
|
2794
3734
|
if args.drop_trivia:
|
|
2795
3735
|
turns = drop_echo_exchanges(turns)
|
|
2796
3736
|
if args.hide_tool_only:
|
|
2797
|
-
turns = [
|
|
2798
|
-
|
|
3737
|
+
turns = [
|
|
3738
|
+
t for t in turns if not (t["role"] == "assistant" and not t["text"].strip())
|
|
3739
|
+
]
|
|
2799
3740
|
return select_range(turns, args)
|
|
2800
3741
|
|
|
2801
3742
|
|
|
@@ -2824,9 +3765,15 @@ def main(argv=None) -> int:
|
|
|
2824
3765
|
return 2
|
|
2825
3766
|
if args.output is None:
|
|
2826
3767
|
args.output = default_output(args, jsonl_path, session_id)
|
|
2827
|
-
return render_and_write(
|
|
2828
|
-
|
|
2829
|
-
|
|
3768
|
+
return render_and_write(
|
|
3769
|
+
turns,
|
|
3770
|
+
session_id,
|
|
3771
|
+
args,
|
|
3772
|
+
all_sessions=args.all_sessions,
|
|
3773
|
+
markdown=args.markdown,
|
|
3774
|
+
open_details=args.open_details,
|
|
3775
|
+
title=args.title,
|
|
3776
|
+
)
|
|
2830
3777
|
|
|
2831
3778
|
|
|
2832
3779
|
def _in_time(t, args):
|
|
@@ -2863,8 +3810,10 @@ def select_range(turns, args):
|
|
|
2863
3810
|
return None
|
|
2864
3811
|
turns = turns[sl]
|
|
2865
3812
|
if len(turns) > 100:
|
|
2866
|
-
print(
|
|
2867
|
-
|
|
3813
|
+
print(
|
|
3814
|
+
f"[lineage] WARN: {len(turns)} turns is large — render may be slow",
|
|
3815
|
+
file=sys.stderr,
|
|
3816
|
+
)
|
|
2868
3817
|
return turns
|
|
2869
3818
|
|
|
2870
3819
|
|
|
@@ -2875,11 +3824,13 @@ def default_output(args, jsonl_path, session_id):
|
|
|
2875
3824
|
return args.output
|
|
2876
3825
|
session_name = discover_session_name(jsonl_path) if jsonl_path else None
|
|
2877
3826
|
if session_name and page_redact(session_name)[0] != session_name:
|
|
2878
|
-
print(
|
|
2879
|
-
|
|
3827
|
+
print(
|
|
3828
|
+
f"[lineage] session name: {page_redact(session_name)[0]} "
|
|
3829
|
+
"(holds a secret-like string; the file is named by the session id)",
|
|
3830
|
+
file=sys.stderr,
|
|
3831
|
+
)
|
|
2880
3832
|
session_name = None
|
|
2881
|
-
slug =
|
|
2882
|
-
else (session_name or session_id[:8]))
|
|
3833
|
+
slug = "all-sessions" if args.all_sessions else (session_name or session_id[:8])
|
|
2883
3834
|
if session_name:
|
|
2884
3835
|
print(f"[lineage] session name: {session_name}", file=sys.stderr)
|
|
2885
3836
|
return f"work/lineage-{slug}.html"
|
|
@@ -2888,31 +3839,50 @@ def default_output(args, jsonl_path, session_id):
|
|
|
2888
3839
|
# The controls that embed, override or isolate text direction (U+202A..U+202E, U+2066..U+2069). In a page they
|
|
2889
3840
|
# reorder what a reader sees and can reach past the element that holds them, so the page shows each as a visible
|
|
2890
3841
|
# [U+XXXX] instead. Plain letters of right-to-left scripts, and the direction marks, stay.
|
|
2891
|
-
_BIDI_CONTROLS = re.compile(
|
|
3842
|
+
_BIDI_CONTROLS = re.compile(
|
|
3843
|
+
"[" + "".join(map(chr, (*range(0x202A, 0x202F), *range(0x2066, 0x206A)))) + "]"
|
|
3844
|
+
)
|
|
2892
3845
|
|
|
2893
3846
|
|
|
2894
3847
|
def _show_bidi_controls(text):
|
|
2895
3848
|
return _BIDI_CONTROLS.sub(lambda m: "[U+%04X]" % ord(m.group()), text)
|
|
2896
3849
|
|
|
2897
3850
|
|
|
2898
|
-
def render_and_write(
|
|
2899
|
-
|
|
2900
|
-
|
|
3851
|
+
def render_and_write(
|
|
3852
|
+
turns,
|
|
3853
|
+
session_id,
|
|
3854
|
+
args,
|
|
3855
|
+
all_sessions=False,
|
|
3856
|
+
markdown=True,
|
|
3857
|
+
open_details=False,
|
|
3858
|
+
title="Session Lineage",
|
|
3859
|
+
summaries=None,
|
|
3860
|
+
prior_redactions=None,
|
|
3861
|
+
):
|
|
2901
3862
|
"""Render the rows, fill the template, self-verify, write, run the reviewer gate.
|
|
2902
3863
|
`prior_redactions`: what --emit-review redacted before the text reached the pack."""
|
|
2903
3864
|
rows, redact_counts, hits, total = render_rows(
|
|
2904
|
-
turns,
|
|
2905
|
-
|
|
2906
|
-
|
|
3865
|
+
turns,
|
|
3866
|
+
session_id,
|
|
3867
|
+
args.redact_extra,
|
|
3868
|
+
args.redact_mode,
|
|
3869
|
+
args.rebuild_summaries,
|
|
3870
|
+
open_details=open_details,
|
|
3871
|
+
markdown=markdown,
|
|
3872
|
+
all_sessions=all_sessions,
|
|
3873
|
+
summaries=summaries,
|
|
3874
|
+
)
|
|
2907
3875
|
for k, v in (prior_redactions or {}).items():
|
|
2908
3876
|
redact_counts[k] = redact_counts.get(k, 0) + v
|
|
2909
3877
|
left = _residual_secrets("\n".join(rows))
|
|
2910
3878
|
if left:
|
|
2911
|
-
print(
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
3879
|
+
print(
|
|
3880
|
+
"[lineage] WARN: the page still holds secret-like text ("
|
|
3881
|
+
+ ", ".join(f"{k}={v}" for k, v in sorted(left.items()))
|
|
3882
|
+
+ "); check the page before sharing it, or run again with the value in LINEAGE_REDACT_EXTRA"
|
|
3883
|
+
" (with --rulebase, add --rebuild-summaries)",
|
|
3884
|
+
file=sys.stderr,
|
|
3885
|
+
)
|
|
2916
3886
|
|
|
2917
3887
|
date_range = ""
|
|
2918
3888
|
dated = [t for t in turns if t.get("ts")]
|
|
@@ -2923,8 +3893,12 @@ def render_and_write(turns, session_id, args, all_sessions=False, markdown=True,
|
|
|
2923
3893
|
|
|
2924
3894
|
html_doc = HTML_TEMPLATE
|
|
2925
3895
|
html_doc = html_doc.replace("{{TITLE}}", html.escape(_show_bidi_controls(title)), 1)
|
|
2926
|
-
html_doc = html_doc.replace(
|
|
2927
|
-
|
|
3896
|
+
html_doc = html_doc.replace(
|
|
3897
|
+
"{{HEADER_TITLE}}", html.escape(_show_bidi_controls(title)), 1
|
|
3898
|
+
)
|
|
3899
|
+
html_doc = html_doc.replace(
|
|
3900
|
+
"{{DATE_RANGE}}", html.escape(_show_bidi_controls(date_range)), 1
|
|
3901
|
+
)
|
|
2928
3902
|
html_doc = html_doc.replace("{{TURNS}}", _show_bidi_controls("\n".join(rows)), 1)
|
|
2929
3903
|
|
|
2930
3904
|
errs = self_verify(html_doc)
|
|
@@ -2938,9 +3912,12 @@ def render_and_write(turns, session_id, args, all_sessions=False, markdown=True,
|
|
|
2938
3912
|
_run_reviewer_gate(turns, out, session_id, args, summaries=summaries)
|
|
2939
3913
|
|
|
2940
3914
|
tool_total = sum(sum(t.get("tools", {}).values()) for t in turns)
|
|
2941
|
-
print(
|
|
2942
|
-
|
|
2943
|
-
|
|
3915
|
+
print(
|
|
3916
|
+
f"[lineage] turns={len(turns)} redacted={sum(redact_counts.values())} "
|
|
3917
|
+
f"tool_calls={tool_total} cache_hits={hits}/{total} "
|
|
3918
|
+
f"size={out.stat().st_size}B output={out}",
|
|
3919
|
+
file=sys.stderr,
|
|
3920
|
+
)
|
|
2944
3921
|
if redact_counts:
|
|
2945
3922
|
details = ", ".join(f"{k}={v}" for k, v in sorted(redact_counts.items()))
|
|
2946
3923
|
print(f"[lineage] redaction breakdown: {details}", file=sys.stderr)
|
|
@@ -2960,6 +3937,7 @@ def _with_pack_stamp(output, pack_output):
|
|
|
2960
3937
|
def _with_timestamp_suffix(path_str):
|
|
2961
3938
|
"""Append `_YYMMDD+HHMM` before the extension (idempotent)."""
|
|
2962
3939
|
import datetime as _dt
|
|
3940
|
+
|
|
2963
3941
|
p = pathlib.Path(path_str)
|
|
2964
3942
|
if re.search(r"_\d{6}\+\d{4}$", p.stem):
|
|
2965
3943
|
return p
|
|
@@ -2974,11 +3952,13 @@ def _gate_samples(bots, summaries, session_id, args):
|
|
|
2974
3952
|
seeded by the turns, so a rerun on the same turns samples the same ones, and a sample
|
|
2975
3953
|
names its turn and shows the head and tail the reviewer saw, fully redacted."""
|
|
2976
3954
|
import random
|
|
3955
|
+
|
|
2977
3956
|
if summaries is None:
|
|
2978
3957
|
out = []
|
|
2979
3958
|
for i, t in enumerate(random.sample(bots, min(5, len(bots)))):
|
|
2980
|
-
s, _ = _page_summary(
|
|
2981
|
-
|
|
3959
|
+
s, _ = _page_summary(
|
|
3960
|
+
t, summaries, session_id, False, args.redact_extra, args.redact_mode
|
|
3961
|
+
)
|
|
2982
3962
|
red_s = review_redact(_MASKED.sub("[REDACTED]", s), args.redact_extra)[0]
|
|
2983
3963
|
red_d = review_redact(t["text"], args.redact_extra)[0][:500]
|
|
2984
3964
|
out.append({"idx": i, "original_detail": red_d, "generated_summary": red_s})
|
|
@@ -2986,13 +3966,29 @@ def _gate_samples(bots, summaries, session_id, args):
|
|
|
2986
3966
|
seed = hashlib.sha256("\x00".join(t["uuid"] for t in bots).encode()).hexdigest()
|
|
2987
3967
|
out = []
|
|
2988
3968
|
for i, t in enumerate(random.Random(seed).sample(bots, min(5, len(bots)))):
|
|
2989
|
-
s =
|
|
2990
|
-
|
|
2991
|
-
|
|
3969
|
+
s = (
|
|
3970
|
+
t.get("sample_summary")
|
|
3971
|
+
or _page_summary(
|
|
3972
|
+
t, summaries, session_id, False, args.redact_extra, args.redact_mode
|
|
3973
|
+
)[0]
|
|
3974
|
+
)
|
|
3975
|
+
detail = (
|
|
3976
|
+
t.get("preview")
|
|
3977
|
+
or _preview(review_redact(t["text"], args.redact_extra)[0])[0]
|
|
3978
|
+
)
|
|
2992
3979
|
gen = review_redact(_MASKED.sub("[REDACTED]", s), args.redact_extra)[0]
|
|
2993
|
-
key = hashlib.sha256(
|
|
2994
|
-
|
|
2995
|
-
|
|
3980
|
+
key = hashlib.sha256(
|
|
3981
|
+
"\x00".join([t["uuid"], detail, gen]).encode()
|
|
3982
|
+
).hexdigest()[:12]
|
|
3983
|
+
out.append(
|
|
3984
|
+
{
|
|
3985
|
+
"idx": i,
|
|
3986
|
+
"id": t["uuid"],
|
|
3987
|
+
"key": key,
|
|
3988
|
+
"original_detail": detail,
|
|
3989
|
+
"generated_summary": gen,
|
|
3990
|
+
}
|
|
3991
|
+
)
|
|
2996
3992
|
return out
|
|
2997
3993
|
|
|
2998
3994
|
|
|
@@ -3015,8 +4011,10 @@ def _renew_samples(path, text, rop):
|
|
|
3015
4011
|
_write_private(path, text)
|
|
3016
4012
|
old = pathlib.Path(rop) if rop else None
|
|
3017
4013
|
if old is not None and _is_verdict_file(old):
|
|
3018
|
-
print(
|
|
3019
|
-
|
|
4014
|
+
print(
|
|
4015
|
+
f"[lineage] WARN: {rop} predates these samples; it is not read",
|
|
4016
|
+
file=sys.stderr,
|
|
4017
|
+
)
|
|
3020
4018
|
_set_aside(old)
|
|
3021
4019
|
|
|
3022
4020
|
|
|
@@ -3027,21 +4025,30 @@ def _refuse_non_verdict(rop):
|
|
|
3027
4025
|
p = pathlib.Path(rop) if rop else None
|
|
3028
4026
|
if p is None or not p.exists() or _is_verdict_file(p):
|
|
3029
4027
|
return
|
|
3030
|
-
if _json_list(p):
|
|
3031
|
-
why = (
|
|
3032
|
-
|
|
4028
|
+
if _json_list(p): # step 8: written again, the same answer fails the same way
|
|
4029
|
+
why = (
|
|
4030
|
+
"it is a JSON array where no entry names idx; if the session wrote the critic's "
|
|
4031
|
+
"answer, have the critic judge the samples again (a failed gate run)"
|
|
4032
|
+
)
|
|
3033
4033
|
else:
|
|
3034
|
-
why = (
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
4034
|
+
why = (
|
|
4035
|
+
"if the session wrote it, write it again as a bare JSON array of "
|
|
4036
|
+
"{idx, id, key, recoverable, reason}"
|
|
4037
|
+
)
|
|
4038
|
+
print(
|
|
4039
|
+
f"[lineage] ERROR: {rop} is not a verdict list; {why}, else give a --reviewer-output "
|
|
4040
|
+
"that does not exist yet",
|
|
4041
|
+
file=sys.stderr,
|
|
4042
|
+
)
|
|
3038
4043
|
raise SystemExit(2)
|
|
3039
4044
|
|
|
3040
4045
|
|
|
3041
4046
|
def _json_list(p):
|
|
3042
4047
|
"""True when `p` is a file holding a JSON array, whatever its entries."""
|
|
3043
4048
|
try:
|
|
3044
|
-
return p.is_file() and isinstance(
|
|
4049
|
+
return p.is_file() and isinstance(
|
|
4050
|
+
json.loads(p.read_text(encoding="utf-8")), list
|
|
4051
|
+
)
|
|
3045
4052
|
except (OSError, ValueError, UnicodeDecodeError, RecursionError):
|
|
3046
4053
|
return False
|
|
3047
4054
|
|
|
@@ -3074,8 +4081,9 @@ def _write_gate_samples(bots, out, summaries, session_id, args):
|
|
|
3074
4081
|
if summaries is not None and getattr(args, "apply_review", None):
|
|
3075
4082
|
pack = pathlib.Path(args.apply_review)
|
|
3076
4083
|
path = pack.with_name(f"{pack.stem}.reviewer-input.json")
|
|
3077
|
-
text = json.dumps(
|
|
3078
|
-
|
|
4084
|
+
text = json.dumps(
|
|
4085
|
+
_gate_samples(bots, summaries, session_id, args), ensure_ascii=False, indent=2
|
|
4086
|
+
)
|
|
3079
4087
|
try:
|
|
3080
4088
|
if summaries is None:
|
|
3081
4089
|
_write_private(path, text)
|
|
@@ -3083,20 +4091,35 @@ def _write_gate_samples(bots, out, summaries, session_id, args):
|
|
|
3083
4091
|
_renew_samples(path, text, args.reviewer_output)
|
|
3084
4092
|
except OSError as e:
|
|
3085
4093
|
if summaries is not None:
|
|
3086
|
-
print(
|
|
4094
|
+
print(
|
|
4095
|
+
f"[lineage] ERROR: cannot write the gate samples {path}: {e}",
|
|
4096
|
+
file=sys.stderr,
|
|
4097
|
+
)
|
|
3087
4098
|
raise SystemExit(2)
|
|
3088
4099
|
return None
|
|
3089
4100
|
print(f"[lineage] reviewer samples: {path}", file=sys.stderr, flush=True)
|
|
3090
4101
|
if summaries is None:
|
|
3091
|
-
print(
|
|
3092
|
-
|
|
3093
|
-
|
|
4102
|
+
print(
|
|
4103
|
+
"[lineage] next: invoke Skill('oh-my-claudecode:critic') with "
|
|
4104
|
+
"the JSON above; expected [{idx, recoverable, reason}, ...] "
|
|
4105
|
+
"(PASS = 5/5 recoverable)",
|
|
4106
|
+
file=sys.stderr,
|
|
4107
|
+
flush=True,
|
|
4108
|
+
)
|
|
3094
4109
|
elif pathlib.Path(args.reviewer_output).exists():
|
|
3095
|
-
print(
|
|
4110
|
+
print(
|
|
4111
|
+
f"[lineage] reading the verdict at {args.reviewer_output}",
|
|
4112
|
+
file=sys.stderr,
|
|
4113
|
+
flush=True,
|
|
4114
|
+
)
|
|
3096
4115
|
else:
|
|
3097
|
-
print(
|
|
3098
|
-
|
|
3099
|
-
|
|
4116
|
+
print(
|
|
4117
|
+
f"[lineage] next: have a critic agent judge the JSON above and write its "
|
|
4118
|
+
f"[{{idx, id, key, recoverable, reason}}, ...] to {args.reviewer_output} "
|
|
4119
|
+
"(one per sample; PASS = all recoverable)",
|
|
4120
|
+
file=sys.stderr,
|
|
4121
|
+
flush=True,
|
|
4122
|
+
)
|
|
3100
4123
|
return path
|
|
3101
4124
|
|
|
3102
4125
|
|
|
@@ -3106,26 +4129,32 @@ def _read_verdict(rop, timeout, set_aside):
|
|
|
3106
4129
|
so a rerun waits for a fresh one instead of reusing it; anything else stays where it is.
|
|
3107
4130
|
Exits 2 when it is missing, unreadable or not a non-empty list."""
|
|
3108
4131
|
import time as _time
|
|
4132
|
+
|
|
3109
4133
|
deadline = _time.time() + (0 if timeout is None else max(1, timeout))
|
|
3110
4134
|
while not rop.exists() and _time.time() < deadline:
|
|
3111
4135
|
_time.sleep(1)
|
|
3112
4136
|
if not rop.exists():
|
|
3113
4137
|
if timeout is None:
|
|
3114
|
-
print(
|
|
3115
|
-
|
|
4138
|
+
print(
|
|
4139
|
+
f"[lineage] ERROR: no verdict at {rop} yet; have a critic judge the samples, "
|
|
4140
|
+
"write its answer there and run this again",
|
|
4141
|
+
file=sys.stderr,
|
|
4142
|
+
)
|
|
3116
4143
|
else:
|
|
3117
|
-
print(
|
|
3118
|
-
|
|
4144
|
+
print(
|
|
4145
|
+
f"[lineage] ERROR: reviewer-output not found within {timeout}s: {rop}",
|
|
4146
|
+
file=sys.stderr,
|
|
4147
|
+
)
|
|
3119
4148
|
raise SystemExit(2)
|
|
3120
4149
|
try:
|
|
3121
4150
|
verdict = json.loads(rop.read_text(encoding="utf-8"))
|
|
3122
4151
|
except (json.JSONDecodeError, OSError, UnicodeDecodeError, RecursionError) as e:
|
|
3123
|
-
print(f"[lineage] ERROR: reviewer-output parse failed: {e}",
|
|
3124
|
-
file=sys.stderr)
|
|
4152
|
+
print(f"[lineage] ERROR: reviewer-output parse failed: {e}", file=sys.stderr)
|
|
3125
4153
|
raise SystemExit(2)
|
|
3126
4154
|
if not isinstance(verdict, list) or not verdict:
|
|
3127
|
-
print(
|
|
3128
|
-
|
|
4155
|
+
print(
|
|
4156
|
+
"[lineage] ERROR: reviewer-output is not a non-empty array", file=sys.stderr
|
|
4157
|
+
)
|
|
3129
4158
|
raise SystemExit(2)
|
|
3130
4159
|
if set_aside:
|
|
3131
4160
|
_set_aside(rop)
|
|
@@ -3148,8 +4177,11 @@ def _set_aside(rop):
|
|
|
3148
4177
|
try:
|
|
3149
4178
|
os.replace(str(rop), str(dest))
|
|
3150
4179
|
except OSError as e:
|
|
3151
|
-
print(
|
|
3152
|
-
|
|
4180
|
+
print(
|
|
4181
|
+
f"[lineage] WARN: could not set the verdict aside ({e}); remove {rop} "
|
|
4182
|
+
"before the next run",
|
|
4183
|
+
file=sys.stderr,
|
|
4184
|
+
)
|
|
3153
4185
|
return
|
|
3154
4186
|
print(f"[lineage] note: verdict moved to {dest}", file=sys.stderr)
|
|
3155
4187
|
|
|
@@ -3158,12 +4190,20 @@ def _verdict_gaps(verdict, samples):
|
|
|
3158
4190
|
"""Why a verdict does not answer these samples, or "": it needs one entry per sample
|
|
3159
4191
|
idx and no other, each carrying the `id` and `key` of that sample (its turn and its
|
|
3160
4192
|
content: an answer to the samples from before a fix carries an old key)."""
|
|
3161
|
-
want = {
|
|
4193
|
+
want = {
|
|
4194
|
+
s["idx"]: s
|
|
4195
|
+
for s in samples
|
|
4196
|
+
if isinstance(s, dict) and type(s.get("idx")) is int
|
|
4197
|
+
}
|
|
3162
4198
|
seen, stray = set(), []
|
|
3163
4199
|
for v in verdict:
|
|
3164
4200
|
idx = v.get("idx") if isinstance(v, dict) else None
|
|
3165
|
-
if (
|
|
3166
|
-
|
|
4201
|
+
if (
|
|
4202
|
+
type(idx) is not int
|
|
4203
|
+
or idx not in want
|
|
4204
|
+
or idx in seen
|
|
4205
|
+
or any(v.get(k) != want[idx][k] for k in ("id", "key") if k in want[idx])
|
|
4206
|
+
):
|
|
3167
4207
|
stray.append(idx)
|
|
3168
4208
|
else:
|
|
3169
4209
|
seen.add(idx)
|
|
@@ -3177,15 +4217,27 @@ def _check_coverage(verdict, samples):
|
|
|
3177
4217
|
"""Exit 2 unless the verdict answers each of the samples in file `samples`."""
|
|
3178
4218
|
try:
|
|
3179
4219
|
gaps = _verdict_gaps(verdict, json.loads(samples.read_text(encoding="utf-8")))
|
|
3180
|
-
except (
|
|
4220
|
+
except (
|
|
4221
|
+
OSError,
|
|
4222
|
+
ValueError,
|
|
4223
|
+
UnicodeDecodeError,
|
|
4224
|
+
AttributeError,
|
|
4225
|
+
RecursionError,
|
|
4226
|
+
) as e:
|
|
3181
4227
|
gaps = f"cannot read the samples: {e}"
|
|
3182
4228
|
if gaps:
|
|
3183
|
-
print(
|
|
3184
|
-
|
|
4229
|
+
print(
|
|
4230
|
+
f"[lineage] ERROR: the verdict does not answer {samples} ({gaps}); have the "
|
|
4231
|
+
"critic judge every sample again",
|
|
4232
|
+
file=sys.stderr,
|
|
4233
|
+
)
|
|
3185
4234
|
for v in verdict:
|
|
3186
4235
|
if isinstance(v, dict) and v.get("recoverable") is not True:
|
|
3187
|
-
print(
|
|
3188
|
-
|
|
4236
|
+
print(
|
|
4237
|
+
f" - not recoverable in it: idx={v.get('idx', '?')}: "
|
|
4238
|
+
f"{v.get('reason', '(no reason)')}",
|
|
4239
|
+
file=sys.stderr,
|
|
4240
|
+
)
|
|
3189
4241
|
raise SystemExit(2)
|
|
3190
4242
|
|
|
3191
4243
|
|
|
@@ -3196,18 +4248,26 @@ def _enforce_gate(rop, timeout, samples=None, reviewed=False):
|
|
|
3196
4248
|
verdict = _read_verdict(rop, timeout, set_aside=reviewed)
|
|
3197
4249
|
if reviewed and samples is not None:
|
|
3198
4250
|
_check_coverage(verdict, samples)
|
|
3199
|
-
fails = [
|
|
3200
|
-
|
|
4251
|
+
fails = [
|
|
4252
|
+
v for v in verdict if not (isinstance(v, dict) and v.get("recoverable") is True)
|
|
4253
|
+
]
|
|
3201
4254
|
if fails:
|
|
3202
|
-
print(
|
|
3203
|
-
|
|
4255
|
+
print(
|
|
4256
|
+
f"[lineage] FAIL: quality gate {len(verdict) - len(fails)}/"
|
|
4257
|
+
f"{len(verdict)} recoverable. Reasons:",
|
|
4258
|
+
file=sys.stderr,
|
|
4259
|
+
)
|
|
3204
4260
|
for f in fails:
|
|
3205
4261
|
if isinstance(f, dict):
|
|
3206
|
-
print(
|
|
3207
|
-
|
|
4262
|
+
print(
|
|
4263
|
+
f" - idx={f.get('idx', '?')}: {f.get('reason', '(no reason)')}",
|
|
4264
|
+
file=sys.stderr,
|
|
4265
|
+
)
|
|
3208
4266
|
raise SystemExit(2)
|
|
3209
|
-
print(
|
|
3210
|
-
|
|
4267
|
+
print(
|
|
4268
|
+
f"[lineage] PASS: quality gate {len(verdict)}/{len(verdict)} recoverable",
|
|
4269
|
+
file=sys.stderr,
|
|
4270
|
+
)
|
|
3211
4271
|
if reviewed and samples is not None:
|
|
3212
4272
|
try:
|
|
3213
4273
|
samples.unlink()
|
|
@@ -3223,17 +4283,23 @@ def _run_reviewer_gate(turns, out, session_id, args, summaries=None):
|
|
|
3223
4283
|
bots = [t for t in turns if t["role"] in ("assistant", "agent")]
|
|
3224
4284
|
reviewed = summaries is not None
|
|
3225
4285
|
if args.skip_reviewer:
|
|
3226
|
-
print(
|
|
3227
|
-
|
|
4286
|
+
print(
|
|
4287
|
+
"[lineage] WARN: --skip-reviewer — quality gate not enforced",
|
|
4288
|
+
file=sys.stderr,
|
|
4289
|
+
)
|
|
3228
4290
|
return
|
|
3229
4291
|
if not bots or (reviewed and not args.reviewer_output):
|
|
3230
4292
|
return
|
|
3231
4293
|
samples = _write_gate_samples(bots, out, summaries, session_id, args)
|
|
3232
4294
|
if args.reviewer_output:
|
|
3233
4295
|
if reviewed and args.reviewer_timeout is None:
|
|
3234
|
-
timeout = None
|
|
4296
|
+
timeout = None # the run after the critic reads the verdict; this one does not wait
|
|
3235
4297
|
else:
|
|
3236
|
-
timeout =
|
|
4298
|
+
timeout = (
|
|
4299
|
+
REVIEWER_TIMEOUT
|
|
4300
|
+
if args.reviewer_timeout is None
|
|
4301
|
+
else args.reviewer_timeout
|
|
4302
|
+
)
|
|
3237
4303
|
_enforce_gate(pathlib.Path(args.reviewer_output), timeout, samples, reviewed)
|
|
3238
4304
|
|
|
3239
4305
|
|