@kaydash9999/banker-plugins 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +69 -0
- package/PRIVACY.md +8 -3
- package/README.md +78 -15
- package/bin/banker.js +154 -32
- package/bin/lib/codex-hooks.mjs +123 -20
- package/bin/lib/codex-names.mjs +4 -0
- package/bin/lib/codex-routes.mjs +204 -0
- package/bin/lib/command-skill.cjs +50 -0
- package/bin/lib/telemetry-config.mjs +2 -1
- package/codex/command-skills/front-qa/SKILL.md +56 -0
- package/codex/command-skills/setup/SKILL.md +87 -0
- package/codex/hooks.json +29 -0
- package/codex/manifest.json +18 -15
- package/codex/skills/graceful-pause/SKILL.md +2 -2
- package/codex/skills/progress/scripts/progress.mjs +44 -14
- package/codex/skills/teamwork/SKILL.md +137 -0
- package/codex/skills/teamwork/scripts/teamwork.mjs +957 -0
- package/codex/transform-matrix.md +5 -3
- package/commands/setup.md +2 -2
- package/hooks/obsidize-hook.mjs +75 -32
- package/hooks/telemetry-count-codex.mjs +303 -35
- package/hooks/update-fetch.mjs +2 -1
- package/hooks/update-notify.mjs +146 -13
- package/package.json +3 -1
- package/skills/3d-intro-build/SKILL.md +101 -20
- package/skills/3d-intro-build/references/NOTICE.md +9 -2
- package/skills/3d-intro-build/references/assemble.mjs +113 -22
- package/skills/3d-intro-build/references/azure-adapter.mjs +110 -24
- package/skills/3d-intro-build/references/clip-post.mjs +443 -0
- package/skills/3d-intro-build/references/dive-mode.mjs +299 -0
- package/skills/3d-intro-build/references/intro-fixes.css +11 -0
- package/skills/3d-intro-build/references/prompts.md +51 -4
- package/skills/3d-intro-build/references/scrub-engine.js +1 -0
- package/skills/3d-intro-build/references/step-nav.css +29 -0
- package/skills/3d-intro-build/references/step-nav.js +277 -106
- package/skills/3d-intro-build/references/verify-intro.mjs +387 -32
- package/skills/3d-intro-build/references/verify.md +6 -0
- package/skills/3d-intro-build/references/video-pool.mjs +142 -12
- package/skills/3d-intro-setup/SKILL.md +20 -7
- package/skills/3d-intro-setup/references/azure-adapter.mjs +110 -24
- package/skills/3d-intro-setup/references/video-pool.mjs +142 -12
- package/skills/all-in-one/SKILL.md +125 -86
- package/skills/anti-drm/SKILL.md +1 -1
- package/skills/anti-drm/scripts/instruction-toggle.mjs +57 -4
- package/skills/arch-diagram/build_pptx_template.py +47 -28
- package/skills/audit-web-page/SKILL.md +1 -1
- package/skills/audit-web-page/harness.py +0 -1
- package/skills/lineage/lineage.py +1435 -533
- package/skills/motion-graphic-make/SKILL.md +1 -1
- package/skills/obsidizer/SKILL.md +20 -5
- package/skills/omc-patch/SKILL.md +1 -1
- package/skills/payload-mon/SKILL.md +2 -2
- package/skills/play-qa/SKILL.md +7 -5
- package/skills/progress-mon/README.md +162 -0
- package/skills/progress-mon/SKILL.md +165 -0
- package/skills/progress-mon/assets/fixture/README.md +1 -0
- package/skills/progress-mon/assets/fixture/arch.json +33 -0
- package/skills/progress-mon/assets/fixture/commits.json +7 -0
- package/skills/progress-mon/assets/fixture/config.json +30 -0
- package/skills/progress-mon/assets/fixture/events.jsonl +7 -0
- package/skills/progress-mon/assets/fixture/flow.json +44 -0
- package/skills/progress-mon/assets/fixture/items.json +14 -0
- package/skills/progress-mon/assets/fixture/req-spec.json +31 -0
- package/skills/progress-mon/assets/fixture/reqs.json +11 -0
- package/skills/progress-mon/assets/fixture/variants/arch-auto/arch.json +58 -0
- package/skills/progress-mon/assets/fixture/variants/check-auto/arch.json +15 -0
- package/skills/progress-mon/assets/fixture/variants/check-overlap/arch.json +14 -0
- package/skills/progress-mon/assets/fixture/variants/history-manual/config.json +13 -0
- package/skills/progress-mon/assets/fixture/variants/history-manual/items.json +7 -0
- package/skills/progress-mon/assets/fixture/variants/workflow-wide/flow.json +35 -0
- package/skills/progress-mon/assets/fixture/workflows/wf_ex1/agent-a01.jsonl +1 -0
- package/skills/progress-mon/assets/fixture/workflows/wf_ex1/agent-a02.jsonl +1 -0
- package/skills/progress-mon/assets/fixture/workflows/wf_ex1/agent-a03.jsonl +1 -0
- package/skills/progress-mon/assets/fixture/workflows/wf_ex1/agent-a04.jsonl +1 -0
- package/skills/progress-mon/assets/fixture/workflows/wf_ex1/journal.jsonl +7 -0
- package/skills/progress-mon/references/arch.md +280 -0
- package/skills/progress-mon/references/authoring.md +199 -0
- package/skills/progress-mon/references/config.md +115 -0
- package/skills/progress-mon/references/flow.md +75 -0
- package/skills/progress-mon/references/inventory.md +58 -0
- package/skills/progress-mon/references/items.md +74 -0
- package/skills/progress-mon/references/req-spec.md +74 -0
- package/skills/progress-mon/references/reqs.md +30 -0
- package/skills/progress-mon/scripts/index.js +31 -0
- package/skills/progress-mon/scripts/pm-check.mjs +526 -0
- package/skills/progress-mon/scripts/pm-config.mjs +341 -0
- package/skills/progress-mon/scripts/pm-data.mjs +394 -0
- package/skills/progress-mon/scripts/pm-html.mjs +77 -0
- package/skills/progress-mon/scripts/pm-memory.mjs +308 -0
- package/skills/progress-mon/scripts/pm-render.mjs +1644 -0
- package/skills/progress-mon/scripts/pm-server.mjs +735 -0
- package/skills/progress-mon/scripts/pm-sources.mjs +501 -0
- package/skills/progress-mon/scripts/progress-mon.mjs +167 -0
- package/skills/ralph-qa/SKILL.md +105 -16
- package/skills/ralph-qa/references/gate-evidence.mjs +457 -0
- package/skills/ralph-qa/references/payload-scan.mjs +19 -2
- package/skills/ralph-qa/references/run-folder.mjs +299 -0
- package/skills/ralph-qa/references/seat-payload.mjs +60 -13
- package/skills/ralph-qa/references/seat-run.mjs +159 -155
- package/skills/remains/SKILL.md +2 -1
- package/skills/remains/scripts/boxes.mjs +6 -2
- package/skills/rfp-author/scripts/md2docx.py +63 -34
- package/skills/setup-bypass-permissions/SKILL.md +2 -2
- package/skills/setup-bypass-permissions/scripts/codex-config.mjs +19 -8
- package/skills/setup-bypass-permissions/scripts/fallback/bypass-permissions.py +66 -15
- package/skills/setup-mcp/SKILL.md +1 -1
- package/skills/setup-omc-hud/SKILL.md +2 -1
- package/skills/setup-omc-hud/assets/codex-config.mjs +19 -8
- package/skills/smart-compact/SKILL.md +1 -1
- package/skills/smart-compact/scripts/codex-gate.mjs +2 -1
- package/skills/theme-selection/SKILL.md +1 -1
- package/skills/tone-compact/SKILL.md +1 -1
- package/skills/tone-compact/scripts/instruction-toggle.mjs +57 -4
- package/skills/ultra-chain/SKILL.md +34 -16
- package/skills/ultra-chain/references/chain-ledger.mjs +111 -3
- package/skills/ultra-init/SKILL.md +30 -22
- package/skills/ultra-ui-qa/SKILL.md +1 -1
- package/skills/ultraqa/NOTICE.md +34 -0
- package/skills/ultraqa/SKILL.md +196 -0
- package/skills/ultraqa/references/gate-evidence.mjs +457 -0
- package/skills/ultraqa/references/run-folder.mjs +299 -0
- package/skills/ultraqa/references/tmux-drive.md +52 -0
- package/skills/ultraqa/references/uqa-run.mjs +339 -0
- package/skills/update-banker/SKILL.md +32 -7
- package/skills/vertical-pptx/SKILL.md +1 -1
- package/skills/vertical-pptx/build_a4p.js +20 -2
- package/skills/vertical-pptx/build_a4p.py +560 -359
|
@@ -9,6 +9,7 @@ self-verifies the output markup. See SKILL.md for the full design and the
|
|
|
9
9
|
Runtime-agnostic: works from any transcript via --session / --from-transcript.
|
|
10
10
|
Auto-discovery assumes Claude Code's ~/.claude/projects/<encoded-cwd>/ layout.
|
|
11
11
|
"""
|
|
12
|
+
|
|
12
13
|
import argparse
|
|
13
14
|
import bisect
|
|
14
15
|
import hashlib
|
|
@@ -29,43 +30,59 @@ from html.parser import HTMLParser
|
|
|
29
30
|
from typing import Any
|
|
30
31
|
|
|
31
32
|
if sys.version_info < (3, 7):
|
|
32
|
-
sys.stderr.write(
|
|
33
|
-
|
|
33
|
+
sys.stderr.write(
|
|
34
|
+
"lineage requires Python 3.7+ (found %s)\n" % sys.version.split()[0]
|
|
35
|
+
)
|
|
34
36
|
sys.exit(2)
|
|
35
37
|
|
|
36
|
-
SCHEMA_VERSION = 2
|
|
37
|
-
SUMMARIZER_VERSION =
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
SCHEMA_VERSION = 2 # cache dir schema (v2: redacted summaries only)
|
|
39
|
+
SUMMARIZER_VERSION = (
|
|
40
|
+
4 # bump when summarize_turn or the page redaction changes (B-3 cache key)
|
|
41
|
+
)
|
|
42
|
+
# 3: cut from redacted text (3.0.2); 4: split secrets joined first (3.0.8)
|
|
43
|
+
USER_FOLD = 400 # collapse user messages longer than this (E-1)
|
|
44
|
+
ECHO_ASK = 40 # echo-exchange: user question length ceiling (A-4)
|
|
45
|
+
ECHO_REPLY = 5 # echo-exchange: assistant reply length ceiling (A-4)
|
|
42
46
|
_BLOCKQUOTE_MAX_DEPTH = 32 # blockquote recursion cap (C-2)
|
|
43
47
|
CACHE_BASE = pathlib.Path.home() / ".cache" / "lineage"
|
|
44
|
-
REVIEW_SCHEMA = "lineage-review/2"
|
|
48
|
+
REVIEW_SCHEMA = "lineage-review/2" # --emit-review pack, read back by --apply-review; 2: llm_key, pages redacted (3.0.2)
|
|
45
49
|
REVIEW_PART_SCHEMA = "lineage-review-part/1"
|
|
46
|
-
REVIEW_PART = 40
|
|
47
|
-
PREVIEW_HEAD = 1500
|
|
50
|
+
REVIEW_PART = 40 # turns per part file: one reviewer's share
|
|
51
|
+
PREVIEW_HEAD = 1500 # a long turn's preview: head + tail, where conclusions sit
|
|
48
52
|
PREVIEW_TAIL = 700
|
|
49
|
-
SUMMARY_MAX = 120
|
|
50
|
-
LLM_CACHE_VERSION = 1
|
|
51
|
-
REVIEWER_TIMEOUT = 60
|
|
52
|
-
DECISIONS_MAX =
|
|
53
|
-
|
|
53
|
+
SUMMARY_MAX = 120 # a reviewer's summary is cut to this, after redaction
|
|
54
|
+
LLM_CACHE_VERSION = 1 # bump to drop every cached reviewer decision
|
|
55
|
+
REVIEWER_TIMEOUT = 60 # --reviewer-timeout default, seconds
|
|
56
|
+
DECISIONS_MAX = (
|
|
57
|
+
1_000_000 # bytes in one decisions file (a 40-turn part answers in some 20 KB)
|
|
58
|
+
)
|
|
59
|
+
DECISIONS_TRIES = 200 # places a wrapped answer's list may start, tried at most
|
|
54
60
|
|
|
55
61
|
# ---------------------------------------------------------------- Noise / classify
|
|
56
62
|
# NOTE: JSONL records carry RAW `<...>` tags. Every regex below uses raw `<`/`>`
|
|
57
63
|
# (NOT the HTML-escaped `<` seen in the display-only change-request document).
|
|
58
|
-
_NOISE_TAGS = (
|
|
59
|
-
|
|
60
|
-
|
|
64
|
+
_NOISE_TAGS = (
|
|
65
|
+
"system-reminder",
|
|
66
|
+
"local-command-caveat",
|
|
67
|
+
"local-command-stdout",
|
|
68
|
+
"local-command-stderr",
|
|
69
|
+
"teammate-message",
|
|
70
|
+
"command-message",
|
|
71
|
+
"command-args",
|
|
72
|
+
"command-name",
|
|
73
|
+
"task-notification",
|
|
74
|
+
)
|
|
61
75
|
_NOISE_BLOCK_RE = re.compile(r"<(%s)\b[^>]*>.*?</\1>" % "|".join(_NOISE_TAGS), re.S)
|
|
62
76
|
_NOISE_CLOSE_RE = re.compile(r"</(%s)>" % "|".join(_NOISE_TAGS))
|
|
63
|
-
_NOISE_OPEN_RE = re.compile(
|
|
77
|
+
_NOISE_OPEN_RE = re.compile(
|
|
78
|
+
r"<(?:%s)\b" % "|".join(_NOISE_TAGS)
|
|
79
|
+
) # leading orphan open (R7)
|
|
64
80
|
|
|
65
81
|
_SKILL_BODY_RE = re.compile(r"^\s*Base directory for this skill:")
|
|
66
82
|
_WORKFLOW_BODY_RE = re.compile(r'^\s*Run the "[^"\n]+" workflow\.')
|
|
67
83
|
_COMPACTION_RE = re.compile(
|
|
68
|
-
r"^\s*This session is being continued from a previous conversation"
|
|
84
|
+
r"^\s*This session is being continued from a previous conversation"
|
|
85
|
+
)
|
|
69
86
|
_HOOK_FEEDBACK_RE = re.compile(r"^\s*Stop hook feedback:")
|
|
70
87
|
_INTERRUPT_RE = re.compile(r"^\s*\[Request interrupted")
|
|
71
88
|
_IMAGE_NOTE_RE = re.compile(r"^\s*\[Image:\s*original\s+\d+x\d+")
|
|
@@ -74,18 +91,48 @@ _SYS_ERROR_RE = re.compile(r"^\s*(?:Login expired|Please run /login|API Error)")
|
|
|
74
91
|
# Manipulation commands: judged by NAME, not by "has no args" (A-4). Plugin
|
|
75
92
|
# commands (name contains ':') are never manipulation — always preserved.
|
|
76
93
|
_HARNESS_CMDS = {
|
|
77
|
-
"agents",
|
|
78
|
-
"
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
94
|
+
"agents",
|
|
95
|
+
"bug",
|
|
96
|
+
"clear",
|
|
97
|
+
"compact",
|
|
98
|
+
"config",
|
|
99
|
+
"context",
|
|
100
|
+
"copy",
|
|
101
|
+
"cost",
|
|
102
|
+
"doctor",
|
|
103
|
+
"effort",
|
|
104
|
+
"exit",
|
|
105
|
+
"export",
|
|
106
|
+
"help",
|
|
107
|
+
"hooks",
|
|
108
|
+
"ide",
|
|
109
|
+
"install-github-app",
|
|
110
|
+
"login",
|
|
111
|
+
"logout",
|
|
112
|
+
"mcp",
|
|
113
|
+
"memory",
|
|
114
|
+
"migrate-installer",
|
|
115
|
+
"model",
|
|
116
|
+
"output-style",
|
|
117
|
+
"permissions",
|
|
118
|
+
"privacy-settings",
|
|
119
|
+
"quit",
|
|
120
|
+
"release-notes",
|
|
121
|
+
"resume",
|
|
122
|
+
"status",
|
|
123
|
+
"statusline",
|
|
124
|
+
"terminal-setup",
|
|
125
|
+
"todos",
|
|
126
|
+
"upgrade",
|
|
127
|
+
"usage",
|
|
128
|
+
"vim",
|
|
83
129
|
}
|
|
84
130
|
# Deliberately NOT in the set: /add-dir (changes readable dirs), /init (writes
|
|
85
131
|
# CLAUDE.md) — these do real work and are kept.
|
|
86
132
|
|
|
87
133
|
_AGENT_WRAP_RE = re.compile(
|
|
88
|
-
r"<(agent-message|teammate-message)\b([^>]*)>(.*?)</\1>", re.S
|
|
134
|
+
r"<(agent-message|teammate-message)\b([^>]*)>(.*?)</\1>", re.S
|
|
135
|
+
)
|
|
89
136
|
_AGENT_FROM_RE = re.compile(r'(?:from|teammate_id)="([^"]+)"')
|
|
90
137
|
_PEER_LEAD_RE = re.compile(r"^\s*Another Claude session sent a message:\s*")
|
|
91
138
|
_BARE_CMD_RE = re.compile(r"^/([\w-]+)\s*$")
|
|
@@ -105,7 +152,7 @@ def clean_user_text(text, drop_harness=True):
|
|
|
105
152
|
"""
|
|
106
153
|
out = _NOISE_CLOSE_RE.sub("", _NOISE_BLOCK_RE.sub("", text)).strip()
|
|
107
154
|
|
|
108
|
-
mb = _BARE_CMD_RE.match(out)
|
|
155
|
+
mb = _BARE_CMD_RE.match(out) # bare `/name` with no wrapper
|
|
109
156
|
if mb and drop_harness and mb.group(1) in _HARNESS_CMDS:
|
|
110
157
|
return ""
|
|
111
158
|
|
|
@@ -140,7 +187,7 @@ def split_agent_message(text):
|
|
|
140
187
|
return None, None
|
|
141
188
|
who = _AGENT_FROM_RE.search(m.group(2) or "")
|
|
142
189
|
body = m.group(3).strip()
|
|
143
|
-
if body.startswith('{"type"'):
|
|
190
|
+
if body.startswith('{"type"'): # idle/status notification
|
|
144
191
|
return (who.group(1) if who else "agent"), ""
|
|
145
192
|
return (who.group(1) if who else "agent"), body
|
|
146
193
|
|
|
@@ -152,10 +199,18 @@ SECRET_PATTERNS = [
|
|
|
152
199
|
("GitHubPAT", re.compile(r"gh[posru]_[A-Za-z0-9]{36}")),
|
|
153
200
|
("Slack", re.compile(r"xox[bpars]-[A-Za-z0-9-]{10,}")),
|
|
154
201
|
("JWT", re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+")),
|
|
155
|
-
(
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
202
|
+
(
|
|
203
|
+
"PrivateKey",
|
|
204
|
+
re.compile(
|
|
205
|
+
r"-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]+?-----END [A-Z ]+PRIVATE KEY-----"
|
|
206
|
+
),
|
|
207
|
+
),
|
|
208
|
+
(
|
|
209
|
+
"Password",
|
|
210
|
+
re.compile(
|
|
211
|
+
r"(?i)(?:password|암호|비번|패스워드)\s*[:=]\s*['\"]([^'\"\n]{4,})['\"]"
|
|
212
|
+
),
|
|
213
|
+
),
|
|
159
214
|
]
|
|
160
215
|
ENTROPY_LONG = re.compile(r"[A-Za-z0-9+/=]{32,}")
|
|
161
216
|
|
|
@@ -194,7 +249,7 @@ class _Trail:
|
|
|
194
249
|
if r.startswith("[REDACTED"):
|
|
195
250
|
out += new * len(r)
|
|
196
251
|
elif r:
|
|
197
|
-
out += pos[s:s + 4] + new * (len(r) - 8) + pos[e - 4:e]
|
|
252
|
+
out += pos[s : s + 4] + new * (len(r) - 8) + pos[e - 4 : e]
|
|
198
253
|
at = e
|
|
199
254
|
self.pos = out + pos[at:]
|
|
200
255
|
|
|
@@ -207,6 +262,7 @@ class _Trail:
|
|
|
207
262
|
if r != m.group(0):
|
|
208
263
|
edits.append((m.start(), m.end(), r))
|
|
209
264
|
return r
|
|
265
|
+
|
|
210
266
|
out = pat.sub(noted, text)
|
|
211
267
|
self.carry(edits)
|
|
212
268
|
return out
|
|
@@ -234,7 +290,9 @@ def _sub_for(trail):
|
|
|
234
290
|
def _shows_more(trail, shown):
|
|
235
291
|
"""Whether the text `trail` follows shows a character of the text it started as that `shown` (a _Trail.shown of
|
|
236
292
|
another text from the same start) has hidden."""
|
|
237
|
-
return (
|
|
293
|
+
return (
|
|
294
|
+
int.from_bytes(trail.shown(), "little") & ~int.from_bytes(shown, "little")
|
|
295
|
+
) != 0
|
|
238
296
|
|
|
239
297
|
|
|
240
298
|
# Characters that cut a value for the patterns while no person reads them as a separator: Unicode's default-ignorable
|
|
@@ -244,14 +302,46 @@ def _shows_more(trail, shown):
|
|
|
244
302
|
# (a lone surrogate became U+FFFD). A value cut by them is put back together where a pattern finds it once they
|
|
245
303
|
# are gone (3.0.8).
|
|
246
304
|
_SPLITTER_RANGES = (
|
|
247
|
-
(0x0000, 0x0008),
|
|
248
|
-
(
|
|
249
|
-
(
|
|
250
|
-
(
|
|
251
|
-
(
|
|
252
|
-
(
|
|
253
|
-
(
|
|
254
|
-
|
|
305
|
+
(0x0000, 0x0008),
|
|
306
|
+
(0x000E, 0x001B),
|
|
307
|
+
(0x007F, 0x0084),
|
|
308
|
+
(0x0086, 0x009F),
|
|
309
|
+
(0x00A0, 0x00A0),
|
|
310
|
+
(0x00AD, 0x00AD),
|
|
311
|
+
(0x034F, 0x034F),
|
|
312
|
+
(0x0600, 0x0605),
|
|
313
|
+
(0x061C, 0x061C),
|
|
314
|
+
(0x06DD, 0x06DD),
|
|
315
|
+
(0x070F, 0x070F),
|
|
316
|
+
(0x0890, 0x0891),
|
|
317
|
+
(0x08E2, 0x08E2),
|
|
318
|
+
(0x115F, 0x1160),
|
|
319
|
+
(0x1680, 0x1680),
|
|
320
|
+
(0x17B4, 0x17B5),
|
|
321
|
+
(0x180B, 0x180F),
|
|
322
|
+
(0x2000, 0x200F),
|
|
323
|
+
(0x202A, 0x202F),
|
|
324
|
+
(0x205F, 0x206F),
|
|
325
|
+
(0x2800, 0x2800),
|
|
326
|
+
(0x3000, 0x3000),
|
|
327
|
+
(0x3164, 0x3164),
|
|
328
|
+
(0xE000, 0xF8FF),
|
|
329
|
+
(0xFE00, 0xFE0F),
|
|
330
|
+
(0xFEFF, 0xFEFF),
|
|
331
|
+
(0xFFA0, 0xFFA0),
|
|
332
|
+
(0xFFF0, 0xFFFD),
|
|
333
|
+
(0x110BD, 0x110BD),
|
|
334
|
+
(0x110CD, 0x110CD),
|
|
335
|
+
(0x13430, 0x1343F),
|
|
336
|
+
(0x1BCA0, 0x1BCA3),
|
|
337
|
+
(0x1D173, 0x1D17A),
|
|
338
|
+
(0xE0000, 0xE0FFF),
|
|
339
|
+
(0xF0000, 0xFFFFD),
|
|
340
|
+
(0x100000, 0x10FFFD),
|
|
341
|
+
)
|
|
342
|
+
_SPLITTERS = "".join(
|
|
343
|
+
chr(a) if a == b else chr(a) + "-" + chr(b) for a, b in _SPLITTER_RANGES
|
|
344
|
+
)
|
|
255
345
|
_SPLITTER_RE = re.compile("[" + _SPLITTERS + "]")
|
|
256
346
|
_UNSPLIT_RE = re.compile("[^" + _SPLITTERS + "]+")
|
|
257
347
|
# A character a value can hold: past a splitter next to a found value, one of these means another value begins.
|
|
@@ -273,7 +363,9 @@ def _matches(text, patterns, entropy):
|
|
|
273
363
|
"""The matches of `patterns` in `text`, and with `entropy` the runs the entropy rule hides."""
|
|
274
364
|
found = [m for pat in patterns for m in pat.finditer(text)]
|
|
275
365
|
if entropy:
|
|
276
|
-
found += [
|
|
366
|
+
found += [
|
|
367
|
+
m for m in ENTROPY_LONG.finditer(text) if shannon_entropy(m.group(0)) >= 4.5
|
|
368
|
+
]
|
|
277
369
|
return found
|
|
278
370
|
|
|
279
371
|
|
|
@@ -281,9 +373,11 @@ def _cut_spans(text, joined, firsts, origins, matches):
|
|
|
281
373
|
"""Where in `text` the values `matches` found in `joined` lie, for those splitters cut or touch. A stretch takes
|
|
282
374
|
in the splitters next to its value when the character past them is no value character (so the value's
|
|
283
375
|
neighbours are those it had in `joined`); a splitter between the value and another value stays."""
|
|
376
|
+
|
|
284
377
|
def at(j): # where joined[j] is in text
|
|
285
378
|
k = bisect.bisect_right(firsts, j) - 1
|
|
286
379
|
return origins[k] + j - firsts[k]
|
|
380
|
+
|
|
287
381
|
spans = []
|
|
288
382
|
for m in matches:
|
|
289
383
|
s, e = at(m.start()), at(m.end() - 1) + 1
|
|
@@ -310,7 +404,9 @@ def _kept(spans, plain):
|
|
|
310
404
|
at += 1
|
|
311
405
|
while heap and heap[0][0] <= s:
|
|
312
406
|
heapq.heappop(heap)
|
|
313
|
-
if not any(
|
|
407
|
+
if not any(
|
|
408
|
+
(o_s <= s and e <= o_e) or not (s <= o_s and o_e <= e) for o_e, o_s in heap
|
|
409
|
+
):
|
|
314
410
|
out.append((s, e))
|
|
315
411
|
return out
|
|
316
412
|
|
|
@@ -333,10 +429,18 @@ def _rejoin_split(text, patterns, entropy=False, trail=None):
|
|
|
333
429
|
if not _SPLITTER_RE.search(text):
|
|
334
430
|
return text
|
|
335
431
|
joined, firsts, origins = _unsplit_runs(text)
|
|
336
|
-
spans = _cut_spans(
|
|
432
|
+
spans = _cut_spans(
|
|
433
|
+
text, joined, firsts, origins, _matches(joined, patterns, entropy)
|
|
434
|
+
)
|
|
337
435
|
if spans:
|
|
338
|
-
spans = _kept(
|
|
339
|
-
|
|
436
|
+
spans = _kept(
|
|
437
|
+
spans, [(m.start(), m.end()) for m in _matches(text, patterns, entropy)]
|
|
438
|
+
)
|
|
439
|
+
drops = [
|
|
440
|
+
(m.start(), m.end(), "")
|
|
441
|
+
for s, e in _merged(sorted(spans))
|
|
442
|
+
for m in _SPLITTER_RE.finditer(text, s, e)
|
|
443
|
+
]
|
|
340
444
|
if trail is not None:
|
|
341
445
|
trail.carry(drops)
|
|
342
446
|
out, at = [], 0
|
|
@@ -350,9 +454,11 @@ def _hide_patterns(out, mode, sub, counts):
|
|
|
350
454
|
"""`out` with each value the page patterns and the entropy rule find replaced by its marker, or by a mask with
|
|
351
455
|
mode mask; `counts` adds them up by name."""
|
|
352
456
|
for name, pat in SECRET_PATTERNS:
|
|
457
|
+
|
|
353
458
|
def repl(m, n=name):
|
|
354
459
|
counts[n] = counts.get(n, 0) + 1
|
|
355
460
|
return _mask(m.group(0)) if mode == "mask" else f"[REDACTED:{n}]"
|
|
461
|
+
|
|
356
462
|
out = sub(pat, repl, out)
|
|
357
463
|
|
|
358
464
|
def ent_repl(m):
|
|
@@ -361,6 +467,7 @@ def _hide_patterns(out, mode, sub, counts):
|
|
|
361
467
|
counts["entropy"] = counts.get("entropy", 0) + 1
|
|
362
468
|
return _mask(s) if mode == "mask" else "[REDACTED:entropy]"
|
|
363
469
|
return s
|
|
470
|
+
|
|
364
471
|
return sub(ENTROPY_LONG, ent_repl, out)
|
|
365
472
|
|
|
366
473
|
|
|
@@ -371,6 +478,7 @@ def _scan_detected(held, sub, counts):
|
|
|
371
478
|
detect-secrets hid anything."""
|
|
372
479
|
from detect_secrets import SecretsCollection
|
|
373
480
|
from detect_secrets.settings import default_settings
|
|
481
|
+
|
|
374
482
|
tf = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
|
|
375
483
|
tmp = tf.name
|
|
376
484
|
try:
|
|
@@ -383,9 +491,10 @@ def _scan_detected(held, sub, counts):
|
|
|
383
491
|
for s in col[fname]:
|
|
384
492
|
sv = s.secret_value
|
|
385
493
|
if sv and sv in held[0]:
|
|
386
|
-
held[0] = sub(
|
|
387
|
-
|
|
388
|
-
|
|
494
|
+
held[0] = sub(
|
|
495
|
+
re.compile(re.escape(sv)), lambda m: "[REDACTED:ds]", held[0]
|
|
496
|
+
)
|
|
497
|
+
counts["ds:detect-secrets"] = counts.get("ds:detect-secrets", 0) + 1
|
|
389
498
|
finally:
|
|
390
499
|
try:
|
|
391
500
|
os.unlink(tmp)
|
|
@@ -400,13 +509,18 @@ def _hide_detected(out, sub, counts):
|
|
|
400
509
|
_scan_detected(held, sub, counts)
|
|
401
510
|
except ImportError:
|
|
402
511
|
if not getattr(redact, "_ds_warned", False):
|
|
403
|
-
print(
|
|
404
|
-
|
|
512
|
+
print(
|
|
513
|
+
"[lineage] WARN: detect-secrets unavailable, using fallback "
|
|
514
|
+
"(pip install 'detect-secrets>=1.5')",
|
|
515
|
+
file=sys.stderr,
|
|
516
|
+
)
|
|
405
517
|
redact._ds_warned = True
|
|
406
518
|
except Exception as e:
|
|
407
519
|
if not getattr(redact, "_ds_err_warned", False):
|
|
408
|
-
print(
|
|
409
|
-
|
|
520
|
+
print(
|
|
521
|
+
f"[lineage] WARN: detect-secrets error ({e}), continuing with fallback",
|
|
522
|
+
file=sys.stderr,
|
|
523
|
+
)
|
|
410
524
|
redact._ds_err_warned = True
|
|
411
525
|
return held[0]
|
|
412
526
|
|
|
@@ -417,8 +531,15 @@ def _hide_extra(out, extra, sub, counts):
|
|
|
417
531
|
for kw in [k.strip() for k in extra.split(",") if k.strip()]:
|
|
418
532
|
pat = re.compile(r"\[REDACTED(?::[^\]\s]*)?\]|" + re.escape(kw), re.IGNORECASE)
|
|
419
533
|
hits = []
|
|
420
|
-
out = sub(
|
|
421
|
-
|
|
534
|
+
out = sub(
|
|
535
|
+
pat,
|
|
536
|
+
lambda m: (
|
|
537
|
+
m.group(0)
|
|
538
|
+
if m.group(0).upper().startswith("[REDACTED")
|
|
539
|
+
else hits.append(1) or "[REDACTED]"
|
|
540
|
+
),
|
|
541
|
+
out,
|
|
542
|
+
)
|
|
422
543
|
if hits:
|
|
423
544
|
counts[f"custom:{kw}"] = len(hits)
|
|
424
545
|
return out
|
|
@@ -440,8 +561,10 @@ def redact(text: str, extra: "str | None" = None, mode: str = "full", trail=None
|
|
|
440
561
|
# follow (the end of an item in an argument list written without spaces). A match starts only
|
|
441
562
|
# there, so no user runs past the place where the next match may start: a long run stays linear,
|
|
442
563
|
# and a user joined to a variable by a - ("$USER"-bot) is read whole.
|
|
443
|
-
_CU_USER = (
|
|
444
|
-
|
|
564
|
+
_CU_USER = (
|
|
565
|
+
r"(?:[^\s:=\"'`(]|[\"'`(](?!-(?:u|[A-Za-z0-9]{1,6}u[\s\"']|-user[\s=\"'])"
|
|
566
|
+
r"|,\\{0,7}[\"']))"
|
|
567
|
+
)
|
|
445
568
|
# The quotes that may open the value, bare or escaped (\" and \\\" in a quoted command, '\'' and
|
|
446
569
|
# '"'"' from bash and shlex.quote, ^" in cmd, `" in PowerShell, $' in bash): up to six in a row,
|
|
447
570
|
# none that closes an item of an argument list.
|
|
@@ -452,9 +575,16 @@ _CU_LEAD = r"(?:[\\^`$]{0,7}[\"'](?!,\\{0,7}[\"'])){0,6}"
|
|
|
452
575
|
# (mktemp -u "${TMPDIR:-/tmp}/x"). A literal password, or a default that is no path or holds a
|
|
453
576
|
# colon, is hidden.
|
|
454
577
|
_CU_REF = r"(?:\$\{?\w+\}?|%\w+%|\$\([^\s()]*(?:\s+[^\s()]+)*\))"
|
|
455
|
-
_CU_SKIP = (
|
|
456
|
-
|
|
457
|
-
|
|
578
|
+
_CU_SKIP = (
|
|
579
|
+
r"(?!"
|
|
580
|
+
+ _CU_LEAD
|
|
581
|
+
+ r"(?:\+[\"']?%[-_0^#]?[A-Za-z%]|(?:\d+:\d+|(?:"
|
|
582
|
+
+ _CU_REF
|
|
583
|
+
+ r"|[^\s:=\"'`(]*\)):"
|
|
584
|
+
+ _CU_REF
|
|
585
|
+
+ r")(?![^\s\"'`;|&)\\^])"
|
|
586
|
+
r"|\$\{\w+:[-=?+]?[/~.][^\s}\"'`(:]*\}[^\s:]*(?!\S)))"
|
|
587
|
+
)
|
|
458
588
|
|
|
459
589
|
# Patterns every page (the reviewed flow and --rulebase) and every part file hides with before the
|
|
460
590
|
# page patterns above, while keys are whole (3.0.2).
|
|
@@ -473,28 +603,52 @@ _MARKER_UNIT = r"\[REDACTED:\w+\]"
|
|
|
473
603
|
|
|
474
604
|
REVIEW_SECRET_PATTERNS = [
|
|
475
605
|
("AnthropicKey", re.compile(r"(?<![A-Za-z0-9])sk-ant-[A-Za-z0-9_-]{20,}")),
|
|
476
|
-
(
|
|
606
|
+
(
|
|
607
|
+
"OpenAIKey",
|
|
608
|
+
re.compile(r"(?<![A-Za-z0-9-])sk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}"),
|
|
609
|
+
),
|
|
477
610
|
("GoogleKey", re.compile(r"(?<![A-Za-z0-9])AIza[0-9A-Za-z_-]{35}")),
|
|
478
611
|
# bounded repeats: a long run of `key` or `a.` would otherwise take quadratic time
|
|
479
|
-
(
|
|
612
|
+
(
|
|
613
|
+
"HexKey",
|
|
614
|
+
re.compile(
|
|
615
|
+
r"(?i)(?:key|token|secret)[\w-]{0,100}\s*[:=]\s*['\"]?[0-9a-f]{32,}"
|
|
616
|
+
),
|
|
617
|
+
),
|
|
480
618
|
# a value, not a path, a variable or a redaction ($PWD is a variable, PGPASSWORD a name);
|
|
481
619
|
# markdown, a table cell or a bracket that closes after the value stays outside the match
|
|
482
|
-
(
|
|
483
|
-
|
|
484
|
-
|
|
620
|
+
(
|
|
621
|
+
"PasswordBare",
|
|
622
|
+
re.compile(
|
|
623
|
+
r"(?i)(?<!\$)(?:password|passwd|passcode|pwd|암호|비번|비밀\s?번호|패스워드)\s*[:=]\s*"
|
|
624
|
+
r"(?![/~$])" + _NOT_REDACTED + r"(?=[^\s'\"]{6})[^\s'\"]*[^\s'\"`|*)]"
|
|
625
|
+
),
|
|
626
|
+
),
|
|
485
627
|
# the password runs to the last @ before the host: it may hold an @ of its own. A marker is
|
|
486
628
|
# no user or password, so a second pass leaves a token-only URL (https://[REDACTED:...]@host).
|
|
487
|
-
(
|
|
488
|
-
|
|
629
|
+
(
|
|
630
|
+
"UrlCredential",
|
|
631
|
+
re.compile(
|
|
632
|
+
r"(?i)[a-z][a-z0-9+.-]{0,30}://(?:"
|
|
633
|
+
+ _MARKER_UNIT
|
|
634
|
+
+ r"|[^\s:@/])*:"
|
|
635
|
+
+ _NOT_MARKED
|
|
636
|
+
+ r"(?!\[REDACTED)[^\s/]{3,}@"
|
|
637
|
+
),
|
|
638
|
+
),
|
|
489
639
|
("Bearer", re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{16,}")),
|
|
490
640
|
# Authorization then `:`, `=`, `=>` or headers["Authorization"] =, a call's ("Authorization",
|
|
491
641
|
# "...") and a HAR name/value pair, or a space before a quote (nginx, Apache)
|
|
492
|
-
(
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
642
|
+
(
|
|
643
|
+
"BasicAuth",
|
|
644
|
+
re.compile(
|
|
645
|
+
r"(?i)\bauthorization"
|
|
646
|
+
r"(?:[\"']?\]?\s*(?:=>|[:=])"
|
|
647
|
+
r"|[\"']\s*,\s*(?:[\"']value[\"']\s*:\s*)?(?=[\"'])"
|
|
648
|
+
r"|\s+(?=[\"']))"
|
|
649
|
+
r"\s*[\"']?basic\s+[A-Za-z0-9+/]{8,}={0,2}"
|
|
650
|
+
),
|
|
651
|
+
),
|
|
498
652
|
# curl's -u and --user flags, alone or last in a bundle (-su, -4u), with a user and password
|
|
499
653
|
# after a space, an = or nothing, quoted together or apart ("user":"pw"), with quotes bare,
|
|
500
654
|
# escaped or nested (_CU_LEAD) or none, in inline code or parentheses, or as two items of an
|
|
@@ -503,15 +657,38 @@ REVIEW_SECRET_PATTERNS = [
|
|
|
503
657
|
# long key as the user with no password, count too. The separators share no character with
|
|
504
658
|
# the user name, so a long run of them stays linear. The values _CU_SKIP names and a
|
|
505
659
|
# host:/path (rsync) are left, and so is a marker an earlier pass left (-u [REDACTED:JWT]).
|
|
506
|
-
(
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
660
|
+
(
|
|
661
|
+
"CurlUser",
|
|
662
|
+
re.compile(
|
|
663
|
+
r"(?<![^\s\"'`(])(?:-u\s*|-[A-Za-z0-9]{1,6}u\s+|--user(?:\s+|=))"
|
|
664
|
+
+ _CU_SKIP
|
|
665
|
+
+ _CU_LEAD
|
|
666
|
+
+ r"(?:(?:"
|
|
667
|
+
+ _MARKER_UNIT
|
|
668
|
+
+ r"|"
|
|
669
|
+
+ _CU_USER
|
|
670
|
+
+ r")*:"
|
|
671
|
+
+ _NOT_MARKED
|
|
672
|
+
+ r"(?!/|\[REDACTED)\S{3,}|"
|
|
673
|
+
+ _CU_USER
|
|
674
|
+
+ r"{15,}:"
|
|
675
|
+
+ _NOT_MARKED
|
|
676
|
+
+ r"(?=[\s\"'`).,;\\]|$))"
|
|
677
|
+
r"|(\\{0,7}[\"'])(?:-[A-Za-z0-9]{0,6}u|--user)\1\s*,\s*(\\{0,7}[\"'])"
|
|
678
|
+
+ _CU_SKIP
|
|
679
|
+
+ r"(?:(?:"
|
|
680
|
+
+ _MARKER_UNIT
|
|
681
|
+
+ r"|"
|
|
682
|
+
+ _CU_USER
|
|
683
|
+
+ r")*:"
|
|
684
|
+
+ _NOT_MARKED
|
|
685
|
+
+ r"(?!/|\[REDACTED)[^\s\"']{3,}|"
|
|
686
|
+
+ _CU_USER
|
|
687
|
+
+ r"{15,}:"
|
|
688
|
+
+ _NOT_MARKED
|
|
689
|
+
+ r")\2"
|
|
690
|
+
),
|
|
691
|
+
),
|
|
515
692
|
]
|
|
516
693
|
|
|
517
694
|
|
|
@@ -521,7 +698,9 @@ def _reviewer_patterns(text, trail=None):
|
|
|
521
698
|
counts, sub = {}, _sub_for(trail)
|
|
522
699
|
for name, pat in REVIEW_SECRET_PATTERNS:
|
|
523
700
|
hits = []
|
|
524
|
-
text = sub(
|
|
701
|
+
text = sub(
|
|
702
|
+
pat, lambda m, marker=f"[REDACTED:{name}]": hits.append(1) or marker, text
|
|
703
|
+
)
|
|
525
704
|
if hits:
|
|
526
705
|
counts[name] = len(hits)
|
|
527
706
|
return text, counts
|
|
@@ -531,8 +710,11 @@ def _rejoin_secrets(text, extra=None, trail=None):
|
|
|
531
710
|
"""`text` with each secret a splitter cuts put back together, for every pattern a page or a reviewer's text is
|
|
532
711
|
hidden with: the reviewer's, the page's, the entropy rule and the --redact-extra keywords (3.0.8)."""
|
|
533
712
|
keywords = [k.strip() for k in (extra or "").split(",") if k.strip()]
|
|
534
|
-
patterns = (
|
|
535
|
-
|
|
713
|
+
patterns = (
|
|
714
|
+
[p for _, p in REVIEW_SECRET_PATTERNS]
|
|
715
|
+
+ [p for _, p in SECRET_PATTERNS]
|
|
716
|
+
+ [re.compile(re.escape(k), re.IGNORECASE) for k in keywords]
|
|
717
|
+
)
|
|
536
718
|
return _rejoin_split(text, patterns, entropy=True, trail=trail)
|
|
537
719
|
|
|
538
720
|
|
|
@@ -601,7 +783,9 @@ def _hidden_by(floor, base):
|
|
|
601
783
|
j = i
|
|
602
784
|
while j < n and pos[j] < 0:
|
|
603
785
|
j += 1
|
|
604
|
-
out.append(
|
|
786
|
+
out.append(
|
|
787
|
+
(pos[i - 1] + 1 if i else 0, pos[j] if j < n else base.n, floor[i:j])
|
|
788
|
+
)
|
|
605
789
|
i = j
|
|
606
790
|
return out
|
|
607
791
|
|
|
@@ -634,7 +818,10 @@ def _with_rejoins(text, extra, one_pass):
|
|
|
634
818
|
[found])."""
|
|
635
819
|
red, founds, trail = _join_first(text, extra, one_pass)
|
|
636
820
|
if trail is None:
|
|
637
|
-
return
|
|
821
|
+
return (
|
|
822
|
+
red,
|
|
823
|
+
founds,
|
|
824
|
+
) # nothing joined: that was the one pass over the text as it stands
|
|
638
825
|
base = _Trail(len(text))
|
|
639
826
|
floor, found = one_pass(text, base)
|
|
640
827
|
joined = _kinds(founds)
|
|
@@ -662,7 +849,9 @@ def review_redact(text, extra=None):
|
|
|
662
849
|
are whole (the entropy rule would otherwise cut a key and leave the rest), then fully
|
|
663
850
|
redacted (never masked, whatever --redact-mode says). A secret a splitter cuts is put back
|
|
664
851
|
together (3.0.8). Returns (text, count)."""
|
|
665
|
-
red, founds = _with_rejoins(
|
|
852
|
+
red, founds = _with_rejoins(
|
|
853
|
+
text, extra, lambda t, trail=None: _review_pass(t, extra, trail)
|
|
854
|
+
)
|
|
666
855
|
return red, sum(sum(f.values()) for f in founds)
|
|
667
856
|
|
|
668
857
|
|
|
@@ -681,7 +870,9 @@ def page_redact(text, extra=None, mode="full"):
|
|
|
681
870
|
splitter cuts is put back together (3.0.8). Returns (text, count_by_kind). A second pass
|
|
682
871
|
over its output finds nothing, but for two values glued with no separator at all (a key
|
|
683
872
|
run into `Bearer`), where the second shows only once the first is a marker, as in 3.0.7."""
|
|
684
|
-
red, founds = _with_rejoins(
|
|
873
|
+
red, founds = _with_rejoins(
|
|
874
|
+
text, extra, lambda t, trail=None: _page_pass(t, extra, mode, trail)
|
|
875
|
+
)
|
|
685
876
|
counts = {}
|
|
686
877
|
for found in founds:
|
|
687
878
|
for k, v in found.items():
|
|
@@ -760,8 +951,21 @@ def _codex_meta(path):
|
|
|
760
951
|
return None
|
|
761
952
|
with open(path, encoding="utf-8") as f:
|
|
762
953
|
obj = json.loads(f.readline())
|
|
763
|
-
return
|
|
764
|
-
|
|
954
|
+
return (
|
|
955
|
+
obj["payload"]
|
|
956
|
+
if obj.get("type") == "session_meta"
|
|
957
|
+
and isinstance(obj.get("payload"), dict)
|
|
958
|
+
else None
|
|
959
|
+
)
|
|
960
|
+
except (
|
|
961
|
+
OSError,
|
|
962
|
+
UnicodeDecodeError,
|
|
963
|
+
ValueError,
|
|
964
|
+
RecursionError,
|
|
965
|
+
KeyError,
|
|
966
|
+
TypeError,
|
|
967
|
+
AttributeError,
|
|
968
|
+
):
|
|
765
969
|
return None
|
|
766
970
|
|
|
767
971
|
|
|
@@ -776,22 +980,32 @@ def _codex_cwd(path):
|
|
|
776
980
|
if isinstance(source, dict) and "subagent" in source:
|
|
777
981
|
return None
|
|
778
982
|
return meta.get("cwd")
|
|
779
|
-
except (
|
|
983
|
+
except (
|
|
984
|
+
OSError,
|
|
985
|
+
UnicodeDecodeError,
|
|
986
|
+
ValueError,
|
|
987
|
+
RecursionError,
|
|
988
|
+
KeyError,
|
|
989
|
+
TypeError,
|
|
990
|
+
AttributeError,
|
|
991
|
+
):
|
|
780
992
|
return None
|
|
781
993
|
|
|
782
994
|
|
|
783
995
|
def auto_discover_jsonl():
|
|
784
996
|
tid = _codex_thread()
|
|
785
|
-
if tid:
|
|
997
|
+
if tid: # inside Codex: this session or nothing, never a Claude Code one
|
|
786
998
|
found = _codex_rollouts(f"rollout-*-{tid}.jsonl")
|
|
787
999
|
return found[-1] if found else None
|
|
788
|
-
projdir =
|
|
789
|
-
pathlib.Path.cwd())
|
|
1000
|
+
projdir = (
|
|
1001
|
+
pathlib.Path.home() / ".claude" / "projects" / encode_cwd(pathlib.Path.cwd())
|
|
1002
|
+
)
|
|
790
1003
|
if not projdir.exists():
|
|
791
1004
|
return None
|
|
792
1005
|
try:
|
|
793
|
-
files = sorted(
|
|
794
|
-
|
|
1006
|
+
files = sorted(
|
|
1007
|
+
projdir.glob("*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True
|
|
1008
|
+
)
|
|
795
1009
|
except OSError:
|
|
796
1010
|
return None
|
|
797
1011
|
return files[0] if files else None
|
|
@@ -802,8 +1016,9 @@ def project_jsonl_files():
|
|
|
802
1016
|
if _codex_thread():
|
|
803
1017
|
here = str(pathlib.Path.cwd())
|
|
804
1018
|
return [p for p in _codex_rollouts("rollout-*.jsonl") if _codex_cwd(p) == here]
|
|
805
|
-
projdir =
|
|
806
|
-
pathlib.Path.cwd())
|
|
1019
|
+
projdir = (
|
|
1020
|
+
pathlib.Path.home() / ".claude" / "projects" / encode_cwd(pathlib.Path.cwd())
|
|
1021
|
+
)
|
|
807
1022
|
if not projdir.exists():
|
|
808
1023
|
return []
|
|
809
1024
|
try:
|
|
@@ -830,7 +1045,9 @@ def session_start(path):
|
|
|
830
1045
|
# ---------------------------------------------------------------- JSONL parse
|
|
831
1046
|
KNOWN_TYPES = {"user", "assistant", "agent-setting", "permission-mode", "summary"}
|
|
832
1047
|
KNOWN_CONTENT_ITEMS = {"text", "tool_use", "tool_result", "thinking"}
|
|
833
|
-
_CTRL_RE = re.compile(
|
|
1048
|
+
_CTRL_RE = re.compile(
|
|
1049
|
+
r"[\x00-\x08\x0b\x0c\x0e-\x1f]"
|
|
1050
|
+
) # strip control chars (keeps \t\n)
|
|
834
1051
|
# What a JSON escape for a lone surrogate (U+D800 to U+DFFF with no pair) leaves in a str. UTF-8 cannot hold one,
|
|
835
1052
|
# so a page, a review pack or a cache key that has it stops with UnicodeEncodeError.
|
|
836
1053
|
_LONE_SURROGATE_RE = re.compile("[" + chr(0xD800) + "-" + chr(0xDFFF) + "]")
|
|
@@ -844,13 +1061,18 @@ def _without_lone_surrogates(value: Any) -> Any:
|
|
|
844
1061
|
if isinstance(value, list):
|
|
845
1062
|
return [_without_lone_surrogates(v) for v in value]
|
|
846
1063
|
if isinstance(value, dict):
|
|
847
|
-
return {
|
|
1064
|
+
return {
|
|
1065
|
+
_without_lone_surrogates(k): _without_lone_surrogates(v)
|
|
1066
|
+
for k, v in value.items()
|
|
1067
|
+
}
|
|
848
1068
|
return value
|
|
849
1069
|
|
|
850
1070
|
|
|
851
1071
|
def _lone_to_fffd(value):
|
|
852
1072
|
"""A str with each lone surrogate as U+FFFD; anything else as it is (no walk into it)."""
|
|
853
|
-
return
|
|
1073
|
+
return (
|
|
1074
|
+
_LONE_SURROGATE_RE.sub(chr(0xFFFD), value) if isinstance(value, str) else value
|
|
1075
|
+
)
|
|
854
1076
|
|
|
855
1077
|
|
|
856
1078
|
def _shown_id(value):
|
|
@@ -862,9 +1084,14 @@ def _shown_id(value):
|
|
|
862
1084
|
def _clean_strings(line_no, role, text, tools, ts, uuid_):
|
|
863
1085
|
"""A record's role, text, tool counts, timestamp and uuid with each lone surrogate as U+FFFD, after one WARN for
|
|
864
1086
|
the line. Only text is touched: a uuid of another shape is replaced later, whole."""
|
|
865
|
-
if not any(
|
|
1087
|
+
if not any(
|
|
1088
|
+
isinstance(s, str) and _LONE_SURROGATE_RE.search(s)
|
|
1089
|
+
for s in (role, text, ts, uuid_, *tools)
|
|
1090
|
+
):
|
|
866
1091
|
return role, text, tools, ts, uuid_
|
|
867
|
-
print(
|
|
1092
|
+
print(
|
|
1093
|
+
f"[WARN] line {line_no}: lone surrogate replaced with U+FFFD", file=sys.stderr
|
|
1094
|
+
)
|
|
868
1095
|
counts = {}
|
|
869
1096
|
for name, n in tools.items():
|
|
870
1097
|
name = _lone_to_fffd(name)
|
|
@@ -879,7 +1106,10 @@ def _turn_id(uuid_, line_no, ts):
|
|
|
879
1106
|
if isinstance(uuid_, str) and uuid_:
|
|
880
1107
|
return uuid_
|
|
881
1108
|
if uuid_ is not None and uuid_ != "":
|
|
882
|
-
print(
|
|
1109
|
+
print(
|
|
1110
|
+
f"[WARN] line {line_no}: uuid that is not text replaced by an id from the line",
|
|
1111
|
+
file=sys.stderr,
|
|
1112
|
+
)
|
|
883
1113
|
return hashlib.sha256(f"{line_no}:{ts}".encode()).hexdigest()[:16]
|
|
884
1114
|
|
|
885
1115
|
|
|
@@ -889,7 +1119,7 @@ def _json_object(raw, line_no):
|
|
|
889
1119
|
something other than an object is skipped, not fatal."""
|
|
890
1120
|
try:
|
|
891
1121
|
obj = json.loads(raw)
|
|
892
|
-
except (ValueError, RecursionError):
|
|
1122
|
+
except (ValueError, RecursionError): # JSONDecodeError is a ValueError
|
|
893
1123
|
print(f"[WARN] line {line_no}: invalid JSON skipped", file=sys.stderr)
|
|
894
1124
|
return None
|
|
895
1125
|
if not isinstance(obj, dict):
|
|
@@ -905,18 +1135,27 @@ def _is_turn_record(obj, seen_unknown_type, unsafe_schema, line_no):
|
|
|
905
1135
|
if obj.get("isSidechain") or obj.get("teamName") or obj.get("agentName"):
|
|
906
1136
|
return False
|
|
907
1137
|
t = obj.get("type")
|
|
908
|
-
if (t and not isinstance(t, str)) or (
|
|
909
|
-
|
|
1138
|
+
if (t and not isinstance(t, str)) or (
|
|
1139
|
+
t in ("user", "assistant") and _odd_message(obj.get("message"))
|
|
1140
|
+
):
|
|
1141
|
+
print(
|
|
1142
|
+
f"[WARN] line {line_no}: record of an unknown shape skipped",
|
|
1143
|
+
file=sys.stderr,
|
|
1144
|
+
)
|
|
910
1145
|
return False
|
|
911
1146
|
if t in ("user", "assistant"):
|
|
912
1147
|
return True
|
|
913
1148
|
if t and t not in KNOWN_TYPES and t not in seen_unknown_type:
|
|
914
1149
|
seen_unknown_type.add(t)
|
|
915
|
-
print(
|
|
1150
|
+
print(
|
|
1151
|
+
f"[WARN] unknown record type {t!r} skipped", file=sys.stderr
|
|
1152
|
+
) # repr: no raw control character
|
|
916
1153
|
if not unsafe_schema and t.startswith("v"):
|
|
917
|
-
print(
|
|
918
|
-
|
|
919
|
-
|
|
1154
|
+
print(
|
|
1155
|
+
f"[ERROR] schema marker {t!r} unrecognized — "
|
|
1156
|
+
"rerun with --unsafe-schema to proceed",
|
|
1157
|
+
file=sys.stderr,
|
|
1158
|
+
)
|
|
920
1159
|
raise SystemExit(2)
|
|
921
1160
|
return False
|
|
922
1161
|
|
|
@@ -934,8 +1173,11 @@ def _odd_message(msg):
|
|
|
934
1173
|
def _odd_item(it, text, name):
|
|
935
1174
|
"""Whether a content item's type, text or tool name has a shape a transcript does not give. A missing or empty
|
|
936
1175
|
text reads as empty and a missing or null tool name as '?'."""
|
|
937
|
-
return (
|
|
938
|
-
|
|
1176
|
+
return (
|
|
1177
|
+
(it is not None and not isinstance(it, str))
|
|
1178
|
+
or (it == "text" and not isinstance(text, str))
|
|
1179
|
+
or (it == "tool_use" and not isinstance(name, str))
|
|
1180
|
+
)
|
|
939
1181
|
|
|
940
1182
|
|
|
941
1183
|
def _take_item(item, line_no, text_parts, tools):
|
|
@@ -946,14 +1188,19 @@ def _take_item(item, line_no, text_parts, tools):
|
|
|
946
1188
|
it, text, name = item.get("type"), item.get("text") or "", item.get("name")
|
|
947
1189
|
name = "?" if name is None else name
|
|
948
1190
|
if _odd_item(it, text, name):
|
|
949
|
-
print(
|
|
1191
|
+
print(
|
|
1192
|
+
f"[WARN] line {line_no}: content item of an unknown shape skipped",
|
|
1193
|
+
file=sys.stderr,
|
|
1194
|
+
)
|
|
950
1195
|
elif it == "text":
|
|
951
1196
|
text_parts.append(text)
|
|
952
1197
|
elif it == "tool_use":
|
|
953
1198
|
tools[name] = tools.get(name, 0) + 1
|
|
954
1199
|
elif it not in KNOWN_CONTENT_ITEMS:
|
|
955
|
-
print(
|
|
956
|
-
|
|
1200
|
+
print(
|
|
1201
|
+
f"[WARN] line {line_no}: unknown content type {it!r} skipped",
|
|
1202
|
+
file=sys.stderr,
|
|
1203
|
+
)
|
|
957
1204
|
|
|
958
1205
|
|
|
959
1206
|
def _content_parts(content, line_no):
|
|
@@ -967,8 +1214,11 @@ def _content_parts(content, line_no):
|
|
|
967
1214
|
_take_item(item, line_no, text_parts, tools)
|
|
968
1215
|
else:
|
|
969
1216
|
if content is not None:
|
|
970
|
-
print(
|
|
971
|
-
|
|
1217
|
+
print(
|
|
1218
|
+
f"[WARN] line {line_no}: unknown content shape "
|
|
1219
|
+
f"({type(content).__name__})",
|
|
1220
|
+
file=sys.stderr,
|
|
1221
|
+
)
|
|
972
1222
|
return None
|
|
973
1223
|
return text_parts, tools
|
|
974
1224
|
|
|
@@ -987,7 +1237,11 @@ def _codex_item_text(item):
|
|
|
987
1237
|
content = item.get("content")
|
|
988
1238
|
if not isinstance(content, list):
|
|
989
1239
|
return ""
|
|
990
|
-
return "\n".join(
|
|
1240
|
+
return "\n".join(
|
|
1241
|
+
c["text"]
|
|
1242
|
+
for c in content
|
|
1243
|
+
if isinstance(c, dict) and isinstance(c.get("text"), str)
|
|
1244
|
+
)
|
|
991
1245
|
|
|
992
1246
|
|
|
993
1247
|
def _codex_tools(p, pt):
|
|
@@ -995,7 +1249,11 @@ def _codex_tools(p, pt):
|
|
|
995
1249
|
input), so the inner tool names count, not the wrapper."""
|
|
996
1250
|
name = p.get("name")
|
|
997
1251
|
name = name if isinstance(name, str) and name else pt[:-5]
|
|
998
|
-
inner =
|
|
1252
|
+
inner = (
|
|
1253
|
+
_CODEX_INNER_TOOL_RE.findall(p.get("input"))
|
|
1254
|
+
if isinstance(p.get("input"), str)
|
|
1255
|
+
else []
|
|
1256
|
+
)
|
|
999
1257
|
counts = {}
|
|
1000
1258
|
for n in inner or [name]:
|
|
1001
1259
|
counts[n] = counts.get(n, 0) + 1
|
|
@@ -1017,9 +1275,18 @@ def _codex_record(obj):
|
|
|
1017
1275
|
pt = p.get("type")
|
|
1018
1276
|
if kind == "event_msg" and pt in ("user_message", "agent_message"):
|
|
1019
1277
|
msg = p.get("message")
|
|
1020
|
-
return (
|
|
1278
|
+
return (
|
|
1279
|
+
("user" if pt == "user_message" else "assistant"),
|
|
1280
|
+
(msg if isinstance(msg, str) else ""),
|
|
1281
|
+
{},
|
|
1282
|
+
)
|
|
1021
1283
|
item = p.get("item")
|
|
1022
|
-
if
|
|
1284
|
+
if (
|
|
1285
|
+
kind == "event_msg"
|
|
1286
|
+
and pt == "item_completed"
|
|
1287
|
+
and isinstance(item, dict)
|
|
1288
|
+
and item.get("type") in _CODEX_ITEM_ROLE
|
|
1289
|
+
):
|
|
1023
1290
|
return _CODEX_ITEM_ROLE[item["type"]], _codex_item_text(item), {}
|
|
1024
1291
|
if kind == "response_item" and isinstance(pt, str) and pt.endswith("_call"):
|
|
1025
1292
|
return "assistant", "", _codex_tools(p, pt)
|
|
@@ -1035,14 +1302,27 @@ def _codex_turn(obj, line_no, session_id, session_name):
|
|
|
1035
1302
|
text = _CTRL_RE.sub("", text.strip())
|
|
1036
1303
|
if not text and not tools:
|
|
1037
1304
|
return None
|
|
1038
|
-
role, text, tools, ts, uuid_ = _clean_strings(
|
|
1305
|
+
role, text, tools, ts, uuid_ = _clean_strings(
|
|
1306
|
+
line_no, role, text, tools, _stamp(obj.get("timestamp")), None
|
|
1307
|
+
)
|
|
1039
1308
|
uuid_ = _turn_id(uuid_, line_no, ts)
|
|
1040
|
-
return {
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1309
|
+
return {
|
|
1310
|
+
"role": role,
|
|
1311
|
+
"text": text,
|
|
1312
|
+
"ts": ts,
|
|
1313
|
+
"uuid": uuid_,
|
|
1314
|
+
"tools": tools,
|
|
1315
|
+
"line_no": line_no,
|
|
1316
|
+
"parts": [text] if text else [],
|
|
1317
|
+
"session": session_id,
|
|
1318
|
+
"session_name": session_name,
|
|
1319
|
+
"meta": False,
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
|
|
1323
|
+
def parse_turns(
|
|
1324
|
+
stream, unsafe_schema: bool = False, session_id=None, session_name=None
|
|
1325
|
+
):
|
|
1046
1326
|
"""Yield dicts: {role, text, ts, uuid, tools, line_no, parts, session,
|
|
1047
1327
|
session_name}.
|
|
1048
1328
|
|
|
@@ -1051,8 +1331,10 @@ def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
|
|
|
1051
1331
|
needs it) and `session` (session guards for merge/echo).
|
|
1052
1332
|
"""
|
|
1053
1333
|
seen_unknown_type = set()
|
|
1054
|
-
codex = None
|
|
1055
|
-
last_said =
|
|
1334
|
+
codex = None # set by the first record: a Codex rollout or a Claude transcript
|
|
1335
|
+
last_said = (
|
|
1336
|
+
None # Codex: (role, text) of the last message, so one said twice counts once
|
|
1337
|
+
)
|
|
1056
1338
|
for line_no, raw in enumerate(stream, 1):
|
|
1057
1339
|
raw = raw.strip()
|
|
1058
1340
|
if not raw:
|
|
@@ -1068,7 +1350,9 @@ def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
|
|
|
1068
1350
|
if said:
|
|
1069
1351
|
last_said = said
|
|
1070
1352
|
continue
|
|
1071
|
-
if obj is None or not _is_turn_record(
|
|
1353
|
+
if obj is None or not _is_turn_record(
|
|
1354
|
+
obj, seen_unknown_type, unsafe_schema, line_no
|
|
1355
|
+
):
|
|
1072
1356
|
continue
|
|
1073
1357
|
msg = obj.get("message") or {}
|
|
1074
1358
|
got = _content_parts(msg.get("content"), line_no)
|
|
@@ -1079,16 +1363,26 @@ def parse_turns(stream, unsafe_schema: bool = False, session_id=None,
|
|
|
1079
1363
|
if not text and not tools:
|
|
1080
1364
|
continue
|
|
1081
1365
|
role = msg.get("role")
|
|
1082
|
-
role =
|
|
1366
|
+
role = (
|
|
1367
|
+
role if role in ("user", "assistant") else obj.get("type")
|
|
1368
|
+
) # the type _is_turn_record let through
|
|
1083
1369
|
role, text, tools, ts, uuid_ = _clean_strings(
|
|
1084
|
-
line_no, role, text, tools, _stamp(obj.get("timestamp")), obj.get("uuid")
|
|
1370
|
+
line_no, role, text, tools, _stamp(obj.get("timestamp")), obj.get("uuid")
|
|
1371
|
+
)
|
|
1085
1372
|
uuid_ = _turn_id(uuid_, line_no, ts)
|
|
1086
1373
|
|
|
1087
|
-
yield {
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1374
|
+
yield {
|
|
1375
|
+
"role": role,
|
|
1376
|
+
"text": text,
|
|
1377
|
+
"ts": ts,
|
|
1378
|
+
"uuid": uuid_,
|
|
1379
|
+
"tools": tools,
|
|
1380
|
+
"line_no": line_no,
|
|
1381
|
+
"parts": [text] if text else [], # R1: always seed parts
|
|
1382
|
+
"session": session_id,
|
|
1383
|
+
"session_name": session_name,
|
|
1384
|
+
"meta": bool(obj.get("isMeta")),
|
|
1385
|
+
} # a harness-injected body
|
|
1092
1386
|
|
|
1093
1387
|
|
|
1094
1388
|
# ---------------------------------------------------------------- Classify (A-1..A-4)
|
|
@@ -1110,9 +1404,11 @@ def classify_turns(turns, drop_trivia=True):
|
|
|
1110
1404
|
if _HOOK_FEEDBACK_RE.match(text) or _INTERRUPT_RE.match(text):
|
|
1111
1405
|
continue
|
|
1112
1406
|
# 2. injected skill/workflow body & harness errors → drop if trivia
|
|
1113
|
-
if drop_trivia and (
|
|
1114
|
-
|
|
1115
|
-
|
|
1407
|
+
if drop_trivia and (
|
|
1408
|
+
_SKILL_BODY_RE.match(text)
|
|
1409
|
+
or _WORKFLOW_BODY_RE.match(text)
|
|
1410
|
+
or _SYS_ERROR_RE.match(text)
|
|
1411
|
+
):
|
|
1116
1412
|
continue
|
|
1117
1413
|
# 3. image note → replace with attachment marker
|
|
1118
1414
|
if _IMAGE_NOTE_RE.match(text):
|
|
@@ -1125,7 +1421,7 @@ def classify_turns(turns, drop_trivia=True):
|
|
|
1125
1421
|
# 5. agent / teammate wrapper → agent bubble
|
|
1126
1422
|
who, body = split_agent_message(text)
|
|
1127
1423
|
if who is not None:
|
|
1128
|
-
if not body:
|
|
1424
|
+
if not body: # idle notification
|
|
1129
1425
|
continue
|
|
1130
1426
|
out.append(dict(t, role="agent", text=body, agent_from=who))
|
|
1131
1427
|
continue
|
|
@@ -1147,8 +1443,12 @@ def merge_assistant_runs(turns):
|
|
|
1147
1443
|
merged = []
|
|
1148
1444
|
for t in turns:
|
|
1149
1445
|
p = merged[-1] if merged else None
|
|
1150
|
-
if (
|
|
1151
|
-
|
|
1446
|
+
if (
|
|
1447
|
+
p
|
|
1448
|
+
and t["role"] == "assistant"
|
|
1449
|
+
and p["role"] == "assistant"
|
|
1450
|
+
and t.get("session") == p.get("session")
|
|
1451
|
+
):
|
|
1152
1452
|
if t["text"]:
|
|
1153
1453
|
p["text"] = (p["text"] + "\n\n" + t["text"]).strip()
|
|
1154
1454
|
p["parts"].append(t["text"])
|
|
@@ -1169,9 +1469,13 @@ def echo_indices(turns):
|
|
|
1169
1469
|
if t["role"] != "user" or len(t["text"]) > ECHO_ASK:
|
|
1170
1470
|
continue
|
|
1171
1471
|
n = turns[i + 1] if i + 1 < len(turns) else None
|
|
1172
|
-
if (
|
|
1173
|
-
|
|
1174
|
-
|
|
1472
|
+
if (
|
|
1473
|
+
n
|
|
1474
|
+
and n["role"] == "assistant"
|
|
1475
|
+
and not n["tools"]
|
|
1476
|
+
and len(n["text"].strip()) <= ECHO_REPLY
|
|
1477
|
+
and n.get("session") == t.get("session")
|
|
1478
|
+
):
|
|
1175
1479
|
drop |= {i, i + 1}
|
|
1176
1480
|
return drop
|
|
1177
1481
|
|
|
@@ -1201,7 +1505,8 @@ def fill_missing_ts(turns):
|
|
|
1201
1505
|
_SENT_SPLIT = re.compile(r"(?<=[.!?…])(?<![0-9][.!?…])(?=\s|$)\s*|\n+")
|
|
1202
1506
|
_INTENT_ONLY_RE = re.compile(
|
|
1203
1507
|
r"^.{0,30}(?:하겠습니다|합니다|해보겠습니다|확인합니다|봅니다|"
|
|
1204
|
-
r"보겠습니다|시작합니다|진행합니다)\.?$"
|
|
1508
|
+
r"보겠습니다|시작합니다|진행합니다)\.?$"
|
|
1509
|
+
)
|
|
1205
1510
|
SHORT_DETAIL = 160
|
|
1206
1511
|
SINGLE_SPLIT_MIN = SHORT_DETAIL
|
|
1207
1512
|
|
|
@@ -1213,7 +1518,7 @@ def _cut(s, n):
|
|
|
1213
1518
|
s = s.strip()
|
|
1214
1519
|
if len(s) <= n:
|
|
1215
1520
|
return s
|
|
1216
|
-
head = _OPEN_MARKER_RE.sub("", s[:max(1, n - 1)])
|
|
1521
|
+
head = _OPEN_MARKER_RE.sub("", s[: max(1, n - 1)]) # a marker cut in two goes whole
|
|
1217
1522
|
return head.rstrip() + "…"
|
|
1218
1523
|
|
|
1219
1524
|
|
|
@@ -1290,10 +1595,14 @@ def _stdin_id_key():
|
|
|
1290
1595
|
CACHE_BASE.mkdir(parents=True, exist_ok=True)
|
|
1291
1596
|
os.chmod(str(CACHE_BASE), 0o700)
|
|
1292
1597
|
# O_EXCL: a name that is already there (a link planted at this guessable name among them) is refused
|
|
1293
|
-
fd = os.open(
|
|
1598
|
+
fd = os.open(
|
|
1599
|
+
str(tmp),
|
|
1600
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0),
|
|
1601
|
+
0o600,
|
|
1602
|
+
)
|
|
1294
1603
|
with os.fdopen(fd, "wb") as f:
|
|
1295
1604
|
f.write(key)
|
|
1296
|
-
os.replace(str(tmp), str(path))
|
|
1605
|
+
os.replace(str(tmp), str(path)) # whole or not at all
|
|
1297
1606
|
except OSError:
|
|
1298
1607
|
try:
|
|
1299
1608
|
os.unlink(str(tmp))
|
|
@@ -1316,17 +1625,38 @@ def cache_dir(session_id: str):
|
|
|
1316
1625
|
def _page_clean_turn(t, extra, mode):
|
|
1317
1626
|
"""`t` with its text and parts as a page shows them, so a summary cut from it cannot cut a
|
|
1318
1627
|
secret in two and keep the half no pattern recognises."""
|
|
1628
|
+
|
|
1319
1629
|
def clean(s):
|
|
1320
1630
|
return page_redact(s, extra, mode)[0]
|
|
1321
|
-
|
|
1631
|
+
|
|
1632
|
+
return dict(
|
|
1633
|
+
t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []]
|
|
1634
|
+
)
|
|
1322
1635
|
|
|
1323
1636
|
|
|
1324
1637
|
# Letters that print as nothing: the Hangul fillers, the braille blank, and the combining marks Unicode ignores by
|
|
1325
1638
|
# default (the grapheme joiner, the Khmer inherent vowels, the Mongolian free variation selectors). With the control,
|
|
1326
1639
|
# format, private-use, unassigned and separator categories, a cache made only of them holds no summary a person can
|
|
1327
1640
|
# read (3.0.5).
|
|
1328
|
-
_BLANK_LETTERS = frozenset(
|
|
1329
|
-
|
|
1641
|
+
_BLANK_LETTERS = frozenset(
|
|
1642
|
+
map(
|
|
1643
|
+
chr,
|
|
1644
|
+
(
|
|
1645
|
+
0x115F,
|
|
1646
|
+
0x1160,
|
|
1647
|
+
0x3164,
|
|
1648
|
+
0xFFA0,
|
|
1649
|
+
0x2800,
|
|
1650
|
+
0x034F,
|
|
1651
|
+
0x17B4,
|
|
1652
|
+
0x17B5,
|
|
1653
|
+
0x180B,
|
|
1654
|
+
0x180C,
|
|
1655
|
+
0x180D,
|
|
1656
|
+
0x180F,
|
|
1657
|
+
),
|
|
1658
|
+
)
|
|
1659
|
+
)
|
|
1330
1660
|
# Cn follows the Unicode table of the Python that runs: a character newer than that table reads as unassigned, so
|
|
1331
1661
|
# a cache made only of such characters is summarized again every run (the page is the same).
|
|
1332
1662
|
_BLANK_CATEGORIES = frozenset(("Cc", "Cf", "Co", "Cn", "Zs", "Zl", "Zp"))
|
|
@@ -1335,7 +1665,11 @@ _BLANK_CATEGORIES = frozenset(("Cc", "Cf", "Co", "Cn", "Zs", "Zl", "Zp"))
|
|
|
1335
1665
|
def _blank(ch):
|
|
1336
1666
|
"""Whether ch prints as nothing: whitespace, a blank-looking letter, or a variation selector.
|
|
1337
1667
|
White space is in the blank categories too (Zs, Zl, Zp, Cc); str.isspace() only answers it sooner."""
|
|
1338
|
-
if
|
|
1668
|
+
if (
|
|
1669
|
+
ch.isspace()
|
|
1670
|
+
or ch in _BLANK_LETTERS
|
|
1671
|
+
or unicodedata.category(ch) in _BLANK_CATEGORIES
|
|
1672
|
+
):
|
|
1339
1673
|
return True
|
|
1340
1674
|
o = ord(ch)
|
|
1341
1675
|
return 0xFE00 <= o <= 0xFE0F or 0xE0100 <= o <= 0xE01EF
|
|
@@ -1346,7 +1680,9 @@ def _has_visible(text):
|
|
|
1346
1680
|
return not all(_blank(ch) for ch in text)
|
|
1347
1681
|
|
|
1348
1682
|
|
|
1349
|
-
CACHE_READ_MAX =
|
|
1683
|
+
CACHE_READ_MAX = (
|
|
1684
|
+
64 * 1024
|
|
1685
|
+
) # bytes: a cached summary or decision is some hundred (120 characters), so this is generous
|
|
1350
1686
|
|
|
1351
1687
|
|
|
1352
1688
|
def _read_small(path, limit=CACHE_READ_MAX):
|
|
@@ -1354,10 +1690,16 @@ def _read_small(path, limit=CACHE_READ_MAX):
|
|
|
1354
1690
|
folder, a larger file). Opened without blocking and without following a link, so a pipe or a link left at a
|
|
1355
1691
|
cache place neither stops a run nor sends it to another file. Windows has neither flag: there the lstat
|
|
1356
1692
|
turns a link away, but a link swapped in between the lstat and the open is followed."""
|
|
1357
|
-
flags = (
|
|
1358
|
-
|
|
1693
|
+
flags = (
|
|
1694
|
+
os.O_RDONLY
|
|
1695
|
+
| getattr(os, "O_NONBLOCK", 0)
|
|
1696
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
1697
|
+
| getattr(os, "O_BINARY", 0)
|
|
1698
|
+
)
|
|
1359
1699
|
try:
|
|
1360
|
-
if not stat.S_ISREG(
|
|
1700
|
+
if not stat.S_ISREG(
|
|
1701
|
+
os.lstat(str(path)).st_mode
|
|
1702
|
+
): # a link (Windows has no O_NOFOLLOW), pipe, folder
|
|
1361
1703
|
return None
|
|
1362
1704
|
fd = os.open(str(path), flags)
|
|
1363
1705
|
with os.fdopen(fd, "rb") as f:
|
|
@@ -1391,8 +1733,13 @@ def _read_cached(p):
|
|
|
1391
1733
|
return None
|
|
1392
1734
|
|
|
1393
1735
|
|
|
1394
|
-
def read_or_summarize(
|
|
1395
|
-
|
|
1736
|
+
def read_or_summarize(
|
|
1737
|
+
turn: dict,
|
|
1738
|
+
session_id: str,
|
|
1739
|
+
rebuild: bool = False,
|
|
1740
|
+
redact_extra=None,
|
|
1741
|
+
redact_mode: str = "full",
|
|
1742
|
+
):
|
|
1396
1743
|
"""Return (summary, cache_hit). Cache key includes the content hash AND the
|
|
1397
1744
|
summarizer version (B-3), so changing the summarizer invalidates old entries.
|
|
1398
1745
|
Cached text is always redacted (secret hygiene). It is cut from the redacted text (3.0.2).
|
|
@@ -1400,10 +1747,12 @@ def read_or_summarize(turn: dict, session_id: str, rebuild: bool = False,
|
|
|
1400
1747
|
digest = hashlib.sha256(
|
|
1401
1748
|
(turn["text"] + "\x00s" + str(SUMMARIZER_VERSION)).encode()
|
|
1402
1749
|
).hexdigest()[:8]
|
|
1403
|
-
try:
|
|
1404
|
-
p = (
|
|
1405
|
-
|
|
1406
|
-
|
|
1750
|
+
try: # ids come from the transcript: keep them in the cache
|
|
1751
|
+
p = (
|
|
1752
|
+
cache_dir(_safe_name(turn.get("session") or session_id))
|
|
1753
|
+
/ f"{_safe_name(turn['uuid'])}-{digest}.md"
|
|
1754
|
+
)
|
|
1755
|
+
except OSError as e: # no writable cache: summarize anyway
|
|
1407
1756
|
_warn_once("_no_cache", f"cache unavailable ({e}); summaries are not cached")
|
|
1408
1757
|
p = None
|
|
1409
1758
|
cached = None if p is None or rebuild else _read_cached(p)
|
|
@@ -1432,7 +1781,8 @@ _SENTINEL = "\x00%d\x00"
|
|
|
1432
1781
|
_SENTINEL_RE = re.compile(r"\x00(\d+)\x00")
|
|
1433
1782
|
_MD_CARVE = re.compile(
|
|
1434
1783
|
r"`(?P<code>[^`\n]+)`"
|
|
1435
|
-
r"|\[(?P<label>[^\]\n]+)\]\((?P<url>https?://[^\s)]+)\)"
|
|
1784
|
+
r"|\[(?P<label>[^\]\n]+)\]\((?P<url>https?://[^\s)]+)\)"
|
|
1785
|
+
)
|
|
1436
1786
|
_MD_BOLD = re.compile(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", re.S)
|
|
1437
1787
|
_MD_DEL = re.compile(r"~~(?=\S)([^<]+?)(?<=\S)~~")
|
|
1438
1788
|
_MD_ITAL = re.compile(r"(?<![*\w])\*(?=\S)([^*<\n]+?)(?<=\S)\*(?![*\w])")
|
|
@@ -1455,7 +1805,8 @@ def _md_inline(escaped):
|
|
|
1455
1805
|
else:
|
|
1456
1806
|
stash.append(
|
|
1457
1807
|
'<a href="%s" target="_blank" rel="noopener noreferrer">%s</a>'
|
|
1458
|
-
% (m.group("url"), m.group("label"))
|
|
1808
|
+
% (m.group("url"), m.group("label"))
|
|
1809
|
+
)
|
|
1459
1810
|
return _SENTINEL % (len(stash) - 1)
|
|
1460
1811
|
|
|
1461
1812
|
s = _MD_CARVE.sub(carve, escaped)
|
|
@@ -1467,16 +1818,25 @@ def _md_inline(escaped):
|
|
|
1467
1818
|
|
|
1468
1819
|
def _is_block_start(lines, i):
|
|
1469
1820
|
line = lines[i]
|
|
1470
|
-
if (
|
|
1471
|
-
|
|
1821
|
+
if (
|
|
1822
|
+
_FENCE_RE.match(line)
|
|
1823
|
+
or _HEAD_RE.match(line)
|
|
1824
|
+
or _HR_RE.match(line)
|
|
1825
|
+
or _QUOTE_RE.match(line)
|
|
1826
|
+
or _LIST_RE.match(line)
|
|
1827
|
+
):
|
|
1472
1828
|
return True
|
|
1473
1829
|
return _is_table_start(lines, i)
|
|
1474
1830
|
|
|
1475
1831
|
|
|
1476
1832
|
def _is_table_start(lines, i):
|
|
1477
1833
|
"""Whether lines[i] starts a table: it has a pipe AND the next line is a delimiter row."""
|
|
1478
|
-
return bool(
|
|
1479
|
-
|
|
1834
|
+
return bool(
|
|
1835
|
+
"|" in lines[i]
|
|
1836
|
+
and i + 1 < len(lines)
|
|
1837
|
+
and "-" in lines[i + 1]
|
|
1838
|
+
and _TABLE_DELIM_RE.match(lines[i + 1])
|
|
1839
|
+
)
|
|
1480
1840
|
|
|
1481
1841
|
|
|
1482
1842
|
def _render_table(lines, i):
|
|
@@ -1488,11 +1848,12 @@ def _render_table(lines, i):
|
|
|
1488
1848
|
i += 1
|
|
1489
1849
|
thead = "".join("<th>%s</th>" % _md_inline(html.escape(c)) for c in header)
|
|
1490
1850
|
body = "".join(
|
|
1491
|
-
"<tr>%s</tr>" % "".join("<td>%s</td>" % _md_inline(html.escape(c))
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
return (
|
|
1495
|
-
|
|
1851
|
+
"<tr>%s</tr>" % "".join("<td>%s</td>" % _md_inline(html.escape(c)) for c in r)
|
|
1852
|
+
for r in rows
|
|
1853
|
+
)
|
|
1854
|
+
return (
|
|
1855
|
+
"<table><thead><tr>%s</tr></thead><tbody>%s</tbody></table>" % (thead, body)
|
|
1856
|
+
), i
|
|
1496
1857
|
|
|
1497
1858
|
|
|
1498
1859
|
def _md_fence(lines, i, fence):
|
|
@@ -1552,8 +1913,12 @@ def _md_block(lines, i, depth):
|
|
|
1552
1913
|
return _md_fence(lines, i, mf.group(1))
|
|
1553
1914
|
mh = _HEAD_RE.match(line)
|
|
1554
1915
|
if mh:
|
|
1555
|
-
hl = max(3, min(6, len(mh.group(1))))
|
|
1556
|
-
return "<h%d>%s</h%d>" % (
|
|
1916
|
+
hl = max(3, min(6, len(mh.group(1)))) # h3..h6 (avoid page-header clash)
|
|
1917
|
+
return "<h%d>%s</h%d>" % (
|
|
1918
|
+
hl,
|
|
1919
|
+
_md_inline(html.escape(mh.group(2).strip())),
|
|
1920
|
+
hl,
|
|
1921
|
+
), i + 1
|
|
1557
1922
|
if _HR_RE.match(line):
|
|
1558
1923
|
return "<hr>", i + 1
|
|
1559
1924
|
if _QUOTE_RE.match(line):
|
|
@@ -1778,8 +2143,13 @@ def _page_summary(t, summaries, session_id, rebuild, redact_extra, redact_mode):
|
|
|
1778
2143
|
rules' (cached)."""
|
|
1779
2144
|
if summaries is not None and t.get("uuid") in summaries:
|
|
1780
2145
|
return summaries[t["uuid"]], None
|
|
1781
|
-
return read_or_summarize(
|
|
1782
|
-
|
|
2146
|
+
return read_or_summarize(
|
|
2147
|
+
t,
|
|
2148
|
+
session_id,
|
|
2149
|
+
rebuild=rebuild,
|
|
2150
|
+
redact_extra=redact_extra,
|
|
2151
|
+
redact_mode=redact_mode,
|
|
2152
|
+
)
|
|
1783
2153
|
|
|
1784
2154
|
|
|
1785
2155
|
def _reviewed_fold(t, summaries):
|
|
@@ -1814,18 +2184,23 @@ def _pills(rs, t, all_sessions):
|
|
|
1814
2184
|
turn `t`."""
|
|
1815
2185
|
if all_sessions and t.get("session") != rs.last_session:
|
|
1816
2186
|
rs.last_session = t.get("session")
|
|
1817
|
-
name = html.escape(
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
2187
|
+
name = html.escape(
|
|
2188
|
+
rs.red(t.get("session_name") or rs.last_session or "session")
|
|
2189
|
+
)
|
|
2190
|
+
rs.rows.append(
|
|
2191
|
+
'<div class="day" data-mark="%s">'
|
|
2192
|
+
'<span class="pill pill-session">%s</span></div>' % (name, name)
|
|
2193
|
+
)
|
|
1821
2194
|
rs.last_date = None
|
|
1822
2195
|
ts = t.get("ts") or ""
|
|
1823
2196
|
if ts:
|
|
1824
2197
|
d = ts[:10]
|
|
1825
2198
|
if d != rs.last_date:
|
|
1826
|
-
rs.rows.append(
|
|
1827
|
-
|
|
1828
|
-
|
|
2199
|
+
rs.rows.append(
|
|
2200
|
+
'<div class="day" data-mark="%s">'
|
|
2201
|
+
'<span class="pill pill-date">%s</span></div>'
|
|
2202
|
+
% (html.escape(d), html.escape(d))
|
|
2203
|
+
)
|
|
1829
2204
|
rs.last_date = d
|
|
1830
2205
|
|
|
1831
2206
|
|
|
@@ -1836,24 +2211,31 @@ def _user_row(rs, t, summaries, time_hm):
|
|
|
1836
2211
|
reviewed = _reviewed_fold(t, summaries)
|
|
1837
2212
|
summary = html.escape(rs.red(reviewed) if reviewed else user_fold_summary(red))
|
|
1838
2213
|
body = render_body(red, markdown=rs.markdown)
|
|
1839
|
-
return (
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
2214
|
+
return (
|
|
2215
|
+
'<div class="row me"><span class="time">%s</span>'
|
|
2216
|
+
'<div class="bubble"><details%s>'
|
|
2217
|
+
'<summary><span class="sum">%s</span></summary>'
|
|
2218
|
+
'<div class="detail">%s</div></details></div></div>'
|
|
2219
|
+
% (time_hm, rs.open_attr, summary, body)
|
|
2220
|
+
)
|
|
2221
|
+
return (
|
|
2222
|
+
'<div class="row me"><span class="time">%s</span>'
|
|
2223
|
+
'<div class="bubble"><div class="plain">%s</div></div></div>'
|
|
2224
|
+
% (time_hm, render_body(red, markdown=rs.markdown))
|
|
2225
|
+
)
|
|
1847
2226
|
|
|
1848
2227
|
|
|
1849
2228
|
def _tools_line(tools, redact_extra):
|
|
1850
2229
|
"""The `🔧 도구 N건` line of a bot bubble ("" without tools), the --redact-extra keywords hidden in the names."""
|
|
1851
2230
|
if not tools:
|
|
1852
2231
|
return ""
|
|
1853
|
-
ts_list = ", ".join(
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
2232
|
+
ts_list = ", ".join(
|
|
2233
|
+
f"{_hide_keywords(k, redact_extra)}×{v}" for k, v in sorted(tools.items())
|
|
2234
|
+
)
|
|
2235
|
+
return '<div class="tools">🔧 도구 %d건: %s</div>' % (
|
|
2236
|
+
sum(tools.values()),
|
|
2237
|
+
html.escape(ts_list),
|
|
2238
|
+
)
|
|
1857
2239
|
|
|
1858
2240
|
|
|
1859
2241
|
def _bot_row(rs, t, summary, time_hm):
|
|
@@ -1864,32 +2246,59 @@ def _bot_row(rs, t, summary, time_hm):
|
|
|
1864
2246
|
role = t["role"]
|
|
1865
2247
|
if role == "agent":
|
|
1866
2248
|
who = html.escape(rs.red(t.get("agent_from") or "agent"))
|
|
1867
|
-
avatar, from_line, row_cls =
|
|
2249
|
+
avatar, from_line, row_cls = (
|
|
2250
|
+
"🤝",
|
|
2251
|
+
'<div class="from">%s</div>' % who,
|
|
2252
|
+
"row bot agent",
|
|
2253
|
+
)
|
|
1868
2254
|
else:
|
|
1869
2255
|
avatar, from_line, row_cls = "C", "", "row bot"
|
|
1870
|
-
return (
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
2256
|
+
return (
|
|
2257
|
+
'<div class="%s"><div class="avatar%s">%s</div>'
|
|
2258
|
+
'<div class="bubble"><details%s>'
|
|
2259
|
+
'<summary><span class="sum">%s</span></summary>'
|
|
2260
|
+
'<div class="detail">%s%s%s</div></details></div>'
|
|
2261
|
+
'<span class="time">%s</span></div>'
|
|
2262
|
+
% (
|
|
2263
|
+
row_cls,
|
|
2264
|
+
" agent" if role == "agent" else "",
|
|
2265
|
+
avatar,
|
|
2266
|
+
rs.open_attr,
|
|
2267
|
+
esc_sum,
|
|
2268
|
+
from_line,
|
|
2269
|
+
detail_body,
|
|
2270
|
+
tools_html,
|
|
2271
|
+
time_hm,
|
|
2272
|
+
)
|
|
2273
|
+
)
|
|
2274
|
+
|
|
2275
|
+
|
|
2276
|
+
def render_rows(
|
|
2277
|
+
turns,
|
|
2278
|
+
session_id,
|
|
2279
|
+
redact_extra,
|
|
2280
|
+
redact_mode,
|
|
2281
|
+
rebuild,
|
|
2282
|
+
open_details=False,
|
|
2283
|
+
markdown=True,
|
|
2284
|
+
all_sessions=False,
|
|
2285
|
+
summaries=None,
|
|
2286
|
+
):
|
|
1881
2287
|
rs = _Rows(redact_extra, redact_mode, open_details, markdown)
|
|
1882
2288
|
for t in turns:
|
|
1883
2289
|
_pills(rs, t, all_sessions)
|
|
1884
2290
|
time_hm = html.escape(_fmt_time(t.get("ts") or ""))
|
|
1885
2291
|
if t["role"] == "mark":
|
|
1886
|
-
rs.rows.append(
|
|
1887
|
-
|
|
2292
|
+
rs.rows.append(
|
|
2293
|
+
'<div class="day"><span class="pill pill-compact">'
|
|
2294
|
+
"⋯ 이전 대화 요약(compaction) ⋯</span></div>"
|
|
2295
|
+
)
|
|
1888
2296
|
elif t["role"] == "user":
|
|
1889
2297
|
rs.rows.append(_user_row(rs, t, summaries, time_hm))
|
|
1890
2298
|
else:
|
|
1891
|
-
summary, hit = _page_summary(
|
|
1892
|
-
|
|
2299
|
+
summary, hit = _page_summary(
|
|
2300
|
+
t, summaries, session_id, rebuild, redact_extra, redact_mode
|
|
2301
|
+
)
|
|
1893
2302
|
if hit is not None:
|
|
1894
2303
|
rs.total += 1
|
|
1895
2304
|
rs.hits += bool(hit)
|
|
@@ -1914,10 +2323,34 @@ def _residual_secrets(rows_html):
|
|
|
1914
2323
|
|
|
1915
2324
|
# ---------------------------------------------------------------- Self-verify (F-1)
|
|
1916
2325
|
_VOID = frozenset(
|
|
1917
|
-
"area base br col embed hr img input link meta param source track wbr".split()
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
2326
|
+
"area base br col embed hr img input link meta param source track wbr".split()
|
|
2327
|
+
)
|
|
2328
|
+
_VERIFY_TAGS = (
|
|
2329
|
+
"details",
|
|
2330
|
+
"div",
|
|
2331
|
+
"p",
|
|
2332
|
+
"ul",
|
|
2333
|
+
"ol",
|
|
2334
|
+
"li",
|
|
2335
|
+
"table",
|
|
2336
|
+
"thead",
|
|
2337
|
+
"tbody",
|
|
2338
|
+
"tr",
|
|
2339
|
+
"th",
|
|
2340
|
+
"td",
|
|
2341
|
+
"blockquote",
|
|
2342
|
+
"pre",
|
|
2343
|
+
"code",
|
|
2344
|
+
"strong",
|
|
2345
|
+
"em",
|
|
2346
|
+
"del",
|
|
2347
|
+
"a",
|
|
2348
|
+
"summary",
|
|
2349
|
+
"h3",
|
|
2350
|
+
"h4",
|
|
2351
|
+
"h5",
|
|
2352
|
+
"h6",
|
|
2353
|
+
)
|
|
1921
2354
|
|
|
1922
2355
|
|
|
1923
2356
|
class _BalanceParser(HTMLParser):
|
|
@@ -1962,8 +2395,12 @@ def self_verify(html_text: str):
|
|
|
1962
2395
|
errs = []
|
|
1963
2396
|
# strip script/style so their bodies aren't parsed as markup (avoids
|
|
1964
2397
|
# false <a> counts from inline JS like `top<a-8`)
|
|
1965
|
-
markup = re.sub(
|
|
1966
|
-
|
|
2398
|
+
markup = re.sub(
|
|
2399
|
+
r"<script\b[\s\S]*?</script>|<style\b[\s\S]*?</style>",
|
|
2400
|
+
"",
|
|
2401
|
+
html_text,
|
|
2402
|
+
flags=re.I,
|
|
2403
|
+
)
|
|
1967
2404
|
try:
|
|
1968
2405
|
HTMLParser().feed(html_text)
|
|
1969
2406
|
except Exception as e:
|
|
@@ -1987,9 +2424,11 @@ def self_verify(html_text: str):
|
|
|
1987
2424
|
errs.append("external script src found")
|
|
1988
2425
|
# Event-handler attributes are detected via the parser (real tag attrs),
|
|
1989
2426
|
# NOT a text regex — escaped content like `<img onerror=` is inert text.
|
|
1990
|
-
for marker, ph in (
|
|
1991
|
-
|
|
1992
|
-
|
|
2427
|
+
for marker, ph in (
|
|
2428
|
+
("<title>{{TITLE}}</title>", "{{TITLE}}"),
|
|
2429
|
+
("<h1>{{HEADER_TITLE}}</h1>", "{{HEADER_TITLE}}"),
|
|
2430
|
+
('id="now">{{DATE_RANGE}}</div>', "{{DATE_RANGE}}"),
|
|
2431
|
+
):
|
|
1993
2432
|
if marker in html_text:
|
|
1994
2433
|
errs.append(f"unsubstituted template placeholder: {ph}")
|
|
1995
2434
|
return errs
|
|
@@ -2028,10 +2467,14 @@ def _write_private(path, text):
|
|
|
2028
2467
|
is never followed, and it is removed when the write fails."""
|
|
2029
2468
|
path = pathlib.Path(path)
|
|
2030
2469
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
2031
|
-
fd, tmp = tempfile.mkstemp(
|
|
2470
|
+
fd, tmp = tempfile.mkstemp(
|
|
2471
|
+
dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp"
|
|
2472
|
+
)
|
|
2032
2473
|
try:
|
|
2033
2474
|
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f:
|
|
2034
|
-
f.write(
|
|
2475
|
+
f.write(
|
|
2476
|
+
_lone_to_fffd(text)
|
|
2477
|
+
) # a title or a file name may still bring one: UTF-8 cannot hold it
|
|
2035
2478
|
os.replace(tmp, str(path))
|
|
2036
2479
|
except BaseException:
|
|
2037
2480
|
try:
|
|
@@ -2045,7 +2488,9 @@ def _write_text_lf(path, text):
|
|
|
2045
2488
|
"""Write `text` to `path` as UTF-8 with LF line ends on every OS. Text mode on Windows
|
|
2046
2489
|
would write CRLF, so the same transcript would give a different page there."""
|
|
2047
2490
|
with open(str(path), "w", encoding="utf-8", newline="\n") as f:
|
|
2048
|
-
f.write(
|
|
2491
|
+
f.write(
|
|
2492
|
+
_lone_to_fffd(text)
|
|
2493
|
+
) # a title or a file name may still bring one: UTF-8 cannot hold it
|
|
2049
2494
|
|
|
2050
2495
|
|
|
2051
2496
|
def _safe_name(s):
|
|
@@ -2057,19 +2502,28 @@ def _safe_name(s):
|
|
|
2057
2502
|
def _decision_context(why, keeps):
|
|
2058
2503
|
"""What a cached decision rests on beside the text: the rules' call on the turn (a
|
|
2059
2504
|
`keep: null` defers to it) and the keep flags the reviewer was told to honour."""
|
|
2060
|
-
return "|".join(
|
|
2061
|
-
|
|
2505
|
+
return "|".join(
|
|
2506
|
+
[
|
|
2507
|
+
why or "",
|
|
2508
|
+
str(keeps.get("keep_trivia") is True),
|
|
2509
|
+
str(keeps.get("keep_tool_only") is True),
|
|
2510
|
+
]
|
|
2511
|
+
)
|
|
2062
2512
|
|
|
2063
2513
|
|
|
2064
2514
|
def _llm_digest(redacted_text, context):
|
|
2065
2515
|
"""The decision cache key: the text with the page's patterns only (as 3.0.1 keyed it, so
|
|
2066
2516
|
no turn is reviewed again for 3.0.2), the cache version and what the decision rests on."""
|
|
2067
|
-
return hashlib.sha256(
|
|
2068
|
-
|
|
2517
|
+
return hashlib.sha256(
|
|
2518
|
+
(redacted_text + "\x00llm" + str(LLM_CACHE_VERSION) + "\x00" + context).encode()
|
|
2519
|
+
).hexdigest()[:12]
|
|
2069
2520
|
|
|
2070
2521
|
|
|
2071
2522
|
def _llm_cache_path(turn_id, session, digest):
|
|
2072
|
-
return
|
|
2523
|
+
return (
|
|
2524
|
+
cache_dir(_safe_name(session))
|
|
2525
|
+
/ f"{_safe_name(turn_id)}-{_safe_name(digest)}-llm.json"
|
|
2526
|
+
)
|
|
2073
2527
|
|
|
2074
2528
|
|
|
2075
2529
|
def read_llm_cache(turn_id, session, digest):
|
|
@@ -2078,9 +2532,11 @@ def read_llm_cache(turn_id, session, digest):
|
|
|
2078
2532
|
null: the reviewer left the turn to the rules."""
|
|
2079
2533
|
try:
|
|
2080
2534
|
path = _llm_cache_path(turn_id, session, digest)
|
|
2081
|
-
except OSError as e:
|
|
2082
|
-
_warn_once(
|
|
2083
|
-
|
|
2535
|
+
except OSError as e: # no cache folder (a read-only HOME, say)
|
|
2536
|
+
_warn_once(
|
|
2537
|
+
"_no_llm_cache",
|
|
2538
|
+
f"reviewer decision cache unavailable ({e}); every turn is reviewed",
|
|
2539
|
+
)
|
|
2084
2540
|
return None
|
|
2085
2541
|
try:
|
|
2086
2542
|
got = json.loads(_read_text(path) or "")
|
|
@@ -2096,11 +2552,16 @@ def read_llm_cache(turn_id, session, digest):
|
|
|
2096
2552
|
|
|
2097
2553
|
def write_llm_cache(turn_id, session, digest, keep, summary):
|
|
2098
2554
|
try:
|
|
2099
|
-
_write_private(
|
|
2100
|
-
|
|
2555
|
+
_write_private(
|
|
2556
|
+
_llm_cache_path(turn_id, session, digest),
|
|
2557
|
+
json.dumps({"keep": keep, "summary": summary}, ensure_ascii=False),
|
|
2558
|
+
)
|
|
2101
2559
|
except OSError as e:
|
|
2102
|
-
_warn_once(
|
|
2103
|
-
|
|
2560
|
+
_warn_once(
|
|
2561
|
+
"_no_llm_cache_write",
|
|
2562
|
+
f"reviewer decisions not cached ({e}); the next "
|
|
2563
|
+
"run reviews this run's turns again (decisions cached before still apply)",
|
|
2564
|
+
)
|
|
2104
2565
|
|
|
2105
2566
|
|
|
2106
2567
|
def heuristic_drops(turns, args):
|
|
@@ -2139,8 +2600,11 @@ def _preview(text):
|
|
|
2139
2600
|
if len(text) <= PREVIEW_HEAD + PREVIEW_TAIL + 200:
|
|
2140
2601
|
return text, False
|
|
2141
2602
|
cut = len(text) - PREVIEW_HEAD - PREVIEW_TAIL
|
|
2142
|
-
return (
|
|
2143
|
-
|
|
2603
|
+
return (
|
|
2604
|
+
text[:PREVIEW_HEAD].rstrip()
|
|
2605
|
+
+ f"\n\n[중간 {cut}자 생략]\n\n"
|
|
2606
|
+
+ text[-PREVIEW_TAIL:].lstrip()
|
|
2607
|
+
), True
|
|
2144
2608
|
|
|
2145
2609
|
|
|
2146
2610
|
def _public_counts(counts):
|
|
@@ -2167,22 +2631,31 @@ def _rule_summary(t, red, args):
|
|
|
2167
2631
|
def _redacted_turn(t, extra):
|
|
2168
2632
|
"""`t` with its text and parts fully redacted, so a summary cut from it cannot cut a
|
|
2169
2633
|
secret in two and keep the half no pattern recognises."""
|
|
2634
|
+
|
|
2170
2635
|
def clean(s):
|
|
2171
2636
|
return review_redact(s, extra)[0]
|
|
2172
|
-
|
|
2637
|
+
|
|
2638
|
+
return dict(
|
|
2639
|
+
t, text=clean(t["text"]), parts=[clean(p) for p in t.get("parts") or []]
|
|
2640
|
+
)
|
|
2173
2641
|
|
|
2174
2642
|
|
|
2175
2643
|
def _review_view(t, red, args):
|
|
2176
2644
|
"""What a reviewer reads beside the preview: the rules' summary and the agent's
|
|
2177
2645
|
name, fully redacted (the page's copies may be masked)."""
|
|
2178
2646
|
if t["role"] in ("assistant", "agent"):
|
|
2179
|
-
summary = review_redact(
|
|
2180
|
-
|
|
2647
|
+
summary = review_redact(
|
|
2648
|
+
summarize_turn(_redacted_turn(t, args.redact_extra)), args.redact_extra
|
|
2649
|
+
)[0]
|
|
2181
2650
|
elif t["role"] == "user" and len(red) > USER_FOLD:
|
|
2182
2651
|
summary = user_fold_summary(review_redact(t["text"], args.redact_extra)[0])
|
|
2183
2652
|
else:
|
|
2184
2653
|
summary = None
|
|
2185
|
-
who =
|
|
2654
|
+
who = (
|
|
2655
|
+
review_redact(t["agent_from"], args.redact_extra)[0]
|
|
2656
|
+
if t.get("agent_from")
|
|
2657
|
+
else None
|
|
2658
|
+
)
|
|
2186
2659
|
return {"summary": summary, "agent_from": who}
|
|
2187
2660
|
|
|
2188
2661
|
|
|
@@ -2196,28 +2669,53 @@ def _pack_turn(n, t, tid, why, session_id, args, counts):
|
|
|
2196
2669
|
for k, v in found.items():
|
|
2197
2670
|
counts[k] = counts.get(k, 0) + v
|
|
2198
2671
|
hit = any(name in found for name, _ in REVIEW_SECRET_PATTERNS)
|
|
2199
|
-
key_text = (
|
|
2200
|
-
|
|
2672
|
+
key_text = (
|
|
2673
|
+
redact(
|
|
2674
|
+
_rejoin_secrets(t["text"], args.redact_extra),
|
|
2675
|
+
extra=args.redact_extra,
|
|
2676
|
+
mode=args.redact_mode,
|
|
2677
|
+
)[0]
|
|
2678
|
+
if hit
|
|
2679
|
+
else red
|
|
2680
|
+
)
|
|
2201
2681
|
preview, clipped = _preview(review_redact(t["text"], args.redact_extra)[0])
|
|
2202
2682
|
session = t.get("session") or session_id
|
|
2203
2683
|
context = _decision_context(why, _keeps(args))
|
|
2204
2684
|
llm_key = _llm_digest(key_text, context)
|
|
2205
2685
|
cached = None if args.rebuild_summaries else read_llm_cache(tid, session, llm_key)
|
|
2206
2686
|
view = _review_view(t, red, args)
|
|
2207
|
-
turn = {
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2687
|
+
turn = {
|
|
2688
|
+
"id": tid,
|
|
2689
|
+
"n": n,
|
|
2690
|
+
"role": t["role"],
|
|
2691
|
+
"ts": t.get("ts") or "",
|
|
2692
|
+
"tools": dict(t.get("tools") or {}),
|
|
2693
|
+
"text": red,
|
|
2694
|
+
"llm_key": llm_key,
|
|
2695
|
+
"redactions": _public_counts(found),
|
|
2696
|
+
"preview": preview,
|
|
2697
|
+
"clipped": clipped,
|
|
2698
|
+
"rule": {
|
|
2699
|
+
"keep": why is None,
|
|
2700
|
+
"why": why,
|
|
2701
|
+
"summary": _rule_summary(t, red, args),
|
|
2702
|
+
"review_summary": view["summary"],
|
|
2703
|
+
},
|
|
2704
|
+
"llm": dict(cached, cached=True) if cached else {"keep": None, "summary": None},
|
|
2705
|
+
}
|
|
2214
2706
|
if t.get("meta"):
|
|
2215
2707
|
turn["meta"] = True
|
|
2216
2708
|
if t["role"] == "user" and _bang_output(t["text"]):
|
|
2217
|
-
turn["bang_output"] =
|
|
2709
|
+
turn["bang_output"] = (
|
|
2710
|
+
True # for the dropped-turn WARN only: no part file carries it
|
|
2711
|
+
)
|
|
2218
2712
|
for k in ("session", "session_name", "agent_from"):
|
|
2219
2713
|
if t.get(k):
|
|
2220
|
-
turn[k] =
|
|
2714
|
+
turn[k] = (
|
|
2715
|
+
t[k]
|
|
2716
|
+
if k == "session"
|
|
2717
|
+
else page_redact(t[k], args.redact_extra, args.redact_mode)[0]
|
|
2718
|
+
)
|
|
2221
2719
|
return turn, view
|
|
2222
2720
|
|
|
2223
2721
|
|
|
@@ -2238,7 +2736,7 @@ def _split_even(items, size):
|
|
|
2238
2736
|
if not items:
|
|
2239
2737
|
return []
|
|
2240
2738
|
step = -(-len(items) // -(-len(items) // size))
|
|
2241
|
-
return [items[k:k + step] for k in range(0, len(items), step)]
|
|
2739
|
+
return [items[k : k + step] for k in range(0, len(items), step)]
|
|
2242
2740
|
|
|
2243
2741
|
|
|
2244
2742
|
def _decisions_name(part_path):
|
|
@@ -2253,8 +2751,12 @@ def _clear_parts(pack_path):
|
|
|
2253
2751
|
prefix = pack_path.stem + ".part-"
|
|
2254
2752
|
samples = f"{pack_path.stem}.reviewer-input.json"
|
|
2255
2753
|
try:
|
|
2256
|
-
olds = [
|
|
2257
|
-
|
|
2754
|
+
olds = [
|
|
2755
|
+
p
|
|
2756
|
+
for p in pack_path.parent.iterdir()
|
|
2757
|
+
if (p.name.startswith(prefix) and p.name.endswith(".json"))
|
|
2758
|
+
or p.name == samples
|
|
2759
|
+
]
|
|
2258
2760
|
except OSError:
|
|
2259
2761
|
return []
|
|
2260
2762
|
failed = []
|
|
@@ -2275,73 +2777,129 @@ def _hide_keywords(s, extra):
|
|
|
2275
2777
|
|
|
2276
2778
|
def _keeps(args):
|
|
2277
2779
|
"""The keep flags the reviewers must honour: with them, the rules keep those turns."""
|
|
2278
|
-
return {
|
|
2780
|
+
return {
|
|
2781
|
+
"keep_trivia": not args.drop_trivia,
|
|
2782
|
+
"keep_tool_only": not args.hide_tool_only,
|
|
2783
|
+
}
|
|
2279
2784
|
|
|
2280
2785
|
|
|
2281
2786
|
def _sheet(k, of, part_path, group, args):
|
|
2282
2787
|
"""Part file `k`: the turns one reviewer reads, without the page-only fields."""
|
|
2283
2788
|
turns = []
|
|
2284
2789
|
for x, view in group:
|
|
2285
|
-
y = {
|
|
2286
|
-
|
|
2287
|
-
|
|
2790
|
+
y = {
|
|
2791
|
+
key: v
|
|
2792
|
+
for key, v in x.items()
|
|
2793
|
+
if key
|
|
2794
|
+
not in (
|
|
2795
|
+
"text",
|
|
2796
|
+
"session",
|
|
2797
|
+
"session_name",
|
|
2798
|
+
"agent_from",
|
|
2799
|
+
"redactions",
|
|
2800
|
+
"llm_key",
|
|
2801
|
+
"bang_output",
|
|
2802
|
+
)
|
|
2803
|
+
}
|
|
2288
2804
|
y["rule"] = {k: v for k, v in x["rule"].items() if k != "review_summary"}
|
|
2289
2805
|
y["rule"]["summary"] = view["summary"]
|
|
2290
|
-
y["tools"] = {
|
|
2291
|
-
|
|
2806
|
+
y["tools"] = {
|
|
2807
|
+
_hide_keywords(name, args.redact_extra): n for name, n in x["tools"].items()
|
|
2808
|
+
}
|
|
2809
|
+
if x["llm"].get("summary"): # cached: redacted for the page, maybe masked
|
|
2292
2810
|
s = _MASKED.sub("[REDACTED]", x["llm"]["summary"])
|
|
2293
2811
|
y["llm"] = dict(x["llm"], summary=review_redact(s, args.redact_extra)[0])
|
|
2294
2812
|
if view["agent_from"]:
|
|
2295
2813
|
y["agent_from"] = view["agent_from"]
|
|
2296
2814
|
turns.append(y)
|
|
2297
|
-
return {
|
|
2298
|
-
|
|
2815
|
+
return {
|
|
2816
|
+
"schema": REVIEW_PART_SCHEMA,
|
|
2817
|
+
"part": k,
|
|
2818
|
+
"of": of,
|
|
2819
|
+
**_keeps(args),
|
|
2820
|
+
"decisions": _decisions_name(part_path),
|
|
2821
|
+
"turns": turns,
|
|
2822
|
+
}
|
|
2299
2823
|
|
|
2300
2824
|
|
|
2301
2825
|
def _pack_header(session_id, output, args, counts):
|
|
2302
2826
|
"""The pack's settings: what --apply-review renders with and the quality gate it runs."""
|
|
2303
|
-
return {
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2827
|
+
return {
|
|
2828
|
+
"schema": REVIEW_SCHEMA,
|
|
2829
|
+
"source": session_id,
|
|
2830
|
+
"output": str(output),
|
|
2831
|
+
"title": args.title,
|
|
2832
|
+
"markdown": args.markdown,
|
|
2833
|
+
"open": args.open_details,
|
|
2834
|
+
"redact_mode": args.redact_mode,
|
|
2835
|
+
"redact_extra": bool(args.redact_extra),
|
|
2836
|
+
"all_sessions": bool(args.all_sessions),
|
|
2837
|
+
"redactions": _public_counts(counts),
|
|
2838
|
+
**_keeps(args),
|
|
2839
|
+
"gate": {
|
|
2840
|
+
"skip_reviewer": bool(args.skip_reviewer),
|
|
2841
|
+
"reviewer_output": args.reviewer_output,
|
|
2842
|
+
"reviewer_timeout": args.reviewer_timeout,
|
|
2843
|
+
},
|
|
2844
|
+
}
|
|
2311
2845
|
|
|
2312
2846
|
|
|
2313
2847
|
def _report_emit(pack_path, items, paths, groups):
|
|
2314
2848
|
dropped = sum(1 for x in items if not x["rule"]["keep"])
|
|
2315
2849
|
cached = sum(1 for x in items if x["llm"].get("cached"))
|
|
2316
|
-
print(
|
|
2317
|
-
|
|
2850
|
+
print(
|
|
2851
|
+
f"[lineage] review pack: {pack_path} (turns={len(items)} "
|
|
2852
|
+
f"rule-dropped={dropped} cached={cached})",
|
|
2853
|
+
file=sys.stderr,
|
|
2854
|
+
)
|
|
2318
2855
|
for k, (p, g) in enumerate(zip(paths, groups), 1):
|
|
2319
2856
|
todo = sum(1 for x, _ in g if not x["llm"].get("cached"))
|
|
2320
|
-
print(
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2857
|
+
print(
|
|
2858
|
+
f"[lineage] part {k}/{len(groups)}: {p} (to review: {todo}) "
|
|
2859
|
+
f"-> {p.with_name(_decisions_name(p))}",
|
|
2860
|
+
file=sys.stderr,
|
|
2861
|
+
)
|
|
2862
|
+
print(
|
|
2863
|
+
f"[lineage] next: write each part's decisions, then --apply-review {pack_path}",
|
|
2864
|
+
file=sys.stderr,
|
|
2865
|
+
)
|
|
2324
2866
|
|
|
2325
2867
|
|
|
2326
2868
|
def emit_review(turns, drops, window, session_id, output, args):
|
|
2327
2869
|
"""Write the pack and one part file per reviewer; print them and the next step. A
|
|
2328
2870
|
--reviewer-output that holds something other than a verdict list stops it here, before
|
|
2329
2871
|
any reviewer runs: the gate would refuse it only after them."""
|
|
2330
|
-
rop =
|
|
2872
|
+
rop = (
|
|
2873
|
+
pathlib.Path(args.reviewer_output)
|
|
2874
|
+
if args.reviewer_output and not args.skip_reviewer
|
|
2875
|
+
else None
|
|
2876
|
+
)
|
|
2331
2877
|
if rop is not None and rop.exists() and not _is_verdict_file(rop):
|
|
2332
|
-
_bad(
|
|
2878
|
+
_bad(
|
|
2879
|
+
f"{rop} is there and is not a verdict list; give a --reviewer-output that does not exist yet"
|
|
2880
|
+
)
|
|
2333
2881
|
return 2
|
|
2334
2882
|
pack_path = pathlib.Path(args.emit_review)
|
|
2335
2883
|
counts, pairs = {}, []
|
|
2336
2884
|
for n, (i, tid) in enumerate(zip(window, _final_ids(turns, window)), 1):
|
|
2337
|
-
pairs.append(
|
|
2885
|
+
pairs.append(
|
|
2886
|
+
_pack_turn(n, turns[i], tid, drops.get(i), session_id, args, counts)
|
|
2887
|
+
)
|
|
2338
2888
|
groups = _split_even(pairs, REVIEW_PART)
|
|
2339
|
-
paths = [
|
|
2340
|
-
|
|
2889
|
+
paths = [
|
|
2890
|
+
pack_path.with_name(f"{pack_path.stem}.part-{k}.json")
|
|
2891
|
+
for k in range(1, len(groups) + 1)
|
|
2892
|
+
]
|
|
2341
2893
|
pack = _pack_header(session_id, _with_timestamp_suffix(output), args, counts)
|
|
2342
|
-
pack["parts"] = [
|
|
2343
|
-
|
|
2344
|
-
|
|
2894
|
+
pack["parts"] = [
|
|
2895
|
+
{
|
|
2896
|
+
"part": k,
|
|
2897
|
+
"file": p.name,
|
|
2898
|
+
"decisions": _decisions_name(p),
|
|
2899
|
+
"ids": [x["id"] for x, _ in g],
|
|
2900
|
+
}
|
|
2901
|
+
for k, (p, g) in enumerate(zip(paths, groups), 1)
|
|
2902
|
+
]
|
|
2345
2903
|
pack["turns"] = [x for x, _ in pairs]
|
|
2346
2904
|
stuck = _clear_parts(pack_path)
|
|
2347
2905
|
if stuck:
|
|
@@ -2350,8 +2908,12 @@ def emit_review(turns, drops, window, session_id, output, args):
|
|
|
2350
2908
|
try:
|
|
2351
2909
|
_write_private(pack_path, json.dumps(pack, ensure_ascii=False, indent=1))
|
|
2352
2910
|
for k, (p, g) in enumerate(zip(paths, groups), 1):
|
|
2353
|
-
_write_private(
|
|
2354
|
-
|
|
2911
|
+
_write_private(
|
|
2912
|
+
p,
|
|
2913
|
+
json.dumps(
|
|
2914
|
+
_sheet(k, len(groups), p, g, args), ensure_ascii=False, indent=1
|
|
2915
|
+
),
|
|
2916
|
+
)
|
|
2355
2917
|
except OSError as e:
|
|
2356
2918
|
_bad(f"cannot write the review pack: {e}")
|
|
2357
2919
|
return 2
|
|
@@ -2360,13 +2922,19 @@ def emit_review(turns, drops, window, session_id, output, args):
|
|
|
2360
2922
|
|
|
2361
2923
|
|
|
2362
2924
|
def _is_pack_turn(x):
|
|
2363
|
-
return (
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2925
|
+
return (
|
|
2926
|
+
isinstance(x, dict)
|
|
2927
|
+
and isinstance(x.get("id"), str)
|
|
2928
|
+
and x.get("role") in ("user", "assistant", "agent", "mark")
|
|
2929
|
+
and isinstance(x.get("text"), str)
|
|
2930
|
+
and isinstance(x.get("llm_key"), str)
|
|
2931
|
+
and isinstance(x.get("rule"), dict)
|
|
2932
|
+
and isinstance(x["rule"].get("keep"), bool)
|
|
2933
|
+
and _is_summary(x["rule"].get("summary"))
|
|
2934
|
+
and isinstance(x.get("llm"), dict)
|
|
2935
|
+
and _is_keep(x["llm"].get("keep"))
|
|
2936
|
+
and _is_summary(x["llm"].get("summary"))
|
|
2937
|
+
)
|
|
2370
2938
|
|
|
2371
2939
|
|
|
2372
2940
|
def _load_pack(path):
|
|
@@ -2375,9 +2943,13 @@ def _load_pack(path):
|
|
|
2375
2943
|
except (OSError, ValueError, UnicodeDecodeError, RecursionError) as e:
|
|
2376
2944
|
return _bad(f"cannot read review pack {path}: {e}")
|
|
2377
2945
|
if not isinstance(pack, dict) or pack.get("schema") != REVIEW_SCHEMA:
|
|
2378
|
-
return _bad(
|
|
2379
|
-
|
|
2380
|
-
|
|
2946
|
+
return _bad(
|
|
2947
|
+
f"{path} is not a {REVIEW_SCHEMA} pack (made by another lineage version? "
|
|
2948
|
+
"run --emit-review again)"
|
|
2949
|
+
)
|
|
2950
|
+
if not isinstance(pack.get("turns"), list) or not all(
|
|
2951
|
+
map(_is_pack_turn, pack["turns"])
|
|
2952
|
+
):
|
|
2381
2953
|
return _bad(f"{path}: malformed turns")
|
|
2382
2954
|
return pack
|
|
2383
2955
|
|
|
@@ -2385,8 +2957,11 @@ def _load_pack(path):
|
|
|
2385
2957
|
def _decision_files(pack_path, pack, explicit):
|
|
2386
2958
|
"""The parts' decisions files that exist beside the pack, then each --decisions.
|
|
2387
2959
|
Those beside the pack are made 0600: the session wrote them with its own umask."""
|
|
2388
|
-
beside = [
|
|
2389
|
-
|
|
2960
|
+
beside = [
|
|
2961
|
+
pack_path.with_name(p["decisions"])
|
|
2962
|
+
for p in pack.get("parts") or []
|
|
2963
|
+
if isinstance(p, dict) and isinstance(p.get("decisions"), str)
|
|
2964
|
+
]
|
|
2390
2965
|
files, seen = [], set()
|
|
2391
2966
|
for f in [f for f in beside if f.exists()] + [pathlib.Path(f) for f in explicit]:
|
|
2392
2967
|
if os.path.realpath(str(f)) not in seen:
|
|
@@ -2402,15 +2977,31 @@ def _decision_files(pack_path, pack, explicit):
|
|
|
2402
2977
|
|
|
2403
2978
|
# Keys of a part file's turns: a list of objects carrying one is the reviewer quoting its
|
|
2404
2979
|
# input, not its answer.
|
|
2405
|
-
_PART_TURN_KEYS = frozenset(
|
|
2406
|
-
|
|
2980
|
+
_PART_TURN_KEYS = frozenset(
|
|
2981
|
+
{
|
|
2982
|
+
"n",
|
|
2983
|
+
"role",
|
|
2984
|
+
"ts",
|
|
2985
|
+
"tools",
|
|
2986
|
+
"text",
|
|
2987
|
+
"preview",
|
|
2988
|
+
"clipped",
|
|
2989
|
+
"rule",
|
|
2990
|
+
"llm",
|
|
2991
|
+
"meta",
|
|
2992
|
+
"agent_from",
|
|
2993
|
+
}
|
|
2994
|
+
)
|
|
2407
2995
|
|
|
2408
2996
|
|
|
2409
2997
|
def _is_answer(d):
|
|
2410
2998
|
"""A decision, extra keys and all; a part turn quoted back (its keep and summary sit
|
|
2411
2999
|
under rule and llm, not on top) is not one."""
|
|
2412
|
-
return (
|
|
2413
|
-
|
|
3000
|
+
return (
|
|
3001
|
+
isinstance(d, dict)
|
|
3002
|
+
and "id" in d
|
|
3003
|
+
and ("keep" in d or "summary" in d or not _PART_TURN_KEYS & d.keys())
|
|
3004
|
+
)
|
|
2414
3005
|
|
|
2415
3006
|
|
|
2416
3007
|
def _decisions_text(text):
|
|
@@ -2429,8 +3020,10 @@ def _decisions_text(text):
|
|
|
2429
3020
|
# and only so many starts (each failed try parses up to the nesting limit)
|
|
2430
3021
|
for k, m in enumerate(re.finditer(r"\[(?=\s*[{\]])", text)):
|
|
2431
3022
|
if k == DECISIONS_TRIES:
|
|
2432
|
-
raise ValueError(
|
|
2433
|
-
|
|
3023
|
+
raise ValueError(
|
|
3024
|
+
f"more than {DECISIONS_TRIES} places a list may start; "
|
|
3025
|
+
"write the bare JSON array"
|
|
3026
|
+
)
|
|
2434
3027
|
try:
|
|
2435
3028
|
got, _ = decoder.raw_decode(text, m.start())
|
|
2436
3029
|
except (ValueError, RecursionError):
|
|
@@ -2448,8 +3041,12 @@ def _decisions_text(text):
|
|
|
2448
3041
|
|
|
2449
3042
|
def _decision_problem(d):
|
|
2450
3043
|
"""Why item `d` of a decisions file is not a decision, or ""."""
|
|
2451
|
-
if not (
|
|
2452
|
-
|
|
3044
|
+
if not (
|
|
3045
|
+
isinstance(d, dict)
|
|
3046
|
+
and isinstance(d.get("id"), str)
|
|
3047
|
+
and _is_keep(d.get("keep"))
|
|
3048
|
+
and _is_summary(d.get("summary"))
|
|
3049
|
+
):
|
|
2453
3050
|
return 'want {"id": str, "keep": true|false|null, "summary": str|null}'
|
|
2454
3051
|
if not _is_answer(d):
|
|
2455
3052
|
return "a turn of the part file, not a decision"
|
|
@@ -2474,13 +3071,20 @@ def _load_decisions(files):
|
|
|
2474
3071
|
why = _decision_problem(d)
|
|
2475
3072
|
if why:
|
|
2476
3073
|
return _bad(f"{f}[{k}]: {why}")
|
|
2477
|
-
cur = merged.setdefault(
|
|
3074
|
+
cur = merged.setdefault(
|
|
3075
|
+
_lone_to_fffd(d["id"]), {}
|
|
3076
|
+
) # a model's JSON may hold a lone surrogate
|
|
2478
3077
|
for key in ("keep", "summary"):
|
|
2479
3078
|
if d.get(key) is not None:
|
|
2480
3079
|
cur[key] = _lone_to_fffd(d[key])
|
|
2481
|
-
if items and all(
|
|
2482
|
-
|
|
2483
|
-
|
|
3080
|
+
if items and all(
|
|
3081
|
+
d.get("keep") is None and d.get("summary") is None for d in items
|
|
3082
|
+
):
|
|
3083
|
+
print(
|
|
3084
|
+
f"[lineage] WARN: {f}: all {len(items)} decisions are null; the rules "
|
|
3085
|
+
"decide those turns",
|
|
3086
|
+
file=sys.stderr,
|
|
3087
|
+
)
|
|
2484
3088
|
return merged
|
|
2485
3089
|
|
|
2486
3090
|
|
|
@@ -2505,9 +3109,11 @@ def _warn_coverage(pack, decided):
|
|
|
2505
3109
|
todo = [i for i in ids if i not in decided]
|
|
2506
3110
|
if todo:
|
|
2507
3111
|
more = ", ..." if len(todo) > 3 else ""
|
|
2508
|
-
print(
|
|
2509
|
-
|
|
2510
|
-
|
|
3112
|
+
print(
|
|
3113
|
+
f"[lineage] WARN: part {p.get('part')}: {len(todo)}/{len(ids)} turns "
|
|
3114
|
+
f"undecided ({', '.join(map(str, todo[:3]))}{more}); the rules decide them",
|
|
3115
|
+
file=sys.stderr,
|
|
3116
|
+
)
|
|
2511
3117
|
|
|
2512
3118
|
|
|
2513
3119
|
def _reviewer_summary(x, d, args, stats):
|
|
@@ -2519,8 +3125,11 @@ def _reviewer_summary(x, d, args, stats):
|
|
|
2519
3125
|
s, n = _clean_summary(raw, args.redact_extra, args.redact_mode)
|
|
2520
3126
|
if n:
|
|
2521
3127
|
stats["hidden"] += n
|
|
2522
|
-
print(
|
|
2523
|
-
|
|
3128
|
+
print(
|
|
3129
|
+
f"[lineage] WARN: the reviewer summary for {x['id']} held {n} "
|
|
3130
|
+
"secret-like string(s); redacted",
|
|
3131
|
+
file=sys.stderr,
|
|
3132
|
+
)
|
|
2524
3133
|
return s
|
|
2525
3134
|
|
|
2526
3135
|
|
|
@@ -2528,7 +3137,8 @@ def _show(x, llm_summary, turns, summaries, stats, prior):
|
|
|
2528
3137
|
"""Put kept pack turn `x` on the page with the summary it shows; `prior` adds up what
|
|
2529
3138
|
--emit-review redacted."""
|
|
2530
3139
|
shows = x["role"] in ("assistant", "agent") or (
|
|
2531
|
-
x["role"] == "user" and len(x["text"]) > USER_FOLD
|
|
3140
|
+
x["role"] == "user" and len(x["text"]) > USER_FOLD
|
|
3141
|
+
)
|
|
2532
3142
|
if llm_summary and not shows:
|
|
2533
3143
|
stats["ignored"] += 1
|
|
2534
3144
|
summary = (llm_summary if shows else None) or x["rule"].get("summary")
|
|
@@ -2541,12 +3151,21 @@ def _show(x, llm_summary, turns, summaries, stats, prior):
|
|
|
2541
3151
|
# what the gate's critic reads: the reviewer's summary, else the rules' cut from the
|
|
2542
3152
|
# fully redacted text (the page's may hold a --redact-mode mask: 4 + 4 characters of a value)
|
|
2543
3153
|
sample = (llm_summary if shows else None) or x["rule"].get("review_summary")
|
|
2544
|
-
turns.append(
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
3154
|
+
turns.append(
|
|
3155
|
+
{
|
|
3156
|
+
"uuid": x["id"],
|
|
3157
|
+
"role": x["role"],
|
|
3158
|
+
"text": x["text"],
|
|
3159
|
+
"ts": x.get("ts") or None,
|
|
3160
|
+
"tools": dict(x.get("tools") or {}),
|
|
3161
|
+
"parts": [],
|
|
3162
|
+
"session": x.get("session"),
|
|
3163
|
+
"session_name": x.get("session_name"),
|
|
3164
|
+
"agent_from": x.get("agent_from"),
|
|
3165
|
+
"preview": x.get("preview"),
|
|
3166
|
+
"sample_summary": sample,
|
|
3167
|
+
}
|
|
3168
|
+
)
|
|
2550
3169
|
|
|
2551
3170
|
|
|
2552
3171
|
def _kept_by_flag(pack, x):
|
|
@@ -2574,8 +3193,12 @@ def _typed_by_user(x):
|
|
|
2574
3193
|
"""Whether pack turn `x` holds words the user typed: a user turn the rules keep that is neither a body the
|
|
2575
3194
|
harness injected (meta) nor what Claude Code writes after a `!` command (bang_output, judged on the text before
|
|
2576
3195
|
redaction: a hidden value at the end of the output takes the closing tags with it)."""
|
|
2577
|
-
return (
|
|
2578
|
-
|
|
3196
|
+
return (
|
|
3197
|
+
x["role"] == "user"
|
|
3198
|
+
and x["rule"]["keep"]
|
|
3199
|
+
and not x.get("meta")
|
|
3200
|
+
and x.get("bang_output") is not True
|
|
3201
|
+
)
|
|
2579
3202
|
|
|
2580
3203
|
|
|
2581
3204
|
def _review_result(pack, merged, args):
|
|
@@ -2584,8 +3207,15 @@ def _review_result(pack, merged, args):
|
|
|
2584
3207
|
reviewer's value first, then the pack's cached one, then the rules'. A keep flag outranks
|
|
2585
3208
|
a reviewer's `false`; the cache still gets what was said."""
|
|
2586
3209
|
turns, summaries, writes, prior, typed = [], {}, [], {}, []
|
|
2587
|
-
stats = {
|
|
2588
|
-
|
|
3210
|
+
stats = {
|
|
3211
|
+
"restored": 0,
|
|
3212
|
+
"dropped": 0,
|
|
3213
|
+
"reviewer": 0,
|
|
3214
|
+
"rule": 0,
|
|
3215
|
+
"ignored": 0,
|
|
3216
|
+
"hidden": 0,
|
|
3217
|
+
"flag-kept": 0,
|
|
3218
|
+
}
|
|
2589
3219
|
for x in pack["turns"]:
|
|
2590
3220
|
d = merged.get(x["id"], {})
|
|
2591
3221
|
llm_keep = _first(d.get("keep"), x["llm"].get("keep"))
|
|
@@ -2593,11 +3223,18 @@ def _review_result(pack, merged, args):
|
|
|
2593
3223
|
stats["flag-kept"] += held
|
|
2594
3224
|
keep = bool(_first(None if held else llm_keep, x["rule"]["keep"]))
|
|
2595
3225
|
if not keep and _typed_by_user(x):
|
|
2596
|
-
typed.append(x["id"])
|
|
3226
|
+
typed.append(x["id"]) # typed by the user: the rules keep these
|
|
2597
3227
|
llm_summary = _reviewer_summary(x, d, args, stats)
|
|
2598
|
-
if x["id"] in merged:
|
|
2599
|
-
writes.append(
|
|
2600
|
-
|
|
3228
|
+
if x["id"] in merged: # reviewed, even if left to the rules
|
|
3229
|
+
writes.append(
|
|
3230
|
+
(
|
|
3231
|
+
x["id"],
|
|
3232
|
+
x.get("session") or pack.get("source"),
|
|
3233
|
+
x["llm_key"],
|
|
3234
|
+
llm_keep,
|
|
3235
|
+
llm_summary,
|
|
3236
|
+
)
|
|
3237
|
+
)
|
|
2601
3238
|
stats["restored"] += keep and not x["rule"]["keep"]
|
|
2602
3239
|
stats["dropped"] += x["rule"]["keep"] and not keep
|
|
2603
3240
|
if keep:
|
|
@@ -2620,31 +3257,42 @@ def _check_pack(pack, merged, args):
|
|
|
2620
3257
|
ids = {x["id"] for x in pack["turns"]}
|
|
2621
3258
|
unknown = [i for i in merged if i not in ids]
|
|
2622
3259
|
if unknown:
|
|
2623
|
-
print(
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
3260
|
+
print(
|
|
3261
|
+
f"[lineage] WARN: decisions name {len(unknown)} turn(s) not in the pack, "
|
|
3262
|
+
f"ignored: {', '.join(_shown_id(i) for i in unknown[:5])}",
|
|
3263
|
+
file=sys.stderr,
|
|
3264
|
+
)
|
|
3265
|
+
_warn_coverage(
|
|
3266
|
+
pack, set(merged) | {x["id"] for x in pack["turns"] if x["llm"].get("cached")}
|
|
3267
|
+
)
|
|
2627
3268
|
if pack.get("redact_mode") in ("full", "mask"):
|
|
2628
3269
|
args.redact_mode = pack["redact_mode"]
|
|
2629
3270
|
if pack.get("redact_extra") is True and not args.redact_extra:
|
|
2630
|
-
print(
|
|
2631
|
-
|
|
3271
|
+
print(
|
|
3272
|
+
"[lineage] WARN: --emit-review had --redact-extra; give the same keywords "
|
|
3273
|
+
"here too, or reviewer summaries miss that redaction",
|
|
3274
|
+
file=sys.stderr,
|
|
3275
|
+
)
|
|
2632
3276
|
_gate_from_pack(args, pack)
|
|
2633
3277
|
|
|
2634
3278
|
|
|
2635
3279
|
def _remove_review_files(pack_path, pack):
|
|
2636
3280
|
"""After a good render the pack, its parts and the decisions beside it go: they hold
|
|
2637
3281
|
the redacted session."""
|
|
2638
|
-
samples =
|
|
3282
|
+
samples = (
|
|
3283
|
+
f"{pack_path.stem}.reviewer-input.json" # a failed gate's, beside the pack
|
|
3284
|
+
)
|
|
2639
3285
|
names = [pack_path.name, samples]
|
|
2640
3286
|
for p in pack.get("parts") or []:
|
|
2641
3287
|
if isinstance(p, dict):
|
|
2642
|
-
names += [
|
|
3288
|
+
names += [
|
|
3289
|
+
p.get(k) for k in ("file", "decisions") if isinstance(p.get(k), str)
|
|
3290
|
+
]
|
|
2643
3291
|
prefix = pack_path.stem + ".part-"
|
|
2644
3292
|
left = []
|
|
2645
3293
|
for name in names:
|
|
2646
3294
|
if name not in (pack_path.name, samples) and not name.startswith(prefix):
|
|
2647
|
-
continue
|
|
3295
|
+
continue # only what --emit-review named
|
|
2648
3296
|
try:
|
|
2649
3297
|
pack_path.with_name(name).unlink()
|
|
2650
3298
|
except FileNotFoundError:
|
|
@@ -2652,8 +3300,10 @@ def _remove_review_files(pack_path, pack):
|
|
|
2652
3300
|
except (OSError, ValueError):
|
|
2653
3301
|
left.append(name)
|
|
2654
3302
|
if left:
|
|
2655
|
-
print(
|
|
2656
|
-
|
|
3303
|
+
print(
|
|
3304
|
+
f"[lineage] WARN: could not remove review files: {', '.join(left)}",
|
|
3305
|
+
file=sys.stderr,
|
|
3306
|
+
)
|
|
2657
3307
|
|
|
2658
3308
|
|
|
2659
3309
|
def _samples_left_unread(pack_path, args):
|
|
@@ -2661,7 +3311,9 @@ def _samples_left_unread(pack_path, args):
|
|
|
2661
3311
|
gives neither a verdict path nor --skip-reviewer: it would pass over that run's verdict."""
|
|
2662
3312
|
left = pack_path.with_name(f"{pack_path.stem}.reviewer-input.json")
|
|
2663
3313
|
if left.exists() and not (args.reviewer_output or args.skip_reviewer):
|
|
2664
|
-
_bad(
|
|
3314
|
+
_bad(
|
|
3315
|
+
f"{left} was left by a gated run; give its --reviewer-output again, or --skip-reviewer"
|
|
3316
|
+
)
|
|
2665
3317
|
return True
|
|
2666
3318
|
return False
|
|
2667
3319
|
|
|
@@ -2676,18 +3328,30 @@ def _review_output(args, pack):
|
|
|
2676
3328
|
|
|
2677
3329
|
def _report_review(stats, turns, pack, typed):
|
|
2678
3330
|
"""What the render took from the reviewers, and the turns typed by the user they dropped."""
|
|
2679
|
-
print(
|
|
2680
|
-
|
|
2681
|
-
|
|
3331
|
+
print(
|
|
3332
|
+
f"[lineage] review applied: shown={len(turns)}/{len(pack['turns'])} "
|
|
3333
|
+
f"restored={stats['restored']} dropped={stats['dropped']} "
|
|
3334
|
+
f"summaries reviewer={stats['reviewer']} rule={stats['rule']}",
|
|
3335
|
+
file=sys.stderr,
|
|
3336
|
+
)
|
|
2682
3337
|
if stats["ignored"]:
|
|
2683
|
-
print(
|
|
2684
|
-
|
|
3338
|
+
print(
|
|
3339
|
+
f"[lineage] note: {stats['ignored']} summaries for turns the page shows "
|
|
3340
|
+
"in full were ignored",
|
|
3341
|
+
file=sys.stderr,
|
|
3342
|
+
)
|
|
2685
3343
|
if stats["flag-kept"]:
|
|
2686
|
-
print(
|
|
2687
|
-
|
|
3344
|
+
print(
|
|
3345
|
+
f"[lineage] note: {stats['flag-kept']} turn(s) a reviewer marked keep: false "
|
|
3346
|
+
"stay, as --keep-trivia or --keep-tool-only asked",
|
|
3347
|
+
file=sys.stderr,
|
|
3348
|
+
)
|
|
2688
3349
|
if typed:
|
|
2689
|
-
print(
|
|
2690
|
-
|
|
3350
|
+
print(
|
|
3351
|
+
f"[lineage] WARN: reviewers dropped {len(typed)} typed user turn(s): {', '.join(_shown_id(i) for i in typed[:10])}"
|
|
3352
|
+
f"{' ...' if len(typed) > 10 else ''}; tell the user",
|
|
3353
|
+
file=sys.stderr,
|
|
3354
|
+
)
|
|
2691
3355
|
|
|
2692
3356
|
|
|
2693
3357
|
def apply_review(args):
|
|
@@ -2708,13 +3372,18 @@ def apply_review(args):
|
|
|
2708
3372
|
if stats["hidden"]:
|
|
2709
3373
|
prior["reviewer-summary"] = stats["hidden"]
|
|
2710
3374
|
try:
|
|
2711
|
-
rc = render_and_write(
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
3375
|
+
rc = render_and_write(
|
|
3376
|
+
turns,
|
|
3377
|
+
pack.get("source") or "session",
|
|
3378
|
+
args,
|
|
3379
|
+
all_sessions=bool(pack.get("all_sessions")),
|
|
3380
|
+
markdown=pack.get("markdown") is not False,
|
|
3381
|
+
open_details=bool(pack.get("open")),
|
|
3382
|
+
title=str(pack.get("title") or "Session Lineage"),
|
|
3383
|
+
summaries=summaries,
|
|
3384
|
+
prior_redactions=prior,
|
|
3385
|
+
)
|
|
3386
|
+
except SystemExit as e: # the quality gate failed
|
|
2718
3387
|
rc = int(e.code) if isinstance(e.code, int) else 2
|
|
2719
3388
|
if rc == 0:
|
|
2720
3389
|
for w in writes:
|
|
@@ -2727,66 +3396,133 @@ def apply_review(args):
|
|
|
2727
3396
|
def build_arg_parser() -> argparse.ArgumentParser:
|
|
2728
3397
|
ap = argparse.ArgumentParser(
|
|
2729
3398
|
prog="lineage",
|
|
2730
|
-
description="Export Claude Code session(s) as a KakaoTalk-style HTML file."
|
|
3399
|
+
description="Export Claude Code session(s) as a KakaoTalk-style HTML file.",
|
|
3400
|
+
)
|
|
2731
3401
|
ap.add_argument("--last", type=int, help="keep last N turns")
|
|
2732
3402
|
ap.add_argument("--from", dest="from_", help="ISO timestamp from (inclusive)")
|
|
2733
3403
|
ap.add_argument("--to", help="ISO timestamp to (inclusive)")
|
|
2734
3404
|
ap.add_argument("--turns", help='range like "10-50" (1-indexed inclusive)')
|
|
2735
|
-
ap.add_argument(
|
|
2736
|
-
|
|
3405
|
+
ap.add_argument(
|
|
3406
|
+
"--output",
|
|
3407
|
+
default=None,
|
|
3408
|
+
help="output HTML path (default: work/lineage-{name}_YYMMDD+HHMM.html)",
|
|
3409
|
+
)
|
|
2737
3410
|
ap.add_argument("--session", help="jsonl path (overrides auto-discover)")
|
|
2738
|
-
ap.add_argument(
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
3411
|
+
ap.add_argument(
|
|
3412
|
+
"--all-sessions",
|
|
3413
|
+
action="store_true",
|
|
3414
|
+
help="merge ALL sessions in the project dir, chronologically",
|
|
3415
|
+
)
|
|
3416
|
+
ap.add_argument(
|
|
3417
|
+
"--from-transcript", help="read transcript from PATH or '-' for stdin"
|
|
3418
|
+
)
|
|
2742
3419
|
ap.add_argument("--redact-extra", help="comma-separated keywords to also redact")
|
|
2743
|
-
ap.add_argument(
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
ap.add_argument(
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
3420
|
+
ap.add_argument(
|
|
3421
|
+
"--redact-mode",
|
|
3422
|
+
choices=["full", "mask"],
|
|
3423
|
+
default="full",
|
|
3424
|
+
help="redaction mode (full=[REDACTED], mask=abcd****wxyz)",
|
|
3425
|
+
)
|
|
3426
|
+
ap.add_argument(
|
|
3427
|
+
"--unsafe-schema",
|
|
3428
|
+
action="store_true",
|
|
3429
|
+
help="proceed even on unknown schema markers",
|
|
3430
|
+
)
|
|
3431
|
+
ap.add_argument(
|
|
3432
|
+
"--rebuild-summaries",
|
|
3433
|
+
action="store_true",
|
|
3434
|
+
help="ignore summary cache and re-summarize",
|
|
3435
|
+
)
|
|
3436
|
+
ap.add_argument(
|
|
3437
|
+
"--purge-cache", action="store_true", help="purge ~/.cache/lineage then exit"
|
|
3438
|
+
)
|
|
3439
|
+
ap.add_argument(
|
|
3440
|
+
"--skip-reviewer",
|
|
3441
|
+
action="store_true",
|
|
3442
|
+
help="skip reviewer quality gate (warn-only)",
|
|
3443
|
+
)
|
|
3444
|
+
ap.add_argument(
|
|
3445
|
+
"--reviewer-output",
|
|
3446
|
+
help="path to Critic response JSON; enforces the quality gate",
|
|
3447
|
+
)
|
|
3448
|
+
ap.add_argument(
|
|
3449
|
+
"--reviewer-timeout",
|
|
3450
|
+
type=int,
|
|
3451
|
+
default=None,
|
|
3452
|
+
help="seconds to wait for reviewer-output (default 60 with --rulebase; "
|
|
3453
|
+
"the reviewed flow waits only when this is given)",
|
|
3454
|
+
)
|
|
2758
3455
|
ap.add_argument("--title", default="Session Lineage", help="HTML title")
|
|
2759
3456
|
_add_review_args(ap)
|
|
2760
3457
|
# Readability defaults are ON. Opt-out flags restore raw/older behavior.
|
|
2761
|
-
ap.add_argument(
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
ap.add_argument(
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
3458
|
+
ap.add_argument(
|
|
3459
|
+
"--hide-tool-only",
|
|
3460
|
+
dest="hide_tool_only",
|
|
3461
|
+
action="store_true",
|
|
3462
|
+
help="(DEFAULT) drop assistant turns that are only tool calls",
|
|
3463
|
+
)
|
|
3464
|
+
ap.add_argument(
|
|
3465
|
+
"--keep-tool-only",
|
|
3466
|
+
dest="hide_tool_only",
|
|
3467
|
+
action="store_false",
|
|
3468
|
+
help="keep tool-only assistant turns (override default)",
|
|
3469
|
+
)
|
|
3470
|
+
ap.add_argument(
|
|
3471
|
+
"--no-markdown",
|
|
3472
|
+
dest="markdown",
|
|
3473
|
+
action="store_false",
|
|
3474
|
+
help="disable markdown rendering (default: render ON)",
|
|
3475
|
+
)
|
|
3476
|
+
ap.add_argument(
|
|
3477
|
+
"--keep-trivia",
|
|
3478
|
+
dest="drop_trivia",
|
|
3479
|
+
action="store_false",
|
|
3480
|
+
help="keep manipulation commands / injected bodies / echo / "
|
|
3481
|
+
"harness errors (default: filter ON)",
|
|
3482
|
+
)
|
|
3483
|
+
ap.add_argument(
|
|
3484
|
+
"--open",
|
|
3485
|
+
dest="open_details",
|
|
3486
|
+
action="store_true",
|
|
3487
|
+
help="render bubbles expanded (default: collapsed)",
|
|
3488
|
+
)
|
|
3489
|
+
ap.add_argument(
|
|
3490
|
+
"--collapse",
|
|
3491
|
+
dest="open_details",
|
|
3492
|
+
action="store_false",
|
|
3493
|
+
help="(DEFAULT) render bubbles collapsed",
|
|
3494
|
+
)
|
|
3495
|
+
ap.set_defaults(
|
|
3496
|
+
hide_tool_only=True, open_details=False, markdown=True, drop_trivia=True
|
|
3497
|
+
)
|
|
2776
3498
|
return ap
|
|
2777
3499
|
|
|
2778
3500
|
|
|
2779
3501
|
def _add_review_args(ap):
|
|
2780
3502
|
"""The flags of the reviewed flow (3.0.0): --emit-review, the session's reviewers,
|
|
2781
3503
|
then --apply-review."""
|
|
2782
|
-
ap.add_argument(
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
3504
|
+
ap.add_argument(
|
|
3505
|
+
"--rulebase",
|
|
3506
|
+
action="store_true",
|
|
3507
|
+
help="one pass, rules only (the default when no review flag is given)",
|
|
3508
|
+
)
|
|
3509
|
+
ap.add_argument(
|
|
3510
|
+
"--emit-review",
|
|
3511
|
+
metavar="PACK",
|
|
3512
|
+
help="write the turns for the session model to review (JSON) and stop",
|
|
3513
|
+
)
|
|
3514
|
+
ap.add_argument(
|
|
3515
|
+
"--apply-review",
|
|
3516
|
+
metavar="PACK",
|
|
3517
|
+
help="render from a review pack and the reviewer's decisions",
|
|
3518
|
+
)
|
|
3519
|
+
ap.add_argument(
|
|
3520
|
+
"--decisions",
|
|
3521
|
+
action="append",
|
|
3522
|
+
default=[],
|
|
3523
|
+
metavar="FILE",
|
|
3524
|
+
help="reviewer decisions [{id, keep, summary}] for --apply-review (repeatable)",
|
|
3525
|
+
)
|
|
2790
3526
|
|
|
2791
3527
|
|
|
2792
3528
|
def _open_stream(path):
|
|
@@ -2797,12 +3533,18 @@ def _load_turns_from(path, unsafe_schema, session_id, session_name):
|
|
|
2797
3533
|
"""Parse one jsonl file, isolating per-file decode errors (D-3)."""
|
|
2798
3534
|
try:
|
|
2799
3535
|
with _open_stream(path) as stream:
|
|
2800
|
-
return list(
|
|
2801
|
-
|
|
2802
|
-
|
|
3536
|
+
return list(
|
|
3537
|
+
parse_turns(
|
|
3538
|
+
stream,
|
|
3539
|
+
unsafe_schema=unsafe_schema,
|
|
3540
|
+
session_id=session_id,
|
|
3541
|
+
session_name=session_name,
|
|
3542
|
+
)
|
|
3543
|
+
)
|
|
2803
3544
|
except (OSError, UnicodeDecodeError) as e:
|
|
2804
|
-
print(
|
|
2805
|
-
|
|
3545
|
+
print(
|
|
3546
|
+
f"[lineage] WARN: skipping unreadable session {path}: {e}", file=sys.stderr
|
|
3547
|
+
)
|
|
2806
3548
|
return []
|
|
2807
3549
|
|
|
2808
3550
|
|
|
@@ -2825,12 +3567,12 @@ def _merge_env_extra(args):
|
|
|
2825
3567
|
"""LINEAGE_REDACT_EXTRA's keywords after --redact-extra's."""
|
|
2826
3568
|
env_extra = os.environ.get("LINEAGE_REDACT_EXTRA", "").strip()
|
|
2827
3569
|
if env_extra:
|
|
2828
|
-
args.redact_extra = ",".join(
|
|
2829
|
-
p for p in (args.redact_extra, env_extra) if p)
|
|
3570
|
+
args.redact_extra = ",".join(p for p in (args.redact_extra, env_extra) if p)
|
|
2830
3571
|
|
|
2831
3572
|
|
|
2832
3573
|
def _purge_cache():
|
|
2833
3574
|
import shutil
|
|
3575
|
+
|
|
2834
3576
|
if CACHE_BASE.exists():
|
|
2835
3577
|
shutil.rmtree(CACHE_BASE)
|
|
2836
3578
|
print(f"[lineage] cache purged: {CACHE_BASE}", file=sys.stderr)
|
|
@@ -2840,10 +3582,22 @@ def _purge_cache():
|
|
|
2840
3582
|
|
|
2841
3583
|
|
|
2842
3584
|
def _run_apply(args):
|
|
2843
|
-
if any(
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
3585
|
+
if any(
|
|
3586
|
+
(
|
|
3587
|
+
args.last,
|
|
3588
|
+
args.from_,
|
|
3589
|
+
args.to,
|
|
3590
|
+
args.turns,
|
|
3591
|
+
args.session,
|
|
3592
|
+
args.all_sessions,
|
|
3593
|
+
args.from_transcript,
|
|
3594
|
+
)
|
|
3595
|
+
):
|
|
3596
|
+
print(
|
|
3597
|
+
"[lineage] note: --apply-review renders the pack; source and range "
|
|
3598
|
+
"flags were applied at --emit-review",
|
|
3599
|
+
file=sys.stderr,
|
|
3600
|
+
)
|
|
2847
3601
|
return apply_review(args)
|
|
2848
3602
|
|
|
2849
3603
|
|
|
@@ -2851,18 +3605,25 @@ def _read_all_sessions(args):
|
|
|
2851
3605
|
"""Every session of the project folder, merged record by record in time order."""
|
|
2852
3606
|
files = project_jsonl_files()
|
|
2853
3607
|
if not files:
|
|
2854
|
-
print(
|
|
2855
|
-
|
|
3608
|
+
print(
|
|
3609
|
+
"[lineage] --all-sessions: no jsonl found in project dir", file=sys.stderr
|
|
3610
|
+
)
|
|
2856
3611
|
return 2, [], "session", None
|
|
2857
3612
|
starts = {p: session_start(p) for p in files}
|
|
2858
3613
|
turns = []
|
|
2859
3614
|
for p in files:
|
|
2860
|
-
name =
|
|
3615
|
+
name = (
|
|
3616
|
+
discover_session_name(p)
|
|
3617
|
+
or (p.stem[-36:] if p.stem.startswith("rollout-") else p.stem)[:8]
|
|
3618
|
+
)
|
|
2861
3619
|
part = _load_turns_from(p, args.unsafe_schema, p.stem, name)
|
|
2862
3620
|
meta = _codex_meta(p)
|
|
2863
3621
|
if meta is not None and not any(t["role"] == "user" for t in part):
|
|
2864
|
-
print(
|
|
2865
|
-
|
|
3622
|
+
print(
|
|
3623
|
+
f"[lineage] WARN: {p.name}: no user turn read from this Codex {meta.get('cli_version', '?')} "
|
|
3624
|
+
"record (a new record shape?); left out",
|
|
3625
|
+
file=sys.stderr,
|
|
3626
|
+
)
|
|
2866
3627
|
continue
|
|
2867
3628
|
fill_missing_ts(part)
|
|
2868
3629
|
for t in part:
|
|
@@ -2870,16 +3631,16 @@ def _read_all_sessions(args):
|
|
|
2870
3631
|
turns.extend(part)
|
|
2871
3632
|
# record-level chronological sort (stable → same-ts keeps file order)
|
|
2872
3633
|
turns.sort(key=lambda t: t.get("ts") or t.get("session_ts") or "9999")
|
|
2873
|
-
print(f"[lineage] --all-sessions: merged {len(files)} sessions",
|
|
2874
|
-
file=sys.stderr)
|
|
3634
|
+
print(f"[lineage] --all-sessions: merged {len(files)} sessions", file=sys.stderr)
|
|
2875
3635
|
return None, turns, "all-sessions", None
|
|
2876
3636
|
|
|
2877
3637
|
|
|
2878
3638
|
def _read_transcript(args):
|
|
2879
3639
|
"""A transcript pasted on stdin ('-') or read from a file."""
|
|
2880
3640
|
if args.from_transcript == "-":
|
|
2881
|
-
turns = list(
|
|
2882
|
-
|
|
3641
|
+
turns = list(
|
|
3642
|
+
parse_turns(sys.stdin, unsafe_schema=args.unsafe_schema, session_id="stdin")
|
|
3643
|
+
)
|
|
2883
3644
|
return None, turns, "stdin", None
|
|
2884
3645
|
jsonl_path = pathlib.Path(args.from_transcript)
|
|
2885
3646
|
session_id = "file-" + jsonl_path.stem
|
|
@@ -2891,13 +3652,15 @@ def _read_discovered(args):
|
|
|
2891
3652
|
"""The newest session of the project folder."""
|
|
2892
3653
|
f = auto_discover_jsonl()
|
|
2893
3654
|
if not f:
|
|
2894
|
-
print(
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
3655
|
+
print(
|
|
3656
|
+
"[lineage] No jsonl found. Use one of:",
|
|
3657
|
+
"\n --session FILE explicit jsonl path",
|
|
3658
|
+
"\n --all-sessions merge all project sessions",
|
|
3659
|
+
"\n --from-transcript - paste transcript via stdin",
|
|
3660
|
+
"\n --from-transcript FILE read transcript from file",
|
|
3661
|
+
"\n(non-ASCII cwd may not auto-encode — use --session)",
|
|
3662
|
+
file=sys.stderr,
|
|
3663
|
+
)
|
|
2901
3664
|
return 2, [], "session", None
|
|
2902
3665
|
turns = _load_turns_from(f, args.unsafe_schema, f.stem, None)
|
|
2903
3666
|
print(f"[lineage] auto-discovered session: {f.name}", file=sys.stderr)
|
|
@@ -2913,9 +3676,12 @@ def _read_input(args):
|
|
|
2913
3676
|
rc, turns, session_id, jsonl_path = _read_source(args)
|
|
2914
3677
|
meta = _codex_meta(jsonl_path) if rc is None and jsonl_path is not None else None
|
|
2915
3678
|
if meta is not None and not any(t["role"] == "user" for t in turns):
|
|
2916
|
-
print(
|
|
2917
|
-
|
|
2918
|
-
|
|
3679
|
+
print(
|
|
3680
|
+
f"[lineage] ERROR: {jsonl_path.name} is a Codex {meta.get('cli_version', '?')} session record, but no "
|
|
3681
|
+
"user turn was read from it; this Codex version may write its record in a new shape "
|
|
3682
|
+
"(nothing written; update banker or report it)",
|
|
3683
|
+
file=sys.stderr,
|
|
3684
|
+
)
|
|
2919
3685
|
return 2, [], session_id, None
|
|
2920
3686
|
return rc, turns, session_id, jsonl_path
|
|
2921
3687
|
|
|
@@ -2929,7 +3695,9 @@ def _read_source(args):
|
|
|
2929
3695
|
return _read_transcript(args)
|
|
2930
3696
|
if args.session:
|
|
2931
3697
|
jsonl_path = pathlib.Path(args.session)
|
|
2932
|
-
turns = _load_turns_from(
|
|
3698
|
+
turns = _load_turns_from(
|
|
3699
|
+
jsonl_path, args.unsafe_schema, jsonl_path.stem, None
|
|
3700
|
+
)
|
|
2933
3701
|
return None, turns, jsonl_path.stem, jsonl_path
|
|
2934
3702
|
return _read_discovered(args)
|
|
2935
3703
|
except SystemExit as e:
|
|
@@ -2940,7 +3708,9 @@ def _stdin_ids(turns):
|
|
|
2940
3708
|
"""stdin → derive per-turn uuid from the content so the caches work (keyed: see _stdin_id_key)."""
|
|
2941
3709
|
key = _stdin_id_key()
|
|
2942
3710
|
for t in turns:
|
|
2943
|
-
t["uuid"] = hmac.new(key, str(t["text"]).encode(), hashlib.sha256).hexdigest()[
|
|
3711
|
+
t["uuid"] = hmac.new(key, str(t["text"]).encode(), hashlib.sha256).hexdigest()[
|
|
3712
|
+
:16
|
|
3713
|
+
]
|
|
2944
3714
|
|
|
2945
3715
|
|
|
2946
3716
|
def _run_emit(turns, session_id, jsonl_path, args):
|
|
@@ -2948,8 +3718,14 @@ def _run_emit(turns, session_id, jsonl_path, args):
|
|
|
2948
3718
|
window = review_window(turns, drops, args)
|
|
2949
3719
|
if window is None:
|
|
2950
3720
|
return 2
|
|
2951
|
-
return emit_review(
|
|
2952
|
-
|
|
3721
|
+
return emit_review(
|
|
3722
|
+
turns,
|
|
3723
|
+
drops,
|
|
3724
|
+
window,
|
|
3725
|
+
session_id,
|
|
3726
|
+
default_output(args, jsonl_path, session_id),
|
|
3727
|
+
args,
|
|
3728
|
+
)
|
|
2953
3729
|
|
|
2954
3730
|
|
|
2955
3731
|
def _rulebase_turns(turns, args):
|
|
@@ -2958,8 +3734,9 @@ def _rulebase_turns(turns, args):
|
|
|
2958
3734
|
if args.drop_trivia:
|
|
2959
3735
|
turns = drop_echo_exchanges(turns)
|
|
2960
3736
|
if args.hide_tool_only:
|
|
2961
|
-
turns = [
|
|
2962
|
-
|
|
3737
|
+
turns = [
|
|
3738
|
+
t for t in turns if not (t["role"] == "assistant" and not t["text"].strip())
|
|
3739
|
+
]
|
|
2963
3740
|
return select_range(turns, args)
|
|
2964
3741
|
|
|
2965
3742
|
|
|
@@ -2988,9 +3765,15 @@ def main(argv=None) -> int:
|
|
|
2988
3765
|
return 2
|
|
2989
3766
|
if args.output is None:
|
|
2990
3767
|
args.output = default_output(args, jsonl_path, session_id)
|
|
2991
|
-
return render_and_write(
|
|
2992
|
-
|
|
2993
|
-
|
|
3768
|
+
return render_and_write(
|
|
3769
|
+
turns,
|
|
3770
|
+
session_id,
|
|
3771
|
+
args,
|
|
3772
|
+
all_sessions=args.all_sessions,
|
|
3773
|
+
markdown=args.markdown,
|
|
3774
|
+
open_details=args.open_details,
|
|
3775
|
+
title=args.title,
|
|
3776
|
+
)
|
|
2994
3777
|
|
|
2995
3778
|
|
|
2996
3779
|
def _in_time(t, args):
|
|
@@ -3027,8 +3810,10 @@ def select_range(turns, args):
|
|
|
3027
3810
|
return None
|
|
3028
3811
|
turns = turns[sl]
|
|
3029
3812
|
if len(turns) > 100:
|
|
3030
|
-
print(
|
|
3031
|
-
|
|
3813
|
+
print(
|
|
3814
|
+
f"[lineage] WARN: {len(turns)} turns is large — render may be slow",
|
|
3815
|
+
file=sys.stderr,
|
|
3816
|
+
)
|
|
3032
3817
|
return turns
|
|
3033
3818
|
|
|
3034
3819
|
|
|
@@ -3039,11 +3824,13 @@ def default_output(args, jsonl_path, session_id):
|
|
|
3039
3824
|
return args.output
|
|
3040
3825
|
session_name = discover_session_name(jsonl_path) if jsonl_path else None
|
|
3041
3826
|
if session_name and page_redact(session_name)[0] != session_name:
|
|
3042
|
-
print(
|
|
3043
|
-
|
|
3827
|
+
print(
|
|
3828
|
+
f"[lineage] session name: {page_redact(session_name)[0]} "
|
|
3829
|
+
"(holds a secret-like string; the file is named by the session id)",
|
|
3830
|
+
file=sys.stderr,
|
|
3831
|
+
)
|
|
3044
3832
|
session_name = None
|
|
3045
|
-
slug =
|
|
3046
|
-
else (session_name or session_id[:8]))
|
|
3833
|
+
slug = "all-sessions" if args.all_sessions else (session_name or session_id[:8])
|
|
3047
3834
|
if session_name:
|
|
3048
3835
|
print(f"[lineage] session name: {session_name}", file=sys.stderr)
|
|
3049
3836
|
return f"work/lineage-{slug}.html"
|
|
@@ -3052,31 +3839,50 @@ def default_output(args, jsonl_path, session_id):
|
|
|
3052
3839
|
# The controls that embed, override or isolate text direction (U+202A..U+202E, U+2066..U+2069). In a page they
|
|
3053
3840
|
# reorder what a reader sees and can reach past the element that holds them, so the page shows each as a visible
|
|
3054
3841
|
# [U+XXXX] instead. Plain letters of right-to-left scripts, and the direction marks, stay.
|
|
3055
|
-
_BIDI_CONTROLS = re.compile(
|
|
3842
|
+
_BIDI_CONTROLS = re.compile(
|
|
3843
|
+
"[" + "".join(map(chr, (*range(0x202A, 0x202F), *range(0x2066, 0x206A)))) + "]"
|
|
3844
|
+
)
|
|
3056
3845
|
|
|
3057
3846
|
|
|
3058
3847
|
def _show_bidi_controls(text):
|
|
3059
3848
|
return _BIDI_CONTROLS.sub(lambda m: "[U+%04X]" % ord(m.group()), text)
|
|
3060
3849
|
|
|
3061
3850
|
|
|
3062
|
-
def render_and_write(
|
|
3063
|
-
|
|
3064
|
-
|
|
3851
|
+
def render_and_write(
|
|
3852
|
+
turns,
|
|
3853
|
+
session_id,
|
|
3854
|
+
args,
|
|
3855
|
+
all_sessions=False,
|
|
3856
|
+
markdown=True,
|
|
3857
|
+
open_details=False,
|
|
3858
|
+
title="Session Lineage",
|
|
3859
|
+
summaries=None,
|
|
3860
|
+
prior_redactions=None,
|
|
3861
|
+
):
|
|
3065
3862
|
"""Render the rows, fill the template, self-verify, write, run the reviewer gate.
|
|
3066
3863
|
`prior_redactions`: what --emit-review redacted before the text reached the pack."""
|
|
3067
3864
|
rows, redact_counts, hits, total = render_rows(
|
|
3068
|
-
turns,
|
|
3069
|
-
|
|
3070
|
-
|
|
3865
|
+
turns,
|
|
3866
|
+
session_id,
|
|
3867
|
+
args.redact_extra,
|
|
3868
|
+
args.redact_mode,
|
|
3869
|
+
args.rebuild_summaries,
|
|
3870
|
+
open_details=open_details,
|
|
3871
|
+
markdown=markdown,
|
|
3872
|
+
all_sessions=all_sessions,
|
|
3873
|
+
summaries=summaries,
|
|
3874
|
+
)
|
|
3071
3875
|
for k, v in (prior_redactions or {}).items():
|
|
3072
3876
|
redact_counts[k] = redact_counts.get(k, 0) + v
|
|
3073
3877
|
left = _residual_secrets("\n".join(rows))
|
|
3074
3878
|
if left:
|
|
3075
|
-
print(
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3879
|
+
print(
|
|
3880
|
+
"[lineage] WARN: the page still holds secret-like text ("
|
|
3881
|
+
+ ", ".join(f"{k}={v}" for k, v in sorted(left.items()))
|
|
3882
|
+
+ "); check the page before sharing it, or run again with the value in LINEAGE_REDACT_EXTRA"
|
|
3883
|
+
" (with --rulebase, add --rebuild-summaries)",
|
|
3884
|
+
file=sys.stderr,
|
|
3885
|
+
)
|
|
3080
3886
|
|
|
3081
3887
|
date_range = ""
|
|
3082
3888
|
dated = [t for t in turns if t.get("ts")]
|
|
@@ -3087,8 +3893,12 @@ def render_and_write(turns, session_id, args, all_sessions=False, markdown=True,
|
|
|
3087
3893
|
|
|
3088
3894
|
html_doc = HTML_TEMPLATE
|
|
3089
3895
|
html_doc = html_doc.replace("{{TITLE}}", html.escape(_show_bidi_controls(title)), 1)
|
|
3090
|
-
html_doc = html_doc.replace(
|
|
3091
|
-
|
|
3896
|
+
html_doc = html_doc.replace(
|
|
3897
|
+
"{{HEADER_TITLE}}", html.escape(_show_bidi_controls(title)), 1
|
|
3898
|
+
)
|
|
3899
|
+
html_doc = html_doc.replace(
|
|
3900
|
+
"{{DATE_RANGE}}", html.escape(_show_bidi_controls(date_range)), 1
|
|
3901
|
+
)
|
|
3092
3902
|
html_doc = html_doc.replace("{{TURNS}}", _show_bidi_controls("\n".join(rows)), 1)
|
|
3093
3903
|
|
|
3094
3904
|
errs = self_verify(html_doc)
|
|
@@ -3102,9 +3912,12 @@ def render_and_write(turns, session_id, args, all_sessions=False, markdown=True,
|
|
|
3102
3912
|
_run_reviewer_gate(turns, out, session_id, args, summaries=summaries)
|
|
3103
3913
|
|
|
3104
3914
|
tool_total = sum(sum(t.get("tools", {}).values()) for t in turns)
|
|
3105
|
-
print(
|
|
3106
|
-
|
|
3107
|
-
|
|
3915
|
+
print(
|
|
3916
|
+
f"[lineage] turns={len(turns)} redacted={sum(redact_counts.values())} "
|
|
3917
|
+
f"tool_calls={tool_total} cache_hits={hits}/{total} "
|
|
3918
|
+
f"size={out.stat().st_size}B output={out}",
|
|
3919
|
+
file=sys.stderr,
|
|
3920
|
+
)
|
|
3108
3921
|
if redact_counts:
|
|
3109
3922
|
details = ", ".join(f"{k}={v}" for k, v in sorted(redact_counts.items()))
|
|
3110
3923
|
print(f"[lineage] redaction breakdown: {details}", file=sys.stderr)
|
|
@@ -3124,6 +3937,7 @@ def _with_pack_stamp(output, pack_output):
|
|
|
3124
3937
|
def _with_timestamp_suffix(path_str):
|
|
3125
3938
|
"""Append `_YYMMDD+HHMM` before the extension (idempotent)."""
|
|
3126
3939
|
import datetime as _dt
|
|
3940
|
+
|
|
3127
3941
|
p = pathlib.Path(path_str)
|
|
3128
3942
|
if re.search(r"_\d{6}\+\d{4}$", p.stem):
|
|
3129
3943
|
return p
|
|
@@ -3138,11 +3952,13 @@ def _gate_samples(bots, summaries, session_id, args):
|
|
|
3138
3952
|
seeded by the turns, so a rerun on the same turns samples the same ones, and a sample
|
|
3139
3953
|
names its turn and shows the head and tail the reviewer saw, fully redacted."""
|
|
3140
3954
|
import random
|
|
3955
|
+
|
|
3141
3956
|
if summaries is None:
|
|
3142
3957
|
out = []
|
|
3143
3958
|
for i, t in enumerate(random.sample(bots, min(5, len(bots)))):
|
|
3144
|
-
s, _ = _page_summary(
|
|
3145
|
-
|
|
3959
|
+
s, _ = _page_summary(
|
|
3960
|
+
t, summaries, session_id, False, args.redact_extra, args.redact_mode
|
|
3961
|
+
)
|
|
3146
3962
|
red_s = review_redact(_MASKED.sub("[REDACTED]", s), args.redact_extra)[0]
|
|
3147
3963
|
red_d = review_redact(t["text"], args.redact_extra)[0][:500]
|
|
3148
3964
|
out.append({"idx": i, "original_detail": red_d, "generated_summary": red_s})
|
|
@@ -3150,13 +3966,29 @@ def _gate_samples(bots, summaries, session_id, args):
|
|
|
3150
3966
|
seed = hashlib.sha256("\x00".join(t["uuid"] for t in bots).encode()).hexdigest()
|
|
3151
3967
|
out = []
|
|
3152
3968
|
for i, t in enumerate(random.Random(seed).sample(bots, min(5, len(bots)))):
|
|
3153
|
-
s =
|
|
3154
|
-
|
|
3155
|
-
|
|
3969
|
+
s = (
|
|
3970
|
+
t.get("sample_summary")
|
|
3971
|
+
or _page_summary(
|
|
3972
|
+
t, summaries, session_id, False, args.redact_extra, args.redact_mode
|
|
3973
|
+
)[0]
|
|
3974
|
+
)
|
|
3975
|
+
detail = (
|
|
3976
|
+
t.get("preview")
|
|
3977
|
+
or _preview(review_redact(t["text"], args.redact_extra)[0])[0]
|
|
3978
|
+
)
|
|
3156
3979
|
gen = review_redact(_MASKED.sub("[REDACTED]", s), args.redact_extra)[0]
|
|
3157
|
-
key = hashlib.sha256(
|
|
3158
|
-
|
|
3159
|
-
|
|
3980
|
+
key = hashlib.sha256(
|
|
3981
|
+
"\x00".join([t["uuid"], detail, gen]).encode()
|
|
3982
|
+
).hexdigest()[:12]
|
|
3983
|
+
out.append(
|
|
3984
|
+
{
|
|
3985
|
+
"idx": i,
|
|
3986
|
+
"id": t["uuid"],
|
|
3987
|
+
"key": key,
|
|
3988
|
+
"original_detail": detail,
|
|
3989
|
+
"generated_summary": gen,
|
|
3990
|
+
}
|
|
3991
|
+
)
|
|
3160
3992
|
return out
|
|
3161
3993
|
|
|
3162
3994
|
|
|
@@ -3179,8 +4011,10 @@ def _renew_samples(path, text, rop):
|
|
|
3179
4011
|
_write_private(path, text)
|
|
3180
4012
|
old = pathlib.Path(rop) if rop else None
|
|
3181
4013
|
if old is not None and _is_verdict_file(old):
|
|
3182
|
-
print(
|
|
3183
|
-
|
|
4014
|
+
print(
|
|
4015
|
+
f"[lineage] WARN: {rop} predates these samples; it is not read",
|
|
4016
|
+
file=sys.stderr,
|
|
4017
|
+
)
|
|
3184
4018
|
_set_aside(old)
|
|
3185
4019
|
|
|
3186
4020
|
|
|
@@ -3191,21 +4025,30 @@ def _refuse_non_verdict(rop):
|
|
|
3191
4025
|
p = pathlib.Path(rop) if rop else None
|
|
3192
4026
|
if p is None or not p.exists() or _is_verdict_file(p):
|
|
3193
4027
|
return
|
|
3194
|
-
if _json_list(p):
|
|
3195
|
-
why = (
|
|
3196
|
-
|
|
4028
|
+
if _json_list(p): # step 8: written again, the same answer fails the same way
|
|
4029
|
+
why = (
|
|
4030
|
+
"it is a JSON array where no entry names idx; if the session wrote the critic's "
|
|
4031
|
+
"answer, have the critic judge the samples again (a failed gate run)"
|
|
4032
|
+
)
|
|
3197
4033
|
else:
|
|
3198
|
-
why = (
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
4034
|
+
why = (
|
|
4035
|
+
"if the session wrote it, write it again as a bare JSON array of "
|
|
4036
|
+
"{idx, id, key, recoverable, reason}"
|
|
4037
|
+
)
|
|
4038
|
+
print(
|
|
4039
|
+
f"[lineage] ERROR: {rop} is not a verdict list; {why}, else give a --reviewer-output "
|
|
4040
|
+
"that does not exist yet",
|
|
4041
|
+
file=sys.stderr,
|
|
4042
|
+
)
|
|
3202
4043
|
raise SystemExit(2)
|
|
3203
4044
|
|
|
3204
4045
|
|
|
3205
4046
|
def _json_list(p):
|
|
3206
4047
|
"""True when `p` is a file holding a JSON array, whatever its entries."""
|
|
3207
4048
|
try:
|
|
3208
|
-
return p.is_file() and isinstance(
|
|
4049
|
+
return p.is_file() and isinstance(
|
|
4050
|
+
json.loads(p.read_text(encoding="utf-8")), list
|
|
4051
|
+
)
|
|
3209
4052
|
except (OSError, ValueError, UnicodeDecodeError, RecursionError):
|
|
3210
4053
|
return False
|
|
3211
4054
|
|
|
@@ -3238,8 +4081,9 @@ def _write_gate_samples(bots, out, summaries, session_id, args):
|
|
|
3238
4081
|
if summaries is not None and getattr(args, "apply_review", None):
|
|
3239
4082
|
pack = pathlib.Path(args.apply_review)
|
|
3240
4083
|
path = pack.with_name(f"{pack.stem}.reviewer-input.json")
|
|
3241
|
-
text = json.dumps(
|
|
3242
|
-
|
|
4084
|
+
text = json.dumps(
|
|
4085
|
+
_gate_samples(bots, summaries, session_id, args), ensure_ascii=False, indent=2
|
|
4086
|
+
)
|
|
3243
4087
|
try:
|
|
3244
4088
|
if summaries is None:
|
|
3245
4089
|
_write_private(path, text)
|
|
@@ -3247,20 +4091,35 @@ def _write_gate_samples(bots, out, summaries, session_id, args):
|
|
|
3247
4091
|
_renew_samples(path, text, args.reviewer_output)
|
|
3248
4092
|
except OSError as e:
|
|
3249
4093
|
if summaries is not None:
|
|
3250
|
-
print(
|
|
4094
|
+
print(
|
|
4095
|
+
f"[lineage] ERROR: cannot write the gate samples {path}: {e}",
|
|
4096
|
+
file=sys.stderr,
|
|
4097
|
+
)
|
|
3251
4098
|
raise SystemExit(2)
|
|
3252
4099
|
return None
|
|
3253
4100
|
print(f"[lineage] reviewer samples: {path}", file=sys.stderr, flush=True)
|
|
3254
4101
|
if summaries is None:
|
|
3255
|
-
print(
|
|
3256
|
-
|
|
3257
|
-
|
|
4102
|
+
print(
|
|
4103
|
+
"[lineage] next: invoke Skill('oh-my-claudecode:critic') with "
|
|
4104
|
+
"the JSON above; expected [{idx, recoverable, reason}, ...] "
|
|
4105
|
+
"(PASS = 5/5 recoverable)",
|
|
4106
|
+
file=sys.stderr,
|
|
4107
|
+
flush=True,
|
|
4108
|
+
)
|
|
3258
4109
|
elif pathlib.Path(args.reviewer_output).exists():
|
|
3259
|
-
print(
|
|
4110
|
+
print(
|
|
4111
|
+
f"[lineage] reading the verdict at {args.reviewer_output}",
|
|
4112
|
+
file=sys.stderr,
|
|
4113
|
+
flush=True,
|
|
4114
|
+
)
|
|
3260
4115
|
else:
|
|
3261
|
-
print(
|
|
3262
|
-
|
|
3263
|
-
|
|
4116
|
+
print(
|
|
4117
|
+
f"[lineage] next: have a critic agent judge the JSON above and write its "
|
|
4118
|
+
f"[{{idx, id, key, recoverable, reason}}, ...] to {args.reviewer_output} "
|
|
4119
|
+
"(one per sample; PASS = all recoverable)",
|
|
4120
|
+
file=sys.stderr,
|
|
4121
|
+
flush=True,
|
|
4122
|
+
)
|
|
3264
4123
|
return path
|
|
3265
4124
|
|
|
3266
4125
|
|
|
@@ -3270,26 +4129,32 @@ def _read_verdict(rop, timeout, set_aside):
|
|
|
3270
4129
|
so a rerun waits for a fresh one instead of reusing it; anything else stays where it is.
|
|
3271
4130
|
Exits 2 when it is missing, unreadable or not a non-empty list."""
|
|
3272
4131
|
import time as _time
|
|
4132
|
+
|
|
3273
4133
|
deadline = _time.time() + (0 if timeout is None else max(1, timeout))
|
|
3274
4134
|
while not rop.exists() and _time.time() < deadline:
|
|
3275
4135
|
_time.sleep(1)
|
|
3276
4136
|
if not rop.exists():
|
|
3277
4137
|
if timeout is None:
|
|
3278
|
-
print(
|
|
3279
|
-
|
|
4138
|
+
print(
|
|
4139
|
+
f"[lineage] ERROR: no verdict at {rop} yet; have a critic judge the samples, "
|
|
4140
|
+
"write its answer there and run this again",
|
|
4141
|
+
file=sys.stderr,
|
|
4142
|
+
)
|
|
3280
4143
|
else:
|
|
3281
|
-
print(
|
|
3282
|
-
|
|
4144
|
+
print(
|
|
4145
|
+
f"[lineage] ERROR: reviewer-output not found within {timeout}s: {rop}",
|
|
4146
|
+
file=sys.stderr,
|
|
4147
|
+
)
|
|
3283
4148
|
raise SystemExit(2)
|
|
3284
4149
|
try:
|
|
3285
4150
|
verdict = json.loads(rop.read_text(encoding="utf-8"))
|
|
3286
4151
|
except (json.JSONDecodeError, OSError, UnicodeDecodeError, RecursionError) as e:
|
|
3287
|
-
print(f"[lineage] ERROR: reviewer-output parse failed: {e}",
|
|
3288
|
-
file=sys.stderr)
|
|
4152
|
+
print(f"[lineage] ERROR: reviewer-output parse failed: {e}", file=sys.stderr)
|
|
3289
4153
|
raise SystemExit(2)
|
|
3290
4154
|
if not isinstance(verdict, list) or not verdict:
|
|
3291
|
-
print(
|
|
3292
|
-
|
|
4155
|
+
print(
|
|
4156
|
+
"[lineage] ERROR: reviewer-output is not a non-empty array", file=sys.stderr
|
|
4157
|
+
)
|
|
3293
4158
|
raise SystemExit(2)
|
|
3294
4159
|
if set_aside:
|
|
3295
4160
|
_set_aside(rop)
|
|
@@ -3312,8 +4177,11 @@ def _set_aside(rop):
|
|
|
3312
4177
|
try:
|
|
3313
4178
|
os.replace(str(rop), str(dest))
|
|
3314
4179
|
except OSError as e:
|
|
3315
|
-
print(
|
|
3316
|
-
|
|
4180
|
+
print(
|
|
4181
|
+
f"[lineage] WARN: could not set the verdict aside ({e}); remove {rop} "
|
|
4182
|
+
"before the next run",
|
|
4183
|
+
file=sys.stderr,
|
|
4184
|
+
)
|
|
3317
4185
|
return
|
|
3318
4186
|
print(f"[lineage] note: verdict moved to {dest}", file=sys.stderr)
|
|
3319
4187
|
|
|
@@ -3322,12 +4190,20 @@ def _verdict_gaps(verdict, samples):
|
|
|
3322
4190
|
"""Why a verdict does not answer these samples, or "": it needs one entry per sample
|
|
3323
4191
|
idx and no other, each carrying the `id` and `key` of that sample (its turn and its
|
|
3324
4192
|
content: an answer to the samples from before a fix carries an old key)."""
|
|
3325
|
-
want = {
|
|
4193
|
+
want = {
|
|
4194
|
+
s["idx"]: s
|
|
4195
|
+
for s in samples
|
|
4196
|
+
if isinstance(s, dict) and type(s.get("idx")) is int
|
|
4197
|
+
}
|
|
3326
4198
|
seen, stray = set(), []
|
|
3327
4199
|
for v in verdict:
|
|
3328
4200
|
idx = v.get("idx") if isinstance(v, dict) else None
|
|
3329
|
-
if (
|
|
3330
|
-
|
|
4201
|
+
if (
|
|
4202
|
+
type(idx) is not int
|
|
4203
|
+
or idx not in want
|
|
4204
|
+
or idx in seen
|
|
4205
|
+
or any(v.get(k) != want[idx][k] for k in ("id", "key") if k in want[idx])
|
|
4206
|
+
):
|
|
3331
4207
|
stray.append(idx)
|
|
3332
4208
|
else:
|
|
3333
4209
|
seen.add(idx)
|
|
@@ -3341,15 +4217,27 @@ def _check_coverage(verdict, samples):
|
|
|
3341
4217
|
"""Exit 2 unless the verdict answers each of the samples in file `samples`."""
|
|
3342
4218
|
try:
|
|
3343
4219
|
gaps = _verdict_gaps(verdict, json.loads(samples.read_text(encoding="utf-8")))
|
|
3344
|
-
except (
|
|
4220
|
+
except (
|
|
4221
|
+
OSError,
|
|
4222
|
+
ValueError,
|
|
4223
|
+
UnicodeDecodeError,
|
|
4224
|
+
AttributeError,
|
|
4225
|
+
RecursionError,
|
|
4226
|
+
) as e:
|
|
3345
4227
|
gaps = f"cannot read the samples: {e}"
|
|
3346
4228
|
if gaps:
|
|
3347
|
-
print(
|
|
3348
|
-
|
|
4229
|
+
print(
|
|
4230
|
+
f"[lineage] ERROR: the verdict does not answer {samples} ({gaps}); have the "
|
|
4231
|
+
"critic judge every sample again",
|
|
4232
|
+
file=sys.stderr,
|
|
4233
|
+
)
|
|
3349
4234
|
for v in verdict:
|
|
3350
4235
|
if isinstance(v, dict) and v.get("recoverable") is not True:
|
|
3351
|
-
print(
|
|
3352
|
-
|
|
4236
|
+
print(
|
|
4237
|
+
f" - not recoverable in it: idx={v.get('idx', '?')}: "
|
|
4238
|
+
f"{v.get('reason', '(no reason)')}",
|
|
4239
|
+
file=sys.stderr,
|
|
4240
|
+
)
|
|
3353
4241
|
raise SystemExit(2)
|
|
3354
4242
|
|
|
3355
4243
|
|
|
@@ -3360,18 +4248,26 @@ def _enforce_gate(rop, timeout, samples=None, reviewed=False):
|
|
|
3360
4248
|
verdict = _read_verdict(rop, timeout, set_aside=reviewed)
|
|
3361
4249
|
if reviewed and samples is not None:
|
|
3362
4250
|
_check_coverage(verdict, samples)
|
|
3363
|
-
fails = [
|
|
3364
|
-
|
|
4251
|
+
fails = [
|
|
4252
|
+
v for v in verdict if not (isinstance(v, dict) and v.get("recoverable") is True)
|
|
4253
|
+
]
|
|
3365
4254
|
if fails:
|
|
3366
|
-
print(
|
|
3367
|
-
|
|
4255
|
+
print(
|
|
4256
|
+
f"[lineage] FAIL: quality gate {len(verdict) - len(fails)}/"
|
|
4257
|
+
f"{len(verdict)} recoverable. Reasons:",
|
|
4258
|
+
file=sys.stderr,
|
|
4259
|
+
)
|
|
3368
4260
|
for f in fails:
|
|
3369
4261
|
if isinstance(f, dict):
|
|
3370
|
-
print(
|
|
3371
|
-
|
|
4262
|
+
print(
|
|
4263
|
+
f" - idx={f.get('idx', '?')}: {f.get('reason', '(no reason)')}",
|
|
4264
|
+
file=sys.stderr,
|
|
4265
|
+
)
|
|
3372
4266
|
raise SystemExit(2)
|
|
3373
|
-
print(
|
|
3374
|
-
|
|
4267
|
+
print(
|
|
4268
|
+
f"[lineage] PASS: quality gate {len(verdict)}/{len(verdict)} recoverable",
|
|
4269
|
+
file=sys.stderr,
|
|
4270
|
+
)
|
|
3375
4271
|
if reviewed and samples is not None:
|
|
3376
4272
|
try:
|
|
3377
4273
|
samples.unlink()
|
|
@@ -3387,17 +4283,23 @@ def _run_reviewer_gate(turns, out, session_id, args, summaries=None):
|
|
|
3387
4283
|
bots = [t for t in turns if t["role"] in ("assistant", "agent")]
|
|
3388
4284
|
reviewed = summaries is not None
|
|
3389
4285
|
if args.skip_reviewer:
|
|
3390
|
-
print(
|
|
3391
|
-
|
|
4286
|
+
print(
|
|
4287
|
+
"[lineage] WARN: --skip-reviewer — quality gate not enforced",
|
|
4288
|
+
file=sys.stderr,
|
|
4289
|
+
)
|
|
3392
4290
|
return
|
|
3393
4291
|
if not bots or (reviewed and not args.reviewer_output):
|
|
3394
4292
|
return
|
|
3395
4293
|
samples = _write_gate_samples(bots, out, summaries, session_id, args)
|
|
3396
4294
|
if args.reviewer_output:
|
|
3397
4295
|
if reviewed and args.reviewer_timeout is None:
|
|
3398
|
-
timeout = None
|
|
4296
|
+
timeout = None # the run after the critic reads the verdict; this one does not wait
|
|
3399
4297
|
else:
|
|
3400
|
-
timeout =
|
|
4298
|
+
timeout = (
|
|
4299
|
+
REVIEWER_TIMEOUT
|
|
4300
|
+
if args.reviewer_timeout is None
|
|
4301
|
+
else args.reviewer_timeout
|
|
4302
|
+
)
|
|
3401
4303
|
_enforce_gate(pathlib.Path(args.reviewer_output), timeout, samples, reviewed)
|
|
3402
4304
|
|
|
3403
4305
|
|