@brandry/claude-jsonl-compressor 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -56
- package/LICENSE +674 -674
- package/NOTICE +8 -8
- package/README.md +666 -611
- package/SKILL.md +230 -345
- package/agents/openai.yaml +7 -7
- package/bin/claude-jsonl-compressor.cjs +4 -4
- package/bin/claude-jsonl-repair-read-pages.cjs +4 -4
- package/bin/run-python.cjs +77 -77
- package/package.json +60 -60
- package/references/claude-jsonl-compression-format.md +635 -578
- package/scripts/claude_session_tools.py +304 -245
- package/scripts/compress_claude_jsonl.py +8608 -8275
- package/scripts/repair_claude_jsonl.py +627 -627
- package/templates/summary_template_en.md +78 -78
|
@@ -1,627 +1,627 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Byte-preserving, explicit compatibility repairs for one Claude JSONL."""
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import argparse
|
|
6
|
-
import collections
|
|
7
|
-
import copy
|
|
8
|
-
import json
|
|
9
|
-
import pathlib
|
|
10
|
-
import sys
|
|
11
|
-
from dataclasses import dataclass, field
|
|
12
|
-
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
13
|
-
|
|
14
|
-
SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
|
|
15
|
-
if str(SCRIPT_DIR) not in sys.path:
|
|
16
|
-
sys.path.insert(0, str(SCRIPT_DIR))
|
|
17
|
-
|
|
18
|
-
import compress_claude_jsonl as ccj
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
JsonObj = Dict[str, Any]
|
|
22
|
-
RULE_NAME = "remove-unsupported-read-pages"
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
@dataclass
|
|
26
|
-
class Member:
|
|
27
|
-
key: str
|
|
28
|
-
start: int
|
|
29
|
-
end: int
|
|
30
|
-
value: "Node"
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
@dataclass
|
|
34
|
-
class Node:
|
|
35
|
-
kind: str
|
|
36
|
-
start: int
|
|
37
|
-
end: int
|
|
38
|
-
members: List[Member] = field(default_factory=list)
|
|
39
|
-
items: List["Node"] = field(default_factory=list)
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
class SpanJsonParser:
|
|
43
|
-
def __init__(self, data: bytes):
|
|
44
|
-
self.data = data
|
|
45
|
-
self.pos = 0
|
|
46
|
-
|
|
47
|
-
def parse(self) -> Node:
|
|
48
|
-
self._ws()
|
|
49
|
-
node = self._value()
|
|
50
|
-
self._ws()
|
|
51
|
-
if self.pos != len(self.data):
|
|
52
|
-
raise ValueError(f"unexpected trailing JSON bytes at offset {self.pos}")
|
|
53
|
-
return node
|
|
54
|
-
|
|
55
|
-
def _ws(self) -> None:
|
|
56
|
-
while self.pos < len(self.data) and self.data[self.pos] in b" \t\r\n":
|
|
57
|
-
self.pos += 1
|
|
58
|
-
|
|
59
|
-
def _string(self) -> Tuple[str, int, int]:
|
|
60
|
-
start = self.pos
|
|
61
|
-
if self.pos >= len(self.data) or self.data[self.pos] != 0x22:
|
|
62
|
-
raise ValueError(f"expected JSON string at offset {self.pos}")
|
|
63
|
-
self.pos += 1
|
|
64
|
-
while self.pos < len(self.data):
|
|
65
|
-
byte = self.data[self.pos]
|
|
66
|
-
if byte == 0x5C:
|
|
67
|
-
self.pos += 2
|
|
68
|
-
continue
|
|
69
|
-
self.pos += 1
|
|
70
|
-
if byte == 0x22:
|
|
71
|
-
raw = self.data[start:self.pos].decode("utf-8", errors="strict")
|
|
72
|
-
value = json.loads(raw)
|
|
73
|
-
if not isinstance(value, str):
|
|
74
|
-
raise ValueError("parsed JSON key is not a string")
|
|
75
|
-
return value, start, self.pos
|
|
76
|
-
raise ValueError(f"unterminated JSON string at offset {start}")
|
|
77
|
-
|
|
78
|
-
def _value(self) -> Node:
|
|
79
|
-
self._ws()
|
|
80
|
-
if self.pos >= len(self.data):
|
|
81
|
-
raise ValueError("unexpected end of JSON")
|
|
82
|
-
byte = self.data[self.pos]
|
|
83
|
-
if byte == 0x7B:
|
|
84
|
-
return self._object()
|
|
85
|
-
if byte == 0x5B:
|
|
86
|
-
return self._array()
|
|
87
|
-
if byte == 0x22:
|
|
88
|
-
_value, start, end = self._string()
|
|
89
|
-
return Node("string", start, end)
|
|
90
|
-
start = self.pos
|
|
91
|
-
while self.pos < len(self.data) and self.data[self.pos] not in b",]} \t\r\n":
|
|
92
|
-
self.pos += 1
|
|
93
|
-
if self.pos == start:
|
|
94
|
-
raise ValueError(f"invalid JSON value at offset {start}")
|
|
95
|
-
json.loads(self.data[start:self.pos].decode("ascii", errors="strict"))
|
|
96
|
-
return Node("scalar", start, self.pos)
|
|
97
|
-
|
|
98
|
-
def _object(self) -> Node:
|
|
99
|
-
start = self.pos
|
|
100
|
-
self.pos += 1
|
|
101
|
-
members: List[Member] = []
|
|
102
|
-
self._ws()
|
|
103
|
-
if self.pos < len(self.data) and self.data[self.pos] == 0x7D:
|
|
104
|
-
self.pos += 1
|
|
105
|
-
return Node("object", start, self.pos, members=members)
|
|
106
|
-
while True:
|
|
107
|
-
self._ws()
|
|
108
|
-
key, key_start, _key_end = self._string()
|
|
109
|
-
self._ws()
|
|
110
|
-
if self.pos >= len(self.data) or self.data[self.pos] != 0x3A:
|
|
111
|
-
raise ValueError(f"expected ':' after object key at offset {self.pos}")
|
|
112
|
-
self.pos += 1
|
|
113
|
-
value = self._value()
|
|
114
|
-
members.append(Member(key, key_start, value.end, value))
|
|
115
|
-
self._ws()
|
|
116
|
-
if self.pos >= len(self.data):
|
|
117
|
-
raise ValueError("unterminated JSON object")
|
|
118
|
-
if self.data[self.pos] == 0x7D:
|
|
119
|
-
self.pos += 1
|
|
120
|
-
return Node("object", start, self.pos, members=members)
|
|
121
|
-
if self.data[self.pos] != 0x2C:
|
|
122
|
-
raise ValueError(f"expected ',' in object at offset {self.pos}")
|
|
123
|
-
self.pos += 1
|
|
124
|
-
|
|
125
|
-
def _array(self) -> Node:
|
|
126
|
-
start = self.pos
|
|
127
|
-
self.pos += 1
|
|
128
|
-
items: List[Node] = []
|
|
129
|
-
self._ws()
|
|
130
|
-
if self.pos < len(self.data) and self.data[self.pos] == 0x5D:
|
|
131
|
-
self.pos += 1
|
|
132
|
-
return Node("array", start, self.pos, items=items)
|
|
133
|
-
while True:
|
|
134
|
-
items.append(self._value())
|
|
135
|
-
self._ws()
|
|
136
|
-
if self.pos >= len(self.data):
|
|
137
|
-
raise ValueError("unterminated JSON array")
|
|
138
|
-
if self.data[self.pos] == 0x5D:
|
|
139
|
-
self.pos += 1
|
|
140
|
-
return Node("array", start, self.pos, items=items)
|
|
141
|
-
if self.data[self.pos] != 0x2C:
|
|
142
|
-
raise ValueError(f"expected ',' in array at offset {self.pos}")
|
|
143
|
-
self.pos += 1
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
def unique_member(node: Node, key: str) -> Optional[Member]:
|
|
147
|
-
matches = [member for member in node.members if member.key == key]
|
|
148
|
-
if len(matches) > 1:
|
|
149
|
-
raise ValueError(f"duplicate JSON key in repair control path: {key}")
|
|
150
|
-
return matches[0] if matches else None
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
def assert_no_duplicate_keys(node: Node) -> None:
|
|
154
|
-
if node.kind == "object":
|
|
155
|
-
counts = collections.Counter(member.key for member in node.members)
|
|
156
|
-
duplicates = sorted(key for key, count in counts.items() if count > 1)
|
|
157
|
-
if duplicates:
|
|
158
|
-
raise ValueError(f"duplicate JSON keys make byte repair ambiguous: {duplicates[:20]}")
|
|
159
|
-
for member in node.members:
|
|
160
|
-
assert_no_duplicate_keys(member.value)
|
|
161
|
-
elif node.kind == "array":
|
|
162
|
-
for item in node.items:
|
|
163
|
-
assert_no_duplicate_keys(item)
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
def member_deletion_span(object_node: Node, target: Member) -> Tuple[int, int]:
|
|
167
|
-
index = object_node.members.index(target)
|
|
168
|
-
if len(object_node.members) == 1:
|
|
169
|
-
return target.start, target.end
|
|
170
|
-
if index < len(object_node.members) - 1:
|
|
171
|
-
return target.start, object_node.members[index + 1].start
|
|
172
|
-
return object_node.members[index - 1].end, target.end
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
def _jsonl_record_spans_with_lines(data: bytes) -> List[Tuple[int, int, int, bytes]]:
|
|
176
|
-
spans: List[Tuple[int, int, int, bytes]] = []
|
|
177
|
-
offset = 0
|
|
178
|
-
physical_line = 1
|
|
179
|
-
while offset < len(data):
|
|
180
|
-
lf_index = data.find(b"\n", offset)
|
|
181
|
-
physical_end = len(data) if lf_index < 0 else lf_index
|
|
182
|
-
content_end = physical_end - 1 if physical_end > offset and data[physical_end - 1] == 0x0D else physical_end
|
|
183
|
-
start = offset
|
|
184
|
-
if start == 0 and data.startswith(b"\xef\xbb\xbf"):
|
|
185
|
-
start = 3
|
|
186
|
-
content = data[start:content_end]
|
|
187
|
-
if content.decode("utf-8", errors="strict").strip():
|
|
188
|
-
spans.append((physical_line, start, content_end, content))
|
|
189
|
-
if lf_index < 0:
|
|
190
|
-
break
|
|
191
|
-
offset = lf_index + 1
|
|
192
|
-
physical_line += 1
|
|
193
|
-
return spans
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
def jsonl_record_spans(data: bytes) -> List[Tuple[int, int, bytes]]:
|
|
197
|
-
return [(start, end, content) for _line, start, end, content in _jsonl_record_spans_with_lines(data)]
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
def _node_at_tool_input(root: Node, block_index: int) -> Tuple[Node, Member]:
|
|
201
|
-
message = unique_member(root, "message")
|
|
202
|
-
if message is None or message.value.kind != "object":
|
|
203
|
-
raise ValueError("matched assistant record has no object message node")
|
|
204
|
-
content = unique_member(message.value, "content")
|
|
205
|
-
if content is None or content.value.kind != "array" or block_index >= len(content.value.items):
|
|
206
|
-
raise ValueError("matched tool_use block has no content-array node")
|
|
207
|
-
block = content.value.items[block_index]
|
|
208
|
-
if block.kind != "object":
|
|
209
|
-
raise ValueError("matched tool_use block is not an object node")
|
|
210
|
-
input_member = unique_member(block, "input")
|
|
211
|
-
if input_member is None or input_member.value.kind != "object":
|
|
212
|
-
raise ValueError("matched Read tool_use has no object input node")
|
|
213
|
-
pages = unique_member(input_member.value, "pages")
|
|
214
|
-
if pages is None:
|
|
215
|
-
raise ValueError("matched Read tool_use pages key has no byte span")
|
|
216
|
-
return input_member.value, pages
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
def plan_read_pages_repairs(
|
|
220
|
-
source_bytes: bytes,
|
|
221
|
-
scope: str = "active-chain",
|
|
222
|
-
resume_leaf_override: Optional[str] = None,
|
|
223
|
-
) -> Dict[str, Any]:
|
|
224
|
-
records, _raw_lines, physical_lines = ccj.parse_jsonl_bytes_with_lines(
|
|
225
|
-
source_bytes,
|
|
226
|
-
source_label="SOURCE_JSONL",
|
|
227
|
-
)
|
|
228
|
-
line_spans = _jsonl_record_spans_with_lines(source_bytes)
|
|
229
|
-
if len(line_spans) != len(records):
|
|
230
|
-
raise ValueError("record/span count mismatch while planning byte repair")
|
|
231
|
-
if [line for line, _start, _end, _bytes in line_spans] != physical_lines:
|
|
232
|
-
raise ValueError("record/span physical line map mismatch while planning byte repair")
|
|
233
|
-
parsed_roots: List[Node] = []
|
|
234
|
-
for _physical_line, _line_start, _line_end, line_bytes in line_spans:
|
|
235
|
-
root = SpanJsonParser(line_bytes).parse()
|
|
236
|
-
assert_no_duplicate_keys(root)
|
|
237
|
-
parsed_roots.append(root)
|
|
238
|
-
topology: Optional[Dict[str, Any]] = None
|
|
239
|
-
if scope == "active-chain":
|
|
240
|
-
topology = ccj.require_resume_leaf_info(records, resume_leaf_override=resume_leaf_override)
|
|
241
|
-
scope_indexes = set(topology.get("activeChainIndexes") or [])
|
|
242
|
-
elif scope == "all":
|
|
243
|
-
scope_indexes = set(range(len(records)))
|
|
244
|
-
else:
|
|
245
|
-
raise ValueError(f"unknown repair scope: {scope}")
|
|
246
|
-
|
|
247
|
-
tool_use_occurrences: Dict[str, List[int]] = collections.defaultdict(list)
|
|
248
|
-
tool_result_occurrences: Dict[str, List[int]] = collections.defaultdict(list)
|
|
249
|
-
for record_index in sorted(scope_indexes):
|
|
250
|
-
for tool_id in ccj.tool_use_ids(records[record_index]):
|
|
251
|
-
tool_use_occurrences[tool_id].append(record_index)
|
|
252
|
-
for tool_id in ccj.tool_result_ids(records[record_index]):
|
|
253
|
-
tool_result_occurrences[tool_id].append(record_index)
|
|
254
|
-
patches: List[Dict[str, Any]] = []
|
|
255
|
-
matches: List[Dict[str, Any]] = []
|
|
256
|
-
seen_target_ids: set = set()
|
|
257
|
-
for record_index in sorted(scope_indexes):
|
|
258
|
-
obj = records[record_index]
|
|
259
|
-
if obj.get("type") != "assistant" or ccj.api_role(obj) != "assistant":
|
|
260
|
-
continue
|
|
261
|
-
message = obj.get("message")
|
|
262
|
-
content = message.get("content") if isinstance(message, dict) else None
|
|
263
|
-
if not isinstance(content, list):
|
|
264
|
-
continue
|
|
265
|
-
physical_line, line_start, _line_end, line_bytes = line_spans[record_index]
|
|
266
|
-
root_node = parsed_roots[record_index]
|
|
267
|
-
for block_index, block in enumerate(content):
|
|
268
|
-
if not isinstance(block, dict) or block.get("type") != "tool_use" or block.get("name") != "Read":
|
|
269
|
-
continue
|
|
270
|
-
tool_input = block.get("input")
|
|
271
|
-
if not isinstance(tool_input, dict) or "pages" not in tool_input:
|
|
272
|
-
continue
|
|
273
|
-
tool_id = block.get("id")
|
|
274
|
-
file_path = tool_input.get("file_path")
|
|
275
|
-
if isinstance(tool_id, str) and len(tool_use_occurrences.get(tool_id, [])) > 1:
|
|
276
|
-
raise ValueError(f"duplicate target tool_use id makes repair ambiguous: {tool_id}")
|
|
277
|
-
if isinstance(tool_id, str) and len(tool_result_occurrences.get(tool_id, [])) > 1:
|
|
278
|
-
raise ValueError(f"duplicate tool_result id makes repair ambiguous: {tool_id}")
|
|
279
|
-
later_results = [
|
|
280
|
-
result_index
|
|
281
|
-
for result_index in tool_result_occurrences.get(tool_id, [])
|
|
282
|
-
if result_index > record_index
|
|
283
|
-
] if isinstance(tool_id, str) else []
|
|
284
|
-
result_index = later_results[0] if len(later_results) == 1 else None
|
|
285
|
-
result_record = records[result_index] if isinstance(result_index, int) else None
|
|
286
|
-
use_session = obj.get("sessionId")
|
|
287
|
-
result_session = result_record.get("sessionId") if isinstance(result_record, dict) else None
|
|
288
|
-
same_session = (
|
|
289
|
-
isinstance(use_session, str)
|
|
290
|
-
and bool(use_session)
|
|
291
|
-
and isinstance(result_session, str)
|
|
292
|
-
and bool(result_session)
|
|
293
|
-
and use_session == result_session
|
|
294
|
-
)
|
|
295
|
-
assistant_uuid = obj.get("uuid")
|
|
296
|
-
source_uuid = ccj.source_tool_assistant_uuid(result_record) if isinstance(result_record, dict) else None
|
|
297
|
-
source_matches = (
|
|
298
|
-
isinstance(assistant_uuid, str)
|
|
299
|
-
and bool(assistant_uuid)
|
|
300
|
-
and source_uuid == assistant_uuid
|
|
301
|
-
)
|
|
302
|
-
paired = len(later_results) == 1 and same_session and source_matches
|
|
303
|
-
eligible = paired and isinstance(file_path, str) and bool(file_path)
|
|
304
|
-
if eligible:
|
|
305
|
-
reason = "eligible"
|
|
306
|
-
elif len(later_results) != 1:
|
|
307
|
-
reason = "pending-tool-result"
|
|
308
|
-
elif not same_session:
|
|
309
|
-
reason = "cross-session-tool-result"
|
|
310
|
-
elif not source_matches:
|
|
311
|
-
reason = "source-assistant-mismatch"
|
|
312
|
-
else:
|
|
313
|
-
reason = "missing-file-path"
|
|
314
|
-
match = {
|
|
315
|
-
"recordLine": physical_line,
|
|
316
|
-
"blockIndex": block_index,
|
|
317
|
-
"toolUseId": tool_id,
|
|
318
|
-
"pairedToolResult": paired,
|
|
319
|
-
"sameSession": same_session,
|
|
320
|
-
"sourceAssistantMatches": source_matches,
|
|
321
|
-
"eligible": eligible,
|
|
322
|
-
"reason": reason,
|
|
323
|
-
}
|
|
324
|
-
matches.append(match)
|
|
325
|
-
if not eligible:
|
|
326
|
-
continue
|
|
327
|
-
if tool_id in seen_target_ids:
|
|
328
|
-
raise ValueError(f"duplicate target tool_use id makes repair ambiguous: {tool_id}")
|
|
329
|
-
seen_target_ids.add(tool_id)
|
|
330
|
-
input_node, pages_member = _node_at_tool_input(root_node, block_index)
|
|
331
|
-
rel_start, rel_end = member_deletion_span(input_node, pages_member)
|
|
332
|
-
patches.append(
|
|
333
|
-
{
|
|
334
|
-
"start": line_start + rel_start,
|
|
335
|
-
"end": line_start + rel_end,
|
|
336
|
-
"recordLine": physical_line,
|
|
337
|
-
"blockIndex": block_index,
|
|
338
|
-
"toolUseId": tool_id,
|
|
339
|
-
}
|
|
340
|
-
)
|
|
341
|
-
ordered = sorted(patches, key=lambda item: (item["start"], item["end"]))
|
|
342
|
-
for left, right in zip(ordered, ordered[1:]):
|
|
343
|
-
if left["end"] > right["start"]:
|
|
344
|
-
raise ValueError("planned repair byte spans overlap")
|
|
345
|
-
return {
|
|
346
|
-
"rule": RULE_NAME,
|
|
347
|
-
"scope": scope,
|
|
348
|
-
"sourceSha256": ccj.sha256_hex(source_bytes),
|
|
349
|
-
"sourceBytes": len(source_bytes),
|
|
350
|
-
"recordCount": len(records),
|
|
351
|
-
"targetMatchCount": len(matches),
|
|
352
|
-
"patchableMatchCount": len(patches),
|
|
353
|
-
"pendingMatchCount": sum(1 for item in matches if item["reason"] == "pending-tool-result"),
|
|
354
|
-
"ineligibleMatchCount": sum(1 for item in matches if not item["eligible"]),
|
|
355
|
-
"matches": matches,
|
|
356
|
-
"patches": ordered,
|
|
357
|
-
"resumeTopology": ccj.public_resume_leaf_info(topology),
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
def apply_patch_plan(source_bytes: bytes, plan: Dict[str, Any]) -> bytes:
|
|
362
|
-
output = source_bytes
|
|
363
|
-
for item in reversed(plan.get("patches") or []):
|
|
364
|
-
start = int(item["start"])
|
|
365
|
-
end = int(item["end"])
|
|
366
|
-
if not (0 <= start < end <= len(output)):
|
|
367
|
-
raise ValueError(f"invalid repair span: {start}:{end}")
|
|
368
|
-
output = output[:start] + output[end:]
|
|
369
|
-
return output
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
def _uuid_parent_signature(records: Sequence[JsonObj]) -> List[Tuple[Any, Any, Any]]:
|
|
373
|
-
return [(obj.get("type"), obj.get("uuid"), obj.get("parentUuid")) for obj in records]
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
def validate_repair(source_bytes: bytes, output_bytes: bytes, plan: Dict[str, Any]) -> Dict[str, Any]:
|
|
377
|
-
errors: List[str] = []
|
|
378
|
-
expected = apply_patch_plan(source_bytes, plan)
|
|
379
|
-
if output_bytes != expected:
|
|
380
|
-
errors.append("output bytes differ outside the planned deletion spans")
|
|
381
|
-
source_records, _ = ccj.parse_jsonl_bytes(source_bytes, source_label="SOURCE_JSONL")
|
|
382
|
-
output_records, _ = ccj.parse_jsonl_bytes(output_bytes, source_label="REPAIRED_JSONL")
|
|
383
|
-
if len(source_records) != len(output_records):
|
|
384
|
-
errors.append("record count changed")
|
|
385
|
-
if _uuid_parent_signature(source_records) != _uuid_parent_signature(output_records):
|
|
386
|
-
errors.append("type/uuid/parentUuid signature changed")
|
|
387
|
-
source_tool_ids = [
|
|
388
|
-
(ccj.tool_use_ids(obj), ccj.tool_result_ids(obj)) for obj in source_records
|
|
389
|
-
]
|
|
390
|
-
output_tool_ids = [
|
|
391
|
-
(ccj.tool_use_ids(obj), ccj.tool_result_ids(obj)) for obj in output_records
|
|
392
|
-
]
|
|
393
|
-
if source_tool_ids != output_tool_ids:
|
|
394
|
-
errors.append("tool_use/tool_result identifier sequence changed")
|
|
395
|
-
second_plan = plan_read_pages_repairs(
|
|
396
|
-
output_bytes,
|
|
397
|
-
scope=str(plan.get("scope") or "active-chain"),
|
|
398
|
-
resume_leaf_override=(plan.get("resumeTopology") or {}).get("selectedLeafUuid")
|
|
399
|
-
if (plan.get("resumeTopology") or {}).get("manualOverride") else None,
|
|
400
|
-
)
|
|
401
|
-
if second_plan.get("patchableMatchCount") != 0:
|
|
402
|
-
errors.append("repair is not idempotent; a second pass still finds patchable matches")
|
|
403
|
-
return {
|
|
404
|
-
"ok": not errors,
|
|
405
|
-
"errors": errors,
|
|
406
|
-
"sourceSha256": ccj.sha256_hex(source_bytes),
|
|
407
|
-
"outputSha256": ccj.sha256_hex(output_bytes),
|
|
408
|
-
"sourceBytes": len(source_bytes),
|
|
409
|
-
"outputBytes": len(output_bytes),
|
|
410
|
-
"removedBytes": len(source_bytes) - len(output_bytes),
|
|
411
|
-
"recordCount": len(output_records),
|
|
412
|
-
"secondPassPatchableMatchCount": second_plan.get("patchableMatchCount"),
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
def public_plan(plan: Dict[str, Any]) -> Dict[str, Any]:
|
|
417
|
-
result = copy.deepcopy(plan)
|
|
418
|
-
result.pop("patches", None)
|
|
419
|
-
return result
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
def publish_repair_candidate(
|
|
423
|
-
path: pathlib.Path,
|
|
424
|
-
source_bytes: bytes,
|
|
425
|
-
output_bytes: bytes,
|
|
426
|
-
plan: Dict[str, Any],
|
|
427
|
-
) -> Dict[str, Any]:
|
|
428
|
-
if ccj.is_under_claude_root(path):
|
|
429
|
-
raise ValueError("repair candidates and reports must be outside the entire .claude directory")
|
|
430
|
-
expected_sha256 = ccj.sha256_hex(output_bytes)
|
|
431
|
-
previous_bytes = path.read_bytes() if path.exists() else None
|
|
432
|
-
try:
|
|
433
|
-
ccj.atomic_write_bytes(path, output_bytes)
|
|
434
|
-
published_bytes = path.read_bytes()
|
|
435
|
-
if published_bytes != output_bytes or ccj.sha256_hex(published_bytes) != expected_sha256:
|
|
436
|
-
raise RuntimeError("published repair candidate bytes differ from the validated repair snapshot")
|
|
437
|
-
validation = validate_repair(source_bytes, published_bytes, plan)
|
|
438
|
-
if not validation.get("ok"):
|
|
439
|
-
raise ValueError(f"published repair candidate validation failed: {validation.get('errors')}")
|
|
440
|
-
full_validation = ccj.validate_jsonl_bytes(published_bytes, source_label=path.name)
|
|
441
|
-
validation["fullTranscriptValidation"] = full_validation
|
|
442
|
-
if not full_validation.get("ok"):
|
|
443
|
-
raise ValueError(
|
|
444
|
-
"published repair candidate full transcript validation failed: "
|
|
445
|
-
f"{full_validation.get('errors')}"
|
|
446
|
-
)
|
|
447
|
-
return validation
|
|
448
|
-
except Exception:
|
|
449
|
-
if previous_bytes is None:
|
|
450
|
-
try:
|
|
451
|
-
path.unlink()
|
|
452
|
-
ccj.fsync_parent_directory(path)
|
|
453
|
-
except FileNotFoundError:
|
|
454
|
-
pass
|
|
455
|
-
else:
|
|
456
|
-
ccj.atomic_write_bytes(path, previous_bytes)
|
|
457
|
-
raise
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
|
461
|
-
parser = argparse.ArgumentParser(
|
|
462
|
-
prog="claude-jsonl-repair-read-pages",
|
|
463
|
-
description="Scan or byte-preservingly remove historical Read.input.pages members from one Claude Code JSONL.",
|
|
464
|
-
)
|
|
465
|
-
parser.add_argument("--version", action="store_true", help="Print package, engine, and report versions")
|
|
466
|
-
parser.add_argument("--input", type=pathlib.Path, help="One source Claude Code session JSONL")
|
|
467
|
-
parser.add_argument("--output", type=pathlib.Path, help="Distinct repaired candidate path")
|
|
468
|
-
parser.add_argument("--scan-only", action="store_true", help="Report matches without writing files")
|
|
469
|
-
parser.add_argument("--replace-original", action="store_true", help="Transactionally replace one closed live .claude/projects session")
|
|
470
|
-
parser.add_argument(
|
|
471
|
-
"--confirm-session-closed",
|
|
472
|
-
action="store_true",
|
|
473
|
-
help="Required caller acknowledgement for --replace-original; this does not detect processes or locks.",
|
|
474
|
-
)
|
|
475
|
-
parser.add_argument("--work-dir", type=pathlib.Path, help="Required external candidate/report directory for --replace-original")
|
|
476
|
-
parser.add_argument("--backup-dir", type=pathlib.Path, help="Optional external numbered-backup directory for --replace-original")
|
|
477
|
-
parser.add_argument("--rule", choices=(RULE_NAME,), default=RULE_NAME, help=f"Repair rule; currently only {RULE_NAME}")
|
|
478
|
-
parser.add_argument("--scope", choices=("active-chain", "all"), default="active-chain", help="Strict active chain by default; all scans every physical branch")
|
|
479
|
-
parser.add_argument("--expect-matches", type=int, help="Require exactly N patchable matches before writing")
|
|
480
|
-
parser.add_argument("--resume-leaf", help="Explicit active-chain recovery leaf override")
|
|
481
|
-
return parser.parse_args(argv)
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
485
|
-
args = parse_args(argv)
|
|
486
|
-
try:
|
|
487
|
-
if args.version:
|
|
488
|
-
print(
|
|
489
|
-
json.dumps(
|
|
490
|
-
{
|
|
491
|
-
"packageVersion": ccj.PACKAGE_VERSION,
|
|
492
|
-
"engineVersion": ccj.CODEX_OFFLINE_COMPRESSION_VERSION,
|
|
493
|
-
"reportSchemaVersion": ccj.REPORT_SCHEMA_VERSION,
|
|
494
|
-
},
|
|
495
|
-
indent=2,
|
|
496
|
-
)
|
|
497
|
-
)
|
|
498
|
-
return 0
|
|
499
|
-
if not args.input:
|
|
500
|
-
raise ValueError("--input is required")
|
|
501
|
-
mode_count = int(args.scan_only) + int(bool(args.output)) + int(args.replace_original)
|
|
502
|
-
if mode_count != 1:
|
|
503
|
-
raise ValueError("choose exactly one mode: --scan-only, --output, or --replace-original")
|
|
504
|
-
if args.replace_original and not args.work_dir:
|
|
505
|
-
raise ValueError("--replace-original requires --work-dir")
|
|
506
|
-
if args.replace_original and not args.confirm_session_closed:
|
|
507
|
-
raise ValueError("--replace-original requires --confirm-session-closed before any live-session writes")
|
|
508
|
-
if args.replace_original:
|
|
509
|
-
if not ccj.is_under_claude_projects(args.input):
|
|
510
|
-
raise ValueError("--replace-original is only for one .claude/projects session JSONL")
|
|
511
|
-
ccj.require_live_session_jsonl(args.input)
|
|
512
|
-
if args.confirm_session_closed and not args.replace_original:
|
|
513
|
-
raise ValueError("--confirm-session-closed is only meaningful with --replace-original")
|
|
514
|
-
if args.backup_dir and not args.replace_original:
|
|
515
|
-
raise ValueError("--backup-dir requires --replace-original")
|
|
516
|
-
if args.expect_matches is not None and args.expect_matches < 0:
|
|
517
|
-
raise ValueError("--expect-matches must be non-negative")
|
|
518
|
-
if args.output and args.output.resolve() == args.input.resolve():
|
|
519
|
-
raise ValueError("--input and --output must be different files")
|
|
520
|
-
for label, process_path in (
|
|
521
|
-
("--output", args.output),
|
|
522
|
-
("--work-dir", args.work_dir),
|
|
523
|
-
("--backup-dir", args.backup_dir),
|
|
524
|
-
):
|
|
525
|
-
if process_path is not None and ccj.is_under_claude_root(process_path):
|
|
526
|
-
raise ValueError(f"{label} process files must be outside the entire .claude directory")
|
|
527
|
-
source_bytes = args.input.read_bytes()
|
|
528
|
-
plan = plan_read_pages_repairs(source_bytes, scope=args.scope, resume_leaf_override=args.resume_leaf)
|
|
529
|
-
if args.expect_matches is not None and plan["patchableMatchCount"] != args.expect_matches:
|
|
530
|
-
raise ValueError(
|
|
531
|
-
f"--expect-matches expected {args.expect_matches}, found {plan['patchableMatchCount']} patchable matches"
|
|
532
|
-
)
|
|
533
|
-
if args.scan_only:
|
|
534
|
-
print(json.dumps(public_plan(plan), ensure_ascii=False, indent=2))
|
|
535
|
-
return 0
|
|
536
|
-
output_bytes = apply_patch_plan(source_bytes, plan)
|
|
537
|
-
validation = validate_repair(source_bytes, output_bytes, plan)
|
|
538
|
-
if not validation.get("ok"):
|
|
539
|
-
raise ValueError(f"repair validation failed: {validation.get('errors')}")
|
|
540
|
-
full_validation = ccj.validate_jsonl_bytes(output_bytes, source_label=args.input.name)
|
|
541
|
-
validation["fullTranscriptValidation"] = full_validation
|
|
542
|
-
if not full_validation.get("ok"):
|
|
543
|
-
raise ValueError(
|
|
544
|
-
"repair candidate full transcript validation failed: "
|
|
545
|
-
f"{full_validation.get('errors')}"
|
|
546
|
-
)
|
|
547
|
-
replacing = False
|
|
548
|
-
if args.replace_original:
|
|
549
|
-
replacing = True
|
|
550
|
-
claude_root = ccj.claude_root_ancestor(args.input)
|
|
551
|
-
if claude_root and ccj.is_same_or_inside(args.work_dir, claude_root):
|
|
552
|
-
raise ValueError("--work-dir must be outside the .claude directory")
|
|
553
|
-
if claude_root and args.backup_dir and ccj.is_same_or_inside(args.backup_dir, claude_root):
|
|
554
|
-
raise ValueError("--backup-dir must be outside the .claude directory")
|
|
555
|
-
args.work_dir.mkdir(parents=True, exist_ok=True)
|
|
556
|
-
output_path = args.work_dir / f"{args.input.stem}.read-pages-repaired.jsonl"
|
|
557
|
-
else:
|
|
558
|
-
output_path = args.output
|
|
559
|
-
validation = publish_repair_candidate(output_path, source_bytes, output_bytes, plan)
|
|
560
|
-
report = {
|
|
561
|
-
"packageVersion": ccj.PACKAGE_VERSION,
|
|
562
|
-
"engineVersion": ccj.CODEX_OFFLINE_COMPRESSION_VERSION,
|
|
563
|
-
"reportSchemaVersion": ccj.REPORT_SCHEMA_VERSION,
|
|
564
|
-
"input": ccj.public_path_label(args.input),
|
|
565
|
-
"output": ccj.public_path_label(output_path),
|
|
566
|
-
"plan": public_plan(plan),
|
|
567
|
-
"validation": validation,
|
|
568
|
-
"replaceOriginal": replacing,
|
|
569
|
-
}
|
|
570
|
-
if replacing:
|
|
571
|
-
replacement = ccj._replace_file_after_validation(
|
|
572
|
-
output_path,
|
|
573
|
-
args.input,
|
|
574
|
-
backup_dir=args.backup_dir,
|
|
575
|
-
expected_source_sha256=plan["sourceSha256"],
|
|
576
|
-
expected_candidate_sha256=validation["outputSha256"],
|
|
577
|
-
)
|
|
578
|
-
backup = replacement["backup_path"]
|
|
579
|
-
report["replacementTarget"] = ccj.public_path_label(args.input)
|
|
580
|
-
report["replacementBackup"] = ccj.public_path_label(backup)
|
|
581
|
-
report["replacementValidation"] = replacement["validation"]
|
|
582
|
-
report["replacementCandidateSha256"] = replacement["candidate_sha256"]
|
|
583
|
-
report["replacementPublishedSha256"] = replacement["published_sha256"]
|
|
584
|
-
report["replacementParentDirectoryFsync"] = replacement["parent_directory_fsync"]
|
|
585
|
-
report["operationState"] = replacement["operation_state"]
|
|
586
|
-
report["replacementCleanupErrors"] = replacement["cleanup_errors"]
|
|
587
|
-
report_path = output_path.with_suffix(output_path.suffix + ".repair.json")
|
|
588
|
-
try:
|
|
589
|
-
ccj.atomic_write_text(report_path, json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
590
|
-
except Exception as report_exc:
|
|
591
|
-
if replacing:
|
|
592
|
-
receipt = {
|
|
593
|
-
"operationState": "committed-report-failed",
|
|
594
|
-
"replacementTarget": ccj.public_path_label(args.input),
|
|
595
|
-
"replacementBackup": report.get("replacementBackup"),
|
|
596
|
-
"replacementCandidate": ccj.public_path_label(output_path),
|
|
597
|
-
"sourceSha256": plan.get("sourceSha256"),
|
|
598
|
-
"candidateSha256": report.get("replacementCandidateSha256"),
|
|
599
|
-
"publishedSha256": report.get("replacementPublishedSha256"),
|
|
600
|
-
"replacementValidationOk": bool((report.get("replacementValidation") or {}).get("ok")),
|
|
601
|
-
"priorOperationState": report.get("operationState"),
|
|
602
|
-
"replacementCleanupErrors": report.get("replacementCleanupErrors", []),
|
|
603
|
-
"reportError": f"{type(report_exc).__name__}: {report_exc}",
|
|
604
|
-
}
|
|
605
|
-
print(json.dumps(receipt, ensure_ascii=False, indent=2))
|
|
606
|
-
ccj.eprint(
|
|
607
|
-
"ERROR: live repair committed, but final report publication failed: "
|
|
608
|
-
f"{report_exc}"
|
|
609
|
-
)
|
|
610
|
-
return 3
|
|
611
|
-
raise
|
|
612
|
-
if report.get("operationState") == "committed-cleanup-failed":
|
|
613
|
-
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
614
|
-
ccj.eprint(
|
|
615
|
-
"ERROR: live repair committed, but transaction cleanup failed; "
|
|
616
|
-
"inspect replacementCleanupErrors and remove only the listed residuals after verification."
|
|
617
|
-
)
|
|
618
|
-
return 3
|
|
619
|
-
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
620
|
-
return 0
|
|
621
|
-
except Exception as exc:
|
|
622
|
-
ccj.eprint(f"ERROR: {exc}")
|
|
623
|
-
return 1
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
if __name__ == "__main__":
|
|
627
|
-
raise SystemExit(main())
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Byte-preserving, explicit compatibility repairs for one Claude JSONL."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import collections
|
|
7
|
+
import copy
|
|
8
|
+
import json
|
|
9
|
+
import pathlib
|
|
10
|
+
import sys
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
13
|
+
|
|
14
|
+
SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
|
|
15
|
+
if str(SCRIPT_DIR) not in sys.path:
|
|
16
|
+
sys.path.insert(0, str(SCRIPT_DIR))
|
|
17
|
+
|
|
18
|
+
import compress_claude_jsonl as ccj
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
JsonObj = Dict[str, Any]
|
|
22
|
+
RULE_NAME = "remove-unsupported-read-pages"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Member:
|
|
27
|
+
key: str
|
|
28
|
+
start: int
|
|
29
|
+
end: int
|
|
30
|
+
value: "Node"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Node:
|
|
35
|
+
kind: str
|
|
36
|
+
start: int
|
|
37
|
+
end: int
|
|
38
|
+
members: List[Member] = field(default_factory=list)
|
|
39
|
+
items: List["Node"] = field(default_factory=list)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class SpanJsonParser:
|
|
43
|
+
def __init__(self, data: bytes):
|
|
44
|
+
self.data = data
|
|
45
|
+
self.pos = 0
|
|
46
|
+
|
|
47
|
+
def parse(self) -> Node:
|
|
48
|
+
self._ws()
|
|
49
|
+
node = self._value()
|
|
50
|
+
self._ws()
|
|
51
|
+
if self.pos != len(self.data):
|
|
52
|
+
raise ValueError(f"unexpected trailing JSON bytes at offset {self.pos}")
|
|
53
|
+
return node
|
|
54
|
+
|
|
55
|
+
def _ws(self) -> None:
|
|
56
|
+
while self.pos < len(self.data) and self.data[self.pos] in b" \t\r\n":
|
|
57
|
+
self.pos += 1
|
|
58
|
+
|
|
59
|
+
def _string(self) -> Tuple[str, int, int]:
|
|
60
|
+
start = self.pos
|
|
61
|
+
if self.pos >= len(self.data) or self.data[self.pos] != 0x22:
|
|
62
|
+
raise ValueError(f"expected JSON string at offset {self.pos}")
|
|
63
|
+
self.pos += 1
|
|
64
|
+
while self.pos < len(self.data):
|
|
65
|
+
byte = self.data[self.pos]
|
|
66
|
+
if byte == 0x5C:
|
|
67
|
+
self.pos += 2
|
|
68
|
+
continue
|
|
69
|
+
self.pos += 1
|
|
70
|
+
if byte == 0x22:
|
|
71
|
+
raw = self.data[start:self.pos].decode("utf-8", errors="strict")
|
|
72
|
+
value = json.loads(raw)
|
|
73
|
+
if not isinstance(value, str):
|
|
74
|
+
raise ValueError("parsed JSON key is not a string")
|
|
75
|
+
return value, start, self.pos
|
|
76
|
+
raise ValueError(f"unterminated JSON string at offset {start}")
|
|
77
|
+
|
|
78
|
+
def _value(self) -> Node:
|
|
79
|
+
self._ws()
|
|
80
|
+
if self.pos >= len(self.data):
|
|
81
|
+
raise ValueError("unexpected end of JSON")
|
|
82
|
+
byte = self.data[self.pos]
|
|
83
|
+
if byte == 0x7B:
|
|
84
|
+
return self._object()
|
|
85
|
+
if byte == 0x5B:
|
|
86
|
+
return self._array()
|
|
87
|
+
if byte == 0x22:
|
|
88
|
+
_value, start, end = self._string()
|
|
89
|
+
return Node("string", start, end)
|
|
90
|
+
start = self.pos
|
|
91
|
+
while self.pos < len(self.data) and self.data[self.pos] not in b",]} \t\r\n":
|
|
92
|
+
self.pos += 1
|
|
93
|
+
if self.pos == start:
|
|
94
|
+
raise ValueError(f"invalid JSON value at offset {start}")
|
|
95
|
+
json.loads(self.data[start:self.pos].decode("ascii", errors="strict"))
|
|
96
|
+
return Node("scalar", start, self.pos)
|
|
97
|
+
|
|
98
|
+
def _object(self) -> Node:
|
|
99
|
+
start = self.pos
|
|
100
|
+
self.pos += 1
|
|
101
|
+
members: List[Member] = []
|
|
102
|
+
self._ws()
|
|
103
|
+
if self.pos < len(self.data) and self.data[self.pos] == 0x7D:
|
|
104
|
+
self.pos += 1
|
|
105
|
+
return Node("object", start, self.pos, members=members)
|
|
106
|
+
while True:
|
|
107
|
+
self._ws()
|
|
108
|
+
key, key_start, _key_end = self._string()
|
|
109
|
+
self._ws()
|
|
110
|
+
if self.pos >= len(self.data) or self.data[self.pos] != 0x3A:
|
|
111
|
+
raise ValueError(f"expected ':' after object key at offset {self.pos}")
|
|
112
|
+
self.pos += 1
|
|
113
|
+
value = self._value()
|
|
114
|
+
members.append(Member(key, key_start, value.end, value))
|
|
115
|
+
self._ws()
|
|
116
|
+
if self.pos >= len(self.data):
|
|
117
|
+
raise ValueError("unterminated JSON object")
|
|
118
|
+
if self.data[self.pos] == 0x7D:
|
|
119
|
+
self.pos += 1
|
|
120
|
+
return Node("object", start, self.pos, members=members)
|
|
121
|
+
if self.data[self.pos] != 0x2C:
|
|
122
|
+
raise ValueError(f"expected ',' in object at offset {self.pos}")
|
|
123
|
+
self.pos += 1
|
|
124
|
+
|
|
125
|
+
def _array(self) -> Node:
|
|
126
|
+
start = self.pos
|
|
127
|
+
self.pos += 1
|
|
128
|
+
items: List[Node] = []
|
|
129
|
+
self._ws()
|
|
130
|
+
if self.pos < len(self.data) and self.data[self.pos] == 0x5D:
|
|
131
|
+
self.pos += 1
|
|
132
|
+
return Node("array", start, self.pos, items=items)
|
|
133
|
+
while True:
|
|
134
|
+
items.append(self._value())
|
|
135
|
+
self._ws()
|
|
136
|
+
if self.pos >= len(self.data):
|
|
137
|
+
raise ValueError("unterminated JSON array")
|
|
138
|
+
if self.data[self.pos] == 0x5D:
|
|
139
|
+
self.pos += 1
|
|
140
|
+
return Node("array", start, self.pos, items=items)
|
|
141
|
+
if self.data[self.pos] != 0x2C:
|
|
142
|
+
raise ValueError(f"expected ',' in array at offset {self.pos}")
|
|
143
|
+
self.pos += 1
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def unique_member(node: Node, key: str) -> Optional[Member]:
|
|
147
|
+
matches = [member for member in node.members if member.key == key]
|
|
148
|
+
if len(matches) > 1:
|
|
149
|
+
raise ValueError(f"duplicate JSON key in repair control path: {key}")
|
|
150
|
+
return matches[0] if matches else None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def assert_no_duplicate_keys(node: Node) -> None:
|
|
154
|
+
if node.kind == "object":
|
|
155
|
+
counts = collections.Counter(member.key for member in node.members)
|
|
156
|
+
duplicates = sorted(key for key, count in counts.items() if count > 1)
|
|
157
|
+
if duplicates:
|
|
158
|
+
raise ValueError(f"duplicate JSON keys make byte repair ambiguous: {duplicates[:20]}")
|
|
159
|
+
for member in node.members:
|
|
160
|
+
assert_no_duplicate_keys(member.value)
|
|
161
|
+
elif node.kind == "array":
|
|
162
|
+
for item in node.items:
|
|
163
|
+
assert_no_duplicate_keys(item)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def member_deletion_span(object_node: Node, target: Member) -> Tuple[int, int]:
|
|
167
|
+
index = object_node.members.index(target)
|
|
168
|
+
if len(object_node.members) == 1:
|
|
169
|
+
return target.start, target.end
|
|
170
|
+
if index < len(object_node.members) - 1:
|
|
171
|
+
return target.start, object_node.members[index + 1].start
|
|
172
|
+
return object_node.members[index - 1].end, target.end
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _jsonl_record_spans_with_lines(data: bytes) -> List[Tuple[int, int, int, bytes]]:
|
|
176
|
+
spans: List[Tuple[int, int, int, bytes]] = []
|
|
177
|
+
offset = 0
|
|
178
|
+
physical_line = 1
|
|
179
|
+
while offset < len(data):
|
|
180
|
+
lf_index = data.find(b"\n", offset)
|
|
181
|
+
physical_end = len(data) if lf_index < 0 else lf_index
|
|
182
|
+
content_end = physical_end - 1 if physical_end > offset and data[physical_end - 1] == 0x0D else physical_end
|
|
183
|
+
start = offset
|
|
184
|
+
if start == 0 and data.startswith(b"\xef\xbb\xbf"):
|
|
185
|
+
start = 3
|
|
186
|
+
content = data[start:content_end]
|
|
187
|
+
if content.decode("utf-8", errors="strict").strip():
|
|
188
|
+
spans.append((physical_line, start, content_end, content))
|
|
189
|
+
if lf_index < 0:
|
|
190
|
+
break
|
|
191
|
+
offset = lf_index + 1
|
|
192
|
+
physical_line += 1
|
|
193
|
+
return spans
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def jsonl_record_spans(data: bytes) -> List[Tuple[int, int, bytes]]:
|
|
197
|
+
return [(start, end, content) for _line, start, end, content in _jsonl_record_spans_with_lines(data)]
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _node_at_tool_input(root: Node, block_index: int) -> Tuple[Node, Member]:
|
|
201
|
+
message = unique_member(root, "message")
|
|
202
|
+
if message is None or message.value.kind != "object":
|
|
203
|
+
raise ValueError("matched assistant record has no object message node")
|
|
204
|
+
content = unique_member(message.value, "content")
|
|
205
|
+
if content is None or content.value.kind != "array" or block_index >= len(content.value.items):
|
|
206
|
+
raise ValueError("matched tool_use block has no content-array node")
|
|
207
|
+
block = content.value.items[block_index]
|
|
208
|
+
if block.kind != "object":
|
|
209
|
+
raise ValueError("matched tool_use block is not an object node")
|
|
210
|
+
input_member = unique_member(block, "input")
|
|
211
|
+
if input_member is None or input_member.value.kind != "object":
|
|
212
|
+
raise ValueError("matched Read tool_use has no object input node")
|
|
213
|
+
pages = unique_member(input_member.value, "pages")
|
|
214
|
+
if pages is None:
|
|
215
|
+
raise ValueError("matched Read tool_use pages key has no byte span")
|
|
216
|
+
return input_member.value, pages
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def plan_read_pages_repairs(
|
|
220
|
+
source_bytes: bytes,
|
|
221
|
+
scope: str = "active-chain",
|
|
222
|
+
resume_leaf_override: Optional[str] = None,
|
|
223
|
+
) -> Dict[str, Any]:
|
|
224
|
+
records, _raw_lines, physical_lines = ccj.parse_jsonl_bytes_with_lines(
|
|
225
|
+
source_bytes,
|
|
226
|
+
source_label="SOURCE_JSONL",
|
|
227
|
+
)
|
|
228
|
+
line_spans = _jsonl_record_spans_with_lines(source_bytes)
|
|
229
|
+
if len(line_spans) != len(records):
|
|
230
|
+
raise ValueError("record/span count mismatch while planning byte repair")
|
|
231
|
+
if [line for line, _start, _end, _bytes in line_spans] != physical_lines:
|
|
232
|
+
raise ValueError("record/span physical line map mismatch while planning byte repair")
|
|
233
|
+
parsed_roots: List[Node] = []
|
|
234
|
+
for _physical_line, _line_start, _line_end, line_bytes in line_spans:
|
|
235
|
+
root = SpanJsonParser(line_bytes).parse()
|
|
236
|
+
assert_no_duplicate_keys(root)
|
|
237
|
+
parsed_roots.append(root)
|
|
238
|
+
topology: Optional[Dict[str, Any]] = None
|
|
239
|
+
if scope == "active-chain":
|
|
240
|
+
topology = ccj.require_resume_leaf_info(records, resume_leaf_override=resume_leaf_override)
|
|
241
|
+
scope_indexes = set(topology.get("activeChainIndexes") or [])
|
|
242
|
+
elif scope == "all":
|
|
243
|
+
scope_indexes = set(range(len(records)))
|
|
244
|
+
else:
|
|
245
|
+
raise ValueError(f"unknown repair scope: {scope}")
|
|
246
|
+
|
|
247
|
+
tool_use_occurrences: Dict[str, List[int]] = collections.defaultdict(list)
|
|
248
|
+
tool_result_occurrences: Dict[str, List[int]] = collections.defaultdict(list)
|
|
249
|
+
for record_index in sorted(scope_indexes):
|
|
250
|
+
for tool_id in ccj.tool_use_ids(records[record_index]):
|
|
251
|
+
tool_use_occurrences[tool_id].append(record_index)
|
|
252
|
+
for tool_id in ccj.tool_result_ids(records[record_index]):
|
|
253
|
+
tool_result_occurrences[tool_id].append(record_index)
|
|
254
|
+
patches: List[Dict[str, Any]] = []
|
|
255
|
+
matches: List[Dict[str, Any]] = []
|
|
256
|
+
seen_target_ids: set = set()
|
|
257
|
+
for record_index in sorted(scope_indexes):
|
|
258
|
+
obj = records[record_index]
|
|
259
|
+
if obj.get("type") != "assistant" or ccj.api_role(obj) != "assistant":
|
|
260
|
+
continue
|
|
261
|
+
message = obj.get("message")
|
|
262
|
+
content = message.get("content") if isinstance(message, dict) else None
|
|
263
|
+
if not isinstance(content, list):
|
|
264
|
+
continue
|
|
265
|
+
physical_line, line_start, _line_end, line_bytes = line_spans[record_index]
|
|
266
|
+
root_node = parsed_roots[record_index]
|
|
267
|
+
for block_index, block in enumerate(content):
|
|
268
|
+
if not isinstance(block, dict) or block.get("type") != "tool_use" or block.get("name") != "Read":
|
|
269
|
+
continue
|
|
270
|
+
tool_input = block.get("input")
|
|
271
|
+
if not isinstance(tool_input, dict) or "pages" not in tool_input:
|
|
272
|
+
continue
|
|
273
|
+
tool_id = block.get("id")
|
|
274
|
+
file_path = tool_input.get("file_path")
|
|
275
|
+
if isinstance(tool_id, str) and len(tool_use_occurrences.get(tool_id, [])) > 1:
|
|
276
|
+
raise ValueError(f"duplicate target tool_use id makes repair ambiguous: {tool_id}")
|
|
277
|
+
if isinstance(tool_id, str) and len(tool_result_occurrences.get(tool_id, [])) > 1:
|
|
278
|
+
raise ValueError(f"duplicate tool_result id makes repair ambiguous: {tool_id}")
|
|
279
|
+
later_results = [
|
|
280
|
+
result_index
|
|
281
|
+
for result_index in tool_result_occurrences.get(tool_id, [])
|
|
282
|
+
if result_index > record_index
|
|
283
|
+
] if isinstance(tool_id, str) else []
|
|
284
|
+
result_index = later_results[0] if len(later_results) == 1 else None
|
|
285
|
+
result_record = records[result_index] if isinstance(result_index, int) else None
|
|
286
|
+
use_session = obj.get("sessionId")
|
|
287
|
+
result_session = result_record.get("sessionId") if isinstance(result_record, dict) else None
|
|
288
|
+
same_session = (
|
|
289
|
+
isinstance(use_session, str)
|
|
290
|
+
and bool(use_session)
|
|
291
|
+
and isinstance(result_session, str)
|
|
292
|
+
and bool(result_session)
|
|
293
|
+
and use_session == result_session
|
|
294
|
+
)
|
|
295
|
+
assistant_uuid = obj.get("uuid")
|
|
296
|
+
source_uuid = ccj.source_tool_assistant_uuid(result_record) if isinstance(result_record, dict) else None
|
|
297
|
+
source_matches = (
|
|
298
|
+
isinstance(assistant_uuid, str)
|
|
299
|
+
and bool(assistant_uuid)
|
|
300
|
+
and source_uuid == assistant_uuid
|
|
301
|
+
)
|
|
302
|
+
paired = len(later_results) == 1 and same_session and source_matches
|
|
303
|
+
eligible = paired and isinstance(file_path, str) and bool(file_path)
|
|
304
|
+
if eligible:
|
|
305
|
+
reason = "eligible"
|
|
306
|
+
elif len(later_results) != 1:
|
|
307
|
+
reason = "pending-tool-result"
|
|
308
|
+
elif not same_session:
|
|
309
|
+
reason = "cross-session-tool-result"
|
|
310
|
+
elif not source_matches:
|
|
311
|
+
reason = "source-assistant-mismatch"
|
|
312
|
+
else:
|
|
313
|
+
reason = "missing-file-path"
|
|
314
|
+
match = {
|
|
315
|
+
"recordLine": physical_line,
|
|
316
|
+
"blockIndex": block_index,
|
|
317
|
+
"toolUseId": tool_id,
|
|
318
|
+
"pairedToolResult": paired,
|
|
319
|
+
"sameSession": same_session,
|
|
320
|
+
"sourceAssistantMatches": source_matches,
|
|
321
|
+
"eligible": eligible,
|
|
322
|
+
"reason": reason,
|
|
323
|
+
}
|
|
324
|
+
matches.append(match)
|
|
325
|
+
if not eligible:
|
|
326
|
+
continue
|
|
327
|
+
if tool_id in seen_target_ids:
|
|
328
|
+
raise ValueError(f"duplicate target tool_use id makes repair ambiguous: {tool_id}")
|
|
329
|
+
seen_target_ids.add(tool_id)
|
|
330
|
+
input_node, pages_member = _node_at_tool_input(root_node, block_index)
|
|
331
|
+
rel_start, rel_end = member_deletion_span(input_node, pages_member)
|
|
332
|
+
patches.append(
|
|
333
|
+
{
|
|
334
|
+
"start": line_start + rel_start,
|
|
335
|
+
"end": line_start + rel_end,
|
|
336
|
+
"recordLine": physical_line,
|
|
337
|
+
"blockIndex": block_index,
|
|
338
|
+
"toolUseId": tool_id,
|
|
339
|
+
}
|
|
340
|
+
)
|
|
341
|
+
ordered = sorted(patches, key=lambda item: (item["start"], item["end"]))
|
|
342
|
+
for left, right in zip(ordered, ordered[1:]):
|
|
343
|
+
if left["end"] > right["start"]:
|
|
344
|
+
raise ValueError("planned repair byte spans overlap")
|
|
345
|
+
return {
|
|
346
|
+
"rule": RULE_NAME,
|
|
347
|
+
"scope": scope,
|
|
348
|
+
"sourceSha256": ccj.sha256_hex(source_bytes),
|
|
349
|
+
"sourceBytes": len(source_bytes),
|
|
350
|
+
"recordCount": len(records),
|
|
351
|
+
"targetMatchCount": len(matches),
|
|
352
|
+
"patchableMatchCount": len(patches),
|
|
353
|
+
"pendingMatchCount": sum(1 for item in matches if item["reason"] == "pending-tool-result"),
|
|
354
|
+
"ineligibleMatchCount": sum(1 for item in matches if not item["eligible"]),
|
|
355
|
+
"matches": matches,
|
|
356
|
+
"patches": ordered,
|
|
357
|
+
"resumeTopology": ccj.public_resume_leaf_info(topology),
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def apply_patch_plan(source_bytes: bytes, plan: Dict[str, Any]) -> bytes:
|
|
362
|
+
output = source_bytes
|
|
363
|
+
for item in reversed(plan.get("patches") or []):
|
|
364
|
+
start = int(item["start"])
|
|
365
|
+
end = int(item["end"])
|
|
366
|
+
if not (0 <= start < end <= len(output)):
|
|
367
|
+
raise ValueError(f"invalid repair span: {start}:{end}")
|
|
368
|
+
output = output[:start] + output[end:]
|
|
369
|
+
return output
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _uuid_parent_signature(records: Sequence[JsonObj]) -> List[Tuple[Any, Any, Any]]:
|
|
373
|
+
return [(obj.get("type"), obj.get("uuid"), obj.get("parentUuid")) for obj in records]
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def validate_repair(source_bytes: bytes, output_bytes: bytes, plan: Dict[str, Any]) -> Dict[str, Any]:
|
|
377
|
+
errors: List[str] = []
|
|
378
|
+
expected = apply_patch_plan(source_bytes, plan)
|
|
379
|
+
if output_bytes != expected:
|
|
380
|
+
errors.append("output bytes differ outside the planned deletion spans")
|
|
381
|
+
source_records, _ = ccj.parse_jsonl_bytes(source_bytes, source_label="SOURCE_JSONL")
|
|
382
|
+
output_records, _ = ccj.parse_jsonl_bytes(output_bytes, source_label="REPAIRED_JSONL")
|
|
383
|
+
if len(source_records) != len(output_records):
|
|
384
|
+
errors.append("record count changed")
|
|
385
|
+
if _uuid_parent_signature(source_records) != _uuid_parent_signature(output_records):
|
|
386
|
+
errors.append("type/uuid/parentUuid signature changed")
|
|
387
|
+
source_tool_ids = [
|
|
388
|
+
(ccj.tool_use_ids(obj), ccj.tool_result_ids(obj)) for obj in source_records
|
|
389
|
+
]
|
|
390
|
+
output_tool_ids = [
|
|
391
|
+
(ccj.tool_use_ids(obj), ccj.tool_result_ids(obj)) for obj in output_records
|
|
392
|
+
]
|
|
393
|
+
if source_tool_ids != output_tool_ids:
|
|
394
|
+
errors.append("tool_use/tool_result identifier sequence changed")
|
|
395
|
+
second_plan = plan_read_pages_repairs(
|
|
396
|
+
output_bytes,
|
|
397
|
+
scope=str(plan.get("scope") or "active-chain"),
|
|
398
|
+
resume_leaf_override=(plan.get("resumeTopology") or {}).get("selectedLeafUuid")
|
|
399
|
+
if (plan.get("resumeTopology") or {}).get("manualOverride") else None,
|
|
400
|
+
)
|
|
401
|
+
if second_plan.get("patchableMatchCount") != 0:
|
|
402
|
+
errors.append("repair is not idempotent; a second pass still finds patchable matches")
|
|
403
|
+
return {
|
|
404
|
+
"ok": not errors,
|
|
405
|
+
"errors": errors,
|
|
406
|
+
"sourceSha256": ccj.sha256_hex(source_bytes),
|
|
407
|
+
"outputSha256": ccj.sha256_hex(output_bytes),
|
|
408
|
+
"sourceBytes": len(source_bytes),
|
|
409
|
+
"outputBytes": len(output_bytes),
|
|
410
|
+
"removedBytes": len(source_bytes) - len(output_bytes),
|
|
411
|
+
"recordCount": len(output_records),
|
|
412
|
+
"secondPassPatchableMatchCount": second_plan.get("patchableMatchCount"),
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def public_plan(plan: Dict[str, Any]) -> Dict[str, Any]:
|
|
417
|
+
result = copy.deepcopy(plan)
|
|
418
|
+
result.pop("patches", None)
|
|
419
|
+
return result
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def publish_repair_candidate(
|
|
423
|
+
path: pathlib.Path,
|
|
424
|
+
source_bytes: bytes,
|
|
425
|
+
output_bytes: bytes,
|
|
426
|
+
plan: Dict[str, Any],
|
|
427
|
+
) -> Dict[str, Any]:
|
|
428
|
+
if ccj.is_under_claude_root(path):
|
|
429
|
+
raise ValueError("repair candidates and reports must be outside the entire .claude directory")
|
|
430
|
+
expected_sha256 = ccj.sha256_hex(output_bytes)
|
|
431
|
+
previous_bytes = path.read_bytes() if path.exists() else None
|
|
432
|
+
try:
|
|
433
|
+
ccj.atomic_write_bytes(path, output_bytes)
|
|
434
|
+
published_bytes = path.read_bytes()
|
|
435
|
+
if published_bytes != output_bytes or ccj.sha256_hex(published_bytes) != expected_sha256:
|
|
436
|
+
raise RuntimeError("published repair candidate bytes differ from the validated repair snapshot")
|
|
437
|
+
validation = validate_repair(source_bytes, published_bytes, plan)
|
|
438
|
+
if not validation.get("ok"):
|
|
439
|
+
raise ValueError(f"published repair candidate validation failed: {validation.get('errors')}")
|
|
440
|
+
full_validation = ccj.validate_jsonl_bytes(published_bytes, source_label=path.name)
|
|
441
|
+
validation["fullTranscriptValidation"] = full_validation
|
|
442
|
+
if not full_validation.get("ok"):
|
|
443
|
+
raise ValueError(
|
|
444
|
+
"published repair candidate full transcript validation failed: "
|
|
445
|
+
f"{full_validation.get('errors')}"
|
|
446
|
+
)
|
|
447
|
+
return validation
|
|
448
|
+
except Exception:
|
|
449
|
+
if previous_bytes is None:
|
|
450
|
+
try:
|
|
451
|
+
path.unlink()
|
|
452
|
+
ccj.fsync_parent_directory(path)
|
|
453
|
+
except FileNotFoundError:
|
|
454
|
+
pass
|
|
455
|
+
else:
|
|
456
|
+
ccj.atomic_write_bytes(path, previous_bytes)
|
|
457
|
+
raise
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
|
461
|
+
parser = argparse.ArgumentParser(
|
|
462
|
+
prog="claude-jsonl-repair-read-pages",
|
|
463
|
+
description="Scan or byte-preservingly remove historical Read.input.pages members from one Claude Code JSONL.",
|
|
464
|
+
)
|
|
465
|
+
parser.add_argument("--version", action="store_true", help="Print package, engine, and report versions")
|
|
466
|
+
parser.add_argument("--input", type=pathlib.Path, help="One source Claude Code session JSONL")
|
|
467
|
+
parser.add_argument("--output", type=pathlib.Path, help="Distinct repaired candidate path")
|
|
468
|
+
parser.add_argument("--scan-only", action="store_true", help="Report matches without writing files")
|
|
469
|
+
parser.add_argument("--replace-original", action="store_true", help="Transactionally replace one closed live .claude/projects session")
|
|
470
|
+
parser.add_argument(
|
|
471
|
+
"--confirm-session-closed",
|
|
472
|
+
action="store_true",
|
|
473
|
+
help="Required caller acknowledgement for --replace-original; this does not detect processes or locks.",
|
|
474
|
+
)
|
|
475
|
+
parser.add_argument("--work-dir", type=pathlib.Path, help="Required external candidate/report directory for --replace-original")
|
|
476
|
+
parser.add_argument("--backup-dir", type=pathlib.Path, help="Optional external numbered-backup directory for --replace-original")
|
|
477
|
+
parser.add_argument("--rule", choices=(RULE_NAME,), default=RULE_NAME, help=f"Repair rule; currently only {RULE_NAME}")
|
|
478
|
+
parser.add_argument("--scope", choices=("active-chain", "all"), default="active-chain", help="Strict active chain by default; all scans every physical branch")
|
|
479
|
+
parser.add_argument("--expect-matches", type=int, help="Require exactly N patchable matches before writing")
|
|
480
|
+
parser.add_argument("--resume-leaf", help="Explicit active-chain recovery leaf override")
|
|
481
|
+
return parser.parse_args(argv)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
485
|
+
args = parse_args(argv)
|
|
486
|
+
try:
|
|
487
|
+
if args.version:
|
|
488
|
+
print(
|
|
489
|
+
json.dumps(
|
|
490
|
+
{
|
|
491
|
+
"packageVersion": ccj.PACKAGE_VERSION,
|
|
492
|
+
"engineVersion": ccj.CODEX_OFFLINE_COMPRESSION_VERSION,
|
|
493
|
+
"reportSchemaVersion": ccj.REPORT_SCHEMA_VERSION,
|
|
494
|
+
},
|
|
495
|
+
indent=2,
|
|
496
|
+
)
|
|
497
|
+
)
|
|
498
|
+
return 0
|
|
499
|
+
if not args.input:
|
|
500
|
+
raise ValueError("--input is required")
|
|
501
|
+
mode_count = int(args.scan_only) + int(bool(args.output)) + int(args.replace_original)
|
|
502
|
+
if mode_count != 1:
|
|
503
|
+
raise ValueError("choose exactly one mode: --scan-only, --output, or --replace-original")
|
|
504
|
+
if args.replace_original and not args.work_dir:
|
|
505
|
+
raise ValueError("--replace-original requires --work-dir")
|
|
506
|
+
if args.replace_original and not args.confirm_session_closed:
|
|
507
|
+
raise ValueError("--replace-original requires --confirm-session-closed before any live-session writes")
|
|
508
|
+
if args.replace_original:
|
|
509
|
+
if not ccj.is_under_claude_projects(args.input):
|
|
510
|
+
raise ValueError("--replace-original is only for one .claude/projects session JSONL")
|
|
511
|
+
ccj.require_live_session_jsonl(args.input)
|
|
512
|
+
if args.confirm_session_closed and not args.replace_original:
|
|
513
|
+
raise ValueError("--confirm-session-closed is only meaningful with --replace-original")
|
|
514
|
+
if args.backup_dir and not args.replace_original:
|
|
515
|
+
raise ValueError("--backup-dir requires --replace-original")
|
|
516
|
+
if args.expect_matches is not None and args.expect_matches < 0:
|
|
517
|
+
raise ValueError("--expect-matches must be non-negative")
|
|
518
|
+
if args.output and args.output.resolve() == args.input.resolve():
|
|
519
|
+
raise ValueError("--input and --output must be different files")
|
|
520
|
+
for label, process_path in (
|
|
521
|
+
("--output", args.output),
|
|
522
|
+
("--work-dir", args.work_dir),
|
|
523
|
+
("--backup-dir", args.backup_dir),
|
|
524
|
+
):
|
|
525
|
+
if process_path is not None and ccj.is_under_claude_root(process_path):
|
|
526
|
+
raise ValueError(f"{label} process files must be outside the entire .claude directory")
|
|
527
|
+
source_bytes = args.input.read_bytes()
|
|
528
|
+
plan = plan_read_pages_repairs(source_bytes, scope=args.scope, resume_leaf_override=args.resume_leaf)
|
|
529
|
+
if args.expect_matches is not None and plan["patchableMatchCount"] != args.expect_matches:
|
|
530
|
+
raise ValueError(
|
|
531
|
+
f"--expect-matches expected {args.expect_matches}, found {plan['patchableMatchCount']} patchable matches"
|
|
532
|
+
)
|
|
533
|
+
if args.scan_only:
|
|
534
|
+
print(json.dumps(public_plan(plan), ensure_ascii=False, indent=2))
|
|
535
|
+
return 0
|
|
536
|
+
output_bytes = apply_patch_plan(source_bytes, plan)
|
|
537
|
+
validation = validate_repair(source_bytes, output_bytes, plan)
|
|
538
|
+
if not validation.get("ok"):
|
|
539
|
+
raise ValueError(f"repair validation failed: {validation.get('errors')}")
|
|
540
|
+
full_validation = ccj.validate_jsonl_bytes(output_bytes, source_label=args.input.name)
|
|
541
|
+
validation["fullTranscriptValidation"] = full_validation
|
|
542
|
+
if not full_validation.get("ok"):
|
|
543
|
+
raise ValueError(
|
|
544
|
+
"repair candidate full transcript validation failed: "
|
|
545
|
+
f"{full_validation.get('errors')}"
|
|
546
|
+
)
|
|
547
|
+
replacing = False
|
|
548
|
+
if args.replace_original:
|
|
549
|
+
replacing = True
|
|
550
|
+
claude_root = ccj.claude_root_ancestor(args.input)
|
|
551
|
+
if claude_root and ccj.is_same_or_inside(args.work_dir, claude_root):
|
|
552
|
+
raise ValueError("--work-dir must be outside the .claude directory")
|
|
553
|
+
if claude_root and args.backup_dir and ccj.is_same_or_inside(args.backup_dir, claude_root):
|
|
554
|
+
raise ValueError("--backup-dir must be outside the .claude directory")
|
|
555
|
+
args.work_dir.mkdir(parents=True, exist_ok=True)
|
|
556
|
+
output_path = args.work_dir / f"{args.input.stem}.read-pages-repaired.jsonl"
|
|
557
|
+
else:
|
|
558
|
+
output_path = args.output
|
|
559
|
+
validation = publish_repair_candidate(output_path, source_bytes, output_bytes, plan)
|
|
560
|
+
report = {
|
|
561
|
+
"packageVersion": ccj.PACKAGE_VERSION,
|
|
562
|
+
"engineVersion": ccj.CODEX_OFFLINE_COMPRESSION_VERSION,
|
|
563
|
+
"reportSchemaVersion": ccj.REPORT_SCHEMA_VERSION,
|
|
564
|
+
"input": ccj.public_path_label(args.input),
|
|
565
|
+
"output": ccj.public_path_label(output_path),
|
|
566
|
+
"plan": public_plan(plan),
|
|
567
|
+
"validation": validation,
|
|
568
|
+
"replaceOriginal": replacing,
|
|
569
|
+
}
|
|
570
|
+
if replacing:
|
|
571
|
+
replacement = ccj._replace_file_after_validation(
|
|
572
|
+
output_path,
|
|
573
|
+
args.input,
|
|
574
|
+
backup_dir=args.backup_dir,
|
|
575
|
+
expected_source_sha256=plan["sourceSha256"],
|
|
576
|
+
expected_candidate_sha256=validation["outputSha256"],
|
|
577
|
+
)
|
|
578
|
+
backup = replacement["backup_path"]
|
|
579
|
+
report["replacementTarget"] = ccj.public_path_label(args.input)
|
|
580
|
+
report["replacementBackup"] = ccj.public_path_label(backup)
|
|
581
|
+
report["replacementValidation"] = replacement["validation"]
|
|
582
|
+
report["replacementCandidateSha256"] = replacement["candidate_sha256"]
|
|
583
|
+
report["replacementPublishedSha256"] = replacement["published_sha256"]
|
|
584
|
+
report["replacementParentDirectoryFsync"] = replacement["parent_directory_fsync"]
|
|
585
|
+
report["operationState"] = replacement["operation_state"]
|
|
586
|
+
report["replacementCleanupErrors"] = replacement["cleanup_errors"]
|
|
587
|
+
report_path = output_path.with_suffix(output_path.suffix + ".repair.json")
|
|
588
|
+
try:
|
|
589
|
+
ccj.atomic_write_text(report_path, json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
590
|
+
except Exception as report_exc:
|
|
591
|
+
if replacing:
|
|
592
|
+
receipt = {
|
|
593
|
+
"operationState": "committed-report-failed",
|
|
594
|
+
"replacementTarget": ccj.public_path_label(args.input),
|
|
595
|
+
"replacementBackup": report.get("replacementBackup"),
|
|
596
|
+
"replacementCandidate": ccj.public_path_label(output_path),
|
|
597
|
+
"sourceSha256": plan.get("sourceSha256"),
|
|
598
|
+
"candidateSha256": report.get("replacementCandidateSha256"),
|
|
599
|
+
"publishedSha256": report.get("replacementPublishedSha256"),
|
|
600
|
+
"replacementValidationOk": bool((report.get("replacementValidation") or {}).get("ok")),
|
|
601
|
+
"priorOperationState": report.get("operationState"),
|
|
602
|
+
"replacementCleanupErrors": report.get("replacementCleanupErrors", []),
|
|
603
|
+
"reportError": f"{type(report_exc).__name__}: {report_exc}",
|
|
604
|
+
}
|
|
605
|
+
print(json.dumps(receipt, ensure_ascii=False, indent=2))
|
|
606
|
+
ccj.eprint(
|
|
607
|
+
"ERROR: live repair committed, but final report publication failed: "
|
|
608
|
+
f"{report_exc}"
|
|
609
|
+
)
|
|
610
|
+
return 3
|
|
611
|
+
raise
|
|
612
|
+
if report.get("operationState") == "committed-cleanup-failed":
|
|
613
|
+
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
614
|
+
ccj.eprint(
|
|
615
|
+
"ERROR: live repair committed, but transaction cleanup failed; "
|
|
616
|
+
"inspect replacementCleanupErrors and remove only the listed residuals after verification."
|
|
617
|
+
)
|
|
618
|
+
return 3
|
|
619
|
+
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
620
|
+
return 0
|
|
621
|
+
except Exception as exc:
|
|
622
|
+
ccj.eprint(f"ERROR: {exc}")
|
|
623
|
+
return 1
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
if __name__ == "__main__":
|
|
627
|
+
raise SystemExit(main())
|