dev-memory-cli 0.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +362 -0
- package/bin/dev-memory.js +590 -0
- package/hooks/README.md +84 -0
- package/hooks/codex-hooks.json +32 -0
- package/hooks/hooks.json +60 -0
- package/lib/assets/tidy_review.html +889 -0
- package/lib/dev_memory_branch.py +853 -0
- package/lib/dev_memory_capture.py +1343 -0
- package/lib/dev_memory_common.py +1934 -0
- package/lib/dev_memory_context.py +129 -0
- package/lib/dev_memory_graduate.py +282 -0
- package/lib/dev_memory_setup.py +256 -0
- package/lib/dev_memory_tidy.py +1622 -0
- package/lib/ui-app.html +1052 -0
- package/lib/ui-server.js +330 -0
- package/package.json +52 -0
- package/scripts/hooks/_common.py +456 -0
- package/scripts/hooks/pre_compact.py +21 -0
- package/scripts/hooks/session_end.py +35 -0
- package/scripts/hooks/session_start.py +51 -0
- package/scripts/hooks/stop.py +35 -0
- package/suite-manifest.json +26 -0
|
@@ -0,0 +1,1622 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
dev-memory-tidy: review and groom existing memory entries.
|
|
4
|
+
|
|
5
|
+
Setup turns lazy-init lumps into structured files (no order → ordered).
|
|
6
|
+
Tidy is the second pass that runs periodically: structured files drift
|
|
7
|
+
over time (stale items, duplicates, mistakes), and tidy lets the user
|
|
8
|
+
review every entry, mark keep/delete/edit, and write the result back.
|
|
9
|
+
|
|
10
|
+
Workflow:
|
|
11
|
+
1. `prepare` — scan the target scope (branch + optional repo), parse
|
|
12
|
+
each .md file into top-level bullet entries, optionally inject
|
|
13
|
+
agent hints (STALE/DUP/OK/UNCLEAR + reason), render a static
|
|
14
|
+
review.html the user opens locally and exports a plan.json from.
|
|
15
|
+
2. `apply` — read the user's plan.json, snapshot the current scope
|
|
16
|
+
into tidy_backup_<ts>/, then rewrite each file according to the
|
|
17
|
+
keep/delete/edit actions per entry.
|
|
18
|
+
|
|
19
|
+
Skipped on purpose:
|
|
20
|
+
- Auto-sync blocks (between AUTO-GENERATED-START/END) are never
|
|
21
|
+
surfaced — they're regenerated by other commands.
|
|
22
|
+
- Manifest.json is not a tidy target; it's structured data, not prose.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import argparse
|
|
26
|
+
import hashlib
|
|
27
|
+
import json
|
|
28
|
+
import shutil
|
|
29
|
+
import sys
|
|
30
|
+
from datetime import datetime, timezone
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
|
|
33
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
34
|
+
|
|
35
|
+
from dev_memory_common import (
|
|
36
|
+
AUTO_END,
|
|
37
|
+
AUTO_START,
|
|
38
|
+
append_log_event,
|
|
39
|
+
ensure_branch_paths_exist,
|
|
40
|
+
join_sections,
|
|
41
|
+
split_sections,
|
|
42
|
+
template_decisions,
|
|
43
|
+
template_glossary,
|
|
44
|
+
template_overview,
|
|
45
|
+
template_pending_promotion,
|
|
46
|
+
template_progress,
|
|
47
|
+
template_repo_decisions,
|
|
48
|
+
template_repo_glossary,
|
|
49
|
+
template_repo_overview,
|
|
50
|
+
template_risks,
|
|
51
|
+
template_unsorted,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
HTML_TEMPLATE_PATH = Path(__file__).resolve().parent / "assets" / "tidy_review.html"
|
|
55
|
+
|
|
56
|
+
# Map file_key -> callable that produces a fresh v2 template body. Used by
|
|
57
|
+
# the reset-file proposal action when an entry-level cleanup is too coarse
|
|
58
|
+
# (e.g. v1 → v2 left H3 skeleton sections that tidy can't surgically remove).
|
|
59
|
+
def _template_for(file_key, branch_name, repo_name):
|
|
60
|
+
table = {
|
|
61
|
+
"overview": lambda: template_overview(branch_name),
|
|
62
|
+
"decisions": lambda: template_decisions(branch_name),
|
|
63
|
+
"progress": lambda: template_progress(branch_name),
|
|
64
|
+
"risks": lambda: template_risks(branch_name),
|
|
65
|
+
"glossary": lambda: template_glossary(branch_name),
|
|
66
|
+
"unsorted": lambda: template_unsorted(),
|
|
67
|
+
"pending_promotion": lambda: template_pending_promotion(),
|
|
68
|
+
"repo_overview": lambda: template_repo_overview(repo_name),
|
|
69
|
+
"repo_decisions": lambda: template_repo_decisions(repo_name),
|
|
70
|
+
"repo_glossary": lambda: template_repo_glossary(repo_name),
|
|
71
|
+
}
|
|
72
|
+
fn = table.get(file_key)
|
|
73
|
+
return fn() if fn else None
|
|
74
|
+
|
|
75
|
+
# Files we tidy. Keys map to dev_memory_common.asset_paths() entries.
|
|
76
|
+
BRANCH_FILE_KEYS = (
|
|
77
|
+
"overview",
|
|
78
|
+
"decisions",
|
|
79
|
+
"progress",
|
|
80
|
+
"risks",
|
|
81
|
+
"glossary",
|
|
82
|
+
"unsorted",
|
|
83
|
+
"pending_promotion",
|
|
84
|
+
)
|
|
85
|
+
REPO_FILE_KEYS = (
|
|
86
|
+
"repo_overview",
|
|
87
|
+
"repo_decisions",
|
|
88
|
+
"repo_glossary",
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
VALID_HINTS = {"STALE", "DUP", "OK", "UNCLEAR", "ORPHAN"}
|
|
92
|
+
|
|
93
|
+
# Default age (days) past which a file is considered "untouched" — entries
|
|
94
|
+
# inside such files get auto-tagged with hint=STALE so the user reviews them.
|
|
95
|
+
# Pulled from log.md's per-action timestamps. Tunable via the CLI flag
|
|
96
|
+
# `--stale-after-days`; the default is conservative because false STALEs
|
|
97
|
+
# annoy users more than missed ones.
|
|
98
|
+
AUTO_STALE_THRESHOLD_DAYS = 30
|
|
99
|
+
|
|
100
|
+
# Minimum token length for ORPHAN detection. Shorter tokens (e.g. "API",
|
|
101
|
+
# "git") match too much across the codebase and produce false negatives —
|
|
102
|
+
# the hint becomes useless. Tunable but rarely worth it.
|
|
103
|
+
AUTO_ORPHAN_MIN_TOKEN_LEN = 4
|
|
104
|
+
VALID_ENTRY_ACTIONS = {"keep", "delete", "edit"}
|
|
105
|
+
VALID_PROPOSAL_TYPES = {"delete-entries", "delete-section", "reset-file", "edit-entries", "delete-block"}
|
|
106
|
+
VALID_PRIORITIES = {"P0", "P1", "P2", "P3", "P4"}
|
|
107
|
+
|
|
108
|
+
# Heuristic: lines like `**Why:** ...`, `**How to apply:** ...`, or
|
|
109
|
+
# `**xxx**:...` are "orphan paragraphs" — narrative attached to the
|
|
110
|
+
# previous bullet but not itself in any bullet list. _parse_blocks absorbs
|
|
111
|
+
# them into the preceding block so a single delete-block takes them out
|
|
112
|
+
# together with the bullets they explain. Two accepted shapes:
|
|
113
|
+
# 1. `**Foo:** rest` — colon inside the bold span (common in CN/EN)
|
|
114
|
+
# 2. `**Foo**: rest` — colon (half- or full-width) outside the bold span
|
|
115
|
+
import re as _re_block
|
|
116
|
+
_ORPHAN_PARAGRAPH_RE = _re_block.compile(
|
|
117
|
+
r"^\s*\*\*[^*\n]+[::]\*\*\s|^\s*\*\*[^*\n]+\*\*\s*[::]"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _now_stamp():
|
|
122
|
+
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _strip_auto_block(body):
|
|
126
|
+
"""Remove the AUTO-GENERATED block from a section body so tidy
|
|
127
|
+
doesn't show machine-managed state to the user. Returns the body
|
|
128
|
+
with the block (and surrounding blank lines) removed."""
|
|
129
|
+
if AUTO_START not in body or AUTO_END not in body:
|
|
130
|
+
return body
|
|
131
|
+
start = body.index(AUTO_START)
|
|
132
|
+
end = body.index(AUTO_END) + len(AUTO_END)
|
|
133
|
+
return (body[:start].rstrip() + "\n" + body[end:].lstrip()).strip()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _parse_entries(body):
|
|
137
|
+
"""Split a section body into top-level bullet entries. A continuation
|
|
138
|
+
line (indented or non-bullet) is appended to the previous entry. Returns
|
|
139
|
+
a list of {entry_idx, text} dicts. Entries that match placeholder
|
|
140
|
+
markers (待补充 etc.) are returned with `is_placeholder=True` so the
|
|
141
|
+
UI can render them faded but still allow delete/edit."""
|
|
142
|
+
entries = []
|
|
143
|
+
current = None
|
|
144
|
+
for line in body.splitlines():
|
|
145
|
+
stripped = line.strip()
|
|
146
|
+
if not stripped:
|
|
147
|
+
if current is not None:
|
|
148
|
+
entries.append(current)
|
|
149
|
+
current = None
|
|
150
|
+
continue
|
|
151
|
+
if stripped.startswith("- "):
|
|
152
|
+
if current is not None:
|
|
153
|
+
entries.append(current)
|
|
154
|
+
current = {"text": stripped[2:].strip(), "raw_lines": [line]}
|
|
155
|
+
elif current is not None:
|
|
156
|
+
# Continuation: keep the original indentation when reconstructing.
|
|
157
|
+
current["text"] = current["text"] + "\n" + stripped
|
|
158
|
+
current["raw_lines"].append(line)
|
|
159
|
+
# Lines before any bullet (free-form prose) are ignored — tidy
|
|
160
|
+
# only operates on bullet-style entries to keep the action set
|
|
161
|
+
# finite. Free-form prose stays untouched in the file.
|
|
162
|
+
if current is not None:
|
|
163
|
+
entries.append(current)
|
|
164
|
+
indexed = []
|
|
165
|
+
for idx, ent in enumerate(entries):
|
|
166
|
+
text = ent["text"].strip()
|
|
167
|
+
is_placeholder = text in ("待补充", "待刷新")
|
|
168
|
+
indexed.append({
|
|
169
|
+
"entry_idx": idx,
|
|
170
|
+
"text": text,
|
|
171
|
+
"is_placeholder": is_placeholder,
|
|
172
|
+
})
|
|
173
|
+
return indexed
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _parse_blocks(body):
|
|
177
|
+
"""Group a section body into semantic 'blocks'. A block is the smallest
|
|
178
|
+
unit an agent typically wants to delete or move as a whole: a top-level
|
|
179
|
+
bullet plus its indented sub-bullets, optionally trailed by bold
|
|
180
|
+
paragraph(s) like `**Why:** ...` that explain it.
|
|
181
|
+
|
|
182
|
+
Why this layer exists: `_parse_entries` treats every bullet line —
|
|
183
|
+
including indented sub-bullets — as a separate entry. That gives
|
|
184
|
+
surgical edits but loses the parent/child relationship. Block parsing
|
|
185
|
+
keeps `_parse_entries` intact and adds a second view that re-groups
|
|
186
|
+
those entries into the semantic unit an agent actually reasons about.
|
|
187
|
+
|
|
188
|
+
Block boundary rules:
|
|
189
|
+
- A top-level `- xxx` (no leading whitespace) starts a new block.
|
|
190
|
+
- Indented bullets ` - yyy` join the current block — they are still
|
|
191
|
+
entries (`_parse_entries` counts them), but they belong to the
|
|
192
|
+
same semantic unit as the top-level bullet above them. The block's
|
|
193
|
+
`entry_idx_range` therefore spans 1 top-level entry + N sub-bullet
|
|
194
|
+
entries.
|
|
195
|
+
- A bold-prefixed paragraph (`**Why:** ...`) immediately following
|
|
196
|
+
the last bullet — possibly across a single blank line — is
|
|
197
|
+
absorbed as the block's tail. Two consecutive blank lines, or any
|
|
198
|
+
other top-level bullet, terminates the tail. Paragraph lines do
|
|
199
|
+
NOT count as entries.
|
|
200
|
+
- Lines before the first bullet are not part of any block (mirrors
|
|
201
|
+
`_parse_entries` which also ignores them).
|
|
202
|
+
|
|
203
|
+
Invariant: every entry produced by `_parse_entries` on the same body
|
|
204
|
+
falls inside exactly one block. `entry_idx_range` is `[start, end]`
|
|
205
|
+
inclusive over those entry indices.
|
|
206
|
+
"""
|
|
207
|
+
lines = body.splitlines()
|
|
208
|
+
blocks = []
|
|
209
|
+
current = None
|
|
210
|
+
pending_blank = False
|
|
211
|
+
entry_counter = -1 # increments on every bullet line (top-level or sub)
|
|
212
|
+
|
|
213
|
+
def push_current():
|
|
214
|
+
nonlocal current
|
|
215
|
+
if current is None:
|
|
216
|
+
return
|
|
217
|
+
while current["raw_lines"] and not current["raw_lines"][-1].strip():
|
|
218
|
+
current["raw_lines"].pop()
|
|
219
|
+
blocks.append(current)
|
|
220
|
+
current = None
|
|
221
|
+
|
|
222
|
+
def is_bullet_line(s):
|
|
223
|
+
return s.startswith("- ") or s == "-"
|
|
224
|
+
|
|
225
|
+
for raw in lines:
|
|
226
|
+
stripped = raw.strip()
|
|
227
|
+
if not stripped:
|
|
228
|
+
if current is not None:
|
|
229
|
+
if pending_blank:
|
|
230
|
+
push_current()
|
|
231
|
+
pending_blank = False
|
|
232
|
+
else:
|
|
233
|
+
pending_blank = True
|
|
234
|
+
current["raw_lines"].append(raw)
|
|
235
|
+
continue
|
|
236
|
+
|
|
237
|
+
is_top_bullet = is_bullet_line(stripped) and not (raw.startswith(" ") or raw.startswith("\t"))
|
|
238
|
+
is_sub_bullet = is_bullet_line(stripped) and (raw.startswith(" ") or raw.startswith("\t"))
|
|
239
|
+
|
|
240
|
+
if is_top_bullet:
|
|
241
|
+
push_current()
|
|
242
|
+
entry_counter += 1
|
|
243
|
+
current = {
|
|
244
|
+
"block_idx": len(blocks),
|
|
245
|
+
"entry_idx_start": entry_counter,
|
|
246
|
+
"entry_idx_end": entry_counter,
|
|
247
|
+
"first_line_preview": stripped[2:].strip()[:160] if stripped.startswith("- ") else stripped[:160],
|
|
248
|
+
"has_orphan_paragraphs": False,
|
|
249
|
+
"raw_lines": [raw],
|
|
250
|
+
}
|
|
251
|
+
pending_blank = False
|
|
252
|
+
continue
|
|
253
|
+
|
|
254
|
+
if current is None:
|
|
255
|
+
# Free-form prose before any bullet — drop (mirrors _parse_entries).
|
|
256
|
+
continue
|
|
257
|
+
|
|
258
|
+
if is_sub_bullet:
|
|
259
|
+
entry_counter += 1
|
|
260
|
+
current["entry_idx_end"] = entry_counter
|
|
261
|
+
current["raw_lines"].append(raw)
|
|
262
|
+
pending_blank = False
|
|
263
|
+
continue
|
|
264
|
+
|
|
265
|
+
# Indented non-bullet continuation (wrapped text under a bullet).
|
|
266
|
+
# _parse_entries treats this as continuation of the previous entry,
|
|
267
|
+
# so it doesn't advance entry_counter — neither do we.
|
|
268
|
+
if raw.startswith(" ") or raw.startswith("\t"):
|
|
269
|
+
current["raw_lines"].append(raw)
|
|
270
|
+
pending_blank = False
|
|
271
|
+
continue
|
|
272
|
+
|
|
273
|
+
# Non-indented, non-blank, not a bullet → candidate orphan paragraph.
|
|
274
|
+
# Absorb if it looks like `**Foo:** ...`. _parse_entries treats it
|
|
275
|
+
# as continuation of the previous (still-open) entry, but only when
|
|
276
|
+
# no blank line has separated them. If a single blank line came
|
|
277
|
+
# between bullet and paragraph, _parse_entries already closed the
|
|
278
|
+
# previous entry — but we still attach to the block (the paragraph
|
|
279
|
+
# belongs to the unit semantically).
|
|
280
|
+
if _ORPHAN_PARAGRAPH_RE.match(raw):
|
|
281
|
+
current["raw_lines"].append(raw)
|
|
282
|
+
current["has_orphan_paragraphs"] = True
|
|
283
|
+
pending_blank = False
|
|
284
|
+
continue
|
|
285
|
+
|
|
286
|
+
# Some other non-bullet prose between bullets — close the current
|
|
287
|
+
# block; this prose is not owned by anything.
|
|
288
|
+
push_current()
|
|
289
|
+
pending_blank = False
|
|
290
|
+
|
|
291
|
+
push_current()
|
|
292
|
+
|
|
293
|
+
return [
|
|
294
|
+
{
|
|
295
|
+
"block_idx": b["block_idx"],
|
|
296
|
+
"entry_idx_range": [b["entry_idx_start"], b["entry_idx_end"]],
|
|
297
|
+
"first_line_preview": b["first_line_preview"],
|
|
298
|
+
"has_orphan_paragraphs": b["has_orphan_paragraphs"],
|
|
299
|
+
"raw_lines": b["raw_lines"],
|
|
300
|
+
}
|
|
301
|
+
for b in blocks
|
|
302
|
+
]
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _block_id(file_key, section_idx, block_idx):
|
|
306
|
+
return f"{file_key}::{section_idx}::block-{block_idx}"
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _block_content_hash(raw_lines):
|
|
310
|
+
"""Compute a deterministic 16-char hex hash of a block's content.
|
|
311
|
+
|
|
312
|
+
`raw_lines` is the list of original source lines that make up the block
|
|
313
|
+
(top-level bullet + sub-bullets + absorbed orphan paragraphs, as captured
|
|
314
|
+
by `_parse_blocks`; `push_current` has already trimmed trailing blanks).
|
|
315
|
+
Joined with `\n` to form a stable byte sequence, SHA-256'd, then
|
|
316
|
+
truncated to 16 hex chars — long enough to keep collisions astronomically
|
|
317
|
+
unlikely across one section, short enough to keep plan.json readable.
|
|
318
|
+
|
|
319
|
+
Used by `apply` to detect that a block's content drifted between
|
|
320
|
+
`prepare` and `apply` (e.g. user edited a bullet but block_idx stayed
|
|
321
|
+
the same), so delete-block can refuse to nuke the wrong unit.
|
|
322
|
+
"""
|
|
323
|
+
raw_text = "\n".join(raw_lines)
|
|
324
|
+
return hashlib.sha256(raw_text.encode("utf-8")).hexdigest()[:16]
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _parse_block_id(bid):
|
|
328
|
+
"""Return (file_key, section_idx, block_idx) or None on bad input."""
|
|
329
|
+
if not isinstance(bid, str):
|
|
330
|
+
return None
|
|
331
|
+
parts = bid.split("::")
|
|
332
|
+
if len(parts) != 3 or not parts[2].startswith("block-"):
|
|
333
|
+
return None
|
|
334
|
+
try:
|
|
335
|
+
return parts[0], int(parts[1]), int(parts[2][len("block-"):])
|
|
336
|
+
except ValueError:
|
|
337
|
+
return None
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _file_label(file_key):
|
|
341
|
+
if file_key.startswith("repo_"):
|
|
342
|
+
return f"repo/{file_key[5:]}.md"
|
|
343
|
+
return f"branch/{file_key}.md"
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _scope_paths(paths, include_repo):
|
|
347
|
+
"""Return ordered (file_key, path) tuples that exist."""
|
|
348
|
+
out = []
|
|
349
|
+
for key in BRANCH_FILE_KEYS:
|
|
350
|
+
p = paths.get(key)
|
|
351
|
+
if p and p.exists():
|
|
352
|
+
out.append((key, p))
|
|
353
|
+
if include_repo:
|
|
354
|
+
for key in REPO_FILE_KEYS:
|
|
355
|
+
p = paths.get(key)
|
|
356
|
+
if p and p.exists():
|
|
357
|
+
out.append((key, p))
|
|
358
|
+
return out
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _scan_scope(paths, include_repo):
|
|
362
|
+
"""Scan target files into a tree of {file, sections, entries}."""
|
|
363
|
+
files = []
|
|
364
|
+
for file_key, path in _scope_paths(paths, include_repo):
|
|
365
|
+
content = path.read_text(encoding="utf-8")
|
|
366
|
+
prefix, sections = split_sections(content)
|
|
367
|
+
section_records = []
|
|
368
|
+
for section_idx, (title, body) in enumerate(sections):
|
|
369
|
+
cleaned = _strip_auto_block(body)
|
|
370
|
+
entries = _parse_entries(cleaned)
|
|
371
|
+
if not entries:
|
|
372
|
+
continue
|
|
373
|
+
section_records.append({
|
|
374
|
+
"section_idx": section_idx,
|
|
375
|
+
"title": title,
|
|
376
|
+
"entries": entries,
|
|
377
|
+
})
|
|
378
|
+
if not section_records:
|
|
379
|
+
continue
|
|
380
|
+
files.append({
|
|
381
|
+
"file_key": file_key,
|
|
382
|
+
"label": _file_label(file_key),
|
|
383
|
+
"path": str(path),
|
|
384
|
+
"sections": section_records,
|
|
385
|
+
})
|
|
386
|
+
return files
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _entry_id(file_key, section_idx, entry_idx):
|
|
390
|
+
return f"{file_key}::{section_idx}::{entry_idx}"
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _flatten_entries(files):
|
|
394
|
+
"""Yield every entry with its stable ID for hint/plan lookup."""
|
|
395
|
+
for f in files:
|
|
396
|
+
for s in f["sections"]:
|
|
397
|
+
for e in s["entries"]:
|
|
398
|
+
yield _entry_id(f["file_key"], s["section_idx"], e["entry_idx"]), f, s, e
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _validate_hints(hints):
|
|
402
|
+
"""Ensure each value is {label: ..., reason: ...} with valid label."""
|
|
403
|
+
if hints is None:
|
|
404
|
+
return {}
|
|
405
|
+
if not isinstance(hints, dict):
|
|
406
|
+
raise ValueError("hints must be a JSON object keyed by entry id")
|
|
407
|
+
cleaned = {}
|
|
408
|
+
for k, v in hints.items():
|
|
409
|
+
if not isinstance(v, dict):
|
|
410
|
+
raise ValueError(f"hint for {k} must be an object")
|
|
411
|
+
label = (v.get("label") or "").strip().upper()
|
|
412
|
+
reason = (v.get("reason") or "").strip()
|
|
413
|
+
if label and label not in VALID_HINTS:
|
|
414
|
+
raise ValueError(f"hint label for {k} must be one of {sorted(VALID_HINTS)}")
|
|
415
|
+
cleaned[k] = {"label": label, "reason": reason}
|
|
416
|
+
return cleaned
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _validate_proposals(raw):
|
|
420
|
+
"""Each proposal: {id, title, reason, actions: [...]}.
|
|
421
|
+
Each action variant: delete-entries / delete-section / reset-file /
|
|
422
|
+
edit-entries. We only sanitise here — actual semantics are interpreted
|
|
423
|
+
at apply time after the user accepts via the HTML."""
|
|
424
|
+
if raw is None:
|
|
425
|
+
return []
|
|
426
|
+
if not isinstance(raw, list):
|
|
427
|
+
raise ValueError("proposals must be a JSON array")
|
|
428
|
+
cleaned = []
|
|
429
|
+
seen_ids = set()
|
|
430
|
+
for i, p in enumerate(raw):
|
|
431
|
+
if not isinstance(p, dict):
|
|
432
|
+
raise ValueError(f"proposal #{i} must be an object")
|
|
433
|
+
pid = (p.get("id") or f"p{i}").strip()
|
|
434
|
+
if pid in seen_ids:
|
|
435
|
+
raise ValueError(f"duplicate proposal id {pid!r}")
|
|
436
|
+
seen_ids.add(pid)
|
|
437
|
+
title = (p.get("title") or "").strip()
|
|
438
|
+
reason = (p.get("reason") or "").strip()
|
|
439
|
+
actions = p.get("actions") or []
|
|
440
|
+
if not title:
|
|
441
|
+
raise ValueError(f"proposal {pid} missing title")
|
|
442
|
+
if not actions:
|
|
443
|
+
raise ValueError(f"proposal {pid} has no actions")
|
|
444
|
+
cleaned_actions = []
|
|
445
|
+
for j, a in enumerate(actions):
|
|
446
|
+
if not isinstance(a, dict):
|
|
447
|
+
raise ValueError(f"proposal {pid} action #{j} must be an object")
|
|
448
|
+
atype = (a.get("type") or "").strip()
|
|
449
|
+
if atype not in VALID_PROPOSAL_TYPES:
|
|
450
|
+
raise ValueError(
|
|
451
|
+
f"proposal {pid} action #{j} has invalid type {atype!r}; "
|
|
452
|
+
f"must be one of {sorted(VALID_PROPOSAL_TYPES)}"
|
|
453
|
+
)
|
|
454
|
+
if atype == "delete-block":
|
|
455
|
+
bid = (a.get("block_id") or "").strip()
|
|
456
|
+
if not bid or _parse_block_id(bid) is None:
|
|
457
|
+
raise ValueError(
|
|
458
|
+
f"proposal {pid} action #{j} delete-block needs block_id of form "
|
|
459
|
+
f"'<file_key>::<section_idx>::block-<N>' (got {bid!r})"
|
|
460
|
+
)
|
|
461
|
+
# Optional content fingerprint. apply will re-hash the block
|
|
462
|
+
# at the resolved location and refuse to delete if the value
|
|
463
|
+
# drifted; absent means "skip the content check".
|
|
464
|
+
ech = a.get("expected_content_hash")
|
|
465
|
+
if ech is not None and not isinstance(ech, str):
|
|
466
|
+
raise ValueError(
|
|
467
|
+
f"proposal {pid} action #{j} delete-block expected_content_hash "
|
|
468
|
+
f"must be a string if provided (got {type(ech).__name__})"
|
|
469
|
+
)
|
|
470
|
+
cleaned_actions.append(a)
|
|
471
|
+
priority = (p.get("priority") or "").strip().upper()
|
|
472
|
+
if priority and priority not in VALID_PRIORITIES:
|
|
473
|
+
raise ValueError(
|
|
474
|
+
f"proposal {pid} priority must be one of {sorted(VALID_PRIORITIES)} or empty, got {priority!r}"
|
|
475
|
+
)
|
|
476
|
+
cleaned.append({
|
|
477
|
+
"id": pid,
|
|
478
|
+
"title": title,
|
|
479
|
+
"reason": reason,
|
|
480
|
+
"priority": priority,
|
|
481
|
+
"actions": cleaned_actions,
|
|
482
|
+
})
|
|
483
|
+
return cleaned
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _collect_blocks(files):
|
|
487
|
+
"""Re-parse each scanned section into blocks. Returns a tuple of
|
|
488
|
+
(blocks_by_file_section, flat_block_index) where:
|
|
489
|
+
|
|
490
|
+
- blocks_by_file_section: {file_key: {section_idx: [block, ...]}}
|
|
491
|
+
with each block carrying its raw_lines (kept internally — not
|
|
492
|
+
emitted in the public JSON to keep that payload light).
|
|
493
|
+
- flat_block_index: list of public block dicts safe to dump as JSON,
|
|
494
|
+
each shaped {id, file, section, first_line_preview, entry_ids,
|
|
495
|
+
has_orphan_paragraphs}.
|
|
496
|
+
|
|
497
|
+
Re-parses on each call rather than threading block info through
|
|
498
|
+
`_scan_scope` so the existing `_scan_scope` shape stays untouched
|
|
499
|
+
(and downstream callers — html template — still see the same JSON).
|
|
500
|
+
"""
|
|
501
|
+
by_fs = {}
|
|
502
|
+
flat = []
|
|
503
|
+
for f in files:
|
|
504
|
+
fk = f["file_key"]
|
|
505
|
+
by_fs[fk] = {}
|
|
506
|
+
# Re-read the section bodies (post _strip_auto_block) to get raw_lines.
|
|
507
|
+
# We can reconstruct them from the file because _scan_scope already
|
|
508
|
+
# filtered out empty sections — but we still need the original body,
|
|
509
|
+
# so re-read the file and re-split.
|
|
510
|
+
content = Path(f["path"]).read_text(encoding="utf-8")
|
|
511
|
+
_, sections = split_sections(content)
|
|
512
|
+
for s in f["sections"]:
|
|
513
|
+
sidx = s["section_idx"]
|
|
514
|
+
if sidx >= len(sections):
|
|
515
|
+
by_fs[fk][sidx] = []
|
|
516
|
+
continue
|
|
517
|
+
_title, body = sections[sidx]
|
|
518
|
+
cleaned = _strip_auto_block(body)
|
|
519
|
+
blocks = _parse_blocks(cleaned)
|
|
520
|
+
by_fs[fk][sidx] = blocks
|
|
521
|
+
for b in blocks:
|
|
522
|
+
start, end = b["entry_idx_range"]
|
|
523
|
+
entry_ids = [_entry_id(fk, sidx, i) for i in range(start, end + 1)]
|
|
524
|
+
flat.append({
|
|
525
|
+
"id": _block_id(fk, sidx, b["block_idx"]),
|
|
526
|
+
"file": f["label"],
|
|
527
|
+
"file_key": fk,
|
|
528
|
+
"section": s["title"],
|
|
529
|
+
"section_idx": sidx,
|
|
530
|
+
"first_line_preview": b["first_line_preview"],
|
|
531
|
+
"entry_ids": entry_ids,
|
|
532
|
+
"has_orphan_paragraphs": b["has_orphan_paragraphs"],
|
|
533
|
+
# 16-char fingerprint of the block's raw source bytes —
|
|
534
|
+
# agent copies this verbatim into plan.json's delete-block
|
|
535
|
+
# action as `expected_content_hash`; apply re-hashes the
|
|
536
|
+
# current file and refuses to delete if the value drifted.
|
|
537
|
+
"content_hash": _block_content_hash(b["raw_lines"]),
|
|
538
|
+
})
|
|
539
|
+
return by_fs, flat
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _render_annotated_md(files, blocks_by_fs, scope_meta, output_path):
|
|
543
|
+
"""Write a single .md mirror with id / block / orphan comment markers
|
|
544
|
+
so an agent can read one file and recover entry id + block boundaries
|
|
545
|
+
without parsing the original markdown twice.
|
|
546
|
+
|
|
547
|
+
Format (see design §1.1):
|
|
548
|
+
# Tidy entries · <branch> · <timestamp>
|
|
549
|
+
## 📄 branch/<file>.md
|
|
550
|
+
### <section title>
|
|
551
|
+
<!-- block: <id> -->
|
|
552
|
+
- bullet <!-- id: <entry-id> -->
|
|
553
|
+
- sub (no id comment — continuation of the entry above)
|
|
554
|
+
**Why:** ... <!-- orphan: paragraph -->
|
|
555
|
+
<!-- /block -->
|
|
556
|
+
|
|
557
|
+
Placeholder entries (`- 待补充` / `- 待刷新`) get an extra
|
|
558
|
+
`<!-- placeholder -->` so the agent can ignore them at a glance.
|
|
559
|
+
AUTO-GENERATED blocks are not surfaced (already stripped upstream).
|
|
560
|
+
"""
|
|
561
|
+
lines = [
|
|
562
|
+
f"# Tidy entries · {scope_meta['branch']} · {scope_meta['generated_at']}",
|
|
563
|
+
"",
|
|
564
|
+
]
|
|
565
|
+
for f in files:
|
|
566
|
+
fk = f["file_key"]
|
|
567
|
+
lines.append(f"<!-- file: {fk} / {f['label']} -->")
|
|
568
|
+
lines.append(f"## 📄 {f['label']}")
|
|
569
|
+
lines.append("")
|
|
570
|
+
for s in f["sections"]:
|
|
571
|
+
sidx = s["section_idx"]
|
|
572
|
+
lines.append(f"<!-- section: {sidx} / {s['title']} -->")
|
|
573
|
+
lines.append(f"### {s['title']}")
|
|
574
|
+
blocks = blocks_by_fs.get(fk, {}).get(sidx, [])
|
|
575
|
+
if not blocks:
|
|
576
|
+
lines.append("")
|
|
577
|
+
continue
|
|
578
|
+
# Build a quick lookup: which entry_idx is the top-level bullet
|
|
579
|
+
# on each raw_line, so we know which line to tag with `<!-- id -->`.
|
|
580
|
+
entry_text_by_idx = {e["entry_idx"]: e for e in s["entries"]}
|
|
581
|
+
for b in blocks:
|
|
582
|
+
lines.append("")
|
|
583
|
+
lines.append(f"<!-- block: {_block_id(fk, sidx, b['block_idx'])} -->")
|
|
584
|
+
start, _end = b["entry_idx_range"]
|
|
585
|
+
# Walk raw_lines, tagging the first top-level bullet with the
|
|
586
|
+
# entry id. Sub-bullets and continuation lines are emitted
|
|
587
|
+
# bare. Orphan paragraph lines get `<!-- orphan: paragraph -->`.
|
|
588
|
+
seen_top_bullet = False
|
|
589
|
+
cur_entry_idx = start
|
|
590
|
+
for raw in b["raw_lines"]:
|
|
591
|
+
stripped = raw.strip()
|
|
592
|
+
if not stripped:
|
|
593
|
+
lines.append(raw)
|
|
594
|
+
continue
|
|
595
|
+
is_top_bullet = (
|
|
596
|
+
stripped.startswith("- ")
|
|
597
|
+
and not (raw.startswith(" ") or raw.startswith("\t"))
|
|
598
|
+
)
|
|
599
|
+
if is_top_bullet:
|
|
600
|
+
if seen_top_bullet:
|
|
601
|
+
cur_entry_idx += 1
|
|
602
|
+
seen_top_bullet = True
|
|
603
|
+
ent = entry_text_by_idx.get(cur_entry_idx, {})
|
|
604
|
+
eid = _entry_id(fk, sidx, cur_entry_idx)
|
|
605
|
+
suffix = f" <!-- id: {eid} -->"
|
|
606
|
+
if ent.get("is_placeholder"):
|
|
607
|
+
suffix += " <!-- placeholder -->"
|
|
608
|
+
lines.append(f"{raw}{suffix}")
|
|
609
|
+
elif _ORPHAN_PARAGRAPH_RE.match(raw):
|
|
610
|
+
lines.append(f"{raw} <!-- orphan: paragraph -->")
|
|
611
|
+
else:
|
|
612
|
+
# Sub-bullet or wrapped continuation — emit bare.
|
|
613
|
+
lines.append(raw)
|
|
614
|
+
lines.append("<!-- /block -->")
|
|
615
|
+
lines.append("")
|
|
616
|
+
lines.append("")
|
|
617
|
+
output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _render_html(files, hints, proposals, scope_meta, output_path):
|
|
621
|
+
"""Inject scan + hints + proposals into the HTML template and write to disk."""
|
|
622
|
+
template = HTML_TEMPLATE_PATH.read_text(encoding="utf-8")
|
|
623
|
+
payload = {
|
|
624
|
+
"scope": scope_meta,
|
|
625
|
+
"files": files,
|
|
626
|
+
"hints": hints,
|
|
627
|
+
"proposals": proposals,
|
|
628
|
+
}
|
|
629
|
+
payload_json = json.dumps(payload, ensure_ascii=False, indent=2)
|
|
630
|
+
rendered = template.replace("__TIDY_DATA_PLACEHOLDER__", payload_json)
|
|
631
|
+
rendered = rendered.replace("__SCOPE_LABEL_PLACEHOLDER__",
|
|
632
|
+
f"{scope_meta['repo_key']} · {scope_meta['branch']}")
|
|
633
|
+
output_path.write_text(rendered, encoding="utf-8")
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
# ---------------------------------------------------------------------------
|
|
637
|
+
# Auto-hint generators (Karpathy lint pattern)
|
|
638
|
+
# ---------------------------------------------------------------------------
|
|
639
|
+
#
|
|
640
|
+
# tidy prepare can run two cheap "positive-signal" passes that bake hints into
|
|
641
|
+
# the review payload before the user opens the HTML:
|
|
642
|
+
#
|
|
643
|
+
# - STALE-by-log : files whose last log.md mention is older than the
|
|
644
|
+
# configured threshold get their entries hinted STALE.
|
|
645
|
+
# Pulls timestamps + targets from log.md, so it costs one
|
|
646
|
+
# file read + a regex scan; no external state.
|
|
647
|
+
# - ORPHAN-glossary : glossary entries whose key phrase doesn't appear
|
|
648
|
+
# anywhere else in the wiki get hinted ORPHAN — the
|
|
649
|
+
# term is dangling, not load-bearing.
|
|
650
|
+
#
|
|
651
|
+
# Both signals are intentionally conservative — false positives waste user
|
|
652
|
+
# attention more than misses. ORPHAN skips tokens shorter than
|
|
653
|
+
# AUTO_ORPHAN_MIN_TOKEN_LEN (short tokens like "API"/"hook" match too much
|
|
654
|
+
# elsewhere); STALE only fires when log.md has a timestamp, never on
|
|
655
|
+
# "file never seen" (cold-start would carpet-bomb the UI).
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
# Reverse of capture's _label(): "branch/decisions.md" / "repo/glossary.md"
|
|
659
|
+
# → file_key. Order matters: pending-promotion has a custom shape, repo_*
|
|
660
|
+
# prefix wins over branch/, and we tolerate the `(mode)` trailer.
|
|
661
|
+
def _file_label_to_key(label):
|
|
662
|
+
if not label:
|
|
663
|
+
return None
|
|
664
|
+
label = label.strip()
|
|
665
|
+
paren = label.find("(")
|
|
666
|
+
if paren > 0:
|
|
667
|
+
label = label[:paren].strip()
|
|
668
|
+
if label == "branch/pending-promotion.md":
|
|
669
|
+
return "pending_promotion"
|
|
670
|
+
if label.startswith("repo/") and label.endswith(".md"):
|
|
671
|
+
return "repo_" + label[len("repo/"):-len(".md")]
|
|
672
|
+
if label.startswith("branch/") and label.endswith(".md"):
|
|
673
|
+
return label[len("branch/"):-len(".md")]
|
|
674
|
+
return None
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
_LOG_HEADER_RE = __import__("re").compile(r"^## \[(?P<ts>[^\]]+)\]")
|
|
678
|
+
_LOG_TARGETS_RE = __import__("re").compile(r"^- targets:\s*(?P<rest>.+)$")
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _parse_log_last_touched(log_path):
|
|
682
|
+
"""Return {file_key: latest_iso_timestamp} by walking log.md once.
|
|
683
|
+
|
|
684
|
+
Algorithm: a log entry block is `## [...]` followed by a few `- key:
|
|
685
|
+
value` lines until the next H2 or EOF. We only care about the `targets`
|
|
686
|
+
detail; each comma-separated file label gets its file_key resolved and
|
|
687
|
+
the block's timestamp recorded. Later timestamps for the same file_key
|
|
688
|
+
overwrite (entries are append-only so later = file position later =
|
|
689
|
+
nothing; we still compare lexicographically since ISO sorts).
|
|
690
|
+
"""
|
|
691
|
+
if log_path is None or not log_path.exists():
|
|
692
|
+
return {}
|
|
693
|
+
last_seen = {}
|
|
694
|
+
current_ts = None
|
|
695
|
+
for raw in log_path.read_text(encoding="utf-8").splitlines():
|
|
696
|
+
m = _LOG_HEADER_RE.match(raw)
|
|
697
|
+
if m:
|
|
698
|
+
current_ts = m.group("ts").strip()
|
|
699
|
+
continue
|
|
700
|
+
if current_ts is None:
|
|
701
|
+
continue
|
|
702
|
+
m = _LOG_TARGETS_RE.match(raw)
|
|
703
|
+
if not m:
|
|
704
|
+
continue
|
|
705
|
+
rest = m.group("rest").strip()
|
|
706
|
+
for chunk in rest.split(","):
|
|
707
|
+
file_key = _file_label_to_key(chunk)
|
|
708
|
+
if not file_key:
|
|
709
|
+
continue
|
|
710
|
+
prev = last_seen.get(file_key)
|
|
711
|
+
if prev is None or current_ts > prev:
|
|
712
|
+
last_seen[file_key] = current_ts
|
|
713
|
+
return last_seen
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
def _iso_to_age_days(iso_ts):
|
|
717
|
+
"""Best-effort: parse an ISO timestamp string into "days ago" relative
|
|
718
|
+
to now. Returns None on parse failure so callers can skip cleanly."""
|
|
719
|
+
if not iso_ts:
|
|
720
|
+
return None
|
|
721
|
+
try:
|
|
722
|
+
# datetime.fromisoformat handles "+00:00" suffixes from now_iso();
|
|
723
|
+
# the Z suffix some legacy callers emit is normalized first.
|
|
724
|
+
s = iso_ts.replace("Z", "+00:00")
|
|
725
|
+
dt = datetime.fromisoformat(s)
|
|
726
|
+
if dt.tzinfo is None:
|
|
727
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
|
728
|
+
delta = datetime.now(timezone.utc) - dt
|
|
729
|
+
return max(0, delta.days)
|
|
730
|
+
except (ValueError, TypeError):
|
|
731
|
+
return None
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _stale_hints_from_log(files, log_paths, *, threshold_days):
|
|
735
|
+
"""Return {entry_id: {label, reason}} for entries inside files whose log
|
|
736
|
+
last-touched age exceeds threshold_days. log_paths is an iterable of
|
|
737
|
+
branch/repo log.md paths — we union them so a file touched recently in
|
|
738
|
+
either layer is considered fresh."""
|
|
739
|
+
last_seen = {}
|
|
740
|
+
for lp in log_paths:
|
|
741
|
+
for fk, ts in _parse_log_last_touched(lp).items():
|
|
742
|
+
prev = last_seen.get(fk)
|
|
743
|
+
if prev is None or ts > prev:
|
|
744
|
+
last_seen[fk] = ts
|
|
745
|
+
|
|
746
|
+
out = {}
|
|
747
|
+
for f in files:
|
|
748
|
+
fk = f["file_key"]
|
|
749
|
+
ts = last_seen.get(fk)
|
|
750
|
+
age_days = _iso_to_age_days(ts)
|
|
751
|
+
if age_days is None or age_days < threshold_days:
|
|
752
|
+
continue
|
|
753
|
+
reason = f"log 上次提到 {age_days} 天前 (> {threshold_days})"
|
|
754
|
+
for s in f["sections"]:
|
|
755
|
+
for e in s["entries"]:
|
|
756
|
+
if e.get("is_placeholder"):
|
|
757
|
+
continue
|
|
758
|
+
eid = _entry_id(fk, s["section_idx"], e["entry_idx"])
|
|
759
|
+
out[eid] = {"label": "STALE", "reason": reason}
|
|
760
|
+
return out
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
_ORPHAN_KEY_PHRASE_TRIM_RE = __import__("re").compile(r"^[\s\-*`]+|[\s`*]+$")
|
|
764
|
+
_ORPHAN_MARKDOWN_BOLD_RE = __import__("re").compile(r"\*\*([^*]+)\*\*")
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def _glossary_key_phrase(entry_text):
|
|
768
|
+
"""Extract the "key phrase" of a glossary entry — the bit other files
|
|
769
|
+
would have to reference for the entry to be load-bearing.
|
|
770
|
+
|
|
771
|
+
Heuristic: take the first line, strip bullet/markdown markers, then take
|
|
772
|
+
the substring before the first `:` or `:` (half/full-width colon).
|
|
773
|
+
Falls back to the trimmed first line if no colon. Returns "" if nothing
|
|
774
|
+
extractable, so callers can skip cleanly.
|
|
775
|
+
"""
|
|
776
|
+
if not entry_text:
|
|
777
|
+
return ""
|
|
778
|
+
first_line = entry_text.splitlines()[0] if entry_text else ""
|
|
779
|
+
s = first_line.strip()
|
|
780
|
+
if s.startswith(("- ", "* ")):
|
|
781
|
+
s = s[2:]
|
|
782
|
+
# Strip bold markdown wrappers if the whole token is wrapped.
|
|
783
|
+
m = _ORPHAN_MARKDOWN_BOLD_RE.match(s)
|
|
784
|
+
if m:
|
|
785
|
+
s = m.group(1) + s[m.end():]
|
|
786
|
+
for sep in (":", ":"):
|
|
787
|
+
idx = s.find(sep)
|
|
788
|
+
if idx > 0:
|
|
789
|
+
s = s[:idx]
|
|
790
|
+
break
|
|
791
|
+
s = _ORPHAN_KEY_PHRASE_TRIM_RE.sub("", s).strip()
|
|
792
|
+
return s
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def _orphan_hints_from_glossary(files, paths):
|
|
796
|
+
"""Return {entry_id: {label, reason}} for glossary entries whose key
|
|
797
|
+
phrase never appears in any other file in the wiki.
|
|
798
|
+
|
|
799
|
+
Why "never": we're after dangling terms, not under-referenced ones. A
|
|
800
|
+
term mentioned even once outside its definition is doing some work; an
|
|
801
|
+
entry that appears literally nowhere else is either superseded, never
|
|
802
|
+
actually used, or a private note that drifted in. Worth a user look.
|
|
803
|
+
"""
|
|
804
|
+
# Build the haystack: full text of every non-glossary file in `paths`.
|
|
805
|
+
glossary_keys = {"glossary", "repo_glossary"}
|
|
806
|
+
haystack_chunks = []
|
|
807
|
+
for key, path in paths.items():
|
|
808
|
+
if key in glossary_keys:
|
|
809
|
+
continue
|
|
810
|
+
if not hasattr(path, "exists") or not path.exists():
|
|
811
|
+
continue
|
|
812
|
+
try:
|
|
813
|
+
haystack_chunks.append(path.read_text(encoding="utf-8"))
|
|
814
|
+
except OSError:
|
|
815
|
+
continue
|
|
816
|
+
haystack = "\n".join(haystack_chunks)
|
|
817
|
+
|
|
818
|
+
out = {}
|
|
819
|
+
for f in files:
|
|
820
|
+
if f["file_key"] not in glossary_keys:
|
|
821
|
+
continue
|
|
822
|
+
for s in f["sections"]:
|
|
823
|
+
for e in s["entries"]:
|
|
824
|
+
if e.get("is_placeholder"):
|
|
825
|
+
continue
|
|
826
|
+
phrase = _glossary_key_phrase(e.get("text", ""))
|
|
827
|
+
if not phrase or len(phrase) < AUTO_ORPHAN_MIN_TOKEN_LEN:
|
|
828
|
+
continue
|
|
829
|
+
if phrase in haystack:
|
|
830
|
+
continue
|
|
831
|
+
eid = _entry_id(f["file_key"], s["section_idx"], e["entry_idx"])
|
|
832
|
+
out[eid] = {
|
|
833
|
+
"label": "ORPHAN",
|
|
834
|
+
"reason": f"术语 '{phrase}' 在其他文件零引用",
|
|
835
|
+
}
|
|
836
|
+
return out
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def _compute_auto_hints(files, paths, *, stale_threshold_days):
|
|
840
|
+
"""Run both auto-hint passes and merge. ORPHAN wins over STALE on the
|
|
841
|
+
same entry — a never-referenced glossary term is a sharper signal than
|
|
842
|
+
"file hasn't been touched in a while" (which would tag the entire
|
|
843
|
+
glossary STALE on a cold log)."""
|
|
844
|
+
log_paths = []
|
|
845
|
+
if paths.get("log"):
|
|
846
|
+
log_paths.append(paths["log"])
|
|
847
|
+
if paths.get("repo_log") and paths.get("repo_log") != paths.get("log"):
|
|
848
|
+
log_paths.append(paths["repo_log"])
|
|
849
|
+
|
|
850
|
+
merged = _stale_hints_from_log(files, log_paths, threshold_days=stale_threshold_days)
|
|
851
|
+
orphan = _orphan_hints_from_glossary(files, paths)
|
|
852
|
+
merged.update(orphan) # ORPHAN beats STALE per the docstring.
|
|
853
|
+
return merged
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def command_prepare(args):
|
|
857
|
+
repo_root, branch_name, branch_key, storage_root, repo_key, repo_dir, branch_dir, paths = ensure_branch_paths_exist(
|
|
858
|
+
args.repo, args.context_dir, args.branch
|
|
859
|
+
)
|
|
860
|
+
include_repo = args.scope == "branch+repo"
|
|
861
|
+
files = _scan_scope(paths, include_repo)
|
|
862
|
+
|
|
863
|
+
user_hints = {}
|
|
864
|
+
if args.hints_json:
|
|
865
|
+
user_hints = _validate_hints(json.loads(args.hints_json))
|
|
866
|
+
elif args.hints_file:
|
|
867
|
+
user_hints = _validate_hints(json.loads(Path(args.hints_file).read_text(encoding="utf-8")))
|
|
868
|
+
|
|
869
|
+
# Auto-hints: derived from log.md timestamps + glossary cross-references.
|
|
870
|
+
# User-supplied hints win on the same entry — user signals are explicit
|
|
871
|
+
# judgment calls and shouldn't be silently overridden by heuristics.
|
|
872
|
+
if args.auto_hints:
|
|
873
|
+
auto = _compute_auto_hints(
|
|
874
|
+
files, paths, stale_threshold_days=args.stale_after_days,
|
|
875
|
+
)
|
|
876
|
+
else:
|
|
877
|
+
auto = {}
|
|
878
|
+
hints = dict(auto)
|
|
879
|
+
hints.update(user_hints)
|
|
880
|
+
|
|
881
|
+
proposals = []
|
|
882
|
+
if args.proposals_json:
|
|
883
|
+
proposals = _validate_proposals(json.loads(args.proposals_json))
|
|
884
|
+
elif args.proposals_file:
|
|
885
|
+
proposals = _validate_proposals(json.loads(Path(args.proposals_file).read_text(encoding="utf-8")))
|
|
886
|
+
|
|
887
|
+
scope_meta = {
|
|
888
|
+
"repo_root": str(repo_root),
|
|
889
|
+
"repo_key": repo_key,
|
|
890
|
+
"branch": branch_name,
|
|
891
|
+
"branch_dir": str(branch_dir),
|
|
892
|
+
"include_repo": include_repo,
|
|
893
|
+
"generated_at": _now_stamp(),
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
review_dir = branch_dir / "tidy_review"
|
|
897
|
+
review_dir.mkdir(parents=True, exist_ok=True)
|
|
898
|
+
out_path = review_dir / f"review_{scope_meta['generated_at']}.html"
|
|
899
|
+
_render_html(files, hints, proposals, scope_meta, out_path)
|
|
900
|
+
|
|
901
|
+
# Build the annotated md mirror — agent reads this single file to recover
|
|
902
|
+
# entry id + block boundaries without re-parsing the original markdown.
|
|
903
|
+
blocks_by_fs, flat_blocks = _collect_blocks(files)
|
|
904
|
+
annotated_md_path = review_dir / f"entries_{scope_meta['generated_at']}.annotated.md"
|
|
905
|
+
_render_annotated_md(files, blocks_by_fs, scope_meta, annotated_md_path)
|
|
906
|
+
|
|
907
|
+
entry_index = []
|
|
908
|
+
for entry_id, f, s, e in _flatten_entries(files):
|
|
909
|
+
entry_index.append({
|
|
910
|
+
"id": entry_id,
|
|
911
|
+
"file": f["label"],
|
|
912
|
+
"section": s["title"],
|
|
913
|
+
"text": e["text"],
|
|
914
|
+
"is_placeholder": e["is_placeholder"],
|
|
915
|
+
})
|
|
916
|
+
|
|
917
|
+
section_index = []
|
|
918
|
+
for f in files:
|
|
919
|
+
for s in f["sections"]:
|
|
920
|
+
section_index.append({
|
|
921
|
+
"file_key": f["file_key"],
|
|
922
|
+
"file": f["label"],
|
|
923
|
+
"section_idx": s["section_idx"],
|
|
924
|
+
"title": s["title"],
|
|
925
|
+
"entry_count": len(s["entries"]),
|
|
926
|
+
})
|
|
927
|
+
|
|
928
|
+
hints_summary = {
|
|
929
|
+
"auto_enabled": bool(args.auto_hints),
|
|
930
|
+
"stale_after_days": args.stale_after_days,
|
|
931
|
+
"auto_count": len(auto),
|
|
932
|
+
"auto_by_label": {
|
|
933
|
+
label: sum(1 for v in auto.values() if v["label"] == label)
|
|
934
|
+
for label in sorted({v["label"] for v in auto.values()})
|
|
935
|
+
} if auto else {},
|
|
936
|
+
"user_count": len(user_hints),
|
|
937
|
+
"total_count": len(hints),
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
print(json.dumps({
|
|
941
|
+
"mode": "prepare",
|
|
942
|
+
"review_html": str(out_path),
|
|
943
|
+
"open_url": f"file://{out_path}",
|
|
944
|
+
"annotated_md": str(annotated_md_path),
|
|
945
|
+
"annotated_md_open": f"file://{annotated_md_path}",
|
|
946
|
+
"scope": scope_meta,
|
|
947
|
+
"hints_summary": hints_summary,
|
|
948
|
+
"entry_count": len(entry_index),
|
|
949
|
+
"entries": entry_index,
|
|
950
|
+
"section_count": len(section_index),
|
|
951
|
+
"sections": section_index,
|
|
952
|
+
"block_count": len(flat_blocks),
|
|
953
|
+
"blocks": flat_blocks,
|
|
954
|
+
"proposal_count": len(proposals),
|
|
955
|
+
"next_steps": [
|
|
956
|
+
"1) Read annotated_md (single Read) to recover entry id + block boundaries + orphan paragraph markers — don't re-read the raw .md files for structure.",
|
|
957
|
+
"2) Decide semantic 'fixes' to propose. Prefer `delete-block` over per-entry `delete-entries` when removing a full semantic unit (block ids are in `blocks[]`).",
|
|
958
|
+
"3) Re-run prepare with --proposals-json or --proposals-file to bake proposal cards into the HTML.",
|
|
959
|
+
"4) Tell the user to open open_url; default each proposal accept/reject; expand to inspect; export plan.json.",
|
|
960
|
+
"5) When plan.json is downloaded, run: dev-memory-cli tidy apply --plan-file <path>.",
|
|
961
|
+
],
|
|
962
|
+
}, ensure_ascii=False, indent=2))
|
|
963
|
+
return 0
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _backup_scope(branch_dir, paths, include_repo, stamp):
|
|
967
|
+
"""Snapshot every file we may touch into tidy_backup_<stamp>/. Files
|
|
968
|
+
are stored under the same relative key naming used internally so
|
|
969
|
+
apply can self-document what was preserved."""
|
|
970
|
+
backup_dir = branch_dir / f"tidy_backup_{stamp}"
|
|
971
|
+
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
972
|
+
saved = []
|
|
973
|
+
for file_key, path in _scope_paths(paths, include_repo):
|
|
974
|
+
dest = backup_dir / f"{file_key}.md"
|
|
975
|
+
shutil.copy2(path, dest)
|
|
976
|
+
saved.append({"file_key": file_key, "src": str(path), "backup": str(dest)})
|
|
977
|
+
(backup_dir / "manifest.json").write_text(
|
|
978
|
+
json.dumps({"saved_at": stamp, "files": saved}, ensure_ascii=False, indent=2),
|
|
979
|
+
encoding="utf-8",
|
|
980
|
+
)
|
|
981
|
+
return backup_dir, saved
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
def _apply_actions_to_section(body, entry_actions):
|
|
985
|
+
"""Rewrite a section body so each entry is either kept, removed,
|
|
986
|
+
or replaced by its `new_text`. entry_actions is keyed by entry_idx.
|
|
987
|
+
Lines outside top-level bullets pass through unchanged."""
|
|
988
|
+
lines = body.splitlines()
|
|
989
|
+
out_blocks = []
|
|
990
|
+
current_block = []
|
|
991
|
+
current_idx = -1
|
|
992
|
+
|
|
993
|
+
def flush():
|
|
994
|
+
nonlocal current_block, current_idx
|
|
995
|
+
if not current_block:
|
|
996
|
+
return
|
|
997
|
+
action = entry_actions.get(current_idx, {"action": "keep"})
|
|
998
|
+
kind = action.get("action", "keep")
|
|
999
|
+
if kind == "delete":
|
|
1000
|
+
pass
|
|
1001
|
+
elif kind == "edit":
|
|
1002
|
+
new_text = (action.get("new_text") or "").strip()
|
|
1003
|
+
if new_text:
|
|
1004
|
+
# Render edited text as a fresh bullet, splitting on
|
|
1005
|
+
# newlines so multi-line edits stay a single entry with
|
|
1006
|
+
# continuation lines indented to depth 2.
|
|
1007
|
+
first, *rest = new_text.split("\n")
|
|
1008
|
+
out_blocks.append("- " + first.strip())
|
|
1009
|
+
for cont in rest:
|
|
1010
|
+
out_blocks.append(" " + cont.strip())
|
|
1011
|
+
else:
|
|
1012
|
+
out_blocks.extend(current_block)
|
|
1013
|
+
current_block = []
|
|
1014
|
+
current_idx = -1
|
|
1015
|
+
|
|
1016
|
+
entry_seen = -1
|
|
1017
|
+
for raw in lines:
|
|
1018
|
+
stripped = raw.strip()
|
|
1019
|
+
if stripped.startswith("- "):
|
|
1020
|
+
flush()
|
|
1021
|
+
entry_seen += 1
|
|
1022
|
+
current_idx = entry_seen
|
|
1023
|
+
current_block = [raw]
|
|
1024
|
+
elif current_block and stripped and (raw.startswith(" ") or raw.startswith("\t")):
|
|
1025
|
+
current_block.append(raw)
|
|
1026
|
+
elif not stripped:
|
|
1027
|
+
flush()
|
|
1028
|
+
out_blocks.append(raw)
|
|
1029
|
+
else:
|
|
1030
|
+
# Free-form prose between bullets — leave alone.
|
|
1031
|
+
flush()
|
|
1032
|
+
out_blocks.append(raw)
|
|
1033
|
+
flush()
|
|
1034
|
+
|
|
1035
|
+
# Trim trailing blank lines for a tidy file.
|
|
1036
|
+
while out_blocks and not out_blocks[-1].strip():
|
|
1037
|
+
out_blocks.pop()
|
|
1038
|
+
rebuilt = "\n".join(out_blocks).strip()
|
|
1039
|
+
if not rebuilt:
|
|
1040
|
+
# Section emptied out entirely — leave the placeholder so the
|
|
1041
|
+
# file remains valid and capture's _section_is_placeholder_only
|
|
1042
|
+
# logic can still detect it.
|
|
1043
|
+
return "- 待补充"
|
|
1044
|
+
return rebuilt
|
|
1045
|
+
|
|
1046
|
+
|
|
1047
|
+
def _rewrite_file(path, plan_actions_by_section):
|
|
1048
|
+
"""For a single file, apply per-section per-entry actions then
|
|
1049
|
+
rewrite the file. Auto-sync blocks pass through untouched."""
|
|
1050
|
+
content = path.read_text(encoding="utf-8")
|
|
1051
|
+
prefix, sections = split_sections(content)
|
|
1052
|
+
new_sections = []
|
|
1053
|
+
for section_idx, (title, body) in enumerate(sections):
|
|
1054
|
+
actions = plan_actions_by_section.get(section_idx)
|
|
1055
|
+
if not actions:
|
|
1056
|
+
new_sections.append((title, body))
|
|
1057
|
+
continue
|
|
1058
|
+
# Preserve auto-sync block: split body at AUTO_START, only tidy
|
|
1059
|
+
# the part before; auto-block goes back as-is.
|
|
1060
|
+
if AUTO_START in body and AUTO_END in body:
|
|
1061
|
+
head_end = body.index(AUTO_START)
|
|
1062
|
+
head = body[:head_end].rstrip()
|
|
1063
|
+
tail = body[head_end:]
|
|
1064
|
+
new_head = _apply_actions_to_section(head, actions)
|
|
1065
|
+
new_body = (new_head + "\n\n" + tail).strip()
|
|
1066
|
+
else:
|
|
1067
|
+
new_body = _apply_actions_to_section(body, actions)
|
|
1068
|
+
new_sections.append((title, new_body))
|
|
1069
|
+
path.write_text(join_sections(prefix, new_sections), encoding="utf-8")
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
def _delete_blocks_from_section(body, block_drops):
|
|
1073
|
+
"""Re-parse the section body into blocks (source of truth at apply
|
|
1074
|
+
time — block_idx from prepare is informational) and drop the listed
|
|
1075
|
+
block indices, returning the rewritten body. Block deletion takes the
|
|
1076
|
+
block's full raw_lines: top-level bullet + any sub-bullets + absorbed
|
|
1077
|
+
orphan paragraphs.
|
|
1078
|
+
|
|
1079
|
+
`block_drops` is a mapping `{block_idx: expected_content_hash | None}`.
|
|
1080
|
+
A non-None hash is compared against the live block's hash after
|
|
1081
|
+
re-parsing; mismatch lands in the invalid list with reason
|
|
1082
|
+
`content_hash_mismatch` and the block is NOT deleted. None skips the
|
|
1083
|
+
content check (back-compat for plans written before the guard existed).
|
|
1084
|
+
|
|
1085
|
+
Returns (rebuilt_body, invalid_entries) where each invalid entry is a
|
|
1086
|
+
dict with at least `block_idx` and `reason`; mismatches also include
|
|
1087
|
+
`expected` and `actual` hash fields for debugging.
|
|
1088
|
+
"""
|
|
1089
|
+
blocks = _parse_blocks(body)
|
|
1090
|
+
invalid = []
|
|
1091
|
+
drop = set()
|
|
1092
|
+
for bidx, expected_hash in block_drops.items():
|
|
1093
|
+
if not (0 <= bidx < len(blocks)):
|
|
1094
|
+
invalid.append({
|
|
1095
|
+
"block_idx": bidx,
|
|
1096
|
+
"reason": "block_idx out of range after re-parse",
|
|
1097
|
+
})
|
|
1098
|
+
continue
|
|
1099
|
+
if expected_hash is not None:
|
|
1100
|
+
actual_hash = _block_content_hash(blocks[bidx]["raw_lines"])
|
|
1101
|
+
if actual_hash != expected_hash:
|
|
1102
|
+
invalid.append({
|
|
1103
|
+
"block_idx": bidx,
|
|
1104
|
+
"reason": "content_hash_mismatch",
|
|
1105
|
+
"expected": expected_hash,
|
|
1106
|
+
"actual": actual_hash,
|
|
1107
|
+
})
|
|
1108
|
+
continue
|
|
1109
|
+
drop.add(bidx)
|
|
1110
|
+
|
|
1111
|
+
# Strategy: replay the original body line-by-line, skipping lines that
|
|
1112
|
+
# belong to a dropped block. Because raw_lines is captured verbatim
|
|
1113
|
+
# from the body during parsing, we can identify the lines by their
|
|
1114
|
+
# position within the body.
|
|
1115
|
+
#
|
|
1116
|
+
# _parse_blocks consumes lines in order; rebuild by walking the same
|
|
1117
|
+
# loop and emitting lines for kept blocks only.
|
|
1118
|
+
lines = body.splitlines()
|
|
1119
|
+
# Re-derive line ownership: walk lines and mark which (if any) block
|
|
1120
|
+
# each line ends up in, using the same logic as _parse_blocks. To
|
|
1121
|
+
# avoid duplicating that logic, fall back to a simpler approach —
|
|
1122
|
+
# re-walk lines and check membership against block raw_lines by index
|
|
1123
|
+
# within the body.
|
|
1124
|
+
#
|
|
1125
|
+
# Trick: every raw line in a block's raw_lines is a verbatim slice of
|
|
1126
|
+
# the original lines list in order. Concatenate kept blocks' raw_lines
|
|
1127
|
+
# plus an explicit blank between them.
|
|
1128
|
+
kept_blocks = [b for i, b in enumerate(blocks) if i not in drop]
|
|
1129
|
+
if not kept_blocks:
|
|
1130
|
+
# Section emptied — preserve any pre-bullet prose if present, else
|
|
1131
|
+
# fall back to the placeholder so capture's
|
|
1132
|
+
# _section_is_placeholder_only still detects it.
|
|
1133
|
+
prefix_lines = []
|
|
1134
|
+
for raw in lines:
|
|
1135
|
+
stripped = raw.strip()
|
|
1136
|
+
if stripped.startswith("- ") and not (raw.startswith(" ") or raw.startswith("\t")):
|
|
1137
|
+
break
|
|
1138
|
+
prefix_lines.append(raw)
|
|
1139
|
+
while prefix_lines and not prefix_lines[-1].strip():
|
|
1140
|
+
prefix_lines.pop()
|
|
1141
|
+
rebuilt = "\n".join(prefix_lines).strip()
|
|
1142
|
+
return (rebuilt + "\n\n- 待补充").strip() if rebuilt else "- 待补充", invalid
|
|
1143
|
+
|
|
1144
|
+
# Walk original lines, emit only those belonging to kept blocks or to
|
|
1145
|
+
# the pre-bullet prefix. To know "belongs to a kept block", track block
|
|
1146
|
+
# boundaries by replaying the parser inline.
|
|
1147
|
+
keep_idx_set = {i for i in range(len(blocks)) if i not in drop}
|
|
1148
|
+
out_lines = []
|
|
1149
|
+
cur_block_idx = -1 # which block index we're currently inside (-1 = prefix)
|
|
1150
|
+
pending_blank = False
|
|
1151
|
+
in_block = False
|
|
1152
|
+
saw_first_bullet = False
|
|
1153
|
+
|
|
1154
|
+
# Helper: emit prefix lines (before any bullet).
|
|
1155
|
+
for raw in lines:
|
|
1156
|
+
stripped = raw.strip()
|
|
1157
|
+
if not saw_first_bullet:
|
|
1158
|
+
# Prefix region — emit until we hit the first top-level bullet.
|
|
1159
|
+
if stripped.startswith("- ") and not (raw.startswith(" ") or raw.startswith("\t")):
|
|
1160
|
+
saw_first_bullet = True
|
|
1161
|
+
cur_block_idx = 0
|
|
1162
|
+
if cur_block_idx in keep_idx_set:
|
|
1163
|
+
out_lines.append(raw)
|
|
1164
|
+
in_block = True
|
|
1165
|
+
else:
|
|
1166
|
+
in_block = False
|
|
1167
|
+
continue
|
|
1168
|
+
out_lines.append(raw)
|
|
1169
|
+
continue
|
|
1170
|
+
|
|
1171
|
+
# We're past the first bullet — emit only if current block is kept.
|
|
1172
|
+
if stripped.startswith("- ") and not (raw.startswith(" ") or raw.startswith("\t")):
|
|
1173
|
+
# New top-level bullet → advance block idx.
|
|
1174
|
+
cur_block_idx += 1
|
|
1175
|
+
in_block = cur_block_idx in keep_idx_set
|
|
1176
|
+
if in_block:
|
|
1177
|
+
# If pending_blank was set inside a kept context, emit it now.
|
|
1178
|
+
out_lines.append(raw)
|
|
1179
|
+
pending_blank = False
|
|
1180
|
+
continue
|
|
1181
|
+
|
|
1182
|
+
if not stripped:
|
|
1183
|
+
# Blank line: emit if we're inside a kept block — but mark
|
|
1184
|
+
# pending so we drop it if the block ends and goes elsewhere.
|
|
1185
|
+
if in_block:
|
|
1186
|
+
out_lines.append(raw)
|
|
1187
|
+
continue
|
|
1188
|
+
|
|
1189
|
+
if in_block:
|
|
1190
|
+
out_lines.append(raw)
|
|
1191
|
+
# else: drop — line belongs to a deleted block (sub-bullet, orphan
|
|
1192
|
+
# paragraph, etc.).
|
|
1193
|
+
|
|
1194
|
+
# Trim trailing blanks.
|
|
1195
|
+
while out_lines and not out_lines[-1].strip():
|
|
1196
|
+
out_lines.pop()
|
|
1197
|
+
rebuilt = "\n".join(out_lines).strip()
|
|
1198
|
+
if not rebuilt:
|
|
1199
|
+
return "- 待补充", invalid
|
|
1200
|
+
return rebuilt, invalid
|
|
1201
|
+
|
|
1202
|
+
|
|
1203
|
+
def _rewrite_file_delete_blocks(path, blocks_by_section):
|
|
1204
|
+
"""Apply delete-block actions to a single file. blocks_by_section is
|
|
1205
|
+
{section_idx: {block_idx: expected_content_hash | None}}. Returns a list
|
|
1206
|
+
of invalid entries (block_idx out of range, or content_hash drift) —
|
|
1207
|
+
each entry is a dict the caller annotates with file_key + section_idx
|
|
1208
|
+
before surfacing to the user."""
|
|
1209
|
+
content = path.read_text(encoding="utf-8")
|
|
1210
|
+
prefix, sections = split_sections(content)
|
|
1211
|
+
invalid_all = []
|
|
1212
|
+
new_sections = []
|
|
1213
|
+
for section_idx, (title, body) in enumerate(sections):
|
|
1214
|
+
drops = blocks_by_section.get(section_idx)
|
|
1215
|
+
if not drops:
|
|
1216
|
+
new_sections.append((title, body))
|
|
1217
|
+
continue
|
|
1218
|
+
if AUTO_START in body and AUTO_END in body:
|
|
1219
|
+
head_end = body.index(AUTO_START)
|
|
1220
|
+
head = body[:head_end].rstrip()
|
|
1221
|
+
tail = body[head_end:]
|
|
1222
|
+
new_head, invalid = _delete_blocks_from_section(head, drops)
|
|
1223
|
+
new_body = (new_head + "\n\n" + tail).strip()
|
|
1224
|
+
else:
|
|
1225
|
+
new_body, invalid = _delete_blocks_from_section(body, drops)
|
|
1226
|
+
for entry in invalid:
|
|
1227
|
+
entry["section_idx"] = section_idx
|
|
1228
|
+
invalid_all.append(entry)
|
|
1229
|
+
new_sections.append((title, new_body))
|
|
1230
|
+
path.write_text(join_sections(prefix, new_sections), encoding="utf-8")
|
|
1231
|
+
return invalid_all
|
|
1232
|
+
|
|
1233
|
+
|
|
1234
|
+
def _delete_sections(path, section_indices):
|
|
1235
|
+
"""Drop the listed top-level (`## `) sections from a file."""
|
|
1236
|
+
content = path.read_text(encoding="utf-8")
|
|
1237
|
+
prefix, sections = split_sections(content)
|
|
1238
|
+
drop = set(section_indices)
|
|
1239
|
+
kept = [(t, b) for i, (t, b) in enumerate(sections) if i not in drop]
|
|
1240
|
+
path.write_text(join_sections(prefix, kept), encoding="utf-8")
|
|
1241
|
+
|
|
1242
|
+
|
|
1243
|
+
def _reset_file(path, file_key, branch_name, repo_name):
|
|
1244
|
+
body = _template_for(file_key, branch_name, repo_name)
|
|
1245
|
+
if body is None:
|
|
1246
|
+
raise ValueError(f"no v2 template registered for file_key {file_key!r}")
|
|
1247
|
+
path.write_text(body, encoding="utf-8")
|
|
1248
|
+
|
|
1249
|
+
|
|
1250
|
+
def command_apply(args):
|
|
1251
|
+
repo_root, branch_name, branch_key, storage_root, repo_key, repo_dir, branch_dir, paths = ensure_branch_paths_exist(
|
|
1252
|
+
args.repo, args.context_dir, args.branch
|
|
1253
|
+
)
|
|
1254
|
+
|
|
1255
|
+
plan = json.loads(Path(args.plan_file).read_text(encoding="utf-8"))
|
|
1256
|
+
actions = plan.get("actions") or []
|
|
1257
|
+
include_repo = (plan.get("scope") or {}).get("include_repo", False)
|
|
1258
|
+
|
|
1259
|
+
# Bucket the actions:
|
|
1260
|
+
# reset_files : file_keys to fully reset (highest precedence)
|
|
1261
|
+
# delete_sections: file_key -> set(section_idx)
|
|
1262
|
+
# delete_blocks: file_key -> section_idx -> set(block_idx)
|
|
1263
|
+
# entry_actions: file_key -> section_idx -> entry_idx -> {action, new_text}
|
|
1264
|
+
reset_files = set()
|
|
1265
|
+
delete_sections = {}
|
|
1266
|
+
delete_blocks = {}
|
|
1267
|
+
entry_actions = {}
|
|
1268
|
+
invalid = []
|
|
1269
|
+
|
|
1270
|
+
for item in actions:
|
|
1271
|
+
# New action types use a "type" field; legacy entry-level actions
|
|
1272
|
+
# use "id" + "action". Distinguish by which key is present.
|
|
1273
|
+
atype = (item.get("type") or "").strip()
|
|
1274
|
+
if atype:
|
|
1275
|
+
if atype == "reset-file":
|
|
1276
|
+
fk = item.get("file_key", "")
|
|
1277
|
+
if fk not in paths:
|
|
1278
|
+
invalid.append({"action": item, "reason": f"unknown file_key {fk!r}"})
|
|
1279
|
+
continue
|
|
1280
|
+
reset_files.add(fk)
|
|
1281
|
+
continue
|
|
1282
|
+
if atype == "delete-section":
|
|
1283
|
+
fk = item.get("file_key", "")
|
|
1284
|
+
if fk not in paths:
|
|
1285
|
+
invalid.append({"action": item, "reason": f"unknown file_key {fk!r}"})
|
|
1286
|
+
continue
|
|
1287
|
+
try:
|
|
1288
|
+
sidx = int(item.get("section_idx"))
|
|
1289
|
+
except (TypeError, ValueError):
|
|
1290
|
+
invalid.append({"action": item, "reason": "section_idx must be int"})
|
|
1291
|
+
continue
|
|
1292
|
+
delete_sections.setdefault(fk, set()).add(sidx)
|
|
1293
|
+
continue
|
|
1294
|
+
if atype == "delete-block":
|
|
1295
|
+
bid = (item.get("block_id") or "").strip()
|
|
1296
|
+
parsed = _parse_block_id(bid)
|
|
1297
|
+
if parsed is None:
|
|
1298
|
+
invalid.append({"action": item, "reason": f"malformed block_id {bid!r}"})
|
|
1299
|
+
continue
|
|
1300
|
+
fk, sidx, bidx = parsed
|
|
1301
|
+
if fk not in paths:
|
|
1302
|
+
invalid.append({"action": item, "reason": f"unknown file_key {fk!r}"})
|
|
1303
|
+
continue
|
|
1304
|
+
# Optional content fingerprint — recorded by prepare and copied
|
|
1305
|
+
# verbatim by the agent. None means "skip the content check"
|
|
1306
|
+
# (back-compat with plans authored before this guard existed).
|
|
1307
|
+
ech = item.get("expected_content_hash")
|
|
1308
|
+
if ech is not None and not isinstance(ech, str):
|
|
1309
|
+
invalid.append({
|
|
1310
|
+
"action": item,
|
|
1311
|
+
"reason": "expected_content_hash must be a string if provided",
|
|
1312
|
+
})
|
|
1313
|
+
continue
|
|
1314
|
+
# If the same block id appears twice with different expected
|
|
1315
|
+
# hashes, the last write wins — that's fine for practical use
|
|
1316
|
+
# (agent should never emit two delete-block actions on the
|
|
1317
|
+
# same id; if they do, both want the block gone anyway).
|
|
1318
|
+
delete_blocks.setdefault(fk, {}).setdefault(sidx, {})[bidx] = ech
|
|
1319
|
+
continue
|
|
1320
|
+
if atype in ("delete-entries", "edit-entries"):
|
|
1321
|
+
# Either form expands to a list of {id, action[, new_text]}.
|
|
1322
|
+
ids = item.get("ids") or []
|
|
1323
|
+
edits = item.get("edits") or []
|
|
1324
|
+
pairs = [(i, "delete", None) for i in ids] + [
|
|
1325
|
+
(e.get("id"), "edit", e.get("new_text")) for e in edits
|
|
1326
|
+
]
|
|
1327
|
+
for eid, kind, new_text in pairs:
|
|
1328
|
+
parsed = _parse_entry_id(eid)
|
|
1329
|
+
if not parsed:
|
|
1330
|
+
invalid.append({"id": eid, "reason": "malformed id"})
|
|
1331
|
+
continue
|
|
1332
|
+
fk, sidx, eidx = parsed
|
|
1333
|
+
if fk not in paths:
|
|
1334
|
+
invalid.append({"id": eid, "reason": f"unknown file_key {fk!r}"})
|
|
1335
|
+
continue
|
|
1336
|
+
entry_actions.setdefault(fk, {}).setdefault(sidx, {})[eidx] = {
|
|
1337
|
+
"action": kind,
|
|
1338
|
+
"new_text": new_text,
|
|
1339
|
+
}
|
|
1340
|
+
continue
|
|
1341
|
+
invalid.append({"action": item, "reason": f"unknown type {atype!r}"})
|
|
1342
|
+
continue
|
|
1343
|
+
|
|
1344
|
+
# Legacy entry-level form.
|
|
1345
|
+
eid = item.get("id") or ""
|
|
1346
|
+
kind = (item.get("action") or "").strip().lower()
|
|
1347
|
+
if kind not in VALID_ENTRY_ACTIONS:
|
|
1348
|
+
invalid.append({"id": eid, "reason": f"unknown action {kind!r}"})
|
|
1349
|
+
continue
|
|
1350
|
+
parsed = _parse_entry_id(eid)
|
|
1351
|
+
if not parsed:
|
|
1352
|
+
invalid.append({"id": eid, "reason": "malformed id"})
|
|
1353
|
+
continue
|
|
1354
|
+
fk, sidx, eidx = parsed
|
|
1355
|
+
if fk not in paths:
|
|
1356
|
+
invalid.append({"id": eid, "reason": f"unknown file_key {fk!r}"})
|
|
1357
|
+
continue
|
|
1358
|
+
if kind == "keep":
|
|
1359
|
+
continue
|
|
1360
|
+
entry_actions.setdefault(fk, {}).setdefault(sidx, {})[eidx] = {
|
|
1361
|
+
"action": kind,
|
|
1362
|
+
"new_text": item.get("new_text"),
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
stamp = _now_stamp()
|
|
1366
|
+
backup_dir, _ = _backup_scope(branch_dir, paths, include_repo, stamp)
|
|
1367
|
+
|
|
1368
|
+
rewritten = []
|
|
1369
|
+
# Apply reset-file first (other ops on the same file are then redundant).
|
|
1370
|
+
for fk in reset_files:
|
|
1371
|
+
path = paths[fk]
|
|
1372
|
+
try:
|
|
1373
|
+
_reset_file(path, fk, branch_name, repo_key.rsplit("-", 1)[0])
|
|
1374
|
+
rewritten.append({"file_key": fk, "path": str(path), "op": "reset-file"})
|
|
1375
|
+
except Exception as exc:
|
|
1376
|
+
invalid.append({"file_key": fk, "op": "reset-file", "reason": str(exc)})
|
|
1377
|
+
# Drop any other ops queued for this file — reset wins.
|
|
1378
|
+
delete_sections.pop(fk, None)
|
|
1379
|
+
delete_blocks.pop(fk, None)
|
|
1380
|
+
entry_actions.pop(fk, None)
|
|
1381
|
+
|
|
1382
|
+
# Then delete-section per file (one rewrite per file).
|
|
1383
|
+
for fk, sidx_set in delete_sections.items():
|
|
1384
|
+
path = paths.get(fk)
|
|
1385
|
+
if not path or not path.exists():
|
|
1386
|
+
invalid.append({"file_key": fk, "op": "delete-section", "reason": "file missing"})
|
|
1387
|
+
continue
|
|
1388
|
+
_delete_sections(path, sidx_set)
|
|
1389
|
+
rewritten.append({"file_key": fk, "path": str(path), "op": f"delete-sections:{sorted(sidx_set)}"})
|
|
1390
|
+
# Block / entry actions targeting these sections are no-ops; prune.
|
|
1391
|
+
if fk in delete_blocks:
|
|
1392
|
+
for sidx in sidx_set:
|
|
1393
|
+
delete_blocks[fk].pop(sidx, None)
|
|
1394
|
+
if not delete_blocks[fk]:
|
|
1395
|
+
delete_blocks.pop(fk, None)
|
|
1396
|
+
if fk in entry_actions:
|
|
1397
|
+
for sidx in sidx_set:
|
|
1398
|
+
entry_actions[fk].pop(sidx, None)
|
|
1399
|
+
if not entry_actions[fk]:
|
|
1400
|
+
entry_actions.pop(fk, None)
|
|
1401
|
+
|
|
1402
|
+
# Then delete-block per file. We re-parse blocks at apply time (prepare's
|
|
1403
|
+
# block_idx is informational); out-of-range block ids or content-hash
|
|
1404
|
+
# mismatches land in `invalid`. Also prune entry-actions that would fall
|
|
1405
|
+
# inside a deleted block so we don't double-touch lines that are already
|
|
1406
|
+
# gone.
|
|
1407
|
+
for fk, sec_blocks in delete_blocks.items():
|
|
1408
|
+
path = paths.get(fk)
|
|
1409
|
+
if not path or not path.exists():
|
|
1410
|
+
invalid.append({"file_key": fk, "op": "delete-block", "reason": "file missing"})
|
|
1411
|
+
continue
|
|
1412
|
+
# Prune entry-actions that fall inside doomed blocks before we delete.
|
|
1413
|
+
# sec_blocks values are now `{block_idx: expected_hash | None}` dicts;
|
|
1414
|
+
# we just need the block_idx keys for the pruning step.
|
|
1415
|
+
if fk in entry_actions:
|
|
1416
|
+
content = path.read_text(encoding="utf-8")
|
|
1417
|
+
_, sections = split_sections(content)
|
|
1418
|
+
for sidx, bidx_map in sec_blocks.items():
|
|
1419
|
+
if sidx >= len(sections):
|
|
1420
|
+
continue
|
|
1421
|
+
body = sections[sidx][1]
|
|
1422
|
+
cleaned = _strip_auto_block(body)
|
|
1423
|
+
blks = _parse_blocks(cleaned)
|
|
1424
|
+
doomed_entry_idx = set()
|
|
1425
|
+
for bidx in bidx_map:
|
|
1426
|
+
if 0 <= bidx < len(blks):
|
|
1427
|
+
s, e = blks[bidx]["entry_idx_range"]
|
|
1428
|
+
for k in range(s, e + 1):
|
|
1429
|
+
doomed_entry_idx.add(k)
|
|
1430
|
+
if sidx in entry_actions[fk]:
|
|
1431
|
+
for k in list(entry_actions[fk][sidx].keys()):
|
|
1432
|
+
if k in doomed_entry_idx:
|
|
1433
|
+
entry_actions[fk][sidx].pop(k, None)
|
|
1434
|
+
if not entry_actions[fk][sidx]:
|
|
1435
|
+
entry_actions[fk].pop(sidx, None)
|
|
1436
|
+
if not entry_actions[fk]:
|
|
1437
|
+
entry_actions.pop(fk, None)
|
|
1438
|
+
invalid_blocks = _rewrite_file_delete_blocks(path, sec_blocks)
|
|
1439
|
+
for entry in invalid_blocks:
|
|
1440
|
+
invalid.append({
|
|
1441
|
+
"op": "delete-block",
|
|
1442
|
+
"file_key": fk,
|
|
1443
|
+
**entry,
|
|
1444
|
+
})
|
|
1445
|
+
rewritten.append({
|
|
1446
|
+
"file_key": fk,
|
|
1447
|
+
"path": str(path),
|
|
1448
|
+
"op": f"delete-blocks:{ {k: sorted(v.keys()) for k, v in sec_blocks.items()} }",
|
|
1449
|
+
})
|
|
1450
|
+
|
|
1451
|
+
# Finally entry-level actions.
|
|
1452
|
+
for fk, sec_actions in entry_actions.items():
|
|
1453
|
+
path = paths.get(fk)
|
|
1454
|
+
if not path or not path.exists():
|
|
1455
|
+
invalid.append({"file_key": fk, "reason": "file missing at apply time"})
|
|
1456
|
+
continue
|
|
1457
|
+
_rewrite_file(path, sec_actions)
|
|
1458
|
+
rewritten.append({"file_key": fk, "path": str(path), "op": "entry-actions"})
|
|
1459
|
+
|
|
1460
|
+
accepted = plan.get("accepted_proposals") or []
|
|
1461
|
+
rejected = plan.get("rejected_proposals") or []
|
|
1462
|
+
custom = plan.get("custom_proposals") or []
|
|
1463
|
+
|
|
1464
|
+
summary_path = branch_dir / "tidy_review" / f"summary_{stamp}.md"
|
|
1465
|
+
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1466
|
+
summary_lines = [
|
|
1467
|
+
f"# Tidy summary {stamp}",
|
|
1468
|
+
"",
|
|
1469
|
+
f"- branch: `{branch_name}`",
|
|
1470
|
+
f"- include_repo: `{include_repo}`",
|
|
1471
|
+
f"- backup: `{backup_dir}`",
|
|
1472
|
+
f"- accepted proposals: {len(accepted)}",
|
|
1473
|
+
f"- rejected proposals: {len(rejected)}",
|
|
1474
|
+
f"- custom proposals: {len(custom)}",
|
|
1475
|
+
f"- actions applied: {len(actions)}",
|
|
1476
|
+
f"- files rewritten: {len(rewritten)}",
|
|
1477
|
+
f"- invalid items: {len(invalid)}",
|
|
1478
|
+
]
|
|
1479
|
+
if plan.get("notes"):
|
|
1480
|
+
summary_lines += ["", "## notes", "", plan["notes"]]
|
|
1481
|
+
if rewritten:
|
|
1482
|
+
summary_lines += ["", "## rewritten", ""]
|
|
1483
|
+
summary_lines += [f"- {r['file_key']} ({r['op']})" for r in rewritten]
|
|
1484
|
+
if custom:
|
|
1485
|
+
summary_lines += [
|
|
1486
|
+
"",
|
|
1487
|
+
"## custom proposals (NOT applied — agent must read user feedback and re-propose)",
|
|
1488
|
+
"",
|
|
1489
|
+
]
|
|
1490
|
+
for c in custom:
|
|
1491
|
+
pid = c.get("proposal_id") or c.get("id") or "?"
|
|
1492
|
+
fb = (c.get("user_feedback") or "").strip()
|
|
1493
|
+
summary_lines += [f"### {pid}", "", fb or "(empty feedback)", ""]
|
|
1494
|
+
if invalid:
|
|
1495
|
+
summary_lines += ["", "## invalid", ""]
|
|
1496
|
+
summary_lines += [f"- {json.dumps(v, ensure_ascii=False)}" for v in invalid]
|
|
1497
|
+
summary_path.write_text("\n".join(summary_lines) + "\n", encoding="utf-8")
|
|
1498
|
+
|
|
1499
|
+
# Event log: one row per apply. Mirror to repo_log if any rewritten file
|
|
1500
|
+
# lives at the repo-shared layer (include_repo plan touched repo_*).
|
|
1501
|
+
touches_repo = any(r["file_key"].startswith("repo_") for r in rewritten)
|
|
1502
|
+
log_details = [
|
|
1503
|
+
("accepted", len(accepted)),
|
|
1504
|
+
("rejected", len(rejected)),
|
|
1505
|
+
("custom", len(custom)),
|
|
1506
|
+
("rewritten", len(rewritten)),
|
|
1507
|
+
("invalid", len(invalid)),
|
|
1508
|
+
("backup", backup_dir.name),
|
|
1509
|
+
]
|
|
1510
|
+
log_summary = (
|
|
1511
|
+
f"accepted={len(accepted)} rewritten={len(rewritten)}"
|
|
1512
|
+
if rewritten or accepted else "no-op"
|
|
1513
|
+
)
|
|
1514
|
+
append_log_event(
|
|
1515
|
+
paths.get("log"),
|
|
1516
|
+
"tidy",
|
|
1517
|
+
kind="apply",
|
|
1518
|
+
summary=log_summary,
|
|
1519
|
+
details=log_details,
|
|
1520
|
+
)
|
|
1521
|
+
if touches_repo and paths.get("repo_log") and paths.get("repo_log") != paths.get("log"):
|
|
1522
|
+
append_log_event(
|
|
1523
|
+
paths.get("repo_log"),
|
|
1524
|
+
"tidy",
|
|
1525
|
+
kind="apply",
|
|
1526
|
+
summary=log_summary,
|
|
1527
|
+
details=log_details,
|
|
1528
|
+
)
|
|
1529
|
+
|
|
1530
|
+
print(json.dumps({
|
|
1531
|
+
"mode": "apply",
|
|
1532
|
+
"branch": branch_name,
|
|
1533
|
+
"branch_dir": str(branch_dir),
|
|
1534
|
+
"backup_dir": str(backup_dir),
|
|
1535
|
+
"accepted_proposals": accepted,
|
|
1536
|
+
"rejected_proposals": rejected,
|
|
1537
|
+
"custom_proposals": custom,
|
|
1538
|
+
"rewritten": rewritten,
|
|
1539
|
+
"invalid": invalid,
|
|
1540
|
+
"summary": str(summary_path),
|
|
1541
|
+
"actions_count": len(actions),
|
|
1542
|
+
"next_steps": (
|
|
1543
|
+
[
|
|
1544
|
+
"Read each custom_proposals user_feedback and decide how to handle it based on the content. "
|
|
1545
|
+
"It is NOT automatic 're-run prepare' — depending on the feedback you may: act inline (capture/tidy subcommand), "
|
|
1546
|
+
"ask the user to clarify, defer with a note, or only when the feedback genuinely calls for "
|
|
1547
|
+
"fresh proposals run another `tidy prepare --proposals-file <revised>`."
|
|
1548
|
+
]
|
|
1549
|
+
if custom else []
|
|
1550
|
+
),
|
|
1551
|
+
}, ensure_ascii=False, indent=2))
|
|
1552
|
+
return 0
|
|
1553
|
+
|
|
1554
|
+
|
|
1555
|
+
def _parse_entry_id(eid):
|
|
1556
|
+
"""Return (file_key, section_idx, entry_idx) or None on bad input."""
|
|
1557
|
+
if not isinstance(eid, str):
|
|
1558
|
+
return None
|
|
1559
|
+
parts = eid.split("::")
|
|
1560
|
+
if len(parts) != 3:
|
|
1561
|
+
return None
|
|
1562
|
+
try:
|
|
1563
|
+
return parts[0], int(parts[1]), int(parts[2])
|
|
1564
|
+
except ValueError:
|
|
1565
|
+
return None
|
|
1566
|
+
|
|
1567
|
+
|
|
1568
|
+
def _add_common_args(p):
|
|
1569
|
+
p.add_argument("--repo", default=".", help="Path inside the target Git repository")
|
|
1570
|
+
p.add_argument("--context-dir", help="User-home storage root. Defaults to ~/.dev-memory/repos")
|
|
1571
|
+
p.add_argument("--branch", help="Branch name. Defaults to the current checked-out branch")
|
|
1572
|
+
|
|
1573
|
+
|
|
1574
|
+
def main():
|
|
1575
|
+
parser = argparse.ArgumentParser(description="Review and tidy existing dev-memory memory entries.")
|
|
1576
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
1577
|
+
|
|
1578
|
+
p_prep = sub.add_parser("prepare", help="Scan scope, render review.html (with optional hints + proposals)")
|
|
1579
|
+
_add_common_args(p_prep)
|
|
1580
|
+
p_prep.add_argument("--scope", choices=("branch", "branch+repo"), default="branch")
|
|
1581
|
+
hint_group = p_prep.add_mutually_exclusive_group()
|
|
1582
|
+
hint_group.add_argument("--hints-json", help="Inline JSON object keyed by entry id with {label, reason}")
|
|
1583
|
+
hint_group.add_argument("--hints-file", help="Path to JSON file with the same shape as --hints-json")
|
|
1584
|
+
p_prep.add_argument(
|
|
1585
|
+
"--no-auto-hints",
|
|
1586
|
+
dest="auto_hints",
|
|
1587
|
+
action="store_false",
|
|
1588
|
+
help="Skip the built-in STALE-by-log + ORPHAN-glossary hint passes",
|
|
1589
|
+
)
|
|
1590
|
+
p_prep.add_argument(
|
|
1591
|
+
"--stale-after-days",
|
|
1592
|
+
type=int,
|
|
1593
|
+
default=AUTO_STALE_THRESHOLD_DAYS,
|
|
1594
|
+
help=f"Auto-hint STALE for files last seen in log >= this many days ago (default {AUTO_STALE_THRESHOLD_DAYS})",
|
|
1595
|
+
)
|
|
1596
|
+
p_prep.set_defaults(auto_hints=True)
|
|
1597
|
+
proposal_group = p_prep.add_mutually_exclusive_group()
|
|
1598
|
+
proposal_group.add_argument(
|
|
1599
|
+
"--proposals-json",
|
|
1600
|
+
help="Inline JSON array of proposals: [{id, title, reason, actions:[...]}]. "
|
|
1601
|
+
"Action types: delete-entries (ids:[...]), delete-section (file_key, section_idx), "
|
|
1602
|
+
"delete-block (block_id), reset-file (file_key), edit-entries (edits:[{id, new_text}]).",
|
|
1603
|
+
)
|
|
1604
|
+
proposal_group.add_argument("--proposals-file", help="Path to JSON file with the same shape as --proposals-json")
|
|
1605
|
+
|
|
1606
|
+
p_apply = sub.add_parser("apply", help="Apply a downloaded plan.json (with backup snapshot)")
|
|
1607
|
+
_add_common_args(p_apply)
|
|
1608
|
+
p_apply.add_argument("--plan-file", required=True)
|
|
1609
|
+
|
|
1610
|
+
args = parser.parse_args()
|
|
1611
|
+
try:
|
|
1612
|
+
return {
|
|
1613
|
+
"prepare": command_prepare,
|
|
1614
|
+
"apply": command_apply,
|
|
1615
|
+
}[args.command](args)
|
|
1616
|
+
except Exception as exc:
|
|
1617
|
+
print(json.dumps({"error": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
|
1618
|
+
return 1
|
|
1619
|
+
|
|
1620
|
+
|
|
1621
|
+
if __name__ == "__main__":
|
|
1622
|
+
raise SystemExit(main())
|