dev-memory-cli 0.18.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1622 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ dev-memory-tidy: review and groom existing memory entries.
4
+
5
+ Setup turns lazy-init lumps into structured files (no order → ordered).
6
+ Tidy is the second pass that runs periodically: structured files drift
7
+ over time (stale items, duplicates, mistakes), and tidy lets the user
8
+ review every entry, mark keep/delete/edit, and write the result back.
9
+
10
+ Workflow:
11
+ 1. `prepare` — scan the target scope (branch + optional repo), parse
12
+ each .md file into top-level bullet entries, optionally inject
13
+ agent hints (STALE/DUP/OK/UNCLEAR + reason), render a static
14
+ review.html the user opens locally and exports a plan.json from.
15
+ 2. `apply` — read the user's plan.json, snapshot the current scope
16
+ into tidy_backup_<ts>/, then rewrite each file according to the
17
+ keep/delete/edit actions per entry.
18
+
19
+ Skipped on purpose:
20
+ - Auto-sync blocks (between AUTO-GENERATED-START/END) are never
21
+ surfaced — they're regenerated by other commands.
22
+ - Manifest.json is not a tidy target; it's structured data, not prose.
23
+ """
24
+
25
+ import argparse
26
+ import hashlib
27
+ import json
28
+ import shutil
29
+ import sys
30
+ from datetime import datetime, timezone
31
+ from pathlib import Path
32
+
33
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
34
+
35
+ from dev_memory_common import (
36
+ AUTO_END,
37
+ AUTO_START,
38
+ append_log_event,
39
+ ensure_branch_paths_exist,
40
+ join_sections,
41
+ split_sections,
42
+ template_decisions,
43
+ template_glossary,
44
+ template_overview,
45
+ template_pending_promotion,
46
+ template_progress,
47
+ template_repo_decisions,
48
+ template_repo_glossary,
49
+ template_repo_overview,
50
+ template_risks,
51
+ template_unsorted,
52
+ )
53
+
54
+ HTML_TEMPLATE_PATH = Path(__file__).resolve().parent / "assets" / "tidy_review.html"
55
+
56
+ # Map file_key -> callable that produces a fresh v2 template body. Used by
57
+ # the reset-file proposal action when an entry-level cleanup is too coarse
58
+ # (e.g. v1 → v2 left H3 skeleton sections that tidy can't surgically remove).
59
+ def _template_for(file_key, branch_name, repo_name):
60
+ table = {
61
+ "overview": lambda: template_overview(branch_name),
62
+ "decisions": lambda: template_decisions(branch_name),
63
+ "progress": lambda: template_progress(branch_name),
64
+ "risks": lambda: template_risks(branch_name),
65
+ "glossary": lambda: template_glossary(branch_name),
66
+ "unsorted": lambda: template_unsorted(),
67
+ "pending_promotion": lambda: template_pending_promotion(),
68
+ "repo_overview": lambda: template_repo_overview(repo_name),
69
+ "repo_decisions": lambda: template_repo_decisions(repo_name),
70
+ "repo_glossary": lambda: template_repo_glossary(repo_name),
71
+ }
72
+ fn = table.get(file_key)
73
+ return fn() if fn else None
74
+
75
+ # Files we tidy. Keys map to dev_memory_common.asset_paths() entries.
76
+ BRANCH_FILE_KEYS = (
77
+ "overview",
78
+ "decisions",
79
+ "progress",
80
+ "risks",
81
+ "glossary",
82
+ "unsorted",
83
+ "pending_promotion",
84
+ )
85
+ REPO_FILE_KEYS = (
86
+ "repo_overview",
87
+ "repo_decisions",
88
+ "repo_glossary",
89
+ )
90
+
91
+ VALID_HINTS = {"STALE", "DUP", "OK", "UNCLEAR", "ORPHAN"}
92
+
93
+ # Default age (days) past which a file is considered "untouched" — entries
94
+ # inside such files get auto-tagged with hint=STALE so the user reviews them.
95
+ # Pulled from log.md's per-action timestamps. Tunable via the CLI flag
96
+ # `--stale-after-days`; the default is conservative because false STALEs
97
+ # annoy users more than missed ones.
98
+ AUTO_STALE_THRESHOLD_DAYS = 30
99
+
100
+ # Minimum token length for ORPHAN detection. Shorter tokens (e.g. "API",
101
+ # "git") match too much across the codebase and produce false negatives —
102
+ # the hint becomes useless. Tunable but rarely worth it.
103
+ AUTO_ORPHAN_MIN_TOKEN_LEN = 4
104
+ VALID_ENTRY_ACTIONS = {"keep", "delete", "edit"}
105
+ VALID_PROPOSAL_TYPES = {"delete-entries", "delete-section", "reset-file", "edit-entries", "delete-block"}
106
+ VALID_PRIORITIES = {"P0", "P1", "P2", "P3", "P4"}
107
+
108
+ # Heuristic: lines like `**Why:** ...`, `**How to apply:** ...`, or
109
+ # `**xxx**:...` are "orphan paragraphs" — narrative attached to the
110
+ # previous bullet but not itself in any bullet list. _parse_blocks absorbs
111
+ # them into the preceding block so a single delete-block takes them out
112
+ # together with the bullets they explain. Two accepted shapes:
113
+ # 1. `**Foo:** rest` — colon inside the bold span (common in CN/EN)
114
+ # 2. `**Foo**: rest` — colon (half- or full-width) outside the bold span
115
+ import re as _re_block
116
+ _ORPHAN_PARAGRAPH_RE = _re_block.compile(
117
+ r"^\s*\*\*[^*\n]+[::]\*\*\s|^\s*\*\*[^*\n]+\*\*\s*[::]"
118
+ )
119
+
120
+
121
+ def _now_stamp():
122
+ return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
123
+
124
+
125
+ def _strip_auto_block(body):
126
+ """Remove the AUTO-GENERATED block from a section body so tidy
127
+ doesn't show machine-managed state to the user. Returns the body
128
+ with the block (and surrounding blank lines) removed."""
129
+ if AUTO_START not in body or AUTO_END not in body:
130
+ return body
131
+ start = body.index(AUTO_START)
132
+ end = body.index(AUTO_END) + len(AUTO_END)
133
+ return (body[:start].rstrip() + "\n" + body[end:].lstrip()).strip()
134
+
135
+
136
+ def _parse_entries(body):
137
+ """Split a section body into top-level bullet entries. A continuation
138
+ line (indented or non-bullet) is appended to the previous entry. Returns
139
+ a list of {entry_idx, text} dicts. Entries that match placeholder
140
+ markers (待补充 etc.) are returned with `is_placeholder=True` so the
141
+ UI can render them faded but still allow delete/edit."""
142
+ entries = []
143
+ current = None
144
+ for line in body.splitlines():
145
+ stripped = line.strip()
146
+ if not stripped:
147
+ if current is not None:
148
+ entries.append(current)
149
+ current = None
150
+ continue
151
+ if stripped.startswith("- "):
152
+ if current is not None:
153
+ entries.append(current)
154
+ current = {"text": stripped[2:].strip(), "raw_lines": [line]}
155
+ elif current is not None:
156
+ # Continuation: keep the original indentation when reconstructing.
157
+ current["text"] = current["text"] + "\n" + stripped
158
+ current["raw_lines"].append(line)
159
+ # Lines before any bullet (free-form prose) are ignored — tidy
160
+ # only operates on bullet-style entries to keep the action set
161
+ # finite. Free-form prose stays untouched in the file.
162
+ if current is not None:
163
+ entries.append(current)
164
+ indexed = []
165
+ for idx, ent in enumerate(entries):
166
+ text = ent["text"].strip()
167
+ is_placeholder = text in ("待补充", "待刷新")
168
+ indexed.append({
169
+ "entry_idx": idx,
170
+ "text": text,
171
+ "is_placeholder": is_placeholder,
172
+ })
173
+ return indexed
174
+
175
+
176
+ def _parse_blocks(body):
177
+ """Group a section body into semantic 'blocks'. A block is the smallest
178
+ unit an agent typically wants to delete or move as a whole: a top-level
179
+ bullet plus its indented sub-bullets, optionally trailed by bold
180
+ paragraph(s) like `**Why:** ...` that explain it.
181
+
182
+ Why this layer exists: `_parse_entries` treats every bullet line —
183
+ including indented sub-bullets — as a separate entry. That gives
184
+ surgical edits but loses the parent/child relationship. Block parsing
185
+ keeps `_parse_entries` intact and adds a second view that re-groups
186
+ those entries into the semantic unit an agent actually reasons about.
187
+
188
+ Block boundary rules:
189
+ - A top-level `- xxx` (no leading whitespace) starts a new block.
190
+ - Indented bullets ` - yyy` join the current block — they are still
191
+ entries (`_parse_entries` counts them), but they belong to the
192
+ same semantic unit as the top-level bullet above them. The block's
193
+ `entry_idx_range` therefore spans 1 top-level entry + N sub-bullet
194
+ entries.
195
+ - A bold-prefixed paragraph (`**Why:** ...`) immediately following
196
+ the last bullet — possibly across a single blank line — is
197
+ absorbed as the block's tail. Two consecutive blank lines, or any
198
+ other top-level bullet, terminates the tail. Paragraph lines do
199
+ NOT count as entries.
200
+ - Lines before the first bullet are not part of any block (mirrors
201
+ `_parse_entries` which also ignores them).
202
+
203
+ Invariant: every entry produced by `_parse_entries` on the same body
204
+ falls inside exactly one block. `entry_idx_range` is `[start, end]`
205
+ inclusive over those entry indices.
206
+ """
207
+ lines = body.splitlines()
208
+ blocks = []
209
+ current = None
210
+ pending_blank = False
211
+ entry_counter = -1 # increments on every bullet line (top-level or sub)
212
+
213
+ def push_current():
214
+ nonlocal current
215
+ if current is None:
216
+ return
217
+ while current["raw_lines"] and not current["raw_lines"][-1].strip():
218
+ current["raw_lines"].pop()
219
+ blocks.append(current)
220
+ current = None
221
+
222
+ def is_bullet_line(s):
223
+ return s.startswith("- ") or s == "-"
224
+
225
+ for raw in lines:
226
+ stripped = raw.strip()
227
+ if not stripped:
228
+ if current is not None:
229
+ if pending_blank:
230
+ push_current()
231
+ pending_blank = False
232
+ else:
233
+ pending_blank = True
234
+ current["raw_lines"].append(raw)
235
+ continue
236
+
237
+ is_top_bullet = is_bullet_line(stripped) and not (raw.startswith(" ") or raw.startswith("\t"))
238
+ is_sub_bullet = is_bullet_line(stripped) and (raw.startswith(" ") or raw.startswith("\t"))
239
+
240
+ if is_top_bullet:
241
+ push_current()
242
+ entry_counter += 1
243
+ current = {
244
+ "block_idx": len(blocks),
245
+ "entry_idx_start": entry_counter,
246
+ "entry_idx_end": entry_counter,
247
+ "first_line_preview": stripped[2:].strip()[:160] if stripped.startswith("- ") else stripped[:160],
248
+ "has_orphan_paragraphs": False,
249
+ "raw_lines": [raw],
250
+ }
251
+ pending_blank = False
252
+ continue
253
+
254
+ if current is None:
255
+ # Free-form prose before any bullet — drop (mirrors _parse_entries).
256
+ continue
257
+
258
+ if is_sub_bullet:
259
+ entry_counter += 1
260
+ current["entry_idx_end"] = entry_counter
261
+ current["raw_lines"].append(raw)
262
+ pending_blank = False
263
+ continue
264
+
265
+ # Indented non-bullet continuation (wrapped text under a bullet).
266
+ # _parse_entries treats this as continuation of the previous entry,
267
+ # so it doesn't advance entry_counter — neither do we.
268
+ if raw.startswith(" ") or raw.startswith("\t"):
269
+ current["raw_lines"].append(raw)
270
+ pending_blank = False
271
+ continue
272
+
273
+ # Non-indented, non-blank, not a bullet → candidate orphan paragraph.
274
+ # Absorb if it looks like `**Foo:** ...`. _parse_entries treats it
275
+ # as continuation of the previous (still-open) entry, but only when
276
+ # no blank line has separated them. If a single blank line came
277
+ # between bullet and paragraph, _parse_entries already closed the
278
+ # previous entry — but we still attach to the block (the paragraph
279
+ # belongs to the unit semantically).
280
+ if _ORPHAN_PARAGRAPH_RE.match(raw):
281
+ current["raw_lines"].append(raw)
282
+ current["has_orphan_paragraphs"] = True
283
+ pending_blank = False
284
+ continue
285
+
286
+ # Some other non-bullet prose between bullets — close the current
287
+ # block; this prose is not owned by anything.
288
+ push_current()
289
+ pending_blank = False
290
+
291
+ push_current()
292
+
293
+ return [
294
+ {
295
+ "block_idx": b["block_idx"],
296
+ "entry_idx_range": [b["entry_idx_start"], b["entry_idx_end"]],
297
+ "first_line_preview": b["first_line_preview"],
298
+ "has_orphan_paragraphs": b["has_orphan_paragraphs"],
299
+ "raw_lines": b["raw_lines"],
300
+ }
301
+ for b in blocks
302
+ ]
303
+
304
+
305
+ def _block_id(file_key, section_idx, block_idx):
306
+ return f"{file_key}::{section_idx}::block-{block_idx}"
307
+
308
+
309
+ def _block_content_hash(raw_lines):
310
+ """Compute a deterministic 16-char hex hash of a block's content.
311
+
312
+ `raw_lines` is the list of original source lines that make up the block
313
+ (top-level bullet + sub-bullets + absorbed orphan paragraphs, as captured
314
+ by `_parse_blocks`; `push_current` has already trimmed trailing blanks).
315
+ Joined with `\n` to form a stable byte sequence, SHA-256'd, then
316
+ truncated to 16 hex chars — long enough to keep collisions astronomically
317
+ unlikely across one section, short enough to keep plan.json readable.
318
+
319
+ Used by `apply` to detect that a block's content drifted between
320
+ `prepare` and `apply` (e.g. user edited a bullet but block_idx stayed
321
+ the same), so delete-block can refuse to nuke the wrong unit.
322
+ """
323
+ raw_text = "\n".join(raw_lines)
324
+ return hashlib.sha256(raw_text.encode("utf-8")).hexdigest()[:16]
325
+
326
+
327
+ def _parse_block_id(bid):
328
+ """Return (file_key, section_idx, block_idx) or None on bad input."""
329
+ if not isinstance(bid, str):
330
+ return None
331
+ parts = bid.split("::")
332
+ if len(parts) != 3 or not parts[2].startswith("block-"):
333
+ return None
334
+ try:
335
+ return parts[0], int(parts[1]), int(parts[2][len("block-"):])
336
+ except ValueError:
337
+ return None
338
+
339
+
340
+ def _file_label(file_key):
341
+ if file_key.startswith("repo_"):
342
+ return f"repo/{file_key[5:]}.md"
343
+ return f"branch/{file_key}.md"
344
+
345
+
346
+ def _scope_paths(paths, include_repo):
347
+ """Return ordered (file_key, path) tuples that exist."""
348
+ out = []
349
+ for key in BRANCH_FILE_KEYS:
350
+ p = paths.get(key)
351
+ if p and p.exists():
352
+ out.append((key, p))
353
+ if include_repo:
354
+ for key in REPO_FILE_KEYS:
355
+ p = paths.get(key)
356
+ if p and p.exists():
357
+ out.append((key, p))
358
+ return out
359
+
360
+
361
+ def _scan_scope(paths, include_repo):
362
+ """Scan target files into a tree of {file, sections, entries}."""
363
+ files = []
364
+ for file_key, path in _scope_paths(paths, include_repo):
365
+ content = path.read_text(encoding="utf-8")
366
+ prefix, sections = split_sections(content)
367
+ section_records = []
368
+ for section_idx, (title, body) in enumerate(sections):
369
+ cleaned = _strip_auto_block(body)
370
+ entries = _parse_entries(cleaned)
371
+ if not entries:
372
+ continue
373
+ section_records.append({
374
+ "section_idx": section_idx,
375
+ "title": title,
376
+ "entries": entries,
377
+ })
378
+ if not section_records:
379
+ continue
380
+ files.append({
381
+ "file_key": file_key,
382
+ "label": _file_label(file_key),
383
+ "path": str(path),
384
+ "sections": section_records,
385
+ })
386
+ return files
387
+
388
+
389
+ def _entry_id(file_key, section_idx, entry_idx):
390
+ return f"{file_key}::{section_idx}::{entry_idx}"
391
+
392
+
393
+ def _flatten_entries(files):
394
+ """Yield every entry with its stable ID for hint/plan lookup."""
395
+ for f in files:
396
+ for s in f["sections"]:
397
+ for e in s["entries"]:
398
+ yield _entry_id(f["file_key"], s["section_idx"], e["entry_idx"]), f, s, e
399
+
400
+
401
+ def _validate_hints(hints):
402
+ """Ensure each value is {label: ..., reason: ...} with valid label."""
403
+ if hints is None:
404
+ return {}
405
+ if not isinstance(hints, dict):
406
+ raise ValueError("hints must be a JSON object keyed by entry id")
407
+ cleaned = {}
408
+ for k, v in hints.items():
409
+ if not isinstance(v, dict):
410
+ raise ValueError(f"hint for {k} must be an object")
411
+ label = (v.get("label") or "").strip().upper()
412
+ reason = (v.get("reason") or "").strip()
413
+ if label and label not in VALID_HINTS:
414
+ raise ValueError(f"hint label for {k} must be one of {sorted(VALID_HINTS)}")
415
+ cleaned[k] = {"label": label, "reason": reason}
416
+ return cleaned
417
+
418
+
419
+ def _validate_proposals(raw):
420
+ """Each proposal: {id, title, reason, actions: [...]}.
421
+ Each action variant: delete-entries / delete-section / reset-file /
422
+ edit-entries. We only sanitise here — actual semantics are interpreted
423
+ at apply time after the user accepts via the HTML."""
424
+ if raw is None:
425
+ return []
426
+ if not isinstance(raw, list):
427
+ raise ValueError("proposals must be a JSON array")
428
+ cleaned = []
429
+ seen_ids = set()
430
+ for i, p in enumerate(raw):
431
+ if not isinstance(p, dict):
432
+ raise ValueError(f"proposal #{i} must be an object")
433
+ pid = (p.get("id") or f"p{i}").strip()
434
+ if pid in seen_ids:
435
+ raise ValueError(f"duplicate proposal id {pid!r}")
436
+ seen_ids.add(pid)
437
+ title = (p.get("title") or "").strip()
438
+ reason = (p.get("reason") or "").strip()
439
+ actions = p.get("actions") or []
440
+ if not title:
441
+ raise ValueError(f"proposal {pid} missing title")
442
+ if not actions:
443
+ raise ValueError(f"proposal {pid} has no actions")
444
+ cleaned_actions = []
445
+ for j, a in enumerate(actions):
446
+ if not isinstance(a, dict):
447
+ raise ValueError(f"proposal {pid} action #{j} must be an object")
448
+ atype = (a.get("type") or "").strip()
449
+ if atype not in VALID_PROPOSAL_TYPES:
450
+ raise ValueError(
451
+ f"proposal {pid} action #{j} has invalid type {atype!r}; "
452
+ f"must be one of {sorted(VALID_PROPOSAL_TYPES)}"
453
+ )
454
+ if atype == "delete-block":
455
+ bid = (a.get("block_id") or "").strip()
456
+ if not bid or _parse_block_id(bid) is None:
457
+ raise ValueError(
458
+ f"proposal {pid} action #{j} delete-block needs block_id of form "
459
+ f"'<file_key>::<section_idx>::block-<N>' (got {bid!r})"
460
+ )
461
+ # Optional content fingerprint. apply will re-hash the block
462
+ # at the resolved location and refuse to delete if the value
463
+ # drifted; absent means "skip the content check".
464
+ ech = a.get("expected_content_hash")
465
+ if ech is not None and not isinstance(ech, str):
466
+ raise ValueError(
467
+ f"proposal {pid} action #{j} delete-block expected_content_hash "
468
+ f"must be a string if provided (got {type(ech).__name__})"
469
+ )
470
+ cleaned_actions.append(a)
471
+ priority = (p.get("priority") or "").strip().upper()
472
+ if priority and priority not in VALID_PRIORITIES:
473
+ raise ValueError(
474
+ f"proposal {pid} priority must be one of {sorted(VALID_PRIORITIES)} or empty, got {priority!r}"
475
+ )
476
+ cleaned.append({
477
+ "id": pid,
478
+ "title": title,
479
+ "reason": reason,
480
+ "priority": priority,
481
+ "actions": cleaned_actions,
482
+ })
483
+ return cleaned
484
+
485
+
486
+ def _collect_blocks(files):
487
+ """Re-parse each scanned section into blocks. Returns a tuple of
488
+ (blocks_by_file_section, flat_block_index) where:
489
+
490
+ - blocks_by_file_section: {file_key: {section_idx: [block, ...]}}
491
+ with each block carrying its raw_lines (kept internally — not
492
+ emitted in the public JSON to keep that payload light).
493
+ - flat_block_index: list of public block dicts safe to dump as JSON,
494
+ each shaped {id, file, section, first_line_preview, entry_ids,
495
+ has_orphan_paragraphs}.
496
+
497
+ Re-parses on each call rather than threading block info through
498
+ `_scan_scope` so the existing `_scan_scope` shape stays untouched
499
+ (and downstream callers — html template — still see the same JSON).
500
+ """
501
+ by_fs = {}
502
+ flat = []
503
+ for f in files:
504
+ fk = f["file_key"]
505
+ by_fs[fk] = {}
506
+ # Re-read the section bodies (post _strip_auto_block) to get raw_lines.
507
+ # We can reconstruct them from the file because _scan_scope already
508
+ # filtered out empty sections — but we still need the original body,
509
+ # so re-read the file and re-split.
510
+ content = Path(f["path"]).read_text(encoding="utf-8")
511
+ _, sections = split_sections(content)
512
+ for s in f["sections"]:
513
+ sidx = s["section_idx"]
514
+ if sidx >= len(sections):
515
+ by_fs[fk][sidx] = []
516
+ continue
517
+ _title, body = sections[sidx]
518
+ cleaned = _strip_auto_block(body)
519
+ blocks = _parse_blocks(cleaned)
520
+ by_fs[fk][sidx] = blocks
521
+ for b in blocks:
522
+ start, end = b["entry_idx_range"]
523
+ entry_ids = [_entry_id(fk, sidx, i) for i in range(start, end + 1)]
524
+ flat.append({
525
+ "id": _block_id(fk, sidx, b["block_idx"]),
526
+ "file": f["label"],
527
+ "file_key": fk,
528
+ "section": s["title"],
529
+ "section_idx": sidx,
530
+ "first_line_preview": b["first_line_preview"],
531
+ "entry_ids": entry_ids,
532
+ "has_orphan_paragraphs": b["has_orphan_paragraphs"],
533
+ # 16-char fingerprint of the block's raw source bytes —
534
+ # agent copies this verbatim into plan.json's delete-block
535
+ # action as `expected_content_hash`; apply re-hashes the
536
+ # current file and refuses to delete if the value drifted.
537
+ "content_hash": _block_content_hash(b["raw_lines"]),
538
+ })
539
+ return by_fs, flat
540
+
541
+
542
+ def _render_annotated_md(files, blocks_by_fs, scope_meta, output_path):
543
+ """Write a single .md mirror with id / block / orphan comment markers
544
+ so an agent can read one file and recover entry id + block boundaries
545
+ without parsing the original markdown twice.
546
+
547
+ Format (see design §1.1):
548
+ # Tidy entries · <branch> · <timestamp>
549
+ ## 📄 branch/<file>.md
550
+ ### <section title>
551
+ <!-- block: <id> -->
552
+ - bullet <!-- id: <entry-id> -->
553
+ - sub (no id comment — continuation of the entry above)
554
+ **Why:** ... <!-- orphan: paragraph -->
555
+ <!-- /block -->
556
+
557
+ Placeholder entries (`- 待补充` / `- 待刷新`) get an extra
558
+ `<!-- placeholder -->` so the agent can ignore them at a glance.
559
+ AUTO-GENERATED blocks are not surfaced (already stripped upstream).
560
+ """
561
+ lines = [
562
+ f"# Tidy entries · {scope_meta['branch']} · {scope_meta['generated_at']}",
563
+ "",
564
+ ]
565
+ for f in files:
566
+ fk = f["file_key"]
567
+ lines.append(f"<!-- file: {fk} / {f['label']} -->")
568
+ lines.append(f"## 📄 {f['label']}")
569
+ lines.append("")
570
+ for s in f["sections"]:
571
+ sidx = s["section_idx"]
572
+ lines.append(f"<!-- section: {sidx} / {s['title']} -->")
573
+ lines.append(f"### {s['title']}")
574
+ blocks = blocks_by_fs.get(fk, {}).get(sidx, [])
575
+ if not blocks:
576
+ lines.append("")
577
+ continue
578
+ # Build a quick lookup: which entry_idx is the top-level bullet
579
+ # on each raw_line, so we know which line to tag with `<!-- id -->`.
580
+ entry_text_by_idx = {e["entry_idx"]: e for e in s["entries"]}
581
+ for b in blocks:
582
+ lines.append("")
583
+ lines.append(f"<!-- block: {_block_id(fk, sidx, b['block_idx'])} -->")
584
+ start, _end = b["entry_idx_range"]
585
+ # Walk raw_lines, tagging the first top-level bullet with the
586
+ # entry id. Sub-bullets and continuation lines are emitted
587
+ # bare. Orphan paragraph lines get `<!-- orphan: paragraph -->`.
588
+ seen_top_bullet = False
589
+ cur_entry_idx = start
590
+ for raw in b["raw_lines"]:
591
+ stripped = raw.strip()
592
+ if not stripped:
593
+ lines.append(raw)
594
+ continue
595
+ is_top_bullet = (
596
+ stripped.startswith("- ")
597
+ and not (raw.startswith(" ") or raw.startswith("\t"))
598
+ )
599
+ if is_top_bullet:
600
+ if seen_top_bullet:
601
+ cur_entry_idx += 1
602
+ seen_top_bullet = True
603
+ ent = entry_text_by_idx.get(cur_entry_idx, {})
604
+ eid = _entry_id(fk, sidx, cur_entry_idx)
605
+ suffix = f" <!-- id: {eid} -->"
606
+ if ent.get("is_placeholder"):
607
+ suffix += " <!-- placeholder -->"
608
+ lines.append(f"{raw}{suffix}")
609
+ elif _ORPHAN_PARAGRAPH_RE.match(raw):
610
+ lines.append(f"{raw} <!-- orphan: paragraph -->")
611
+ else:
612
+ # Sub-bullet or wrapped continuation — emit bare.
613
+ lines.append(raw)
614
+ lines.append("<!-- /block -->")
615
+ lines.append("")
616
+ lines.append("")
617
+ output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
618
+
619
+
620
+ def _render_html(files, hints, proposals, scope_meta, output_path):
621
+ """Inject scan + hints + proposals into the HTML template and write to disk."""
622
+ template = HTML_TEMPLATE_PATH.read_text(encoding="utf-8")
623
+ payload = {
624
+ "scope": scope_meta,
625
+ "files": files,
626
+ "hints": hints,
627
+ "proposals": proposals,
628
+ }
629
+ payload_json = json.dumps(payload, ensure_ascii=False, indent=2)
630
+ rendered = template.replace("__TIDY_DATA_PLACEHOLDER__", payload_json)
631
+ rendered = rendered.replace("__SCOPE_LABEL_PLACEHOLDER__",
632
+ f"{scope_meta['repo_key']} · {scope_meta['branch']}")
633
+ output_path.write_text(rendered, encoding="utf-8")
634
+
635
+
636
+ # ---------------------------------------------------------------------------
637
+ # Auto-hint generators (Karpathy lint pattern)
638
+ # ---------------------------------------------------------------------------
639
+ #
640
+ # tidy prepare can run two cheap "positive-signal" passes that bake hints into
641
+ # the review payload before the user opens the HTML:
642
+ #
643
+ # - STALE-by-log : files whose last log.md mention is older than the
644
+ # configured threshold get their entries hinted STALE.
645
+ # Pulls timestamps + targets from log.md, so it costs one
646
+ # file read + a regex scan; no external state.
647
+ # - ORPHAN-glossary : glossary entries whose key phrase doesn't appear
648
+ # anywhere else in the wiki get hinted ORPHAN — the
649
+ # term is dangling, not load-bearing.
650
+ #
651
+ # Both signals are intentionally conservative — false positives waste user
652
+ # attention more than misses. ORPHAN skips tokens shorter than
653
+ # AUTO_ORPHAN_MIN_TOKEN_LEN (short tokens like "API"/"hook" match too much
654
+ # elsewhere); STALE only fires when log.md has a timestamp, never on
655
+ # "file never seen" (cold-start would carpet-bomb the UI).
656
+
657
+
658
+ # Reverse of capture's _label(): "branch/decisions.md" / "repo/glossary.md"
659
+ # → file_key. Order matters: pending-promotion has a custom shape, repo_*
660
+ # prefix wins over branch/, and we tolerate the `(mode)` trailer.
661
+ def _file_label_to_key(label):
662
+ if not label:
663
+ return None
664
+ label = label.strip()
665
+ paren = label.find("(")
666
+ if paren > 0:
667
+ label = label[:paren].strip()
668
+ if label == "branch/pending-promotion.md":
669
+ return "pending_promotion"
670
+ if label.startswith("repo/") and label.endswith(".md"):
671
+ return "repo_" + label[len("repo/"):-len(".md")]
672
+ if label.startswith("branch/") and label.endswith(".md"):
673
+ return label[len("branch/"):-len(".md")]
674
+ return None
675
+
676
+
677
+ _LOG_HEADER_RE = __import__("re").compile(r"^## \[(?P<ts>[^\]]+)\]")
678
+ _LOG_TARGETS_RE = __import__("re").compile(r"^- targets:\s*(?P<rest>.+)$")
679
+
680
+
681
+ def _parse_log_last_touched(log_path):
682
+ """Return {file_key: latest_iso_timestamp} by walking log.md once.
683
+
684
+ Algorithm: a log entry block is `## [...]` followed by a few `- key:
685
+ value` lines until the next H2 or EOF. We only care about the `targets`
686
+ detail; each comma-separated file label gets its file_key resolved and
687
+ the block's timestamp recorded. Later timestamps for the same file_key
688
+ overwrite (entries are append-only so later = file position later =
689
+ nothing; we still compare lexicographically since ISO sorts).
690
+ """
691
+ if log_path is None or not log_path.exists():
692
+ return {}
693
+ last_seen = {}
694
+ current_ts = None
695
+ for raw in log_path.read_text(encoding="utf-8").splitlines():
696
+ m = _LOG_HEADER_RE.match(raw)
697
+ if m:
698
+ current_ts = m.group("ts").strip()
699
+ continue
700
+ if current_ts is None:
701
+ continue
702
+ m = _LOG_TARGETS_RE.match(raw)
703
+ if not m:
704
+ continue
705
+ rest = m.group("rest").strip()
706
+ for chunk in rest.split(","):
707
+ file_key = _file_label_to_key(chunk)
708
+ if not file_key:
709
+ continue
710
+ prev = last_seen.get(file_key)
711
+ if prev is None or current_ts > prev:
712
+ last_seen[file_key] = current_ts
713
+ return last_seen
714
+
715
+
716
+ def _iso_to_age_days(iso_ts):
717
+ """Best-effort: parse an ISO timestamp string into "days ago" relative
718
+ to now. Returns None on parse failure so callers can skip cleanly."""
719
+ if not iso_ts:
720
+ return None
721
+ try:
722
+ # datetime.fromisoformat handles "+00:00" suffixes from now_iso();
723
+ # the Z suffix some legacy callers emit is normalized first.
724
+ s = iso_ts.replace("Z", "+00:00")
725
+ dt = datetime.fromisoformat(s)
726
+ if dt.tzinfo is None:
727
+ dt = dt.replace(tzinfo=timezone.utc)
728
+ delta = datetime.now(timezone.utc) - dt
729
+ return max(0, delta.days)
730
+ except (ValueError, TypeError):
731
+ return None
732
+
733
+
734
+ def _stale_hints_from_log(files, log_paths, *, threshold_days):
735
+ """Return {entry_id: {label, reason}} for entries inside files whose log
736
+ last-touched age exceeds threshold_days. log_paths is an iterable of
737
+ branch/repo log.md paths — we union them so a file touched recently in
738
+ either layer is considered fresh."""
739
+ last_seen = {}
740
+ for lp in log_paths:
741
+ for fk, ts in _parse_log_last_touched(lp).items():
742
+ prev = last_seen.get(fk)
743
+ if prev is None or ts > prev:
744
+ last_seen[fk] = ts
745
+
746
+ out = {}
747
+ for f in files:
748
+ fk = f["file_key"]
749
+ ts = last_seen.get(fk)
750
+ age_days = _iso_to_age_days(ts)
751
+ if age_days is None or age_days < threshold_days:
752
+ continue
753
+ reason = f"log 上次提到 {age_days} 天前 (> {threshold_days})"
754
+ for s in f["sections"]:
755
+ for e in s["entries"]:
756
+ if e.get("is_placeholder"):
757
+ continue
758
+ eid = _entry_id(fk, s["section_idx"], e["entry_idx"])
759
+ out[eid] = {"label": "STALE", "reason": reason}
760
+ return out
761
+
762
+
763
+ _ORPHAN_KEY_PHRASE_TRIM_RE = __import__("re").compile(r"^[\s\-*`]+|[\s`*]+$")
764
+ _ORPHAN_MARKDOWN_BOLD_RE = __import__("re").compile(r"\*\*([^*]+)\*\*")
765
+
766
+
767
+ def _glossary_key_phrase(entry_text):
768
+ """Extract the "key phrase" of a glossary entry — the bit other files
769
+ would have to reference for the entry to be load-bearing.
770
+
771
+ Heuristic: take the first line, strip bullet/markdown markers, then take
772
+ the substring before the first `:` or `:` (half/full-width colon).
773
+ Falls back to the trimmed first line if no colon. Returns "" if nothing
774
+ extractable, so callers can skip cleanly.
775
+ """
776
+ if not entry_text:
777
+ return ""
778
+ first_line = entry_text.splitlines()[0] if entry_text else ""
779
+ s = first_line.strip()
780
+ if s.startswith(("- ", "* ")):
781
+ s = s[2:]
782
+ # Strip bold markdown wrappers if the whole token is wrapped.
783
+ m = _ORPHAN_MARKDOWN_BOLD_RE.match(s)
784
+ if m:
785
+ s = m.group(1) + s[m.end():]
786
+ for sep in (":", ":"):
787
+ idx = s.find(sep)
788
+ if idx > 0:
789
+ s = s[:idx]
790
+ break
791
+ s = _ORPHAN_KEY_PHRASE_TRIM_RE.sub("", s).strip()
792
+ return s
793
+
794
+
795
+ def _orphan_hints_from_glossary(files, paths):
796
+ """Return {entry_id: {label, reason}} for glossary entries whose key
797
+ phrase never appears in any other file in the wiki.
798
+
799
+ Why "never": we're after dangling terms, not under-referenced ones. A
800
+ term mentioned even once outside its definition is doing some work; an
801
+ entry that appears literally nowhere else is either superseded, never
802
+ actually used, or a private note that drifted in. Worth a user look.
803
+ """
804
+ # Build the haystack: full text of every non-glossary file in `paths`.
805
+ glossary_keys = {"glossary", "repo_glossary"}
806
+ haystack_chunks = []
807
+ for key, path in paths.items():
808
+ if key in glossary_keys:
809
+ continue
810
+ if not hasattr(path, "exists") or not path.exists():
811
+ continue
812
+ try:
813
+ haystack_chunks.append(path.read_text(encoding="utf-8"))
814
+ except OSError:
815
+ continue
816
+ haystack = "\n".join(haystack_chunks)
817
+
818
+ out = {}
819
+ for f in files:
820
+ if f["file_key"] not in glossary_keys:
821
+ continue
822
+ for s in f["sections"]:
823
+ for e in s["entries"]:
824
+ if e.get("is_placeholder"):
825
+ continue
826
+ phrase = _glossary_key_phrase(e.get("text", ""))
827
+ if not phrase or len(phrase) < AUTO_ORPHAN_MIN_TOKEN_LEN:
828
+ continue
829
+ if phrase in haystack:
830
+ continue
831
+ eid = _entry_id(f["file_key"], s["section_idx"], e["entry_idx"])
832
+ out[eid] = {
833
+ "label": "ORPHAN",
834
+ "reason": f"术语 '{phrase}' 在其他文件零引用",
835
+ }
836
+ return out
837
+
838
+
839
+ def _compute_auto_hints(files, paths, *, stale_threshold_days):
840
+ """Run both auto-hint passes and merge. ORPHAN wins over STALE on the
841
+ same entry — a never-referenced glossary term is a sharper signal than
842
+ "file hasn't been touched in a while" (which would tag the entire
843
+ glossary STALE on a cold log)."""
844
+ log_paths = []
845
+ if paths.get("log"):
846
+ log_paths.append(paths["log"])
847
+ if paths.get("repo_log") and paths.get("repo_log") != paths.get("log"):
848
+ log_paths.append(paths["repo_log"])
849
+
850
+ merged = _stale_hints_from_log(files, log_paths, threshold_days=stale_threshold_days)
851
+ orphan = _orphan_hints_from_glossary(files, paths)
852
+ merged.update(orphan) # ORPHAN beats STALE per the docstring.
853
+ return merged
854
+
855
+
856
+ def command_prepare(args):
857
+ repo_root, branch_name, branch_key, storage_root, repo_key, repo_dir, branch_dir, paths = ensure_branch_paths_exist(
858
+ args.repo, args.context_dir, args.branch
859
+ )
860
+ include_repo = args.scope == "branch+repo"
861
+ files = _scan_scope(paths, include_repo)
862
+
863
+ user_hints = {}
864
+ if args.hints_json:
865
+ user_hints = _validate_hints(json.loads(args.hints_json))
866
+ elif args.hints_file:
867
+ user_hints = _validate_hints(json.loads(Path(args.hints_file).read_text(encoding="utf-8")))
868
+
869
+ # Auto-hints: derived from log.md timestamps + glossary cross-references.
870
+ # User-supplied hints win on the same entry — user signals are explicit
871
+ # judgment calls and shouldn't be silently overridden by heuristics.
872
+ if args.auto_hints:
873
+ auto = _compute_auto_hints(
874
+ files, paths, stale_threshold_days=args.stale_after_days,
875
+ )
876
+ else:
877
+ auto = {}
878
+ hints = dict(auto)
879
+ hints.update(user_hints)
880
+
881
+ proposals = []
882
+ if args.proposals_json:
883
+ proposals = _validate_proposals(json.loads(args.proposals_json))
884
+ elif args.proposals_file:
885
+ proposals = _validate_proposals(json.loads(Path(args.proposals_file).read_text(encoding="utf-8")))
886
+
887
+ scope_meta = {
888
+ "repo_root": str(repo_root),
889
+ "repo_key": repo_key,
890
+ "branch": branch_name,
891
+ "branch_dir": str(branch_dir),
892
+ "include_repo": include_repo,
893
+ "generated_at": _now_stamp(),
894
+ }
895
+
896
+ review_dir = branch_dir / "tidy_review"
897
+ review_dir.mkdir(parents=True, exist_ok=True)
898
+ out_path = review_dir / f"review_{scope_meta['generated_at']}.html"
899
+ _render_html(files, hints, proposals, scope_meta, out_path)
900
+
901
+ # Build the annotated md mirror — agent reads this single file to recover
902
+ # entry id + block boundaries without re-parsing the original markdown.
903
+ blocks_by_fs, flat_blocks = _collect_blocks(files)
904
+ annotated_md_path = review_dir / f"entries_{scope_meta['generated_at']}.annotated.md"
905
+ _render_annotated_md(files, blocks_by_fs, scope_meta, annotated_md_path)
906
+
907
+ entry_index = []
908
+ for entry_id, f, s, e in _flatten_entries(files):
909
+ entry_index.append({
910
+ "id": entry_id,
911
+ "file": f["label"],
912
+ "section": s["title"],
913
+ "text": e["text"],
914
+ "is_placeholder": e["is_placeholder"],
915
+ })
916
+
917
+ section_index = []
918
+ for f in files:
919
+ for s in f["sections"]:
920
+ section_index.append({
921
+ "file_key": f["file_key"],
922
+ "file": f["label"],
923
+ "section_idx": s["section_idx"],
924
+ "title": s["title"],
925
+ "entry_count": len(s["entries"]),
926
+ })
927
+
928
+ hints_summary = {
929
+ "auto_enabled": bool(args.auto_hints),
930
+ "stale_after_days": args.stale_after_days,
931
+ "auto_count": len(auto),
932
+ "auto_by_label": {
933
+ label: sum(1 for v in auto.values() if v["label"] == label)
934
+ for label in sorted({v["label"] for v in auto.values()})
935
+ } if auto else {},
936
+ "user_count": len(user_hints),
937
+ "total_count": len(hints),
938
+ }
939
+
940
+ print(json.dumps({
941
+ "mode": "prepare",
942
+ "review_html": str(out_path),
943
+ "open_url": f"file://{out_path}",
944
+ "annotated_md": str(annotated_md_path),
945
+ "annotated_md_open": f"file://{annotated_md_path}",
946
+ "scope": scope_meta,
947
+ "hints_summary": hints_summary,
948
+ "entry_count": len(entry_index),
949
+ "entries": entry_index,
950
+ "section_count": len(section_index),
951
+ "sections": section_index,
952
+ "block_count": len(flat_blocks),
953
+ "blocks": flat_blocks,
954
+ "proposal_count": len(proposals),
955
+ "next_steps": [
956
+ "1) Read annotated_md (single Read) to recover entry id + block boundaries + orphan paragraph markers — don't re-read the raw .md files for structure.",
957
+ "2) Decide semantic 'fixes' to propose. Prefer `delete-block` over per-entry `delete-entries` when removing a full semantic unit (block ids are in `blocks[]`).",
958
+ "3) Re-run prepare with --proposals-json or --proposals-file to bake proposal cards into the HTML.",
959
+ "4) Tell the user to open open_url; default each proposal accept/reject; expand to inspect; export plan.json.",
960
+ "5) When plan.json is downloaded, run: dev-memory-cli tidy apply --plan-file <path>.",
961
+ ],
962
+ }, ensure_ascii=False, indent=2))
963
+ return 0
964
+
965
+
966
+ def _backup_scope(branch_dir, paths, include_repo, stamp):
967
+ """Snapshot every file we may touch into tidy_backup_<stamp>/. Files
968
+ are stored under the same relative key naming used internally so
969
+ apply can self-document what was preserved."""
970
+ backup_dir = branch_dir / f"tidy_backup_{stamp}"
971
+ backup_dir.mkdir(parents=True, exist_ok=True)
972
+ saved = []
973
+ for file_key, path in _scope_paths(paths, include_repo):
974
+ dest = backup_dir / f"{file_key}.md"
975
+ shutil.copy2(path, dest)
976
+ saved.append({"file_key": file_key, "src": str(path), "backup": str(dest)})
977
+ (backup_dir / "manifest.json").write_text(
978
+ json.dumps({"saved_at": stamp, "files": saved}, ensure_ascii=False, indent=2),
979
+ encoding="utf-8",
980
+ )
981
+ return backup_dir, saved
982
+
983
+
984
+ def _apply_actions_to_section(body, entry_actions):
985
+ """Rewrite a section body so each entry is either kept, removed,
986
+ or replaced by its `new_text`. entry_actions is keyed by entry_idx.
987
+ Lines outside top-level bullets pass through unchanged."""
988
+ lines = body.splitlines()
989
+ out_blocks = []
990
+ current_block = []
991
+ current_idx = -1
992
+
993
+ def flush():
994
+ nonlocal current_block, current_idx
995
+ if not current_block:
996
+ return
997
+ action = entry_actions.get(current_idx, {"action": "keep"})
998
+ kind = action.get("action", "keep")
999
+ if kind == "delete":
1000
+ pass
1001
+ elif kind == "edit":
1002
+ new_text = (action.get("new_text") or "").strip()
1003
+ if new_text:
1004
+ # Render edited text as a fresh bullet, splitting on
1005
+ # newlines so multi-line edits stay a single entry with
1006
+ # continuation lines indented to depth 2.
1007
+ first, *rest = new_text.split("\n")
1008
+ out_blocks.append("- " + first.strip())
1009
+ for cont in rest:
1010
+ out_blocks.append(" " + cont.strip())
1011
+ else:
1012
+ out_blocks.extend(current_block)
1013
+ current_block = []
1014
+ current_idx = -1
1015
+
1016
+ entry_seen = -1
1017
+ for raw in lines:
1018
+ stripped = raw.strip()
1019
+ if stripped.startswith("- "):
1020
+ flush()
1021
+ entry_seen += 1
1022
+ current_idx = entry_seen
1023
+ current_block = [raw]
1024
+ elif current_block and stripped and (raw.startswith(" ") or raw.startswith("\t")):
1025
+ current_block.append(raw)
1026
+ elif not stripped:
1027
+ flush()
1028
+ out_blocks.append(raw)
1029
+ else:
1030
+ # Free-form prose between bullets — leave alone.
1031
+ flush()
1032
+ out_blocks.append(raw)
1033
+ flush()
1034
+
1035
+ # Trim trailing blank lines for a tidy file.
1036
+ while out_blocks and not out_blocks[-1].strip():
1037
+ out_blocks.pop()
1038
+ rebuilt = "\n".join(out_blocks).strip()
1039
+ if not rebuilt:
1040
+ # Section emptied out entirely — leave the placeholder so the
1041
+ # file remains valid and capture's _section_is_placeholder_only
1042
+ # logic can still detect it.
1043
+ return "- 待补充"
1044
+ return rebuilt
1045
+
1046
+
1047
+ def _rewrite_file(path, plan_actions_by_section):
1048
+ """For a single file, apply per-section per-entry actions then
1049
+ rewrite the file. Auto-sync blocks pass through untouched."""
1050
+ content = path.read_text(encoding="utf-8")
1051
+ prefix, sections = split_sections(content)
1052
+ new_sections = []
1053
+ for section_idx, (title, body) in enumerate(sections):
1054
+ actions = plan_actions_by_section.get(section_idx)
1055
+ if not actions:
1056
+ new_sections.append((title, body))
1057
+ continue
1058
+ # Preserve auto-sync block: split body at AUTO_START, only tidy
1059
+ # the part before; auto-block goes back as-is.
1060
+ if AUTO_START in body and AUTO_END in body:
1061
+ head_end = body.index(AUTO_START)
1062
+ head = body[:head_end].rstrip()
1063
+ tail = body[head_end:]
1064
+ new_head = _apply_actions_to_section(head, actions)
1065
+ new_body = (new_head + "\n\n" + tail).strip()
1066
+ else:
1067
+ new_body = _apply_actions_to_section(body, actions)
1068
+ new_sections.append((title, new_body))
1069
+ path.write_text(join_sections(prefix, new_sections), encoding="utf-8")
1070
+
1071
+
1072
+ def _delete_blocks_from_section(body, block_drops):
1073
+ """Re-parse the section body into blocks (source of truth at apply
1074
+ time — block_idx from prepare is informational) and drop the listed
1075
+ block indices, returning the rewritten body. Block deletion takes the
1076
+ block's full raw_lines: top-level bullet + any sub-bullets + absorbed
1077
+ orphan paragraphs.
1078
+
1079
+ `block_drops` is a mapping `{block_idx: expected_content_hash | None}`.
1080
+ A non-None hash is compared against the live block's hash after
1081
+ re-parsing; mismatch lands in the invalid list with reason
1082
+ `content_hash_mismatch` and the block is NOT deleted. None skips the
1083
+ content check (back-compat for plans written before the guard existed).
1084
+
1085
+ Returns (rebuilt_body, invalid_entries) where each invalid entry is a
1086
+ dict with at least `block_idx` and `reason`; mismatches also include
1087
+ `expected` and `actual` hash fields for debugging.
1088
+ """
1089
+ blocks = _parse_blocks(body)
1090
+ invalid = []
1091
+ drop = set()
1092
+ for bidx, expected_hash in block_drops.items():
1093
+ if not (0 <= bidx < len(blocks)):
1094
+ invalid.append({
1095
+ "block_idx": bidx,
1096
+ "reason": "block_idx out of range after re-parse",
1097
+ })
1098
+ continue
1099
+ if expected_hash is not None:
1100
+ actual_hash = _block_content_hash(blocks[bidx]["raw_lines"])
1101
+ if actual_hash != expected_hash:
1102
+ invalid.append({
1103
+ "block_idx": bidx,
1104
+ "reason": "content_hash_mismatch",
1105
+ "expected": expected_hash,
1106
+ "actual": actual_hash,
1107
+ })
1108
+ continue
1109
+ drop.add(bidx)
1110
+
1111
+ # Strategy: replay the original body line-by-line, skipping lines that
1112
+ # belong to a dropped block. Because raw_lines is captured verbatim
1113
+ # from the body during parsing, we can identify the lines by their
1114
+ # position within the body.
1115
+ #
1116
+ # _parse_blocks consumes lines in order; rebuild by walking the same
1117
+ # loop and emitting lines for kept blocks only.
1118
+ lines = body.splitlines()
1119
+ # Re-derive line ownership: walk lines and mark which (if any) block
1120
+ # each line ends up in, using the same logic as _parse_blocks. To
1121
+ # avoid duplicating that logic, fall back to a simpler approach —
1122
+ # re-walk lines and check membership against block raw_lines by index
1123
+ # within the body.
1124
+ #
1125
+ # Trick: every raw line in a block's raw_lines is a verbatim slice of
1126
+ # the original lines list in order. Concatenate kept blocks' raw_lines
1127
+ # plus an explicit blank between them.
1128
+ kept_blocks = [b for i, b in enumerate(blocks) if i not in drop]
1129
+ if not kept_blocks:
1130
+ # Section emptied — preserve any pre-bullet prose if present, else
1131
+ # fall back to the placeholder so capture's
1132
+ # _section_is_placeholder_only still detects it.
1133
+ prefix_lines = []
1134
+ for raw in lines:
1135
+ stripped = raw.strip()
1136
+ if stripped.startswith("- ") and not (raw.startswith(" ") or raw.startswith("\t")):
1137
+ break
1138
+ prefix_lines.append(raw)
1139
+ while prefix_lines and not prefix_lines[-1].strip():
1140
+ prefix_lines.pop()
1141
+ rebuilt = "\n".join(prefix_lines).strip()
1142
+ return (rebuilt + "\n\n- 待补充").strip() if rebuilt else "- 待补充", invalid
1143
+
1144
+ # Walk original lines, emit only those belonging to kept blocks or to
1145
+ # the pre-bullet prefix. To know "belongs to a kept block", track block
1146
+ # boundaries by replaying the parser inline.
1147
+ keep_idx_set = {i for i in range(len(blocks)) if i not in drop}
1148
+ out_lines = []
1149
+ cur_block_idx = -1 # which block index we're currently inside (-1 = prefix)
1150
+ pending_blank = False
1151
+ in_block = False
1152
+ saw_first_bullet = False
1153
+
1154
+ # Helper: emit prefix lines (before any bullet).
1155
+ for raw in lines:
1156
+ stripped = raw.strip()
1157
+ if not saw_first_bullet:
1158
+ # Prefix region — emit until we hit the first top-level bullet.
1159
+ if stripped.startswith("- ") and not (raw.startswith(" ") or raw.startswith("\t")):
1160
+ saw_first_bullet = True
1161
+ cur_block_idx = 0
1162
+ if cur_block_idx in keep_idx_set:
1163
+ out_lines.append(raw)
1164
+ in_block = True
1165
+ else:
1166
+ in_block = False
1167
+ continue
1168
+ out_lines.append(raw)
1169
+ continue
1170
+
1171
+ # We're past the first bullet — emit only if current block is kept.
1172
+ if stripped.startswith("- ") and not (raw.startswith(" ") or raw.startswith("\t")):
1173
+ # New top-level bullet → advance block idx.
1174
+ cur_block_idx += 1
1175
+ in_block = cur_block_idx in keep_idx_set
1176
+ if in_block:
1177
+ # If pending_blank was set inside a kept context, emit it now.
1178
+ out_lines.append(raw)
1179
+ pending_blank = False
1180
+ continue
1181
+
1182
+ if not stripped:
1183
+ # Blank line: emit if we're inside a kept block — but mark
1184
+ # pending so we drop it if the block ends and goes elsewhere.
1185
+ if in_block:
1186
+ out_lines.append(raw)
1187
+ continue
1188
+
1189
+ if in_block:
1190
+ out_lines.append(raw)
1191
+ # else: drop — line belongs to a deleted block (sub-bullet, orphan
1192
+ # paragraph, etc.).
1193
+
1194
+ # Trim trailing blanks.
1195
+ while out_lines and not out_lines[-1].strip():
1196
+ out_lines.pop()
1197
+ rebuilt = "\n".join(out_lines).strip()
1198
+ if not rebuilt:
1199
+ return "- 待补充", invalid
1200
+ return rebuilt, invalid
1201
+
1202
+
1203
+ def _rewrite_file_delete_blocks(path, blocks_by_section):
1204
+ """Apply delete-block actions to a single file. blocks_by_section is
1205
+ {section_idx: {block_idx: expected_content_hash | None}}. Returns a list
1206
+ of invalid entries (block_idx out of range, or content_hash drift) —
1207
+ each entry is a dict the caller annotates with file_key + section_idx
1208
+ before surfacing to the user."""
1209
+ content = path.read_text(encoding="utf-8")
1210
+ prefix, sections = split_sections(content)
1211
+ invalid_all = []
1212
+ new_sections = []
1213
+ for section_idx, (title, body) in enumerate(sections):
1214
+ drops = blocks_by_section.get(section_idx)
1215
+ if not drops:
1216
+ new_sections.append((title, body))
1217
+ continue
1218
+ if AUTO_START in body and AUTO_END in body:
1219
+ head_end = body.index(AUTO_START)
1220
+ head = body[:head_end].rstrip()
1221
+ tail = body[head_end:]
1222
+ new_head, invalid = _delete_blocks_from_section(head, drops)
1223
+ new_body = (new_head + "\n\n" + tail).strip()
1224
+ else:
1225
+ new_body, invalid = _delete_blocks_from_section(body, drops)
1226
+ for entry in invalid:
1227
+ entry["section_idx"] = section_idx
1228
+ invalid_all.append(entry)
1229
+ new_sections.append((title, new_body))
1230
+ path.write_text(join_sections(prefix, new_sections), encoding="utf-8")
1231
+ return invalid_all
1232
+
1233
+
1234
+ def _delete_sections(path, section_indices):
1235
+ """Drop the listed top-level (`## `) sections from a file."""
1236
+ content = path.read_text(encoding="utf-8")
1237
+ prefix, sections = split_sections(content)
1238
+ drop = set(section_indices)
1239
+ kept = [(t, b) for i, (t, b) in enumerate(sections) if i not in drop]
1240
+ path.write_text(join_sections(prefix, kept), encoding="utf-8")
1241
+
1242
+
1243
+ def _reset_file(path, file_key, branch_name, repo_name):
1244
+ body = _template_for(file_key, branch_name, repo_name)
1245
+ if body is None:
1246
+ raise ValueError(f"no v2 template registered for file_key {file_key!r}")
1247
+ path.write_text(body, encoding="utf-8")
1248
+
1249
+
1250
+ def command_apply(args):
1251
+ repo_root, branch_name, branch_key, storage_root, repo_key, repo_dir, branch_dir, paths = ensure_branch_paths_exist(
1252
+ args.repo, args.context_dir, args.branch
1253
+ )
1254
+
1255
+ plan = json.loads(Path(args.plan_file).read_text(encoding="utf-8"))
1256
+ actions = plan.get("actions") or []
1257
+ include_repo = (plan.get("scope") or {}).get("include_repo", False)
1258
+
1259
+ # Bucket the actions:
1260
+ # reset_files : file_keys to fully reset (highest precedence)
1261
+ # delete_sections: file_key -> set(section_idx)
1262
+ # delete_blocks: file_key -> section_idx -> set(block_idx)
1263
+ # entry_actions: file_key -> section_idx -> entry_idx -> {action, new_text}
1264
+ reset_files = set()
1265
+ delete_sections = {}
1266
+ delete_blocks = {}
1267
+ entry_actions = {}
1268
+ invalid = []
1269
+
1270
+ for item in actions:
1271
+ # New action types use a "type" field; legacy entry-level actions
1272
+ # use "id" + "action". Distinguish by which key is present.
1273
+ atype = (item.get("type") or "").strip()
1274
+ if atype:
1275
+ if atype == "reset-file":
1276
+ fk = item.get("file_key", "")
1277
+ if fk not in paths:
1278
+ invalid.append({"action": item, "reason": f"unknown file_key {fk!r}"})
1279
+ continue
1280
+ reset_files.add(fk)
1281
+ continue
1282
+ if atype == "delete-section":
1283
+ fk = item.get("file_key", "")
1284
+ if fk not in paths:
1285
+ invalid.append({"action": item, "reason": f"unknown file_key {fk!r}"})
1286
+ continue
1287
+ try:
1288
+ sidx = int(item.get("section_idx"))
1289
+ except (TypeError, ValueError):
1290
+ invalid.append({"action": item, "reason": "section_idx must be int"})
1291
+ continue
1292
+ delete_sections.setdefault(fk, set()).add(sidx)
1293
+ continue
1294
+ if atype == "delete-block":
1295
+ bid = (item.get("block_id") or "").strip()
1296
+ parsed = _parse_block_id(bid)
1297
+ if parsed is None:
1298
+ invalid.append({"action": item, "reason": f"malformed block_id {bid!r}"})
1299
+ continue
1300
+ fk, sidx, bidx = parsed
1301
+ if fk not in paths:
1302
+ invalid.append({"action": item, "reason": f"unknown file_key {fk!r}"})
1303
+ continue
1304
+ # Optional content fingerprint — recorded by prepare and copied
1305
+ # verbatim by the agent. None means "skip the content check"
1306
+ # (back-compat with plans authored before this guard existed).
1307
+ ech = item.get("expected_content_hash")
1308
+ if ech is not None and not isinstance(ech, str):
1309
+ invalid.append({
1310
+ "action": item,
1311
+ "reason": "expected_content_hash must be a string if provided",
1312
+ })
1313
+ continue
1314
+ # If the same block id appears twice with different expected
1315
+ # hashes, the last write wins — that's fine for practical use
1316
+ # (agent should never emit two delete-block actions on the
1317
+ # same id; if they do, both want the block gone anyway).
1318
+ delete_blocks.setdefault(fk, {}).setdefault(sidx, {})[bidx] = ech
1319
+ continue
1320
+ if atype in ("delete-entries", "edit-entries"):
1321
+ # Either form expands to a list of {id, action[, new_text]}.
1322
+ ids = item.get("ids") or []
1323
+ edits = item.get("edits") or []
1324
+ pairs = [(i, "delete", None) for i in ids] + [
1325
+ (e.get("id"), "edit", e.get("new_text")) for e in edits
1326
+ ]
1327
+ for eid, kind, new_text in pairs:
1328
+ parsed = _parse_entry_id(eid)
1329
+ if not parsed:
1330
+ invalid.append({"id": eid, "reason": "malformed id"})
1331
+ continue
1332
+ fk, sidx, eidx = parsed
1333
+ if fk not in paths:
1334
+ invalid.append({"id": eid, "reason": f"unknown file_key {fk!r}"})
1335
+ continue
1336
+ entry_actions.setdefault(fk, {}).setdefault(sidx, {})[eidx] = {
1337
+ "action": kind,
1338
+ "new_text": new_text,
1339
+ }
1340
+ continue
1341
+ invalid.append({"action": item, "reason": f"unknown type {atype!r}"})
1342
+ continue
1343
+
1344
+ # Legacy entry-level form.
1345
+ eid = item.get("id") or ""
1346
+ kind = (item.get("action") or "").strip().lower()
1347
+ if kind not in VALID_ENTRY_ACTIONS:
1348
+ invalid.append({"id": eid, "reason": f"unknown action {kind!r}"})
1349
+ continue
1350
+ parsed = _parse_entry_id(eid)
1351
+ if not parsed:
1352
+ invalid.append({"id": eid, "reason": "malformed id"})
1353
+ continue
1354
+ fk, sidx, eidx = parsed
1355
+ if fk not in paths:
1356
+ invalid.append({"id": eid, "reason": f"unknown file_key {fk!r}"})
1357
+ continue
1358
+ if kind == "keep":
1359
+ continue
1360
+ entry_actions.setdefault(fk, {}).setdefault(sidx, {})[eidx] = {
1361
+ "action": kind,
1362
+ "new_text": item.get("new_text"),
1363
+ }
1364
+
1365
+ stamp = _now_stamp()
1366
+ backup_dir, _ = _backup_scope(branch_dir, paths, include_repo, stamp)
1367
+
1368
+ rewritten = []
1369
+ # Apply reset-file first (other ops on the same file are then redundant).
1370
+ for fk in reset_files:
1371
+ path = paths[fk]
1372
+ try:
1373
+ _reset_file(path, fk, branch_name, repo_key.rsplit("-", 1)[0])
1374
+ rewritten.append({"file_key": fk, "path": str(path), "op": "reset-file"})
1375
+ except Exception as exc:
1376
+ invalid.append({"file_key": fk, "op": "reset-file", "reason": str(exc)})
1377
+ # Drop any other ops queued for this file — reset wins.
1378
+ delete_sections.pop(fk, None)
1379
+ delete_blocks.pop(fk, None)
1380
+ entry_actions.pop(fk, None)
1381
+
1382
+ # Then delete-section per file (one rewrite per file).
1383
+ for fk, sidx_set in delete_sections.items():
1384
+ path = paths.get(fk)
1385
+ if not path or not path.exists():
1386
+ invalid.append({"file_key": fk, "op": "delete-section", "reason": "file missing"})
1387
+ continue
1388
+ _delete_sections(path, sidx_set)
1389
+ rewritten.append({"file_key": fk, "path": str(path), "op": f"delete-sections:{sorted(sidx_set)}"})
1390
+ # Block / entry actions targeting these sections are no-ops; prune.
1391
+ if fk in delete_blocks:
1392
+ for sidx in sidx_set:
1393
+ delete_blocks[fk].pop(sidx, None)
1394
+ if not delete_blocks[fk]:
1395
+ delete_blocks.pop(fk, None)
1396
+ if fk in entry_actions:
1397
+ for sidx in sidx_set:
1398
+ entry_actions[fk].pop(sidx, None)
1399
+ if not entry_actions[fk]:
1400
+ entry_actions.pop(fk, None)
1401
+
1402
+ # Then delete-block per file. We re-parse blocks at apply time (prepare's
1403
+ # block_idx is informational); out-of-range block ids or content-hash
1404
+ # mismatches land in `invalid`. Also prune entry-actions that would fall
1405
+ # inside a deleted block so we don't double-touch lines that are already
1406
+ # gone.
1407
+ for fk, sec_blocks in delete_blocks.items():
1408
+ path = paths.get(fk)
1409
+ if not path or not path.exists():
1410
+ invalid.append({"file_key": fk, "op": "delete-block", "reason": "file missing"})
1411
+ continue
1412
+ # Prune entry-actions that fall inside doomed blocks before we delete.
1413
+ # sec_blocks values are now `{block_idx: expected_hash | None}` dicts;
1414
+ # we just need the block_idx keys for the pruning step.
1415
+ if fk in entry_actions:
1416
+ content = path.read_text(encoding="utf-8")
1417
+ _, sections = split_sections(content)
1418
+ for sidx, bidx_map in sec_blocks.items():
1419
+ if sidx >= len(sections):
1420
+ continue
1421
+ body = sections[sidx][1]
1422
+ cleaned = _strip_auto_block(body)
1423
+ blks = _parse_blocks(cleaned)
1424
+ doomed_entry_idx = set()
1425
+ for bidx in bidx_map:
1426
+ if 0 <= bidx < len(blks):
1427
+ s, e = blks[bidx]["entry_idx_range"]
1428
+ for k in range(s, e + 1):
1429
+ doomed_entry_idx.add(k)
1430
+ if sidx in entry_actions[fk]:
1431
+ for k in list(entry_actions[fk][sidx].keys()):
1432
+ if k in doomed_entry_idx:
1433
+ entry_actions[fk][sidx].pop(k, None)
1434
+ if not entry_actions[fk][sidx]:
1435
+ entry_actions[fk].pop(sidx, None)
1436
+ if not entry_actions[fk]:
1437
+ entry_actions.pop(fk, None)
1438
+ invalid_blocks = _rewrite_file_delete_blocks(path, sec_blocks)
1439
+ for entry in invalid_blocks:
1440
+ invalid.append({
1441
+ "op": "delete-block",
1442
+ "file_key": fk,
1443
+ **entry,
1444
+ })
1445
+ rewritten.append({
1446
+ "file_key": fk,
1447
+ "path": str(path),
1448
+ "op": f"delete-blocks:{ {k: sorted(v.keys()) for k, v in sec_blocks.items()} }",
1449
+ })
1450
+
1451
+ # Finally entry-level actions.
1452
+ for fk, sec_actions in entry_actions.items():
1453
+ path = paths.get(fk)
1454
+ if not path or not path.exists():
1455
+ invalid.append({"file_key": fk, "reason": "file missing at apply time"})
1456
+ continue
1457
+ _rewrite_file(path, sec_actions)
1458
+ rewritten.append({"file_key": fk, "path": str(path), "op": "entry-actions"})
1459
+
1460
+ accepted = plan.get("accepted_proposals") or []
1461
+ rejected = plan.get("rejected_proposals") or []
1462
+ custom = plan.get("custom_proposals") or []
1463
+
1464
+ summary_path = branch_dir / "tidy_review" / f"summary_{stamp}.md"
1465
+ summary_path.parent.mkdir(parents=True, exist_ok=True)
1466
+ summary_lines = [
1467
+ f"# Tidy summary {stamp}",
1468
+ "",
1469
+ f"- branch: `{branch_name}`",
1470
+ f"- include_repo: `{include_repo}`",
1471
+ f"- backup: `{backup_dir}`",
1472
+ f"- accepted proposals: {len(accepted)}",
1473
+ f"- rejected proposals: {len(rejected)}",
1474
+ f"- custom proposals: {len(custom)}",
1475
+ f"- actions applied: {len(actions)}",
1476
+ f"- files rewritten: {len(rewritten)}",
1477
+ f"- invalid items: {len(invalid)}",
1478
+ ]
1479
+ if plan.get("notes"):
1480
+ summary_lines += ["", "## notes", "", plan["notes"]]
1481
+ if rewritten:
1482
+ summary_lines += ["", "## rewritten", ""]
1483
+ summary_lines += [f"- {r['file_key']} ({r['op']})" for r in rewritten]
1484
+ if custom:
1485
+ summary_lines += [
1486
+ "",
1487
+ "## custom proposals (NOT applied — agent must read user feedback and re-propose)",
1488
+ "",
1489
+ ]
1490
+ for c in custom:
1491
+ pid = c.get("proposal_id") or c.get("id") or "?"
1492
+ fb = (c.get("user_feedback") or "").strip()
1493
+ summary_lines += [f"### {pid}", "", fb or "(empty feedback)", ""]
1494
+ if invalid:
1495
+ summary_lines += ["", "## invalid", ""]
1496
+ summary_lines += [f"- {json.dumps(v, ensure_ascii=False)}" for v in invalid]
1497
+ summary_path.write_text("\n".join(summary_lines) + "\n", encoding="utf-8")
1498
+
1499
+ # Event log: one row per apply. Mirror to repo_log if any rewritten file
1500
+ # lives at the repo-shared layer (include_repo plan touched repo_*).
1501
+ touches_repo = any(r["file_key"].startswith("repo_") for r in rewritten)
1502
+ log_details = [
1503
+ ("accepted", len(accepted)),
1504
+ ("rejected", len(rejected)),
1505
+ ("custom", len(custom)),
1506
+ ("rewritten", len(rewritten)),
1507
+ ("invalid", len(invalid)),
1508
+ ("backup", backup_dir.name),
1509
+ ]
1510
+ log_summary = (
1511
+ f"accepted={len(accepted)} rewritten={len(rewritten)}"
1512
+ if rewritten or accepted else "no-op"
1513
+ )
1514
+ append_log_event(
1515
+ paths.get("log"),
1516
+ "tidy",
1517
+ kind="apply",
1518
+ summary=log_summary,
1519
+ details=log_details,
1520
+ )
1521
+ if touches_repo and paths.get("repo_log") and paths.get("repo_log") != paths.get("log"):
1522
+ append_log_event(
1523
+ paths.get("repo_log"),
1524
+ "tidy",
1525
+ kind="apply",
1526
+ summary=log_summary,
1527
+ details=log_details,
1528
+ )
1529
+
1530
+ print(json.dumps({
1531
+ "mode": "apply",
1532
+ "branch": branch_name,
1533
+ "branch_dir": str(branch_dir),
1534
+ "backup_dir": str(backup_dir),
1535
+ "accepted_proposals": accepted,
1536
+ "rejected_proposals": rejected,
1537
+ "custom_proposals": custom,
1538
+ "rewritten": rewritten,
1539
+ "invalid": invalid,
1540
+ "summary": str(summary_path),
1541
+ "actions_count": len(actions),
1542
+ "next_steps": (
1543
+ [
1544
+ "Read each custom_proposals user_feedback and decide how to handle it based on the content. "
1545
+ "It is NOT automatic 're-run prepare' — depending on the feedback you may: act inline (capture/tidy subcommand), "
1546
+ "ask the user to clarify, defer with a note, or only when the feedback genuinely calls for "
1547
+ "fresh proposals run another `tidy prepare --proposals-file <revised>`."
1548
+ ]
1549
+ if custom else []
1550
+ ),
1551
+ }, ensure_ascii=False, indent=2))
1552
+ return 0
1553
+
1554
+
1555
+ def _parse_entry_id(eid):
1556
+ """Return (file_key, section_idx, entry_idx) or None on bad input."""
1557
+ if not isinstance(eid, str):
1558
+ return None
1559
+ parts = eid.split("::")
1560
+ if len(parts) != 3:
1561
+ return None
1562
+ try:
1563
+ return parts[0], int(parts[1]), int(parts[2])
1564
+ except ValueError:
1565
+ return None
1566
+
1567
+
1568
+ def _add_common_args(p):
1569
+ p.add_argument("--repo", default=".", help="Path inside the target Git repository")
1570
+ p.add_argument("--context-dir", help="User-home storage root. Defaults to ~/.dev-memory/repos")
1571
+ p.add_argument("--branch", help="Branch name. Defaults to the current checked-out branch")
1572
+
1573
+
1574
+ def main():
1575
+ parser = argparse.ArgumentParser(description="Review and tidy existing dev-memory memory entries.")
1576
+ sub = parser.add_subparsers(dest="command", required=True)
1577
+
1578
+ p_prep = sub.add_parser("prepare", help="Scan scope, render review.html (with optional hints + proposals)")
1579
+ _add_common_args(p_prep)
1580
+ p_prep.add_argument("--scope", choices=("branch", "branch+repo"), default="branch")
1581
+ hint_group = p_prep.add_mutually_exclusive_group()
1582
+ hint_group.add_argument("--hints-json", help="Inline JSON object keyed by entry id with {label, reason}")
1583
+ hint_group.add_argument("--hints-file", help="Path to JSON file with the same shape as --hints-json")
1584
+ p_prep.add_argument(
1585
+ "--no-auto-hints",
1586
+ dest="auto_hints",
1587
+ action="store_false",
1588
+ help="Skip the built-in STALE-by-log + ORPHAN-glossary hint passes",
1589
+ )
1590
+ p_prep.add_argument(
1591
+ "--stale-after-days",
1592
+ type=int,
1593
+ default=AUTO_STALE_THRESHOLD_DAYS,
1594
+ help=f"Auto-hint STALE for files last seen in log >= this many days ago (default {AUTO_STALE_THRESHOLD_DAYS})",
1595
+ )
1596
+ p_prep.set_defaults(auto_hints=True)
1597
+ proposal_group = p_prep.add_mutually_exclusive_group()
1598
+ proposal_group.add_argument(
1599
+ "--proposals-json",
1600
+ help="Inline JSON array of proposals: [{id, title, reason, actions:[...]}]. "
1601
+ "Action types: delete-entries (ids:[...]), delete-section (file_key, section_idx), "
1602
+ "delete-block (block_id), reset-file (file_key), edit-entries (edits:[{id, new_text}]).",
1603
+ )
1604
+ proposal_group.add_argument("--proposals-file", help="Path to JSON file with the same shape as --proposals-json")
1605
+
1606
+ p_apply = sub.add_parser("apply", help="Apply a downloaded plan.json (with backup snapshot)")
1607
+ _add_common_args(p_apply)
1608
+ p_apply.add_argument("--plan-file", required=True)
1609
+
1610
+ args = parser.parse_args()
1611
+ try:
1612
+ return {
1613
+ "prepare": command_prepare,
1614
+ "apply": command_apply,
1615
+ }[args.command](args)
1616
+ except Exception as exc:
1617
+ print(json.dumps({"error": str(exc)}, ensure_ascii=False), file=sys.stderr)
1618
+ return 1
1619
+
1620
+
1621
+ if __name__ == "__main__":
1622
+ raise SystemExit(main())