dev-memory-cli 0.22.1 → 0.23.0

package/lib/dev_memory_capture.py

@@ -16,9 +16,11 @@ Subcommands:
 
 import argparse
 import difflib
+import fcntl
 import json
 import os
 import sys
+from contextlib import contextmanager
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -27,6 +29,7 @@ from dev_memory_common import (
     AUTO_END,
     AUTO_START,
     PLACEHOLDER_MARKERS,
+    atomic_write_text,
     append_log_event,
     append_to_section,
     asset_paths,
@@ -34,9 +37,11 @@ from dev_memory_common import (
     collect_git_facts,
     ensure_branch_paths_exist,
     get_head_commit,
+    get_branch_paths,
     get_setup_completed,
     is_cross_branch_candidate,
     join_sections,
+    limit_markdown_entries,
     list_missing_docs,
     merged_focus_areas,
     now_iso,
@@ -101,7 +106,7 @@ def _emit_capture_log(paths, *, action, kind_label, summary, touched, extra_deta
         )
 
 
-def _append_with_separator(path, title, body):
+def _append_with_separator(path, title, body, *, enforce_limit=True, max_entries=None):
     """Like append_to_section but always inserts a blank line between the
     existing section body and the new entry. Drops placeholder-only bodies
     instead of padding them with a blank line.
@@ -112,6 +117,7 @@ def _append_with_separator(path, title, body):
     matched = False
     updated = []
     body_stripped = body.strip()
+    pruned = 0
 
     def _is_placeholder_only(text):
         lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
@@ -125,13 +131,21 @@ def _append_with_separator(path, title, body):
                 combined = body_stripped
             else:
                 combined = existing_body.rstrip() + "\n\n" + body_stripped
+            if enforce_limit:
+                combined, pruned = limit_markdown_entries(combined, max_entries=max_entries)
             updated.append((existing_title, combined))
             matched = True
         else:
             updated.append((existing_title, existing_body))
     if not matched:
-        updated.append((title, body_stripped))
-    path.write_text(join_sections(prefix, updated), encoding="utf-8")
+        bounded, pruned = (
+            limit_markdown_entries(body_stripped, max_entries=max_entries)
+            if enforce_limit
+            else (body_stripped, 0)
+        )
+        updated.append((title, bounded))
+    atomic_write_text(path, join_sections(prefix, updated))
+    return pruned
 
 
 # ---------------------------------------------------------------------------
@@ -760,7 +774,7 @@ def _entry_mutation(paths, eid, *, new_text=None, delete=False):
 
     new_sections = list(sections)
     new_sections[section_idx] = (section_title, new_body)
-    target_path.write_text(join_sections(prefix, new_sections), encoding="utf-8")
+    atomic_write_text(target_path, join_sections(prefix, new_sections))
     return {
         "id": eid,
         "file_key": file_key,
@@ -1090,7 +1104,7 @@ def _prune_repo_shared_section(path, section_title, *, max_entries, max_entry_ch
         updated.append((existing_title, new_body))
     if not changed:
         return None
-    path.write_text(join_sections(prefix, updated), encoding="utf-8")
+    atomic_write_text(path, join_sections(prefix, updated))
     return result
 
 
@@ -1125,6 +1139,43 @@ def prune_repo_shared_memory(paths, *, max_entries=None, max_entry_chars=None):
     return touched
 
 
+def prune_bounded_memory(paths, *, max_entries=None):
+    """Apply the general semantic-entry cap after a multi-operation summary."""
+    max_entries = _int_env("DEV_MEMORY_MAX_ENTRIES", 200) if max_entries is None else max_entries
+    targets = {
+        (spec["file"], spec["section"])
+        for spec in KIND_MAP.values()
+        if spec.get("default_mode") == "append"
+    }
+    touched = []
+    for file_key, section_title in sorted(targets):
+        path = paths.get(file_key)
+        if not path or not path.exists():
+            continue
+        content = path.read_text(encoding="utf-8")
+        prefix, sections = split_sections(content)
+        updated = []
+        changed = False
+        for title, body in sections:
+            if title.strip() != section_title.strip():
+                updated.append((title, body))
+                continue
+            bounded, pruned = limit_markdown_entries(body, max_entries=max_entries)
+            updated.append((title, bounded))
+            if pruned:
+                changed = True
+                touched.append({
+                    "file": _label(file_key),
+                    "section": title,
+                    "mode": "prune",
+                    "pruned": pruned,
+                    "max_entries": max_entries,
+                })
+        if changed:
+            atomic_write_text(path, join_sections(prefix, updated))
+    return touched
+
+
 # ---------------------------------------------------------------------------
 # Write primitives
 # ---------------------------------------------------------------------------
@@ -1140,7 +1191,7 @@ def _resolve_target(paths, kind, title_override=None):
     return spec["file"], target_path, section_title
 
 
-def _write_one(paths, kind, body, title_override=None, *, mode_override=None):
+def _write_one(paths, kind, body, title_override=None, *, mode_override=None, enforce_limit=True):
     """Write a body into the kind's target file. Picks append vs upsert from
     KIND_MAP[kind].default_mode unless overridden. progress.md uses a
     dedicated upsert that preserves the auto-sync marker.
@@ -1159,13 +1210,28 @@ def _write_one(paths, kind, body, title_override=None, *, mode_override=None):
         body_to_write = body.strip()
         if not body_to_write.startswith(("- ", "* ", "#")):
             body_to_write = "- " + body_to_write
-        _append_with_separator(target_path, section_title, body_to_write)
+        entry_limit = (
+            _REPO_SHARED_MAX_ENTRIES
+            if file_key in {"repo_decisions", "repo_glossary"}
+            else _int_env("DEV_MEMORY_MAX_ENTRIES", 200)
+        )
+        pruned = _append_with_separator(
+            target_path,
+            section_title,
+            body_to_write,
+            enforce_limit=enforce_limit,
+            max_entries=entry_limit,
+        )
     else:
         if file_key == "progress":
             upsert_progress_section(target_path, section_title, body)
         else:
             upsert_markdown_section(target_path, section_title, body)
-    return {"file": _label(file_key), "section": section_title, "mode": mode}
+    result = {"file": _label(file_key), "section": section_title, "mode": mode}
+    if mode == "append" and pruned:
+        result["pruned"] = pruned
+        result["max_entries"] = entry_limit
+    return result
 
 
 def _label(file_key):
@@ -1216,6 +1282,8 @@ def _decision_content(item):
     if isinstance(item, str):
         return item.strip()
     if isinstance(item, dict):
+        if not _decision_summary(item):
+            return ""
         return decision_body(item)
     return ""
 
@@ -1819,7 +1887,13 @@ def command_apply_summary_output(args):
                     "dedup_hint": hint,
                 })
                 return
-        rec = _write_one(paths, kind, body, mode_override=mode_override)
+        rec = _write_one(
+            paths,
+            kind,
+            body,
+            mode_override=mode_override,
+            enforce_limit=False,
+        )
         touched.append(rec)
         _maybe_worktree_writeback(
             worktree_writeback,
@@ -1894,6 +1968,51 @@ def command_apply_summary_output(args):
             "reason": item.get("reason"),
         })
 
+    # Repair schema placeholder prose left by malformed historical summary
+    # output. Valid entries are top-level bullets; these exact unbulleted
+    # words are generator scaffolding and should never survive in memory.
+    placeholder_words = {
+        "decision", "summary", "reason", "impact", "risk", "mitigation",
+        "term", "definition", "name", "url", "note", "label", "path", "content",
+    }
+    for file_key in ("decisions", "risks", "glossary", "repo_glossary"):
+        path = paths[file_key]
+        if not path.exists():
+            continue
+        content = path.read_text(encoding="utf-8")
+        prefix, sections = split_sections(content)
+        updated = []
+        removed = 0
+        for section_title, section_body in sections:
+            kept_lines = []
+            section_removed = 0
+            for line in section_body.splitlines():
+                stripped = line.strip().strip(":：").lower()
+                is_bullet = line.lstrip().startswith(("- ", "* ", "+ "))
+                if stripped in placeholder_words and not is_bullet:
+                    section_removed += 1
+                    continue
+                kept_lines.append(line)
+            if section_removed:
+                rec = {
+                    "file": _label(file_key),
+                    "section": section_title,
+                    "mode": "remove-placeholder-orphans",
+                    "removed": section_removed,
+                }
+                touched.append(rec)
+                actions.append({"op": "remove-placeholder-orphans", **rec})
+                removed += section_removed
+            updated.append((section_title, "\n".join(kept_lines).strip()))
+        if removed:
+            atomic_write_text(path, join_sections(prefix, updated))
+
+    bounded = prune_bounded_memory(paths)
+    if bounded:
+        touched.extend(bounded)
+        for rec in bounded:
+            actions.append({"op": "prune-memory", **rec})
+
     pruned = prune_repo_shared_memory(paths)
     if pruned:
         touched.extend(pruned)
@@ -2103,6 +2222,20 @@ def _add_common_args(parser):
     parser.add_argument("--branch", help="Branch name. Defaults to the current checked-out branch")
 
 
+@contextmanager
+def _repo_command_lock(args):
+    """Serialize capture commands per repo so read-modify-write stays coherent."""
+    _, _, _, _, _, repo_dir, _ = get_branch_paths(args.repo, args.context_dir, args.branch)
+    lock_path = repo_dir / "jobs" / "capture.lock"
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a+", encoding="utf-8") as lock_stream:
+        fcntl.flock(lock_stream.fileno(), fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(lock_stream.fileno(), fcntl.LOCK_UN)
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="Unified write entrypoint for dev-memory repo+branch memory (merges old sync + update).",
@@ -2115,7 +2248,11 @@ def main():
     suggest = subparsers.add_parser("suggest-kind", help="Dry-run classify a content string")
     suggest.add_argument("--content", help="Inline content to classify")
     suggest.add_argument("--content-file", help="File containing content to classify")
-    suggest.add_argument("--already-setup", action="store_true", help="Hint that setup is completed (shifts default from unsorted to progress)")
+    suggest.add_argument(
+        "--already-setup",
+        action="store_true",
+        help="Accepted for compatibility; ambiguous content remains unsorted",
+    )
     suggest.add_argument("--branch-name", help="Branch name for cross-branch candidate check")
 
     classify = subparsers.add_parser("classify", help="Classify content against the current branch context")
@@ -2219,7 +2356,11 @@ def main():
             "sync-working-tree": command_sync_working_tree,
             "record-head": command_record_head,
         }
-        return handlers[args.command](args)
+        handler = handlers[args.command]
+        if args.command == "suggest-kind":
+            return handler(args)
+        with _repo_command_lock(args):
+            return handler(args)
     except Exception as exc:
         print(json.dumps({"error": str(exc)}, ensure_ascii=False), file=sys.stderr)
         return 1

package/lib/dev_memory_common.py

@@ -6,6 +6,7 @@ import os
 import re
 import shutil
 import subprocess
+import tempfile
 import time
 import uuid
 from collections import Counter
@@ -511,11 +512,29 @@ def asset_paths(repo_dir, branch_dir):
 
 def ensure_file(path, content):
     if not path.exists():
-        path.write_text(content, encoding="utf-8")
+        atomic_write_text(path, content)
+
+
+def atomic_write_text(path, content):
+    """Atomically replace a UTF-8 text file without exposing partial bytes."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    mode = (path.stat().st_mode & 0o777) if path.exists() else 0o644
+    fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
+    tmp_path = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as stream:
+            stream.write(content)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.chmod(tmp_path, mode)
+        os.replace(tmp_path, path)
+    finally:
+        tmp_path.unlink(missing_ok=True)
 
 
 def write_json(path, payload):
-    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    atomic_write_text(path, json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
 
 
 def read_json(path):
@@ -1234,8 +1253,7 @@ def ensure_branch_paths_exist(repo, context_dir=None, branch=None):
 # Heuristic classifier (capture routing)
 # ---------------------------------------------------------------------------
 
-# Order matters: the first pattern that matches wins. Keep decisions/risks
-# ahead of progress since their signals are more specific.
+# Order matters: the first pattern that matches wins.
 _CLASSIFY_PATTERNS = [
     ("decision", re.compile(r"结论[:：]|决[定议][:：]|不再|改为|采用|废弃|选择.+?不选|abandoned|adopt")),
     ("risk", re.compile(r"阻塞|注意|坑|失败|风险|卡住|gotcha|caveat|warning")),
@@ -1244,19 +1262,18 @@ _CLASSIFY_PATTERNS = [
 
 
 def classify_content(text, *, already_setup=False):
-    """Classify free-form content into one of decisions/progress/risks/
-    glossary/unsorted. Used by capture when the caller doesn't pass --kind.
+    """Classify free-form content into decision/risk/glossary/unsorted.
 
-    Before setup, ambiguous content falls to `unsorted` so the user can sort
-    it later via setup merge. After setup, the default shifts to `progress`
-    because the user has signaled they want aggressive categorization.
+    ``already_setup`` is retained for callers that still pass the setup state,
+    but ambiguous content always stays in ``unsorted``. Ephemeral progress is
+    intentionally derived from Git instead of being a semantic capture kind.
     """
     if not text or not text.strip():
         return "unsorted"
     for label, pattern in _CLASSIFY_PATTERNS:
         if pattern.search(text):
             return label
-    return "progress" if already_setup else "unsorted"
+    return "unsorted"
 
 
 # Branch-name tokens that appear in generic technical content too often to
@@ -1707,7 +1724,7 @@ def ensure_progress_auto_block(path):
     else:
         updated = content.rstrip() + auto_section
 
-    path.write_text(updated + ("" if updated.endswith("\n") else "\n"), encoding="utf-8")
+    atomic_write_text(path, updated + ("" if updated.endswith("\n") else "\n"))
     return path.read_text(encoding="utf-8")
 
 
@@ -1764,7 +1781,7 @@ def upsert_markdown_section(path, title, body):
             updated.append((existing_title, existing_body))
     if not replaced:
         updated.append((title, body))
-    path.write_text(join_sections(prefix, updated), encoding="utf-8")
+    atomic_write_text(path, join_sections(prefix, updated))
 
 
 def _section_is_placeholder_only(text):
@@ -1774,7 +1791,37 @@ def _section_is_placeholder_only(text):
     return all(any(marker in line for marker in PLACEHOLDER_MARKERS) for line in lines)
 
 
-def append_to_section(path, title, body):
+def memory_max_entries():
+    value = os.environ.get("DEV_MEMORY_MAX_ENTRIES", "200").strip()
+    try:
+        return max(1, int(value))
+    except ValueError:
+        return 200
+
+
+def limit_markdown_entries(body, max_entries=None):
+    """Bound an accumulated markdown section while retaining newest entries.
+
+    Accumulation sections are stored oldest-to-newest so entry ids remain
+    stable. Only top-level bullet blocks count as entries; continuation lines
+    stay attached to their bullet.
+    """
+    limit = memory_max_entries() if max_entries is None else max(1, int(max_entries))
+    lines = (body or "").strip().splitlines()
+    starts = [idx for idx, line in enumerate(lines) if re.match(r"^[-*]\s+", line)]
+    if len(starts) <= limit:
+        return (body or "").strip(), 0
+    blocks = []
+    for pos, start in enumerate(starts):
+        end = starts[pos + 1] if pos + 1 < len(starts) else len(lines)
+        blocks.append("\n".join(lines[start:end]).strip())
+    preamble = "\n".join(lines[:starts[0]]).strip()
+    kept = blocks[-limit:]
+    bounded = "\n\n".join(([preamble] if preamble else []) + kept).strip()
+    return bounded, len(blocks) - len(kept)
+
+
+def append_to_section(path, title, body, *, max_entries=None):
     content = path.read_text(encoding="utf-8") if path.exists() else ""
     prefix, sections = split_sections(content)
     target = title.strip()
@@ -1786,13 +1833,15 @@ def append_to_section(path, title, body):
                 combined = body.strip()
             else:
                 combined = (existing_body.rstrip() + "\n" + body.strip()).strip()
+            combined, _pruned = limit_markdown_entries(combined, max_entries=max_entries)
             updated.append((existing_title, combined))
             matched = True
         else:
             updated.append((existing_title, existing_body))
     if not matched:
-        updated.append((title, body.strip()))
-    path.write_text(join_sections(prefix, updated), encoding="utf-8")
+        bounded, _pruned = limit_markdown_entries(body.strip(), max_entries=max_entries)
+        updated.append((title, bounded))
+    atomic_write_text(path, join_sections(prefix, updated))
 
 
 def upsert_progress_section(path, title, body):
@@ -1819,7 +1868,7 @@ def upsert_progress_section(path, title, body):
     if not replaced:
         updated.append((title, body))
     rewritten = join_sections(prefix, updated).rstrip() + "\n\n" + marker + after
-    path.write_text(rewritten, encoding="utf-8")
+    atomic_write_text(path, rewritten)
 
 
 def sync_progress(paths, facts):
@@ -1834,7 +1883,7 @@ def sync_progress(paths, facts):
     )
     current = ensure_progress_auto_block(paths["progress"])
     updated = replace_auto_block(current, build_auto_block(facts))
-    paths["progress"].write_text(updated, encoding="utf-8")
+    atomic_write_text(paths["progress"], updated)
 
 
 # ---------------------------------------------------------------------------
@@ -1888,7 +1937,7 @@ def append_log_event(log_path, action, *, kind=None, summary=None, details=None)
         return
     if not log_path.exists():
         log_path.parent.mkdir(parents=True, exist_ok=True)
-        log_path.write_text(template_log("unknown"), encoding="utf-8")
+        atomic_write_text(log_path, template_log("unknown"))
 
     header_parts = [f"## [{now_iso()}] {action}"]
     if kind:
@@ -1908,7 +1957,7 @@ def append_log_event(log_path, action, *, kind=None, summary=None, details=None)
     existing = log_path.read_text(encoding="utf-8")
     if not existing.endswith("\n"):
         existing += "\n"
-    log_path.write_text(existing + "\n" + block, encoding="utf-8")
+    atomic_write_text(log_path, existing + "\n" + block)
 
 
 # ---------------------------------------------------------------------------

package/lib/dev_memory_graduate.py

@@ -123,7 +123,7 @@ def _apply_entries(target_path, entries):
         if not section:
             raise ValueError(f"harvest entry missing 'section': {entry}")
         if mode == "append":
-            append_to_section(target_path, section, body)
+            append_to_section(target_path, section, body, max_entries=20)
         elif mode == "replace":
             upsert_markdown_section(target_path, section, body)
         else: