dev-memory-cli 0.18.2 → 0.18.3

package/lib/dev_memory_common.py

@@ -54,7 +54,7 @@ LEGACY_V1_FILES = ("development.md", "context.md", "sources.md")
 # Bottom-up clustering of changed-file paths into a small set of "focus
 # directories". The cluster never grows larger than this many entries; a
 # higher number gives finer granularity at the cost of a longer hint list.
-FOCUS_AREA_LIMIT = 5
+FOCUS_AREA_LIMIT = 10
 
 
 def now_iso():
@@ -1339,10 +1339,11 @@ def summarize_scopes(paths):
 
 def _initial_parent(path_str):
     """File path → its immediate parent directory key. Files at repo root use
-    '.', everything else uses the POSIX-style parent dir string."""
+    their own file path as the focus key instead of '.', everything else uses
+    the POSIX-style parent dir string."""
     parent = Path(path_str).parent
     if str(parent) in ("", "."):
-        return "."
+        return Path(path_str).as_posix()
     return parent.as_posix()
 
 
@@ -1368,8 +1369,8 @@ def summarize_focus_areas(paths, limit=None):
     count (the "dominant cluster" — most files concentrated under a single
     deeper subtree), and merge only those originals into it. Buckets that
     were already at depth 1 or '.' are left untouched, so isolated shallow
-    directories survive. Tie-break: deeper rolled-up key wins, then
-    lexicographic.
+    directories and root-level files survive. Tie-break: deeper rolled-up key
+    wins, then lexicographic.
     """
     if limit is None:
         limit = FOCUS_AREA_LIMIT
@@ -1405,19 +1406,22 @@ def summarize_focus_areas(paths, limit=None):
 RECENT_COMMITS_FOR_FOCUS = 10
 
 
-def _files_in_recent_commits(repo_root, count):
-    """Files touched by the last `count` non-merge commits on HEAD."""
-    out = git_lines(
-        [
-            "log",
-            f"-n{count}",
-            "--no-merges",
-            "--name-only",
-            "--pretty=format:",
-        ],
-        cwd=repo_root,
-        check=False,
-    )
+def _files_in_recent_commits(repo_root, count, base_ref=None):
+    """Files touched by recent non-merge commits on the current branch.
+
+    When a default base is known, bound the range to ``base..HEAD`` so noisy
+    commits already merged into the base branch do not pollute focus areas.
+    """
+    cmd = [
+        "log",
+        f"-n{count}",
+        "--no-merges",
+        "--name-only",
+        "--pretty=format:",
+    ]
+    if base_ref:
+        cmd.append(f"{base_ref}..HEAD")
+    out = git_lines(cmd, cwd=repo_root, check=False)
     seen = []
     seen_set = set()
     for line in out:
@@ -1432,9 +1436,8 @@ def collect_git_facts(repo_root, branch_name, _storage_root=None):
     working_tree_files = git_lines(["diff", "--name-only"], cwd=repo_root)
     staged_files = git_lines(["diff", "--cached", "--name-only"], cwd=repo_root)
     untracked_files = git_lines(["ls-files", "--others", "--exclude-standard"], cwd=repo_root)
-    recent_commit_files = _files_in_recent_commits(repo_root, RECENT_COMMITS_FOR_FOCUS)
-
     default_base = detect_default_base(repo_root)
+    recent_commit_files = _files_in_recent_commits(repo_root, RECENT_COMMITS_FOR_FOCUS, default_base)
 
     all_paths = sorted(set(working_tree_files + staged_files + untracked_files + recent_commit_files))
     return {
@@ -1456,25 +1459,24 @@ def merged_focus_areas(new_paths, existing, limit=None):
     tree signal.
 
     Algorithm (matches user's mental model — "specific signals over wide
-    parents, stale entries kept ranked low until limit prunes them"):
+    parents; stale entries disappear once current git facts no longer cover
+    them"):
 
       1. Internally dedupe `existing`: when a wider parent dir (e.g. `apps`)
-         coexists with its specific descendant (e.g. `apps/x/y/prompts`),
-         drop the wider parent. The wider parent is almost always a leftover
-         from an earlier roll-up that swallowed too much; the specific
-         descendants carry the real signal.
+         coexists with either an existing or freshly-derived specific
+         descendant (e.g. `apps/x/y/prompts`), drop the wider parent. The wider
+         parent is almost always a leftover from an earlier roll-up that
+         swallowed too much; the specific descendants carry the real signal.
       2. Score every surviving existing dir by how many `new_paths` it
-         actually covers (0 means stale, e.g. `.vscode` after a one-shot
-         IDE write — but we keep it in the ranking, only the final limit
-         truncation prunes it from the bottom).
-      3. Compute `uncovered` = paths not under any surviving existing dir.
+         actually covers. 0 means stale, e.g. `.vscode` after a one-shot IDE
+         write; stale entries are not carried forward.
+      3. Compute `uncovered` = paths not under any active existing dir.
       4. Roll up the uncovered paths' immediate parents into ≤ remaining
          budget buckets, **forbidding roll-up into any ancestor of an
          existing dir** so the output cannot regenerate the over-wide
          parent we just removed in step 1.
-      5. Merge existing weights with the rolled-up uncovered buckets,
-         sort by weight desc, truncate to `limit`. Stale 0-weight entries
-         naturally sit at the bottom and get culled when buckets > limit.
+      5. Merge active existing weights with the rolled-up uncovered buckets,
+         sort by weight desc, truncate to `limit`.
     """
     if limit is None:
         limit = FOCUS_AREA_LIMIT
@@ -1486,24 +1488,31 @@ def merged_focus_areas(new_paths, existing, limit=None):
             return "/" not in path
         return path == dir_ or path.startswith(dir_ + "/")
 
+    fresh_focus = summarize_focus_areas(new_paths, limit=limit)
+
     def _drop_wider_parents(items):
-        items = list(dict.fromkeys(items))  # dedupe preserving order
-        return [d for d in items if not any(other != d and _is_under(other, d) for other in items)]
+        items = [d for d in dict.fromkeys(items) if d != "."]  # dedupe preserving order
+        comparison_items = items + fresh_focus
+        return [d for d in items if not any(other != d and _is_under(other, d) for other in comparison_items)]
 
     # (1) Drop wider parent entries inside existing.
     deduped_existing = _drop_wider_parents(existing)
 
-    # (2) Real coverage weight per surviving existing dir.
+    # (2) Real coverage weight per surviving existing dir. Drop stale existing
+    # dirs before computing uncovered paths or slot budget; otherwise stale
+    # manifest entries still force new hot spots to roll up too far.
     existing_weights = {d: sum(1 for p in new_paths if _is_under(p, d)) for d in deduped_existing}
+    active_existing_weights = {d: w for d, w in existing_weights.items() if w > 0}
 
-    # (3) Paths not yet covered by any existing dir.
-    uncovered = [p for p in new_paths if not any(_is_under(p, d) for d in deduped_existing)]
+    # (3) Paths not yet covered by any active existing dir.
+    active_existing_dirs = set(active_existing_weights)
+    uncovered = [p for p in new_paths if not any(_is_under(p, d) for d in active_existing_dirs)]
 
     # (4) Forbidden roll-up targets = any ancestor of a surviving existing
     # dir. Without this guard the roll-up of `uncovered` could produce a
     # parent like `apps` and we'd be back where we started.
     forbidden_ancestors = set()
-    for d in deduped_existing:
+    for d in active_existing_dirs:
         if d in ("", "."):
             continue
         parent = Path(d).parent
@@ -1517,7 +1526,7 @@ def merged_focus_areas(new_paths, existing, limit=None):
             parent = parent.parent
 
     # Reserve slots for existing entries; uncovered fills the rest.
-    remaining_budget = max(1, limit - len(existing_weights))
+    remaining_budget = max(1, limit - len(active_existing_weights))
 
     new_buckets = Counter()
     for p in uncovered:
@@ -1558,16 +1567,10 @@ def merged_focus_areas(new_paths, existing, limit=None):
     #   - new_quota      = floor(limit/2) → reserved for fresh buckets
     #   - either side can borrow unused slots from the other (so we never
     #     leave a slot empty just to enforce the partition).
-    #   - stale existing (weight=0) is a fallback that only fills slots left
-    #     after both quotas are settled — they ride at the bottom and get
-    #     pruned naturally.
     existing_active = sorted(
-        ((d, w) for d, w in existing_weights.items() if w > 0),
+        active_existing_weights.items(),
         key=lambda kv: (-kv[1], kv[0]),
     )
-    existing_stale = sorted(
-        (d for d, w in existing_weights.items() if w == 0),
-    )
     new_items = sorted(new_buckets.items(), key=lambda kv: (-kv[1], kv[0]))
 
     existing_quota = (limit + 1) // 2  # ceil(limit/2)
@@ -1584,26 +1587,24 @@ def merged_focus_areas(new_paths, existing, limit=None):
     if new_overflow > 0:
         existing_take.extend(existing_active[existing_quota:existing_quota + new_overflow])
 
-    final = []
+    selected = []
     seen = set()
-    for d, _ in existing_take + new_take:
-        if d in seen:
-            continue
-        seen.add(d)
-        final.append(d)
-    for d in existing_stale:
-        if len(final) >= limit:
-            break
+    for d, weight in existing_take + new_take:
         if d in seen:
             continue
         seen.add(d)
-        final.append(d)
-    return final[:limit]
+        selected.append((d, weight))
+    selected.sort(key=lambda kv: (-kv[1], kv[0]))
+    return [d for d, _ in selected[:limit]]
 
 
 def build_auto_block(facts):
     base_line = facts["default_base"] or "未检测到 origin/HEAD"
     head_line = facts["last_seen_head"] or "尚未检测到 HEAD"
+    if facts["default_base"]:
+        history_cmd = f"git log --oneline -n {RECENT_COMMITS_FOR_FOCUS} --no-merges {facts['default_base']}..HEAD"
+    else:
+        history_cmd = f"git log --oneline -n {RECENT_COMMITS_FOR_FOCUS} --no-merges"
     focus_lines = render_bullets(facts["focus_areas"], empty_text="- 当前未检测到改动目录", wrap_code=True)
     scope_lines = render_bullets(
         [f"{item['scope']} ({item['files']} files)" for item in facts["scope_summary"]],
@@ -1620,7 +1621,7 @@ def build_auto_block(facts):
         "#### 顶层改动范围\n\n"
         f"{scope_lines}\n\n"
         "#### 按需查看提交历史\n\n"
-        f"- `git log --oneline -n {RECENT_COMMITS_FOR_FOCUS} --no-merges`\n"
+        f"- `{history_cmd}`\n"
         "- `git diff --name-only`\n"
     )
 

package/lib/dev_memory_graduate.py

@@ -10,6 +10,8 @@ the heuristic missed, but the primary signal is pending-promotion.
 """
 
 import argparse
+import contextlib
+import fcntl
 import json
 import sys
 from datetime import datetime, timezone
@@ -130,21 +132,11 @@ def _apply_entries(target_path, entries):
     return n
 
 
-def command_apply(args):
-    if detect_no_git_mode(args.repo):
-        print(json.dumps({"error": "no-git mode: graduate has nothing to archive"}, ensure_ascii=False))
-        return 1
-
-    repo_root, branch_name, branch_key, _, repo_key, repo_dir, branch_dir = get_branch_paths(args.repo, args.context_dir, args.branch)
-
-    if not branch_dir.exists():
-        print(json.dumps({"error": f"branch memory not initialized: {branch_dir}"}, ensure_ascii=False))
-        return 1
-
-    harvest = read_json(Path(args.harvest_file))
+def _load_harvest(harvest_file):
+    harvest_path = Path(harvest_file)
+    harvest = read_json(harvest_path)
     if not harvest:
-        print(json.dumps({"error": f"harvest file empty or missing: {args.harvest_file}"}, ensure_ascii=False))
-        return 1
+        raise ValueError(f"harvest file empty or missing: {harvest_file}")
 
     # Reject unknown top-level keys explicitly. The most common failure mode
     # is a pre-v2 harvest.json still using repo_context / repo_sources —
@@ -160,13 +152,74 @@ def command_apply(args):
                 " (v1 schema? repo_context → repo_decisions or repo_glossary; "
                 "repo_sources → repo_glossary; see references/harvest-schema.md)"
             )
-        print(
-            json.dumps(
-                {"error": f"unknown harvest key(s): {unknown_keys}{legacy_hint}"},
-                ensure_ascii=False,
+        raise ValueError(f"unknown harvest key(s): {unknown_keys}{legacy_hint}")
+    return harvest
+
+
+def _resolve_apply_context(args):
+    if detect_no_git_mode(args.repo):
+        raise ValueError("no-git mode: graduate has nothing to archive")
+
+    repo_root, branch_name, branch_key, _, repo_key, repo_dir, branch_dir = get_branch_paths(
+        args.repo,
+        args.context_dir,
+        args.branch,
+    )
+    if not branch_dir.exists():
+        raise ValueError(f"branch memory not initialized: {branch_dir}")
+
+    return {
+        "repo_root": repo_root,
+        "branch_name": branch_name,
+        "branch_key": branch_key,
+        "repo_key": repo_key,
+        "repo_dir": repo_dir,
+        "branch_dir": branch_dir,
+    }
+
+
+def _graduate_apply_lock_path(repo_dir):
+    return Path(repo_dir) / "jobs" / "graduate-apply.lock"
+
+
+@contextlib.contextmanager
+def _graduate_apply_queue(repo_dir):
+    """Serialize graduate apply per repo memory directory.
+
+    Pre-flight validation happens before this context. The lock only protects
+    repo-shared writes + branch archive moves, so independent invocations can
+    still fail fast on bad harvest files before waiting behind a long apply.
+    """
+    lock_path = _graduate_apply_lock_path(repo_dir)
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a+", encoding="utf-8") as lock_fh:
+        waited = False
+        try:
+            fcntl.flock(lock_fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            waited = True
+            print(
+                f"[dev-memory][graduate] another apply is running; queued on {lock_path}",
+                file=sys.stderr,
             )
-        )
-        return 1
+            fcntl.flock(lock_fh.fileno(), fcntl.LOCK_EX)
+        try:
+            yield waited
+        finally:
+            fcntl.flock(lock_fh.fileno(), fcntl.LOCK_UN)
+
+
+def _apply_validated(context, harvest, *, queue_waited=False):
+    repo_root = context["repo_root"]
+    branch_name = context["branch_name"]
+    branch_key = context["branch_key"]
+    repo_dir = context["repo_dir"]
+    branch_dir = context["branch_dir"]
+
+    # The branch may have been archived by an earlier queued apply while this
+    # process was waiting. Re-check inside the critical section before writing.
+    if not branch_dir.exists():
+        raise ValueError(f"branch memory not initialized: {branch_dir}")
 
     paths = asset_paths(repo_dir, branch_dir)
 
@@ -226,10 +279,20 @@ def command_apply(args):
         "harvested_total": total,
         "archive_summary": str((archive_dst or branch_dir) / "archive_summary.md"),
         "archived_to": str(archive_dst) if archive_dst else None,
+        "queue_waited": queue_waited,
     }, ensure_ascii=False, indent=2))
     return 0
 
 
+def command_apply(args):
+    # Pre-flight first: bad input should return immediately even if another
+    # session is currently applying a valid graduate harvest.
+    context = _resolve_apply_context(args)
+    harvest = _load_harvest(args.harvest_file)
+    with _graduate_apply_queue(context["repo_dir"]) as waited:
+        return _apply_validated(context, harvest, queue_waited=waited)
+
+
 def command_index(args):
     if detect_no_git_mode(args.repo):
         print(json.dumps({"error": "no-git mode: no archive index"}, ensure_ascii=False))

package/lib/dev_memory_summary.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def _read_json(path):
+    return json.loads(Path(path).read_text(encoding="utf-8"))
+
+
+def _text_from_content(content):
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return ""
+    parts = []
+    for item in content:
+        if not isinstance(item, dict):
+            continue
+        typ = item.get("type")
+        if typ in ("tool_use", "tool_result", "function_call", "function_call_output"):
+            continue
+        text = item.get("text")
+        if isinstance(text, str):
+            parts.append(text)
+    return "\n".join(parts).strip()
+
+
+def _extract_claude(obj):
+    typ = obj.get("type")
+    if typ not in ("user", "assistant"):
+        return None
+    msg = obj.get("message") if isinstance(obj.get("message"), dict) else {}
+    role = msg.get("role") or typ
+    text = _text_from_content(msg.get("content"))
+    if not text:
+        return None
+    return {
+        "source": "claude",
+        "role": role,
+        "timestamp": obj.get("timestamp"),
+        "uuid": obj.get("uuid"),
+        "text": text,
+    }
+
+
+def _extract_codex(obj):
+    if obj.get("type") != "response_item":
+        return None
+    payload = obj.get("payload")
+    if not isinstance(payload, dict):
+        return None
+    if payload.get("type") != "message":
+        return None
+    role = payload.get("role")
+    if role not in ("user", "assistant"):
+        return None
+    text = _text_from_content(payload.get("content"))
+    if not text:
+        return None
+    return {
+        "source": "codex",
+        "role": role,
+        "timestamp": obj.get("timestamp"),
+        "text": text,
+    }
+
+
+def _iter_core_messages(transcript_path):
+    if not transcript_path:
+        return []
+    path = Path(transcript_path).expanduser()
+    if not path.exists():
+        return []
+    out = []
+    with path.open(encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+            except Exception:
+                continue
+            msg = _extract_claude(obj) or _extract_codex(obj)
+            if msg:
+                out.append(msg)
+    return out
+
+
+def _truncate(text, max_chars):
+    text = (text or "").strip()
+    if len(text) <= max_chars:
+        return text
+    return text[: max_chars - 3].rstrip() + "..."
+
+
+def _is_nonsemantic_user_text(text):
+    stripped = (text or "").strip()
+    if not stripped:
+        return True
+    markers = (
+        "<local-command-caveat>",
+        "<command-name>",
+        "<command-message>",
+        "<command-args>",
+        "Your tool call was malformed",
+    )
+    return any(marker in stripped for marker in markers)
+
+
+def _memory_file(path, max_chars, name=None):
+    p = Path(path)
+    if not p.exists():
+        return None
+    text = p.read_text(encoding="utf-8")
+    item = {
+        "name": name or p.name,
+        "content": _truncate(text, max_chars),
+    }
+    if len(text) > max_chars:
+        item["truncated"] = True
+    return item
+
+
+def _summary_job(job):
+    transcript_state = job.get("transcript_state") if isinstance(job.get("transcript_state"), dict) else {}
+    previous_job = job.get("previous_job") if isinstance(job.get("previous_job"), dict) else {}
+    return {
+        "repo_root": job.get("repo_root"),
+        "transcript_state": {
+            key: transcript_state.get(key)
+            for key in ("size", "mtime_ms")
+            if transcript_state.get(key) is not None
+        },
+        "previous_processed": previous_job.get("processed") if isinstance(previous_job.get("processed"), dict) else None,
+    }
+
+
+def extract_core_payload(
+    job,
+    *,
+    max_messages=40,
+    max_message_chars=2000,
+    max_memory_chars=6000,
+    since_size=0,
+    include_message_metadata=False,
+):
+    branch_dir = Path(job["branch_dir"])
+    repo_dir = Path(job["repo_dir"])
+    memory_paths = [
+        ("branch/progress.md", branch_dir / "progress.md"),
+        ("branch/risks.md", branch_dir / "risks.md"),
+        ("branch/decisions.md", branch_dir / "decisions.md"),
+        ("branch/glossary.md", branch_dir / "glossary.md"),
+        ("branch/overview.md", branch_dir / "overview.md"),
+        ("repo/decisions.md", repo_dir / "repo" / "decisions.md"),
+        ("repo/glossary.md", repo_dir / "repo" / "glossary.md"),
+    ]
+    messages = _iter_core_messages(job.get("transcript_path"))
+    if since_size:
+        # JSONL is line-oriented but byte offsets can land mid-line. Keep this
+        # option conservative for now: expose cursor metadata and let workers
+        # use recent-tail until a precise offset reader is needed.
+        pass
+    messages = [m for m in messages if not _is_nonsemantic_user_text(m["text"])]
+    recent = messages[-max_messages:]
+    core_messages = []
+    for m in recent:
+        item = {
+            "role": m["role"],
+            "text": _truncate(m["text"], max_message_chars),
+        }
+        if include_message_metadata:
+            item = {**m, "text": item["text"]}
+        core_messages.append(item)
+
+    return {
+        "job": _summary_job(job),
+        "existing_memory": [
+            item
+            for item in (
+                _memory_file(path, max_memory_chars, name=name)
+                for name, path in memory_paths
+            )
+            if item is not None
+        ],
+        "core_messages": core_messages,
+        "stats": {
+            "core_message_count": len(messages),
+            "returned_core_message_count": len(recent),
+        },
+    }
+
+
+def command_extract_core(args):
+    job = _read_json(args.job)
+    payload = extract_core_payload(
+        job,
+        max_messages=args.max_messages,
+        max_message_chars=args.max_message_chars,
+        max_memory_chars=args.max_memory_chars,
+        since_size=args.since_size,
+        include_message_metadata=args.include_message_metadata,
+    )
+    print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Session summary helper commands.")
+    sub = parser.add_subparsers(dest="command", required=True)
+    p = sub.add_parser("extract-core", help="Extract core transcript messages and current memory for a summary job")
+    p.add_argument("job", help="Path to a session-summary pending job JSON")
+    p.add_argument("--max-messages", type=int, default=40)
+    p.add_argument("--max-message-chars", type=int, default=2000)
+    p.add_argument("--max-memory-chars", type=int, default=6000)
+    p.add_argument("--since-size", type=int, default=0, help="Reserved cursor hint for future byte-offset extraction")
+    p.add_argument(
+        "--include-message-metadata",
+        action="store_true",
+        help="Include source/timestamp/uuid fields in core_messages for debugging",
+    )
+
+    args = parser.parse_args()
+    if args.command == "extract-core":
+        command_extract_core(args)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dev-memory-cli",
-  "version": "0.18.2",
+  "version": "0.18.3",
   "description": "CLI for dev-memory hooks and repo-local setup (formerly @xluos/dev-assets-cli)",
   "license": "MIT",
   "repository": {