@sushanglewis/lincoln 1.6.3 → 1.7.0

package/framework/scripts/lincoln_mr_mine.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Lincoln MR / PR knowledge miner.
+
+Extracts decision-related review comments from a merged pull request and writes
+structured decision notes to the issue work package. These notes stay in
+issue-<N>/docs/decisions/ until the sync-knowledge stage promotes them to
+knowledge/.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+DECISION_KEYWORDS = ["decision", "decided", "agreed", "rejected", "note:", "approve", "approved"]
+
+
+def _run_gh(args: list[str]) -> dict[str, Any] | list[Any]:
+    """Call the gh CLI and return parsed JSON."""
+    cmd = ["gh", *args]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise RuntimeError(result.stderr.strip() or "gh CLI failed")
+    return json.loads(result.stdout or "[]")
+
+
+def _extract_decision_snippets(text: str) -> list[str]:
+    """Pull decision-oriented sentences or paragraphs from a comment body."""
+    if not text:
+        return []
+    snippets = []
+    for paragraph in re.split(r"\n\s*\n", text.strip()):
+        paragraph = paragraph.strip()
+        if not paragraph:
+            continue
+        lowered = paragraph.lower()
+        if any(keyword in lowered for keyword in DECISION_KEYWORDS):
+            snippets.append(paragraph)
+    return snippets
+
+
+def _sanitize_filename(text: str) -> str:
+    """Reduce a comment snippet to a safe filename slug."""
+    text = text.lower()
+    text = re.sub(r"[^a-z0-9]+", "-", text)
+    text = text.strip("-")
+    return text[:60]
+
+
+def mine_pr_decisions(
+    owner: str,
+    repo: str,
+    pr_number: int,
+    issue_number: int | None = None,
+    output_dir: Path | None = None,
+) -> list[Path]:
+    """Mine decision comments from a PR and write them to the issue package."""
+    if output_dir is None:
+        if issue_number is None:
+            raise ValueError("issue_number or output_dir is required")
+        output_dir = Path(f"issue-{issue_number}") / "docs" / "decisions"
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    comments = _run_gh(
+        ["api", f"repos/{owner}/{repo}/pulls/{pr_number}/comments"]
+    )
+    review_comments = _run_gh(
+        ["api", f"repos/{owner}/{repo}/pulls/{pr_number}/reviews"]
+    )
+
+    if isinstance(comments, dict):
+        comments = [comments]
+    if isinstance(review_comments, dict):
+        review_comments = [review_comments]
+
+    all_comments = list(comments) + list(review_comments)
+
+    written: list[Path] = []
+    seen: set[str] = set()
+
+    for comment in all_comments:
+        body = comment.get("body") or ""
+        snippets = _extract_decision_snippets(body)
+        for snippet in snippets:
+            if snippet in seen:
+                continue
+            seen.add(snippet)
+            slug = _sanitize_filename(snippet)
+            filename = f"pr-{pr_number}-{slug}.md"
+            path = output_dir / filename
+            if path.exists():
+                continue
+
+            content = f"""# PR {pr_number} Decision Note
+
+**Source:** {owner}/{repo}#{pr_number}
+**Author:** {comment.get('author', {}).get('login', 'unknown')}
+**Created:** {comment.get('createdAt', 'unknown')}
+**Comment ID:** {comment.get('id', 'unknown')}
+
+## Snippet
+
+{snippet}
+
+## Keywords
+
+{', '.join(DECISION_KEYWORDS)}
+
+---
+*Auto-generated by scripts/lincoln_mr_mine.py*
+"""
+            path.write_text(content, encoding="utf-8")
+            written.append(path)
+
+    return written
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Mine PR review decisions")
+    parser.add_argument("--owner", required=True)
+    parser.add_argument("--repo", required=True)
+    parser.add_argument("--pr", type=int, required=True)
+    parser.add_argument("--issue", type=int, default=None)
+    parser.add_argument("--output-dir", type=Path, default=None)
+    args = parser.parse_args(argv)
+
+    paths = mine_pr_decisions(
+        owner=args.owner,
+        repo=args.repo,
+        pr_number=args.pr,
+        issue_number=args.issue,
+        output_dir=args.output_dir,
+    )
+    for path in paths:
+        print(path)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/framework/scripts/lincoln_recall.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""Lincoln recall precheck.
+
+Given a task description, scores knowledge base documents by keyword overlap
+(and optionally by embedding similarity) and returns the top-K matches.
+Designed to be called from on-session-start.sh before injecting context.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+DEFAULT_KNOWLEDGE_DIR = "knowledge"
+DEFAULT_TOP_K = 5
+
+
+def _tokenize(text: str) -> set[str]:
+    """Extract lowercase alphanumeric tokens."""
+    return set(re.findall(r"[a-zA-Z0-9一-鿿]+", text.lower()))
+
+
+def _load_documents(knowledge_dir: Path) -> list[dict[str, Any]]:
+    docs: list[dict[str, Any]] = []
+    if not knowledge_dir.exists():
+        return docs
+    for path in sorted(knowledge_dir.rglob("*.md")):
+        try:
+            text = path.read_text(encoding="utf-8")
+            docs.append(
+                {
+                    "path": str(path.relative_to(knowledge_dir)),
+                    "tokens": _tokenize(text),
+                    "title": path.stem,
+                    "_text": text[:2000],
+                }
+            )
+        except Exception:
+            continue
+    return docs
+
+
+def _score_by_keywords(query_tokens: set[str], doc: dict[str, Any]) -> float:
+    if not query_tokens:
+        return 0.0
+    doc_tokens = doc["tokens"]
+    if not doc_tokens:
+        return 0.0
+    overlap = len(query_tokens & doc_tokens)
+    return overlap / max(len(query_tokens), len(doc_tokens))
+
+
+def _try_embedding_score(query: str, docs: list[dict[str, Any]]) -> list[float] | None:
+    """Attempt embedding-based scoring if sentence-transformers is installed."""
+    try:
+        from sentence_transformers import SentenceTransformer, util  # type: ignore
+
+        model = SentenceTransformer("all-MiniLM-L6-v2")
+        query_embedding = model.encode(query, convert_to_tensor=True)
+        doc_texts = [doc.get("_text", "") for doc in docs]
+        doc_embeddings = model.encode(doc_texts, convert_to_tensor=True)
+        scores = util.cos_sim(query_embedding, doc_embeddings)[0].tolist()
+        return list(scores)
+    except Exception:
+        return None
+
+
+def recall(
+    query: str,
+    knowledge_dir: Path | None = None,
+    top_k: int = DEFAULT_TOP_K,
+    use_embedding: bool = True,
+) -> dict[str, Any]:
+    """Return top-K knowledge documents for a query."""
+    knowledge_dir = knowledge_dir or Path(DEFAULT_KNOWLEDGE_DIR)
+    docs = _load_documents(knowledge_dir)
+    query_tokens = _tokenize(query)
+
+    if use_embedding:
+        embedding_scores = _try_embedding_score(query, docs)
+    else:
+        embedding_scores = None
+
+    results = []
+    for idx, doc in enumerate(docs):
+        keyword_score = _score_by_keywords(query_tokens, doc)
+        if embedding_scores is not None:
+            score = 0.5 * keyword_score + 0.5 * embedding_scores[idx]
+        else:
+            score = keyword_score
+        results.append(
+            {
+                "path": doc["path"],
+                "title": doc["title"],
+                "score": round(score, 4),
+                "method": "embedding+keywords" if embedding_scores else "keywords",
+            }
+        )
+
+    results.sort(key=lambda r: r["score"], reverse=True)
+    return {
+        "query": query,
+        "top_k": top_k,
+        "docs": [r for r in results[:top_k] if r["score"] > 0],
+        "method": "embedding+keywords" if embedding_scores else "keywords",
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Recall relevant knowledge documents")
+    parser.add_argument("--query", required=True)
+    parser.add_argument("--knowledge-dir", type=Path, default=None)
+    parser.add_argument("--top-k", type=int, default=DEFAULT_TOP_K)
+    parser.add_argument("--format", choices=["json", "markdown"], default="json")
+    parser.add_argument("--no-embedding", action="store_true", default=False)
+    args = parser.parse_args(argv)
+
+    result = recall(
+        query=args.query,
+        knowledge_dir=args.knowledge_dir,
+        top_k=args.top_k,
+        use_embedding=not args.no_embedding,
+    )
+
+    if args.format == "json":
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        print(f"# 相关知识（{result['method']}）\n")
+        if result["docs"]:
+            for doc in result["docs"]:
+                print(f"- [{doc['title']}]({doc['path']}) — score {doc['score']}")
+        else:
+            print("_未找到相关知识。_")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/framework/scripts/lincoln_trace.py

@@ -31,7 +31,7 @@ from scripts.lincoln_paths import (  # noqa: E402
     process_package_root,
 )
 
-TRACE_SCHEMA_VERSION = "1.0.0"
+TRACE_SCHEMA_VERSION = "2.0.0"
 
 # Tools whose invocations are not traced to avoid recursion or noise.
 NO_TRACE_TOOLS = {"Read", "Grep", "Glob"}
@@ -46,6 +46,26 @@ TASK_TOOLS = frozenset({
     "TaskStop",
 })
 
+# All valid trace categories. Tool categories are auto-derived; lifecycle categories
+# are written explicitly by stage_loader and hooks.
+VALID_CATEGORIES = frozenset({
+    "tool",
+    "skill",
+    "agent",
+    "read",
+    "write",
+    "edit",
+    "bash",
+    "task",
+    "mcp",
+    "handoff",
+    "other",
+    "stage_lifecycle",
+    "gate",
+    "session",
+    "condensation",
+})
+
 # Maximum length stored for a Bash command summary in the trace entry.
 COMMAND_SUMMARY_MAX_LEN = 500
 
@@ -202,6 +222,11 @@ def _resolve_trace_file(state: dict[str, Any] | None, state_path: Path | None) -
     return trace_dir / "lc-trace.jsonl"
 
 
+def get_trace_file(state: dict[str, Any] | None = None, state_path: Path | None = None) -> Path:
+    """Public accessor for the resolved trace file path."""
+    return _resolve_trace_file(state, state_path)
+
+
 def _get_run_id(state: dict[str, Any] | None, run_id: str | None) -> str:
     if run_id:
         return run_id
@@ -250,12 +275,26 @@ def _build_trace_target(tool: str, args_summary: dict[str, Any]) -> str:
     return str(args_summary.get("path", args_summary.get("name", "")))
 
 
+def _get_node_id(state: dict[str, Any] | None, node_id: str | None) -> str | None:
+    if node_id:
+        return node_id
+    if state:
+        current_run = state.get("current_run", {})
+        # Find the in_progress node for the current stage.
+        current_stage = current_run.get("current_stage", "")
+        for node in state.get("nodes", []):
+            if node.get("stage_id") == current_stage and node.get("status") == "in_progress":
+                return str(node.get("node_id", "")) or None
+    return None
+
+
 def _build_trace_entry(
     tool: str,
     args_summary: dict[str, Any],
     exit_code: int,
     run_id: str,
     stage: str,
+    node_id: str | None = None,
 ) -> dict[str, Any]:
     return {
         "schema_version": TRACE_SCHEMA_VERSION,
@@ -263,6 +302,7 @@ def _build_trace_entry(
         "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
         "run_id": run_id,
         "stage": stage,
+        "node_id": node_id,
         "tool": tool,
         "category": categorize_tool(tool),
         "target": _build_trace_target(tool, args_summary),
@@ -290,8 +330,9 @@ def append_trace_entry(
     exit_code: int,
     stage: str | None = None,
     run_id: str | None = None,
+    node_id: str | None = None,
 ) -> Path | None:
-    """Append a single trace entry to the appropriate JSONL file.
+    """Append a single trace entry for a tool invocation.
 
     The write is atomic with respect to other processes on the same host: the
     file is locked while appending, so concurrent parent and child agents do
@@ -303,9 +344,84 @@ def append_trace_entry(
     state = _load_state(state_path)
     resolved_run_id = _get_run_id(state, run_id)
     resolved_stage = _get_stage(state, stage)
+    resolved_node_id = _get_node_id(state, node_id)
     trace_file = _resolve_trace_file(state, state_path)
     args_summary = redact_args(tool, _safe_json_loads(args))
-    entry = _build_trace_entry(tool, args_summary, exit_code, resolved_run_id, resolved_stage)
+    entry = _build_trace_entry(
+        tool, args_summary, exit_code, resolved_run_id, resolved_stage, resolved_node_id
+    )
+    _append_trace_line(trace_file, entry)
+    return trace_file
+
+
+def append_event_entry(
+    state_path: Path | None,
+    category: str,
+    payload: dict[str, Any],
+    stage: str | None = None,
+    run_id: str | None = None,
+    node_id: str | None = None,
+) -> Path | None:
+    """Append a non-tool lifecycle event to the trace.
+
+    Categories must be one of VALID_CATEGORIES. The payload is stored under
+    ``args_summary`` for backward compatibility with existing readers.
+    """
+    if category not in VALID_CATEGORIES:
+        raise ValueError(f"Invalid trace category: {category!r}")
+
+    state = _load_state(state_path)
+    resolved_run_id = _get_run_id(state, run_id)
+    resolved_stage = _get_stage(state, stage)
+    resolved_node_id = _get_node_id(state, node_id)
+    trace_file = _resolve_trace_file(state, state_path)
+
+    entry: dict[str, Any] = {
+        "schema_version": TRACE_SCHEMA_VERSION,
+        "sequence_id": generate_sequence_id(),
+        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "run_id": resolved_run_id,
+        "stage": resolved_stage,
+        "node_id": resolved_node_id,
+        "tool": "",
+        "category": category,
+        "target": str(payload.get("event_type", payload.get("type", "")))[:200],
+        "exit_code": int(payload.get("exit_code", 0)),
+        "args_summary": payload,
+    }
+    _append_trace_line(trace_file, entry)
+    return trace_file
+
+
+def append_event_to_file(
+    trace_file: Path,
+    category: str,
+    payload: dict[str, Any],
+    stage: str | None = None,
+    run_id: str | None = None,
+    node_id: str | None = None,
+) -> Path:
+    """Append a non-tool lifecycle event to an explicit trace file path.
+
+    This variant is useful when the caller has already resolved the trace file
+    (e.g. the condenser) and wants to avoid re-resolving it from state.
+    """
+    if category not in VALID_CATEGORIES:
+        raise ValueError(f"Invalid trace category: {category!r}")
+
+    entry: dict[str, Any] = {
+        "schema_version": TRACE_SCHEMA_VERSION,
+        "sequence_id": generate_sequence_id(),
+        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "run_id": run_id or "unknown",
+        "stage": stage or "unknown",
+        "node_id": node_id or "unknown",
+        "tool": "",
+        "category": category,
+        "target": str(payload.get("event_type", payload.get("type", "")))[:200],
+        "exit_code": int(payload.get("exit_code", 0)),
+        "args_summary": payload,
+    }
     _append_trace_line(trace_file, entry)
     return trace_file
 

package/framework/scripts/security_analyzer.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""Risk analyzer for Lincoln pre-tool-use security gate.
+
+Loads `.claude/policies/security.yaml` and evaluates a tool call against the
+configured policies. Designed to be invoked from `.claude/hooks/pre-tool-use.sh`
+and from generated harness-specific hook scripts.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+DEFAULT_POLICY_PATH = ".claude/policies/security.yaml"
+
+
+def load_policy(root: Path) -> dict[str, Any]:
+    """Load the risk policy YAML, returning an empty policy if missing."""
+    path = root / DEFAULT_POLICY_PATH
+    if not path.exists():
+        return {"policies": []}
+    return yaml.safe_load(path.read_text(encoding="utf-8")) or {"policies": []}
+
+
+def _normalize_command(command: str) -> str:
+    """Strip leading whitespace and collapse internal spaces for matching."""
+    return re.sub(r"\s+", " ", command.strip())
+
+
+def _split_command_segments(command: str) -> list[str]:
+    """Split a shell command on chain operators so each segment is matched.
+
+    Handles `&&`, `||`, `;`, and `|`.
+    """
+    segments = re.split(r"\s*(&&|\|\||;|\|)\s*", command)
+    return [_normalize_command(seg) for seg in segments if seg and not re.match(r"^(&&|\|\||;|\|)$", seg.strip())]
+
+
+def _strip_command_prefix(segment: str) -> str:
+    """Remove common leading prefixes that should not hide the real command.
+
+    Examples: `sudo rm foo`, `env VAR=1 rm foo`, `cd /tmp && rm foo`.
+    """
+    segment = _normalize_command(segment)
+    while re.match(r"^[A-Za-z_][A-Za-z0-9_]*=\S+\s+", segment):
+        segment = _normalize_command(re.sub(r"^[A-Za-z_][A-Za-z0-9_]*=\S+\s+", "", segment))
+    if re.match(r"^(sudo|doas)\s+", segment):
+        segment = _normalize_command(re.sub(r"^(sudo|doas)\s+", "", segment))
+    if re.match(r"^cd\s+\S+\s*(&&|;|\|)", segment):
+        segment = _normalize_command(re.sub(r"^cd\s+\S+\s*(&&|;|\|)", "", segment))
+    return segment
+
+
+def _command_matches(condition: str, command: str) -> bool:
+    """Evaluate a human-readable condition against a bash command."""
+    lowered = condition.lower()
+    segments = _split_command_segments(command)
+
+    def _any_segment(pattern: str) -> bool:
+        return any(re.search(pattern, _strip_command_prefix(seg)) for seg in segments)
+
+    if "rm -rf" in lowered or 'command matches "rm -rf"' in lowered:
+        if _any_segment(r"\brm\s+-rf\b"):
+            return True
+
+    if 'command matches "rm -r" or "rm -rf"' in lowered:
+        if _any_segment(r"\brm\s+-r(?:f)?\b"):
+            return True
+
+    if 'command matches any of ["rm", "rmdir"]' in lowered:
+        if _any_segment(r"(?:^|\s)(?:rm\b|rmdir\b)"):
+            return True
+
+    if 'command matches "git push"' in lowered:
+        if _any_segment(r"\bgit\s+push\b"):
+            return True
+
+    if "external http client" in lowered or "curl, wget" in lowered:
+        if _any_segment(r"\b(?:curl|wget|httpie)\b"):
+            return True
+
+    if 'command matches "rm"' in lowered and "rmdir" not in lowered:
+        if _any_segment(r"\brm\b"):
+            return True
+
+    return False
+
+
+def _resolve_target_path(target: str, root: Path) -> Path:
+    """Resolve a Write/Edit target to an absolute, canonical path."""
+    path = Path(target)
+    if not path.is_absolute():
+        path = root / path
+    try:
+        return path.resolve()
+    except (OSError, ValueError):
+        return path.absolute()
+
+
+def _target_matches(condition: str, target: str, process_slug: str, root: Path) -> bool:
+    """Evaluate a file-target condition using canonical absolute paths."""
+    lowered = condition.lower()
+    resolved = _resolve_target_path(target, root)
+    try:
+        package_root = (root / process_slug).resolve()
+    except (OSError, ValueError):
+        package_root = (root / process_slug).absolute()
+
+    if "outside current process_slug" in lowered:
+        try:
+            resolved.relative_to(package_root)
+            return False
+        except ValueError:
+            return True
+
+    if "inside current process_slug" in lowered:
+        try:
+            resolved.relative_to(package_root)
+            return True
+        except ValueError:
+            return False
+
+    return False
+
+
+def _evaluate_policy(policy: dict[str, Any], tool_name: str, tool_args: dict[str, Any], process_slug: str, root: Path) -> bool:
+    """Return True if the policy applies to this tool call."""
+    tools = [t.strip() for t in policy.get("tool", "").split("|")]
+    if tool_name not in tools:
+        return False
+
+    condition = policy.get("condition", "")
+    if tool_name == "Bash":
+        command = tool_args.get("command", "")
+        return _command_matches(condition, command)
+
+    if tool_name in ("Write", "Edit"):
+        target = tool_args.get("file_path") or tool_args.get("path") or ""
+        return _target_matches(condition, target, process_slug, root)
+
+    return False
+
+
+def _log_security_event(
+    log_dir: Path,
+    tool_name: str,
+    tool_args: dict[str, Any],
+    result: dict[str, Any],
+) -> None:
+    """Append a JSON line to issue-<N>/logs/security.log."""
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / "security.log"
+    entry = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "tool": tool_name,
+        "level": result["level"],
+        "policy": result.get("policy", ""),
+        "action": "blocked" if result["confirm_required"] else "allowed",
+        "args": {k: v for k, v in tool_args.items() if k in ("command", "file_path", "path")},
+    }
+    with log_file.open("a", encoding="utf-8") as f:
+        f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+
+def analyze(
+    root: Path,
+    tool_name: str,
+    tool_args: dict[str, Any],
+    process_slug: str = "",
+    log_dir: str | None = None,
+) -> dict[str, Any]:
+    """Analyze a tool call and return risk assessment."""
+    if os.environ.get("LINCOLN_SECURITY_MODE") == "permissive":
+        return {
+            "level": "low",
+            "policy": "",
+            "message": "Security analyzer is in permissive mode.",
+            "confirm_required": False,
+            "log": False,
+        }
+
+    policy_data = load_policy(root)
+    policies = policy_data.get("policies", [])
+
+    for policy in policies:
+        if not _evaluate_policy(policy, tool_name, tool_args, process_slug, root):
+            continue
+
+        result = {
+            "level": policy.get("level", "unknown"),
+            "policy": policy.get("name", ""),
+            "message": policy.get("message", ""),
+            "confirm_required": bool(policy.get("confirm", False)),
+            "log": bool(policy.get("log", False)),
+        }
+        if log_dir:
+            _log_security_event(Path(log_dir), tool_name, tool_args, result)
+        return result
+
+    return {
+        "level": "low",
+        "policy": "",
+        "message": "",
+        "confirm_required": False,
+        "log": False,
+    }
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Analyze tool call risk")
+    parser.add_argument("--root", type=Path, default=Path.cwd())
+    parser.add_argument("--tool", required=True)
+    parser.add_argument("--args", default="{}")
+    parser.add_argument("--process-slug", default="")
+    parser.add_argument("--log-dir", default=None)
+    args = parser.parse_args()
+
+    tool_args = json.loads(args.args)
+    result = analyze(
+        args.root,
+        args.tool,
+        tool_args,
+        process_slug=args.process_slug,
+        log_dir=args.log_dir,
+    )
+    print(json.dumps(result, ensure_ascii=False))