@sushanglewis/lincoln 1.6.2 → 1.7.0

package/framework/scripts/lincoln_render.py

@@ -189,6 +189,91 @@ def _escape_markdown_for_script(markdown: str) -> str:
     return markdown.replace("</script>", "<\\/script>")
 
 
+def _derive_uid_from_target(target: str) -> str:
+    """Derive a stable UID from the target filename."""
+    name = Path(target).name
+    # Strip a single well-known extension.
+    for ext in (".html", ".htm"):
+        if name.lower().endswith(ext):
+            return name[: -len(ext)]
+    return name
+
+
+def _detect_version_from_markdown(source: str | Path) -> str | None:
+    """Extract a version marker from Markdown text or a Markdown file path."""
+    text = ""
+    if isinstance(source, Path):
+        if not source.exists():
+            return None
+        try:
+            text = source.read_text(encoding="utf-8")
+        except Exception:
+            return None
+    else:
+        text = source
+    for line in text.splitlines()[:20]:
+        m = VERSION_COMMENT_RE.search(line)
+        if m:
+            return m.group(1)
+    return None
+
+
+def _derive_defaults(
+    markdown_path: str | Path | None,
+    target: str,
+    title: str,
+    nav_label: str | None,
+    nav_group: str | None,
+    version: str | None,
+    uid: str | None,
+    stage_mark: str | None,
+    stage_id: str,
+) -> dict[str, str]:
+    """Return coarse defaults for page-level placeholders.
+
+    The CLI already provides defaults for some fields; this helper centralizes
+    the "derive from title/target/markdown" logic so both single-page and batch
+    rendering behave the same way.
+    """
+    md_path: Path | None = None
+    md_content: str | None = None
+    if markdown_path:
+        if isinstance(markdown_path, Path):
+            md_path = markdown_path
+        elif "\n" in markdown_path or len(markdown_path) > 260 or not (PROJECT_ROOT / markdown_path).exists():
+            # Treat as inline Markdown content rather than a file path.
+            md_content = markdown_path
+        else:
+            md_path = PROJECT_ROOT / markdown_path
+
+    derived_title = title
+    derived_nav_label = nav_label if nav_label is not None else title
+    derived_nav_group = nav_group if nav_group is not None else "Docs"
+    derived_version = version if version is not None else ""
+    derived_uid = uid if uid is not None else ""
+    derived_stage_mark = stage_mark if stage_mark is not None else stage_id
+
+    if not derived_version:
+        if md_path and md_path.exists():
+            derived_version = _detect_version_from_markdown(md_path) or ""
+        elif md_content:
+            derived_version = _detect_version_from_markdown(md_content) or ""
+    if not derived_version:
+        derived_version = "v1.0"
+
+    if not derived_uid:
+        derived_uid = _derive_uid_from_target(target)
+
+    return {
+        "title": derived_title,
+        "nav_label": derived_nav_label,
+        "nav_group": derived_nav_group,
+        "version": derived_version,
+        "uid": derived_uid,
+        "stage_mark": derived_stage_mark,
+    }
+
+
 def _html_attr(value: str) -> str:
     return html.escape(value, quote=True)
 
@@ -461,6 +546,86 @@ def _ensure_version_comment(html: str, version: str) -> str:
     return html + marker
 
 
+GLOB_CHARS = frozenset("*?[]")
+
+
+def _is_glob_pattern(pattern: str) -> bool:
+    return any(ch in GLOB_CHARS for ch in pattern)
+
+
+def _render_stage_batch(stage_id: str, state_file: str | Path | None) -> list[str]:
+    """Render all missing artifact pages declared by a stage definition.
+
+    Only exact (non-glob) patterns are rendered automatically. If a matching
+    .md file exists next to the target, it is used as the Markdown source.
+    Returns the list of rendered target paths.
+    """
+    stage_data = _load_stage(stage_id)
+    runtime = _load_runtime_variables(state_file)
+    variables: dict[str, str] = {k: "" for k in RUNTIME_VARIABLES}
+    variables.update(runtime)
+
+    raw_templates = stage_data.get("artifact_templates", {}) or {}
+    interpolated_templates = {
+        _replace_variables(str(pattern), variables): str(template_path)
+        for pattern, template_path in raw_templates.items()
+    }
+
+    rendered: list[str] = []
+    for pattern, template_path in interpolated_templates.items():
+        if _is_glob_pattern(pattern):
+            print(f"Skipping glob pattern in batch render: {pattern}")
+            continue
+
+        target_path = PROJECT_ROOT / pattern
+        if target_path.exists():
+            continue
+
+        # Look for a matching Markdown file (e.g. design-review.html -> design-review.md).
+        md_path = target_path.with_suffix(".md")
+        markdown_source = ""
+        if md_path.exists():
+            markdown_source = md_path.read_text(encoding="utf-8")
+
+        defaults = _derive_defaults(
+            markdown_path=md_path if markdown_source else None,
+            target=pattern,
+            title="",
+            nav_label=None,
+            nav_group=None,
+            version=None,
+            uid=None,
+            stage_mark=None,
+            stage_id=stage_id,
+        )
+
+        try:
+            html = render_page(
+                stage_id=stage_id,
+                target=pattern,
+                title=defaults["title"],
+                nav_label=defaults["nav_label"],
+                nav_group=defaults["nav_group"],
+                version=defaults["version"],
+                uid=defaults["uid"],
+                stage_mark=defaults["stage_mark"],
+                markdown_source=markdown_source,
+                page_data=None,
+                extra_variables=None,
+                state_file=state_file,
+            )
+        except Exception as exc:
+            print(f"Failed to render {pattern}: {exc}", file=sys.stderr)
+            continue
+
+        target_path.parent.mkdir(parents=True, exist_ok=True)
+        atomic_write_text(target_path, html)
+        print(f"Rendered {target_path}")
+        rendered.append(pattern)
+
+    return rendered
+
+
 PAGE_UID_META_RE = re.compile(r'<meta\s+name="page-uid"', re.IGNORECASE)
 
 
@@ -488,6 +653,24 @@ def render_page(
     """Render a single HTML page and return the rendered text."""
     stage_data = _load_stage(stage_id)
 
+    defaults = _derive_defaults(
+        markdown_path=markdown_source if markdown_source else None,
+        target=target,
+        title=title,
+        nav_label=nav_label,
+        nav_group=nav_group,
+        version=version,
+        uid=uid,
+        stage_mark=stage_mark,
+        stage_id=stage_id,
+    )
+    title = defaults["title"]
+    nav_label = defaults["nav_label"]
+    nav_group = defaults["nav_group"]
+    version = defaults["version"]
+    uid = defaults["uid"]
+    stage_mark = defaults["stage_mark"]
+
     runtime = _load_runtime_variables(state_file)
     variables: dict[str, str] = {k: "" for k in RUNTIME_VARIABLES}
     variables.update(runtime)
@@ -548,15 +731,15 @@ def _build_argument_parser() -> argparse.ArgumentParser:
     )
     parser.add_argument("--stage", required=True, help="Stage ID (e.g. clarify)")
     parser.add_argument(
-        "--target", required=True, help="Repo-relative target artifact path"
+        "--target", default="", help="Repo-relative target artifact path"
     )
     parser.add_argument("--title", default="", help="Page title")
-    parser.add_argument("--nav-label", default="", help="Navigation label")
-    parser.add_argument("--nav-group", default="Docs", help="Navigation group")
-    parser.add_argument("--version", default="", help="Version marker (e.g. v1.0)")
-    parser.add_argument("--uid", default="", help="Stable page/doc UID")
+    parser.add_argument("--nav-label", default=None, help="Navigation label (defaults to title)")
+    parser.add_argument("--nav-group", default=None, help="Navigation group (defaults to Docs)")
+    parser.add_argument("--version", default=None, help="Version marker (defaults to v1.0 or from Markdown)")
+    parser.add_argument("--uid", default=None, help="Stable page/doc UID (defaults to target filename)")
     parser.add_argument(
-        "--stage-mark", default="", help="Stage label to display (defaults to stage ID)"
+        "--stage-mark", default=None, help="Stage label to display (defaults to stage ID)"
     )
     parser.add_argument(
         "--markdown",
@@ -579,6 +762,11 @@ def _build_argument_parser() -> argparse.ArgumentParser:
         default=None,
         help="Path to workflow-stage.yaml (defaults to canonical discovery)",
     )
+    parser.add_argument(
+        "--render-stage",
+        action="store_true",
+        help="Render all missing artifact pages declared by --stage (requires --state-file)",
+    )
     parser.add_argument(
         "--dry-run",
         action="store_true",
@@ -601,6 +789,21 @@ def main(argv: list[str] | None = None) -> int:
     parser = _build_argument_parser()
     args = parser.parse_args(argv)
 
+    if args.render_stage:
+        if not args.state_file:
+            print("--render-stage requires --state-file", file=sys.stderr)
+            return 1
+        try:
+            _render_stage_batch(args.stage, args.state_file)
+        except Exception as exc:
+            print(f"Batch render failed: {exc}", file=sys.stderr)
+            return 1
+        return 0
+
+    if not args.target:
+        print("--target is required unless using --render-stage", file=sys.stderr)
+        return 1
+
     markdown_source = ""
     if args.markdown:
         md_path = PROJECT_ROOT / args.markdown

package/framework/scripts/lincoln_trace.py

@@ -31,7 +31,7 @@ from scripts.lincoln_paths import (  # noqa: E402
     process_package_root,
 )
 
-TRACE_SCHEMA_VERSION = "1.0.0"
+TRACE_SCHEMA_VERSION = "2.0.0"
 
 # Tools whose invocations are not traced to avoid recursion or noise.
 NO_TRACE_TOOLS = {"Read", "Grep", "Glob"}
@@ -46,6 +46,26 @@ TASK_TOOLS = frozenset({
     "TaskStop",
 })
 
+# All valid trace categories. Tool categories are auto-derived; lifecycle categories
+# are written explicitly by stage_loader and hooks.
+VALID_CATEGORIES = frozenset({
+    "tool",
+    "skill",
+    "agent",
+    "read",
+    "write",
+    "edit",
+    "bash",
+    "task",
+    "mcp",
+    "handoff",
+    "other",
+    "stage_lifecycle",
+    "gate",
+    "session",
+    "condensation",
+})
+
 # Maximum length stored for a Bash command summary in the trace entry.
 COMMAND_SUMMARY_MAX_LEN = 500
 
@@ -202,6 +222,11 @@ def _resolve_trace_file(state: dict[str, Any] | None, state_path: Path | None) -
     return trace_dir / "lc-trace.jsonl"
 
 
+def get_trace_file(state: dict[str, Any] | None = None, state_path: Path | None = None) -> Path:
+    """Public accessor for the resolved trace file path."""
+    return _resolve_trace_file(state, state_path)
+
+
 def _get_run_id(state: dict[str, Any] | None, run_id: str | None) -> str:
     if run_id:
         return run_id
@@ -250,12 +275,26 @@ def _build_trace_target(tool: str, args_summary: dict[str, Any]) -> str:
     return str(args_summary.get("path", args_summary.get("name", "")))
 
 
+def _get_node_id(state: dict[str, Any] | None, node_id: str | None) -> str | None:
+    if node_id:
+        return node_id
+    if state:
+        current_run = state.get("current_run", {})
+        # Find the in_progress node for the current stage.
+        current_stage = current_run.get("current_stage", "")
+        for node in state.get("nodes", []):
+            if node.get("stage_id") == current_stage and node.get("status") == "in_progress":
+                return str(node.get("node_id", "")) or None
+    return None
+
+
 def _build_trace_entry(
     tool: str,
     args_summary: dict[str, Any],
     exit_code: int,
     run_id: str,
     stage: str,
+    node_id: str | None = None,
 ) -> dict[str, Any]:
     return {
         "schema_version": TRACE_SCHEMA_VERSION,
@@ -263,6 +302,7 @@ def _build_trace_entry(
         "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
         "run_id": run_id,
         "stage": stage,
+        "node_id": node_id,
         "tool": tool,
         "category": categorize_tool(tool),
         "target": _build_trace_target(tool, args_summary),
@@ -290,8 +330,9 @@ def append_trace_entry(
     exit_code: int,
     stage: str | None = None,
     run_id: str | None = None,
+    node_id: str | None = None,
 ) -> Path | None:
-    """Append a single trace entry to the appropriate JSONL file.
+    """Append a single trace entry for a tool invocation.
 
     The write is atomic with respect to other processes on the same host: the
     file is locked while appending, so concurrent parent and child agents do
@@ -303,9 +344,84 @@ def append_trace_entry(
     state = _load_state(state_path)
     resolved_run_id = _get_run_id(state, run_id)
     resolved_stage = _get_stage(state, stage)
+    resolved_node_id = _get_node_id(state, node_id)
     trace_file = _resolve_trace_file(state, state_path)
     args_summary = redact_args(tool, _safe_json_loads(args))
-    entry = _build_trace_entry(tool, args_summary, exit_code, resolved_run_id, resolved_stage)
+    entry = _build_trace_entry(
+        tool, args_summary, exit_code, resolved_run_id, resolved_stage, resolved_node_id
+    )
+    _append_trace_line(trace_file, entry)
+    return trace_file
+
+
+def append_event_entry(
+    state_path: Path | None,
+    category: str,
+    payload: dict[str, Any],
+    stage: str | None = None,
+    run_id: str | None = None,
+    node_id: str | None = None,
+) -> Path | None:
+    """Append a non-tool lifecycle event to the trace.
+
+    Categories must be one of VALID_CATEGORIES. The payload is stored under
+    ``args_summary`` for backward compatibility with existing readers.
+    """
+    if category not in VALID_CATEGORIES:
+        raise ValueError(f"Invalid trace category: {category!r}")
+
+    state = _load_state(state_path)
+    resolved_run_id = _get_run_id(state, run_id)
+    resolved_stage = _get_stage(state, stage)
+    resolved_node_id = _get_node_id(state, node_id)
+    trace_file = _resolve_trace_file(state, state_path)
+
+    entry: dict[str, Any] = {
+        "schema_version": TRACE_SCHEMA_VERSION,
+        "sequence_id": generate_sequence_id(),
+        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "run_id": resolved_run_id,
+        "stage": resolved_stage,
+        "node_id": resolved_node_id,
+        "tool": "",
+        "category": category,
+        "target": str(payload.get("event_type", payload.get("type", "")))[:200],
+        "exit_code": int(payload.get("exit_code", 0)),
+        "args_summary": payload,
+    }
+    _append_trace_line(trace_file, entry)
+    return trace_file
+
+
+def append_event_to_file(
+    trace_file: Path,
+    category: str,
+    payload: dict[str, Any],
+    stage: str | None = None,
+    run_id: str | None = None,
+    node_id: str | None = None,
+) -> Path:
+    """Append a non-tool lifecycle event to an explicit trace file path.
+
+    This variant is useful when the caller has already resolved the trace file
+    (e.g. the condenser) and wants to avoid re-resolving it from state.
+    """
+    if category not in VALID_CATEGORIES:
+        raise ValueError(f"Invalid trace category: {category!r}")
+
+    entry: dict[str, Any] = {
+        "schema_version": TRACE_SCHEMA_VERSION,
+        "sequence_id": generate_sequence_id(),
+        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "run_id": run_id or "unknown",
+        "stage": stage or "unknown",
+        "node_id": node_id or "unknown",
+        "tool": "",
+        "category": category,
+        "target": str(payload.get("event_type", payload.get("type", "")))[:200],
+        "exit_code": int(payload.get("exit_code", 0)),
+        "args_summary": payload,
+    }
     _append_trace_line(trace_file, entry)
     return trace_file
 

package/framework/scripts/security_analyzer.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""Risk analyzer for Lincoln pre-tool-use security gate.
+
+Loads `.claude/policies/security.yaml` and evaluates a tool call against the
+configured policies. Designed to be invoked from `.claude/hooks/pre-tool-use.sh`
+and from generated harness-specific hook scripts.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+DEFAULT_POLICY_PATH = ".claude/policies/security.yaml"
+
+
+def load_policy(root: Path) -> dict[str, Any]:
+    """Load the risk policy YAML, returning an empty policy if missing."""
+    path = root / DEFAULT_POLICY_PATH
+    if not path.exists():
+        return {"policies": []}
+    return yaml.safe_load(path.read_text(encoding="utf-8")) or {"policies": []}
+
+
+def _normalize_command(command: str) -> str:
+    """Strip leading whitespace and collapse internal spaces for matching."""
+    return re.sub(r"\s+", " ", command.strip())
+
+
+def _split_command_segments(command: str) -> list[str]:
+    """Split a shell command on chain operators so each segment is matched.
+
+    Handles `&&`, `||`, `;`, and `|`.
+    """
+    segments = re.split(r"\s*(&&|\|\||;|\|)\s*", command)
+    return [_normalize_command(seg) for seg in segments if seg and not re.match(r"^(&&|\|\||;|\|)$", seg.strip())]
+
+
+def _strip_command_prefix(segment: str) -> str:
+    """Remove common leading prefixes that should not hide the real command.
+
+    Examples: `sudo rm foo`, `env VAR=1 rm foo`, `cd /tmp && rm foo`.
+    """
+    segment = _normalize_command(segment)
+    while re.match(r"^[A-Za-z_][A-Za-z0-9_]*=\S+\s+", segment):
+        segment = _normalize_command(re.sub(r"^[A-Za-z_][A-Za-z0-9_]*=\S+\s+", "", segment))
+    if re.match(r"^(sudo|doas)\s+", segment):
+        segment = _normalize_command(re.sub(r"^(sudo|doas)\s+", "", segment))
+    if re.match(r"^cd\s+\S+\s*(&&|;|\|)", segment):
+        segment = _normalize_command(re.sub(r"^cd\s+\S+\s*(&&|;|\|)", "", segment))
+    return segment
+
+
+def _command_matches(condition: str, command: str) -> bool:
+    """Evaluate a human-readable condition against a bash command."""
+    lowered = condition.lower()
+    segments = _split_command_segments(command)
+
+    def _any_segment(pattern: str) -> bool:
+        return any(re.search(pattern, _strip_command_prefix(seg)) for seg in segments)
+
+    if "rm -rf" in lowered or 'command matches "rm -rf"' in lowered:
+        if _any_segment(r"\brm\s+-rf\b"):
+            return True
+
+    if 'command matches "rm -r" or "rm -rf"' in lowered:
+        if _any_segment(r"\brm\s+-r(?:f)?\b"):
+            return True
+
+    if 'command matches any of ["rm", "rmdir"]' in lowered:
+        if _any_segment(r"(?:^|\s)(?:rm\b|rmdir\b)"):
+            return True
+
+    if 'command matches "git push"' in lowered:
+        if _any_segment(r"\bgit\s+push\b"):
+            return True
+
+    if "external http client" in lowered or "curl, wget" in lowered:
+        if _any_segment(r"\b(?:curl|wget|httpie)\b"):
+            return True
+
+    if 'command matches "rm"' in lowered and "rmdir" not in lowered:
+        if _any_segment(r"\brm\b"):
+            return True
+
+    return False
+
+
+def _resolve_target_path(target: str, root: Path) -> Path:
+    """Resolve a Write/Edit target to an absolute, canonical path."""
+    path = Path(target)
+    if not path.is_absolute():
+        path = root / path
+    try:
+        return path.resolve()
+    except (OSError, ValueError):
+        return path.absolute()
+
+
+def _target_matches(condition: str, target: str, process_slug: str, root: Path) -> bool:
+    """Evaluate a file-target condition using canonical absolute paths."""
+    lowered = condition.lower()
+    resolved = _resolve_target_path(target, root)
+    try:
+        package_root = (root / process_slug).resolve()
+    except (OSError, ValueError):
+        package_root = (root / process_slug).absolute()
+
+    if "outside current process_slug" in lowered:
+        try:
+            resolved.relative_to(package_root)
+            return False
+        except ValueError:
+            return True
+
+    if "inside current process_slug" in lowered:
+        try:
+            resolved.relative_to(package_root)
+            return True
+        except ValueError:
+            return False
+
+    return False
+
+
+def _evaluate_policy(policy: dict[str, Any], tool_name: str, tool_args: dict[str, Any], process_slug: str, root: Path) -> bool:
+    """Return True if the policy applies to this tool call."""
+    tools = [t.strip() for t in policy.get("tool", "").split("|")]
+    if tool_name not in tools:
+        return False
+
+    condition = policy.get("condition", "")
+    if tool_name == "Bash":
+        command = tool_args.get("command", "")
+        return _command_matches(condition, command)
+
+    if tool_name in ("Write", "Edit"):
+        target = tool_args.get("file_path") or tool_args.get("path") or ""
+        return _target_matches(condition, target, process_slug, root)
+
+    return False
+
+
+def _log_security_event(
+    log_dir: Path,
+    tool_name: str,
+    tool_args: dict[str, Any],
+    result: dict[str, Any],
+) -> None:
+    """Append a JSON line to issue-<N>/logs/security.log."""
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / "security.log"
+    entry = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "tool": tool_name,
+        "level": result["level"],
+        "policy": result.get("policy", ""),
+        "action": "blocked" if result["confirm_required"] else "allowed",
+        "args": {k: v for k, v in tool_args.items() if k in ("command", "file_path", "path")},
+    }
+    with log_file.open("a", encoding="utf-8") as f:
+        f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+
+def analyze(
+    root: Path,
+    tool_name: str,
+    tool_args: dict[str, Any],
+    process_slug: str = "",
+    log_dir: str | None = None,
+) -> dict[str, Any]:
+    """Analyze a tool call and return risk assessment."""
+    if os.environ.get("LINCOLN_SECURITY_MODE") == "permissive":
+        return {
+            "level": "low",
+            "policy": "",
+            "message": "Security analyzer is in permissive mode.",
+            "confirm_required": False,
+            "log": False,
+        }
+
+    policy_data = load_policy(root)
+    policies = policy_data.get("policies", [])
+
+    for policy in policies:
+        if not _evaluate_policy(policy, tool_name, tool_args, process_slug, root):
+            continue
+
+        result = {
+            "level": policy.get("level", "unknown"),
+            "policy": policy.get("name", ""),
+            "message": policy.get("message", ""),
+            "confirm_required": bool(policy.get("confirm", False)),
+            "log": bool(policy.get("log", False)),
+        }
+        if log_dir:
+            _log_security_event(Path(log_dir), tool_name, tool_args, result)
+        return result
+
+    return {
+        "level": "low",
+        "policy": "",
+        "message": "",
+        "confirm_required": False,
+        "log": False,
+    }
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Analyze tool call risk")
+    parser.add_argument("--root", type=Path, default=Path.cwd())
+    parser.add_argument("--tool", required=True)
+    parser.add_argument("--args", default="{}")
+    parser.add_argument("--process-slug", default="")
+    parser.add_argument("--log-dir", default=None)
+    args = parser.parse_args()
+
+    tool_args = json.loads(args.args)
+    result = analyze(
+        args.root,
+        args.tool,
+        tool_args,
+        process_slug=args.process_slug,
+        log_dir=args.log_dir,
+    )
+    print(json.dumps(result, ensure_ascii=False))