arkaos 4.0.2 → 4.1.0

package/core/hooks/pre_tool_use.py

@@ -0,0 +1,267 @@
+"""PreToolUse — consolidated entrypoint (PR-6 v4.1.0 hook hygiene).
+
+Replaces the 5-8 python3/jq spawns of the old ``pre-tool-use.sh`` heredocs
+with ONE python process. Gate order is preserved exactly:
+
+    1. KB-first research gate  (core/workflow/research_gate.py)
+    2. Specialist-dispatch gate (core/workflow/specialist_enforcer.py)
+    3. Fast allow for non-flow-gated tools
+    4. CostGovernor budget check (stdlib-only — MUST run even when the
+       yaml-needing enforcers degrade; see ADR 2026-07-04-cost-governor)
+    5. Evidence-flow gate (core/workflow/flow_enforcer.py)
+
+Behavior contract (unchanged from the bash version):
+    - allow  → no stdout, exit 0 (nudges/warnings on stderr)
+    - deny   → stderr message + permissionDecision=deny JSON, exit 2
+    - env bypasses (ARKA_BYPASS_FLOW, ARKA_BYPASS_KB_FIRST) and feature
+      flags are honored inside the delegated modules
+    - every module import is lazy try/except: a PyYAML-less python3
+      degrades each gate to allow, identical to the old heredocs
+
+The transcript is parsed at most ONCE per invocation and shared between
+the specialist and flow gates via their ``messages=`` params.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from core.hooks._shared import (
+    emit_deny_json,
+    ensure_root_on_path,
+    get_str,
+    read_stdin_json,
+    resolve_arkaos_root,
+)
+
+_FLOW_GATED_TOOLS = frozenset({
+    "Write", "Edit", "MultiEdit", "NotebookEdit", "Task", "Skill", "Bash",
+})
+_SPECIALIST_TOOLS = frozenset({"Write", "Edit", "MultiEdit", "NotebookEdit"})
+_ASSISTANT_WINDOW = 20
+
+
+def _query_hint(tool_input: dict) -> str:
+    """Mirror `jq -r '.tool_input.query // .prompt // .url' | head -c 500`."""
+    for key in ("query", "prompt", "url"):
+        value = tool_input.get(key)
+        if value is not None and value is not False:
+            return str(value)[:500]
+    return ""
+
+
+class _MessagesOnce:
+    """Parse the transcript at most once, lazily, across both gates.
+
+    ``peek()`` never triggers a read — gates that have their own cheaper
+    early-outs (feature flag off, marker-cache hit) pass ``peek()`` so the
+    module self-loads only when it really needs the transcript. ``load()``
+    is called by the first gate that has already decided it needs messages.
+    """
+
+    def __init__(self, transcript_path: str):
+        self._path = transcript_path
+        self._messages: list[str] | None = None
+
+    def peek(self) -> list[str] | None:
+        return self._messages
+
+    def load(self) -> list[str] | None:
+        if self._messages is None:
+            try:
+                from core.workflow.flow_enforcer import (
+                    _load_last_assistant_messages,
+                )
+                self._messages = _load_last_assistant_messages(
+                    self._path, _ASSISTANT_WINDOW
+                )
+            except Exception:
+                return None
+        return self._messages
+
+
+def _deny(stderr_msg: str) -> int:
+    if stderr_msg:
+        print(stderr_msg, file=sys.stderr)
+    emit_deny_json(stderr_msg)
+    return 2
+
+
+def _kb_gate(root: str, tool_name: str, session_id: str, query: str) -> int | None:
+    """KB-first gate. Returns 2 on deny, None to continue."""
+    if not (Path(root) / "core" / "workflow" / "research_gate.py").is_file():
+        return None
+    try:
+        from core.workflow.research_gate import (
+            evaluate_research_gate,
+            record_telemetry,
+        )
+    except Exception:
+        return None  # kb-gate-import-failed → allow (old heredoc contract)
+    decision = evaluate_research_gate(
+        tool_name=tool_name, session_id=session_id, query=query
+    )
+    try:
+        record_telemetry(session_id=session_id, tool=tool_name, decision=decision)
+    except Exception:
+        pass
+    if not decision.allow:
+        return _deny(decision.to_stderr_message())
+    if decision.nudge and decision.to_stderr_message():
+        print(decision.to_stderr_message(), file=sys.stderr)
+    return None
+
+
+def _specialist_gate(
+    root: str,
+    tool_name: str,
+    transcript_path: str,
+    session_id: str,
+    cwd: str,
+    tool_input: dict,
+    messages: _MessagesOnce,
+) -> int | None:
+    """Specialist-dispatch gate. Returns 2 on deny, None to continue."""
+    if tool_name not in _SPECIALIST_TOOLS:
+        return None
+    module_path = Path(root) / "core" / "workflow" / "specialist_enforcer.py"
+    if not module_path.is_file():
+        return None
+    try:
+        from core.workflow.specialist_enforcer import (
+            _feature_flag_on,
+            evaluate,
+            record_telemetry,
+        )
+    except Exception:
+        return None  # specialist-import-failed → allow
+    # Load (and share) the transcript only when the gate will actually
+    # scan it — flag-off sessions keep the zero-read fast path.
+    shared = messages.peek()
+    if shared is None and _feature_flag_on():
+        shared = messages.load()
+    decision = evaluate(
+        tool_name=tool_name,
+        transcript_path=transcript_path,
+        session_id=session_id,
+        cwd=cwd,
+        tool_input=tool_input,
+        messages=shared,
+    )
+    try:
+        record_telemetry(
+            session_id=session_id,
+            tool=tool_name,
+            decision=decision,
+            cwd=cwd,
+            target_file=str(tool_input.get("file_path", "")),
+            model_requested=str(tool_input.get("model", "")),
+        )
+    except Exception:
+        pass
+    if not decision.allow:
+        return _deny(decision.to_stderr_message())
+    return None
+
+
+def _budget_check(session_id: str) -> tuple[bool, str]:
+    """CostGovernor gate (stdlib-only). Returns (allow, warning)."""
+    try:
+        from core.runtime.cost_governor import check as _check
+        gov = _check(session_id)
+        return gov.allow, gov.to_warning()
+    except Exception:
+        return True, ""
+
+
+def _flow_gate(
+    root: str,
+    tool_name: str,
+    transcript_path: str,
+    session_id: str,
+    cwd: str,
+    tool_input: dict,
+    messages: _MessagesOnce,
+) -> int:
+    """Budget + evidence-flow gate. Returns final exit code."""
+    # Budget fires even when the flow enforcer degrades (PR-5 contract).
+    budget_allow, budget_warning = _budget_check(session_id)
+    if budget_warning:
+        print(budget_warning, file=sys.stderr)
+    if not budget_allow:
+        return _deny(budget_warning)
+
+    try:
+        from core.workflow.flow_enforcer import evaluate, record_telemetry
+    except Exception:
+        return 0  # enforcer-import-failed → allow
+    decision = evaluate(
+        tool_name=tool_name,
+        transcript_path=transcript_path,
+        session_id=session_id,
+        cwd=cwd,
+        tool_input=tool_input,
+        # peek() only: when the specialist gate already parsed, reuse it;
+        # otherwise let evaluate() keep its lazy early-outs (feature flag,
+        # env bypass, classifier flag, marker cache) before reading.
+        messages=messages.peek(),
+    )
+    try:
+        record_telemetry(
+            session_id=session_id, tool=tool_name, decision=decision, cwd=cwd
+        )
+    except Exception:
+        pass
+    if decision.allow:
+        return 0
+    return _deny(decision.to_stderr_message())
+
+
+def main(stdin_json: dict | None = None) -> int:
+    if stdin_json is None:
+        stdin_json, _ = read_stdin_json()
+    root = resolve_arkaos_root()
+    ensure_root_on_path(root)
+
+    tool_name = get_str(stdin_json, "tool_name")
+    transcript_path = get_str(stdin_json, "transcript_path")
+    session_id = get_str(stdin_json, "session_id")
+    cwd = get_str(stdin_json, "cwd")
+    tool_input = stdin_json.get("tool_input")
+    if not isinstance(tool_input, dict):
+        tool_input = {}
+
+    messages = _MessagesOnce(transcript_path)
+
+    code = _kb_gate(root, tool_name, session_id, _query_hint(tool_input))
+    if code is not None:
+        return code
+
+    code = _specialist_gate(
+        root, tool_name, transcript_path, session_id, cwd, tool_input, messages
+    )
+    if code is not None:
+        return code
+
+    # Fast allow: not a flow-gated tool (Bash stays — classified per-command
+    # by the enforcer via bash_is_effect()).
+    if tool_name not in _FLOW_GATED_TOOLS:
+        return 0
+    if not (Path(root) / "core" / "workflow" / "flow_enforcer.py").is_file():
+        return 0
+
+    return _flow_gate(
+        root, tool_name, transcript_path, session_id, cwd, tool_input, messages
+    )
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except SystemExit:
+        raise
+    except Exception:
+        # Fail open — the bash version piped every heredoc through
+        # `2>/dev/null` and exited 0 on internal errors.
+        raise SystemExit(0)

package/core/hooks/stop.py

@@ -0,0 +1,395 @@
+"""Stop — consolidated entrypoint (PR-6 v4.1.0 hook hygiene).
+
+Replaces the ~16 python3/jq spawn sites of the old ``stop.sh`` with ONE
+python process. WARN mode: this hook NEVER blocks — always exits 0.
+
+Section order preserved exactly:
+    1. DNA fidelity check (persona from last dispatch/routing marker)
+    2. stop_hook_active loop guard
+    3. Native usage cost capture (runs BEFORE the WF_MARKER gate — the
+       llm-cost blind spot covers ALL sessions)
+    4. WF_MARKER gate (/tmp/arkaos-wf-required/<session>)
+    5. Flow completion validation + soft-block checks (meta_tag_found,
+       [arka:meta], kb-cite, sycophancy, closing marker, skill proposer,
+       kb=N reconciliation) → telemetry entry (mode: warn)
+    6. Gate checkpoint persistence (core/workflow/gate_checkpoint.py)
+    7. Auto-documentor enqueue (flow-required + QG APPROVED + external
+       research heuristic)
+    8. Belt-and-braces WF_MARKER removal
+
+The transcript is read from disk ONCE and shared across native-usage,
+flow validation, gate checkpoint, and the auto-doc external-research
+heuristic (the old hook parsed it four times).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+
+from core.hooks._shared import (
+    ensure_root_on_path,
+    get_str,
+    read_stdin_json,
+    resolve_arkaos_root,
+    safe_session_id,
+)
+
+_DISPATCH_RE = re.compile(
+    r"\[arka:dispatch\][ \t]*[A-Za-z0-9_-]+[ \t]*->[ \t]*([A-Za-z0-9_-]+)",
+    re.IGNORECASE,
+)
+_ROUTING_RE = re.compile(
+    r"\[arka:routing\][ \t]*[A-Za-z0-9_-]+[ \t]*->[ \t]*([A-Za-z0-9_-]+)",
+    re.IGNORECASE,
+)
+
+_EXTERNAL_MARKERS = (
+    "WebFetch", "WebSearch", "mcp__context7", "mcp__firecrawl",
+    "http://", "https://",
+)
+
+
+def _read_transcript(transcript_path: str) -> str | None:
+    """Single shared disk read for the whole hook invocation."""
+    if not transcript_path:
+        return None
+    try:
+        return Path(transcript_path).read_text(
+            encoding="utf-8", errors="replace"
+        )
+    except OSError:
+        return None
+
+
+def _extract_persona(assistant_msg: str) -> str:
+    """Dispatch marker wins over routing; latest match wins."""
+    dispatch_hits = _DISPATCH_RE.findall(assistant_msg)
+    if dispatch_hits:
+        return dispatch_hits[-1].lower()
+    routing_hits = _ROUTING_RE.findall(assistant_msg)
+    if routing_hits:
+        return routing_hits[-1].lower()
+    return ""
+
+
+def _dna_fidelity(assistant_msg: str, session_id: str) -> None:
+    if not assistant_msg or not session_id:
+        return
+    persona = _extract_persona(assistant_msg)
+    if not persona:
+        return
+    try:
+        from core.governance.dna_fidelity import check_fidelity, record_fidelity
+        violations = check_fidelity(persona, assistant_msg)
+        record_fidelity(persona, session_id, violations)
+    except Exception:
+        pass
+
+
+def _native_usage(transcript_path: str, session_id: str, raw: str | None) -> None:
+    if not transcript_path or not session_id:
+        return
+    try:
+        from core.runtime.native_usage import record_native_usage
+        record_native_usage(transcript_path, session_id, raw_text=raw)
+    except Exception:
+        pass
+
+
+def _write_tmp_state(subdir: str, safe_sid: str, payload: dict) -> None:
+    """Owner-only /tmp state file (umask 0o077 — PR25 v2.46.1)."""
+    prev_umask = os.umask(0o077)
+    try:
+        state_dir = Path("/tmp") / subdir
+        state_dir.mkdir(parents=True, exist_ok=True)
+        (state_dir / f"{safe_sid}.json").write_text(
+            json.dumps(payload), encoding="utf-8"
+        )
+    finally:
+        os.umask(prev_umask)
+
+
+def _flow_checks(
+    session_id: str, transcript_path: str, cwd: str,
+    effort_level: str, raw: str | None,
+) -> None:
+    """Sections 5-6: soft-block checks + telemetry + gate checkpoint."""
+    try:
+        from core.workflow.flow_enforcer import (
+            TELEMETRY_PATH,
+            _load_last_assistant_messages,
+            clear_flow_required,
+        )
+    except Exception:
+        return
+
+    # Parse once; the last message drives every check, the 20-window
+    # feeds the gate checkpoint.
+    messages = _load_last_assistant_messages(
+        transcript_path, n=20, raw_text=raw
+    )
+    last = messages[-1] if messages else ""
+
+    gate4 = bool(re.search(r"\[arka:gate:4\]", last, re.IGNORECASE))
+    phase13 = bool(re.search(r"\[arka:phase:13\]", last, re.IGNORECASE))
+    trivial = bool(re.search(r"\[arka:trivial\]", last, re.IGNORECASE))
+    closing_ok = gate4 or phase13 or trivial
+
+    # v4.1.0 — persist gate transitions for structured resume.
+    try:
+        from core.workflow.gate_checkpoint import checkpoint as _gate_checkpoint
+        _gate_checkpoint(
+            transcript_path, session_id, project=cwd, messages=messages
+        )
+    except Exception:
+        pass
+
+    meta_tag_found = bool(re.search(r"\[arka:meta\]", last, re.IGNORECASE))
+
+    sycophancy_signals: list = []
+    sycophancy_confidence = 0.0
+    is_sycophantic = False
+    try:
+        from core.governance.sycophancy_detector import detect_sycophancy
+        sv = detect_sycophancy(last)
+        sycophancy_signals = sv.signals
+        sycophancy_confidence = sv.confidence
+        is_sycophantic = sv.is_sycophantic
+    except Exception:
+        pass
+
+    cite_passed = True
+    cite_reason = "trivial"
+    cite_count = 0
+    cite_topic_score = 0.0
+    safe_sid: str | None = None
+    try:
+        from core.governance.kb_cite_check import check_citation
+        cr = check_citation(last)
+        cite_passed = cr.passed
+        cite_reason = cr.reason
+        cite_count = cr.citation_count
+        cite_topic_score = cr.topic_score
+        safe_sid = safe_session_id(session_id)
+        if safe_sid:
+            _write_tmp_state("arkaos-cite", safe_sid, {
+                "passed": cr.passed,
+                "reason": cr.reason,
+                "suggestion": cr.suggestion,
+                "citation_count": cr.citation_count,
+                "topic_score": cr.topic_score,
+            })
+    except Exception:
+        pass
+
+    try:
+        from core.governance.skill_proposer import evaluate as _eval_skill
+        _eval_skill(last)
+    except Exception:
+        pass
+
+    meta_passed = True
+    meta_reason = "trivial"
+    try:
+        from core.governance.meta_tag_check import check_meta_tag
+        mr = check_meta_tag(last)
+        meta_passed = mr.passed
+        meta_reason = mr.reason
+        if safe_sid:
+            _write_tmp_state("arkaos-meta", safe_sid, {
+                "passed": mr.passed,
+                "reason": mr.reason,
+                "suggestion": mr.suggestion,
+            })
+    except Exception:
+        pass
+
+    # Structural honesty PR-2 — kb=N reconciliation (warn-only).
+    kb_reported = None
+    kb_injected = None
+    kb_inflated = False
+    try:
+        from core.governance.meta_tag_check import (
+            parse_reported_kb,
+            reconcile_kb_count,
+        )
+        kb_reported = parse_reported_kb(last)
+        if safe_sid:
+            injected_path = (
+                Path("/tmp/arkaos-kb-injected") / f"{safe_sid}.json"
+            )
+            if injected_path.exists():
+                injected_data = json.loads(
+                    injected_path.read_text(encoding="utf-8")
+                )
+                raw_injected = injected_data.get("kb_injected")
+                kb_injected = (
+                    int(raw_injected) if raw_injected is not None else None
+                )
+        reconciled = reconcile_kb_count(kb_reported, kb_injected)
+        kb_reported = reconciled["kb_reported"]
+        kb_injected = reconciled["kb_injected"]
+        kb_inflated = reconciled["kb_inflated"]
+    except Exception:
+        pass
+
+    closing_check_passed = True
+    closing_check_reason = "trivial"
+    try:
+        from core.governance.closing_marker_check import check_closing_marker
+        cmr = check_closing_marker(last)
+        closing_check_passed = cmr.passed
+        closing_check_reason = cmr.reason
+        if safe_sid:
+            _write_tmp_state("arkaos-closing", safe_sid, {
+                "passed": cmr.passed,
+                "reason": cmr.reason,
+                "suggestion": cmr.suggestion,
+            })
+    except Exception:
+        pass
+
+    entry = {
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "session_id": session_id,
+        "cwd": cwd,
+        "event": "stop-hook-flow-check",
+        "closing_marker_found": closing_ok,
+        "phase13": phase13,
+        "trivial": trivial,
+        "meta_tag_found": meta_tag_found,
+        "sycophancy_is_flagged": is_sycophantic,
+        "sycophancy_signals": sycophancy_signals,
+        "sycophancy_confidence": sycophancy_confidence,
+        "kb_cite_passed": cite_passed,
+        "kb_cite_reason": cite_reason,
+        "kb_cite_count": cite_count,
+        "kb_cite_topic_score": cite_topic_score,
+        "meta_tag_check_passed": meta_passed,
+        "meta_tag_check_reason": meta_reason,
+        "kb_reported": kb_reported,
+        "kb_injected": kb_injected,
+        "kb_inflated": kb_inflated,
+        "closing_marker_check_passed": closing_check_passed,
+        "closing_marker_check_reason": closing_check_reason,
+        "effort_level": effort_level,
+        "mode": "warn",
+    }
+    try:
+        TELEMETRY_PATH.parent.mkdir(parents=True, exist_ok=True)
+        with TELEMETRY_PATH.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(entry) + "\n")
+    except Exception:
+        pass
+
+    try:
+        clear_flow_required(session_id)
+    except Exception:
+        pass
+
+
+def _auto_doc_enqueue(
+    session_id: str, transcript_path: str, raw: str | None
+) -> None:
+    """Section 7: fire-and-forget auto-documentor job enqueue."""
+    try:
+        from core.jobs.auto_doc_worker import enqueue_job
+        from core.workflow.flow_enforcer import _load_last_assistant_messages
+    except Exception:
+        return
+    if not session_id or not transcript_path:
+        return
+
+    try:
+        msgs = _load_last_assistant_messages(transcript_path, 1, raw_text=raw)
+        last = msgs[-1] if msgs else ""
+    except Exception:
+        last = ""
+
+    qg_approved = bool(re.search(r"\[arka:qg:approved\]", last, re.IGNORECASE))
+    if not qg_approved:
+        qg_log = Path.home() / ".arkaos" / "telemetry" / "qg.jsonl"
+        if qg_log.exists():
+            try:
+                for line in reversed(
+                    qg_log.read_text(encoding="utf-8").splitlines()
+                ):
+                    if not line.strip():
+                        continue
+                    try:
+                        rec = json.loads(line)
+                    except Exception:
+                        continue
+                    if rec.get("session_id") == session_id:
+                        qg_approved = (
+                            rec.get("verdict", "").upper() == "APPROVED"
+                        )
+                        break
+            except Exception:
+                pass
+    if not qg_approved:
+        return
+
+    data = raw if raw is not None else ""
+    if not any(marker in data for marker in _EXTERNAL_MARKERS):
+        return
+    try:
+        enqueue_job(session_id, transcript_path, "APPROVED")
+    except Exception:
+        pass
+
+
+def main(stdin_json: dict | None = None) -> int:
+    if stdin_json is None:
+        stdin_json, _ = read_stdin_json()
+    root = resolve_arkaos_root()
+    ensure_root_on_path(root)
+
+    session_id = get_str(stdin_json, "session_id")
+    transcript_path = get_str(stdin_json, "transcript_path")
+    stop_hook_active = get_str(stdin_json, "stop_hook_active")
+    cwd = get_str(stdin_json, "cwd")
+    effort_level = get_str(stdin_json, "effort", "level") or os.environ.get(
+        "CLAUDE_EFFORT", ""
+    )
+    assistant_msg = get_str(stdin_json, "assistant_message")
+
+    _dna_fidelity(assistant_msg, session_id)
+
+    # Prevent infinite loops when Stop was triggered by its own decision.
+    if stop_hook_active == "true":
+        return 0
+
+    raw = _read_transcript(transcript_path)
+
+    _native_usage(transcript_path, session_id, raw)
+
+    # Only evaluate sessions where the classifier flagged creation intent.
+    wf_marker = Path("/tmp/arkaos-wf-required") / session_id if session_id else None
+    if wf_marker is None or not safe_session_id(session_id) or not wf_marker.is_file():
+        return 0
+
+    if (Path(root) / "core" / "workflow" / "flow_enforcer.py").is_file():
+        _flow_checks(session_id, transcript_path, cwd, effort_level, raw)
+
+    _auto_doc_enqueue(session_id, transcript_path, raw)
+
+    # Belt-and-braces marker removal (session id already allowlisted).
+    try:
+        wf_marker.unlink(missing_ok=True)
+    except OSError:
+        pass
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except SystemExit:
+        raise
+    except Exception:
+        raise SystemExit(0)  # Stop hook must never break the turn