arkaos 4.0.2 → 4.1.0

package/core/synapse/graph_context_layer.py

@@ -0,0 +1,212 @@
+"""Synapse layer L2.7 — Graphify grounding context injection.
+
+When the working project has a Graphify code knowledge graph
+(``graphify-out/graph.json``, generated locally by the ``graphify`` CLI
+via tree-sitter), this layer matches the user's prompt keywords against
+graph node labels and injects the top matches as grounded context:
+node label, confidence tag and the exact ``source_location``. The model
+then cites real code structure instead of inventing it.
+
+Honesty contract (PR-3 v4.1 — Graphify grounding):
+  - Only ``EXTRACTED`` nodes are injected as-is. ``INFERRED`` nodes are
+    included with an explicit ``(inferred)`` suffix. ``AMBIGUOUS`` nodes
+    are never injected.
+  - ``source_location`` is NEVER truncated — a clipped citation is worse
+    than no citation.
+  - No graph → the layer contributes nothing (inert, zero tokens).
+
+Feature flag: ``synapse.graphContext`` in ``~/.arkaos/config.json``
+(default ``true`` — mirrors how L2.5 reads ``synapse.l25KbContext``).
+``ARKA_BYPASS_L27=1`` env disables for debugging.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any, Optional
+
+from core.synapse.layers import Layer, LayerResult, PromptContext
+from core.synapse.pattern_library_layer import _extract_keywords
+
+# graph.json can grow to many MB on large repos. Loading it inside a
+# per-prompt hook with a <100ms Synapse budget would blow both latency
+# and memory, so beyond this cap we skip with an explicit metrics tag
+# instead of parsing. (`graphify . --update` keeps graphs incremental;
+# a >10MB graph is a signal the project should shard extraction, not
+# that the hook should pay for it.)
+_MAX_GRAPH_BYTES = 10 * 1024 * 1024
+
+_MAX_NODES = 5
+_MAX_PARENT_WALK = 3  # cwd + up to 3 parent dirs
+_MAX_TOKENS_EST = 400  # ~4 chars/token budget cap for the whole block
+
+
+def _graph_flag_on() -> bool:
+    if os.environ.get("ARKA_BYPASS_L27", "").strip() == "1":
+        return False
+    config_path = Path.home() / ".arkaos" / "config.json"
+    if not config_path.exists():
+        return True
+    try:
+        data = json.loads(config_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return True
+    synapse_cfg = data.get("synapse") or {}
+    return bool(synapse_cfg.get("graphContext", True))
+
+
+def _locate_graph(cwd: str) -> Optional[Path]:
+    """Find ``graphify-out/graph.json`` in cwd or up to 3 parent dirs."""
+    if not cwd:
+        return None
+    current = Path(cwd)
+    if not current.is_dir():
+        return None
+    for candidate in [current, *current.parents[:_MAX_PARENT_WALK]]:
+        graph = candidate / "graphify-out" / "graph.json"
+        if graph.is_file():
+            return graph
+    return None
+
+
+def _node_degrees(edges: list) -> dict[str, int]:
+    degrees: dict[str, int] = {}
+    for edge in edges:
+        if not isinstance(edge, dict):
+            continue
+        for key in ("source", "target"):
+            node_id = edge.get(key)
+            if isinstance(node_id, str) and node_id:
+                degrees[node_id] = degrees.get(node_id, 0) + 1
+    return degrees
+
+
+def _score_nodes(
+    nodes: list, keywords: list[str], degrees: dict[str, int]
+) -> list[tuple[int, int, dict]]:
+    """Rank nodes by keyword-match count, then degree. Drops non-matches."""
+    scored: list[tuple[int, int, dict]] = []
+    for node in nodes:
+        if not isinstance(node, dict):
+            continue
+        confidence = str(node.get("confidence", "")).upper()
+        if confidence not in ("EXTRACTED", "INFERRED"):
+            continue  # AMBIGUOUS (or untagged) nodes are never injected
+        label = str(node.get("label") or node.get("id") or "").strip()
+        if not label:
+            continue
+        haystack = label.lower()
+        matches = sum(1 for kw in keywords if kw in haystack)
+        if matches == 0:
+            continue
+        degree = degrees.get(str(node.get("id", "")), 0)
+        scored.append((matches, degree, node))
+    scored.sort(key=lambda row: (-row[0], -row[1]))
+    return scored
+
+
+def _format_node_line(node: dict) -> str:
+    confidence = str(node.get("confidence", "")).upper()
+    label = str(node.get("label") or node.get("id") or "").strip()
+    # source_location is the citation — NEVER truncate it.
+    location = str(node.get("source_location", "")).strip() or "(no source_location)"
+    line = f"- {label} [{confidence}] — {location}"
+    if confidence == "INFERRED":
+        line += " (inferred)"
+    return line
+
+
+def _format_graph_block(lines: list[str]) -> str:
+    header = (
+        f"[arka:graph-context:{len(lines)}] Graphify grounded nodes for this "
+        f"prompt (cite source_location; EXTRACTED = verified from code):"
+    )
+    return "\n".join([header, *lines])
+
+
+class GraphContextLayer(Layer):
+    """L2.7 — inject Graphify graph nodes matching the user prompt."""
+
+    def __init__(self, max_nodes: int = _MAX_NODES) -> None:
+        self._max_nodes = max_nodes
+
+    @property
+    def id(self) -> str:
+        return "L2.7"
+
+    @property
+    def name(self) -> str:
+        return "GraphContext"
+
+    @property
+    def cache_ttl(self) -> int:
+        return 30
+
+    @property
+    def priority(self) -> int:
+        return 26  # right after L2.5 KBContext (25), before ProjectLayer (30)
+
+    def compute(self, ctx: PromptContext) -> LayerResult:
+        start = time.time()
+        if not ctx.user_input or not _graph_flag_on():
+            return self._empty(start)
+        graph_path = _locate_graph(ctx.cwd)
+        if graph_path is None:
+            return self._empty(start)
+        try:
+            if graph_path.stat().st_size > _MAX_GRAPH_BYTES:
+                # See _MAX_GRAPH_BYTES: too big to parse inside the hook
+                # budget — surface a metrics note instead of silence.
+                return self._empty(start, tag="[graph-context:skipped size>10MB]")
+            data = json.loads(graph_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError, UnicodeDecodeError):
+            return self._empty(start)
+        return self._build_result(ctx, data, start)
+
+    def _build_result(self, ctx: PromptContext, data: Any, start: float) -> LayerResult:
+        if not isinstance(data, dict):
+            return self._empty(start)
+        keywords = _extract_keywords(ctx.user_input)
+        if not keywords:
+            return self._empty(start)
+        nodes = data.get("nodes") or []
+        edges = data.get("edges") or []
+        scored = _score_nodes(nodes, keywords, _node_degrees(edges))
+        lines = self._capped_lines(scored)
+        if not lines:
+            return self._empty(start)
+        content = _format_graph_block(lines)
+        return LayerResult(
+            layer_id=self.id,
+            tag=f"[graph-context:{len(lines)}]",
+            content=content,
+            tokens_est=max(1, len(content) // 4),
+            compute_ms=int((time.time() - start) * 1000),
+            cached=False,
+        )
+
+    def _capped_lines(self, scored: list[tuple[int, int, dict]]) -> list[str]:
+        """Top-N node lines, kept under the ~400-token block budget."""
+        lines: list[str] = []
+        budget_chars = _MAX_TOKENS_EST * 4
+        used = 0
+        for _, _, node in scored[: self._max_nodes]:
+            line = _format_node_line(node)
+            if lines and used + len(line) > budget_chars:
+                break
+            lines.append(line)
+            used += len(line)
+        return lines
+
+    def _empty(self, start: float, tag: str = "") -> LayerResult:
+        return LayerResult(
+            layer_id=self.id,
+            tag=tag,
+            content="",
+            tokens_est=0,
+            compute_ms=int((time.time() - start) * 1000),
+            cached=False,
+        )

package/core/synapse/kb_cache.py

@@ -248,7 +248,10 @@ class KBSessionCache:
                     "text": r.get("text", "")[:300],
                     "source": r.get("source", ""),
                     "heading": r.get("heading", ""),
+                    # None = keyword-degraded hit (no similarity exists);
+                    # preserved as-is so cached entries stay honest.
                     "score": r.get("score", 0.0),
+                    "retrieval": r.get("retrieval", ""),
                 }
                 for r in results
             ],

package/core/synapse/layers.py

@@ -661,7 +661,12 @@ class KnowledgeRetrievalLayer(Layer):
 
         content = " | ".join(parts)
         chunk_count = len(snippets) + (len(overlapping) if overlapping else 0)
+        # RAG honesty (PR-3 v4.1): keyword-degraded results must never be
+        # presented as semantic similarity — label the tag explicitly.
+        degraded = any(r.get("retrieval") == "keyword-degraded" for r in results)
         tag = f"[knowledge:{chunk_count} chunks]"
+        if degraded:
+            tag = f"[knowledge:{chunk_count} chunks degraded=keyword]"
         ms = int((time.time() - start) * 1000)
         tokens_est = len(content.split())
 
@@ -879,19 +884,27 @@ def _extract_wikilinks(raw: str, limit: int = 3) -> list[str]:
     return seen
 
 
-def _format_kb_block(notes: list[dict]) -> str:
+def _format_kb_block(notes: list[dict], degraded: bool = False) -> str:
     lines: list[str] = [
         f"[arka:kb-context] O teu cérebro (Obsidian) tem {len(notes)} "
         f"nota{'s' if len(notes) != 1 else ''} relevante{'s' if len(notes) != 1 else ''} "
         f"para este pedido:",
         "",
     ]
+    if degraded:
+        lines.insert(1, "")
+        lines.insert(
+            1,
+            "Atenção: correspondência por palavras-chave (pesquisa semântica "
+            "indisponível) — NÃO é similaridade semântica.",
+        )
     for note in notes:
         title = note.get("title", "")
         path = note.get("path", "")
         excerpt = note.get("excerpt", "")
         relates = note.get("relates", []) or []
-        lines.append(f"- [[{title}]] (path: `{path}`)")
+        suffix = " (inferred — not authoritative)" if note.get("inferred") else ""
+        lines.append(f"- [[{title}]]{suffix} (path: `{path}`)")
         if excerpt:
             lines.append(f"  Excerto: {excerpt}")
         if relates:
@@ -950,13 +963,53 @@ def _load_fallback_notes(vault_path: Optional[Path]) -> list[dict]:
     return notes
 
 
-def _build_note_entry(raw: str, title: str, path: str, score: float) -> dict:
+_GROUNDING_INFERRED_RE = re.compile(r"^grounding:\s*inferred\s*$", re.MULTILINE)
+
+
+def _frontmatter_marks_inferred(raw: str) -> bool:
+    """Cheap check: does the YAML frontmatter carry `grounding: inferred`?
+
+    Parses ONLY the frontmatter block (the note content is already in
+    hand) — no YAML library, no full-document scan. Dreaming-written
+    notes carry this marker (see core/cognition/dreaming.py, PR-3 v4.1).
+    """
+    match = _FRONTMATTER_RE.match(raw or "")
+    if not match:
+        return False
+    return bool(_GROUNDING_INFERRED_RE.search(match.group(0)))
+
+
+def _hit_is_inferred(hit: dict) -> bool:
+    """Inferred check for a vector-store hit.
+
+    Chunk text has frontmatter stripped by the chunker, so check the hit
+    metadata first, then read just the head of the source file (cheap:
+    frontmatter lives in the first bytes).
+    """
+    metadata = hit.get("metadata") or {}
+    if isinstance(metadata, dict) and metadata.get("grounding") == "inferred":
+        return True
+    source = hit.get("source", "") or ""
+    if not source:
+        return False
+    try:
+        with open(source, "r", encoding="utf-8", errors="ignore") as fh:
+            head = fh.read(2048)
+    except OSError:
+        return False
+    return _frontmatter_marks_inferred(head)
+
+
+def _build_note_entry(
+    raw: str, title: str, path: str, score: float, inferred: bool = False
+) -> dict:
     return {
         "title": title,
         "path": path,
         "excerpt": _extract_excerpt(raw),
         "relates": _extract_wikilinks(raw),
         "score": float(score),
+        "inferred": inferred,
     }
 
 
@@ -965,7 +1018,24 @@ def _note_from_vector_hit(hit: dict) -> dict:
     raw = hit.get("text", "") or ""
     title = hit.get("heading") or Path(source).stem or "note"
     score_val = hit.get("score", 0.0) or 0.0
-    return _build_note_entry(raw, str(title), str(source), float(score_val))
+    return _build_note_entry(
+        raw, str(title), str(source), float(score_val),
+        inferred=_hit_is_inferred(hit),
+    )
+
+
+def _apply_grounding_policy(notes: list[dict], max_notes: int) -> list[dict]:
+    """Quarantine inferred notes (Dreaming output) from grounded context.
+
+    Policy (PR-3 v4.1): inferred notes are EXCLUDED by default; they are
+    only included — explicitly suffixed `(inferred — not authoritative)`
+    by the formatter — when fewer than 2 grounded notes matched.
+    """
+    grounded = [n for n in notes if not n.get("inferred")]
+    if len(grounded) >= 2:
+        return grounded[:max_notes]
+    inferred = [n for n in notes if n.get("inferred")]
+    return (grounded + inferred)[:max_notes]
 
 
 class KBContextLayer(Layer):
@@ -1034,54 +1104,68 @@ class KBContextLayer(Layer):
         except Exception:
             pass
 
-    def _retrieve(self, prompt: str) -> list[dict]:
+    def _retrieve(self, prompt: str) -> tuple[list[dict], bool]:
+        """Return (notes, degraded). Degraded = keyword-only retrieval.
+
+        Degraded hits carry no similarity score, so the min_similarity
+        threshold does not apply to them — they are included but labeled
+        (never presented as semantic matches).
+        """
         hits = _vector_search(self._store, prompt, top_k=self._max_notes * 2)
+        degraded = any(h.get("retrieval") == "keyword-degraded" for h in hits)
         notes: list[dict] = []
         for h in hits:
-            score = float(h.get("score", 0.0) or 0.0)
-            if score < self._min_similarity:
-                continue
+            if not degraded:
+                score = float(h.get("score", 0.0) or 0.0)
+                if score < self._min_similarity:
+                    continue
             notes.append(_note_from_vector_hit(h))
-            if len(notes) >= self._max_notes:
-                break
+        notes = _apply_grounding_policy(notes, self._max_notes)
         if notes:
-            return notes
+            return notes, degraded
         candidates = _load_fallback_notes(self._vault_path)
         if not candidates:
-            return []
+            return [], False
         picked = _jaccard_fallback(
-            prompt, candidates, self._max_notes, self._min_similarity
+            prompt, candidates, self._max_notes * 2, self._min_similarity
         )
-        return [
-            _build_note_entry(n["raw"], n["title"], n["path"], 0.0)
+        fallback_notes = [
+            _build_note_entry(
+                n["raw"], n["title"], n["path"], 0.0,
+                inferred=_frontmatter_marks_inferred(n["raw"]),
+            )
             for n in picked
         ]
+        return _apply_grounding_policy(fallback_notes, self._max_notes), False
 
     def build(self, prompt: str) -> Optional[str]:
         """Public entrypoint — returns the formatted block or None."""
         if not prompt or not _l25_feature_flag_on():
             return None
-        notes = self._retrieve(prompt[:2000])
+        notes, degraded = self._retrieve(prompt[:2000])
         if not notes:
             return None
-        return _format_kb_block(notes[: self._max_notes])
+        return _format_kb_block(notes[: self._max_notes], degraded=degraded)
 
     def compute(self, ctx: PromptContext) -> LayerResult:
         start = time.time()
         if not ctx.user_input or not _l25_feature_flag_on():
             return self._empty(start)
         try:
-            notes = self._retrieve(ctx.user_input[:2000])
+            notes, degraded = self._retrieve(ctx.user_input[:2000])
         except Exception:
             return self._empty(start)
         self._record(ctx, len(notes))
         if not notes:
             return self._empty(start)
-        block = _format_kb_block(notes[: self._max_notes])
+        block = _format_kb_block(notes[: self._max_notes], degraded=degraded)
         ms = int((time.time() - start) * 1000)
+        tag = f"[kb-context:{len(notes)}]"
+        if degraded:
+            tag = f"[kb-context:{len(notes)} degraded=keyword]"
         return LayerResult(
             layer_id=self.id,
-            tag=f"[kb-context:{len(notes)}]",
+            tag=tag,
             content=block,
             tokens_est=len(block.split()),
             compute_ms=ms,

package/core/workflow/enforcer.py

@@ -1,15 +1,16 @@
 """Workflow enforcement engine.
 
-Central enforcement engine that evaluates all 14 NON-NEGOTIABLE
-Constitution rules against tool operations. Returns violations
-with severity classification (BLOCK, ESCALATE, WARN).
+Evaluates the disk-verifiable Constitution rules in
+``core.workflow.rules_registry`` against tool operations. Only rules
+whose state can be read from disk live in that registry; behavioral
+rules are enforced by hooks/telemetry (see the registry docstring).
+Returns violations with severity classification (BLOCK, ESCALATE, WARN).
 
 BLOCK severity = operation is halted until resolved
 ESCALATE severity = operation continues but alerts Tier 0
 WARN severity = non-blocking advisory
 """
 
-import re
 from typing import Any
 
 from core.workflow.rules_registry import RULES_REGISTRY, RuleDefinition
@@ -94,18 +95,6 @@ def _build_context(
     }
 
 
-def _is_code_file(file_path: str) -> bool:
-    """Check if file is a code file subject to enforcement."""
-    code_extensions = {".py", ".js", ".ts", ".vue", ".php", ".jsx", ".tsx"}
-    return any(file_path.endswith(ext) for ext in code_extensions)
-
-
-def _is_text_file(file_path: str) -> bool:
-    """Check if file is a text file subject to human-writing checks."""
-    text_extensions = {".md", ".txt", ".json", ".yml", ".yaml"}
-    return any(file_path.endswith(ext) for ext in text_extensions)
-
-
 def enforce(tool_name: str, context: dict[str, Any]) -> EnforcementResult:
     """Evaluate all applicable rules against the given context.
 

package/core/workflow/flow_enforcer.py

@@ -1,4 +1,4 @@
-"""Mandatory 13-phase flow enforcement for write-mutation tools.
+"""Evidence-flow enforcement for write-mutation tools.
 
 Invoked by the Claude Code `PreToolUse` hook. Decides whether a `Write`,
 `Edit`, or `MultiEdit` tool call may proceed, based on markers observed
@@ -8,7 +8,9 @@ Design contract:
 - Stateless transcript parse (no /tmp state for decisions).
 - Side effects limited to reading the transcript path supplied by the hook.
 - Signals permission when the assistant has emitted one of the flow markers:
-  `[arka:routing]`, `[arka:trivial]`, or `[arka:phase:`.
+  `[arka:routing]`, `[arka:trivial]`, or `[arka:gate:` (v4.1 evidence flow).
+  The legacy 13-phase `[arka:phase:` marker remains accepted during the
+  v4.1 deprecation window.
 - Respects `ARKA_BYPASS_FLOW=1` env var (installer/`/arka update` internal).
 - Honors feature flag `hooks.hardEnforcement` in `~/.arkaos/config.json`.
 - Gated tool list is closed: anything outside it is always allowed.
@@ -186,7 +188,9 @@ def bash_is_effect(command: str) -> bool:
 
 ROUTING_RE = re.compile(r"\[arka:routing\]\s*[\w-]+\s*->\s*\w+", re.IGNORECASE)
 TRIVIAL_RE = re.compile(r"\[arka:trivial\]\s*\S+", re.IGNORECASE)
-PHASE_RE = re.compile(r"\[arka:phase:\d+\]", re.IGNORECASE)
+GATE_RE = re.compile(r"\[arka:gate:[1-4]\]", re.IGNORECASE)
+# Legacy 13-phase marker — accepted during the v4.1 deprecation window.
+PHASE_RE = re.compile(r"\[arka:phase:\d+(\.\d+)?\]", re.IGNORECASE)
 
 # Re-export for backward compatibility with any external importers that
 # relied on the module-level symbols before the core.shared extraction.
@@ -298,13 +302,21 @@ def _extract_text(content: object) -> str:
     return ""
 
 
-def _load_last_assistant_messages(transcript_path: str, n: int) -> list[str]:
-    """Read the last `n` assistant messages from a JSONL transcript."""
-    path = Path(transcript_path) if transcript_path else None
-    if path is None or not path.exists():
-        return []
+def _load_last_assistant_messages(
+    transcript_path: str, n: int, raw_text: str | None = None
+) -> list[str]:
+    """Read the last `n` assistant messages from a JSONL transcript.
+
+    ``raw_text`` lets callers that already read the transcript (PR-6 hook
+    consolidation — parse once per hook invocation) skip the disk read.
+    """
+    if raw_text is None:
+        path = Path(transcript_path) if transcript_path else None
+        if path is None or not path.exists():
+            return []
+        raw_text = path.read_text(encoding="utf-8", errors="replace")
     messages: list[str] = []
-    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+    for line in raw_text.splitlines():
         if not line.strip():
             continue
         try:
@@ -329,14 +341,19 @@ def _scan_markers(messages: list[str]) -> tuple[str | None, str | None]:
     phase_observed: str | None = None
     for text in messages:
         if phase_observed is None:
+            gate_match = GATE_RE.search(text)
             phase_match = PHASE_RE.search(text)
-            if phase_match:
+            if gate_match:
+                phase_observed = gate_match.group(0)
+            elif phase_match:
                 phase_observed = phase_match.group(0)
         if marker_found is None:
             if ROUTING_RE.search(text):
                 marker_found = "routing"
             elif TRIVIAL_RE.search(text):
                 marker_found = "trivial"
+            elif GATE_RE.search(text):
+                marker_found = "gate"
             elif PHASE_RE.search(text):
                 marker_found = "phase"
     return marker_found, phase_observed
@@ -348,6 +365,7 @@ def evaluate(
     session_id: str = "",
     cwd: str = "",
     tool_input: dict | None = None,
+    messages: list[str] | None = None,
 ) -> Decision:
     """Decide whether a tool call may proceed.
 
@@ -357,6 +375,10 @@ def evaluate(
     PR11 v2.33.0 expanded the gated set beyond Write/Edit/MultiEdit to
     cover all EFFECT tools (NotebookEdit, Task, Skill) and to classify
     Bash commands per-command via ``bash_is_effect``.
+
+    ``messages`` (PR-6 hook consolidation): pre-parsed assistant messages.
+    When None (default) the transcript is read from ``transcript_path`` —
+    backward compatible with all existing callers.
     """
     is_gated = tool_name in EFFECT_TOOLS_ALWAYS
     if not is_gated and tool_name == "Bash":
@@ -386,7 +408,10 @@ def evaluate(
             marker_found=cached_type or None,
         )
 
-    messages = _load_last_assistant_messages(transcript_path, ASSISTANT_WINDOW)
+    if messages is None:
+        messages = _load_last_assistant_messages(
+            transcript_path, ASSISTANT_WINDOW
+        )
     marker_found, phase_observed = _scan_markers(messages)
 
     if marker_found is None: