@heytherevibin/skillforge 0.2.1 → 0.8.0

package/python/app/mcp_contract.py

@@ -0,0 +1,121 @@
+"""Shared MCP response shape for ``route_skills`` (and CLI parity).
+
+Versioned ``_meta`` with ``sources[]`` and ``budget``. Schema **1.1** adds
+per-chunk ``sources`` when ``context_items`` (RAG chunks) are returned from Phase 1.
+Schema **1.2** adds ``kind: file`` sources and project chunk char counts in ``budget``.
+Schema **1.4** adds optional ``context_redaction`` (hit counts when scrubbing is on).
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Mapping, Protocol
+
+from app.redaction import redaction_enabled, redact_display_path
+
+
+class _SkillBody(Protocol):
+    name: str
+    body: str
+
+
+MCP_RESPONSE_SCHEMA_VERSION = "1.4"
+
+
+def build_route_skills_meta(
+    *,
+    result: dict[str, Any],
+    picked_names: list[str],
+    user_id: str,
+    db_path: Path | str,
+    skills_map: Mapping[str, _SkillBody],
+    response_text: str,
+    error: str | None = None,
+    context_items: list[dict[str, Any]] | None = None,
+    fusion: dict[str, Any] | None = None,
+    context_redaction: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build ``_meta`` for a route_skills-style response (success or structured error)."""
+    sources: list[dict[str, Any]] = []
+    chars_skill = 0
+    chars_project = 0
+
+    if context_items:
+        for c in context_items:
+            tlen = len(c.get("text") or "")
+            ref_path = c.get("path")
+            if ref_path:
+                row = {
+                    "kind": "file",
+                    "ref": ref_path,
+                    "line_start": c.get("line_start"),
+                    "line_end": c.get("line_end"),
+                    "score": round(float(c.get("score", 0.0)), 6),
+                }
+                if c.get("mmr_rank") is not None:
+                    row["mmr_rank"] = int(c["mmr_rank"])
+                sources.append(row)
+                chars_project += tlen
+            else:
+                row = {
+                    "kind": "skill",
+                    "ref": c["skill"],
+                    "line_start": c.get("line_start"),
+                    "line_end": c.get("line_end"),
+                    "score": round(float(c.get("score", 0.0)), 6),
+                }
+                if c.get("mmr_rank") is not None:
+                    row["mmr_rank"] = int(c["mmr_rank"])
+                sources.append(row)
+                chars_skill += tlen
+    else:
+        for n in picked_names:
+            s = skills_map.get(n)
+            if s is not None:
+                sources.append({
+                    "kind": "skill",
+                    "ref": n,
+                    "line_start": None,
+                    "line_end": None,
+                    "score": None,
+                })
+                chars_skill += len(s.body)
+
+    chars_body = chars_skill + chars_project
+
+    candidates_raw = result.get("candidates") or []
+    candidates_preview: list[dict[str, Any]] = []
+    for item in candidates_raw[:15]:
+        if isinstance(item, tuple) and len(item) == 2:
+            sk, sc = item
+            name = getattr(sk, "name", None)
+            if name is not None:
+                candidates_preview.append({"name": name, "score": round(float(sc), 6)})
+
+    meta: dict[str, Any] = {
+        "schema_version": MCP_RESPONSE_SCHEMA_VERSION,
+        "sources": sources,
+        "budget": {
+            "chars_skill_bodies": chars_skill,
+            "chars_project_chunks": chars_project,
+            "chars_context_items_total": chars_body,
+            "chars_response_total": len(response_text),
+            "est_tokens_approx": max(1, len(response_text) // 4),
+        },
+        "picked": list(picked_names),
+        "reasoning": result.get("reasoning"),
+        "session_id": result.get("session_id"),
+        "user_id": user_id,
+        "rerouted": result.get("rerouted"),
+        "change_pct": round(float(result.get("change", 0)) * 100, 1),
+        "route_ms": round(float(result.get("route_ms", 0)), 1),
+        "orchestrator_db": redact_display_path(db_path) if redaction_enabled() else str(db_path),
+        "candidates_preview": candidates_preview,
+        "context_items_count": len(context_items or []),
+    }
+    if fusion is not None and fusion.get("enabled"):
+        meta["fusion"] = fusion
+    if context_redaction is not None:
+        meta["context_redaction"] = context_redaction
+    if error:
+        meta["error"] = error
+    return meta

package/python/app/mcp_server.py

@@ -2,11 +2,11 @@
 MCP server for skillforge.
 
 Exposes skill routing as MCP tools so MCP-aware clients (Claude Desktop,
-Claude Code, Cursor, etc.) can use the orchestrator without running the
-HTTP server.
+Claude Code, Cursor, etc.) can use the orchestrator locally.
 
 Tools exposed:
   route_skills / skillforge_bootstrap — routing (+ optional project materialize).
+  search_skills / explain_route / get_skill — retrieval, debugging, deterministic fetch.
   materialize_project — .cursor/rules, docs/SKILLFORGE-PRD.md, CLAUDE.md block.
   list_skills, skill_feedback, skill_referenced, disable_skill.
 
@@ -25,7 +25,10 @@ from pathlib import Path
 
 from app.db_paths import resolve_orchestrator_db
 from app.main import (
+    TOP_K_CANDIDATES,
+    MAX_ACTIVE_SKILLS,
     build_router_and_skills,
+    format_context_items_markdown,
     init_db,
     load_all_skills,
     log_event,
@@ -36,6 +39,10 @@ from app.main import (
     Router,
 )
 from app.materialize import materialize_project_files
+from app.mcp_contract import MCP_RESPONSE_SCHEMA_VERSION, build_route_skills_meta
+from app.redaction import redaction_enabled, redact_display_path
+from app.route_policies import load_route_policies_config, merge_policy_includes
+from app.routing_signals import build_route_query_text
 
 
 def _env_truthy(name: str, default: str = "1") -> bool:
@@ -82,7 +89,7 @@ class MCPServer:
         self._db_cache: dict[str, sqlite3.Connection] = {}
 
     def _mcp_user_id(self, args: dict) -> str:
-        """Per-tool user namespace for weights/sessions/events (aligned with HTTP bearer user id)."""
+        """Per-tool user namespace for weights/sessions/events."""
         raw = (
             args.get("user_id")
             or os.getenv("SKILLFORGE_MCP_USER_ID", "")
@@ -97,6 +104,15 @@ class MCPServer:
         env = os.getenv("SKILLFORGE_PROJECT_ROOT", "").strip()
         return env or None
 
+    @staticmethod
+    def _include_project_rag_from_args(args: dict) -> bool:
+        v = args.get("include_project_rag")
+        if v is True:
+            return True
+        if isinstance(v, str) and v.strip().lower() in ("1", "true", "yes"):
+            return True
+        return False
+
     def _get_con(self, args: dict):
         path = resolve_orchestrator_db(self._project_root_from_args(args))
         key = str(path)
@@ -173,7 +189,7 @@ class MCPServer:
         return {
             "protocolVersion": "2024-11-05",
             "capabilities": caps,
-            "serverInfo": {"name": "skillforge", "version": "0.2.1"},
+            "serverInfo": {"name": "skillforge", "version": "0.7.1"},
         }
 
     def handle_tools_list(self, params):
@@ -183,11 +199,14 @@ class MCPServer:
                     "name": "route_skills",
                     "description": (
                         "Route the user's prompt to the most relevant skills from the catalog "
-                        "and return their full SKILL.md bodies. The client should inject the "
-                        "returned content into the LLM's context. Returns up to 7 skills. "
-                        "Pass project_root (workspace path) for per-repo SQLite in .skillforge/ "
-                        "and learning; else use env SKILLFORGE_PROJECT_ROOT or global data dir. "
-                        "Optional session_id for reroute stats; optional user_id for multi-user."
+                        "and return SKILL.md context (full body or RAG chunks per CONTEXT_MODE). "
+                        "Returns up to 7 skills. "
+                        "Pass project_root for per-repo SQLite in .skillforge/ and learning. "
+                        "Optional include_project_rag merges top chunks from `skillforge index` into context. "
+                        "On success, _meta includes schema_version ("
+                        f"{MCP_RESPONSE_SCHEMA_VERSION}), sources[] (kind skill or file), "
+                        "budget (chars_skill_bodies, chars_project_chunks), fusion (MMR when combined index+RAG), "
+                        "candidates_preview, context_items_count."
                     ),
                     "inputSchema": {
                         "type": "object",
@@ -197,6 +216,14 @@ class MCPServer:
                                 "type": "string",
                                 "description": "Repo/workspace root — stores orchestrator state in .skillforge/",
                             },
+                            "include_project_rag": {
+                                "type": "boolean",
+                                "description": (
+                                    "If true, append top chunks from the project index in the same DB "
+                                    "(see skillforge index). Requires project_root."
+                                ),
+                                "default": False,
+                            },
                             "conversation": {
                                 "type": "array",
                                 "description": "Optional recent messages for context",
@@ -208,12 +235,80 @@ class MCPServer:
                             },
                             "user_id": {
                                 "type": "string",
-                                "description": "Logical user id for weights/sessions/events (same as HTTP user id string)",
+                                "description": "Logical user id for weights/sessions/events",
                             },
                         },
                         "required": ["prompt"],
                     },
                 },
+                {
+                    "name": "search_skills",
+                    "description": (
+                        "Embedding-only retrieval: top skills for a query with similarity scores "
+                        "and descriptions (no Haiku, no full route). Use to explore the catalog."
+                    ),
+                    "inputSchema": {
+                        "type": "object",
+                        "properties": {
+                            "query": {"type": "string", "description": "Search query or task text"},
+                            "limit": {
+                                "type": "integer",
+                                "description": f"Max skills to return (default {TOP_K_CANDIDATES})",
+                            },
+                            "project_root": {"type": "string"},
+                            "user_id": {"type": "string"},
+                        },
+                        "required": ["query"],
+                    },
+                },
+                {
+                    "name": "explain_route",
+                    "description": (
+                        "Debug routing: embedding facets for the shortlist (same query text as route_skills when "
+                        "`conversation` is passed — conversation-aware when SKILLFORGE_ROUTER_CONV_MAX_TURNS > 0), "
+                        "optional Haiku rerank, Haiku/embedding-only pick with reasoning, and policy merge audit. "
+                        "Does not write sessions or increment uses."
+                    ),
+                    "inputSchema": {
+                        "type": "object",
+                        "properties": {
+                            "prompt": {"type": "string"},
+                            "conversation": {"type": "array", "items": {"type": "object"}},
+                            "limit": {
+                                "type": "integer",
+                                "description": "Max shortlist rows in facets (default TOP_K)",
+                            },
+                            "project_root": {"type": "string"},
+                            "user_id": {"type": "string"},
+                        },
+                        "required": ["prompt"],
+                    },
+                },
+                {
+                    "name": "get_skill",
+                    "description": (
+                        "Load one skill by name: full SKILL.md body or a short summary. "
+                        "Use for deterministic workflows when you already know the skill name."
+                    ),
+                    "inputSchema": {
+                        "type": "object",
+                        "properties": {
+                            "skill_name": {"type": "string"},
+                            "format": {
+                                "type": "string",
+                                "enum": ["full", "summary"],
+                                "description": "summary = description + first ~8k chars of body",
+                                "default": "full",
+                            },
+                            "max_chars": {
+                                "type": "integer",
+                                "description": "If > 0, truncate body to this many characters",
+                                "default": 0,
+                            },
+                        },
+                        "required": ["skill_name"],
+                    },
+                },
                 {
                     "name": "list_skills",
                     "description": (
@@ -317,6 +412,11 @@ class MCPServer:
                             "session_id": {"type": "string"},
                             "user_id": {"type": "string"},
                             "merge": {"type": "boolean", "default": True},
+                            "include_project_rag": {
+                                "type": "boolean",
+                                "description": "Same as route_skills: merge indexed project file chunks into context.",
+                                "default": False,
+                            },
                         },
                         "required": ["prompt", "project_root"],
                     },
@@ -330,6 +430,12 @@ class MCPServer:
 
         if name == "route_skills":
             return await self._tool_route_skills(args)
+        if name == "search_skills":
+            return self._tool_search_skills(args)
+        if name == "explain_route":
+            return await self._tool_explain_route(args)
+        if name == "get_skill":
+            return self._tool_get_skill(args)
         if name == "list_skills":
             return self._tool_list_skills(args)
         if name == "skill_feedback":
@@ -349,8 +455,25 @@ class MCPServer:
         conversation = args.get("conversation", [])
         session_id = args.get("session_id") or None
         user_id = self._mcp_user_id(args)
+        pr = self._project_root_from_args(args)
+        db_path = resolve_orchestrator_db(pr)
+
         if not prompt.strip():
-            return {"content": [{"type": "text", "text": "No prompt provided."}]}
+            err_text = "No prompt provided."
+            return {
+                "content": [{"type": "text", "text": err_text}],
+                "isError": True,
+                "_meta": build_route_skills_meta(
+                    result={"candidates": []},
+                    picked_names=[],
+                    user_id=user_id,
+                    db_path=db_path,
+                    skills_map=self.skills or {},
+                    response_text=err_text,
+                    error="empty_prompt",
+                ),
+            }
+
         con = self._get_con(args)
         result = await run_route_turn(
             con,
@@ -359,11 +482,12 @@ class MCPServer:
             conversation,
             user_id=user_id,
             session_id=session_id,
+            project_root=pr,
+            include_project_rag=self._include_project_rag_from_args(args),
         )
         picked_names = result["picked_names"]
         reasoning = result["reasoning"]
-        pr = self._project_root_from_args(args)
-        db_path = resolve_orchestrator_db(pr)
+        context_items = result.get("context_items") or []
         if pr:
             try:
                 d = Path(pr).expanduser().resolve() / ".skillforge"
@@ -375,35 +499,184 @@ class MCPServer:
                     "reasoning": reasoning,
                     "route_ms": round(result["route_ms"], 1),
                     "user_id": user_id,
+                    "schema_version": MCP_RESPONSE_SCHEMA_VERSION,
+                    "context_mode": self.router.context_mode,
+                    "context_items_count": len(context_items),
+                    "project_rag_items_count": (result.get("event") or {}).get("project_rag_items_count", 0),
                 }
                 (d / "last_route.json").write_text(json.dumps(snap, indent=2), encoding="utf-8")
             except OSError:
                 pass
 
-        # Build response: a header explaining what was loaded, then the skill bodies
+        db_disp = redact_display_path(db_path) if redaction_enabled() else str(db_path)
         blocks = [
-            f"# Skillforge — routed {len(picked_names)} skill(s)",
-            f"_DB:_ `{db_path}`",
+            f"# Skillforge — routed {len(picked_names)} skill(s); context=`{self.router.context_mode}`",
+            f"_DB:_ `{db_disp}`",
             f"_Reasoning: {reasoning}_" if reasoning else "",
             "",
         ]
-        for n in picked_names:
-            s = self.skills.get(n)
-            if s:
-                blocks.append(f"---\n## Skill: {s.name}\n\n{s.body}\n")
-        if not picked_names:
+        if context_items:
+            blocks.append(format_context_items_markdown(context_items))
+        elif not picked_names:
             blocks.append("_No skills matched this prompt closely enough to load._")
+        response_text = "\n".join(b for b in blocks if b is not None)
+        meta = build_route_skills_meta(
+            result=result,
+            picked_names=picked_names,
+            user_id=user_id,
+            db_path=db_path,
+            skills_map=self.skills,
+            response_text=response_text,
+            context_items=context_items,
+            fusion=(result.get("event") or {}).get("context_fusion"),
+            context_redaction=(result.get("event") or {}).get("context_redaction"),
+        )
         return {
-            "content": [{"type": "text", "text": "\n".join(b for b in blocks if b is not None)}],
+            "content": [{"type": "text", "text": response_text}],
+            "_meta": meta,
+        }
+
+    def _tool_search_skills(self, args):
+        query = (args.get("query") or "").strip()
+        user_id = self._mcp_user_id(args)
+        pr = self._project_root_from_args(args)
+        db_path = resolve_orchestrator_db(pr)
+        if not query:
+            return {
+                "content": [{"type": "text", "text": "query is required."}],
+                "isError": True,
+            }
+        try:
+            limit = int(args.get("limit") or TOP_K_CANDIDATES)
+        except (TypeError, ValueError):
+            limit = TOP_K_CANDIDATES
+        limit = max(1, min(limit, 50))
+        con = self._get_con(args)
+        facets = self.router.shortlist_with_facets(query, con, k=limit, user_id=user_id)
+        lines = ["# search_skills — embedding shortlist", ""]
+        for f in facets:
+            lines.append(
+                f"- **{f['name']}** (cos {f['cosine_similarity']}, score {f['routing_score']}): "
+                f"{(f.get('description_preview') or '')[:220]}"
+            )
+        text = "\n".join(lines)
+        return {
+            "content": [{"type": "text", "text": text}],
             "_meta": {
-                "picked": picked_names,
-                "reasoning": reasoning,
-                "session_id": result["session_id"],
-                "user_id": user_id,
-                "rerouted": result["rerouted"],
-                "change_pct": round(result["change"] * 100, 1),
-                "route_ms": round(result["route_ms"], 1),
-                "orchestrator_db": str(db_path),
+                "schema_version": MCP_RESPONSE_SCHEMA_VERSION,
+                "tool": "search_skills",
+                "orchestrator_db": redact_display_path(db_path) if redaction_enabled() else str(db_path),
+                "results": facets,
+                "count": len(facets),
+            },
+        }
+
+    async def _tool_explain_route(self, args):
+        prompt = (args.get("prompt") or "").strip()
+        conversation = args.get("conversation") or []
+        user_id = self._mcp_user_id(args)
+        pr = self._project_root_from_args(args)
+        db_path = resolve_orchestrator_db(pr)
+        if not prompt:
+            return {
+                "content": [{"type": "text", "text": "prompt is required."}],
+                "isError": True,
+            }
+        try:
+            limit = int(args.get("limit") or TOP_K_CANDIDATES)
+        except (TypeError, ValueError):
+            limit = TOP_K_CANDIDATES
+        limit = max(1, min(limit, 50))
+        con = self._get_con(args)
+        route_query = build_route_query_text(prompt, conversation)
+        facets = self.router.shortlist_with_facets(route_query, con, k=limit, user_id=user_id)
+        candidates = self.router.shortlist(route_query, con, user_id=user_id)
+        candidates = await self.router.rerank_candidates_haiku(route_query, conversation, candidates)
+        picked, reasoning = await self.router.pick_final(
+            prompt, conversation, candidates, route_query=route_query
+        )
+        policies_cfg = load_route_policies_config(pr)
+        merged, policy_audit = merge_policy_includes(
+            prompt,
+            list(picked),
+            policies_cfg,
+            self.router._by_name,
+            con,
+            user_id,
+            max_active=MAX_ACTIVE_SKILLS,
+        )
+        router_mode = "full" if self.router.anthropic else "embedding-only"
+        explain = {
+            "schema_version": MCP_RESPONSE_SCHEMA_VERSION,
+            "tool": "explain_route",
+            "orchestrator_db": redact_display_path(db_path) if redaction_enabled() else str(db_path),
+            "router_mode": router_mode,
+            "embedding_shortlist": facets,
+            "picked_before_policy": list(picked),
+            "picked_after_policy": merged,
+            "router_reasoning": reasoning,
+            "policy": {
+                "rules_loaded": len(policies_cfg.get("rules") or [])
+                if isinstance(policies_cfg.get("rules"), list)
+                else 0,
+                "audit": policy_audit,
+            },
+        }
+        lines = [
+            "# explain_route — routing diagnostics (no DB writes)",
+            "",
+            f"**Router:** {router_mode}",
+            f"**Picked (router):** {', '.join(picked) if picked else '_(none)_'}",
+            f"**After policies:** {', '.join(merged) if merged else '_(none)_'}",
+            f"**Reasoning:** {reasoning}" if reasoning else "**Reasoning:** _(n/a)_",
+            "",
+            "## Shortlist (embedding)",
+        ]
+        for f in facets[:15]:
+            lines.append(
+                f"- `{f['name']}` cos={f['cosine_similarity']} weight={f['learned_weight']} "
+                f"score={f['routing_score']}"
+            )
+        if policy_audit:
+            lines.extend(["", "## Policy audit"])
+            for row in policy_audit[:30]:
+                lines.append(f"- {row}")
+        body = "\n".join(lines)
+        return {"content": [{"type": "text", "text": body}], "_meta": explain}
+
+    def _tool_get_skill(self, args):
+        name = (args.get("skill_name") or "").strip()
+        fmt = (args.get("format") or "full").strip().lower()
+        if fmt not in ("full", "summary"):
+            fmt = "full"
+        max_chars = args.get("max_chars")
+        try:
+            mc = int(max_chars) if max_chars is not None else 0
+        except (TypeError, ValueError):
+            mc = 0
+        if not name or name not in self.skills:
+            return {
+                "content": [{"type": "text", "text": f"Unknown skill: {name or '(empty)'}"}],
+                "isError": True,
+            }
+        s = self.skills[name]
+        if fmt == "summary":
+            body = f"{s.description}\n\n---\n\n{(s.body or '')[:8000]}"
+        else:
+            body = s.body or ""
+        if mc > 0:
+            body = body[:mc]
+        header = f"# get_skill: `{name}`\n**Source:** {s.source} · **format:** {fmt}\n\n"
+        text = header + body
+        return {
+            "content": [{"type": "text", "text": text}],
+            "_meta": {
+                "schema_version": MCP_RESPONSE_SCHEMA_VERSION,
+                "tool": "get_skill",
+                "skill_name": name,
+                "source": s.source,
+                "format": fmt,
+                "chars": len(body),
             },
         }
 
@@ -514,6 +787,7 @@ class MCPServer:
                 "conversation": conversation,
                 "session_id": session_id,
                 "user_id": user_id,
+                "include_project_rag": self._include_project_rag_from_args(args),
             }
         )
         if route.get("isError"):